@@ -101,6 +101,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
101101 usable (decrypted) size still matches the requested volume size. */
102102 private static final long LUKS2_HEADER_RESERVE_BYTES = 16L << 20 ; // 16 MiB
103103
104+ /** Suffix of the per-template base image encrypted roots are cloned from. */
105+ private static final String LUKS_CLONE_BASE_SUFFIX = "-luks" ;
106+
104107 private static final Set <StoragePoolType > QEMU_IMG_MANAGED_POOL_TYPES = Set .of (StoragePoolType .NetworkFilesystem , StoragePoolType .Filesystem , StoragePoolType .SharedMountPoint );
105108
106109 public LibvirtStorageAdaptor (StorageLayer storage ) {
@@ -1200,6 +1203,7 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag
12001203 }
12011204 logger .info ("Successfully unprotected and removed any remaining snapshots (" + snaps .size () + ") of "
12021205 + pool .getSourceDir () + "/" + uuid + " Continuing to remove the RBD image" );
1206+ removeLuksCloneBaseIfPresent (rbd , uuid );
12031207 } catch (RbdException e ) {
12041208 logger .error ("Failed to remove snapshot with exception: " + e .toString () +
12051209 ", RBD error: " + ErrorCode .getErrorMessage (e .getReturnValue ()));
@@ -1526,54 +1530,35 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template,
15261530
15271531 /**
15281532 * Option A (thin CoW encrypted root), used when the template already lives on the same RBD cluster
1529- * as the destination pool. Per the Ceph "Image Encryption" clone recipe: grow the template base to
1530- * reserve LUKS2- header space, snapshot+protect that grown state, clone from it, apply a LUKS2 header,
1531- * then resize the clone to the requested size. The inherited (plaintext) template data stays readable
1532- * through the clone's encryption, and the clone is a thin CoW image (only the header is written) .
1533+ * as the destination pool: clone a per-template base image that carries LUKS2-header room, apply a
1534+ * LUKS2 header to the clone, then grow it to the requested size. The base is a dense copy of the
1535+ * template, not the template itself, because a hole in the parent of an encrypted clone reads as
1536+ * garbage once a write copies up the object around it. It is built once per template per pool .
15331537 *
15341538 * @return the encrypted CoW clone, or {@code null} if the Ceph operations failed
15351539 */
15361540 private KVMPhysicalDisk createEncryptedRootCoWClone (KVMPhysicalDisk template , KVMStoragePool destPool ,
15371541 String newUuid , KVMPhysicalDisk disk , byte [] passphrase ) {
1538- String luksReservedSnapshotName = rbdTemplateSnapName + "-luks" ;
1542+ final String cloneBaseImage = template . getName () + LUKS_CLONE_BASE_SUFFIX ;
15391543 Rados radosConnection = null ;
15401544 IoCTX ioContext = null ;
1541- Rbd rbdClient = null ;
1542- RbdImage templateImage = null ;
15431545 try {
15441546 radosConnection = new Rados (destPool .getAuthUserName ());
15451547 radosConnection .confSet ("mon_host" , destPool .getSourceHost () + ":" + destPool .getSourcePort ());
15461548 radosConnection .confSet ("key" , destPool .getAuthSecret ());
15471549 radosConnection .confSet ("client_mount_timeout" , "30" );
15481550 radosConnection .connect ();
15491551 ioContext = radosConnection .ioCtxCreate (destPool .getSourceDir ());
1550- rbdClient = new Rbd (ioContext );
1551- templateImage = rbdClient .open (template .getName ());
1552- boolean luksSnapshotExists = false ;
1553- for (RbdSnapInfo snapshotInfo : templateImage .snapList ()) {
1554- if (luksReservedSnapshotName .equals (snapshotInfo .name )) {
1555- luksSnapshotExists = true ;
1556- break ;
1557- }
1558- }
1559- if (!luksSnapshotExists ) {
1560- templateImage .resize (template .getVirtualSize () + LUKS2_HEADER_RESERVE_BYTES );
1561- templateImage .snapCreate (luksReservedSnapshotName );
1562- templateImage .snapProtect (luksReservedSnapshotName );
1563- logger .debug ("Prepared LUKS-reserved template snapshot {}@{}" , template .getName (), luksReservedSnapshotName );
1552+ Rbd rbdClient = new Rbd (ioContext );
1553+ if (!isLuksCloneBaseReady (rbdClient , cloneBaseImage )) {
1554+ buildLuksCloneBase (rbdClient , template , destPool , cloneBaseImage );
15641555 }
1565- rbdClient .clone (template .getName (), luksReservedSnapshotName , ioContext , newUuid , RBD_FEATURES , rbdOrder );
1556+ rbdClient .clone (cloneBaseImage , rbdTemplateSnapName , ioContext , newUuid , RBD_FEATURES , rbdOrder );
1557+ logger .debug ("Cloned {}/{}@{} to {}" , destPool .getSourceDir (), cloneBaseImage , rbdTemplateSnapName , newUuid );
15661558 } catch (RadosException | RbdException e ) {
15671559 logger .error ("Failed to create encrypted CoW clone {}: {}" , newUuid , e .getMessage ());
15681560 return null ;
15691561 } finally {
1570- if (rbdClient != null && templateImage != null ) {
1571- try {
1572- rbdClient .close (templateImage );
1573- } catch (RbdException ignored ) {
1574- // best-effort close of the template handle
1575- }
1576- }
15771562 if (radosConnection != null && ioContext != null ) {
15781563 radosConnection .ioCtxDestroy (ioContext );
15791564 }
@@ -1589,6 +1574,114 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV
15891574 return disk ;
15901575 }
15911576
1577+ /** Does {@code cloneBaseImage} exist with its protected snapshot? It gets its name only once complete. */
1578+ private boolean isLuksCloneBaseReady (Rbd rbdClient , String cloneBaseImage ) {
1579+ RbdImage image = null ;
1580+ try {
1581+ image = rbdClient .open (cloneBaseImage );
1582+ for (RbdSnapInfo snapshotInfo : image .snapList ()) {
1583+ if (rbdTemplateSnapName .equals (snapshotInfo .name )) {
1584+ return image .snapIsProtected (snapshotInfo .name );
1585+ }
1586+ }
1587+ return false ;
1588+ } catch (RbdException e ) {
1589+ // not there yet, or not readable: it gets built
1590+ return false ;
1591+ } finally {
1592+ if (image != null ) {
1593+ try {
1594+ rbdClient .close (image );
1595+ } catch (RbdException ignored ) {
1596+ // best-effort close of the base image handle
1597+ }
1598+ }
1599+ }
1600+ }
1601+
1602+ /**
1603+ * Build the base image: template size plus LUKS2-header room, a dense copy of the template, and a
1604+ * protected snapshot. It is built under a staging name and renamed at the end, so a half-written
1605+ * base is never cloned; a host that loses the rename to another one discards its copy.
1606+ */
1607+ private void buildLuksCloneBase (Rbd rbdClient , KVMPhysicalDisk template , KVMStoragePool destPool ,
1608+ String cloneBaseImage ) throws RbdException {
1609+ final String stagingImage = cloneBaseImage + "." + UUID .randomUUID ();
1610+ logger .info ("Building dense LUKS clone base {}/{} from template {} as {}" , destPool .getSourceDir (),
1611+ cloneBaseImage , template .getName (), stagingImage );
1612+ rbdClient .create (stagingImage , template .getVirtualSize () + LUKS2_HEADER_RESERVE_BYTES , RBD_FEATURES , rbdOrder );
1613+ boolean renamed = false ;
1614+ try {
1615+ new RbdEncryption ().copyTemplateDense (destPool .getSourceHost (), destPool .getSourcePort (),
1616+ destPool .getAuthUserName (), destPool .getAuthSecret (), destPool .getSourceDir (),
1617+ template .getName (), stagingImage );
1618+ RbdImage stagedImage = rbdClient .open (stagingImage );
1619+ try {
1620+ stagedImage .snapCreate (rbdTemplateSnapName );
1621+ stagedImage .snapProtect (rbdTemplateSnapName );
1622+ } finally {
1623+ rbdClient .close (stagedImage );
1624+ }
1625+ try {
1626+ rbdClient .rename (stagingImage , cloneBaseImage );
1627+ renamed = true ;
1628+ logger .info ("Prepared dense LUKS clone base {}/{}" , destPool .getSourceDir (), cloneBaseImage );
1629+ } catch (RbdException e ) {
1630+ logger .info ("Another host prepared LUKS clone base {}/{} first; discarding {}" ,
1631+ destPool .getSourceDir (), cloneBaseImage , stagingImage );
1632+ }
1633+ } finally {
1634+ if (!renamed ) {
1635+ removeRbdImageQuietly (rbdClient , stagingImage );
1636+ }
1637+ }
1638+ if (!renamed && !isLuksCloneBaseReady (rbdClient , cloneBaseImage )) {
1639+ throw new RbdException ("LUKS clone base " + destPool .getSourceDir () + "/" + cloneBaseImage
1640+ + " could not be prepared and no other host has prepared it" );
1641+ }
1642+ }
1643+
1644+ /** Remove a template's LUKS clone base image along with the template, if it has one. */
1645+ private void removeLuksCloneBaseIfPresent (Rbd rbdClient , String volumeName ) {
1646+ final String cloneBaseImage = volumeName + LUKS_CLONE_BASE_SUFFIX ;
1647+ RbdImage image ;
1648+ try {
1649+ image = rbdClient .open (cloneBaseImage );
1650+ } catch (RbdException e ) {
1651+ // everything that is not a template of an encrypted root has no base image
1652+ return ;
1653+ }
1654+ try {
1655+ rbdClient .close (image );
1656+ } catch (RbdException ignored ) {
1657+ // best-effort close of the probe handle
1658+ }
1659+ logger .info ("Removing LUKS clone base image {} along with {}" , cloneBaseImage , volumeName );
1660+ removeRbdImageQuietly (rbdClient , cloneBaseImage );
1661+ }
1662+
1663+ /** Remove an RBD image and its snapshots, logging instead of failing. */
1664+ private void removeRbdImageQuietly (Rbd rbdClient , String imageName ) {
1665+ try {
1666+ RbdImage image = rbdClient .open (imageName );
1667+ try {
1668+ for (RbdSnapInfo snapshotInfo : image .snapList ()) {
1669+ if (image .snapIsProtected (snapshotInfo .name )) {
1670+ image .snapUnprotect (snapshotInfo .name );
1671+ }
1672+ image .snapRemove (snapshotInfo .name );
1673+ }
1674+ } finally {
1675+ rbdClient .close (image );
1676+ }
1677+ rbdClient .remove (imageName );
1678+ logger .debug ("Removed RBD image {}" , imageName );
1679+ } catch (RbdException e ) {
1680+ // e.g. a base image still has clones of live encrypted roots
1681+ logger .warn ("Could not remove RBD image {}: {}" , imageName , e .getMessage ());
1682+ }
1683+ }
1684+
15921685 /**
15931686 * Option B (full-copy encrypted root), used when the template is not on the same RBD cluster (e.g. first
15941687 * use from secondary storage). Create an empty image, apply a LUKS2 header, then import the template
0 commit comments