From 6110c07eae7e93f457ca314fefa6ac2b748d5db5 Mon Sep 17 00:00:00 2001 From: sp16743 Date: Fri, 9 Oct 2026 16:59:17 +0530 Subject: [PATCH 1/3] Bug Fix:CSTACKEX-327:-Concurrent iSCSI VM deployment fails intermittently: post-clone LUN resize returns 404 --- .../storage/service/UnifiedSANStrategy.java | 55 +++++++++++++------ .../storage/utils/OntapStorageConstants.java | 3 + 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java index a632ecf78fb8..6bf8bb9b0348 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java @@ -295,33 +295,54 @@ private void validateCreatedLun(Lun lun, String requestName, String operation) { * Grows an existing LUN to {@code sizeInBytes}. * *

Needed after cloning a cached template, because a clone inherits the size of its source - * while the service offering may ask for a larger disk.

+ * while the service offering may ask for a larger disk. ONTAP can temporarily return 404 when + * PATCH follows clone creation before the new LUN is visible to the update path, so PATCH is + * retried with the UUID returned by the clone response.

*/ @Override public void resizeCloudStackVolume(CloudStackVolume cloudstackVolume, long sizeInBytes) { - if (cloudstackVolume == null || cloudstackVolume.getLun() == null || cloudstackVolume.getLun().getUuid() == null) { + if (cloudstackVolume == null || cloudstackVolume.getLun() == null + || cloudstackVolume.getLun().getUuid() == null) { logger.error("resizeCloudStackVolume: Lun resize failed. Invalid request: {}", cloudstackVolume); throw new CloudRuntimeException("Failed to resize Lun, invalid request"); } if (sizeInBytes <= 0) { throw new CloudRuntimeException("Failed to resize Lun, invalid size " + sizeInBytes); } + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + LunSpace lunSpace = new LunSpace(); + lunSpace.setSize(sizeInBytes); + Lun patch = new Lun(); + patch.setSpace(lunSpace); + + int maxRetries = OntapStorageConstants.ONTAP_LUN_RESIZE_MAX_RETRIES; + int pollIntervalMs = OntapStorageConstants.ONTAP_LUN_RESIZE_RETRY_INTERVAL_MS; String lunUuid = cloudstackVolume.getLun().getUuid(); - logger.trace("resizeCloudStackVolume: Resizing Lun {} to {} bytes", lunUuid, sizeInBytes); - try { - String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); - LunSpace lunSpace = new LunSpace(); - lunSpace.setSize(sizeInBytes); - Lun patch = new Lun(); - patch.setSpace(lunSpace); - JobResponse response = sanFeignClient.updateLun(authHeader, lunUuid, patch); - pollJobIfPresent(response, "resize Lun [" + lunUuid + "]"); - logger.debug("resizeCloudStackVolume: Lun {} resized to {} bytes", lunUuid, sizeInBytes); - } catch (FeignException e) { - throw new CloudRuntimeException("Failed to resize Lun: " + e.getMessage(), e); - } catch (Exception e) { - logger.error("Exception occurred while resizing LUN: {}, Exception: {}", lunUuid, e.getMessage()); - throw new CloudRuntimeException("Failed to resize Lun: " + e.getMessage()); + for (int attempt = 1; ; attempt++) { + logger.trace("resizeCloudStackVolume: Resizing Lun {} to {} bytes", lunUuid, sizeInBytes); + try { + JobResponse response = sanFeignClient.updateLun(authHeader, lunUuid, patch); + pollJobIfPresent(response, "resize Lun [" + lunUuid + "]"); + logger.debug("resizeCloudStackVolume: Lun {} resized to {} bytes", lunUuid, sizeInBytes); + return; + } catch (FeignException e) { + if (e.status() != 404 || attempt >= maxRetries) { + logger.error("FeignException occurred while resizing LUN: {}, Status: {}, Exception: {}", + lunUuid, e.status(), e.getMessage()); + throw new CloudRuntimeException("Failed to resize Lun: " + e.getMessage(), e); + } + logger.debug("resizeCloudStackVolume: PATCH for Lun {} returned 404, retry {}/{}", + lunUuid, attempt, maxRetries); + } catch (Exception e) { + logger.error("Exception occurred while resizing LUN: {}, Exception: {}", lunUuid, e.getMessage()); + throw new CloudRuntimeException("Failed to resize Lun: " + e.getMessage(), e); + } + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new CloudRuntimeException("Interrupted while waiting for Lun to accept resize", ie); + } } } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java index 9cf9b7356b15..b824d7e363e7 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java @@ -147,6 +147,9 @@ public class OntapStorageConstants { public static final int ONTAP_CG_JOB_POLL_INTERVAL_MS = 2000; public static final int ONTAP_CG_SNAPSHOT_RESOLVE_MAX_RETRIES = 30; public static final int ONTAP_CG_SNAPSHOT_RESOLVE_POLL_INTERVAL_MS = 1000; + /** Retry settings for a newly cloned LUN that is not yet visible to the resize API. */ + public static final int ONTAP_LUN_RESIZE_MAX_RETRIES = 30; + public static final int ONTAP_LUN_RESIZE_RETRY_INTERVAL_MS = 1000; public static final int ONTAP_SFSR_JOB_MAX_RETRIES = 60; public static final int ONTAP_SFSR_JOB_POLL_INTERVAL_MS = 2000; public static final int ONTAP_SNAPSHOT_DELETE_JOB_MAX_RETRIES = 30; From 54dc776a9de10a7f69435c39a1e62665f1c9827e Mon Sep 17 00:00:00 2001 From: sp16743 Date: Fri, 9 Oct 2026 17:41:07 +0530 Subject: [PATCH 2/3] Remove the extra LUN resize comment. --- .../apache/cloudstack/storage/service/UnifiedSANStrategy.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java index 6bf8bb9b0348..f09db476425d 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java @@ -295,9 +295,7 @@ private void validateCreatedLun(Lun lun, String requestName, String operation) { * Grows an existing LUN to {@code sizeInBytes}. * *

Needed after cloning a cached template, because a clone inherits the size of its source - * while the service offering may ask for a larger disk. ONTAP can temporarily return 404 when - * PATCH follows clone creation before the new LUN is visible to the update path, so PATCH is - * retried with the UUID returned by the clone response.

+ * while the service offering may ask for a larger disk.

*/ @Override public void resizeCloudStackVolume(CloudStackVolume cloudstackVolume, long sizeInBytes) { From 62c8ec755b0b4f58de481d8aca708eef07b096dd Mon Sep 17 00:00:00 2001 From: sp16743 Date: Fri, 9 Oct 2026 22:07:10 +0530 Subject: [PATCH 3/3] changes --- .../apache/cloudstack/storage/service/UnifiedSANStrategy.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java index f09db476425d..844c99d80cf0 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java @@ -299,8 +299,7 @@ private void validateCreatedLun(Lun lun, String requestName, String operation) { */ @Override public void resizeCloudStackVolume(CloudStackVolume cloudstackVolume, long sizeInBytes) { - if (cloudstackVolume == null || cloudstackVolume.getLun() == null - || cloudstackVolume.getLun().getUuid() == null) { + if (cloudstackVolume == null || cloudstackVolume.getLun() == null || cloudstackVolume.getLun().getUuid() == null) { logger.error("resizeCloudStackVolume: Lun resize failed. Invalid request: {}", cloudstackVolume); throw new CloudRuntimeException("Failed to resize Lun, invalid request"); }