From de50d8ae71f84e5167148aea3aa199a0e6e7304d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 7 Sep 2026 16:57:30 +0530 Subject: [PATCH 1/5] engine-schema: upgrade path for 24.0.0 Adds a 4.23.0 to 24.0.0 upgrade path. Signed-off-by: Abhishek Kumar --- .../cloud/upgrade/DatabaseUpgradeChecker.java | 6 ++-- .../cloud/upgrade/dao/Upgrade42300to2400.java | 30 +++++++++++++++++++ .../db/schema-42300to2400-cleanup.sql | 20 +++++++++++++ .../META-INF/db/schema-42300to2400.sql | 20 +++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java create mode 100644 engine/schema/src/main/resources/META-INF/db/schema-42300to2400-cleanup.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index c3a982aa70e5..a431b63deabb 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -91,11 +91,12 @@ import com.cloud.upgrade.dao.Upgrade42020to42030; import com.cloud.upgrade.dao.Upgrade42030to42040; import com.cloud.upgrade.dao.Upgrade42040to42100; -import com.cloud.upgrade.dao.Upgrade42100to42200; -import com.cloud.upgrade.dao.Upgrade42200to42210; import com.cloud.upgrade.dao.Upgrade420to421; +import com.cloud.upgrade.dao.Upgrade42100to42200; import com.cloud.upgrade.dao.Upgrade421to430; +import com.cloud.upgrade.dao.Upgrade42200to42210; import com.cloud.upgrade.dao.Upgrade42210to42300; +import com.cloud.upgrade.dao.Upgrade42300to2400; import com.cloud.upgrade.dao.Upgrade430to440; import com.cloud.upgrade.dao.Upgrade431to440; import com.cloud.upgrade.dao.Upgrade432to440; @@ -248,6 +249,7 @@ public DatabaseUpgradeChecker() { .next("4.21.0.0", new Upgrade42100to42200()) .next("4.22.0.0", new Upgrade42200to42210()) .next("4.22.1.0", new Upgrade42210to42300()) + .next("4.23.0.0", new Upgrade42300to2400()) .build(); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java new file mode 100644 index 000000000000..ce217cef9e75 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +public class Upgrade42300to2400 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { + + @Override + public String[] getUpgradableVersionRange() { + return new String[]{"4.23.0.0", "24.0.0"}; + } + + @Override + public String getUpgradedVersion() { + return "24.0.0"; + } +} diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400-cleanup.sql new file mode 100644 index 000000000000..861a038fe7a7 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400-cleanup.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade cleanup from 4.23.0.0 to 24.0.0 +--; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql new file mode 100644 index 000000000000..7c11013a17d2 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.23.0.0 to 24.0.0 +--; From bfd20021d5bcdab02d118d60c30c3840c830086c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 9 Sep 2026 18:47:00 +0530 Subject: [PATCH 2/5] fix CS version Signed-off-by: Abhishek Kumar --- .../cloud/upgrade/DatabaseUpgradeChecker.java | 9 ++- .../cloudstack/veeam/api/dto/Version.java | 8 ++- .../cloudstack/utils/CloudStackVersion.java | 62 +++++++++++++++---- .../utils/CloudStackVersionTest.java | 43 +++++++++++-- 4 files changed, 102 insertions(+), 20 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index a431b63deabb..f3ec76a27c7a 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -515,8 +515,13 @@ protected void doUpgrades(GlobalLock lock) { String csVersion = parseSystemVmMetadata(); final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion); final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); - SystemVmTemplateRegistration.CS_MAJOR_VERSION = sysVmVersion.getMajorRelease() + "." + sysVmVersion.getMinorRelease(); - SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease()); + if (sysVmVersion.usesNewVersioning()) { + SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(sysVmVersion.getMajorRelease()); + SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getMajorRelease()); + } else { + SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.format("%d.%d", sysVmVersion.getMajorRelease(), sysVmVersion.getMinorRelease()); + SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease()); + } LOGGER.info("DB version = {} Code Version = {}", dbVersion, currentVersion); diff --git a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java index 7b7d80a0f16c..2d14443e2d58 100644 --- a/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java +++ b/plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java @@ -87,8 +87,12 @@ public static Version fromPackageAndCSVersion(boolean complete) { } version.setMajor(String.valueOf(csVersion.getMajorRelease())); version.setMinor(String.valueOf(csVersion.getMinorRelease())); - version.setBuild(String.valueOf(csVersion.getPatchRelease())); - version.setRevision(String.valueOf(csVersion.getSecurityRelease())); + if (csVersion.usesNewVersioning()) { + version.setBuild(String.valueOf(csVersion.getSecurityRelease())); + } else { + version.setBuild(String.valueOf(csVersion.getPatchRelease())); + version.setRevision(String.valueOf(csVersion.getSecurityRelease())); + } return version; } } diff --git a/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java b/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java index e29bd9c4e17b..8eb4c6ab9289 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java @@ -39,22 +39,23 @@ */ public final class CloudStackVersion implements Comparable { - private final static Pattern NUMBER_VERSION_FORMAT = Pattern.compile("(\\d+\\.){2}(\\d+\\.)?\\d+"); - private final static Pattern FULL_VERSION_FORMAT = Pattern.compile("(\\d+\\.){2}(\\d+\\.)?\\d+(-[a-zA-Z]+)?(-\\d+)?(-SNAPSHOT)?"); + private final static Pattern NUMBER_VERSION_FORMAT = Pattern.compile("\\d+\\.\\d+\\.\\d+(?:\\.\\d+)?"); + private final static Pattern FULL_VERSION_FORMAT = Pattern.compile("\\d+\\.\\d+\\.\\d+(?:\\.\\d+)?(?:-[a-zA-Z]+)?(?:-\\d+)?(?:-SNAPSHOT)?"); + private final static int NEW_VERSIONING_CUTOVER_MAJOR_VERSION = 24; private final int majorRelease; private final int minorRelease; - private final int patchRelease; + private final Integer patchRelease; private final Integer securityRelease; - private CloudStackVersion(final int majorRelease, final int minorRelease, final int patchRelease, final Integer securityRelease) { + private CloudStackVersion(final int majorRelease, final int minorRelease, final Integer patchRelease, final Integer securityRelease) { super(); checkArgument(majorRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a majorRelease greater than 0."); checkArgument(minorRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a minorRelease greater than 0."); - checkArgument(patchRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a patchRelease greater than 0."); - checkArgument((securityRelease != null && securityRelease >= 0) || (securityRelease == null), + checkArgument(patchRelease == null || patchRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a patchRelease greater than 0."); + checkArgument(securityRelease == null || securityRelease >= 0, CloudStackVersion.class.getName() + "(int, int, int, Integer) requires a null securityRelease or a non-null value greater than 0."); this.majorRelease = majorRelease; @@ -69,11 +70,13 @@ private CloudStackVersion(final int majorRelease, final int minorRelease, final * Parses a String representation of a version that conforms one of the following * formats into a CloudStackVersion instance: *
    - *
  • <major>.<minor>.<patch>.<security>
    • - *
  • <major>.<minor>.<patch>.<security>.<security>
    • - *
  • <major>.<minor>.<patch>.<security>.<security>-<any string>
    • + *
  • <major>.<minor>.<patch> (legacy, deprecated since 24.0.0, allowed only below major version 24)
    • + *
  • <major>.<minor>.<patch>.<security> (legacy, deprecated since 24.0.0, allowed only below major version 24)
    • + *
  • <major>.<minor>.<security release> (for versions >= 24.0.0)
    • *
* + * Legacy patch-based formats remain supported for backward compatibility. + * * If the string contains a suffix that begins with a "-" character, then the "-" and all characters following it * will be dropped. * @@ -91,7 +94,7 @@ public static CloudStackVersion parse(final String value) { checkArgument(StringUtils.isNotBlank(trimmedValue), CloudStackVersion.class.getName() + ".parse(String) requires a non-blank value"); checkArgument(NUMBER_VERSION_FORMAT.matcher(trimmedValue).matches(), CloudStackVersion.class.getName() + ".parse(String) passed " + - value + ", but requires a value in the format of int.int.int(.int)(-)"); + value + ", but requires a value in the format of int.int.int(.int)(-)"); final String[] components = trimmedValue.split("\\."); @@ -100,8 +103,26 @@ public static CloudStackVersion parse(final String value) { final int majorRelease = Integer.valueOf(components[0]); final int minorRelease = Integer.valueOf(components[1]); - final int patchRelease = Integer.valueOf(components[2]); - final Integer securityRelease = components.length == 3 ? null : Integer.valueOf(components[3]); + final int thirdComponent = Integer.valueOf(components[2]); + + final int patchRelease; + final Integer securityRelease; + + if (components.length == 4) { + checkArgument(isLegacyVersioning(majorRelease), CloudStackVersion.class.getName() + ".parse(String) passed " + value + + ", but major versions at or above 24 do not support legacy int.int.int.int format"); + // Deprecated legacy format: major.minor.patch.security + patchRelease = thirdComponent; + securityRelease = Integer.valueOf(components[3]); + } else if (isNewVersioning(majorRelease)) { + // New format: major.minor.securityRelease (patch dropped) + patchRelease = 0; + securityRelease = thirdComponent; + } else { + // Deprecated legacy format: major.minor.patch + patchRelease = thirdComponent; + securityRelease = null; + } return new CloudStackVersion(majorRelease, minorRelease, patchRelease, securityRelease); @@ -207,6 +228,14 @@ private static ImmutableList normalizeVersionValues(final ImmutableList } + private static boolean isLegacyVersioning(final int majorRelease) { + return majorRelease < NEW_VERSIONING_CUTOVER_MAJOR_VERSION; + } + + private static boolean isNewVersioning(final int majorRelease) { + return majorRelease >= NEW_VERSIONING_CUTOVER_MAJOR_VERSION; + } + /** * * @return The components of this version as an {@link ImmutableList} in order of major release, minor release, @@ -244,6 +273,10 @@ public Integer getSecurityRelease() { return securityRelease; } + public boolean usesNewVersioning() { + return isNewVersioning(majorRelease); + } + @Override public boolean equals(final Object thatObject) { @@ -270,6 +303,11 @@ public int hashCode() { @Override public String toString() { + // Canonicalize cutover-and-later versions to major.minor.securityRelease. + if (securityRelease != null && patchRelease == 0 && isNewVersioning(majorRelease)) { + return Joiner.on(".").join(ImmutableList.of(majorRelease, minorRelease, securityRelease)); + } + return Joiner.on(".").join(asList()); } diff --git a/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java b/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java index dabaf9bc97d3..4d0b4cb0439b 100644 --- a/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java +++ b/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java @@ -36,7 +36,11 @@ public final class CloudStackVersionTest { "1.2.3, 1.2.3", "1.2.3.4, 1.2.3.4", "1.2.3-12, 1.2.3", - "1.2.3.4-14, 1.2.3.4" + "1.2.3.4-14, 1.2.3.4", + "23.9.5, 23.9.5", + "24.0.0, 24.0.0", + "24.0.1, 24.0.1", + "25.1.1, 25.1.1" }) public void testValidParse(final String inputValue, final String expectedVersion) { final CloudStackVersion version = CloudStackVersion.parse(inputValue); @@ -44,6 +48,28 @@ public void testValidParse(final String inputValue, final String expectedVersion assertEquals(expectedVersion, version.toString()); } + @Test + public void testParseComponentMappingForLegacyAndNewVersioning() { + final CloudStackVersion legacyVersion = CloudStackVersion.parse("23.9.5"); + assertEquals(23, legacyVersion.getMajorRelease()); + assertEquals(9, legacyVersion.getMinorRelease()); + assertEquals(5, legacyVersion.getPatchRelease()); + Assert.assertNull(legacyVersion.getSecurityRelease()); + + final CloudStackVersion newVersion = CloudStackVersion.parse("24.0.1"); + assertEquals(24, newVersion.getMajorRelease()); + assertEquals(0, newVersion.getMinorRelease()); + // Patch is retained as 0 to represent "no patch" in the new major.minor.security scheme. + assertEquals(0, newVersion.getPatchRelease()); + assertEquals(Integer.valueOf(1), newVersion.getSecurityRelease()); + + final CloudStackVersion futureNewVersion = CloudStackVersion.parse("25.1.1"); + assertEquals(25, futureNewVersion.getMajorRelease()); + assertEquals(1, futureNewVersion.getMinorRelease()); + assertEquals(0, futureNewVersion.getPatchRelease()); + assertEquals(Integer.valueOf(1), futureNewVersion.getSecurityRelease()); + } + @Test(expected = IllegalArgumentException.class) @DataProvider({ "1.2", @@ -52,7 +78,10 @@ public void testValidParse(final String inputValue, final String expectedVersion "aaaa", "", " ", - "1.2.3.4.5" + "1.2.3.4.5", + "24.0.0.1", + "25.0.0.1", + "26.2.3.4" }) public void testInvalidParse(final String invalidValue) { CloudStackVersion.parse(invalidValue); @@ -147,7 +176,9 @@ public void testEqualCompareDirect(final String value, final String thatValue) { "1.2.3.4-10, 1.0.0.0-5", "1.2.3-10, 1.0.0-5", "1.2.3.4, 1.0.0.0-5", - "1.2.3.4-10, 1.0.0" + "1.2.3.4-10, 1.0.0", + "24.0.2, 24.0.1", + "24.1.0, 24.0.9" }) public void testGreaterThanAndLessThanCompareTo(final String value, final String thatValue) { @@ -178,7 +209,9 @@ public void testGreaterThanAndLessThanCompareTo(final String value, final String "1.2.3.4-10, 1.0.0.0-5", "1.2.3-10, 1.0.0-5", "1.2.3.4, 1.0.0.0-5", - "1.2.3.4-10, 1.0.0" + "1.2.3.4-10, 1.0.0", + "24.0.2, 24.0.1", + "24.1.0, 24.0.9" }) public void testGreaterThanAndLessThanCompareDirect(final String value, final String thatValue) { @@ -213,6 +246,7 @@ private void verifyGetVMwareParentVersion(String hypervisorVersion, String expec Assert.assertEquals(CloudStackVersion.getVMwareParentVersion(hypervisorVersion), expectedParentVersion); } } + @Test public void testGetParentVersion() { verifyGetVMwareParentVersion(null, null); @@ -223,5 +257,6 @@ public void testGetParentVersion() { verifyGetVMwareParentVersion("8.0.0", "8.0"); verifyGetVMwareParentVersion("8.0.0.2", "8.0"); verifyGetVMwareParentVersion("8.0.1.0", "8.0.1"); + verifyGetVMwareParentVersion("24.1.1", "24.1"); } } From eda84a6d5bca4ab43457fc3c95ac8f92a08753c7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 10 Sep 2026 13:28:02 +0530 Subject: [PATCH 3/5] fix upgrade unit tests for cutover Signed-off-by: Abhishek Kumar --- .../upgrade/DatabaseUpgradeCheckerTest.java | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java index 884398cf410d..763f78fe6949 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java @@ -25,6 +25,7 @@ import javax.sql.DataSource; import org.apache.cloudstack.utils.CloudStackVersion; +import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Test; import org.junit.Before; import org.junit.After; @@ -214,10 +215,10 @@ public void testFindUpgradePath452to490() { @Test public void testCalculateUpgradePathUnknownDbVersion() { - final CloudStackVersion dbVersion = CloudStackVersion.parse("4.99.0.0"); + final CloudStackVersion dbVersion = CloudStackVersion.parse("99.0.0"); assertNotNull(dbVersion); - final CloudStackVersion currentVersion = CloudStackVersion.parse("4.99.1.0"); + final CloudStackVersion currentVersion = CloudStackVersion.parse("99.1.0"); assertNotNull(currentVersion); final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); @@ -234,7 +235,7 @@ public void testCalculateUpgradePathFromKnownDbVersion() { final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.0.0"); assertNotNull(dbVersion); - final CloudStackVersion currentVersion = CloudStackVersion.parse("4.99.1.0"); + final CloudStackVersion currentVersion = CloudStackVersion.parse("99.1.0"); assertNotNull(currentVersion); final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); @@ -268,10 +269,7 @@ public void testCalculateUpgradePathFromLatestDbVersion() { final CloudStackVersion dbVersion = checker.getLatestVersion(); assertNotNull(dbVersion); - final CloudStackVersion currentVersion = CloudStackVersion.parse(dbVersion.getMajorRelease() + "." - + dbVersion.getMinorRelease() + "." - + dbVersion.getPatchRelease() + "." - + (dbVersion.getSecurityRelease() + 1)); + final CloudStackVersion currentVersion = getNextSecurityRelease(dbVersion); assertNotNull(currentVersion); final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); @@ -293,10 +291,7 @@ public void testCalculateUpgradePathFrom41800toNextSecurityRelease() { final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); assertNotNull(upgrades); - final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "." - + currentVersion.getMinorRelease() + "." - + currentVersion.getPatchRelease() + "." - + (currentVersion.getSecurityRelease() + 1)); + final CloudStackVersion nextSecurityRelease = getNextSecurityRelease(currentVersion); assertNotNull(nextSecurityRelease); final DbUpgrade[] upgradesToNext = checker.calculateUpgradePath(dbVersion, nextSecurityRelease); @@ -306,16 +301,26 @@ public void testCalculateUpgradePathFrom41800toNextSecurityRelease() { assertTrue(upgradesToNext[upgradesToNext.length - 1] instanceof NoopDbUpgrade); } + private static CloudStackVersion getNextSecurityRelease(CloudStackVersion version, int increment) { + String nextSecurityReleaseVersionStr = version.getMajorRelease() + "." + + version.getMinorRelease() + "." + + (version.usesNewVersioning() ? "" : version.getPatchRelease() + ".") + + (version.getSecurityRelease() + increment); + + return CloudStackVersion.parse(nextSecurityReleaseVersionStr); + } + + private static CloudStackVersion getNextSecurityRelease(CloudStackVersion version) { + return getNextSecurityRelease(version, 1); + } + @Test public void testCalculateUpgradePathFromSecurityReleaseToLatest() { final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.2.0"); // a EOL version assertNotNull(dbVersion); - final CloudStackVersion oldSecurityRelease = CloudStackVersion.parse(dbVersion.getMajorRelease() + "." - + dbVersion.getMinorRelease() + "." - + dbVersion.getPatchRelease() + "." - + (dbVersion.getSecurityRelease() + 100)); + final CloudStackVersion oldSecurityRelease = getNextSecurityRelease(dbVersion, 100); assertNotNull(oldSecurityRelease); // fake security release 4.17.2.100 final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); @@ -347,10 +352,7 @@ public void testCalculateUpgradePathFromSecurityReleaseToNextSecurityRelease() { final CloudStackVersion currentVersion = checker.getLatestVersion(); assertNotNull(currentVersion); - final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "." - + currentVersion.getMinorRelease() + "." - + currentVersion.getPatchRelease() + "." - + (currentVersion.getSecurityRelease() + 1)); + final CloudStackVersion nextSecurityRelease = getNextSecurityRelease(currentVersion); assertNotNull(nextSecurityRelease); // fake security release final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); From f334927e118f95204402aba147a76a8b45685cff Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 10 Sep 2026 16:04:15 +0530 Subject: [PATCH 4/5] fix imports Signed-off-by: Abhishek Kumar --- .../upgrade/DatabaseUpgradeCheckerTest.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java index 763f78fe6949..3810d03161d9 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java @@ -16,21 +16,25 @@ // under the License. package com.cloud.upgrade; -import java.sql.SQLException; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + import java.lang.reflect.Field; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import javax.sql.DataSource; import org.apache.cloudstack.utils.CloudStackVersion; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.junit.Test; -import org.junit.Before; import org.junit.After; +import org.junit.Before; +import org.junit.Test; import org.junit.runner.RunWith; - import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Mockito; @@ -56,15 +60,8 @@ import com.cloud.upgrade.dao.Upgrade471to480; import com.cloud.upgrade.dao.Upgrade480to481; import com.cloud.upgrade.dao.Upgrade490to4910; - import com.cloud.utils.db.TransactionLegacy; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertArrayEquals; - @RunWith(MockitoJUnitRunner.class) public class DatabaseUpgradeCheckerTest { From a0019617bf015fc6a14eb2718f0156633cb84c64 Mon Sep 17 00:00:00 2001 From: Ramgopal Nagaboina Date: Fri, 11 Sep 2026 17:02:58 -0400 Subject: [PATCH 5/5] metrics: add host usage history Store host stats samples in a new host_stats table and expose them through a new listHostsUsageHistory API, the host counterpart of listVirtualMachinesUsageHistory, listSystemVmsUsageHistory and listVolumesUsageHistory. Samples are written on every host stats collection and pruned by the new host.stats.max.retention.time setting (minutes, default 720). Setting it to 0 or less disables storing and pruning. --- .../cloud/agent/api/HostStatsEntryBase.java | 134 ++++++++++++++++ .../main/java/com/cloud/host/HostStatsVO.java | 88 +++++++++++ .../java/com/cloud/host/dao/HostStatsDao.java | 42 +++++ .../com/cloud/host/dao/HostStatsDaoImpl.java | 115 ++++++++++++++ ...spring-engine-schema-core-daos-context.xml | 1 + .../META-INF/db/schema-42300to2400.sql | 12 ++ .../api/ListHostsUsageHistoryCmd.java | 71 +++++++++ .../cloudstack/metrics/MetricsService.java | 3 + .../metrics/MetricsServiceImpl.java | 147 ++++++++++++++++++ .../response/HostMetricsStatsResponse.java | 53 +++++++ .../metrics/MetricsServiceImplTest.java | 110 +++++++++++++ .../java/com/cloud/server/StatsCollector.java | 61 ++++++++ .../com/cloud/server/StatsCollectorTest.java | 64 ++++++++ test/integration/smoke/test_metrics_api.py | 23 +++ 14 files changed, 924 insertions(+) create mode 100644 core/src/main/java/com/cloud/agent/api/HostStatsEntryBase.java create mode 100644 engine/schema/src/main/java/com/cloud/host/HostStatsVO.java create mode 100644 engine/schema/src/main/java/com/cloud/host/dao/HostStatsDao.java create mode 100644 engine/schema/src/main/java/com/cloud/host/dao/HostStatsDaoImpl.java create mode 100644 plugins/metrics/src/main/java/org/apache/cloudstack/api/ListHostsUsageHistoryCmd.java create mode 100644 plugins/metrics/src/main/java/org/apache/cloudstack/response/HostMetricsStatsResponse.java diff --git a/core/src/main/java/com/cloud/agent/api/HostStatsEntryBase.java b/core/src/main/java/com/cloud/agent/api/HostStatsEntryBase.java new file mode 100644 index 000000000000..2ce3f326401b --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/HostStatsEntryBase.java @@ -0,0 +1,134 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.host.HostStats; + +/** + * Serializable host-stats payload persisted as JSON in {@code host_stats.host_stats_data}. Unlike + * {@link HostStatsEntry} it carries no {@code HostVO}, so serialization does not pull in a host entity. + */ +public class HostStatsEntryBase implements HostStats { + + private long hostId; + private String entityType; + private double cpuUtilization; + private double averageLoad; + private double networkReadKBs; + private double networkWriteKBs; + private double totalMemoryKBs; + private double freeMemoryKBs; + + public HostStatsEntryBase() { + } + + public HostStatsEntryBase(long hostId, String entityType, double cpuUtilization, double averageLoad, + double networkReadKBs, double networkWriteKBs, double totalMemoryKBs, double freeMemoryKBs) { + this.hostId = hostId; + this.entityType = entityType; + this.cpuUtilization = cpuUtilization; + this.averageLoad = averageLoad; + this.networkReadKBs = networkReadKBs; + this.networkWriteKBs = networkWriteKBs; + this.totalMemoryKBs = totalMemoryKBs; + this.freeMemoryKBs = freeMemoryKBs; + } + + public long getHostId() { + return hostId; + } + + public void setHostId(long hostId) { + this.hostId = hostId; + } + + @Override + public String getEntityType() { + return entityType; + } + + public void setEntityType(String entityType) { + this.entityType = entityType; + } + + @Override + public double getCpuUtilization() { + return cpuUtilization; + } + + public void setCpuUtilization(double cpuUtilization) { + this.cpuUtilization = cpuUtilization; + } + + @Override + public double getLoadAverage() { + return averageLoad; + } + + public void setAverageLoad(double averageLoad) { + this.averageLoad = averageLoad; + } + + @Override + public double getNetworkReadKBs() { + return networkReadKBs; + } + + public void setNetworkReadKBs(double networkReadKBs) { + this.networkReadKBs = networkReadKBs; + } + + @Override + public double getNetworkWriteKBs() { + return networkWriteKBs; + } + + public void setNetworkWriteKBs(double networkWriteKBs) { + this.networkWriteKBs = networkWriteKBs; + } + + @Override + public double getTotalMemoryKBs() { + return totalMemoryKBs; + } + + public void setTotalMemoryKBs(double totalMemoryKBs) { + this.totalMemoryKBs = totalMemoryKBs; + } + + @Override + public double getFreeMemoryKBs() { + return freeMemoryKBs; + } + + public void setFreeMemoryKBs(double freeMemoryKBs) { + this.freeMemoryKBs = freeMemoryKBs; + } + + @Override + public double getUsedMemory() { + return (totalMemoryKBs - freeMemoryKBs) * 1024; + } + + @Override + public HostStats getHostStats() { + return this; + } +} diff --git a/engine/schema/src/main/java/com/cloud/host/HostStatsVO.java b/engine/schema/src/main/java/com/cloud/host/HostStatsVO.java new file mode 100644 index 000000000000..30869fa5cbef --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/host/HostStatsVO.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +/** One persisted historical host-stats sample. */ +@Entity +@Table(name = "host_stats") +public class HostStatsVO { + + @Id + @Column(name = "id", updatable = false, nullable = false) + protected long id; + + @Column(name = "host_id", updatable = false, nullable = false) + protected Long hostId; + + @Column(name = "mgmt_server_id", updatable = false, nullable = false) + protected Long mgmtServerId; + + @Column(name = "timestamp", updatable = false) + @Temporal(value = TemporalType.TIMESTAMP) + protected Date timestamp; + + @Column(name = "host_stats_data", updatable = false, nullable = false, length = 65535) + protected String hostStatsData; + + public HostStatsVO(Long hostId, Long mgmtServerId, Date timestamp, String hostStatsData) { + this.hostId = hostId; + this.mgmtServerId = mgmtServerId; + this.timestamp = timestamp; + this.hostStatsData = hostStatsData; + } + + public HostStatsVO() { + + } + + public long getId() { + return id; + } + + public Long getHostId() { + return hostId; + } + + public Long getMgmtServerId() { + return mgmtServerId; + } + + public Date getTimestamp() { + return timestamp; + } + + public String getHostStatsData() { + return hostStatsData; + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "hostId", "mgmtServerId", "timestamp", "hostStatsData"); + } + +} diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostStatsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostStatsDao.java new file mode 100644 index 000000000000..2a4bdd982f60 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostStatsDao.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host.dao; + +import java.util.Date; +import java.util.List; + +import com.cloud.host.HostStatsVO; +import com.cloud.utils.db.GenericDao; + +/** DAO for the host_stats table. */ +public interface HostStatsDao extends GenericDao { + + List findByHostId(long hostId); + + List findByHostIdAndTimestampGreaterThanEqual(long hostId, Date time); + + List findByHostIdAndTimestampLessThanEqual(long hostId, Date time); + + List findByHostIdAndTimestampBetween(long hostId, Date startTime, Date endTime); + + /** + * Expunges all host stats older than {@code limitDate}. + * @param limitPerQuery max rows removed per query; 0 or negative means no limit. + */ + void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery); + +} diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostStatsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostStatsDaoImpl.java new file mode 100644 index 000000000000..ac2211e24771 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostStatsDaoImpl.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host.dao; + +import java.util.Date; +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.stereotype.Component; + +import com.cloud.host.HostStatsVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; + +/** DAO for the host_stats table. */ +@Component +public class HostStatsDaoImpl extends GenericDaoBase implements HostStatsDao { + + protected Logger logger = LogManager.getLogger(getClass()); + + protected SearchBuilder hostIdSearch; + protected SearchBuilder hostIdTimestampGreaterThanEqualSearch; + protected SearchBuilder hostIdTimestampLessThanEqualSearch; + protected SearchBuilder hostIdTimestampBetweenSearch; + protected SearchBuilder timestampSearch; + + @PostConstruct + protected void init() { + hostIdSearch = createSearchBuilder(); + hostIdSearch.and("hostId", hostIdSearch.entity().getHostId(), Op.EQ); + hostIdSearch.done(); + + hostIdTimestampGreaterThanEqualSearch = createSearchBuilder(); + hostIdTimestampGreaterThanEqualSearch.and("hostId", hostIdTimestampGreaterThanEqualSearch.entity().getHostId(), Op.EQ); + hostIdTimestampGreaterThanEqualSearch.and("timestamp", hostIdTimestampGreaterThanEqualSearch.entity().getTimestamp(), Op.GTEQ); + hostIdTimestampGreaterThanEqualSearch.done(); + + hostIdTimestampLessThanEqualSearch = createSearchBuilder(); + hostIdTimestampLessThanEqualSearch.and("hostId", hostIdTimestampLessThanEqualSearch.entity().getHostId(), Op.EQ); + hostIdTimestampLessThanEqualSearch.and("timestamp", hostIdTimestampLessThanEqualSearch.entity().getTimestamp(), Op.LTEQ); + hostIdTimestampLessThanEqualSearch.done(); + + hostIdTimestampBetweenSearch = createSearchBuilder(); + hostIdTimestampBetweenSearch.and("hostId", hostIdTimestampBetweenSearch.entity().getHostId(), Op.EQ); + hostIdTimestampBetweenSearch.and("timestamp", hostIdTimestampBetweenSearch.entity().getTimestamp(), Op.BETWEEN); + hostIdTimestampBetweenSearch.done(); + + timestampSearch = createSearchBuilder(); + timestampSearch.and("timestamp", timestampSearch.entity().getTimestamp(), Op.LT); + timestampSearch.done(); + } + + @Override + public List findByHostId(long hostId) { + SearchCriteria sc = hostIdSearch.create(); + sc.setParameters("hostId", hostId); + return listBy(sc); + } + + @Override + public List findByHostIdAndTimestampGreaterThanEqual(long hostId, Date time) { + SearchCriteria sc = hostIdTimestampGreaterThanEqualSearch.create(); + sc.setParameters("hostId", hostId); + sc.setParameters("timestamp", time); + return listBy(sc); + } + + @Override + public List findByHostIdAndTimestampLessThanEqual(long hostId, Date time) { + SearchCriteria sc = hostIdTimestampLessThanEqualSearch.create(); + sc.setParameters("hostId", hostId); + sc.setParameters("timestamp", time); + return listBy(sc); + } + + @Override + public List findByHostIdAndTimestampBetween(long hostId, Date startTime, Date endTime) { + SearchCriteria sc = hostIdTimestampBetweenSearch.create(); + sc.setParameters("hostId", hostId); + sc.setParameters("timestamp", startTime, endTime); + return listBy(sc); + } + + @Override + public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) { + SearchCriteria sc = timestampSearch.create(); + sc.setParameters("timestamp", limitDate); + + logger.debug(String.format("Starting to remove all host_stats rows older than [%s].", limitDate)); + + long totalRemoved = batchExpunge(sc, limitPerQuery); + + logger.info(String.format("Removed a total of [%s] host_stats rows older than [%s].", totalRemoved, limitDate)); + } + +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 932db538f30b..adc7b4b49b9d 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -227,6 +227,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index 7c11013a17d2..773257f1e8af 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -18,3 +18,15 @@ --; -- Schema upgrade from 4.23.0.0 to 24.0.0 --; + +-- Add host_stats table for the host usage history +CREATE TABLE IF NOT EXISTS `cloud`.`host_stats` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `host_id` bigint unsigned NOT NULL, + `mgmt_server_id` bigint unsigned NOT NULL, + `timestamp` datetime NOT NULL, + `host_stats_data` text NOT NULL, + PRIMARY KEY (`id`), + KEY `i_host_stats__host_id` (`host_id`), + KEY `i_host_stats__timestamp` (`timestamp`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='historical per-host stats samples'; diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/api/ListHostsUsageHistoryCmd.java b/plugins/metrics/src/main/java/org/apache/cloudstack/api/ListHostsUsageHistoryCmd.java new file mode 100644 index 000000000000..b565ff04146a --- /dev/null +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/api/ListHostsUsageHistoryCmd.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api; + +import java.util.List; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.response.HostMetricsStatsResponse; + +@APICommand(name = "listHostsUsageHistory", description = "Lists host stats", responseObject = HostMetricsStatsResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "24.0.0", + authorized = {RoleType.Admin}) +public class ListHostsUsageHistoryCmd extends BaseResourceUsageHistoryCmd { + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = HostResponse.class, description = "The ID of the host.") + private Long id; + + @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = HostResponse.class, description = "The IDs of the hosts, mutually exclusive with id.") + private List ids; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "Name of the host (a substring match is made against the parameter value returning the data for all matching hosts).") + private String name; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public List getIds() { + return ids; + } + + public String getName() { + return name; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ListResponse response = metricsService.searchForHostMetricsStats(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsService.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsService.java index bb7763688385..d81ab2db37f1 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsService.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsService.java @@ -19,6 +19,7 @@ import java.util.List; +import org.apache.cloudstack.api.ListHostsUsageHistoryCmd; import org.apache.cloudstack.api.ListSystemVMsUsageHistoryCmd; import org.apache.cloudstack.api.ListVMsUsageHistoryCmd; import org.apache.cloudstack.api.ListVolumesUsageHistoryCmd; @@ -34,6 +35,7 @@ import org.apache.cloudstack.response.ClusterMetricsResponse; import org.apache.cloudstack.response.DbMetricsResponse; import org.apache.cloudstack.response.HostMetricsResponse; +import org.apache.cloudstack.response.HostMetricsStatsResponse; import org.apache.cloudstack.response.InfrastructureResponse; import org.apache.cloudstack.response.ManagementServerMetricsResponse; import org.apache.cloudstack.response.StoragePoolMetricsResponse; @@ -58,6 +60,7 @@ public interface MetricsService extends PluggableService { ListResponse searchForVmMetricsStats(ListVMsUsageHistoryCmd cmd); ListResponse searchForSystemVmMetricsStats(ListSystemVMsUsageHistoryCmd cmd); ListResponse searchForVolumeMetricsStats(ListVolumesUsageHistoryCmd cmd); + ListResponse searchForHostMetricsStats(ListHostsUsageHistoryCmd cmd); List listVolumeMetrics(List volumeResponses); List listVmMetrics(List vmResponses); List listStoragePoolMetrics(List poolResponses); diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index 0321d7c08d9e..194d6851b768 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -37,6 +37,7 @@ import org.apache.cloudstack.api.ListClustersMetricsCmd; import org.apache.cloudstack.api.ListDbMetricsCmd; import org.apache.cloudstack.api.ListHostsMetricsCmd; +import org.apache.cloudstack.api.ListHostsUsageHistoryCmd; import org.apache.cloudstack.api.ListInfrastructureCmd; import org.apache.cloudstack.api.ListMgmtsMetricsCmd; import org.apache.cloudstack.api.ListStoragePoolsMetricsCmd; @@ -67,6 +68,7 @@ import org.apache.cloudstack.response.ClusterMetricsResponse; import org.apache.cloudstack.response.DbMetricsResponse; import org.apache.cloudstack.response.HostMetricsResponse; +import org.apache.cloudstack.response.HostMetricsStatsResponse; import org.apache.cloudstack.response.HostMetricsSummary; import org.apache.cloudstack.response.InfrastructureResponse; import org.apache.cloudstack.response.ManagementServerMetricsResponse; @@ -88,6 +90,7 @@ import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; +import com.cloud.agent.api.HostStatsEntryBase; import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmStatsEntryBase; import com.cloud.alert.AlertManager; @@ -110,8 +113,11 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.HostStats; +import com.cloud.host.HostStatsVO; +import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostStatsDao; import com.cloud.network.router.VirtualRouter; import com.cloud.org.Cluster; import com.cloud.projects.Project; @@ -184,6 +190,8 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements private VolumeDao volumeDao; @Inject private VolumeStatsDao volumeStatsDao; + @Inject + protected HostStatsDao hostStatsDao; @Inject private ObjectStoreDao objectStoreDao; @@ -248,6 +256,144 @@ public ListResponse searchForVolumeMetricsStats(List return createVolumeMetricsStatsResponse(volumeList, volumeStatsList); } + /** + * Searches for host stats based on the {@code ListHostsUsageHistoryCmd} parameters. + * + * @param cmd the {@link ListHostsUsageHistoryCmd} specifying what should be searched. + * @return the list of host metrics stats found. + */ + @Override + public ListResponse searchForHostMetricsStats(ListHostsUsageHistoryCmd cmd) { + Pair, Integer> hostList = searchForHostsInternal(cmd); + Map> hostStatsList = searchForHostMetricsStatsInternal(cmd.getStartDate(), cmd.getEndDate(), hostList.first()); + return createHostMetricsStatsResponse(hostList, hostStatsList); + } + + /** + * Searches routing hosts based on {@code ListHostsUsageHistoryCmd} parameters. + * + * @param cmd the {@link ListHostsUsageHistoryCmd} specifying the parameters. + * @return the list of hosts and the total count. + */ + protected Pair, Integer> searchForHostsInternal(ListHostsUsageHistoryCmd cmd) { + Filter searchFilter = new Filter(HostVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + List ids = getIdsListFromCmd(cmd.getId(), cmd.getIds()); + String name = cmd.getName(); + String keyword = cmd.getKeyword(); + + SearchBuilder sb = hostDao.createSearchBuilder(); + sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ); + + SearchCriteria sc = sb.create(); + sc.setParameters("type", Host.Type.Routing); + if (CollectionUtils.isNotEmpty(ids)) { + sc.setParameters("idIN", ids.toArray()); + } + if (StringUtils.isNotBlank(name)) { + sc.setParameters("name", "%" + name + "%"); + } + if (StringUtils.isNotBlank(keyword)) { + SearchCriteria ssc = hostDao.createSearchCriteria(); + ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + sc.addAnd("name", SearchCriteria.Op.SC, ssc); + } + + return hostDao.searchAndCount(sc, searchFilter); + } + + /** + * Searches stats for a list of hosts, based on date filtering parameters. + * + * @param startDate the start date for which stats should be searched. + * @param endDate the end date for which stats should be searched. + * @param hostList the list of hosts for which stats should be searched. + * @return the key-value map in which keys are host IDs and values are lists of host stats. + */ + protected Map> searchForHostMetricsStatsInternal(Date startDate, Date endDate, List hostList) { + Map> hostStatsVOList = new HashMap<>(); + validateDateParams(startDate, endDate); + + for (HostVO hostVO : hostList) { + Long hostId = hostVO.getId(); + hostStatsVOList.put(hostId, findHostStatsAccordingToDateParams(hostId, startDate, endDate)); + } + + return hostStatsVOList; + } + + /** + * Finds stats for a specific host based on date parameters. + * + * @param hostId the specific host. + * @param startDate the start date to filtering. + * @param endDate the end date to filtering. + * @return the list of stats for the specified host. + */ + protected List findHostStatsAccordingToDateParams(Long hostId, Date startDate, Date endDate) { + if (startDate != null && endDate != null) { + return hostStatsDao.findByHostIdAndTimestampBetween(hostId, startDate, endDate); + } + if (startDate != null) { + return hostStatsDao.findByHostIdAndTimestampGreaterThanEqual(hostId, startDate); + } + if (endDate != null) { + return hostStatsDao.findByHostIdAndTimestampLessThanEqual(hostId, endDate); + } + return hostStatsDao.findByHostId(hostId); + } + + /** + * Creates a {@code ListResponse}. For each host, this joins essential host info + * with its respective list of stats. + * + * @param hostList the list of hosts and the total count. + * @param hostStatsList the respective list of stats. + * @return the list of responses that was created. + */ + protected ListResponse createHostMetricsStatsResponse(Pair, Integer> hostList, + Map> hostStatsList) { + List responses = new ArrayList<>(); + for (HostVO hostVO : hostList.first()) { + HostMetricsStatsResponse hostMetricsStatsResponse = new HostMetricsStatsResponse(); + hostMetricsStatsResponse.setObjectName("host"); + hostMetricsStatsResponse.setId(hostVO.getUuid()); + hostMetricsStatsResponse.setName(hostVO.getName()); + hostMetricsStatsResponse.setStats(createHostStatsResponse(hostStatsList.get(hostVO.getId()))); + responses.add(hostMetricsStatsResponse); + } + + ListResponse response = new ListResponse<>(); + response.setResponses(responses, hostList.second()); + return response; + } + + /** + * Creates a {@code List} from a given {@code List}. + * + * @param hostStatsList the list of host stats. + * @return the list of responses that was created. + */ + protected List createHostStatsResponse(List hostStatsList) { + List statsResponseList = new ArrayList<>(); + DecimalFormat decimalFormat = new DecimalFormat("#.##"); + for (HostStatsVO hostStats : hostStatsList) { + StatsResponse response = new StatsResponse(); + response.setTimestamp(hostStats.getTimestamp()); + + HostStatsEntryBase statsEntry = gson.fromJson(hostStats.getHostStatsData(), HostStatsEntryBase.class); + response.setCpuUsed(decimalFormat.format(statsEntry.getCpuUtilization()) + "%"); + response.setNetworkKbsRead((long) statsEntry.getNetworkReadKBs()); + response.setNetworkKbsWrite((long) statsEntry.getNetworkWriteKBs()); + response.setMemoryKBs((long) statsEntry.getTotalMemoryKBs()); + response.setMemoryIntFreeKBs((long) statsEntry.getFreeMemoryKBs()); + + statsResponseList.add(response); + } + return statsResponseList; + } + /** * Outputs the parameters that should be used for access control in the query of a resource to * {@code permittedAccounts} and {@code domainIdRecursiveListProject}. @@ -1194,6 +1340,7 @@ public List> getCommands() { cmdList.add(ListVMsUsageHistoryCmd.class); cmdList.add(ListSystemVMsUsageHistoryCmd.class); cmdList.add(ListVolumesUsageHistoryCmd.class); + cmdList.add(ListHostsUsageHistoryCmd.class); // separate Admin commands cmdList.add(ListVMsMetricsCmdByAdmin.class); return cmdList; diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/response/HostMetricsStatsResponse.java b/plugins/metrics/src/main/java/org/apache/cloudstack/response/HostMetricsStatsResponse.java new file mode 100644 index 000000000000..2b3ba89929f5 --- /dev/null +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/response/HostMetricsStatsResponse.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.response; + +import java.util.List; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.response.StatsResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class HostMetricsStatsResponse extends BaseResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "The ID of the host") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "The name of the host") + private String name; + + @SerializedName("stats") + @Param(description = "The list of host stats", responseObject = StatsResponse.class) + private List stats; + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setStats(List stats) { + this.stats = stats; + } +} diff --git a/plugins/metrics/src/test/java/org/apache/cloudstack/metrics/MetricsServiceImplTest.java b/plugins/metrics/src/test/java/org/apache/cloudstack/metrics/MetricsServiceImplTest.java index 9184d7444105..f3d888814047 100644 --- a/plugins/metrics/src/test/java/org/apache/cloudstack/metrics/MetricsServiceImplTest.java +++ b/plugins/metrics/src/test/java/org/apache/cloudstack/metrics/MetricsServiceImplTest.java @@ -28,7 +28,9 @@ import org.apache.cloudstack.api.ListVMsUsageHistoryCmd; import org.apache.cloudstack.api.ListVolumesUsageHistoryCmd; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StatsResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.response.HostMetricsStatsResponse; import org.apache.cloudstack.response.VmMetricsStatsResponse; import org.apache.commons.lang3.time.DateUtils; import org.junit.Assert; @@ -43,7 +45,11 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.agent.api.HostStatsEntryBase; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.host.HostStatsVO; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostStatsDao; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; @@ -57,6 +63,8 @@ import com.cloud.vm.VmStatsVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VmStatsDao; +import com.google.gson.Gson; +import com.google.gson.JsonObject; @RunWith(MockitoJUnitRunner.class) @@ -122,6 +130,10 @@ public class MetricsServiceImplTest { SearchCriteria volumeSearchCriteriaMock; @Mock Filter filterMock; + @Mock + HostStatsDao hostStatsDaoMock; + @Mock + HostVO hostVOMock; private void prepareSearchCriteriaWhenUseSetParameters() { @@ -330,6 +342,104 @@ public void findVmStatsAccordingToDateParamsTestWithNoDate() { Mockito.verify(vmStatsDaoMock).findByVmId(Mockito.anyLong()); } + @Test + public void searchForHostMetricsStatsInternalTestWithAPopulatedListOfHosts() { + Mockito.doReturn(new ArrayList()).when(spy).findHostStatsAccordingToDateParams( + Mockito.anyLong(), Mockito.any(), Mockito.any()); + Mockito.doReturn(1L).when(hostVOMock).getId(); + Map> expected = new HashMap<>(); + expected.put(1L, new ArrayList<>()); + + Map> result = spy.searchForHostMetricsStatsInternal(null, null, Arrays.asList(hostVOMock)); + + Mockito.verify(spy).findHostStatsAccordingToDateParams(1L, null, null); + Assert.assertEquals(expected, result); + } + + @Test + public void searchForHostMetricsStatsInternalTestWithAnEmptyListOfHosts() { + Map> result = spy.searchForHostMetricsStatsInternal(null, null, new ArrayList<>()); + + Mockito.verify(spy, Mockito.never()).findHostStatsAccordingToDateParams( + Mockito.anyLong(), Mockito.any(), Mockito.any()); + Assert.assertTrue(result.isEmpty()); + } + + @Test(expected = InvalidParameterValueException.class) + public void searchForHostMetricsStatsInternalTestWithEndDateBeforeStartDate() { + Date startDate = new Date(); + + spy.searchForHostMetricsStatsInternal(startDate, DateUtils.addSeconds(startDate, -1), Arrays.asList(hostVOMock)); + } + + @Test + public void findHostStatsAccordingToDateParamsTestWithStartDateAndEndDate() { + Date startDate = new Date(); + Date endDate = DateUtils.addSeconds(startDate, 1); + + spy.findHostStatsAccordingToDateParams(1L, startDate, endDate); + + Mockito.verify(hostStatsDaoMock).findByHostIdAndTimestampBetween(1L, startDate, endDate); + } + + @Test + public void findHostStatsAccordingToDateParamsTestWithOnlyStartDate() { + Date startDate = new Date(); + + spy.findHostStatsAccordingToDateParams(1L, startDate, null); + + Mockito.verify(hostStatsDaoMock).findByHostIdAndTimestampGreaterThanEqual(1L, startDate); + } + + @Test + public void findHostStatsAccordingToDateParamsTestWithOnlyEndDate() { + Date endDate = new Date(); + + spy.findHostStatsAccordingToDateParams(1L, null, endDate); + + Mockito.verify(hostStatsDaoMock).findByHostIdAndTimestampLessThanEqual(1L, endDate); + } + + @Test + public void findHostStatsAccordingToDateParamsTestWithNoDate() { + spy.findHostStatsAccordingToDateParams(1L, null, null); + + Mockito.verify(hostStatsDaoMock).findByHostId(1L); + } + + @Test + public void createHostMetricsStatsResponseTestWithValidInput() { + Mockito.doReturn(1L).when(hostVOMock).getId(); + Mockito.doReturn("host-uuid").when(hostVOMock).getUuid(); + Mockito.doReturn("host-name").when(hostVOMock).getName(); + Map> statsMap = new HashMap<>(); + statsMap.put(1L, new ArrayList<>()); + + ListResponse result = spy.createHostMetricsStatsResponse( + new Pair<>(Arrays.asList(hostVOMock), 5), statsMap); + + Assert.assertEquals(Integer.valueOf(5), result.getCount()); + Assert.assertEquals(1, result.getResponses().size()); + } + + @Test + public void createHostStatsResponseTestMapsTheStoredValues() { + Date timestamp = new Date(); + HostStatsEntryBase entry = new HostStatsEntryBase(1L, "host", 12.345, 0.5, 100.0, 200.0, 4096.0, 1024.0); + HostStatsVO hostStatsVO = new HostStatsVO(1L, 2L, timestamp, new Gson().toJson(entry)); + + List result = spy.createHostStatsResponse(Arrays.asList(hostStatsVO)); + + Assert.assertEquals(1, result.size()); + JsonObject response = new Gson().toJsonTree(result.get(0)).getAsJsonObject(); + Assert.assertTrue(response.has("timestamp")); + Assert.assertEquals("12.35%", response.get("cpuused").getAsString()); + Assert.assertEquals(100L, response.get("networkkbsread").getAsLong()); + Assert.assertEquals(200L, response.get("networkkbswrite").getAsLong()); + Assert.assertEquals(4096L, response.get("memorykbs").getAsLong()); + Assert.assertEquals(1024L, response.get("memoryintfreekbs").getAsLong()); + } + @Test public void createVmMetricsStatsResponseTestWithValidInput() { Mockito.doReturn("").when(userVmVOMock).getUuid(); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 456792d14b75..787205ed502e 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -84,6 +84,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.GetStorageStatsCommand; import com.cloud.agent.api.HostStatsEntry; +import com.cloud.agent.api.HostStatsEntryBase; import com.cloud.agent.api.VgpuTypesInfo; import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmNetworkStatsEntry; @@ -110,7 +111,9 @@ import com.cloud.host.HostStats; import com.cloud.host.HostVO; import com.cloud.host.Status; +import com.cloud.host.HostStatsVO; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostStatsDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.as.AutoScaleManager; @@ -301,6 +304,9 @@ public String toString() { protected static ConfigKey vmDiskStatsMaxRetentionTime = new ConfigKey<>("Advanced", Integer.class, "vm.disk.stats.max.retention.time", "720", "The maximum time (in minutes) for keeping VM disks stats records in the database. The VM disks stats cleanup process will be disabled if this is set to 0 or less than 0.", true); + protected static ConfigKey hostStatsMaxRetentionTime = new ConfigKey<>("Advanced", Integer.class, "host.stats.max.retention.time", "720", + "The maximum time (in minutes) for keeping host stats records in the database. Host stats are not stored and the cleanup process is disabled if this is set to 0 or less than 0.", true); + private static StatsCollector s_instance = null; private static Gson gson = new Gson(); @@ -322,6 +328,8 @@ public String toString() { @Inject protected VmStatsDao vmStatsDao; @Inject + protected HostStatsDao hostStatsDao; + @Inject private VolumeDao _volsDao; @Inject protected VolumeStatsDao volumeStatsDao; @@ -510,6 +518,8 @@ protected void init(Map configs) { _executor.scheduleWithFixedDelay(new VmStatsCleaner(), DEFAULT_INITIAL_DELAY, 60000L, TimeUnit.MILLISECONDS); + _executor.scheduleWithFixedDelay(new HostStatsCleaner(), DEFAULT_INITIAL_DELAY, 60000L, TimeUnit.MILLISECONDS); + _executor.scheduleWithFixedDelay(new VolumeStatsCleaner(), DEFAULT_INITIAL_DELAY, 60000L, TimeUnit.MILLISECONDS); scheduleCollection(MANAGEMENT_SERVER_STATUS_COLLECTION_INTERVAL, new ManagementServerCollector(), 1L); @@ -676,12 +686,17 @@ protected void runInContext() { logger.debug(String.format("HostStatsCollector is running to process %d UP hosts", hosts.size())); Map metrics = new HashMap<>(); + boolean persistHostStats = hostStatsMaxRetentionTime.value() > 0; + Date timestamp = new Date(); for (HostVO host : hosts) { HostStatsEntry hostStatsEntry = (HostStatsEntry) _resourceMgr.getHostStatistics(host); if (hostStatsEntry != null) { hostStatsEntry.setHostVo(host); metrics.put(hostStatsEntry.getHostId(), hostStatsEntry); _hostStats.put(host.getId(), hostStatsEntry); + if (persistHostStats) { + persistHostStats(hostStatsEntry, timestamp); + } } else { logger.warn("The Host stats is null for host: {}", host); } @@ -1316,6 +1331,17 @@ protected void runInContext() { } } + class HostStatsCleaner extends ManagedContextRunnable{ + @Override + protected void runInContext() { + try { + cleanUpHostStats(); + } catch (RuntimeException e) { + logger.error("Error trying to clean up host stats", e); + } + } + } + class VolumeStatsCleaner extends ManagedContextRunnable{ @Override protected void runInContext() { @@ -2001,6 +2027,23 @@ protected void persistVirtualMachineStats(VmStatsEntry statsForCurrentIteration, vmStatsDao.persist(vmStatsVO); } + /** + * Persists the host stats of the current collection in the host_stats table. + * + * @param statsForCurrentIteration the host metrics to persist. + * @param timestamp the time that will be stamped. + */ + protected void persistHostStats(HostStatsEntry statsForCurrentIteration, Date timestamp) { + HostStatsEntryBase hostStats = new HostStatsEntryBase(statsForCurrentIteration.getHostId(), + statsForCurrentIteration.getEntityType(), statsForCurrentIteration.getCpuUtilization(), + statsForCurrentIteration.getLoadAverage(), statsForCurrentIteration.getNetworkReadKBs(), + statsForCurrentIteration.getNetworkWriteKBs(), statsForCurrentIteration.getTotalMemoryKBs(), + statsForCurrentIteration.getFreeMemoryKBs()); + HostStatsVO hostStatsVO = new HostStatsVO(statsForCurrentIteration.getHostId(), msId, timestamp, gson.toJson(hostStats)); + logger.trace(String.format("Recording host stats: [%s].", hostStatsVO.toString())); + hostStatsDao.persist(hostStatsVO); + } + private String getVmDiskStatsEntryAsString(VmDiskStatsEntry statsForCurrentIteration, Hypervisor.HypervisorType hypervisorType) { VmDiskStatsEntry entry; if (Hypervisor.HypervisorType.KVM.equals(hypervisorType)) { @@ -2051,6 +2094,23 @@ protected void cleanUpVirtualMachineStats() { vmStatsDao.removeAllByTimestampLessThan(limit, DELETE_QUERY_BATCH_SIZE.value()); } + /** + * Removes the oldest host stats records according to the global + * parameter {@code host.stats.max.retention.time}. + */ + protected void cleanUpHostStats() { + Integer maxRetentionTime = hostStatsMaxRetentionTime.value(); + if (maxRetentionTime <= 0) { + logger.debug(String.format("Skipping host stats cleanup. The [%s] parameter [%s] is set to 0 or less than 0.", + ConfigKey.Scope.decodeAsCsv(hostStatsMaxRetentionTime.getScopeBitmask()), hostStatsMaxRetentionTime.toString())); + return; + } + logger.trace("Removing older host stats records."); + Date now = new Date(); + Date limit = DateUtils.addMinutes(now, -maxRetentionTime); + hostStatsDao.removeAllByTimestampLessThan(limit, DELETE_QUERY_BATCH_SIZE.value()); + } + /** * Removes the oldest Volume stats records according to the global * parameter {@code vm.disk.stats.max.retention.time}. @@ -2245,6 +2305,7 @@ public String getConfigComponentName() { public ConfigKey[] getConfigKeys() { return new ConfigKey[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri, vmStatsIncrementMetrics, vmStatsMaxRetentionTime, vmStatsCollectUserVMOnly, vmDiskStatsRetentionEnabled, vmDiskStatsMaxRetentionTime, + hostStatsMaxRetentionTime, MANAGEMENT_SERVER_STATUS_COLLECTION_INTERVAL, DATABASE_SERVER_STATUS_COLLECTION_INTERVAL, DATABASE_SERVER_LOAD_HISTORY_RETENTION_NUMBER}; diff --git a/server/src/test/java/com/cloud/server/StatsCollectorTest.java b/server/src/test/java/com/cloud/server/StatsCollectorTest.java index cb00d1652c9a..e10552e1208e 100644 --- a/server/src/test/java/com/cloud/server/StatsCollectorTest.java +++ b/server/src/test/java/com/cloud/server/StatsCollectorTest.java @@ -64,11 +64,15 @@ import com.cloud.agent.api.GetStorageStatsAnswer; import com.cloud.agent.api.GetStorageStatsCommand; +import com.cloud.agent.api.HostStatsEntry; +import com.cloud.agent.api.HostStatsEntryBase; import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmStatsEntry; import com.cloud.dc.Vlan.VlanType; import com.cloud.dc.VlanVO; import com.cloud.dc.dao.VlanDao; +import com.cloud.host.HostStatsVO; +import com.cloud.host.dao.HostStatsDaoImpl; import com.cloud.hypervisor.Hypervisor; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; @@ -101,6 +105,12 @@ public class StatsCollectorTest { @Mock VmStatsDaoImpl vmStatsDaoMock; + @Mock + HostStatsDaoImpl hostStatsDaoMock; + + @Captor + ArgumentCaptor hostStatsVOCaptor = ArgumentCaptor.forClass(HostStatsVO.class); + @Mock VmStatsEntry statsForCurrentIterationMock; @@ -146,6 +156,7 @@ public class StatsCollectorTest { public void setUp() throws Exception { closeable = MockitoAnnotations.openMocks(this); statsCollector.vmStatsDao = vmStatsDaoMock; + statsCollector.hostStatsDao = hostStatsDaoMock; statsCollector.volumeStatsDao = volumeStatsDao; Field msStatsGsonField = StatsCollector.class.getDeclaredField("msStatsGson"); msStatsGsonField.setAccessible(true); @@ -394,6 +405,59 @@ public void volumeStatsCleanerTestCatchesCloudRuntimeExceptionAndKeepsRunning() Mockito.verify(statsCollector).cleanUpVolumeStats(); } + // host stats persistence + retention + + private void setHostStatsMaxRetentionTimeValue(String value) { + StatsCollector.hostStatsMaxRetentionTime = new ConfigKey("Advanced", Integer.class, "host.stats.max.retention.time", value, + "The maximum time (in minutes) for keeping host stats records in the database. The host stats cleanup process will be disabled if this is set to 0 or less than 0.", true); + } + + @Test + public void cleanUpHostStatsTestIsDisabled() { + setHostStatsMaxRetentionTimeValue("0"); + + statsCollector.cleanUpHostStats(); + + Mockito.verify(hostStatsDaoMock, Mockito.never()).removeAllByTimestampLessThan(Mockito.any(), Mockito.anyLong()); + } + + @Test + public void cleanUpHostStatsTestIsEnabled() { + setHostStatsMaxRetentionTimeValue("1"); + + statsCollector.cleanUpHostStats(); + + Mockito.verify(hostStatsDaoMock).removeAllByTimestampLessThan(Mockito.any(), Mockito.anyLong()); + } + + @Test + public void persistHostStatsTestPersistsSuccessfully() { + statsCollector.msId = 7L; + Date timestamp = new Date(); + // hostId, cpuUtilization, networkReadKBs, networkWriteKBs, entityType, totalMemoryKBs, freeMemoryKBs, xapiMemoryUsageKBs, averageLoad + HostStatsEntry statsForCurrentIteration = new HostStatsEntry(5L, 10.0, 20.0, 30.0, "host", 1000.0, 400.0, 0.0, 2.0); + Mockito.doReturn(new HostStatsVO()).when(hostStatsDaoMock).persist(Mockito.any()); + + statsCollector.persistHostStats(statsForCurrentIteration, timestamp); + + Mockito.verify(hostStatsDaoMock).persist(hostStatsVOCaptor.capture()); + HostStatsVO actual = hostStatsVOCaptor.getValue(); + Assert.assertEquals(Long.valueOf(5L), actual.getHostId()); + Assert.assertEquals(Long.valueOf(7L), actual.getMgmtServerId()); + Assert.assertEquals(timestamp, actual.getTimestamp()); + HostStatsEntryBase persisted = gson.fromJson(actual.getHostStatsData(), HostStatsEntryBase.class); + Assert.assertEquals(5L, persisted.getHostId()); + Assert.assertEquals("host", persisted.getEntityType()); + Assert.assertEquals(10.0, persisted.getCpuUtilization(), 0); + Assert.assertEquals(2.0, persisted.getLoadAverage(), 0); + Assert.assertEquals(20.0, persisted.getNetworkReadKBs(), 0); + Assert.assertEquals(30.0, persisted.getNetworkWriteKBs(), 0); + Assert.assertEquals(1000.0, persisted.getTotalMemoryKBs(), 0); + Assert.assertEquals(400.0, persisted.getFreeMemoryKBs(), 0); + // Lean payload must NOT carry a HostVO blob. + Assert.assertFalse(actual.getHostStatsData().contains("hostVo")); + } + @Test public void persistVirtualMachineStatsTestPersistsSuccessfully() { statsCollector.msId = 1L; diff --git a/test/integration/smoke/test_metrics_api.py b/test/integration/smoke/test_metrics_api.py index ab2644fc1aad..85f8e122bc2f 100644 --- a/test/integration/smoke/test_metrics_api.py +++ b/test/integration/smoke/test_metrics_api.py @@ -547,6 +547,29 @@ def test_list_volumes_metrics_history(self): return + @attr(tags = ["advanced", "advancedns", "smoke", "basic"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_list_hosts_metrics_history(self): + cmd = listHostsUsageHistory.listHostsUsageHistoryCmd() + now = datetime.datetime.now() - datetime.timedelta(minutes=15) + start_time = now.strftime("%Y-%m-%d %H:%M:%S") + cmd.startdate = start_time + + result = self.apiclient.listHostsUsageHistory(cmd)[0] + + self.assertTrue(hasattr(result, 'stats')) + self.assertTrue(type(result.stats) == list and len(result.stats) > 0) + stats = result.stats[0] + self.assertTrue(hasattr(stats, 'cpuused')) + self.assertTrue(hasattr(stats, 'memorykbs')) + self.assertTrue(hasattr(stats, 'memoryintfreekbs')) + self.assertTrue(hasattr(stats, 'networkkbsread')) + self.assertTrue(hasattr(stats, 'networkkbswrite')) + self.assertTrue(hasattr(stats, 'timestamp')) + self.assertTrue(self.valid_date(stats.timestamp)) + + return + def validate_vm_stats(self, stats): self.assertTrue(hasattr(stats, 'cpuused')) self.assertTrue(hasattr(stats, 'diskiopstotal'))