<fix>[vm]: add MetadataImpact

compute/src/main/java/org/zstack/compute/vm/CleanupVmInstanceMetadataOnPrimaryStorageGC.java

@@ -0,0 +1,64 @@
+package org.zstack.compute.vm;
+
+import org.zstack.core.cloudbus.CloudBusCallBack;
+import org.zstack.core.gc.GC;
+import org.zstack.core.gc.GCCompletion;
+import org.zstack.core.gc.TimeBasedGarbageCollector;
+import org.zstack.header.message.MessageReply;
+import org.zstack.header.storage.primary.CleanupVmInstanceMetadataOnPrimaryStorageMsg;
+import org.zstack.header.storage.primary.PrimaryStorageConstant;
+import org.zstack.header.storage.primary.PrimaryStorageVO;
+import org.zstack.utils.Utils;
+import org.zstack.utils.logging.CLogger;
+
+public class CleanupVmInstanceMetadataOnPrimaryStorageGC extends TimeBasedGarbageCollector {
+    private static final CLogger logger = Utils.getLogger(CleanupVmInstanceMetadataOnPrimaryStorageGC.class);
+
+    @GC
+    public String primaryStorageUuid;
+    @GC
+    public String vmUuid;
+    @GC
+    public String rootVolumeUuid;
+    @GC
+    public String metadataPath;
+    @GC
+    public String hostUuid;
+
+    public static String getGCName(String vmUuid) {
+        return String.format("gc-cleanup-vm-metadata-%s", vmUuid);
+    }
+
+    @Override
+    protected void triggerNow(GCCompletion completion) {
+        if (!dbf.isExist(primaryStorageUuid, PrimaryStorageVO.class)) {
+            logger.debug(String.format("[MetadataCleanupGC] primary storage[uuid:%s] no longer exists, " +
+                    "cancel gc for vm[uuid:%s]", primaryStorageUuid, vmUuid));
+            completion.cancel();
+            return;
+        }
+
+        CleanupVmInstanceMetadataOnPrimaryStorageMsg msg = new CleanupVmInstanceMetadataOnPrimaryStorageMsg();
+        msg.setPrimaryStorageUuid(primaryStorageUuid);
+        msg.setVmUuid(vmUuid);
+        msg.setRootVolumeUuid(rootVolumeUuid);
+        msg.setMetadataPath(metadataPath);
+        msg.setHostUuid(hostUuid);
+
+        bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, primaryStorageUuid);
+        bus.send(msg, new CloudBusCallBack(completion) {
+            @Override
+            public void run(MessageReply reply) {
+                if (reply.isSuccess()) {
+                    logger.info(String.format("[MetadataCleanupGC] successfully cleaned up metadata " +
+                            "for vm[uuid:%s] on ps[uuid:%s]", vmUuid, primaryStorageUuid));
+                    completion.success();
+                } else {
+                    logger.warn(String.format("[MetadataCleanupGC] failed to clean up metadata " +
+                            "for vm[uuid:%s] on ps[uuid:%s]: %s", vmUuid, primaryStorageUuid, reply.getError()));
+                    completion.fail(reply.getError());
+                }
+            }
+        });
+    }
+}

compute/src/main/java/org/zstack/compute/vm/VmExpungeMetadataFlow.java

@@ -0,0 +1,118 @@
+package org.zstack.compute.vm;
+
+import org.springframework.beans.factory.annotation.Autowire;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Configurable;
+import org.zstack.core.cloudbus.CloudBus;
+import org.zstack.core.cloudbus.CloudBusCallBack;
+import org.zstack.core.componentloader.PluginRegistry;
+import org.zstack.core.db.Q;
+import org.zstack.header.core.workflow.FlowTrigger;
+import org.zstack.header.core.workflow.NoRollbackFlow;
+import org.zstack.header.message.MessageReply;
+import org.zstack.header.storage.primary.CleanupVmInstanceMetadataOnPrimaryStorageMsg;
+import org.zstack.header.storage.primary.PrimaryStorageConstant;
+import org.zstack.header.storage.primary.PrimaryStorageVO;
+import org.zstack.header.storage.primary.PrimaryStorageVO_;
+import org.zstack.header.vm.VmInstanceConstant;
+import org.zstack.header.vm.VmInstanceSpec;
+import org.zstack.header.vm.metadata.VmMetadataPathBuildExtensionPoint;
+import org.zstack.header.volume.VolumeInventory;
+import org.zstack.utils.Utils;
+import org.zstack.utils.logging.CLogger;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
+public class VmExpungeMetadataFlow extends NoRollbackFlow {
+    private static final CLogger logger = Utils.getLogger(VmExpungeMetadataFlow.class);
+
+    @Autowired
+    private CloudBus bus;
+    @Autowired
+    private PluginRegistry pluginRgty;
+
+    @Override
+    public void run(FlowTrigger trigger, Map data) {
+        final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString());
+        if (spec == null || spec.getVmInventory() == null) {
+            logger.warn("[MetadataExpunge] missing VmInstanceSpec or VmInventory, skip metadata cleanup");
+            trigger.next();
+            return;
+        }
+
+        final String vmUuid = spec.getVmInventory().getUuid();
+
+        VolumeInventory rootVolume = spec.getVmInventory().getRootVolume();
+        String psUuid = rootVolume != null ? rootVolume.getPrimaryStorageUuid() : null;
+        if (psUuid == null) {
+            logger.debug(String.format("[MetadataExpunge] vm[uuid:%s] root volume has no primaryStorageUuid, " +
+                    "skipping metadata cleanup", vmUuid));
+            trigger.next();
+            return;
+        }
+
+
+        String psType = Q.New(PrimaryStorageVO.class).select(PrimaryStorageVO_.type).eq(PrimaryStorageVO_.uuid, psUuid).findValue();
+        VmMetadataPathBuildExtensionPoint ext = pluginRgty.getExtensionFromMap(psType, VmMetadataPathBuildExtensionPoint.class);
+        if (ext == null) {
+            trigger.next();
+            return;
+        }
+        final String metadataPath;
+        try {
+            metadataPath = ext.buildVmMetadataPath(psUuid, vmUuid);
+        } catch (Exception e) {
+            logger.warn(String.format("[MetadataExpunge] failed to build metadata path for vm[uuid:%s] on ps[uuid:%s], " +
+                    "skip metadata cleanup: %s", vmUuid, psUuid, e.getMessage()));
+            trigger.next();
+            return;
+        }
+
+        String rootVolumeUuid = rootVolume.getUuid();
+        CleanupVmInstanceMetadataOnPrimaryStorageMsg cmsg = new CleanupVmInstanceMetadataOnPrimaryStorageMsg();
+        cmsg.setPrimaryStorageUuid(psUuid);
+        cmsg.setVmUuid(vmUuid);
+        cmsg.setMetadataPath(metadataPath);
+        cmsg.setRootVolumeUuid(rootVolumeUuid);
+
+        String hostUuid = spec.getVmInventory().getHostUuid();
+        if (hostUuid == null) {
+            hostUuid = spec.getVmInventory().getLastHostUuid();
+        }
+        cmsg.setHostUuid(hostUuid);
+
+        final String finalPsUuid = psUuid;
+        final String finalHostUuid = hostUuid;
+
+        bus.makeTargetServiceIdByResourceUuid(cmsg, PrimaryStorageConstant.SERVICE_ID, psUuid);
+        bus.send(cmsg, new CloudBusCallBack(trigger) {
+            @Override
+            public void run(MessageReply reply) {
+                if (reply.isSuccess()) {
+                    logger.info(String.format("[MetadataExpunge] successfully deleted metadata for vm[uuid:%s] on ps[uuid:%s]",
+                            vmUuid, finalPsUuid));
+                } else {
+                    logger.warn(String.format("[MetadataExpunge] failed to delete metadata for vm[uuid:%s] on ps[uuid:%s]: %s, " +
+                            "submitting GC job for retry", vmUuid, finalPsUuid, reply.getError()));
+                    submitGC(finalPsUuid, vmUuid, rootVolumeUuid, metadataPath, finalHostUuid);
+                }
+                trigger.next();
+            }
+        });
+    }
+
+    private void submitGC(String psUuid, String vmUuid, String rootVolumeUuid, String metadataPath, String hostUuid) {
+        CleanupVmInstanceMetadataOnPrimaryStorageGC gc = new CleanupVmInstanceMetadataOnPrimaryStorageGC();
+        gc.NAME = CleanupVmInstanceMetadataOnPrimaryStorageGC.getGCName(vmUuid);
+        gc.primaryStorageUuid = psUuid;
+        gc.vmUuid = vmUuid;
+        gc.rootVolumeUuid = rootVolumeUuid;
+        gc.metadataPath = metadataPath;
+        gc.hostUuid = hostUuid;
+        gc.deduplicateSubmit(TimeUnit.HOURS.toSeconds(8), TimeUnit.SECONDS);
+
+        logger.info(String.format("[MetadataExpunge] submitted GC job [%s] for vm[uuid:%s] on ps[uuid:%s]", gc.NAME, vmUuid, psUuid));
+    }
+}

compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java

@@ -133,4 +133,9 @@ public class VmGlobalConfig {
     @GlobalConfigValidation(validValues = {"None", "AuthenticAMD"})
     @BindResourceConfig(value = {VmInstanceVO.class})
     public static GlobalConfig VM_CPUID_VENDOR = new GlobalConfig(CATEGORY, "vm.cpuid.vendor");
+
+    @GlobalConfigValidation(validValues = {"true", "false"})
+    public static GlobalConfig VM_METADATA_ENABLED = new GlobalConfig(CATEGORY, "vm.metadata.enabled");
+
+    public static GlobalConfig VM_METADATA_LAST_REFRESH_VERSION = new GlobalConfig(CATEGORY, "vm.metadata.lastRefreshVersion");
 }

compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java

@@ -29,8 +29,10 @@
 import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO_;
 import org.zstack.header.vm.*;
 import org.zstack.header.vm.cdrom.*;
+import org.zstack.header.vm.metadata.APIRegisterVmInstanceFromMetadataMsg;
 import org.zstack.header.vm.devices.VmInstanceResourceMetadataGroupVO;
 import org.zstack.header.vm.devices.VmInstanceResourceMetadataGroupVO_;
+import org.zstack.header.vm.metadata.VmInstanceMetadataConstants;
 import org.zstack.header.volume.*;
 import org.zstack.network.l2.L2NetworkHostUtils;
 import org.zstack.resourceconfig.ResourceConfigFacade;
@@ -166,6 +168,8 @@ else if (msg instanceof APIAttachVmNicToVmMsg) {
             validate((APIConvertTemplatedVmInstanceToVmInstanceMsg) msg);
         } else if (msg instanceof APIDeleteTemplatedVmInstanceMsg) {
             validate((APIDeleteTemplatedVmInstanceMsg) msg);
+        } else if (msg instanceof APIRegisterVmInstanceFromMetadataMsg) {
+            validate((APIRegisterVmInstanceFromMetadataMsg) msg);
         }
 
         if (msg instanceof NewVmInstanceMessage2) {
@@ -1318,4 +1322,42 @@ private void validate(APIFstrimVmMsg msg) {
         }
         msg.setHostUuid(t.get(1, String.class));
     }
+
+    private void validate(APIRegisterVmInstanceFromMetadataMsg msg) {
+        String path = msg.getMetadataPath();
+        if (path == null || path.isEmpty()) {
+            throw new ApiMessageInterceptionException(argerr("metadataPath cannot be empty"));
+        }
+
+        validateMetadataPath(path);
+
+        if (!(path.endsWith(VmInstanceMetadataConstants.FILE_METADATA_SUFFIX) || path.endsWith(VmInstanceMetadataConstants.SBLK_METADATA_LV_SUFFIX))) {
+            throw new ApiMessageInterceptionException(argerr("metadataPath must end with %s or %s",
+                    VmInstanceMetadataConstants.FILE_METADATA_SUFFIX, VmInstanceMetadataConstants.SBLK_METADATA_LV_SUFFIX));
+        }
+    }
+
+    private void validateMetadataPath(String path) {
+        if (!path.startsWith("/")) {
+            throw new ApiMessageInterceptionException(argerr("metadataPath must be an absolute path starting with '/'"));
+        }
+
+        if (path.contains("..")) {
+            throw new ApiMessageInterceptionException(argerr(
+                    "metadataPath contains illegal '..' sequence, path traversal is not allowed"));
+        }
+
+        if (path.contains("//")) {
+            throw new ApiMessageInterceptionException(argerr(
+                    "metadataPath contains illegal '//' sequence"));
+        }
+
+        // Only allow safe characters: alphanumeric, hyphen, underscore, period, forward slash
+        String safePattern = "^[a-zA-Z0-9_\\-./]+$";
+        if (!path.matches(safePattern)) {
+            throw new ApiMessageInterceptionException(argerr(
+                    "metadataPath contains illegal characters, only alphanumeric characters, " +
+                            "hyphen (-), underscore (_), period (.), and forward slash (/) are allowed"));
+        }
+    }
 }

compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java

@@ -307,4 +307,8 @@ public String desensitizeTag(SystemTag systemTag, String tag) {
     }
 
     public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class);
+
+    public static String VM_METADATA_REGISTERING_MN_UUID_TOKEN = "registeringMnUuid";
+    public static PatternedSystemTag VM_METADATA_REGISTERING_MN_UUID = new PatternedSystemTag(
+            String.format("vmMetadata::registeringMnUuid::{%s}", VM_METADATA_REGISTERING_MN_UUID_TOKEN), VmInstanceVO.class);
 }

conf/db/upgrade/V5.0.0__schema.sql

@@ -0,0 +1,24 @@
+CREATE TABLE IF NOT EXISTS `zstack`.`VmMetadataDirtyVO` (
+    `vmInstanceUuid` VARCHAR(32) NOT NULL,
+    `managementNodeUuid` VARCHAR(32) DEFAULT NULL,
+    `dirtyVersion` BIGINT NOT NULL DEFAULT 1,
+    `lastClaimTime` TIMESTAMP NULL DEFAULT NULL,
+    `storageStructureChange` TINYINT(1) NOT NULL DEFAULT 0,
+    `retryCount` INT NOT NULL DEFAULT 0,
+    `nextRetryTime` TIMESTAMP NULL DEFAULT NULL,
+    `lastOpDate` timestamp on update CURRENT_TIMESTAMP,
+    `createDate` timestamp,
+    PRIMARY KEY (`vmInstanceUuid`),
+    CONSTRAINT `fkVmMetadataDirtyVOVmInstanceEO` FOREIGN KEY (`vmInstanceUuid`)  REFERENCES `VmInstanceEO` (`uuid`) ON DELETE CASCADE,
+    CONSTRAINT `fkVmMetadataDirtyVOManagementNodeVO` FOREIGN KEY (`managementNodeUuid`) REFERENCES `ManagementNodeVO` (`uuid`) ON DELETE SET NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+CREATE TABLE IF NOT EXISTS `zstack`.`VmMetadataFingerprintVO` (
+    `vmInstanceUuid` VARCHAR(32) NOT NULL,
+    `metadataSnapshot` LONGTEXT,
+    `lastFlushTime` TIMESTAMP NULL DEFAULT NULL,
+    `lastFlushFailed` TINYINT(1) NOT NULL DEFAULT 0,
+    `staleRecoveryCount` INT NOT NULL DEFAULT 0,
+    PRIMARY KEY (`vmInstanceUuid`),
+    CONSTRAINT `fkVmMetadataFingerprintVOVmInstanceEO` FOREIGN KEY (`vmInstanceUuid`)  REFERENCES `VmInstanceEO` (`uuid`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

conf/globalConfig/vm.xml

@@ -317,4 +317,21 @@
 		<type>java.lang.Boolean</type>
 		<defaultValue>false</defaultValue>
 	</config>
+
+	<config>
+		<category>vm</category>
+		<name>vm.metadata.enabled</name>
+		<description>enable vm metadata</description>
+		<type>java.lang.Boolean</type>
+		<defaultValue>false</defaultValue>
+	</config>
+
+	<config>
+		<category>vm</category>
+		<name>vm.metadata.lastRefreshVersion</name>
+		<description>Last completed upgrade refresh version, prevents duplicate triggers across MNs. Internal use only</description>
+		<type>java.lang.String</type>
+		<defaultValue>5.0.0</defaultValue>
+	</config>
+
 </globalConfig>

conf/persistence.xml

@@ -211,5 +211,7 @@
         <class>org.zstack.header.resourceattribute.entity.ResourceAttributeKeyResourceTypeVO</class>
         <class>org.zstack.header.resourceattribute.entity.ResourceAttributeConstraintVO</class>
         <class>org.zstack.softwarePackage.header.SoftwarePackageVO</class>
+        <class>org.zstack.header.vm.metadata.VmMetadataDirtyVO</class>
+        <class>org.zstack.header.vm.metadata.VmMetadataFingerprintVO</class>
     </persistence-unit>
 </persistence>

conf/serviceConfig/primaryStorage.xml

@@ -84,4 +84,7 @@
     <message>
         <name>org.zstack.header.storage.primary.APIAddStorageProtocolMsg</name>
     </message>
-</service>
+    <message>
+        <name>org.zstack.header.storage.primary.APIScanVmInstanceMetadataFromPrimaryStorageMsg</name>
+    </message>
+</service>

conf/serviceConfig/vmInstance.xml

@@ -264,4 +264,16 @@
     <message>
         <name>org.zstack.header.vm.APIDeleteTemplatedVmInstanceMsg</name>
     </message>
+    <message>
+        <name>org.zstack.header.vm.metadata.APICleanupVmInstanceMetadataMsg</name>
+    </message>
+    <message>
+        <name>org.zstack.header.vm.metadata.APIRegisterVmInstanceFromMetadataMsg</name>
+    </message>
+    <message>
+        <name>org.zstack.header.vm.metadata.APIUpdateVmInstanceMetadataMsg</name>
+    </message>
+    <message>
+        <name>org.zstack.header.vm.metadata.APIGetVmInstanceMetadataFromPrimaryStorageMsg</name>
+    </message>
 </service>

conf/springConfigXml/VmInstanceManager.xml

@@ -118,6 +118,7 @@
                 <value>org.zstack.compute.vm.VmExpungeRootVolumeFlow</value>
                 <value>org.zstack.compute.vm.VmExpungeMemoryVolumeFlow</value>
                 <value>org.zstack.compute.vm.VmExpungeCacheVolumeFlow</value>
+                <value>org.zstack.compute.vm.VmExpungeMetadataFlow</value>
             </list>
         </property>
         <property name="pauseVmWorkFlowElements">

header/src/main/java/org/zstack/header/storage/primary/APIScanVmInstanceMetadataFromPrimaryStorageMsg.java

@@ -0,0 +1,35 @@
+package org.zstack.header.storage.primary;
+
+import org.springframework.http.HttpMethod;
+import org.zstack.header.message.APIParam;
+import org.zstack.header.message.APISyncCallMessage;
+import org.zstack.header.rest.RestRequest;
+
+@RestRequest(
+        path = "/primary-storage/vm-instances/metadata/scan",
+        method = HttpMethod.GET,
+        responseClass = APIScanVmInstanceMetadataFromPrimaryStorageReply.class
+)
+public class APIScanVmInstanceMetadataFromPrimaryStorageMsg extends APISyncCallMessage implements PrimaryStorageMessage {
+    @APIParam(resourceType = PrimaryStorageVO.class)
+    private String uuid;
+
+    public String getUuid() {
+        return uuid;
+    }
+
+    public void setUuid(String uuid) {
+        this.uuid = uuid;
+    }
+
+    @Override
+    public String getPrimaryStorageUuid() {
+        return uuid;
+    }
+
+    public static APIScanVmInstanceMetadataFromPrimaryStorageMsg __example__() {
+        APIScanVmInstanceMetadataFromPrimaryStorageMsg msg = new APIScanVmInstanceMetadataFromPrimaryStorageMsg();
+        msg.setUuid(uuid());
+        return msg;
+    }
+}