Introduce new native backup provider (KNIB)

agent/conf/agent.properties

@@ -457,3 +457,16 @@ iscsi.session.cleanup.enabled=false
 
 # Instance conversion VIRT_V2V_TMPDIR env var
 #convert.instance.env.virtv2v.tmpdir=
+
+# Timeout (in seconds) for QCOW2 delta merge operations, mainly used for classic volume snapshots, disk-only VM snapshots on file-based storage, and the KNIB plugin.
+# If a value of 0 or less is informed, the default will be used.
+# qcow2.delta.merge.timeout=259200
+
+
+# Maximum number of backup validation jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many validations as possible will be done at
+# the same time.
+# backup.validation.max.concurrent.operations.per.host=
+
+# Maximum number of backup compression jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many compressions as possible will be
+# done at the same time.
+# backup.compression.max.concurrent.operations.per.host=

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -156,7 +156,8 @@ public class AgentProperties{
     public static final Property<Integer> CMDS_TIMEOUT = new Property<>("cmds.timeout", 7200);
 
     /**
-     * The timeout (in seconds) for the snapshot merge operation, mainly used for classic volume snapshots and disk-only VM snapshots on file-based storage.<br>
+     * The timeout (in seconds) for QCOW2 delta merge operations, mainly used for classic volume snapshots, disk-only VM snapshots on file-based storage, and the KNIB plugin.
+     * If a value of 0 or less is informed, the default will be used.<br>
      * This configuration is only considered if libvirt.events.enabled is also true. <br>
      * Data type: Integer.<br>
      * Default value: <code>259200</code>
@@ -886,6 +887,18 @@ public Property<Integer> getWorkers() {
     public static final Property<Boolean> CREATE_FULL_CLONE = new Property<>("create.full.clone", false);
 
 
+    /**
+     * Maximum number of backup validation jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many validations as possible will be done at
+     * the same time.
+     */
+    public static final Property<Integer> BACKUP_VALIDATION_MAX_CONCURRENT_OPERATIONS_PER_HOST = new Property<>("backup.validation.max.concurrent.operations.per.host", null, Integer.class);
+
+    /**
+     * Maximum number of backup compression jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many compressions as possible will be
+     * done at the same time.
+     */
+    public static final Property<Integer> BACKUP_COMPRESSION_MAX_CONCURRENT_OPERATIONS_PER_HOST = new Property<>("backup.compression.max.concurrent.operations.per.host", null, Integer.class);
+
     public static class Property <T>{
         private String name;
         private T defaultValue;

api/src/main/java/com/cloud/agent/api/to/DataObjectType.java

@@ -19,5 +19,5 @@
 package com.cloud.agent.api.to;
 
 public enum DataObjectType {
-    VOLUME, SNAPSHOT, TEMPLATE, ARCHIVE
+    VOLUME, SNAPSHOT, TEMPLATE, ARCHIVE, BACKUP
 }

api/src/main/java/com/cloud/event/EventTypes.java

@@ -643,6 +643,7 @@ public class EventTypes {
     public static final String EVENT_VM_BACKUP_USAGE_METRIC = "BACKUP.USAGE.METRIC";
     public static final String EVENT_VM_BACKUP_EDIT = "BACKUP.OFFERING.EDIT";
     public static final String EVENT_VM_CREATE_FROM_BACKUP = "VM.CREATE.FROM.BACKUP";
+    public static final String EVENT_SCREENSHOT_DOWNLOAD = "BACKUP.VALIDATION.SCREENSHOT.DOWNLOAD";
 
     // external network device events
     public static final String EVENT_EXTERNAL_NVP_CONTROLLER_ADD = "PHYSICAL.NVPCONTROLLER.ADD";

api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java

@@ -20,6 +20,7 @@
 import java.util.Map;
 
 import org.apache.cloudstack.backup.Backup;
+import org.apache.cloudstack.backup.BackupProvider;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.Command;
@@ -94,10 +95,10 @@ public interface HypervisorGuru extends Adapter {
     Map<String, String> getClusterSettings(long vmId);
 
     VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId, long accountId, long userId,
-                                                  String vmInternalName, Backup backup) throws Exception;
+                                                  String vmInternalName, Backup backup, BackupProvider backupProvider) throws Exception;
 
     boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backup.VolumeInfo volumeInfo,
-                                                 VirtualMachine vm, long poolId, Backup backup) throws Exception;
+                                                 VirtualMachine vm, long poolId, Backup backup, BackupProvider backupProvider) throws Exception;
     /**
      * Will generate commands to migrate a vm to a pool. For now this will only work for stopped VMs on Vmware.
      *

api/src/main/java/com/cloud/storage/Storage.java

@@ -35,7 +35,8 @@ public static enum ImageFormat {
         VDI(true, true, false, "vdi"),
         TAR(false, false, false, "tar"),
         ZIP(false, false, false, "zip"),
-        DIR(false, false, false, "dir");
+        DIR(false, false, false, "dir"),
+        PNG(false, false, false, "png");
 
         private final boolean supportThinProvisioning;
         private final boolean supportSparse;

api/src/main/java/com/cloud/storage/Volume.java

@@ -60,7 +60,9 @@ enum State {
         UploadError(false, "Volume upload encountered some error"),
         UploadAbandoned(false, "Volume upload is abandoned since the upload was never initiated within a specified time"),
         Attaching(true, "The volume is attaching to a VM from Ready state."),
-        Restoring(true, "The volume is being restored from backup.");
+        Restoring(true, "The volume is being restored from backup."),
+        Consolidating(true, "The volume is being flattened."),
+        RestoreError(false, "The volume restore encountered an error.");
 
         boolean _transitional;
 
@@ -153,6 +155,10 @@ public String getDescription() {
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Destroy, Event.RestoreRequested, Restoring, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Restoring, Event.RestoreSucceeded, Ready, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Restoring, Event.RestoreFailed, Ready, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(Ready, Event.ConsolidationRequested, Consolidating, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(Consolidating, Event.OperationSucceeded, Ready, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(Consolidating, Event.OperationFailed, RestoreError, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(RestoreError, Event.RestoreFailed, RestoreError, null));
         }
     }
 
@@ -179,7 +185,8 @@ enum Event {
         OperationTimeout,
         RestoreRequested,
         RestoreSucceeded,
-        RestoreFailed;
+        RestoreFailed,
+        ConsolidationRequested
     }
 
     /**

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -107,7 +107,7 @@ public interface VolumeApiService {
 
     Volume attachVolumeToVM(AttachVolumeCmd command);
 
-    Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId, Boolean allowAttachForSharedFS);
+    Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId, Boolean allowAttachForSharedFS, boolean allowAttachOnRestoring);
 
     Volume detachVolumeViaDestroyVM(long vmId, long volumeId);
 
@@ -182,7 +182,7 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
 
     boolean validateConditionsToReplaceDiskOfferingOfVolume(Volume volume, DiskOffering newDiskOffering, StoragePool destPool);
 
-    Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge);
+    Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge, Boolean countDisplayFalseInResourceCount);
 
     void destroyVolume(long volumeId);
 

api/src/main/java/com/cloud/user/ResourceLimitService.java

@@ -259,7 +259,7 @@ void checkVolumeResourceLimitForDiskOfferingChange(Account owner, Boolean displa
     void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException;
 
     void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
-    void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
+    void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering, Boolean countDisplayFalseInResourceCount);
 
     void updateVmResourceCountForTemplateChange(long accountId, Boolean display, ServiceOffering offering, VirtualMachineTemplate currentTemplate, VirtualMachineTemplate newTemplate);
 
@@ -274,8 +274,8 @@ void updateVolumeResourceCountForDiskOfferingChange(long accountId, Boolean disp
     void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
     void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
     void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) throws ResourceAllocationException;
-    void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template);
-    void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template);
+    void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Boolean countDisplayFalseInResourceLimit);
+    void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Boolean countDisplayFalseInResourceCount);
 
     void checkVmResourceLimitsForServiceOfferingChange(Account owner, Boolean display, Long currentCpu, Long newCpu,
             Long currentMemory, Long newMemory, ServiceOffering currentOffering, ServiceOffering newOffering, VirtualMachineTemplate template) throws ResourceAllocationException;

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -73,11 +73,13 @@ public interface UserVmService {
     /**
      * Destroys one virtual machine
      *
-     * @param cmd the API Command Object containg the parameters to use for this service action
+     * @param cmd
+     *         the API Command Object containg the parameters to use for this service action
+     * @param checkExpunge
      * @throws ConcurrentOperationException
      * @throws ResourceUnavailableException
      */
-    UserVm destroyVm(DestroyVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException;
+    UserVm destroyVm(DestroyVMCmd cmd, boolean checkExpunge) throws ResourceUnavailableException, ConcurrentOperationException;
 
     /**
      * Destroys one virtual machine

api/src/main/java/com/cloud/vm/VirtualMachine.java

@@ -58,7 +58,10 @@ public enum State {
         Error(false, "VM is in error"),
         Unknown(false, "VM state is unknown."),
         Shutdown(false, "VM state is shutdown from inside"),
-        Restoring(true, "VM is being restored from backup");
+        Restoring(true, "VM is being restored from backup"),
+        BackingUp(true, "VM is being backed up"),
+        BackupError(false, "VM backup is in a inconsistent state. Operator should analyse the logs and restore the VM"),
+        RestoreError(false, "VM restore left the VM in a inconsistent state. Operator should analyse the logs and restore the VM");
 
         private final boolean _transitional;
         String _description;
@@ -131,6 +134,14 @@ public static StateMachine2<State, VirtualMachine.Event, VirtualMachine> getStat
             s_fsm.addTransition(new Transition<State, Event>(State.Destroyed, Event.RestoringRequested, State.Restoring, null));
             s_fsm.addTransition(new Transition<State, Event>(State.Restoring, Event.RestoringSuccess, State.Stopped, null));
             s_fsm.addTransition(new Transition<State, Event>(State.Restoring, Event.RestoringFailed, State.Stopped, null));
+            s_fsm.addTransition(new Transition<>(State.Running, Event.BackupRequested, State.BackingUp, null));
+            s_fsm.addTransition(new Transition<>(State.Stopped, Event.BackupRequested, State.BackingUp, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.BackupSucceededRunning, State.Running, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.BackupSucceededStopped, State.Stopped, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.OperationFailedToError, State.BackupError, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.OperationFailedToRunning, State.Running, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.OperationFailedToStopped, State.Stopped, null));
+            s_fsm.addTransition(new Transition<State, Event>(State.RestoreError, Event.RestoringFailed, State.RestoreError, null));
 
             s_fsm.addTransition(new Transition<State, Event>(State.Starting, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running, Arrays.asList(new Impact[]{Impact.USAGE})));
             s_fsm.addTransition(new Transition<State, Event>(State.Stopping, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running, null));
@@ -209,6 +220,8 @@ public enum Event {
         ExpungeOperation,
         OperationSucceeded,
         OperationFailed,
+        OperationFailedToRunning,
+        OperationFailedToStopped,
         OperationFailedToError,
         OperationRetry,
         AgentReportShutdowned,
@@ -218,6 +231,9 @@ public enum Event {
         RestoringRequested,
         RestoringFailed,
         RestoringSuccess,
+        BackupRequested,
+        BackupSucceededStopped,
+        BackupSucceededRunning,
 
         // added for new VMSync logic
         FollowAgentPowerOnReport,

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -130,4 +130,13 @@ public interface VmDetailConstants {
     String EXTERNAL_DETAIL_PREFIX = "External:";
     String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details";
     String CLOUDSTACK_VLAN = "cloudstack.vlan";
+
+    // KNIB specific
+    String LINKED_VOLUMES_SECONDARY_STORAGE_UUIDS = "linkedVolumesSecondaryStorageUuids";
+    String VALIDATION_COMMAND = "backupValidationCommand";
+    String VALIDATION_COMMAND_ARGUMENTS = "backupValidationCommandArguments";
+    String VALIDATION_COMMAND_EXPECTED_RESULT = "backupValidationCommandExpectedResult";
+    String VALIDATION_COMMAND_TIMEOUT = "backupValidationCommandTimeout";
+    String VALIDATION_SCREENSHOT_WAIT = "backupValidationScreenshotWait";
+    String VALIDATION_BOOT_TIMEOUT = "backupValidationBootTimeout";
 }

api/src/main/java/org/apache/cloudstack/alert/AlertService.java

@@ -83,6 +83,9 @@ private AlertType(short type, String name, boolean isDefault) {
         public static final AlertType ALERT_TYPE_VPN_GATEWAY_OBSOLETE_PARAMETERS = new AlertType((short)34, "ALERT.S2S.VPN.GATEWAY.OBSOLETE.PARAMETERS", true, true);
         public static final AlertType ALERT_TYPE_BACKUP_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, "ALERT.STORAGE.BACKUP", true);
         public static final AlertType ALERT_TYPE_OBJECT_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, "ALERT.STORAGE.OBJECT", true);
+        public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_FAILED = new AlertType((short)35, "ALERT.BACKUP.VALIDATION.FAILED", true, true);
+        public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_UNABLE_TO_VALIDATE = new AlertType((short)36, "ALERT.BACKUP.VALIDATION.UNABLE.TO.VALIDATE", true, true);
+        public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_CLEANUP_FAILED = new AlertType((short)37, "ALERT.BACKUP.VALIDATION.CLEANUP_FAILED", true, true);
 
         public short getType() {
             return type;

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -521,6 +521,7 @@ public class ApiConstants {
     public static final String QUALIFIERS = "qualifiers";
     public static final String QUERY_FILTER = "queryfilter";
     public static final String QUIESCE_VM = "quiescevm";
+    public static final String QUICK_RESTORE = "quickrestore";
     public static final String SCHEDULE = "schedule";
     public static final String SCHEDULE_ID = "scheduleid";
     public static final String SCOPE = "scope";
@@ -570,6 +571,8 @@ public class ApiConstants {
     public static final String STATE = "state";
     public static final String STATS = "stats";
     public static final String STATUS = "status";
+    public static final String COMPRESSION_STATUS = "compressionstatus";
+    public static final String VALIDATION_STATUS = "validationstatus";
     public static final String STORAGE_TYPE = "storagetype";
     public static final String STORAGE_POLICY = "storagepolicy";
     public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
@@ -660,6 +663,7 @@ public class ApiConstants {
     public static final String ETCD_SERVICE_OFFERING_NAME = "etcdofferingname";
     public static final String REMOVE_VLAN = "removevlan";
     public static final String VLAN_ID = "vlanid";
+    public static final String ISOLATED = "isolated";
     public static final String ISOLATED_PVLAN = "isolatedpvlan";
     public static final String ISOLATED_PVLAN_TYPE = "isolatedpvlantype";
     public static final String ISOLATION_URI = "isolationuri";
@@ -1176,6 +1180,7 @@ public class ApiConstants {
     public static final String CLEAN_UP_EXTRA_CONFIG = "cleanupextraconfig";
     public static final String CLEAN_UP_PARAMETERS = "cleanupparameters";
     public static final String VIRTUAL_SIZE = "virtualsize";
+    public static final String UNCOMPRESSED_SIZE = "uncompressedsize";
     public static final String NETSCALER_CONTROLCENTER_ID = "netscalercontrolcenterid";
     public static final String NETSCALER_SERVICEPACKAGE_ID = "netscalerservicepackageid";
     public static final String FETCH_ROUTER_HEALTH_CHECK_RESULTS = "fetchhealthcheckresults";
@@ -1365,6 +1370,10 @@ public class ApiConstants {
 
     public static final String VMWARE_DC = "vmwaredc";
 
+    public static final String PARAMETER_DESCRIPTION_ISOLATED_BACKUPS = "Whether the backup will be isolated, defaults to false. " +
+            "Isolated backups are always created as full backups in independent chains. Therefore, they will never depend on any existing backup chain " +
+            "and no backup chain will depend on them. Currently only supported for the KNIB provider.";
+
     public static final String CSS = "css";
 
     public static final String JSON_CONFIGURATION = "jsonconfiguration";
@@ -1385,6 +1394,26 @@ public class ApiConstants {
     public static final String OBSOLETE_PARAMETERS = "obsoleteparameters";
     public static final String EXCLUDED_PARAMETERS = "excludedparameters";
 
+    public static final String COMPRESS = "compress";
+
+    public static final String VALIDATE = "validate";
+
+    public static final String VALIDATION_STEPS = "validationsteps";
+
+    public static final String ALLOW_QUICK_RESTORE = "allowquickrestore";
+
+    public static final String ALLOW_EXTRACT_FILE = "allowextractfile";
+
+    public static final String BACKUP_CHAIN_SIZE = "backupchainsize";
+
+    public static final String COMPRESSION_LIBRARY = "compressionlibrary";
+    public static final String ATTEMPTS = "attempts";
+
+    public static final String EXECUTING = "executing";
+
+    public static final String SCHEDULED = "scheduled";
+    public static final String SCHEDULED_DATE = "scheduleddate";
+
     /**
      * This enum specifies IO Drivers, each option controls specific policies on I/O.
      * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0).

api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java

@@ -22,6 +22,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.cloudstack.api.response.NativeBackupOfferingResponse;
+import org.apache.cloudstack.backup.NativeBackupOffering;
 import org.apache.cloudstack.api.response.ConsoleSessionResponse;
 import org.apache.cloudstack.consoleproxy.ConsoleSession;
 import org.apache.cloudstack.acl.apikeypair.ApiKeyPair;
@@ -591,4 +593,6 @@ List<TemplateResponse> createTemplateResponses(ResponseView view, VirtualMachine
     ApiKeyPairResponse createKeyPairResponse(ApiKeyPair keyPair);
 
     ListResponse<BaseRolePermissionResponse> createKeypairPermissionsResponse(List<ApiKeyPairPermission> permissions);
+
+    NativeBackupOfferingResponse createNativeBackupOfferingResponse(NativeBackupOffering offering);
 }