CloudStack Backup & Recovery Framework (#3553)

Author: yadvr

.travis.yml

@@ -46,6 +46,7 @@ env:
              smoke/test_affinity_groups
              smoke/test_affinity_groups_projects
              smoke/test_async_job
+             smoke/test_backup_recovery_dummy
              smoke/test_create_list_domain_account_project
              smoke/test_create_network
              smoke/test_deploy_vgpu_enabled_vm

api/src/main/java/com/cloud/event/EventTypes.java

@@ -478,6 +478,17 @@ public class EventTypes {
     public static final String EVENT_VM_SNAPSHOT_OFF_PRIMARY = "VMSNAPSHOT.OFF_PRIMARY";
     public static final String EVENT_VM_SNAPSHOT_REVERT = "VMSNAPSHOT.REVERTTO";
 
+    // Backup and Recovery events
+    public static final String EVENT_VM_BACKUP_IMPORT_OFFERING = "BACKUP.IMPORT.OFFERING";
+    public static final String EVENT_VM_BACKUP_OFFERING_ASSIGN = "BACKUP.OFFERING.ASSIGN";
+    public static final String EVENT_VM_BACKUP_OFFERING_REMOVE = "BACKUP.OFFERING.REMOVE";
+    public static final String EVENT_VM_BACKUP_CREATE = "BACKUP.CREATE";
+    public static final String EVENT_VM_BACKUP_RESTORE = "BACKUP.RESTORE";
+    public static final String EVENT_VM_BACKUP_DELETE = "BACKUP.DELETE";
+    public static final String EVENT_VM_BACKUP_RESTORE_VOLUME_TO_VM = "BACKUP.RESTORE.VOLUME.TO.VM";
+    public static final String EVENT_VM_BACKUP_SCHEDULE_CONFIGURE = "BACKUP.SCHEDULE.CONFIGURE";
+    public static final String EVENT_VM_BACKUP_SCHEDULE_DELETE = "BACKUP.SCHEDULE.DELETE";
+
     // external network device events
     public static final String EVENT_EXTERNAL_NVP_CONTROLLER_ADD = "PHYSICAL.NVPCONTROLLER.ADD";
     public static final String EVENT_EXTERNAL_NVP_CONTROLLER_DELETE = "PHYSICAL.NVPCONTROLLER.DELETE";

api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java

@@ -19,13 +19,14 @@
 import java.util.List;
 import java.util.Map;
 
-import com.cloud.storage.StoragePool;
+import org.apache.cloudstack.backup.Backup;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.storage.StoragePool;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.Adapter;
 import com.cloud.vm.NicProfile;
@@ -86,6 +87,11 @@ public interface HypervisorGuru extends Adapter {
 
     Map<String, String> getClusterSettings(long vmId);
 
+    VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId, long accountId, long userId,
+                                                  String vmInternalName, Backup backup) throws Exception;
+
+    boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backup.VolumeInfo volumeInfo,
+                                                 VirtualMachine vm, long poolId, Backup backup) throws Exception;
     /**
      * Will generate commands to migrate a vm to a pool. For now this will only work for stopped VMs on Vmware.
      *

api/src/main/java/com/cloud/server/ResourceTag.java

@@ -29,6 +29,7 @@ public enum ResourceObjectType {
         ISO(true, false),
         Volume(true, true),
         Snapshot(true, false),
+        Backup(true, false),
         Network(true, true),
         Nic(false, true),
         LoadBalancer(true, true),

api/src/main/java/com/cloud/vm/VirtualMachine.java

@@ -16,18 +16,21 @@
 // under the License.
 package com.cloud.vm;
 
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cloudstack.acl.ControlledEntity;
+import org.apache.cloudstack.api.Displayable;
+import org.apache.cloudstack.backup.Backup;
+import org.apache.cloudstack.kernel.Partition;
+
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.utils.fsm.StateMachine2;
 import com.cloud.utils.fsm.StateMachine2.Transition;
 import com.cloud.utils.fsm.StateMachine2.Transition.Impact;
 import com.cloud.utils.fsm.StateObject;
-import org.apache.cloudstack.acl.ControlledEntity;
-import org.apache.cloudstack.api.Displayable;
-import org.apache.cloudstack.kernel.Partition;
-
-import java.util.Arrays;
-import java.util.Date;
-import java.util.Map;
 
 /**
  * VirtualMachine describes the properties held by a virtual machine
@@ -319,6 +322,12 @@ public boolean isUsedBySystem() {
 
     Long getDiskOfferingId();
 
+    Long getBackupOfferingId();
+
+    String getBackupExternalId();
+
+    List<Backup.VolumeInfo> getBackupVolumeList();
+
     Type getType();
 
     HypervisorType getHypervisorType();

api/src/main/java/org/apache/cloudstack/api/ApiCommandJobType.java

@@ -23,6 +23,7 @@ public enum ApiCommandJobType {
     Volume,
     ConsoleProxy,
     Snapshot,
+    Backup,
     Template,
     Iso,
     SystemVm,

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -33,6 +33,9 @@ public class ApiConstants {
     public static final String APPLIED = "applied";
     public static final String LIST_LB_VMIPS = "lbvmips";
     public static final String AVAILABLE = "available";
+    public static final String BACKUP_ID = "backupid";
+    public static final String BACKUP_OFFERING_NAME = "backupofferingname";
+    public static final String BACKUP_OFFERING_ID = "backupofferingid";
     public static final String BITS = "bits";
     public static final String BOOTABLE = "bootable";
     public static final String BIND_DN = "binddn";
@@ -144,6 +147,7 @@ public class ApiConstants {
     public static final String EXTRA_DHCP_OPTION_NAME = "extradhcpoptionname";
     public static final String EXTRA_DHCP_OPTION_CODE = "extradhcpoptioncode";
     public static final String EXTRA_DHCP_OPTION_VALUE = "extradhcpvalue";
+    public static final String EXTERNAL = "external";
     public static final String FENCE = "fence";
     public static final String FETCH_LATEST = "fetchlatest";
     public static final String FIRSTNAME = "firstname";
@@ -366,6 +370,7 @@ public class ApiConstants {
     public static final String VALUE = "value";
     public static final String VIRTUAL_MACHINE_ID = "virtualmachineid";
     public static final String VIRTUAL_MACHINE_IDS = "virtualmachineids";
+    public static final String VIRTUAL_MACHINE_NAME = "virtualmachinename";
     public static final String VIRTUAL_MACHINE_ID_IP = "vmidipmap";
     public static final String VIRTUAL_MACHINE_COUNT = "virtualmachinecount";
     public static final String USAGE_ID = "usageid";
@@ -385,6 +390,7 @@ public class ApiConstants {
     public static final String VNET = "vnet";
     public static final String IS_VOLATILE = "isvolatile";
     public static final String VOLUME_ID = "volumeid";
+    public static final String VOLUMES = "volumes";
     public static final String ZONE = "zone";
     public static final String ZONE_ID = "zoneid";
     public static final String ZONE_NAME = "zonename";
@@ -531,6 +537,7 @@ public class ApiConstants {
     public static final String REQUIRED = "required";
     public static final String RESTART_REQUIRED = "restartrequired";
     public static final String ALLOW_USER_CREATE_PROJECTS = "allowusercreateprojects";
+    public static final String ALLOW_USER_DRIVEN_BACKUPS = "allowuserdrivenbackups";
     public static final String CONSERVE_MODE = "conservemode";
     public static final String TRAFFIC_TYPE_IMPLEMENTOR = "traffictypeimplementor";
     public static final String KEYWORD = "keyword";
@@ -780,7 +787,7 @@ public enum HostDetails {
     }
 
     public enum VMDetails {
-        all, group, nics, stats, secgrp, tmpl, servoff, diskoff, iso, volume, min, affgrp;
+        all, group, nics, stats, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp;
     }
 
     public enum DomainDetails {

api/src/main/java/org/apache/cloudstack/api/BaseBackupListCmd.java

@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cloudstack.api.response.BackupOfferingResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.backup.BackupOffering;
+import org.apache.cloudstack.context.CallContext;
+
+public abstract class BaseBackupListCmd extends BaseListCmd {
+
+    protected void setupResponseBackupOfferingsList(final List<BackupOffering> offerings, final Integer count) {
+        final ListResponse<BackupOfferingResponse> response = new ListResponse<>();
+        final List<BackupOfferingResponse> responses = new ArrayList<>();
+        for (final BackupOffering offering : offerings) {
+            if (offering == null) {
+                continue;
+            }
+            BackupOfferingResponse backupOfferingResponse = _responseGenerator.createBackupOfferingResponse(offering);
+            responses.add(backupOfferingResponse);
+        }
+        response.setResponses(responses, count);
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccount().getId();
+    }
+}

api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java

@@ -22,8 +22,6 @@
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
-import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.cloudstack.affinity.AffinityGroup;
 import org.apache.cloudstack.affinity.AffinityGroupResponse;
 import org.apache.cloudstack.api.ApiConstants.HostDetails;
@@ -36,6 +34,8 @@
 import org.apache.cloudstack.api.response.AutoScalePolicyResponse;
 import org.apache.cloudstack.api.response.AutoScaleVmGroupResponse;
 import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse;
+import org.apache.cloudstack.api.response.BackupOfferingResponse;
+import org.apache.cloudstack.api.response.BackupScheduleResponse;
 import org.apache.cloudstack.api.response.CapacityResponse;
 import org.apache.cloudstack.api.response.ClusterResponse;
 import org.apache.cloudstack.api.response.ConditionResponse;
@@ -88,6 +88,7 @@
 import org.apache.cloudstack.api.response.ResourceCountResponse;
 import org.apache.cloudstack.api.response.ResourceLimitResponse;
 import org.apache.cloudstack.api.response.ResourceTagResponse;
+import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
 import org.apache.cloudstack.api.response.SSHKeyPairResponse;
 import org.apache.cloudstack.api.response.SecurityGroupResponse;
 import org.apache.cloudstack.api.response.ServiceOfferingResponse;
@@ -111,6 +112,7 @@
 import org.apache.cloudstack.api.response.UsageRecordResponse;
 import org.apache.cloudstack.api.response.UserResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.api.response.BackupResponse;
 import org.apache.cloudstack.api.response.VMSnapshotResponse;
 import org.apache.cloudstack.api.response.VirtualRouterProviderResponse;
 import org.apache.cloudstack.api.response.VlanIpRangeResponse;
@@ -119,7 +121,11 @@
 import org.apache.cloudstack.api.response.VpcResponse;
 import org.apache.cloudstack.api.response.VpnUsersResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
+import org.apache.cloudstack.backup.BackupOffering;
+import org.apache.cloudstack.backup.Backup;
+import org.apache.cloudstack.backup.BackupSchedule;
 import org.apache.cloudstack.config.Configuration;
+import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule;
 import org.apache.cloudstack.region.PortableIp;
 import org.apache.cloudstack.region.PortableIpRange;
@@ -467,6 +473,12 @@ List<TemplateResponse> createTemplateResponses(ResponseView view, VirtualMachine
 
     SSHKeyPairResponse createSSHKeyPairResponse(SSHKeyPair sshkeyPair, boolean privatekey);
 
+    BackupResponse createBackupResponse(Backup backup);
+
+    BackupScheduleResponse createBackupScheduleResponse(BackupSchedule backup);
+
+    BackupOfferingResponse createBackupOfferingResponse(BackupOffering policy);
+
     ManagementServerResponse createManagementResponse(ManagementServerHost mgmt);
 
     List<RouterHealthCheckResultResponse> createHealthCheckResponse(VirtualMachine router, List<RouterHealthCheckResult> healthCheckResults);

api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteBackupOfferingCmd.java

@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api.command.admin.backup;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.BackupOfferingResponse;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.backup.BackupManager;
+import org.apache.cloudstack.context.CallContext;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+
+@APICommand(name = DeleteBackupOfferingCmd.APINAME,
+        description = "Deletes a backup offering",
+        responseObject = SuccessResponse.class, since = "4.14.0",
+        authorized = {RoleType.Admin})
+public class DeleteBackupOfferingCmd extends BaseCmd {
+    public static final String APINAME = "deleteBackupOffering";
+
+    @Inject
+    private BackupManager backupManager;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    ////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ID,
+            type = CommandType.UUID,
+            entityType = BackupOfferingResponse.class,
+            required = true,
+            description = "ID of the backup offering")
+    private Long id;
+
+    /////////////////////////////////////////////////////
+    /////////////////// Accessors ///////////////////////
+    /////////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        if (backupManager.deleteBackupOffering(getId())) {
+            SuccessResponse response = new SuccessResponse(getCommandName());
+            setResponseObject(response);
+        } else {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to remove backup offering: " + getId());
+        }
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccount().getId();
+    }
+}