[#6858]Platform: Preflight checks for NFS backup storage configuration (

#6971)

* Heading:
[#6858]Platform: Preflight checks for NFS backup storage configuration
Description:
When user initiates a manual backup -or- a scheduled backup, check whether
NFS path is a valid path on each of the database nodes before starting the backup task .
Fail the task right away before creating the snapshots for the backup.
Testing:
Configured the NFS backup storage PATH and created a AWS cloud provider
Then using cloud provider, created a multinode/single node universe
Ran sample Apps, which created table for backup.
Then after tried to create backup.
It will fail immediately if the storage path is not mounted on anyone of
the node.

Added unit Tests as well. It's working fine.

* Heading:
[#6858]Platform: Preflight checks for NFS backup storage configuration
Description:
When user initiates a manual backup -or- a scheduled backup, check whether
NFS path is a valid path on each of the database nodes before starting the backup task .
Fail the task right away before creating the snapshots for the backup.
Testing:
Configured the NFS backup storage PATH and created a AWS cloud provider
Then using cloud provider, created a multinode/single node universe
Ran sample Apps, which created table for backup.
Then after tried to create backup.
It will fail immediately if the storage path is not mounted on anyone of
the node.

Added unit Tests as well. It's working fine.

* Heading:
[#6858]Platform: Preflight checks for NFS backup storage configuration
Description:
When user initiates a manual backup -or- a scheduled backup, check whether
NFS path is a valid path on each of the database nodes before starting the backup task .
Fail the task right away before creating the snapshots for the backup.
Testing:
Configured the NFS backup storage PATH and created a AWS cloud provider
Then using cloud provider, created a multinode/single node universe
Ran sample Apps, which created table for backup.
Then after tried to create backup.
It will fail immediately if the storage path is not mounted on anyone of
the node.

Added unit Tests as well. It's working fine.

Co-authored-by: jitendra-12113 <jitedra.kumar@hashedin.com>

managed/devops/bin/yb_backup.py

@@ -709,13 +709,32 @@ def parse_arguments(self):
             '--restore_keys_destination', required=False,
             help="Location to download universe encryption keys backup file to"
         )
+        parser.add_argument(
+            '--nfs_storage_path', required=False, help="NFS storage mount path")
         logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
         self.args = parser.parse_args()
 
     def post_process_arguments(self):
         if self.args.verbose:
             logging.info("Parsed arguments: {}".format(vars(self.args)))
 
+        if self.args.storage_type == 'nfs':
+            logging.info('Checking whether NFS backup storage path mounted on TServers or not')
+            pool = ThreadPool(self.args.parallelism)
+            tablets_by_leader_ip = []
+
+            output = self.run_yb_admin(['list_all_tablet_servers'])
+            for line in output.splitlines():
+                if LEADING_UUID_RE.match(line):
+                    fields = split_by_space(line)
+                    ip_port = fields[1]
+                    state = fields[3]
+                    (ip, port) = ip_port.split(':')
+                    if state == 'ALIVE':
+                        tablets_by_leader_ip.append(ip)
+            tserver_ips = list(tablets_by_leader_ip)
+            SingleArgParallelCmd(self.find_nfs_storage, tserver_ips).run(pool)
+
         self.args.backup_location = self.args.backup_location or self.args.s3bucket
         options = BackupOptions(self.args)
         self.cloud_cfg_file_path = os.path.join(self.get_tmp_dir(), CLOUD_CFG_FILE_NAME)
@@ -1610,6 +1629,20 @@ def delete_bucket_obj(self):
         else:
             self.run_program(del_cmd)
 
+    def find_nfs_storage(self, tserver_ip):
+        """
+        Finds the NFS storage path mounted on the given tserver.
+        if we don't find storage path mounted on given tserver IP we
+        raise exception
+        :param tserver_ip: tablet server ip
+        """
+        try:
+            self.run_ssh_cmd(['find', self.args.nfs_storage_path], tserver_ip)
+        except Exception as ex:
+            raise BackupException(
+                ('Did not find nfs backup storage path: %s mounted on tablet server %s'
+                % (self.args.nfs_storage_path, tserver_ip)))
+
     def upload_metadata_and_checksum(self, src_path, dest_path):
         """
         Upload metadata file and checksum file to the target backup location.
@@ -1761,7 +1794,6 @@ def backup_table(self):
         Creates a backup of the given table by creating a snapshot and uploading it to the provided
         backup location.
         """
-
         if not self.args.keyspace:
             raise BackupException('Need to specify --keyspace')
 

managed/src/main/java/com/yugabyte/yw/common/TableManager.java

@@ -234,6 +234,10 @@ private void addCommonCommandArgs(BackupTableParams backupTableParams, AccessKey
     commandArgs.add(backupTableParams.storageLocation);
     commandArgs.add("--storage_type");
     commandArgs.add(customerConfig.name.toLowerCase());
+    if(customerConfig.name.toLowerCase().equals("nfs")) {
+      commandArgs.add("--nfs_storage_path");
+      commandArgs.add(customerConfig.getData().get("BACKUP_LOCATION").asText());
+    }
     if (nodeToNodeTlsEnabled) {
       commandArgs.add("--certs_dir");
       commandArgs.add(getCertsDir(region, provider));

managed/src/test/java/com/yugabyte/yw/common/TableManagerTest.java

@@ -263,6 +263,10 @@ private List<String> getExpectedBackupTableCommand(
     cmd.add(backupTableParams.storageLocation);
     cmd.add("--storage_type");
     cmd.add(storageType);
+    if(storageType.equals("nfs")) {
+      cmd.add("--nfs_storage_path");
+      cmd.add("/foo/bar");
+    }
     if (userIntent.enableNodeToNodeEncrypt) {
       cmd.add("--certs_dir");
       cmd.add(testProvider.code.equals("kubernetes") ? K8S_CERT_PATH : VM_CERT_PATH);