Changed to two interfaces scow-sync-start and scow-sync-query

README.md

@@ -1,23 +1,32 @@
 # Scow-Sync
-A file transfer system backend on SCOW
+A file transfer system tool on SCOW
 
 ## Install
 
 ### dependencies
+
 - python3
 
 - pip3
 
 - paramiko 3.0.0
 
+- psutil 5.9.4 
+
 ### install globally
-Clone the repository in a directory that all users have access to, then execute`sudo bash install.sh`
+
+Clone the repository in a directory that sudoer have access to, then execute`sudo bash install.sh`. 
+
+Attention that this will create /tmp/scow-sync directory which is used to stored the temp files of the transfer process.
 
 ## Usage
-Usage:
+
+### start
+
+You can use the following command for transfer, but the command will return immediately and write the transfer ID(for your query, you can see next) and process ID to stdout.
 
 ```bash
-python3 scow-sync-start [-h] [-a ADDRESS] [-u USER] [-s SOURCE] [-d DESTINATION] [-p PORT] [-k SSHKEY_PATH]
+scow-sync-start [-h] [-a ADDRESS] [-u USER] [-s SOURCE] [-d DESTINATION] [-p PORT] [-k SSHKEY_PATH]
 ```
 
 Optional arguments:
@@ -34,5 +43,20 @@ Optional arguments:
 
   `-p PORT, --port PORT`  port of the server
 
-  `-k SSHKEY_PATH, --sshkey-path PATH`  path to id_rsa file
+  `-k SSHKEY_PATH, --sshkey_path PATH`  path to id_rsa file
+
+### query
+
+You can use the following command to view the real-time transfer process, including file path, progress, speed and remaining time.
+
+```bash
+scow-sync-query [-f transfer_id] [-p pid]
+```
+
+Optional arguments:
+
+  `-f TRANSFER_ID, --transfer_id TRANSFER_ID`  the id of this transfer
+
+  `-p PID, --pid PID` the process id of this transfer
+
 

install.sh

@@ -1,8 +1,16 @@
 #!/bin/bash
 # install scow-sync globally by add soft link in /usr/local/bin
 # usage: sudo bash install.sh
+pip3 install psutil==5.9.4
 pip3 install paramiko==3.0.0
 local_path=$(pwd)
-chmod +x $local_path/scow-sync
+chmod +x $local_path/scow-sync-start
+chmod +X $local_path/scow-sync-query
+ln -s $local_path/scow-sync-start /usr/bin/scow-sync-start
+ln -s $local_path/scow-sync-query /usr/bin/scow-sync-query
 ln -s $local_path/scow-sync /usr/bin/scow-sync
-chmod +x /usr/bin/scow-sync
+chmod +x /usr/bin/scow-sync-start
+chmod +x /usr/bin/scow-sync-query
+mkdir /tmp
+mkdir /tmp/scow-sync
+chmod 777 /tmp/scow-sync

scow-sync.py → scow-sync

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 '''
 Code entry to run the scow-sync
 '''

scow-sync-query.py → scow-sync-query

@@ -1,9 +1,12 @@
+#!/usr/bin/python3
 '''
 query thr progress of file transfered
 '''
 from argparse import ArgumentParser
 import os
 import sys
+import fcntl
+import select
 import subprocess
 import psutil
 
@@ -25,7 +28,7 @@ def get_args_parser(self) -> ArgumentParser:
         '''
         # fid
         self.parser.add_argument(
-            '-f', '--files_transfer_id',
+            '-f', '--transfer_id',
             type=int,
             help='the id of the file transfer this time'
         )
@@ -40,20 +43,25 @@ def get_args_parser(self) -> ArgumentParser:
 
 if __name__ == '__main__':
     args = PidArgsParser().get_args_parser().parse_args()
-    fid = args.files_transfer_id
+    fid = args.transfer_id
     pid = args.pid
 
-    filename = f'./tmp/{fid}.out'
+    filename = f'/tmp/scow-sync/{fid}.out'
     process = psutil.Process(pid)
 
+    # 
     if os.path.exists(filename):
         cmd = ["tail", "-f", filename]
         popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        while popen.poll() is None and process.is_running():
-            stdout = popen.stdout
-            if stdout:
-                output = stdout.readline()
-                if output:
+        stdout = popen.stdout
+        if stdout:
+            stdout_fd = stdout.fileno()
+            flags = fcntl.fcntl(stdout_fd, fcntl.F_GETFL)
+            fcntl.fcntl(stdout_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+            while popen.poll() is None and process.is_running():
+                ready, _, _ = select.select([stdout], [], [], 0)
+                if ready:
+                    output = stdout.readline()
                     sys.stdout.write(output.decode().strip())
                     sys.stdout.write('\n')
                     sys.stdout.flush()

scow-sync-start.py → scow-sync-start

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 '''
 Start a file transfer asynchronously
 '''
@@ -8,15 +9,19 @@
 
 if __name__ == '__main__':
     args = ArgsParser().get_args_parser().parse_args()
-    cmd = f'python3 scow-sync.py -a {args.address} -u {args.user} -s {args.source} -d {args.destination} -m {args.max_depth} -p {args.port} -k {args.sshkey_path}'
+    cmd = f'scow-sync -a {args.address} -u {args.user} -s {args.source} -d {args.destination} -m {args.max_depth} -p {args.port} -k {args.sshkey_path}'
 
     # 此次传输的重定向文件，使用hashlib生成
     hash_object = hashlib.md5(cmd.encode())
     hash_value = int.from_bytes(hash_object.digest(), byteorder='big')
     modulus = 10**9 + 7;
     unique_id = hash_value % modulus
 
-    popen = Popen(cmd, shell=True, stdout=open(f'./tmp/{unique_id}.out', 'w'))
-
+    popen = Popen(cmd, shell=True, stdout=open(f'/tmp/scow-sync/{unique_id}.out', 'w'))
+    pid = popen.pid
     sys.stdout.write(str(unique_id))
+    sys.stdout.write('\n')
+    sys.stdout.write(str(pid))
+    sys.stdout.write('\n')
+
     exit(0)

scowsync.py

@@ -118,3 +118,4 @@ def transfer_files(self):
                 self.thread_pool.submit(
                     self.__transfer_file, entity_file.subpath)
         self.thread_pool.shutdown()
+        exit(0)