Polling Based Job Cancellation

server/dive_tasks/utils.py

@@ -7,6 +7,7 @@
 import subprocess
 from subprocess import Popen
 import tempfile
+import time
 from typing import List, Tuple
 from urllib import request
 from urllib.parse import urlencode, urljoin
@@ -87,6 +88,17 @@ def stream_subprocess(
             stdout=subprocess.PIPE,
             stderr=stderr_file,
         )
+        while process.poll() is None:  # while there is no return code
+            time.sleep(60)  # every minute we check to see if the process is in the cancelling state
+            manager.refreshStatus()
+            print(manager.status)
+            if manager.status == JobStatus.CANCELING:
+                manager.write('Killing subprocess')
+                process.send_signal(signal.SIGTERM)
+                process.send_signal(signal.SIGKILL)
+                time.sleep(10)
+                # VIAME doesn't respond to the above signals we need to interrupt
+                process.send_signal(signal.SIGINT)
 
         if process.stdout is None:
             raise RuntimeError("Stdout must not be none")
@@ -99,11 +111,14 @@ def stream_subprocess(
                 stdout += line_str
 
             # Cancel the subprocess if the status is cancelling
+            # note this only checks when there is stdout from the subprocess
             if check_canceled(task, context, force=False) or manager.status == JobStatus.CANCELING:
                 # Can never be sure what signal a process will respond to.
                 process.send_signal(signal.SIGTERM)
                 process.send_signal(signal.SIGKILL)
-
+                # VIAME doesn't respond to the above signals we need to interrupt
+                time.sleep(10)
+                process.send_signal(signal.SIGINT)
         # flush logs
         manager._flush()
         # Wait for exit up to 30 seconds after kill