replace all unix-specific subprocess calls such that app runs on Windows

afctl/_version.py

@@ -495,7 +495,7 @@ def get_versions():
         # versionfile_source is the relative path from the top of the source
         # tree (where the .git directory might live) to this file. Invert
         # this to find the root from __file__.
-        for i in cfg.versionfile_source.split('/'):
+        for i in cfg.versionfile_source.split(os.path.sep):
             root = os.path.dirname(root)
     except NameError:
         return {"version": "0+unknown", "full-revisionid": None,

afctl/parser_helpers.py

@@ -1,16 +1,19 @@
 from afctl.utils import Utility
 import os
-import subprocess
+import shutil
 from termcolor import colored
 from afctl.exceptions import AfctlParserException
+import git
+
+SEP = os.path.sep
 
 class ParserHelpers():
 
     @staticmethod
     def get_project_file_names(name):
         try:
             pwd = os.getcwd()
-            main_dir = pwd if name == '.' else os.path.join(pwd, name.lstrip('/').rstrip('/'))
+            main_dir = pwd if name == '.' else os.path.join(pwd, name.lstrip(SEP).rstrip(SEP))
             project_name = os.path.basename(main_dir)
             config_dir = Utility.CONSTS['config_dir']
             config_file = Utility.project_config(project_name)
@@ -35,11 +38,10 @@ def get_project_file_names(name):
     @staticmethod
     def add_git_config(files):
         try:
-            origin = subprocess.run(['git', '--git-dir={}'.format(os.path.join(files['main_dir'], '.git')), 'config',
-                                     '--get', 'remote.origin.url'],stdout=subprocess.PIPE)
-            origin = origin.stdout.decode('utf-8')[:-1]
-            if origin == '':
-                subprocess.run(['git', 'init', files['main_dir']])
+            try:
+                origin = git.Repo(os.path.join(files['main_dir']))
+            except git.exc.InvalidGitRepositoryError:
+                origin = git.Repo.init(os.path.join(files['main_dir']))
                 print(colored("Git origin is not set for this repository. Run 'afctl config global -o <origin>'", 'yellow'))
             else:
                 print("Updating git origin.")
@@ -71,19 +73,17 @@ def generate_all(files):
             #STEP - 2: create config file
             ParserHelpers.generate_config_file(files)
 
-            subprocess.run(['cp', '{}/templates/gitignore.txt'.format(os.path.dirname(os.path.abspath(__file__))),
-                            sub_file['.gitignore']])
-
+            shutil.copyfile('{}/templates/gitignore.txt'.format(os.path.dirname(os.path.abspath(__file__))),
+                            sub_file['.gitignore'])
         except Exception as e:
             raise AfctlParserException(e)
 
 
     @staticmethod
     def generate_config_file(files):
         try:
-            subprocess.run(['cp', '{}/plugins/deployments/deployment_config.yml'.format(os.path.dirname(os.path.abspath(__file__))),
-                            files['config_file']])
-
+            shutil.copyfile('{}/plugins/deployments/deployment_config.yml'.format(os.path.dirname(os.path.abspath(__file__))),
+                            files['config_file'])
             ParserHelpers.add_git_config(files)
 
         except Exception as e:

afctl/parsers.py

@@ -5,7 +5,6 @@
 from afctl import __version__
 from afctl.utils import Utility
 from afctl.exceptions import AfctlParserException
-import subprocess
 from afctl.plugins.deployments.deployment_config import DeploymentConfig
 from afctl.parser_helpers import ParserHelpers
 from termcolor import colored
@@ -123,9 +122,10 @@ def generate(cls, args):
             elif args.type == "module":
                 path = "{}/{}/dags/{}".format(project_path, project_name, args.n)
                 test_path = "{}/tests/{}".format(project_path, args.n)
-                mod_val = subprocess.call(['mkdir', path])
-                test_val = subprocess.call(['mkdir', test_path])
-                if mod_val != 0 or test_val != 0:
+                try:
+                    os.makedirs(path, exist_ok=True)
+                    os.makedirs(test_path, exist_ok=True)
+                except:
                     cls.parser.error(colored("Unable to generate.", 'red'))
 
             print(colored("Generated successfully.", 'green'))
@@ -286,4 +286,4 @@ def act_on_configs(cls, args, project_name):
                 Utility.print_file(Utility.project_config(project_name))
 
         except Exception as e:
-            AfctlParserException(e)
+            AfctlParserException(e)

afctl/plugins/deployments/docker/deployment_config.py

@@ -4,7 +4,7 @@
 import os
 import yaml
 from afctl.utils import Utility
-import subprocess
+import docker
 
 # Yaml Structure
 #   deployment:
@@ -47,8 +47,10 @@ def deploy_project(cls, args, config_file):
             with open(Utility.project_config(config_file)) as file:
                 config = yaml.full_load(file)
 
-            val = subprocess.call(['docker', 'info'])
-            if val != 0:
+            try:
+                client = docker.from_env()
+                client.info()
+            except:
                 return True, "Docker is not running. Please start docker."
 
             if args.d:

afctl/plugins/deployments/qubole/qubole_utils.py

@@ -1,25 +1,17 @@
-import subprocess
 from afctl.exceptions import AfctlDeploymentException
 from mako.template import Template
 from qds_sdk.qubole import Qubole
 from qds_sdk.commands import ShellCommand
 from urllib.parse import urlparse
-
+import git
 
 class QuboleUtils():
 
     @staticmethod
     def fetch_latest_commit(origin, branch):
         try:
-            commit = subprocess.run(['git', 'ls-remote', origin, 'refs/heads/{}'.format(branch), '|', 'cut', '-f', '1'],
-                                    stdout=subprocess.PIPE)
-            commit = commit.stdout.decode('utf-8')
-
-            if commit == '':
-                return None
-
-            return commit.split('\t')[0]
-
+            repo = git.Repo('.')
+            return repo.remotes.origin.refs[0].commit.hexsha
         except Exception as e:
             raise AfctlDeploymentException(e)
 
@@ -95,9 +87,8 @@ def generate_configs(configs, args):
             env = "{}/api".format(config['env'].rstrip('/'))
             cluster = config['cluster']
             token = config['token']
-            branch = subprocess.run(['git', 'symbolic-ref', '--short', 'HEAD'], stdout=subprocess.PIPE).stdout.decode(
-                'utf-8')[:-1]
-
+            repo = git.Repo('.')
+            branch = repo.active_branch.name
             return {
                 'name': name,
                 'env': env,

afctl/templates/dag_template.py

@@ -8,8 +8,8 @@ def dag_template(name, config_name):
 
 default_args = {
 'owner': '${config_name}',
+'start_date': datetime.now() - timedelta(days=1),
 # 'depends_on_past': ,
-# 'start_date': ,
 # 'email': ,
 # 'email_on_failure': ,
 # 'email_on_retry': ,
@@ -23,4 +23,4 @@ def dag_template(name, config_name):
 """
     )
 
-    return template.render_unicode(name=name, config_name=config_name)
+    return template.render_unicode(name=name, config_name=config_name)

afctl/tests/deployment_tests/test_local_deployment.py

@@ -1,7 +1,9 @@
 from afctl.plugins.deployments.docker.deployment_config import DockerDeploymentConfig
 from afctl.tests.utils import clean_up, PROJECT_NAME, PROJECT_CONFIG_DIR
 import pytest
-import os, subprocess
+import os, pathlib, tempfile
+
+TMP = tempfile.gettempdir()
 
 
 class TestLocalDeployment:
@@ -10,12 +12,12 @@ class TestLocalDeployment:
     def create_project(self):
         clean_up(PROJECT_NAME)
         clean_up(PROJECT_CONFIG_DIR)
-        main_dir = os.path.join('/tmp', PROJECT_NAME)
-        subprocess.run(['mkdir', main_dir])
-        subprocess.run(['mkdir', PROJECT_CONFIG_DIR])
-        subprocess.run(['mkdir', os.path.join(main_dir, 'deployments')])
+        main_dir = os.path.join(TMP, PROJECT_NAME)
+        os.makedirs(main_dir, exist_ok=True)
+        os.makedirs(PROJECT_CONFIG_DIR, exist_ok=True)
+        os.makedirs(os.path.join(main_dir, 'deployments'), exist_ok=True)
         config_file = "{}.yml".format(os.path.join(PROJECT_CONFIG_DIR, PROJECT_NAME))
-        subprocess.run(['touch', config_file])
+        pathlib.Path(config_file).touch()
         config_file_content = """
 global:
     airflow_version:
@@ -51,4 +53,4 @@ def test_docker_compose_generation(self, create_project):
         current_output = open(config_file).read()
         expected_output = expected_output.replace(" ", "")
         current_output = current_output.replace(" ", "")
-        assert expected_output == current_output
+        assert expected_output == current_output

afctl/tests/parser_tests/test_parser_utils.py

@@ -1,40 +1,44 @@
 from afctl.utils import Utility
 import pytest
-import os, subprocess
+import os, pathlib, shutil, tempfile
 from afctl.tests.utils import create_path_and_clean, PROJECT_NAME, PROJECT_CONFIG_DIR, clean_up
 
+SEP = os.path.sep
+TMP = tempfile.gettempdir()
+TMP_NO_SEP = TMP.replace(SEP,'')
+
 class TestUtils:
 
     @pytest.fixture(scope='function')
     def clean_tmp_dir(self):
-        parent = ['/tmp']
+        parent = [TMP]
         child = ['one', 'two', 'three']
         create_path_and_clean(parent, child)
         yield
         create_path_and_clean(parent, child)
 
     # create_dirs
     def test_create_dir(self, clean_tmp_dir):
-        parent = ['/tmp']
+        parent = [TMP]
         child = ['one', 'two', 'three']
         dirs = Utility.create_dirs(parent, child)
-        assert dirs['one'] == '/tmp/one'
-        assert os.path.exists(dirs['one']) is True
-        assert dirs['two'] == '/tmp/two'
-        assert os.path.exists(dirs['two']) is True
-        assert dirs['three'] == '/tmp/three'
-        assert os.path.exists(dirs['three']) is True
+        assert dirs['one'] == SEP.join([TMP, 'one'])
+        assert os.path.isdir(dirs['one'])
+        assert dirs['two'] == SEP.join([TMP, 'two'])
+        assert os.path.isdir(dirs['two'])
+        assert dirs['three'] == SEP.join([TMP, 'three'])
+        assert os.path.isdir(dirs['three'])
 
     # create_files
     def test_create_files(self, clean_tmp_dir):
-        parent = ['/tmp']
+        parent = [TMP]
         child = ['one', 'two', 'three']
         dirs = Utility.create_files(parent, child)
-        assert dirs['one'] == '/tmp/one'
+        assert dirs['one'] == SEP.join([TMP, 'one'])
         assert os.path.exists(dirs['one']) is True
-        assert dirs['two'] == '/tmp/two'
+        assert dirs['two'] == SEP.join([TMP, 'two'])
         assert os.path.exists(dirs['two']) is True
-        assert dirs['three'] == '/tmp/three'
+        assert dirs['three'] == SEP.join([TMP, 'three'])
         assert os.path.exists(dirs['three']) is True
 
     # project_config
@@ -47,7 +51,7 @@ def test_return_project_config_file(self):
     # generate_dag_template
     def test_generate_dag_template(self):
         project_name = "tes_project"
-        path = "/tmp"
+        path = TMP
         dag = "test"
         Utility.generate_dag_template(project_name, dag, path)
         expected_output = """ 
@@ -56,8 +60,8 @@ def test_generate_dag_template(self):
 
 default_args = {
 'owner': 'tes_project',
+'start_date': datetime.now() - timedelta(days=1),
 # 'depends_on_past': ,
-# 'start_date': ,
 # 'email': ,
 # 'email_on_failure': ,
 # 'email_on_retry': ,
@@ -76,26 +80,26 @@ def test_generate_dag_template(self):
 
     @pytest.fixture(scope='function')
     def create_project(self):
-        path = '/tmp/one/two/three'
-        subprocess.run(['mkdir', '-p', path])
-        file_path = '/tmp/one/two/.afctl_project'
-        subprocess.run(['touch', file_path])
+        path = os.path.sep.join([TMP, 'one', 'two', 'three'])
+        os.makedirs(path, exist_ok=True)
+        file_path = os.path.sep.join([TMP, 'one', 'two', '.afctl_project'])
+        pathlib.Path(file_path).touch()
         yield
-        subprocess.run(['rm', '-rf', path])
+        shutil.rmtree(path)
 
     # find_project
     def test_find_project(self, create_project):
-        path = '/tmp/one/two/three'
+        path = os.path.sep.join([TMP, 'one', 'two', 'three'])
         project = Utility.find_project(path)
         assert project[0] == 'two'
-        assert project[1] == '/tmp/one/two'
+        assert project[1] == os.path.sep.join([TMP, 'one', 'two'])
 
 
     @pytest.fixture(scope='function')
     def create_config_file(self):
-        subprocess.run(['mkdir', PROJECT_CONFIG_DIR])
+        os.mkdir(PROJECT_CONFIG_DIR)
         file_path = os.path.join(PROJECT_CONFIG_DIR, PROJECT_NAME)+'.yml'
-        subprocess.run(['touch', file_path])
+        pathlib.Path(file_path).touch()
         yml_template = """
 parent:
     child1:
@@ -160,4 +164,4 @@ def test_add_and_update_configs(self, create_config_file):
         current_output = open(config_file).read()
         expected_output = expected_output.replace(" ", "")
         current_output = current_output.replace(" ", "")
-        assert expected_output == current_output
+        assert expected_output == current_output

afctl/tests/utils.py

@@ -1,5 +1,5 @@
 import os
-import subprocess
+import shutil
 import itertools
 
 PROJECT_NAME = 'test_project'
@@ -20,7 +20,10 @@ def __init__(self, type=None, origin=None, token=None, version=None, env=None, c
 
 def clean_up(project_file):
     if os.path.exists(project_file):
-        subprocess.run(['rm', '-rf', project_file])
+        if os.path.isdir(project_file):
+            shutil.rmtree(project_file, ignore_errors=True)
+        else:
+            os.unlink(project_file)
 
 
 def check_paths(parent, child):