Ashishonce/multiplerunmultiprocessing

.cloud/.azure/run.json

@@ -1,3 +1,16 @@
+[{
+    "experiment_name": "<your-experiment-name>",
+    "tags": {"<your-run-tag-key>": "<your-run-tag-value>"},
+    "wait_for_completion": true,
+    "runconfig_python_file": "<your-runconfig-python-file>",
+    "runconfig_python_function_name": "<your-runconfig-python-function-name>",
+    "runconfig_yaml_file": "<your-runconfig-yaml-file>",
+    "pipeline_yaml_file": "<your-pipeline-yaml-file>",
+    "pipeline_publish": false,
+    "pipeline_name": "<your-pipeline-name>",
+    "pipeline_version": "<your-pipeline-version>",
+    "pipeline_continue_on_step_failure": false
+},
 {
     "experiment_name": "<your-experiment-name>",
     "tags": {"<your-run-tag-key>": "<your-run-tag-value>"},
@@ -11,3 +24,4 @@
     "pipeline_version": "<your-pipeline-version>",
     "pipeline_continue_on_step_failure": false
 }
+]

.github/workflows/python.yml

@@ -21,3 +21,11 @@ jobs:
           pip install flake8
           flake8 --ignore E501 code/main.py
           flake8 --ignore E501 code/utils.py
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Test with pytest
+        run: |
+          pip install pytest
+          pytest

.gitignore

@@ -1,5 +1,129 @@
-################################################################################
-# This .gitignore file was automatically created by Microsoft(R) Visual Studio.
-################################################################################
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
 
-/.vs
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

code/main.py

@@ -1,5 +1,6 @@
 import os
 import json
+import multiprocessing
 
 from azureml.core import Workspace, Experiment
 from azureml.core.authentication import ServicePrincipalAuthentication
@@ -11,70 +12,8 @@
 from utils import AMLConfigurationException, AMLExperimentConfigurationException, required_parameters_provided, mask_parameter, convert_to_markdown, load_pipeline_yaml, load_runconfig_yaml, load_runconfig_python
 
 
-def main():
-    # Loading input values
-    print("::debug::Loading input values")
-    parameters_file = os.environ.get("INPUT_PARAMETERS_FILE", default="run.json")
-    azure_credentials = os.environ.get("INPUT_AZURE_CREDENTIALS", default="{}")
-    try:
-        azure_credentials = json.loads(azure_credentials)
-    except JSONDecodeError:
-        print("::error::Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS")
-        raise AMLConfigurationException(f"Incorrect or poorly formed output from azure credentials saved in AZURE_CREDENTIALS secret. See setup in https://github.com/Azure/aml-workspace/blob/master/README.md")
-
-    # Checking provided parameters
-    print("::debug::Checking provided parameters")
-    required_parameters_provided(
-        parameters=azure_credentials,
-        keys=["tenantId", "clientId", "clientSecret"],
-        message="Required parameter(s) not found in your azure credentials saved in AZURE_CREDENTIALS secret for logging in to the workspace. Please provide a value for the following key(s): "
-    )
-
-    # Mask values
-    print("::debug::Masking parameters")
-    mask_parameter(parameter=azure_credentials.get("tenantId", ""))
-    mask_parameter(parameter=azure_credentials.get("clientId", ""))
-    mask_parameter(parameter=azure_credentials.get("clientSecret", ""))
-    mask_parameter(parameter=azure_credentials.get("subscriptionId", ""))
-
-    # Loading parameters file
-    print("::debug::Loading parameters file")
-    parameters_file_path = os.path.join(".cloud", ".azure", parameters_file)
-    try:
-        with open(parameters_file_path) as f:
-            parameters = json.load(f)
-    except FileNotFoundError:
-        print(f"::debug::Could not find parameter file in {parameters_file_path}. Please provide a parameter file in your repository if you do not want to use default settings (e.g. .cloud/.azure/run.json).")
-        parameters = {}
-
-    # Loading Workspace
-    print("::debug::Loading AML Workspace")
-    sp_auth = ServicePrincipalAuthentication(
-        tenant_id=azure_credentials.get("tenantId", ""),
-        service_principal_id=azure_credentials.get("clientId", ""),
-        service_principal_password=azure_credentials.get("clientSecret", "")
-    )
-    config_file_path = os.environ.get("GITHUB_WORKSPACE", default=".cloud/.azure")
-    config_file_name = "aml_arm_config.json"
-    try:
-        ws = Workspace.from_config(
-            path=config_file_path,
-            _file_name=config_file_name,
-            auth=sp_auth
-        )
-    except AuthenticationException as exception:
-        print(f"::error::Could not retrieve user token. Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS: {exception}")
-        raise AuthenticationException
-    except AuthenticationError as exception:
-        print(f"::error::Microsoft REST Authentication Error: {exception}")
-        raise AuthenticationError
-    except AdalError as exception:
-        print(f"::error::Active Directory Authentication Library Error: {exception}")
-        raise AdalError
-    except ProjectSystemException as exception:
-        print(f"::error::Workspace authorizationfailed: {exception}")
-        raise ProjectSystemException
-
+def submitRun(args):
+    ws, parameters = args[0], args[1]
     # Create experiment
     print("::debug::Creating experiment")
     try:
@@ -187,6 +126,78 @@ def main():
         print(f"::error::Could not register pipeline because you did not pass a pipeline to the action")
 
     print("::debug::Successfully finished Azure Machine Learning Train Action")
+    return True
+
+
+def main():
+    # Loading input values
+    print("::debug::Loading input values")
+    parameters_file = os.environ.get("INPUT_PARAMETERS_FILE", default="run.json")
+    azure_credentials = os.environ.get("INPUT_AZURE_CREDENTIALS", default="{}")
+    try:
+        azure_credentials = json.loads(azure_credentials)
+    except JSONDecodeError:
+        print("::error::Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS")
+        raise AMLConfigurationException(f"Incorrect or poorly formed output from azure credentials saved in AZURE_CREDENTIALS secret. See setup in https://github.com/Azure/aml-workspace/blob/master/README.md")
+
+    # Checking provided parameters
+    print("::debug::Checking provided parameters")
+    required_parameters_provided(
+        parameters=azure_credentials,
+        keys=["tenantId", "clientId", "clientSecret"],
+        message="Required parameter(s) not found in your azure credentials saved in AZURE_CREDENTIALS secret for logging in to the workspace. Please provide a value for the following key(s): "
+    )
+
+    # Mask values
+    print("::debug::Masking parameters")
+    mask_parameter(parameter=azure_credentials.get("tenantId", ""))
+    mask_parameter(parameter=azure_credentials.get("clientId", ""))
+    mask_parameter(parameter=azure_credentials.get("clientSecret", ""))
+    mask_parameter(parameter=azure_credentials.get("subscriptionId", ""))
+
+    # Loading parameters file
+    print("::debug::Loading parameters file")
+    parameters_file_path = os.path.join(".cloud", ".azure", parameters_file)
+    try:
+        with open(parameters_file_path) as f:
+            parameters = json.load(f)
+    except FileNotFoundError:
+        print(f"::debug::Could not find parameter file in {parameters_file_path}. Please provide a parameter file in your repository if you do not want to use default settings (e.g. .cloud/.azure/run.json).")
+        parameters = [{}]  # even if user doesn't have run.json file, we want to check the defaul directory, so want to run it at least once.
+
+    # Loading Workspace
+    print("::debug::Loading AML Workspace")
+    sp_auth = ServicePrincipalAuthentication(
+        tenant_id=azure_credentials.get("tenantId", ""),
+        service_principal_id=azure_credentials.get("clientId", ""),
+        service_principal_password=azure_credentials.get("clientSecret", "")
+    )
+    config_file_path = os.environ.get("GITHUB_WORKSPACE", default=".cloud/.azure")
+    config_file_name = "aml_arm_config.json"
+    try:
+        ws = Workspace.from_config(
+            path=config_file_path,
+            _file_name=config_file_name,
+            auth=sp_auth
+        )
+    except AuthenticationException as exception:
+        print(f"::error::Could not retrieve user token. Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS: {exception}")
+        raise AuthenticationException
+    except AuthenticationError as exception:
+        print(f"::error::Microsoft REST Authentication Error: {exception}")
+        raise AuthenticationError
+    except AdalError as exception:
+        print(f"::error::Active Directory Authentication Library Error: {exception}")
+        raise AdalError
+    except ProjectSystemException as exception:
+        print(f"::error::Workspace authorizationfailed: {exception}")
+        raise ProjectSystemException
+
+    # check here the number of cpus and create pool accordingly
+    pool = multiprocessing.Pool(processes=8)
+    tasks = [(ws, parameter) for parameter in parameters]
+    results = pool.map(submitRun, tasks)
+    print(results)
 
 
 if __name__ == "__main__":

requirements.txt

@@ -0,0 +1,2 @@
+azureml-sdk==1.1.5
+jsonschema==3.2.0

tests/test_basic.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import os
+import pytest
+import sys
+import json
+from azureml.exceptions import AuthenticationException, ProjectSystemException, AzureMLException, UserErrorException
+
+@pytest.fixture(autouse=True)
+def set_env_variables():
+    sys.path.append(os.path.join(os.path.dirname(__file__), '../code/'))
+    os.environ["GITHUB_REPOSITORY"] ="https://github.com/test/"
+    os.environ["GITHUB_REF"] ="test_basic"
+    os.environ["INPUT_PARAMETERS_FILE"] ="ashkuma"
+    os.environ["INPUT_AZURE_CREDENTIALS"] = ' { "clientId": "123", "clientSecret": "123", "subscriptionId": "123", "tenantId": "123"}'
+    os.environ["GITHUB_WORKSPACE"] ="./"
+
+    yield
+    # a test may override the base info so after yield it resets
+    os.environ["GITHUB_REPOSITORY"] ="https://github.com/test/"
+    os.environ["GITHUB_REF"] ="test_basic"
+    os.environ["INPUT_PARAMETERS_FILE"] ="ashkuma"
+    os.environ["INPUT_AZURE_CREDENTIALS"] = ' { "clientId": "123", "clientSecret": "123", "subscriptionId": "123", "tenantId": "123"}'
+    os.environ["GITHUB_WORKSPACE"] ="./"
+
+
+def test_credentialErrorException():
+    """unit Test to verify that in azure credentials are not present code must raise the error """
+    os.environ["INPUT_AZURE_CREDENTIALS"] = ''
+    import main
+    import utils
+    with pytest.raises(utils.AMLConfigurationException):
+        assert main.main()
+
+def test_parametersfileError():
+    """unit Test to verify that if  aml_arm_config.json file is not present the raise this error """
+    os.environ["INPUT_PARAMETERS_FILE"] = 'wrongfile.json'
+    import main
+    with pytest.raises(UserErrorException):
+        assert main.main()
+
+
+if __name__ == "__main__":
+    pass