adding task level cache and run level cache, fixed issue topological_…

…generations_without_scheduler stoping the workflow from stopping when tasks become blocked

.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+- repo: local
+  hooks:
+    - id: make-black
+      name: make-black
+      entry: make black
+      language: system
+      types: [ python ]
+      require_serial: true
+    - id: make-unit-tests
+      name: make-unit-tests
+      entry: make unit-tests
+      language: system
+      types: [ python ]
+      require_serial: true

Makefile.CodeQuality

@@ -1,9 +1,15 @@
-.PHONY: black pycodestyle
+.PHONY: black pycodestyle importanize
 
 ## @Code_quality Runs black on the checked out code
 black:
 	poetry run black servicecatalog_puppet
 
+## @Code_quality Runs importanize on the checked out code
+importanize:
+	poetry run importanize servicecatalog_puppet
+
 ## @Code_quality Runs pycodestyle on the the checked out code
 pycodestyle:
 	poetry run pycodestyle --statistics -qq servicecatalog_puppet
+
+pre-commit: importanize black unit-tests

pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "aws-service-catalog-puppet"
-version = "0.219.0"
+version = "0.220.0"
 description = "Making it easier to deploy ServiceCatalog products"
 classifiers = ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Natural Language :: English"]
 homepage = "https://service-catalog-tools-workshop.com/"
@@ -31,6 +31,7 @@ nose2 = "^0.10.0"
 
 [tool.poetry.group.dev.dependencies]
 viztracer = "^0.15.4"
+importanize = "^0.7.0"
 
 [tool.dephell.main]
 versioning = "semver"

servicecatalog_puppet/aws.py

@@ -11,6 +11,7 @@
 
 from servicecatalog_puppet import config, constants
 
+
 logger = logging.getLogger(__file__)
 
 

servicecatalog_puppet/cli.py

@@ -2,28 +2,33 @@
 #  SPDX-License-Identifier: Apache-2.0
 
 import json
-from servicecatalog_puppet import serialisation_utils
 import os
 import sys
 from datetime import datetime
 
 import click
 import yaml
 
-from servicecatalog_puppet import config, remote_config
-from servicecatalog_puppet import constants
-from servicecatalog_puppet import environmental_variables
-from servicecatalog_puppet.commands import bootstrap as bootstrap_commands
-from servicecatalog_puppet.commands import graph as graph_commands
-from servicecatalog_puppet.commands import management as management_commands
-from servicecatalog_puppet.commands import manifest as manifest_commands
-from servicecatalog_puppet.commands import misc as misc_commands
-from servicecatalog_puppet.commands import orgs as orgs_commands
-from servicecatalog_puppet.commands import show_codebuilds as show_codebuilds_commands
-from servicecatalog_puppet.commands import show_pipelines as show_pipelines_commands
-from servicecatalog_puppet.commands import spoke_management as spoke_management_commands
-from servicecatalog_puppet.commands import task_reference as task_reference_commands
-from servicecatalog_puppet.commands import version as version_commands
+from servicecatalog_puppet import (
+    config,
+    constants,
+    environmental_variables,
+    remote_config,
+    serialisation_utils,
+)
+from servicecatalog_puppet.commands import (
+    bootstrap as bootstrap_commands,
+    graph as graph_commands,
+    management as management_commands,
+    manifest as manifest_commands,
+    misc as misc_commands,
+    orgs as orgs_commands,
+    show_codebuilds as show_codebuilds_commands,
+    show_pipelines as show_pipelines_commands,
+    spoke_management as spoke_management_commands,
+    task_reference as task_reference_commands,
+    version as version_commands,
+)
 
 
 @click.group()
@@ -440,12 +445,36 @@ def setup_config(
         if regions != ""
         else remote_config.get_regions(puppet_account_id_to_use, home_region_to_use)
     )
-    if os.environ.get(environmental_variables.CACHE_INVALIDATOR):
+    if os.environ.get(environmental_variables.TASK_IDEMPOTENCY_TOKEN):
         click.echo(
-            f"Found existing {environmental_variables.CACHE_INVALIDATOR}: {os.environ.get(environmental_variables.CACHE_INVALIDATOR)}"
+            f"Found existing {environmental_variables.TASK_IDEMPOTENCY_TOKEN}: {os.environ.get(environmental_variables.TASK_IDEMPOTENCY_TOKEN)}"
         )
     else:
-        os.environ[environmental_variables.CACHE_INVALIDATOR] = str(datetime.now())
+        task_idempotency_token = remote_config.get_task_idempotency_token(
+            puppet_account_id_to_use, home_region_to_use
+        )
+        if task_idempotency_token is None:
+            task_idempotency_token = str(datetime.now())
+            click.echo(f"Created new task_idempotency_token: {task_idempotency_token}")
+        else:
+            click.echo(
+                f"Found existing task_idempotency_token: {task_idempotency_token}"
+            )
+        os.environ[
+            environmental_variables.TASK_IDEMPOTENCY_TOKEN
+        ] = task_idempotency_token
+
+    run_idempotency_token = os.environ.get(
+        environmental_variables.RUN_IDEMPOTENCY_TOKEN
+    )
+    if run_idempotency_token:
+        click.echo(f"Found existing run_idempotency_token: {run_idempotency_token}")
+    else:
+        run_idempotency_token = str(datetime.now())
+        os.environ[
+            environmental_variables.RUN_IDEMPOTENCY_TOKEN
+        ] = run_idempotency_token
+        click.echo(f"Created new run_idempotency_token: {run_idempotency_token}")
 
     os.environ[environmental_variables.SHOULD_USE_SNS] = (
         str(remote_config.get_should_use_sns(puppet_account_id_to_use, home_region))

servicecatalog_puppet/commands/bootstrap.py

@@ -3,7 +3,6 @@
 
 import io
 import json
-from servicecatalog_puppet import serialisation_utils
 import os
 import zipfile
 from threading import Thread
@@ -12,10 +11,10 @@
 import click
 from betterboto import client as betterboto_client
 
-from servicecatalog_puppet import config, constants
+from servicecatalog_puppet import config, constants, serialisation_utils
 from servicecatalog_puppet.template_builder.hub import (
-    bootstrap_region as hub_bootstrap_region,
     bootstrap as hub_bootstrap,
+    bootstrap_region as hub_bootstrap_region,
 )
 
 

servicecatalog_puppet/commands/graph.py

@@ -2,6 +2,7 @@
 #  SPDX-License-Identifier: Apache-2.0
 from servicecatalog_puppet import serialisation_utils
 
+
 colors = dict()
 
 default_color = "#88000022"

servicecatalog_puppet/commands/management.py

@@ -1,7 +1,6 @@
 #  Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #  SPDX-License-Identifier: Apache-2.0
 import glob
-from servicecatalog_puppet import serialisation_utils
 import os
 import shutil
 import zipfile
@@ -16,11 +15,12 @@
 from jinja2 import Template
 
 from servicecatalog_puppet import (
-    constants,
-    config,
     asset_helpers,
-    viz_template,
+    config,
+    constants,
     print_utils,
+    serialisation_utils,
+    viz_template,
 )
 
 

servicecatalog_puppet/commands/manifest.py

@@ -5,17 +5,21 @@
 import os
 
 import click
+import networkx as nx
 import requests
 import yamale
 import yaml
 from betterboto import client as betterboto_client
-import networkx as nx
-from servicecatalog_puppet import asset_helpers
-from servicecatalog_puppet import config
-from servicecatalog_puppet import constants
-from servicecatalog_puppet import print_utils
-from servicecatalog_puppet import manifest_utils
-from servicecatalog_puppet import serialisation_utils
+
+from servicecatalog_puppet import (
+    asset_helpers,
+    config,
+    constants,
+    manifest_utils,
+    print_utils,
+    serialisation_utils,
+)
+
 
 logger = logging.getLogger(__name__)
 

servicecatalog_puppet/commands/misc.py

@@ -2,7 +2,6 @@
 #  SPDX-License-Identifier: Apache-2.0
 
 import json
-from servicecatalog_puppet import serialisation_utils
 import logging
 import os
 import time
@@ -12,11 +11,8 @@
 import click
 from betterboto import client as betterboto_client
 
-from servicecatalog_puppet import (
-    aws,
-    config,
-    constants,
-)
+from servicecatalog_puppet import aws, config, constants, serialisation_utils
+
 
 logger = logging.getLogger(constants.PUPPET_LOGGER_NAME)
 
@@ -34,12 +30,12 @@ def cli(info, info_line_numbers):
 
     if info:
         logging.basicConfig(
-            format="%(levelname)s %(process)s %(threadName)s %(message)s",
+            format="%(levelname)s %(processName)s %(threadName)s %(message)s",
             level=logging.INFO,
         )
     if info_line_numbers:
         logging.basicConfig(
-            format="%(asctime)s %(levelname)s %(process)s %(threadName)s [%(filename)s:%(lineno)d] %(message)s",
+            format="%(asctime)s %(levelname)s %(processName)s %(threadName)s [%(filename)s:%(lineno)d] %(message)s",
             datefmt="%Y-%m-%d:%H:%M:%S",
             level=logging.INFO,
         )

servicecatalog_puppet/commands/orgs.py

@@ -7,7 +7,8 @@
 from betterboto import client as betterboto_client
 from jinja2 import Template
 
-from servicecatalog_puppet import constants, asset_helpers
+from servicecatalog_puppet import asset_helpers, constants
+
 
 logger = logging.getLogger(constants.PUPPET_LOGGER_NAME)
 

servicecatalog_puppet/commands/show_codebuilds.py

@@ -1,14 +1,14 @@
 #  Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #  SPDX-License-Identifier: Apache-2.0
 import json
-from servicecatalog_puppet import serialisation_utils
 import logging
 import operator
 
 import click
 from betterboto import client as betterboto_client
 
-from servicecatalog_puppet import constants
+from servicecatalog_puppet import constants, serialisation_utils
+
 
 logger = logging.getLogger(constants.PUPPET_LOGGER_NAME)
 

servicecatalog_puppet/commands/show_pipelines.py

@@ -2,14 +2,13 @@
 #  SPDX-License-Identifier: Apache-2.0
 import datetime
 import json
-from servicecatalog_puppet import serialisation_utils
 
 import click
 import terminaltables
 from betterboto import client as betterboto_client
 
-from servicecatalog_puppet import constants
-from servicecatalog_puppet import utils
+from servicecatalog_puppet import constants, serialisation_utils, utils
+
 
 result_look_up = dict(Failed="red", Succeeded="green")
 trigger_look_up = dict(StartPipelineExecution="Manual Full Run")

servicecatalog_puppet/commands/spoke_management.py

@@ -7,14 +7,13 @@
 from betterboto import client as betterboto_client
 from jinja2 import Template
 
-from servicecatalog_puppet import asset_helpers
-from servicecatalog_puppet import config
-from servicecatalog_puppet import constants
+from servicecatalog_puppet import asset_helpers, config, constants
 from servicecatalog_puppet.workflow import (
     management as management_tasks,
-    runner as runner,
+    runner,
 )
 
+
 logger = logging.getLogger(constants.PUPPET_LOGGER_NAME)