Add support for python up to 3.11 (#265)

* Add support for python up to 3.11

* update pre-commit

* Fix None type check from typing Optional

* Disable pylint deprecated option

* Force certain names in exceptions

* PR cleanup tasks

Author: kddejong

.github/workflows/pr-ci.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python: [ 3.7, 3.8, 3.9 ]
+        python: [ "3.8", "3.9", "3.10", "3.11" ]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python }}
@@ -34,11 +34,11 @@ jobs:
     - name: pre-commit checks
       run: |
         pre-commit run --all-files
-    - name: End to End Resource Packaging Test Python 3.6
-      run: ./e2e-test.sh python36
-    - name: End to End Resource Packaging Test Python 3.7
-      run: ./e2e-test.sh python37
     - name: End to End Resource Packaging Test Python 3.8
       run: ./e2e-test.sh python38
     - name: End to End Resource Packaging Test Python 3.9
       run: ./e2e-test.sh python39
+    - name: End to End Resource Packaging Test Python 3.10
+      run: ./e2e-test.sh python310
+    - name: End to End Resource Packaging Test Python 3.11
+      run: ./e2e-test.sh python311

.pre-commit-config.yaml

@@ -7,12 +7,12 @@ repos:
   - id: isort
     # language_version: python3.6
 - repo: https://github.com/psf/black
-  rev: 22.8.0
+  rev: 23.9.1
   hooks:
   - id: black
     exclude: templates/
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.1.0
+  rev: v4.4.0
   hooks:
   - id: check-case-conflict
   - id: end-of-file-fixer
@@ -28,7 +28,7 @@ repos:
   - id: check-merge-conflict
   - id: check-yaml
 - repo: https://github.com/pycqa/flake8
-  rev: "5.0.4"
+  rev: 6.1.0
   hooks:
   - id: flake8
     additional_dependencies:
@@ -41,18 +41,18 @@ repos:
     # language_version: python3.6
     exclude: templates/
 - repo: https://github.com/pre-commit/pygrep-hooks
-  rev: v1.9.0
+  rev: v1.10.0
   hooks:
   - id: python-check-blanket-noqa
   - id: python-check-mock-methods
   - id: python-no-log-warn
 - repo: https://github.com/PyCQA/bandit
-  rev: "1.7.1"
+  rev: 1.7.5
   hooks:
   - id: bandit
     files: ^(src|python)/
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.812
+  rev: v1.5.1
   hooks:
   - id: mypy
     files: ^src/

.pylintrc

@@ -9,7 +9,6 @@ persistent=yes
 disable=
     missing-docstring,          # not everything needs a docstring
     fixme,                      # work in progress
-    bad-continuation,           # clashes with black
     too-few-public-methods,     # triggers when inheriting
     ungrouped-imports,          # clashes with isort
     duplicate-code,             # broken, setup.py

python/rpdk/python/codegen.py

@@ -39,15 +39,15 @@ def validate_no(value):
     return value.lower() not in ("n", "no")
 
 
-class Python36LanguagePlugin(LanguagePlugin):
+class _PythonLanguagePlugin(LanguagePlugin):
     MODULE_NAME = __name__
-    NAME = "python36"
-    RUNTIME = "python3.6"
+    NAME = ""
+    RUNTIME = ""
     HOOK_ENTRY_POINT = "{}.handlers.hook"
     RESOURCE_ENTRY_POINT = "{}.handlers.resource"
     TEST_ENTRY_POINT = "{}.handlers.test_entrypoint"
     CODE_URI = "build/"
-    DOCKER_TAG = 3.6
+    DOCKER_TAG = ""
 
     def __init__(self):
         self.env = self._setup_jinja_env(
@@ -237,9 +237,8 @@ def _generate_target_models(self, project):
             target_name = "".join(
                 [s.capitalize() for s in target_namespace]
             )  # awssqsqueue -> AwsSqsQueue
-            target_model_file = "{}.py".format(
-                "_".join(target_namespace)
-            )  # awssqsqueue -> aws_sqs_queue.py
+            target_model_file = f'{"_".join(target_namespace)}.py'
+            # awssqsqueue -> aws_sqs_queue.py
 
             models = resolve_models(target_schema, target_name)
 
@@ -425,19 +424,25 @@ def _pip_build(cls, base_path):
         LOG.debug("--- pip stderr:\n%s", completed_proc.stderr)
 
 
-class Python37LanguagePlugin(Python36LanguagePlugin):
-    NAME = "python37"
-    RUNTIME = "python3.7"
-    DOCKER_TAG = 3.7
-
-
-class Python38LanguagePlugin(Python36LanguagePlugin):
+class Python38LanguagePlugin(_PythonLanguagePlugin):
     NAME = "python38"
     RUNTIME = "python3.8"
     DOCKER_TAG = 3.8
 
 
-class Python39LanguagePlugin(Python36LanguagePlugin):
+class Python39LanguagePlugin(_PythonLanguagePlugin):
     NAME = "python39"
     RUNTIME = "python3.9"
     DOCKER_TAG = 3.9
+
+
+class Python310LanguagePlugin(_PythonLanguagePlugin):
+    NAME = "python310"
+    RUNTIME = "python3.10"
+    DOCKER_TAG = 3.10
+
+
+class Python311LanguagePlugin(_PythonLanguagePlugin):
+    NAME = "python311"
+    RUNTIME = "python3.11"
+    DOCKER_TAG = 3.11

python/rpdk/python/parser.py

@@ -2,8 +2,8 @@ def setup_subparser(subparsers, parents, python_version, python_version_number):
     parser = subparsers.add_parser(
         python_version,
         description=(
-            "This sub command generates IDE and build files for Python "
-            "{}".format(python_version_number)
+            "This sub command generates IDE and build files for "
+            f"Python {python_version_number}"
         ),
         parents=parents,
     )
@@ -30,17 +30,17 @@ def setup_subparser(subparsers, parents, python_version, python_version_number):
     return parser
 
 
-def setup_subparser_python36(subparsers, parents):
-    return setup_subparser(subparsers, parents, "python36", "3.6")
-
-
-def setup_subparser_python37(subparsers, parents):
-    return setup_subparser(subparsers, parents, "python37", "3.7")
-
-
 def setup_subparser_python38(subparsers, parents):
     return setup_subparser(subparsers, parents, "python38", "3.8")
 
 
 def setup_subparser_python39(subparsers, parents):
     return setup_subparser(subparsers, parents, "python39", "3.9")
+
+
+def setup_subparser_python310(subparsers, parents):
+    return setup_subparser(subparsers, parents, "python310", "3.10")
+
+
+def setup_subparser_python311(subparsers, parents):
+    return setup_subparser(subparsers, parents, "python311", "3.11")

setup.py

@@ -36,20 +36,20 @@ def find_version(*file_paths):
     # package_data -> use MANIFEST.in instead
     include_package_data=True,
     zip_safe=True,
-    python_requires=">=3.6",
+    python_requires=">=3.8",
     install_requires=["cloudformation-cli>=0.2.26", "types-dataclasses>=0.1.5"],
     entry_points={
         "rpdk.v1.languages": [
+            "python311 = rpdk.python.codegen:Python311LanguagePlugin",
+            "python310 = rpdk.python.codegen:Python310LanguagePlugin",
             "python39 = rpdk.python.codegen:Python39LanguagePlugin",
             "python38 = rpdk.python.codegen:Python38LanguagePlugin",
-            "python37 = rpdk.python.codegen:Python37LanguagePlugin",
-            "python36 = rpdk.python.codegen:Python36LanguagePlugin",
         ],
         "rpdk.v1.parsers": [
+            "python311 = rpdk.python.parser:setup_subparser_python311",
+            "python310 = rpdk.python.parser:setup_subparser_python310",
             "python39 = rpdk.python.parser:setup_subparser_python39",
             "python38 = rpdk.python.parser:setup_subparser_python38",
-            "python37 = rpdk.python.parser:setup_subparser_python37",
-            "python36 = rpdk.python.parser:setup_subparser_python36",
         ],
     },
     license="Apache License 2.0",
@@ -63,8 +63,10 @@ def find_version(*file_paths):
         "Topic :: Software Development :: Code Generators",
         "Operating System :: OS Independent",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     keywords="Amazon Web Services AWS CloudFormation",
 )

src/cloudformation_cli_python_lib/hook.py

@@ -91,7 +91,8 @@ def _invoke_handler(  # pylint: disable=too-many-arguments
             handler = self._handlers[invocation_point]
         except KeyError:
             return ProgressEvent.failed(
-                HandlerErrorCode.InternalFailure, f"No handler for {invocation_point}"
+                HandlerErrorCode.InternalFailure,
+                f"No handler for {invocation_point.name}",
             )
 
         return handler(session, request, callback_context, type_configuration)
@@ -257,10 +258,10 @@ def print_or_log(message: str) -> None:
             print_or_log("Handler error")
             progress = e.to_progress_event()
         except Exception as e:  # pylint: disable=broad-except
-            print_or_log("Exception caught {0}".format(e))
+            print_or_log(f"Exception caught {e}")
             progress = ProgressEvent.failed(HandlerErrorCode.InternalFailure)
         except BaseException as e:  # pylint: disable=broad-except
-            print_or_log("Base exception caught (this is usually bad) {0}".format(e))
+            print_or_log(f"Base exception caught (this is usually bad) {e}")
             progress = ProgressEvent.failed(HandlerErrorCode.InternalFailure)
 
         # use the raw event_data as a last-ditch attempt to call back if the

src/cloudformation_cli_python_lib/identifier_utils.py

@@ -20,6 +20,7 @@ def generate_resource_identifier(
     max_length: int,
 ) -> str:
     if max_length < MIN_PHYSICAL_RESOURCE_ID_LENGTH:
+        # pylint: disable=broad-exception-raised
         raise Exception(
             f"Cannot generate resource IDs shorter than\
                {MIN_PHYSICAL_RESOURCE_ID_LENGTH} characters."

src/cloudformation_cli_python_lib/interface.py

@@ -10,6 +10,7 @@
 
 class _AutoName(Enum):
     @staticmethod
+    # pylint: disable=arguments-differ
     def _generate_next_value_(
         name: str, _start: int, _count: int, _last_values: List[str]
     ) -> str:

src/cloudformation_cli_python_lib/metrics.py

@@ -140,7 +140,7 @@ def __init__(self, session: SessionProxy, hook_type: str, account_id: str) -> No
         self._account_id = account_id
         self._namespace = self._make_hook_namespace(hook_type, account_id)
 
-    # pylint: disable=arguments-differ
+    # pylint: disable=arguments-differ,arguments-renamed
     def publish_exception_metric(  # type: ignore
         self,
         timestamp: datetime.datetime,

src/cloudformation_cli_python_lib/recast.py

@@ -129,7 +129,7 @@ def _field_to_type(field: Any, key: str, classes: Dict[str, Any]) -> Any:  # noq
     # Assuming that the union is generated from typing.Optional, so only
     # contains one type and None
     # pylint: disable=unidiomatic-typecheck
-    fields = [t for t in possible_types if type(None) != t]
+    fields = [t for t in possible_types if type(None) is not t]
     if len(fields) != 1:
         raise InvalidRequest(f"Cannot process type {field} for field {key}")
     field = fields[0]

src/cloudformation_cli_python_lib/resource.py

@@ -89,7 +89,7 @@ def _invoke_handler(
             handler = self._handlers[action]
         except KeyError:
             return ProgressEvent.failed(
-                HandlerErrorCode.InternalFailure, f"No handler for {action}"
+                HandlerErrorCode.InternalFailure, f"No handler for {action.name}"
             )
         progress = handler(session, request, callback_context)
         is_in_progress = progress.status == OperationStatus.IN_PROGRESS
@@ -227,10 +227,10 @@ def print_or_log(message: str) -> None:
             print_or_log("Handler error")
             progress = e.to_progress_event()
         except Exception as e:  # pylint: disable=broad-except
-            print_or_log("Exception caught {0}".format(e))
+            print_or_log(f"Exception caught {e}")
             progress = ProgressEvent.failed(HandlerErrorCode.InternalFailure)
         except BaseException as e:  # pylint: disable=broad-except
-            print_or_log("Base exception caught (this is usually bad) {0}".format(e))
+            print_or_log(f"Base exception caught (this is usually bad) {e}")
             progress = ProgressEvent.failed(HandlerErrorCode.InternalFailure)
 
         if progress.result:  # pragma: no cover

src/setup.py

@@ -29,6 +29,10 @@
         "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     keywords="Amazon Web Services AWS CloudFormation",
 )

tests/lib/metrics_test.py

@@ -382,4 +382,4 @@ def test_metrics_publisher_proxy_add_metrics_publisher_none_safe():
     proxy = MetricsPublisherProxy()
     proxy.add_metrics_publisher(None, None)
     proxy.add_hook_metrics_publisher(None, None, None)
-    assert proxy._publishers == []  # pylint: disable=protected-access
+    assert not proxy._publishers  # pylint: disable=protected-access

tests/lib/utils_test.py

@@ -141,7 +141,7 @@ def test_hook_handler_request_serde_roundtrip():
             else json.loads(v)
             if k.endswith("Credentials")
             else v
-            for k, v in payload[k].items()
+            for k, v in v.items()
             if v is not None and k not in undesired
         }
         if k in ("requestData", "requestContext")

tests/plugin/codegen_test.py

@@ -13,7 +13,7 @@
 from rpdk.python.codegen import (
     SUPPORT_LIB_NAME,
     SUPPORT_LIB_PKG,
-    Python36LanguagePlugin as PythonLanguagePlugin,
+    _PythonLanguagePlugin as PythonLanguagePlugin,
     validate_no,
 )
 from shutil import copyfile