Python 3.12 Upgrade (#4164)

* Python version upgraded from 3.8 to 3.11 * Python version changes done in build_docker_images.yml file * Testng 3.8 version test results * Testing 3.8 tests results * python 3.11 testing * Final Commit for python upgrade * requirements.txt updated for testing * bullseye to bookworm python update * service bus version update * rolledback requirement.txt file * resource procesor change * Bookworm to Bullseye Changes * 3.12 version upgrade for python * airlockfiles changes * mlflow changes * remove stuff * add in 'the' * update * update requirements * updaet * update * update * update pytest versions * update starlette * update * update mlflow * update * update vmss_porter * update * update core version * python updates * update StrEnum * fix api messages * add in user, edit output.sh, revert schema_extra * updates * update oydantic version * update * update * update * linting * update * update changelog * remove pydantic chnages * update * update * update cli dependencies * update version * update * remove setuptools test * remove --------- Co-authored-by: Neosoft <gaurav.mishra@cloudkubed.com> Co-authored-by: gauravmishrack <gaurav.mishra@cloudkued.com> Co-authored-by: Tim Allen <tallenp@me.com>
microsoft · Dec 13, 2024 · 6c7a450 · 6c7a450
1 parent fbc4b55
commit 6c7a450
Show file tree

Hide file tree

Showing 30 changed files with 95 additions and 94 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,7 +1,7 @@
 # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.166.1/containers/python-3/.devcontainer/base.Dockerfile
 
-# [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6
-ARG VARIANT="3.8"
+# [Choice] Python version: 3.11 3.12, 3.13
+ARG VARIANT="3.12"
 ARG TARGETPLATFORM="linux/amd64"
 FROM --platform="${TARGETPLATFORM}" mcr.microsoft.com/vscode/devcontainers/python:dev-${VARIANT}-bullseye
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,7 @@
 
 **BREAKING CHANGES & MIGRATIONS**:
 * InnerEye and MLFlow bundles depreciated and removed from main. If you wish to update and deploy these worksapce services they can be retrieved from release 0.19.1. ([#4127](https://github.com/microsoft/AzureTRE/issues/4127))
+* Upgrade Python version from 3.8 to 3.12 ([#3949](https://github.com/microsoft/AzureTRE/issues/3949))
 
 FEATURES:
 
@@ -17,6 +18,7 @@ ENHANCEMENTS:
 * Update Guacamole version and dependencies ([#4140](https://github.com/microsoft/AzureTRE/issues/4140))
 * Add partial (core resources only) support for customer-managed keys  ([#4141](https://github.com/microsoft/AzureTRE/issues/4142),  [#4144](https://github.com/microsoft/AzureTRE/issues/4144))
 * Update the Azure CLI version to 2.67.0 in dev container and vmss ([#4157](https://github.com/microsoft/AzureTRE/pull/4157))
+* Upgrade Python version from 3.8 to 3.12 ([#3949](https://github.com/microsoft/AzureTRE/issues/3949))
 * Move Github PR bot commands into main documentation ([#4167](https://github.com/microsoft/AzureTRE/pull/4167))
 * Block Authentication with keys to CosmosDB SQL account ([#4175](https://github.com/microsoft/AzureTRE/pull/4175))
 * Add support for customer-managed keys encryption in base workspace ([#4161](https://github.com/microsoft/AzureTRE/pull/4161))

diff --git a/airlock_processor/Dockerfile b/airlock_processor/Dockerfile
@@ -1,6 +1,6 @@
 # To enable ssh & remote debugging on app service change the base image to the one below
-# FROM mcr.microsoft.com/azure-functions/python:4-python3.8-appservice AS base
-FROM mcr.microsoft.com/azure-functions/python:4-python3.8-slim AS base
+# FROM mcr.microsoft.com/azure-functions/python:4-python3.8-appservice as base
+FROM mcr.microsoft.com/azure-functions/python:4-python3.12 AS base
 
 COPY requirements.txt /
 RUN pip install --no-cache-dir -r /requirements.txt

diff --git a/airlock_processor/_version.py b/airlock_processor/_version.py
@@ -1 +1 @@
-__version__ = "0.7.3"
+__version__ = "0.8.0"
diff --git a/airlock_processor/requirements-dev.txt b/airlock_processor/requirements-dev.txt
@@ -1,3 +1,3 @@
 # Dev requirements
-pytest==7.2.0
-mock==5.0.0
+pytest==8.3.3
+mock==5.1.0
diff --git a/airlock_processor/requirements.txt b/airlock_processor/requirements.txt
@@ -1,8 +1,8 @@
 # Do not include azure-functions-worker as it may conflict with the Azure Functions platform
-azure-core==1.30.0
+azure-core==1.31.0
 azure-functions==1.17.0
-azure-storage-blob==12.19.0
-azure-identity==1.16.1
-azure-mgmt-storage==21.1.0
-azure-mgmt-resource==23.0.1
-pydantic==1.10.13
+azure-storage-blob==12.23.1
+azure-identity==1.19.0
+azure-mgmt-storage==21.2.1
+azure-mgmt-resource==23.2.0
+pydantic==1.10.19
diff --git a/api_app/Dockerfile b/api_app/Dockerfile
@@ -1,4 +1,5 @@
-FROM python:3.8-slim-bullseye AS base
+FROM python:3.12-slim-bullseye AS base
+
 COPY requirements.txt /.
 RUN pip3 install --no-cache-dir -r requirements.txt
 

diff --git a/api_app/_version.py b/api_app/_version.py
@@ -1 +1 @@
-__version__ = "0.19.4"
+__version__ = "0.20.0"
diff --git a/api_app/models/domain/airlock_operations.py b/api_app/models/domain/airlock_operations.py
@@ -1,8 +1,7 @@
 from pydantic import Field
 from pydantic.types import UUID4
-from pydantic.schema import Optional
 from models.domain.azuretremodel import AzureTREModel
-from typing import List
+from typing import List, Optional
 from models.domain.airlock_request import AirlockFile
 
 

diff --git a/api_app/models/domain/airlock_request.py b/api_app/models/domain/airlock_request.py
@@ -1,13 +1,12 @@
-from enum import Enum
-from typing import List, Dict
+from enum import StrEnum
+from typing import List, Dict, Optional
 
 from models.domain.azuretremodel import AzureTREModel
 from pydantic import Field, validator
-from pydantic.schema import Optional
 from resources import strings
 
 
-class AirlockRequestStatus(str, Enum):
+class AirlockRequestStatus(StrEnum):
     """
     Airlock Resource status
     """
@@ -25,12 +24,12 @@ class AirlockRequestStatus(str, Enum):
     Failed = strings.AIRLOCK_RESOURCE_STATUS_FAILED
 
 
-class AirlockRequestType(str, Enum):
+class AirlockRequestType(StrEnum):
     Import = strings.AIRLOCK_REQUEST_TYPE_IMPORT
     Export = strings.AIRLOCK_REQUEST_TYPE_EXPORT
 
 
-class AirlockActions(str, Enum):
+class AirlockActions(StrEnum):
     Review = strings.AIRLOCK_ACTION_REVIEW
     Cancel = strings.AIRLOCK_ACTION_CANCEL
     Submit = strings.AIRLOCK_ACTION_SUBMIT
@@ -41,7 +40,7 @@ class AirlockFile(AzureTREModel):
     size: float = Field(title="size", description="size of the file in bytes")
 
 
-class AirlockReviewDecision(str, Enum):
+class AirlockReviewDecision(StrEnum):
     Approved = strings.AIRLOCK_REVIEW_DECISION_APPROVED
     Rejected = strings.AIRLOCK_REVIEW_DECISION_REJECTED
 

diff --git a/api_app/models/domain/costs.py b/api_app/models/domain/costs.py
@@ -1,17 +1,17 @@
 from datetime import datetime, timedelta, date
 from typing import List, Optional
 from pydantic import BaseModel
-from enum import Enum
+from enum import StrEnum
 import random
 import uuid
 
 
-class GranularityEnum(str, Enum):
+class GranularityEnum(StrEnum):
     daily = "Daily"
     none = "None"
 
 
-class CurrencyEnum(str, Enum):
+class CurrencyEnum(StrEnum):
     USD = "USD"
     ILS = "ILS"
 

diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py
@@ -1,4 +1,4 @@
-from enum import Enum
+from enum import StrEnum
 from typing import List, Optional
 
 from pydantic import Field
@@ -9,7 +9,7 @@
 from resources import strings
 
 
-class Status(str, Enum):
+class Status(StrEnum):
     """
     Operation status
     """

diff --git a/api_app/models/domain/request_action.py b/api_app/models/domain/request_action.py
@@ -1,7 +1,7 @@
-from enum import Enum
+from enum import StrEnum
 
 
-class RequestAction(str, Enum):
+class RequestAction(StrEnum):
     Install = "install"
     UnInstall = "uninstall"
     Upgrade = "upgrade"
diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py
@@ -1,12 +1,12 @@
-from enum import Enum
+from enum import StrEnum
 from typing import Optional, Union, List
 from pydantic import BaseModel, Field, validator
 from models.domain.azuretremodel import AzureTREModel
 from models.domain.request_action import RequestAction
 from resources import strings
 
 
-class ResourceType(str, Enum):
+class ResourceType(StrEnum):
     """
     Type of resource to deploy
     """

diff --git a/api_app/models/domain/shared_service.py b/api_app/models/domain/shared_service.py
@@ -5,4 +5,4 @@ class SharedService(Resource):
     """
     Shared service request
     """
-    resourceType = ResourceType.SharedService
+    resourceType: ResourceType = ResourceType.SharedService
diff --git a/api_app/models/domain/user_resource.py b/api_app/models/domain/user_resource.py
@@ -11,4 +11,4 @@ class UserResource(Resource):
     ownerId: str = Field("", title="Owner of the user resource")
     parentWorkspaceServiceId: str = Field("", title="Parent Workspace Service ID", description="Service target Workspace Service id")
     azureStatus: dict = Field({}, title="Azure Status", description="Azure status, varies per user resource")
-    resourceType = ResourceType.UserResource
+    resourceType: ResourceType = ResourceType.UserResource
diff --git a/api_app/models/domain/user_resource_template.py b/api_app/models/domain/user_resource_template.py
@@ -6,4 +6,4 @@
 
 class UserResourceTemplate(ResourceTemplate):
     parentWorkspaceService: str = Field("", title="Parent Workspace Service", description="The parent workspace service under which services with this template can be created")
-    resourceType = ResourceType.UserResource
+    resourceType: ResourceType = ResourceType.UserResource
diff --git a/api_app/models/domain/workspace.py b/api_app/models/domain/workspace.py
@@ -16,7 +16,7 @@ class Workspace(Resource):
     Workspace request
     """
     workspaceURL: str = Field("", title="Workspace URL", description="Main endpoint for workspace users")
-    resourceType = ResourceType.Workspace
+    resourceType: ResourceType = ResourceType.Workspace
 
 
 class WorkspaceAuth(AzureTREModel):

diff --git a/api_app/models/domain/workspace_service.py b/api_app/models/domain/workspace_service.py
@@ -8,4 +8,4 @@ class WorkspaceService(Resource):
     Workspace service request
     """
     workspaceId: str = Field("", title="Workspace ID", description="Service target Workspace id")
-    resourceType = ResourceType.WorkspaceService
+    resourceType: ResourceType = ResourceType.WorkspaceService
diff --git a/api_app/models/schemas/status.py b/api_app/models/schemas/status.py
@@ -1,12 +1,12 @@
-from enum import Enum
+from enum import StrEnum
 from typing import List
 
 from pydantic import BaseModel
 
 from resources import strings
 
 
-class StatusEnum(str, Enum):
+class StatusEnum(StrEnum):
     ok = strings.OK
     not_ok = strings.NOT_OK
 

diff --git a/api_app/models/schemas/workspace.py b/api_app/models/schemas/workspace.py
@@ -1,4 +1,4 @@
-from enum import Enum
+from enum import StrEnum
 from typing import List
 
 from pydantic import BaseModel, Field
@@ -23,7 +23,7 @@ def get_sample_workspace(workspace_id: str, spec_workspace_id: str = "0001") ->
     }
 
 
-class AuthProvider(str, Enum):
+class AuthProvider(StrEnum):
     """
     Auth Provider
     """

diff --git a/api_app/requirements-dev.txt b/api_app/requirements-dev.txt
@@ -1,5 +1,5 @@
 # Dev requirements
-pytest-asyncio==0.21.1
+pytest-asyncio==0.24.0
 httpx==0.25.0
 mock==5.1.0
-pytest==7.4.3
+pytest==8.3.3
diff --git a/api_app/requirements.txt b/api_app/requirements.txt
@@ -1,25 +1,25 @@
-aiohttp==3.9.4
-azure-core==1.30.0
-azure-cosmos==4.5.1
-azure-eventgrid==4.15.0
-azure-identity==1.16.1
-azure-mgmt-compute==30.3.0
-azure-mgmt-cosmosdb==9.3.0
+aiohttp==3.10.10
+azure-core==1.31.0
+azure-cosmos==4.7.0
+azure-eventgrid==4.20.0
+azure-identity==1.19.0
+azure-mgmt-compute==33.0.0
+azure-mgmt-cosmosdb==9.6.0
 azure-mgmt-costmanagement==4.0.1
-azure-mgmt-resource==23.0.1
-azure-mgmt-storage==21.1.0
-azure-monitor-opentelemetry~=1.6
-azure-servicebus==7.11.3
-azure-storage-blob==12.19.0
-fastapi==0.110.0
-fastapi-utils==0.2.1
-gunicorn==22.0.0
+azure-mgmt-resource==23.2.0
+azure-mgmt-storage==21.2.1
+azure-monitor-opentelemetry==1.6.4
+azure-servicebus==7.12.3
+azure-storage-blob==12.23.1
+fastapi==0.115.3
+gunicorn==23.0.0
 jsonschema[format_nongpl]==4.19.1
-msal==1.26.0
-opentelemetry.instrumentation.logging>=0.44b0
-pandas==2.0.3
-PyJWT==2.8.0
-pytz==2022.7
-python-dateutil==2.8.2
+msal==1.31.0
+opentelemetry-instrumentation-logging==0.49b2
+pandas==2.2.2
+PyJWT==2.9.0
+pytz==2024.2
+python-dateutil==2.9.0
 semantic-version==2.10.0
-uvicorn[standard]==0.23.2
+uvicorn[standard]==0.31.0
+pydantic==1.10.19
diff --git a/cli/requirements.txt b/cli/requirements.txt
@@ -1,11 +1,11 @@
 # if you update this file, update the install_requires in setup.py as well
 click==8.1.3
-httpx~=0.23.0
-msal==1.26.0
+httpx==0.25.0
+msal==1.31.0
 jmespath==1.0.1
 tabulate==0.9.0
 pygments==2.16.1
-PyJWT==2.8.0
-azure-cli-core==2.57.0
-azure-identity==1.16.1
-aiohttp==3.9.4
+PyJWT==2.9.0
+azure-cli-core==2.65.0
+azure-identity==1.19.0
+aiohttp==3.10.10
diff --git a/cli/setup.py b/cli/setup.py
@@ -4,7 +4,7 @@
 from setuptools import setup
 
 PROJECT = 'azure-tre-cli'
-VERSION = '0.2.3'
+VERSION = '0.2.4'
 
 try:
     long_description = open('README.md', 'rt').read()
@@ -42,14 +42,14 @@
     install_requires=[
         "click==8.1.3",
         "httpx==0.25.0",
-        "msal==1.26.0",
+        "msal==1.31.0",
         "jmespath==1.0.1",
         "tabulate==0.9.0",
         "pygments==2.16.1",
-        "PyJWT==2.8.0",
-        "azure-cli-core==2.57.0",
-        "azure-identity==1.14.1",
-        "aiohttp==3.9.4"
+        "PyJWT==2.9.0",
+        "azure-cli-core==2.65.0",
+        "azure-identity==1.19.0",
+        "aiohttp==3.10.10"
     ],
 
     namespace_packages=[],

diff --git a/e2e_tests/requirements.txt b/e2e_tests/requirements.txt
@@ -1,8 +1,8 @@
 # API
 httpx==0.25.0
-pytest==7.4.3
-pytest-asyncio==0.21.1
-starlette==0.36.3
+pytest==8.3.3
+pytest-asyncio==0.24.0
+starlette==0.41.2
 pytest-timeout==2.2.0
 pytest-xdist==3.3.1
 backoff==2.2.1
diff --git a/resource_processor/_version.py b/resource_processor/_version.py
@@ -1 +1 @@
-__version__ = "0.9.5"
+__version__ = "0.10.0"
diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile
@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM python:3.8-slim-bullseye
+FROM python:3.12-slim-bullseye
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
@@ -34,10 +34,10 @@ ENV PORTER_HOME_V0 ${PORTER_HOME}
 # can't be in a non default path
 # ARG PORTER_HOME_V1=/home/$USERNAME/.porter-v1/
 ARG PORTER_HOME_V1=/root/.porter/
-ARG PORTER_VERSION=v1.0.15
-ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.2
-ARG PORTER_AZ_MIXIN_VERSION=v1.0.1
-ARG PORTER_AZURE_PLUGIN_VERSION=v1.2.0
+ARG PORTER_VERSION=v1.1.1
+ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.3
+ARG PORTER_AZ_MIXIN_VERSION=v1.0.2
+ARG PORTER_AZURE_PLUGIN_VERSION=v1.2.3
 COPY scripts/porter-v1.sh /tmp/
 RUN export PORTER_VERSION=${PORTER_VERSION} \
     PORTER_TERRAFORM_MIXIN_VERSION=${PORTER_TERRAFORM_MIXIN_VERSION} \

diff --git a/resource_processor/vmss_porter/requirements.txt b/resource_processor/vmss_porter/requirements.txt
@@ -1,6 +1,6 @@
-aiohttp==3.9.4
-azure-cli-core==2.57.0
-azure-identity==1.16.1
-azure-monitor-opentelemetry~=1.6
-azure-servicebus==7.11.3
-opentelemetry.instrumentation.logging>=0.44b0
+aiohttp==3.10.10
+azure-cli-core==2.65.0
+azure-identity==1.19.0
+azure-monitor-opentelemetry==1.6.4
+azure-servicebus==7.12.3
+opentelemetry-instrumentation-logging==0.49b2