Fix get_result(wait=True) and support Python 3.12 (#1949)

* Support Python 3.12 and drop Python <3.10

* Remove Codecov from CI

* API: refactor and fix `get_result(wait=True)`

The previous `GET /dispatches/{dispatch_id}` endpoint was trying to do
too much. Its responsibilities are now separated into two endpoints:

* `GET /dispatches`: bulk query dispatch summaries (including status)
with options to filter by `dispatch_id`, sort chronologically, and
also limit the output to status only.

* `GET /dispatches/{dispatch_id}`: download manifest

To achieve the desired behavior of `get_result(id, wait=True)`, the
client

1. Polls the dispatch status by querying the first endpoint.

2. Downloads the manifest after the dispatch has reached a final
status.

The server no longer returns 503 errors when the dispatch is not yet
"ready". A 503 status code is not entirely accurate here because it is
intended to convey temporary service unavailablity resulting from
server overload or rate limiting. However, the fact that the workflow
is still running does not indicate any fault of the server.

These changes will allow `get_result(dispatch_id, wait=True)` to wait
as long as required instead of erroring out after some time.

Supporting improvements:

DAL: Add sorting and pagination to Controller

DAL: improve bulk get when retrieving only some columns

Directly select the specified columns instead of retrieving the whole
ORM entities and deferring column loading using load_only

* Remove unused code

* Update more python versions

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/release.yml

@@ -164,7 +164,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: "3.10"
 
       - name: Install Python dependencies
         run: |
@@ -174,7 +174,7 @@ jobs:
       - name: Set up Node
         uses: actions/setup-node@v2
         with:
-          node-version: 16
+          node-version: 20
 
       - name: Build Webapp
         run: |

.github/workflows/requirements.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: "3.10"
 
       - name: Install covalent
         run: python -m pip install .
@@ -47,7 +47,6 @@ jobs:
           --ignore-module=covalent_ui.*
           --ignore-module=covalent.*
           --ignore-module=urllib3.*
-          --ignore-module=pkg_resources
           --ignore-module=covalent/_dispatcher_plugins
           --ignore-module=covalent/_shared_files
           --ignore-file=covalent/quantum/**

.github/workflows/test_matrix.json

@@ -1,20 +1,4 @@
 [
-  {
-    "name": "Debian 12 / Python 3.9 / Dask",
-    "os": "ubuntu-latest",
-    "container": "ghcr.io/agnostiqhq/covalent-dev/debian12-py39:latest",
-    "backend": "dask",
-    "experimental": false,
-    "trigger": ["schedule", "workflow_dispatch"]
-  },
-  {
-    "name": "Debian 12 / Python 3.9 / Local",
-    "os": "ubuntu-latest",
-    "container": "ghcr.io/agnostiqhq/covalent-dev/debian12-py39:latest",
-    "backend": "local",
-    "experimental": false,
-    "trigger": ["schedule", "workflow_dispatch"]
-  },
   {
     "name": "Debian 12 / Python 3.10 / Dask",
     "os": "ubuntu-latest",
@@ -48,147 +32,163 @@
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 12 / Python 3.9 / Dask",
-    "os": "macos-12",
-    "python-version": "3.9",
+    "name": "Debian 12 / Python 3.12 / Dask",
+    "os": "ubuntu-latest",
+    "container": "ghcr.io/agnostiqhq/covalent-dev/debian12-py312:latest",
     "backend": "dask",
     "experimental": false,
-    "trigger": ["schedule", "workflow_dispatch"]
+    "trigger": ["push", "schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 12 / Python 3.9 / Local",
-    "os": "macos-12",
-    "python-version": "3.9",
+    "name": "Debian 12 / Python 3.12 / Local",
+    "os": "ubuntu-latest",
+    "container": "ghcr.io/agnostiqhq/covalent-dev/debian12-py312:latest",
     "backend": "local",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 12 / Python 3.10 / Dask",
-    "os": "macos-12",
+    "name": "MacOS 13 / Python 3.10 / Dask",
+    "os": "macos-13",
     "python-version": "3.10",
     "backend": "dask",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 12 / Python 3.10 / Local",
-    "os": "macos-12",
+    "name": "MacOS 13 / Python 3.10 / Local",
+    "os": "macos-13",
     "python-version": "3.10",
     "backend": "local",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 12 / Python 3.11 / Dask",
-    "os": "macos-12",
+    "name": "MacOS 13 / Python 3.11 / Dask",
+    "os": "macos-13",
     "python-version": "3.11",
     "backend": "dask",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 12 / Python 3.11 / Local",
-    "os": "macos-12",
+    "name": "MacOS 13 / Python 3.11 / Local",
+    "os": "macos-13",
     "python-version": "3.11",
     "backend": "local",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 13 / Python 3.9 / Dask",
+    "name": "MacOS 13 / Python 3.12 / Dask",
     "os": "macos-13",
-    "python-version": "3.9",
+    "python-version": "3.12",
     "backend": "dask",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 13 / Python 3.9 / Local",
+    "name": "MacOS 13 / Python 3.11 / Local",
     "os": "macos-13",
-    "python-version": "3.9",
+    "python-version": "3.11",
     "backend": "local",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 13 / Python 3.10 / Dask",
-    "os": "macos-13",
+    "name": "MacOS 14 / Python 3.10 / Dask",
+    "os": "macos-14",
     "python-version": "3.10",
     "backend": "dask",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 13 / Python 3.10 / Local",
-    "os": "macos-12",
+    "name": "MacOS 14 / Python 3.10 / Local",
+    "os": "macos-14",
     "python-version": "3.10",
     "backend": "local",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 13 / Python 3.11 / Dask",
-    "os": "macos-12",
+    "name": "MacOS 14 / Python 3.11 / Dask",
+    "os": "macos-14",
     "python-version": "3.11",
     "backend": "dask",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 13 / Python 3.11 / Local",
-    "os": "macos-13",
+    "name": "MacOS 14 / Python 3.11 / Local",
+    "os": "macos-14",
     "python-version": "3.11",
     "backend": "local",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 14 / Python 3.9 / Dask",
+    "name": "MacOS 14 / Python 3.12 / Dask",
     "os": "macos-14",
-    "python-version": "3.9",
+    "python-version": "3.12",
     "backend": "dask",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 14 / Python 3.9 / Local",
+    "name": "MacOS 14 / Python 3.12 / Local",
     "os": "macos-14",
-    "python-version": "3.9",
+    "python-version": "3.12",
     "backend": "local",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 14 / Python 3.10 / Dask",
-    "os": "macos-14",
+    "name": "MacOS 15 / Python 3.10 / Dask",
+    "os": "macos-15",
     "python-version": "3.10",
     "backend": "dask",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 14 / Python 3.10 / Local",
-    "os": "macos-14",
+    "name": "MacOS 15 / Python 3.10 / Local",
+    "os": "macos-15",
     "python-version": "3.10",
     "backend": "local",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 14 / Python 3.11 / Dask",
-    "os": "macos-12",
+    "name": "MacOS 15 / Python 3.11 / Dask",
+    "os": "macos-15",
     "python-version": "3.11",
     "backend": "dask",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
   },
   {
-    "name": "MacOS 14 / Python 3.11 / Local",
-    "os": "macos-14",
+    "name": "MacOS 15 / Python 3.11 / Local",
+    "os": "macos-15",
     "python-version": "3.11",
     "backend": "local",
     "experimental": false,
     "trigger": ["schedule", "workflow_dispatch"]
+  },
+  {
+    "name": "MacOS 15 / Python 3.12 / Dask",
+    "os": "macos-15",
+    "python-version": "3.12",
+    "backend": "dask",
+    "experimental": false,
+    "trigger": ["schedule", "workflow_dispatch"]
+  },
+  {
+    "name": "MacOS 15 / Python 3.12 / Local",
+    "os": "macos-15",
+    "python-version": "3.12",
+    "backend": "local",
+    "experimental": false,
+    "trigger": ["schedule", "workflow_dispatch"]
   }
 ]

.github/workflows/tests.yml

@@ -64,8 +64,6 @@ jobs:
       COVALENT_SERVER_IFACE_ANY: true
     container: ${{ matrix.container }}
     continue-on-error: ${{ matrix.experimental }}
-    outputs:
-      codecov: ${{ steps.local-codecov.outputs.local_codecov }}
     steps:
       - name: Check out the default branch
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.commit_sha == ''
@@ -309,85 +307,6 @@ jobs:
       - name: Dump Covalent logs
         run: covalent logs
 
-      - name: Upload SDK report to Codecov
-        id: upload-sdk-report
-        if: >
-          env.RECOMMENDED_PLATFORM
-          && (github.event_name == 'workflow_call'
-          || steps.sdk-coverage.outcome == 'success')
-        uses: codecov/codecov-action@v3
-        with:
-          files: ./sdk_coverage.xml
-          flags: SDK
-          name: "SDK Unit Tests"
-          fail_ci_if_error: true
-
-      - name: Upload Dispatcher report to Codecov
-        id: upload-dispatcher-report
-        if: >
-          env.RECOMMENDED_PLATFORM
-          && (github.event_name == 'workflow_call'
-          || steps.dispatcher-coverage.outcome == 'success')
-        uses: codecov/codecov-action@v3
-        with:
-          files: ./dispatcher_coverage.xml
-          flags: Dispatcher
-          name: "Dispatcher Unit Tests"
-          fail_ci_if_error: true
-
-      - name: Upload Functional report to Codecov
-        id: upload-functional-report
-        if: >
-          env.RECOMMENDED_PLATFORM
-          && steps.functional-coverage.outcome == 'success'
-        uses: codecov/codecov-action@v3
-        with:
-          files: ./functional_tests_coverage.xml
-          flags: Functional_Tests
-          name: "Functional Tests"
-          fail_ci_if_error: true
-
-      - name: Upload UI backend report to Codecov
-        id: upload-ui-backend-report
-        if: >
-          env.RECOMMENDED_PLATFORM
-          && (github.event_name == 'workflow_call'
-          || steps.ui-backend-coverage.outcome == 'success')
-        uses: codecov/codecov-action@v3
-        with:
-          files: ./ui_backend_coverage.xml
-          flags: UI_Backend
-          name: "UI Backend Unit Tests"
-          fail_ci_if_error: true
-
-      - name: Upload UI frontend report to Codecov
-        id: upload-ui-frontend-report
-        if: >
-          env.RECOMMENDED_PLATFORM
-          && (github.event_name == 'workflow_call'
-          || steps.ui-frontend-tests.outcome == 'success')
-        uses: codecov/codecov-action@v3
-        with:
-          files: ./covalent_ui/webapp/coverage/clover.xml
-          flags: UI_Frontend
-          name: "UI Frontend Unit Tests"
-          fail_ci_if_error: true
-
-      - name: Local Codecov
-        id: local-codecov
-        if: >
-          env.RECOMMENDED_PLATFORM
-          && github.event_name == 'workflow_call'
-        run: |
-          if ${{ steps.upload-sdk-report.outcome == 'skipped'
-            && steps.upload-dispatcher-report.outcome == 'skipped'
-            && steps.upload-functional-report.outcome == 'skipped'
-            && steps.upload-ui-backend-report.outcome == 'skipped'
-            && steps.upload-ui-frontend-report.outcome  == 'skipped' }} ; then
-            local_codecov=true
-          fi
-          echo "::set-output name=local_codecov::${local_codecov}"
-
       - name: Alert Slack
         if: >
           github.event_name == 'workflow_call'
@@ -400,19 +319,3 @@ jobs:
           SLACK_COLOR: ${{ job.status }}
           SLACK_TITLE: ":warning:  Attention Required  :warning:"
           SLACK_WEBHOOK: ${{ secrets.SLACK_ALERT_WEBHOOK }}
-
-  codecov_patch:
-    needs: tests
-    name: "codecov/patch"
-    if: needs.tests.outputs.codecov == 'true'
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo "No report needed."
-
-  codecov_project:
-    needs: tests
-    name: "codecov/project"
-    if: needs.tests.outputs.codecov == 'true'
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo "No report needed."

.pre-commit-config.yaml

@@ -58,7 +58,7 @@ repos:
           )$
 
   - repo: https://github.com/hadialqattan/pycln
-    rev: v2.4.0
+    rev: v2.5.0
     hooks:
       - id: pycln
         args: [--config=pyproject.toml]