chore: another attempt at fixing pip-compile-multi

both @john-bodley and I have tried to fix the pip-compile-multi flow
that we need to manage our complex network of python dependencies.

Last attempt was here -> #26944

Here I'm reopening the PR on the main fork so we can collaborate on it
more easily.

requirements/base.in

@@ -20,3 +20,5 @@
 urllib3>=1.26.18
 werkzeug>=3.0.1
 numexpr>=2.9.0
+urllib3>=1.25.4,<1.27 # botocore 1.34.32
+referencing>=0.28.0,<0.32.0 # jsonschema-path 0.3.2

requirements/docker.txt

@@ -10,13 +10,13 @@
     # via
     #   -r requirements/base.in
     #   -r requirements/docker.in
-gevent==23.9.1
+gevent==24.2.1
     # via apache-superset
 psycopg2-binary==2.9.6
     # via apache-superset
-zope-event==4.5.0
+zope-event==5.0
     # via gevent
-zope-interface==5.4.0
+zope-interface==6.2
     # via gevent
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/integration.txt

@@ -5,69 +5,76 @@
 #
 #    pip-compile-multi
 #
-build==0.10.0
+build==1.1.1
     # via pip-tools
-cachetools==5.3.2
+cachetools==5.3.3
     # via tox
-cfgv==3.3.1
+cfgv==3.4.0
     # via pre-commit
-chardet==5.1.0
+chardet==5.2.0
     # via tox
-click==8.1.3
+click==8.1.7
     # via
     #   pip-compile-multi
     #   pip-tools
 colorama==0.4.6
     # via tox
-distlib==0.3.6
+distlib==0.3.8
     # via virtualenv
-filelock==3.12.2
+filelock==3.13.1
     # via
     #   tox
     #   virtualenv
-identify==2.5.24
+identify==2.5.35
     # via pre-commit
-nodeenv==1.7.0
+importlib-metadata==7.0.1
+    # via build
+nodeenv==1.8.0
     # via pre-commit
-packaging==23.1
+packaging==23.2
     # via
     #   build
     #   pyproject-api
     #   tox
 pip-compile-multi==2.6.3
     # via -r requirements/integration.in
-pip-tools==7.3.0
+pip-tools==7.4.0
     # via pip-compile-multi
-platformdirs==3.8.1
+platformdirs==4.2.0
     # via
     #   tox
     #   virtualenv
-pluggy==1.2.0
+pluggy==1.4.0
     # via tox
-pre-commit==3.3.3
+pre-commit==3.6.2
     # via -r requirements/integration.in
-pyproject-api==1.5.2
+pyproject-api==1.6.1
     # via tox
 pyproject-hooks==1.0.0
-    # via build
+    # via
+    #   build
+    #   pip-tools
 pyyaml==6.0.1
     # via pre-commit
 tomli==2.0.1
     # via
     #   build
     #   pip-tools
     #   pyproject-api
+    #   pyproject-hooks
     #   tox
 toposort==1.10
     # via pip-compile-multi
-tox==4.6.4
+tox==4.14.0
     # via -r requirements/integration.in
-virtualenv==20.23.1
+virtualenv==20.25.1
     # via
     #   pre-commit
     #   tox
-wheel==0.40.0
+wheel==0.42.0
     # via pip-tools
+zipp==3.17.0
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

requirements/local.txt

@@ -6,7 +6,7 @@
 #    pip-compile-multi
 #
 -r development.txt
--e file:.
+-e file:///Users/max/code/superset
     # via
     #   -r requirements/base.in
     #   -r requirements/development.in

requirements/testing.txt

@@ -12,135 +12,129 @@
     #   -r requirements/base.in
     #   -r requirements/development.in
     #   -r requirements/testing.in
-cmdstanpy==1.1.0
+cmdstanpy==1.2.1
     # via prophet
-contourpy==1.0.7
+contourpy==1.2.0
     # via matplotlib
-coverage[toml]==7.2.5
+coverage[toml]==7.4.3
     # via pytest-cov
-cycler==0.11.0
+cycler==0.12.1
     # via matplotlib
-db-dtypes==1.1.1
+db-dtypes==1.2.0
     # via pandas-gbq
-docker==6.1.1
+docker==7.0.0
     # via -r requirements/testing.in
 flask-testing==0.8.1
     # via -r requirements/testing.in
-fonttools==4.43.0
+fonttools==4.49.0
     # via matplotlib
-freezegun==1.2.2
+freezegun==1.4.0
     # via -r requirements/testing.in
-google-api-core[grpc]==2.11.0
+google-api-core[grpc]==2.17.1
     # via
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
     #   google-cloud-core
     #   pandas-gbq
     #   sqlalchemy-bigquery
-google-auth-oauthlib==1.0.0
+google-auth-oauthlib==1.2.0
     # via
     #   pandas-gbq
     #   pydata-google-auth
-google-cloud-bigquery==3.10.0
+google-cloud-bigquery==3.18.0
     # via
     #   apache-superset
     #   pandas-gbq
     #   sqlalchemy-bigquery
-google-cloud-bigquery-storage==2.19.1
-    # via
-    #   pandas-gbq
-    #   sqlalchemy-bigquery
-google-cloud-core==2.3.2
+google-cloud-bigquery-storage==2.24.0
+    # via pandas-gbq
+google-cloud-core==2.4.1
     # via google-cloud-bigquery
 google-crc32c==1.5.0
     # via google-resumable-media
-google-resumable-media==2.5.0
+google-resumable-media==2.7.0
     # via google-cloud-bigquery
-googleapis-common-protos==1.59.0
+googleapis-common-protos==1.62.0
     # via
     #   google-api-core
     #   grpcio-status
-grpcio==1.60.1
+grpcio==1.62.0
     # via
     #   -r requirements/testing.in
     #   google-api-core
-    #   google-cloud-bigquery
     #   grpcio-status
-grpcio-status==1.60.1
+grpcio-status==1.62.0
     # via google-api-core
 iniconfig==2.0.0
     # via pytest
-jsonschema-spec==0.1.4
+jsonschema-path==0.3.2
     # via openapi-spec-validator
-kiwisolver==1.4.4
+kiwisolver==1.4.5
     # via matplotlib
-matplotlib==3.7.1
+lazy-object-proxy==1.10.0
+    # via openapi-spec-validator
+matplotlib==3.8.3
     # via prophet
 oauthlib==3.2.2
     # via requests-oauthlib
-openapi-schema-validator==0.4.4
+openapi-schema-validator==0.6.2
     # via openapi-spec-validator
-openapi-spec-validator==0.5.6
+openapi-spec-validator==0.7.1
     # via -r requirements/testing.in
-pandas-gbq==0.19.1
+pandas-gbq==0.21.0
     # via apache-superset
 parameterized==0.9.0
     # via -r requirements/testing.in
 pathable==0.4.3
-    # via jsonschema-spec
+    # via jsonschema-path
 playwright==1.41.2
     # via apache-superset
 prophet==1.1.5
     # via apache-superset
-proto-plus==1.22.2
-    # via
-    #   google-cloud-bigquery
-    #   google-cloud-bigquery-storage
-protobuf==4.23.0
+proto-plus==1.23.0
+    # via google-cloud-bigquery-storage
+protobuf==4.25.3
     # via
     #   google-api-core
-    #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
     #   googleapis-common-protos
     #   grpcio-status
     #   proto-plus
-pydata-google-auth==1.7.0
+pydata-google-auth==1.8.2
     # via pandas-gbq
 pyee==11.0.1
     # via playwright
-pyfakefs==5.2.2
+pyfakefs==5.3.5
     # via -r requirements/testing.in
 pyhive[presto]==0.7.0
     # via apache-superset
-pytest==7.3.1
+pytest==8.0.2
     # via
     #   -r requirements/testing.in
     #   pytest-cov
     #   pytest-mock
-pytest-cov==4.0.0
+pytest-cov==4.1.0
     # via -r requirements/testing.in
-pytest-mock==3.10.0
+pytest-mock==3.12.0
     # via -r requirements/testing.in
-pytz-deprecation-shim==0.1.0.post0
-    # via tzlocal
 requests-oauthlib==1.3.1
     # via google-auth-oauthlib
 rfc3339-validator==0.1.4
     # via openapi-schema-validator
-sqlalchemy-bigquery==1.6.1
+sqlalchemy-bigquery==1.10.0
     # via apache-superset
+stanio==0.3.0
+    # via cmdstanpy
 statsd==4.0.1
     # via -r requirements/testing.in
-tqdm==4.65.0
+tqdm==4.66.2
     # via
     #   cmdstanpy
     #   prophet
 trino==0.328.0
     # via apache-superset
-tzlocal==4.3
+tzlocal==5.2
     # via trino
-websocket-client==1.5.1
-    # via docker
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip