chore: move to uv

.github/workflows/unit-tests.yml

@@ -45,18 +45,20 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: 'requirements**.txt'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install -r requirements-dataproc.txt
-          pip install -r requirements-dev.txt
+      # Install uv
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            **/uv.lock
+            **/pyproject.toml
+      - name: Sync dependencies (prod + dataproc)
+        run: uv sync --locked 
       - name: Check Ruff Format
-        run: ruff format --check v03_pipeline --diff
+        run: uv run ruff format --check v03_pipeline --diff
       - name: Check Ruff
-        run: ruff check --output-format github
+        run: uv run ruff check --output-format github
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
         with:
@@ -69,5 +71,5 @@ jobs:
           export PYSPARK_SUBMIT_ARGS='--driver-memory 8G pyspark-shell'
           export CLICKHOUSE_DATABASE=test
           export LOCAL_DISK_MOUNT_PATH=.
-          nosetests --with-coverage --cover-package v03_pipeline v03_pipeline
-          coverage report --omit '*test*' --fail-under=90
+          uv run nosetests --with-coverage --cover-package v03_pipeline v03_pipeline
+          uv run coverage report --omit '*test*' --fail-under=90

pyproject.toml

@@ -2,29 +2,54 @@
 name = "seqr-loading-pipeline"
 dynamic = ['version']
 requires-python = ">=3.11"
+dependencies = [
+    "hail==0.2.136",
+    "luigi==3.5.2",
+    "gnomad==0.8.0",
+    "aiofiles==24.1.0",
+    "pydantic==2.8.2",
+    "clickhouse-driver==0.2.9",
+    "slacker==0.14.0",
+    "ipywidgets==8.0.6",  # matches pinned version in hail init_notebook.py
+    "ipykernel==6.22.0",  # matches pinned version in hail init_notebook.py
+    "notebook==6.5.6",    # matches pinned version in hail init_notebook.py
+    "nbconvert==7.3.1",   # matches pinned version in hail init_notebook.py
+    # Git/URL installs (PEP 508 direct references)
+    "gnomad-qc @ git+https://github.com/broadinstitute/gnomad_qc.git@v4.1",
+    "onnxconverter-common @ git+https://github.com/microsoft/onnxconverter-common.git@f7a8eb699caa6f6a78d3bdfbcaf5dfbd43569ebd",
+
+    # Dataproc
+    "pip==23.2.1",
+    "google-cloud-dataproc==5.16.0",
+    "google-cloud-bigquery==3.29.0",
+    "google-api-python-client==2.159.0",
+]
+
+[dependency-groups]
+prod = [
+    "sqlalchemy<2.0.0" # This is required for the Luigi UI but nothing else.
+]
+dev = [
+    "coverage>=7.2.2",
+    "nose-py3>=1.6.3", # bug in regular nose here: https://github.com/nose-devs/nose/issues/1099#issuecomment-577647469
+    "responses>=0.23.1",
+    "ruff==0.11.3",
+    "shellcheck-py>=0.10.0",
+    "pysam",
+    "pyarrow",
+]
 
 [tool.setuptools]
 include-package-data = true
 
 [tool.setuptools.package-data]
-"v03_pipeline.var" = [
-    "liftover/*.over.chain.gz"
-]
-
-[tool.setuptools.dynamic]
-version = {attr = "v03_pipeline.__version__"}
+"v03_pipeline.var" = ["liftover/*.over.chain.gz"]
 
 [tool.setuptools.packages.find]
 include = ["v03_pipeline*"]
 exclude = ["v03_pipeline.bin", "v03_pipeline.deploy", "v03_pipeline*test*"]
 namespaces = false
 
-[tool.mypy]
-packages = 'v03_pipeline'
-python_version = "3.11"
-ignore_missing_imports = true
-strict = true
-
 [tool.ruff]
 extend-exclude = [
     'download_and_create_reference_datasets/v02/*',