Move to pyproject.toml and unify workflows

.github/workflows/push.yaml → .github/workflows/ci_cd.yaml

@@ -1,14 +1,55 @@
-name: Push
+name: CI/CD
 
 on:
+  pull_request:
+    branches: [main]
   push:
-    branches:
-      - main
+    branches: [main]
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    name: Lint
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install black
+      - name: Check formatting
+        run: black . -l 79 --check
+
+  test:
+    name: Build and Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all tags and branches
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Install package
+        run: pip install -e ".[dev]"
+      - name: Download data inputs
+        run: make download
+        env:
+          POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}
+      - name: Build datasets
+        run: make data
+      - name: Run tests
+        run: pytest
+
   docker:
     name: Docker
     runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -22,58 +63,38 @@ jobs:
         run: docker build . -f docker/policyengine_us_data.Dockerfile -t ghcr.io/policyengine/policyengine-us-data:latest
       - name: Push container
         run: docker push ghcr.io/policyengine/policyengine-us-data:latest
-  build:
-    name: Build and test
+
+  upload-and-publish:
+    name: Upload Data and Publish Package
     runs-on: ubuntu-latest
+    needs: [lint, test]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all tags and branches
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Install package
-        run: make install
+        run: pip install -e ".[dev]"
       - name: Download data inputs
         run: make download
         env:
           POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}
       - name: Build datasets
         run: make data
-      - name: Run tests
-        run: make test
       - name: Upload data
         run: make upload
         env:
           POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}
-
-  lint:
-    runs-on: ubuntu-latest
-    name: Lint
-    steps:
-      - uses: actions/checkout@v4
-      - name: Check formatting
-        uses: "lgeiger/black-action@master"
-        with:
-          args: ". -l 79 --check"
-  publish:
-    runs-on: ubuntu-latest
-    name: Publish
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.12
-      - name: Install package
-        run: make install
       - name: Build package
-        run: make build
-      - name: Publish a Python distribution to PyPI
+        run: python -m build
+      - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.PYPI }}
           skip-existing: true
-

Makefile

@@ -1,13 +1,15 @@
+.PHONY: all format test install download upload docker documentation data clean build
+
 all: data test
 
 format:
 	black . -l 79
 
 test:
-	pytest policyengine_us_data/tests
+	pytest
 
 install:
-	pip install -e .[dev]
+	pip install -e ".[dev]"
 
 download:
 	python policyengine_us_data/data_storage/download_public_prerequisites.py
@@ -26,7 +28,11 @@ data:
 	python policyengine_us_data/datasets/cps/enhanced_cps.py
 
 clean:
-	rm policyengine_us_data/data_storage/puf_2015.csv
-	rm policyengine_us_data/data_storage/demographics_2015.csv
+	rm -f policyengine_us_data/data_storage/puf_2015.csv
+	rm -f policyengine_us_data/data_storage/demographics_2015.csv
+
 build:
-	python setup.py sdist bdist_wheel
+	python -m build
+
+publish:
+	twine upload dist/*

pyproject.toml

@@ -0,0 +1,65 @@
+[build-system]
+requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "policyengine_us_data"
+dynamic = ["version"]
+description = "A package to create representative microdata for the US."
+readme = "README.md"
+authors = [
+    {name = "PolicyEngine", email = "hello@policyengine.org"},
+]
+license = {file = "LICENSE"}
+requires-python = ">=3.6"
+dependencies = [
+    "policyengine_core",
+    "tables",
+    "survey_enhance",
+    "torch",
+    "requests",
+    "tqdm",
+    "tabulate",
+]
+
+[project.optional-dependencies]
+dev = [
+    "black",
+    "pytest",
+    "policyengine_us==1.71.1",
+    "streamlit",
+]
+
+[tool.setuptools]
+packages = ["policyengine_us_data"]
+include-package-data = true
+
+[tool.setuptools.package-data]
+"policyengine_us_data" = ["**/*"]
+
+[tool.setuptools_scm]
+write_to = "policyengine_us_data/_version.py"
+
+[tool.pytest.ini_options]
+addopts = "-v"
+testpaths = [
+    "tests",
+]
+
+[tool.black]
+line-length = 79
+target-version = ['py311']
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+  # directories
+  \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | build
+  | dist
+)/
+'''