Project script fixes, Justfile

.env.example

@@ -0,0 +1,3 @@
+# DJANGO_STUBS_VENV=.venv
+# DJANGO_VERSION=3.2
+# PYTEST_ADDOPTS=-vv

.gitignore

@@ -1,5 +1,6 @@
 *.egg-info
 .DS_Store
+.env
 .idea/
 .mypy_cache/
 .pytest_cache/
@@ -10,3 +11,4 @@ out/
 pip-wheel-metadata/
 stubgen/
 build/
+dist/

CONTRIBUTING.md

@@ -31,6 +31,30 @@ Additionally, the following resources might be useful:
 As a first step you will need to fork this repository and clone your fork locally.
 In order to be able to continously sync your fork with the origin repository's master branch, you will need to set up an upstream master. To do so follow this [official github guide](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/syncing-a-fork).
 
+You can now follow instructions in subsequent sections of this document to manually install dependencies, run tests, etc., _or_ you can [install `just`](https://just.systems/man/en/) to run our `Justfile`. For example:
+
+```console
+> just
+Available recipes:
+    ls              # list available recipes
+    help            # print help info & list available recipes
+    full            # refresh setup, run checks & builds
+    clean           # remove development artifacts
+    setup           # setup up project development environment
+    draft-stubs     # generate new draft stub files
+    lint            # run all linters & checkers
+    test            # run all tests
+    build           # build project packages
+    release         # build & release project packages
+
+> just --show full
+# refresh setup, run checks & builds
+full: clean setup lint test build
+
+> just full
+... # proceeds to create virtualenv, install dependencies, run tests, etc.
+```
+
 ### Dependency Setup
 
 After your repository is setup you will then need to create and activate a git ignored virtual env, e.g.:

Justfile

@@ -0,0 +1,191 @@
+#!/usr/bin/env -S just --justfile
+
+# Config
+# =====================================================================
+
+set dotenv-load := true
+set shell := [ "bash", "-euo", "pipefail", "-c" ]
+
+# these names are short because they will be used a lot
+FROM := invocation_directory()
+HERE := justfile_directory()
+SELF := justfile()
+SHELL := file_stem(`test -n "$0" && echo "$0" || status fish-path`)
+
+# more explicit names are exported
+export JUSTFILE := SELF
+export JUSTFILE_DIR := HERE
+export JUST_INVOCATION_DIR := FROM
+export JUST_INVOCATION_SHELL := SHELL
+
+# Handy bits
+t := "true"
+f := "false"
+
+# Path of .env file (not configurable)
+dotenv := HERE / ".env"
+dotexample := HERE / ".env.example"
+
+# The Python executable used to create the project virtualenv
+srcpy := env_var_or_default("DJANGO_STUBS_PYTHON", "python3")
+
+# The path at which to create the project virtualenv
+venv_dir := env_var_or_default("DJANGO_STUBS_VENV", HERE / ".venv")
+venv_bin := venv_dir / "bin"
+venv_act := venv_bin / "activate" + if SHELL =~ '^(fi|c)sh$' { "." + SHELL } else { "" }
+
+# The Python executable of the project virtualenv
+pyexe := venv_bin / "python"
+
+# The version of Django for which to test and build
+export DJANGO_VERSION := env_var_or_default("DJANGO_VERSION", "3.2")
+
+export PATH := venv_bin + ":" + env_var("PATH")
+
+
+# Aliases
+# =====================================================================
+
+alias h := help
+alias t := test
+alias rm := clean
+alias clear := clean
+
+
+# Recipes
+# =====================================================================
+
+## General
+## --------------------------------------------------------------------
+
+# run this recipe if no arguments are given (by virtue of it being the *first* recipe)
+@_default: ls
+
+# list available recipes
+@ls:
+  "{{ SELF }}" --list --unsorted
+
+# print help info & list available recipes
+@help: && ls
+  "{{ SELF }}" --help
+
+
+## Development
+## --------------------------------------------------------------------
+
+# refresh setup, run checks & builds
+full: clean setup lint test build
+
+# remove development artifacts
+clean: _clean-precommit _clean-setuptools _clean-project
+
+# uninstall pre-commit hooks and clean up artifacts
+_clean-precommit:
+  -pre-commit uninstall
+  -pre-commit clean
+  -pre-commit gc
+
+# run setuptools clean command, delete other artifacts
+_clean-setuptools:
+  -"{{ if path_exists(pyexe) == t { pyexe } else { srcpy } }}" setup.py clean --all
+  -cd django_stubs_ext && "{{ if path_exists(pyexe) == t { pyexe } else { srcpy } }}" setup.py clean --all
+  -rm -rf {.,django_stubs_ext}/*.egg-info
+  -rm -rf {.,django_stubs_ext}/build
+  -rm -rf {.,django_stubs_ext}/dist
+  -rm -rf {.,django_stubs_ext}/django-stubs-*.*.?*
+
+# remove all artifacts not removed by other cleaners
+_clean-project:
+  -rm -rf "{{ venv_dir }}"
+  -rm -rf {.,django_stubs_ext}/.*cache
+  @# Remove the dotenv file if it is no different from the example one.
+  -! diff -q "{{ dotenv }}" "{{ dotexample }}" >/dev/null 2>&1 || rm "{{ dotenv }}"
+  @# TODO: Consider deleting these safely by using a prompt & --noinput switch.
+  @echo "Manually delete the following, if needed:"
+  @printf -- "    %s\n" $(python scripts/paths.py)
+
+
+# setup up project development environment
+setup:
+  @# @test ! -e "{{ pyexe }}" && "{{ SELF }}" _setup || "{{ SELF }}" _install
+  @"{{ SELF }}" {{ if path_exists(pyexe) == f { "_setup" } else { "_install" } }}
+  @echo "If needed, generate draft stubs by running 'just draft-stubs"
+
+# create virtualenv, install requirements
+_setup: && _install
+  test -e "{{ dotenv }}" || cp "{{ dotexample }}" "{{ dotenv }}"
+  test ! -e "{{ venv_dir }}" || rm -rf "{{ venv_dir }}"
+  "{{ srcpy }}" -m venv "{{ venv_dir }}"
+
+_install:
+  "{{ pyexe }}" -m pip install -U pip setuptools wheel
+  "{{ pyexe }}" -m pip install -r requirements.txt
+  "{{ pyexe }}" -m pre_commit install --install-hooks
+
+# generate new draft stub files
+draft-stubs:
+  "{{ pyexe }}" scripts/stubgen-django.py --django_version "$DJANGO_VERSION"
+
+
+## Lint & Test
+## --------------------------------------------------------------------
+
+# run all linters & checkers
+lint: lint-precommit lint-mypy
+
+# run pre-commit hooks
+lint-precommit *ARGS="--all-files":
+  "{{ pyexe }}" -m pre_commit run {{ ARGS }}
+
+# typecheck project w/ mypy
+lint-mypy:
+  @# see `.github/workflows/tests.yml:jobs.mypy-self-check.steps`
+  "{{ pyexe }}" -m mypy --strict mypy_django_plugin
+  "{{ pyexe }}" -m mypy --strict django_stubs_ext
+  "{{ pyexe }}" -m mypy --strict scripts
+  "{{ pyexe }}" -m mypy --cache-dir=/dev/null --no-incremental django-stubs
+
+# run all tests
+test: test-py test-dj
+
+# run unit tests w/ pytest
+test-py *ARGS:
+  @# see `.github/workflows/tests.yml:jobs.test.steps`
+  "{{ pyexe }}" -m pytest {{ ARGS }}
+
+# typecheck Django tests w/ mypy using project stubs
+test-dj:
+  @# see `.github/workflows/tests.yml:jobs.typecheck.steps`
+  "{{ pyexe }}" scripts/typecheck_tests.py --django_version "$DJANGO_VERSION"
+
+
+## Build
+## --------------------------------------------------------------------
+
+# build project packages
+build: build-pkg build-pkg-ext
+
+# build django-stubs package
+build-pkg:
+  "{{ pyexe }}" -m pip install --upgrade setuptools wheel twine
+  "{{ pyexe }}" setup.py check sdist bdist_wheel
+
+# build django-stubs-ext package
+build-pkg-ext:
+  "{{ pyexe }}" -m pip install --upgrade setuptools wheel twine
+  cd django_stubs_ext && "{{ pyexe }}" setup.py check sdist bdist_wheel
+
+
+## Release
+## --------------------------------------------------------------------
+
+# build & release project packages
+release: full release-pkg release-pkg-ext
+
+# build & release django-stubs package
+release-pkg:
+  . "{{ venv_act }}" && ./scripts/release.sh
+
+# build & release django-stubs-ext package
+release-pkg-ext:
+  cd django_stubs_ext && . "{{ venv_act }}" && ./release.sh

django_stubs_ext/release.sh

@@ -6,7 +6,7 @@ then
   if [[ "$VIRTUAL_ENV" != ""  ]]
   then
     pip install --upgrade setuptools wheel twine
-    python setup.py sdist bdist_wheel
+    python setup.py check sdist bdist_wheel
     twine upload dist/*
     rm -rf dist/ build/
   else

mypy.ini

@@ -15,6 +15,11 @@ warn_unreachable = True
 disallow_untyped_defs = true
 disallow_incomplete_defs = true
 
+exclude = (?x)(
+        # prevent build artifacts from causing "duplicate module found" errors
+        /build/lib/
+    )
+
 plugins =
     mypy_django_plugin.main
 

scripts/paths.py

@@ -2,3 +2,9 @@
 
 PROJECT_DIRECTORY = Path(__file__).parent.parent
 DJANGO_SOURCE_DIRECTORY = PROJECT_DIRECTORY / "django-source"  # type: Path
+STUBGEN_OUTPUT_DIRECTORY = PROJECT_DIRECTORY / "stubgen"  # type: Path
+
+
+if __name__ == "__main__":
+    print(DJANGO_SOURCE_DIRECTORY)
+    print(STUBGEN_OUTPUT_DIRECTORY)

scripts/release.sh

@@ -6,7 +6,7 @@ then
   if [[ "$VIRTUAL_ENV" != ""  ]]
   then
     pip install --upgrade setuptools wheel twine
-    python setup.py sdist bdist_wheel
+    python setup.py check sdist bdist_wheel
     twine upload dist/*
     rm -rf dist/ build/
   else

scripts/stubgen-django.py

@@ -3,13 +3,14 @@
 from mypy.stubgen import generate_stubs, parse_options
 
 from scripts.git_helpers import checkout_django_branch
-from scripts.paths import DJANGO_SOURCE_DIRECTORY
+from scripts.paths import DJANGO_SOURCE_DIRECTORY, STUBGEN_OUTPUT_DIRECTORY
 
 if __name__ == "__main__":
     parser = ArgumentParser()
     parser.add_argument("--django_version", required=True)
     parser.add_argument("--commit_sha", required=False)
     args = parser.parse_args()
     checkout_django_branch(args.django_version, args.commit_sha)
-    stubgen_options = parse_options([f"{DJANGO_SOURCE_DIRECTORY}", "-o=stubgen"])
+    django_root = DJANGO_SOURCE_DIRECTORY / "django"
+    stubgen_options = parse_options([f"{django_root}", f"-o={STUBGEN_OUTPUT_DIRECTORY}"])
     generate_stubs(stubgen_options)