From 2e88539d450e5c28d6d7e9b08524eed9f3c60bfb Mon Sep 17 00:00:00 2001 From: Teo Zosa Date: Fri, 9 Dec 2022 00:15:31 +0900 Subject: [PATCH] :recycle: Extract make command groups TeoZosa/cookiecutter-cruft-poetry-tox-pre-commit-ci-cd#805 --- .makefile/docs.mk | 17 ++ .makefile/help.mk | 61 +++++++ .makefile/project_orchestration.mk | 106 ++++++++++++ .makefile/test.mk | 82 ++++++++++ .makefile/version_tags.mk | 26 +++ Makefile | 251 ++--------------------------- 6 files changed, 304 insertions(+), 239 deletions(-) create mode 100644 .makefile/docs.mk create mode 100644 .makefile/help.mk create mode 100644 .makefile/project_orchestration.mk create mode 100644 .makefile/test.mk create mode 100644 .makefile/version_tags.mk diff --git a/.makefile/docs.mk b/.makefile/docs.mk new file mode 100644 index 00000000..1cd78628 --- /dev/null +++ b/.makefile/docs.mk @@ -0,0 +1,17 @@ +################################################################################# +# COMMANDS # +################################################################################# +.PHONY: docs-% +## Build documentation in the format specified after `-` +## (e.g., +## `make docs-html` builds the docs in HTML format, +## `make docs-confluence` builds and publishes the docs on confluence (see `docs/source/conf.py` for details), +## `make docs-clean` cleans the docs build directory) +docs-%: LAUNCH_DOCS_PREVIEW ?= true +docs-%: + $(MAKE) $* LAUNCH_DOCS_PREVIEW=$(LAUNCH_DOCS_PREVIEW) -C docs + +.PHONY: test-docs +## Test documentation format/syntax +test-docs: + poetry run tox -e docs diff --git a/.makefile/help.mk b/.makefile/help.mk new file mode 100644 index 00000000..c781b6db --- /dev/null +++ b/.makefile/help.mk @@ -0,0 +1,61 @@ +################################################################################# +# Self Documenting Commands # +################################################################################# + +# Inspired by +# sed script explained: +# /^##/: +# * save line in hold space +# * purge line +# * Loop: +# * append newline + line to hold space +# * go to next line +# * if line starts with doc comment, strip comment character off and loop +# * remove target prerequisites +# * append hold space (+ newline) to line +# * replace newline plus comments by `---` +# * print line +# Separate expressions are necessary because labels cannot be delimited by +# semicolon; see +.PHONY: help +help: +ifdef DESCRIPTION + @echo "$$(tput bold)Description:$$(tput sgr0)" && echo "$$DESCRIPTION" && echo +endif + @echo "$$(tput bold)Available rules:$$(tput sgr0)" + @echo + @sed -n -e "/^## / { \ + h; \ + s/.*//; \ + :doc" \ + -e "H; \ + n; \ + s/^## //; \ + t doc" \ + -e "s/:.*//; \ + G; \ + s/\\n## /---/; \ + s/\\n/ /g; \ + p; \ + }" ${MAKEFILE_LIST} \ + | LC_ALL='C' sort --ignore-case \ + | awk -F '---' \ + -v ncol=$$(tput cols) \ + -v indent=19 \ + -v col_on="$$(tput setaf 6)" \ + -v col_off="$$(tput sgr0)" \ + '{ \ + printf "%s%*s%s ", col_on, -indent, $$1, col_off; \ + n = split($$2, words, " "); \ + line_length = ncol - indent; \ + for (i = 1; i <= n; i++) { \ + line_length -= length(words[i]) + 1; \ + if (line_length <= 0) { \ + line_length = ncol - indent - length(words[i]) - 1; \ + printf "\n%*s ", -indent, " "; \ + } \ + printf "%s ", words[i]; \ + } \ + printf "\n"; \ + }' \ + | more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars') diff --git a/.makefile/project_orchestration.mk b/.makefile/project_orchestration.mk new file mode 100644 index 00000000..125a7728 --- /dev/null +++ b/.makefile/project_orchestration.mk @@ -0,0 +1,106 @@ +################################################################################# +# HELPER TARGETS # +################################################################################# +.PHONY: _validate-poetry-installation +_validate-poetry-installation: +ifeq ($(shell command -v poetry),) + @echo "poetry could not be found!" + @echo "Please install poetry!" + @echo "Ex.: 'curl -sSL \ + https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - \ + && source $$HOME/.local/env'" + @echo "see:" + @echo "- https://python-poetry.org/docs/#installation" + @echo "Note: 'pyenv' recommended for Python version management" + @echo "see:" + @echo "- https://github.com/pyenv/pyenv" + @echo "- https://python-poetry.org/docs/managing-environments/" + false +else + @echo "Using $(shell poetry --version) in $(shell which poetry)" +endif + +.PHONY: update-pyproject-deps-to-latest +# Based on: https://github.com/python-poetry/poetry/issues/461#issuecomment-920663114 +## Updates all dependencies in `pyproject.toml` to their latest versions +update-pyproject-deps-to-latest: MAIN_DEPS_EXPORT_FILE:=main_deps.txt +update-pyproject-deps-to-latest: DEV_DEPS_EXPORT_FILE:=dev_deps.txt +update-pyproject-deps-to-latest: DOCS_DEPS_EXPORT_FILE:=docs_deps.txt +update-pyproject-deps-to-latest: + poetry show --outdated --tree --only main | grep -v -e "[├,│,└].*" | cut -d " " -f 1 | sed 's/$$/\@latest/g' > $(MAIN_DEPS_EXPORT_FILE) + poetry show --outdated --tree --only dev | grep -v -e "[├,│,└].*" | cut -d " " -f 1 | sed 's/$$/\@latest/g' > $(DEV_DEPS_EXPORT_FILE) + poetry show --outdated --tree --only docs | grep -v -e "[├,│,└].*" | cut -d " " -f 1 | sed 's/$$/\@latest/g' > $(DOCS_DEPS_EXPORT_FILE) + cat $(MAIN_DEPS_EXPORT_FILE) | xargs poetry add -vvv --group=main + cat $(DEV_DEPS_EXPORT_FILE) | xargs poetry add -vvv --group=dev + cat $(DOCS_DEPS_EXPORT_FILE) | xargs poetry add -vvv --group=docs + rm $(MAIN_DEPS_EXPORT_FILE) $(DEV_DEPS_EXPORT_FILE) $(DOCS_DEPS_EXPORT_FILE) + +.PHONY: update-dependencies +## Update and install Python dependencies, +## updating packages in `poetry.lock` with any newer versions +## that adhere to `pyproject.toml` version range constraints +update-dependencies: + poetry update --lock +ifneq (${CI}, true) + $(MAKE) install-dependencies +endif + +.PHONY: install-dependencies +## Install Python dependencies specified in `poetry.lock` +install-dependencies: + poetry install --no-interaction --no-root --with dev,docs -vv + +.PHONY: install-project +## Install {} source code (in editable mode) +install-project: + poetry install --only-root --no-interaction -vv + $(MAKE) clean + +.PHONY: clean +## Delete all compiled Python files +clean: + find . -type f -name "*.py[co]" -delete + find . -type d -name "__pycache__" -delete + # Clean up files in source directories that may have been generated from C extension compilation + find . -type f -name "*.so" -delete -maxdepth 2 + find . -type f -name "*.pyd" -delete -maxdepth 2 + +.PHONY: get-project-version-number +## Echo project's canonical version number +get-project-version-number: + @poetry version --short + +################################################################################# +# COMMANDS # +################################################################################# +.PHONY: provision-environment +## Set up a Python virtual environment with installed project dependencies +provision-environment: _validate-poetry-installation install-dependencies install-project + +.PHONY: install-pre-commit-hooks +## Install git pre-commit hooks locally +install-pre-commit-hooks: + poetry run pre-commit install + +# Note: The new version should ideally be a valid semver string or a valid bump rule: +# "patch", "minor", "major", "prepatch", "preminor", "premajor", "prerelease". +.PHONY: bump-commit-and-push-project-version-number-% +## *ATOMICALLY*: +## 1.) Bump the version of the project +## 2.) Write the new version back to `pyproject.toml` (assuming a valid bump rule is provided) +## 3.) Commit the version bump to VCS +## 4.) Push the commit to the remote repository +## (e.g., +## `bump-commit-and-push-project-version-number-patch`, +## `bump-commit-and-push-project-version-number-minor`, +## etc.) +bump-commit-and-push-project-version-number-%: VERSION_NUM_FILE:=pyproject.toml +bump-commit-and-push-project-version-number-%: + # shell out to ensure next line gets updated version number; + # directly running `poetry version $*` will cause next line to NOT pick up the version bump + @echo "$(shell poetry version $*)" + @export NEW_VER_NUM=$(shell $(MAKE) get-project-version-number) && \ + export COMMIT_MSG=":bookmark: Bump version number to \`$${NEW_VER_NUM}\`" && \ + git commit $(VERSION_NUM_FILE) -m "$${COMMIT_MSG}" && \ + git push \ + || git checkout HEAD -- $(VERSION_NUM_FILE) # Rollback `VERSION_NUM_FILE` file on failure diff --git a/.makefile/test.mk b/.makefile/test.mk new file mode 100644 index 00000000..225f1706 --- /dev/null +++ b/.makefile/test.mk @@ -0,0 +1,82 @@ +################################################################################# +# HELPER TARGETS # +################################################################################# +.PHONY: generate-requirements +## Generate project requirements files from `pyproject.toml` +generate-requirements: + poetry export --format requirements.txt --without-hashes --output requirements.txt # subset + poetry export --with dev --format requirements.txt --without-hashes --output requirements-dev.txt # superset w/o docs + poetry export --with dev,docs --format requirements.txt --without-hashes --output requirements-all.txt # superset + +.PHONY: clean-requirements +## Clean generated project requirements files +clean-requirements: + find . -type f -name "requirements*.txt" -delete -maxdepth 1 + +.PHONY: tox-% +## Run specified tox testenvs +tox-%: generate-requirements + poetry run tox -e "$*" -- $(POSARGS) + $(MAKE) clean-requirements + +################################################################################# +# COMMANDS # +################################################################################# +.PHONY: test +ifeq (${CI}, true) +test: export TOX_PARALLEL_NO_SPINNER=1 +endif +## Run pre-defined test suite via tox +test: generate-requirements + poetry run tox --parallel + $(MAKE) clean-requirements + +.PHONY: test-% +## Run Python version-specific tests (e.g., `make test-py39`) +test-%: + $(MAKE) tox-$*,coverage + +.PHONY: performance-benchmarks +## Run performance benchmark tests +performance-benchmarks: + $(MAKE) performance-benchmarks-"{pure_python,c_library}" + +.PHONY: performance-benchmarks-% +# Run library-specific (viz. Python or C) performance benchmark tests +performance-benchmarks-%: + $(MAKE) tox-"py3{8,9}-benchmark-$*" + +# Mutation testing modifies the code in small ways that should produce incorrect semantics +# If a test suite is sufficiently strong, this "mutated" code should caught by the suite, +# thus causing tests to fail. +.PHONY: test-mutations +## Run mutation testing to validate test suite robustness +test-mutations: + $(MAKE) tox-mutmut + +.PHONY: lint +## Run full static analysis suite for local development +lint: + $(MAKE) scan-dependencies + $(MAKE) pre-commit + +.PHONY: scan-dependencies +## Scan dependencies for security vulnerabilities +scan-dependencies: + $(MAKE) tox-security + +.PHONY: pre-commit +## Lint using *ALL* pre-commit hooks +## (see `.pre-commit-config.yaml`) +pre-commit: + # Note: Running through `tox` since some hooks rely on finding their executables + # in the `.tox/precommit/bin` directory and to provide an extra layer of isolation + # for reproducibility. + $(MAKE) tox-precommit POSARGS=$(PRECOMMIT_HOOK_ID) + +.PHONY: pre-commit-% +## Lint using a *SINGLE* specific pre-commit hook (e.g., `make pre-commit-mypy`) +## (see `.pre-commit-config.yaml`) +pre-commit-%: export SKIP= # Reset `SKIP` env var to force single hooks to always run +pre-commit-%: + $(MAKE) pre-commit PRECOMMIT_HOOK_ID=$* diff --git a/.makefile/version_tags.mk b/.makefile/version_tags.mk new file mode 100644 index 00000000..57c87050 --- /dev/null +++ b/.makefile/version_tags.mk @@ -0,0 +1,26 @@ +# Implementation reference: https://github.com/kubeflow/kubeflow/blob/a349284acf99723aa822143eaed43314aa001048/components/centraldashboard/Makefile#L25 + +################################################################################# +# GLOBALS # +################################################################################# +# List any changed files (excluding submodules) +CHANGED_FILES := $(shell git diff --name-only) + +ifeq ($(strip $(CHANGED_FILES)),) +GIT_VERSION := $(shell git describe --tags --long --always) +else +diff_checksum := $(shell git diff | shasum -a 256 | cut -c -6) +GIT_VERSION := $(shell git describe --tags --long --always --dirty)-$(diff_checksum) +endif +TAG := $(shell date +v%Y%m%d)-$(GIT_VERSION) + +################################################################################# +# COMMANDS # +################################################################################# +.PHONY: strong-version-tag +strong-version-tag: + @echo $(TAG) + +.PHONY: strong-version-tag-dateless +strong-version-tag-dateless: + @echo $(GIT_VERSION) diff --git a/Makefile b/Makefile index 1a1515f6..e08aabc9 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,13 @@ define DESCRIPTION Code quality (testing, linting/auto-formatting, etc.) and local execution orchestration for $(PROJECT_NAME). endef +export DESCRIPTION ################################################################################# # CONFIGURATIONS # ################################################################################# +NUM_PROCS:=$(shell nproc --all) MAKEFLAGS += --warn-undefined-variables SHELL := bash .SHELLFLAGS := -eu -o pipefail -c @@ -21,248 +23,19 @@ SHELL := bash PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) PROJECT_NAME := $(shell basename $(PROJECT_DIR)) -# List any changed files (excluding submodules) -CHANGED_FILES := $(shell git diff --name-only) - -ifeq ($(strip $(CHANGED_FILES)),) -GIT_VERSION := $(shell git describe --tags --long --always) -else -diff_checksum := $(shell git diff | shasum -a 256 | cut -c -6) -GIT_VERSION := $(shell git describe --tags --long --always --dirty)-$(diff_checksum) -endif -TAG := $(shell date +v%Y%m%d)-$(GIT_VERSION) - -################################################################################# -# HELPER TARGETS # -################################################################################# - -.PHONY: strong-version-tag -strong-version-tag: get-make-var-TAG - -.PHONY: strong-version-tag-dateless -strong-version-tag-dateless: get-make-var-GIT_VERSION - -.PHONY: get-make-var-% -get-make-var-%: - @echo $($*) - -.PHONY: _validate-poetry-installation -_validate-poetry-installation: -ifeq ($(shell command -v poetry),) - @echo "poetry could not be found!" - @echo "Please install poetry!" - @echo "Ex.: 'curl -sSL \ - https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - \ - && source $$HOME/.local/env'" - @echo "see:" - @echo "- https://python-poetry.org/docs/#installation" - @echo "Note: 'pyenv' recommended for Python version management" - @echo "see:" - @echo "- https://github.com/pyenv/pyenv" - @echo "- https://python-poetry.org/docs/managing-environments/" - false -else - @echo "Using $(shell poetry --version) in $(shell which poetry)" -endif - -.PHONY: update-dependencies -## Update and install Python dependencies, -## updating packages in `poetry.lock` with any newer versions -## that adhere to `pyproject.toml` version range constraints -update-dependencies: - poetry update --lock -ifneq (${CI}, true) - $(MAKE) install-dependencies -endif - -.PHONY: install-dependencies -## Install Python dependencies specified in `poetry.lock` -install-dependencies: - poetry install --no-interaction --no-root --with docs -vv - -.PHONY: install-project -## Install {} source code (in editable mode) -install-project: - poetry install --no-interaction -vv - $(MAKE) clean - -.PHONY: generate-requirements -## Generate project requirements files from `pyproject.toml` -generate-requirements: - poetry export --format requirements.txt --without-hashes --output requirements.txt # subset - poetry export --with dev --format requirements.txt --without-hashes --output requirements-dev.txt # superset w/o docs - poetry export --with dev,docs --format requirements.txt --without-hashes --output requirements-all.txt # superset - -.PHONY: clean-requirements -## Clean generated project requirements files -clean-requirements: - find . -type f -name "requirements*.txt" -delete -maxdepth 1 - -.PHONY: clean -## Delete all compiled Python files -clean: - find . -type f -name "*.py[co]" -delete - find . -type d -name "__pycache__" -delete - ################################################################################# # COMMANDS # ################################################################################# -.PHONY: provision-environment -## Set up a Python virtual environment with installed project dependencies -provision-environment: _validate-poetry-installation install-dependencies install-project +# Commands' help text generation +# When users invoke `make help` at the command line, any `##` prefixed lines that +# immediately precede a rule will render as that rule's help text +include .makefile/help.mk +.DEFAULT_GOAL := help # alias `make` to `make help` -.PHONY: install-pre-commit-hooks -## Install git pre-commit hooks locally -install-pre-commit-hooks: - poetry run pre-commit install +# Note: must come before other includes +include .makefile/version_tags.mk # "Strong" version tag for package image tags -.PHONY: get-project-version-number -## Echo project's canonical version number -get-project-version-number: - @poetry version --short - -# Note: The new version should ideally be a valid semver string or a valid bump rule: -# "patch", "minor", "major", "prepatch", "preminor", "premajor", "prerelease". -.PHONY: bump-commit-and-push-project-version-number-% -## *ATOMICALLY*: -## 1.) Bump the version of the project -## 2.) Write the new version back to `pyproject.toml` (assuming a valid bump rule is provided) -## 3.) Commit the version bump to VCS -## 4.) Push the commit to the remote repository -## (e.g., -## `bump-commit-and-push-project-version-number-patch`, -## `bump-commit-and-push-project-version-number-minor`, -## etc.) -bump-commit-and-push-project-version-number-%: VERSION_NUM_FILE:=pyproject.toml -bump-commit-and-push-project-version-number-%: - # shell out to ensure next line gets updated version number; - # directly running `poetry version $*` will cause next line to NOT pick up the version bump - @echo "$(shell poetry version $*)" - @export NEW_VER_NUM=$(shell $(MAKE) get-project-version-number) && \ - export COMMIT_MSG=":bookmark: Bump version number to \`$${NEW_VER_NUM}\`" && \ - git commit $(VERSION_NUM_FILE) -m "$${COMMIT_MSG}" && \ - git push \ - || git checkout HEAD -- $(VERSION_NUM_FILE) # Rollback `VERSION_NUM_FILE` file on failure - -.PHONY: tox-% -## Run specified tox testenvs -tox-%: generate-requirements - poetry run tox -e "$*" -- $(POSARGS) - $(MAKE) clean-requirements - -.PHONY: test -ifeq (${CI}, true) -test: export TOX_PARALLEL_NO_SPINNER=1 -endif -## Run pre-defined test suite via tox -test: generate-requirements - poetry run tox --parallel - $(MAKE) clean-requirements - -.PHONY: test-% -## Run Python version-specific tests (e.g., `make test-py39`) -test-%: - $(MAKE) tox-$*,coverage - -.PHONY: lint -## Run full static analysis suite for local development -lint: - $(MAKE) scan-dependencies - $(MAKE) pre-commit - -.PHONY: scan-dependencies -## Scan dependencies for security vulnerabilities -scan-dependencies: - $(MAKE) tox-security - -.PHONY: pre-commit -## Lint using *ALL* pre-commit hooks -## (see `.pre-commit-config.yaml`) -pre-commit: - # Note: Running through `tox` since some hooks rely on finding their executables - # in the `.tox/precommit/bin` directory and to provide an extra layer of isolation - # for reproducibility. - $(MAKE) tox-precommit POSARGS=$(PRECOMMIT_HOOK_ID) - -.PHONY: pre-commit-% -## Lint using a *SINGLE* specific pre-commit hook (e.g., `make pre-commit-mypy`) -## (see `.pre-commit-config.yaml`) -pre-commit-%: export SKIP= # Reset `SKIP` env var to force single hooks to always run -pre-commit-%: - $(MAKE) pre-commit PRECOMMIT_HOOK_ID=$* - -.PHONY: docs-% -## Build documentation in the format specified after `-` -## (e.g., -## `make docs-html` builds the docs in HTML format, -## `make docs-confluence` builds and publishes the docs on confluence (see `docs/source/conf.py` for details), -## `make docs-clean` cleans the docs build directory) -docs-%: LAUNCH_DOCS_PREVIEW ?= true -docs-%: - $(MAKE) $* LAUNCH_DOCS_PREVIEW=$(LAUNCH_DOCS_PREVIEW) -C docs - -################################################################################# -# Self Documenting Commands # -################################################################################# - -.DEFAULT_GOAL := help - -# Inspired by -# sed script explained: -# /^##/: -# * save line in hold space -# * purge line -# * Loop: -# * append newline + line to hold space -# * go to next line -# * if line starts with doc comment, strip comment character off and loop -# * remove target prerequisites -# * append hold space (+ newline) to line -# * replace newline plus comments by `---` -# * print line -# Separate expressions are necessary because labels cannot be delimited by -# semicolon; see -export DESCRIPTION -.PHONY: help -help: -ifdef DESCRIPTION - @echo "$$(tput bold)Description:$$(tput sgr0)" && echo "$$DESCRIPTION" && echo -endif - @echo "$$(tput bold)Available rules:$$(tput sgr0)" - @echo - @sed -n -e "/^## / { \ - h; \ - s/.*//; \ - :doc" \ - -e "H; \ - n; \ - s/^## //; \ - t doc" \ - -e "s/:.*//; \ - G; \ - s/\\n## /---/; \ - s/\\n/ /g; \ - p; \ - }" ${MAKEFILE_LIST} \ - | LC_ALL='C' sort --ignore-case \ - | awk -F '---' \ - -v ncol=$$(tput cols) \ - -v indent=19 \ - -v col_on="$$(tput setaf 6)" \ - -v col_off="$$(tput sgr0)" \ - '{ \ - printf "%s%*s%s ", col_on, -indent, $$1, col_off; \ - n = split($$2, words, " "); \ - line_length = ncol - indent; \ - for (i = 1; i <= n; i++) { \ - line_length -= length(words[i]) + 1; \ - if (line_length <= 0) { \ - line_length = ncol - indent - length(words[i]) - 1; \ - printf "\n%*s ", -indent, " "; \ - } \ - printf "%s ", words[i]; \ - } \ - printf "\n"; \ - }' \ - | more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars') +include .makefile/docs.mk # Documentation building commands +include .makefile/project_orchestration.mk # Project/virtual environment orchestration commands +include .makefile/test.mk # Application testing + code quality and security scanning commands