Prepare isort and black formatters and checks, but don't enable them …

…yet.

.devcontainer/devcontainer.json

@@ -73,8 +73,9 @@
                 "lextudio.restructuredtext",
                 "matangover.mypy",
                 "ms-azuretools.vscode-docker",
-                "ms-python.autopep8",
-                "ms-python.flake8",
+                // TODO: Enable additional formatter extensions:
+                //"ms-python.black-formatter",
+                //"ms-python.isort",
                 "ms-python.pylint",
                 "ms-python.python",
                 "ms-python.vscode-pylance",

.vscode/extensions.json

@@ -16,8 +16,9 @@
         "lextudio.restructuredtext",
         "matangover.mypy",
         "ms-azuretools.vscode-docker",
-        "ms-python.autopep8",
-        "ms-python.flake8",
+	// TODO: Enable additional formatter extensions:
+        //"ms-python.black-formatter",
+        //"ms-python.isort",
         "ms-python.pylint",
         "ms-python.python",
         "ms-python.vscode-pylance",

.vscode/settings.json

@@ -129,9 +129,10 @@
         "--experimental"
     ],
     "[python]": {
-        "editor.defaultFormatter": "ms-python.autopep8",
-        "editor.formatOnSave": true,
-        "editor.formatOnSaveMode": "modifications"
+        // TODO: Enable black formatter
+        //"editor.defaultFormatter": "ms-python.black-formatter",
+        //"editor.formatOnSave": true,
+        //"editor.formatOnSaveMode": "modifications"
     },
     // See Also .vscode/launch.json for environment variable args to pytest during debug sessions.
     // For the rest, see setup.cfg

Makefile

@@ -69,7 +69,9 @@ ifneq (,$(filter format,$(MAKECMDGOALS)))
 endif
 
 build/format.${CONDA_ENV_NAME}.build-stamp: build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
-# TODO: add isort and black formatters
+# TODO: enable isort and black formatters
+#build/format.${CONDA_ENV_NAME}.build-stamp: build/isort.${CONDA_ENV_NAME}.build-stamp
+#build/format.${CONDA_ENV_NAME}.build-stamp: build/black.${CONDA_ENV_NAME}.build-stamp
 build/format.${CONDA_ENV_NAME}.build-stamp:
 	touch $@
 
@@ -93,10 +95,117 @@ build/licenseheaders.${CONDA_ENV_NAME}.build-stamp:
 		-x mlos_bench/setup.py mlos_core/setup.py mlos_viz/setup.py
 	touch $@
 
+.PHONY: isort
+isort: build/isort.${CONDA_ENV_NAME}.build-stamp
+
+ifneq (,$(filter isort,$(MAKECMDGOALS)))
+    FORMAT_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
+endif
+
+build/isort.${CONDA_ENV_NAME}.build-stamp: build/isort.mlos_core.${CONDA_ENV_NAME}.build-stamp
+build/isort.${CONDA_ENV_NAME}.build-stamp: build/isort.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+build/isort.${CONDA_ENV_NAME}.build-stamp: build/isort.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+build/isort.${CONDA_ENV_NAME}.build-stamp:
+	touch $@
+
+# NOTE: when using pattern rules (involving %) we can only add one line of
+# prerequisities, so we use this pattern to compose the list as variables.
+
+# Both isort and licenseheaders alter files, so only run one at a time, by
+# making licenseheaders an order-only prerequisite.
+ISORT_COMMON_PREREQS :=
+ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
+ISORT_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+endif
+ISORT_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
+ISORT_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/isort.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/isort.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/isort.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+build/isort.%.${CONDA_ENV_NAME}.build-stamp: $(ISORT_COMMON_PREREQS)
+	# Reformat python file imports with isort.
+	conda run -n ${CONDA_ENV_NAME} isort --verbose --only-modified --atomic -j0 $(filter %.py,$?)
+	touch $@
+
+.PHONY: black
+black: build/black.${CONDA_ENV_NAME}.build-stamp
+
+ifneq (,$(filter black,$(MAKECMDGOALS)))
+    FORMAT_PREREQS += build/black.${CONDA_ENV_NAME}.build-stamp
+endif
+
+build/black.${CONDA_ENV_NAME}.build-stamp: build/black.mlos_core.${CONDA_ENV_NAME}.build-stamp
+build/black.${CONDA_ENV_NAME}.build-stamp: build/black.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+build/black.${CONDA_ENV_NAME}.build-stamp: build/black.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+build/black.${CONDA_ENV_NAME}.build-stamp:
+	touch $@
+
+# Both black, licenseheaders, and isort all alter files, so only run one at a time, by
+# making licenseheaders and isort an order-only prerequisite.
+BLACK_COMMON_PREREQS :=
+ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
+BLACK_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+endif
+ifneq (,$(filter format isort,$(MAKECMDGOALS)))
+BLACK_COMMON_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
+endif
+BLACK_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
+BLACK_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/black.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/black.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/black.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+build/black.%.${CONDA_ENV_NAME}.build-stamp: $(BLACK_COMMON_PREREQS)
+	# Reformat python files with black.
+	conda run -n ${CONDA_ENV_NAME} black $(filter %.py,$?)
+	touch $@
 
 .PHONY: check
 check: pycodestyle pydocstyle pylint mypy # cspell markdown-link-check
-# TODO: add isort and black checks
+# TODO: Enable isort and black checks
+#check: isort-check black-check pycodestyle pydocstyle pylint mypy # cspell markdown-link-check
+
+.PHONY: black-check
+black-check: build/black-check.mlos_core.${CONDA_ENV_NAME}.build-stamp
+black-check: build/black-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+black-check: build/black-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+
+# Make sure black format rules run before black-check rules.
+build/black-check.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/black-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/black-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+BLACK_CHECK_COMMON_PREREQS := build/conda-env.${CONDA_ENV_NAME}.build-stamp
+BLACK_CHECK_COMMON_PREREQS += $(FORMAT_PREREQS)
+BLACK_CHECK_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/black-check.%.${CONDA_ENV_NAME}.build-stamp: $(BLACK_CHECK_COMMON_PREREQS)
+	# Check for import sort order.
+	# Note: if this fails use "make format" or "make black" to fix it.
+	conda run -n ${CONDA_ENV_NAME} black --verbose --check --diff $(filter %.py,$?)
+	touch $@
+
+.PHONY: isort-check
+isort-check: build/isort-check.mlos_core.${CONDA_ENV_NAME}.build-stamp
+isort-check: build/isort-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+isort-check: build/isort-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+
+# Make sure isort format rules run before isort-check rules.
+build/isort-check.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/isort-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/isort-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+ISORT_CHECK_COMMON_PREREQS := build/conda-env.${CONDA_ENV_NAME}.build-stamp
+ISORT_CHECK_COMMON_PREREQS += $(FORMAT_PREREQS)
+ISORT_CHECK_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/isort-check.%.${CONDA_ENV_NAME}.build-stamp: $(ISORT_CHECK_COMMON_PREREQS)
+	# Note: if this fails use "make format" or "make isort" to fix it.
+	conda run -n ${CONDA_ENV_NAME} isort --only-modified --check --diff -j0 $(filter %.py,$?)
+	touch $@
 
 .PHONY: pycodestyle
 pycodestyle: build/pycodestyle.mlos_core.${CONDA_ENV_NAME}.build-stamp

conda-envs/mlos-3.10.yml

@@ -7,6 +7,7 @@ dependencies:
   # All other dependencies for the mlos modules come from pip.
   - pip
   - pylint
+  - black
   - pycodestyle
   - pydocstyle
   - flake8
@@ -27,6 +28,7 @@ dependencies:
     - autopep8>=1.7.0
     - bump2version
     - check-jsonschema
+    - isort
     - licenseheaders
     - mypy
     - pandas-stubs

conda-envs/mlos-3.11.yml

@@ -7,6 +7,7 @@ dependencies:
   # All other dependencies for the mlos modules come from pip.
   - pip
   - pylint
+  - black
   - pycodestyle
   - pydocstyle
   - flake8
@@ -27,6 +28,7 @@ dependencies:
     - autopep8>=1.7.0
     - bump2version
     - check-jsonschema
+    - isort
     - licenseheaders
     - mypy
     - pandas-stubs

conda-envs/mlos-3.8.yml

@@ -7,6 +7,7 @@ dependencies:
   # All other dependencies for the mlos modules come from pip.
   - pip
   - pylint
+  - black
   - pycodestyle
   - pydocstyle
   - flake8
@@ -27,6 +28,7 @@ dependencies:
     - autopep8>=1.7.0
     - bump2version
     - check-jsonschema
+    - isort
     - licenseheaders
     - mypy
     - pandas-stubs

conda-envs/mlos-3.9.yml

@@ -7,6 +7,7 @@ dependencies:
   # All other dependencies for the mlos modules come from pip.
   - pip
   - pylint
+  - black
   - pycodestyle
   - pydocstyle
   - flake8
@@ -27,6 +28,7 @@ dependencies:
     - autopep8>=1.7.0
     - bump2version
     - check-jsonschema
+    - isort
     - licenseheaders
     - mypy
     - pandas-stubs

conda-envs/mlos-windows.yml

@@ -8,6 +8,7 @@ dependencies:
   # All other dependencies for the mlos modules come from pip.
   - pip
   - pylint
+  - black
   - pycodestyle
   - pydocstyle
   - flake8
@@ -30,6 +31,7 @@ dependencies:
     - autopep8>=1.7.0
     - bump2version
     - check-jsonschema
+    - isort
     - licenseheaders
     - mypy
     - pandas-stubs

conda-envs/mlos.yml

@@ -7,6 +7,7 @@ dependencies:
   # All other dependencies for the mlos modules come from pip.
   - pip
   - pylint
+  - black
   - pycodestyle
   - pydocstyle
   - flake8
@@ -26,6 +27,7 @@ dependencies:
     - autopep8>=1.7.0
     - bump2version
     - check-jsonschema
+    - isort
     - licenseheaders
     - mypy
     - pandas-stubs

pyproject.toml

@@ -0,0 +1,9 @@
+[tool.black]
+line-length = 88
+target-version = ["py38", "py39", "py310", "py311", "py312"]
+include = '\.pyi?$'
+
+[tool.isort]
+profile = "black"
+py_version = 311
+src_paths = ["mlos_core", "mlos_bench", "mlos_viz"]