refactor: update dev container and pod (#477)

* fix: remove syntax directive from conda Dockerfile

* fix: update devcontainer for compatibility with dp

- Update devcontainer name to "pyrovelocity-dev".
- Update dockerfile path to "../dockerfiles/Dockerfile.conda".
- Remove the "docker-in-docker" feature.

* fix: add `.envrc` to `.gitignore`

* feat: add dockerignore

* fix: update vscode settings and extensions

* fix: add make targets to install direnv and just

* feat: add example envrc

* feat: add justfile for devcontainer

---------

Co-authored-by: pyrovelocity[bot] <144943013+pyrovelocity-bot@users.noreply.github.com>

.devcontainer/devcontainer.json

@@ -1,12 +1,9 @@
 {
-  "name": "pyrovelocity",
+  "name": "pyrovelocity-dev",
   "build": {
-    "dockerfile": "../dockerfiles/Dockerfile.conda.cpu",
+    "dockerfile": "../dockerfiles/Dockerfile.conda",
     "context": ".."
   },
-  "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
-  },
   "customizations": {
     "vscode": {
       "extensions": [

.dockerignore

@@ -0,0 +1,121 @@
+#
+archive/
+
+#
+.DS_Store
+
+#
+monkeytype.sqlite3
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+.envrc
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# IDE settings
+*.code-workspace
+.VSCodeCounter
+.idea/
+*.pdf
+*.tif
+*.h5ad
+.nox/

.envrc.example

@@ -0,0 +1,12 @@
+export CONTAINER_BUILDER=podman # path to container management executable
+export GITHUB_USERNAME=username # github username associated to uploading startup scripts as github gists
+export GITHUB_ORG_NAME=pinellolab # name of the github org or user containing the github repository with code for development
+export GITHUB_REPO_NAME=pyrovelocity # name of a github repository with a conda environment yaml file
+export GITHUB_BRANCH_NAME=master # name of github repository branch to checkout
+export POD_DISK_SIZE=400Gi # size of the PVC to mount to the devpod
+export POD_ACCELERATOR_TYPE=nvidia-tesla-t4 # cluster-specific node selector for accelerator type | nvidia-l4 | nvidia-tesla-a100
+export POD_MIN_CPU=16 # cpu resource request for kubernetes pod
+export POD_MIN_MEM=64Gi # ram resource request for kubernetes pod
+export POD_MAX_CPU=30 # cpu resource limit for kubernetes pod
+export POD_MAX_MEM=96Gi # ram resource limit for kubernetes pod
+export POD_MAX_ACCEL=1 # accelerator resource limit for kubernetes pod

.gitignore

@@ -91,6 +91,7 @@ celerybeat-schedule
 
 # dotenv
 .env
+.envrc
 
 # virtualenv
 .venv
@@ -112,7 +113,6 @@ ENV/
 
 # IDE settings
 *.code-workspace
-.vscode/
 .VSCodeCounter
 .idea/
 *.pdf

.vscode/extensions.json

@@ -0,0 +1,22 @@
+{
+  "recommendations": [
+    "donjayamanne.python-extension-pack",
+    "redhat.vscode-yaml",
+    "sclu1034.justfile",
+    "ms-vscode.makefile-tools",
+    "hashicorp.terraform",
+    "ms-python.black-formatter",
+    "googlecloudtools.cloudcode",
+    "ms-kubernetes-tools.vscode-kubernetes-tools",
+    "ms-vsliveshare.vsli",
+    "eamodio.gitlens",
+    "GitHub.vscode-pull-request-github",
+    "github.vscode-github-actions",
+    "ms-azuretools.vscode-docker",
+    "ms-toolsai.jupyter",
+    "iterative.dvc",
+    "njzy.stats-bar",
+    "vscode-icons-team.vscode-icons",
+    "poimandres.theme-poimandres"
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.formatOnSave": true
+  }
+}

Makefile

@@ -19,6 +19,45 @@ help: ## Display this help. (Default)
 help_sort: ## Display alphabetized version of help.
 	@grep -hE '^[A-Za-z0-9_ \-]*?:.*##.*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
+#-------------
+# system / dev
+#-------------
+
+install_direnv: ## Install direnv to `/usr/local/bin`. Check script before execution: https://direnv.net/ .
+	@which direnv > /dev/null || \
+	(curl -sfL https://direnv.net/install.sh | bash && \
+	sudo install -c -m 0755 direnv /usr/local/bin && \
+	rm -f ./direnv)
+	@echo "see https://direnv.net/docs/hook.html"
+
+install_just: ## Install just. Check script before execution: https://just.systems/ .
+	@which cargo > /dev/null || (curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh)
+	@cargo install just
+
+precommit: ## Run pre-commit hooks using nox.
+	nox -x -rs pre-commit
+
+env_print: ## Print a subset of environment variables defined in ".envrc" file.
+	env | grep "TF_VAR\|GITHUB\|GH_\|GCP_\|MLFLOW" | sort
+
+approve_prs: ## Approve github pull requests from bots: PR_ENTRIES="2-5 10 12-18"
+	for entry in $(PR_ENTRIES); do \
+		if [[ "$$entry" == *-* ]]; then \
+			start=$${entry%-*}; \
+			end=$${entry#*-}; \
+			for pr in $$(seq $$start $$end); do \
+				@gh pr review $$pr --approve; \
+			done; \
+		else \
+			@gh pr review $$entry --approve; \
+		fi; \
+	done
+
+
+#----------------
+# web application
+#----------------
+
 st: ## Run streamlit app in local environment.
 	streamlit run app/app.py \
 	--server.port=8080 \
@@ -96,9 +135,3 @@ ifdef GCP_RUN_SERVICE_NAME
 else
 	@echo 'Run "make help" and define necessary variables'
 endif
-
-precommit: ## Run pre-commit hooks using nox.
-	nox -x -rs pre-commit
-
-env_print: ## Print a subset of environment variables defined in ".env" file.
-	env | grep "TF_VAR\|GITHUB\|GH_\|GCP_\|MLFLOW" | sort

dockerfiles/Dockerfile.conda

@@ -1,4 +1,3 @@
-# syntax=docker/dockerfile:1
 FROM condaforge/mambaforge:23.1.0-1
 LABEL maintainer="pyrovelocity team"
 

justfile

@@ -0,0 +1,119 @@
+# Default command when 'just' is run without arguments
+# Run 'just <command>' to execute a command.
+default: list
+
+# Display help
+help:
+  @printf "\nSee Makefile targets for just and direnv installation."
+  @printf "\nRun 'just -n <command>' to print what would be executed...\n\n"
+  @just --list --unsorted
+  @echo "\n...by running 'just <command>'.\n"
+  @echo "This message is printed by 'just help'."
+  @echo "Just 'just' will just list the available recipes.\n"
+
+# List just recipes
+list:
+  @just --list --unsorted
+
+# List evaluated just variables
+vars:
+  @just --evaluate
+
+builder := env_var_or_default('BUILDER', 'podman')
+container_user := "runner"
+container_home := "/home" / container_user
+container_work := container_home / "work"
+git_username := env_var_or_default('GITHUB_USERNAME', 'pinellolab')
+git_org_name := env_var_or_default('GITHUB_ORG_NAME', 'pinellolab')
+git_repo_name := env_var_or_default('GITHUB_REPO_NAME', 'pyrovelocity')
+git_branch_name := env_var_or_default('GITHUB_BRANCH_NAME', 'master')
+container_registry := "ghcr.io/" + git_org_name + "/"
+pod_accelerator_type := env_var_or_default('POD_ACCELERATOR_TYPE', 'nvidia-tesla-t4')
+accelerator_node_selector := "gpu-type=" + pod_accelerator_type
+
+container_type := "dev" # or "app"
+container_image := if container_type == "dev" {
+    "pyrovelocitygpu"
+  } else if container_type == "app" {
+    "pyrovelocityapp"
+  } else {
+    error("container_type must be either 'dev' or 'app'")
+  }
+container_tag := "latest"
+
+pod_source_type := env_var_or_default('POD_SOURCE_TYPE', 'git')
+pod_git_provider := env_var_or_default('POD_GIT_PROVIDER', 'github')
+pod_disk_size := env_var_or_default('POD_DISK_SIZE', '400Gi')
+pod_min_cpu := env_var_or_default('POD_MIN_CPU', '16')
+pod_min_mem := env_var_or_default('POD_MIN_MEM', '64Gi')
+pod_max_cpu := env_var_or_default('POD_MAX_CPU', '32')
+pod_max_mem := env_var_or_default('POD_MAX_MEM', '96Gi')
+pod_max_accel := env_var_or_default('POD_MAX_ACCEL', '1')
+pod_resources := "requests.cpu=" + pod_min_cpu + ",requests.memory=" + pod_min_mem + ",limits.cpu=" + pod_max_cpu + ",limits.memory=" + pod_max_mem + ",limits.nvidia.com/gpu=" + pod_max_accel
+
+architecture := if arch() == "x86_64" {
+    "amd64"
+  } else if arch() == "aarch64" {
+    "arm64"
+  } else {
+    error("unsupported architecture must be amd64 or arm64")
+  }
+
+opsys := if os() == "macos" {
+    "darwin"
+  } else if os() == "linux" {
+    "linux"
+  } else {
+    error("unsupported operating system must be darwin or linux")
+  }
+
+devpod_release := "latest" # or "v0.3.7" or "v0.4.0-alpha.4"
+
+devpod_binary_url := if devpod_release == "latest" {
+  "https://github.com/loft-sh/devpod/releases/latest/download/devpod-" + opsys + "-" + architecture
+} else {
+  "https://github.com/loft-sh/devpod/releases/download/" + devpod_release + "/devpod-" + opsys + "-" + architecture
+}
+
+# Install devpod (check/set: devpod_release)
+[unix]
+install-devpod:
+  curl -L -o devpod {{devpod_binary_url}} && \
+  sudo install -c -m 0755 devpod /usr/local/bin && \
+  rm -f devpod
+  which devpod
+  devpod version
+
+# Print devpod info
+devpod:
+  devpod version && echo
+  devpod context list
+  devpod provider list
+  devpod list
+
+# Install and use devpod kubernetes provider
+provider:
+  devpod provider add kubernetes --silent || true \
+  && devpod provider use kubernetes
+
+# Run latest container_image in current kube context
+pod:
+  devpod up \
+  --debug \
+  --devcontainer-image {{container_registry}}{{container_image}}:{{container_tag}} \
+  --provider kubernetes \
+  --ide vscode \
+  --open-ide \
+  --source {{pod_source_type}}:https://{{pod_git_provider}}.com/{{git_username}}/{{git_repo_name}} \
+  --provider-option DISK_SIZE={{pod_disk_size}} \
+  --provider-option NODE_SELECTOR={{accelerator_node_selector}} \
+  --provider-option RESOURCES={{pod_resources}} \
+  {{container_image}}
+
+# Stop devpod (check container_image hasn't changed with -n)
+stop:
+  devpod stop {{container_image}}
+
+# Delete devpod (check container_image hasn't changed with -n)
+delete:
+  devpod delete {{container_image}}