add conda and apptainer support

Author: poldrack

.dockerignore

@@ -1,29 +1,44 @@
 OUTPUT_DIR := /Users/poldrack/data_unsynced/BCBS/simple_workflow/wf_snakemake
+DOCKER_IMAGE := poldrack/simple-workflow
+DOCKER_TAG := latest
 
-.PHONY: run report graph
+.PHONY: run run-conda run-apptainer report graph clean docker-build docker-push
 
 clean:
 	-rm -rf .snakemake
 	-rm -rf $(OUTPUT_DIR)/*
 
 run:
-	uv run snakemake --cores 1 --config output_dir=$(OUTPUT_DIR)
+	uv run snakemake --cores 1 -d $(OUTPUT_DIR)
 
 run-conda:
-	uv run snakemake --cores 1 --sdm conda --config output_dir=$(OUTPUT_DIR)
+	uv run snakemake --cores 1 --sdm conda -d $(OUTPUT_DIR)
+
+run-apptainer:
+	uv run snakemake --cores 1 --sdm apptainer -d $(OUTPUT_DIR)
 
 lint:
-	snakemake --sdm conda --lint --cores 1 --config output_dir=$(OUTPUT_DIR)
+	snakemake --lint --cores 1 -d $(OUTPUT_DIR)
 
 dryrun:
-	snakemake --dry-run --cores 1 --config output_dir=$(OUTPUT_DIR)
+	snakemake --dry-run --cores 1 -d $(OUTPUT_DIR)
+
+dryrun-apptainer:
+	snakemake --dry-run --cores 1 --sdm apptainer -d $(OUTPUT_DIR)
 
 report:
-	snakemake --report $(OUTPUT_DIR)/report.html --config output_dir=$(OUTPUT_DIR)
+	snakemake --report $(OUTPUT_DIR)/report.html -d $(OUTPUT_DIR)
 
 graph:
-	snakemake --rulegraph --config output_dir=$(OUTPUT_DIR) --cores 2 | dot -Tpng -Gdpi=300  > output/rulegraph.png
+	snakemake --rulegraph -d $(OUTPUT_DIR) --cores 2 | dot -Tpng -Gdpi=300 > $(OUTPUT_DIR)/rulegraph.png
 
 export-env:
 	-mkdir envs
 	conda env export -n bettercode > envs/bettercode.yml
+
+# Container image management
+docker-build:
+	docker build -t $(DOCKER_IMAGE):$(DOCKER_TAG) .
+
+docker-push:
+	docker push $(DOCKER_IMAGE):$(DOCKER_TAG)

snakemake_workflow/Makefile

@@ -9,147 +9,147 @@ This workflow demonstrates Snakemake features with a simple pandas-based analysi
 
 Usage:
     # Run full workflow
-    snakemake --cores 1 --config output_dir=/path/to/output
+    snakemake --cores 1 -d /path/to/output
+
+    # Run with container (requires apptainer/singularity)
+    snakemake --cores 1 --sdm apptainer -d /path/to/output
 
     # Dry run
-    snakemake -n --config output_dir=/path/to/output
+    snakemake -n -d /path/to/output
 
     # Generate report
-    snakemake --report report.html --config output_dir=/path/to/output
+    snakemake --report report.html -d /path/to/output
 """
 
-from pathlib import Path
-
 from snakemake.utils import min_version
 
 min_version("8.0")
 
 
+# Base directory (where Snakefile is located)
+BASEDIR = workflow.basedir
+
+
 # Load configuration
-configfile: "config/config.yaml"
+configfile: f"{BASEDIR}/config/config.yaml"
 
 
 # Global report
-report: "report/workflow.rst"
-
+report: f"{BASEDIR}/report/workflow.rst"
 
-# Validate required config
-if config.get("output_dir") is None:
-    raise ValueError("output_dir must be provided via --config output_dir=/path/to/output")
 
-OUTPUT_DIR = Path(config["output_dir"])
-DATA_DIR = OUTPUT_DIR / "data"
-RESULTS_DIR = OUTPUT_DIR / "results"
-FIGURES_DIR = OUTPUT_DIR / "figures"
-LOGS_DIR = OUTPUT_DIR / "logs"
+# Container image for all rules (used with --sdm apptainer)
+container: config["container"]
 
 
-# Create output directories at workflow start
-onstart:
-    shell(f"mkdir -p {DATA_DIR} {RESULTS_DIR} {FIGURES_DIR} {LOGS_DIR}")
+# Output directories (relative to working directory set via -d)
+DATA_DIR = "data"
+RESULTS_DIR = "results"
+FIGURES_DIR = "figures"
+LOGS_DIR = "logs"
 
 
 # Default target
 rule all:
     input:
-        FIGURES_DIR / "correlation_heatmap.png",
+        f"{FIGURES_DIR}/correlation_heatmap.png",
 
 
 # Step 1a: Download meaningful variables data
 rule download_meaningful_variables:
     output:
-        DATA_DIR / "meaningful_variables.csv",
+        f"{DATA_DIR}/meaningful_variables.csv",
     params:
         url=config["meaningful_variables_url"],
     log:
-        OUTPUT_DIR / "logs" / "download_meaningful_variables.log",
+        f"{LOGS_DIR}/download_meaningful_variables.log",
     conda:
-        "envs/simple.yml"
+        f"{BASEDIR}/envs/simple.yml"
     script:
-        "scripts/download_data.py"
+        f"{BASEDIR}/scripts/download_data.py"
 
 
 # Step 1b: Download demographics data
 rule download_demographics:
     output:
-        DATA_DIR / "demographics.csv",
+        f"{DATA_DIR}/demographics.csv",
     params:
         url=config["demographics_url"],
     log:
-        OUTPUT_DIR / "logs" / "download_demographics.log",
+        f"{LOGS_DIR}/download_demographics.log",
     conda:
-        "envs/simple.yml"
+        f"{BASEDIR}/envs/simple.yml"
     script:
-        "scripts/download_data.py"
+        f"{BASEDIR}/scripts/download_data.py"
 
 
 # Step 2a: Filter meaningful variables to numerical columns
 rule filter_meaningful_variables:
     input:
-        DATA_DIR / "meaningful_variables.csv",
+        f"{DATA_DIR}/meaningful_variables.csv",
     output:
-        DATA_DIR / "meaningful_variables_numerical.csv",
+        f"{DATA_DIR}/meaningful_variables_numerical.csv",
     log:
-        OUTPUT_DIR / "logs" / "filter_meaningful_variables.log",
+        f"{LOGS_DIR}/filter_meaningful_variables.log",
     conda:
-        "envs/simple.yml"
+        f"{BASEDIR}/envs/simple.yml"
     script:
-        "scripts/filter_data.py"
+        f"{BASEDIR}/scripts/filter_data.py"
 
 
 # Step 2b: Filter demographics to numerical columns
 rule filter_demographics:
     input:
-        DATA_DIR / "demographics.csv",
+        f"{DATA_DIR}/demographics.csv",
     output:
-        DATA_DIR / "demographics_numerical.csv",
+        f"{DATA_DIR}/demographics_numerical.csv",
     log:
-        OUTPUT_DIR / "logs" / "filter_demographics.log",
+        f"{LOGS_DIR}/filter_demographics.log",
     conda:
-        "envs/simple.yml"
+        f"{BASEDIR}/envs/simple.yml"
     script:
-        "scripts/filter_data.py"
+        f"{BASEDIR}/scripts/filter_data.py"
 
 
 # Step 3: Join the two datasets
 rule join_datasets:
     input:
-        meaningful_vars=DATA_DIR / "meaningful_variables_numerical.csv",
-        demographics=DATA_DIR / "demographics_numerical.csv",
+        meaningful_vars=f"{DATA_DIR}/meaningful_variables_numerical.csv",
+        demographics=f"{DATA_DIR}/demographics_numerical.csv",
     output:
-        DATA_DIR / "joined_data.csv",
+        f"{DATA_DIR}/joined_data.csv",
     log:
-        OUTPUT_DIR / "logs" / "join_datasets.log",
+        f"{LOGS_DIR}/join_datasets.log",
     conda:
-        "envs/simple.yml"
+        f"{BASEDIR}/envs/simple.yml"
     script:
-        "scripts/join_data.py"
+        f"{BASEDIR}/scripts/join_data.py"
 
 
 # Step 4: Compute correlation matrix
 rule compute_correlation:
     input:
-        DATA_DIR / "joined_data.csv",
+        f"{DATA_DIR}/joined_data.csv",
     output:
-        RESULTS_DIR / "correlation_matrix.csv",
+        f"{RESULTS_DIR}/correlation_matrix.csv",
     params:
         method=config["correlation_method"],
     log:
-        OUTPUT_DIR / "logs" / "compute_correlation.log",
+        f"{LOGS_DIR}/compute_correlation.log",
     conda:
-        "envs/simple.yml"
+        f"{BASEDIR}/envs/simple.yml"
     script:
-        "scripts/compute_correlation.py"
+        f"{BASEDIR}/scripts/compute_correlation.py"
 
 
 # Step 5: Generate clustered heatmap
 rule generate_heatmap:
     input:
-        RESULTS_DIR / "correlation_matrix.csv",
+        f"{RESULTS_DIR}/correlation_matrix.csv",
     output:
         report(
-            FIGURES_DIR / "correlation_heatmap.png",
-            caption="report/heatmap.rst",
+            f"{FIGURES_DIR}/correlation_heatmap.png",
+            caption=f"{BASEDIR}/report/heatmap.rst",
             category="Results",
         ),
     params:
@@ -158,8 +158,8 @@ rule generate_heatmap:
         vmin=config["heatmap"]["vmin"],
         vmax=config["heatmap"]["vmax"],
     log:
-        OUTPUT_DIR / "logs" / "generate_heatmap.log",
+        f"{LOGS_DIR}/generate_heatmap.log",
     conda:
-        "envs/simple.yml"
+        f"{BASEDIR}/envs/simple.yml"
     script:
-        "scripts/generate_heatmap.py"
+        f"{BASEDIR}/scripts/generate_heatmap.py"

snakemake_workflow/Snakefile

@@ -1,5 +1,8 @@
 # Configuration for the simple correlation Snakemake workflow
 
+# Container image (used with --sdm apptainer)
+container: "docker://poldrack/simple-workflow:latest"
+
 # Data URLs
 meaningful_variables_url: "https://raw.githubusercontent.com/IanEisenberg/Self_Regulation_Ontology/refs/heads/master/Data/Complete_02-16-2019/meaningful_variables_clean.csv"
 demographics_url: "https://raw.githubusercontent.com/IanEisenberg/Self_Regulation_Ontology/refs/heads/master/Data/Complete_02-16-2019/demographics.csv"

snakemake_workflow/config/config.yaml

@@ -1,8 +1,8 @@
-name: bettercode
+name: simple_workflow
 channels:
   - conda-forge
 dependencies:
-  - numpy=2.4.0
-  - pandas=2.3.3
-  - matplotlib==3.10.8
-  - seaborn==0.13.2
+  - python>=3.10
+  - pandas
+  - matplotlib
+  - seaborn

snakemake_workflow/envs/simple.yml

@@ -1,10 +1,13 @@
 """Snakemake script for computing correlation matrix."""
 
+import sys
 from pathlib import Path
 
 import pandas as pd
 
-from bettercode.simple_workflow import compute_correlation_matrix
+# Add workflow directory to path for local simple_workflow module
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from simple_workflow import compute_correlation_matrix
 
 def main():
     """Compute Spearman correlation matrix."""

snakemake_workflow/scripts/compute_correlation.py

@@ -1,10 +1,13 @@
 """Snakemake script for filtering data to numerical columns."""
 
+import sys
 from pathlib import Path
 
 import pandas as pd
 
-from bettercode.simple_workflow import filter_numerical_columns
+# Add workflow directory to path for local simple_workflow module
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from simple_workflow import filter_numerical_columns
 
 def main():
     """Filter data to numerical columns."""

snakemake_workflow/scripts/filter_data.py

@@ -1,12 +1,15 @@
 """Snakemake script for generating clustered heatmap."""
 
+import sys
 from pathlib import Path
 
 import matplotlib.pyplot as plt
 import pandas as pd
 import seaborn as sns
 
-from bettercode.simple_workflow import generate_clustered_heatmap
+# Add workflow directory to path for local simple_workflow module
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from simple_workflow import generate_clustered_heatmap
 
 
 def main():

snakemake_workflow/scripts/generate_heatmap.py

@@ -1,10 +1,13 @@
 """Snakemake script for joining two dataframes."""
 
+import sys
 from pathlib import Path
 
 import pandas as pd
 
-from bettercode.simple_workflow import join_dataframes
+# Add workflow directory to path for local simple_workflow module
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from simple_workflow import join_dataframes
 
 def main():
     """Join the two datasets."""