Feat/containerize dnase

.gitignore

@@ -32,3 +32,7 @@ environments/
 work
 genome_build
 tmp
+
+*.sif
+containers/module_files/*
+programs/*

containers/Makefile

@@ -0,0 +1,63 @@
+# Makefile for building BWA process containers
+#
+# Usage:
+#   make all           - Build all containers
+#   make align         - Build just the align container
+#   make clean         - Remove all .sif files
+#   make list          - List all available targets
+
+# Container definitions and their corresponding .sif files
+CONTAINERS = align samtools-1.12 pysam-samtools mark_duplicates insert_size spot_score density_files macs2 bwa-aggregate-base
+
+# Directory where containers are stored
+CONTAINER_DIR = bwa
+
+# Default target
+.PHONY: all
+all: $(addprefix $(CONTAINER_DIR)/, $(addsuffix .sif, $(CONTAINERS)))
+
+# Pattern rule for building .sif files from .def files
+$(CONTAINER_DIR)/%.sif: $(CONTAINER_DIR)/%.def
+	@echo "Building container: $@"
+	apptainer build --force "$@" "$<"
+	@echo "Successfully built: $@"
+
+# Individual container targets for convenience
+.PHONY: align samtools-1.12 pysam-samtools mark_duplicates insert_size spot_score density_files macs2 bwa-aggregate-base
+align: $(CONTAINER_DIR)/align.sif
+samtools-1.12: $(CONTAINER_DIR)/samtools-1.12.sif
+pysam-samtools: $(CONTAINER_DIR)/pysam-samtools.sif
+mark_duplicates: $(CONTAINER_DIR)/mark_duplicates.sif
+insert_size: $(CONTAINER_DIR)/insert_size.sif
+density_files: $(CONTAINER_DIR)/density_files.sif
+spot_score: $(CONTAINER_DIR)/spot_score.sif
+macs2: $(CONTAINER_DIR)/macs2.sif
+bwa-aggregate-base: $(CONTAINER_DIR)/bwa-aggregate-base.sif
+
+# Clean target to remove all .sif files
+.PHONY: clean
+clean:
+	@echo "Removing all .sif files..."
+	rm -f $(CONTAINER_DIR)/*.sif
+	@echo "Clean complete."
+
+# List available targets
+.PHONY: list
+list:
+	@echo "Available container targets:"
+	@echo "  all              - Build all containers"
+	@echo "  align            - Build align container"
+	@echo "  samtools-1.12    - Build samtools-1.12 container"
+	@echo "  pysam-samtools   - Build pysam-samtools container"
+	@echo "  mark_duplicates  - Build mark_duplicates container"
+	@echo "  insert_size      - Build insert_size container"
+	@echo "  density_files    - Build density_files container"
+	@echo "  spot_score       - Build spot_score container"
+	@echo "  macs2            - Build macs2 container"
+	@echo "  bwa-aggregate-base - Build bwa-aggregate-base container"
+	@echo "  clean            - Remove all .sif files"
+	@echo "  list             - Show this help"
+
+# Help target
+.PHONY: help
+help: list

containers/bwa/align.def

@@ -0,0 +1,46 @@
+Bootstrap: docker
+From: ubuntu:20.04
+
+%post -c /bin/bash
+    # Install Environment Modules, sed, and findutils
+    apt-get update && apt-get install -y environment-modules sed findutils libncurses5
+
+    # Fix the hardcoded paths in all modulefiles
+    find /etc/modulefiles -type f -exec sed -i 's|/net/module/sw|/opt/sw|g' {} +
+
+    # Load modules
+    source /etc/profile.d/modules.sh
+    export MODULEPATH=/etc/modulefiles
+
+    # save current ENV to file
+    env > /tmp/env_before_modules.txt
+    module load bwa/0.7.12 samtools/1.3
+
+    # diff env to store modified variables in APPTAINER_ENVIRONMENT
+    env > /tmp/env_after_modules.txt
+    diff /tmp/env_before_modules.txt /tmp/env_after_modules.txt | grep '^> ' | sed 's/^> //' >> "$APPTAINER_ENVIRONMENT"
+
+
+%files
+    # Copy only the modulefiles needed for align process
+    module_files/modulefiles/bwa /etc/modulefiles/bwa
+    module_files/modulefiles/samtools /etc/modulefiles/samtools
+
+    # Copy only the software needed for align process
+    module_files/sw/bwa /opt/sw/bwa
+    module_files/sw/samtools /opt/sw/samtools
+
+%environment
+    # Set the path to the custom modulefiles
+    export MODULEPATH=/etc/modulefiles
+
+%runscript
+    # This script runs when the container is executed
+    echo "BWA Align container - modules: bwa/0.7.12, samtools/1.3"
+    echo "You can use 'module avail' and 'module load' commands."
+    exec /bin/bash
+
+%test
+    # Test basic functionality
+    bwa 2>&1 | grep -q "Usage"
+    samtools --version | grep -q "samtools 1.3"

containers/bwa/bwa-aggregate-base.def

@@ -0,0 +1,116 @@
+Bootstrap: docker
+From: ubuntu:18.04  # We use 18.04 to get libgfortan3
+
+%post -c /bin/bash
+    # Install Environment Modules, sed, and findutils
+    apt-get update && apt-get install -y environment-modules sed findutils libncurses5 libgfortran3 tcl
+
+    # Fix the hardcoded paths in all modulefiles
+    find /etc/modulefiles -type f -exec sed -i 's|/net/module/sw|/opt/sw|g' {} +
+
+    # Load modules
+    source /etc/profile.d/modules.sh
+    export MODULEPATH=/etc/modulefiles
+
+    # save current ENV to file
+    env > /tmp/env_before_modules.txt
+    module load openssl-dev/1.0.1t libpng/1.6.21 mysql-connector/6.1.6 htslib/1.3.1 bedops/2.4.35-typical samtools/1.3 modwt/1.0 kentutil/302 hotspot2/2.1.1 jdk/1.8.0_92 gcc/4.7.2 R/3.2.5 picard/2.8.1 git/2.3.3 coreutils/8.25 bedtools/2.25.0 python/3.5.1 pysam/0.9.0 htslib/1.6.0 numpy/1.11.0 atlas-lapack/3.10.2 scipy/1.0.0 scikit-learn/0.18.1 preseq/2.0.3 gsl/2.4
+
+    # diff env to store modified variables in APPTAINER_ENVIRONMENT
+    env > /tmp/env_after_modules.txt
+    diff /tmp/env_before_modules.txt /tmp/env_after_modules.txt | grep '^> ' | sed 's/^> //' | sed -e 's/\([^=]*\)=\(.*\)/export \1="\2"/' | sed 's/&/\\&/g' >> "$APPTAINER_ENVIRONMENT"
+
+
+%files
+    # Copy all the modulefiles needed for this container
+    module_files/modulefiles/openssl-dev/1.0.1t /etc/modulefiles/openssl-dev/1.0.1t
+    module_files/modulefiles/libpng/1.6.21 /etc/modulefiles/libpng/1.6.21
+    module_files/modulefiles/mysql-connector/6.1.6 /etc/modulefiles/mysql-connector/6.1.6
+    module_files/modulefiles/htslib/1.3.1 /etc/modulefiles/htslib/1.3.1
+    module_files/modulefiles/htslib/1.6.0 /etc/modulefiles/htslib/1.6.0
+    module_files/modulefiles/bedops/2.4.35-typical /etc/modulefiles/bedops/2.4.35-typical
+    module_files/modulefiles/samtools/1.3 /etc/modulefiles/samtools/1.3
+    module_files/modulefiles/modwt/1.0 /etc/modulefiles/modwt/1.0
+    module_files/modulefiles/kentutil/302 /etc/modulefiles/kentutil/302
+    module_files/modulefiles/hotspot2/2.1.1 /etc/modulefiles/hotspot2/2.1.1
+    module_files/modulefiles/jdk/1.8.0_92 /etc/modulefiles/jdk/1.8.0_92
+    module_files/modulefiles/gcc/4.7.2 /etc/modulefiles/gcc/4.7.2
+    module_files/modulefiles/R/3.2.5 /etc/modulefiles/R/3.2.5
+    module_files/modulefiles/picard/2.8.1 /etc/modulefiles/picard/2.8.1
+    module_files/modulefiles/git/2.3.3 /etc/modulefiles/git/2.3.3
+    module_files/modulefiles/coreutils/8.25 /etc/modulefiles/coreutils/8.25
+    module_files/modulefiles/bedtools/2.25.0 /etc/modulefiles/bedtools/2.25.0
+    module_files/modulefiles/python/3.5.1 /etc/modulefiles/python/3.5.1
+    module_files/modulefiles/pysam/0.9.0 /etc/modulefiles/pysam/0.9.0
+    module_files/modulefiles/numpy/1.11.0 /etc/modulefiles/numpy/1.11.0
+    module_files/modulefiles/atlas-lapack/3.10.2 /etc/modulefiles/atlas-lapack/3.10.2
+    module_files/modulefiles/scipy/1.0.0 /etc/modulefiles/scipy/1.0.0
+    module_files/modulefiles/scikit-learn/0.18.1 /etc/modulefiles/scikit-learn/0.18.1
+    module_files/modulefiles/preseq/2.0.3 /etc/modulefiles/preseq/2.0.3
+    module_files/modulefiles/gsl/2.4 /etc/modulefiles/gsl/2.4
+    module_files/modulefiles/zlib/1.2.8 /etc/modulefiles/zlib/1.2.8
+    module_files/modulefiles/libssh2/1.7.0 /etc/modulefiles/libssh2/1.7.0
+    module_files/modulefiles/curl/7.49.1 /etc/modulefiles/curl/7.49.1
+
+    # Copy all the software needed for this container
+    module_files/sw/openssl-dev/1.0.1t /opt/sw/openssl-dev/1.0.1t
+    module_files/sw/libpng/1.6.21 /opt/sw/libpng/1.6.21
+    module_files/sw/mysql-connector/6.1.6 /opt/sw/mysql-connector/6.1.6
+    module_files/sw/htslib/1.3.1 /opt/sw/htslib/1.3.1
+    module_files/sw/htslib/1.6.0 /opt/sw/htslib/1.6.0
+    module_files/sw/bedops/2.4.35-typical /opt/sw/bedops/2.4.35-typical
+    module_files/sw/samtools/1.3 /opt/sw/samtools/1.3
+    module_files/sw/modwt/1.0 /opt/sw/modwt/1.0
+    module_files/sw/kentutil/302 /opt/sw/kentutil/302
+    module_files/sw/hotspot2/2.1.1 /opt/sw/hotspot2/2.1.1
+    module_files/sw/jdk/1.8.0_92 /opt/sw/jdk/1.8.0_92
+    module_files/sw/gcc/4.7.2 /opt/sw/gcc/4.7.2
+    module_files/sw/R/3.2.5 /opt/sw/R/3.2.5
+    module_files/sw/picard/2.8.1 /opt/sw/picard/2.8.1
+    module_files/sw/git/2.3.3 /opt/sw/git/2.3.3
+    module_files/sw/coreutils/8.25 /opt/sw/coreutils/8.25
+    module_files/sw/bedtools/2.25.0 /opt/sw/bedtools/2.25.0
+    module_files/sw/python/3.5.1 /opt/sw/python/3.5.1
+    module_files/sw/pysam/0.9.0 /opt/sw/pysam/0.9.0
+    module_files/sw/numpy/1.11.0 /opt/sw/numpy/1.11.0
+    module_files/sw/atlas-lapack/3.10.2 /opt/sw/atlas-lapack/3.10.2
+    module_files/sw/scipy/1.0.0 /opt/sw/scipy/1.0.0
+    module_files/sw/scikit-learn/0.18.1 /opt/sw/scikit-learn/0.18.1
+    module_files/sw/preseq/2.0.3 /opt/sw/preseq/2.0.3
+    module_files/sw/gsl/2.4 /opt/sw/gsl/2.4
+    module_files/sw/zlib/1.2.8 /opt/sw/zlib/1.2.8
+    module_files/sw/libssh2/1.7.0 /opt/sw/libssh2/1.7.0
+    module_files/sw/curl/7.49.1 /opt/sw/curl/7.49.1
+
+%environment
+    # Set the path to the custom modulefiles
+    export MODULEPATH=/etc/modulefiles
+
+%runscript
+    # This script runs when the container is executed
+    echo "bwa-aggregate-base container"
+    exec /bin/bash
+
+%test
+    # Test basic functionality
+    samtools --version | grep -q "samtools"
+    python3.5 --version | grep -q "Python 3.5"
+    R --version | grep -q "R version"
+    java -version 2>&1 | grep -q "java version"
+    gcc --version | grep -q "gcc"
+    git --version | grep -q "git version"
+    bedtools --version | grep -q "bedtools"
+    preseq 2>&1 | grep -q "Usage"
+
+    # Test Python scientific stack
+    python3.5 -c "import pysam; print('pysam import successful')"
+    python3.5 -c "import numpy; print('numpy import successful')"
+    python3.5 -c "import scipy; print('scipy import successful')"
+    python3.5 -c "import sklearn; print('scikit-learn import successful')"
+
+    # Test bedops tools
+    which convert2bed
+    which bedops
+
+    # Test that picard is available
+    picard 2>&1 | grep -q MarkDuplicates

containers/bwa/density_files.def

@@ -0,0 +1,67 @@
+Bootstrap: docker
+From: ubuntu:20.04
+
+%post -c /bin/bash
+    set -e
+    # Install Environment Modules, sed, and findutils
+    apt-get update && apt-get install -y environment-modules sed findutils libncurses5 libcurl4
+
+    # Fix the hardcoded paths in all modulefiles
+    find /etc/modulefiles -type f -exec sed -i 's|/net/module/sw|/opt/sw|g' {} +
+
+    # Load modules
+    source /etc/profile.d/modules.sh
+    export MODULEPATH=/etc/modulefiles
+
+    # save current ENV to file
+    env > /tmp/env_before_modules.txt
+    module load openssl-dev/1.0.1t libpng/1.6.21 git/2.3.3 mysql-connector/6.1.6 bedops/2.4.35-typical samtools/1.3 htslib/1.6.0 kentutil/302
+
+    # diff env to store modified variables in APPTAINER_ENVIRONMENT
+    env > /tmp/env_after_modules.txt
+    diff /tmp/env_before_modules.txt /tmp/env_after_modules.txt | grep '^> ' | sed 's/^> //' | sed -e 's/\([^=]*\)=\(.*\)/\1="\2"/' >> "$APPTAINER_ENVIRONMENT"
+
+
+%files
+    # Copy only the modulefiles needed for this container
+    module_files/modulefiles/openssl-dev /etc/modulefiles/openssl-dev
+    module_files/modulefiles/libpng /etc/modulefiles/libpng
+    module_files/modulefiles/git /etc/modulefiles/git
+    module_files/modulefiles/mysql-connector /etc/modulefiles/mysql-connector
+    module_files/modulefiles/bedops /etc/modulefiles/bedops
+    module_files/modulefiles/samtools /etc/modulefiles/samtools
+    module_files/modulefiles/htslib /etc/modulefiles/htslib
+    module_files/modulefiles/kentutil /etc/modulefiles/kentutil
+
+    # Copy only the software needed for this container
+    module_files/sw/openssl-dev /opt/sw/openssl-dev
+    module_files/sw/libpng /opt/sw/libpng
+    module_files/sw/git /opt/sw/git
+    module_files/sw/mysql-connector /opt/sw/mysql-connector
+    module_files/sw/bedops /opt/sw/bedops
+    module_files/sw/samtools /opt/sw/samtools
+    module_files/sw/htslib /opt/sw/htslib
+    module_files/sw/kentutil /opt/sw/kentutil
+
+%environment
+    # Set the path to the custom modulefiles
+    export MODULEPATH=/etc/modulefiles
+
+%runscript
+    # This script runs when the container is executed
+    echo "density_files container - modules: bedops/2.4.35-typical, samtools/1.3, htslib/1.6.0, kentutil/302"
+    exec /bin/bash
+
+%test
+    # Test basic functionality
+    samtools --version | grep -q "samtools"
+    git --version | grep -q "git version"
+    bgzip --version | grep -q "bgzip"
+
+    # Test bedops tools
+    convert2bed --version
+    bedops --version
+
+    # Test kentutil tools (common ones)
+    bigWigToBedGraph 2>&1 | grep -q usage
+    bedGraphToBigWig 2>&1 | grep -q usage

containers/bwa/insert_size.def

@@ -0,0 +1,58 @@
+Bootstrap: docker
+From: ubuntu:16.04  # We use 16.04 to get libgfortran3 and readline4
+
+%post -c /bin/bash
+    # Install Environment Modules, sed, and findutils
+    apt-get update && apt-get install -y environment-modules sed findutils libncurses5 libgomp1 libgfortran3 tcl libreadline-dev
+    #curl http://archive.ubuntu.com/ubuntu/pool/main/r/readline6/libreadline6_6.3-8ubuntu2_amd64.deb -O libreadline6_6.3-8ubuntu2_amd64.deb
+    #dpkg -i libreadline6_6.3-8ubuntu2_amd64.deb
+
+    # Fix the hardcoded paths in all modulefiles
+    find /etc/modulefiles /opt/sw -type f -exec sed -i 's|/net/module/sw|/opt/sw|g' {} +
+
+    # Load modules
+    source /etc/profile.d/modules.sh
+    export MODULEPATH=/etc/modulefiles
+
+    # save current ENV to file
+    env > /tmp/env_before_modules.txt
+    module load jdk/1.8.0_92 picard/2.8.1 samtools/1.3 R/3.2.5
+
+    # diff env to store modified variables in APPTAINER_ENVIRONMENT
+    env > /tmp/env_after_modules.txt
+    diff /tmp/env_before_modules.txt /tmp/env_after_modules.txt | grep '^> ' | sed 's/^> //' | sed -e 's/\([^=]*\)=\(.*\)/\1="\2"/' >> "$APPTAINER_ENVIRONMENT"
+
+    # fix libreadline dependency for R
+    #ln -s /usr/lib/x86_64-linux-gnu/libreadline.so.7 /usr/lib/x86_64-linux-gnu/libreadline.so.6
+
+%files
+    # Copy only the modulefiles needed for this container
+    module_files/modulefiles/jdk /etc/modulefiles/jdk
+    module_files/modulefiles/picard /etc/modulefiles/picard
+    module_files/modulefiles/samtools /etc/modulefiles/samtools
+    module_files/modulefiles/R /etc/modulefiles/R
+
+    # Copy only the software needed for this container
+    module_files/sw/jdk /opt/sw/jdk
+    module_files/sw/picard /opt/sw/picard
+    module_files/sw/samtools/1.3 /opt/sw/samtools/1.3
+    module_files/sw/R/3.2.5 /opt/sw/R/3.2.5
+
+%environment
+    # Set the path to the custom modulefiles
+    export MODULEPATH=/etc/modulefiles
+
+%runscript
+    # This script runs when the container is executed
+    echo "insert_size container - modules: jdk/1.8.0_92, picard/2.8.1, samtools/1.3, R/3.2.5"
+    exec /bin/bash
+
+%test
+    # Test basic functionality
+    java -version 2>&1 | grep -q "java version"
+    samtools --version | grep -q "samtools"
+    R --version | grep -q "R version"
+    Rscript --version 2>&1 | grep -q "R scripting"
+
+    # Test that picard is available
+    picard CollectInsertSizeMetrics --version 2>&1 | grep -q 2.8.1