From 7c8a2dfda37fa0930987326c8a69864b6f64a8b2 Mon Sep 17 00:00:00 2001 From: skchronicles Date: Thu, 15 Sep 2022 14:58:52 -0400 Subject: [PATCH] Moving default resources to OpenOmics shared group area and adding enhancements for increased portability --- config/cluster.json | 1 - config/containers.json | 13 +++---------- config/genome.json | 14 +++++++------- src/run.py | 2 +- src/run.sh | 4 ++-- 5 files changed, 13 insertions(+), 21 deletions(-) diff --git a/config/cluster.json b/config/cluster.json index 4daccc3..7f3692a 100644 --- a/config/cluster.json +++ b/config/cluster.json @@ -2,7 +2,6 @@ "__default__": { "threads": "4", "mem": "8g", - "partition": "norm", "time": "0-04:00:00", "gres": "lscratch:64" }, diff --git a/config/containers.json b/config/containers.json index ce677ba..768528d 100644 --- a/config/containers.json +++ b/config/containers.json @@ -1,13 +1,6 @@ { "images": { - "cutadapt": "/data/NCBR/dev/SIFs/ccbr_cutadapt_1.18_v032219.sif", - "fastq_screen": "/data/NCBR/dev/SIFs/ccbr_fastq_screen_0.13.0_v2.0.sif", - "fastqc": "/data/NCBR/dev/SIFs/ccbr_fastqc_0.11.9_v1.1.sif", - "fastqvalidator": "/data/NCBR/dev/SIFs/ccbr_fastqvalidator_v0.1.0.sif", - "kraken": "/data/NCBR/dev/SIFs/ccbr_kraken_v2.1.1_v0.0.1.sif", - "metaquast": "/data/NCBR/dev/SIFs/quast_5.0.2--py27pl526ha92aebf_0.sif", - "metavirs": "docker://skchronicles/metavirs:v0.1.0", - "multiqc": "/data/NCBR/dev/SIFs/ccbr_multiqc_1.9_v0.0.1.sif", - "python": "/data/NCBR/dev/SIFs/ccbr_python_v0.0.1.sif" + "fastqvalidator": "docker://nciccbr/ccbr_fastqvalidator:v0.1.0", + "metavirs": "docker://skchronicles/metavirs:v0.1.0" } -} \ No newline at end of file +} diff --git a/config/genome.json b/config/genome.json index 4866e34..0e86605 100644 --- a/config/genome.json +++ b/config/genome.json @@ -1,12 +1,12 @@ { "references": { "adapters": "resources/TruSeq_and_nextera_adapters.fa", - "host_bowtie2_index": "/data/NCBR/references/metavirs/bowtie2/", - "kraken2_viral_db": "/data/NCBR/references/metavirs/kraken2/viral/", - "kronatools": "/data/NCBR/references/metavirs/kronatax_0921/", - "CAT_db": "/data/NCBR/references/metavirs/CAT_prepare_20210107/2021-01-07_CAT_database", - "CAT_taxonomy": "/data/NCBR/references/metavirs/CAT_prepare_20210107/2021-01-07_taxonomy", - "CAT_diamond": "/data/NCBR/references/metavirs/CAT_prepare_20210107/Diamond_2.0.6/diamond", - "ncbi_viral_fasta": "/data/NCBR/references/metavirs/ncbi/viral_oneline.fa" + "host_bowtie2_index": "/data/OpenOmics/references/metavirs/bowtie2/", + "kraken2_viral_db": "/data/OpenOmics/references/metavirs/kraken2/viral/", + "kronatools": "/data/OpenOmics/references/metavirs/kronatax_0921/", + "CAT_db": "/data/OpenOmics/references/metavirs/CAT_prepare_20210107/2021-01-07_CAT_database", + "CAT_taxonomy": "/data/OpenOmics/references/metavirs/CAT_prepare_20210107/2021-01-07_taxonomy", + "CAT_diamond": "/data/OpenOmics/references/metavirs/CAT_prepare_20210107/Diamond_2.0.6/diamond", + "ncbi_viral_fasta": "/data/OpenOmics/references/metavirs/ncbi/viral_oneline.fa" } } diff --git a/src/run.py b/src/run.py index cf6c806..e30de97 100644 --- a/src/run.py +++ b/src/run.py @@ -715,7 +715,7 @@ def runner(mode, outdir, alt_cache, logger, additional_bind_paths = None, # Look into later: it maybe worth # replacing Popen subprocess with a direct # snakemake API call: https://snakemake.readthedocs.io/en/stable/api_reference/snakemake.html - # CLUSTER_OPTS="'sbatch --gres {cluster.gres} --cpus-per-task {cluster.threads} -p {cluster.partition} \ + # CLUSTER_OPTS="'sbatch --gres {cluster.gres} --cpus-per-task {cluster.threads} \ # -t {cluster.time} --mem {cluster.mem} --job-name={params.rname} -e $SLURMDIR/slurm-%j_{params.rname}.out \ # -o $SLURMDIR/slurm-%j_{params.rname}.out'" # sbatch --parsable -J "$2" --gres=lscratch:500 --time=5-00:00:00 --mail-type=BEGIN,END,FAIL \ diff --git a/src/run.sh b/src/run.sh index 5dd61e5..02747e1 100755 --- a/src/run.sh +++ b/src/run.sh @@ -192,7 +192,7 @@ function submit(){ # --keep-remote --restart-times 3 -j 50 --use-singularity # --singularity-args -B {}.format({bindpaths}) --local-cores 24 SLURM_DIR="$3/logfiles/slurmfiles" - CLUSTER_OPTS="sbatch --gres {cluster.gres} --cpus-per-task {cluster.threads} -p {cluster.partition} -t {cluster.time} --mem {cluster.mem} --job-name={params.rname} -e $SLURM_DIR/slurm-%j_{params.rname}.out -o $SLURM_DIR/slurm-%j_{params.rname}.out" + CLUSTER_OPTS="sbatch --gres {cluster.gres} --cpus-per-task {cluster.threads} -t {cluster.time} --mem {cluster.mem} --job-name={params.rname} -e $SLURM_DIR/slurm-%j_{params.rname}.out -o $SLURM_DIR/slurm-%j_{params.rname}.out" # Check if NOT running on Biowulf # Assumes other clusters do NOT # have GRES for local node disk, @@ -207,7 +207,7 @@ function submit(){ # SLURM is not configured to use # GRES, remove prefix single quote if [[ ${6#\'} != /lscratch* ]]; then - CLUSTER_OPTS="sbatch --cpus-per-task {cluster.threads} -p {cluster.partition} -t {cluster.time} --mem {cluster.mem} --job-name={params.rname} -e $SLURM_DIR/slurm-%j_{params.rname}.out -o $SLURM_DIR/slurm-%j_{params.rname}.out" + CLUSTER_OPTS="sbatch --cpus-per-task {cluster.threads} -t {cluster.time} --mem {cluster.mem} --job-name={params.rname} -e $SLURM_DIR/slurm-%j_{params.rname}.out -o $SLURM_DIR/slurm-%j_{params.rname}.out" fi # Create sbacth script to build index cat << EOF > kickoff.sh