diff --git a/CHANGELOG.md b/CHANGELOG.md index 384497c..69f955c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [[1.2.1](https://github.com/sanger-tol/readmapping/releases/tag/1.2.1)] - [2024-02-29] + +### Enhancements & fixes + +- Increased the memory requests for reruns of BWAMEM2_MEM and SAMTOOLS_SORMADUP. + ## [[1.2.0](https://github.com/sanger-tol/readmapping/releases/tag/1.2.0)] – Norwegian Ridgeback - [2023-12-19] ### Enhancements & fixes diff --git a/conf/base.config b/conf/base.config index cdffac4..bfd327b 100644 --- a/conf/base.config +++ b/conf/base.config @@ -4,33 +4,6 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Increasing the number of CPUs often gives diminishing returns, so we increase it - following a logarithm curve. Example: - - 0 < value <= 1: start + step - - 1 < value <= 2: start + 2*step - - 2 < value <= 4: start + 3*step - - 4 < value <= 8: start + 4*step - In order to support re-runs, the step increase may be multiplied by the attempt - number prior to calling this function. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -// Modified logarithm function that doesn't return negative numbers -def positive_log(value, base) { - if (value <= 1) { - return 0 - } else { - return Math.log(value)/Math.log(base) - } -} - -def log_increase_cpus(start, step, value, base) { - return check_max(start + step * (1 + Math.ceil(positive_log(value, base))), 'cpus') -} - - process { errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } @@ -82,7 +55,7 @@ process { withName: 'SAMTOOLS_SORMADUP' { cpus = { log_increase_cpus(2, 6*task.attempt, 1, 2) } - memory = { check_max( 10.GB + 0.6.GB * Math.ceil( meta.read_count / 100000000 ) * task.attempt, 'memory' ) } + memory = { check_max( 4.GB + 850.MB * log_increase_cpus(2, 6*task.attempt, 1, 2) * task.attempt + 0.6.GB * Math.ceil( meta.read_count / 100000000 ), 'memory' ) } time = { check_max( 2.h * Math.ceil( meta.read_count / 100000000 ) * task.attempt / log_increase_cpus(2, 6*task.attempt, 1, 2), 'time' ) } } @@ -113,7 +86,7 @@ process { time = { check_max( 3.h * task.attempt * Math.ceil(positive_log(meta2.genome_size/100000, 10)) * Math.ceil(meta.read_count/1000000000) * 12 / log_increase_cpus(6, 6*task.attempt, meta.read_count/1000000000, 2), 'time' ) } // Base RAM usage is about 6 times the genome size. Each thread takes an additional 800 MB RAM // Memory usage of SAMTOOLS_VIEW is negligible. - memory = { check_max( 6.GB * Math.ceil(meta2.genome_size / 1000000000) + 800.MB * log_increase_cpus(6, 6*task.attempt, meta.read_count/1000000000, 2), 'memory' ) } + memory = { check_max( 6.GB * Math.ceil(meta2.genome_size / 1000000000) + 800.MB * task.attempt * log_increase_cpus(6, 6*task.attempt, meta.read_count/1000000000, 2), 'memory' ) } } withName: MINIMAP2_ALIGN { diff --git a/nextflow.config b/nextflow.config index f9d5027..03de7e7 100644 --- a/nextflow.config +++ b/nextflow.config @@ -183,7 +183,7 @@ manifest { description = 'Pipeline to map reads generated using different sequencing technologies against a genome assembly.' mainScript = 'main.nf' nextflowVersion = '!>=22.10.1' - version = '1.2.0' + version = '1.2.1' doi = '10.5281/zenodo.6563577' } @@ -222,3 +222,30 @@ def check_max(obj, type) { } } } + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Increasing the number of CPUs often gives diminishing returns, so we increase it + following a logarithm curve. Example: + - 0 < value <= 1: start + step + - 1 < value <= 2: start + 2*step + - 2 < value <= 4: start + 3*step + - 4 < value <= 8: start + 4*step + In order to support re-runs, the step increase may be multiplied by the attempt + number prior to calling this function. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +// Modified logarithm function that doesn't return negative numbers +def positive_log(value, base) { + if (value <= 1) { + return 0 + } else { + return Math.log(value)/Math.log(base) + } +} + +def log_increase_cpus(start, step, value, base) { + return check_max(start + step * (1 + Math.ceil(positive_log(value, base))), 'cpus') +} +