Merge pull request #7 from cokelaer/dev

Set the singularity container and fix falco

Author: cokelaer

.github/workflows/main.yml

@@ -38,7 +38,6 @@ jobs:
       run: |
         # $CONDA is an environment variable pointing to the root of the miniconda directory
         echo $CONDA/bin >> $GITHUB_PATH
-        #conda update ruamel_yaml
 
     - name: conda
       run: |

README.rst

@@ -25,14 +25,12 @@ This is is the **fastqc** pipeline from the `Sequana <https://sequana.readthedoc
 Installation
 ~~~~~~~~~~~~
 
-You must install Sequana first (use --upgrade to get the latest version installed)::
-
-    pip install sequana --upgrade
-
-Then, just install this package::
+sequana_fastqc is based on Python3, just install the package as follows::
 
     pip install sequana_fastqc --upgrade
 
+You will need third-party software such as fastqc. Please see below for details.
+
 Usage
 ~~~~~
 
@@ -47,10 +45,10 @@ To know more about the options (e.g., add a different pattern to restrict the
 execution to a subset of the input files, change the output/working directory,
 etc)::
 
-    sequana_pipelines_fastqc --help
-    sequana_pipelines_fastqc --input-directory DATAPATH
+    sequana_fastqc --help
+    sequana_fastqc --input-directory DATAPATH
 
-This creates a directory **fastq**. You just need to execute the pipeline::
+This creates a directory **fastqc**. You just need to execute the pipeline::
 
     cd fastqc
     sh fastqc.sh  # for a local run
@@ -83,7 +81,7 @@ then, prepare the pipeline::
 Just open the HTML entry called summary.html. A multiqc report is also available. 
 You will get expected images such as the following one:
 
-.. image:: https://github.com/sequana/fastqc/blob/master/doc/summary.png?raw=true
+.. image:: https://github.com/sequana/fastqc/blob/main/doc/summary.png?raw=true
 
 Please see the `Wiki <https://github.com/sequana/fastqc/wiki>`_ for more examples and features.
 
@@ -94,15 +92,17 @@ This pipelines requires the following executable(s):
 
 - fastqc
 - falco (optional)
-- sequana (Python: pip install sequana)
 
-For Linux users, we provide a singularity image available through damona::
 
-    pip install damona
-    damona install fastqc
-    # and add the ~/.config/damona/bin path to your binary PATH 
+For Linux users, we provide singularity images available through within the **damona** project (https://damona.readthedocs.io). 
+
+To mak use of them, initiliase the pipeline with the --use-singularity option and everything should be downloaded
+automatically for you, which also guarantees reproducibility::
+
+    sequana_fastqc --input-directory data --use-singularity
+
 
-.. image:: https://raw.githubusercontent.com/sequana/fastqc/master/sequana_pipelines/fastqc/dag.png
+.. image:: https://raw.githubusercontent.com/sequana/fastqc/main/sequana_pipelines/fastqc/dag.png
 
 
 Details
@@ -124,14 +124,16 @@ ready-to-use HTML reports, etc
 Rules and configuration details
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Here is the `latest documented configuration file <https://raw.githubusercontent.com/sequana/fastqc/master/sequana_pipelines/fastqc/config.yaml>`_
+Here is the `latest documented configuration file <https://raw.githubusercontent.com/sequana/fastqc/main/sequana_pipelines/fastqc/config.yaml>`_
 to be used with the pipeline. Each rule used in the pipeline may have a section in the configuration file. 
 
 Changelog
 ~~~~~~~~~
 ========= ====================================================================
 Version   Description
 ========= ====================================================================
+1.6.0     * Fixed falco output error and use singularity containers
+1.5.0     * removed modules completely.
 1.4.2     * simplified pipeline (suppress setup and use existing wrapper)
 1.4.1     * simplified pipeline with wrappers/rules
 1.4.0     * This version uses sequana 0.12.0 and new sequana-wrappers 

requirements.txt

@@ -1,2 +1,2 @@
 sequana>=0.12.1
-sequana_pipetools>=0.7.2
+sequana_pipetools>=0.9.1

sequana_pipelines/fastqc/config.yaml

@@ -11,34 +11,12 @@
 input_directory: '.'
 input_pattern: '*fastq.gz'
 
-################################################################################
-# extra_prefixes_to_strip
-#
-# In most cases, the input_directory and input_pattern (and input_read_tag for paired analysis)
-# allows us to retrieve unique sample names. We automatically removed common prefixes for you.
-#
-# Sometimes, some prefixes are not common to all samples but still need to be removed
-# to get unique sample names or better output. You can provide extra prefixes to be removed
-# by uncommenting and filling the field extra_prefixes_to_strip.
-#
-# For instance, if you have files called prefix.mess.A.fastq.gz and prefix.B.fastq.gz
-# 'prefix.' will be removed automatically because it is common, but not 'mess'. 
-# use thoses prefixes in the left to right order ['prefix', 'mess'] or ['prefix.mess']
-#
-# extra_prefixes_to_strip: []
 
-
-################################################################################
-# sample_pattern
-#
-# You may have trailing words that are in sample names but not wanted. Consider
-# the filename A_mess.fastq.gz, you can get rid of _mess if it appears in all 
-# samples using a pattern as follows '{sample}_mess.fastq.gz
-#
-# uncomment and fill to use this option. Be aware that prefixes are not removed
-# if you use sample_pattern
-#
-# sample_pattern: '{sample}_mess.fastq.gz'
+# See sequana_pipetools.readthedocs.io for details about these 2 options
+# common prefixes are removed. addition prefixes may be removed here
+#extra_prefixes_to_strip = []
+# in special cases, sample names can be extracted with a pattern
+#sample_pattern: '{sample}.fastq.gz'
 
 ##############################################################################
 # general section

sequana_pipelines/fastqc/fastqc.rules

@@ -76,6 +76,8 @@ if 'general' in config and 'method_choice' in config['general'] and \
         params:
             options=config['falco']['options'],
             working_directory="samples/{sample}"
+        container:
+            "https://zenodo.org/record/7014954/files/falco_1.0.0.img"
         resources:
             **config['falco']['resources']
         wrapper:
@@ -97,6 +99,8 @@ else:
             working_directory="samples/{sample}"
         resources:
             **config['fastqc']['resources']
+        container:
+            "https://zenodo.org/record/7015004/files/fastqc_0.11.9-py3.img"
         wrapper:
             "main/wrappers/fastqc"
     __multiqc__input = expand("samples/{sample}/fastqc.done", sample=manager.samples)
@@ -174,7 +178,7 @@ include: sm.modules['rulegraph']
 
 
 rule plotting_and_stats:
-    input: expand("samples/{sample}/" + f"{METHOD}.done", sample=manager.samples)
+    input: expand("samples/{sample}/" + f"{METHOD}.log", sample=manager.samples)
     output: "outputs/summary.png", "outputs/summary.json"
     resources:
         **config["multiqc"]["resources"]

sequana_pipelines/fastqc/main.py

@@ -108,9 +108,7 @@ def main(args=None):
         manager.exists(cfg.input_directory)
     # finalise the command and save it; copy the snakemake. update the config
     # file and save it.
-    # Since input can be bam, fast5, mgi, the standard FastQFactory may not
-    # work, so we only check presence of input files. 
-    manager.teardown(check_fastq_files=False)
+    manager.teardown()
 
     if options.run:
         subprocess.Popen(["sh", '{}.sh'.format(NAME)], cwd=options.workdir)

setup.py

@@ -1,12 +1,10 @@
-# -*- coding: utf-8 -*-
-# License: 3-clause BSD
 from setuptools import setup, find_namespace_packages
 from setuptools.command.develop import develop
 from setuptools.command.install import install
 import subprocess
 
 _MAJOR               = 1
-_MINOR               = 5
+_MINOR               = 6
 _MICRO               = 0
 version              = '%d.%d.%d' % (_MAJOR, _MINOR, _MICRO)
 release              = '%d.%d' % (_MAJOR, _MINOR)

test/test_main.py

@@ -26,7 +26,7 @@ def test_standalone_script():
     m.main()
 
 
-def test_full():
+def test_full_fastqc():
 
     with tempfile.TemporaryDirectory() as directory:
         wk = directory
@@ -48,3 +48,23 @@ def test_full():
 def test_version():
     cmd = "sequana_fastqc --version"
     subprocess.call(cmd.split())
+
+def test_full_falco():
+
+    with tempfile.TemporaryDirectory() as directory:
+        wk = directory
+
+        cmd = "sequana_fastqc --input-directory {} "
+        cmd += "--working-directory {} --run-mode local --force --method falco"
+        cmd = cmd.format(sharedir, wk)
+        subprocess.call(cmd.split())
+
+
+        cmd = "snakemake -s fastqc.rules --wrapper-prefix https://raw.githubusercontent.com/sequana/sequana-wrappers/  -p --cores 2 "
+
+        stat = subprocess.call(cmd.split(), cwd=wk)
+
+        assert os.path.exists(wk + "/summary.html")
+        assert os.path.exists(wk + "/tree.html")
+        assert os.path.exists(wk + "/multiqc/multiqc_report.html")
+