refactoring 2023_illumina_sequencing_c -> 2024_illumina_sequencing_d

_run_gbs_qc

@@ -281,9 +281,9 @@ function get_run_opts() {
    OUT_ROOT=""
    SNP_ENGINE=tassel        # the only one supported at this point
 
-   NOVASEQ_ROOT=/dataset/2023_illumina_sequencing_c/active
-   NOVASEQ_PROCESSING_ROOT=/dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs
-   NOVASEQ_BCLCONVERT_ROOT=/dataset/2023_illumina_sequencing_c/scratch/postprocessing/illumina/novaseq
+   NOVASEQ_ROOT=/dataset/2024_illumina_sequencing_d/active
+   NOVASEQ_PROCESSING_ROOT=/dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs
+   NOVASEQ_BCLCONVERT_ROOT=/dataset/2024_illumina_sequencing_d/scratch/postprocessing/illumina/novaseq
    mkdir -p $NOVASEQ_PROCESSING_ROOT
    mkdir -p $NOVASEQ_BCLCONVERT_ROOT
 

add_sample_sheet_header.py

@@ -78,7 +78,7 @@ def get_options():
 
 cat /dataset/hiseq/active/191021_D00390_0510_BCE3UBANXX/SampleSheet.csv | ./add_sample_sheet_header.py --sequencing_platform hiseq -H  /dataset/gseq_processing/active/bin/gbs_prism/etc/sample_sheet_header.csv
 
-cat /dataset/2023_illumina_sequencing_c/scratch/220426_A01439_0069_BHNFW2DRXY/HNFW2DRXY.csv | ./add_sample_sheet_header.py
+cat /dataset/2024_illumina_sequencing_d/scratch/220426_A01439_0069_BHNFW2DRXY/HNFW2DRXY.csv | ./add_sample_sheet_header.py
 
 
 """

ag_gbs_qc_prism.sh

@@ -421,7 +421,7 @@ export GBS_PRISM_BIN=$GBS_PRISM_BIN
 export SEQ_PRISMS_BIN=$SEQ_PRISMS_BIN
 export MELSEQ_PRISM_BIN=$GBS_PRISM_BIN/../melseq_prism
 
-DEMUX_ROOT=/dataset/2023_illumina_sequencing_c/scratch/microbiome_fasta/qc/$libname
+DEMUX_ROOT=/dataset/2024_illumina_sequencing_d/scratch/microbiome_fasta/qc/$libname
 mkdir -p \$DEMUX_ROOT
 
 # get keyfile needed

autostart_gbs_qc

@@ -15,7 +15,7 @@ This is used to auto-start a run. It will poll for a landmark file indicating up
 \n
 usage example (currently run in a screen session) :\n
 \n
-autostart_gbs_qc 200511_D00390_0545_BHFMY3BCX3 2>&1 | tee /dataset/2023_illumina_sequencing_c/scratch/postprocessing/autostart_logs/200511_D00390_0545_BHFMY3BCX3.log \n
+autostart_gbs_qc 200511_D00390_0545_BHFMY3BCX3 2>&1 | tee /dataset/2024_illumina_sequencing_d/scratch/postprocessing/autostart_logs/200511_D00390_0545_BHFMY3BCX3.log \n
 "
    while getopts ":h" opt; do
    case $opt in
@@ -82,14 +82,14 @@ function send_mail() {
 function get_landmark() {
    RUN=$1
    landmark=""
-   if [ -f /dataset/2023_illumina_sequencing_c/active/$RUN/RTAComplete.txt ]; then
-      landmark=/dataset/2023_illumina_sequencing_c/active/$RUN/RTAComplete.txt
+   if [ -f /dataset/2024_illumina_sequencing_d/active/$RUN/RTAComplete.txt ]; then
+      landmark=/dataset/2024_illumina_sequencing_d/active/$RUN/RTAComplete.txt
    fi
 }
 
 function get_digest() {
    RUN=$1
-   digest=`ls -lR /dataset/2023_illumina_sequencing_c/active/$RUN/ | md5sum -b `
+   digest=`ls -lR /dataset/2024_illumina_sequencing_d/active/$RUN/ | md5sum -b `
    echo $digest
 }
 

genotype_prism.sh

@@ -25,8 +25,8 @@ function get_opts() {
 usage :\n 
 ./genotype_prism.sh  [-h] [-n] [-d] [-x KGD_tassel] [-p genotyping parameters] [-m hapmap_folder] [-o outfolder] folder\n
 example:\n
-./genotype_prism.sh -x KGD_tassel3 /dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs/weevils_gbsx\n
-./genotype_prism.sh -x KGD_tassel3 -p pooled /dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs/pooled_worms\n
+./genotype_prism.sh -x KGD_tassel3 /dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs/weevils_gbsx\n
+./genotype_prism.sh -x KGD_tassel3 -p pooled /dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs/pooled_worms\n
 EOF
 )
    while getopts ":nhfC:x:p:m:o:" opt; do

import_hiseq_reads_tags_cv.sh

@@ -16,7 +16,7 @@ help_text="\n
 
 DRY_RUN=no
 RUN_NAME=""
-BUILD_ROOT=/dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs
+BUILD_ROOT=/dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs
 
 while getopts "hr:d:" opt; do
   case $opt in

import_kgd_stats.sh

@@ -8,7 +8,7 @@ help_text="\n
 "
 
 RUN_NAME=""
-BUILD_ROOT=/dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs 
+BUILD_ROOT=/dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs 
 
 while getopts "hr:d:" opt; do
   case $opt in
@@ -111,7 +111,7 @@ function import_data() {
 
 
    # import cohort level data 
-   # currently just from the KGD stdout file - e.g. /dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs/221020_A01439_0127_AHMKVMDRX2/SQ1951.all.deer.PstI/SQ1951.all.deer.PstI.KGD_tassel3.KGD.stdout
+   # currently just from the KGD stdout file - e.g. /dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs/221020_A01439_0127_AHMKVMDRX2/SQ1951.all.deer.PstI/SQ1951.all.deer.PstI.KGD_tassel3.KGD.stdout
    files=`ls $RUN_PATH/*/*.KGD_tassel3.KGD.stdout | egrep -v "\/OLD|_OLD"`
    for kgd_stdout in $files; do
       gupdate --explain -t lab_report -p "name=import_gbs_kgd_cohort_stats;file=$kgd_stdout" $RUN_NAME 

make_clientcohort_pages.py

@@ -57,7 +57,7 @@ def get_cohorts(options):
 
 
 def generate_run_plot(options):
-    BASEDIR=options["basedir"]   # e.g. /bifo/scratch/2023_illumina_sequencing_c/postprocessing/gbs    
+    BASEDIR=options["basedir"]   # e.g. /bifo/scratch/2024_illumina_sequencing_d/postprocessing/gbs    
     stats = {
         "found file count" : 0,
         "no file count" : 0,
@@ -308,7 +308,7 @@ def get_options():
     parser.add_argument('-o', '--output_filename' , dest='output_filename', default="peacock.html", type=str, help="name of output file")
     parser.add_argument('-U', '--hapmap_subfolder_name' , dest='hapmap_subfolder_name', default="hapMap", type=str, help="name of hapmap subfolder")
     parser.add_argument('-K', '--kgd_subfolder_name' , dest='kgd_subfolder_name', default="KGD", type=str, help="name of KGD sub-folder name")
-    parser.add_argument('-b', '--basedir' , dest='basedir', default="/dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs", type=str, help="base dir of original output")
+    parser.add_argument('-b', '--basedir' , dest='basedir', default="/dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs", type=str, help="base dir of original output")
 
 
 

run_kgd.sh

@@ -2,13 +2,13 @@
 #
 # stub for running KGD. It takes two required arguments and 1 optional argument
 # 1. The KGD folder - this will be working folder
-#    e.g. /dataset/2023_illumina_sequencing_c/scratch/postprocessing/151113_D00390_0239_BC808AANXX.gbs/SQ0139.processed_sample/uneak/KGD
+#    e.g. /dataset/2024_illumina_sequencing_d/scratch/postprocessing/151113_D00390_0239_BC808AANXX.gbs/SQ0139.processed_sample/uneak/KGD
 # 2. The "geno_method" (prev called samptype)  , "default" or "pooled" (default for diploid)
 # 3. Optional hapMap folder name (e.g. if want to use filtered hapMap data)
 # so e.g. run as 
-# run_kgd.sh /dataset/2023_illumina_sequencing_c/scratch/postprocessing/151113_D00390_0239_BC808AANXX.gbs/SQ0139.processed_sample/uneak/KGD diploid
+# run_kgd.sh /dataset/2024_illumina_sequencing_d/scratch/postprocessing/151113_D00390_0239_BC808AANXX.gbs/SQ0139.processed_sample/uneak/KGD diploid
 # or (non-default mapMap folder)
-# run_kgd.sh /dataset/2023_illumina_sequencing_c/scratch/postprocessing/151113_D00390_0239_BC808AANXX.gbs/SQ0139.processed_sample/uneak/KGD diploid filtered_hapMap
+# run_kgd.sh /dataset/2024_illumina_sequencing_d/scratch/postprocessing/151113_D00390_0239_BC808AANXX.gbs/SQ0139.processed_sample/uneak/KGD diploid filtered_hapMap
 
 KGD_WORKING=$1
 KGD_METHOD=$2

utils/SelfRelDepthtoHTML.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 #
 # make an HTML page to view the output of SelfRelDepth.r together with the orginal plot
-# - the output should be sent to /bifo/scratch/2023_illumina_sequencing_c/postprocessing/ so that the paths
+# - the output should be sent to /bifo/scratch/2024_illumina_sequencing_d/postprocessing/ so that the paths
 # work when opened 
 #
 import sys
@@ -106,18 +106,18 @@ def generate_run_plot():
 
     for record in record_array:
         # these are like
-        # /bifo/scratch/2023_illumina_sequencing_c/postprocessing/180130_D00390_0343_BCBG7MANXX.gbs/SQ0618.processed_sample/uneak/PstI.PstI.cohort/KGD -0.06391309 2.17314e-15
+        # /bifo/scratch/2024_illumina_sequencing_d/postprocessing/180130_D00390_0343_BCBG7MANXX.gbs/SQ0618.processed_sample/uneak/PstI.PstI.cohort/KGD -0.06391309 2.17314e-15
         #/dataset/gseq_processing/scratch/gbs/140624_D00390_0044_BH9PEBADXX/SQ0001.all.PstI.PstI/KGD     -0.05257605     0.1317527       cattle  PstI    PstI    13
         #/dataset/gseq_processing/scratch/gbs/140624_D00390_0044_BH9PEBADXX/SQ0001.all.PstI.PstI/KGD.orig        -0.05257605     0.1317527       cattle  PstI    PstI    13
 
         #print record
-        #['/bifo/scratch/2023_illumina_sequencing_c/postprocessing/180627_D00390_0375_BCCHBJANXX.gbs/SQ2741.processed_sample/uneak/all.ApeKI.ApeKI.cohort/KGD',
+        #['/bifo/scratch/2024_illumina_sequencing_d/postprocessing/180627_D00390_0375_BCCHBJANXX.gbs/SQ2741.processed_sample/uneak/all.ApeKI.ApeKI.cohort/KGD',
         # '-0.5114565', '1.276835e-12', 'white', 'clover', 'ApeKI', 'ApeKI', '96']
         #print "DEBUG processing %s"%record
 
         (path, slope,pval,species, gbs_cohort, enzyme, count) = record
         # need to fix up path - from this
-        # /bifo/scratch/2023_illumina_sequencing_c/postprocessing/180130_D00390_0343_BCBG7MANXX.gbs/SQ0618.processed_sample/uneak/PstI.PstI.cohort/KGD
+        # /bifo/scratch/2024_illumina_sequencing_d/postprocessing/180130_D00390_0343_BCBG7MANXX.gbs/SQ0618.processed_sample/uneak/PstI.PstI.cohort/KGD
         # to this
         # 180816_D00390_0393_ACCRBRANXX.gbs/SQ0782.processed_sample/uneak/all.GOAT.PstI.cohort/KGD/GHWdgm.05diagdepth.png
         #
@@ -129,7 +129,7 @@ def generate_run_plot():
             relpath=os.path.relpath(path, "/dataset/gseq_processing/scratch/gbs")
             image_path=os.path.join(relpath, "GHWdgm.05diagdepth.png")
         else:
-            relpath=os.path.relpath(path, "/dataset/2023_illumina_sequencing_c/scratch/postprocessing")
+            relpath=os.path.relpath(path, "/dataset/2024_illumina_sequencing_d/scratch/postprocessing")
             image_path=os.path.join("old_plots", relpath,"GHWdgm.05diagdepth.png")
 
 
@@ -138,16 +138,16 @@ def generate_run_plot():
 
         run_match=re.search("/dataset/gseq_processing/scratch/gbs/([^\/]+)/",path)
         if run_match is None:
-            run_match=re.search("/dataset/2023_illumina_sequencing_c/scratch/postprocessing/([^\/]+)\.gbs/",path)
+            run_match=re.search("/dataset/2024_illumina_sequencing_d/scratch/postprocessing/([^\/]+)\.gbs/",path)
             if run_match is None:
                 print "Error could not parse run from %s"%path
                 continue
 
         run=run_match.groups()[0]
 
-        if os.path.exists(os.path.join("/dataset/2023_illumina_sequencing_c/scratch/postprocessing/", "%s_plots.html"%run)):
-            plots_page="\\\\isamba\\dataset\\2023_illumina_sequencing_c\\scratch\\postprocessing\\%s_plots.html"%run
-            path="<a href=\"%s\" target=plots_page>%s</a>"%(plots_page, os.path.join("/dataset/2023_illumina_sequencing_c/scratch/postprocessing/", "%s_plots.html"%run))
+        if os.path.exists(os.path.join("/dataset/2024_illumina_sequencing_d/scratch/postprocessing/", "%s_plots.html"%run)):
+            plots_page="\\\\isamba\\dataset\\2024_illumina_sequencing_d\\scratch\\postprocessing\\%s_plots.html"%run
+            path="<a href=\"%s\" target=plots_page>%s</a>"%(plots_page, os.path.join("/dataset/2024_illumina_sequencing_d/scratch/postprocessing/", "%s_plots.html"%run))
         else:
             plots_page="file:///\\\\isamba\\" + path[1:].replace("/","\\")
             path="<a href=\"%s\" target=plots_page>%s</a>"%(plots_page, path)

utils/annotateSelfRelDepth.py

@@ -20,14 +20,14 @@ def annotate():
     #/dataset/gseq_processing/scratch/gbs/181005_D00390_0407_BCCV91ANXX/SQ0807.all.DEER.PstI/KGD -0.02888578 0.3695895
     #/dataset/gseq_processing/scratch/gbs/181005_D00390_0407_BCCV91ANXX/SQ2766.all.ApeKI.ApeKI/KGD -0.1362936 0.2429048
 
-    #/bifo/scratch/2023_illumina_sequencing_c/postprocessing/180810_D00390_0392_BCCR4LANXX.gbs/SQ0772.processed_sample/uneak/all.DEER.PstI.cohort/KGD -2.523152 5.655549e-05
-    #/bifo/scratch/2023_illumina_sequencing_c/postprocessing/180810_D00390_0392_BCCR4LANXX.gbs/SQ0772.processed_sample/uneak/all.GOAT.PstI.cohort/KGD -0.03751884 0.002106262
-    #/bifo/scratch/2023_illumina_sequencing_c/postprocessing/180810_D00390_0392_BCCR4LANXX.gbs/SQ0775.processed_sample/uneak/all.Cattle.PstI.cohort/KGD 0.007961025 0.7796325
+    #/bifo/scratch/2024_illumina_sequencing_d/postprocessing/180810_D00390_0392_BCCR4LANXX.gbs/SQ0772.processed_sample/uneak/all.DEER.PstI.cohort/KGD -2.523152 5.655549e-05
+    #/bifo/scratch/2024_illumina_sequencing_d/postprocessing/180810_D00390_0392_BCCR4LANXX.gbs/SQ0772.processed_sample/uneak/all.GOAT.PstI.cohort/KGD -0.03751884 0.002106262
+    #/bifo/scratch/2024_illumina_sequencing_d/postprocessing/180810_D00390_0392_BCCR4LANXX.gbs/SQ0775.processed_sample/uneak/all.Cattle.PstI.cohort/KGD 0.007961025 0.7796325
     #
     # and
-    # /bifo/scratch/2023_illumina_sequencing_c/postprocessing/171218_D00390_0337_BCBG3AANXX.gbs/SQ0575.processed_sample/uneak/PstI.PstI.cohort
+    # /bifo/scratch/2024_illumina_sequencing_d/postprocessing/171218_D00390_0337_BCBG3AANXX.gbs/SQ0575.processed_sample/uneak/PstI.PstI.cohort
     # and
-    # /dataset/2023_illumina_sequencing_c/scratch/postprocessing/150224_D00390_0217_AC4UAUACXX.gbs/SQ0056.processed_sample/uneak/all.PstI.PstI.cohort/KGD -0.2113675 2.645513e-25
+    # /dataset/2024_illumina_sequencing_d/scratch/postprocessing/150224_D00390_0217_AC4UAUACXX.gbs/SQ0056.processed_sample/uneak/all.PstI.PstI.cohort/KGD -0.2113675 2.645513e-25
     # from this :
     # flowcell = CCR4LANXX
     # libraryprepid = 772 etc

utils/augment_singleton_hapmap.py

@@ -71,12 +71,12 @@ def augment_hmc(filename, options):
 
 def get_options():
     description = """
-this script takes one of the files generated by tassel such as /dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs/230901_A01439_0210_AHL5HCDRX3/SQ2220.all.fallow_deer.PstI/hapMap/HapMap.hmc.txt and
+this script takes one of the files generated by tassel such as /dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs/230901_A01439_0210_AHL5HCDRX3/SQ2220.all.fallow_deer.PstI/hapMap/HapMap.hmc.txt and
 duplicates all of the samples twice , witgh a fake name, and adjusts the totals columns. It is mainly intended to allow analysis and reporting of singleton samples in the usual way.
 
 Example :
 
-iramohio-01$ head -3 /dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs/230901_A01439_0210_AHL5HCDRX3/SQ2220.all.fallow_deer.PstI/hapMap/HapMap.hmc.txt
+iramohio-01$ head -3 /dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs/230901_A01439_0210_AHL5HCDRX3/SQ2220.all.fallow_deer.PstI/hapMap/HapMap.hmc.txt
 rs      qc1021959-1_HL5HCDRX3_1_2220_X4 qc1021959-1_HL5HCDRX3_2_2220_X4 HetCount_allele1        HetCount_allele2        Count_allele1   Count_allele2   Frequency
 TP1     3|1     2|4     5       5       5       5       0.500
 TP2     2|4     6|1     8       5       8       5       0.615
@@ -101,7 +101,7 @@ def get_options():
     long_description = """
     Example :
 
-augment_singleton_hapmap.py  /dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs/230901_A01439_0210_AHL5HCDRX3/SQ2220.all.fallow_deer.PstI/hapMap/HapMap.hmc.txt 
+augment_singleton_hapmap.py  /dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs/230901_A01439_0210_AHL5HCDRX3/SQ2220.all.fallow_deer.PstI/hapMap/HapMap.hmc.txt 
 
     """
     parser = argparse.ArgumentParser(description=description, epilog=long_description, formatter_class = argparse.RawDescriptionHelpFormatter)

utils/debug_barcodes.sh

@@ -4,7 +4,7 @@
 #
 if [ -z "$1" ]; then
    echo "examples : 
-debug_barcodes.sh /bifo/scratch/2023_illumina_sequencing_c/postprocessing/gbs/230623_A01439_0188_AHFFGNDRX3/SQ2128.all.microbes.PstI/./Illumina/SQ2128_HFFGNDRX3_s_1_fastq.txt.gz
+debug_barcodes.sh /bifo/scratch/2024_illumina_sequencing_d/postprocessing/gbs/230623_A01439_0188_AHFFGNDRX3/SQ2128.all.microbes.PstI/./Illumina/SQ2128_HFFGNDRX3_s_1_fastq.txt.gz
 "
    exit 1
 elif [ ! -f $1 ]; then

utils/get_historical_beta_plus.sh

@@ -11,10 +11,10 @@ export SEQ_PRISMS_BIN=/dataset/gseq_processing/active/bin/gbs_prism/seq_prisms
 
 #
 # we will do the work here - i.e. don't over-write the original GBS processing
-WORKDIR=/dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs_redo_with_latest_KGD_for_beta
+WORKDIR=/dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs_redo_with_latest_KGD_for_beta
 
 # for this lot...
-for cohort_path in /dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs/23*/SQ*; do
+for cohort_path in /dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs/23*/SQ*; do
 
    # parse the cohort and run names from the path 
    cohort=`basename $cohort_path`
@@ -28,13 +28,13 @@ for cohort_path in /dataset/2023_illumina_sequencing_c/scratch/postprocessing/gb
 
    echo "processing $cohort_path"
 
-   # patch the orignal callback - e.g. /dataset/2023_illumina_sequencing_c/scratch/postprocessing/gbs/230414_A01439_0164_AHW5YGDRX2/SQ2081.all.deer.PstI/SQ2081.all.deer.PstI.KGD_tassel3.genotype_prism.sh
+   # patch the orignal callback - e.g. /dataset/2024_illumina_sequencing_d/scratch/postprocessing/gbs/230414_A01439_0164_AHW5YGDRX2/SQ2081.all.deer.PstI/SQ2081.all.deer.PstI.KGD_tassel3.genotype_prism.sh
    original_callback=$cohort_path/${cohort}.KGD_tassel3.genotype_prism.sh
    redo_callback=$WORKDIR/$run/$cohort/${cohort}.KGD_tassel3.genotype_prism.sh
 
    echo "writing $WORKDIR/$run/$cohort/${cohort}.KGD_tassel3.genotype_prism.sh"
    mkdir -p $WORKDIR/$run/$cohort/
-   cat $original_callback | sed 's/\/hiseq\//\/2023_illumina_sequencing_c\//g' | sed 's/postprocessing\/gbs/postprocessing\/gbs_redo/g' > $redo_callback
+   cat $original_callback | sed 's/\/hiseq\//\/2024_illumina_sequencing_d\//g' | sed 's/postprocessing\/gbs/postprocessing\/gbs_redo/g' > $redo_callback
    chmod +x $redo_callback
 
    # copy over or symlink some dependencies that will be needed 

utils/parse_sacct_cpu.py

@@ -87,7 +87,7 @@ def get_options():
 
 cat 2023-08-01.sacct.txt  | ./parse_sacct_cpu.py  --sequencing_platform hiseq -H  /dataset/gseq_processing/active/bin/gbs_prism/etc/sample_sheet_header.csv
 
-cat /dataset/2023_illumina_sequencing_c/scratch/220426_A01439_0069_BHNFW2DRXY/HNFW2DRXY.csv | ./add_sample_sheet_header.py
+cat /dataset/2024_illumina_sequencing_d/scratch/220426_A01439_0069_BHNFW2DRXY/HNFW2DRXY.csv | ./add_sample_sheet_header.py
 
 
 """