add pararun

pararun is used to run a cmd for all the files with same surfix using similar cmds

Author: Runsheng

gbrowser_script/maker_re.py

@@ -0,0 +1,135 @@
+#! /usr/bin/env python
+import argparse
+
+def gff_to_list(gfffile):
+    gene_l=[]
+    
+    with open(gfffile, "r") as f:
+        lines=f.readlines()
+        for line in lines:
+            if line[0]!="#" and len(line)>18:
+                gene_l.append(line)
+
+    return gene_l
+
+
+def name_to_list(filename):
+    """
+    the name is like Bp_scaf_6090-3.8
+    Bp_scaf_6090 is contig name
+    3.8 is the mRNA name
+    """
+    name_l=[]
+    with open(filename) as f:
+        lines=f.readlines()
+        for line in lines:
+            name_l.append(line.strip())
+    return name_l
+
+
+def gene_format(gene_l):
+    """
+    format the gff line to blocks, 
+    {gene name, [mRNAline, cdsline...]}
+    
+    the gff need to be sorted and all gene/mRNA/exon/cds/UTR is together
+    """
+    
+    gene_d={}
+    for line in gene_l:
+        line_l=line.split("\t")
+        
+        gff_type=line_l[2]
+        attributes=line_l[-1].strip()
+        
+        if gff_type=="gene":
+            for attribute in attributes.split(";"):
+                if "ID" in attribute:
+                    kk=attribute.split("=")[1]
+                    gene_d[kk]=[]
+                    gene_d[kk].append(line)
+        
+        else:
+            for attribute in attributes.split(";"):
+                if "Parent" in attribute:
+                    parent=attribute.split("=")[1]
+                    if kk in parent:
+                        gene_d[kk].append(line)
+    return gene_d
+
+
+def gene_select(gene_d,name_l):
+    """
+    select by key of gene_d
+    """
+    gff_new={}
+    name_set=set(name_l)
+    for key in gene_d.keys():
+        name=key.split("-")[1]+"-"+key.split("-")[-1]
+        if name in name_set:
+            gff_new[key]=gene_d[key]
+    return gff_new
+    
+
+def gene_name_replace(gff_d):
+    """
+    replace the long name to short one, 
+    from "maker-Bp_scaf_46564-exonerate_est2genome-gene-0.0" to "Bp_scaf_46564-0.0"
+    """
+    gff_namere={}
+    
+    name_d={}
+    for key in gff_d.keys():
+        name=key.split(":")[1]
+        name_re=key.split("-")[1]+"-"+key.split("-")[-1]
+        name_d[key]=(name,name_re)
+    
+    for k, v in gff_d.items():
+        v_re=[]
+        for line in v:
+            line_new=line.replace(name_d[k][0], name_d[k][1])
+            v_re.append(line_new)
+        gff_namere[k]=v_re
+    
+    return gff_namere
+
+
+def gff_write(gff_d, out):
+    
+    def write_line():
+        fw.write(line)
+    
+    fw=open(out,"w")
+    for k in gff_d.keys():
+        #print k
+        v=gff_d[k]
+        for line in v:
+            write_line()
+    fw.close()
+
+
+    
+if __name__=="__main__":  
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--gff_in",help="the sorted maker gff file")
+    parser.add_argument("--namefile", help="the genes you want to keep in new gff file")
+    parser.add_argument("--gff_out", help="the name of output gff file")
+    args = parser.parse_args()
+
+    try:
+        gene_l=gff_to_list(args.gff_in)
+        name_l=name_to_list(args.namefile)
+    except Exception:
+        raise IOERROR("Error occurred, please make sure the gff and name file is in your path")
+
+    gene_d=gene_format(gene_l)
+    gff_new=gene_select(gene_d,name_l)
+    gff_re=gene_name_replace(gff_new)
+    
+    print "The old gff file contains %d genes and the new gff file contains % d genes." % (len(gene_d), len(gff_new))
+    
+    try:
+        gff_write(gff_re, args.gff_out)
+    
+    except Exception:
+        print "Error occurred, please check you have write permission to your disk"

pararun.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time    : 8/3/2020 4:37 PM
+# @Author  : Runsheng     
+# @File    : pararun.py
+
+"""
+A general para runner for the functions with only one input file/para
+"""
+
+from __future__ import print_function
+import os
+import argparse
+import subprocess
+from glob import glob
+import logging
+import sys
+import signal
+import multiprocessing
+
+
+def myexe(cmd, timeout=0):
+    """
+    a simple wrap of the shell
+    mainly used to run the bwa mem mapping and samtool orders
+    """
+    def setupAlarm():
+        signal.signal(signal.SIGALRM, alarmHandler)
+        signal.alarm(timeout)
+
+    def alarmHandler(signum, frame):
+        sys.exit(1)
+
+    proc=subprocess.Popen(cmd, shell=True, preexec_fn=setupAlarm,
+                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,cwd=os.getcwd())
+    out, err=proc.communicate()
+    print(err)
+    return out, err, proc.returncode
+
+
+def parmap(f, X, nprocs=multiprocessing.cpu_count()):
+    """
+    a function to use mutip map inside a function
+    modified from stackoverflow, 3288595
+    :param f:
+    :param X:
+    :param nprocs: core, if not given, use all core
+    :return:
+    """
+    q_in = multiprocessing.Queue(1)
+    q_out = multiprocessing.Queue()
+
+    proc = [multiprocessing.Process(target=fun, args=(f, q_in, q_out))
+            for _ in range(nprocs)]
+    for p in proc:
+        p.daemon = True
+        p.start()
+
+    sent = [q_in.put((i, x)) for i, x in enumerate(X)]
+    [q_in.put((None, None)) for _ in range(nprocs)]
+    res = [q_out.get() for _ in range(len(sent))]
+
+    [p.join() for p in proc]
+
+    return [x for i, x in sorted(res)]
+
+
+def fun(f, q_in, q_out):
+    """
+    for parmap
+    :param f:
+    :param q_in:
+    :param q_out:
+    :return:
+    """
+    while True:
+        i, x = q_in.get()
+        if i is None:
+            break
+        q_out.put((i, f(x)))
+
+
+def path_split(pathstring):
+    laststring = pathstring.split("/")[-1]
+    if "." in laststring:
+        return laststring.split(".")[0]
+    else:
+        return laststring.split(".")[0]
+
+
+def get_prefix(bamlist):
+    prefix_l = []
+
+    for i in bamlist:
+        prefix_l.append(path_split(i))
+    return prefix_l
+
+
+if __name__=="__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f", "--folder",help="the folder containing all files")
+    parser.add_argument("-s", "--suffix", help="the suffix of the files")
+    parser.add_argument("-p", "--core",default=10, help="the cores used")
+    parser.add_argument("-c", "--cmd", default="ls", help="the cmd to run, with prefix as prefix")
+
+    args = parser.parse_args()
+
+    ## main run code
+    seqdir = args.folder
+    os.chdir(seqdir)
+    logging.info("Move to {dir}".format(dir=seqdir))
+    file_s = glob("*."+args.suffix)
+    prefix_l= (get_prefix(file_s))
+
+    def run_one(prefix):
+        file_one=prefix+args.suffix
+        cmd_run=args.cmd.replace("prefix", prefix)
+        cmd_new = """cd {seqdir}
+           {cmd_run}""".format(cmd_run=cmd_run, seqdir=seqdir)
+        print(cmd_new)
+        return myexe(cmd_new)
+
+    parmap(run_one, prefix_l, nprocs=int(args.core))
+