Added metabolights.sh (requires curlftpfs) bash script to test mzml2i…

…sa on every study of MetaboLights (EBI) containing .mzML

README.md

@@ -68,10 +68,10 @@ print mm.meta_json
 
 ## Metabolights
 To test the converter with (real) Metabolights studies, run
-```python
+```bash
 python scripts/metabolights.py <size>
 ```
-where size is the maximum size in GiB you can allocate to download files
+where size is the maximum size in GiB you can allocate to download files.
 
 
 

mzml2isa/mzml.py

@@ -85,7 +85,7 @@ def __init__(self, in_file):
 
         terms['source_file'] = {
             'MS:1000767': {'attribute': False, 'name':'Native spectrum identifier format', 'plus1': False, 'value':False, 'soft': False},
-            'MS:1000561': {'attribute': False, 'name':'Raw data file checksum type', 'plus1': True, 'value':True, 'soft': False},
+        #!# 'MS:1000561': {'attribute': False, 'name':'Raw data file checksum type', 'plus1': True, 'value':True, 'soft': False},
             'MS:1000560': {'attribute': False, 'name':'Raw data file format', 'plus1': False, 'value':False, 'soft': False}
         }
 
@@ -246,7 +246,7 @@ def _instrument_byref(self):
                         self.meta['Instrument serial number'] = {'value': ie.attrib['value']}
 
 
-        soft_ref = pyxpath(self, XPATHS['ic_soft_ref'])[0].attrib["ref"]
+        soft_ref = pyxpath(self, XPATHS['ic_soft_ref'])[0].attrib[self.env['softwareRef']]
 
         # Get associated software
         self.software(soft_ref, 'Instrument')
@@ -289,7 +289,7 @@ def _instrument_nested(self):
 
 
 
-        soft_ref = pyxpath(self, XPATHS['ic_soft_ref'])[0].attrib["ref"]
+        soft_ref = pyxpath(self, XPATHS['ic_soft_ref'])[0].attrib[self.env['softwareRef']]
         # Get associated software
         self.software(soft_ref, 'Instrument')
 
@@ -357,19 +357,25 @@ def derived(self):
 
         scan_window_cv = pyxpath(self, XPATHS['scan_window_cv'])
 
-        minmz_l = []
-        maxmz_l = []
+        try: #case with detection range
 
-        for i in scan_window_cv:
+            minmz_l = []
+            maxmz_l = []
 
-            if i.attrib['accession'] == 'MS:1000501':
-                minmz_l.append(float(i.attrib['value']))
-            if i.attrib['accession'] == 'MS:1000500':
-                maxmz_l.append(float(i.attrib['value']))
+            for i in scan_window_cv:
 
-        minmz = str(int(min(minmz_l)))
-        maxmz = str(int(max(maxmz_l)))
-        mzrange = minmz + " - " + maxmz
+                if i.attrib['accession'] == 'MS:1000501':
+                    minmz_l.append(float(i.attrib['value']))
+                if i.attrib['accession'] == 'MS:1000500':
+                    maxmz_l.append(float(i.attrib['value']))
+
+            minmz = str(int(min(minmz_l)))
+            maxmz = str(int(max(maxmz_l)))
+            mzrange = minmz + " - " + maxmz
+
+        except ValueError: #Case with windowed target
+
+            mzrange = ''
 
         #######################
         # Get timerange
@@ -399,13 +405,13 @@ def derived(self):
 
         in_dir = os.path.dirname(self.in_file)
 
-        self.meta['Raw Spectral Data File'] = {'value': os.path.join(in_dir, raw_file)}
+        self.meta['Raw Spectral Data File'] = {'value': os.path.basename(raw_file)}
         self.meta['MS Assay Name'] = {'value': os.path.splitext(os.path.basename(self.in_file))[0]}
         self.meta['Number of scans'] = {'value': int(scan_num)}
         self.meta['Scan m/z range'] = {'value': mzrange}
         self.meta['Scan polarity'] = {'value': polarity}
         self.meta['Time range'] = {'value': timerange}
-        self.meta['Derived Spectral Data File'] = {'value': self.in_file} # mzML file name
+        self.meta['Derived Spectral Data File'] = {'value': os.path.basename(self.in_file)} # mzML file name
         self.meta['Sample Name'] = {'value': os.path.splitext(os.path.basename(self.in_file))[0]} # mzML file name
 
     def isa_tab_compatible(self):
@@ -470,9 +476,17 @@ def build_env(self):
 
         # check if softwareRef or instrumentSoftwareRef
         if self.tree.find('{root}/{instrument}List/{instrument}/s:softwareRef[@ref]'.format(**self.env), self.ns) is not None:
-            self.env['softwareRef'] = 's:softwareRef'
+            self.env['software'] = 's:softwareRef'
+            self.env['softwareRef'] = 'ref'
+
         elif self.tree.find('{root}/{instrument}List/{instrument}/s:instrumentSoftwareRef[@ref]'.format(**self.env), self.ns) is not None:
-            self.env['softwareRef'] = 's:instrumentSoftwareRef'
+            self.env['software'] = 's:instrumentSoftwareRef'
+            self.env['softwareRef'] = 'ref'
+
+        elif self.tree.find('{root}/s:dataProcessingList/s:dataProcessing/s:processingMethod[@softwareRef]'.format(**self.env), self.ns) is not None:
+            self.env['software'] = 's:processingMethod'
+            self.env['softwareRef'] = 'sofwareRef'
+            self.env['dataProcessing'] = 's:dataProcessing'
 
 
         if self.tree.find('{root}/s:referenceableParamGroupList/s:referenceableParamGroup/s:cvParam[@accession="MS:1000529"]'.format(**self.env), self.ns) is not None:
@@ -485,4 +499,3 @@ def build_env(self):
 
 
 
-

mzml2isa/versionutils.py

@@ -31,7 +31,7 @@
 XPATHS =      {'ic_ref':            '{root}/{instrument}List/{instrument}/s:referenceableParamGroupRef[@ref]',
                'ic_elements':       '{root}/s:referenceableParamGroupList/s:referenceableParamGroup',
                'ic_nest':           '{root}/{instrument}List/{instrument}/s:cvParam[@accession]',
-               'ic_soft_ref':       '{root}/{instrument}List/{instrument}/{softwareRef}[@ref]',
+               'ic_soft_ref':       '{root}/{instrument}List/{instrument}/{software}[@{softwareRef}]',
                'software_elements': '{root}/s:softwareList/s:software',
                'sp_cv':             '{root}/s:run/s:spectrumList/s:spectrum/s:cvParam',
                'scan_window_cv':    '{root}/s:run/s:spectrumList/s:spectrum/{scanList}/s:scan/{scanWindow}List/{scanWindow}/s:cvParam',                   

scripts/metabolights.py

@@ -64,7 +64,7 @@ def human_readable(size,precision=2):
 ## Download studies
 print('Downloading study files (max {} GiB):'.format(MAX_SIZE))
 total_dl_size, total_dl_studies, total_files = 0, 0, 0
-for study in sorted(size_dict, key=size_dict.__getitem__):
+for study in sorted(size_dict, key=size_dict.__getitem__)[15:]:
 
     ## check if next study is too large for max size
     if total_dl_size + size_dict[study] > MAX_SIZE * (2**30):
@@ -98,4 +98,4 @@ def human_readable(size,precision=2):
     ftp.cwd('..')
 
 ftp.close()
-print('Downloaded {} of data in total ({} studies, {} files).'.format(human_readable(total_dl_size), total_dl_studies, total_files))
+print('Downloaded {} of data in total ({} studies, {} files).'.format(human_readable(total_dl_size), total_dl_studies, total_files))

scripts/metabolights.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+echo "             _        _           _ _       _     _       ";
+echo "  /\/\   ___| |_ __ _| |__   ___ | (_) __ _| |__ | |_ ___ ";
+echo " /    \ / _ \ __/ _\` | '_ \ / _ \| | |/ _\` | '_ \| __/ __|";
+echo "/ /\/\ \  __/ || (_| | |_) | (_) | | | (_| | | | | |_\__ \\";
+echo "\/    \/\___|\__\__,_|_.__/ \___/|_|_|\__, |_| |_|\__|___/";
+echo "                  ftp.ebi.ac.uk       |___/       EBI     ";
+echo "                                                          ";
+
+
+/usr/bin/python3 -c "
+import json
+import urllib.request as rq
+study_url = 'http://ftp.ebi.ac.uk/pub/databases/metabolights/study_file_extensions/ml_file_extension.json'
+req = rq.Request(study_url)
+con = rq.urlopen(req)
+e = json.JSONDecoder()
+study = e.decode(con.read().decode('utf-8'))
+mzml_studies = [k['id'] for k in study if '.mzML' in k['extensions']]
+for study in mzml_studies:
+    print(study)
+" > scripts/mzml_studies.txt
+
+
+[ -d example_files/metabolights ] || mkdir example_files/metabolights
+[ -n "$(ls -A example_files/metabolights)" ] || curlftpfs ftp.ebi.ac.uk/pub/databases/metabolights/studies/public example_files/metabolights
+
+[ -d out_folder/metabolights ] || mkdir out_folder/metabolights
+while read study; do
+	mzml2isa -i example_files/metabolights/$study -o out_folder/metabolights -s ${study#*/*/*}
+	[ -d out_folder/metabolights/$study ] || echo $study >> fails.txt
+done < scripts/mzml_studies.txt
+
+echo "Completed parsing... Press ENTER to unmount fuse MetaboLights"
+read
+fusermount -u example_files/metabolights

setup.py

@@ -13,27 +13,16 @@
 
 """
 def main():
-    #if sys.version_info[0] != 2 and sys.version_info[1] != 7:
-    #    raise Error("Python-2.7.8 is required")
     example_files = glob.glob(os.path.join('scripts', '*.*'))
     example_path = os.path.join('scripts')
 
     setup( name="mzml_2_isa",
     version= "0.1",
-    description= "mzML to ISA-tab parsing tool",
-    author= "Thomas Lawson",
-    author_email= "tnl495@bham.ac.uk",
-    url= "http://www.biosciences.bham.ac.uk/labs/viant/",
-    platforms = ['Linux (ubuntu), Windows'],
     keywords = ['Metabolomics', 'Mass spectrometry', 'metabolites', 'ISA Tab', 'mzML', 'parsing'],
-    packages=['mzml2isa'],
-    package_data={'': ['default/*.txt','*.obo']},
 
     data_files=[(example_path, example_files)],
     long_description="mzml2isa - mzML to ISA-tab parsing tool",)
 
-if __name__ == "__main__":
-    main()
 """
 
 ## SETUPTOOLS VERSION
@@ -45,11 +34,9 @@ def main():
 
     py_modules=['mzml2isa'],
 
-
     author= "Thomas Lawson",
     author_email= "tnl495@bham.ac.uk",
 
-
     description="mzML to ISA-tab parsing tool",
     long_description=open('README.md').read(),
 
@@ -59,8 +46,6 @@ def main():
 
     url='http://www.biosciences.bham.ac.uk/labs/viant/',
 
-
-
     classifiers=[
     "Programming Language :: Python",
     "Programming Language :: Python :: 2",
@@ -72,7 +57,6 @@ def main():
     "Operating System :: OS Independent",
     ],
 
-
     entry_points = {
         'console_scripts': [
             'mzml2isa = mzml2isa.parsing:run',
@@ -82,8 +66,5 @@ def main():
 
     keywords = ['Metabolomics', 'Mass spectrometry', 'metabolites', 'ISA Tab', 'mzML', 'parsing'],
 
-
-
-
 )