Merge pull request #845 from squirrelo/issue-835

Issue 835

Author: ElDeveloper

qiita_db/analysis.py

@@ -17,7 +17,6 @@
 # -----------------------------------------------------------------------------
 from __future__ import division
 from collections import defaultdict
-from binascii import crc32
 from os.path import join
 
 from future.utils import viewitems
@@ -31,7 +30,7 @@
 from .study import Study
 from .exceptions import QiitaDBStatusError  # QiitaDBNotImplementedError
 from .util import (convert_to_id, get_work_base_dir,
-                   get_mountpoint, get_table_cols)
+                   get_mountpoint, get_table_cols, insert_filepaths)
 
 
 class Analysis(QiitaStatusObject):
@@ -738,24 +737,11 @@ def _add_file(self, filename, filetype, data_type=None, conn_handler=None):
         conn_handler = conn_handler if conn_handler is not None \
             else SQLConnectionHandler()
 
-        # get required bookkeeping data for DB
-        _, base_fp = get_mountpoint(self._table, conn_handler=conn_handler)[0]
-        fptypeid = convert_to_id(filetype, "filepath_type", conn_handler)
-        fullpath = join(base_fp, filename)
-        with open(fullpath, 'rb') as f:
-            checksum = crc32(f.read()) & 0xffffffff
-
-        analysis_dd_id, _ = get_mountpoint("analysis",
-                                           conn_handler=conn_handler)[0]
-
-        # add  file to analysis
-        sql = ("INSERT INTO qiita.filepath (filepath, filepath_type_id, "
-               "checksum, checksum_algorithm_id, data_directory_id) VALUES "
-               "(%s, %s, %s, %s, %s) RETURNING filepath_id")
-        # magic number 1 is for crc32 checksum algorithm
-        fpid = conn_handler.execute_fetchone(sql, (fullpath, fptypeid,
-                                                   checksum, 1,
-                                                   analysis_dd_id))[0]
+        filetype_id = convert_to_id(filetype, 'filepath_type', conn_handler)
+        _, mp = get_mountpoint('analysis', conn_handler)[0]
+        fpid = insert_filepaths([
+            (join(mp, filename), filetype_id)], -1, 'analysis', 'filepath',
+            conn_handler, move_files=False)[0]
 
         col = ""
         dtid = ""

qiita_db/support_files/patches/14.sql

@@ -1,2 +1,4 @@
+--Feb 11, 2015
+--Placeholder for python to run
 -- we don't really need an SQL patch, but having nothing here breaks the build
 select 42;

qiita_db/support_files/patches/15.sql

@@ -0,0 +1,4 @@
+--Feb 11, 2015
+--Placeholder for python to run
+-- we don't really need an SQL patch, but having nothing here breaks the build
+select 42;

qiita_db/support_files/patches/python_patches/15.py

@@ -0,0 +1,26 @@
+# Feb 11, 2015
+# This changes all analysis files to be relative path instead of absolute
+
+from os.path import basename, dirname
+
+from qiita_db.util import get_mountpoint
+from qiita_db.sql_connection import SQLConnectionHandler
+
+conn_handler = SQLConnectionHandler()
+
+filepaths = conn_handler.execute_fetchall(
+    'SELECT f.* from qiita.filepath f JOIN qiita.analysis_filepath afp ON '
+    'f.filepath_id = afp.filepath_id')
+
+# retrieve relative filepaths as dictionary for matching
+mountpoints = {m[1].rstrip('/\\'): m[0] for m in get_mountpoint(
+    'analysis', conn_handler=conn_handler, retrieve_all=True)}
+
+for filepath in filepaths:
+    filename = basename(filepath['filepath'])
+    # find the ID of the analysis filepath used
+    mp_id = mountpoints[dirname(filepath['filepath']).rstrip('/\\')]
+    conn_handler.execute(
+        'UPDATE qiita.filepath SET filepath = %s, data_directory_id = %s WHERE'
+        ' filepath_id = %s',
+        [filename, mp_id, filepath['filepath_id']])

qiita_db/test/test_analysis.py

@@ -1,4 +1,5 @@
 from unittest import TestCase, main
+from os import remove
 from os.path import exists, join
 from datetime import datetime
 from shutil import move
@@ -37,6 +38,10 @@ def tearDown(self):
         with open(self.map_fp, 'w') as f:
                 f.write("")
 
+        fp = join(get_mountpoint('analysis')[0][1], 'testfile.txt')
+        if exists(fp):
+            remove(fp)
+
         mp = get_mountpoint("processed_data")[0][1]
         study2fp = join(mp, "2_2_study_1001_closed_reference_otu_table.biom")
         if exists(study2fp):
@@ -427,6 +432,22 @@ def test_build_files_raises_value_error(self):
         with self.assertRaises(ValueError):
             self.analysis.build_files(-10)
 
+    def test_add_file(self):
+        fp = join(get_mountpoint('analysis')[0][1], 'testfile.txt')
+        with open(fp, 'w') as f:
+            f.write('testfile!')
+        self.analysis._add_file('testfile.txt', 'plain_text', '18S')
+
+        obs = self.conn_handler.execute_fetchall(
+            'SELECT * FROM qiita.filepath WHERE filepath_id = 19')
+        exp = [[19, 'testfile.txt', 9, '3675007573', 1, 1]]
+        self.assertEqual(obs, exp)
+
+        obs = self.conn_handler.execute_fetchall(
+            'SELECT * FROM qiita.analysis_filepath WHERE filepath_id = 19')
+        exp = [[1, 19, 2]]
+        self.assertEqual(obs, exp)
+
 
 if __name__ == "__main__":
     main()

qiita_db/util.py

@@ -608,8 +608,8 @@ def get_mountpoint(mount_type, conn_handler=None, retrieve_all=False):
             "SELECT data_directory_id, mountpoint, subdirectory FROM "
             "qiita.data_directory WHERE data_type='%s' and active=true"
             % mount_type)]
-
-    return [(d, join(get_db_files_base_dir(), m, s)) for d, m, s in result]
+    basedir = get_db_files_base_dir()
+    return [(d, join(basedir, m, s)) for d, m, s in result]
 
 
 def insert_filepaths(filepaths, obj_id, table, filepath_table, conn_handler,
@@ -627,7 +627,7 @@ def insert_filepaths(filepaths, obj_id, table, filepath_table, conn_handler,
             Id of the object calling the functions. Disregarded if move_files
             is False
         table : str
-            Table that holds the file data. Disregarded if move_files is False
+            Table that holds the file data.
         filepath_table : str
             Table that holds the filepath information
         conn_handler : SQLConnectionHandler