Merge pull request GoogleCloudPlatform#221 from pkattamuri/feature/hi…

…ve-bigquery Feature/hive-bigquery Changed static filenames
epishova · Mar 26, 2019 · 870cdff · 870cdff
2 parents fa66cef + c576420
commit 870cdff
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 6 deletions.
diff --git a/tools/hive-bigquery/bigquery_table_model.py b/tools/hive-bigquery/bigquery_table_model.py
@@ -18,6 +18,7 @@
 import logging
 import os
 from collections import OrderedDict
+from uuid import uuid4
 
 from properties_reader import PropertiesReader
 
@@ -65,12 +66,13 @@ def table_name(self):
     @property
     def schema(self):
         if self._table_details['schema'] is None:
+            filename = 'bq_schema_{}.json'.format(uuid4())
             os.system(
-                'bq show --format=prettyjson {0}.{1} > bq_schema.json'.format(
-                    self.dataset_id, self.table_name))
-            with open('bq_schema.json', 'r') as file_content:
+                'bq show --format=prettyjson {0}.{1} > {2}'.format(
+                    self.dataset_id, self.table_name, filename))
+            with open(filename, 'r') as file_content:
                 schema = json.load(file_content)
-            os.remove('bq_schema.json')
+            os.remove(filename)
             self._table_details['schema'] = schema['schema']['fields']
         return self._table_details['schema']
 

diff --git a/tools/hive-bigquery/gcs_storage_component.py b/tools/hive-bigquery/gcs_storage_component.py
@@ -190,7 +190,7 @@ def stage_to_gcs(self, mysql_component, bq_component, hive_table_model,
                 if file_name not in file_info.keys():
                     file_info[file_name] = source_location
             source_locations = ' '.join(file_info.values())
-            filename = "file_info.json"
+            filename = "file_info_{}.json".format(uuid4())
             # Dictionary of file names and their locations
             with open(filename, "w") as file_content:
                 file_content.write(str(file_info))

diff --git a/tools/hive-bigquery/hive_component.py b/tools/hive-bigquery/hive_component.py
@@ -170,7 +170,7 @@ def list_hdfs_files(location):
             List: List of the underlying data files.
         """
 
-        file_name = "migration_temp_file.txt"
+        file_name = "hdfs_files_{}.txt".format(uuid4())
         status_code = os.system(
             "hdfs dfs -ls {0} > {1}".format(location, file_name))
         if status_code: