Update BQ Benchmark Code to Python3 (#410)

* using avro python 3 lib * changing sys.maxint to sys.maxsize (changed in python 3) * changing cStringIO to io * Revert "changing cStringIO to io" This reverts commit f1bca24. * Revert "changing sys.maxint to sys.maxsize (changed in python 3)" This reverts commit de62286. * removing future imports * adding 2to3 changes * fixing new errors in file load coad * removing collections import that is no longer needed * use fastavro to address #412 * failing tests if no project_id provided - otherwise tests fail and dont give an obvious reason * taking out sort(), we want field types in the order they appear * fixing order of fields * going back to using counter * fixing order of fields * fixing order of fields Co-authored-by: Jacob Ferriero <jferriero@google.com>
GoogleCloudPlatform · Feb 27, 2020 · 1f43137 · 1f43137
1 parent acfda77
commit 1f43137
Show file tree

Hide file tree

Showing 23 changed files with 195 additions and 400 deletions.
diff --git a/examples/bq_file_load_benchmark/benchmark_tools/benchmark_result_util.py b/examples/bq_file_load_benchmark/benchmark_tools/benchmark_result_util.py
diff --git a/examples/bq_file_load_benchmark/bq_file_load_benchmark.py b/examples/bq_file_load_benchmark/bq_file_load_benchmark.py
@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 import argparse
 import logging
 

diff --git a/examples/bq_file_load_benchmark/generic_benchmark_tools/avro_util.py b/examples/bq_file_load_benchmark/generic_benchmark_tools/avro_util.py
@@ -1,6 +1,17 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import json
 
 import avro.schema
@@ -55,6 +66,6 @@ def get_avro_translated_schema(self):
             'name': self.schema_name,
             'fields': fields,
         }
-        avro_schema = avro.schema.parse(json.dumps(schema_dict))
+        avro_schema = avro.schema.Parse(json.dumps(schema_dict))
 
         return avro_schema
diff --git a/examples/bq_file_load_benchmark/generic_benchmark_tools/benchmark_result_util.py b/examples/bq_file_load_benchmark/generic_benchmark_tools/benchmark_result_util.py
@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 from abc import ABC, abstractmethod
 import logging
 import os

diff --git a/examples/bq_file_load_benchmark/generic_benchmark_tools/bucket_util.py b/examples/bq_file_load_benchmark/generic_benchmark_tools/bucket_util.py
@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+
 from concurrent.futures import ThreadPoolExecutor
 import itertools
 import logging

diff --git a/examples/bq_file_load_benchmark/generic_benchmark_tools/file_constants.py b/examples/bq_file_load_benchmark/generic_benchmark_tools/file_constants.py
@@ -17,10 +17,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 from google.cloud import bigquery
 
 FILE_CONSTANTS = {

diff --git a/examples/bq_file_load_benchmark/generic_benchmark_tools/parquet_util.py b/examples/bq_file_load_benchmark/generic_benchmark_tools/parquet_util.py
@@ -12,10 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import pyarrow as pa
 
 

diff --git a/examples/bq_file_load_benchmark/generic_benchmark_tools/schema_creator.py b/examples/bq_file_load_benchmark/generic_benchmark_tools/schema_creator.py
@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 import json
 import logging
 import os

diff --git a/examples/bq_file_load_benchmark/generic_benchmark_tools/staging_table_generator.py b/examples/bq_file_load_benchmark/generic_benchmark_tools/staging_table_generator.py
@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 from concurrent.futures import ThreadPoolExecutor
 import itertools
 import logging