Fix relative importing cause error (#2823)

typhoonzero · web-flow · commit 655629903cf8 · 2020-08-13T15:06:33.000+08:00
* fix relative importing cause error

* clean up
diff --git a/go/codegen/alps/template_train.go b/go/codegen/alps/template_train.go
@@ -41,7 +41,7 @@ from alps.framework.column.column import (DenseColumn, GroupedSparseColumn,
 from alps.framework.engine import LocalEngine
 from alps.framework.experiment import EstimatorBuilder
 from alps.io.base import OdpsConf
-from runtime import db
+from runtime.model import db
 from runtime.alps.train import train
 from runtime.tensorflow.get_tf_version import tf_is_version2
 
diff --git a/go/codegen/pai/template_tf.go b/go/codegen/pai/template_tf.go
@@ -66,9 +66,9 @@ type requirementsFiller struct {
 const tfImportsText = `
 import tensorflow as tf
 from runtime.tensorflow import is_tf_estimator
-from runtime.import_model import import_model
+from runtime.tensorflow.import_model import import_model
 try:
-	from runtime import oss
+	from runtime.model import oss
 	from runtime.pai.pai_distributed import define_tf_flags, set_oss_environs
 except:
 	pass # PAI is not always needed
diff --git a/go/codegen/pai/template_xgboost.go b/go/codegen/pai/template_xgboost.go
@@ -31,7 +31,7 @@ import json
 import copy
 import runtime.xgboost as xgboost_extended
 from runtime.xgboost.predict import pred
-from runtime import oss
+from runtime.model import oss
 from runtime.pai.pai_distributed import define_tf_flags, set_oss_environs
 
 FLAGS = define_tf_flags()
@@ -102,7 +102,7 @@ if os.environ.get('DISPLAY', '') == '':
 
 import json
 from runtime.xgboost.explain import explain
-from runtime import oss
+from runtime.model import oss
 from runtime.pai.pai_distributed import define_tf_flags, set_oss_environs
 
 FLAGS = define_tf_flags()
@@ -162,7 +162,7 @@ const xgbEvalTemplateText = `
 import json
 import runtime.xgboost as xgboost_extended
 from runtime.xgboost.evaluate import evaluate
-from runtime import oss
+from runtime.model import oss
 from runtime.pai.pai_distributed import define_tf_flags, set_oss_environs
 
 FLAGS = define_tf_flags()
diff --git a/go/codegen/xgboost/template_train.go b/go/codegen/xgboost/template_train.go
@@ -50,7 +50,7 @@ if "{{.IsPAI}}" == "true":
     set_oss_environs(FLAGS)
 
 if "{{.IsPAI}}" == "true" and "{{.LoadPreTrainedModel}}" == "true":
-    from runtime import oss
+    from runtime.model import oss
     oss.load_file("{{.OSSModelDirToLoad}}", "my_model")
 
 model_params = json.loads('''{{.ModelParamsJSON}}''')
@@ -99,7 +99,7 @@ FLAGS = define_tf_flags()
 set_oss_environs(FLAGS)
 
 if "{{.IsPAI}}" == "true" and "{{.LoadPreTrainedModel}}" == "true":
-	from runtime import oss
+	from runtime.model import oss
 	oss.load_file("{{.OSSModelDirToLoad}}", "my_model")
 
 model_params = json.loads('''{{.ModelParamsJSON}}''')
diff --git a/python/runtime/alisa/submitter.py b/python/runtime/alisa/submitter.py
@@ -13,8 +13,8 @@
 
 import os
 
-from runtime import oss
 from runtime.diagnostics import SQLFlowDiagnostic
+from runtime.model import oss
 from runtime.pai.submitter import gen_rand_string
 
 AlisaTaskTypePAI = 0
diff --git a/python/runtime/explainer.py b/python/runtime/explainer.py
@@ -16,7 +16,7 @@
 import matplotlib
 # The default backend
 import matplotlib.pyplot as plt
-from runtime.oss import copyfileobj
+from runtime.model.oss import copyfileobj
 
 # TODO(shendiaomo): extract common code from tensorflow/explain.py
 # and xgboost/explain.py
diff --git a/python/runtime/model/model.py b/python/runtime/model/model.py
@@ -16,7 +16,7 @@
 import tempfile
 from enum import Enum
 
-import runtime.oss as oss
+from runtime.model import oss
 from runtime.model.db import read_with_generator, write_with_generator
 from runtime.model.tar import unzip_dir, zip_dir
 
diff --git a/python/runtime/model/model_test.py b/python/runtime/model/model_test.py
@@ -15,7 +15,7 @@
 import tempfile
 import unittest
 
-import runtime.oss as oss
+import runtime.model.oss as oss
 from runtime.model import EstimatorType, Model
 from runtime.testing import get_datasource
 
diff --git a/python/runtime/model/oss.py b/python/runtime/model/oss.py
diff --git a/python/runtime/optimize/optflow.py b/python/runtime/optimize/optflow.py
@@ -19,11 +19,11 @@
 import oss2
 import requests
 import six
+from runtime.model.oss import get_bucket
 from runtime.optimize.model_generation import (
     assert_are_valid_tokens, find_matched_aggregation_function_brackets,
     generate_objective_and_constraint_expr, try_convert_comparision_token,
     try_convert_to_aggregation_function, update_by_column_names)
-from runtime.oss import get_bucket
 
 __all__ = [
     'run_optimize_on_optflow',
diff --git a/python/runtime/pai/submitter.py b/python/runtime/pai/submitter.py
@@ -20,10 +20,9 @@
 import tempfile
 from os import path
 
-from runtime import db, oss
+from runtime import db
 from runtime.diagnostics import SQLFlowDiagnostic
-from runtime.model import EstimatorType
-from runtime.oss import delete_oss_dir_recursive
+from runtime.model import EstimatorType, oss
 from runtime.pai import cluster_conf
 from runtime.pai.kmeans import get_train_kmeans_pai_cmd
 from runtime.pai.random_forest import get_train_random_forest_pai_cmd
@@ -254,7 +253,7 @@ def get_project(datasource):
 
 def clean_oss_model_path(oss_path):
     bucket = oss.get_models_bucket()
-    delete_oss_dir_recursive(bucket, oss_path)
+    oss.delete_oss_dir_recursive(bucket, oss_path)
 
 
 def max_compute_table_url(table):
@@ -545,7 +544,7 @@ def create_predict_result_table(datasource, select, result_table, label_column,
         label_column: name of the label column, if not exist in select
             result, we will add a int column in the result table
         train_label_column: name of the label column when training
-        model_type: type of model defined in runtime.oss
+        model_type: type of model defined in runtime.model.oss
     """
     conn = db.connect_with_data_source(datasource)
     db.execute(conn, "DROP TABLE IF EXISTS %s" % result_table)
diff --git a/python/runtime/pai/tensorflow/evaluate.py b/python/runtime/pai/tensorflow/evaluate.py
@@ -14,12 +14,12 @@
 import sys
 
 import tensorflow as tf
-from runtime import oss
-from runtime.import_model import import_model
+from runtime.model import oss
 from runtime.pai.pai_distributed import define_tf_flags
 from runtime.tensorflow import is_tf_estimator
 from runtime.tensorflow.evaluate import (estimator_evaluate, keras_evaluate,
                                          write_result_metrics)
+from runtime.tensorflow.import_model import import_model
 from runtime.tensorflow.input_fn import get_dataset_fn
 from runtime.tensorflow.keras_with_feature_column_input import \
     init_model_with_feature_column
diff --git a/python/runtime/pai/tensorflow/explain.py b/python/runtime/pai/tensorflow/explain.py
@@ -17,10 +17,10 @@
 import matplotlib
 import pandas as pd
 import tensorflow as tf
-from runtime import oss
-from runtime.import_model import import_model
+from runtime.model import oss
 from runtime.tensorflow import is_tf_estimator
 from runtime.tensorflow.explain import explain_boosted_trees, explain_dnns
+from runtime.tensorflow.import_model import import_model
 from runtime.tensorflow.input_fn import input_fn
 from runtime.tensorflow.keras_with_feature_column_input import \
     init_model_with_feature_column
diff --git a/python/runtime/pai/tensorflow/predict.py b/python/runtime/pai/tensorflow/predict.py
@@ -12,9 +12,10 @@
 # limitations under the License
 
 import tensorflow as tf
-from runtime import db, oss
-from runtime.import_model import import_model
+from runtime import db
+from runtime.model import oss
 from runtime.tensorflow import is_tf_estimator
+from runtime.tensorflow.import_model import import_model
 from runtime.tensorflow.predict import estimator_predict, keras_predict
 
 
diff --git a/python/runtime/pai/tensorflow/train.py b/python/runtime/pai/tensorflow/train.py
@@ -14,13 +14,12 @@
 import os
 import types
 
-from runtime import oss
-from runtime.import_model import import_model
-from runtime.model import collect_metadata
+from runtime.model import collect_metadata, oss
 from runtime.pai.pai_distributed import define_tf_flags, set_oss_environs
 from runtime.pai.tensorflow.train_estimator import estimator_train_and_save
 from runtime.pai.tensorflow.train_keras import keras_train_and_save
 from runtime.tensorflow.get_tf_model_type import is_tf_estimator
+from runtime.tensorflow.import_model import import_model
 from runtime.tensorflow.input_fn import get_dataset_fn
 from runtime.tensorflow.set_log_level import set_log_level
 
diff --git a/python/runtime/pai/tensorflow/train_keras.py b/python/runtime/pai/tensorflow/train_keras.py
@@ -12,8 +12,7 @@
 # limitations under the License.
 
 import tensorflow as tf
-from runtime import oss
-from runtime.model import save_metadata
+from runtime.model import oss, save_metadata
 from runtime.pai.pai_distributed import (
     dump_into_tf_config, make_distributed_info_without_evaluator)
 from runtime.seeding import get_tf_random_seed
diff --git a/python/runtime/pai/xgboost/evaluate.py b/python/runtime/pai/xgboost/evaluate.py
@@ -12,7 +12,7 @@
 # limitations under the License
 
 import runtime.xgboost as xgboost_extended
-from runtime import oss
+from runtime.model import oss
 from runtime.pai.pai_distributed import define_tf_flags
 from runtime.xgboost.evaluate import evaluate as _evaluate
 
diff --git a/python/runtime/pai/xgboost/explain.py b/python/runtime/pai/xgboost/explain.py
@@ -12,7 +12,7 @@
 # limitations under the License
 
 import runtime.xgboost as xgboost_extended
-from runtime import oss
+from runtime.model import oss
 from runtime.pai.pai_distributed import define_tf_flags
 from runtime.xgboost.explain import explain as explain_xgb
 
diff --git a/python/runtime/pai/xgboost/predict.py b/python/runtime/pai/xgboost/predict.py
@@ -14,7 +14,7 @@
 import copy
 
 import runtime.xgboost as xgboost_extended
-from runtime import oss
+from runtime.model import oss
 from runtime.pai.pai_distributed import define_tf_flags
 from runtime.xgboost.predict import pred
 
diff --git a/python/runtime/pai/xgboost/train.py b/python/runtime/pai/xgboost/train.py
@@ -11,7 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from runtime import oss
+from runtime.model import oss
 from runtime.pai.pai_distributed import define_tf_flags
 from runtime.xgboost.feature_column import ComposedColumnTransformer
 from runtime.xgboost.train import dist_train
diff --git a/python/runtime/tensorflow/evaluate.py b/python/runtime/tensorflow/evaluate.py
@@ -14,9 +14,9 @@
 import sys
 
 from runtime.db import buffered_db_writer, connect_with_data_source
-from runtime.import_model import import_model
 from runtime.tensorflow import metrics
 from runtime.tensorflow.get_tf_model_type import is_tf_estimator
+from runtime.tensorflow.import_model import import_model
 from runtime.tensorflow.input_fn import get_dataset_fn
 from runtime.tensorflow.keras_with_feature_column_input import \
     init_model_with_feature_column
diff --git a/python/runtime/tensorflow/explain.py b/python/runtime/tensorflow/explain.py
@@ -21,8 +21,8 @@
 import tensorflow as tf
 from runtime import explainer
 from runtime.db import buffered_db_writer, connect_with_data_source
-from runtime.import_model import import_model
 from runtime.tensorflow.get_tf_version import tf_is_version2
+from runtime.tensorflow.import_model import import_model
 from runtime.tensorflow.input_fn import input_fn
 from runtime.tensorflow.keras_with_feature_column_input import \
     init_model_with_feature_column
diff --git a/python/runtime/tensorflow/import_model.py b/python/runtime/tensorflow/import_model.py
diff --git a/python/runtime/tensorflow/input_fn.py b/python/runtime/tensorflow/input_fn.py
@@ -218,7 +218,9 @@ def dataset_input_fn():
                            pai_table=pai_table,
                            num_workers=num_workers,
                            worker_id=worker_id)
-        dataset = dataset.cache("cache_train")
+        # NOTE(typhoonzero): on PAI some times cache to a file may cause
+        # "lockfile already exists" error.
+        dataset = dataset.cache()
         if shuffle_size is not None:
             dataset = dataset.shuffle(shuffle_size)
         dataset = dataset.batch(batch_size)
diff --git a/python/runtime/tensorflow/predict.py b/python/runtime/tensorflow/predict.py
@@ -17,9 +17,9 @@
 import numpy as np
 import tensorflow as tf
 from runtime import db
-from runtime.import_model import import_model
 from runtime.tensorflow.get_tf_model_type import is_tf_estimator
 from runtime.tensorflow.get_tf_version import tf_is_version2
+from runtime.tensorflow.import_model import import_model
 from runtime.tensorflow.input_fn import (get_dtype,
                                          parse_sparse_feature_predict,
                                          tf_generator)
diff --git a/python/runtime/tensorflow/train.py b/python/runtime/tensorflow/train.py
@@ -15,9 +15,9 @@
 import os
 import types
 
-from runtime.import_model import import_model
 from runtime.model import collect_metadata
 from runtime.tensorflow.get_tf_model_type import is_tf_estimator
+from runtime.tensorflow.import_model import import_model
 from runtime.tensorflow.input_fn import get_dataset_fn
 from runtime.tensorflow.set_log_level import set_log_level
 from runtime.tensorflow.train_estimator import estimator_train_and_save
diff --git a/python/runtime/xgboost/train.py b/python/runtime/xgboost/train.py
@@ -17,8 +17,9 @@
 import runtime.pai.pai_distributed as pai_dist
 import six
 import xgboost as xgb
-from runtime import oss as pai_model_store
-from runtime.model import collect_metadata, save_metadata
+from runtime.model import collect_metadata
+from runtime.model import oss as pai_model_store
+from runtime.model import save_metadata
 from runtime.xgboost.dataset import xgb_dataset
 from runtime.xgboost.pai_rabit import PaiXGBoostTracker, PaiXGBoostWorker
 
