fix(samples): Switched the AutoML sample from BQ to GCS datasets (#4501)

The BigQuery dataset has been deleted. See GoogleCloudPlatform/python-docs-samples#4553
kubeflow · Sep 16, 2020 · 3fd72e4 · 3fd72e4
1 parent c32ea23
commit 3fd72e4
Showing 1 changed file with 18 additions and 17 deletions.
diff --git a/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb b/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb
@@ -33,8 +33,9 @@
     "\n",
     "automl_create_dataset_for_tables_op        = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/create_dataset_for_tables/component.yaml')\n",
     "automl_import_data_from_bigquery_source_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/import_data_from_bigquery/component.yaml')\n",
+    "automl_import_data_from_gcs_op             = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/import_data_from_gcs/component.yaml')\n",
     "automl_create_model_for_tables_op          = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/create_model_for_tables/component.yaml')\n",
-    "automl_prediction_service_batch_predict_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/prediction_service_batch_predict/component.yaml')\n",
+    "automl_prediction_service_batch_predict_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/dc8dc3301c8a590231289cf9537b4dc08089957b/components/gcp/automl/prediction_service_batch_predict/component.yaml')\n",
     "automl_split_dataset_table_column_names_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/split_dataset_table_column_names/component.yaml')"
    ]
   },
@@ -47,15 +48,15 @@
     "# Define the pipeline\n",
     "import kfp\n",
     "\n",
-    "def retail_product_stockout_prediction_pipeline(\n",
+    "def retail_product_stockout_prediction_pipeline_gcs(\n",
     "    gcp_project_id: str,\n",
     "    gcp_region: str,\n",
     "    batch_predict_gcs_output_uri_prefix: str,\n",
-    "    dataset_bq_input_uri: str = 'bq://product-stockout.product_stockout.stockout',\n",
-    "    dataset_display_name: str = 'stockout_data',\n",
+    "    dataset_gcs_input_uris: list = ['gs://kubeflow-pipelines-regional-us-central1/mirror/cloud-ml-data/automl-tables/notebooks/stockout.csv'],\n",
+    "    dataset_display_name: str = 'stockout_data_gcs',\n",
     "    target_column_name: str = 'Stockout',\n",
     "    model_display_name: str = 'stockout_model',\n",
-    "    batch_predict_bq_input_uri: str = 'bq://product-stockout.product_stockout.batch_prediction_inputs',\n",
+    "    batch_predict_gcs_input_uris: list = ['gs://kubeflow-pipelines-regional-us-central1/mirror/cloud-ml-data/automl-tables/notebooks/batch_prediction_inputs.csv'],\n",
     "    train_budget_milli_node_hours: 'Integer' = 1000,\n",
     "):\n",
     "    # Create dataset\n",
@@ -66,9 +67,9 @@
     "    )\n",
     "\n",
     "    # Import data\n",
-    "    import_data_task = automl_import_data_from_bigquery_source_op(\n",
+    "    import_data_task = automl_import_data_from_gcs_op(\n",
     "        dataset_path=create_dataset_task.outputs['dataset_path'],\n",
-    "        input_uri=dataset_bq_input_uri,\n",
+    "        input_uris=dataset_gcs_input_uris,\n",
     "    )\n",
     "    \n",
     "    # Prepare column schemas\n",
@@ -94,7 +95,8 @@
     "    # Batch prediction\n",
     "    batch_predict_task = automl_prediction_service_batch_predict_op(\n",
     "        model_path=create_model_task.outputs['model_path'],\n",
-    "        bq_input_uri=batch_predict_bq_input_uri, \n",
+    "        #bq_input_uri=batch_predict_bq_input_uri,\n",
+    "        gcs_input_uris=batch_predict_gcs_input_uris,\n",
     "        gcs_output_uri_prefix=batch_predict_gcs_output_uri_prefix,\n",
     "    )\n",
     "    \n",
@@ -113,18 +115,17 @@
    "outputs": [],
    "source": [
     "# Run the pipeline\n",
+    "import json\n",
     "\n",
-    "# Get the GCP location of your project.\n",
-    "from google.cloud import automl\n",
-    "location_path = automl.AutoMlClient().location_path(PROJECT_ID, COMPUTE_REGION)\n",
-    "\n",
-    "kfp.run_pipeline_func_on_cluster(\n",
-    "    retail_product_stockout_prediction_pipeline,\n",
+    "kfp_endpoint=None\n",
+    "kfp.Client(host=kfp_endpoint).create_run_from_pipeline_func(\n",
+    "    retail_product_stockout_prediction_pipeline_gcs,\n",
     "    arguments=dict(\n",
     "        gcp_project_id=PROJECT_ID,\n",
     "        gcp_region=COMPUTE_REGION,\n",
-    "        dataset_bq_input_uri='bq://product-stockout.product_stockout.stockout',\n",
-    "        batch_predict_bq_input_uri='bq://product-stockout.product_stockout.batch_prediction_inputs',\n",
+    "        dataset_display_name='stockout_data_gcs2',  # Change this every time there is new data\n",
+    "        dataset_gcs_input_uris=json.dumps(['gs://kubeflow-pipelines-regional-us-central1/mirror/cloud-ml-data/automl-tables/notebooks/stockout.csv']),\n",
+    "        batch_predict_gcs_input_uris=json.dumps(['gs://kubeflow-pipelines-regional-us-central1/mirror/cloud-ml-data/automl-tables/notebooks/batch_prediction_inputs.csv']),\n",
     "        batch_predict_gcs_output_uri_prefix=batch_predict_gcs_output_uri_prefix,\n",
     "    )\n",
     ")"
@@ -152,4 +153,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}