diff --git a/components/gcp/bigquery/query/README.md b/components/gcp/bigquery/query/README.md index 96995d6f3c4..64bda7d265c 100644 --- a/components/gcp/bigquery/query/README.md +++ b/components/gcp/bigquery/query/README.md @@ -52,11 +52,7 @@ output_gcs_path | The path to the Cloud Storage bucket containing the query outp To use the component, the following requirements must be met: * The BigQuery API is enabled. -* The component is running under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow Pipeline cluster. For example: - - ``` - bigquery_query_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * The Kubeflow user service account is a member of the `roles/bigquery.admin` role of the project. * The Kubeflow user service account is a member of the `roles/storage.objectCreator `role of the Cloud Storage output bucket. @@ -125,7 +121,6 @@ OUTPUT_PATH = '{}/bigquery/query/questions.csv'.format(GCS_WORKING_DIR) ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Bigquery query pipeline', @@ -147,7 +142,7 @@ def pipeline( table_id=table_id, output_gcs_path=output_gcs_path, dataset_location=dataset_location, - job_config=job_config).apply(gcp.use_gcp_secret('user-gcp-sa')) + job_config=job_config) ``` #### Compile the pipeline diff --git a/components/gcp/bigquery/query/sample.ipynb b/components/gcp/bigquery/query/sample.ipynb index 7791b398f2b..fa7b38f5823 100644 --- a/components/gcp/bigquery/query/sample.ipynb +++ b/components/gcp/bigquery/query/sample.ipynb @@ -57,11 +57,7 @@ "To use the component, the following requirements must be met:\n", "\n", "* The BigQuery API is enabled.\n", - "* The component is running under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow Pipeline cluster. For example:\n", - "\n", - " ```\n", - " bigquery_query_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", + "* The component can authenticate to use GCP APIs. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* The Kubeflow user service account is a member of the `roles/bigquery.admin` role of the project.\n", "* The Kubeflow user service account is a member of the `roles/storage.objectCreator `role of the Cloud Storage output bucket.\n", "\n", @@ -179,7 +175,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Bigquery query pipeline',\n", @@ -201,7 +196,7 @@ " table_id=table_id, \n", " output_gcs_path=output_gcs_path, \n", " dataset_location=dataset_location, \n", - " job_config=job_config).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " job_config=job_config)" ] }, { @@ -301,4 +296,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/dataflow/launch_python/README.md b/components/gcp/dataflow/launch_python/README.md index 484549984ef..e013177200f 100644 --- a/components/gcp/dataflow/launch_python/README.md +++ b/components/gcp/dataflow/launch_python/README.md @@ -63,14 +63,11 @@ job_id | The ID of the Cloud Dataflow job that is created. ## Cautions & requirements To use the components, the following requirements must be met: - Cloud Dataflow API is enabled. -- The component is running under a secret Kubeflow user service account in a Kubeflow Pipelines cluster. For example: - ``` - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` -The Kubeflow user service account is a member of: -- `roles/dataflow.developer` role of the project. -- `roles/storage.objectViewer` role of the Cloud Storage Objects `python_file_path` and `requirements_file_path`. -- `roles/storage.objectCreator` role of the Cloud Storage Object `staging_dir`. +- The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. +- The Kubeflow user service account is a member of: + - `roles/dataflow.developer` role of the project. + - `roles/storage.objectViewer` role of the Cloud Storage Objects `python_file_path` and `requirements_file_path`. + - `roles/storage.objectCreator` role of the Cloud Storage Object `staging_dir`. ## Detailed description The component does several things during the execution: @@ -221,7 +218,6 @@ OUTPUT_FILE = '{}/wc/wordcount.out'.format(GCS_STAGING_DIR) ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataflow launch python pipeline', @@ -243,7 +239,7 @@ def pipeline( staging_dir = staging_dir, requirements_file_path = requirements_file_path, args = args, - wait_interval = wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval = wait_interval) ``` #### Compile the pipeline diff --git a/components/gcp/dataflow/launch_python/sample.ipynb b/components/gcp/dataflow/launch_python/sample.ipynb index 4d3b2000616..c7b00e84ce2 100644 --- a/components/gcp/dataflow/launch_python/sample.ipynb +++ b/components/gcp/dataflow/launch_python/sample.ipynb @@ -47,14 +47,11 @@ "## Cautions & requirements\n", "To use the components, the following requirements must be met:\n", "- Cloud Dataflow API is enabled.\n", - "- The component is running under a secret Kubeflow user service account in a Kubeflow Pipeline cluster. For example:\n", - "```\n", - "component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - "```\n", - "The Kubeflow user service account is a member of:\n", - "- `roles/dataflow.developer` role of the project.\n", - "- `roles/storage.objectViewer` role of the Cloud Storage Objects `python_file_path` and `requirements_file_path`.\n", - "- `roles/storage.objectCreator` role of the Cloud Storage Object `staging_dir`. \n", + "- The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", + "- The Kubeflow user service account is a member of:\n", + " - `roles/dataflow.developer` role of the project.\n", + " - `roles/storage.objectViewer` role of the Cloud Storage Objects `python_file_path` and `requirements_file_path`.\n", + " - `roles/storage.objectCreator` role of the Cloud Storage Object `staging_dir`. \n", "\n", "## Detailed description\n", "The component does several things during the execution:\n", @@ -295,7 +292,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataflow launch python pipeline',\n", @@ -317,7 +313,7 @@ " staging_dir = staging_dir, \n", " requirements_file_path = requirements_file_path, \n", " args = args,\n", - " wait_interval = wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " wait_interval = wait_interval)" ] }, { @@ -417,4 +413,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/dataflow/launch_template/README.md b/components/gcp/dataflow/launch_template/README.md index 68808d55c0d..f1d5b3eb264 100644 --- a/components/gcp/dataflow/launch_template/README.md +++ b/components/gcp/dataflow/launch_template/README.md @@ -37,11 +37,8 @@ job_id | The id of the Cloud Dataflow job that is created. To use the component, the following requirements must be met: - Cloud Dataflow API is enabled. -- The component is running under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow Pipeline cluster. For example: - ``` - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` -* The Kubeflow user service account is a member of: +- The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. +- The Kubeflow user service account is a member of: - `roles/dataflow.developer` role of the project. - `roles/storage.objectViewer` role of the Cloud Storage Object `gcs_path.` - `roles/storage.objectCreator` role of the Cloud Storage Object `staging_dir.` @@ -102,7 +99,6 @@ OUTPUT_PATH = '{}/out/wc'.format(GCS_WORKING_DIR) ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataflow launch template pipeline', @@ -128,7 +124,7 @@ def pipeline( location = location, validate_only = validate_only, staging_dir = staging_dir, - wait_interval = wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval = wait_interval)) ``` #### Compile the pipeline diff --git a/components/gcp/dataflow/launch_template/sample.ipynb b/components/gcp/dataflow/launch_template/sample.ipynb index 312ad02d802..d880c7a9a12 100644 --- a/components/gcp/dataflow/launch_template/sample.ipynb +++ b/components/gcp/dataflow/launch_template/sample.ipynb @@ -42,11 +42,8 @@ "\n", "To use the component, the following requirements must be met:\n", "- Cloud Dataflow API is enabled.\n", - "- The component is running under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow Pipeline cluster. For example:\n", - " ```\n", - " component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", - "* The Kubeflow user service account is a member of:\n", + "- The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", + "- The Kubeflow user service account is a member of:\n", " - `roles/dataflow.developer` role of the project.\n", " - `roles/storage.objectViewer` role of the Cloud Storage Object `gcs_path.`\n", " - `roles/storage.objectCreator` role of the Cloud Storage Object `staging_dir.` \n", @@ -155,7 +152,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataflow launch template pipeline',\n", @@ -181,7 +177,7 @@ " location = location, \n", " validate_only = validate_only,\n", " staging_dir = staging_dir,\n", - " wait_interval = wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " wait_interval = wait_interval)" ] }, { @@ -282,4 +278,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/dataproc/create_cluster/README.md b/components/gcp/dataproc/create_cluster/README.md index ecc4d066b11..89350b1ab65 100644 --- a/components/gcp/dataproc/create_cluster/README.md +++ b/components/gcp/dataproc/create_cluster/README.md @@ -62,11 +62,7 @@ Note: You can recycle the cluster by using the [Dataproc delete cluster componen To use the component, you must: * Set up the GCP project by following these [steps](https://cloud.google.com/dataproc/docs/guides/setup-project). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - - ``` - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the following types of access to the Kubeflow user service account: * Read access to the Cloud Storage buckets which contain the initialization action files. * The role, `roles/dataproc.editor`, on the project. @@ -114,7 +110,6 @@ EXPERIMENT_NAME = 'Dataproc - Create Cluster' ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataproc create cluster pipeline', @@ -140,7 +135,7 @@ def dataproc_create_cluster_pipeline( config_bucket=config_bucket, image_version=image_version, cluster=cluster, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` #### Compile the pipeline diff --git a/components/gcp/dataproc/create_cluster/sample.ipynb b/components/gcp/dataproc/create_cluster/sample.ipynb index 3eccef858c7..9daf0d7ea98 100644 --- a/components/gcp/dataproc/create_cluster/sample.ipynb +++ b/components/gcp/dataproc/create_cluster/sample.ipynb @@ -46,11 +46,7 @@ "\n", "To use the component, you must:\n", "* Set up the GCP project by following these [steps](https://cloud.google.com/dataproc/docs/guides/setup-project).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```\n", - " component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the following types of access to the Kubeflow user service account:\n", " * Read access to the Cloud Storage buckets which contains initialization action files.\n", " * The role, `roles/dataproc.editor` on the project.\n", @@ -137,7 +133,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataproc create cluster pipeline',\n", @@ -163,7 +158,7 @@ " config_bucket=config_bucket, \n", " image_version=image_version, \n", " cluster=cluster, \n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " wait_interval=wait_interval)" ] }, { @@ -248,4 +243,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/dataproc/delete_cluster/README.md b/components/gcp/dataproc/delete_cluster/README.md index ed752124e89..11a35e24aed 100644 --- a/components/gcp/dataproc/delete_cluster/README.md +++ b/components/gcp/dataproc/delete_cluster/README.md @@ -43,11 +43,7 @@ ML workflow: ## Cautions & requirements To use the component, you must: * Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - - ``` - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the Kubeflow user service account the role, `roles/dataproc.editor`, on the project. ## Detailed description @@ -98,7 +94,6 @@ EXPERIMENT_NAME = 'Dataproc - Delete Cluster' ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataproc delete cluster pipeline', @@ -112,7 +107,7 @@ def dataproc_delete_cluster_pipeline( dataproc_delete_cluster_op( project_id=project_id, region=region, - name=name).apply(gcp.use_gcp_secret('user-gcp-sa')) + name=name) ``` #### Compile the pipeline diff --git a/components/gcp/dataproc/delete_cluster/sample.ipynb b/components/gcp/dataproc/delete_cluster/sample.ipynb index ea1a5044a6a..548d962e0a0 100644 --- a/components/gcp/dataproc/delete_cluster/sample.ipynb +++ b/components/gcp/dataproc/delete_cluster/sample.ipynb @@ -33,11 +33,7 @@ "## Cautions & requirements\n", "To use the component, you must:\n", "* Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```\n", - " component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project.\n", "\n", "## Detailed description\n", @@ -125,7 +121,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataproc delete cluster pipeline',\n", @@ -139,7 +134,7 @@ " dataproc_delete_cluster_op(\n", " project_id=project_id, \n", " region=region, \n", - " name=name).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " name=name)" ] }, { diff --git a/components/gcp/dataproc/submit_hadoop_job/README.md b/components/gcp/dataproc/submit_hadoop_job/README.md index 02068db1d69..7eb96623a2a 100644 --- a/components/gcp/dataproc/submit_hadoop_job/README.md +++ b/components/gcp/dataproc/submit_hadoop_job/README.md @@ -60,11 +60,7 @@ job_id | The ID of the created job. | String To use the component, you must: * Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project). * [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - - ```python - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the Kubeflow user service account the role, `roles/dataproc.editor`, on the project. ## Detailed description @@ -135,7 +131,6 @@ Caution: This will remove all blob files under `OUTPUT_GCS_PATH`. ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataproc submit Hadoop job pipeline', @@ -164,7 +159,7 @@ def dataproc_submit_hadoop_job_pipeline( args=args, hadoop_job=hadoop_job, job=job, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` #### Compile the pipeline diff --git a/components/gcp/dataproc/submit_hadoop_job/sample.ipynb b/components/gcp/dataproc/submit_hadoop_job/sample.ipynb index cddcec83f9b..de3364a1f6f 100644 --- a/components/gcp/dataproc/submit_hadoop_job/sample.ipynb +++ b/components/gcp/dataproc/submit_hadoop_job/sample.ipynb @@ -46,11 +46,7 @@ "To use the component, you must:\n", "* Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project).\n", "* [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```python\n", - " component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project.\n", "\n", "## Detailed description\n", @@ -186,7 +182,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataproc submit Hadoop job pipeline',\n", @@ -215,7 +210,7 @@ " args=args, \n", " hadoop_job=hadoop_job, \n", " job=job, \n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " wait_interval=wait_interval)" ] }, { @@ -316,4 +311,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/dataproc/submit_hive_job/README.md b/components/gcp/dataproc/submit_hive_job/README.md index aa72c8b8340..148f2e5a720 100644 --- a/components/gcp/dataproc/submit_hive_job/README.md +++ b/components/gcp/dataproc/submit_hive_job/README.md @@ -52,11 +52,7 @@ job_id | The ID of the created job. | String To use the component, you must: * Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project). * [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - - ``` - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project. ## Detailed description @@ -128,7 +124,6 @@ EXPERIMENT_NAME = 'Dataproc - Submit Hive Job' ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataproc submit Hive job pipeline', @@ -154,7 +149,7 @@ def dataproc_submit_hive_job_pipeline( script_variables=script_variables, hive_job=hive_job, job=job, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` diff --git a/components/gcp/dataproc/submit_hive_job/sample.ipynb b/components/gcp/dataproc/submit_hive_job/sample.ipynb index 4340483deb4..fe90c907946 100644 --- a/components/gcp/dataproc/submit_hive_job/sample.ipynb +++ b/components/gcp/dataproc/submit_hive_job/sample.ipynb @@ -39,11 +39,7 @@ "To use the component, you must:\n", "* Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project).\n", "* [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```\n", - " component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project.\n", "\n", "## Detailed description\n", @@ -156,7 +152,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataproc submit Hive job pipeline',\n", @@ -182,7 +177,7 @@ " script_variables=script_variables, \n", " hive_job=hive_job, \n", " job=job, \n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", + " wait_interval=wait_interval)\n", " " ] }, @@ -267,4 +262,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/dataproc/submit_pig_job/README.md b/components/gcp/dataproc/submit_pig_job/README.md index 301dac8effa..5cf40ec503b 100644 --- a/components/gcp/dataproc/submit_pig_job/README.md +++ b/components/gcp/dataproc/submit_pig_job/README.md @@ -58,11 +58,7 @@ job_id | The ID of the created job. | String To use the component, you must: * Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project). * [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - - ``` - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the Kubeflow user service account the role, `roles/dataproc.editor`, on the project. ## Detailed description @@ -124,7 +120,6 @@ EXPERIMENT_NAME = 'Dataproc - Submit Pig Job' ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataproc submit Pig job pipeline', @@ -150,7 +145,7 @@ def dataproc_submit_pig_job_pipeline( script_variables=script_variables, pig_job=pig_job, job=job, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` diff --git a/components/gcp/dataproc/submit_pig_job/sample.ipynb b/components/gcp/dataproc/submit_pig_job/sample.ipynb index d8b7a084fca..fb29a471490 100644 --- a/components/gcp/dataproc/submit_pig_job/sample.ipynb +++ b/components/gcp/dataproc/submit_pig_job/sample.ipynb @@ -42,11 +42,7 @@ "To use the component, you must:\n", "* Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project).\n", "* [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```\n", - " component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project.\n", "\n", "## Detailed description\n", @@ -146,7 +142,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataproc submit Pig job pipeline',\n", @@ -172,7 +167,7 @@ " script_variables=script_variables, \n", " pig_job=pig_job, \n", " job=job, \n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", + " wait_interval=wait_interval)\n", " " ] }, @@ -257,4 +252,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/dataproc/submit_pyspark_job/README.md b/components/gcp/dataproc/submit_pyspark_job/README.md index b26d11c5468..51ae678d173 100644 --- a/components/gcp/dataproc/submit_pyspark_job/README.md +++ b/components/gcp/dataproc/submit_pyspark_job/README.md @@ -54,11 +54,7 @@ job_id | The ID of the created job. | String To use the component, you must: * Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project). * [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - - ``` - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project. ## Detailed description @@ -120,7 +116,6 @@ EXPERIMENT_NAME = 'Dataproc - Submit PySpark Job' ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataproc submit PySpark job pipeline', @@ -144,7 +139,7 @@ def dataproc_submit_pyspark_job_pipeline( args=args, pyspark_job=pyspark_job, job=job, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` diff --git a/components/gcp/dataproc/submit_pyspark_job/sample.ipynb b/components/gcp/dataproc/submit_pyspark_job/sample.ipynb index ff292ac1c9b..43362238bca 100644 --- a/components/gcp/dataproc/submit_pyspark_job/sample.ipynb +++ b/components/gcp/dataproc/submit_pyspark_job/sample.ipynb @@ -42,11 +42,7 @@ "To use the component, you must:\n", "* Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project).\n", "* [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```\n", - " component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project.\n", "\n", "## Detailed description\n", @@ -157,7 +153,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataproc submit PySpark job pipeline',\n", @@ -181,7 +176,7 @@ " args=args, \n", " pyspark_job=pyspark_job, \n", " job=job, \n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", + " wait_interval=wait_interval)\n", " " ] }, @@ -266,4 +261,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/dataproc/submit_spark_job/README.md b/components/gcp/dataproc/submit_spark_job/README.md index e6b26098bfd..18b43c8ad1d 100644 --- a/components/gcp/dataproc/submit_spark_job/README.md +++ b/components/gcp/dataproc/submit_spark_job/README.md @@ -66,13 +66,7 @@ To use the component, you must: * Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project). * [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts), in a Kubeflow cluster. For example: - - ``` - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` - - +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project. @@ -139,7 +133,6 @@ EXPERIMENT_NAME = 'Dataproc - Submit Spark Job' ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataproc submit Spark job pipeline', @@ -165,7 +158,7 @@ def dataproc_submit_spark_job_pipeline( args=args, spark_job=spark_job, job=job, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` diff --git a/components/gcp/dataproc/submit_spark_job/sample.ipynb b/components/gcp/dataproc/submit_spark_job/sample.ipynb index 67a8f168bf4..5d85964752e 100644 --- a/components/gcp/dataproc/submit_spark_job/sample.ipynb +++ b/components/gcp/dataproc/submit_spark_job/sample.ipynb @@ -50,13 +50,7 @@ "\n", "* Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project).\n", "* [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```\n", - " component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", - "\n", - "\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project.\n", "\n", "\n", @@ -159,7 +153,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataproc submit Spark job pipeline',\n", @@ -185,7 +178,7 @@ " args=args, \n", " spark_job=spark_job, \n", " job=job, \n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", + " wait_interval=wait_interval)\n", " " ] }, @@ -271,4 +264,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/dataproc/submit_sparksql_job/README.md b/components/gcp/dataproc/submit_sparksql_job/README.md index 53d6d90824c..ecb3ddd9ad7 100644 --- a/components/gcp/dataproc/submit_sparksql_job/README.md +++ b/components/gcp/dataproc/submit_sparksql_job/README.md @@ -53,10 +53,7 @@ job_id | The ID of the created job. | String To use the component, you must: * Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project). * [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - ``` - component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the Kubeflow user service account the role, `roles/dataproc.editor`, on the project. ## Detailed Description @@ -124,7 +121,6 @@ EXPERIMENT_NAME = 'Dataproc - Submit SparkSQL Job' ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='Dataproc submit SparkSQL job pipeline', @@ -150,7 +146,7 @@ def dataproc_submit_sparksql_job_pipeline( script_variables=script_variables, sparksql_job=sparksql_job, job=job, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` diff --git a/components/gcp/dataproc/submit_sparksql_job/sample.ipynb b/components/gcp/dataproc/submit_sparksql_job/sample.ipynb index 6ec702737eb..fee13655c07 100644 --- a/components/gcp/dataproc/submit_sparksql_job/sample.ipynb +++ b/components/gcp/dataproc/submit_sparksql_job/sample.ipynb @@ -40,10 +40,7 @@ "To use the component, you must:\n", "* Set up a GCP project by following this [guide](https://cloud.google.com/dataproc/docs/guides/setup-project).\n", "* [Create a new cluster](https://cloud.google.com/dataproc/docs/guides/create-cluster).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "```\n", - "component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - "```\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the Kubeflow user service account the role `roles/dataproc.editor` on the project.\n", "\n", "## Detailed Description\n", @@ -152,7 +149,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='Dataproc submit SparkSQL job pipeline',\n", @@ -178,7 +174,7 @@ " script_variables=script_variables, \n", " sparksql_job=sparksql_job, \n", " job=job, \n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", + " wait_interval=wait_interval)\n", " " ] }, @@ -263,4 +259,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/ml_engine/batch_predict/README.md b/components/gcp/ml_engine/batch_predict/README.md index 37454a6fb12..ae148b8daf5 100644 --- a/components/gcp/ml_engine/batch_predict/README.md +++ b/components/gcp/ml_engine/batch_predict/README.md @@ -57,13 +57,7 @@ output_path | The output path of the batch prediction job | GCSPath To use the component, you must: * Set up a cloud environment by following this [guide](https://cloud.google.com/ml-engine/docs/tensorflow/getting-started-training-prediction#setup). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - - ```python - mlengine_predict_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` - - +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the following types of access to the Kubeflow user service account: * Read access to the Cloud Storage buckets which contains the input data. * Write access to the Cloud Storage bucket of the output directory. @@ -132,7 +126,6 @@ OUTPUT_GCS_PATH = GCS_WORKING_DIR + '/batch_predict/output/' ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='CloudML batch predict pipeline', @@ -161,7 +154,7 @@ def pipeline( output_data_format=output_data_format, prediction_input=prediction_input, job_id_prefix=job_id_prefix, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` #### Compile the pipeline diff --git a/components/gcp/ml_engine/batch_predict/sample.ipynb b/components/gcp/ml_engine/batch_predict/sample.ipynb index 3e8dd1de3d6..976c7653cae 100644 --- a/components/gcp/ml_engine/batch_predict/sample.ipynb +++ b/components/gcp/ml_engine/batch_predict/sample.ipynb @@ -62,13 +62,7 @@ "To use the component, you must:\n", "\n", "* Set up a cloud environment by following this [guide](https://cloud.google.com/ml-engine/docs/tensorflow/getting-started-training-prediction#setup).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```python\n", - " mlengine_predict_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", - "\n", - "\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the following types of access to the Kubeflow user service account:\n", " * Read access to the Cloud Storage buckets which contains the input data.\n", " * Write access to the Cloud Storage bucket of the output directory.\n", @@ -185,7 +179,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='CloudML batch predict pipeline',\n", @@ -214,7 +207,7 @@ " output_data_format=output_data_format, \n", " prediction_input=prediction_input, \n", " job_id_prefix=job_id_prefix,\n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " wait_interval=wait_interval)" ] }, { @@ -315,4 +308,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/ml_engine/deploy/README.md b/components/gcp/ml_engine/deploy/README.md index be4224283c5..9bf7961c61e 100644 --- a/components/gcp/ml_engine/deploy/README.md +++ b/components/gcp/ml_engine/deploy/README.md @@ -70,14 +70,7 @@ The accepted file formats are: To use the component, you must: * [Set up the cloud environment](https://cloud.google.com/ml-engine/docs/tensorflow/getting-started-training-prediction#setup). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - - ``` - ```python - mlengine_deploy_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - - ``` - +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant read access to the Cloud Storage bucket that contains the trained model to the Kubeflow user service account. ## Detailed description @@ -136,7 +129,6 @@ TRAINED_MODEL_PATH = 'gs://ml-pipeline-playground/samples/ml_engine/census/train ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='CloudML deploy pipeline', @@ -163,7 +155,7 @@ def pipeline( version=version, replace_existing_version=replace_existing_version, set_default=set_default, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` #### Compile the pipeline diff --git a/components/gcp/ml_engine/deploy/sample.ipynb b/components/gcp/ml_engine/deploy/sample.ipynb index b17b610fbdc..828d9ec770d 100644 --- a/components/gcp/ml_engine/deploy/sample.ipynb +++ b/components/gcp/ml_engine/deploy/sample.ipynb @@ -75,14 +75,7 @@ "To use the component, you must:\n", "\n", "* [Set up the cloud environment](https://cloud.google.com/ml-engine/docs/tensorflow/getting-started-training-prediction#setup).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```\n", - " ```python\n", - " mlengine_deploy_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - "\n", - " ```\n", - "\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant read access to the Cloud Storage bucket that contains the trained model to the Kubeflow user service account.\n", "\n", "## Detailed description\n", @@ -176,7 +169,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='CloudML deploy pipeline',\n", @@ -203,7 +195,7 @@ " version=version, \n", " replace_existing_version=replace_existing_version, \n", " set_default=set_default, \n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " wait_interval=wait_interval)" ] }, { @@ -288,4 +280,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/components/gcp/ml_engine/train/README.md b/components/gcp/ml_engine/train/README.md index cf03e6cdb00..43a95883250 100644 --- a/components/gcp/ml_engine/train/README.md +++ b/components/gcp/ml_engine/train/README.md @@ -73,12 +73,7 @@ The component accepts two types of inputs: To use the component, you must: * Set up a cloud environment by following this [guide](https://cloud.google.com/ml-engine/docs/tensorflow/getting-started-training-prediction#setup). -* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example: - - ``` - mlengine_train_op(...).apply(gcp.use_gcp_secret('user-gcp-sa')) - ``` - +* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details. * Grant the following access to the Kubeflow user service account: * Read access to the Cloud Storage buckets which contain the input data, packages, or Docker images. * Write access to the Cloud Storage bucket of the output directory. @@ -160,7 +155,6 @@ rm -fr ./cloudml-samples-master/ ./master.zip ./dist ```python import kfp.dsl as dsl -import kfp.gcp as gcp import json @dsl.pipeline( name='CloudML training pipeline', @@ -199,7 +193,7 @@ def pipeline( worker_image_uri=worker_image_uri, training_input=training_input, job_id_prefix=job_id_prefix, - wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa')) + wait_interval=wait_interval) ``` #### Compile the pipeline diff --git a/components/gcp/ml_engine/train/sample.ipynb b/components/gcp/ml_engine/train/sample.ipynb index a6dcce6235d..765cd7c85c1 100644 --- a/components/gcp/ml_engine/train/sample.ipynb +++ b/components/gcp/ml_engine/train/sample.ipynb @@ -56,12 +56,7 @@ "To use the component, you must:\n", "\n", "* Set up a cloud environment by following this [guide](https://cloud.google.com/ml-engine/docs/tensorflow/getting-started-training-prediction#setup).\n", - "* Run the component under a secret [Kubeflow user service account](https://www.kubeflow.org/docs/started/getting-started-gke/#gcp-service-accounts) in a Kubeflow cluster. For example:\n", - "\n", - " ```\n", - " mlengine_train_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", - " ```\n", - "\n", + "* The component can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", "* Grant the following access to the Kubeflow user service account: \n", " * Read access to the Cloud Storage buckets which contain the input data, packages, or Docker images.\n", " * Write access to the Cloud Storage bucket of the output directory.\n", @@ -219,7 +214,6 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "@dsl.pipeline(\n", " name='CloudML training pipeline',\n", @@ -258,7 +252,7 @@ " worker_image_uri=worker_image_uri, \n", " training_input=training_input, \n", " job_id_prefix=job_id_prefix, \n", - " wait_interval=wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " wait_interval=wait_interval)" ] }, { @@ -360,4 +354,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/samples/contrib/image-captioning-gcp/Image Captioning TF 2.0.ipynb b/samples/contrib/image-captioning-gcp/Image Captioning TF 2.0.ipynb index 5a8a27259a5..db02802207e 100644 --- a/samples/contrib/image-captioning-gcp/Image Captioning TF 2.0.ipynb +++ b/samples/contrib/image-captioning-gcp/Image Captioning TF 2.0.ipynb @@ -154,8 +154,7 @@ "source": [ "import kfp\n", "import kfp.dsl as dsl\n", - "from kfp import compiler\n", - "from kfp.gcp import use_gcp_secret" + "from kfp import compiler" ] }, { @@ -827,7 +826,11 @@ "source": [ "## Create and run pipeline\n", "### Create pipeline\n", - "The pipeline parameters are specified below in the `caption pipeline` function signature. Using the value `'default'` for the output directories saves them in a subdirectory of `GCS_DATASET_PATH`. Use `use_gcp_secret('user-gcp-sa')` to give read/write permissions for the storage buckets. " + "The pipeline parameters are specified below in the `caption pipeline` function signature. Using the value `'default'` for the output directories saves them in a subdirectory of `GCS_DATASET_PATH`.\n", + "\n", + "### Requirements\n", + "* The pipeline can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", + "* Read/write permissions for the storage buckets." ] }, { @@ -859,15 +862,13 @@ " dataset_path, \n", " output_dir=preprocessing_output_dir,\n", " batch_size=preprocessing_batch_size, \n", - " num_examples=num_examples).apply(\n", - " use_gcp_secret('user-gcp-sa'))\n", + " num_examples=num_examples)\n", " \n", " tokenize_captions_task = tokenize_captions_op(\n", " dataset_path, \n", " preprocessing_img_task.output, \n", " output_dir=tokenizing_output_dir, \n", - " top_k=vocab_size).apply(\n", - " use_gcp_secret('user-gcp-sa'))\n", + " top_k=vocab_size)\n", " \n", " model_train_task = model_train_op(\n", " dataset_path, \n", @@ -878,8 +879,7 @@ " batch_size=training_batch_size, \n", " embedding_dim=embedding_dim, \n", " units=hidden_state_size, \n", - " epochs=epochs).apply(\n", - " use_gcp_secret('user-gcp-sa'))\n", + " epochs=epochs)\n", " \n", " predict_task = predict_op(\n", " dataset_path,\n", @@ -888,8 +888,14 @@ " preprocess_output_dir=preprocessing_output_dir,\n", " valid_output_dir=validation_output_dir,\n", " embedding_dim=embedding_dim,\n", - " units=hidden_state_size).apply(\n", - " use_gcp_secret('user-gcp-sa'))" + " units=hidden_state_size)\n", + "\n", + " # The pipeline should be able to authenticate to GCP.\n", + " # Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", + " #\n", + " # For example, you may uncomment the following lines to use GSA keys.\n", + " # from kfp.gcp import use_gcp_secret\n", + " # kfp.dsl.get_pipeline_conf().add_op_transformer(use_gcp_secret('user-gcp-sa'))" ] }, { @@ -944,4 +950,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/samples/contrib/local_development_quickstart/Local Development Quickstart.ipynb b/samples/contrib/local_development_quickstart/Local Development Quickstart.ipynb index bf04f7ef1bf..f508f813013 100644 --- a/samples/contrib/local_development_quickstart/Local Development Quickstart.ipynb +++ b/samples/contrib/local_development_quickstart/Local Development Quickstart.ipynb @@ -140,7 +140,7 @@ "metadata": {}, "source": [ "### 1.3 Define pipeline\n", - "Note that when accessing google cloud file system, you need to apply proper GCP secret. This can be done by calling `apply(use_gcp_secret('user-gcp-sa'))` after a component function." + "Note that when accessing google cloud file system, you need to make sure the pipeline can authenticate to GCP. Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details." ] }, { @@ -150,12 +150,16 @@ "outputs": [], "source": [ "import kfp.dsl as dsl\n", - "from kfp.gcp import use_gcp_secret\n", "\n", "# Defines the pipeline.\n", "@dsl.pipeline(name='List GCS blobs', description='Lists GCS blobs.')\n", "def pipeline_func(bucket_name):\n", - " list_blobs_task = list_blobs_op(bucket_name).apply(use_gcp_secret('user-gcp-sa'))\n", + " list_blobs_task = list_blobs_op(bucket_name)\n", + " # Use the following commented code instead if you want to use GSA key for authentication.\n", + " #\n", + " # from kfp.gcp import use_gcp_secret\n", + " # list_blobs_task = list_blobs_op(bucket_name).apply(use_gcp_secret('user-gcp-sa'))\n", + " # Same for below.", "\n", "# Compile the pipeline to a file.\n", "import kfp.compiler as compiler\n", @@ -353,7 +357,6 @@ "source": [ "import datetime\n", "import kfp.compiler as compiler\n", - "from kfp.gcp import use_gcp_secret\n", "\n", "# Define the pipeline\n", "@kfp.dsl.pipeline(\n", @@ -361,7 +364,7 @@ " description='Takes a GCS bucket name as input and lists the blobs.'\n", ")\n", "def pipeline_func(bucket='Enter your bucket name here.'):\n", - " list_blobs_task = list_gcs_blobs_op('List', bucket).apply(use_gcp_secret('user-gcp-sa'))\n", + " list_blobs_task = list_gcs_blobs_op('List', bucket)\n", "\n", "# Compile the pipeline to a file.\n", "filename = 'tmp/pipelines/list_blobs{dt:%Y%m%d_%H%M%S}.pipeline.tar.gz'.format(\n", @@ -555,8 +558,8 @@ " description='Takes a GCS bucket name views a CSV input file in the bucket.'\n", ")\n", "def pipeline_func(bucket='Enter your bucket name here.'):\n", - " list_blobs_task = list_gcs_blobs_op('List', bucket).apply(use_gcp_secret('user-gcp-sa'))\n", - " view_input_task = view_input_op('View', list_blobs_task.outputs['blobs']).apply(use_gcp_secret('user-gcp-sa'))\n", + " list_blobs_task = list_gcs_blobs_op('List', bucket)\n", + " view_input_task = view_input_op('View', list_blobs_task.outputs['blobs'])\n", "\n", "# Compile the pipeline to a file.\n", "filename = 'tmp/pipelines/quickstart_pipeline{dt:%Y%m%d_%H%M%S}.pipeline.tar.gz'.format(\n", diff --git a/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb b/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb index 7a0e0c33fde..3a03c00d8c3 100644 --- a/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb +++ b/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb @@ -98,8 +98,12 @@ " gcs_output_uri_prefix=batch_predict_gcs_output_uri_prefix,\n", " )\n", " \n", - " from kfp.gcp import use_gcp_secret\n", - " kfp.dsl.get_pipeline_conf().add_op_transformer(use_gcp_secret('user-gcp-sa'))" + " # The pipeline should be able to authenticate to GCP.\n", + " # Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", + " #\n", + " # For example, you may uncomment the following lines to use GSA keys.\n", + " # from kfp.gcp import use_gcp_secret\n", + " # kfp.dsl.get_pipeline_conf().add_op_transformer(use_gcp_secret('user-gcp-sa'))" ] }, { @@ -148,4 +152,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/sdk/python/kfp/gcp.py b/sdk/python/kfp/gcp.py index f7ae33049a4..41de43f874f 100644 --- a/sdk/python/kfp/gcp.py +++ b/sdk/python/kfp/gcp.py @@ -16,20 +16,10 @@ V1NodeSelector, V1NodeSelectorTerm, V1NodeSelectorRequirement, V1PreferredSchedulingTerm def use_gcp_secret(secret_name='user-gcp-sa', secret_file_path_in_volume=None, volume_name=None, secret_volume_mount_path='/secret/gcp-credentials'): - """An operator that configures the container to use GCP service account. + """An operator that configures the container to use GCP service account by service account key + stored in a Kubernetes secret. - The user-gcp-sa secret is created as part of the kubeflow deployment that - stores the access token for kubeflow user service account. - - With this service account, the container has a range of GCP APIs to - access to. This service account is automatically created as part of the - kubeflow deployment. - - For the list of the GCP APIs this service account can access to, check - https://github.com/kubeflow/kubeflow/blob/7b0db0d92d65c0746ac52b000cbc290dac7c62b1/deployment/gke/deployment_manager_configs/iam_bindings_template.yaml#L18 - - If you want to call the GCP APIs in a different project, grant the kf-user - service account access permission. + For cluster setup and alternatives to using service account key, check https://www.kubeflow.org/docs/gke/authentication-pipelines/. """ # permitted values for secret_name = ['admin-gcp-sa', 'user-gcp-sa']