Skip to content

Commit

Permalink
Added CLI functionality to quickstart
Browse files Browse the repository at this point in the history
  • Loading branch information
bradmiro committed Jan 27, 2020
1 parent e7ebcdc commit be445cf
Showing 1 changed file with 31 additions and 13 deletions.
44 changes: 31 additions & 13 deletions dataproc/quickstart/quickstart.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,24 @@
# limitations under the License.

# [START dataproc_quickstart]
"""This quickstart sample walks a user through creating a Cloud Dataproc
cluster, submitting a PySpark job from Google Cloud Storage to the
cluster, reading the output of the job and deleting the cluster, all
using the Python client library.
Usage:
python3 quickstart.py --project_id <PROJECT_ID> --region <REGION> \
--cluster_name <CLUSTER_NAME> --job_file_path <GCS_JOB_FILE_PATH>
"""

import argparse
import time

from google.cloud import dataproc_v1 as dataproc
from google.cloud import storage


def quickstart(project_id, region, cluster_name, job_file_path):
"""This quickstart sample walks a user through creating a Cloud Dataproc
cluster, submitting a PySpark job from Google Cloud Storage to the
cluster, reading the output of the job and deleting the cluster, all
using the Python client library.
Args:
project_id (string): Project to use for creating resources.
region (string): Region where the resources should live.
cluster_name (string): Name to use for creating a cluster.
job_file_path (string): Job in GCS to execute against the cluster.
"""

# Create the cluster client.
cluster_client = dataproc.ClusterControllerClient(client_options={
'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region)
Expand Down Expand Up @@ -125,4 +124,23 @@ def quickstart(project_id, region, cluster_name, job_file_path):
operation.result()

print('Cluster {} successfully deleted.'.format(cluster_name))
# [END dataproc_quickstart]


if __name__ == "__main__":
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('--project_id', type=str,
help='Project to use for creating resources.')
parser.add_argument('--region', type=str,
help='Region where the resources should live.')
parser.add_argument('--cluster_name', type=str,
help='Name to use for creating a cluster')
parser.add_argument('--job_file_path', type=str,
help='Job in GCS to execute against the cluster.')

args = parser.parse_args()
quickstart(args.project_id, args.region,
args.cluster_name, args.job_file_path)
# [END dataproc_quickstart]

0 comments on commit be445cf

Please sign in to comment.