diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py new file mode 100644 index 000000000000..d893a142fa1e --- /dev/null +++ b/dataproc/create_cluster.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_cluster(project_id, region, cluster_name): + # [START dataproc_create_cluster] + from google.cloud import dataproc_v1 as dataproc + + # TODO(developer): Uncomment and set the following variables + # project_id = 'YOUR_PROJECT_ID' + # region = 'YOUR_CLUSTER_REGION' + # cluster_name = 'YOUR_CLUSTER_NAME' + + # Create a client with the endpoint set to the desired cluster region + client = dataproc.ClusterControllerClient(client_options={ + 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) + }) + + # Create the cluster config + cluster = { + 'project_id': project_id, + 'cluster_name': cluster_name, + 'config': { + 'master_config': { + 'num_instances': 1, + 'machine_type_uri': 'n1-standard-1' + }, + 'worker_config': { + 'num_instances': 2, + 'machine_type_uri': 'n1-standard-1' + } + } + } + + # Create the cluster + operation = client.create_cluster(project_id, region, cluster) + result = operation.result() + + # Output a success message + print('Cluster created successfully: {}'.format(result.cluster_name)) + # [END dataproc_create_cluster] diff --git a/dataproc/create_cluster_test.py b/dataproc/create_cluster_test.py new file mode 100644 index 000000000000..d58a1d0b6557 --- /dev/null +++ b/dataproc/create_cluster_test.py @@ -0,0 +1,44 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid +import pytest + +from google.cloud import dataproc_v1 as dataproc + +import create_cluster + +PROJECT_ID = os.environ['GCLOUD_PROJECT'] +REGION = 'us-central1' +CLUSTER_NAME = 'test-cluster-{}'.format(str(uuid.uuid4())) + + +@pytest.fixture(autouse=True) +def teardown(): + yield + + client = dataproc.ClusterControllerClient(client_options={ + 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(REGION) + }) + # Client library function + client.delete_cluster(PROJECT_ID, REGION, CLUSTER_NAME) + + +def test_cluster_create(capsys): + # Wrapper function for client library function + create_cluster.create_cluster(PROJECT_ID, REGION, CLUSTER_NAME) + + out, _ = capsys.readouterr() + assert CLUSTER_NAME in out diff --git a/dataproc/dataproc_e2e_test.py b/dataproc/dataproc_e2e_donttest.py similarity index 100% rename from dataproc/dataproc_e2e_test.py rename to dataproc/dataproc_e2e_donttest.py diff --git a/dataproc/requirements.txt b/dataproc/requirements.txt index 81a0a72bf2b8..0ffe752a6eaa 100644 --- a/dataproc/requirements.txt +++ b/dataproc/requirements.txt @@ -3,4 +3,4 @@ google-auth==1.6.3 google-auth-httplib2==0.0.3 google-cloud==0.34.0 google-cloud-storage==1.19.1 -google-cloud-dataproc==0.5.0 +google-cloud-dataproc==0.6.1