Automate new project creation (kubeflow#318)

* project creation

* merge master

* add readme

* fix lint

* lint

project_creation/README.md

@@ -0,0 +1,7 @@
+# Creating Projects Through Deployment Manager
+
+This folder contains configurations to create a new GCP project for deploying Kubeflow via Deployment Manager.
+
+The files are modified based on [the example of GCP Deployment Manager](https://github.com/GoogleCloudPlatform/deploymentmanager-samples/tree/master/examples/v2/project_creation).
+
+Project [kf-gcp-deploy0](https://pantheon.corp.google.com/kubernetes/workload?project=kf-gcp-deploy0&organizationId=714441643818) creates and owns the new projects, which are all in folder [gcp-deploy](https://pantheon.corp.google.com/projectselector2/kubernetes/list?folder=838562927550&supportedpurview=project)

project_creation/apis.py

@@ -0,0 +1,46 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Enables APIs on a specified project."""
+
+
+def GenerateConfig(context):
+  """Generates config."""
+
+  project_id = context.properties['project']
+  billing = context.properties['billing']
+  concurrent_api_activation = context.properties['concurrent_api_activation']
+
+  resources = []
+  for index, api in enumerate(context.properties['apis']):
+    depends_on = [project_id, billing]
+    # Serialize the activation of all the apis by making api_n depend on api_n-1
+    if (not concurrent_api_activation) and index != 0:
+      depends_on.append(
+          ApiResourceName(project_id, context.properties['apis'][index-1]))
+    resources.append({
+        'name': ApiResourceName(project_id, api),
+        'type': 'deploymentmanager.v2.virtual.enableService',
+        'metadata': {
+            'dependsOn': depends_on
+        },
+        'properties': {
+            'consumerId': 'project:' + project_id,
+            'serviceName': api
+        }
+    })
+  return {'resources': resources}
+
+
+def ApiResourceName(project_id, api_name):
+  return project_id + '-' + api_name

project_creation/config.yaml

@@ -0,0 +1,79 @@
+imports:
+- path: project.py
+
+resources:
+  # The "name" property below will be the ID of the new project
+  # If you want your project to have a different name, use the "project-name"
+  # property.
+  - name: kf-gcp-deploy-test5
+    type: project.py
+    properties:
+        # Change this to your organization ID.
+        # organization-id: "ORG_ID"
+        # You can also create the project in a folder.
+        # If both organization-id and parent-folder-id are provided,
+        # the project will be created in parent-folder-id.
+        #
+        # folder "kubeflow.org/gcp-deploy"
+        parent-folder-id: "838562927550"
+
+        # Change the following to your organization's billing account
+        # Billing account is "Kubeflow billed to Google"
+        billing-account-name: billingAccounts/01AF53-DC4A1B-4B1AA1
+
+        # The apis to enable in the new project.
+        # To see the possible APIs, use: gcloud services list --available
+        apis:
+          - compute.googleapis.com
+          - deploymentmanager.googleapis.com
+          - storage-component.googleapis.com
+          - monitoring.googleapis.com
+          - logging.googleapis.com
+          - containerregistry.googleapis.com
+          - container.googleapis.com
+
+        # The service accounts you want to create in the project
+        # service-accounts:
+          # - my-service-account-1
+          # - my-service-account-2
+
+        bucket-export-settings:
+            create-bucket: true
+            # If using an already existing bucket, specify this
+            # bucket: <my bucket name>
+
+        # Makes the service account that Deployment Manager would use in the
+        # generated project when making deployments in this new project a
+        # project owner.
+        set-dm-service-account-as-owner: true
+
+        # The patches to apply to the project's IAM policy. Note that these are
+        # always applied as a patch to the project's current IAM policy, not as a
+        # diff with the existing properties stored in DM. This means that removing
+        # a binding from the 'add' section will not remove the binding on the
+        # project during the next update. Instead it must be added to the 'remove'
+        # section.
+        iam-policy-patch:
+            # These are the bindings to add.
+            add:
+              - role: roles/owner
+                members:
+                  # NOTE: The DM service account that is creating this project will
+                  # automatically be added as an owner.
+                  #
+                  # Cloud service account owned by "kf-gcp-deploy0".
+                  - serviceAccount:459682233032@cloudservices.gserviceaccount.com
+                  # Account used by prober test.
+                  - serviceAccount:kubeflow-prober@kubeflow-prober-deploy.iam.gserviceaccount.com
+              #- role: roles/viewer
+                #members:
+                  #- user:iamtester@deployment-manager.net
+            # The bindings to remove. Note that these are idempotent, in the sense
+            # that any binding here that is not actually on the project is considered
+            # to have been removed successfully.
+            #remove:
+              #- role: roles/owner
+                #members:
+                  # This is already not on the project, but in case it shows up, let's
+                  # remove it.
+                  #- serviceAccount:1234567890@cloudservices.gserviceaccount.com

project_creation/project.py

@@ -0,0 +1,226 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Creates a single project with specified service accounts and APIs enabled."""
+
+import sys
+from apis import ApiResourceName
+
+#pylint: disable-msg=too-many-statements,too-many-branches
+def GenerateConfig(context):
+  """Generates config."""
+
+  project_id = context.env['name']
+  billing_name = 'billing_' + project_id
+
+  if not IsProjectParentValid(context.properties):
+    sys.exit(('Invalid [organization-id, parent-folder-id], '
+              'must specify at least one. If setting up Shared VPC, you '
+              'must specify at least organization-id.'))
+
+  parent_type = ''
+  parent_id = ''
+
+  if 'parent-folder-id' in context.properties:
+    parent_type = 'folder'
+    parent_id = context.properties['parent-folder-id']
+  else:
+    parent_type = 'organization'
+    parent_id = context.properties['organization-id']
+
+  if 'project-name' in context.properties:
+    project_name = context.properties['project-name']
+  else:
+    project_name = project_id
+
+  resources = [{
+      'name': project_id,
+      'type': 'cloudresourcemanager.v1.project',
+      'properties': {
+          'name': project_name,
+          'projectId': project_id,
+          'parent': {
+              'type': parent_type,
+              'id': parent_id
+          }
+      }
+  }, {
+      'name': billing_name,
+      'type': 'deploymentmanager.v2.virtual.projectBillingInfo',
+      'metadata': {
+          'dependsOn': [project_id]
+      },
+      'properties': {
+          'name': 'projects/' + project_id,
+          'billingAccountName': context.properties['billing-account-name']
+      }
+  }, {
+      'name': 'apis',
+      'type': 'apis.py',
+      'properties': {
+          'project': project_id,
+          'billing': billing_name,
+          'apis': context.properties['apis'],
+          'concurrent_api_activation':
+              context.properties['concurrent_api_activation']
+      }
+  }, {
+      'name': 'service-accounts',
+      'type': 'service-accounts.py',
+      'properties': {
+          'project': project_id,
+          'service-accounts': context.properties['service-accounts']
+      }
+  }]
+  if (context.properties.get('iam-policy-patch') or
+      context.properties.get('set-dm-service-account-as-owner')):
+    iam_policy_patch = context.properties.get('iam-policy-patch', {})
+    if iam_policy_patch.get('add'):
+      policies_to_add = iam_policy_patch['add']
+    else:
+      policies_to_add = []
+    if iam_policy_patch.get('remove'):
+      policies_to_remove = iam_policy_patch['remove']
+    else:
+      policies_to_remove = []
+
+    if context.properties.get('set-dm-service-account-as-owner'):
+      svc_acct = 'serviceAccount:{}@cloudservices.gserviceaccount.com'.format(
+        '$(ref.{}.projectNumber)'.format(project_id)
+      )
+
+      # Merge the default DM service account into the owner role if it exists
+      owner_idx = [bind['role'] == 'roles/owner' for bind in policies_to_add]
+      try:
+        # Determine where in policies_to_add the owner role is.
+        idx = owner_idx.index(True)
+      except ValueError:
+        # If the owner role is not defined just append to what to add.
+        policies_to_add.append({'role': 'roles/owner', 'members': [svc_acct]})
+      else:
+        # Append the default DM service account to the owner role members
+        if svc_acct not in policies_to_add[idx]['members']:
+          policies_to_add[idx]['members'].append(svc_acct)
+
+    get_iam_policy_dependencies = [project_id]
+    for api in context.properties['apis']:
+      get_iam_policy_dependencies.append(ApiResourceName(project_id, api))
+
+    resources.extend([{
+        # Get the IAM policy first so that we do not remove any existing bindings.
+        'name': 'get-iam-policy-' + project_id,
+        'action': 'gcp-types/cloudresourcemanager-v1:cloudresourcemanager.projects.getIamPolicy',
+        'properties': {
+          'resource': project_id,
+        },
+        'metadata': {
+          'dependsOn': get_iam_policy_dependencies,
+          'runtimePolicy': ['UPDATE_ALWAYS']
+        }
+    }, {
+        # Set the IAM policy patching the existing policy with what ever is currently in the
+        # config.
+        'name': 'patch-iam-policy-' + project_id,
+        'action': 'gcp-types/cloudresourcemanager-v1:cloudresourcemanager.projects.setIamPolicy',
+        'properties': {
+          'resource': project_id,
+          'policy': '$(ref.get-iam-policy-' + project_id + ')',
+          'gcpIamPolicyPatch': {
+             'add': policies_to_add,
+             'remove': policies_to_remove
+          }
+        }
+    }])
+  if context.properties.get('bucket-export-settings'):
+    bucket_name = None
+    action_dependency = [project_id,
+                         ApiResourceName(project_id, 'compute.googleapis.com')]
+    if context.properties['bucket-export-settings'].get('create-bucket'):
+      bucket_name = project_id + '-export-bucket'
+      resources.append({
+          'name': bucket_name,
+          'type': 'gcp-types/storage-v1:buckets',
+          'properties': {
+              'project': project_id,
+              'name': bucket_name
+          },
+          'metadata': {
+              'dependsOn': [project_id,
+                            ApiResourceName(
+                                project_id, 'storage-component.googleapis.com')]
+          }
+      })
+      action_dependency.append(bucket_name)
+    else:
+      bucket_name = context.properties['bucket-export-settings']['bucket-name']
+    resources.append({
+        'name': 'set-export-bucket',
+        'action': 'gcp-types/compute-v1:compute.projects.setUsageExportBucket',
+        'properties': {
+            'project': project_id,
+            'bucketName': 'gs://' + bucket_name
+        },
+        'metadata': {
+            'dependsOn': action_dependency
+        }
+    })
+  if context.properties.get('shared_vpc_host'):
+    resources.append({
+      'name': project_id + '-xpn-host',
+      'type': 'compute.beta.xpnHost',
+      'properties': {
+          'organization-id': context.properties['organization-id'],
+          'billing-account-name': context.properties['billing-account-name'],
+          'project': project_id,
+      },
+      'metadata': {
+          'dependsOn': [
+              ApiResourceName(project_id, 'compute.googleapis.com'),
+              project_id,
+          ],
+      }
+    })
+  if context.properties.get('shared_vpc_service_of'):
+    resources.append({
+      'name': project_id + '-xpn-service-' +
+      context.properties['shared_vpc_service_of'],
+      'type': 'compute.beta.xpnResource',
+      'properties': {
+          'organization-id': context.properties['organization-id'],
+          'billing-account-name': context.properties['billing-account-name'],
+          'project': [context.properties['shared_vpc_service_of']],
+          'xpnResource': {
+              'id': project_id,
+              'type': 'PROJECT',
+          },
+      },
+      'metadata': {
+          'dependsOn': [
+              ApiResourceName(project_id, 'compute.googleapis.com'),
+              project_id,
+              context.properties['shared_vpc_service_of'] + '-xpn-host',
+          ],
+      }
+    })
+
+  return {'resources': resources}
+
+def IsProjectParentValid(properties):
+  """ A helper function to validate that the project is either under a folder
+      or under an organization.
+      If we are setting up Shared VPC, we always need organization-id.
+      If not, we can work with either organization-id or parent-folder-id.
+  """
+  if ('shared_vpc_service_of' in properties or properties['shared_vpc_host']):
+    return 'organization-id' in properties
+  return 'organization-id' in properties or 'parent-folder-id' in properties