Merge branch 'master' into master

Author: engelke

.kokoro/common.cfg

@@ -9,7 +9,7 @@ build_file: "python-docs-samples/.kokoro/trampoline.sh"
 # Use the Python worker docker image.
 env_vars: {
     key: "TRAMPOLINE_IMAGE"
-    value: "gcr.io/cloud-devrel-kokoro-resources/python@sha256:4b6ba8c199e96248980db4538065cddeea594138b9b9fb2d0388603922087747"
+    value: "gcr.io/cloud-devrel-kokoro-resources/python@sha256:e11a459d01e5dcd3613fda35c7c94edfecfe911ed79c078580ff59de300b1938"
 }
 
 # Specify project ID

.kokoro/presubmit_tests_trace.cfg

@@ -0,0 +1,15 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Download secrets from Cloud Storage.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
+
+# Tell the trampoline which build file to use.
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-docs-samples/.kokoro/system_tests.sh"
+}
+
+env_vars: {
+    key: "NOX_SESSION"
+    value: "trace and py36 and not appengine"
+}

.kokoro/system_tests_trace.cfg

@@ -0,0 +1,15 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Download secrets from Cloud Storage.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
+
+# Tell the trampoline which build file to use.
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-docs-samples/.kokoro/system_tests.sh"
+}
+
+env_vars: {
+    key: "NOX_SESSION"
+    value: "trace and py36 and not appengine"
+}

.travis.yml

@@ -32,4 +32,4 @@ install:
 - pip install --upgrade nox
 - pip install --upgrade git+https://github.com/dhermes/ci-diff-helper.git
 script:
-- "./scripts/travis.sh"
+- "./scripts/travis.sh"

appengine/flexible/pubsub/main.py

@@ -24,7 +24,7 @@
 app = Flask(__name__)
 
 # Configure the following environment variables via app.yaml
-# This is used in the push request handler to veirfy that the request came from
+# This is used in the push request handler to verify that the request came from
 # pubsub and originated from a trusted source.
 app.config['PUBSUB_VERIFICATION_TOKEN'] = \
     os.environ['PUBSUB_VERIFICATION_TOKEN']

appengine/standard/endpoints-frameworks-v2/echo/requirements.txt

@@ -1,2 +1,2 @@
-google-endpoints==4.7.0
+google-endpoints==4.8.0
 google-endpoints-api-management==1.11.0

appengine/standard/mail/app.yaml

@@ -2,13 +2,11 @@ runtime: python27
 api_version: 1
 threadsafe: yes
 
-# [START bounce_service]
 # [START mail_service]
 inbound_services:
 - mail
+- mail_bounce # Handle bounced mail notifications
 # [END mail_service]
-- mail_bounce
-# [END bounce_service]
 
 handlers:
 - url: /user/.+

appengine/standard/pubsub/main.py

@@ -25,7 +25,7 @@
 app = Flask(__name__)
 
 # Configure the following environment variables via app.yaml
-# This is used in the push request handler to veirfy that the request came from
+# This is used in the push request handler to verify that the request came from
 # pubsub and originated from a trusted source.
 app.config['PUBSUB_VERIFICATION_TOKEN'] = \
     os.environ['PUBSUB_VERIFICATION_TOKEN']

appengine/standard_python37/bigquery/main.py

@@ -39,5 +39,8 @@ def main():
 
 
 if __name__ == '__main__':
-    app.run(debug=True)
+    # This is used when running locally only. When deploying to Google App
+    # Engine, a webserver process such as Gunicorn will serve the app. This
+    # can be configured by adding an `entrypoint` to app.yaml.
+    app.run(host='127.0.0.1', port=8080, debug=True)
 # [END gae_python37_bigquery]

appengine/standard_python37/custom-server/app.yaml

@@ -0,0 +1,4 @@
+# [START gae_python37_custom_runtime]
+runtime: python37
+entrypoint: uwsgi --http-socket :8080 --wsgi-file main.py --callable app --master --processes 1 --threads 2
+# [END gae_python37_custom_runtime]

appengine/standard_python37/custom-server/main.py

@@ -0,0 +1,26 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask import Flask
+
+app = Flask(__name__)
+
+
+@app.route('/')
+def main():
+    return 'Hello, World!'
+
+
+if __name__ == '__main__':
+    app.run(debug=True)

appengine/standard_python37/custom-server/requirements.txt

@@ -0,0 +1,2 @@
+uwsgi==2.0.17.1
+flask==1.0.2

appengine/standard_python37/redis/main.py

@@ -39,5 +39,7 @@ def index():
 
 
 if __name__ == '__main__':
-    # This is used when running locally.
+    # This is used when running locally only. When deploying to Google App
+    # Engine, a webserver process such as Gunicorn will serve the app. This
+    # can be configured by adding an `entrypoint` to app.yaml.
     app.run(host='127.0.0.1', port=8080, debug=True)

appengine/standard_python37/spanner/main.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# [START gae_python37_cloudsql_mysql]
+# [START gae_python37_cloud_spanner]
 import os
 
 from flask import Flask
@@ -33,3 +33,4 @@ def main():
     results = list(cursor)
 
     return 'Query Result: {}'.format(results[0][0])
+# [END gae_python37_cloud_spanner]

bigquery/cloud-client/user_credentials.py

@@ -23,20 +23,18 @@
 import argparse
 
 
-def run_query(credentials, project, query):
-    from google.cloud import bigquery
-
-    client = bigquery.Client(project=project, credentials=credentials)
-    query_job = client.query(query)
-
-    # Print the results.
-    for row in query_job.result():  # Wait for the job to complete.
-        print(row)
-
-
-def authenticate_and_query(project, query, launch_browser=True):
+def main(project, launch_browser=True):
+    # [START bigquery_auth_user_flow]
     from google_auth_oauthlib import flow
 
+    # TODO: Uncomment the line below to set the `launch_browser` variable.
+    # launch_browser = True
+    #
+    # The `launch_browser` boolean variable indicates if a local server is used
+    # as the callback URL in the auth flow. A value of `True` is recommended,
+    # but a local server does not work if accessing the application remotely,
+    # such as over SSH or from a remote Jupyter notebook.
+
     appflow = flow.InstalledAppFlow.from_client_secrets_file(
         'client_secrets.json',
         scopes=['https://www.googleapis.com/auth/bigquery'])
@@ -46,7 +44,33 @@ def authenticate_and_query(project, query, launch_browser=True):
     else:
         appflow.run_console()
 
-    run_query(appflow.credentials, project, query)
+    credentials = appflow.credentials
+    # [END bigquery_auth_user_flow]
+
+    # [START bigquery_auth_user_query]
+    from google.cloud import bigquery
+
+    # TODO: Uncomment the line below to set the `project` variable.
+    # project = 'user-project-id'
+    #
+    # The `project` variable defines the project to be billed for query
+    # processing. The user must have the bigquery.jobs.create permission on
+    # this project to run a query. See:
+    # https://cloud.google.com/bigquery/docs/access-control#permissions
+
+    client = bigquery.Client(project=project, credentials=credentials)
+
+    query_string = """SELECT name, SUM(number) as total
+    FROM `bigquery-public-data.usa_names.usa_1910_current`
+    WHERE name = 'William'
+    GROUP BY name;
+    """
+    query_job = client.query(query_string)
+
+    # Print the results.
+    for row in query_job.result():  # Wait for the job to complete.
+        print("{}: {}".format(row['name'], row['total']))
+    # [END bigquery_auth_user_query]
 
 
 if __name__ == '__main__':
@@ -58,9 +82,8 @@ def authenticate_and_query(project, query, launch_browser=True):
         help='Use a local server flow to authenticate. ',
         action='store_true')
     parser.add_argument('project', help='Project to use for BigQuery billing.')
-    parser.add_argument('query', help='BigQuery SQL Query.')
 
     args = parser.parse_args()
 
-    authenticate_and_query(
-        args.project, args.query, launch_browser=args.launch_browser)
+    main(
+        args.project, launch_browser=args.launch_browser)

bigquery/cloud-client/user_credentials_test.py

@@ -18,7 +18,7 @@
 import mock
 import pytest
 
-from user_credentials import authenticate_and_query
+from user_credentials import main
 
 
 PROJECT = os.environ['GCLOUD_PROJECT']
@@ -36,6 +36,7 @@ def mock_flow():
 
 
 def test_auth_query_console(mock_flow, capsys):
-    authenticate_and_query(PROJECT, 'SELECT 1+1;', launch_browser=False)
+    main(PROJECT, launch_browser=False)
     out, _ = capsys.readouterr()
-    assert '2' in out
+    # Fun fact: William P. Wood was the 1st director of the US Secret Service.
+    assert 'William' in out

composer/tools/copy_environment.py

@@ -28,7 +28,6 @@
 from __future__ import print_function
 
 import argparse
-import ast
 import base64
 import contextlib
 import json
@@ -40,6 +39,7 @@
 import tempfile
 import time
 import uuid
+from distutils.spawn import find_executable
 
 from cryptography import fernet
 import google.auth
@@ -52,6 +52,7 @@
 from six.moves import configparser
 
 DEFAULT_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"]
+EXECUTABLES = ['gcsfuse', 'cloud_sql_proxy', 'mysql', 'gcloud', 'gsutil']
 
 
 def parse_args():
@@ -294,8 +295,9 @@ def create_service_account_key(iam_client, project, service_account_name):
         )
         .execute()
     )
-    service_account_key_decoded = ast.literal_eval(
+    service_account_key_decoded = json.loads(
         base64.b64decode(service_account_key.get("privateKeyData", ""))
+              .decode("utf-8")
     )
     time.sleep(5)
     return service_account_key_decoded
@@ -333,16 +335,10 @@ def get_sql_instance_service_account(sql_client, project, instance):
 
 
 def grant_rw_permissions(gcs_bucket, service_account):
-    if subprocess.call(
-        [
-            "gsutil",
-            "acl",
-            "ch",
-            "-u",
-            service_account + ":O",
-            "gs://" + gcs_bucket.name,
-        ]
-    ):
+    try:
+        gcs_bucket.acl.user(service_account).grant_owner()
+        gcs_bucket.acl.save()
+    except Exception:
         print(
             "Failed to set acls for service account {} on bucket {}.".format(
                 service_account, gcs_bucket.name
@@ -542,7 +538,10 @@ def import_data(
         if proxy_subprocess:
             proxy_subprocess.kill()
         if fuse_dir:
-            subprocess.call(["fusermount", "-u", fuse_dir])
+            try:
+                subprocess.call(["fusermount", "-u", fuse_dir])
+            except OSError:
+                subprocess.call(["umount", fuse_dir])
         if tmp_dir_name:
             shutil.rmtree(tmp_dir_name)
 
@@ -571,7 +570,9 @@ def copy_database(project, existing_env, new_env, running_as_service_account):
     try:
         # create default creds clients
         default_credentials, _ = google.auth.default(scopes=DEFAULT_SCOPES)
-        storage_client = storage.Client(credentials=default_credentials)
+        storage_client = storage.Client(
+            project=project, credentials=default_credentials
+        )
         iam_client = discovery.build(
             "iam", "v1", credentials=default_credentials
         )
@@ -717,8 +718,19 @@ def clone_environment(
     )
 
 
+def check_executables():
+    not_found = [
+        executable for executable in EXECUTABLES
+        if not find_executable(executable)
+    ]
+    if not_found:
+        print('Required executables not found: {}'.format(' '.join(not_found)))
+        sys.exit(1)
+
+
 if __name__ == "__main__":
     args = parse_args()
+    check_executables()
     clone_environment(
         args.project,
         args.location,

composer/tools/copy_environment_test.py

@@ -22,7 +22,7 @@
 
 def test_grant_rw_permissions_fails_gracefully(monkeypatch, capsys):
     mock_call = mock.Mock()
-    mock_call.return_value = 1
+    mock_call.side_effect = RuntimeError()
     monkeypatch.setattr(subprocess, 'call', mock_call)
     monkeypatch.setattr(time, 'sleep', lambda sec: None)
     from . import copy_environment