diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/lint/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/common.cfg new file mode 100644 index 000000000..5d9c326e8 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "lint" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/lint/continuous.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/continuous.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/lint/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/periodic.cfg new file mode 100644 index 000000000..50fec9649 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/lint/presubmit.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/common.cfg new file mode 100644 index 000000000..1317612e8 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.6" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/continuous.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/continuous.cfg new file mode 100644 index 000000000..7218af149 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/continuous.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic.cfg new file mode 100644 index 000000000..50fec9649 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/presubmit.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/common.cfg new file mode 100644 index 000000000..322c83c17 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.7" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/continuous.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/continuous.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/periodic.cfg new file mode 100644 index 000000000..50fec9649 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/presubmit.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/common.cfg new file mode 100644 index 000000000..992f9f0d7 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.8" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/continuous.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/continuous.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic.cfg new file mode 100644 index 000000000..50fec9649 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/presubmit.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/test-samples.sh b/synthtool/gcp/templates/python_library/.kokoro/test-samples.sh new file mode 100755 index 000000000..16859b1fa --- /dev/null +++ b/synthtool/gcp/templates/python_library/.kokoro/test-samples.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +cd github/{{ metadata['repo']['repo'].split('/')[1] }} + +# Run periodic samples tests at latest release +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + LATEST_RELEASE=$(git describe --abbrev=0 --tags) + git checkout $LATEST_RELEASE +fi + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Install nox +python3.6 -m pip install --upgrade --quiet nox + +# Use secrets acessor service account to get secrets +if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then + gcloud auth activate-service-account \ + --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ + --project="cloud-devrel-kokoro-resources" +fi + +# This script will create 3 files: +# - testing/test-env.sh +# - testing/service-account.json +# - testing/client-secrets.json +./scripts/decrypt-secrets.sh + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json + +# For cloud-run session, we activate the service account for gcloud sdk. +gcloud auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" + +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +echo -e "\n******************** TESTING PROJECTS ********************" + +# Switch to 'fail at end' to allow all tests to complete before exiting. +set +e +# Use RTN to return a non-zero value if the test fails. +RTN=0 +ROOT=$(pwd) +# Find all requirements.txt in the samples directory (may break on whitespace). +for file in samples/**/requirements.txt; do + cd "$ROOT" + # Navigate to the project folder. + file=$(dirname "$file") + cd "$file" + + echo "------------------------------------------------------------" + echo "- testing $file" + echo "------------------------------------------------------------" + + # Use nox to execute the tests for the project. + python3.6 -m nox -s "$RUN_TESTS_SESSION" + EXIT=$? + + # If this is a periodic build, send the test log to the Build Cop Bot. + # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/buildcop. + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/buildcop + $KOKORO_GFILE_DIR/linux_amd64/buildcop + fi + + if [[ $EXIT -ne 0 ]]; then + RTN=1 + echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" + else + echo -e "\n Testing completed.\n" + fi + +done +cd "$ROOT" + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit "$RTN" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/samples/AUTHORING_GUIDE.md b/synthtool/gcp/templates/python_library/samples/AUTHORING_GUIDE.md new file mode 100644 index 000000000..55c97b32f --- /dev/null +++ b/synthtool/gcp/templates/python_library/samples/AUTHORING_GUIDE.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/samples/CONTRIBUTING.md b/synthtool/gcp/templates/python_library/samples/CONTRIBUTING.md new file mode 100644 index 000000000..34c882b6f --- /dev/null +++ b/synthtool/gcp/templates/python_library/samples/CONTRIBUTING.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/scripts/decrypt-secrets.sh b/synthtool/gcp/templates/python_library/scripts/decrypt-secrets.sh new file mode 100755 index 000000000..ff599eb2a --- /dev/null +++ b/synthtool/gcp/templates/python_library/scripts/decrypt-secrets.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT=$( dirname "$DIR" ) + +# Work from the project root. +cd $ROOT + +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +PROJECT_ID="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +gcloud secrets versions access latest --secret="python-docs-samples-test-env" \ + > testing/test-env.sh +gcloud secrets versions access latest \ + --secret="python-docs-samples-service-account" \ + > testing/service-account.json +gcloud secrets versions access latest \ + --secret="python-docs-samples-client-secrets" \ + > testing/client-secrets.json \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/scripts/readme-gen/readme_gen.py.j2 b/synthtool/gcp/templates/python_library/scripts/readme-gen/readme_gen.py.j2 new file mode 100644 index 000000000..d309d6e97 --- /dev/null +++ b/synthtool/gcp/templates/python_library/scripts/readme-gen/readme_gen.py.j2 @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generates READMEs using configuration defined in yaml.""" + +import argparse +import io +import os +import subprocess + +import jinja2 +import yaml + + +jinja_env = jinja2.Environment( + trim_blocks=True, + loader=jinja2.FileSystemLoader( + os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')))) + +README_TMPL = jinja_env.get_template('README.tmpl.rst') + + +def get_help(file): + return subprocess.check_output(['python', file, '--help']).decode() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('source') + parser.add_argument('--destination', default='README.rst') + + args = parser.parse_args() + + source = os.path.abspath(args.source) + root = os.path.dirname(source) + destination = os.path.join(root, args.destination) + + jinja_env.globals['get_help'] = get_help + + with io.open(source, 'r') as f: + config = yaml.load(f) + + # This allows get_help to execute in the right directory. + os.chdir(root) + + output = README_TMPL.render(config) + + with io.open(destination, 'w') as f: + f.write(output) + + +if __name__ == '__main__': + main() diff --git a/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/README.tmpl.rst b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/README.tmpl.rst new file mode 100644 index 000000000..a39dc774d --- /dev/null +++ b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/README.tmpl.rst @@ -0,0 +1,89 @@ +{%- raw -%} +{# The following line is a lie. BUT! Once jinja2 is done with it, it will + become truth! #} +.. This file is automatically generated. Do not edit this file directly. + +{{product.name}} Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/README.rst + + +This directory contains samples for {{product.name}}. {{product.description}} + +{{description}} + +.. _{{product.name}}: {{product.url}} + +{% if required_api_url %} +To run the sample, you need to enable the API at: {{required_api_url}} +{% endif %} + +{% if required_role %} +To run the sample, you need to have `{{required_role}}` role. +{% endif %} + +{{other_required_steps}} + +{% if setup %} +Setup +------------------------------------------------------------------------------- + +{% for section in setup %} + +{% include section + '.tmpl.rst' %} + +{% endfor %} +{% endif %} + +{% if samples %} +Samples +------------------------------------------------------------------------------- + +{% for sample in samples %} +{{sample.name}} ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +{% if not sample.hide_cloudshell_button %} +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/{{sample.file}},{{folder}}/README.rst +{% endif %} + + +{{sample.description}} + +To run this sample: + +.. code-block:: bash + + $ python {{sample.file}} +{% if sample.show_help %} + + {{get_help(sample.file)|indent}} +{% endif %} + + +{% endfor %} +{% endif %} + +{% if cloud_client_library %} + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + +{% endif %} + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ +{%- endraw -%} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/auth.tmpl.rst b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/auth.tmpl.rst new file mode 100644 index 000000000..1446b94a5 --- /dev/null +++ b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/auth.tmpl.rst @@ -0,0 +1,9 @@ +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started diff --git a/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/auth_api_key.tmpl.rst b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/auth_api_key.tmpl.rst new file mode 100644 index 000000000..11957ce27 --- /dev/null +++ b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/auth_api_key.tmpl.rst @@ -0,0 +1,14 @@ +Authentication +++++++++++++++ + +Authentication for this service is done via an `API Key`_. To obtain an API +Key: + +1. Open the `Cloud Platform Console`_ +2. Make sure that billing is enabled for your project. +3. From the **Credentials** page, create a new **API Key** or use an existing + one for your project. + +.. _API Key: + https://developers.google.com/api-client-library/python/guide/aaa_apikeys +.. _Cloud Console: https://console.cloud.google.com/project?_ diff --git a/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/install_deps.tmpl.rst b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/install_deps.tmpl.rst new file mode 100644 index 000000000..a0406dba8 --- /dev/null +++ b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -0,0 +1,29 @@ +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ diff --git a/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/install_portaudio.tmpl.rst b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/install_portaudio.tmpl.rst new file mode 100644 index 000000000..5ea33d18c --- /dev/null +++ b/synthtool/gcp/templates/python_library/scripts/readme-gen/templates/install_portaudio.tmpl.rst @@ -0,0 +1,35 @@ +Install PortAudio ++++++++++++++++++ + +Install `PortAudio`_. This is required by the `PyAudio`_ library to stream +audio from your computer's microphone. PyAudio depends on PortAudio for cross-platform compatibility, and is installed differently depending on the +platform. + +* For Mac OS X, you can use `Homebrew`_:: + + brew install portaudio + + **Note**: if you encounter an error when running `pip install` that indicates + it can't find `portaudio.h`, try running `pip install` with the following + flags:: + + pip install --global-option='build_ext' \ + --global-option='-I/usr/local/include' \ + --global-option='-L/usr/local/lib' \ + pyaudio + +* For Debian / Ubuntu Linux:: + + apt-get install portaudio19-dev python-all-dev + +* Windows may work without having to install PortAudio explicitly (it will get + installed with PyAudio). + +For more details, see the `PyAudio installation`_ page. + + +.. _PyAudio: https://people.csail.mit.edu/hubert/pyaudio/ +.. _PortAudio: http://www.portaudio.com/ +.. _PyAudio installation: + https://people.csail.mit.edu/hubert/pyaudio/#downloads +.. _Homebrew: http://brew.sh diff --git a/synthtool/gcp/templates/python_library/testing/.gitignore b/synthtool/gcp/templates/python_library/testing/.gitignore new file mode 100644 index 000000000..b05fbd630 --- /dev/null +++ b/synthtool/gcp/templates/python_library/testing/.gitignore @@ -0,0 +1,3 @@ +test-env.sh +service-account.json +client-secrets.json \ No newline at end of file diff --git a/synthtool/gcp/templates/python_samples/README.rst b/synthtool/gcp/templates/python_samples/README.rst new file mode 100644 index 000000000..6bd2ca589 --- /dev/null +++ b/synthtool/gcp/templates/python_samples/README.rst @@ -0,0 +1,75 @@ +{# The following line is a lie. BUT! Once jinja2 is done with it, it will + become truth! #} +.. This file is automatically generated. Do not edit this file directly. + +{{product.name}} Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/README.rst + + +This directory contains samples for {{product.name}}. {{product.description}} + +{{description}} + +.. _{{product.name}}: {{product.url}} + +{% if setup %} +Setup +------------------------------------------------------------------------------- + +{% for section in setup %} + +{% include section + '.tmpl.rst' %} + +{% endfor %} +{% endif %} + +{% if samples %} +Samples +------------------------------------------------------------------------------- + +{% for sample in samples %} +{{sample.name}} ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/{{sample.file}},{{folder}}/README.rst + + +{{sample.description}} + +To run this sample: + +.. code-block:: bash + + $ python {{sample.file}} +{% if sample.show_help %} + + {{get_help(sample.abs_path)|indent}} +{% endif %} + + +{% endfor %} +{% endif %} + +{% if cloud_client_library %} + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + +{% endif %} + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/synthtool/gcp/templates/python_samples/auth.tmpl.rst b/synthtool/gcp/templates/python_samples/auth.tmpl.rst new file mode 100644 index 000000000..1446b94a5 --- /dev/null +++ b/synthtool/gcp/templates/python_samples/auth.tmpl.rst @@ -0,0 +1,9 @@ +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started diff --git a/synthtool/gcp/templates/python_samples/auth_api_key.tmpl.rst b/synthtool/gcp/templates/python_samples/auth_api_key.tmpl.rst new file mode 100644 index 000000000..11957ce27 --- /dev/null +++ b/synthtool/gcp/templates/python_samples/auth_api_key.tmpl.rst @@ -0,0 +1,14 @@ +Authentication +++++++++++++++ + +Authentication for this service is done via an `API Key`_. To obtain an API +Key: + +1. Open the `Cloud Platform Console`_ +2. Make sure that billing is enabled for your project. +3. From the **Credentials** page, create a new **API Key** or use an existing + one for your project. + +.. _API Key: + https://developers.google.com/api-client-library/python/guide/aaa_apikeys +.. _Cloud Console: https://console.cloud.google.com/project?_ diff --git a/synthtool/gcp/templates/python_samples/install_deps.tmpl.rst b/synthtool/gcp/templates/python_samples/install_deps.tmpl.rst new file mode 100644 index 000000000..275d64989 --- /dev/null +++ b/synthtool/gcp/templates/python_samples/install_deps.tmpl.rst @@ -0,0 +1,29 @@ +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 3.6+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ diff --git a/synthtool/gcp/templates/python_samples/install_portaudio.tmpl.rst b/synthtool/gcp/templates/python_samples/install_portaudio.tmpl.rst new file mode 100644 index 000000000..5ea33d18c --- /dev/null +++ b/synthtool/gcp/templates/python_samples/install_portaudio.tmpl.rst @@ -0,0 +1,35 @@ +Install PortAudio ++++++++++++++++++ + +Install `PortAudio`_. This is required by the `PyAudio`_ library to stream +audio from your computer's microphone. PyAudio depends on PortAudio for cross-platform compatibility, and is installed differently depending on the +platform. + +* For Mac OS X, you can use `Homebrew`_:: + + brew install portaudio + + **Note**: if you encounter an error when running `pip install` that indicates + it can't find `portaudio.h`, try running `pip install` with the following + flags:: + + pip install --global-option='build_ext' \ + --global-option='-I/usr/local/include' \ + --global-option='-L/usr/local/lib' \ + pyaudio + +* For Debian / Ubuntu Linux:: + + apt-get install portaudio19-dev python-all-dev + +* Windows may work without having to install PortAudio explicitly (it will get + installed with PyAudio). + +For more details, see the `PyAudio installation`_ page. + + +.. _PyAudio: https://people.csail.mit.edu/hubert/pyaudio/ +.. _PortAudio: http://www.portaudio.com/ +.. _PyAudio installation: + https://people.csail.mit.edu/hubert/pyaudio/#downloads +.. _Homebrew: http://brew.sh diff --git a/synthtool/gcp/templates/python_samples/noxfile.py.j2 b/synthtool/gcp/templates/python_samples/noxfile.py.j2 new file mode 100644 index 000000000..b23055f14 --- /dev/null +++ b/synthtool/gcp/templates/python_samples/noxfile.py.j2 @@ -0,0 +1,225 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + 'ignored_versions': ["2.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + 'gcloud_project_env': 'GCLOUD_PROJECT', + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append('.') + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG['gcloud_project_env'] + # This should error out if not set. + ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] + ret['GCLOUD_PROJECT'] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG['envs']) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + "." + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip("SKIPPED: {} tests are disabled for this sample.".format( + session.python + )) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/synthtool/languages/python.py b/synthtool/languages/python.py index 06622959e..9ca05e7cf 100644 --- a/synthtool/languages/python.py +++ b/synthtool/languages/python.py @@ -13,10 +13,19 @@ # limitations under the License. import re +import sys +import yaml +from pathlib import Path +from typing import Any, Dict import synthtool as s +from synthtool import log, shell, _tracked_paths +from synthtool.gcp.common import CommonTemplates +from synthtool.sources import templates +PathOrStr = templates.PathOrStr + PB2_HEADER = r"""(\# -\*- coding: utf-8 -\*-\n)(\# Generated by the protocol buffer compiler\. DO NOT EDIT!.*?# source: .*?\.proto)""" PB2_GRPC_HEADER = r"""(\# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!) (import grpc)""" @@ -36,6 +45,11 @@ # See the License for the specific language governing permissions and # limitations under the License.""" +SAMPLES_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] +IGNORED_VERSIONS = ["2.7"] + +SAMPLES_TEMPLATE_PATH = Path(CommonTemplates()._template_root) / "python_samples" + def fix_pb2_headers(*, proto_root: str = "**/*_pb2.py") -> None: s.replace( @@ -52,3 +66,74 @@ def fix_pb2_grpc_headers(*, proto_root: str = "**/*_pb2_grpc.py") -> None: PB2_GRPC_HEADER, fr"{LICENSE}\n\n\g<1>\n\n\g<2>", # add line breaks to avoid stacking replacements ) + + +def _get_help(filename: str) -> str: + """Function used by sample readmegen""" + return shell.run(["python", filename, "--help"]).stdout + + +def _get_sample_readme_metadata(sample_dir: Path) -> dict: + sample_readme = sample_dir / "README.rst.in" + + sample_metadata = {} + if sample_readme.exists(): + requirements = str(Path(sample_dir / "requirements.txt").resolve()) + log.debug( + f"Installing requirements at {requirements} to generate {sample_readme}" + ) + shell.run([sys.executable, "-m", "pip", "install", "-r", requirements]) + + with open(sample_readme) as f: + sample_metadata = yaml.load(f, Loader=yaml.SafeLoader) + for sample in sample_metadata["samples"]: + # add absolute path to metadata so `python foo.py --help` succeeds + sample["abs_path"] = Path(sample_dir / (sample["file"])).resolve() + + return sample_metadata + + +def py_samples(*, root: PathOrStr = None, skip_readmes: bool = False) -> None: + """ + Find all samples projects and render templates. + Samples projects always have a 'requirements.txt' file and may also have + README.rst.in + + Args: + root (Union[Path, str]): The samples directory root. + skip_readmes (bool): If true, do not generate readmes. + """ + in_client_library = Path("samples").exists() and Path("setup.py").exists() + if root is None: + if in_client_library: + root = "samples" + else: + root = "." + + excludes = [] + if skip_readmes: + excludes.append("README.rst") + t = templates.TemplateGroup(SAMPLES_TEMPLATE_PATH, excludes=excludes) + + t.env.globals["get_help"] = _get_help # for sample readmegen + + for req in Path(root).glob("**/requirements.txt"): + sample_project_dir = req.parent + log.info(f"Generating templates for samples project '{sample_project_dir}'") + + excludes = ["**/*tmpl*"] # .tmpl. files are partial templates + + sample_readme_metadata: Dict[str, Any] = {} + if not skip_readmes: + sample_readme_metadata = _get_sample_readme_metadata(sample_project_dir) + # Don't generate readme if there's no metadata + if sample_readme_metadata == {}: + excludes.append("**/README.rst") + + if Path(sample_project_dir / "noxfile_config.py").exists(): + # Don't overwrite existing noxfile configs + excludes.append("**/noxfile_config.py") + + result = t.render(subdir=sample_project_dir, **sample_readme_metadata) + _tracked_paths.add(result) + s.copy([result], excludes=excludes) diff --git a/synthtool/sources/templates.py b/synthtool/sources/templates.py index 9d5338a38..dd1b5f048 100644 --- a/synthtool/sources/templates.py +++ b/synthtool/sources/templates.py @@ -66,8 +66,8 @@ def __init__(self, location: PathOrStr) -> None: self.source_path = Path(location) self.dir = tmp.tmpdir() - def render(self, template_name: str, **kwargs) -> Path: - return _render_to_path(self.env, template_name, self.dir, kwargs) + def render(self, template_name: str, subdir: PathOrStr = ".", **kwargs) -> Path: + return _render_to_path(self.env, template_name, self.dir / subdir, kwargs) class TemplateGroup: @@ -76,11 +76,11 @@ def __init__(self, location: PathOrStr, excludes: List[str] = []) -> None: self.dir = tmp.tmpdir() self.excludes = excludes - def render(self, **kwargs) -> Path: + def render(self, subdir: PathOrStr = ".", **kwargs) -> Path: for template_name in self.env.list_templates(): if template_name not in self.excludes: print(template_name) - _render_to_path(self.env, template_name, self.dir, kwargs) + _render_to_path(self.env, template_name, self.dir / subdir, kwargs) else: print(f"Skipping: {template_name}") diff --git a/tests/test_templates.py b/tests/test_templates.py index 20d4a28ca..42534e91b 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -34,6 +34,15 @@ def test_render(): assert result.read_text() == "Hello, world!\n" +def test_render_with_subdir(): + t = templates.Templates(FIXTURES) + result = t.render("example.j2", name="world", subdir="foo/bar") + + assert result.match("**/foo/bar/example") + assert result.name == "example" + assert result.read_text() == "Hello, world!\n" + + def test_render_group(): t = templates.TemplateGroup(FIXTURES / "group") result = t.render(var_a="hello", var_b="world") @@ -42,6 +51,14 @@ def test_render_group(): assert (result / "subdir" / "2.txt").read_text() == "world\n" +def test_render_group_with_subdir(): + t = templates.TemplateGroup(FIXTURES / "group") + result = t.render(subdir="foo/bar", var_a="hello", var_b="world") + + assert (result / "foo/bar" / "1.txt").read_text() == "hello\n" + assert (result / "foo/bar" / "subdir" / "2.txt").read_text() == "world\n" + + def test_render_preserve_mode(): """ Test that rendering templates correctly preserve file modes.