Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: update templates #289

Merged
merged 28 commits into from
Mar 25, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
63800b0
changes without context
yoshi-automation Mar 24, 2021
01b31be
docs: add proto-plus to intersphinx mapping
yoshi-automation Mar 24, 2021
35d5a94
chore: add type hint check
yoshi-automation Mar 24, 2021
29e53ce
chore: add blacken to template
yoshi-automation Mar 24, 2021
c8a9c1d
fix: address lint issues
yoshi-automation Mar 24, 2021
999c04d
docs(python): update intersphinx for grpc and auth
yoshi-automation Mar 24, 2021
a34476c
docs(python): fix intersphinx link for google-auth
yoshi-automation Mar 24, 2021
3216c97
chore: add config / docs for 'pre-commit' support
yoshi-automation Mar 24, 2021
6be0e6f
chore(deps): update precommit hook pre-commit/pre-commit-hooks to v3.3.0
yoshi-automation Mar 24, 2021
6920f9c
chore: update noxfile.py.j2
yoshi-automation Mar 24, 2021
e6b3ee6
chore(deps): update precommit hook pre-commit/pre-commit-hooks to v3.4.0
yoshi-automation Mar 24, 2021
d7bcbc6
docs(python): document adding Python 3.9 support, dropping 3.5 support
yoshi-automation Mar 24, 2021
352798a
chore: exclude `.nox` directories from linting
yoshi-automation Mar 24, 2021
56167b1
chore(python): fix column sizing issue in docs
yoshi-automation Mar 24, 2021
950ba76
chore(python): use 'http' in LICENSE
yoshi-automation Mar 24, 2021
c80243a
chore(python): skip docfx in main presubmit
yoshi-automation Mar 24, 2021
cd0dcb2
chore: add missing quotation mark
yoshi-automation Mar 24, 2021
e20c499
chore: add 3.9 to noxfile template
yoshi-automation Mar 24, 2021
372d953
build(python): make `NOX_SESSION` optional
yoshi-automation Mar 24, 2021
a3196a3
chore: Add header checker config to python library synth
yoshi-automation Mar 24, 2021
f537e26
build: migrate to flakybot
yoshi-automation Mar 24, 2021
d70eaf7
chore(python): include py.typed files in release
yoshi-automation Mar 24, 2021
4ddc7a0
docs: update python contributing guide
yoshi-automation Mar 24, 2021
ead3db7
build(python): enable flakybot on library unit and system tests
yoshi-automation Mar 24, 2021
be4ac51
chore: add pre-commit-config to renovate ignore paths
yoshi-automation Mar 24, 2021
dc9423a
chore(python): add kokoro configs for periodic builds against head
yoshi-automation Mar 24, 2021
19611eb
chore(deps): update precommit hook pycqa/flake8 to v3.9.0
yoshi-automation Mar 24, 2021
7fc65ea
fix: revert changes that break unit and docs tests
larkee Mar 24, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .flake8
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ exclude =
*_pb2.py

# Standard linting exemptions.
**/.nox/**
__pycache__,
.git,
*.pyc,
Expand Down
15 changes: 15 additions & 0 deletions .github/header-checker-lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{"allowedCopyrightHolders": ["Google LLC"],
"allowedLicenses": ["Apache-2.0", "MIT", "BSD-3"],
"ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt"],
"sourceFileExtensions": [
"ts",
"js",
"java",
"sh",
"Dockerfile",
"yaml",
"py",
"html",
"txt"
]
}
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,10 @@ docs.metadata

# Virtual environment
env/

# Test logs
coverage.xml
sponge_log.xml
*sponge_log.xml

# System test environment variables.
system_tests/local_test_setup
Expand Down
26 changes: 20 additions & 6 deletions .kokoro/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@

set -eo pipefail

cd github/python-spanner
if [[ -z "${PROJECT_ROOT:-}" ]]; then
PROJECT_ROOT="github/python-spanner"
fi

cd "${PROJECT_ROOT}"

# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1
Expand All @@ -33,16 +37,26 @@ export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=true

# Remove old nox
python3.6 -m pip uninstall --yes --quiet nox-automation
python3 -m pip uninstall --yes --quiet nox-automation

# Install nox
python3.6 -m pip install --upgrade --quiet nox
python3.6 -m nox --version
python3 -m pip install --upgrade --quiet nox
python3 -m nox --version

# If this is a continuous build, send the test log to the FlakyBot.
# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
cleanup() {
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot
}
trap cleanup EXIT HUP
fi

# If NOX_SESSION is set, it only runs the specified session,
# otherwise run all the sessions.
if [[ -n "${NOX_SESSION:-}" ]]; then
python3.6 -m nox -s "${NOX_SESSION:-}"
python3 -m nox -s ${NOX_SESSION:-}
else
python3.6 -m nox
python3 -m nox
fi
11 changes: 11 additions & 0 deletions .kokoro/docs/docs-presubmit.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,14 @@ env_vars: {
key: "TRAMPOLINE_IMAGE_UPLOAD"
value: "false"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-spanner/.kokoro/build.sh"
}

# Only run this nox session.
env_vars: {
key: "NOX_SESSION"
value: "docs docfx"
}
11 changes: 11 additions & 0 deletions .kokoro/samples/python3.6/periodic-head.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "INSTALL_LIBRARY_FROM_SOURCE"
value: "True"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
}
11 changes: 11 additions & 0 deletions .kokoro/samples/python3.7/periodic-head.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "INSTALL_LIBRARY_FROM_SOURCE"
value: "True"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
}
11 changes: 11 additions & 0 deletions .kokoro/samples/python3.8/periodic-head.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "INSTALL_LIBRARY_FROM_SOURCE"
value: "True"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
}
28 changes: 28 additions & 0 deletions .kokoro/test-samples-against-head.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/bin/bash
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A customized test runner for samples.
#
# For periodic builds, you can specify this file for testing against head.

# `-e` enables the script to automatically fail when a command fails
# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero
set -eo pipefail
# Enables `**` to include files nested inside sub-folders
shopt -s globstar

cd github/python-spanner

exec .kokoro/test-samples-impl.sh
102 changes: 102 additions & 0 deletions .kokoro/test-samples-impl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
#!/bin/bash
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# `-e` enables the script to automatically fail when a command fails
# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero
set -eo pipefail
# Enables `**` to include files nested inside sub-folders
shopt -s globstar

# Exit early if samples directory doesn't exist
if [ ! -d "./samples" ]; then
echo "No tests run. `./samples` not found"
exit 0
fi

# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1

# Debug: show build environment
env | grep KOKORO

# Install nox
python3.6 -m pip install --upgrade --quiet nox

# Use secrets acessor service account to get secrets
if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
gcloud auth activate-service-account \
--key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \
--project="cloud-devrel-kokoro-resources"
fi

# This script will create 3 files:
# - testing/test-env.sh
# - testing/service-account.json
# - testing/client-secrets.json
./scripts/decrypt-secrets.sh

source ./testing/test-env.sh
export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json

# For cloud-run session, we activate the service account for gcloud sdk.
gcloud auth activate-service-account \
--key-file "${GOOGLE_APPLICATION_CREDENTIALS}"

export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json

echo -e "\n******************** TESTING PROJECTS ********************"

# Switch to 'fail at end' to allow all tests to complete before exiting.
set +e
# Use RTN to return a non-zero value if the test fails.
RTN=0
ROOT=$(pwd)
# Find all requirements.txt in the samples directory (may break on whitespace).
for file in samples/**/requirements.txt; do
cd "$ROOT"
# Navigate to the project folder.
file=$(dirname "$file")
cd "$file"

echo "------------------------------------------------------------"
echo "- testing $file"
echo "------------------------------------------------------------"

# Use nox to execute the tests for the project.
python3.6 -m nox -s "$RUN_TESTS_SESSION"
EXIT=$?

# If this is a periodic build, send the test log to the FlakyBot.
# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot
fi

if [[ $EXIT -ne 0 ]]; then
RTN=1
echo -e "\n Testing failed: Nox returned a non-zero exit code. \n"
else
echo -e "\n Testing completed.\n"
fi

done
cd "$ROOT"

# Workaround for Kokoro permissions issue: delete secrets
rm testing/{test-env.sh,client-secrets.json,service-account.json}

exit "$RTN"
96 changes: 16 additions & 80 deletions .kokoro/test-samples.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# The default test runner for samples.
#
# For periodic builds, we rewinds the repo to the latest release, and
# run test-samples-impl.sh.

# `-e` enables the script to automatically fail when a command fails
# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero
Expand All @@ -24,87 +28,19 @@ cd github/python-spanner

# Run periodic samples tests at latest release
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
# preserving the test runner implementation.
cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh"
echo "--- IMPORTANT IMPORTANT IMPORTANT ---"
echo "Now we rewind the repo back to the latest release..."
LATEST_RELEASE=$(git describe --abbrev=0 --tags)
git checkout $LATEST_RELEASE
fi

# Exit early if samples directory doesn't exist
if [ ! -d "./samples" ]; then
echo "No tests run. `./samples` not found"
exit 0
fi

# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1

# Debug: show build environment
env | grep KOKORO

# Install nox
python3.6 -m pip install --upgrade --quiet nox

# Use secrets acessor service account to get secrets
if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
gcloud auth activate-service-account \
--key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \
--project="cloud-devrel-kokoro-resources"
fi

# This script will create 3 files:
# - testing/test-env.sh
# - testing/service-account.json
# - testing/client-secrets.json
./scripts/decrypt-secrets.sh

source ./testing/test-env.sh
export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json

# For cloud-run session, we activate the service account for gcloud sdk.
gcloud auth activate-service-account \
--key-file "${GOOGLE_APPLICATION_CREDENTIALS}"

export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json

echo -e "\n******************** TESTING PROJECTS ********************"

# Switch to 'fail at end' to allow all tests to complete before exiting.
set +e
# Use RTN to return a non-zero value if the test fails.
RTN=0
ROOT=$(pwd)
# Find all requirements.txt in the samples directory (may break on whitespace).
for file in samples/**/requirements.txt; do
cd "$ROOT"
# Navigate to the project folder.
file=$(dirname "$file")
cd "$file"

echo "------------------------------------------------------------"
echo "- testing $file"
echo "------------------------------------------------------------"

# Use nox to execute the tests for the project.
python3.6 -m nox -s "$RUN_TESTS_SESSION"
EXIT=$?

# If this is a periodic build, send the test log to the FlakyBot.
# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot
echo "The current head is: "
echo $(git rev-parse --verify HEAD)
echo "--- IMPORTANT IMPORTANT IMPORTANT ---"
# move back the test runner implementation if there's no file.
if [ ! -f .kokoro/test-samples-impl.sh ]; then
cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh
fi
fi

if [[ $EXIT -ne 0 ]]; then
RTN=1
echo -e "\n Testing failed: Nox returned a non-zero exit code. \n"
else
echo -e "\n Testing completed.\n"
fi

done
cd "$ROOT"

# Workaround for Kokoro permissions issue: delete secrets
rm testing/{test-env.sh,client-secrets.json,service-account.json}

exit "$RTN"
exec .kokoro/test-samples-impl.sh
17 changes: 17 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.4.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- repo: https://github.com/psf/black
rev: 19.10b0
hooks:
- id: black
- repo: https://gitlab.com/pycqa/flake8
rev: 3.9.0
hooks:
- id: flake8
Loading