From 52ce03252f96783106ee79bc9de0ac61c86ea038 Mon Sep 17 00:00:00 2001 From: Daniel Azuma Date: Wed, 7 Apr 2021 11:05:30 -0700 Subject: [PATCH] chore: Update release scripts to use Trampoline V2 and Secret Manager (#3649) --- .kokoro/build.bat | 11 - .kokoro/build.sh | 4 - .kokoro/osx.sh | 4 - .kokoro/populate-secrets.sh | 76 ++++ .kokoro/release.cfg | 94 ---- .kokoro/release.sh | 21 + .kokoro/release/common.cfg | 51 +-- .kokoro/trampoline.bat | 10 - .kokoro/trampoline.sh | 4 - .kokoro/trampoline_v2.sh | 489 +++++++++++++++++++++ {rakelib => .toys/kokoro/.lib}/releaser.rb | 70 ++- .toys/kokoro/release-all-generated.rb | 34 ++ .toys/kokoro/release.rb | 28 ++ .trampolinerc | 48 ++ Rakefile | 95 ---- rakelib/devsite/devsite_builder.rb | 126 ------ rakelib/devsite/link_checker.rb | 64 --- rakelib/devsite/repo_metadata.rb | 56 --- 18 files changed, 758 insertions(+), 527 deletions(-) delete mode 100644 .kokoro/build.bat delete mode 100755 .kokoro/build.sh delete mode 100644 .kokoro/osx.sh create mode 100644 .kokoro/populate-secrets.sh delete mode 100644 .kokoro/release.cfg create mode 100755 .kokoro/release.sh delete mode 100644 .kokoro/trampoline.bat delete mode 100644 .kokoro/trampoline.sh create mode 100644 .kokoro/trampoline_v2.sh rename {rakelib => .toys/kokoro/.lib}/releaser.rb (64%) create mode 100644 .toys/kokoro/release-all-generated.rb create mode 100644 .toys/kokoro/release.rb create mode 100644 .trampolinerc delete mode 100644 Rakefile delete mode 100644 rakelib/devsite/devsite_builder.rb delete mode 100644 rakelib/devsite/link_checker.rb delete mode 100644 rakelib/devsite/repo_metadata.rb diff --git a/.kokoro/build.bat b/.kokoro/build.bat deleted file mode 100644 index c76f2bc2fce..00000000000 --- a/.kokoro/build.bat +++ /dev/null @@ -1,11 +0,0 @@ -REM This file runs tests for merges, PRs, and nightlies. - -SET url="https://raw.githubusercontent.com/googleapis/google-cloud-ruby/master/.kokoro/build.bat" - -SET "download=powershell -C Invoke-WebRequest -Uri %url% -OutFile master-build.bat" - -SET EXIT_STATUS=1 - -%download% && master-build.bat && SET EXIT_STATUS=0 - -EXIT %EXIT_STATUS% diff --git a/.kokoro/build.sh b/.kokoro/build.sh deleted file mode 100755 index 7f4e00fbb9e..00000000000 --- a/.kokoro/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -script_url="https://raw.githubusercontent.com/googleapis/google-cloud-ruby/master/.kokoro/build.sh" -curl -o master-build.sh $script_url && source master-build.sh diff --git a/.kokoro/osx.sh b/.kokoro/osx.sh deleted file mode 100644 index 8be8ec05445..00000000000 --- a/.kokoro/osx.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -script_url="https://raw.githubusercontent.com/googleapis/google-cloud-ruby/master/.kokoro/osx.sh" -curl -o master-osx.sh $script_url && source master-osx.sh diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh new file mode 100644 index 00000000000..aab0ec38634 --- /dev/null +++ b/.kokoro/populate-secrets.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Copyright 2020 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is called in the early stage of `trampoline_v2.sh` to +# populate secrets needed for the CI builds. + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + +# Populates requested secrets set in SECRET_MANAGER_KEYS + +# In Kokoro CI builds, we use the service account attached to the +# Kokoro VM. This means we need to setup auth on other CI systems. +# For local run, we just use the gcloud command for retrieving the +# secrets. + +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + GCLOUD_COMMANDS=( + "docker" + "run" + "--entrypoint=gcloud" + "--volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR}" + "gcr.io/google.com/cloudsdktool/cloud-sdk" + ) + if [[ "${TRAMPOLINE_CI:-}" == "kokoro" ]]; then + SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" + else + echo "Authentication for this CI system is not implemented yet." + exit 2 + # TODO: Determine appropriate SECRET_LOCATION and the GCLOUD_COMMANDS. + fi +else + # For local run, use /dev/shm or temporary directory for + # KOKORO_GFILE_DIR. + if [[ -d "/dev/shm" ]]; then + export KOKORO_GFILE_DIR=/dev/shm + else + export KOKORO_GFILE_DIR=$(mktemp -d -t ci-XXXXXXXX) + fi + SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" + GCLOUD_COMMANDS=("gcloud") +fi + +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} + +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + "${GCLOUD_COMMANDS[@]}" \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret $key > \ + "$SECRET_LOCATION/$key" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + exit 2 + fi +done diff --git a/.kokoro/release.cfg b/.kokoro/release.cfg deleted file mode 100644 index d7cff25fca5..00000000000 --- a/.kokoro/release.cfg +++ /dev/null @@ -1,94 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Fetch the token needed for reporting release status to GitHub -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "yoshi-automation-github-key" - } - } -} - -# Fetch magictoken to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "releasetool-magictoken" - backend_type: FASTCONFIGPUSH - } - } -} - -# Fetch api key to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "magic-github-proxy-api-key" - backend_type: FASTCONFIGPUSH - } - } -} - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "docuploader_service_account" - } - } -} - -# Download resources for system tests (service account key, etc.) -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-ruby" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "google-api-ruby-client/.kokoro/trampoline.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/release" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/google-api-ruby-client/.kokoro/build.sh" -} - -env_vars: { - key: "TRAMPOLINE_SCRIPT" - value: "trampoline_v1.py" -} - -env_vars: { - key: "JOB_TYPE" - value: "release" -} - -env_vars: { - key: "OS" - value: "linux" -} - -env_vars: { - key: "REPO_DIR" - value: "github/google-api-ruby-client" -} - -env_vars: { - key: "PACKAGE" - value: "google-api-client" -} diff --git a/.kokoro/release.sh b/.kokoro/release.sh new file mode 100755 index 00000000000..1e81b37f867 --- /dev/null +++ b/.kokoro/release.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -eo pipefail + +# Install gems in the user directory because the default install directory +# is in a read-only location. +export GEM_HOME=$HOME/.gem +export PATH=$GEM_HOME/bin:$PATH + +python3 -m pip install git+https://github.com/googleapis/releasetool +python3 -m pip install gcp-docuploader +gem install --no-document toys +bundle install + +if [ "$PACKAGE" = "ALL_GENERATED" ]; then + # This is not called from autorelease, so don't run publish-reporter-script + toys kokoro release-all-generated < /dev/null +else + python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script + toys kokoro release "$PACKAGE" < /dev/null +fi diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 34a76ea43ab..9db4023278e 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -17,28 +17,6 @@ before_action { } } -# Fetch magictoken to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "releasetool-magictoken" - backend_type: FASTCONFIGPUSH - } - } -} - -# Fetch api key to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "magic-github-proxy-api-key" - backend_type: FASTCONFIGPUSH - } - } -} - before_action { fetch_keystore { keystore_resource { @@ -55,35 +33,20 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-ruby" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "google-api-ruby-client/.kokoro/trampoline.sh" +build_file: "google-api-ruby-client/.kokoro/trampoline_v2.sh" # Configure the docker image for kokoro-trampoline. env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/release" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/google-api-ruby-client/.kokoro/build.sh" -} - -env_vars: { - key: "TRAMPOLINE_SCRIPT" - value: "trampoline_v1.py" -} - -env_vars: { - key: "JOB_TYPE" - value: "release" + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/release" } env_vars: { - key: "OS" - value: "linux" + key: "TRAMPOLINE_BUILD_FILE" + value: ".kokoro/release.sh" } env_vars: { - key: "REPO_DIR" - value: "github/google-api-ruby-client" + key: "SECRET_MANAGER_KEYS" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } diff --git a/.kokoro/trampoline.bat b/.kokoro/trampoline.bat deleted file mode 100644 index 1634b0710d0..00000000000 --- a/.kokoro/trampoline.bat +++ /dev/null @@ -1,10 +0,0 @@ - -SET url="https://raw.githubusercontent.com/googleapis/google-cloud-ruby/master/.kokoro/trampoline.bat" - -SET "download=powershell -C Invoke-WebRequest -Uri %url% -OutFile master-trampoline.bat" - -SET EXIT_STATUS=1 - -%download% && master-trampoline.bat && SET EXIT_STATUS=0 - -EXIT %EXIT_STATUS% diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh deleted file mode 100644 index 4325866e0a3..00000000000 --- a/.kokoro/trampoline.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -script_url="https://raw.githubusercontent.com/googleapis/google-cloud-ruby/master/.kokoro/trampoline.sh" -curl -o master-trampoline.sh $script_url && source master-trampoline.sh diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100644 index 00000000000..ef6972b4703 --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,489 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# If you want to make a change to this file, consider doing so at: +# https://github.com/googlecloudplatform/docker-ci-helper +# +# This script is for running CI builds. For Kokoro builds, we +# set this script to `build_file` field in the Kokoro configuration. + +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. +# +# Here is an example for running this script. +# TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:10-user \ +# TRAMPOLINE_BUILD_FILE=.kokoro/system-test.sh \ +# .kokoro/trampoline_v2.sh + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.10" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For Flaky Bot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + "KOKORO_BUILD_ARTIFACTS_SUBDIR" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}" | grep "${TRAMPOLINE_IMAGE%:*}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/rakelib/releaser.rb b/.toys/kokoro/.lib/releaser.rb similarity index 64% rename from rakelib/releaser.rb rename to .toys/kokoro/.lib/releaser.rb index d98a795b995..58065fad990 100644 --- a/rakelib/releaser.rb +++ b/.toys/kokoro/.lib/releaser.rb @@ -3,11 +3,48 @@ require "rubygems" class Releaser + @loaded_env = false + + def self.load_env + return if @loaded_env + + if ::ENV["KOKORO_GFILE_DIR"] + service_account = "#{::ENV['KOKORO_GFILE_DIR']}/service-account.json" + raise "#{service_account} is not a file" unless ::File.file? service_account + ::ENV["GOOGLE_APPLICATION_CREDENTIALS"] = service_account + + filename = "#{::ENV['KOKORO_GFILE_DIR']}/ruby_env_vars.json" + raise "#{filename} is not a file" unless ::File.file? filename + env_vars = ::JSON.parse ::File.read filename + env_vars.each { |k, v| ::ENV[k] ||= v } + end + + if ::ENV["KOKORO_KEYSTORE_DIR"] + ::ENV["DOCS_CREDENTIALS"] ||= "#{::ENV['KOKORO_KEYSTORE_DIR']}/73713_docuploader_service_account" + ::ENV["GITHUB_TOKEN"] ||= "#{::ENV['KOKORO_KEYSTORE_DIR']}/73713_yoshi-automation-github-key" + end + + @loaded_env = true + end + + def self.lookup_current_versions + versions = {} + lines = `gem search '^google-apis-.*'`.split("\n") + lines.each do |line| + if line =~ /^(google-apis-\w+) \(([\d.]+)\)/ + versions[Regexp.last_match[1]] = Regexp.last_match[2] + end + end + raise "Something went wrong getting all current gem versions" if versions.empty? + versions + end + def initialize gem_name, gem_dir, rubygems_api_token: nil, docs_staging_bucket: nil, docuploader_credentials: nil, - dry_run: false + dry_run: false, + current_versions: nil @gem_name = gem_name @gem_dir = File.expand_path(gem_dir) @rubygems_api_token = rubygems_api_token || ENV["RUBYGEMS_API_TOKEN"] @@ -17,6 +54,7 @@ def initialize gem_name, gem_dir, @docuploader_credentials = File.join(ENV["KOKORO_KEYSTORE_DIR"], "73713_docuploader_service_account") end @dry_run = dry_run ? true : false + @current_versions = current_versions || {} @bundle_updated = false end @@ -35,31 +73,33 @@ def needs_gem_publish? end def publish_gem - Dir.chdir(gem_dir) do - FileUtils.rm_rf("pkg") + puts "**** Starting publish_gem for #{gem_name}" + Dir.chdir gem_dir do + FileUtils.rm_rf "pkg" isolate_bundle do execute "bundle exec rake build" end built_gem_path = "pkg/#{gem_name}-#{gem_version}.gem" - raise "Failed to build #{built_gem_path}" unless File.file?(built_gem_path) + raise "Failed to build #{built_gem_path}" unless File.file? built_gem_path if dry_run? puts "**** In dry run mode. Skipping gem publish of #{gem_name}" return end - response = gems_client.push(File.new(built_gem_path)) + response = gems_client.push File.new built_gem_path puts response raise "Failed to publish gem" unless response.include? "Successfully registered gem:" end end def publish_docs - Dir.chdir(gem_dir) do - FileUtils.rm_rf("doc") - FileUtils.rm_rf(".yardoc") + puts "**** Starting publish_docs for #{gem_name}" + Dir.chdir gem_dir do + FileUtils.rm_rf "doc" + FileUtils.rm_rf ".yardoc" isolate_bundle do execute "bundle exec rake yard" end - Dir.chdir("doc") do + Dir.chdir "doc" do execute "python3 -m docuploader create-metadata" \ " --name #{gem_name}" \ " --distribution-name #{gem_name}" \ @@ -83,7 +123,7 @@ def publish_docs def current_rubygems_version @current_rubygems_version ||= begin - gems_client.info(gem_name)["version"] + @current_versions[gem_name] || gems_client.info(gem_name)["version"] rescue Gems::NotFound "0.0.0" end @@ -91,8 +131,8 @@ def current_rubygems_version def gem_version @gem_version ||= begin - version_content = File.read(version_file_path) - match = /\s(?:GEM_)?VERSION = "([\d\.]+)"/.match(version_content) + version_content = File.read version_file_path + match = /\s(?:GEM_)?VERSION = "([\d\.]+)"/.match version_content raise "Unable to find version constant in #{version_file_path}" unless match match[1] end @@ -127,7 +167,7 @@ def isolate_bundle yield end if defined?(Bundler) - if Bundler.respond_to?(:with_unbundled_env) + if Bundler.respond_to? :with_unbundled_env Bundler.with_unbundled_env(&block) else Bundler.with_clean_env(&block) @@ -137,8 +177,8 @@ def isolate_bundle end end - def execute(cmd) + def execute cmd puts cmd - raise "Error executing command" unless system(cmd) + raise "Error executing command" unless system cmd end end diff --git a/.toys/kokoro/release-all-generated.rb b/.toys/kokoro/release-all-generated.rb new file mode 100644 index 00000000000..fdcc6539ddf --- /dev/null +++ b/.toys/kokoro/release-all-generated.rb @@ -0,0 +1,34 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include :exec, e: true +include :gems + +flag :dry_run, default: ::ENV["RELEASE_DRY_RUN"] == "true" + +def run + gem "gems", "~> 1.2" + require "releaser" + Releaser.load_env + current_versions = Releaser.lookup_current_versions + Dir.glob("generated/google-apis-*") do |gem_dir| + gem_name = File.basename gem_dir + puts "Checking #{gem_name}..." + releaser = Releaser.new gem_name, gem_dir, current_versions: current_versions, dry_run: dry_run + if releaser.needs_gem_publish? + releaser.publish_gem + releaser.publish_docs + end + end +end diff --git a/.toys/kokoro/release.rb b/.toys/kokoro/release.rb new file mode 100644 index 00000000000..657db61f5d2 --- /dev/null +++ b/.toys/kokoro/release.rb @@ -0,0 +1,28 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include :exec, e: true +include :gems + +required_arg :package +flag :dry_run, default: ::ENV["RELEASE_DRY_RUN"] == "true" + +def run + gem "gems", "~> 1.2" + require "releaser" + Releaser.load_env + releaser = Releaser.new package, package, dry_run: dry_run + releaser.publish_gem if releaser.needs_gem_publish? + releaser.publish_docs +end diff --git a/.trampolinerc b/.trampolinerc new file mode 100644 index 00000000000..3a9764cad6b --- /dev/null +++ b/.trampolinerc @@ -0,0 +1,48 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add required env vars here. +required_envvars+=( +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + "AUTORELEASE_PR" "RELEASE_DRY_RUN" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE="" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi + +# Secret Manager secrets. +source ${PROJECT_ROOT}/.kokoro/populate-secrets.sh diff --git a/Rakefile b/Rakefile deleted file mode 100644 index ba2872e31d4..00000000000 --- a/Rakefile +++ /dev/null @@ -1,95 +0,0 @@ -require "json" - -namespace :kokoro do - task :load_env_vars do - load_env_vars - end - - task :dry_run do - ENV["DRY_RUN"] = "true" - end - - task :presubmit do - Rake::Task["kokoro:run_tests"].invoke - end - - task :continuous do - Rake::Task["kokoro:run_tests"].invoke - end - - task :nightly do - Rake::Task["kokoro:run_tests"].invoke - end - - task :post do - require_relative "rakelib/devsite/link_checker.rb" - - link_checker = LinkChecker.new - link_checker.run - exit link_checker.exit_status - end - - task :release do - require_relative "rakelib/releaser.rb" - - load_env_vars if ENV["KOKORO_GFILE_DIR"] - package = ENV["PACKAGE"] - raise "PACKAGE missing" unless package - dry_run = ENV["DRY_RUN"] - - if package == "ALL_GENERATED" - Dir.glob("generated/google-apis-*") do |gem_dir| - gem_name = File.basename(gem_dir) - releaser = Releaser.new(gem_name, gem_dir, dry_run: dry_run) - if releaser.needs_gem_publish? - releaser.publish_gem - releaser.publish_docs - end - end - else - releaser = Releaser.new(package, package, dry_run: dry_run) - releaser.publish_gem if releaser.needs_gem_publish? - releaser.publish_docs - end - end - - task :run_tests do - gem_directories = ["google-apis-core", "google-apis-generator"] - gem_directories << "google-api-client" unless RUBY_VERSION.start_with? "3." - gem_directories.each do |dir| - next unless File.file?(File.join(dir, "Gemfile")) - cd dir do - Bundler.with_clean_env do - sh "bundle install" - sh "bundle exec rake spec" - end - end - end - end -end - -def load_env_vars - service_account = "#{ENV['KOKORO_GFILE_DIR']}/service-account.json" - raise "#{service_account} is not a file" unless File.file? service_account - ENV["GOOGLE_APPLICATION_CREDENTIALS"] = service_account - filename = "#{ENV['KOKORO_GFILE_DIR']}/ruby_env_vars.json" - raise "#{filename} is not a file" unless File.file? filename - env_vars = JSON.parse File.read(filename) - env_vars.each do |k, v| - if ENV[k] - puts "Ignoring #{k} because it is already set" - else - puts "Setting #{k}" - end - ENV[k] ||= v - end -end - -def header str, token = "#" - line_length = str.length + 8 - puts - puts token * line_length - puts "#{token * 3} #{str} #{token * 3}" - puts token * line_length - puts -end diff --git a/rakelib/devsite/devsite_builder.rb b/rakelib/devsite/devsite_builder.rb deleted file mode 100644 index 61c23a4b22f..00000000000 --- a/rakelib/devsite/devsite_builder.rb +++ /dev/null @@ -1,126 +0,0 @@ -require "pathname" -require "tmpdir" - -require_relative "repo_metadata.rb" - -class DevsiteBuilder - def initialize build_tag = nil - @build_tag = build_tag || latest_tag - @output_dir = "doc" - end - - def build tag - checkout_tag tag - doc_path = tmp_dir + @output_dir - FileUtils.remove_dir doc_path if Dir.exist? doc_path - markup = "--markup markdown --markup-provider redcarpet" - - Dir.chdir tmp_dir do - cmds = ["-o #{@output_dir}", markup] - cmd "yard --verbose #{cmds.join ' '}" - end - metadata.build doc_path - end - - def upload - Dir.chdir tmp_dir + @output_dir do - opts = [ - "--credentials=#{ENV['KOKORO_KEYSTORE_DIR']}/73713_docuploader_service_account", - "--staging-bucket=#{ENV.fetch 'STAGING_BUCKET', 'docs-staging'}", - "--metadata-file=./docs.metadata" - ] - cmd "python3 -m docuploader upload . #{opts.join ' '}" - end - end - - def publish tag = nil - build(tag || @build_tag) - upload - end - - def publish_if_missing tag = nil - tag ||= @build_tag - puts tag - puts missing? tag - publish tag if missing? tag - end - - def missing? tag - require "httparty" - - url = "https://googleapis.dev/ruby/google-api-client/v#{version tag}/index.html" - response = HTTParty.get url - response.code != 200 - rescue StandardError - true - end - - def cmd line - puts line - output = `#{line}` - puts output - output - end - - def metadata - return @metadata if @metadata - - metadata_json = "#{tmp_dir}/.repo-metadata.json" - @metadata = RepoMetadata.from_source metadata_json if File.file? metadata_json - @metadata ||= RepoMetadata.from_source "name" => "google-api-client", - "distribution-name" => "google-api-client", - "language" => "ruby" - @metadata["version"] = "v#{version @build_tag}" - @metadata - end - - def checkout_tag git_tag - Dir.chdir tmp_dir do - `git checkout tags/#{git_tag} -b v#{version git_tag}` - end - end - - def version git_tag - m = git_tag.match(/(\d+\.\d+\.\d+)/) - return m if m.nil? - m[0] - end - - def versions - Dir.chdir tmp_dir do - tags.map { |t| version t }.reject(&:nil?).sort_by { |v| Gem::Version.new v }.reverse - end - end - - def tags - Dir.chdir tmp_dir do - `git tag`.split "\n" - end - end - - def latest_version - @latest_version ||= versions.first - end - - def latest_tag - @latest_tag ||= tags.select { |t| t.include? latest_version }.min_by(&:size) - end - - def tmp_dir - return @tmp_dir if @tmp_dir - - tmp = Dir.tmpdir - dir_name = "google-api-ruby-client" - @tmp_dir = Pathname.new(tmp) + dir_name - FileUtils.remove_dir @tmp_dir if Dir.exist? @tmp_dir - - Dir.chdir tmp do - `git clone https://github.com/googleapis/google-api-ruby-client.git` - end - Dir.chdir @tmp_dir do - `git fetch` - end - - @tmp_dir - end -end diff --git a/rakelib/devsite/link_checker.rb b/rakelib/devsite/link_checker.rb deleted file mode 100644 index 5842b85774f..00000000000 --- a/rakelib/devsite/link_checker.rb +++ /dev/null @@ -1,64 +0,0 @@ -require "open3" - -class LinkChecker - def initialize - @failed = false - end - - def run - job_info - git_commit = ENV.fetch "KOKORO_GITHUB_COMMIT", "master" - - markdown_files = Dir.glob "**/*.md" - broken_markdown_links = check_links(markdown_files, - "https://github.com/googleapis/google-api-ruby-client/tree/#{git_commit}", - " --skip '^(?!(\\Wruby.*google|.*google.*\\Wruby|.*cloud\\.google\\.com))'") - - broken_devsite_links = check_links(["google-api-client"], - "https://googleapis.dev/ruby", - "/latest/ --recurse --skip https:.*github.*") - - puts_broken_links broken_markdown_links - puts_broken_links broken_devsite_links - end - - def check_links location_list, base, tail - broken_links = Hash.new { |h, k| h[k] = [] } - location_list.each do |location| - out, err, st = Open3.capture3 "npx linkinator #{base}/#{location}#{tail}" - puts out - unless st.to_i.zero? - @failed = true - puts err - end - checked_links = out.split "\n" - checked_links.select! { |link| link =~ /\[\d+\]/ && !link.include?("[200]") } - unless checked_links.empty? - @failed = true - broken_links[location] += checked_links - end - end - broken_links - end - - def puts_broken_links link_hash - link_hash.each do |location, links| - puts "#{location} contains the following broken links:" - links.each { |link| puts " #{link}" } - puts "" - end - end - - def job_info - line_length = "Using Ruby - #{RUBY_VERSION}".length + 8 - puts - puts "#" * line_length - puts "### Using Ruby - #{RUBY_VERSION} ###" - puts "#" * line_length - puts - end - - def exit_status - @failed ? 1 : 0 - end -end diff --git a/rakelib/devsite/repo_metadata.rb b/rakelib/devsite/repo_metadata.rb deleted file mode 100644 index fa06a1fcaec..00000000000 --- a/rakelib/devsite/repo_metadata.rb +++ /dev/null @@ -1,56 +0,0 @@ -require "json" - -class RepoMetadata - attr_accessor :data - - def initialize data - @data = data - normalize_data! - end - - def allowed_fields - [ - "name", "version", "language", "distribution-name", - "product-page", "github-repository", "issue-tracker" - ] - end - - def build output_directory - fields = @data.to_a.map { |kv| "--#{kv[0]} #{kv[1]}" } - Dir.chdir output_directory do - cmd "python3 -m docuploader create-metadata #{fields.join ' '}" - end - end - - def normalize_data! - @data.delete_if { |k, _| !allowed_fields.include?(k) } - end - - def [] key - data[key] - end - - def []= key, value - @data[key] = value - end - - def cmd line - puts line - output = `#{line}` - puts output - output - end - - def self.from_source source - if source.is_a? RepoMetadata - data = source.data - elsif source.is_a? Hash - data = source - elsif File.file? source - data = JSON.parse File.read(source) - else - raise "Source must be a path, hash, or RepoMetadata instance" - end - RepoMetadata.new data - end -end