diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 09d23a106b..f6ef6f1609 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -15,26 +15,68 @@ # build from the root of this repo: FROM gcr.io/cloud-devrel-public-resources/python -# install tools +ARG SYNTHTOOL_COMMITTISH=63cc541da2c45fcfca2136c43e638da1fbae174d +ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 +ENV HOME=/home + +# install OS tools RUN apt-get update && apt-get install -y \ unzip openjdk-17-jdk rsync maven jq \ && apt-get clean -COPY library_generation /src - +# use python 3.11 (the base image has several python versions; here we define the default one) RUN rm $(which python3) RUN ln -s $(which python3.11) /usr/local/bin/python RUN ln -s $(which python3.11) /usr/local/bin/python3 RUN python -m pip install --upgrade pip -RUN cd /src && python -m pip install -r requirements.txt -RUN cd /src && python -m pip install . -# set dummy git credentials for empty commit used in postprocessing -RUN git config --global user.email "cloud-java-bot@google.com" -RUN git config --global user.name "Cloud Java Bot" +# copy source code +COPY library_generation /src -WORKDIR /workspace -RUN chmod 750 /workspace -RUN chmod 750 /src/generate_repo.py +# install scripts as a python package +WORKDIR /src +RUN python -m pip install -r requirements.txt +RUN python -m pip install . + +# install synthtool +WORKDIR /tools +RUN git clone https://github.com/googleapis/synthtool +WORKDIR /tools/synthtool +RUN git checkout "${SYNTHTOOL_COMMITTISH}" +RUN python3 -m pip install --no-deps -e . +RUN python3 -m pip install -r requirements.in -CMD [ "/src/generate_repo.py" ] +# Install nvm with node and npm +ENV NODE_VERSION 20.12.0 +WORKDIR /home +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash +RUN chmod o+rx /home/.nvm +ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin +ENV PATH=${PATH}:${NODE_PATH} +RUN node --version +RUN npm --version + +# install the owl-bot CLI +WORKDIR /tools +RUN git clone https://github.com/googleapis/repo-automation-bots +WORKDIR /tools/repo-automation-bots/packages/owl-bot +RUN git checkout "${OWLBOT_CLI_COMMITTISH}" +RUN npm i && npm run compile && npm link +RUN owl-bot copy-code --version +RUN chmod -R o+rx ${NODE_PATH} +RUN ln -sf ${NODE_PATH}/* /usr/local/bin + +# allow users to access the script folders +RUN chmod -R o+rx /src + +# set dummy git credentials for the empty commit used in postprocessing +# we use system so all users using the container will use this configuration +RUN git config --system user.email "cloud-java-bot@google.com" +RUN git config --system user.name "Cloud Java Bot" + +# allow read-write for /home and execution for binaries in /home/.nvm +RUN chmod -R a+rw /home +RUN chmod -R a+rx /home/.nvm + +WORKDIR /workspace +ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ] diff --git a/.github/workflows/anaylze_dependency.yaml b/.github/workflows/analyze_dependency.yaml similarity index 98% rename from .github/workflows/anaylze_dependency.yaml rename to .github/workflows/analyze_dependency.yaml index 1030e469d1..22b350d53c 100644 --- a/.github/workflows/anaylze_dependency.yaml +++ b/.github/workflows/analyze_dependency.yaml @@ -25,7 +25,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: temurin - java-version: 11 + java-version: 17 cache: maven - name: Set up Maven uses: stCarolas/setup-maven@v4.5 diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index c51f8ed7b0..8feec8a9d6 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -3,14 +3,38 @@ on: branches: - main pull_request: - paths: - - library_generation/** workflow_dispatch: name: verify_library_generation jobs: - integration_tests: + should-run-library-generation-tests: runs-on: ubuntu-22.04 + outputs: + should_run: ${{ steps.get_changed_directories.outputs.should_run }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: get changed directories in the pull request + id: get_changed_directories + shell: bash + run: | + set -ex + git checkout "${base_ref}" + git checkout "${head_ref}" + changed_directories="$(git diff --name-only ${base_ref} ${head_ref})" + if [[ ${changed_directories} =~ "library_generation/" ]]; then + echo "should_run=true" >> $GITHUB_OUTPUT + else + echo "should_run=false" >> $GITHUB_OUTPUT + fi + env: + base_ref: ${{ github.event.pull_request.base.ref }} + head_ref: ${{ github.event.pull_request.head.ref }} + library-generation-integration-tests: + runs-on: ubuntu-22.04 + needs: should-run-library-generation-tests + if: needs.should-run-library-generation-tests.outputs.should_run == 'true' steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -41,12 +65,14 @@ jobs: run: | set -x python -m unittest library_generation/test/integration_tests.py - unit_tests: + library-generation-unit-tests: + runs-on: ${{ matrix.os }} + needs: should-run-library-generation-tests + if: needs.should-run-library-generation-tests.outputs.should_run == 'true' strategy: matrix: java: [ 8 ] os: [ ubuntu-22.04, macos-12 ] - runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: install utils (macos) @@ -88,8 +114,10 @@ jobs: run: | set -x python -m unittest discover -s library_generation/test/ -p "*unit_tests.py" - lint-shell: + library-generation-lint-shell: runs-on: ubuntu-22.04 + needs: should-run-library-generation-tests + if: needs.should-run-library-generation-tests.outputs.should_run == 'true' steps: - uses: actions/checkout@v4 - name: Run ShellCheck @@ -98,8 +126,10 @@ jobs: scandir: 'library_generation' format: tty severity: error - lint-python: + library-generation-lint-python: runs-on: ubuntu-22.04 + needs: should-run-library-generation-tests + if: needs.should-run-library-generation-tests.outputs.should_run == 'true' steps: - uses: actions/checkout@v4 - name: install python dependencies diff --git a/gapic-generator-java/pom.xml b/gapic-generator-java/pom.xml index 923191445c..5b1145d0bd 100644 --- a/gapic-generator-java/pom.xml +++ b/gapic-generator-java/pom.xml @@ -351,7 +351,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.5.2 + 3.5.3 package diff --git a/gax-java/dependencies.properties b/gax-java/dependencies.properties index 6519fdfe96..e76c827179 100644 --- a/gax-java/dependencies.properties +++ b/gax-java/dependencies.properties @@ -83,5 +83,5 @@ maven.org_mockito_mockito_core=org.mockito:mockito-core:2.28.2 maven.org_hamcrest_hamcrest_core=org.hamcrest:hamcrest-core:1.3 maven.com_google_truth_truth=com.google.truth:truth:1.4.2 maven.com_googlecode_java_diff_utils_diffutils=com.googlecode.java-diff-utils:diffutils:1.3.0 -maven.net_bytebuddy_byte_buddy=net.bytebuddy:byte-buddy:1.14.13 +maven.net_bytebuddy_byte_buddy=net.bytebuddy:byte-buddy:1.14.14 maven.org_objenesis_objenesis=org.objenesis:objenesis:2.6 diff --git a/gax-java/gax/src/main/java/com/google/api/gax/core/InstantiatingExecutorProvider.java b/gax-java/gax/src/main/java/com/google/api/gax/core/InstantiatingExecutorProvider.java index 1faee91a4a..25c8213f02 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/core/InstantiatingExecutorProvider.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/core/InstantiatingExecutorProvider.java @@ -34,6 +34,8 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; /** * InstantiatingChannelProvider is an ExecutorProvider which constructs a new @@ -41,6 +43,9 @@ */ @AutoValue public abstract class InstantiatingExecutorProvider implements ExecutorProvider { + private static final Logger LOGGER = + Logger.getLogger(InstantiatingExecutorProvider.class.getName()); + // Thread factory to use to create our worker threads private static final ThreadFactory DEFAULT_THREAD_FACTORY = new ThreadFactory() { @@ -95,6 +100,8 @@ public static Builder newBuilder() { public static Builder newIOBuilder() { int numCpus = Runtime.getRuntime().availableProcessors(); int numThreads = IO_THREAD_MULTIPLIER * Math.max(MIN_THREAD_AMOUNT, numCpus); + LOGGER.log( + Level.CONFIG, String.format("Thread Pool for requests has Core Pool Size: %d", numThreads)); return new AutoValue_InstantiatingExecutorProvider.Builder() .setExecutorThreadCount(numThreads) diff --git a/library_generation/DEVELOPMENT.md b/library_generation/DEVELOPMENT.md new file mode 100644 index 0000000000..cede089ae9 --- /dev/null +++ b/library_generation/DEVELOPMENT.md @@ -0,0 +1,171 @@ +> [!IMPORTANT] +> All examples assume you are inside the `library_generation` folder + + +# Linting + +When contributing, ensure your changes to python code have a valid +format. + +``` +python -m pip install black +black . +``` + +# Running the integration tests + +The integration tests build the docker image declared in +`.cloudbuild/library_generation/library_generation.Dockerfile`, pull GAPIC +repositories, generate the libraries and compares the results with the source +code declared in a "golden branch" of the repo. + +It requires docker and python 3.x to be installed. + +``` +python -m pip install . +python -m pip install -r requirements.txt +python -m unittest test/integration_tests.py +``` + +# Running the unit tests + +The unit tests of the hermetic build scripts are contained in several scripts, +corresponding to a specific component. Every unit test script ends with +`unit_tests.py`. To avoid them specifying them +individually, we can use the following command: + +```bash +python -m unittest discover -s test/ -p "*unit_tests.py" +``` + +> [!NOTE] +> The output of this command may look erratic during the first 30 seconds. +> This is normal. After the tests are done, an "OK" message should be shown. + +# Running the scripts in your local environment + +Although the scripts are designed to be run in a Docker container, you can also +run them directly. This section explains how to run the entrypoint script +(`library_generation/cli/entry_point.py`). + +## Installing prerequisites + +In order to run the generation scripts directly, there are a few tools we +need to install beforehand. + +### Install synthtool + +It requires python 3.x to be installed. +You will need to specify a committish of the synthtool repo in order to have +your generation results matching exactly what the docker image would produce. +You can achieve this by inspecting `SYNTHTOOL_COMMITISH` in +`.cloudbuild/library_generation/library_generation.Dockerfile`. + +```bash +# obtained from .cloudbuild/library_generation/library_generation.Dockerfile +export SYNTHTOOL_COMMITTISH=6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 +``` + +```bash +git clone https://github.com/googleapis/synthtool +cd synthtool +git checkout "${SYNTHTOOL_COMMITTISH}" +python -m pip install --require-hashes -r requirements.txt +python -m pip install --no-deps -e . +python -m synthtool --version +``` + +### Install the owl-bot CLI + +Requires node.js to be installed. +Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script) +for NVM, Node.js's version manager. + +After you install it, you can install the owl-bot CLI with the following +commands: +```bash +git clone https://github.com/googleapis/repo-automation-bots +cd repo-automation-bots/packages/owl-bot +npm i && npm run compile && npm link +owl-bot copy-code --version +``` + +The key step is `npm link`, which will make the command available in you current +shell session. + +## Running the script +The entrypoint script (`library_generation/cli/entry_point.py`) allows you to +update the target repository with the latest changes starting from the +googleapis committish declared in `generation_config.yaml`. + +### Download the repo +For example, google-cloud-java +``` +git clone https://github.com/googleapis/google-cloud-java +export path_to_repo="$(pwd)/google-cloud-java" +``` + +### Install the scripts +``` +python -m pip install . +``` + +### Run the script +``` +python cli/entry_point.py generate --repository-path="${path_to_repo}" +``` + + +# Running the scripts using the docker container image +This is convenient in order to avoid installing the dependencies manually. + +> [!IMPORTANT] +> From now, the examples assume you are in the root of your sdk-platform-java +> folder + +## Build the docker image +```bash +docker build --file .cloudbuild/library_generation/library_generation.Dockerfile --iidfile image-id . +``` + +This will create an `image-id` file at the root of the repo with the hash ID of +the image. + +## Run the docker image +The docker image will perform changes on its internal `/workspace` folder, to which you +need to map a folder on your host machine (i.e. map your downloaded repo to this +folder). + +To run the docker container on the google-cloud-java repo, you must run: +```bash +docker run -u "$(id -u)":"$(id -g)" -v/path/to/google-cloud-java:/workspace $(cat image-id) +``` + + * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating + yourself. This avoids folder ownership changes since it runs as root by + default. + * `-v/path/to/google-cloud-java:/workspace` maps the host machine's + google-cloud-java folder to the /workspace folder. The image is configured to + perform changes in this directory + * `$(cat image-id)` obtains the image ID created in the build step + +## Debug the created containers +If you are working on changing the way the containers are created, you may want +to inspect the containers to check the setup. It would be convenient in such +case to have a text editor/viewer available. You can achieve this by modifying +the Dockerfile as follows: + +```docker +# install OS tools +RUN apt-get update && apt-get install -y \ + unzip openjdk-17-jdk rsync maven jq less vim \ + && apt-get clean +``` + +We add `less` and `vim` as text tools for further inspection. + +You can also run a shell in a new container by running: + +```bash +docker run --rm -it -u=$(id -u):$(id -g) -v/path/to/google-cloud-java:/workspace --entrypoint="bash" $(cat image-id) +``` diff --git a/library_generation/README.md b/library_generation/README.md index 4a51f665ef..b94fe2de53 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -98,8 +98,6 @@ They are shared by library level parameters. | grpc_version | No | inferred from the generator if not specified | | googleapis-commitish | Yes | | | libraries_bom_version | Yes | | -| owlbot-cli-image | Yes | | -| synthtool-commitish | Yes | | | template_excludes | Yes | | ### Library level parameters @@ -149,8 +147,6 @@ gapic_generator_version: 2.34.0 protoc_version: 25.2 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 libraries_bom_version: 26.37.0 -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 destination_path: google-cloud-java template_excludes: - ".github/*" diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index a2f8d9f401..ae5b535e3e 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -112,8 +112,6 @@ def generate_composed_library( "", versions_file, owlbot_cli_source_folder, - config.owlbot_cli_image, - config.synthtool_commitish, str(config.is_monorepo()).lower(), config_path, ], diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index 2a5d453981..bc9c5e3f20 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -32,8 +32,6 @@ def __init__( gapic_generator_version: str, googleapis_commitish: str, libraries_bom_version: str, - owlbot_cli_image: str, - synthtool_commitish: str, template_excludes: list[str], libraries: list[LibraryConfig], grpc_version: Optional[str] = None, @@ -42,8 +40,6 @@ def __init__( self.gapic_generator_version = gapic_generator_version self.googleapis_commitish = googleapis_commitish self.libraris_bom_version = libraries_bom_version - self.owlbot_cli_image = owlbot_cli_image - self.synthtool_commitish = synthtool_commitish self.template_excludes = template_excludes self.libraries = libraries self.grpc_version = grpc_version @@ -145,10 +141,6 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig: libraries_bom_version=__required( config, "libraries_bom_version", REPO_LEVEL_PARAMETER ), - owlbot_cli_image=__required(config, "owlbot_cli_image", REPO_LEVEL_PARAMETER), - synthtool_commitish=__required( - config, "synthtool_commitish", REPO_LEVEL_PARAMETER - ), template_excludes=__required(config, "template_excludes", REPO_LEVEL_PARAMETER), libraries=parsed_libraries, ) diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 37f50152ce..789737b2a5 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -70,7 +70,12 @@ echo "Fixing missing license headers..." python3 "${scripts_root}/owlbot/src/fix-license-headers.py" echo "...done" -# ensure formatting on all .java files in the repository +# Ensure formatting on all .java files in the repository. +# Here we manually set the user.home system variable. Unfortunately, Maven +# user.home inference involves the /etc/passwd file (confirmed empirically), +# instead of the presumable $HOME env var, which may not work properly +# when `docker run`ning with the -u flag because we may incur in users +# not registered in the container's /etc/passwd file echo "Reformatting source..." -mvn fmt:format -V --batch-mode --no-transfer-progress +mvn fmt:format -Duser.home="${HOME}" -V --batch-mode --no-transfer-progress echo "...done" diff --git a/library_generation/owlbot/templates/java_library/samples/pom.xml b/library_generation/owlbot/templates/java_library/samples/pom.xml index add08dc502..e653958eb4 100644 --- a/library_generation/owlbot/templates/java_library/samples/pom.xml +++ b/library_generation/owlbot/templates/java_library/samples/pom.xml @@ -38,7 +38,7 @@ org.apache.maven.plugins maven-deploy-plugin - 3.1.1 + 3.1.2 true diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 397716c843..411c93fd08 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -15,12 +15,9 @@ # 3 - versions_file: path to file containing versions to be applied to the poms # 4 - owlbot_cli_source_folder: alternative folder with a structure exactly like # googleapis-gen. It will be used instead of preprocessed_sources_path if -# 5 - owlbot_cli_image_sha: SHA of the image containing the OwlBot CLI -# 6 - synthtool_commitish: Commit SHA of the synthtool repo -# provided -# 7 - is_monorepo: whether this library is a monorepo, which implies slightly +# 5 - is_monorepo: whether this library is a monorepo, which implies slightly # different logic -# 8 - configuration_yaml_path: path to the configuration yaml containing library +# 6 - configuration_yaml_path: path to the configuration yaml containing library # generation information for this library set -exo pipefail scripts_root=$(dirname "$(readlink -f "$0")") @@ -29,15 +26,13 @@ postprocessing_target=$1 preprocessed_sources_path=$2 versions_file=$3 owlbot_cli_source_folder=$4 -owlbot_cli_image_sha=$5 -synthtool_commitish=$6 -is_monorepo=$7 -configuration_yaml_path=$8 +is_monorepo=$5 +configuration_yaml_path=$6 owlbot_yaml_file_name=".OwlBot-hermetic.yaml" source "${scripts_root}"/utils/utilities.sh -declare -a required_inputs=("postprocessing_target" "versions_file" "owlbot_cli_image_sha" "synthtool_commitish" "is_monorepo") +declare -a required_inputs=("postprocessing_target" "versions_file" "is_monorepo") for required_input in "${required_inputs[@]}"; do if [[ -z "${!required_input}" ]]; then echo "missing required ${required_input} argument, please specify one" @@ -84,70 +79,22 @@ else fi # Default values for running copy-code directly from host -repo_bindings="-v ${postprocessing_target}:/workspace" repo_workspace="/workspace" preprocessed_libraries_binding="${owlbot_cli_source_folder}" -# When running docker inside docker, we run into the issue of volume bindings -# being mapped from the host machine to the child container (instead of the -# parent container to child container) because we bind the `docker.sock` socket -# to the parent container (i.e. docker calls use the host's filesystem context) -# see https://serverfault.com/a/819371 -# We solve this by referencing environment variables that will be -# set to produce the correct volume mapping. -# -# The workflow is: to check if we are in a docker container (via passed env var) -# and use managed volumes (docker volume create) instead of bindings -# (-v /path:/other-path). The volume names are also received as env vars. - -if [[ -n "${RUNNING_IN_DOCKER}" ]]; then - set -u # temporarily fail on unset variables - repo_bindings="${REPO_BINDING_VOLUMES}" - set +u - library_name=$(echo "${postprocessing_target}" | rev | cut -d'/' -f1 | rev) - repo_workspace="/workspace/" - if [[ "${is_monorepo}" == "true" ]]; then - monorepo_name=$(echo "${postprocessing_target}" | rev | cut -d'/' -f2 | rev) - repo_workspace+="${monorepo_name}/" - fi - repo_workspace+="${library_name}" -fi +pushd "${postprocessing_target}" -docker run --rm \ - --user "$(id -u)":"$(id -g)" \ - ${repo_bindings} \ - -v "/tmp:/tmp" \ - -w "${repo_workspace}" \ - --env HOME=/tmp \ - gcr.io/cloud-devrel-public-resources/owlbot-cli@"${owlbot_cli_image_sha}" \ - copy-code \ +owl-bot copy-code \ --source-repo-commit-hash=none \ - --source-repo="${preprocessed_libraries_binding}" \ + --source-repo="${owlbot_cli_source_folder}" \ --config-file="${owlbot_yaml_relative_path}" + # clean the custom owlbot yaml if [[ "${is_monorepo}" == "true" ]]; then rm "${postprocessing_target}/.OwlBot.hermetic.yaml" fi -# we clone the synthtool library and manually build it -mkdir -p /tmp/synthtool -pushd /tmp/synthtool - -if [ ! -d "synthtool" ]; then - git clone https://github.com/googleapis/synthtool.git -fi -git config --global --add safe.directory /tmp/synthtool/synthtool -pushd "synthtool" - -git fetch --all -git reset --hard "${synthtool_commitish}" - -python3 -m pip install -e . -python3 -m pip install -r requirements.in -popd # synthtool -popd # temp dir - # run the postprocessor echo 'running owl-bot post-processor' pushd "${postprocessing_target}" diff --git a/library_generation/requirements.txt b/library_generation/requirements.txt index 627f36fd03..be42b1fea3 100644 --- a/library_generation/requirements.txt +++ b/library_generation/requirements.txt @@ -4,7 +4,7 @@ attrs==23.2.0 black==24.3.0 click==8.1.7 gitdb==4.0.11 -GitPython==3.1.41 +GitPython==3.1.43 Jinja2==3.1.3 lxml==5.2.1 MarkupSafe==2.1.5 @@ -14,7 +14,7 @@ pathspec==0.12.1 platformdirs==4.1.0 PyYAML==6.0.1 smmap==5.0.1 -typing==3.10.0.0 +typing==3.7.4.3 parameterized==0.9.0 # used in parameterized test colorlog==6.8.2 watchdog==4.0.0 diff --git a/library_generation/templates/root-pom.xml.j2 b/library_generation/templates/root-pom.xml.j2 index c48e8b71ef..a713cefa2c 100644 --- a/library_generation/templates/root-pom.xml.j2 +++ b/library_generation/templates/root-pom.xml.j2 @@ -25,7 +25,7 @@ org.apache.maven.plugins maven-deploy-plugin - 3.1.1 + 3.1.2 true diff --git a/library_generation/test/generate_pr_description_unit_tests.py b/library_generation/test/generate_pr_description_unit_tests.py index c977728859..959c442805 100644 --- a/library_generation/test/generate_pr_description_unit_tests.py +++ b/library_generation/test/generate_pr_description_unit_tests.py @@ -61,8 +61,6 @@ def test_generate_pr_description_with_same_googleapis_commits(self): gapic_generator_version="", googleapis_commitish=commit_sha, libraries_bom_version="", - owlbot_cli_image="", - synthtool_commitish="", template_excludes=[], grpc_version="", protoc_version="", diff --git a/library_generation/test/generate_repo_unit_tests.py b/library_generation/test/generate_repo_unit_tests.py index a947e29f5e..9269fb08a1 100644 --- a/library_generation/test/generate_repo_unit_tests.py +++ b/library_generation/test/generate_repo_unit_tests.py @@ -46,8 +46,6 @@ def __get_an_empty_generation_config() -> GenerationConfig: gapic_generator_version="", googleapis_commitish="", libraries_bom_version="", - synthtool_commitish="", - owlbot_cli_image="", template_excludes=[], libraries=[], ) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 356bdec447..e3f950300e 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -65,31 +65,25 @@ def test_entry_point_running_in_container(self): config_files = self.__get_config_files(config_dir) for repo, config_file in config_files: config = from_yaml(config_file) + repo_location = f"{output_dir}/{repo}" + config_location = f"{golden_dir}/../{repo}" # 1. pull repository repo_dest = self.__pull_repo_to( - Path(f"{output_dir}/{repo}"), repo, committish_map[repo] + Path(repo_location), repo, committish_map[repo] ) # 2. prepare golden files library_names = self.__get_library_names_from_config(config) self.__prepare_golden_files( config=config, library_names=library_names, repo_dest=repo_dest ) - # 3. bind repository and configuration to docker volumes - self.__bind_device_to_volumes( - volume_name=f"repo-{repo}", device_dir=f"{output_dir}/{repo}" - ) - self.__bind_device_to_volumes( - volume_name=f"config-{repo}", device_dir=f"{golden_dir}/../{repo}" - ) - repo_volumes = f"-v repo-{repo}:/workspace/{repo} -v config-{repo}:/workspace/config-{repo}" - # 4. run entry_point.py in docker container + # 3. run entry_point.py in docker container self.__run_entry_point_in_docker_container( - repo=repo, - repo_volumes=repo_volumes, + repo_location=repo_location, + config_location=config_location, baseline_config=baseline_config_name, current_config=current_config_name, ) - # 5. compare generation result with golden files + # 4. compare generation result with golden files print( "Generation finished successfully. " "Will now compare differences between generated and existing " @@ -233,71 +227,35 @@ def __prepare_golden_files( else: copy_tree(f"{repo_dest}", f"{golden_dir}/{library_name}") - @classmethod - def __bind_device_to_volumes(cls, volume_name: str, device_dir: str): - # We use a volume to hold the repositories used in the integration - # tests. This is because the test container creates a child container - # using the host machine's docker socket, meaning that we can only - # reference volumes created from within the host machine (i.e. the - # machine running this script). - # - # To summarize, we create a special volume that can be referenced both - # in the main container and in any child containers created by this one. - - # use subprocess.run because we don't care about the return value (we - # want to remove the volume in any case). - subprocess.run(["docker", "volume", "rm", volume_name]) - subprocess.check_call( - [ - "docker", - "volume", - "create", - "--name", - volume_name, - "--opt", - "type=none", - "--opt", - f"device={device_dir}", - "--opt", - "o=bind", - ] - ) - @classmethod def __run_entry_point_in_docker_container( cls, - repo: str, - repo_volumes: str, + repo_location: str, + config_location: str, baseline_config: str, current_config: str, ): + # we use the calling user to prevent the mapped volumes from changing + # owners + user_id = shell_call("id -u") + group_id = shell_call("id -g") subprocess.check_call( [ "docker", "run", + "-u", + f"{user_id}:{group_id}", "--rm", "-v", - f"repo-{repo}:/workspace/{repo}", - "-v", - f"config-{repo}:/workspace/config-{repo}", - "-v", - "/tmp:/tmp", + f"{repo_location}:/workspace/repo", "-v", - "/var/run/docker.sock:/var/run/docker.sock", - "-e", - "RUNNING_IN_DOCKER=true", - "-e", - f"REPO_BINDING_VOLUMES={repo_volumes}", + f"{config_location}:/workspace/config", "-w", - "/src", + "/workspace/repo", image_tag, - "python", - "/src/cli/entry_point.py", - "generate", - f"--baseline-generation-config-path=/workspace/config-{repo}/{baseline_config}", - f"--current-generation-config-path=/workspace/config-{repo}/{current_config}", - f"--repository-path=/workspace/{repo}", - ] + f"--baseline-generation-config-path=/workspace/config/{baseline_config}", + f"--current-generation-config-path=/workspace/config/{current_config}", + ], ) @classmethod diff --git a/library_generation/test/model/config_change_unit_tests.py b/library_generation/test/model/config_change_unit_tests.py index 323edd3cc0..8349a7e1fa 100644 --- a/library_generation/test/model/config_change_unit_tests.py +++ b/library_generation/test/model/config_change_unit_tests.py @@ -240,8 +240,6 @@ def __get_a_gen_config( gapic_generator_version="", googleapis_commitish=googleapis_commitish, libraries_bom_version="", - owlbot_cli_image="", - synthtool_commitish="", template_excludes=[], grpc_version="", protoc_version="", diff --git a/library_generation/test/model/generation_config_unit_test.py b/library_generation/test/model/generation_config_unit_test.py index 2c8b3fc93d..d9efb494c0 100644 --- a/library_generation/test/model/generation_config_unit_test.py +++ b/library_generation/test/model/generation_config_unit_test.py @@ -46,13 +46,6 @@ def test_from_yaml_succeeds(self): "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish ) self.assertEqual("26.37.0", config.libraris_bom_version) - self.assertEqual( - "sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409", - config.owlbot_cli_image, - ) - self.assertEqual( - "6612ab8f3afcd5e292aecd647f0fa68812c9f5b5", config.synthtool_commitish - ) self.assertEqual( [ ".github/*", @@ -115,8 +108,6 @@ def test_is_monorepo_with_one_library_returns_false(self): gapic_generator_version="", googleapis_commitish="", libraries_bom_version="", - owlbot_cli_image="", - synthtool_commitish="", template_excludes=[], libraries=[library_1], ) @@ -127,8 +118,6 @@ def test_is_monorepo_with_two_libraries_returns_true(self): gapic_generator_version="", googleapis_commitish="", libraries_bom_version="", - owlbot_cli_image="", - synthtool_commitish="", template_excludes=[], libraries=[library_1, library_2], ) diff --git a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml index 6a100856b7..2b13ebb50f 100644 --- a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml @@ -2,8 +2,6 @@ gapic_generator_version: 2.38.1 protoc_version: 25.2 googleapis_commitish: a17d4caf184b050d50cacf2b0d579ce72c31ce74 libraries_bom_version: 26.37.0 -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 63cc541da2c45fcfca2136c43e638da1fbae174d template_excludes: - ".github/*" - ".kokoro/*" @@ -56,4 +54,4 @@ libraries: api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. GAPICs: - proto_path: google/cloud/cloudcontrolspartner/v1 - - proto_path: google/cloud/cloudcontrolspartner/v1beta \ No newline at end of file + - proto_path: google/cloud/cloudcontrolspartner/v1beta diff --git a/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml index e34916a344..995ab31eb9 100644 --- a/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml @@ -2,8 +2,6 @@ gapic_generator_version: 2.38.1 protoc_version: 25.2 googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc libraries_bom_version: 26.37.0 -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 63cc541da2c45fcfca2136c43e638da1fbae174d template_excludes: - ".github/*" - ".kokoro/*" @@ -56,4 +54,4 @@ libraries: api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. GAPICs: - proto_path: google/cloud/cloudcontrolspartner/v1 - - proto_path: google/cloud/cloudcontrolspartner/v1beta \ No newline at end of file + - proto_path: google/cloud/cloudcontrolspartner/v1beta diff --git a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml index f1eb4948d5..784206521f 100644 --- a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml +++ b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml @@ -1,8 +1,6 @@ gapic_generator_version: 2.37.0 protoc_version: 25.2 googleapis_commitish: 9868a57470a969ffa1d21194a5c05d7a6e4e98cc -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: a6fb7d5f072b75698af1cbf06c5b001565753cfb template_excludes: - ".gitignore" - ".kokoro/presubmit/integration.cfg" diff --git a/library_generation/test/resources/test-config/config_without_owlbot.yaml b/library_generation/test/resources/test-config/config_without_owlbot.yaml deleted file mode 100644 index 0d1bb7deea..0000000000 --- a/library_generation/test/resources/test-config/config_without_owlbot.yaml +++ /dev/null @@ -1,10 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/config_without_synthtool.yaml b/library_generation/test/resources/test-config/config_without_synthtool.yaml deleted file mode 100644 index 820c53032e..0000000000 --- a/library_generation/test/resources/test-config/config_without_synthtool.yaml +++ /dev/null @@ -1,11 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/config_without_temp_excludes.yaml b/library_generation/test/resources/test-config/config_without_temp_excludes.yaml index 54dab6449d..0d1bb7deea 100644 --- a/library_generation/test/resources/test-config/config_without_temp_excludes.yaml +++ b/library_generation/test/resources/test-config/config_without_temp_excludes.yaml @@ -1,8 +1,6 @@ gapic_generator_version: 2.34.0 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 libraries_bom_version: 26.37.0 -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 libraries: - api_shortname: apigeeconnect name_pretty: Apigee Connect diff --git a/library_generation/test/resources/test-config/generation_config.yaml b/library_generation/test/resources/test-config/generation_config.yaml index 8930e982da..6bd83e20d2 100644 --- a/library_generation/test/resources/test-config/generation_config.yaml +++ b/library_generation/test/resources/test-config/generation_config.yaml @@ -2,8 +2,6 @@ gapic_generator_version: 2.34.0 protoc_version: 25.2 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 libraries_bom_version: 26.37.0 -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 template_excludes: - ".github/*" - ".kokoro/*" diff --git a/library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml b/library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml index 8347287bc3..44a1df9409 100644 --- a/library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml +++ b/library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml @@ -25,7 +25,7 @@ org.apache.maven.plugins maven-deploy-plugin - 3.1.1 + 3.1.2 true diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index 7e8eccbcf8..736eba26e4 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -372,8 +372,6 @@ def __get_a_gen_config( gapic_generator_version="", googleapis_commitish="", libraries_bom_version="", - owlbot_cli_image="", - synthtool_commitish="", template_excludes=[ ".github/*", ".kokoro/*", diff --git a/library_generation/test/utils/generation_config_comparator_unit_tests.py b/library_generation/test/utils/generation_config_comparator_unit_tests.py index 04d9661fcd..fa466ac34d 100644 --- a/library_generation/test/utils/generation_config_comparator_unit_tests.py +++ b/library_generation/test/utils/generation_config_comparator_unit_tests.py @@ -40,8 +40,6 @@ def setUp(self) -> None: gapic_generator_version="", googleapis_commitish="", libraries_bom_version="", - owlbot_cli_image="", - synthtool_commitish="", template_excludes=[], grpc_version="", protoc_version="", @@ -51,8 +49,6 @@ def setUp(self) -> None: gapic_generator_version="", googleapis_commitish="", libraries_bom_version="", - owlbot_cli_image="", - synthtool_commitish="", template_excludes=[], grpc_version="", protoc_version="", @@ -107,34 +103,6 @@ def test_compare_config_libraries_bom_update(self): self.assertEqual("libraris_bom_version", config_change.changed_param) self.assertEqual("26.37.0", config_change.current_value) - def test_compare_config_owlbot_cli_update(self): - self.baseline_config.owlbot_cli_image = "image_version_123" - self.current_config.owlbot_cli_image = "image_version_456" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("owlbot_cli_image", config_change.changed_param) - self.assertEqual("image_version_456", config_change.current_value) - - def test_compare_config_synthtool_update(self): - self.baseline_config.synthtool_commitish = "commit123" - self.current_config.synthtool_commitish = "commit456" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("synthtool_commitish", config_change.changed_param) - self.assertEqual("commit456", config_change.current_value) - def test_compare_protobuf_update(self): self.baseline_config.protoc_version = "3.25.2" self.current_config.protoc_version = "3.27.0" diff --git a/pom.xml b/pom.xml index bb18ea81cd..a26c57a42a 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ org.apache.maven.plugins maven-deploy-plugin - 3.1.1 + 3.1.2 true