diff --git a/.gitignore b/.gitignore index 201526885b..05901afab0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,6 @@ target/ # library generation **/output/ **/googleapis -library_generation/test/**/golden*/ library_generation/test/resources/test_monorepo_postprocessing/ **/*egg-info/ **/build/ diff --git a/library_generation/README.md b/library_generation/README.md index c0469d1d9c..dfa013e3d0 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -55,24 +55,24 @@ of versions.txt. For each module (e.g. `google-cloud-java/java-asset`), the following files/folders will be created/modified: -| Name | Notes | -|:----------------------------|:-------------------------------------------------------------------------| -| google-*/ | Source code generated by gapic-generator-java | -| google-*/pom.xml | Only be generated if it does not exist | -| grpc-*/ | Source code generated by grpc generator, one per each version | -| grpc-*/pom.xml | Only be generated if it does not exist | -| proto-*/ | Source code generated by Protobuf default compiler, one per each version | -| proto-*/pom.xml | Only be generated if it does not exist | -| samples/snippets/generated/ | Only be generated if `include_samples` is set to true | -| google-*-bom/pom.xml | Library BOM, only be generated if it does not exist | -| pom.xml | Library parent BOM, only be generated if it does not exist | -| .repo-metadata.json | Always generated from inputs | -| .OwlBot.yaml | Only be generated from a template if it does not exist | -| owlbot.py | Only be generated from a template if it does not exist | -| README.md | Always generated from inputs | -| gapic-libraries-bom/pom.xml | Always generated from inputs | -| pom.xml (repo root dir) | Always generated from inputs | -| versions.txt | New entries will be added if they don’t exist | +| Name | Notes | +|:------------------------------------|:-------------------------------------------------------------------------| +| google-*/ | Source code generated by gapic-generator-java | +| google-*/pom.xml | Only be generated if it does not exist | +| grpc-*/ | Source code generated by grpc generator, one per each version | +| grpc-*/pom.xml | Only be generated if it does not exist | +| proto-*/ | Source code generated by Protobuf default compiler, one per each version | +| proto-*/pom.xml | Only be generated if it does not exist | +| samples/snippets/generated/ | Only be generated if `include_samples` is set to true | +| google-*-bom/pom.xml | Library BOM, only be generated if it does not exist | +| pom.xml | Library parent BOM, only be generated if it does not exist | +| .repo-metadata.json | Always generated from inputs | +| .OwlBot-hermetic.yaml | Only be generated from a template if it does not exist | +| owlbot.py | Only be generated from a template if it does not exist | +| README.md | Always generated from inputs | +| gapic-libraries-bom/pom.xml | Always generated from inputs | +| pom.xml (repo root dir) | Always generated from inputs | +| versions.txt | New entries will be added if they don’t exist | ### googleapis commit history @@ -231,7 +231,7 @@ $repository_path | |_samples | | |_snippets | | | |_generated -| |_.OwlBot.yaml +| |_.OwlBot-hermetic.yaml | |_.repo-metadata.json | |_owlbot.py | |_pom.xml @@ -275,7 +275,7 @@ $repository_path | |_samples | | |_snippets | | | |_generated -| |_.OwlBot.yaml +| |_.OwlBot-hermetic.yaml | |_.repo-metadata.json | |_owlbot.py | |_pom.xml diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 0b6fdd5fca..12f21dec9c 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -72,7 +72,7 @@ def generate_composed_library( build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve() print(f"build_file_folder: {build_file_folder}") gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path) - # generate prerequisite files (.repo-metadata.json, .OwlBot.yaml, + # generate prerequisite files (.repo-metadata.json, .OwlBot-hermetic.yaml, # owlbot.py) here because transport is parsed from BUILD.bazel, # which lives in a versioned proto_path. util.generate_prerequisite_files( diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index e7eb91c179..37f50152ce 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -27,19 +27,10 @@ set -ex scripts_root=$1 versions_file=$2 configuration_yaml=$3 +is_monorepo=$4 -# This script can be used to process HW libraries and monorepo -# (google-cloud-java) libraries, which require a slightly different treatment -# monorepo folders have an .OwlBot.yaml file in the module folder (e.g. -# java-asset/.OwlBot.yaml), whereas HW libraries have the yaml in -# `.github/.OwlBot.yaml` -monorepo="false" -if [[ -f "$(pwd)/.OwlBot.yaml" ]]; then - monorepo="true" -fi - -if [[ "${monorepo}" == "true" ]]; then +if [[ "${is_monorepo}" == "true" ]]; then mv owl-bot-staging/* temp rm -rd owl-bot-staging/ mv temp owl-bot-staging @@ -50,7 +41,7 @@ fi # apply repo templates echo "Rendering templates" -python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}" +python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${is_monorepo}" # templates as well as retrieving files from owl-bot-staging echo "Retrieving files from owl-bot-staging directory..." @@ -66,7 +57,7 @@ echo "...done" # write or restore pom.xml files echo "Generating missing pom.xml..." -python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "${monorepo}" +python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "${is_monorepo}" echo "...done" # write or restore clirr-ignored-differences.xml diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 480b7e5170..397716c843 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -9,7 +9,7 @@ # has the following requirements # - a .repo-metadata.json file must be present # - an owlbot.py file must be present -# - an .OwlBot.yaml file must be present +# - an .OwlBot-hermetic.yaml file must be present # 2 - preprocessed_sources_path: used to transfer the raw grpc, proto and gapic # libraries into the postprocessing_target via copy-code # 3 - versions_file: path to file containing versions to be applied to the poms @@ -33,6 +33,7 @@ owlbot_cli_image_sha=$5 synthtool_commitish=$6 is_monorepo=$7 configuration_yaml_path=$8 +owlbot_yaml_file_name=".OwlBot-hermetic.yaml" source "${scripts_root}"/utils/utilities.sh @@ -44,7 +45,7 @@ for required_input in "${required_inputs[@]}"; do fi done -for owlbot_file in ".repo-metadata.json" "owlbot.py" ".OwlBot.yaml" +for owlbot_file in ".repo-metadata.json" "owlbot.py" "${owlbot_yaml_file_name}" do if [[ $(find "${postprocessing_target}" -name "${owlbot_file}" | wc -l) -eq 0 ]]; then echo "necessary file for postprocessing '${owlbot_file}' was not found in postprocessing_target" @@ -59,10 +60,10 @@ if [[ -z "${owlbot_cli_source_folder}" ]]; then fi -# we determine the location of the .OwlBot.yaml file by checking if the target +# we determine the location of the .OwlBot-hermetic.yaml file by checking if the target # folder is a monorepo folder or not if [[ "${is_monorepo}" == "true" ]]; then - # the deep-remove-regex and deep-preserve-regex of the .OwlBot.yaml + # the deep-remove-regex and deep-preserve-regex of the .OwlBot-hermetic.yaml # files in the monorepo libraries assume that `copy-code` is run # from the root of the monorepo. However, we call `copy-code` from inside each # library, so a path like `/java-asset/google-.*/src` will not have @@ -74,12 +75,12 @@ if [[ "${is_monorepo}" == "true" ]]; then # - "/google-.*/src" library_name=$(basename "${postprocessing_target}") - cat "${postprocessing_target}/.OwlBot.yaml" \ + cat "${postprocessing_target}/${owlbot_yaml_file_name}" \ | sed "s/- \"\/${library_name}/ - \"/" \ > "${postprocessing_target}/.OwlBot.hermetic.yaml" owlbot_yaml_relative_path=".OwlBot.hermetic.yaml" else - owlbot_yaml_relative_path=".github/.OwlBot.yaml" + owlbot_yaml_relative_path=".github/${owlbot_yaml_file_name}" fi # Default values for running copy-code directly from host @@ -150,5 +151,5 @@ popd # temp dir # run the postprocessor echo 'running owl-bot post-processor' pushd "${postprocessing_target}" -bash "${scripts_root}/owlbot/bin/entrypoint.sh" "${scripts_root}" "${versions_file}" "${configuration_yaml_path}" +bash "${scripts_root}/owlbot/bin/entrypoint.sh" "${scripts_root}" "${versions_file}" "${configuration_yaml_path}" "${is_monorepo}" popd # postprocessing_target diff --git a/library_generation/test/resources/goldens/.OwlBot-golden.yaml b/library_generation/test/resources/goldens/.OwlBot-hermetic-golden.yaml similarity index 100% rename from library_generation/test/resources/goldens/.OwlBot-golden.yaml rename to library_generation/test/resources/goldens/.OwlBot-hermetic-golden.yaml diff --git a/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml index 839f80996e..96f1a67292 100644 --- a/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml @@ -1,8 +1,8 @@ -gapic_generator_version: 2.37.0 +gapic_generator_version: 2.38.1 protobuf_version: 25.2 googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e +synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 template_excludes: - ".github/*" - ".kokoro/*" diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index d2472486e3..fae52e023b 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -359,8 +359,8 @@ def test_generate_prerequisite_files_non_monorepo_success(self): # since this is a single library, we treat this as HW repository, # meaning that the owlbot yaml will be inside a .github folder file_comparator.compare_files( - f"{library_path}/.github/.OwlBot.yaml", - f"{library_path}/.OwlBot-golden.yaml", + f"{library_path}/.github/.OwlBot-hermetic.yaml", + f"{library_path}/.OwlBot-hermetic-golden.yaml", ) file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" @@ -374,8 +374,8 @@ def test_generate_prerequisite_files_monorepo_success(self): f"{library_path}/.repo-metadata-monorepo-golden.json", ) file_comparator.compare_files( - f"{library_path}/.OwlBot.yaml", - f"{library_path}/.OwlBot-golden.yaml", + f"{library_path}/.OwlBot-hermetic.yaml", + f"{library_path}/.OwlBot-hermetic-golden.yaml", ) file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" @@ -432,7 +432,7 @@ def __setup_prerequisite_files( library_path = f"{resources_dir}/goldens" files = [ f"{library_path}/.repo-metadata.json", - f"{library_path}/.OwlBot.yaml", + f"{library_path}/.OwlBot-hermetic.yaml", f"{library_path}/owlbot.py", ] cleanup(files) diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py index 9610430cd7..1456ede48f 100755 --- a/library_generation/utils/utilities.py +++ b/library_generation/utils/utilities.py @@ -280,8 +280,8 @@ def generate_prerequisite_files( with open(f"{library_path}/{json_file}", "w") as fp: json.dump(repo_metadata, fp, indent=2) - # generate .OwlBot.yaml - owlbot_yaml_file = ".OwlBot.yaml" + # generate .OwlBot-hermetic.yaml + owlbot_yaml_file = ".OwlBot-hermetic.yaml" path_to_owlbot_yaml_file = ( f"{library_path}/{owlbot_yaml_file}" if config.is_monorepo() diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh index f0bdaeee01..1c3feb937b 100755 --- a/library_generation/utils/utilities.sh +++ b/library_generation/utils/utilities.sh @@ -276,7 +276,7 @@ get_proto_path_from_preprocessed_sources() { # for a pre-processed library stored in $preprocessed_sources_path, a folder # tree is built on $target_folder so it looks like a googleapis-gen folder and -# is therefore consumable by an .OwlBot.yaml file +# is therefore consumable by OwlBot CLI build_owlbot_cli_source_folder() { local postprocessing_target=$1 local target_folder=$2 @@ -288,7 +288,7 @@ build_owlbot_cli_source_folder() { owlbot_staging_folder="${postprocessing_target}/owl-bot-staging" mkdir -p "${owlbot_staging_folder}" - # By default (thanks to generation templates), .OwlBot.yaml `deep-copy` section + # By default (thanks to generation templates), .OwlBot-hermetic.yaml `deep-copy` section # references a wildcard pattern matching a folder # ending with `-java` at the leaf of proto_path. We then use a generated-java # folder that will be picked up by copy-code