From 5a153e016dd78efb5187372a9dde34ed6380ea43 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 28 May 2022 17:19:14 +0200 Subject: [PATCH] Replace generation of docker volumes to be done from python (#23985) The pre-commit to generate docker volumes in docker compose file is now written in Python and it also uses the newer "volume:" syntax to define the volumes mounted in the docker-compose. (cherry picked from commit 882535a8a2699af7d1d079ecebd8c31aa7fbaba9) --- .pre-commit-config.yaml | 7 +- .../airflow_breeze/commands/main_command.py | 4 +- .../src/airflow_breeze/utils/path_utils.py | 13 +- scripts/ci/docker-compose/local.yml | 128 +++++++++++++----- scripts/ci/pre_commit/pre_commit_flake8.py | 1 - .../pre_commit/pre_commit_local_yml_mounts.py | 56 ++++++++ .../pre_commit/pre_commit_local_yml_mounts.sh | 43 ------ scripts/ci/pre_commit/pre_commit_mypy.py | 1 - 8 files changed, 167 insertions(+), 86 deletions(-) create mode 100755 scripts/ci/pre_commit/pre_commit_local_yml_mounts.py delete mode 100755 scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 61043bc116895..8f4039f1e9050 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -350,10 +350,11 @@ repos: additional_dependencies: ['rich>=12.4.1', 'rich-click'] - id: update-local-yml-file name: Update mounts in the local yml file - entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh - language: system - files: ^scripts/ci/libraries/_local_mounts\.sh$|^scripts/ci/docker_compose/local\.yml$ + entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.py + language: python + files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$ pass_filenames: false + additional_dependencies: ['rich>=12.4.1'] - id: update-setup-cfg-file name: Update setup.cfg file with all licenses entry: ./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py index 4d805e7b7a1ca..8ed451b0e5073 100644 --- a/dev/breeze/src/airflow_breeze/commands/main_command.py +++ b/dev/breeze/src/airflow_breeze/commands/main_command.py @@ -37,7 +37,7 @@ option_use_packages_from_dist, option_verbose, ) -from airflow_breeze.utils.path_utils import create_directories +from airflow_breeze.utils.path_utils import create_directories_and_files @click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']}) @@ -64,6 +64,6 @@ def main(ctx: click.Context, **kwargs): from airflow_breeze.commands.developer_commands import shell - create_directories() + create_directories_and_files() if not ctx.invoked_subcommand: ctx.forward(shell, extra_args={}) diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index de2065b678273..49d9d3bf9ceef 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -142,7 +142,11 @@ def print_warning_if_setup_changed() -> bool: Prints warning if detected airflow sources are not the ones that Breeze was installed with. :return: True if warning was printed. """ - package_hash = get_package_setup_metadata_hash() + try: + package_hash = get_package_setup_metadata_hash() + except ModuleNotFoundError as e: + if "importlib_metadata" in e.msg: + return False sources_hash = get_installation_sources_config_metadata_hash() if sources_hash != package_hash: installation_sources = get_installation_airflow_sources() @@ -251,9 +255,9 @@ def find_airflow_sources_root_to_operate_on() -> Path: BREEZE_SOURCES_ROOT = AIRFLOW_SOURCES_ROOT / "dev" / "breeze" -def create_directories() -> None: +def create_directories_and_files() -> None: """ - Creates all directories that are needed for Breeze to work. + Creates all directories and files that are needed for Breeze to work via docker-compose. Checks if setup has been updates since last time and proposes to upgrade if so. """ BUILD_CACHE_DIR.mkdir(parents=True, exist_ok=True) @@ -263,3 +267,6 @@ def create_directories() -> None: LOGS_DIR.mkdir(parents=True, exist_ok=True) DIST_DIR.mkdir(parents=True, exist_ok=True) OUTPUT_LOG.mkdir(parents=True, exist_ok=True) + (AIRFLOW_SOURCES_ROOT / ".bash_aliases").touch() + (AIRFLOW_SOURCES_ROOT / ".bash_history").touch() + (AIRFLOW_SOURCES_ROOT / ".inputrc").touch() diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 0e37d0c34a0a8..e356cb873611f 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -26,38 +26,100 @@ services: # or those that might be useful to see in the host as output of the # tests (such as logs) volumes: - - mypy-cache-volume:/opt/airflow/.mypy_cache/ - # START automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh - - ../../../.bash_aliases:/root/.bash_aliases:cached - - ../../../.bash_history:/root/.bash_history:cached - - ../../../.coveragerc:/opt/airflow/.coveragerc:cached - - ../../../.dockerignore:/opt/airflow/.dockerignore:cached - - ../../../.flake8:/opt/airflow/.flake8:cached - - ../../../.github:/opt/airflow/.github:cached - - ../../../.inputrc:/root/.inputrc:cached - - ../../../.rat-excludes:/opt/airflow/.rat-excludes:cached - - ../../../RELEASE_NOTES.rst:/opt/airflow/RELEASE_NOTES.rst:cached - - ../../../LICENSE:/opt/airflow/LICENSE:cached - - ../../../MANIFEST.in:/opt/airflow/MANIFEST.in:cached - - ../../../NOTICE:/opt/airflow/NOTICE:cached - - ../../../airflow:/opt/airflow/airflow:cached - - ../../../provider_packages:/opt/airflow/provider_packages:cached - - ../../../dags:/opt/airflow/dags:cached - - ../../../dev:/opt/airflow/dev:cached - - ../../../docs:/opt/airflow/docs:cached - - ../../../hooks:/opt/airflow/hooks:cached - - ../../../logs:/root/airflow/logs:cached - - ../../../pyproject.toml:/opt/airflow/pyproject.toml:cached - - ../../../pytest.ini:/opt/airflow/pytest.ini:cached - - ../../../scripts:/opt/airflow/scripts:cached - - ../../../scripts/docker/entrypoint_ci.sh:/entrypoint:cached - - ../../../setup.cfg:/opt/airflow/setup.cfg:cached - - ../../../setup.py:/opt/airflow/setup.py:cached - - ../../../tests:/opt/airflow/tests:cached - - ../../../kubernetes_tests:/opt/airflow/kubernetes_tests:cached - - ../../../docker_tests:/opt/airflow/docker_tests:cached - - ../../../chart:/opt/airflow/chart:cached - - ../../../metastore_browser:/opt/airflow/metastore_browser:cached - # END automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh + - type: volume + source: mypy-cache-volume + target: /opt/airflow/.mypy_cache/ + # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py + - type: bind + source: ../../../.bash_aliases + target: /root/.bash_aliases + - type: bind + source: ../../../.bash_history + target: /root/.bash_history + - type: bind + source: ../../../.coveragerc + target: /opt/airflow/.coveragerc + - type: bind + source: ../../../.dockerignore + target: /opt/airflow/.dockerignore + - type: bind + source: ../../../.flake8 + target: /opt/airflow/.flake8 + - type: bind + source: ../../../.github + target: /opt/airflow/.github + - type: bind + source: ../../../.inputrc + target: /root/.inputrc + - type: bind + source: ../../../.rat-excludes + target: /opt/airflow/.rat-excludes + - type: bind + source: ../../../RELEASE_NOTES.rst + target: /opt/airflow/RELEASE_NOTES.rst + - type: bind + source: ../../../LICENSE + target: /opt/airflow/LICENSE + - type: bind + source: ../../../MANIFEST.in + target: /opt/airflow/MANIFEST.in + - type: bind + source: ../../../NOTICE + target: /opt/airflow/NOTICE + - type: bind + source: ../../../airflow + target: /opt/airflow/airflow + - type: bind + source: ../../../provider_packages + target: /opt/airflow/provider_packages + - type: bind + source: ../../../dags + target: /opt/airflow/dags + - type: bind + source: ../../../dev + target: /opt/airflow/dev + - type: bind + source: ../../../docs + target: /opt/airflow/docs + - type: bind + source: ../../../hooks + target: /opt/airflow/hooks + - type: bind + source: ../../../logs + target: /root/airflow/logs + - type: bind + source: ../../../pyproject.toml + target: /opt/airflow/pyproject.toml + - type: bind + source: ../../../pytest.ini + target: /opt/airflow/pytest.ini + - type: bind + source: ../../../scripts + target: /opt/airflow/scripts + - type: bind + source: ../../../scripts/docker/entrypoint_ci.sh + target: /entrypoint + - type: bind + source: ../../../setup.cfg + target: /opt/airflow/setup.cfg + - type: bind + source: ../../../setup.py + target: /opt/airflow/setup.py + - type: bind + source: ../../../tests + target: /opt/airflow/tests + - type: bind + source: ../../../kubernetes_tests + target: /opt/airflow/kubernetes_tests + - type: bind + source: ../../../docker_tests + target: /opt/airflow/docker_tests + - type: bind + source: ../../../chart + target: /opt/airflow/chart + - type: bind + source: ../../../metastore_browser + target: /opt/airflow/metastore_browser + # END automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py volumes: mypy-cache-volume: diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py b/scripts/ci/pre_commit/pre_commit_flake8.py index 3d4a56060f673..7adba2ffc656b 100755 --- a/scripts/ci/pre_commit/pre_commit_flake8.py +++ b/scripts/ci/pre_commit/pre_commit_flake8.py @@ -32,7 +32,6 @@ GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") if __name__ == '__main__': - os.environ['SKIP_BREEZE_UPGRADE_CHECK'] = "true" sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) from airflow_breeze.branch_defaults import AIRFLOW_BRANCH from airflow_breeze.global_constants import MOUNT_SELECTED diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py new file mode 100755 index 0000000000000..391214fa296e3 --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import sys +from pathlib import Path + +AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve() + +sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported +sys.path.insert(0, str(AIRFLOW_SOURCES_DIR)) # make sure setup is imported from Airflow +sys.path.insert( + 0, str(AIRFLOW_SOURCES_DIR / "dev" / "breeze" / "src") +) # make sure setup is imported from Airflow +# flake8: noqa: F401 + +from common_precommit_utils import insert_documentation # isort: skip + +sys.path.append(str(AIRFLOW_SOURCES_DIR)) + +MOUNTS_HEADER = ( + ' # START automatically generated volumes from ' + 'NECESSARY_HOST_VOLUMES in docker_command_utils.py' +) +MOUNTS_FOOTER = ( + ' # END automatically generated volumes from ' 'NECESSARY_HOST_VOLUMES in docker_command_utils.py' +) + +if __name__ == '__main__': + from airflow_breeze.utils.docker_command_utils import NECESSARY_HOST_VOLUMES + + local_mount_file_path = AIRFLOW_SOURCES_DIR / 'scripts' / 'ci' / 'docker-compose' / 'local.yml' + PREFIX = ' ' + volumes = [] + for (src, dest) in NECESSARY_HOST_VOLUMES: + volumes.extend( + [ + PREFIX + "- type: bind\n", + PREFIX + f" source: ../../../{src}\n", + PREFIX + f" target: {dest}\n", + ] + ) + insert_documentation(local_mount_file_path, volumes, MOUNTS_HEADER, MOUNTS_FOOTER) diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh deleted file mode 100755 index dbf82b72412dd..0000000000000 --- a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -export PRINT_INFO_FROM_SCRIPTS="false" -export SKIP_CHECK_REMOTE_IMAGE="true" - -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - -TMP_OUTPUT=$(mktemp) -readonly TMP_OUTPUT - -# Remove temp file if it's hanging around -traps::add_trap "rm -rf -- '${TMP_OUTPUT}' 2>/dev/null" EXIT HUP INT TERM - -LOCAL_YML_FILE="${AIRFLOW_SOURCES}/scripts/ci/docker-compose/local.yml" -readonly LOCAL_YML_FILE - -lead_marker=' # START automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh' -tail_marker=' # END automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh' - -local_mounts::generate_local_mounts_list " - ../../../" - -sed "/$lead_marker/q" "${LOCAL_YML_FILE}" > "${TMP_OUTPUT}" - -printf '%s\n' "${LOCAL_MOUNTS[@]}" >> "${TMP_OUTPUT}" -sed -ne "/$tail_marker/,\$ p" "${LOCAL_YML_FILE}" >> "${TMP_OUTPUT}" - -mv "${TMP_OUTPUT}" "${LOCAL_YML_FILE}" diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py index 00758563252cf..9d92b99e80edf 100755 --- a/scripts/ci/pre_commit/pre_commit_mypy.py +++ b/scripts/ci/pre_commit/pre_commit_mypy.py @@ -33,7 +33,6 @@ GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") if __name__ == '__main__': - os.environ['SKIP_BREEZE_UPGRADE_CHECK'] = "true" sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) from airflow_breeze.branch_defaults import AIRFLOW_BRANCH from airflow_breeze.global_constants import MOUNT_SELECTED