From 945b988cc255ba96ac2252a6084c2a662c935497 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 2 Feb 2020 11:13:04 +0100 Subject: [PATCH] [AIRFLOW-6662] install dumb init (#7300) * Revert "[AIRFLOW-6662] Switch to --init docker flag for signal propagation (#7278)" This reverts commit d1bf343ffec505270d4b2ee4b0e9fa5dbbedd891. * [AIRFLOW-6662] return back the dumb-init - installed by apt We had stability problems with tests with --init flag so we are going back to it --- Dockerfile | 6 +- scripts/ci/_utils.sh | 61 ++++++++----------- scripts/ci/docker-compose/backend-mysql.yml | 1 - .../ci/docker-compose/backend-postgres.yml | 1 - .../docker-compose/integration-cassandra.yml | 1 - .../docker-compose/integration-kerberos.yml | 1 - .../ci/docker-compose/integration-mongo.yml | 1 - .../docker-compose/integration-openldap.yml | 1 - .../docker-compose/integration-rabbitmq.yml | 1 - .../ci/docker-compose/integration-redis.yml | 1 - 10 files changed, 29 insertions(+), 46 deletions(-) diff --git a/Dockerfile b/Dockerfile index d965a606024ee..24434cad0c668 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,7 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ apt-utils \ build-essential \ dirmngr \ + dumb-init \ freetds-bin \ freetds-dev \ git \ @@ -353,6 +354,9 @@ RUN \ && pip uninstall --yes apache-airflow; \ fi +# Link dumb-init for backwards compatibility (so that older images also work) +RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init + # Install NPM dependencies here. The NPM dependencies don't change that often and we already have pip # installed dependencies in case of CI optimised build, so it is ok to install NPM deps here # Rather than after setup.py is added. @@ -431,6 +435,6 @@ ENV PATH="${HOME}:${PATH}" EXPOSE 8080 -ENTRYPOINT ["/entrypoint.sh"] +ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint.sh"] CMD ["--help"] diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index 9ebcc71cf1a66..fffb373eca448 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -640,30 +640,27 @@ function run_flake8() { if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "${FILES[@]}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -682,31 +679,29 @@ function run_bats_tests() { function run_docs() { docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - --entrypoint \ - "/opt/airflow/docs/build.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/docs/build.sh" \ | tee -a "${OUTPUT_LOG}" } function run_check_license() { docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \ | tee -a "${OUTPUT_LOG}" } @@ -714,31 +709,27 @@ function run_mypy() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "airflow" "tests" "docs" \ + "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "airflow" "tests" "docs" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ - "${FILES[@]}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -747,30 +738,27 @@ function run_pylint_main() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "${FILES[@]}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -780,30 +768,27 @@ function run_pylint_tests() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "${FILES[@]}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -856,13 +841,15 @@ function filter_out_files_from_pylint_todo_list() { function refresh_pylint_todo() { docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint /opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_CI_IMAGE}" | tee -a "${OUTPUT_LOG}" + --rm \ + "${AIRFLOW_CI_IMAGE}" \ + /opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \ + | tee -a "${OUTPUT_LOG}" } function rebuild_all_images_if_needed_and_confirmed() { diff --git a/scripts/ci/docker-compose/backend-mysql.yml b/scripts/ci/docker-compose/backend-mysql.yml index d2bfd8dad9e49..d9773aac1a97b 100644 --- a/scripts/ci/docker-compose/backend-mysql.yml +++ b/scripts/ci/docker-compose/backend-mysql.yml @@ -25,7 +25,6 @@ services: depends_on: - mysql mysql: - init: true image: mysql:5.6 environment: - MYSQL_ALLOW_EMPTY_PASSWORD=true diff --git a/scripts/ci/docker-compose/backend-postgres.yml b/scripts/ci/docker-compose/backend-postgres.yml index 9cfdc7b54b44f..10a6fd5227bbb 100644 --- a/scripts/ci/docker-compose/backend-postgres.yml +++ b/scripts/ci/docker-compose/backend-postgres.yml @@ -25,7 +25,6 @@ services: depends_on: - postgres postgres: - init: true image: postgres:9.6 environment: - POSTGRES_USER=postgres diff --git a/scripts/ci/docker-compose/integration-cassandra.yml b/scripts/ci/docker-compose/integration-cassandra.yml index 304c36c9d51f8..12a5537156b30 100644 --- a/scripts/ci/docker-compose/integration-cassandra.yml +++ b/scripts/ci/docker-compose/integration-cassandra.yml @@ -18,7 +18,6 @@ version: "2.2" services: cassandra: - init: true image: cassandra:3.0 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/integration-kerberos.yml b/scripts/ci/docker-compose/integration-kerberos.yml index 79d841f21a2fa..6714a104f5819 100644 --- a/scripts/ci/docker-compose/integration-kerberos.yml +++ b/scripts/ci/docker-compose/integration-kerberos.yml @@ -18,7 +18,6 @@ version: "2.2" services: kerberos: - init: true image: godatadriven/krb5-kdc-server hostname: kerberos domainname: example.com diff --git a/scripts/ci/docker-compose/integration-mongo.yml b/scripts/ci/docker-compose/integration-mongo.yml index 06dfea113b18a..5d48c5022761e 100644 --- a/scripts/ci/docker-compose/integration-mongo.yml +++ b/scripts/ci/docker-compose/integration-mongo.yml @@ -18,7 +18,6 @@ version: "2.2" services: mongo: - init: true image: mongo:3 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/integration-openldap.yml b/scripts/ci/docker-compose/integration-openldap.yml index 44e2c2f2de50e..cf8cfd3019c4b 100644 --- a/scripts/ci/docker-compose/integration-openldap.yml +++ b/scripts/ci/docker-compose/integration-openldap.yml @@ -18,7 +18,6 @@ version: "2.2" services: openldap: - init: true image: osixia/openldap:1.2.0 command: --copy-service environment: diff --git a/scripts/ci/docker-compose/integration-rabbitmq.yml b/scripts/ci/docker-compose/integration-rabbitmq.yml index 1864122f8d34b..d14db8aaef984 100644 --- a/scripts/ci/docker-compose/integration-rabbitmq.yml +++ b/scripts/ci/docker-compose/integration-rabbitmq.yml @@ -18,7 +18,6 @@ version: "2.2" services: rabbitmq: - init: true image: rabbitmq:3.7 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/integration-redis.yml b/scripts/ci/docker-compose/integration-redis.yml index 819a35416f99c..6893af7074860 100644 --- a/scripts/ci/docker-compose/integration-redis.yml +++ b/scripts/ci/docker-compose/integration-redis.yml @@ -18,7 +18,6 @@ version: "2.2" services: redis: - init: true image: redis:5.0.1 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source