Add 3.8 to the test matrices (#8836)

.github/workflows/ci.yml

@@ -123,7 +123,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8]
     env:
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       CI_JOB_TYPE: "Prod image"
@@ -228,7 +228,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     needs: [static-checks-1, static-checks-2, pyfiles]
     strategy:
       matrix:
-        python-version: [3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8]
         postgres-version: [9.6, 10]
         test-type: [Core, Integration]
       fail-fast: false
@@ -260,7 +260,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     needs: [static-checks-1, static-checks-2, pyfiles]
     strategy:
       matrix:
-        python-version: [3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8]
         mysql-version: [5.7]
         test-type: [Core, Integration]
       fail-fast: false
@@ -292,7 +292,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     needs: [static-checks-1, static-checks-2, pyfiles]
     strategy:
       matrix:
-        python-version: [3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8]
         test-type: [Core, Integration]
       fail-fast: false
     env:
@@ -354,7 +354,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8]
       fail-fast: false
     env:
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
@@ -386,7 +386,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     if: github.ref == 'refs/heads/master' && github.event_name != 'schedule'
     strategy:
       matrix:
-        python-version: [3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8]
     env:
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       CI_JOB_TYPE: "Prod image"
@@ -417,7 +417,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
       github.event_name != 'schedule'
     strategy:
       matrix:
-        python-version: [3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8]
     env:
       PULL_PYTHON_BASE_IMAGES_FROM_CACHE: "false"
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}

BREEZE.rst

@@ -796,7 +796,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of:
 
-                 3.6 3.7
+                 3.6 3.7 3.8
 
   -a, --install-airflow-version <INSTALL_AIRFLOW_VERSION>
           If specified, installs Airflow directly from PIP released version. This happens at
@@ -886,7 +886,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of:
 
-                 3.6 3.7
+                 3.6 3.7 3.8
 
   -I, --production-image
           Use production image for entering the environment and builds (not for tests).
@@ -931,7 +931,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of:
 
-                 3.6 3.7
+                 3.6 3.7 3.8
 
   -v, --verbose
           Show verbose information about executed commands (enabled by default for running test).
@@ -1094,7 +1094,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of:
 
-                 3.6 3.7
+                 3.6 3.7 3.8
 
 
   ####################################################################################################
@@ -1120,7 +1120,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of:
 
-                 3.6 3.7
+                 3.6 3.7 3.8
 
   -F, --force-build-images
           Forces building of the local docker images. The images are rebuilt
@@ -1242,7 +1242,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of:
 
-                 3.6 3.7
+                 3.6 3.7 3.8
 
   -b, --backend <BACKEND>
           Backend to use for tests - it determines which database is used.
@@ -1290,7 +1290,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of:
 
-                 3.6 3.7
+                 3.6 3.7 3.8
 
   -b, --backend <BACKEND>
           Backend to use for tests - it determines which database is used.
@@ -1339,7 +1339,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of:
 
-                 3.6 3.7
+                 3.6 3.7 3.8
 
   -F, --force-build-images
           Forces building of the local docker images. The images are rebuilt
@@ -1506,7 +1506,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           Python version used for the image. This is always major/minor version.
           One of:
 
-                 3.6 3.7
+                 3.6 3.7 3.8
 
   ****************************************************************************************************
    Choose backend to run for Airflow

Dockerfile.ci

@@ -220,11 +220,17 @@ ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH}
 # We do not have to always reinstall it from the scratch.
 # This can be reinstalled from latest master by increasing PIP_DEPENDENCIES_EPOCH_NUMBER.
 # And is automatically reinstalled from the scratch every month
+
+ARG AIRFLOW_PY_VERSION_REQ_FILE="https://raw.githubusercontent.com/${AIRFLOW_REPO}/${AIRFLOW_BRANCH}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV AIRFLOW_PY_VERSION_REQ_FILE_STATUS="$(curl -s --head -w %{http_code} ${AIRFLOW_PY_VERSION_REQ_FILE} -o /dev/null)"
+RUN echo $AIRFLOW_PY_VERSION_REQ_FILE_STATUS;
+
 RUN \
-    if [[ "${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" == "true" ]]; then \
-        pip install \
+    if [[ "${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" == "true" && "${AIRFLOW_PY_VERSION_REQ_FILE_STATUS}" == "200" ]]; then \
+        echo "Using optimized build." \
+        && pip install \
         "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
-            --constraint "https://raw.githubusercontent.com/${AIRFLOW_REPO}/${AIRFLOW_BRANCH}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt" \
+            --constraint ${AIRFLOW_PY_VERSION_REQ_FILE} \
         && pip uninstall --yes apache-airflow; \
     fi
 

README.md

@@ -61,7 +61,7 @@ Apache Airflow is tested with:
 
 ### Master version (2.0.0dev)
 
-* Python versions: 3.6, 3.7
+* Python versions: 3.6, 3.7, 3.8
 * Postgres DB: 9.6, 10
 * MySQL DB: 5.7
 * Sqlite - latest stable (it is used mainly for development purpose)
@@ -101,7 +101,7 @@ produce unusable Airflow installation.
 
 In order to have repeatable installation, however, starting from **Airflow 1.10.10** we also keep a set of
 "known-to-be-working" requirement files in the `requirements` folder. Those "known-to-be-working"
-requirements are per major/minor python version (3.6/3.7). You can use them as constraint files
+requirements are per major/minor python version (3.6/3.7/3.8). You can use them as constraint files
 when installing Airflow from PyPI. Note that you have to specify correct Airflow version and python versions
 in the URL.
 

airflow/__init__.py

@@ -43,6 +43,7 @@
 login: Optional[Callable] = None
 
 PY37 = sys.version_info >= (3, 7)
+PY38 = sys.version_info >= (3, 8)
 
 
 def __getattr__(name):

airflow/providers/google/cloud/hooks/text_to_speech.py

@@ -56,10 +56,12 @@ def get_conn(self) -> TextToSpeechClient:
         :rtype: google.cloud.texttospeech_v1.TextToSpeechClient
         """
         if not self._client:
+            # pylint: disable=unexpected-keyword-arg
             self._client = TextToSpeechClient(
                 credentials=self._get_credentials(),
                 client_info=self.client_info
             )
+            # pylint: enable=unexpected-keyword-arg
 
         return self._client
 
@@ -96,6 +98,8 @@ def synthesize_speech(
         """
         client = self.get_conn()
         self.log.info("Synthesizing input: %s", input_data)
+        # pylint: disable=unexpected-keyword-arg
         return client.synthesize_speech(
             input_=input_data, voice=voice, audio_config=audio_config, retry=retry, timeout=timeout
         )
+        # pylint: enable=unexpected-keyword-arg

breeze

@@ -68,7 +68,7 @@ function setup_default_breeze_variables() {
     # We have different versions of images depending on the python version used. We keep up with the
     # Latest patch-level changes in Python (this is done automatically during CI builds) so we have
     # To only take into account MAJOR and MINOR version of python. This variable keeps the major/minor
-    # version of python in X.Y format (3.6, 3.7 etc).
+    # version of python in X.Y format (3.6, 3.7, 3.8 etc).
     export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(read_from_file PYTHON_MAJOR_MINOR_VERSION)}"
 
     # When we generate documentation for README files, we want to force the width of terminal so that
@@ -203,7 +203,7 @@ function initialize_virtualenv() {
             if [[ ${SYSTEM} == "Darwin" ]]; then
                 echo "  brew install sqlite mysql postgresql openssl"
             else
-                echo "  sudo apt install build-essentials python3.6-dev python3.7-dev python-dev openssl \\"
+                echo "  sudo apt install build-essentials python3.6-dev python3.7-dev python3.8-dev python-dev openssl \\"
                 echo "              sqlite sqlite-dev default-libmysqlclient-dev libmysqld-dev postgresql"
             fi
             echo

breeze-complete

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS="3.6 3.7"
+_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS="3.6 3.7 3.8"
 _BREEZE_ALLOWED_BACKENDS="sqlite mysql postgres"
 _BREEZE_ALLOWED_INTEGRATIONS="cassandra kerberos mongo openldap presto rabbitmq redis"
 _BREEZE_ALLOWED_KUBERNETES_MODES="image git"

requirements/requirements-python3.6.txt

@@ -15,7 +15,7 @@ Flask==1.1.2
 GitPython==3.1.3
 HeapDict==1.0.1
 JPype1==0.7.5
-JayDeBeApi==1.2.1
+JayDeBeApi==1.2.2
 Jinja2==2.10.3
 Mako==1.1.3
 Markdown==2.6.11
@@ -45,7 +45,7 @@ apispec==1.3.3
 appdirs==1.4.4
 argcomplete==1.11.1
 asn1crypto==1.3.0
-astroid==2.3.3
+astroid==2.4.1
 async-generator==1.10
 async-timeout==3.0.1
 atlasclient==1.0.0
@@ -61,7 +61,7 @@ azure-mgmt-containerinstance==1.5.0
 azure-mgmt-datalake-nspkg==3.0.1
 azure-mgmt-datalake-store==0.5.0
 azure-mgmt-nspkg==3.0.2
-azure-mgmt-resource==9.0.0
+azure-mgmt-resource==10.0.0
 azure-nspkg==3.0.2
 azure-storage-blob==2.1.0
 azure-storage-common==2.1.0
@@ -72,9 +72,9 @@ beautifulsoup4==4.7.1
 billiard==3.6.3.0
 black==19.10b0
 blinker==1.4
-boto3==1.13.22
+boto3==1.13.23
 boto==2.49.0
-botocore==1.16.22
+botocore==1.16.23
 bowler==0.8.0
 cached-property==1.5.1
 cachetools==4.1.0
@@ -139,11 +139,11 @@ gcsfs==0.6.2
 gevent==20.5.2
 gitdb==4.0.5
 google-ads==4.0.0
-google-api-core==1.17.0
+google-api-core==1.19.0
 google-api-python-client==1.9.1
 google-auth-httplib2==0.0.3
 google-auth-oauthlib==0.4.1
-google-auth==1.16.0
+google-auth==1.16.1
 google-cloud-automl==0.10.0
 google-cloud-bigquery-datatransfer==1.0.0
 google-cloud-bigquery==1.24.0
@@ -181,7 +181,7 @@ hmsclient==0.1.1
 httplib2==0.18.1
 humanize==0.5.1
 hvac==0.10.3
-identify==1.4.18
+identify==1.4.19
 idna-ssl==1.1.0
 idna==2.9
 ijson==2.6.1
@@ -225,7 +225,7 @@ monotonic==1.5
 more-itertools==8.3.0
 moto==1.3.14
 msgpack==1.0.0
-msrest==0.6.14
+msrest==0.6.15
 msrestazure==0.6.3
 multi-key-dict==2.0.3
 multidict==4.7.6
@@ -335,7 +335,7 @@ setproctitle==1.1.10
 sh==1.13.1
 simple-salesforce==1.0.0
 six==1.15.0
-slackclient==2.7.0
+slackclient==2.7.1
 smmap==3.0.4
 snowballstemmer==2.0.0
 snowflake-connector-python==2.2.7
@@ -388,7 +388,7 @@ virtualenv==20.0.21
 watchtower==0.7.3
 wcwidth==0.2.3
 websocket-client==0.57.0
-wrapt==1.11.2
+wrapt==1.12.1
 xmltodict==0.12.0
 yamllint==1.23.0
 yandexcloud==0.40.0