Auto-apply apply_default decorator (#15667)

* Auto-apply apply_default decorator

* Automatically patch warnings stacklevel inside Operator constructors

This makes two changes to make the apply_defaults transparent to
warnings:

1. It removes `__call__` from the Metaclass (which added one frame)
2. It "patches" warnings.warn _in the function scope_ to automatically
   adjust the stacklevel

The reason for this is that now that apply_defaults is called
"magically" from the Metaclass we're adding extra frames to the
callstack. By locally patching warnings like this it means that user
code doesn't have to be aware of what we've done.

(I wish Python had a native way of handling this case as it crops up all
the time. Perhaps I should submit a PEP for it with a better design)

* Don't test provider packges against 2.0 anymore

Since we have changed the dep to need Airflow 2.1, testing against 2.0
no longer makes any sense.

And since 2.1 is not yet out, we have disabled this specific step for
now

Co-authored-by: Kamil Breguła <kamil.bregula@polidea.com>

Author: ashb

.github/workflows/ci.yml

@@ -553,11 +553,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         env:
           USE_AIRFLOW_VERSION: "wheel"
           PACKAGE_FORMAT: "wheel"
-      - name: "Install and test provider packages and airflow on Airflow 2.0 files"
+      - name: "Install and test provider packages and airflow on Airflow 2.1 files"
         run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
         env:
-          USE_AIRFLOW_VERSION: "2.0.0"
+          USE_AIRFLOW_VERSION: "2.1.0"
           PACKAGE_FORMAT: "wheel"
+        if: false
 
   prepare-test-provider-packages-sdist:
     timeout-minutes: 40

UPDATING.md

@@ -71,6 +71,10 @@ https://developers.google.com/style/inclusive-documentation
 
 -->
 
+### `@apply_default` decorator isn't longer necessary
+
+This decorator is now automatically added to all operators via the metaclass on BaseOperator
+
 ### Change the configuration options for field masking
 
 We've improved masking for sensitive data in Web UI and logs. As part of it, the following configurations have been changed:

airflow/contrib/operators/adls_to_gcs.py

@@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.transfers.adls_to_gcs.ADLSToGCSOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)

airflow/contrib/operators/awsbatch_operator.py

@@ -53,7 +53,7 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.amazon.aws.operators.batch.AwsBatchOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 

airflow/contrib/operators/bigquery_operator.py

@@ -51,6 +51,6 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.operators.bigquery.BigQueryExecuteQueryOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)

airflow/contrib/operators/bigquery_table_delete_operator.py

@@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)

airflow/contrib/operators/bigquery_to_gcs.py

@@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.transfers.bigquery_to_gcs.BigQueryToGCSOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)

airflow/contrib/operators/cassandra_to_gcs.py

@@ -42,6 +42,6 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.transfers.cassandra_to_gcs.CassandraToGCSOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)

airflow/contrib/operators/dataflow_operator.py

@@ -43,7 +43,7 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.operators.dataflow.DataflowCreateJavaJobOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs):
             Please use
             `airflow.providers.google.cloud.operators.dataflow.DataflowCreatePythonJobOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -77,6 +77,6 @@ def __init__(self, *args, **kwargs):
             Please use
             `airflow.providers.google.cloud.operators.dataflow.DataflowTemplatedJobStartOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)

airflow/contrib/operators/dataproc_operator.py

@@ -52,7 +52,7 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.operators.dataproc.DataprocCreateClusterOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -68,7 +68,7 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.operators.dataproc.DataprocDeleteClusterOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -84,7 +84,7 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.operators.dataproc.DataprocScaleClusterOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -101,7 +101,7 @@ def __init__(self, *args, **kwargs):
             Please use
             `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitHadoopJobOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -118,7 +118,7 @@ def __init__(self, *args, **kwargs):
             Please use
             `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitHiveJobOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -134,7 +134,7 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.operators.dataproc.DataprocJobBaseOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -150,7 +150,7 @@ def __init__(self, *args, **kwargs):
             """This class is deprecated.
             Please use `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitPigJobOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -167,7 +167,7 @@ def __init__(self, *args, **kwargs):
             Please use
             `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitPySparkJobOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -184,7 +184,7 @@ def __init__(self, *args, **kwargs):
             Please use
             `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitSparkJobOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -201,7 +201,7 @@ def __init__(self, *args, **kwargs):
             Please use
             `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitSparkSqlJobOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -220,7 +220,7 @@ def __init__(self, *args, **kwargs):
             `airflow.providers.google.cloud.operators.dataproc
             .DataprocInstantiateInlineWorkflowTemplateOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)
 
@@ -239,6 +239,6 @@ def __init__(self, *args, **kwargs):
             `airflow.providers.google.cloud.operators.dataproc
             .DataprocInstantiateWorkflowTemplateOperator`.""",
             DeprecationWarning,
-            stacklevel=3,
+            stacklevel=2,
         )
         super().__init__(*args, **kwargs)