Add doc and example dag for AWS CloudFormation Operators

airflow/providers/amazon/aws/example_dags/example_cloudformation.py

@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from datetime import datetime
+from json import dumps
+
+from airflow import DAG
+from airflow.models.baseoperator import chain
+from airflow.providers.amazon.aws.operators.cloud_formation import (
+    CloudFormationCreateStackOperator,
+    CloudFormationDeleteStackOperator,
+)
+from airflow.providers.amazon.aws.sensors.cloud_formation import (
+    CloudFormationCreateStackSensor,
+    CloudFormationDeleteStackSensor,
+)
+
+CLOUDFORMATION_STACK_NAME = 'example-stack-name'
+# The CloudFormation template must have at least one resource to be usable, this example uses SQS
+# as a free and serverless option.
+CLOUDFORMATION_TEMPLATE = {
+    'Description': 'Stack from Airflow CloudFormation example DAG',
+    'Resources': {
+        'ExampleQueue': {
+            'Type': 'AWS::SQS::Queue',
+        }
+    },
+}
+CLOUDFORMATION_CREATE_PARAMETERS = {
+    'StackName': CLOUDFORMATION_STACK_NAME,
+    'TemplateBody': dumps(CLOUDFORMATION_TEMPLATE),
+    'TimeoutInMinutes': 2,
+    'OnFailure': 'DELETE',  # Don't leave stacks behind if creation fails.
+}
+
+with DAG(
+    dag_id='example_cloudformation',
+    schedule_interval=None,
+    start_date=datetime(2021, 1, 1),
+    tags=['example'],
+    catchup=False,
+) as dag:
+
+    # [START howto_operator_cloudformation_create_stack]
+    create_stack = CloudFormationCreateStackOperator(
+        task_id='create_stack',
+        stack_name=CLOUDFORMATION_STACK_NAME,
+        cloudformation_parameters=CLOUDFORMATION_CREATE_PARAMETERS,
+    )
+    # [END howto_operator_cloudformation_create_stack]
+
+    # [START howto_sensor_cloudformation_create_stack]
+    wait_for_stack_create = CloudFormationCreateStackSensor(
+        task_id='wait_for_stack_creation', stack_name=CLOUDFORMATION_STACK_NAME
+    )
+    # [END howto_sensor_cloudformation_create_stack]
+
+    # [START howto_operator_cloudformation_delete_stack]
+    delete_stack = CloudFormationDeleteStackOperator(
+        task_id='delete_stack', stack_name=CLOUDFORMATION_STACK_NAME
+    )
+    # [END howto_operator_cloudformation_delete_stack]
+
+    # [START howto_sensor_cloudformation_delete_stack]
+    wait_for_stack_delete = CloudFormationDeleteStackSensor(
+        task_id='wait_for_stack_deletion', trigger_rule='all_done', stack_name=CLOUDFORMATION_STACK_NAME
+    )
+    # [END howto_sensor_cloudformation_delete_stack]
+
+    chain(create_stack, wait_for_stack_create, delete_stack, wait_for_stack_delete)

airflow/providers/amazon/aws/operators/cloud_formation.py

@@ -29,11 +29,13 @@ class CloudFormationCreateStackOperator(BaseOperator):
     """
     An operator that creates a CloudFormation stack.
 
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudFormationCreateStackOperator`
+
+
     :param stack_name: stack name (templated)
     :param cloudformation_parameters: parameters to be passed to CloudFormation.
-
-        .. seealso::
-            https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudformation.html#CloudFormation.Client.create_stack
     :param aws_conn_id: aws connection to uses
     """
 
@@ -63,8 +65,10 @@ class CloudFormationDeleteStackOperator(BaseOperator):
     :param stack_name: stack name (templated)
     :param cloudformation_parameters: parameters to be passed to CloudFormation.
 
-        .. seealso::
-            https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudformation.html#CloudFormation.Client.delete_stack
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudFormationDeleteStackOperator`
+
     :param aws_conn_id: aws connection to uses
     """
 

airflow/providers/amazon/aws/sensors/cloud_formation.py

@@ -35,6 +35,11 @@ class CloudFormationCreateStackSensor(BaseSensorOperator):
     """
     Waits for a stack to be created successfully on AWS CloudFormation.
 
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudFormationCreateStackSensor`
+
+
     :param stack_name: The name of the stack to wait for (templated)
     :param aws_conn_id: ID of the Airflow connection where credentials and extra configuration are
         stored
@@ -68,6 +73,11 @@ class CloudFormationDeleteStackSensor(BaseSensorOperator):
     """
     Waits for a stack to be deleted successfully on AWS CloudFormation.
 
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudFormationDeleteStackSensor`
+
+
     :param stack_name: The name of the stack to wait for (templated)
     :param aws_conn_id: ID of the Airflow connection where credentials and extra configuration are
         stored

airflow/providers/amazon/provider.yaml

@@ -51,6 +51,8 @@ integrations:
   - integration-name: Amazon CloudFormation
     external-doc-url: https://aws.amazon.com/cloudformation/
     logo: /integration-logos/aws/AWS-CloudFormation_light-bg@4x.png
+    how-to-guide:
+      - /docs/apache-airflow-providers-amazon/operators/cloudformation.rst
     tags: [aws]
   - integration-name: Amazon CloudWatch Logs
     external-doc-url: https://aws.amazon.com/cloudwatch/

docs/apache-airflow-providers-amazon/operators/cloudformation.rst

@@ -0,0 +1,98 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+AWS CloudFormation Operators
+============================
+
+`AWS CloudFormation <https://aws.amazon.com/cloudformation/>`__ enables you to create and provision AWS
+infrastructure deployments predictably and repeatedly. It helps you leverage AWS products such as Amazon
+EC2, Amazon Elastic Block Store, Amazon SNS, Elastic Load Balancing, and Auto Scaling to build highly
+reliable, highly scalable, cost-effective applications in the cloud without worrying about creating and
+configuring the underlying AWS infrastructure. AWS CloudFormation enables you to use a template file to
+create and delete a collection of resources together as a single unit (a stack).
+
+Prerequisite Tasks
+------------------
+
+.. include:: _partials/prerequisite_tasks.rst
+
+.. _howto/operator:CloudFormationCreateStackOperator:
+
+AWS CloudFormation Create Stack Operator
+""""""""""""""""""""""""""""""""""""""""
+
+To create a new AWS CloudFormation stack use
+:class:`~airflow.providers.amazon.aws.operators.cloud_formation.CloudFormationCreateStackOperator`.
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_cloudformation.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudformation_create_stack]
+    :end-before: [END howto_operator_cloudformation_create_stack]
+
+
+.. _howto/operator:CloudFormationCreateStackSensor:
+
+AWS CloudFormation Create Stack Sensor
+""""""""""""""""""""""""""""""""""""""
+
+To wait on the state of an AWS CloudFormation stack creation until it reaches a terminal state you can use
+:class:`~airflow.providers.amazon.aws.sensors.cloud_formation.CloudFormationCreateStackSensor`
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_cloudformation.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_sensor_cloudformation_create_stack]
+    :end-before: [END howto_sensor_cloudformation_create_stack]
+
+.. _howto/operator:CloudFormationDeleteStackOperator:
+
+AWS CloudFormation Delete Stack Operator
+""""""""""""""""""""""""""""""""""""""""
+
+To delete an AWS CloudFormation stack you can use
+:class:`~airflow.providers.amazon.aws.operators.cloud_formation.CloudFormationDeleteStackOperator`.
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_cloudformation.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudformation_delete_stack]
+    :end-before: [END howto_operator_cloudformation_delete_stack]
+
+
+.. _howto/operator:CloudFormationDeleteStackSensor:
+
+AWS CloudFormation Delete Stack Sensor
+""""""""""""""""""""""""""""""""""""""
+
+To wait on the state of an AWS CloudFormation stack deletion until it reaches a terminal state you can use
+use :class:`~airflow.providers.amazon.aws.sensors.cloud_formation.CloudFormationDeleteStackSensor`
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_cloudformation.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_sensor_cloudformation_delete_stack]
+    :end-before: [END howto_sensor_cloudformation_delete_stack]
+
+
+Reference
+---------
+
+For further information, look at:
+
+* `Boto3 Library Documentation for CloudFormation <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudformation.html>`__