Merge pull request #1 from tuanavu/development

Create first tutorial dag

Author: tuanavu

README.md

@@ -1,2 +1,58 @@
-# airflow-tutorial
 Airflow tutorial
+---
+
+Airflow tutorial
+
+## Getting Started
+
+These instructions will get you a copy of the project up and running on your local machine for development and testing purposes.
+
+- Clone this repo
+- Install the prerequisites
+- Run the service
+- Check http://localhost:5000
+- Done! :tada:
+
+### Prerequisites
+
+- Install [Docker](https://www.docker.com/)
+- Install [Docker Compose](https://docs.docker.com/compose/install/)
+- Following the Airflow release from [Python Package Index](https://pypi.python.org/pypi/apache-airflow)
+
+### Usage
+
+Run the web service with docker
+
+```
+docker-compose up -d
+
+# Build the image
+# docker-compose up -d --build
+```
+
+Check http://localhost:8080/
+
+- `docker-compose logs` - Displays log output
+- `docker-compose ps` - List containers
+- `docker-compose down` - Stop containers
+
+## Other commands
+
+If you want to run other airflow sub-commands, you can do so like this:
+
+- `docker-compose run --rm webserver airflow list_dags` - List dags
+- `docker-compose run --rm webserver airflow test [DAG_ID] [TASK_ID] [EXECUTION_DATE]` - Test specific task
+
+## Connect to database
+
+If you want to use Ad hoc query, make sure you've configured connections:
+Go to Admin -> Connections and Edit "postgres_default" set this values:
+- Host : postgres
+- Schema : airflow
+- Login : airflow
+- Password : airflow
+
+
+## Credits
+
+- [docker-airflow](https://github.com/puckel/docker-airflow/tree/1.10.0-5)

dags/tutorial.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+### Tutorial Documentation
+Documentation that goes along with the Airflow tutorial located
+[here](https://airflow.incubator.apache.org/tutorial.html)
+"""
+from datetime import timedelta
+
+import airflow
+from airflow import DAG
+from airflow.operators.bash_operator import BashOperator
+
+# These args will get passed on to each operator
+# You can override them on a per-task basis during operator initialization
+default_args = {
+    'owner': 'airflow',
+    'depends_on_past': False,
+    'start_date': airflow.utils.dates.days_ago(2),
+    'email': ['airflow@example.com'],
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'retries': 1,
+    'retry_delay': timedelta(minutes=5),
+    # 'queue': 'bash_queue',
+    # 'pool': 'backfill',
+    # 'priority_weight': 10,
+    # 'end_date': datetime(2016, 1, 1),
+    # 'wait_for_downstream': False,
+    # 'dag': dag,
+    # 'adhoc':False,
+    # 'sla': timedelta(hours=2),
+    # 'execution_timeout': timedelta(seconds=300),
+    # 'on_failure_callback': some_function,
+    # 'on_success_callback': some_other_function,
+    # 'on_retry_callback': another_function,
+    # 'trigger_rule': u'all_success'
+}
+
+dag = DAG(
+    'tutorial',
+    default_args=default_args,
+    description='A simple tutorial DAG',
+    schedule_interval=timedelta(days=1),
+)
+
+# t1, t2 and t3 are examples of tasks created by instantiating operators
+t1 = BashOperator(
+    task_id='print_date',
+    bash_command='date',
+    dag=dag,
+)
+
+t1.doc_md = """\
+#### Task Documentation
+You can document your task using the attributes `doc_md` (markdown),
+`doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets
+rendered in the UI's Task Instance Details page.
+![img](http://montcs.bloomu.edu/~bobmon/Semesters/2012-01/491/import%20soul.png)
+"""
+
+dag.doc_md = __doc__
+
+t2 = BashOperator(
+    task_id='sleep',
+    depends_on_past=False,
+    bash_command='sleep 5',
+    dag=dag,
+)
+
+templated_command = """
+{% for i in range(5) %}
+    echo "{{ ds }}"
+    echo "{{ macros.ds_add(ds, 7)}}"
+    echo "{{ params.my_param }}"
+{% endfor %}
+"""
+
+t3 = BashOperator(
+    task_id='templated',
+    depends_on_past=False,
+    bash_command=templated_command,
+    params={'my_param': 'Parameter I passed in'},
+    dag=dag,
+)
+
+t1 >> [t2, t3]

docker-compose.yml

@@ -0,0 +1,37 @@
+version: '3'
+services:
+  postgres:
+    image: postgres:9.6
+    environment:
+      - POSTGRES_USER=airflow
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+    ports:
+      - "5432:5432"
+
+  webserver:
+    image: puckel/docker-airflow:1.10.0-5
+    build:
+      context: https://github.com/puckel/docker-airflow.git#1.10.0-5
+      dockerfile: Dockerfile
+      args:
+        AIRFLOW_DEPS: gcp_api,s3        
+    restart: always
+    depends_on:
+      - postgres
+    environment:
+      - LOAD_EX=n
+      - EXECUTOR=Local
+      - FERNET_KEY=jsDPRErfv8Z_eVTnGfF8ywd19j4pyqE3NpdUBA_oRTo=
+    volumes:
+      - ./dags:/usr/local/airflow/dags
+      # Uncomment to include custom plugins
+      # - ./plugins:/usr/local/airflow/plugins
+    ports:
+      - "8080:8080"
+    command: webserver
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
+      interval: 30s
+      timeout: 30s
+      retries: 3