Starting (#4)

* Initial commit

* Black + isort

* Update docker-compose.yaml

* Some Dockerfile changes + test addition

* Test changes

* Fixing

* Tests addition

* Changes to Dockerfile

* Update Dockerfile

* Start.sh addition

* Add local run

* Postgres addition

* Update Dockerfile

---------

Co-authored-by: Stanislav Roslavtsev <jalchinn@yandex.ru>

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.11
+ENV AIRFLOW__CORE__LOAD_EXAMPLES=false
+ENV DEBIAN_FRONTEND=noninteractive
+ENV AIRFLOW__CORE__EXECUTOR=LocalExecutor
+
+# Linux block
+COPY requirements.txt .
+RUN apt-get update \
+    && apt-get -yq install \
+        postgresql \
+        postgresql-contrib \
+    && python3 -m pip install --no-cache-dir -r ./requirements.txt
+
+# Run block
+COPY --chmod=+x ./start.sh /start.sh
+CMD /start.sh
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8080" ]

dags.py

@@ -0,0 +1,34 @@
+from datetime import datetime, timedelta
+
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+
+with DAG(
+    dag_id="git_pull_pipelines",
+    start_date=datetime(2022, 1, 1),
+    schedule_interval=timedelta(minutes=5),
+    catchup=False,
+    tags=["infra"],
+    default_args={"owner": "infra", "retries": 3, "retry_delay": timedelta(minutes=5)},
+) as dag:
+    BashOperator(
+        task_id="git_update",
+        bash_command=f"cd /root/airflow/dags/dwh-pipelines "
+        f"&& git reset --hard"
+        f"&& git submodule update --init --recursive ",
+    )
+
+with DAG(
+    dag_id="git_pull_tests",
+    start_date=datetime(2022, 1, 1),
+    schedule_interval=timedelta(minutes=5),
+    catchup=False,
+    tags=["infra"],
+    default_args={"owner": "infra", "retries": 3, "retry_delay": timedelta(minutes=5)},
+) as test_dag:
+    BashOperator(
+        task_id="git_update",
+        bash_command=f"cd /root/airflow/dags/airflow_test "
+        f"&& git fetch && git pull "
+        f"&& git submodule update --init --recursive ",
+    )

docker-compose.yaml

@@ -0,0 +1,23 @@
+version: '3.8'
+
+services:
+  airflow:
+    build: .
+    ports:
+      - 8080:8080
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      - AIRFLOW_ENV=DEV
+      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres@postgres:5432/postgres
+    volumes:
+      - ./start_local:/start_inc.sh
+
+  postgres:
+    image: postgres:latest
+    restart: always
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+    ports:
+      - 5432:5432

requirements.txt

@@ -0,0 +1,12 @@
+requests
+pandas
+beautifulsoup4
+gunicorn
+cryptography
+celery
+flower
+psycopg2_binary
+apache-airflow
+apache-airflow-providers-common-sql
+apache-airflow-providers-postgres
+google-api-python-client

start.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Если папки с дагами не существует, то создай и склонируй туда наш реполиторий с дагами
+[[ -d /root/airflow/dags/dwh-pipelines ]] \
+    || mkdir -p /root/airflow/dags/dwh-pipelines \
+    && git clone --recurse-submodules -b main https://github.com/profcomff/dwh-pipelines.git /root/airflow/dags/dwh-pipelines
+
+[[ -d /root/airflow/dags/airflow_test ]] \
+    || mkdir -p /root/airflow/dags/airflow_test \
+    && git clone --recurse-submodules -b main https://github.com/Men-of-Honest-Fate/airflow_test.git /root/airflow/dags/airflow_test
+
+# Инициализируй БД или проведи миграции для обновления
+airflow db init
+
+# Если файл доопределений существует, выполни его тоже
+[ -f /start_inc.sh ] && source /start_inc.sh
+
+airflow webserver && airflow scheduler

start_local.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+airflow users create \
+    --username admin \
+    --firstname Local \
+    --lastname User \
+    --role Admin \
+    --email user@localhost