Merge remote-tracking branch 'origin/master' into prod

.env.template

@@ -1,4 +1,4 @@
-# get the complete env from other volunteers, please
+AIRFLOW_HOME=/opt/airflow
 BIGQUERY_PROJECT=pycontw-225217
-DATATEAM_DISCORD_WEBHOOK=<>
-GOOGLE_APPLICATION_CREDENTIALS=/usr/local/airflow/service-account.json
+GOOGLE_APPLICATION_CREDENTIALS=/opt/airflow/service-account.json
+AIRFLOW__CORE__FERNET_KEY=paste-your-fernet-key-here

.github/workflows/dockerimage.yml

@@ -1,65 +1,45 @@
 name: Docker Image CI
-
 on:
   push:
     branches: [ master, prod ]
   pull_request:
     branches: [ master, prod ]
 env:
-  RC_NAME: davidtnfsh/pycon_etl
-
+  RC_NAME: asia-east1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/data-team/pycon-etl
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Login to docker hub
-      uses: actions-hub/docker/login@master
-      env:
-        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-
+    - uses: actions/checkout@v4
+    - name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v1
+      with:
+        credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+    - name: Configure docker to use gcloud command-line tool as a credential helper
+      run: |
+        gcloud auth configure-docker asia-east1-docker.pkg.dev
     - name: Pull cache
       run: |
-        docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-        docker pull ${RC_NAME}:cache
-
+        docker pull ${RC_NAME}:cache || true
     - name: Build the Docker image
-      if: always()
       run: |
-        docker build -t ${RC_NAME}:${GITHUB_SHA} --cache-from ${RC_NAME}:cache .
-        docker tag ${RC_NAME}:${GITHUB_SHA} ${RC_NAME}:cache
+        docker build -t ${RC_NAME}:cache --cache-from ${RC_NAME}:cache .
         docker build -t ${RC_NAME}:test --cache-from ${RC_NAME}:cache -f Dockerfile.test .
-        docker tag ${RC_NAME}:${GITHUB_SHA} ${RC_NAME}:staging
-        docker tag ${RC_NAME}:${GITHUB_SHA} ${RC_NAME}:latest
-
     - name: Run test
       run: |
-        docker run -d --rm -p 8080:8080 --name airflow -v $(pwd)/dags:/usr/local/airflow/dags -v $(pwd)/fixtures:/usr/local/airflow/fixtures ${RC_NAME}:test webserver
+        docker run -d --rm -p 8080:8080 --name airflow -v $(pwd)/dags:/opt/airflow/dags -v $(pwd)/fixtures:/opt/airflow/fixtures ${RC_NAME}:test webserver
         sleep 10
-        docker exec airflow bash -c "airflow test OPENING_CRAWLER_V1 CRAWLER 2020-01-01"
-        docker exec airflow bash -c "airflow test QUESTIONNAIRE_2_BIGQUERY TRANSFORM_data_questionnaire 2020-09-29"
-
-    - name: Push Cache to docker registry
-      uses: actions-hub/docker@master
-      if: always()
-      with:
-        args: push ${RC_NAME}:cache
-
-    - name: Push GITHUB_SHA to docker registry
-      uses: actions-hub/docker@master
-      if: always()
-      with:
-        args: push ${RC_NAME}:${GITHUB_SHA}
-
-    - name: Push staging to docker registry
-      uses: actions-hub/docker@master
-      if: ${{ github.ref == 'refs/heads/master' }} && success()
-      with:
-        args: push ${RC_NAME}:staging
-
-    - name: Push prod version to docker registry
-      uses: actions-hub/docker@master
+    - name: Push cache to Google Container Registry
+      if: success()
+      run: |
+        docker push ${RC_NAME}:cache
+    - name: Push staging to Google Container Registry
+      if: github.ref == 'refs/heads/master' && success()
+      run: |
+        docker tag ${RC_NAME}:cache ${RC_NAME}:staging
+        docker push ${RC_NAME}:staging
+    - name: Push prod version to Google Container Registry
       if: github.ref == 'refs/heads/prod' && success()
-      with:
-        args: push ${RC_NAME}:latest
+      run: |
+        docker tag ${RC_NAME}:cache ${RC_NAME}:latest
+        docker push ${RC_NAME}:latest

.github/workflows/python.yml

@@ -2,38 +2,36 @@ name: Python CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 env:
-  POETRY_VIRTUALENVS_CREATE: false
   AIRFLOW_TEST_MODE: true
 jobs:
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.8
+      - uses: actions/checkout@v2
 
-    - name: Install dependencies
-      run: |
-        pip install poetry==1.1.14
-        poetry config experimental.new-installer false
-        poetry install
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
 
-    - name: Run linters
-      run: make lint
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt -r requirements-dev.txt -c constraints-3.8.txt
+          pip install black==19.10b0 click==7.1.2
 
-    - name: Run test
-      run: make test
+      - name: Run linters
+        run: make lint
 
-    - name: Coverage
-      run: make coverage
+      - name: Run test
+        run: make test
 
+      - name: Coverage
+        run: make coverage
 
     # CD part
     # - name: Push dags to GCS

.gitignore

@@ -1,4 +1,5 @@
 # project stuff
+.env.production
 .env.staging
 env
 env.sh

Dockerfile

@@ -1,24 +1,33 @@
-FROM puckel/docker-airflow:1.10.9
+ARG AIRFLOW_VERSION=1.10.15
+ARG PYTHON_VERSION=3.8
+
+FROM apache/airflow:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
+
 USER root
-ENV POETRY_VIRTUALENVS_CREATE=false \
-    POETRY_CACHE_DIR='/var/cache/pypoetry' \
-    GOOGLE_APPLICATION_CREDENTIALS='/usr/local/airflow/service-account.json'
 
-RUN apt-get update \
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 \
+    && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys B7B3B788A8D3785C \
+    && apt-get update \
     && apt-get install -y --no-install-recommends git \
     # 1. if you don't need postgres, remember to remove postgresql-dev and sqlalchemy
     # 2. libglib2.0-0 libsm6 libxext6 libxrender-dev libgl1-mesa-dev are required by opencv
     # 3. git is required by pip install git+https
-    && pip install --no-cache-dir poetry==1.1.7 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-COPY pyproject.toml pyproject.toml
-COPY poetry.lock poetry.lock
-RUN python -m poetry install --no-interaction --no-ansi --no-dev \
-    # Cleaning poetry installation's cache for production:
-    && rm -rf "$POETRY_CACHE_DIR" \
-    && pip uninstall -yq poetry
+COPY entrypoint.sh /entrypoint.sh
+
+RUN chmod +x /entrypoint.sh
+
 USER airflow
-COPY dags /usr/local/airflow/dags
-COPY airflow.cfg airflow.cfg
+
+COPY ./requirements.txt ${AIRFLOW_HOME}/requirements.txt
+COPY ./constraints-3.8.txt ${AIRFLOW_HOME}/constraints-3.8.txt
+
+RUN pip install --no-cache-dir -r ${AIRFLOW_HOME}/requirements.txt --constraint constraints-3.8.txt
+
+COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
+
+COPY --chown=airflow:root dags ${AIRFLOW_HOME}/dags
+
+ENTRYPOINT ["/entrypoint.sh"]

Dockerfile.test

@@ -1,5 +1,36 @@
-FROM davidtnfsh/pycon_etl:prod
-ENV AIRFLOW_TEST_MODE=True
-ENV FERNET_KEY="uMnRC6ingT/WjPzPiXLvbWTJYzaA3sJdJRVkinceVp4="
+ARG AIRFLOW_VERSION=1.10.15
+ARG PYTHON_VERSION=3.8
+
+FROM apache/airflow:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
+
 USER root
-COPY airflow.test.cfg airflow.cfg
+
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 \
+    && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys B7B3B788A8D3785C \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends git \
+    # 1. if you don't need postgres, remember to remove postgresql-dev and sqlalchemy
+    # 2. libglib2.0-0 libsm6 libxext6 libxrender-dev libgl1-mesa-dev are required by opencv
+    # 3. git is required by pip install git+https
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY entrypoint.sh /entrypoint.sh
+
+RUN chmod +x /entrypoint.sh
+
+USER airflow
+
+COPY ./requirements.txt ${AIRFLOW_HOME}/requirements.txt
+COPY ./requirements-dev.txt ${AIRFLOW_HOME}/requirements-dev.txt
+COPY ./constraints-3.8.txt ${AIRFLOW_HOME}/constraints-3.8.txt
+
+RUN pip install --no-cache-dir -r ${AIRFLOW_HOME}/requirements.txt -r ${AIRFLOW_HOME}/requirements-dev.txt --constraint constraints-3.8.txt
+
+COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
+
+COPY --chown=airflow:root dags ${AIRFLOW_HOME}/dags
+
+ENV AIRFLOW_TEST_MODE=True
+
+ENTRYPOINT ["/entrypoint.sh"]

Makefile

@@ -1,17 +1,30 @@
-VENV_PREFIX=poetry run
-
 lint:
-	$(VENV_PREFIX) black . --check
-	$(VENV_PREFIX) isort --check-only .
-	$(VENV_PREFIX) flake8 .
-	$(VENV_PREFIX) mypy dags/ tests/
+	black . --check
+	isort --check-only .
+	flake8 .
+	mypy dags/ tests/
 
 format:
-	$(VENV_PREFIX) black .
-	$(VENV_PREFIX) isort .
+	black .
+	isort .
 
 test:
-	PYTHONPATH=./dags $(VENV_PREFIX) pytest
+	PYTHONPATH=./dags pytest
 
 coverage:
-	PYTHONPATH=./dags $(VENV_PREFIX) pytest --cov=dags tests
+	PYTHONPATH=./dags pytest --cov=dags tests
+
+build-dev:
+	docker-compose -f ./docker-compose-dev.yml build
+
+deploy-dev:
+	docker-compose -f ./docker-compose-dev.yml up -d
+
+down-dev:
+	docker-compose -f ./docker-compose-dev.yml down
+
+deploy-prod:
+	docker-compose -f ./docker-compose.yml up -d
+
+down-prod:
+	docker-compose -f ./docker-compose.yml down