diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d384ce0..0436bcb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,17 +5,64 @@ on: [push] jobs: build: runs-on: ubuntu-latest + container: python:3.8.7 + + services: + postgres: + image: postgres:10.8 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: 3.8.6 + + - name: Install postgresql-client package + run: | + apt-get update + apt-get install --yes --no-install-recommends postgresql-client + + - name: Create database + run: | + createdb airflow + env: + PGUSER: postgres + PGPASSWORD: postgres + PGHOST: postgres + - name: Install dependencies run: | + python -m venv env + . env/bin activate pip --version + pip install wheel pip install -r requirements.txt --use-deprecated=legacy-resolver + pip list + + - name: db init + env: + AIRFLOW__CORE__UNIT_TEST_MODE: True + AIRFLOW__CORE__DAGS_FOLDER: src/dags + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgres://postgres:postgres@postgres/airflow + AIRFLOW__CORE__EXECUTOR: LocalExecutor + run: | + . env/bin activate + airflow initdb + - name: Test with pytest run: | + . env/bin activate pytest + + env: + AIRFLOW__CORE__UNIT_TEST_MODE: True + AIRFLOW__CORE__DAGS_FOLDER: src/dags + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgres://postgres:postgres@postgres/airflow + AIRFLOW__CORE__EXECUTOR: LocalExecutor + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 750b303..0000000 --- a/requirements.txt +++ /dev/null @@ -1,117 +0,0 @@ -alembic==1.4.3 -apache-airflow>=1.10.12 -apispec==1.3.3 -appdirs==1.4.4 -argcomplete==1.12.1 -attrs>=20.1.0 -Babel==2.8.1 -CacheControl==0.12.6 -cached-property==1.5.2 -cachy==0.3.0 -cattrs==1.1.1 -certifi==2020.11.8 -cffi==1.14.3 -chardet==3.0.4 -cleo==0.8.1 -click==7.1.2 -clikit==0.6.2 -colorama==0.4.4 -colorlog==4.0.2 -configparser==3.5.3 -crashtest==0.3.1 -croniter==0.3.36 -cryptography==3.2.1 -defusedxml==0.6.0 -dill==0.3.3 -distlib==0.3.1 -dnspython==2.0.0 -docutils==0.16 -email-validator==1.1.2 -filelock==3.0.12 -Flask==1.1.2 -Flask-Admin==1.5.4 -Flask-AppBuilder==2.3.4 -Flask-Babel==1.0.0 -Flask-Caching==1.3.3 -Flask-JWT-Extended==3.25.0 -Flask-Login==0.4.1 -Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.4 -flask-swagger==0.2.14 -Flask-WTF==0.14.3 -funcsigs==1.0.2 -future==0.18.2 -graphviz==0.14.2 -gunicorn==20.0.4 -html5lib==1.1 -idna==2.10 -iso8601==0.1.13 -itsdangerous==1.1.0 -jeepney==0.5.0 -Jinja2==2.11.2 -json-merge-patch==0.2 -jsonschema==3.2.0 -keyring==21.5.0 -lazy-object-proxy==1.5.1 -lockfile==0.12.2 -Mako==1.1.3 -Markdown==2.6.11 -MarkupSafe==1.1.1 -marshmallow==2.21.0 -marshmallow-enum==1.5.1 -marshmallow-sqlalchemy==0.24.0 -msgpack==1.0.0 -natsort==7.0.1 -numpy==1.19.4 -packaging==20.4 -pandas==1.1.4 -pastel==0.2.1 -#pendulum==1.4.4 -pexpect==4.8.0 -pkginfo==1.6.1 -poetry==1.1.4 -poetry-core==1.0.0 -prison==0.1.3 -psutil==5.7.3 -ptyprocess==0.6.0 -pycparser==2.20 -Pygments==2.7.2 -PyJWT==1.7.1 -pylev==1.3.0 -pyparsing==2.4.7 -pyrsistent==0.17.3 -python-daemon==2.2.4 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-nvd3==0.15.0 -python-slugify==4.0.1 -python3-openid==3.2.0 -pytz==2020.4 -pytzdata==2020.1 -PyYAML==5.3.1 -requests==2.25.0 -requests-toolbelt==0.9.1 -SecretStorage==3.2.0 -setproctitle==1.1.10 -shellingham==1.3.2 -six==1.15.0 -SQLAlchemy==1.3.20 -SQLAlchemy-JSONField==0.9.0 -SQLAlchemy-Utils==0.36.8 -tabulate==0.8.7 -tenacity==4.12.0 -text-unidecode==1.3 -thrift==0.13.0 -tomlkit==0.7.0 -tzlocal==1.5.1 -unicodecsv==0.14.1 -urllib3==1.26.0 -virtualenv==20.1.0 -webencodings==0.5.1 -Werkzeug==0.16.1 -WTForms==2.3.3 -zope.deprecation==4.4.0 -python-dotenv -airtable==0.3.1 -psycopg2-binary==2.8.6 -pytest==6.2.1 diff --git a/tests/dags/test_tv_and_movies.py b/tests/dags/test_tv_and_movies.py index 4321b8f..0999636 100644 --- a/tests/dags/test_tv_and_movies.py +++ b/tests/dags/test_tv_and_movies.py @@ -1,9 +1,12 @@ import airflow from airflow.models import DagBag from airflow.configuration import conf +from os import environ conf.load_test_config() +environ['AIRTABLE_API_KEY'] = 'xxxx' + def test_dagbag_imports(): '''