Merge pull request #42 from puckel/custom-pip-p

Add pip install at run for custom python package to avoid full rebuild of the image

Dockerfile

@@ -1,4 +1,4 @@
-# VERSION 1.7.1.3-2
+# VERSION 1.7.1.3-3
 # AUTHOR: Matthieu "Puckel_" Roisil
 # DESCRIPTION: Basic Airflow container
 # BUILD: docker build --rm -t puckel/docker-airflow
@@ -25,7 +25,6 @@ ENV LC_ALL  en_US.UTF-8
 
 RUN set -ex \
     && buildDeps=' \
-        python-pip \
         python-dev \
         libkrb5-dev \
         libsasl2-dev \
@@ -39,6 +38,7 @@ RUN set -ex \
     && apt-get update -yqq \
     && apt-get install -yqq --no-install-recommends \
         $buildDeps \
+        python-pip \
         apt-utils \
         curl \
         netcat \

README.md

@@ -56,6 +56,12 @@ Go to Admin -> Connections and Edit "mysql_default" set this values (equivalent
 
 Check [Airflow Documentation](http://pythonhosted.org/airflow/)
 
+## Install custom python package
+
+- Create a file "requirements.txt" with the dedired python modules
+- Mount this file as a volume `-v $(pwd)/requirements.txt:/requirements.txt`
+- The entrypoint.sh script execute the pip install command (with --user option)
+
 ## UI Links
 
 - Airflow: [localhost:8080](http://localhost:8080/)

config/airflow.cfg

@@ -17,8 +17,7 @@ remote_base_log_folder =
 remote_log_conn_id =
 # Use server-side encryption for logs stored in S3
 encrypt_s3_logs = False
-# deprecated option for remote log storage, use remote_base_log_folder instead!
-# s3_log_folder =
+# remote_base_log_folder =
 
 # The executor class that airflow should use. Choices include
 # SequentialExecutor, LocalExecutor, CeleryExecutor

docker-compose-CeleryExecutor.yml

@@ -19,7 +19,7 @@ services:
             - POSTGRES_DB=airflow
 
     webserver:
-        image: puckel/docker-airflow:1.7.1.3-2
+        image: puckel/docker-airflow:1.7.1.3-3
         restart: always
         depends_on:
             - postgres
@@ -34,7 +34,7 @@ services:
         command: webserver
 
     flower:
-        image: puckel/docker-airflow:1.7.1.3-2
+        image: puckel/docker-airflow:1.7.1.3-3
         restart: always
         depends_on:
             - rabbitmq
@@ -45,7 +45,7 @@ services:
         command: flower
 
     scheduler:
-        image: puckel/docker-airflow:1.7.1.3-2
+        image: puckel/docker-airflow:1.7.1.3-3
         restart: always
         depends_on:
             - webserver
@@ -57,7 +57,7 @@ services:
         command: scheduler -n 5
 
     worker:
-        image: puckel/docker-airflow:1.7.1.3-2
+        image: puckel/docker-airflow:1.7.1.3-3
         restart: always
         depends_on:
             - scheduler

docker-compose-LocalExecutor.yml

@@ -8,7 +8,7 @@ services:
             - POSTGRES_DB=airflow
 
     webserver:
-        image: puckel/docker-airflow:1.7.1.3-2
+        image: puckel/docker-airflow:1.7.1.3-3
         restart: always
         depends_on:
             - postgres
@@ -22,12 +22,13 @@ services:
         command: webserver
 
     scheduler:
-        image: puckel/docker-airflow:1.7.1.3-2
+        image: puckel/docker-airflow:1.7.1.3-3
         restart: always
         depends_on:
             - webserver
         # volumes:
-        #     - /localpath/to/dags:/usr/local/airflow/dags
+            # - ./requirements.txt:/requirements.txt:ro
+            # - /localpath/to/dags:/usr/local/airflow/dags
         environment:
             # - LOAD_EX=n
             - EXECUTOR=Local

script/entrypoint.sh

@@ -14,6 +14,11 @@ if [ "x$LOAD_EX" = "xn" ]; then
     sed -i "s/load_examples = True/load_examples = False/" "$AIRFLOW_HOME"/airflow.cfg
 fi
 
+# Install custome python package if requirements.txt is present
+if [ -e "/requirements.txt" ]; then
+    $(which pip) install --user -r /requirements.txt
+fi
+
 # Generate Fernet key
 sed -i "s|\$FERNET_KEY|$FERNET_KEY|" "$AIRFLOW_HOME"/airflow.cfg