airflow-cluster moved to extra-quickstarters

Author: gerardcl

airflow-cluster/Jenkinsfile

@@ -0,0 +1,29 @@
+def odsNamespace = env.ODS_NAMESPACE ?: 'ods'
+def odsGitRef = env.ODS_GIT_REF ?: 'master'
+def odsImageTag = env.ODS_IMAGE_TAG ?: 'latest'
+
+library("ods-jenkins-shared-library@${odsGitRef}")
+
+def dockerRegistry
+
+node {
+  dockerRegistry = env.DOCKER_REGISTRY
+}
+
+odsQuickstarterPipeline(
+  imageStreamTag: "${odsNamespace}/jenkins-agent-airflow:${odsImageTag}",
+) { context ->
+
+  odsQuickstarterStageCopyFiles(context)
+
+  stage('Setup OpenShift resources') {
+    sh "cd ${context.sourceDir} ; sh ./custom-create-components.sh \
+      -p ${context.projectId} -c ${context.componentId} -b ${context.gitUrlHttp.split('/' + context.projectId)[0]} \
+      -r ${dockerRegistry} -gr ${odsGitRef} \
+      -oa 'your-openshift-apihost' -oc 'https://your-openshift-console' "
+  }
+
+  odsQuickstarterStageRenderJenkinsfile(context)
+
+  odsQuickstarterStageRenderSonarProperties(context)
+}

airflow-cluster/Jenkinsfile.template

@@ -0,0 +1,69 @@
+// See https://www.opendevstack.org/ods-documentation/ for usage and customization.
+
+@Library('ods-jenkins-shared-library@@ods_git_ref@') _
+
+odsComponentPipeline(
+    imageStreamTag: '@ods_namespace@/jenkins-agent-airflow:@ods_image_tag@',
+    componentId: 'airflow-worker',
+    testResults : 'artifacts',
+    branchToEnvironmentMapping: [
+        'master': 'dev',
+        // 'release/': 'test'
+    ]
+) { context ->
+    stageDAGTest(context)
+    stageUnitTest(context)
+    stageBuild(context)
+    odsComponentStageScanWithSonar(context)
+    odsComponentStageBuildOpenShiftImage(context, [buildTimeoutMinutes: 25])
+    stageTagImage(context)
+    stagePublishDAGs(context)
+}
+
+def stageDAGTest(def context) {
+    stage('DAG Integrity Tests') {
+        sh 'sh test_dag_integrity.sh'
+    }
+}
+
+def stageUnitTest(def context) {
+    stage('Unit Tests') {
+        sh "pip install -i ${context.nexusHostWithBasicAuth}/repository/pypi-all/simple --trusted-host ${context.nexusHostWithoutScheme} -r src/requirements.txt --user"
+        sh 'sh test_all.sh'
+    }
+}
+
+def stageBuild(def context) {
+    stage('Build') {
+        sh 'sh build.sh'
+    }
+}
+
+def stageTagImage(def context) {
+    stage('Tag build') {
+        if (!context.environment) {
+            println("Skipping for empty environment ...")
+            return
+        }
+        sh(
+            script: "oc -n ${context.targetProject} tag ${context.componentId}:${context.tagversion} ${context.componentId}:latest",
+            label: "Update latest tag of is/${context.componentId} to ${context.tagversion}"
+        )
+    }
+}
+
+def stagePublishDAGs(def context) {
+    stage('Publish DAGs') {
+        def airflow_webserver_info = sh(returnStdout: true, script: "oc get pods --sort-by=.status.startTime --no-headers -l component=airflow-webserver -n ${context.targetProject} | tail -n1").trim().split(/\s+/)
+        def airflow_scheduler_info = sh(returnStdout: true, script: "oc get pods --sort-by=.status.startTime --no-headers -l component=airflow-scheduler -n ${context.targetProject} | tail -n1").trim().split(/\s+/)
+
+        if (airflow_webserver_info[2] != "Running" || airflow_scheduler_info[2] != "Running") {
+            error("Airflow cluster is not running or does not exist")
+        }
+
+        sh "oc rsync --no-perms=true --delete=true --exclude=.keep --exclude=__pycache__ src/dags ${airflow_webserver_info[0]}:/opt/app-root/src/airflow/ -n ${context.targetProject}"
+        sh "oc rsync --no-perms=true --delete=true --exclude=.keep --exclude=__pycache__ src/dags ${airflow_scheduler_info[0]}:/opt/app-root/src/airflow/ -n ${context.targetProject}"
+        sh "oc rsync --no-perms=true --delete=true --exclude=.keep --exclude=__pycache__ src/dag_deps ${airflow_webserver_info[0]}:/opt/app-root/src/airflow/ -n ${context.targetProject}"
+        sh "oc rsync --no-perms=true --delete=true --exclude=.keep --exclude=__pycache__ src/dag_deps ${airflow_scheduler_info[0]}:/opt/app-root/src/airflow/ -n ${context.targetProject}"
+    }
+}

airflow-cluster/README.md

@@ -0,0 +1,5 @@
+# AirFlow Openshift Cluster Boilerplate
+
+Documentation is located in our [official documentation](https://www.opendevstack.org/ods-documentation/ods-quickstarters/latest/index.html)
+
+Please update documentation in the [antora page directory](https://github.com/opendevstack/ods-quickstarters/tree/master/docs/modules/ROOT/pages)

airflow-cluster/base-images/README.md

@@ -0,0 +1,8 @@
+# Airflow base images
+
+Images used in [ods-quickstarters/airflow-cluster](https://github.com/opendevstack/ods-quickstarters/tree/master/airflow-cluster)
+
+## Contents
+
+1. [Airflow](airflow) for OpenDevStack
+2. [Elasticsearch](elasticsearch) for OpenDevStack

airflow-cluster/base-images/airflow/Airflow Architecture Diagram.png

@@ -0,0 +1,138 @@
+# RedHat image should be specified in the build config
+# FROM registry.access.redhat.com/rhscl/python-36-rhel7
+
+# Centos is used for enabling local builds
+FROM centos/python-36-centos7
+
+ARG AIRFLOW_VERSION=1.10.3
+ARG FILE_BEATS_VERSION=7.0.0
+
+LABEL maintainer="Hugo Wruck Schneider <hugo.wruck_schneider@boehringer-ingelheim.com>"
+
+ENV SUMMARY="OpenDevStack Provided Airflow ${AIRFLOW_VERSION} Base Image compatible with OpenShift"
+ENV DESCRIPTION="OpenDevStack Provided Airflow ${AIRFLOW_VERSION} Base Image compatible with OpenShift. This image \
+should be used for running airflow's webserver and scheduler and as base image for building the worker's image \
+\
+Packages being used: \
+    - Airflow ${AIRFLOW_VERSION}\
+    - FileBeat ${FILE_BEATS_VERSION}\
+"
+LABEL   "name"="openshift3/airflow" \
+        "summary"=$SUMMARY \
+        "description"=$DESCRIPTION \
+        "version"=$AIRFLOW_VERSION \
+        "io.openshift.tags"="python36,python,airflow,airflow1102" \
+        "io.k8s.description"=$DESCRIPTION \
+        "io.openshift.expose-services"="8080:http" \
+        "io.k8s.display-name"="Airflow ${AIRFLOW_VERSION}" \
+        "com.redhat.component"="airflow-rhel7-docker"
+
+ARG PYTHON_DEPS="Flask==1.0.0 tzlocal==1.5.1 urllib3==1.25.3 thrift==0.11.0 tabulate==0.8.3 six==1.12.0 PyYAML==5.1.2 \
+                    pytzdata==2019.2 python-dateutil==2.8.0 Pygments==2.4.2 pyasn1-modules==0.2.6 psutil==5.6.3"
+
+ARG AIRFLOW_DEPS=""
+
+ENV AIRFLOW_HOME=$HOME/airflow
+ENV SLUGIFY_USES_TEXT_UNIDECODE yes
+
+ARG NEXUS_USERNAME
+ARG NEXUS_PASSWORD
+ARG NEXUS_URL
+
+# Define en_US.
+ENV LANGUAGE en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV LC_CTYPE en_US.UTF-8
+ENV LC_MESSAGES en_US.UTF-8
+
+
+USER root
+
+ADD https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILE_BEATS_VERSION}-x86_64.rpm ${HOME}
+
+RUN buildDeps=' \
+        freetds-devel \
+        krb5-devel \
+        cyrus-sasl-devel \
+        openssl-devel \
+        libffi-devel \
+        postgresql-devel \
+        git \
+    ' \
+    && pipDeps="\
+       pytz==2019.2 \
+       pyOpenSSL==19.0.0 \
+       paramiko==2.6.0 \
+       sshtunnel==0.1.5 \
+       ndg-httpsclient==0.5.1 \
+       pyasn1==0.4.6 \
+       requests-oauthlib==1.1.0 \
+       apache-airflow[crypto,postgres,hive,jdbc,mysql,ssh,kubernetes,elasticsearch${AIRFLOW_DEPS:+,}${AIRFLOW_DEPS}]==${AIRFLOW_VERSION} \
+    " \
+    && rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \
+    && yum install --assumeyes \
+        $buildDeps \
+        freetds \
+        mariadb-libs \
+        curl \
+        rsync \
+        nmap-ncat \
+    && localedef -c -f UTF-8 -i en_US en_US.UTF-8 \
+    && if [[ ! -z ${NEXUS_URL} ]]; \
+        then pip install -U -i https://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_URL:8}/repository/pypi-all/simple pip setuptools wheel \
+        && pip install -i https://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_URL:8}/repository/pypi-all/simple $pipDeps; \
+        else pip install -U pip setuptools wheel \
+        && pip install $pipDeps; \
+    fi \
+    && yum localinstall --assumeyes ${HOME}/filebeat-${FILE_BEATS_VERSION}-x86_64.rpm \
+    && yum clean all \
+    && rm -rf /var/cache/yum
+
+COPY config/filebeat.yml /etc/filebeat/filebeat.yml
+
+### Log Stash - END
+
+COPY scripts/entrypoint.sh /entrypoint.sh
+COPY config/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
+
+RUN mkdir -p ${AIRFLOW_HOME}/dags \
+    && mkdir -p ${AIRFLOW_HOME}/plugins \
+    && mkdir -p ${AIRFLOW_HOME}/dag_deps
+
+COPY dist/oauth ${HOME}/oauth
+COPY dist/openshift_plugin ${HOME}/openshift_plugin
+
+RUN pip install ${HOME}/oauth \
+    && CFLAGS=-std=c99 pip install ${HOME}/openshift_plugin \
+    && chgrp -R 0 ${AIRFLOW_HOME} \
+    && chmod -R g=u ${AIRFLOW_HOME} \
+    && chmod +x /entrypoint.sh \
+    && chmod g+w /etc/passwd \
+    && mkdir -p /var/lib/filebeat \
+    && chgrp -R 0 /var/lib/filebeat  \
+    && chmod -R g=u /var/lib/filebeat  \
+    && mkdir -p /var/log/filebeat \
+    && chgrp -R 0 /var/log/filebeat  \
+    && chmod -R g=u /var/log/filebeat  \
+    && chmod g+r /etc/filebeat/filebeat.yml \
+    && chgrp -R 0 /etc/pki/ca-trust \
+    && chmod -R g=u /etc/pki/ca-trust
+
+RUN if [ -n "${PYTHON_DEPS}" ]; then \
+        if [[ ! -z ${NEXUS_URL} ]]; \
+            then pip install -i https://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_URL:8}/repository/pypi-all/simple -U ${PYTHON_DEPS}; \
+            else pip install -U ${PYTHON_DEPS}; \
+        fi \
+    fi
+
+USER 1001
+
+ENV PYTHONPATH="${AIRFLOW_HOME}/dag_deps:${PYTHONPATH}"
+
+# Only the port for the webserver (8080) is exposed. Exposing the ports for flower
+# and for the http server on the worker is not advisable but if need this should be changed.
+EXPOSE 8080
+
+WORKDIR ${AIRFLOW_HOME}
+ENTRYPOINT ["/entrypoint.sh"]

airflow-cluster/base-images/airflow/Dockerfile

@@ -0,0 +1,132 @@
+# AirFlow OpenDevStack Shared Image
+
+This image provides AirFlow 1.10.3 for OpenShift.
+
+## Setup
+
+The AirFlow setup that this image provides is based on the KubernetesExecutor and will
+start worker pods on demand. It is also setup to use an ElasticSearch instance as the log
+repository for all workers. As illustrated bellow:
+
+![Airflow Architecture](Airflow Architecture Diagram.png?raw=true "Airflow Architecture")
+
+To setup the whole infrastructure the Airlfow QuickStarter should be used.
+
+## Contents
+
+The image contains all dependencies and Airflow extras to run Airflow 1.10.3 in this setup.
+It also includes FileBeat 7.0.0 to send all log files to ElasticSearch
+
+### Generic OAuth backend
+
+A generic OAuth backend is included in this image to enabled Airflow to
+authenticate against any OAuth server including the one from OpenShift.
+
+### OpenShift Plugin
+
+An OpenShift Plugin was added to address compatibility and security issues
+between the Kubernetes Executor and Openshift.
+
+This plugin includes two views.
+
+* One for inspecting the pods that are and/or were part of the cluster
+* A second one to sync DAG information from the worker image back to
+the webserver and scheduler
+
+## Security
+
+### Authentication
+
+The authentication method is enabled and uses OpenShift OAuth to authenticate
+users in the WebServer.
+
+## Configuration
+
+None Airflow configuration options were changed and they can be used as documented
+in the [Airflow Documentation](https://airflow.apache.org/project.html)
+
+Besides all Airflow configuration, the OAuth backend and the OpenShift Plugin
+include a small set of configuration.
+
+#### OAuth configuration
+
+The configuration section is `oauth` and must have the followin keys:
+
+```ini
+[oauth]
+# base_url contains alwayes the value of the OpenShift API url
+base_url = https://your.openshift.api.url
+
+# client_id must have the service account name which serves as OAuth client
+client_id = system:serviceaccount:your-namespace:your-service-account-name
+
+
+# oauth_callback_route should not change unless you know what you are doing
+oauth_callback_route = /oauth2callback
+
+# authorize_url authorization api of OpenShift.
+authorize_url = https://your.openshift.api.url/oauth/authorize
+
+# access_token_url token api of OpenShift.
+access_token_url = https://your.openshift.api.url/oauth/token
+
+# access_token_method Method which should be used for calling the API
+access_token_method = POST
+
+# user_info_url User information API of OpenShift
+user_info_url = https://your.openshift.api.url/apis/user.openshift.io/v1/users/~
+
+# username_key and email_key are path inside the reply of user_info_url where
+# the username and email of the user can be found
+username_key = metadata.name
+email_key = metadata.name
+
+# oauth_permission_backend The OAuth authorization backend
+oauth_permission_backend=airflow_oauth.contrib.auth.backends.openshift_permission_backend
+```
+
+NOTE: if SSL verification fails when calling oauth callback, users can import the host certificate chain using `AIRFLOW_HOSTS_TO_TRUST` or set
+environment variable `AIRFLOW__HTTP_CLIENT__INSECURE` to `true`.
+
+### OpenShift Plugin Settings
+
+The section `openshift_plugin` supports the plugin and must have the followin keys:
+
+```ini
+[openshift_plugin]
+# OpenShift roles of an user which will allow access to Airflow
+access_roles=role1,role2
+# OpenShift roles of an user which will allow superuser access to Airflow
+superuser_roles=role1
+# Base OpenShift console url for build the links to airflow resources
+openshift_console_url=https://localhost
+```
+
+## Environemnt Variables
+
+
+A set of Environment Variables where included to allow a easy way of using Config Maps
+and Secrets. They are:
+
+| Name | Type | Description|
+|-----|------|------------|
+|**AIRFLOW_COMMAND**| String (Required) | Airflow command that should be executed or empty for a custom commands. It will define if the container will behave as the webserver, schediler and so on.  The availabled options are: 'webserver', 'worker', 'scheduler', 'flower' or 'version' |
+|AIRFLOW_HOSTS_TO_TRUST| String | A semi-colon separeted list of hosts to be trusted in the format `hostname1:port1;hostname2:port2;...` . This host will have their certificate chain automaticaly trusted |
+|POSTGRES_HOST| String (Required) | Postgresql host to which Airflow should connect |
+|POSTGRES_PORT|String| Postgresql port to which Airflow should connect. Default: 5432 |
+|POSTGRES_USER|String (Required) | Postgresql username which Airflow should use |
+|POSTGRES_PASSWORD|String (Required)| Postgresql password which Airflow should use |
+|POSTGRES_DATABASE|String| Database name that should be used. Default: airflow|
+|START_FILE_BEAT| 1 or 0 | 0 if FileBeat service should not be started. Default: 1 (it should be stared) |
+|ELASTICSEARCH_URL| URL | ElasticSearch url to which Airflow should connect|
+|ELASTICSEARCH_USERNAME| String | ElasticSearch username which Airflow should use |
+|ELASTICSEARCH_PASSWORD| String | ElasticSearch password which Airflow should use |
+
+
+Since multiple deployment configs will use the same configuration and the values, besides
+the `AIRFLOW_COMMAND`, it is advisable to create a config map and mount it to each
+Airflow deployment config.
+
+**All Airflow configuration, including OpenShift Plugin configuration,
+can be done using environment variables** as documented in
+https://airflow.apache.org/howto/set-config.html.