Apache airflow ironbank added (rapidfort#331)

* Added apache airflow/ironbank image directory Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> * Minor Update Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> * Minor change(Used k8s) Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> * Docker-compose working Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> * Added airflow/ironbank image dorectory Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> * Cleanup Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> * Updated workflows Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> * Updates as per reviewdog Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> * Cleanup selenium tests Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> * Added .rfignore and verified its content. Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> --------- Signed-off-by: Kirandeep Singh <kirandeep@rapidfort.com> Co-authored-by: Kirandeep Singh <kirandeep@rapidfort.com>
uandhgroup · Aug 10, 2023 · 7a9017d · 7a9017d
1 parent 7998919
commit 7a9017d
Show file tree

Hide file tree

Showing 12 changed files with 316 additions and 1 deletion.
diff --git a/.github/workflows/image_run_pr_v3.yml b/.github/workflows/image_run_pr_v3.yml
@@ -76,6 +76,70 @@ jobs:
            -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \
            -d '{ "rapidfort/airflow": 1 }' \
            https://data-receiver.rapidfort.com/counts/internal_image_pulls
+  airflow-ib:
+    runs-on: ubuntu-latest
+
+    environment: actions-cicd-pr
+
+    steps:
+      - uses: actions/checkout@v3
+
+
+      - name: Check if test required
+        id: test-required
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 scripts/tests_required.py airflow-ib airflow/airflow/ironbank output.txt
+          cat output.txt >> $GITHUB_OUTPUT
+
+      - name: Start minikube
+
+        if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }}
+
+        with:
+          memory: 6g
+        uses: medyagh/setup-minikube@master
+      - name: Check k8s cluster !
+
+        if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }}
+
+        run: kubectl get pods -A
+      - name: Setup ubuntu
+        env:
+          RF_ACCESS_ID: ${{ secrets.RF_ACCESS_ID }}
+          RF_SECRET_ACCESS_KEY: ${{ secrets.RF_SECRET_ACCESS_KEY }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          RF_PLATFORM_HOST: ${{ secrets.RF_PLATFORM_HOST }}
+
+        if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }}
+
+        run: ./scripts/setup.sh
+
+      - name: Create images for testing PR
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          RAPIDFORT_ACCOUNT: ${{ secrets.RAPIDFORT_ACCOUNT }}
+          IB_DOCKER_USERNAME: ${{ secrets.IB_DOCKER_USERNAME }}
+          IB_DOCKER_PASSWORD: ${{ secrets.IB_DOCKER_PASSWORD }}
+        if: ${{ steps.test-required.outputs.TEST_REQUIRED == 'yes' }}
+        run: python3 community_images/common/orchestrator/main.py hourly_run airflow/airflow/ironbank
+
+
+      - name: Report pulls
+        env:
+          PULL_COUNTER_MAGIC_TOKEN: ${{ secrets.PULL_COUNTER_MAGIC_TOKEN }}
+
+        if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }}
+
+        run: |
+          curl -X POST \
+           -H "Accept: application/json" \
+           -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \
+           -d '{ "rapidfort/airflow-ib": 1 }' \
+           https://data-receiver.rapidfort.com/counts/internal_image_pulls
   apache:
     runs-on: ubuntu-latest
 

diff --git a/.github/workflows/image_run_v3.yml b/.github/workflows/image_run_v3.yml
@@ -59,6 +59,54 @@ jobs:
            -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \
            -d '{ "rapidfort/airflow": 1 }' \
            https://data-receiver.rapidfort.com/counts/internal_image_pulls
+  airflow-ib:
+    runs-on: ubuntu-latest
+
+    environment: actions-cicd
+
+    steps:
+      - uses: actions/checkout@v3
+
+
+      - name: Start minikube
+
+        with:
+          memory: 6g
+        uses: medyagh/setup-minikube@master
+      - name: Check k8s cluster !
+
+        run: kubectl get pods -A
+      - name: Setup ubuntu
+        env:
+          RF_ACCESS_ID: ${{ secrets.RF_ACCESS_ID }}
+          RF_SECRET_ACCESS_KEY: ${{ secrets.RF_SECRET_ACCESS_KEY }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          RF_PLATFORM_HOST: ${{ secrets.RF_PLATFORM_HOST }}
+
+        run: ./scripts/setup.sh
+
+      - name: Create images and publish to dockerhub
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          RAPIDFORT_ACCOUNT: ${{ secrets.RAPIDFORT_ACCOUNT }}
+          IB_DOCKER_USERNAME: ${{ secrets.IB_DOCKER_USERNAME }}
+          IB_DOCKER_PASSWORD: ${{ secrets.IB_DOCKER_PASSWORD }}
+        if: github.event_name != 'pull_request'
+        run: python3 community_images/common/orchestrator/main.py hourly_run airflow/airflow/ironbank --publish
+
+
+      - name: Report pulls
+        env:
+          PULL_COUNTER_MAGIC_TOKEN: ${{ secrets.PULL_COUNTER_MAGIC_TOKEN }}
+
+        run: |
+          curl -X POST \
+           -H "Accept: application/json" \
+           -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \
+           -d '{ "rapidfort/airflow-ib": 1 }' \
+           https://data-receiver.rapidfort.com/counts/internal_image_pulls
   apache:
     runs-on: ubuntu-latest
 

diff --git a/.github/workflows/monitor.yml b/.github/workflows/monitor.yml
@@ -14,6 +14,8 @@ jobs:
     steps:
       - name: Pull Apache Airflow
         run: docker pull rapidfort/airflow
+      - name: Pull Apache Airflow Ironbank
+        run: docker pull rapidfort/airflow-ib
       - name: Pull Apache Airflow Scheduler
         run: docker pull rapidfort/airflow-scheduler
       - name: Pull Apache Airflow Worker
@@ -160,5 +162,5 @@ jobs:
           curl -X POST \
            -H "Accept: application/json" \
            -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \
-           -d '{ "rapidfort/airflow": 1 ,  "rapidfort/airflow-scheduler": 1 ,  "rapidfort/airflow-worker": 1 ,  "rapidfort/apache": 1 ,  "rapidfort/apache2-ib": 1 ,  "rapidfort/apache-official": 1 ,  "rapidfort/cassandra-official": 1 ,  "rapidfort/consul": 1 ,  "rapidfort/consul-ib": 1 ,  "rapidfort/consul-official": 1 ,  "rapidfort/couchdb": 1 ,  "rapidfort/couchdb_3-ib": 1 ,  "rapidfort/couchdb-official": 1 ,  "rapidfort/curl": 1 ,  "rapidfort/elasticsearch": 1 ,  "rapidfort/elasticsearch-official": 1 ,  "rapidfort/envoy": 1 ,  "rapidfort/envoy-official": 1 ,  "rapidfort/etcd": 1 ,  "rapidfort/etcd-ib": 1 ,  "rapidfort/fluentd": 1 ,  "rapidfort/fluentd-ib": 1 ,  "rapidfort/fluentd-official": 1 ,  "rapidfort/ghost": 1 ,  "rapidfort/grafana-ib": 1 ,  "rapidfort/haproxy": 1 ,  "rapidfort/haproxy24-ib": 1 ,  "rapidfort/haproxy-official": 1 ,  "rapidfort/influxdb": 1 ,  "rapidfort/kong": 1 ,  "rapidfort/mariadb": 1 ,  "rapidfort/mariadb-ib": 1 ,  "rapidfort/mariadb-official": 1 ,  "rapidfort/memcached": 1 ,  "rapidfort/memcached-ib": 1 ,  "rapidfort/memcached-official": 1 ,  "rapidfort/microsoft-sql-server-2019-ib": 1 ,  "rapidfort/mongodb": 1 ,  "rapidfort/mongodb-ib": 1 ,  "rapidfort/mongodb-official": 1 ,  "rapidfort/mysql": 1 ,  "rapidfort/mysql8-ib": 1 ,  "rapidfort/mysql-official": 1 ,  "rapidfort/nats": 1 ,  "rapidfort/nats-ib": 1 ,  "rapidfort/nats-official": 1 ,  "rapidfort/nginx": 1 ,  "rapidfort/nginx-ib": 1 ,  "rapidfort/nginx-official": 1 ,  "rapidfort/oncall": 1 ,  "rapidfort/postgresql": 1 ,  "rapidfort/postgresql12-ib": 1 ,  "rapidfort/postgresql-official": 1 ,  "rapidfort/prometheus": 1 ,  "rapidfort/prometheus-ib": 1 ,  "rapidfort/rabbitmq": 1 ,  "rapidfort/redis": 1 ,  "rapidfort/redis-cluster": 1 ,  "rapidfort/redis6-ib": 1 ,  "rapidfort/redis-official": 1 ,  "rapidfort/telegraf": 1 ,  "rapidfort/traefik": 1 ,  "rapidfort/traefik-ib": 1 ,  "rapidfort/vault": 1 ,  "rapidfort/wordpress": 1 ,  "rapidfort/wordpress-ib": 1 ,  "rapidfort/yourls": 1 ,  "rapidfort/zookeeper": 1 ,  "rapidfort/zookeeper-ib": 1 ,  "rapidfort/zookeeper-official": 1  }' \
+           -d '{ "rapidfort/airflow": 1 ,  "rapidfort/airflow-ib": 1 ,  "rapidfort/airflow-scheduler": 1 ,  "rapidfort/airflow-worker": 1 ,  "rapidfort/apache": 1 ,  "rapidfort/apache2-ib": 1 ,  "rapidfort/apache-official": 1 ,  "rapidfort/cassandra-official": 1 ,  "rapidfort/consul": 1 ,  "rapidfort/consul-ib": 1 ,  "rapidfort/consul-official": 1 ,  "rapidfort/couchdb": 1 ,  "rapidfort/couchdb_3-ib": 1 ,  "rapidfort/couchdb-official": 1 ,  "rapidfort/curl": 1 ,  "rapidfort/elasticsearch": 1 ,  "rapidfort/elasticsearch-official": 1 ,  "rapidfort/envoy": 1 ,  "rapidfort/envoy-official": 1 ,  "rapidfort/etcd": 1 ,  "rapidfort/etcd-ib": 1 ,  "rapidfort/fluentd": 1 ,  "rapidfort/fluentd-ib": 1 ,  "rapidfort/fluentd-official": 1 ,  "rapidfort/ghost": 1 ,  "rapidfort/grafana-ib": 1 ,  "rapidfort/haproxy": 1 ,  "rapidfort/haproxy24-ib": 1 ,  "rapidfort/haproxy-official": 1 ,  "rapidfort/influxdb": 1 ,  "rapidfort/kong": 1 ,  "rapidfort/mariadb": 1 ,  "rapidfort/mariadb-ib": 1 ,  "rapidfort/mariadb-official": 1 ,  "rapidfort/memcached": 1 ,  "rapidfort/memcached-ib": 1 ,  "rapidfort/memcached-official": 1 ,  "rapidfort/microsoft-sql-server-2019-ib": 1 ,  "rapidfort/mongodb": 1 ,  "rapidfort/mongodb-ib": 1 ,  "rapidfort/mongodb-official": 1 ,  "rapidfort/mysql": 1 ,  "rapidfort/mysql8-ib": 1 ,  "rapidfort/mysql-official": 1 ,  "rapidfort/nats": 1 ,  "rapidfort/nats-ib": 1 ,  "rapidfort/nats-official": 1 ,  "rapidfort/nginx": 1 ,  "rapidfort/nginx-ib": 1 ,  "rapidfort/nginx-official": 1 ,  "rapidfort/oncall": 1 ,  "rapidfort/postgresql": 1 ,  "rapidfort/postgresql12-ib": 1 ,  "rapidfort/postgresql-official": 1 ,  "rapidfort/prometheus": 1 ,  "rapidfort/prometheus-ib": 1 ,  "rapidfort/rabbitmq": 1 ,  "rapidfort/redis": 1 ,  "rapidfort/redis-cluster": 1 ,  "rapidfort/redis6-ib": 1 ,  "rapidfort/redis-official": 1 ,  "rapidfort/telegraf": 1 ,  "rapidfort/traefik": 1 ,  "rapidfort/traefik-ib": 1 ,  "rapidfort/vault": 1 ,  "rapidfort/wordpress": 1 ,  "rapidfort/wordpress-ib": 1 ,  "rapidfort/yourls": 1 ,  "rapidfort/zookeeper": 1 ,  "rapidfort/zookeeper-ib": 1 ,  "rapidfort/zookeeper-official": 1  }' \
            https://data-receiver.rapidfort.com/counts/internal_image_pulls
diff --git a/.github/workflows/readme_updater.yml b/.github/workflows/readme_updater.yml
@@ -50,6 +50,14 @@ jobs:
           repository: rapidfort/airflow
           readme-filepath: ./community_images/airflow/airflow/bitnami/README.md.dockerhub
           short-description: RapidFort optimized, hardened image for Apache Airflow
+      - name: Docker Hub Description Apache Airflow Ironbank
+        uses: peter-evans/dockerhub-description@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          repository: rapidfort/airflow-ib
+          readme-filepath: ./community_images/airflow/airflow/ironbank/README.md.dockerhub
+          short-description: RapidFort optimized, hardened image for Apache Airflow Ironbank
       - name: Docker Hub Description Apache Airflow Scheduler
         uses: peter-evans/dockerhub-description@v3
         with:

diff --git a/README.md b/README.md
@@ -112,6 +112,7 @@ We’ve optimized and hardened some of the most popular container images on Dock
 | [Grafana Ironbank][ grafana-ib-github-link]| <a href="https://frontrow.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Fgrafana%2Fgrafana?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=grafana-ib&utm_content=landing_get_full_report_button"> <img src="/contrib/github_button_3.svg" alt="View Report" height="25" /> </a> | <a href="https://hub.docker.com/r/rapidfort/grafana-ib"> <img src="/contrib/view_on_dockerhub.svg" alt="View on Dockerhub" height="25" /> </a> | <b> 1,696 </b> |
 | [TRAEFIK Ironbank][ traefik-ib-github-link]| <a href="https://frontrow.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Ftraefik%2Ftraefik?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=traefik-ib&utm_content=landing_get_full_report_button"> <img src="/contrib/github_button_3.svg" alt="View Report" height="25" /> </a> | <a href="https://hub.docker.com/r/rapidfort/traefik-ib"> <img src="/contrib/view_on_dockerhub.svg" alt="View on Dockerhub" height="25" /> </a> | <b> 1,576 </b> |
 | [NATS Ironbank][ nats-ib-github-link]| <a href="https://frontrow.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Fsynadia%2Fnats-server?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=nats-ib&utm_content=landing_get_full_report_button"> <img src="/contrib/github_button_3.svg" alt="View Report" height="25" /> </a> | <a href="https://hub.docker.com/r/rapidfort/nats-ib"> <img src="/contrib/view_on_dockerhub.svg" alt="View on Dockerhub" height="25" /> </a> | <b> 1,095 </b> |
+| [Apache Airflow Ironbank][ airflow-ib-github-link]| <a href="https://frontrow.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Fapache%2Fairflow%2Fairflow?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=airflow-ib&utm_content=landing_get_full_report_button"> <img src="/contrib/github_button_3.svg" alt="View Report" height="25" /> </a> | <a href="https://hub.docker.com/r/rapidfort/airflow-ib"> <img src="/contrib/view_on_dockerhub.svg" alt="View on Dockerhub" height="25" /> </a> | <b>  </b> |
 
 <h2 id="supported-iron-bank-containers">What IronBank containers are supported?</h2>
 
@@ -139,6 +140,7 @@ We’ve optimized and hardened some of the most popular container images on Iron
 | [Grafana Ironbank][ grafana-ib-github-link]| <a href="https://frontrow.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Fgrafana%2Fgrafana?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=grafana-ib&utm_content=landing_get_full_report_button"> <img src="/contrib/github_button_3.svg" alt="View Report" height="25" /> </a> | <a href="https://hub.docker.com/r/rapidfort/grafana-ib"> <img src="/contrib/view_on_dockerhub.svg" alt="View on Dockerhub" height="25" /> </a> | <b> 1,696 </b> |
 | [TRAEFIK Ironbank][ traefik-ib-github-link]| <a href="https://frontrow.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Ftraefik%2Ftraefik?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=traefik-ib&utm_content=landing_get_full_report_button"> <img src="/contrib/github_button_3.svg" alt="View Report" height="25" /> </a> | <a href="https://hub.docker.com/r/rapidfort/traefik-ib"> <img src="/contrib/view_on_dockerhub.svg" alt="View on Dockerhub" height="25" /> </a> | <b> 1,576 </b> |
 | [NATS Ironbank][ nats-ib-github-link]| <a href="https://frontrow.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Fsynadia%2Fnats-server?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=nats-ib&utm_content=landing_get_full_report_button"> <img src="/contrib/github_button_3.svg" alt="View Report" height="25" /> </a> | <a href="https://hub.docker.com/r/rapidfort/nats-ib"> <img src="/contrib/view_on_dockerhub.svg" alt="View on Dockerhub" height="25" /> </a> | <b> 1,095 </b> |
+| [Apache Airflow Ironbank][ airflow-ib-github-link]| <a href="https://frontrow.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Fapache%2Fairflow%2Fairflow?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=airflow-ib&utm_content=landing_get_full_report_button"> <img src="/contrib/github_button_3.svg" alt="View Report" height="25" /> </a> | <a href="https://hub.docker.com/r/rapidfort/airflow-ib"> <img src="/contrib/view_on_dockerhub.svg" alt="View on Dockerhub" height="25" /> </a> | <b>  </b> |
 
 ### How to use Community Images
 
@@ -223,6 +225,10 @@ Learn more about container optimization at [RapidFort.com][rf-link-additonal-res
 [airflow-dh-img-pulls-badge]: https://img.shields.io/docker/pulls/rapidfort/airflow?logo=docker&logoColor=white
 [airflow-rf-link]:https://frontrow.rapidfort.com/app/community/imageinfo/docker.io%2Fbitnami%2Fairflow?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=airflow&utm_content=landing_view_report
 
+[airflow-ib-github-link]: https://github.com/rapidfort/community-images/tree/main/community_images/airflow/airflow/ironbank
+[airflow-ib-dh-img-pulls-badge]: https://img.shields.io/docker/pulls/rapidfort/airflow-ib?logo=docker&logoColor=white
+[airflow-ib-rf-link]:https://frontrow.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Fapache%2Fairflow%2Fairflow?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=airflow-ib&utm_content=landing_view_report
+
 [airflow-scheduler-github-link]: https://github.com/rapidfort/community-images/tree/main/community_images/airflow/airflow-scheduler/bitnami
 [airflow-scheduler-dh-img-pulls-badge]: https://img.shields.io/docker/pulls/rapidfort/airflow-scheduler?logo=docker&logoColor=white
 [airflow-scheduler-rf-link]:https://frontrow.rapidfort.com/app/community/imageinfo/docker.io%2Fbitnami%2Fairflow-scheduler?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=airflow-scheduler&utm_content=landing_view_report

diff --git a/builder.lst b/builder.lst
@@ -66,3 +66,4 @@ prometheus/ironbank
 etcd/ironbank
 traefik/ironbank
 nats/ironbank
+airflow/airflow/ironbank