Merge pull request #26 from Aquaveo/gilabmaster

Gitlabmaster

Author: msouff

.gitignore

@@ -182,5 +182,11 @@ dmypy.json
 # pytype static role analyzer
 .pytype/
 
+# vscode
+.vscode/
+
 ## Misc
 sample_data/**
+/aws_config.json
+/app/return-periods.zarr
+/app/package-metadata-table.parquet

.gitlab-ci.yml

@@ -2,80 +2,43 @@ stages:
   - Build
   - Deploy
 
+include:
+  - project: "tethys/kubernetes/gl-k8s-integration"
+    ref: 1.6.1
+    file: "/helm-check.yml"
+  - project: "tethys/kubernetes/gl-k8s-integration"
+    ref: 1.6.1
+    file: "/kaniko-build.yml"
+
 ### Build ###
 "Check Helm":
   stage: Build
-  image: docker.aquaveo.com/tethys/kubernetes/helm-docker:0.7.1
   variables:
-    GIT_SUBMODULE_STRATEGY: none
+    HELM_REPO_URL: https://eccr.ecmwf.int
+    HELM_REPO_CHANNEL: geoglows_api
+    HELM_REPO_USERNAME: $ECCR_USER
+    HELM_REPO_PASSWORD: $ECCR_PASSWORD
     HELM_CHART: $CI_PROJECT_DIR/helm/gsprestapi
   script:
     # Ensure all required variables are set
     - ": \"${HELM_CHART:?must be set}\""
-    - helm lint $HELM_CHART
     - chmod +x ./check_helm_chart
     - ./check_helm_chart $HELM_CHART
-  except:
-   - tags
-   - master
 
 "Update Helm":
   stage: Build
-  image: docker.aquaveo.com/tethys/kubernetes/helm-docker:0.7.1
   variables:
-    GIT_SUBMODULE_STRATEGY: none
-    HELM_REPO_USERNAME: aquaveo
+    HELM_REPO_URL: https://eccr.ecmwf.int
+    HELM_REPO_CHANNEL: geoglows_api
+    HELM_REPO_USERNAME: $ECCR_USER
+    HELM_REPO_PASSWORD: $ECCR_PASSWORD
     HELM_CHART: $CI_PROJECT_DIR/helm/gsprestapi
   script:
     # Ensure all required variables are set
     - ": \"${HELM_CHART:?must be set}\""
-    - ": \"${HELM_KEY_PASSPHRASE:?must be set}\""
-    - ": \"${HELM_REPO_PASSWORD:?must be set}\""
     - helm lint $HELM_CHART
     - chmod +x ./check_helm_chart
     - ./check_helm_chart $HELM_CHART --upload
-  only:
-    - master
-
-.kaniko_build:
-  stage: Build
-  image:
-    name: gcr.io/kaniko-project/executor:debug-v0.23.0
-    entrypoint: [""]
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-  before_script:
-    - >
-      echo "{\"auths\":{
-        \"$CI_REGISTRY\": {\"username\":\"$CI_REGISTRY_USER\", \"password\":\"$CI_REGISTRY_PASSWORD\"},
-        \"eccr.ecmwf.int/harbor\": {\"username\":\"$ECCR_USER\", \"password\":\"$ECCR_PASSWORD\"}
-      }}" > /kaniko/.docker/config.json
-  script:
-    # Ensure all required variables are set
-    - ": \"${KANIKO_DOCKERFILE:?must be set}\""
-    - ": \"${KANIKO_CONTEXT:?must be set}\""
-    - ": \"${KANIKO_DESTINATIONS:?must be set}\""
-    # Run Kaniko
-    - export KANIKO_CMD="/kaniko/executor
-        --context $KANIKO_CONTEXT
-        --dockerfile $KANIKO_DOCKERFILE
-        $(for DEST in $KANIKO_DESTINATIONS; do echo "--destination $DEST "; done)
-        --cache=true
-        --cache-repo $CI_REGISTRY_IMAGE/cache
-        --force"
-    - echo $KANIKO_CMD
-    - $KANIKO_CMD
-
-"Build Commit":
-  extends: .kaniko_build
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-    KANIKO_CONTEXT: $CI_PROJECT_DIR
-    KANIKO_DOCKERFILE: $CI_PROJECT_DIR/Dockerfile
-    KANIKO_DESTINATIONS: $CI_REGISTRY_IMAGE/dev:$CI_COMMIT_SHORT_SHA
-
-  except:
-    - tags
 
 "Build Tag":
   extends: .kaniko_build
@@ -86,6 +49,12 @@ stages:
     KANIKO_DESTINATIONS: >
       eccr.ecmwf.int/geoglows_api/geoglows_api:$CI_COMMIT_TAG
       eccr.ecmwf.int/geoglows_api/geoglows_api:latest
+  before_script:
+    - >
+      echo "{\"auths\":{
+        \"$CI_REGISTRY\": {\"username\":\"$CI_REGISTRY_USER\", \"password\":\"$CI_REGISTRY_PASSWORD\"},
+        \"eccr.ecmwf.int/harbor\": {\"username\":\"$ECCR_USER\", \"password\":\"$ECCR_PASSWORD\"}
+      }}" > /kaniko/.docker/config.json
   only:
     - tags
 
@@ -95,13 +64,19 @@ stages:
     KANIKO_CONTEXT: $CI_PROJECT_DIR
     KANIKO_DOCKERFILE: $CI_PROJECT_DIR/Dockerfile
     KANIKO_DESTINATIONS: eccr.ecmwf.int/geoglows_api/geoglows_api:stable
+  before_script:
+    - >
+      echo "{\"auths\":{
+        \"$CI_REGISTRY\": {\"username\":\"$CI_REGISTRY_USER\", \"password\":\"$CI_REGISTRY_PASSWORD\"},
+        \"eccr.ecmwf.int/harbor\": {\"username\":\"$ECCR_USER\", \"password\":\"$ECCR_PASSWORD\"}
+      }}" > /kaniko/.docker/config.json
   only:
     - /^stable$/
 
 ### DEPLOY ###
 "Deploy to Staging":
   stage: Deploy
-  image: docker.aquaveo.com/tethys/kubernetes/helm-docker:2.0.0-rc1
+  image: aquaveollc/helm-docker:0.3.1
   dependencies: [] # Deploys do not depend on the Build phase artifacts
   variables:
     GIT_SUBMODULE_STRATEGY: none
@@ -115,7 +90,7 @@ stages:
 
 "Deploy to Production":
   stage: Deploy
-  image: docker.aquaveo.com/tethys/kubernetes/helm-docker:2.0.0-rc1
+  image: aquaveollc/helm-docker:0.3.1
   dependencies: [] # Deploys do not depend on the Build phase artifacts
   variables:
     GIT_SUBMODULE_STRATEGY: none
@@ -126,5 +101,3 @@ stages:
   when: manual
   only:
     - tags
-
-

Dockerfile

@@ -1,44 +1,42 @@
-FROM continuumio/miniconda3:latest
+FROM mambaorg/micromamba
 
-ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PATH=/opt/conda/envs/gsp_api/bin:$PATH API_PREFIX=/api
+USER root
 
-# For Development On Analytics Only:
-# ENV AWS_ACCESS_KEY_ID=
-# ENV AWS_SECRET_ACCESS_KEY=
-# ENV AWS_LOG_GROUP_NAME=
-# ENV AWS_LOG_STREAM_NAME=
-# ENV AWS_REGION=
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV PATH=/opt/conda/envs/app-env/bin:$PATH
+ENV API_PREFIX=/api
+ENV PYTHONPATH=$PYTHONPATH:/app
 
-RUN mkdir /var/uwsgi
+COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yaml /environment.yaml
+COPY startup.sh /startup.sh
+COPY app /app
 
-RUN apt-get update -qq && apt-get install -yqq supervisor vim
+WORKDIR /
 
-COPY ./environment.yml ./startup.sh ./
+RUN mkdir -p /var/log/uwsgi
+RUN apt-get update && apt-get install -y --no-install-recommends curl vim awscli && rm -rf /var/lib/apt/lists/*
 
-RUN conda config --set channel_priority strict && \
-    conda config --add channels conda-forge && \
-    conda env create -f environment.yml && \
-    echo "conda activate gsp_api" >> ~/.bashrc
+# startup.sh is a helper script
+RUN chmod +x /startup.sh
+RUN micromamba create -n app-env --yes --file "environment.yaml" && micromamba clean --all --yes
 
-RUN mkdir -p /mnt/output/forecasts && \
-    mkdir -p /mnt/output/era-interim && \
-    mkdir -p /mnt/output/era-5 && \
-    mkdir -p /mnt/output/forecast-records
+# download a copy of the package metadata table
+RUN wget http://geoglows-v2.s3-us-west-2.amazonaws.com/tables/v2-model-table.parquet -O /app/package-metadata-table.parquet
+ENV PYGEOGLOWS_METADATA_TABLE_PATH=/app/package-metadata-table.parquet
 
-# COPY ./sample_data/forecasts /mnt/output/forecasts
-# COPY ./sample_data/era-interim /mnt/output/era-interim
-# COPY ./sample_data/era-5 /mnt/output/era-5
-# COPY ./sample_data/forecast-records /mnt/output/forecast-records
+# download a copy of the package metadata table with extra attributes
+RUN wget http://geoglows-v2.s3-us-west-2.amazonaws.com/tables/package-metadata-table.parquet -O /app/extra-metadata-table.parquet
+ENV PYGEOGLOWS_EXTRA_METADATA_TABLE_PATH=/app/extra-metadata-table.parquet
 
-# Copy API code
-COPY ./GSP_API /app/GSP_API/
-COPY ./supervisord.conf /etc/supervisor/conf.d/uwsgi.conf
-
-# startup.sh is a helper script
-RUN chmod +x /startup.sh
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
+ENV AWS_LOG_GROUP_NAME=geoglows.ecmwf.int
+ENV AWS_LOG_STREAM_NAME=rest_api_metrics
+ENV AWS_REGION=eu-central-1
 
 # Expose the port that is to be used when calling your API
 EXPOSE 80
+
 HEALTHCHECK --interval=1m --timeout=3s --start-period=20s \
   CMD curl -f http://localhost/ || exit 1
-ENTRYPOINT [ "/startup.sh" ]
+
+ENTRYPOINT [ "/startup.sh" ]