Merge branch 'master' into feat/add-vlmd-submission-tools-cli

.github/workflows/build_and_push_python_image.yml

@@ -4,6 +4,7 @@ on:
   push:
     paths:
     - python3.9-data-science/Dockerfile
+    - .github/workflows/build_and_push_python_image.yml
 
 jobs:
   python_3-9:

.github/workflows/build_brh_notebook_image.yml

@@ -4,6 +4,7 @@ on:
   push:
     paths:
     - BRH-notebooks/*/**
+    - .github/workflows/build_brh_notebook_image.yml
 
 jobs:
   push-image:

.github/workflows/build_heal_notebook_image.yml

@@ -4,6 +4,7 @@ on:
   push:
     paths:
     - HEAL-notebooks/*/**
+    - .github/workflows/build_heal_notebook_image.yml
 
 jobs:
   push-image:

.github/workflows/build_jupyter_nextflow_image.yml

@@ -0,0 +1,22 @@
+name: Build Push to Quay and ECR
+
+on:
+  push:
+    paths:
+    - jupyter-nextflow/Dockerfile
+    - .github/workflows/build_jupyter_nextflow_image.yml
+
+jobs:
+  jupyter-nextflow:
+    name: Build and Push jupyter-nextflow
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./jupyter-nextflow/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./jupyter-nextflow"
+      OVERRIDE_REPO_NAME: "jupyter-notebook"
+      OVERRIDE_TAG_NAME: "nextflow-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}

.github/workflows/build_push_stata_gen3_licensed.yml

@@ -0,0 +1,69 @@
+name: Push Stata Gen3-licensed Image to quay
+
+on:
+  push:
+    paths:
+      - jupyter-pystata-gen3-licensed/**
+      - .github/workflows/build_push_stata_gen3_licensed.yml
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 30000
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+      - uses: actions/checkout@v2
+      - uses: prewk/s3-cp-action@v0.1.1
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          SOURCE: 's3://ctds-stata/Stata17Linux64.tar.gz'
+          DEST: './jupyter-pystata-gen3-licensed/resources/'
+
+      - name: Extract branch name
+        shell: bash
+        run: echo "branch=$(echo $(echo ${GITHUB_REF#refs/*/} | tr / _))" >> $GITHUB_OUTPUT
+        id: extract_branch
+
+      - name: Determine image to build
+        id: parse_image
+        shell: python
+        run: |
+          import os
+
+          build_target = "jupyter-pystata-licensed"
+          print(f"Will trigger build for: {build_target}")
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+            print(f'build_target={build_target}', file=fh)
+
+      - if: ${{ steps.parse_image.outputs.build_target }}
+        name: Sanitize image name
+        id: sanitize_name
+        run: |
+          IMAGE_NAME=$( sed 's/[^[:alnum:]]/_/g' <<< ${{ steps.parse_image.outputs.build_target }} );
+          echo "image_name=$IMAGE_NAME" >> $GITHUB_OUTPUT
+
+      - name: Build Image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: jupyter-pystata-gen3-licensed
+          tags: ${{ steps.extract_branch.outputs.branch }}
+          dockerfiles: ./jupyter-pystata-gen3-licensed/Dockerfile
+
+      - name: Push To quay.io
+        id: push-to-quay
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/cdis
+          username: ${{ secrets.QUAY_SERVICE_ACCOUNT_USER }}
+          password: ${{ secrets.QUAY_SERVICE_ACCOUNT_PASSWORD }}

.github/workflows/push_stata_image.yml → .github/workflows/build_push_stata_user_licensed.yml

@@ -1,10 +1,10 @@
-name: Push Stata Image to quay
+name: Push Stata User-licensed Image to quay
 
 on:
   push:
     paths:
-      - jupyter-pystata/**
-      - .github/workflows/push_stata_image.yml
+      - jupyter-pystata-user-licensed/**
+      - .github/workflows/build_push_stata_user_licensed.yml
   workflow_dispatch:
 
 jobs:
@@ -25,20 +25,20 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SOURCE: 's3://ctds-stata/Stata17Linux64.tar.gz'
-          DEST: './jupyter-pystata/resources/'
+          DEST: './jupyter-pystata-user-licensed/resources/'
 
       - name: Extract branch name
         shell: bash
-        run: echo "branch=$(echo $(echo ${GITHUB_REF#refs/heads/} | tr  / _))" >> $GITHUB_OUTPUT
+        run: echo "::set-output name=branch::$(echo $(echo ${GITHUB_REF#refs/*/} | tr / _))"
         id: extract_branch
 
       - name: Build Image
         id: build-image
         uses: redhat-actions/buildah-build@v2
         with:
-          image: stata-heal
-          tags: ${{ steps.extract_branch.outputs.branch }} ${{ github.sha }}
-          dockerfiles: ./jupyter-pystata/Dockerfile
+          image: jupyter-pystata-user-licensed
+          tags: ${{ steps.extract_branch.outputs.branch }}
+          dockerfiles: ./jupyter-pystata-user-licensed/Dockerfile
 
       - name: Push To quay.io
         id: push-to-quay

jupyter-nextflow/Dockerfile

@@ -0,0 +1,62 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+FROM quay.io/cdis/scipy-notebook:2023-03-13
+USER root
+
+#
+# R pre-requisites
+#  https://github.com/rocker-org/rocker/blob/master/r-base/Dockerfile
+#
+## Configure default locale, see https://github.com/rocker-org/rocker/issues/19
+RUN echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
+	&& locale-gen en_US.utf8 \
+	&& /usr/sbin/update-locale LANG=en_US.UTF-8
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV R_BASE_VERSION 3.6.3
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        autoconf \
+        ca-certificates \
+        fonts-dejavu \
+        fonts-texgyre \
+        gfortran \
+        gcc \
+        jq \
+        less \
+        libssl3 \
+        libssl-dev \
+        libbz2-dev \
+        libcurl4-gnutls-dev \
+        liblzma-dev \
+        locales \
+        openjdk-11-jdk \
+        make \
+        automake \
+        perl \
+        pkg-config \
+        tzdata \
+        vim \
+        zlib1g \
+        zlib1g-dev
+
+RUN apt-get clean \
+ 	&& rm -rf /var/lib/apt/lists/*
+
+#
+# !!!!!NOTE!!!!!
+# Do not install conda, python, pip, etc dependencies as root
+# It screws up directory permissions
+#
+USER $NB_UID
+WORKDIR /home/$NB_USER
+
+RUN pip install --upgrade nbconvert
+
+COPY --chown=jovyan:users resources/custom.js /home/$NB_USER/.jupyter/custom/
+COPY --chown=jovyan:users resources/jupyter_notebook_config.py /home/$NB_USER/.jupyter/tmp.py
+RUN cat /home/$NB_USER/.jupyter/tmp.py >> /home/$NB_USER/.jupyter/jupyter_notebook_config.py && rm /home/$NB_USER/.jupyter/tmp.py
+
+RUN conda install -c bioconda nextflow

jupyter-nextflow/resources/custom.js

@@ -0,0 +1,4 @@
+// do not open notebooks in a new tab: https://github.com/jupyter/notebook/issues/4115
+define(['base/js/namespace'], function(Jupyter){
+    Jupyter._target = '_self';
+})

jupyter-nextflow/resources/jupyter_notebook_config.py

@@ -0,0 +1,8 @@
+import os
+
+c.NotebookApp.tornado_settings = {
+    "headers": {
+        "Content-Security-Policy": "frame-ancestors self %s"
+        % os.getenv("FRAME_ANCESTORS", "")
+    }
+}

jupyter-pystata-gen3-licensed/Dockerfile

@@ -0,0 +1,23 @@
+FROM quay.io/cdis/jupyter-pystata-user-licensed:1.2.0
+
+USER root
+RUN apt-get update
+RUN apt-get install -y firefox
+RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz
+RUN tar -xvzf geckodriver*
+RUN mv geckodriver /bin/
+
+COPY jupyter-pystata-gen3-licensed/resources/wait_for_license.sh /tmp/
+COPY jupyter-pystata-gen3-licensed/resources/setup_licensed_notebook.py /tmp/
+RUN chmod 777 /tmp/wait_for_license.sh /tmp/setup_licensed_notebook.py
+
+USER $NB_USER
+RUN pip3 install selenium
+
+# Pin stata_setup to avoid error on splash parameter
+RUN pip3 uninstall --yes stata-setup
+RUN pip3 install stata-setup==0.1.2
+
+# Remove the notebook created in jupyter-pystata-user-licensed
+RUN rm $HOME/Stata.ipynb
+COPY jupyter-pystata-gen3-licensed/resources/licensed_stata_session.ipynb $HOME

jupyter-pystata-gen3-licensed/README.md

@@ -0,0 +1,70 @@
+## Licensed Stata Workspaces
+
+---
+
+For general information about Gen3 Stata workspaces, see the
+[Stata workspaces README](https://github.com/uc-cdis/containers/tree/master/jupyter-pystata-user-licensed)
+
+### Licensing
+
+Stata software requires a license to run.
+This container waits for a license provided by an external job.
+It then runs a script which launces a jupyter notebook, runs its first cell in order to initialize a STATA session, then deletes the license.
+This is to prevent the user from accessing the license directly.
+
+The current external job for license distribution is the
+[`distribute-licenses-job`](https://github.com/uc-cdis/cloud-automation/blob/master/kube/services/jobs/distribute-licenses-job.yaml) in the
+[cloud-automation repository](https://github.com/uc-cdis/cloud-automation).
+
+#### License file creation
+
+You will need
+
+* a license PDF from Stata
+* access to the Stata `stinit` function
+
+One way to access the Stata function is to exec into a pod that is running Stata,
+possibly a user-licensed instance. From the bash shell, invoke the Stata function
+
+```
+stinit
+```
+
+Follow the instructions and add information from your license PDF.
+This will generate a license string. The license string will likely have characters
+separated by exclamation marks (!). **Save the string!**
+
+#### License secret creation
+
+The license secret can be managed by [`g3auto`](https://github.com/uc-cdis/cloud-automation/blob/9042162/doc/secrets.md).
+
+Store one or more copies of the license string in the file
+`g3auto/stata-workspace-gen3-license/stata_license.txt` in your commons.
+Run the secret creation command
+
+```
+gen3 secrets sync
+```
+
+Verify that your secret has been created
+
+```
+kubectl get secret stata-workspace-gen3-license-g3auto
+```
+
+### Local development
+
+To build, enter the root directory of this repo and run:
+```
+docker build -t stata-licensed -f jupyter-pystata-gen3-licensed/Dockerfile .
+
+docker run --name stata-licensed -p 8888:8888 stata-licensed /tmp/wait_for_license.sh --NotebookApp.base_url=/lw-workspace/proxy/ --NotebookApp.password='' --NotebookApp.token=''
+```
+
+(You will need a local copy of `Stata17Linux64.tar.gz`.)
+
+Then, with your license `stata.lic`,
+
+```
+docker cp stata.lic stata-licensed:/usr/local/stata17/stata.lic
+```

jupyter-pystata/resources/Stata.ipynb → jupyter-pystata-gen3-licensed/resources/licensed_stata_session.ipynb

@@ -1,34 +1,48 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Licensed STATA Notebook Workspace\n",
+    "This notebook runs a licensed STATA MP session, managed by the Gen3 platform.\n",
+    "\n",
+    "The license supplied to this workspace session belongs to the University of Chicago and is not for use outside of the Gen3 platform.\n",
+    "\n",
+    "Because licenses are protected and limited, this workspace is limited to a single running STATA session, which has already been initialized via the cell below.\n",
+    "\n",
+    "Users who have an existing STATA license are encouraged to bring it to a self-supplied STATA workspace."
+   ],
+   "metadata": {}
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "import stata_setup\n",
     "stata_setup.config(\"/usr/local/stata17\", \"mp\")"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "%%stata\n",
     ". describe"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "%%mata\n",
     "sqrt(4)"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {}
   }
  ],
  "metadata": {