Testing new xcache with new pregenerated scitoken for it (available /…

…etc/cmsaf-secrets/access_token)

Work from John Thiltges and Carl

coffea_casa/coffea_casa.py

@@ -23,7 +23,9 @@
 CERT_FILE = SECRETS_DIR / "hostcert.pem"
 HOME_DIR = Path.home()
 # XCache
+# REMOVE ME (backward compatibity for now)
 XCACHE_FILE = SECRETS_DIR / "xcache_token"
+XCACHE_SCITOKEN_FILE = SECRETS_DIR / "access_token"
 # pip
 PIP_REQUIREMENTS = HOME_DIR / "requirements.txt"
 # conda, with yml/yaml both supported
@@ -142,7 +144,7 @@ def _modify_job_kwargs(cls,
         if (CA_FILE.is_file() and CERT_FILE.is_file() and cls.security().get_connection_args("scheduler")["require_encryption"]):
             job_config["protocol"] = "tls://"
             job_config["security"] = cls.security()
-            input_files += [CA_FILE, CERT_FILE, XCACHE_FILE]
+            input_files += [CA_FILE, CERT_FILE, XCACHE_FILE, XCACHE_SCITOKEN_FILE]
         else:
             raise KeyError("Please check with system administarator why you do not have a certificate.")
         files = ", ".join(str(path) for path in input_files)

docker/Dockerfile.cc-analysis-ubuntu

@@ -9,8 +9,7 @@ ARG NB_GID="11265"
 # XCACHE
 ARG XCACHE_HOST="red-xcache1.unl.edu"
 ARG CERT_DIR="/etc/cmsaf-secrets"
-# XCACHE settings for Servicex
-ARG CACHE_PREFIX="red-xcache1.unl.edu"
+# FIX ME AFTER TEST:
 ARG BEARER_TOKEN_FILE=$CERT_DIR"/xcache_token"
 
 # Hack for GH Actions
@@ -103,7 +102,6 @@ RUN mamba install --yes \
         --force-pkgs-dirs \
         --yes
 
-#
 RUN pip install --no-cache-dir \
     supervisor \
     correctionlib \
@@ -114,15 +112,50 @@ RUN pip install --no-cache-dir \
     tornado==6.2 \
     aiostream
 
+# REMOVE ME AFTER TEST:
 # ------- xrootd-authz-plugin -------------------------------
-RUN cd /tmp && \
-    git clone https://github.com/bbockelm/xrdcl-authz-plugin.git && \
-    cd xrdcl-authz-plugin && \
-    mkdir build && \
-    cd  build && \
-    cmake /tmp/xrdcl-authz-plugin -DCMAKE_INSTALL_PREFIX=/opt/conda && \
-    make && \
-    make install
+#RUN cd /tmp && \
+#    # ------- xrdcl-authz-plugin -------------------------------
+#    git clone https://github.com/bbockelm/xrdcl-authz-plugin.git && \
+#    cd xrdcl-authz-plugin && \
+#    mkdir build && \
+#    cd  build && \
+#    cmake /tmp/xrdcl-authz-plugin -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} && \
+#    make && \
+#    make install
+
+RUN chmod 755 /etc/grid-security/certificates
+
+RUN cat << EOF > /etc/grid-security/certificates/hcc-flatiron.pem
+-----BEGIN CERTIFICATE-----
+MIIC+DCCAeCgAwIBAgIQKnsyFkDsuVdGOdnQXStdeTANBgkqhkiG9w0BAQsFADAW
+MRQwEgYDVQQDEwtJbnRlcm5hbCBDQTAeFw0yMjEyMTMxNjMzMDVaFw0zMjEyMTAx
+NjMzMDVaMBYxFDASBgNVBAMTC0ludGVybmFsIENBMIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAxxLTYp6aOjpwJp7FMgEqk0ZOOQ0BzC5htlSnaP/y/2l0
+ExD1z/moV5qEWt/T53GslBsGmezVbYBI7Eijb6sOiM4Bj6o4AALdYbUdrkqf+0LR
+aV6dP8BTnJtwCVDpUgSlPABiDYAteIoKYjd1+cGLvXlOqdrmr20WQFcykK3OsNCd
+vgC9In2goAQuaUxcLoOgr+I3SQ6EuYtMhJYh+XqhpIc/2tE29ORHHj0VRIG+xQ8H
+3nZvVofB6cEcCPIWvuMh+OVb90hwB3+X429Obupr0WFafdw9de3fi0Y03q4VMJQf
+L4CB9xp1S+0qGLJ8hdo4R1L+AEOh9ilEPSigPvkZcwIDAQABo0IwQDAOBgNVHQ8B
+Af8EBAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUz/QKVJPhNVUsAH6Z
+6LgehKWREsIwDQYJKoZIhvcNAQELBQADggEBAFjo/OcNe2PuQlef6y55QiIPl/X5
+WS35qBZJYo4ME9LdFxF8sYd8BwCyVhW+uvDes/EiYVGHQJfLQjaSSoMKHjWqcDF0
+KrlKMiqxnpFeJrVc2mlDEqMa/2LE7ln05t9jHhUH0bXvUfXqDsmXStzUYaNf5iDT
+6EwdT070gIdL32NSzT2pnRsnAfiAqJngSiiW1ehKCZuqH5zR0UK80dDiNcnFs/rF
+Pv2TukoTv8rNDbDjw3Zk1Q/+xnuL6Cji+OHSjqVRrryS7scn8G3QA1nFK1d+mm0h
+tWATifspL3kF4eC/jcAY0bT+D2LUMxwaNfM5oEncZiwMME2UYMr9D4hALzI=
+-----END CERTIFICATE-----
+EOF
+
+RUN ln -s /etc/grid-security/certificates/hcc-flatiron.pem /etc/grid-security/certificates/80d1fda9.0
+
+# REMOVE ME AFTER TEST:
+# xcache setup
+#ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/"
+#ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClAuthzPlugin.so"
+# TODO: RETEST IF WE STILL NEED THIS 
+ENV LD_LIBRARY_PATH="/opt/conda/lib/:$LD_LIBRARY_PATH"
+ENV PATH="/opt/conda/bin/:$PATH"
 
 USER root
 # Setup supervisord files
@@ -148,12 +181,6 @@ RUN cd /opt/conda/lib/python3.8/site-packages/distributed && \
     patch -p2 < 0004-Add-possibility-to-setup-external_adress-for-schedul.patch
     # && patch -p2 < 0005-Add-patch-from-John-Thiltges.patch
 
-# xcache setup
-ENV XRD_PLUGINCONFDIR="/opt/conda/etc/xrootd/client.plugins.d/"
-ENV LD_LIBRARY_PATH="/opt/conda/lib/:$LD_LIBRARY_PATH"
-ENV XRD_PLUGIN="/opt/conda/lib/libXrdClAuthzPlugin.so"
-ENV PATH="/opt/conda/bin/:$PATH"
-
 # FIXME: we have a wrong path, let's make a link.
 # cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH
 # /opt/conda/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games

docker/Dockerfile.cc-ubuntu

@@ -20,6 +20,7 @@ ARG REGISTRY
 ARG WORKER_IMAGE="${REGISTRY}/${PROJECT}/cc-analysis-ubuntu"
 # Secrets
 ARG CERT_DIR="/etc/cmsaf-secrets"
+# FIX ME AFTER TEST:
 ARG BEARER_TOKEN_FILE=$CERT_DIR"/xcache_token"
 # Configure Labextention Dask Cluster factory
 ARG DASK_ROOT_CONFIG="/opt/dask"
@@ -32,9 +33,7 @@ ARG COLLECTOR_NAME="Nebraska T2"
 ARG UID_DOMAIN="unl.edu"
 ARG SCHEDD_HOST="t3.unl.edu"
 # XCACHE
-ARG XCACHE_HOST="red-xcache1.unl.edu"
-# XCACHE settings for Servicex
-ARG CACHE_PREFIX="red-xcache1.unl.edu"
+# ARG XCACHE_HOST="red-xcache1.unl.edu"
 
 # Hack for GH Actions
 ARG GITHUB_ACTIONS="false"
@@ -69,16 +68,47 @@ RUN pip install --no-cache-dir \
     # coffea casa jobqueue modules
     coffea_casa -U
 
+# REMOVE ME AFTER TEST:
 # ------- xrootd-authz-plugin -------------------------------
-RUN cd /tmp && \
-    # ------- xrdcl-authz-plugin -------------------------------
-    git clone https://github.com/bbockelm/xrdcl-authz-plugin.git && \
-    cd xrdcl-authz-plugin && \
-    mkdir build && \
-    cd  build && \
-    cmake /tmp/xrdcl-authz-plugin -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} && \
-    make && \
-    make install
+#RUN cd /tmp && \
+#    # ------- xrdcl-authz-plugin -------------------------------
+#    git clone https://github.com/bbockelm/xrdcl-authz-plugin.git && \
+#    cd xrdcl-authz-plugin && \
+#    mkdir build && \
+#    cd  build && \
+#    cmake /tmp/xrdcl-authz-plugin -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} && \
+#    make && \
+#    make install
+
+RUN chmod 755 /etc/grid-security/certificates
+
+RUN cat << EOF > /etc/grid-security/certificates/hcc-flatiron.pem
+-----BEGIN CERTIFICATE-----
+MIIC+DCCAeCgAwIBAgIQKnsyFkDsuVdGOdnQXStdeTANBgkqhkiG9w0BAQsFADAW
+MRQwEgYDVQQDEwtJbnRlcm5hbCBDQTAeFw0yMjEyMTMxNjMzMDVaFw0zMjEyMTAx
+NjMzMDVaMBYxFDASBgNVBAMTC0ludGVybmFsIENBMIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAxxLTYp6aOjpwJp7FMgEqk0ZOOQ0BzC5htlSnaP/y/2l0
+ExD1z/moV5qEWt/T53GslBsGmezVbYBI7Eijb6sOiM4Bj6o4AALdYbUdrkqf+0LR
+aV6dP8BTnJtwCVDpUgSlPABiDYAteIoKYjd1+cGLvXlOqdrmr20WQFcykK3OsNCd
+vgC9In2goAQuaUxcLoOgr+I3SQ6EuYtMhJYh+XqhpIc/2tE29ORHHj0VRIG+xQ8H
+3nZvVofB6cEcCPIWvuMh+OVb90hwB3+X429Obupr0WFafdw9de3fi0Y03q4VMJQf
+L4CB9xp1S+0qGLJ8hdo4R1L+AEOh9ilEPSigPvkZcwIDAQABo0IwQDAOBgNVHQ8B
+Af8EBAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUz/QKVJPhNVUsAH6Z
+6LgehKWREsIwDQYJKoZIhvcNAQELBQADggEBAFjo/OcNe2PuQlef6y55QiIPl/X5
+WS35qBZJYo4ME9LdFxF8sYd8BwCyVhW+uvDes/EiYVGHQJfLQjaSSoMKHjWqcDF0
+KrlKMiqxnpFeJrVc2mlDEqMa/2LE7ln05t9jHhUH0bXvUfXqDsmXStzUYaNf5iDT
+6EwdT070gIdL32NSzT2pnRsnAfiAqJngSiiW1ehKCZuqH5zR0UK80dDiNcnFs/rF
+Pv2TukoTv8rNDbDjw3Zk1Q/+xnuL6Cji+OHSjqVRrryS7scn8G3QA1nFK1d+mm0h
+tWATifspL3kF4eC/jcAY0bT+D2LUMxwaNfM5oEncZiwMME2UYMr9D4hALzI=
+-----END CERTIFICATE-----
+EOF
+
+RUN ln -s /etc/grid-security/certificates/hcc-flatiron.pem /etc/grid-security/certificates/80d1fda9.0
+
+# REMOVE ME AFTER TEST:
+# xcache setup
+#ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/"
+#ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClAuthzPlugin.so"
 
 # Coffea_casa - > jobqueue-coffea-casa.yaml
 COPY dask/jobqueue-coffea-casa.yaml dask/dask_tls.yaml ${DASK_ROOT_CONFIG}/
@@ -111,10 +141,6 @@ RUN rm -rf /tmp/* \
     && (find ${CONDA_DIR}/lib/python*/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \
     && rm -rf ${CONDA_DIR}/pkgs
 
-# xcache setup
-ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/"
-ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClAuthzPlugin.so"
-
 # FIXME: add better layering for preparation of env
 ADD prepare-env/prepare-env-cc.sh /usr/local/bin/prepare-env.sh
 RUN chmod ugo+x /usr/local/bin/prepare-env.sh

docker/prepare-env/prepare-env-cc-analysis.sh

@@ -66,9 +66,17 @@ if [[ ! -v COFFEA_CASA_SIDECAR ]]; then
   if [[ -f "$_CONDOR_JOB_IWD/condor_token" ]]; then
       mkdir -p /home/$NB_USER/.condor/tokens.d/ && cp $_CONDOR_JOB_IWD/condor_token /home/$NB_USER/.condor/tokens.d/condor_token
   fi
+
+  # Bearer token (overwrite value preconfigured for k8s)
+  #if [[ -f "$_CONDOR_JOB_IWD/xcache_token" ]]; then
+  #    export BEARER_TOKEN_FILE="$_CONDOR_JOB_IWD/xcache_token"
+  #fi
+
+  # REMOVE ME AFTER TEST:
   # Bearer token (overwrite value preconfigured for k8s)
-  if [[ -f "$_CONDOR_JOB_IWD/xcache_token" ]]; then
-      export BEARER_TOKEN_FILE="$_CONDOR_JOB_IWD/xcache_token"
+  if [[ -f "$_CONDOR_JOB_IWD/access_token" ]]; then
+      chmod 600 $_CONDOR_JOB_IWD/access_token
+      export BEARER_TOKEN_FILE="$_CONDOR_JOB_IWD/access_token"
   fi
 
   if [[ -f "$_CONDOR_JOB_IWD/ceph.conf" ]]; then

docker/prepare-env/prepare-env-cc.sh

@@ -45,19 +45,13 @@ else
   echo "Skyhook was not configured. Please add next env values: SKYHOOK_CEPH_KEYRING SKYHOOK_CEPH_UUIDGEN SKYHOOK_CLUSTER_ADDR SKYHOOK_PUBLIC_ADDR SKYHOOK_MON_HOST in helm charts."
 fi
 
-# Configure oidc-agent for token management
-#echo "eval \`oidc-keychain\`" >> ~/.bashrc
-#eval `oidc-keychain`
-#oidc-gen coffea-casa --issuer $IAM_SERVER \
-#               --client-id $IAM_CLIENT_ID \ # https://cms-auth.web.cern.ch/
-#               --client-secret $IAM_CLIENT_SECRET \
-#               --rt $REFRESH_TOKEN \
-#               --confirm-yes \
-#               --scope "openid profile email wlcg wlcg.groups" \
-#               --redirect-uri  http://localhost:8843 \
-#               --pw-cmd "echo \"DUMMY PWD\""
-
-#while true; do oidc-token coffea-casa --time 1200 > /tmp/token; sleep 600; done &
+# REMOVE ME AFTER TEST:
+if [[ -f "/etc/cmsaf-secrets/access_token" ]]; then
+  cat /etc/cmsaf-secrets/access_token > /tmp/access_token
+  chmod 600 /tmp/access_token
+  # Redefine BEARER_TOKEN_FILE
+  BEARER_TOKEN_FILE="/tmp/access_token"
+fi
 
 # Check environment
 if [ -e "$HOME/environment.yml" ]; then