Enable PyPI proxy and consume via Linux VMs (#1303)

* Added files.pythonhosted whitelist * Added pypi nexus source * Added pip source set up in VM cloudinit * Fixed proxy conf naming * Pip install working for linux
microsoft · Feb 15, 2022 · 5cc52f2 · 5cc52f2
1 parent 632078e
commit 5cc52f2
Show file tree

Hide file tree

Showing 13 changed files with 85 additions and 22 deletions.
diff --git a/docs/azure-tre-overview/shared-services/nexus.md b/docs/azure-tre-overview/shared-services/nexus.md
@@ -22,7 +22,7 @@ Nexus will be deployed as part of the main TRE terraform deployment. A configura
 3. To fetch Python packages from the PyPI proxy, a researcher can use pip install while specifying the proxy server:
 
     ```bash
-    pip install packagename --index-url https://nexus-<TRE_ID>.azurewebsites.net/repository/pypi-proxy-repo/simple
+    pip install packagename --index-url https://nexus-<TRE_ID>.azurewebsites.net/repository/apt-pypi/simple
     ```
 
 ## Network requirements

diff --git a/docs/tre-templates/workspace-services/inner-eye.md b/docs/tre-templates/workspace-services/inner-eye.md
@@ -64,7 +64,7 @@ This will provision Base Workspace, with AML service and InnerEye service, inclu
     sudo apt-get install git-lfs
     git lfs install
     git lfs pull
-    export PIP_INDEX_URL=https://nexus-<TRE_ID>.azurewebsites.net/repository/pypi-proxy-repo/simple
+    export PIP_INDEX_URL=https://nexus-<TRE_ID>.azurewebsites.net/repository/apt-pypi/simple
     conda init
     conda env create --file environment.yml
     conda activate InnerEye

diff --git a/scripts/apt-pypi_proxy_conf.json b/scripts/apt-pypi_proxy_conf.json
@@ -0,0 +1,34 @@
+{
+    "name": "apt-pypi",
+    "online": true,
+    "storage": {
+        "blobStoreName": "default",
+        "strictContentTypeValidation": true,
+        "write_policy": "ALLOW"
+    },
+    "proxy": {
+        "remoteUrl": "https://pypi.org",
+        "contentMaxAge": 1440,
+        "metadataMaxAge": 1440
+    },
+    "negativeCache": {
+        "enabled": true,
+        "timeToLive": 1440
+    },
+    "httpClient": {
+        "blocked": false,
+        "autoBlock": true,
+        "connection": {
+            "retries": 0,
+            "userAgentSuffix": "string",
+            "timeout": 60,
+            "enableCircularRedirects": false,
+            "enableCookies": false,
+            "useTrustStore": false
+        }
+    },
+    "apt": {
+        "distribution": "bionic",
+        "flat": false
+    }
+}
diff --git a/scripts/configure_nexus.sh b/scripts/configure_nexus.sh
@@ -63,7 +63,7 @@ if [ -z "$NEXUS_PASS" ]; then
 fi
 
 #Check if the repo already exists
- export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/ubuntu-proxy-repo" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null)
+ export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/ubuntu" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null)
 
  if [[ ${STATUS_CODE} == 404 ]]
   then
@@ -76,7 +76,7 @@ fi
  fi
 
  #Check if the repo already exists
- export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/ubuntu-security-proxy-repo" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null)
+ export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/ubuntu-security" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null)
 
  if [[ ${STATUS_CODE} == 404 ]]
   then
@@ -89,7 +89,7 @@ fi
  fi
 
 #Check if the repo already exists
-export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/pypi-proxy-repo" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null)
+export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/apt-pypi" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null)
 
 if [[ ${STATUS_CODE} == 404 ]]
  then
@@ -98,5 +98,18 @@ if [[ ${STATUS_CODE} == 404 ]]
     $NEXUS_URL/service/rest/v1/repositories/apt/proxy \
     -H 'accept: application/json' \
     -H 'Content-Type: application/json' \
+    -d '@./scripts/apt-pypi_proxy_conf.json'
+fi
+
+#Check if the repo already exists
+export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/pypi" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null)
+
+if [[ ${STATUS_CODE} == 404 ]]
+ then
+    # Let's create pypi proxy
+    curl -iu admin:$NEXUS_PASS -XPOST \
+    $NEXUS_URL/service/rest/v1/repositories/pypi/proxy \
+    -H 'accept: application/json' \
+    -H 'Content-Type: application/json' \
     -d '@./scripts/pypi_proxy_conf.json'
 fi
diff --git a/scripts/pypi_proxy_conf.json b/scripts/pypi_proxy_conf.json
@@ -1,5 +1,5 @@
 {
-    "name": "pypi-proxy-repo",
+    "name": "pypi",
     "online": true,
     "storage": {
         "blobStoreName": "default",
@@ -26,9 +26,5 @@
             "enableCookies": false,
             "useTrustStore": false
         }
-    },
-    "apt": {
-        "distribution": "bionic",
-        "flat": false
     }
-}
+}
diff --git a/scripts/ubuntu_proxy_conf.json b/scripts/ubuntu_proxy_conf.json
@@ -1,5 +1,5 @@
 {
-  "name": "ubuntu-proxy-repo",
+  "name": "ubuntu",
   "online": true,
   "storage": {
       "blobStoreName": "default",

diff --git a/scripts/ubuntu_security_proxy_conf.json b/scripts/ubuntu_security_proxy_conf.json
@@ -1,5 +1,5 @@
 {
-  "name": "ubuntu-security-proxy-repo",
+  "name": "ubuntu-security",
   "online": true,
   "storage": {
       "blobStoreName": "default",

diff --git a/templates/shared_services/sonatype-nexus/terraform/variables.tf b/templates/shared_services/sonatype-nexus/terraform/variables.tf
@@ -17,5 +17,5 @@ variable "nexus_storage_limit" {
 variable "nexus_allowed_fqdns" {
   type        = string
   description = "comma seperated string of allowed FQDNs for Nexus"
-  default     = "*pypi.org,security.ubuntu.com,archive.ubuntu.com"
+  default     = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com"
 }
diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml
@@ -1,5 +1,5 @@
 name: tre-service-guacamole-linuxvm
-version: 0.1.0
+version: 0.1.2
 description: "An Azure TRE User Resource Template for Guacamole (Linux)"
 registry: azuretre
 dockerfile: Dockerfile.tmpl

diff --git a/...zure-linuxvm/terraform/sources_config.yml → ...-linuxvm/terraform/apt_sources_config.yml b/...zure-linuxvm/terraform/sources_config.yml → ...-linuxvm/terraform/apt_sources_config.yml
@@ -3,14 +3,14 @@ apt:
   primary:
     - arches:
         - default
-      uri: '${nexus_proxy_url}/ubuntu-proxy-repo/'
+      uri: '${nexus_proxy_url}/repository/ubuntu/'
 
   security:
     - arches:
         - default
-      uri: '${nexus_proxy_url}/ubuntu-security-proxy-repo/'
+      uri: '${nexus_proxy_url}/repository/ubuntu-security/'
   sources_list: |
       deb [trusted=yes] $PRIMARY $RELEASE main restricted universe multiverse
       deb [trusted=yes] $PRIMARY $RELEASE-updates main restricted universe multiverse
       deb [trusted=yes] $SECURITY $RELEASE main restricted universe multiverse
-      deb [trusted=yes] ${nexus_proxy_url}/pypi-proxy-repo/ $RELEASE main restricted universe multiverse
+      deb [trusted=yes] ${nexus_proxy_url}/repository/apt-pypi/ $RELEASE main restricted universe multiverse
diff --git a/.../workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf b/.../workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf
@@ -73,7 +73,12 @@ data "template_cloudinit_config" "config" {
 
   part {
     content_type = "text/cloud-config"
-    content      = data.template_file.sources_config.rendered
+    content      = data.template_file.apt_sources_config.rendered
+  }
+
+  part {
+    content_type = "text/x-shellscript"
+    content      = data.template_file.pypi_sources_config.rendered
   }
 
   part {
@@ -89,10 +94,17 @@ data "template_file" "rdp_config" {
   }
 }
 
-data "template_file" "sources_config" {
-  template = file("${path.module}/sources_config.yml")
+data "template_file" "pypi_sources_config" {
+  template = file("${path.module}/pypi_sources_config.sh")
+  vars = {
+    nexus_proxy_url = local.nexus_proxy_url
+  }
+}
+
+data "template_file" "apt_sources_config" {
+  template = file("${path.module}/apt_sources_config.yml")
   vars = {
-    nexus_proxy_url = "https://nexus-${var.tre_id}.azurewebsites.net/repository"
+    nexus_proxy_url = local.nexus_proxy_url
   }
 }
 

diff --git a/...s/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf b/...s/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf
@@ -8,6 +8,7 @@ locals {
   core_resource_group_name       = "rg-${var.tre_id}"
   vm_name                        = "linuxvm${local.short_service_id}"
   keyvault_name                  = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}")
+  nexus_proxy_url                = "https://nexus-${var.tre_id}.azurewebsites.net"
   image_ref = {
     "Ubuntu 18.04" = {
       "publisher"  = "canonical"

diff --git a/...ervices/guacamole/user_resources/guacamole-azure-linuxvm/terraform/pypi_sources_config.sh b/...ervices/guacamole/user_resources/guacamole-azure-linuxvm/terraform/pypi_sources_config.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+sudo tee /etc/pip.conf > dev/null <<'EOF'
+[global]
+index = ${nexus_proxy_url}/repository/pypi/pypi
+index-url = ${nexus_proxy_url}/repository/pypi/simple
+trusted-host = ${nexus_proxy_url}
+EOF