diff --git a/docs/azure-tre-overview/shared-services/nexus.md b/docs/azure-tre-overview/shared-services/nexus.md index d15ab9096b..03be5e27d0 100644 --- a/docs/azure-tre-overview/shared-services/nexus.md +++ b/docs/azure-tre-overview/shared-services/nexus.md @@ -22,7 +22,7 @@ Nexus will be deployed as part of the main TRE terraform deployment. A configura 3. To fetch Python packages from the PyPI proxy, a researcher can use pip install while specifying the proxy server: ```bash - pip install packagename --index-url https://nexus-.azurewebsites.net/repository/pypi-proxy-repo/simple + pip install packagename --index-url https://nexus-.azurewebsites.net/repository/apt-pypi/simple ``` ## Network requirements diff --git a/docs/tre-templates/workspace-services/inner-eye.md b/docs/tre-templates/workspace-services/inner-eye.md index c45a52e456..aa2f220152 100644 --- a/docs/tre-templates/workspace-services/inner-eye.md +++ b/docs/tre-templates/workspace-services/inner-eye.md @@ -64,7 +64,7 @@ This will provision Base Workspace, with AML service and InnerEye service, inclu sudo apt-get install git-lfs git lfs install git lfs pull - export PIP_INDEX_URL=https://nexus-.azurewebsites.net/repository/pypi-proxy-repo/simple + export PIP_INDEX_URL=https://nexus-.azurewebsites.net/repository/apt-pypi/simple conda init conda env create --file environment.yml conda activate InnerEye diff --git a/scripts/apt-pypi_proxy_conf.json b/scripts/apt-pypi_proxy_conf.json new file mode 100644 index 0000000000..f43a20ebdd --- /dev/null +++ b/scripts/apt-pypi_proxy_conf.json @@ -0,0 +1,34 @@ +{ + "name": "apt-pypi", + "online": true, + "storage": { + "blobStoreName": "default", + "strictContentTypeValidation": true, + "write_policy": "ALLOW" + }, + "proxy": { + "remoteUrl": "https://pypi.org", + "contentMaxAge": 1440, + "metadataMaxAge": 1440 + }, + "negativeCache": { + "enabled": true, + "timeToLive": 1440 + }, + "httpClient": { + "blocked": false, + "autoBlock": true, + "connection": { + "retries": 0, + "userAgentSuffix": "string", + "timeout": 60, + "enableCircularRedirects": false, + "enableCookies": false, + "useTrustStore": false + } + }, + "apt": { + "distribution": "bionic", + "flat": false + } +} diff --git a/scripts/configure_nexus.sh b/scripts/configure_nexus.sh index f09dd66fe3..cbbb869297 100755 --- a/scripts/configure_nexus.sh +++ b/scripts/configure_nexus.sh @@ -63,7 +63,7 @@ if [ -z "$NEXUS_PASS" ]; then fi #Check if the repo already exists - export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/ubuntu-proxy-repo" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) + export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/ubuntu" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) if [[ ${STATUS_CODE} == 404 ]] then @@ -76,7 +76,7 @@ fi fi #Check if the repo already exists - export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/ubuntu-security-proxy-repo" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) + export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/ubuntu-security" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) if [[ ${STATUS_CODE} == 404 ]] then @@ -89,7 +89,7 @@ fi fi #Check if the repo already exists -export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/pypi-proxy-repo" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) +export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/apt-pypi" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) if [[ ${STATUS_CODE} == 404 ]] then @@ -98,5 +98,18 @@ if [[ ${STATUS_CODE} == 404 ]] $NEXUS_URL/service/rest/v1/repositories/apt/proxy \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ + -d '@./scripts/apt-pypi_proxy_conf.json' +fi + +#Check if the repo already exists +export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" "${NEXUS_URL}/service/rest/v1/repositories/apt/proxy/pypi" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) + +if [[ ${STATUS_CODE} == 404 ]] + then + # Let's create pypi proxy + curl -iu admin:$NEXUS_PASS -XPOST \ + $NEXUS_URL/service/rest/v1/repositories/pypi/proxy \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ -d '@./scripts/pypi_proxy_conf.json' fi diff --git a/scripts/pypi_proxy_conf.json b/scripts/pypi_proxy_conf.json index 9cd7511fff..b91058e5d7 100644 --- a/scripts/pypi_proxy_conf.json +++ b/scripts/pypi_proxy_conf.json @@ -1,5 +1,5 @@ { - "name": "pypi-proxy-repo", + "name": "pypi", "online": true, "storage": { "blobStoreName": "default", @@ -26,9 +26,5 @@ "enableCookies": false, "useTrustStore": false } - }, - "apt": { - "distribution": "bionic", - "flat": false } -} \ No newline at end of file +} diff --git a/scripts/ubuntu_proxy_conf.json b/scripts/ubuntu_proxy_conf.json index 7494b58a96..b24fd8ea3e 100644 --- a/scripts/ubuntu_proxy_conf.json +++ b/scripts/ubuntu_proxy_conf.json @@ -1,5 +1,5 @@ { - "name": "ubuntu-proxy-repo", + "name": "ubuntu", "online": true, "storage": { "blobStoreName": "default", diff --git a/scripts/ubuntu_security_proxy_conf.json b/scripts/ubuntu_security_proxy_conf.json index 3cb4cca0b9..ea9c6998e2 100644 --- a/scripts/ubuntu_security_proxy_conf.json +++ b/scripts/ubuntu_security_proxy_conf.json @@ -1,5 +1,5 @@ { - "name": "ubuntu-security-proxy-repo", + "name": "ubuntu-security", "online": true, "storage": { "blobStoreName": "default", diff --git a/templates/shared_services/sonatype-nexus/terraform/variables.tf b/templates/shared_services/sonatype-nexus/terraform/variables.tf index 2932ec6bf7..dd574d3c13 100644 --- a/templates/shared_services/sonatype-nexus/terraform/variables.tf +++ b/templates/shared_services/sonatype-nexus/terraform/variables.tf @@ -17,5 +17,5 @@ variable "nexus_storage_limit" { variable "nexus_allowed_fqdns" { type = string description = "comma seperated string of allowed FQDNs for Nexus" - default = "*pypi.org,security.ubuntu.com,archive.ubuntu.com" + default = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com" } diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index 156e4b3591..5c14e3703c 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,5 +1,5 @@ name: tre-service-guacamole-linuxvm -version: 0.1.0 +version: 0.1.2 description: "An Azure TRE User Resource Template for Guacamole (Linux)" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/sources_config.yml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/apt_sources_config.yml similarity index 63% rename from templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/sources_config.yml rename to templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/apt_sources_config.yml index 750b9d1fdd..0a234c6ff3 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/sources_config.yml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/apt_sources_config.yml @@ -3,14 +3,14 @@ apt: primary: - arches: - default - uri: '${nexus_proxy_url}/ubuntu-proxy-repo/' + uri: '${nexus_proxy_url}/repository/ubuntu/' security: - arches: - default - uri: '${nexus_proxy_url}/ubuntu-security-proxy-repo/' + uri: '${nexus_proxy_url}/repository/ubuntu-security/' sources_list: | deb [trusted=yes] $PRIMARY $RELEASE main restricted universe multiverse deb [trusted=yes] $PRIMARY $RELEASE-updates main restricted universe multiverse deb [trusted=yes] $SECURITY $RELEASE main restricted universe multiverse - deb [trusted=yes] ${nexus_proxy_url}/pypi-proxy-repo/ $RELEASE main restricted universe multiverse + deb [trusted=yes] ${nexus_proxy_url}/repository/apt-pypi/ $RELEASE main restricted universe multiverse diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf index 18984b2cfb..dede5f26a9 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf @@ -73,7 +73,12 @@ data "template_cloudinit_config" "config" { part { content_type = "text/cloud-config" - content = data.template_file.sources_config.rendered + content = data.template_file.apt_sources_config.rendered + } + + part { + content_type = "text/x-shellscript" + content = data.template_file.pypi_sources_config.rendered } part { @@ -89,10 +94,17 @@ data "template_file" "rdp_config" { } } -data "template_file" "sources_config" { - template = file("${path.module}/sources_config.yml") +data "template_file" "pypi_sources_config" { + template = file("${path.module}/pypi_sources_config.sh") + vars = { + nexus_proxy_url = local.nexus_proxy_url + } +} + +data "template_file" "apt_sources_config" { + template = file("${path.module}/apt_sources_config.yml") vars = { - nexus_proxy_url = "https://nexus-${var.tre_id}.azurewebsites.net/repository" + nexus_proxy_url = local.nexus_proxy_url } } diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf index 51d1e7fa39..e7019d4043 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf @@ -8,6 +8,7 @@ locals { core_resource_group_name = "rg-${var.tre_id}" vm_name = "linuxvm${local.short_service_id}" keyvault_name = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}") + nexus_proxy_url = "https://nexus-${var.tre_id}.azurewebsites.net" image_ref = { "Ubuntu 18.04" = { "publisher" = "canonical" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/pypi_sources_config.sh b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/pypi_sources_config.sh new file mode 100644 index 0000000000..6d70862655 --- /dev/null +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/pypi_sources_config.sh @@ -0,0 +1,7 @@ +#!/bin/bash +sudo tee /etc/pip.conf > dev/null <<'EOF' +[global] +index = ${nexus_proxy_url}/repository/pypi/pypi +index-url = ${nexus_proxy_url}/repository/pypi/simple +trusted-host = ${nexus_proxy_url} +EOF