Merge pull request #76 from adobeethoshelloworld/ETHOS-39859

adobejmong · web-flow · commit 8226e9240ef5 · 2023-02-02T15:56:15.000-08:00
feat(ETHOS-39859): add support to update /etc/apt/sources.list
diff --git a/Makefile b/Makefile
@@ -0,0 +1,12 @@
+.PHONY: build-shunit2 shunit2
+
+SHUNIT2_IMG ?= shunit2
+SHUNIT2_VERSION ?= 0.1.0
+
+TAG_NAME=$(SHUNIT2_IMG):$(SHUNIT2_VERSION)
+
+build-shunit2:
+	cd tests/sh && docker build -t $(TAG_NAME) -f Dockerfile.shunit2 ../..
+
+shunit2: build-shunit2
+	docker run -it $(TAG_NAME)
diff --git a/README.md b/README.md
@@ -7,14 +7,16 @@ https://hub.docker.com/r/behance/docker-base/tags/
 
 Provides base OS, security patches, and tools for quick and easy spinup.
 
-
 ### Variants
 
-* Ubuntu 18.04 LTS available, tagged as `-VERSION#-ubuntu-18.04`
-* Ubuntu 20.04 LTS available, tagged as `-VERSION#-ubuntu-20.04`
-* Ubuntu 22.04 LTS available, tagged as `-VERSION#-ubuntu-22.04`
-* Alpine builds available, tagged as `-alpine` **DEPRECATED**
-* Centos 7 builds available, tagged as `-centos-7` **DEPRECATED**
+| OS            | Tag                         | Notes              | 
+| ------------- | --------------------------- | ------------------ |
+| Ubuntu 20.04  |  `-VERSION#-ubuntu-20.04`   | Current            |
+| Ubuntu 22.04  |  `-VERSION#-ubuntu-22.04`   | Current            |
+| Alpine        |  `-alpine`                  | **DEPRECATED**     |
+| CentOS 7      |  `-centos-7`                | **DEPRECATED**     |
+
+`Alpine`, `CentOS 7` will be removed November 2022.
 
 ### Tools
 
diff --git a/container/root/scripts/install_s6.sh b/container/root/scripts/install_s6.sh
@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
-# Add S6 for zombie reaping, boot-time coordination, signal transformation/distribution
+# Add S6 for zombie reaping, boot-time coordination, signal transformation /
+# distribution
 # @see https://github.com/just-containers/s6-overlay
 #
 # Downloads, verifies, and extracts
@@ -11,26 +12,38 @@ ARCH="$(archstring --x64 amd64 --arm64 aarch64)"
 
 S6_NAME=s6-overlay-${ARCH}.tar.gz
 S6_VERSION=${S6_VERSION:="v2.1.0.2"}
-PUBLIC_KEY=6101B2783B2FD161
+# PUBLIC_KEY=6101B2783B2FD161
 
-curl -fL https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/${S6_NAME} -o /tmp/${S6_NAME}
-curl -fL https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/${S6_NAME}.sig -o /tmp/${S6_NAME}.sig
+S6_OVERLAY_BASE_URL=https://github.com/just-containers/s6-overlay/
 
-gpg --keyserver pgp.surfnet.nl --recv-keys $PUBLIC_KEY
-gpg --verify /tmp/${S6_NAME}.sig /tmp/${S6_NAME}
+curl -fL "${S6_OVERLAY_BASE_URL}releases/download/${S6_VERSION}/${S6_NAME}" \
+  -o "/tmp/${S6_NAME}"
+curl -fL "${S6_OVERLAY_BASE_URL}releases/download/${S6_VERSION}/${S6_NAME}.sig" \
+  -o "/tmp/${S6_NAME}.sig"
+
+# 2022-08-28: commented out since this always keeps failing will need to
+# revisit. With newer versions of s6-overlay, the way to validate is check
+# the checksum
+# @see https://github.com/just-containers/s6-overlay#verifying-downloads
+#gpg --import /tmp/s6-gpg-pub-key
+#gpg --keyserver pgp.surfnet.nl --recv-keys $PUBLIC_KEY
+#gpg --verify "/tmp/${S6_NAME}.sig" "/tmp/${S6_NAME}"
 
 # Special handling - CentOS >= 7 + Ubuntu >= 20.04
 # @see https://github.com/just-containers/s6-overlay#bin-and-sbin-are-symlinks
-# Need to also exclude the symlink included in s6-overlay-amd64.tar.gz as the symlink would otherwise overwrite the binary
+# Need to also exclude the symlink included in s6-overlay-amd64.tar.gz as the
+# symlink would otherwise overwrite the binary
 # $ tar tvzf s6-overlay-amd64.tar.gz |grep execlineb
 # -rwxr-xr-x root/root     33856 2019-03-21 12:29 ./bin/execlineb
 # lrwxrwxrwx root/root         0 2019-03-21 12:40 ./usr/bin/execlineb -> /bin/execlineb
 
 if [[ -L /bin ]]; then
-  tar xzf /tmp/${S6_NAME} -C / --exclude="./bin" --exclude="./usr/bin/execlineb"
-  tar xzf /tmp/${S6_NAME} -C /usr ./bin --exclude="./usr/bin/execlineb"
+  tar xzf "/tmp/${S6_NAME}" -C \
+    / --exclude="./bin" --exclude="./usr/bin/execlineb"
+  tar xzf "/tmp/${S6_NAME}" -C \
+    /usr ./bin --exclude="./usr/bin/execlineb"
 else
-  tar xzf /tmp/${S6_NAME} -C /
+  tar xzf "/tmp/${S6_NAME}" -C /
 fi
 
-rm /tmp/${S6_NAME} && rm /tmp/${S6_NAME}.sig
+rm "/tmp/${S6_NAME}" && rm "/tmp/${S6_NAME}.sig"
diff --git a/container/root/scripts/ubuntu/update_apt_sources.sh b/container/root/scripts/ubuntu/update_apt_sources.sh
@@ -0,0 +1,47 @@
+#!/bin/bash -e
+
+# https://jira.corp.adobe.com/browse/ETHOS-28973
+# Add the ability to switch to a different mirror as needed
+
+APT_SOURCE_LIST=${APT_SOURCE_LIST:=/etc/apt/sources.list}
+
+update_apt_sources() {
+    local _src="$1" _mirror="$2"
+
+    # Mirror URL must be set and must begin with http
+    if [[ -z "$_mirror" ]]; then
+        return
+    fi
+    
+    if [[ "$_mirror" == http* ]]; then
+        sed -i -e "s|${_src}|${_mirror}|g" "${APT_SOURCE_LIST}"
+    else
+        echo "warning: Unsupported mirror url: $_mirror"
+    fi
+}
+
+if [[ ! -f "$APT_SOURCE_LIST" ]]; then
+    echo "failure: $APT_SOURCE_LIST does not exist!"
+    exit 1
+fi
+
+if [[ -n "$UBUNTU_ARCHIVE_MIRROR_URL" ]]; then
+    update_apt_sources "http://archive.ubuntu.com/ubuntu/" \
+        "$UBUNTU_ARCHIVE_MIRROR_URL"
+fi
+
+if [[ -n "$UBUNTU_SECURITY_MIRROR_URL" ]]; then
+    update_apt_sources "http://security.ubuntu.com/ubuntu/" \
+        "$UBUNTU_SECURITY_MIRROR_URL"
+fi
+
+if [[ -n "$UBUNTU_PORTS_MIRROR_URL" ]]; then
+    update_apt_sources "http://ports.ubuntu.com/ubuntu-ports/" \
+        "$UBUNTU_PORTS_MIRROR_URL"
+fi
+
+# Useful when you need to debug the contents of sources.list
+if [[ -n "$DEBUG" ]]; then
+    echo "*** DEBUG is set. Dumping $APT_SOURCE_LIST ***"
+    grep -v '#' "$APT_SOURCE_LIST"
+fi
diff --git a/docs/mirrors.md b/docs/mirrors.md
@@ -0,0 +1,63 @@
+# Mirrors
+
+By default, Ubuntu ships with the following configuration:
+
+```shell
+root@979e866735eb:/# grep -v '#' /etc/apt/sources.list
+deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted
+deb http://ports.ubuntu.com/ubuntu-ports/ focal universe
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates universe
+deb http://ports.ubuntu.com/ubuntu-ports/ focal multiverse
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates multiverse
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted universe multiverse
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-security universe
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-security multiverse
+```
+
+There might be a need for you to replace the sources URL (as it was with
+[ETHOS-28973]) and re-point to a different mirror.
+
+One way to tackle this problem is to simply add something similar to the
+following before you perform an `apt-get -y update`:
+
+```shell
+RUN sed -i \
+    -e 's|http://archive.ubuntu.com/ubuntu|https://your/remote|g' \
+    -e 's|http://security.ubuntu.com/ubuntu|https://your/remote|g' \
+    -e 's|http://ports.ubuntu.com/ubuntu-ports|https://your/remote|g' \
+"/etc/apt/sources.list"
+```
+
+# update_apt_sources.sh
+
+For Blessed Base Containers which inherit from this `docker-base`, we use
+the [update_apt_sources.sh] script to re-point the sources to our Corporate
+mirrors on demand.
+
+To use the script, you would set one or more of the following environment.
+When set, the script will search and replace the URL with the value that you
+specify:
+
+| Environment Variable          | Search and replace URL if present          |
+| ----------------------------- | ------------------------------------------ | 
+| `UBUNTU_PORTS_MIRROR_URL`     | http://ports.ubuntu.com/ubuntu-ports/      |
+| `UBUNTU_ARCHIVE_MIRROR_URL`   | http://archive.ubuntu.com/ubuntu/          |
+| `UBUNTU_SECURITY_MIRROR_URL`  | http://security.ubuntu.com/ubuntu/         |
+
+For example, in your `Dockerfile`
+
+```
+COPY ./container/root /
+
+# Set environment
+ENV UBUNTU_PORTS_MIRROR_URL=https://foo/my/mirror
+
+# Run the replacement before an apt-get
+RUN /bin/bash -e /scripts/ubuntu/test_update_apt_sources.sh && \
+    apt-get -y update && apt-get -y upgrade \
+    [...snip...]
+```
+
+[update_apt_sources.sh]: ../container/root/scripts/ubuntu/test_update_apt_sources.sh
diff --git a/docs/tests.md b/docs/tests.md
@@ -0,0 +1,37 @@
+# Tests
+
+## shunit2
+
+Starting August 28, 2022, all shell scripts should ideally have a corresponding
+`shunit2` test.
+
+See [shunit2] for usage and documentation.
+
+Guidelines:
+
+* Tests are stored in `tests/sh`
+* Path to the test should mirror the actual location of the original script
+* Name of the script should be `test_` + name of actual script to test
+
+For an example, see [test_update_apt_sources.sh].
+
+To run all tests:
+
+```shell
+make shunit2
+```
+
+To run a specific test file (useful to target a specific test)
+
+```shell
+RUN_TEST_FILE=tests/sh/container/root/scripts/ubuntu/test_update_apt_sources.sh make shunit2
+```
+
+To run all tests in a specific directory:
+
+```shell
+RUN_TEST_DIR=tests/sh/container/root/scripts/ubuntu make shunit2
+```
+
+[shunit2]: https://github.com/kward/shunit2
+[test_update_apt_sources.sh]: ../tests/sh/container/root/scripts/ubuntu/test_update_apt_sources.sh
diff --git a/tests/sh/Dockerfile.shunit2 b/tests/sh/Dockerfile.shunit2
@@ -0,0 +1,12 @@
+FROM ubuntu:22.04
+
+WORKDIR /
+
+# Copy the top-level directory that we wish to test
+COPY ./container/root /
+
+# Copy our tests
+COPY ./tests /tests
+
+# Run the shunit2 tests
+CMD ["/bin/bash", "/tests/sh/run_tests.sh"]
diff --git a/tests/sh/container/root/scripts/ubuntu/CHANGELOG.md b/tests/sh/container/root/scripts/ubuntu/CHANGELOG.md
@@ -0,0 +1,5 @@
+# tests CHANGELOG
+
+# 0.1.0
+
+- Add this folder
diff --git a/tests/sh/container/root/scripts/ubuntu/test_update_apt_sources.sh b/tests/sh/container/root/scripts/ubuntu/test_update_apt_sources.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+
+setUp() {
+    # Get the path to this test script
+    SHUNIT2_TEST_SCRIPT=$0
+    # Replace the path tests/sh and strip out the test_ part of the name
+    SCRIPT_TO_TEST=$(echo "$SHUNIT2_TEST_SCRIPT" | \
+        sed 's|tests/sh/container/root||' | sed 's|test_||')
+    # Temp location to store script output for validation purposes
+    TEST_OUTPUT=/tmp/$$.out
+
+    # Fake copy of the real apt sources list
+    FAKE_APT_SOURCE_LIST=/tmp/foo.list
+}
+
+tearDown() {
+    # Clean up
+    if [[ -f "$TEST_OUTPUT" ]]; then
+        rm -f "$TEST_OUTPUT"
+    fi
+}
+
+create_fake_source_list() {
+    # Helper to create a temporary source list for testing purposes
+    local _prefix="$1"
+
+    # Create a fake source list for testing purposes
+    cat << EOF > "/tmp/${_prefix}.list"
+deb http://${_prefix}.ubuntu.com/ubuntu/ focal main restricted
+
+deb http://${_prefix}.ubuntu.com/ubuntu-ports/ focal-updates main restricted
+
+deb http://${_prefix}.ubuntu.com/ubuntu-ports/ focal universe
+deb http://${_prefix}.ubuntu.com/ubuntu-ports/ focal-updates universe
+EOF
+}
+
+test_fail_when_sources_list_does_not_exist() {
+    APT_SOURCE_LIST=/tmp/foo \
+        bash "$SCRIPT_TO_TEST" 2>&1 | tee -a "$TEST_OUTPUT" > /dev/null
+    # cat "$TEST_OUTPUT"
+
+    grep "failure: /tmp/foo does not exist!" "$TEST_OUTPUT" > /dev/null
+    assertTrue "Expecting failure message" "[ $? -eq 0 ]"
+}
+
+test_non_http_mirror() {
+    UBUNTU_ARCHIVE_MIRROR_URL=ftp://foo.bar \
+        bash "$SCRIPT_TO_TEST" 2>&1 | tee -a "$TEST_OUTPUT" > /dev/null
+    # cat "$TEST_OUTPUT"
+
+    grep "warning: Unsupported mirror url: ftp://foo.bar" "$TEST_OUTPUT" \
+        > /dev/null
+    assertTrue "Expecting warning message" "[ $? -eq 0 ]"
+}
+
+test_update_apt_source_ports() {
+    # Make a copy of the original sources.list.
+    # Typical contents of this file looks like:
+    # root@979e866735eb:/# grep -v '#' /etc/apt/sources.list
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal universe
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates universe
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal multiverse
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates multiverse
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted universe multiverse
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal-security universe
+    # deb http://ports.ubuntu.com/ubuntu-ports/ focal-security multiverse
+    cp /etc/apt/sources.list "$FAKE_APT_SOURCE_LIST"
+
+    # Do our search and replace
+    APT_SOURCE_LIST="$FAKE_APT_SOURCE_LIST" \
+    UBUNTU_PORTS_MIRROR_URL=https://foo/bar/ports-ubuntu-remote \
+        bash "$SCRIPT_TO_TEST" 2>&1 | tee -a "$TEST_OUTPUT" > /dev/null
+
+    # Confirm that ports.ubuntu.com no longer exist
+    grep "http://ports.ubuntu.com/ubuntu-ports" "$FAKE_APT_SOURCE_LIST" \
+        > /dev/null
+    assertTrue "Expecting warning message" "[ $? -ne 0 ]"
+
+    # Confirm that we completed our search and replace
+    local _total=""
+    _total=$(grep -c "$UBUNTU_PORTS_MIRROR_URL" "$FAKE_APT_SOURCE_LIST")
+    assertTrue "Expecting ports url to be replaced" "[ $_total -gt 0 ]"
+}
+
+test_update_apt_source_archive() {
+    create_fake_source_list "archive"
+
+    # Do our search and replace
+    APT_SOURCE_LIST="/tmp/archive.list" \
+    UBUNTU_ARCHIVE_MIRROR_URL=https://foo/bar/archive-ubuntu-remote \
+        bash "$SCRIPT_TO_TEST" 2>&1 | tee -a "$TEST_OUTPUT" > /dev/null
+
+    local _total=""
+    _total=$(grep -c "$UBUNTU_ARCHIVE_MIRROR_URL" "$FAKE_APT_SOURCE_LIST")
+    assertTrue "Expecting archive url to be replaced" "[ $_total -gt 0 ]"
+}
+
+test_update_apt_source_security() {
+    create_fake_source_list "security"
+
+    # Do our search and replace
+    APT_SOURCE_LIST="/tmp/security.list" \
+    UBUNTU_ARCHIVE_MIRROR_URL=https://foo/bar/security-ubuntu-remote \
+        bash "$SCRIPT_TO_TEST" 2>&1 | tee -a "$TEST_OUTPUT" > /dev/null
+
+    local _total=""
+    _total=$(grep -c "$UBUNTU_ARCHIVE_MIRROR_URL" "$FAKE_APT_SOURCE_LIST")
+    assertTrue "Expecting url to be replaced" "[ $_total -gt 0 ]"
+}
+
+# Load shunit2 as the last line
+. /tests/sh/vendor/shunit2/shunit2
diff --git a/tests/sh/run_tests.sh b/tests/sh/run_tests.sh
diff --git a/tests/sh/vendor/shunit2 b/tests/sh/vendor/shunit2
diff --git a/tests/sh/vendor/shunit2-2.1.8/shunit2 b/tests/sh/vendor/shunit2-2.1.8/shunit2

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +# tests CHANGELOG
++
 +# 0.1.0
++
 +- Add this folder