Clean up more Dockerfile templates and optimize removing/extracting multiple files

eng/dockerfile-templates/Dockerfile.linux.download-dotnet

@@ -5,38 +5,67 @@
         extract-paths   : (optional) Paths within the tarball to extract
         out-file        : (optional) Name of the output file
 
-        The following arguments are mutually exclusive:
-            sha               : (optional) The checksum of the downloaded file
-            sha-url           : (optional) URL to download checksum file
-            sha-url-aggregate : (optional) URL to download aggregate checksum file
+        The following arguments groups are mutually exclusive:
+
+        sha                  : (optional) The checksum of the downloaded file
+
+        sha-url              : (optional) URL to download checksum file
+        sha-url-is-aggregate : (optional) True if the checksum file contains more than one checksum,
+                               but we only want to validate the checksum of the file we downloaded
         ^
 
+    set shaFunction to "512" ^
     set urlParts to split(ARGS["download-url"], "/") ^
     set fileName to
         when(ARGS["out-file"],
             ARGS["out-file"],
             urlParts[len(urlParts) - 1]) ^
+
     set fileNameParts to split(fileName, ".") ^
-    set fileExtension to fileNameParts[len(fileNameParts) - 1]
-
-}}{{InsertTemplate("Dockerfile.linux.download-validate-file", [
-    "url": ARGS["download-url"],
-    "out-file": ARGS["out-file"],
-    "sha": ARGS["sha"],
-    "sha-url": ARGS["sha-url"],
-    "sha-url-aggregate": ARGS["sha-url-aggregate"],
-    "sha-var-name": ARGS["sha-var-name"]
-])}}{{
+    set fileExtension to fileNameParts[len(fileNameParts) - 1] ^
+
+    set shaUrlParts to split(ARGS["sha-url"], "/") ^
+    set shaFileName to shaUrlParts[len(shaUrlParts) - 1] ^
+
+    set filesToDownload to
+        when(ARGS["sha-url"],
+            [
+                ["url": ARGS["download-url"]],
+                ["url": ARGS["sha-url"]]
+            ],
+            [
+                ["url": ARGS["download-url"], "out-file": ARGS["out-file"]]
+            ]
+        )
+
+}}{{
+    InsertTemplate("Dockerfile.linux.download-files", [
+        "files": filesToDownload
+    ])
+}}{{
+if (ARGS["sha"] || ARGS["sha-url"]): \
+&& {{
+    InsertTemplate("Dockerfile.linux.validate-checksum", [
+        "file": fileName,
+        "sha-function": shaFunction,
+        "sha": ARGS["sha"],
+        "sha-var-name": ARGS["sha-var-name"],
+        "sha-file": shaFileName,
+        "sha-file-is-aggregate": ARGS["sha-url-is-aggregate"]
+    ])
+}}}}{{
 if ARGS["extract-to"]: \
 && mkdir --parents {{ARGS["extract-to"]}} \
-&& {{if (fileExtension = "zip"):{{
-    InsertTemplate("Dockerfile.linux.extract-zip", [
-        "file": fileName,
-        "dest-dir": ARGS["extract-to"],
-        "extract-paths": ARGS["extract-paths"]
-])}}^else:{{
-    InsertTemplate("Dockerfile.linux.extract-tarball", [
+&& {{
+    InsertTemplate("Dockerfile.linux.extract-file", [
         "file": fileName,
         "dest-dir": ARGS["extract-to"],
         "extract-paths": ARGS["extract-paths"]
-])}}}}}}
+    ])
+}}}}{{
+if (ARGS["sha-url"]): \
+&& rm \
+    {{fileName}} \
+    {{shaFileName}}^
+else: \
+&& rm {{fileName}}}}

eng/dockerfile-templates/Dockerfile.linux.download-files

@@ -54,7 +54,7 @@
     set numFiles to len(ARGS["files"]) ^
     set indent to when(numFiles > 1, "    ", "")
 
-}}{{downloadCommand}} {{if (numFiles > 1): \
+}}{{downloadCommand}} {{if (numFiles > 1):\
 }}{{
 for i, file in ARGS["files"]:{{indent}}{{
     set isLastFile to (i = len(ARGS["files"]) - 1)

eng/dockerfile-templates/Dockerfile.linux.extract-file

@@ -0,0 +1,18 @@
+{{
+    _ Extracts a file (zip or tarball) based on its extension
+
+    ARGS:
+        file: File to extract
+        dest-dir: Destination directory
+        extract-paths: (optional) Paths within the archive to extract
+        ^
+
+    set fileNameParts to split(ARGS["file"], ".") ^
+    set fileExtension to fileNameParts[len(fileNameParts) - 1]
+
+
+}}{{
+if (fileExtension = "zip")
+    :unzip {{ARGS["file"]}}{{if ARGS["dest-dir"]: -d {{ARGS["dest-dir"]}}}}{{if ARGS["extract-paths"]: {{join(ARGS["extract-paths"], " ")}}}}^
+else
+    :tar --gzip --extract --no-same-owner --file {{ARGS["file"]}}{{if ARGS["dest-dir"]: --directory {{ARGS["dest-dir"]}}}}{{if ARGS["extract-paths"]: {{join(ARGS["extract-paths"], " ")}}}}}}

eng/dockerfile-templates/Dockerfile.linux.validate-checksum

@@ -3,25 +3,25 @@
 
     ARGS:
         Required by all option groups:
-        sha-function        : SHA function to use (e.g. 256, 384, 512), not optional.
+        sha-function          : SHA function to use (e.g. 256, 384, 512), not optional.
 
         The following argument groups are mutually exclusive:
 
-        file                : Path to the file to validate.
-        sha                 : Expected checksum of the downloaded file.
-        sha-var-name        : (optional) Name of variable to store the checksum in. If not provided,
-                              the checksum will be written inline with the call to the sha*sum check
-                              tool.
-
-        sha-file            : Checksum file to validate against. The file should contain a single
-                              line with the expected checksum and the file name to validate,
-                              separated by one or two spaces. See `man cksum` or `man sha512sum` for
-                              details.
-
-        sha-file-aggregate  : File containing a list of checksums. The file may contain one or more
-                              lines with expected checksums and file names separated by one or two
-                              spaces. One line must have a filename matching the file you wish to
-                              validate. See `man cksum` or `man sha512sum` for details.
+        file                  : Path to the file to validate.
+        sha                   : Expected checksum of the downloaded file.
+        sha-var-name          : (optional) Name of variable to store the checksum in. If not provided,
+                                the checksum will be written inline with the call to the sha*sum check
+                                tool.
+
+        sha-file              : Checksum file to validate against. The file should contain a single
+                                line with the expected checksum and the file name to validate,
+                                separated by one or two spaces. See `man cksum` or `man sha512sum` for
+                                details.
+
+        sha-file-is-aggregate : File containing a list of checksums. The file may contain one or more
+                                lines with expected checksums and file names separated by one or two
+                                spaces. One line must have a filename matching the file you wish to
+                                validate. See `man cksum` or `man sha512sum` for details.
         ^
 
     set isAlpine to find(OS_VERSION, "alpine") >= 0 ^
@@ -31,12 +31,14 @@
         ARGS["sha"])
 
 }}{{
-if ARGS["sha"]:{{if ARGS["sha-var-name"]:{{ARGS["sha-var-name"]}}='{{ARGS["sha"]}}' \
+if ARGS["sha"]:{{
+if ARGS["sha-var-name"]:{{ARGS["sha-var-name"]}}='{{ARGS["sha"]}}' \
 && }}echo "{{shaValue}}  {{ARGS["file"]}}" | sha{{ARGS["sha-function"]}}sum -c -^
-elif (ARGS["sha-file"])
-    :sha{{ARGS["sha-function"]}}sum -c {{ARGS["sha-file"]}}^
-elif (ARGS["sha-file-aggregate"])
-    :if (isAlpine)
-        :awk -v file="{{ARGS["file"]}}" '{gsub(/\r/, "")} $2 == file' ${{ARGS["sha-file-aggregate"]}} | sha512sum -c^
+elif (ARGS["file"]):{{
+    if (ARGS["sha-file-is-aggregate"]):{{
+        if (isAlpine)
+            :awk -v file="{{ARGS["file"]}}" '{gsub(/\r/, "")} $2 == file' {{ARGS["sha-file"]}} | sha{{ARGS["sha-function"]}}sum -c^
+        else
+            :sha{{ARGS["sha-function"]}}sum -c {{ARGS["sha-file"]}} --ignore-missing}}^
     else
-        :sha512sum --ignore-missing --check {{ARGS["sha-file"]}}}}
+        :sha{{ARGS["sha-function"]}}sum -c {{ARGS["sha-file"]}}}}}}

eng/dockerfile-templates/monitor/Dockerfile.linux.install-extensions

@@ -37,13 +37,16 @@
         "sha-function": "512"
     ], "    ")}} \
     && mkdir --parents {{installDir}} \
-    && {{InsertTemplate("../Dockerfile.linux.extract-tarball", [
+    && {{InsertTemplate("../Dockerfile.linux.extract-file", [
         "file": azBlobFileName,
         "dest-dir": installDir,
         "extract-paths": ARGS["extract-paths"]
     ], "    ")}} \
-    && {{InsertTemplate("../Dockerfile.linux.extract-tarball", [
+    && {{InsertTemplate("../Dockerfile.linux.extract-file", [
         "file": s3FileName,
         "dest-dir": installDir,
         "extract-paths": ARGS["extract-paths"]
-    ], "    ")}}
+    ], "    ")}} \
+    && rm \
+        {{azBlobFileName}} \
+        {{s3FileName}}

eng/dockerfile-templates/runtime-deps/Dockerfile.chiseled-ubuntu

@@ -8,10 +8,12 @@
     set osVersionNumber to split(OS_ARCH_HYPHENATED, "-")[1] ^
 
     set chiselUrl to VARIABLES[cat("chisel|", dotnetVersion, "|", ARCH_SHORT, "|url")] ^
+    set chiselFileName to "chisel.tar.gz" ^
     set shaFunction to "384" ^
     set chiselSha to VARIABLES[cat("chisel|", dotnetVersion, "|", ARCH_SHORT, "|sha", shaFunction)] ^
-    set rocksToolboxUrl to VARIABLES[cat("rocks-toolbox|", dotnetVersion, "|url")] ^
+    set rocksToolboxBaseUrl to VARIABLES[cat("rocks-toolbox|", dotnetVersion, "|url")] ^
     set rocksToolboxRef to VARIABLES[cat("rocks-toolbox|", dotnetVersion, "|version")] ^
+    set rocksToolboxUrl to cat(rocksToolboxBaseUrl, "/", rocksToolboxRef, "/chisel-wrapper") ^
 
     set installerImageTag to cat(ARCH_VERSIONED, "/buildpack-deps:", osVersionBase, "-curl") ^
 
@@ -62,20 +64,25 @@
 
 RUN apt-get update && apt-get install -y file
 
-RUN {{InsertTemplate("../Dockerfile.linux.download-validate-file",
-    [
-        "out-file": "chisel.tar.gz",
+RUN {{InsertTemplate("../Dockerfile.linux.download-files", ["files": [[
         "url": chiselUrl,
-        "sha": chiselSha,
-        "sha-var-name": cat("chisel_sha", shaFunction),
+        "out-file": chiselFileName
+    ]]], "    ")}} \
+    && {{InsertTemplate("../Dockerfile.linux.validate-checksum", [
+        "file": chiselFileName,
         "sha-function": shaFunction,
+        "sha": chiselSha,
+        "sha-var-name": cat("chisel_sha", shaFunction)
     ], "    ")}} \
-    && tar -xzf chisel.tar.gz -C /usr/bin/ chisel \
-    && {{InsertTemplate("../Dockerfile.linux.download-validate-file",
-    [
-        "out-file": "/usr/bin/chisel-wrapper",
-        "url": cat(rocksToolboxUrl, "/", rocksToolboxRef, "/chisel-wrapper")
+    && {{InsertTemplate("../Dockerfile.linux.extract-file", [
+        "file": chiselFileName,
+        "dest-dir": "/usr/bin/"
     ], "    ")}} \
+    && rm {{chiselFileName}} \
+    && {{InsertTemplate("../Dockerfile.linux.download-files", ["files": [[
+        "url": rocksToolboxUrl,
+        "out-file": "/usr/bin/chisel-wrapper"
+    ]]], "    ")}} \
     && chmod 755 /usr/bin/chisel-wrapper
 
 RUN {{InsertTemplate("Dockerfile.linux.non-root-user-staged", [

eng/dockerfile-templates/sdk/Dockerfile.linux.install-powershell

@@ -7,13 +7,16 @@
     set url to cat("https://powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net/tool/$powershell_version/", nupkgName)
 }}# Install PowerShell global tool
 RUN powershell_version={{VARIABLES[cat("powershell|", dotnetVersion, "|build-version")]}} \
-    && {{InsertTemplate("../Dockerfile.linux.download-validate-file",
-        [
-            "out-file": nupkgName,
-            "url": url,
-            "sha": VARIABLES[join(["powershell", dotnetVersion, shaPlatform, "sha"], "|")],
-            "sha-var-name": "powershell_sha512"
-        ], "    ")}} \
+    && {{InsertTemplate("../Dockerfile.linux.download-files", ["files": [[
+        "url": url,
+        "out-file": nupkgName
+    ]]], "    ")}} \
+    && {{InsertTemplate("../Dockerfile.linux.validate-checksum", [
+        "file": nupkgName,
+        "sha-function": "512",
+        "sha-var-name": "powershell_sha512",
+        "sha": VARIABLES[join(["powershell", dotnetVersion, shaPlatform, "sha"], "|")]
+    ], "    ")}} \
     && mkdir --parents /usr/share/powershell \
     && dotnet tool install --add-source / --tool-path /usr/share/powershell --version $powershell_version PowerShell.{{nupkgPlatform}} \
     && dotnet nuget locals all --clear \

src/monitor/8.1/azurelinux-distroless/amd64/Dockerfile

@@ -11,7 +11,7 @@ RUN tdnf install -y \
 
 # Retrieve .NET Monitor extensions
 RUN dotnet_monitor_extension_version=8.1.1 \
-    && curl --fail --show-error --location  \
+    && curl --fail --show-error --location \
         --output dotnet-monitor-egress-azureblobstorage.tar.gz https://ci.dot.net/public/diagnostics/monitor/8.1.1-servicing.25257.8/dotnet-monitor-egress-azureblobstorage-$dotnet_monitor_extension_version-linux-x64.tar.gz \
         --output dotnet-monitor-egress-s3storage.tar.gz https://ci.dot.net/public/diagnostics/monitor/8.1.1-servicing.25257.8/dotnet-monitor-egress-s3storage-$dotnet_monitor_extension_version-linux-x64.tar.gz \
     && dotnet_monitor_azureblobstorage_sha512='072f71d2127a3302cd91677373d6e4b96af56a0b031b1215199e07657e6fc1965eed015fc8b8485d41735bd417d20d4d5007269ef5da022bcdda46551aa6388c' \
@@ -20,9 +20,10 @@ RUN dotnet_monitor_extension_version=8.1.1 \
     && echo "$dotnet_monitor_s3storage_sha512  dotnet-monitor-egress-s3storage.tar.gz" | sha512sum -c - \
     && mkdir --parents /app \
     && tar --gzip --extract --no-same-owner --file dotnet-monitor-egress-azureblobstorage.tar.gz --directory /app \
-    && rm dotnet-monitor-egress-azureblobstorage.tar.gz \
     && tar --gzip --extract --no-same-owner --file dotnet-monitor-egress-s3storage.tar.gz --directory /app \
-    && rm dotnet-monitor-egress-s3storage.tar.gz
+    && rm \
+        dotnet-monitor-egress-azureblobstorage.tar.gz \
+        dotnet-monitor-egress-s3storage.tar.gz
 
 
 # .NET Monitor image