Use external debian packagemanager

- use debian_versions.bzl and debian_archives.bzl generated by debian_package_manager - os_release which was previously generated by package manager is now just a new rule - Remove the custom deb repo rule from use (keep the code in here for now till we figure out if anyone is using it, so we can move it somewhere)
GoogleContainerTools · Jan 19, 2022 · 71836ee · 71836ee
1 parent ebfa82e
commit 71836ee
Show file tree

Hide file tree

Showing 34 changed files with 2,413 additions and 927 deletions.
diff --git a/.github/workflows/update-deb-package-snapshots.yml b/.github/workflows/update-deb-package-snapshots.yml
@@ -13,10 +13,12 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v2
 
-    - name: Setup Bazel
-      uses: abhinavsingh/setup-bazel@1fe920bf5df3791aab606c06a3608f4bb600c4f2
+
+    - name: Set up Go 1.16.x
+      uses: actions/setup-go@v2
       with:
-        version: 4.2.1
+        go-version: 1.16.x
+      id: go
 
     - name: Run update script
       run: ./updateWorkspaceSnapshots.sh

diff --git a/WORKSPACE b/WORKSPACE
@@ -45,215 +45,17 @@ load(
     "SHA256s",
     "VERSIONS",
 )
+load(":debian_archives.bzl", debian_repositories = "repositories")
 
-[
-    dpkg_src(
-        name = arch + "_" + name,
-        arch = arch,
-        distro = distro,
-        sha256 = SHA256s[arch][name]["main"],
-        snapshot = DEBIAN_SNAPSHOT,
-        url = "https://snapshot.debian.org/archive",
-    )
-    for arch in ARCHITECTURES
-    for (name, distro) in VERSIONS
-]
+debian_repositories()
 
-[
-    dpkg_src(
-        name = arch + "_" + name + "_updates",
-        arch = arch,
-        distro = distro + "-updates",
-        sha256 = SHA256s[arch][name]["updates"],
-        snapshot = DEBIAN_SNAPSHOT,
-        url = "https://snapshot.debian.org/archive",
-    )
-    for arch in ARCHITECTURES
-    for (name, distro) in VERSIONS
-]
+load(":busybox_archives.bzl", busybox_repositories = "repositories")
 
-[
-    dpkg_src(
-        name = arch + "_" + name + "_security",
-        package_prefix = "https://snapshot.debian.org/archive/debian-security/{}/".format(DEBIAN_SECURITY_SNAPSHOT),
-        packages_url = "https://snapshot.debian.org/archive/debian-security/{}/dists/{}/updates/main/binary-{}/Packages.xz".format(DEBIAN_SECURITY_SNAPSHOT, distro, arch),
-        sha256 = SHA256s[arch][name]["security"],
-    )
-    for arch in ARCHITECTURES
-    for (name, distro) in VERSIONS
-    if "debian10" == name
-    if "security" in SHA256s[arch][name]
-]
+busybox_repositories()
 
-# debian11 has a slightly different structure for security on snapshots
-[
-    dpkg_src(
-        name = arch + "_" + name + "_security",
-        package_prefix = "https://snapshot.debian.org/archive/debian-security/{}/".format(DEBIAN_SECURITY_SNAPSHOT),
-        packages_url = "https://snapshot.debian.org/archive/debian-security/{}/dists/{}-security/main/binary-{}/Packages.xz".format(DEBIAN_SECURITY_SNAPSHOT, distro, arch),
-        sha256 = SHA256s[arch][name]["security"],
-    )
-    for arch in ARCHITECTURES
-    for (name, distro) in VERSIONS
-    if "debian11" == name
-    if "security" in SHA256s[arch][name]
-]
+load(":node_archives.bzl", node_repositories = "repositories")
 
-[
-    dpkg_src(
-        name = arch + "_" + name + "_backports",
-        arch = arch,
-        distro = distro + "-backports",
-        sha256 = SHA256s[arch][name]["backports"],
-        snapshot = DEBIAN_SNAPSHOT,
-        url = "https://snapshot.debian.org/archive",
-    )
-    for arch in ARCHITECTURES
-    for (name, distro) in VERSIONS
-    if "backports" in SHA256s[arch][name]
-]
-
-[
-    dpkg_list(
-        name = "package_bundle_" + arch + "_debian11",
-        packages = [
-            "base-files",
-            "ca-certificates",
-            "libc6",
-            "libc-bin",
-            "libssl1.1",
-            "netbase",
-            "openssl",
-            "tzdata",
-
-            # c++
-            "libgcc-s1",
-            "libgomp1",
-            "libstdc++6",
-        ] + ([
-            # python only builds on amd64/arm64
-            "dash",
-            "libbz2-1.0",
-            "libcom-err2",
-            "libcrypt1",  # TODO: glibc library for -lcrypt; maybe should be in cc?
-            "libdb5.3",
-            "libexpat1",
-            "libffi7",
-            "libgssapi-krb5-2",
-            "libk5crypto3",
-            "libkeyutils1",
-            "libkrb5-3",
-            "libkrb5support0",
-            "liblzma5",
-            "libmpdec3",
-            "libncursesw6",
-            "libnsl2",
-            "libpython3.9-minimal",
-            "libpython3.9-stdlib",
-            "libreadline8",
-            "libsqlite3-0",
-            "libtinfo6",
-            "libtirpc3",
-            "libuuid1",
-            "python3-distutils",
-            "python3.9-minimal",
-            "zlib1g",
-            # java only builds on amd64/arm64
-            "fontconfig-config",
-            "fonts-dejavu-core",
-            "libbrotli1",
-            "libexpat1",
-            "libfontconfig1",
-            "libfreetype6",
-            "libglib2.0-0",
-            "libgraphite2-3",
-            "libharfbuzz0b",
-            "libjpeg62-turbo",
-            "liblcms2-2",
-            "libpcre3",
-            "libpng16-16",
-            "libuuid1",
-            "openjdk-11-jdk-headless",
-            "openjdk-11-jre-headless",
-            "openjdk-17-jdk-headless",  # 11 and 17 should share the same "base"
-            "openjdk-17-jre-headless",
-            "zlib1g",
-        ] if arch in BASE_ARCHITECTURES else []),
-        sources = [
-            "@" + arch + "_debian11_security//file:Packages.json",
-            "@" + arch + "_debian11_updates//file:Packages.json",
-            "@" + arch + "_debian11//file:Packages.json",
-        ],
-    )
-    for arch in ARCHITECTURES
-]
-
-[
-    dpkg_list(
-        name = "package_bundle_" + arch + "_debian10",
-        packages = [
-            "libc6",
-            "libc-bin",
-            "base-files",
-            "ca-certificates",
-            "openssl",
-            "libssl1.1",
-            "libbz2-1.0",
-            "libdb5.3",
-            "libffi6",
-            "liblzma5",
-            "libreadline7",
-            "libsqlite3-0",
-            "mime-support",
-            "netbase",
-            "readline-common",
-            "tzdata",
-
-            #c++
-            "libgcc1",
-            "libgomp1",
-            "libstdc++6",
-        ] + ([
-            # python3 only builds on amd64/arm64
-            "dash",
-            "libexpat1",
-            "libmpdec2",
-            "libpython3.7-minimal",
-            "libpython3.7-stdlib",
-            "libtinfo6",
-            "libuuid1",
-            "libncursesw6",
-            "python3-distutils",
-            "python3.7-minimal",
-            "zlib1g",
-        ] if arch in BASE_ARCHITECTURES else []) + ([
-            # java only builds on amd64
-            "zlib1g",
-            "libjpeg62-turbo",
-            "libexpat1",
-            "libpng16-16",
-            "liblcms2-2",
-            "libfreetype6",
-            "fonts-dejavu-core",
-            "fontconfig-config",
-            "libfontconfig1",
-            "libuuid1",
-            "openjdk-11-jre-headless",
-            "openjdk-11-jdk-headless",
-            "libgraphite2-3",
-            "libharfbuzz0b",
-            "libglib2.0-0",
-            "libpcre3",
-            "zlib1g",
-        ] if arch == "amd64" else []),
-        sources = [
-            "@" + arch + "_debian10_security//file:Packages.json",
-            "@" + arch + "_debian10_updates//file:Packages.json",
-            "@" + arch + "_debian10//file:Packages.json",
-        ],
-    )
-    for arch in ARCHITECTURES
-]
+node_repositories()
 
 # For Jetty
 http_archive(
@@ -265,102 +67,6 @@ http_archive(
     urls = ["https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-distribution/9.4.43.v20210629/jetty-distribution-9.4.43.v20210629.tar.gz"],
 )
 
-# Node (https://nodejs.org/en/about/releases/)
-# Follow Node's maintainence schedule and support all LTS versions that are not end of life
-http_archive(
-    name = "nodejs12_amd64",
-    build_file = "//nodejs:BUILD.nodejs",
-    sha256 = "860c481f0e7963cbe5afa669d9e5deefa773fb67da571823945ac79a3ea76d3c",
-    strip_prefix = "node-v12.22.9-linux-x64/",
-    type = "tar.gz",
-    urls = ["https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.gz"],
-)
-
-http_archive(
-    name = "nodejs14_amd64",
-    build_file = "//nodejs:BUILD.nodejs",
-    sha256 = "bd96f88e054801d1368787f7eaf77b49cd052b9543c56bd6bc0bfc90310e2756",
-    strip_prefix = "node-v14.18.3-linux-x64/",
-    type = "tar.gz",
-    urls = ["https://nodejs.org/dist/v14.18.3/node-v14.18.3-linux-x64.tar.gz"],
-)
-
-http_archive(
-    name = "nodejs16_amd64",
-    build_file = "//nodejs:BUILD.nodejs",
-    sha256 = "a0f23911d5d9c371e95ad19e4e538d19bffc0965700f187840eb39a91b0c3fb0",
-    strip_prefix = "node-v16.13.2-linux-x64/",
-    type = "tar.gz",
-    urls = ["https://nodejs.org/dist/v16.13.2/node-v16.13.2-linux-x64.tar.gz"],
-)
-
-http_archive(
-    name = "nodejs12_arm64",
-    build_file = "//nodejs:BUILD.nodejs",
-    sha256 = "307aa26c68600e2f73d699e58a15c59ea06928e4a348cd5a216278d9f2ee0d6c",
-    strip_prefix = "node-v12.22.9-linux-arm64/",
-    type = "tar.gz",
-    urls = ["https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-arm64.tar.gz"],
-)
-
-http_archive(
-    name = "nodejs14_arm64",
-    build_file = "//nodejs:BUILD.nodejs",
-    sha256 = "2d071ca1bc1d0ea1eb259e79b81ebb4387237b2f77b3cf616806534e0030eaa8",
-    strip_prefix = "node-v14.18.3-linux-arm64/",
-    type = "tar.gz",
-    urls = ["https://nodejs.org/dist/v14.18.3/node-v14.18.3-linux-arm64.tar.gz"],
-)
-
-http_archive(
-    name = "nodejs16_arm64",
-    build_file = "//nodejs:BUILD.nodejs",
-    sha256 = "e87d7c173d7c70672d71cc816ffe0baea2b0458cb7f96c248560410e9cd37522",
-    strip_prefix = "node-v16.13.2-linux-arm64/",
-    type = "tar.gz",
-    urls = ["https://nodejs.org/dist/v16.13.2/node-v16.13.2-linux-arm64.tar.gz"],
-)
-
-# For the debug image
-http_file(
-    name = "busybox_amd64",
-    executable = True,
-    sha256 = "51fcb60efbdf3e579550e9ab893730df56b33d0cc928a2a6467bd846cdfef7d8",
-    urls = ["https://busybox.net/downloads/binaries/1.31.0-defconfig-multiarch-musl/busybox-x86_64"],
-)
-
-http_file(
-    name = "busybox_arm",
-    executable = True,
-    sha256 = "cd04052b8b6885f75f50b2a280bfcbf849d8710c8e61d369c533acf307eda064",
-    urls = ["https://busybox.net/downloads/binaries/1.31.0-defconfig-multiarch-musl/busybox-armv7l"],
-)
-
-http_file(
-    name = "busybox_arm64",
-    executable = True,
-    sha256 = "141adb1b625a6f44c4b114f76b4387b4ea4f7ab802b88eb40e0d2f6adcccb1c3",
-    urls = ["https://busybox.net/downloads/binaries/1.31.0-defconfig-multiarch-musl/busybox-armv8l"],
-)
-
-http_file(
-    name = "busybox_s390x",
-    executable = True,
-    sha256 = "48d13ac057046b95ba58921958be639cc3a179ac888cdd65aacd7a69139aa857",
-    urls = ["https://busybox.net/downloads/binaries/1.31.0-defconfig-multiarch-musl/busybox-s390x"],
-)
-
-# To update ppc64le busybox binary (#723)
-# Get the latest commit hash from dist-ppc64le branch of docker-library repo.
-# Substitute it in the link: https://github.com/docker-library/busybox/raw/<latest-commit-hash>/stable/musl/busybox.tar.xz
-# Update the sha256 value. Since github api doesn't give sha256 value, it can be obtained using sha256sum command.
-http_file(
-    name = "busybox_ppc64le",
-    executable = True,
-    sha256 = "469297ea9293df0dcb6c3f8d344eaf9f9b6ec1732696ffe86994f87c3600423b",
-    urls = ["https://github.com/docker-library/busybox/raw/c0125333c4c3dfa4b9e5fd9fe6fbb875242f3613/stable/musl/busybox.tar.xz"],
-)
-
 # Docker rules.
 http_archive(
     name = "io_bazel_rules_docker",

diff --git a/base/base.bzl b/base/base.bzl
@@ -1,28 +1,30 @@
 # defines a function to replicate the container images for different distributions
 load("@io_bazel_rules_docker//container:container.bzl", "container_image")
 load("@io_bazel_rules_docker//contrib:test.bzl", "container_test")
-load(":distro.bzl", "DISTRO_PACKAGES", "DISTRO_REPOSITORY")
 load("//cacerts:cacerts.bzl", "cacerts")
 load("//:checksums.bzl", "ARCHITECTURES")
 load("@io_bazel_rules_go//go:def.bzl", "go_binary")
 
 NONROOT = 65532
 
+def deb_file(arch, distro, package):
+    return "@" + arch + "_" + distro + "_" + package + "//file"
+
 # Replicate everything for all distroless suffixes
 def distro_components(distro):
     for arch in ARCHITECTURES:
         cacerts(
             name = "cacerts_" + arch + "_" + distro,
-            deb = DISTRO_PACKAGES[arch][distro]["ca-certificates"],
+            deb = deb_file(arch, distro, "ca-certificates"),
         )
 
         for (user, uid, workdir) in [("root", 0, "/"), ("nonroot", NONROOT, "/home/nonroot")]:
             container_image(
                 name = "static_" + user + "_" + arch + "_" + distro,
                 debs = [
-                    DISTRO_PACKAGES[arch][distro]["base-files"],
-                    DISTRO_PACKAGES[arch][distro]["netbase"],
-                    DISTRO_PACKAGES[arch][distro]["tzdata"],
+                    deb_file(arch, distro, "base-files"),
+                    deb_file(arch, distro, "netbase"),
+                    deb_file(arch, distro, "tzdata"),
                 ],
                 architecture = arch,
                 env = {
@@ -42,7 +44,7 @@ def distro_components(distro):
                     # directory with specific permissions.
                     ":tmp.tar",
                     ":nsswitch.tar",
-                    DISTRO_REPOSITORY[arch][distro] + "//file:os_release.tar",
+                    "//os_release:os_release_" + distro + ".tar",
                     ":cacerts_" + arch + "_" + distro + ".tar",
                 ],
                 user = "%d" % uid,
@@ -54,9 +56,9 @@ def distro_components(distro):
                 architecture = arch,
                 base = ":static_" + user + "_" + arch + "_" + distro,
                 debs = [
-                    DISTRO_PACKAGES[arch][distro]["libc6"],
-                    DISTRO_PACKAGES[arch][distro]["libssl1.1"],
-                    DISTRO_PACKAGES[arch][distro]["openssl"],
+                    deb_file(arch, distro, "libc6"),
+                    deb_file(arch, distro, "libssl1.1"),
+                    deb_file(arch, distro, "openssl"),
                 ],
             )