Bug 1654994 - Switch full symbols archive to tar.zst. r=firefox-build…

…-system-reviewers,rstewart,tomprince

Differential Revision: https://phabricator.services.mozilla.com/D85252

Makefile.in

@@ -199,8 +199,8 @@ endif
 
 .PHONY: symbolsfullarchive
 symbolsfullarchive: prepsymbolsarchive
-	$(RM) '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
-	$(call py_action,symbols_archive,'$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' \
+	$(RM) '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst'
+	$(call py_action,symbols_archive,'$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst' \
                                      $(abspath $(DIST)/crashreporter-symbols) \
                                      --full-archive)
 
@@ -224,7 +224,7 @@ endif
 
 uploadsymbols:
 ifdef MOZ_CRASHREPORTER
-	$(PYTHON3) -u $(topsrcdir)/toolkit/crashreporter/tools/upload_symbols.py '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
+	$(PYTHON3) -u $(topsrcdir)/toolkit/crashreporter/tools/upload_symbols.py '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst'
 endif
 
 .PHONY: update-packaging

build/.lldbinit.in

@@ -11,7 +11,7 @@ settings set symbols.enable-external-lookup true
 settings append target.exec-search-paths @topobjdir@/toolkit/library
 settings append target.exec-search-paths @topobjdir@/mozglue/build
 
-# This is where artifact builds unpacks "crashreporter-symbols-full.zip" uncompressed ELF debug symbols.
+# This is where artifact builds unpacks "crashreporter-symbols-full" uncompressed ELF debug symbols.
 settings append target.debug-file-search-paths @topobjdir@/dist/crashreporter-symbols
 
 # These are specific paths encoded into Mozilla's automation outputs.

python/mozbuild/mozbuild/artifacts.py

@@ -159,7 +159,7 @@ def __init__(self, log=None,
         self._substs = substs
         self._symbols_archive_suffix = None
         if download_symbols == 'full':
-            self._symbols_archive_suffix = 'crashreporter-symbols-full.zip'
+            self._symbols_archive_suffix = 'crashreporter-symbols-full.tar.zst'
         elif download_symbols:
             self._symbols_archive_suffix = 'crashreporter-symbols.zip'
         self._mozbuild = mozbuild

taskcluster/ci/upload-symbols/kind.yml

@@ -2,7 +2,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
-loader: taskgraph.loader.single_dep:loader
+loader: taskgraph.loader.multi_dep:loader
+
+group-by: platform
 
 transforms:
     - taskgraph.transforms.upload_symbols:transforms
@@ -11,24 +13,36 @@ transforms:
 
 kind-dependencies:
     - build
+    - toolchain
+
+primary-dependency: build
 
 not-for-build-platforms:
     # Address Sanitizer builds don't have symbols to upload.
     - linux64-asan-reporter-shippable/opt
     - win64-asan-reporter-shippable/opt
+    - linux-geckodriver/opt
+    - linux64-geckodriver/opt
+    - macosx64-geckodriver/opt
+    - win32-geckodriver/opt
+    - win64-geckodriver/opt
 
 job-template:
     description: Upload Symbols
     worker-type: b-linux
     worker:
-        docker-image: {in-tree: "lint"}
+        docker-image: {in-tree: "debian10-base"}
         max-run-time: 900
         env:
             # {level} gets replaced in the upload_symbols transform
             SYMBOL_SECRET: "project/releng/gecko/build/level-{level}/gecko-symbol-upload"
+            MAKECAB: /builds/worker/fetches/makecab/makecab
     run:
         using: mach
-        mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py <build/public/build/target.crashreporter-symbols-full.zip>"}
+        mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py <build/public/build/target.crashreporter-symbols-full.tar.zst>"}
         sparse-profile: upload-symbols
     scopes:
         - secrets:get:project/releng/gecko/build/level-{level}/gecko-symbol-upload
+    fetches:
+        toolchain:
+            - linux64-makecab

taskcluster/taskgraph/transforms/upload_symbols.py

@@ -40,6 +40,7 @@ def check_nightlies(config, tasks):
 def fill_template(config, tasks):
     for task in tasks:
         dep = task['primary-dependency']
+        task.pop('dependent-tasks', None)
 
         # Fill out the dynamic fields in the task description
         task['label'] = dep.label + '-upload-symbols'

toolkit/crashreporter/tools/symbolstore.py

@@ -752,32 +752,10 @@ def ShouldProcess(self, file):
     def CopyDebug(self, file, debug_file, guid, code_file, code_id):
         file = locate_pdb(file)
 
-        def compress(path):
-            compressed_file = path[:-1] + "_"
-            # ignore makecab's output
-            makecab = buildconfig.substs["MAKECAB"]
-            wine = buildconfig.substs.get("WINE")
-            if wine and makecab.lower().endswith(".exe"):
-                cmd = [wine, makecab]
-            else:
-                cmd = [makecab]
-            success = subprocess.call(
-                cmd + ["-D", "CompressionType=MSZIP", path, compressed_file],
-                stdout=open(os.devnull, "w"),
-                stderr=subprocess.STDOUT,
-            )
-            if success == 0 and os.path.exists(compressed_file):
-                os.unlink(path)
-                return True
-            return False
-
         rel_path = os.path.join(debug_file, guid, debug_file).replace("\\", "/")
         full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
         shutil.copyfile(file, full_path)
-        if compress(full_path):
-            print(rel_path[:-1] + "_")
-        else:
-            print(rel_path)
+        print(rel_path)
 
         # Copy the binary file as well
         if code_file and code_id:
@@ -793,10 +771,7 @@ def compress(path):
                     if e.errno != errno.EEXIST:
                         raise
                 shutil.copyfile(full_code_path, full_path)
-                if compress(full_path):
-                    print(rel_path[:-1] + "_")
-                else:
-                    print(rel_path)
+                print(rel_path)
 
     def SourceServerIndexing(self, debug_file, guid, sourceFileStream, vcs_root):
         # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
@@ -862,9 +837,7 @@ def CopyDebug(self, file, debug_file, guid, code_file, code_id):
             rel_path = os.path.join(debug_file, guid, debug_file + ".dbg")
             full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
             shutil.move(file_dbg, full_path)
-            # gzip the shipped debug files
-            os.system("gzip -4 -f %s" % full_path)
-            print(rel_path + ".gz")
+            print(rel_path)
         else:
             if os.path.isfile(file_dbg):
                 os.unlink(file_dbg)
@@ -989,16 +962,15 @@ def GenerateDSYM(self, file):
     def CopyDebug(self, file, debug_file, guid, code_file, code_id):
         """ProcessFile has already produced a dSYM bundle, so we should just
         copy that to the destination directory. However, we'll package it
-        into a .tar.bz2 because the debug symbols are pretty huge, and
-        also because it's a bundle, so it's a directory. |file| here is the
+        into a .tar because it's a bundle, so it's a directory. |file| here is
         the original filename."""
         dsymbundle = file + ".dSYM"
         rel_path = os.path.join(
-            debug_file, guid, os.path.basename(dsymbundle) + ".tar.bz2"
+            debug_file, guid, os.path.basename(dsymbundle) + ".tar"
         )
         full_path = os.path.abspath(os.path.join(self.symbol_path, rel_path))
         success = subprocess.call(
-            ["tar", "cjf", full_path, os.path.basename(dsymbundle)],
+            ["tar", "cf", full_path, os.path.basename(dsymbundle)],
             cwd=os.path.dirname(dsymbundle),
             stdout=open(os.devnull, "w"),
             stderr=subprocess.STDOUT,

toolkit/crashreporter/tools/unit-symbolstore.py

@@ -192,7 +192,6 @@ def mock_dsymutil(args, **kwargs):
         d.Process(os.path.join(self.test_dir, add_extension(["foo"])[0]))
         self.assertEqual(1, len(copied))
 
-    @patch.dict("buildconfig.substs._dict", {"MAKECAB": "makecab"})
     def test_copy_debug_copies_binaries(self):
         """
         Test that CopyDebug copies binaries as well on Windows.
@@ -219,7 +218,7 @@ def mock_compress(args, **kwargs):
         d.Process(test_file)
         self.assertTrue(
             os.path.isfile(
-                os.path.join(self.symbol_dir, code_file, code_id, code_file[:-1] + "_")
+                os.path.join(self.symbol_dir, code_file, code_id, code_file)
             )
         )
 

toolkit/mozapps/installer/packager.mk

@@ -60,9 +60,9 @@ ifdef MOZ_ARTIFACT_BUILD_SYMBOLS
 	cd $(DIST)/crashreporter-symbols && \
           zip -r5D '../$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip' . -i '*.sym' -i '*.txt'
 ifeq ($(MOZ_ARTIFACT_BUILD_SYMBOLS),full)
-	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
-	cd $(DIST)/crashreporter-symbols && \
-          zip -r5D '../$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' .
+	$(call py_action,symbols_archive,'$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst' \
+                                     $(abspath $(DIST)/crashreporter-symbols) \
+                                     --full-archive)
 endif
 endif # MOZ_ARTIFACT_BUILD_SYMBOLS
 endif # MOZ_AUTOMATION

toolkit/mozapps/installer/upload-files.mk

@@ -380,7 +380,7 @@ UPLOAD_FILES= \
   $(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \
   $(call QUOTED_WILDCARD,$(PKG_JSSHELL)) \
-  $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip) \
+  $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst) \
   $(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup.exe) \
   $(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup-stub.exe) \
   $(call QUOTED_WILDCARD,$(topsrcdir)/toolchains.json) \