From c4e45fdeb3b4b877459c1bfeaeab197d7566a281 Mon Sep 17 00:00:00 2001 From: Wes Kocher Date: Wed, 16 Aug 2017 17:26:08 -0700 Subject: [PATCH] Backed out 4 changesets (bug 1259832) for apparently scheduling nightlies on every push/platform a=backout Backed out changeset 7781a37a4db0 (bug 1259832) Backed out changeset 73bf88110b38 (bug 1259832) Backed out changeset 14d18d7cf454 (bug 1259832) Backed out changeset a02695cbf576 (bug 1259832) MozReview-Commit-ID: 4Tf6jBcvKLq --- Makefile.in | 4 - browser/config/mozconfigs/linux32/artifact | 3 +- .../config/mozconfigs/linux32/debug-artifact | 3 +- browser/config/mozconfigs/linux64/artifact | 3 +- .../config/mozconfigs/linux64/debug-artifact | 3 +- browser/config/mozconfigs/macosx64/artifact | 3 +- .../config/mozconfigs/macosx64/debug-artifact | 3 +- browser/config/mozconfigs/win32/artifact | 3 +- .../config/mozconfigs/win32/debug-artifact | 3 +- browser/config/mozconfigs/win64/artifact | 3 +- .../config/mozconfigs/win64/debug-artifact | 3 +- build/moz-automation.mk | 3 - build/mozconfig.artifact.automation | 6 - build/mozconfig.automation | 1 - build/upload_generated_sources.py | 156 ------------------ .../action/package_generated_sources.py | 30 ---- python/mozbuild/mozbuild/generated_sources.py | 73 -------- .../ci/upload-generated-sources/kind.yml | 33 ---- taskcluster/docs/kinds.rst | 5 - .../transforms/upload_generated_sources.py | 41 ----- toolkit/crashreporter/tools/symbolstore.py | 32 +--- .../crashreporter/tools/unit-symbolstore.py | 22 --- toolkit/mozapps/installer/package-name.mk | 3 - toolkit/mozapps/installer/upload-files.mk | 1 - 24 files changed, 21 insertions(+), 419 deletions(-) delete mode 100644 build/mozconfig.artifact.automation delete mode 100644 build/upload_generated_sources.py delete mode 100644 python/mozbuild/mozbuild/action/package_generated_sources.py delete mode 100644 python/mozbuild/mozbuild/generated_sources.py delete mode 100644 taskcluster/ci/upload-generated-sources/kind.yml delete mode 100644 taskcluster/taskgraph/transforms/upload_generated_sources.py diff --git a/Makefile.in b/Makefile.in index dd0d5ae5330f4..0372837e6a9dc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -305,10 +305,6 @@ endif update-packaging: $(MAKE) -C tools/update-packaging -.PHONY: package-generated-sources -package-generated-sources: - $(call py_action,package_generated_sources,'$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)') - #XXX: this is a hack, since we don't want to clobber for MSVC # PGO support, but we can't do this test in client.mk ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_) diff --git a/browser/config/mozconfigs/linux32/artifact b/browser/config/mozconfigs/linux32/artifact index 5e394ecc1bd85..560d86bb2c2bb 100644 --- a/browser/config/mozconfigs/linux32/artifact +++ b/browser/config/mozconfigs/linux32/artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 . "$topsrcdir/browser/config/mozconfigs/linux32/common-opt" . "$topsrcdir/build/mozconfig.common.override" diff --git a/browser/config/mozconfigs/linux32/debug-artifact b/browser/config/mozconfigs/linux32/debug-artifact index 33c43f22a23c6..52b49cee9a809 100644 --- a/browser/config/mozconfigs/linux32/debug-artifact +++ b/browser/config/mozconfigs/linux32/debug-artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 . "$topsrcdir/build/unix/mozconfig.linux32" . "$topsrcdir/build/mozconfig.common.override" diff --git a/browser/config/mozconfigs/linux64/artifact b/browser/config/mozconfigs/linux64/artifact index 5f2673fb53437..51b82b384b9e1 100644 --- a/browser/config/mozconfigs/linux64/artifact +++ b/browser/config/mozconfigs/linux64/artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 . "$topsrcdir/browser/config/mozconfigs/linux64/common-opt" . "$topsrcdir/build/mozconfig.common.override" diff --git a/browser/config/mozconfigs/linux64/debug-artifact b/browser/config/mozconfigs/linux64/debug-artifact index 5ee6c3db3d6ee..db0a435ff4b2b 100644 --- a/browser/config/mozconfigs/linux64/debug-artifact +++ b/browser/config/mozconfigs/linux64/debug-artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 . "$topsrcdir/build/unix/mozconfig.linux" . "$topsrcdir/build/mozconfig.common.override" diff --git a/browser/config/mozconfigs/macosx64/artifact b/browser/config/mozconfigs/macosx64/artifact index 43a1e31361ba3..93497630ff7af 100644 --- a/browser/config/mozconfigs/macosx64/artifact +++ b/browser/config/mozconfigs/macosx64/artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 # Needed to set SourceRepository in application.ini (used by Talos) export MOZILLA_OFFICIAL=1 diff --git a/browser/config/mozconfigs/macosx64/debug-artifact b/browser/config/mozconfigs/macosx64/debug-artifact index fcf65b6d9afca..a625e99207625 100644 --- a/browser/config/mozconfigs/macosx64/debug-artifact +++ b/browser/config/mozconfigs/macosx64/debug-artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 . "$topsrcdir/build/macosx/mozconfig.common" . "$topsrcdir/build/mozconfig.common.override" diff --git a/browser/config/mozconfigs/win32/artifact b/browser/config/mozconfigs/win32/artifact index 2fc6ad43103fc..d18bedd6e892a 100644 --- a/browser/config/mozconfigs/win32/artifact +++ b/browser/config/mozconfigs/win32/artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 # Needed to set SourceRepository in application.ini (used by Talos) export MOZILLA_OFFICIAL=1 diff --git a/browser/config/mozconfigs/win32/debug-artifact b/browser/config/mozconfigs/win32/debug-artifact index 20cdabd99762e..1d7ab24ba3b1b 100644 --- a/browser/config/mozconfigs/win32/debug-artifact +++ b/browser/config/mozconfigs/win32/debug-artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 . "$topsrcdir/browser/config/mozconfigs/common" . "$topsrcdir/build/mozconfig.win-common" diff --git a/browser/config/mozconfigs/win64/artifact b/browser/config/mozconfigs/win64/artifact index 8b3ada2ec1c8e..9934248448df6 100644 --- a/browser/config/mozconfigs/win64/artifact +++ b/browser/config/mozconfigs/win64/artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 # Needed to set SourceRepository in application.ini (used by Talos) export MOZILLA_OFFICIAL=1 diff --git a/browser/config/mozconfigs/win64/debug-artifact b/browser/config/mozconfigs/win64/debug-artifact index d2bda199d30ef..bb9dc9fa8a992 100644 --- a/browser/config/mozconfigs/win64/debug-artifact +++ b/browser/config/mozconfigs/win64/debug-artifact @@ -1,4 +1,5 @@ -. "$topsrcdir/build/mozconfig.artifact.automation" +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_L10N_CHECK=0 . "$topsrcdir/browser/config/mozconfigs/win64/common-win64" . "$topsrcdir/browser/config/mozconfigs/common" diff --git a/build/moz-automation.mk b/build/moz-automation.mk index 0ba739038d46d..e7665a882ed34 100644 --- a/build/moz-automation.mk +++ b/build/moz-automation.mk @@ -27,7 +27,6 @@ tier_MOZ_AUTOMATION_INSTALLER = installer tier_MOZ_AUTOMATION_PACKAGE = package tier_MOZ_AUTOMATION_PACKAGE_TESTS = package-tests tier_MOZ_AUTOMATION_UPDATE_PACKAGING = update-packaging -tier_MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES = package-generated-sources tier_MOZ_AUTOMATION_UPLOAD_SYMBOLS = uploadsymbols tier_MOZ_AUTOMATION_UPLOAD = upload @@ -42,7 +41,6 @@ moz_automation_symbols = \ MOZ_AUTOMATION_PACKAGE \ MOZ_AUTOMATION_INSTALLER \ MOZ_AUTOMATION_UPDATE_PACKAGING \ - MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES \ MOZ_AUTOMATION_L10N_CHECK \ MOZ_AUTOMATION_UPLOAD \ $(NULL) @@ -62,7 +60,6 @@ automation/upload: automation/package automation/upload: automation/package-tests automation/upload: automation/buildsymbols automation/upload: automation/update-packaging -automation/upload: automation/package-generated-sources # The installer and packager all run stage-package, and may conflict # with each other. diff --git a/build/mozconfig.artifact.automation b/build/mozconfig.artifact.automation deleted file mode 100644 index 90bb2ab41500e..0000000000000 --- a/build/mozconfig.artifact.automation +++ /dev/null @@ -1,6 +0,0 @@ -# Common options for artifact builds to set automation steps. -# This gets included before mozconfig.automation. - -MOZ_AUTOMATION_BUILD_SYMBOLS=0 -MOZ_AUTOMATION_L10N_CHECK=0 -MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0 diff --git a/build/mozconfig.automation b/build/mozconfig.automation index f85cafcb26573..c959c3fc859de 100644 --- a/build/mozconfig.automation +++ b/build/mozconfig.automation @@ -15,7 +15,6 @@ mk_add_options "export MOZ_AUTOMATION_PACKAGE=${MOZ_AUTOMATION_PACKAGE-1}" mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=${MOZ_AUTOMATION_PACKAGE_TESTS-1}" mk_add_options "export MOZ_AUTOMATION_INSTALLER=${MOZ_AUTOMATION_INSTALLER-0}" mk_add_options "export MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-0}" -mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=${MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES-1}" mk_add_options "export MOZ_AUTOMATION_UPLOAD=${MOZ_AUTOMATION_UPLOAD-1}" mk_add_options "export MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-0}" diff --git a/build/upload_generated_sources.py b/build/upload_generated_sources.py deleted file mode 100644 index b24f509375203..0000000000000 --- a/build/upload_generated_sources.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env/python -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function, unicode_literals - -import argparse -from concurrent.futures import ThreadPoolExecutor -from contextlib import contextmanager -import gzip -import io -import logging -from mozbuild.base import MozbuildObject -from mozbuild.generated_sources import ( - get_filename_with_digest, - get_s3_region_and_bucket, -) -import os -from Queue import Queue -import requests -import sys -import tarfile -from threading import Event, Thread -import time - -# Arbitrary, should probably measure this. -NUM_WORKER_THREADS = 10 -log = logging.getLogger('upload-generated-sources') -log.setLevel(logging.INFO) - - -@contextmanager -def timed(): - ''' - Yield a function that provides the elapsed time in seconds since this - function was called. - ''' - start = time.time() - def elapsed(): - return time.time() - start - yield elapsed - - -def gzip_compress(data): - ''' - Apply gzip compression to `data` and return the result as a `BytesIO`. - ''' - b = io.BytesIO() - with gzip.GzipFile(fileobj=b, mode='w') as f: - f.write(data) - b.flush() - b.seek(0) - return b - - -def upload_worker(queue, event, bucket, session_args): - ''' - Get `(name, contents)` entries from `queue` and upload `contents` - to S3 with gzip compression using `name` as the key, prefixed with - the SHA-512 digest of `contents` as a hex string. If an exception occurs, - set `event`. - ''' - try: - import boto3 - session = boto3.session.Session(**session_args) - s3 = session.client('s3') - while True: - if event.is_set(): - # Some other thread hit an exception. - return - (name, contents) = queue.get() - pathname = get_filename_with_digest(name, contents) - compressed = gzip_compress(contents) - extra_args = { - 'ContentEncoding': 'gzip', - 'ContentType': 'text/plain', - } - log.info('Uploading "{}" ({} bytes)'.format(pathname, len(compressed.getvalue()))) - with timed() as elapsed: - s3.upload_fileobj(compressed, bucket, pathname, ExtraArgs=extra_args) - log.info('Finished uploading "{}" in {:0.3f}s'.format(pathname, elapsed())) - queue.task_done() - except Exception: - log.exception('Thread encountered exception:') - event.set() - - -def do_work(artifact, region, bucket): - session_args = {'region_name': region} - session = requests.Session() - if 'TASK_ID' in os.environ: - level = os.environ.get('MOZ_SCM_LEVEL', '1') - secrets_url = 'http://taskcluster/secrets/v1/secret/project/releng/gecko/build/level-{}/gecko-generated-sources-upload'.format(level) - log.info('Using AWS credentials from the secrets service: "{}"'.format(secrets_url)) - res = session.get(secrets_url) - res.raise_for_status() - secret = res.json() - session_args.update( - aws_access_key_id=secret['secret']['AWS_ACCESS_KEY_ID'], - aws_secret_access_key=secret['secret']['AWS_SECRET_ACCESS_KEY'], - ) - else: - log.info('Trying to use your AWS credentials..') - - - # First, fetch the artifact containing the sources. - log.info('Fetching generated sources artifact: "{}"'.format(artifact)) - with timed() as elapsed: - res = session.get(artifact) - log.info('Fetch HTTP status: {}, {} bytes downloaded in {:0.3f}s'.format(res.status_code, len(res.content), elapsed())) - res.raise_for_status() - # Create a queue and worker threads for uploading. - q = Queue() - event = Event() - log.info('Creating {} worker threads'.format(NUM_WORKER_THREADS)) - for i in range(NUM_WORKER_THREADS): - t = Thread(target=upload_worker, args=(q, event, bucket, session_args)) - t.daemon = True - t.start() - with tarfile.open(fileobj=io.BytesIO(res.content), mode='r|gz') as tar: - # Next, process each file. - for entry in tar: - if event.is_set(): - break - log.info('Queueing "{}"'.format(entry.name)) - q.put((entry.name, tar.extractfile(entry).read())) - # Wait until all uploads are finished. - # We don't use q.join() here because we want to also monitor event. - while q.unfinished_tasks: - if event.wait(0.1): - log.error('Worker thread encountered exception, exiting...') - break - - -def main(argv): - logging.basicConfig(format='%(levelname)s - %(threadName)s - %(message)s') - parser = argparse.ArgumentParser( - description='Upload generated source files in ARTIFACT to BUCKET in S3.') - parser.add_argument('artifact', - help='generated-sources artifact from build task') - args = parser.parse_args(argv) - region, bucket = get_s3_region_and_bucket() - - config = MozbuildObject.from_environment() - config._activate_virtualenv() - config.virtualenv_manager.install_pip_package('boto3==1.4.4') - - with timed() as elapsed: - do_work(region=region, bucket=bucket, artifact=args.artifact) - log.info('Finished in {:.03f}s'.format(elapsed())) - return 0 - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/package_generated_sources.py b/python/mozbuild/mozbuild/action/package_generated_sources.py deleted file mode 100644 index 59b1f889622e2..0000000000000 --- a/python/mozbuild/mozbuild/action/package_generated_sources.py +++ /dev/null @@ -1,30 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function, unicode_literals - -import argparse -import json -import os.path -import sys - -import buildconfig -from mozpack.archive import create_tar_gz_from_files -from mozbuild.generated_sources import get_generated_sources - - -def main(argv): - parser = argparse.ArgumentParser( - description='Produce archive of generated sources') - parser.add_argument('outputfile', help='File to write output to') - args = parser.parse_args(argv) - - - files = dict(get_generated_sources()) - with open(args.outputfile, 'wb') as fh: - create_tar_gz_from_files(fh, files, compresslevel=5) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/generated_sources.py b/python/mozbuild/mozbuild/generated_sources.py deleted file mode 100644 index be3711039f541..0000000000000 --- a/python/mozbuild/mozbuild/generated_sources.py +++ /dev/null @@ -1,73 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import hashlib -import json -import os - -from mozpack.files import FileFinder -import mozpack.path as mozpath - - -def sha512_digest(data): - ''' - Generate the SHA-512 digest of `data` and return it as a hex string. - ''' - return hashlib.sha512(data).hexdigest() - - -def get_filename_with_digest(name, contents): - ''' - Return the filename that will be used to store the generated file - in the S3 bucket, consisting of the SHA-512 digest of `contents` - joined with the relative path `name`. - ''' - digest = sha512_digest(contents) - return mozpath.join(digest, name) - - -def get_generated_sources(): - ''' - Yield tuples of `(objdir-rel-path, file)` for generated source files - in this objdir, where `file` is either an absolute path to the file or - a `mozpack.File` instance. - ''' - import buildconfig - - # First, get the list of generated sources produced by the build backend. - gen_sources = os.path.join(buildconfig.topobjdir, 'generated-sources.json') - with open(gen_sources, 'rb') as f: - data = json.load(f) - for f in data['sources']: - yield f, mozpath.join(buildconfig.topobjdir, f) - # Next, return all the files in $objdir/ipc/ipdl/_ipdlheaders. - base = 'ipc/ipdl/_ipdlheaders' - finder = FileFinder(mozpath.join(buildconfig.topobjdir, base)) - for p, f in finder.find('**/*.h'): - yield mozpath.join(base, p), f - # Next, return any Rust source files that were generated into the Rust - # object directory. - rust_build_kind = 'debug' if buildconfig.substs.get('MOZ_DEBUG_RUST') else 'release' - base = mozpath.join('toolkit/library', - buildconfig.substs['RUST_TARGET'], - rust_build_kind, - 'build') - finder = FileFinder(mozpath.join(buildconfig.topobjdir, base)) - for p, f in finder.find('**/*.rs'): - yield mozpath.join(base, p), f - - -def get_s3_region_and_bucket(): - ''' - Return a tuple of (region, bucket) giving the AWS region and S3 - bucket to which generated sources should be uploaded. - ''' - region = 'us-west-2' - level = os.environ.get('MOZ_SCM_LEVEL', '1') - bucket = { - '1': 'gecko-generated-sources-l1', - '2': 'gecko-generated-sources-l2', - '3': 'gecko-generated-sources', - }[level] - return (region, bucket) diff --git a/taskcluster/ci/upload-generated-sources/kind.yml b/taskcluster/ci/upload-generated-sources/kind.yml deleted file mode 100644 index 77ac1b911d503..0000000000000 --- a/taskcluster/ci/upload-generated-sources/kind.yml +++ /dev/null @@ -1,33 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -loader: taskgraph.loader.single_dep:loader - -transforms: - - taskgraph.transforms.upload_generated_sources:transforms - - taskgraph.transforms.job:transforms - - taskgraph.transforms.task:transforms - -kind-dependencies: - - build - -only-for-attributes: - - nightly - -job-template: - description: Upload generated source files from build - worker-type: aws-provisioner-v1/gecko-t-linux-xlarge - treeherder: - symbol: Ugs - kind: build - worker: - docker-image: {in-tree: "lint"} - max-run-time: 600 - run: - using: run-task - command: > - cd /home/worker/checkouts/gecko && - ./mach python build/upload_generated_sources.py ${ARTIFACT_URL} - scopes: - - secrets:get:project/releng/gecko/build/level-{level}/gecko-generated-sources-upload diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index 40410f98242ae..035db56beb575 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -67,11 +67,6 @@ upload-symbols Upload-symbols tasks run after builds and upload the symbols files generated by build tasks to Socorro for later use in crash analysis. -upload-generated-sources --------------- - -Upload-generated-sources tasks run after builds and upload source files that were generated as part of the build process to an s3 bucket for later use in links from crash reports or when debugging shipped builds. - valgrind -------- diff --git a/taskcluster/taskgraph/transforms/upload_generated_sources.py b/taskcluster/taskgraph/transforms/upload_generated_sources.py deleted file mode 100644 index cdc422ba09f76..0000000000000 --- a/taskcluster/taskgraph/transforms/upload_generated_sources.py +++ /dev/null @@ -1,41 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Transform the upload-generated-files task description template, - taskcluster/ci/upload-generated-sources/kind.yml -into an actual task description. -""" - -from __future__ import absolute_import, print_function, unicode_literals - -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.taskcluster import get_artifact_url - - -transforms = TransformSequence() - - -@transforms.add -def add_task_info(config, jobs): - for job in jobs: - dep_task = job['dependent-task'] - del job['dependent-task'] - - # Add a dependency on the build task. - job['dependencies'] = {'build': dep_task.label} - # Label the job to match the build task it's uploading from. - job['label'] = dep_task.label.replace("build-", "upload-generated-sources-") - # Copy over some bits of treeherder metdata from the build task. - dep_th = dep_task.task['extra']['treeherder'] - plat = '{}/{}'.format(dep_th['machine']['platform'], dep_task.attributes.get('build_type')) - job['treeherder']['platform'] = plat - job['treeherder']['tier'] = dep_th['tier'] - # Add an environment variable pointing at the artifact from the build. - artifact_url = get_artifact_url('', - 'public/build/target.generated-files.tar.gz') - job['worker'].setdefault('env', {})['ARTIFACT_URL'] = { - 'task-reference': artifact_url - } - - yield job diff --git a/toolkit/crashreporter/tools/symbolstore.py b/toolkit/crashreporter/tools/symbolstore.py index ed98d522d4ec5..64be72505f09d 100755 --- a/toolkit/crashreporter/tools/symbolstore.py +++ b/toolkit/crashreporter/tools/symbolstore.py @@ -41,12 +41,6 @@ from optparse import OptionParser -from mozbuild.util import memoize -from mozbuild.generated_sources import ( - get_filename_with_digest, - get_generated_sources, - get_s3_region_and_bucket, -) from mozpack.copier import FileRegistry from mozpack.manifests import ( InstallManifest, @@ -337,15 +331,6 @@ def make_file_mapping(install_manifests): file_mapping[abs_dest] = src.path return file_mapping -@memoize -def get_generated_file_s3_path(filename, rel_path, bucket): - """Given a filename, return a path formatted similarly to - GetVCSFilename but representing a file available in an s3 bucket.""" - with open(filename, 'rb') as f: - path = get_filename_with_digest(rel_path, f.read()) - return 's3:{bucket}:{path}:'.format(bucket=bucket, path=path) - - def GetPlatformSpecificDumper(**kwargs): """This function simply returns a instance of a subclass of Dumper that is appropriate for the current platform.""" @@ -391,8 +376,6 @@ def __init__(self, dump_syms, symbol_path, copy_debug=False, vcsinfo=False, srcsrv=False, - generated_files=None, - s3_bucket=None, file_mapping=None): # popen likes absolute paths, at least on windows self.dump_syms = os.path.abspath(dump_syms) @@ -406,8 +389,6 @@ def __init__(self, dump_syms, symbol_path, self.copy_debug = copy_debug self.vcsinfo = vcsinfo self.srcsrv = srcsrv - self.generated_files = generated_files or {} - self.s3_bucket = s3_bucket self.file_mapping = file_mapping or {} # Add a static mapping for Rust sources. target_os = buildconfig.substs['OS_ARCH'] @@ -515,12 +496,7 @@ def ProcessFileWork(self, file, arch_num, arch, vcs_root, dsymbundle=None): if filename in self.file_mapping: filename = self.file_mapping[filename] if self.vcsinfo: - gen_path = self.generated_files.get(filename) - if gen_path and self.s3_bucket: - filename = get_generated_file_s3_path(filename, gen_path, self.s3_bucket) - rootname = '' - else: - (filename, rootname) = GetVCSFilename(filename, self.srcdirs) + (filename, rootname) = GetVCSFilename(filename, self.srcdirs) # sets vcs_root in case the loop through files were to end on an empty rootname if vcs_root is None: if rootname: @@ -691,7 +667,6 @@ def SourceServerIndexing(self, debug_file, guid, sourceFileStream, vcs_root): os.remove(stream_output_path) return result - class Dumper_Linux(Dumper): objcopy = os.environ['OBJCOPY'] if 'OBJCOPY' in os.environ else 'objcopy' def ShouldProcess(self, file): @@ -865,9 +840,6 @@ def main(): parser.error(str(e)) exit(1) file_mapping = make_file_mapping(manifests) - generated_files = {os.path.join(buildconfig.topobjdir, f): f - for (f, _) in get_generated_sources()} - _, bucket = get_s3_region_and_bucket() dumper = GetPlatformSpecificDumper(dump_syms=args[0], symbol_path=args[1], copy_debug=options.copy_debug, @@ -875,8 +847,6 @@ def main(): srcdirs=options.srcdir, vcsinfo=options.vcsinfo, srcsrv=options.srcsrv, - generated_files=generated_files, - s3_bucket=bucket, file_mapping=file_mapping) dumper.Process(args[2]) diff --git a/toolkit/crashreporter/tools/unit-symbolstore.py b/toolkit/crashreporter/tools/unit-symbolstore.py index 3461456861d34..c8452aefcee2c 100644 --- a/toolkit/crashreporter/tools/unit-symbolstore.py +++ b/toolkit/crashreporter/tools/unit-symbolstore.py @@ -214,28 +214,6 @@ def testVCSFilenameEnv(self): symbolstore.GetVCSFilename(filename, [self.test_dir])[0]) -# SHA-512 of a zero-byte file -EMPTY_SHA512 = 'cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e' - - -class TestGeneratedFilePath(HelperMixin, unittest.TestCase): - def setUp(self): - HelperMixin.setUp(self) - - def tearDown(self): - HelperMixin.tearDown(self) - - def test_generated_file_path(self): - # Make an empty generated file - g = os.path.join(self.test_dir, 'generated') - rel_path = 'a/b/generated' - with open(g, 'wb') as f: - pass - expected = 's3:bucket:{}/{}:'.format(EMPTY_SHA512, - rel_path) - self.assertEqual(expected, symbolstore.get_generated_file_s3_path(g, rel_path, 'bucket')) - - if target_platform() == 'WINNT': class TestFixFilenameCase(HelperMixin, unittest.TestCase): def test_fix_filename_case(self): diff --git a/toolkit/mozapps/installer/package-name.mk b/toolkit/mozapps/installer/package-name.mk index d572c87a7e3ab..90d707f1bca4d 100644 --- a/toolkit/mozapps/installer/package-name.mk +++ b/toolkit/mozapps/installer/package-name.mk @@ -80,9 +80,6 @@ PKG_SRCPACK_PATH = SYMBOL_FULL_ARCHIVE_BASENAME = $(PKG_BASENAME).crashreporter-symbols-full SYMBOL_ARCHIVE_BASENAME = $(PKG_BASENAME).crashreporter-symbols -# Generated file package naming -GENERATED_SOURCE_FILE_PACKAGE = $(PKG_BASENAME).generated-files.tar.gz - # Code coverage package naming CODE_COVERAGE_ARCHIVE_BASENAME = $(PKG_BASENAME).code-coverage-gcno diff --git a/toolkit/mozapps/installer/upload-files.mk b/toolkit/mozapps/installer/upload-files.mk index fd4ce1bd4528b..b66d9b222bc2a 100644 --- a/toolkit/mozapps/installer/upload-files.mk +++ b/toolkit/mozapps/installer/upload-files.mk @@ -399,7 +399,6 @@ UPLOAD_FILES= \ $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(WP_TEST_PACKAGE)) \ $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GTEST_PACKAGE)) \ $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \ - $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)) \ $(call QUOTED_WILDCARD,$(MOZ_SOURCESTAMP_FILE)) \ $(call QUOTED_WILDCARD,$(MOZ_BUILDINFO_FILE)) \ $(call QUOTED_WILDCARD,$(MOZ_BUILDID_INFO_TXT_FILE)) \