export_tarball.py

#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
This tool creates a tarball with all the sources, but without .svn directories.

It can also remove files which are not strictly required for build, so that
the resulting tarball can be reasonably small (last time it was ~110 MB).

Example usage:

export_tarball.py /foo/bar

The above will create file /foo/bar.tar.bz2.
"""

import optparse
import os
import subprocess
import sys
import tarfile


NONESSENTIAL_DIRS = (
    'breakpad/src/processor/testdata',
    'chrome/browser/resources/tracing/tests',
    'chrome/common/extensions/docs',
    'chrome/tools/test/reference_build',
    'courgette/testdata',
    'data',
    'native_client/src/trusted/service_runtime/testdata',
    'src/chrome/test/data',
    'o3d/documentation',
    'o3d/samples',
    'o3d/tests',
    'ppapi/examples',
    'ppapi/native_client/tests',
    'third_party/angle/samples/gles2_book',
    'third_party/findbugs',
    'third_party/hunspell_dictionaries',
    'third_party/hunspell/tests',
    'third_party/lighttpd',
    'third_party/sqlite/src/test',
    'third_party/sqlite/test',
    'third_party/vc_80',
    'third_party/xdg-utils/tests',
    'third_party/yasm/source/patched-yasm/modules/arch/x86/tests',
    'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests',
    'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests',
    'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests',
    'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests',
    'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests',
    'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests',
    'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests',
    'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests',
    'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests',
    'third_party/WebKit/LayoutTests',
    'third_party/WebKit/Source/JavaScriptCore/tests',
    'third_party/WebKit/Source/WebCore/ChangeLog',
    'third_party/WebKit/Source/WebKit2',
    'third_party/WebKit/Tools/Scripts',
    'tools/gyp/test',
    'v8/test',
    'webkit/data/layout_tests',
    'webkit/tools/test/reference_build',
)

TESTDIRS = (
    'chrome/test/data',
    'content/test/data',
    'media/test/data',
    'net/data',
)


def GetSourceDirectory():
  return os.path.realpath(
    os.path.join(os.path.dirname(__file__), '..', '..', '..', 'src'))


# Workaround lack of the exclude parameter in add method in python-2.4.
# TODO(phajdan.jr): remove the workaround when it's not needed on the bot.
class MyTarFile(tarfile.TarFile):
  def set_remove_nonessential_files(self, remove):
    self.__remove_nonessential_files = remove

  def set_verbose(self, verbose):
    self.__verbose = verbose

  def __report_skipped(self, name):
    if self.__verbose:
      print 'D\t%s' % name

  def __report_added(self, name):
    if self.__verbose:
      print 'A\t%s' % name

  def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
    head, tail = os.path.split(name)
    if tail in ('.svn', '.git'):
      self.__report_skipped(name)
      return

    if self.__remove_nonessential_files:
      # WebKit change logs take quite a lot of space. This saves ~10 MB
      # in a bzip2-compressed tarball.
      if 'ChangeLog' in name:
        self.__report_skipped(name)
        return

      # Remove contents of non-essential directories, but preserve gyp files,
      # so that build/gyp_chromium can work.
      for nonessential_dir in (NONESSENTIAL_DIRS + TESTDIRS):
        dir_path = os.path.join(GetSourceDirectory(), nonessential_dir)
        if (name.startswith(dir_path) and
            os.path.isfile(name) and
            'gyp' not in name):
          self.__report_skipped(name)
          return

    self.__report_added(name)
    tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive)


def main(argv):
  parser = optparse.OptionParser()
  parser.add_option("--basename")
  parser.add_option("--remove-nonessential-files",
                    dest="remove_nonessential_files",
                    action="store_true", default=False)
  parser.add_option("--test-data", action="store_true")
  # TODO(phajdan.jr): Remove --xz option when it's not needed for compatibility.
  parser.add_option("--xz", action="store_true")
  parser.add_option("--verbose", action="store_true", default=False)
  parser.add_option("--progress", action="store_true", default=False)

  options, args = parser.parse_args(argv)

  if len(args) != 1:
    print 'You must provide only one argument: output file name'
    print '(without .tar.xz extension).'
    return 1

  if not os.path.exists(GetSourceDirectory()):
    print 'Cannot find the src directory ' + GetSourceDirectory()
    return 1

  # These two commands are from src/DEPS; please keep them in sync.
  if subprocess.call(['python', 'build/util/lastchange.py', '-o',
                      'build/util/LASTCHANGE'], cwd=GetSourceDirectory()) != 0:
    print 'Could not run build/util/lastchange.py to update LASTCHANGE.'
    return 1
  if subprocess.call(['python', 'build/util/lastchange.py', '-s',
                      'third_party/WebKit', '-o',
                      'build/util/LASTCHANGE.blink'],
                     cwd=GetSourceDirectory()) != 0:
    print 'Could not run build/util/lastchange.py to update LASTCHANGE.blink.'
    return 1

  output_fullname = args[0] + '.tar'
  output_basename = options.basename or os.path.basename(args[0])

  archive = MyTarFile.open(output_fullname, 'w')
  archive.set_remove_nonessential_files(options.remove_nonessential_files)
  archive.set_verbose(options.verbose)
  try:
    if options.test_data:
      for directory in TESTDIRS:
        archive.add(os.path.join(GetSourceDirectory(), directory),
                    arcname=os.path.join(output_basename, directory))
    else:
      archive.add(GetSourceDirectory(), arcname=output_basename)
  finally:
    archive.close()

  if options.progress:
    sys.stdout.flush()
    pv = subprocess.Popen(
        ['pv', '--force', output_fullname],
        stdout=subprocess.PIPE,
        stderr=sys.stdout)
    with open(output_fullname + '.xz', 'w') as f:
      rc = subprocess.call(['xz', '-9', '-'], stdin=pv.stdout, stdout=f)
    pv.wait()
  else:
    rc = subprocess.call(['xz', '-9', output_fullname])

  if rc != 0:
    print 'xz -9 failed!'
    return 1

  return 0


if __name__ == "__main__":
  sys.exit(main(sys.argv[1:]))