Skip to content

Commit

Permalink
Simplify the setup required for unified compilation
Browse files Browse the repository at this point in the history
  • Loading branch information
@sethfowler
sethfowler committed Jun 6, 2017
1 parent 1e4f00f commit bebe48d
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 156 deletions.
1 change: 0 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ tags
extensions/*
otherMakefiles.am
unified-compilation.am
regenerate-unified-compilation.am
*.d
*.output
*.json.p4
Expand Down
2 changes: 0 additions & 2 deletions Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@ AM_CPPFLAGS += -DCONFIG_PKGDATADIR=\"$(pkgdatadir)\"

TOOLSDIR=$(srcdir)/tools
GENTESTS=$(TOOLSDIR)/gen-tests.py
GEN_UNIFIED_MAKEFILE=$(TOOLSDIR)/gen-unified-makefile.py
GEN_UNIFIED_CPP=$(TOOLSDIR)/gen-unified-sources.py

################ Utility functions

Expand Down
8 changes: 2 additions & 6 deletions bootstrap.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,8 @@
set -e # exit on error
./find-makefiles.sh # creates otherMakefiles.am, included in Makefile.am

# Generate the unified compilation makefile, which is included in Makefile.am.
# This needs to be done before automake runs. See the source code of the tool
# for more discussion.
tools/gen-unified-makefile.py --max-chunk-size 10 \
--regenerate-with regenerate-unified-compilation.am \
-o unified-compilation.am
# Generates unified compilation rules; needs to be done before automake runs.
./tools/autounify --max-chunk-size 10

mkdir -p extensions # place where additional back-ends are expected
echo "Running autoconf/configure tools"
Expand Down
199 changes: 114 additions & 85 deletions tools/gen-unified-makefile.py → tools/autounify
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,20 +53,8 @@
# effect.
#
# Basic usage of this script is to run it from the root of the source tree
# before automake is run, passing '-o unified-compilation.am' to specify the
# output file. The output file then needs to be included from your Makefile.am.
# The script requires that Makefile.am also set a GEN_UNIFIED_CPP
# variable which points to the script that will actually create the generated
# '.cpp' files at build time. (That's `gen-unified-sources.py`.) Doing it this way
# allows this tool to run before the build directory has been created, and
# avoids generating '.cpp' files that aren't needed by the targets we're
# building.
#
# If you want to support automatically regenerating the unified compilation
# information when one of the input automake files changes, pass
# '--regenerate-with regenerate-unified-compilation.am' when you run this script
# and include the resulting file in your Makefile.am. You'll also need to set a
# GEN_UNIFIED_MAKEFILE variable which points to this script.
# before automake is run, and include the output file (by default
# 'unified-compilation.am') from your Makefile.am.
#
# Information about other command line arguments is available by passing
# '--help' to the script.
Expand All @@ -81,13 +69,24 @@
import stat
import sys

def find_automake_files(excluded_files):
def find_automake_files(included_paths, excluded_paths):
included_files = [p for p in included_paths if os.path.isfile(p)]
included_dirs = [p for p in included_paths if os.path.isdir(p)]
excluded_files = [p for p in excluded_paths if os.path.isfile(p)]
excluded_dirs = [p for p in excluded_paths if os.path.isdir(p)]

files = []
for path_prefix, _, filenames in os.walk('.', followlinks=True):
abs_path_prefix = os.path.abspath(path_prefix)
if any([abs_path_prefix.startswith(p) for p in excluded_dirs]):
continue
for filename in fnmatch.filter(filenames, '*.am'):
path = os.path.join(path_prefix, filename)
relative_path = os.path.relpath(path, '.')
if relative_path not in excluded_files:
abs_path = os.path.abspath(path)
if (not any([abs_path_prefix.startswith(p) for p in included_dirs]) and
abs_path not in included_files):
continue
if abs_path not in excluded_files:
files.append(path)

return files
Expand Down Expand Up @@ -132,10 +131,10 @@ def compute_file_sets(automake_files):
# supported in filenames, so don't get too crazy. =) 'foo_SOURCES' is also
# detected for error reporting; see below.
unified_re = re.compile(
r"^(?P<target>\w+?)" # The target name ('foo').
r"(?P<type>_UNIFIED|_NONUNIFIED|_SOURCES)" # The type of file set.
r" *(?:\+|:|::|\?)?\=" # Some sort of assignment or append.
r"(?P<files>(( |\t|\\\n)*[^ \t\\\n]+)*)$", # A possibly-multiline sequence of files.
r"^(?P<target>\w+?)" # The target name ('foo').
r"(?P<type>_UNIFIED|_NONUNIFIED|_SOURCES)" # The type of file set.
r" *(?:\+|:|::|\?)?\=" # Some sort of assignment or append.
r"(?P<files>(( |\t|\\\n)*[^ \t\\\n]+)*) *$", # A possibly-multiline sequence of files.
re.MULTILINE)

# This regular expression is used for splitting the sequence of files matched by
Expand Down Expand Up @@ -188,17 +187,41 @@ def compute_file_sets(automake_files):
raise Exception('The same files are present in both {}_UNIFIED and '
'{}_NONUNIFIED: {}'.format(target, duplicated_files))

return (unified_targets, unified_file_sets, nonunified_file_sets)
return (unified_targets, unified_file_sets, nonunified_file_sets,
'BUILT' in targets_with_sources_vars)

def generate_preamble(output, script_name, max_chunk_size):
def generate_preamble(output, script_name, max_chunk_size, have_BUILT_SOURCES):
print('###################################################################', file=output)
print('# Unified compilation automake file.', file=output)
print('# Unified compilation rules.', file=output)
print('# Generated by:', script_name, file=output)
print('# Maximum chunk size:', max_chunk_size, file=output)
print('# DO NOT EDIT. Changes will be overwritten.', file=output)
print('###################################################################', file=output)
print(file=output)

# Helper to generate a unified .cpp file. The output file is updated only if
# its contents would actually change.
print('define _unified__gen_cpp', file=output)
print('\tOUT="$$(mktemp -t unified__gen_cpp.tmp.XXXXXXXX)"; \\', file=output)
print('\techo "// Generated. DO NOT EDIT. Changes will be overwritten." > "$$OUT" ; \\', file=output)
print("\tfor item in $(2); do \\", file=output)
print('\t case "$$item" in \\', file=output)
print('\t *.c | *.cpp) echo "#include \\"$$item\\"" >> "$$OUT" ;; \\', file=output)
print('\t *.h | *.hpp) echo "// WARNING: Skipping header file: $$item" >> "$$OUT" ;; \\', file=output)
print('\t *) echo "// WARNING: Skipping non-C/C++ source file: $$item" >> "$$OUT" ;; \\', file=output)
print('\t esac; \\', file=output)
print('\tdone; \\', file=output)
print('\tif ! cmp -s "$(1)" "$$OUT"; then cp -f "$$OUT" "$(1)"; fi; \\', file=output)
print('\trm -f "$$OUT"', file=output)
print('endef', file=output)
print(file=output)

# If the input files don't define BUILT_SOURCES, define it, since we
# generate rules that append to it.
if not have_BUILT_SOURCES:
print('BUILT_SOURCES = ', file=output)
print(file=output)

def chunk(max_chunk_size, file_set):
""" Split `file_set` into chunks with at most `max_chunk_size` elements. """
if len(file_set) == 0:
Expand All @@ -219,65 +242,67 @@ def generate_rules(output, target, unified_file_chunks, nonunified_file_set):

# For each chunk of unified files, we generate a rule that creates a unified
# .cpp file which #include's all those files, and then add the unified .cpp
# file to 'foo_SOURCES'. The generation of the unified .cpp file is actually
# performed at build time using a tool specified in Makefile.am by setting
# GEN_UNIFIED_CPP. This allows this script to be run before the build
# directory is created.
# file to 'foo_SOURCES'.
for index in xrange(0, len(unified_file_chunks)):
files_in_chunk = ' '.join(unified_file_chunks[index])
chunk_target = 'unified-sources-{}-{}.cpp'.format(target, index)
tmp_file = '.{}.tmp'.format(chunk_target)
stamp_file = '.{}.stamp'.format(chunk_target)

# If any file in the chunk has changed, a recompile is required.
# Regenerate the unified .cpp file unconditionally.
print('{}: {}'.format(chunk_target, files_in_chunk), file=output)
print('\t@$(GEN_UNIFIED_CPP) {} > $@'.format(files_in_chunk), file=output)
# If the unified .cpp doesn't already exist, we need to generate it.
# This is a one time thing; further updates use the rule below.
print('{}:'.format(chunk_target), file=output)
print('\t@if test ! -e $@; then $(call _unified__gen_cpp,$@,{}); fi'
.format(files_in_chunk), file=output)
print(file=output)

# If the automake rules we're generating have changed, a recompile may
# be required. We regenerate the unified .cpp file, but we don't replace
# the previous version (and trigger a recompile) unless something
# actually changed. Regardless, we update the stamp file for this chunk
# to indicate that we've performed this check.
print('{}: $(GEN_UNIFIED_CPP) {}'.format(stamp_file, output.name), file=output)
print('{}: {}'.format(stamp_file, output.name), file=output)
print('\t@$(call _unified__gen_cpp,{},{})'
.format(chunk_target, files_in_chunk), file=output)
print('\t@touch $@', file=output)
print('\t@$(GEN_UNIFIED_CPP) {} > {}'.format(files_in_chunk, tmp_file), file=output)
print('\t@if ! cmp -s {0} {1}; then cp -f {0} {1}; fi'
.format(tmp_file, chunk_target), file=output)
print('\t@rm -f {}'.format(tmp_file), file=output)
print(file=output)

print('{}_SOURCES += {}'.format(target, chunk_target), file=output)
print('BUILT_SOURCES += {}'.format(stamp_file), file=output)
print(file=output)

def generate_regen_file(regen, script_name, args, automake_files):
# Generate an automake file which will rerun this script with the same
# arguments when the input automake files change. We can't put these rules
# in the main output automake file because it would create a circular
# dependency between the rules in the file and the file itself; make
# identifies and ignores such dependencies.
print('###################################################################', file=regen)
print('# Utility makefile to regenerate unified compilation information.', file=regen)
print('# Generated by: {}'.format(script_name), file=regen)
print('# DO NOT EDIT. Changes will be overwritten.', file=regen)
print('###################################################################', file=regen)
print(file=regen)
def generate_regen_rules(output, script_name, args, automake_files):
# Generate rules which will rerun this script with the same arguments when
# the input automake files change. A '.stamp' file is used as indirection to
# avoid creating a circular dependency between the file we're generating and
# a rule that it contains; Make detects and rejects such rules.
stamp_file = '.unified-compilation.stamp'

# The dependencies need to be $(srcdir) relative. The script should've either
# been run from $(srcdir), or the user should've passed '--root $(srcdir)'
# at the command line, so we can just rewrite the paths relative to '.'.
target = '$(srcdir)/' + os.path.relpath(args.output, '.')
dependencies = ' '.join(['$(srcdir)/' + os.path.relpath(f, '.') for f in automake_files])

print('{}: $(GEN_UNIFIED_MAKEFILE) {}'.format(target, dependencies), file=regen)
print('\t@$(GEN_UNIFIED_MAKEFILE) --max-chunk-size {} --root $(srcdir) --regenerate-with {} -o {}'
.format(args.max_chunk_size, args.regen, args.output), file=regen)
print(file=regen)
included = ' '.join(args.include)
excluded = ' '.join(args.exclude)

print('{}: {} {}'.format(stamp_file, script_name, dependencies), file=output)
print('\t@{} --max-chunk-size {} --root $(srcdir) --include {} --exclude {} -o {}'
.format(script_name, args.max_chunk_size, included, excluded, args.output),
file=output)
print('\t@touch $@', file=output)
print(file=output)

# We want to add an additional dependency to Makefile.in, but otherwise keep
# the rule that automake generates intact. If automake realizes that we
# wrote a rule with Makefile.in as a target, though, it thinks we want to
# override the default rule for Makefile.in and it doesn't generate it. To
# trick it, we prepend _AUTOMAKE_IGNORE to the target; automake doesn't
# understand this and generates the default version of the rule as usual.
print('$(AUTOMAKE_IGNORE)$(srcdir)/Makefile.in: {}'.format(stamp_file), file=output)
print(file=output)

def main(argv):
script_name = os.path.basename(sys.argv[0])
script_name = os.path.abspath(sys.argv[0])

parser = argparse.ArgumentParser(description='Search for automake files and '
'generate a unified compilation '
Expand All @@ -290,57 +315,61 @@ def main(argv):
'faster; decrease to make incremental rebuilds faster. '
'Defaults to 10.')
parser.add_argument('--root', nargs='?', default='.',
type=str, metavar='DIR',
type=str, metavar='ROOTDIR',
help='Search for input automake files in subdirectories '
'of DIR. Automake files in DIR itself are ignored. '
'This should be the same as $(srcdir) in your '
'top-level Makefile.am. The output files are also '
'specified relative to this directory. Defaults to '
'the current directory.')
parser.add_argument('--regenerate-with', nargs='?', dest='regen',
type=str, metavar='REGEN.am',
help='Write rules into REGEN.am to rerun this tool when '
'the input automake files change.')
parser.add_argument('-o', required=True, dest='output',
'of ROOTDIR. This must be the same as $(srcdir) '
'in Makefile.am. The paths in all other arguments '
'are specified relative to this directory. '
'Defaults to the current directory.')
parser.add_argument('--include', nargs='*', type=str,
default=['.'], metavar='PATH',
help='Include only the provided files or directories '
'when searching for input automake files. Defaults '
'to ROOTDIR.')
parser.add_argument('--exclude', nargs='*', type=str,
default=[], metavar='PATH',
help='Ignore the provided files or directories when '
'searching for input automake files. Takes '
'precedence over --include. Paths are specified '
'relative to ROOTDIR.')
parser.add_argument('-o', '--out', dest='output',
default='unified-compilation.am',
type=str, metavar='OUTPUT.am',
help='Write the unified compilation automake file into '
'OUTPUT.am.')
'OUTPUT.am. The path is specified relative to '
'ROOTDIR. Defaults to "unified-compilation.am".')
args = parser.parse_args()

print('Generating unified compilation rules')

# Change to the source tree root and open the output files. The order is
# important since the output files are specified relative to the root.
# Change to the source tree root and open the output file. The order is
# important since the output file are specified relative to the root.
os.chdir(args.root)
output = open(args.output, 'w')
regen = open(args.regen, 'w') if args.regen else None

# Start by locating all automake files in the source tree. We don't need to
# do anything clever about e.g. excluding automake files that we don't
# intend on actually including in the makefile, because we're just
# generating rules at this point. The rules won't actually be invoked unless
# unless there's some target that depends on them. (We do have to exclude
# the files we're generating, though, or else we'll confuse ourselves.)
excluded_files = [args.output, args.regen]
automake_files = find_automake_files(excluded_files)
# Start by locating all automake files in the source tree that haven't been
# excluded by the user. Regardless of the user's preferences, we always need
# to exclude the file we're generating; including it would break the logic
# in this script.
included_paths = [os.path.abspath(p) for p in args.include]
excluded_paths = [os.path.abspath(p) for p in [args.output] + args.exclude]
automake_files = find_automake_files(included_paths, excluded_paths)

# Now we compute the file sets. This is designed to fit right in with
# automake's approach. We look for make variables named 'foo_UNIFIED' (the
# files which should be compiled in unified mode) and 'foo_NONUNIFIED' (the
# files which should be compiled separately); the makefile we generate will
# set 'foo_SOURCES' with appropriate dependencies, and automake/make will do
# the rest.
(unified_targets, unified_file_sets, nonunified_file_sets) = compute_file_sets(automake_files)
(unified_targets, unified_file_sets, nonunified_file_sets, have_BUILT_SOURCES) = \
compute_file_sets(automake_files)

# We now have enough information to generate the automake file.
generate_preamble(output, script_name, args.max_chunk_size)
generate_preamble(output, script_name, args.max_chunk_size, have_BUILT_SOURCES)
generate_regen_rules(output, script_name, args, automake_files)
for target in unified_targets:
unified_file_chunks = chunk(args.max_chunk_size, unified_file_sets[target])
generate_rules(output, target, unified_file_chunks, nonunified_file_sets[target])

# If requested, generate rules to rerun this tool when the inputs change.
if regen is not None:
generate_regen_file(regen, script_name, args, automake_files)

if __name__ == "__main__":
main(sys.argv)
Loading

0 comments on commit bebe48d

Please sign in to comment.