From a96386381576d8da64b140f1833d161d9aa9cce6 Mon Sep 17 00:00:00 2001 From: Mikhail Bautin Date: Fri, 15 Jun 2018 15:26:51 -0700 Subject: [PATCH] Reduce verbosity of PostgreSQL build Summary: - Reduce verbosity of PostgreSQL build. Setting YB_VERBOSE=1 will enable the verbose level of output. - Disable readline in ASAN/TSAN mode. - Use `-fdebug-prefix-map` to make debugging PostgreSQL code easier. Debug information should point to original source files, not to files we've copied to the build directory. - Add a `--skip-postgres` option to `yb_build.sh` to save 9 seconds on running configure/make/make install on a fully built PostgreSQL codebase. Test Plan: Jenkins Reviewers: hector, sergei Reviewed By: sergei Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D4975 --- python/yb/build_postgres.py | 83 +++++++++++++++++++++++++++---------- requirements.txt | 2 +- yb_build.sh | 5 +++ 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/python/yb/build_postgres.py b/python/yb/build_postgres.py index 04d2c459cce6..34b82cd1224a 100755 --- a/python/yb/build_postgres.py +++ b/python/yb/build_postgres.py @@ -13,10 +13,11 @@ from subprocess import check_call -from yugabyte_pycommon import init_logging, run_program, WorkDirContext, mkdir_p, quote_for_bash +from yugabyte_pycommon import init_logging, run_program, WorkDirContext, mkdir_p, quote_for_bash, \ + is_verbose_mode from yb import common_util -from yb.common_util import YB_SRC_ROOT, get_build_type_from_build_root +from yb.common_util import YB_SRC_ROOT, get_build_type_from_build_root, get_bool_env_var REMOVE_CONFIG_CACHE_MSG_RE = re.compile(r'error: run.*\brm config[.]cache\b.*and start over') @@ -36,6 +37,14 @@ def filter_compiler_flags(compiler_flags, allow_error_on_warning): flag for flag in compiler_flags.split() if not is_error_on_warning_flag(flag)]) +def get_path_variants(path): + """ + Returns a list of different variants (just an absolute path vs. all symlinks resolved) for the + given path. + """ + return sorted(set([os.path.abspath(path), os.path.realpath(path)])) + + class PostgresBuilder: def __init__(self): self.args = None @@ -99,18 +108,29 @@ def set_env_vars(self, step): else: del os.environ[var_name] - additional_c_cxx_flags = ( - ' -Wno-error=missing-prototypes' + - ' -Wno-error=unused-function' + - ' -DHAVE__BUILTIN_CONSTANT_P=1' + - ' -DUSE_SSE42_CRC32C=1' - ) + additional_c_cxx_flags = [ + '-Wno-error=missing-prototypes', + '-Wno-error=unused-function', + '-DHAVE__BUILTIN_CONSTANT_P=1', + '-DUSE_SSE42_CRC32C=1', + ] + + # Tell gdb to pretend that we're compiling the code in the $YB_SRC_ROOT/src/postgres + # directory. + build_path_variants = get_path_variants(self.pg_build_root) + src_path_variants = get_path_variants(self.postgres_src_dir) + additional_c_cxx_flags += [ + '-fdebug-prefix-map=%s=%s' % (build_path, source_path) + for build_path in build_path_variants + for source_path in src_path_variants + ] + if self.compiler_type == 'gcc': - additional_c_cxx_flags += ' -Wno-error=maybe-uninitialized' + additional_c_cxx_flags.append('-Wno-error=maybe-uninitialized') for var_name in ['CFLAGS', 'CXXFLAGS']: os.environ[var_name] = filter_compiler_flags( - os.environ.get(var_name, '') + additional_c_cxx_flags, + os.environ.get(var_name, '') + ' ' + ' '.join(additional_c_cxx_flags), allow_error_on_warning=(step == 'make')) for env_var_name in ['MAKEFLAGS']: @@ -130,10 +150,11 @@ def set_env_vars(self, step): for ldflags_var_name in ['LDFLAGS', 'LDFLAGS_EX']: os.environ[ldflags_var_name] += ' -lm' - # CPPFLAGS are C preprocessor flags, CXXFLAGS are C++ flags. - for env_var_name in ['CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'LDFLAGS', 'LDFLAGS_EX', 'LIBS']: - if env_var_name in os.environ: - logging.info("%s: %s", env_var_name, os.environ[env_var_name]) + if is_verbose_mode(): + # CPPFLAGS are C preprocessor flags, CXXFLAGS are C++ flags. + for env_var_name in ['CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'LDFLAGS', 'LDFLAGS_EX', 'LIBS']: + if env_var_name in os.environ: + logging.info("%s: %s", env_var_name, os.environ[env_var_name]) # PostgreSQL builds pretty fast, and we don't want to use our remote compilation over SSH # for it as it might have issues with parallelism. @@ -165,7 +186,8 @@ def sync_postgres_source(self): '--exclude', '*.rej', self.postgres_src_dir + '/', self.pg_build_root], capture_output=False) - logging.info("Successfully synced postgres source code") + if is_verbose_mode(): + logging.info("Successfully synced postgres source code") def clean_postgres(self): logging.info("Removing the postgres build and installation directories") @@ -175,7 +197,8 @@ def clean_postgres(self): run_program(['rm', '-rf', dir_to_delete], capture_output=False) def configure_postgres(self): - logging.info("Running configure in the postgres build directory") + if is_verbose_mode(): + logging.info("Running configure in the postgres build directory") # Don't enable -Werror when running configure -- that can affect the resulting # configuration. self.set_env_vars('configure') @@ -186,6 +209,11 @@ def configure_postgres(self): '--enable-depend', # We're enabling debug symbols for all types of builds. '--enable-debug'] + + # We get readline-related errors in ASAN/TSAN, so let's disable readline there. + if self.build_type in ['asan', 'tsan']: + configure_cmd_line += ['--without-readline'] + if self.build_type != 'release': configure_cmd_line += ['--enable-cassert'] configure_result = run_program(configure_cmd_line, error_ok=True) @@ -194,7 +222,7 @@ def configure_postgres(self): for line in configure_result.stderr.splitlines(): if REMOVE_CONFIG_CACHE_MSG_RE.search(line.strip()): logging.info("Configure failed because of stale config.cache, re-running.") - run_program('rm -f config.cache', capture_output=True) + run_program('rm -f config.cache') rerun_configure = True break @@ -204,7 +232,15 @@ def configure_postgres(self): configure_result = run_program(configure_cmd_line) - configure_result.print_output_to_stdout() + if is_verbose_mode(): + configure_result.print_output_to_stdout() + for output_type in ['out', 'err']: + output_path = os.path.join(self.pg_build_root, 'configure.' + output_type) + output_content = getattr(configure_result, 'std' + output_type) + with open(output_path, 'w') as out_f: + out_f.write(output_content) + if output_content.strip(): + logging.info("Wrote std%s of configure to %s", output_type, output_path) logging.info("Successfully ran configure in the postgres build directory") @@ -236,12 +272,17 @@ def make_postgres(self): run_program(['chmod', 'u+x', make_script_path]) # Actually run Make. - logging.info("Running make in the %s directory", work_dir) - run_program(make_cmd, capture_output=False) - run_program(['make', 'install'], capture_output=False) + if is_verbose_mode(): + logging.info("Running make in the %s directory", work_dir) + run_program(make_cmd) + run_program(['make', 'install']) logging.info("Successfully ran make in the %s directory", work_dir) def run(self): + if get_bool_env_var('YB_SKIP_POSTGRES_BUILD'): + logging.info("Skipping PostgreSQL build (YB_SKIP_POSTGRES_BUILD is set)") + return + self.parse_args() self.build_postgres() diff --git a/requirements.txt b/requirements.txt index dc74dc6ab2c9..d9d5bc560924 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ PyYAML==3.12 six==1.11.0 -yugabyte_pycommon==1.7.4 +yugabyte_pycommon==1.8.0 diff --git a/yb_build.sh b/yb_build.sh index f20bacf759fa..ba40f6bfb0fb 100755 --- a/yb_build.sh +++ b/yb_build.sh @@ -152,6 +152,8 @@ Options: generated. Only works in non-release mode. --clean-postgres Do a clean build of the PostgreSQL subtree. + --no-postgres, --skip-postgres, --np, --sp + Skip PostgreSQL build -- Pass all arguments after -- to repeat_unit_test. Build types: @@ -830,6 +832,9 @@ while [[ $# -gt 0 ]]; do --clean-postgres) clean_postgres=true ;; + --no-postgres|--skip-postgres|--np|--sp) + export YB_SKIP_POSTGRES_BUILD=1 + ;; *) echo "Invalid option: '$1'" >&2 exit 1