diff --git a/build-support/build_postgres b/build-support/build_postgres index 900a16970359..84991bdddff6 100755 --- a/build-support/build_postgres +++ b/build-support/build_postgres @@ -26,5 +26,5 @@ handle_predefined_build_root activate_virtualenv set_sanitizer_runtime_options -export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-} +set_pythonpath "$YB_SRC_ROOT"/python/yb/build_postgres.py "$@" diff --git a/build-support/common-build-env.sh b/build-support/common-build-env.sh index fb861e169182..527c90a4d1b3 100644 --- a/build-support/common-build-env.sh +++ b/build-support/common-build-env.sh @@ -2095,6 +2095,27 @@ VIRTUALENV DEBUGGING export VIRTUAL_ENV } +set_pythonpath_called=false + +set_pythonpath() { + if [[ $set_pythonpath_called == "true" ]]; then + return + fi + set_pythonpath_called=true + + if [[ ! -d ${YB_SRC_ROOT:-} ]]; then + fatal "YB_SRC_ROOT is not set or does not exist; ${YB_SRC_ROOT:-undefined}" + fi + + local new_entry=$YB_SRC_ROOT/python + if [[ -z ${PYTHONPATH:-} ]]; then + PYTHONPATH=$new_entry + else + PYTHONPATH=$new_entry:$PYTHONPATH + fi + export PYTHONPATH +} + log_file_existence() { expect_num_args 1 "$@" local file_name=$1 diff --git a/build-support/dependency_graph b/build-support/dependency_graph index 3278bc29653b..af0eeb1298f9 100755 --- a/build-support/dependency_graph +++ b/build-support/dependency_graph @@ -16,5 +16,5 @@ set -euo pipefail . "${BASH_SOURCE%/*}/common-build-env.sh" activate_virtualenv -export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-} +set_pythonpath "$YB_SRC_ROOT"/python/yb/dependency_graph.py "$@" diff --git a/build-support/fossa_analysis b/build-support/fossa_analysis new file mode 100755 index 000000000000..beb4df0d62c6 --- /dev/null +++ b/build-support/fossa_analysis @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Copyright (c) Yugabyte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. + +set -euo pipefail +. "${BASH_SOURCE%/*}/common-build-env.sh" + +activate_virtualenv +set_pythonpath + +if [[ -n ${BUILD_ROOT:-} ]]; then + handle_predefined_build_root_quietly=true + predefined_build_root=$BUILD_ROOT + handle_predefined_build_root +fi +find_or_download_thirdparty + +python3 "$YB_SRC_ROOT"/python/yb/fossa_analysis.py "$@" diff --git a/build-support/gen_initial_sys_catalog_snapshot_wrapper b/build-support/gen_initial_sys_catalog_snapshot_wrapper index a79b6d71bbe7..2b9678e69d96 100755 --- a/build-support/gen_initial_sys_catalog_snapshot_wrapper +++ b/build-support/gen_initial_sys_catalog_snapshot_wrapper @@ -16,5 +16,5 @@ set -euo pipefail . "${BASH_SOURCE%/*}/common-build-env.sh" activate_virtualenv -export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-} +set_pythonpath "$YB_SRC_ROOT"/python/yb/gen_initial_sys_catalog_snapshot.py "$@" diff --git a/build-support/run_pvs_studio_analyzer b/build-support/run_pvs_studio_analyzer index e2dea7f0e29a..954ddae2b485 100755 --- a/build-support/run_pvs_studio_analyzer +++ b/build-support/run_pvs_studio_analyzer @@ -16,5 +16,5 @@ set -euo pipefail . "${BASH_SOURCE%/*}/common-build-env.sh" activate_virtualenv -export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-} -"$YB_SRC_ROOT"/python/yb/run_pvs_studio_analyzer.py "$@" +set_pythonpath +python3 "$YB_SRC_ROOT"/python/yb/run_pvs_studio_analyzer.py "$@" diff --git a/build-support/thirdparty_url_centos.txt b/build-support/thirdparty_url_centos.txt index dd203bf1a447..e12a1b682c94 100644 --- a/build-support/thirdparty_url_centos.txt +++ b/build-support/thirdparty_url_centos.txt @@ -1 +1 @@ -https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210192532-45c97f45f1-centos7-linuxbrew/yugabyte-db-thirdparty-v20210210192532-45c97f45f1-centos7-linuxbrew.tar.gz \ No newline at end of file +https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402083441-ee4e2e453b-centos7-linuxbrew/yugabyte-db-thirdparty-v20210402083441-ee4e2e453b-centos7-linuxbrew.tar.gz \ No newline at end of file diff --git a/build-support/thirdparty_url_centos_clang11.txt b/build-support/thirdparty_url_centos_clang11.txt index 9c18e2dd697b..fc603d364a7c 100644 --- a/build-support/thirdparty_url_centos_clang11.txt +++ b/build-support/thirdparty_url_centos_clang11.txt @@ -1 +1 @@ -https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210306075844-471ef8e125-centos7-clang11/yugabyte-db-thirdparty-v20210306075844-471ef8e125-centos7-clang11.tar.gz \ No newline at end of file +https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402093407-ee4e2e453b-centos7-clang11/yugabyte-db-thirdparty-v20210402093407-ee4e2e453b-centos7-clang11.tar.gz \ No newline at end of file diff --git a/build-support/thirdparty_url_centos_gcc8.txt b/build-support/thirdparty_url_centos_gcc8.txt index 98517d38cc37..7166e56d8496 100644 --- a/build-support/thirdparty_url_centos_gcc8.txt +++ b/build-support/thirdparty_url_centos_gcc8.txt @@ -1 +1 @@ -https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210222846-45c97f45f1-centos7-devtoolset-8/yugabyte-db-thirdparty-v20210210222846-45c97f45f1-centos7-devtoolset-8.tar.gz \ No newline at end of file +https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402122231-ee4e2e453b-centos7-devtoolset-8/yugabyte-db-thirdparty-v20210402122231-ee4e2e453b-centos7-devtoolset-8.tar.gz \ No newline at end of file diff --git a/build-support/thirdparty_url_centos_gcc9.txt b/build-support/thirdparty_url_centos_gcc9.txt index 383ec71455ff..b7f028d076c9 100644 --- a/build-support/thirdparty_url_centos_gcc9.txt +++ b/build-support/thirdparty_url_centos_gcc9.txt @@ -1 +1 @@ -https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210211756-45c97f45f1-centos7-devtoolset-9/yugabyte-db-thirdparty-v20210210211756-45c97f45f1-centos7-devtoolset-9.tar.gz \ No newline at end of file +https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402130708-ee4e2e453b-centos7-devtoolset-9/yugabyte-db-thirdparty-v20210402130708-ee4e2e453b-centos7-devtoolset-9.tar.gz \ No newline at end of file diff --git a/build-support/thirdparty_url_mac.txt b/build-support/thirdparty_url_mac.txt index 0cdb8a42be7e..35457a491d16 100644 --- a/build-support/thirdparty_url_mac.txt +++ b/build-support/thirdparty_url_mac.txt @@ -1 +1 @@ -https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210192517-45c97f45f1-macos/yugabyte-db-thirdparty-v20210210192517-45c97f45f1-macos.tar.gz \ No newline at end of file +https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402050931-ee4e2e453b-macos/yugabyte-db-thirdparty-v20210402050931-ee4e2e453b-macos.tar.gz \ No newline at end of file diff --git a/build-support/thirdparty_url_ubuntu.txt b/build-support/thirdparty_url_ubuntu.txt index 462c961e8e4f..3198ee7f2448 100644 --- a/build-support/thirdparty_url_ubuntu.txt +++ b/build-support/thirdparty_url_ubuntu.txt @@ -1 +1 @@ -https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210212603-45c97f45f1-ubuntu1804-gcc7/yugabyte-db-thirdparty-v20210210212603-45c97f45f1-ubuntu1804-gcc7.tar.gz \ No newline at end of file +https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402084227-ee4e2e453b-ubuntu1804-gcc7/yugabyte-db-thirdparty-v20210402084227-ee4e2e453b-ubuntu1804-gcc7.tar.gz \ No newline at end of file diff --git a/python/yb/common_util.py b/python/yb/common_util.py index 41df8d0a94ea..f897b64fe481 100644 --- a/python/yb/common_util.py +++ b/python/yb/common_util.py @@ -13,6 +13,8 @@ import subprocess import shlex +from typing import Any + MODULE_DIR = os.path.dirname(os.path.realpath(__file__)) YB_SRC_ROOT = os.path.realpath(os.path.join(MODULE_DIR, '..', '..')) @@ -24,6 +26,9 @@ _YB_THIRDPARTY_DIR = os.path.realpath( os.environ.get("YB_THIRDPARTY_DIR", os.path.join(YB_SRC_ROOT, 'thirdparty'))) +GLOBAL_DOWNLOAD_CACHE_DIR = '/opt/yb-build/download_cache' +attempted_to_create_download_cache_dir = False + def get_thirdparty_dir(): global _YB_THIRDPARTY_DIR @@ -284,3 +289,27 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for env_var_name, saved_value in self.saved_env_vars.items(): dict_set_or_del(os.environ, env_var_name, saved_value) + + +def get_download_cache_dir(): + global attempted_to_create_download_cache_dir + if not os.path.exists(GLOBAL_DOWNLOAD_CACHE_DIR) and not attempted_to_create_download_cache_dir: + attempted_to_create_download_cache_dir = True + try: + original_umask = os.umask(0) + os.makedirs(GLOBAL_DOWNLOAD_CACHE_DIR, 0o777) + except Exception as ex: + logging.exception(f"Could not create directory {GLOBAL_DOWNLOAD_CACHE_DIR}") + finally: + os.umask(original_umask) + if os.path.isdir(GLOBAL_DOWNLOAD_CACHE_DIR) and os.access(GLOBAL_DOWNLOAD_CACHE_DIR, os.W_OK): + return GLOBAL_DOWNLOAD_CACHE_DIR + return os.path.expanduser('~/.cache/downloads') + + +def load_yaml_file(yaml_path: str) -> Any: + # Import the yaml module locally so that the common_util module is still usable outside of any + # virtualenv where yaml is preinstalled. + import yaml + with open(yaml_path) as yaml_file: + return yaml.safe_load(yaml_file) diff --git a/python/yb/fossa_analysis.py b/python/yb/fossa_analysis.py new file mode 100644 index 000000000000..d995b38eae1e --- /dev/null +++ b/python/yb/fossa_analysis.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Copyright (c) Yugabyte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. + +import os +import yaml +import subprocess +import argparse +import tempfile +import atexit +import logging +import shlex +import packaging +import re +import time + +from packaging import version + +from yb.common_util import ( + YB_SRC_ROOT, get_thirdparty_dir, get_download_cache_dir, load_yaml_file, init_env, shlex_join +) + +from downloadutil.downloader import Downloader +from downloadutil.download_config import DownloadConfig + + +FOSSA_VERSION_RE = re.compile(r'^fossa-cli version ([^ ]+) .*$') +MIN_FOSSA_CLI_VERSION = '1.1.7' + + +def should_include_fossa_module(name: str) -> bool: + return not name.startswith(('llvm', 'gmock', 'cassandra-cpp-driver', 'bison', 'flex')) + + +def main(): + parser = argparse.ArgumentParser( + description='Run FOSSA analysis (open source license compliance).') + parser.add_argument('--verbose', action='store_true', help='Enable verbose output') + parser.add_argument( + 'fossa_cli_args', + nargs='*', + help='These arguments are passed directly to fossa-cli') + args = parser.parse_args() + init_env(args.verbose) + + fossa_cmd_line = ['fossa', 'analyze'] + fossa_cmd_line.extend(args.fossa_cli_args) + + should_upload = not any( + arg in args.fossa_cli_args for arg in ('--show-output', '--output', '-o')) + + if not should_upload and not os.getenv('FOSSA_API_KEY'): + # --output is used for local analysis only, without uploading the results. In all other + # cases we would like . + raise RuntimeError('FOSSA_API_KEY must be specified in order to upload analysis results.') + + logging.info( + f"FOSSA CLI command line (except the configuration path): {shlex_join(fossa_cmd_line)}") + + fossa_version_str = subprocess.check_output(['fossa', '--version']).decode('utf-8') + fossa_version_match = FOSSA_VERSION_RE.match(fossa_version_str) + if not fossa_version_match: + raise RuntimeError(f"Cannot parse fossa-cli version: {fossa_version_str}") + fossa_version = fossa_version_match.group(1) + if version.parse(fossa_version) < version.parse(MIN_FOSSA_CLI_VERSION): + raise RuntimeError( + f"fossa-cli version too old: {fossa_version} " + f"(expected {MIN_FOSSA_CLI_VERSION} or later)") + + download_cache_path = get_download_cache_dir() + logging.info(f"Using the download cache directory {download_cache_path}") + download_config = DownloadConfig( + verbose=args.verbose, + cache_dir_path=download_cache_path + ) + downloader = Downloader(download_config) + + fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa.yml') + fossa_yml_data = load_yaml_file(fossa_yml_path) + modules = fossa_yml_data['analyze']['modules'] + + thirdparty_dir = get_thirdparty_dir() + fossa_modules_path = os.path.join(thirdparty_dir, 'fossa_modules.yml') + + seen_urls = set() + + start_time_sec = time.time() + if os.path.exists(fossa_modules_path): + thirdparty_fossa_modules_data = load_yaml_file(fossa_modules_path) + for thirdparty_module_data in thirdparty_fossa_modules_data: + fossa_module_data = thirdparty_module_data['fossa_module'] + module_name = fossa_module_data['name'] + if not should_include_fossa_module(module_name): + continue + fossa_module_yb_metadata = thirdparty_module_data['yb_metadata'] + expected_sha256 = fossa_module_yb_metadata['sha256sum'] + url = fossa_module_yb_metadata['url'] + if url in seen_urls: + # Due to a bug in some versions of yugabyte-db-thirdparty scripts, as of 04/20/2021 + # we may include the same dependency twince in the fossa_modules.yml file. We just + # skip the duplicates here. + continue + seen_urls.add(url) + + logging.info(f"Adding module from {url}") + downloaded_path = downloader.download_url( + url, + download_parent_dir_path=None, # Download to cache directly. + verify_checksum=True, + expected_sha256=expected_sha256 + ) + fossa_module_data['target'] = downloaded_path + modules.append(fossa_module_data) + + effective_fossa_yml_path = os.path.join(os.getenv('BUILD_ROOT'), 'effective_fossa.yml') + with open(effective_fossa_yml_path, 'w') as effective_fossa_yml_file: + effective_fossa_yml_file.write(yaml.dump(fossa_yml_data, default_flow_style=False)) + + logging.info(f"Wrote the expanded FOSSA file to {effective_fossa_yml_path}") + else: + logging.warning( + f"File {fossa_modules_path} does not exist. Some C/C++ dependencies will be missing " + f"from FOSSA analysis.") + + effective_fossa_yml_path = fossa_yml_path + + fossa_cmd_line += ['--config', effective_fossa_yml_path] + + elapsed_time_sec = time.time() - start_time_sec + logging.info("Generated the effective FOSSA configuration file in %.1f sec", elapsed_time_sec) + logging.info(f"Running command: {shlex_join(fossa_cmd_line)})") + subprocess.check_call(fossa_cmd_line) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 89455723248e..a2bf375fcf9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ compiledb psutil distro boto -overrides \ No newline at end of file +overrides +downloadutil diff --git a/requirements_frozen.txt b/requirements_frozen.txt index b6c9d934effc..9503771150fc 100644 --- a/requirements_frozen.txt +++ b/requirements_frozen.txt @@ -9,3 +9,5 @@ psutil==5.5.1 distro==1.5.0 boto==2.49.0 overrides==3.1.0 +downloadutil==1.0.2 +packaging==20.9 diff --git a/thirdparty b/thirdparty index 45c97f45f1ea..ee4e2e453b5b 160000 --- a/thirdparty +++ b/thirdparty @@ -1 +1 @@ -Subproject commit 45c97f45f1ea2f0898b87f434439c8356c119761 +Subproject commit ee4e2e453b5b83fb045b7a755f25305206a4e806 diff --git a/yb_release b/yb_release index 228ac7df219e..176e6635fc07 100755 --- a/yb_release +++ b/yb_release @@ -16,5 +16,5 @@ set -euo pipefail . "${BASH_SOURCE%/*}/build-support/common-build-env.sh" activate_virtualenv -export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-} +set_pythonpath python3 "$YB_SRC_ROOT"/build-support/yb_release.py "$@"