[#8120] Include C/C++ dependenices into FOSSA analysis

Summary:
Include C/C++ dependenices into FOSSA analysis for open source license compliance.
This is done by taking the special YAML metadata file that we've included in the third-party
dependencies archive and creating "raw" FOSSA modules from each dependency's tar.gz file, excluding
some of the dependencies that are only used during build time.

Test Plan: Jenkins: compile only

Reviewers: jharveysmith, steve.varnau

Reviewed By: jharveysmith, steve.varnau

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D11319

build-support/build_postgres

@@ -26,5 +26,5 @@ handle_predefined_build_root
 activate_virtualenv
 set_sanitizer_runtime_options
 
-export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-}
+set_pythonpath
 "$YB_SRC_ROOT"/python/yb/build_postgres.py "$@"

build-support/common-build-env.sh

@@ -2095,6 +2095,27 @@ VIRTUALENV DEBUGGING
   export VIRTUAL_ENV
 }
 
+set_pythonpath_called=false
+
+set_pythonpath() {
+  if [[ $set_pythonpath_called == "true" ]]; then
+    return
+  fi
+  set_pythonpath_called=true
+
+  if [[ ! -d ${YB_SRC_ROOT:-} ]]; then
+    fatal "YB_SRC_ROOT is not set or does not exist; ${YB_SRC_ROOT:-undefined}"
+  fi
+
+  local new_entry=$YB_SRC_ROOT/python
+  if [[ -z ${PYTHONPATH:-} ]]; then
+    PYTHONPATH=$new_entry
+  else
+    PYTHONPATH=$new_entry:$PYTHONPATH
+  fi
+  export PYTHONPATH
+}
+
 log_file_existence() {
   expect_num_args 1 "$@"
   local file_name=$1

build-support/dependency_graph

@@ -16,5 +16,5 @@ set -euo pipefail
 . "${BASH_SOURCE%/*}/common-build-env.sh"
 
 activate_virtualenv
-export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-}
+set_pythonpath
 "$YB_SRC_ROOT"/python/yb/dependency_graph.py "$@"

build-support/fossa_analysis

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Copyright (c) Yugabyte, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.  See the License for the specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+. "${BASH_SOURCE%/*}/common-build-env.sh"
+
+activate_virtualenv
+set_pythonpath
+
+if [[ -n ${BUILD_ROOT:-} ]]; then
+  handle_predefined_build_root_quietly=true
+  predefined_build_root=$BUILD_ROOT
+  handle_predefined_build_root
+fi
+find_or_download_thirdparty
+
+python3 "$YB_SRC_ROOT"/python/yb/fossa_analysis.py "$@"

build-support/gen_initial_sys_catalog_snapshot_wrapper

@@ -16,5 +16,5 @@ set -euo pipefail
 . "${BASH_SOURCE%/*}/common-build-env.sh"
 
 activate_virtualenv
-export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-}
+set_pythonpath
 "$YB_SRC_ROOT"/python/yb/gen_initial_sys_catalog_snapshot.py "$@"

build-support/run_pvs_studio_analyzer

@@ -16,5 +16,5 @@ set -euo pipefail
 . "${BASH_SOURCE%/*}/common-build-env.sh"
 
 activate_virtualenv
-export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-}
-"$YB_SRC_ROOT"/python/yb/run_pvs_studio_analyzer.py "$@"
+set_pythonpath
+python3 "$YB_SRC_ROOT"/python/yb/run_pvs_studio_analyzer.py "$@"

build-support/thirdparty_url_centos.txt

@@ -1 +1 @@
-https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210192532-45c97f45f1-centos7-linuxbrew/yugabyte-db-thirdparty-v20210210192532-45c97f45f1-centos7-linuxbrew.tar.gz
+https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402083441-ee4e2e453b-centos7-linuxbrew/yugabyte-db-thirdparty-v20210402083441-ee4e2e453b-centos7-linuxbrew.tar.gz

build-support/thirdparty_url_centos_clang11.txt

@@ -1 +1 @@
-https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210306075844-471ef8e125-centos7-clang11/yugabyte-db-thirdparty-v20210306075844-471ef8e125-centos7-clang11.tar.gz
+https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402093407-ee4e2e453b-centos7-clang11/yugabyte-db-thirdparty-v20210402093407-ee4e2e453b-centos7-clang11.tar.gz

build-support/thirdparty_url_centos_gcc8.txt

@@ -1 +1 @@
-https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210222846-45c97f45f1-centos7-devtoolset-8/yugabyte-db-thirdparty-v20210210222846-45c97f45f1-centos7-devtoolset-8.tar.gz
+https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402122231-ee4e2e453b-centos7-devtoolset-8/yugabyte-db-thirdparty-v20210402122231-ee4e2e453b-centos7-devtoolset-8.tar.gz

build-support/thirdparty_url_centos_gcc9.txt

@@ -1 +1 @@
-https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210211756-45c97f45f1-centos7-devtoolset-9/yugabyte-db-thirdparty-v20210210211756-45c97f45f1-centos7-devtoolset-9.tar.gz
+https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402130708-ee4e2e453b-centos7-devtoolset-9/yugabyte-db-thirdparty-v20210402130708-ee4e2e453b-centos7-devtoolset-9.tar.gz

build-support/thirdparty_url_mac.txt

@@ -1 +1 @@
-https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210192517-45c97f45f1-macos/yugabyte-db-thirdparty-v20210210192517-45c97f45f1-macos.tar.gz
+https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402050931-ee4e2e453b-macos/yugabyte-db-thirdparty-v20210402050931-ee4e2e453b-macos.tar.gz

build-support/thirdparty_url_ubuntu.txt

@@ -1 +1 @@
-https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210210212603-45c97f45f1-ubuntu1804-gcc7/yugabyte-db-thirdparty-v20210210212603-45c97f45f1-ubuntu1804-gcc7.tar.gz
+https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20210402084227-ee4e2e453b-ubuntu1804-gcc7/yugabyte-db-thirdparty-v20210402084227-ee4e2e453b-ubuntu1804-gcc7.tar.gz

python/yb/common_util.py

@@ -13,6 +13,8 @@
 import subprocess
 import shlex
 
+from typing import Any
+
 
 MODULE_DIR = os.path.dirname(os.path.realpath(__file__))
 YB_SRC_ROOT = os.path.realpath(os.path.join(MODULE_DIR, '..', '..'))
@@ -24,6 +26,9 @@
 _YB_THIRDPARTY_DIR = os.path.realpath(
         os.environ.get("YB_THIRDPARTY_DIR", os.path.join(YB_SRC_ROOT, 'thirdparty')))
 
+GLOBAL_DOWNLOAD_CACHE_DIR = '/opt/yb-build/download_cache'
+attempted_to_create_download_cache_dir = False
+
 
 def get_thirdparty_dir():
     global _YB_THIRDPARTY_DIR
@@ -284,3 +289,27 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         for env_var_name, saved_value in self.saved_env_vars.items():
             dict_set_or_del(os.environ, env_var_name, saved_value)
+
+
+def get_download_cache_dir():
+    global attempted_to_create_download_cache_dir
+    if not os.path.exists(GLOBAL_DOWNLOAD_CACHE_DIR) and not attempted_to_create_download_cache_dir:
+        attempted_to_create_download_cache_dir = True
+        try:
+            original_umask = os.umask(0)
+            os.makedirs(GLOBAL_DOWNLOAD_CACHE_DIR, 0o777)
+        except Exception as ex:
+            logging.exception(f"Could not create directory {GLOBAL_DOWNLOAD_CACHE_DIR}")
+        finally:
+            os.umask(original_umask)
+    if os.path.isdir(GLOBAL_DOWNLOAD_CACHE_DIR) and os.access(GLOBAL_DOWNLOAD_CACHE_DIR, os.W_OK):
+        return GLOBAL_DOWNLOAD_CACHE_DIR
+    return os.path.expanduser('~/.cache/downloads')
+
+
+def load_yaml_file(yaml_path: str) -> Any:
+    # Import the yaml module locally so that the common_util module is still usable outside of any
+    # virtualenv where yaml is preinstalled.
+    import yaml
+    with open(yaml_path) as yaml_file:
+        return yaml.safe_load(yaml_file)

python/yb/fossa_analysis.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+# Copyright (c) Yugabyte, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.  See the License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import yaml
+import subprocess
+import argparse
+import tempfile
+import atexit
+import logging
+import shlex
+import packaging
+import re
+import time
+
+from packaging import version
+
+from yb.common_util import (
+    YB_SRC_ROOT, get_thirdparty_dir, get_download_cache_dir, load_yaml_file, init_env, shlex_join
+)
+
+from downloadutil.downloader import Downloader
+from downloadutil.download_config import DownloadConfig
+
+
+FOSSA_VERSION_RE = re.compile(r'^fossa-cli version ([^ ]+) .*$')
+MIN_FOSSA_CLI_VERSION = '1.1.7'
+
+
+def should_include_fossa_module(name: str) -> bool:
+    return not name.startswith(('llvm', 'gmock', 'cassandra-cpp-driver', 'bison', 'flex'))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Run FOSSA analysis (open source license compliance).')
+    parser.add_argument('--verbose', action='store_true', help='Enable verbose output')
+    parser.add_argument(
+        'fossa_cli_args',
+        nargs='*',
+        help='These arguments are passed directly to fossa-cli')
+    args = parser.parse_args()
+    init_env(args.verbose)
+
+    fossa_cmd_line = ['fossa', 'analyze']
+    fossa_cmd_line.extend(args.fossa_cli_args)
+
+    should_upload = not any(
+        arg in args.fossa_cli_args for arg in ('--show-output', '--output', '-o'))
+
+    if not should_upload and not os.getenv('FOSSA_API_KEY'):
+        # --output is used for local analysis only, without uploading the results. In all other
+        # cases we would like .
+        raise RuntimeError('FOSSA_API_KEY must be specified in order to upload analysis results.')
+
+    logging.info(
+        f"FOSSA CLI command line (except the configuration path): {shlex_join(fossa_cmd_line)}")
+
+    fossa_version_str = subprocess.check_output(['fossa', '--version']).decode('utf-8')
+    fossa_version_match = FOSSA_VERSION_RE.match(fossa_version_str)
+    if not fossa_version_match:
+        raise RuntimeError(f"Cannot parse fossa-cli version: {fossa_version_str}")
+    fossa_version = fossa_version_match.group(1)
+    if version.parse(fossa_version) < version.parse(MIN_FOSSA_CLI_VERSION):
+        raise RuntimeError(
+            f"fossa-cli version too old: {fossa_version} "
+            f"(expected {MIN_FOSSA_CLI_VERSION} or later)")
+
+    download_cache_path = get_download_cache_dir()
+    logging.info(f"Using the download cache directory {download_cache_path}")
+    download_config = DownloadConfig(
+        verbose=args.verbose,
+        cache_dir_path=download_cache_path
+    )
+    downloader = Downloader(download_config)
+
+    fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa.yml')
+    fossa_yml_data = load_yaml_file(fossa_yml_path)
+    modules = fossa_yml_data['analyze']['modules']
+
+    thirdparty_dir = get_thirdparty_dir()
+    fossa_modules_path = os.path.join(thirdparty_dir, 'fossa_modules.yml')
+
+    seen_urls = set()
+
+    start_time_sec = time.time()
+    if os.path.exists(fossa_modules_path):
+        thirdparty_fossa_modules_data = load_yaml_file(fossa_modules_path)
+        for thirdparty_module_data in thirdparty_fossa_modules_data:
+            fossa_module_data = thirdparty_module_data['fossa_module']
+            module_name = fossa_module_data['name']
+            if not should_include_fossa_module(module_name):
+                continue
+            fossa_module_yb_metadata = thirdparty_module_data['yb_metadata']
+            expected_sha256 = fossa_module_yb_metadata['sha256sum']
+            url = fossa_module_yb_metadata['url']
+            if url in seen_urls:
+                # Due to a bug in some versions of yugabyte-db-thirdparty scripts, as of 04/20/2021
+                # we may include the same dependency twince in the fossa_modules.yml file. We just
+                # skip the duplicates here.
+                continue
+            seen_urls.add(url)
+
+            logging.info(f"Adding module from {url}")
+            downloaded_path = downloader.download_url(
+                url,
+                download_parent_dir_path=None,  # Download to cache directly.
+                verify_checksum=True,
+                expected_sha256=expected_sha256
+            )
+            fossa_module_data['target'] = downloaded_path
+            modules.append(fossa_module_data)
+
+        effective_fossa_yml_path = os.path.join(os.getenv('BUILD_ROOT'), 'effective_fossa.yml')
+        with open(effective_fossa_yml_path, 'w') as effective_fossa_yml_file:
+            effective_fossa_yml_file.write(yaml.dump(fossa_yml_data, default_flow_style=False))
+
+        logging.info(f"Wrote the expanded FOSSA file to {effective_fossa_yml_path}")
+    else:
+        logging.warning(
+            f"File {fossa_modules_path} does not exist. Some C/C++ dependencies will be missing "
+            f"from FOSSA analysis.")
+
+        effective_fossa_yml_path = fossa_yml_path
+
+    fossa_cmd_line += ['--config', effective_fossa_yml_path]
+
+    elapsed_time_sec = time.time() - start_time_sec
+    logging.info("Generated the effective FOSSA configuration file in %.1f sec", elapsed_time_sec)
+    logging.info(f"Running command: {shlex_join(fossa_cmd_line)})")
+    subprocess.check_call(fossa_cmd_line)
+
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -5,4 +5,5 @@ compiledb
 psutil
 distro
 boto
-overrides
+overrides
+downloadutil

requirements_frozen.txt

@@ -9,3 +9,5 @@ psutil==5.5.1
 distro==1.5.0
 boto==2.49.0
 overrides==3.1.0
+downloadutil==1.0.2
+packaging==20.9

yb_release

@@ -16,5 +16,5 @@ set -euo pipefail
 . "${BASH_SOURCE%/*}/build-support/common-build-env.sh"
 
 activate_virtualenv
-export PYTHONPATH=$YB_SRC_ROOT/python:${PYTHONPATH:-}
+set_pythonpath
 python3 "$YB_SRC_ROOT"/build-support/yb_release.py "$@"