generate_notice.py

#!/usr/bin/env python3
import glob
import os
import datetime
import argparse
import json
import csv
import re
import pdb
import copy


def read_file(filename):

    if not os.path.isfile(filename):
        print("File not found {}".format(filename))
        return ""

    with open(filename, 'r', encoding='utf_8') as f:
        return f.read()


def read_go_mod(vendor_dir):
    lines = []
    with open(os.path.join(vendor_dir, "modules.txt"), encoding="utf_8") as f:
        lines = f.readlines()

    deps = []
    for line in lines:
        if line.startswith("# "):
            line = line[2:]
            elems = line.split(" ")
            if len(elems) == 2:
                data = _get_version_info(elems)
                deps.append(data)
                continue

            if " => " in line:
                data = _get_replaced_dep_data(line)
                deps.append(data)

    return deps


def _get_version_info(elems):
    data = {}
    data["path"] = elems[0]

    version_info = elems[1].rstrip("\n")
    if version_info.endswith("+incompatible"):
        version_info = version_info[:-len("+incompatible")]

    if len(version_info) < 30:
        data["version"] = version_info
        return data

    revision_elems = version_info.split("-")
    if len(revision_elems) != 3:
        raise ValueError("unexpected number of elements")

    if revision_elems[0] != "v0.0.0":
        data["version"] = revision_elems[0]
    data["revision"] = revision_elems[2]

    return data


def _get_replaced_dep_data(line):
    original, fork = line.split(" => ")
    elems = fork.split(" ")
    fork_data = _get_version_info(elems)
    elems = original.split(" ")
    data = _get_version_info(elems)

    if "path" in fork_data:
        data["overwrite-path"] = fork_data["path"]
    if "version" in fork_data:
        data["overwrite-version"] = fork_data["version"]
    if "revision" in fork_data:
        data["overwrite-revision"] = fork_data["revision"]

    if "revision" in data and data["revision"] == "000000000000":
        del(data["revision"])

    return data


def get_library_path(license):
    """
    Get the contents up to the vendor folder.
    """
    split = license.split(os.sep)
    for i, word in reversed(list(enumerate(split))):
        if word == "vendor":
            return "/".join(split[i + 1:])
    return "/".join(split)


def gather_dependencies(vendor_dir, overrides=None):
    dependencies = {}   # lib_path -> [array of lib]
    libs = read_go_mod(vendor_dir)

    # walk looking for LICENSE files
    for root, dirs, filenames in os.walk("./vendor"):
        licenses = get_licenses(root)
        for filename in licenses:
            lib_path = get_library_path(root)
            lib_search = [l for l in libs if l["path"].startswith(lib_path)]
            if len(lib_search) == 0:
                print("WARNING: No version information found for: {}".format(lib_path))
                lib = {"path": lib_path}
            else:
                lib = copy.deepcopy(lib_search[0])

            lib["license_file"] = os.path.join(root, filename)

            lib["license_contents"] = read_file(lib["license_file"])
            lib["license_summary"] = detect_license_summary(lib["license_contents"])
            if lib["license_summary"] == "UNKNOWN":
                print("WARNING: Unknown license for: {}".format(lib_path))

            revision = overrides.get(lib_path, {}).get("revision")
            if revision:
                lib["revision"] = revision

            if lib_path not in dependencies:
                dependencies[lib_path] = [lib]
            else:
                dependencies[lib_path].append(lib)

        # don't walk down into another vendor dir
        if "vendor" in dirs:
            dirs.remove("vendor")

    return dependencies


# Allow to skip files that could match the `LICENSE` pattern but does not have any license information.
SKIP_FILES = [
    # AWS lambda go defines that some part of the code is APLv2 and other on a MIT Modified license.
    "./vendor/github.com/aws/aws-lambda-go/LICENSE-SUMMARY"
]


def get_licenses(folder):
    """
    Get a list of license files from a given directory.
    """
    licenses = []
    for filename in sorted(os.listdir(folder)):
        if filename.startswith("LICENSE") and "docs" not in filename and os.path.join(folder, filename) not in SKIP_FILES:
            licenses.append(filename)
        elif filename.startswith("APLv2"):  # gorhill/cronexpr
            licenses.append(filename)
        elif filename in ("COPYING",):  # BurntSushi/toml
            licenses.append(filename)
    return licenses


def has_license(folder):
    """
    Checks if a particular repo has a license files.

    There are two cases accepted:
        * The folder contains a LICENSE
        * The parent folder contains a LICENSE
        * The folder only contains subdirectories AND all these
          subdirectories contain a LICENSE
        * The folder only contains subdirectories AND all these
          subdirectories contain  subdirectories which contain a LICENSE (ex Azure folder)
    """

    if len(get_licenses(folder)) > 0:
        return True, ""
    elif len(get_licenses(os.path.join(folder, os.pardir))) > 0:  # For go.opencensus.io.
        return True, ""

    for subdir in os.listdir(folder):
        if not os.path.isdir(os.path.join(folder, subdir)):
            return False, folder
        if len(get_licenses(os.path.join(folder, subdir))) > 0:
            continue
        for dir in os.listdir(os.path.join(folder, subdir)):
            if not os.path.isdir(os.path.join(folder, subdir, dir)):
                return False, subdir
            if len(get_licenses(os.path.join(folder, subdir, dir))) == 0:
                return False, os.path.join(folder, subdir, dir)

    return True, ""


def check_all_have_license_files(vendor_dir):
    """
    Checks that everything in the vendor folders has a license one way
    or the other. This doesn't collect the licenses, because the code that
    collects the licenses needs to walk the full tree. This one makes sure
    that every folder in the `vendor` directories has at least one license.
    """
    issues = []
    for root, dirs, filenames in os.walk(vendor_dir):
        depth = 2
        if root.count(os.sep) - vendor_dir.count(os.sep) == depth:  # two levels deep
            # Two level deep means folders like `github.com/elastic`.
            # look for the license in root but also one level up
            ok, issue = has_license(root)
            if not ok:
                depth += 1

                if depth > 5:
                    print("No license in: {}".format(issue))
                    issues.append(issue)

    if len(issues) > 0:
        raise Exception("I have found licensing issues in the following folders: {}"
                        .format(issues))


def write_notice_file(f, beat, copyright, dependencies):

    now = datetime.datetime.now()

    # Add header
    f.write("{}\n".format(beat))
    f.write("Copyright 2014-{0} {1}\n".format(now.year, copyright))
    f.write("\n")
    f.write("This product includes software developed by The Apache Software \n" +
            "Foundation (http://www.apache.org/).\n\n")

    # Add licenses for 3rd party libraries
    f.write("==========================================================================\n")
    f.write("Third party libraries used by the {} project:\n".format(beat))
    f.write("==========================================================================\n\n")

    # Sort licenses by package path, ignore upper / lower case
    for key in sorted(dependencies, key=str.lower):
        for lib in dependencies[key]:
            f.write("\n--------------------------------------------------------------------\n")
            f.write("Dependency: {}\n".format(key))
            if "version" in lib:
                f.write("Version: {}\n".format(lib["version"]))
            if "revision" in lib:
                f.write("Revision: {}\n".format(lib["revision"]))
            if "overwrite-path" in lib:
                f.write("Overwrite: {}\n".format(lib["overwrite-path"]))
            if "overwrite-version" in lib:
                f.write("Overwrite-Version: {}\n".format(lib["overwrite-version"]))
            if "overwrite-revision" in lib:
                f.write("Overwrite-Revision: {}\n".format(lib["overwrite-revision"]))
            f.write("License type (autodetected): {}\n".format(lib["license_summary"]))
            f.write("{}:\n".format(lib["license_file"]))
            f.write("--------------------------------------------------------------------\n")
            if lib["license_summary"] != "Apache-2.0":
                f.write(lib["license_contents"])
            else:
                # it's an Apache License, so include only the NOTICE file
                f.write("Apache License 2.0\n\n")

                # Skip NOTICE files which are not needed
                if os.path.join(os.path.dirname(lib["license_file"])) in SKIP_NOTICE:
                    continue

                for notice_file in glob.glob(os.path.join(os.path.dirname(lib["license_file"]), "NOTICE*")):
                    notice_file_hdr = "-------{}-----\n".format(os.path.basename(notice_file))
                    f.write(notice_file_hdr)
                    f.write(read_file(notice_file))


def write_csv_file(csvwriter, dependencies):
    csvwriter.writerow(["name", "url", "version", "revision", "license"])
    for key in sorted(dependencies, key=str.lower):
        for lib in dependencies[key]:
            csvwriter.writerow([key, get_url(key), lib.get("version", ""), lib.get("revision", ""),
                                lib["license_summary"]])


def get_url(repo):
    words = repo.split("/")
    if words[0] != "github.com":
        return repo
    return "https://github.com/{}/{}".format(words[1], words[2])


def create_notice(filename, beat, copyright, vendor_dir, csvfile, overrides=None):
    dependencies = gather_dependencies(vendor_dir, overrides=overrides)
    if not csvfile:
        with open(filename, "w+", encoding='utf_8') as f:
            write_notice_file(f, beat, copyright, dependencies)
            print("Available at {}".format(filename))
    else:
        with open(csvfile, "w") as f:
            csvwriter = csv.writer(f)
            write_csv_file(csvwriter, dependencies)
            print("Available at {}".format(csvfile))
    return dependencies


APACHE2_LICENSE_TITLES = [
    "Apache License 2.0",
    "Apache License Version 2.0",
    "Apache License, Version 2.0",
    "licensed under the Apache 2.0 license",  # github.com/zmap/zcrypto
    re.sub(r"\s+", " ", """Apache License
    ==============

    _Version 2.0, January 2004_"""),
]

MIT_LICENSES = [
    re.sub(r"\s+", " ", """Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
    """),
    re.sub(r"\s+", " ", """Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies."""),
    re.sub(r"\s+", " ", """Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
    """),
    re.sub(r"\s+", " ", """Permission is hereby granted, free of charge, to any person obtaining a copy of this
software and associated documentation files (the "Software"), to deal in the Software
without restriction, including without limitation the rights to use, copy, modify,
merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    """),
]

BSD_LICENSE_CONTENTS = [
    re.sub(r"\s+", " ", """Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:"""),
    re.sub(r"\s+", " ", """Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer."""),
    re.sub(r"\s+", " ", """Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.
""")]

BSD_LICENSE_3_CLAUSE = [
    re.sub(r"\s+", " ", """Neither the name of"""),
    re.sub(r"\s+", " ", """nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.""")
]

BSD_LICENSE_4_CLAUSE = [
    re.sub(r"\s+", " ", """All advertising materials mentioning features or use of this software
   must display the following acknowledgement"""),
]

CC_SA_4_LICENSE_TITLE = [
    "Creative Commons Attribution-ShareAlike 4.0 International"
]

ECLIPSE_PUBLIC_LICENSE_TITLES = [
    "Eclipse Public License - v 1.0"
]

LGPL_3_LICENSE_TITLE = [
    "GNU LESSER GENERAL PUBLIC LICENSE Version 3"
]

MPL_LICENSE_TITLES = [
    "Mozilla Public License Version 2.0",
    "Mozilla Public License, version 2.0"
]

UNIVERSAL_PERMISSIVE_LICENSE_TITLES = [
    "The Universal Permissive License (UPL), Version 1.0"
]

ISC_LICENSE_TITLE = [
    "ISC License",
]


# return SPDX identifiers from https://spdx.org/licenses/
def detect_license_summary(content):
    # replace all white spaces with a single space
    content = re.sub(r"\s+", ' ', content)
    # replace smart quotes with less intelligent ones
    content = content.replace(bytes(b'\xe2\x80\x9c').decode(), '"').replace(bytes(b'\xe2\x80\x9d').decode(), '"')
    if any(sentence in content[0:1000] for sentence in APACHE2_LICENSE_TITLES):
        return "Apache-2.0"
    if any(sentence in content[0:1000] for sentence in MIT_LICENSES):
        return "MIT"
    if all(sentence in content[0:1000] for sentence in BSD_LICENSE_CONTENTS):
        if all(sentence in content[0:1000] for sentence in BSD_LICENSE_3_CLAUSE):
            if all(sentence in content[0:1000] for sentence in BSD_LICENSE_4_CLAUSE):
                return "BSD-4-Clause"
            return "BSD-3-Clause"
        else:
            return "BSD-2-Clause"
    if any(sentence in content[0:300] for sentence in MPL_LICENSE_TITLES):
        return "MPL-2.0"
    if any(sentence in content[0:3000] for sentence in CC_SA_4_LICENSE_TITLE):
        return "CC-BY-SA-4.0"
    if any(sentence in content[0:3000] for sentence in LGPL_3_LICENSE_TITLE):
        return "LGPL-3.0"
    if any(sentence in content[0:1500] for sentence in UNIVERSAL_PERMISSIVE_LICENSE_TITLES):
        return "UPL-1.0"
    if any(sentence in content[0:1500] for sentence in ECLIPSE_PUBLIC_LICENSE_TITLES):
        return "EPL-1.0"
    if any(sentence in content[0:1500] for sentence in ISC_LICENSE_TITLE):
        return "ISC"
    if any(sentence in content[0:1500] for sentence in ECLIPSE_PUBLIC_LICENSE_TITLES):
        return "EPL-1.0"
    return "UNKNOWN"


ACCEPTED_LICENSES = [
    "Apache-2.0",
    "BSD-4-Clause",
    "BSD-3-Clause",
    "BSD-2-Clause",
    "EPL-1.0",
    "MIT",
    "MPL-2.0",
    "UPL-1.0",
    "ISC",
]
SKIP_NOTICE = []

if __name__ == "__main__":

    parser = argparse.ArgumentParser(
        description="Generate the NOTICE file from all vendor directories available in a given directory")
    parser.add_argument("vendor",
                        help="directory where to search for vendor directories")
    parser.add_argument("-b", "--beat", default="Elastic Beats",
                        help="Beat name")
    parser.add_argument("-c", "--copyright", default="Elasticsearch BV",
                        help="copyright owner")
    parser.add_argument("--csv", dest="csvfile",
                        help="Output to a csv file")
    parser.add_argument("-e", "--excludes", default=["dev-tools", "build"],
                        help="List of top directories to exclude")
    # no need to be generic for now, no other transitive dependency information available
    parser.add_argument("--beats-origin", type=argparse.FileType('r'),
                        help="path to beats vendor.json")
    parser.add_argument("-s", "--skip-notice", default=[],
                        help="List of NOTICE files to skip")
    args = parser.parse_args()

    cwd = os.getcwd()
    notice = os.path.join(cwd, "NOTICE.txt")
    vendor_dir = "./vendor"

    excludes = args.excludes
    if not isinstance(excludes, list):
        excludes = [excludes]
    SKIP_NOTICE = args.skip_notice

    overrides = {}  # revision overrides only for now
    if args.beats_origin:
        govendor = json.load(args.beats_origin)
        overrides = {package['path']: package for package in govendor["package"]}

    print("Get the licenses available from {}".format(vendor_dir))
    check_all_have_license_files(vendor_dir)
    dependencies = create_notice(notice, args.beat, args.copyright, vendor_dir, args.csvfile, overrides=overrides)

    # check that all licenses are accepted
    for _, deps in dependencies.items():
        for dep in deps:
            if dep["license_summary"] not in ACCEPTED_LICENSES:
                raise Exception("Dependency {} has invalid license {}"
                                .format(dep["path"], dep["license_summary"]))