Find license notices in Go module archives

tools/licensecheck/BUILD

@@ -27,5 +27,6 @@ go_test(
         "dependencies.yaml",
         "//governance:licensing.yaml",
     ],
+    embedsrcs = glob(["testdata/*.txt"]),
     library = ":licensecheck",
 )

tools/licensecheck/licensecheck.go

@@ -651,7 +651,8 @@ func fetchLicense(d dep) (*fetched, error) {
 	return f, nil
 }
 
-// licenseFileNames are candidate top-level license files, in priority order.
+// licenseFileNames are candidate root paths for GitHub raw-file requests.
+// Module ZIPs provide a directory listing and discover suffixed notices below.
 var licenseFileNames = []string{
 	"LICENSE", "LICENSE.txt", "LICENSE.md", "LICENSE.TXT", "LICENSE.MIT",
 	"LICENSE-APACHE-2.0.txt", "LICENCE", "COPYING", "License", "license.md",
@@ -660,7 +661,7 @@ var licenseFileNames = []string{
 var pseudoVersionRE = regexp.MustCompile(`\d{14}-([0-9a-f]{12})$`)
 
 // fetchGoModule downloads a module from the Go module proxy and classifies
-// the license file at its root.
+// the license notices at its root.
 func fetchGoModule(path, version string) (*fetched, error) {
 	escPath, err := module.EscapePath(path)
 	if err != nil {
@@ -706,38 +707,62 @@ func fetchGoModule(path, version string) (*fetched, error) {
 		return nil, fmt.Errorf("cannot download module zip: %w", err)
 	}
 	zipSum := sha256.Sum256(zipBody)
-	zipReader, err := zip.NewReader(bytes.NewReader(zipBody), int64(len(zipBody)))
+	licenses, err := moduleLicenses(zipBody, path+"@"+version+"/")
+	if err != nil {
+		return nil, err
+	}
+	return &fetched{commit: commit, sha256: hex.EncodeToString(zipSum[:]), license: licenses}, nil
+}
+
+// moduleLicenses classifies each root license notice, including files that split
+// grants between names such as LICENSE-BSD and LICENSE-MIT. A README may carry
+// the grant when there are no primary notices; it must not override them.
+func moduleLicenses(body []byte, prefix string) (Licenses, error) {
+	z, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
 	if err != nil {
 		return nil, fmt.Errorf("cannot read module zip: %w", err)
 	}
-	rootFiles := make(map[string]*zip.File)
-	prefix := path + "@" + version + "/"
-	for _, f := range zipReader.File {
-		if rest, ok := strings.CutPrefix(f.Name, prefix); ok && !strings.Contains(rest, "/") {
-			rootFiles[rest] = f
+	var notices, readmes []*zip.File
+	for _, f := range z.File {
+		name, ok := strings.CutPrefix(f.Name, prefix)
+		if !ok || strings.Contains(name, "/") || !f.Mode().IsRegular() {
+			continue
 		}
-	}
-	for _, name := range licenseFileNames {
-		f, ok := rootFiles[name]
-		if !ok {
+		// Go source files such as license.go are not license notices.
+		if strings.HasSuffix(name, ".go") {
 			continue
 		}
-		r, err := f.Open()
-		if err != nil {
-			return nil, err
+		stem := strings.ToUpper(name)
+		if i := strings.IndexAny(stem, ".-_"); i >= 0 {
+			stem = stem[:i]
 		}
-		text, err := io.ReadAll(r)
-		r.Close()
+		switch stem {
+		case "LICENSE", "LICENCE", "COPYING":
+			notices = append(notices, f)
+		case "README":
+			readmes = append(readmes, f)
+		}
+	}
+	if len(notices) == 0 {
+		notices = readmes
+	}
+	if len(notices) == 0 {
+		return nil, errors.New("no license notice at module root")
+	}
+	var licenses Licenses
+	for _, f := range notices {
+		text, err := fs.ReadFile(z, f.Name)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("cannot read %s: %w", f.Name, err)
 		}
-		license, err := classify(string(text))
+		ids, err := classify(string(text))
 		if err != nil {
-			return nil, fmt.Errorf("%s: %w", name, err)
+			return nil, fmt.Errorf("%s: %w", strings.TrimPrefix(f.Name, prefix), err)
 		}
-		return &fetched{commit: commit, sha256: hex.EncodeToString(zipSum[:]), license: license}, nil
+		licenses = append(licenses, ids...)
 	}
-	return nil, errors.New("no license file at module root")
+	slices.Sort(licenses)
+	return slices.Compact(licenses), nil
 }
 
 var githubURLRegexps = []*regexp.Regexp{

tools/licensecheck/licensecheck_test.go

@@ -15,6 +15,9 @@
 package licensecheck
 
 import (
+	"archive/zip"
+	"bytes"
+	_ "embed"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -23,6 +26,117 @@ import (
 	"testing"
 )
 
+// Verbatim notices from the modules that exposed incomplete discovery:
+// https://github.com/JohnCGriffin/overflow/blob/46fa312c3/README.md
+// https://github.com/aymanbagabas/go-udiff/tree/4608934d2
+var (
+	//go:embed testdata/overflow-readme.txt
+	overflowReadme string
+	//go:embed testdata/udiff-bsd.txt
+	udiffBSD string
+	//go:embed testdata/udiff-mit.txt
+	udiffMIT string
+)
+
+func TestModuleLicenses(t *testing.T) {
+	const prefix = "example.com/module@v1.0.0/"
+	for _, test := range []struct {
+		name    string
+		files   map[string]string
+		want    Licenses
+		wantErr string
+	}{
+		{
+			name: "split notices",
+			files: map[string]string{
+				prefix + "LICENSE-BSD": udiffBSD,
+				prefix + "LICENSE-MIT": udiffMIT,
+			},
+			want: Licenses{"BSD-3-Clause", "MIT"},
+		},
+		{
+			name: "source files alongside notices",
+			files: map[string]string{
+				prefix + "LICENSE":         udiffMIT,
+				prefix + "LICENSE.docs":    udiffBSD,
+				prefix + "license.go":      "package licensecheck\n",
+				prefix + "license_test.go": "package licensecheck\n",
+			},
+			want: Licenses{"BSD-3-Clause", "MIT"},
+		},
+		{
+			name: "unrecognized notice alongside known notice",
+			files: map[string]string{
+				prefix + "LICENSE":     "All rights reserved. Do not redistribute.",
+				prefix + "LICENSE-MIT": udiffMIT,
+			},
+			wantErr: "LICENSE: cannot classify license text",
+		},
+		{
+			name:  "readme grant",
+			files: map[string]string{prefix + "README.md": overflowReadme},
+			want:  Licenses{"MIT"},
+		},
+		{
+			name: "primary notices take precedence",
+			files: map[string]string{
+				prefix + "License_BSD": udiffBSD,
+				prefix + "README.md":   overflowReadme,
+			},
+			want: Licenses{"BSD-3-Clause"},
+		},
+		{
+			name: "unrecognized primary notice",
+			files: map[string]string{
+				prefix + "COPYING":   "All rights reserved. Do not redistribute.",
+				prefix + "README.md": overflowReadme,
+			},
+			wantErr: "cannot classify license text",
+		},
+		{
+			name:    "readme reference is not a grant",
+			files:   map[string]string{prefix + "README": "See https://opensource.org/licenses/MIT for background."},
+			wantErr: "cannot classify license text",
+		},
+		{
+			name: "root membership",
+			files: map[string]string{
+				prefix + "source.go":               udiffMIT,
+				prefix + "nested/LICENSE":          udiffMIT,
+				"example.com/other@v1.0.0/LICENSE": udiffMIT,
+			},
+			wantErr: "no license notice at module root",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var body bytes.Buffer
+			z := zip.NewWriter(&body)
+			for name, text := range test.files {
+				w, err := z.Create(name)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if _, err := w.Write([]byte(text)); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if err := z.Close(); err != nil {
+				t.Fatal(err)
+			}
+			got, err := moduleLicenses(body.Bytes(), prefix)
+			if test.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), test.wantErr) {
+					t.Fatalf("moduleLicenses = %v, %v, want error %q", got, err, test.wantErr)
+				}
+				return
+			}
+			if err != nil || !slices.Equal(got, test.want) {
+				t.Fatalf("moduleLicenses = %v, %v, want %v", got, err, test.want)
+			}
+		})
+	}
+}
+
 func TestClassify(t *testing.T) {
 	for _, test := range []struct {
 		name, text string

tools/licensecheck/testdata/overflow-readme.txt

@@ -0,0 +1,84 @@
+[![Build Status](https://travis-ci.org/JohnCGriffin/overflow.png)](https://travis-ci.org/JohnCGriffin/overflow)
+# overflow
+Check for int/int8/int16/int64/int32 integer overflow in Golang arithmetic.
+### Install
+```
+go get github.com/johncgriffin/overflow
+```
+Note that because Go has no template types, the majority of repetitive code is 
+generated by overflow_template.sh.  If you have to change an
+algorithm, change it there and regenerate the Go code via:
+```
+go generate
+```
+### Synopsis
+
+```
+package main
+
+import "fmt"
+import "math"
+import "github.com/JohnCGriffin/overflow"
+
+func main() {
+
+	addend := math.MaxInt64 - 5
+
+	for i := 0; i < 10; i++ {
+		sum, ok := overflow.Add(addend, i)
+		fmt.Printf("%v+%v -> (%v,%v)\n",
+			addend, i, sum, ok)
+	}
+
+}
+```
+yields the output
+```
+9223372036854775802+0 -> (9223372036854775802,true)
+9223372036854775802+1 -> (9223372036854775803,true)
+9223372036854775802+2 -> (9223372036854775804,true)
+9223372036854775802+3 -> (9223372036854775805,true)
+9223372036854775802+4 -> (9223372036854775806,true)
+9223372036854775802+5 -> (9223372036854775807,true)
+9223372036854775802+6 -> (0,false)
+9223372036854775802+7 -> (0,false)
+9223372036854775802+8 -> (0,false)
+9223372036854775802+9 -> (0,false)
+```
+
+For int, int64, and int32 types, provide Add, Add32, Add64, Sub, Sub32, Sub64, etc.  
+Unsigned types not covered at the moment, but such additions are welcome.
+
+### Stay calm and panic
+
+There's a good case to be made that a panic is an unidiomatic but proper response.  Iff you
+believe that there's no valid way to continue your program after math goes wayward, you can
+use the easier Addp, Mulp, Subp, and Divp versions which return the normal result or panic.
+
+
+- - -
+MIT License
+
+Copyright (c) 2017 John C. Griffin, 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+
+

tools/licensecheck/testdata/udiff-bsd.txt

@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

tools/licensecheck/testdata/udiff-mit.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Ayman Bagabas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.