Add LicenseRef support

lib/utils.js

@@ -416,20 +416,18 @@ function joinExpressions(expressions) {
   return SPDX.normalize(joinedExpressionString)
 }
 
-function normalizeLicenseExpression(licenseExpression, logger) {
-  if (!licenseExpression) return null
-
-  const licenseVisitor = rawLicenseExpression => {
-    const mappedLicenseExpression = scancodeMap.get(rawLicenseExpression)
-    const licenseExpression = mappedLicenseExpression ? mappedLicenseExpression : rawLicenseExpression
-
-    return SPDX.normalizeSingle(licenseExpression)
-  }
-
-  const parsed = SPDX.parse(licenseExpression, licenseVisitor)
+function normalizeLicenseExpression(
+  rawLicenseExpression,
+  logger,
+  licenseRefLookup = token => token && scancodeMap.get(token)
+) {
+  if (!rawLicenseExpression) return null
+
+  const licenseVisitor = licenseExpression =>
+    scancodeMap.get(licenseExpression) || SPDX.normalizeSingle(licenseExpression)
+  const parsed = SPDX.parse(rawLicenseExpression, licenseVisitor, licenseRefLookup)
   const result = SPDX.stringify(parsed)
-
-  if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${licenseExpression}`)
+  if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${rawLicenseExpression}`)
 
   return result
 }

package.json

@@ -66,7 +66,7 @@
     "semver": "7.6.0",
     "serialize-error": "^2.1.0",
     "spdx-expression-parse": "github:clearlydefined/spdx-expression-parse.js#fork",
-    "spdx-license-list": "^6.6.0",
+    "spdx-license-list": "^6.9.0",
     "swagger-ui-express": "^4.0.1",
     "throat": "^4.1.0",
     "tiny-attribution-generator": "0.1.2",

providers/summary/scancode/legacy-summarizer.js

@@ -83,7 +83,7 @@ class ScanCodeLegacySummarizer {
 
   _readLicenseExpressionFromSummary(harvested) {
     const licenseExpression = get(harvested, 'content.summary.packages[0].license_expression')
-    const result = licenseExpression && normalizeLicenseExpression(licenseExpression, this.logger)
+    const result = licenseExpression && normalizeLicenseExpression(licenseExpression, this.logger, null)
     return result?.includes('NOASSERTION') ? null : result
   }
 
@@ -196,7 +196,7 @@ class ScanCodeLegacySummarizer {
   _createExpressionFromLicense(license) {
     const rule = license.matched_rule
     if (!rule || !rule.license_expression) return SPDX.normalize(license.spdx_license_key)
-    return normalizeLicenseExpression(rule.license_expression, this.logger)
+    return normalizeLicenseExpression(rule.license_expression, this.logger, null)
   }
 }
 

test/lib/util.js

@@ -880,3 +880,57 @@ describe('Utils buildSourceUrl', () => {
     expect(result).to.eq('https://pypi.org/project/zuul/3.3.0/')
   })
 })
+
+describe('normalizeLicenseExpression', () => {
+  it('should normalize license', () => {
+    const expression = 'MIT AND GPL-3.0'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('MIT AND GPL-3.0')
+  })
+  it('should normalize license to SPDX equivalent', () => {
+    /* 
+      NOTE: If this fails in tests for generated scancode map workflow PR, it is incorrect if it is expecting a LicenseRef.  
+      There is an SPDX valid license which does not require a LicenseRef meaning this test is correct as is.
+    */
+    const expression = 'net-snmp'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('Net-SNMP')
+  })
+  it('should normalize single licenseRef', () => {
+    const expression = 'afpl-9.0'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('LicenseRef-scancode-afpl-9.0')
+  })
+  it('should normalize license and licenseRef', () => {
+    const expression = 'afl-1.1 AND afpl-9.0'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('AFL-1.1 AND LicenseRef-scancode-afpl-9.0')
+  })
+  it('should normalize licenseRef and license', () => {
+    const expression = 'afpl-9.0 AND MIT'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND MIT')
+  })
+  it('should normalize licenseRef and licenseRef', () => {
+    const expression = 'afpl-9.0 AND activestate-community'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community')
+  })
+  it('should normalize licenseRef and licenseRef or licenseRef', () => {
+    const expression = 'afpl-9.0 AND activestate-community OR ac3filter'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq(
+      'LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community OR LicenseRef-scancode-ac3filter'
+    )
+  })
+  it('should normalize INVALID to NOASSERTION', () => {
+    const mockLogger = {
+      info: message => {
+        console.log(message)
+      }
+    }
+    const expression = 'INVALID'
+    const result = utils.normalizeLicenseExpression(expression, mockLogger)
+    expect(result).to.eq('NOASSERTION')
+  })
+})

test/providers/summary/scancode/new-summarizer.js

@@ -58,7 +58,7 @@ describe('ScanCodeNewSummarizer basic compatability', () => {
       const coordinates = { type: 'pypi', provider: 'pypi' }
       const harvestData = getHarvestData(scancodeVersion, 'pypi-complex-declared-license')
       const result = summarizer.summarize(coordinates, harvestData)
-      assert.equal(result.licensed.declared, 'HPND')
+      assert.equal(result.licensed.declared, 'LicenseRef-scancode-secret-labs-2011')
     }
   })