Do not create empty URLs for npms #2996

Reference: #2996
Reference: aboutcode-org/scancode.io#448
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/packagedcode/npm.py

@@ -226,8 +226,7 @@ def parse(cls, location):
 
         if not package.download_url:
             # Only add a synthetic download URL if there is none from the dist mapping.
-            dnl_url = npm_download_url(package.namespace, package.name, package.version)
-            package.download_url = dnl_url.strip()
+            package.download_url = npm_download_url(package.namespace, package.name, package.version)
 
         # licenses are a tad special with many different data structures
         lic = package_data.get('license')
@@ -781,12 +780,15 @@ def npm_homepage_url(namespace, name, registry='https://www.npmjs.com/package'):
 
     >>> expected = 'https://yarnpkg.com/en/package/@ang/angular'
     >>> assert npm_homepage_url('@ang', 'angular', 'https://yarnpkg.com/en/package') == expected
+
+    >>> assert not npm_homepage_url(None, None)
     """
-    if namespace:
-        ns_name = f'{namespace}/{name}'
-    else:
-        ns_name = name
-    return f'{registry}/{ns_name}'
+    if name:
+        if namespace:
+            ns_name = f'{namespace}/{name}'
+        else:
+            ns_name = name
+        return f'{registry}/{ns_name}'
 
 
 def npm_download_url(namespace, name, version, registry='https://registry.npmjs.org'):
@@ -803,13 +805,15 @@ def npm_download_url(namespace, name, version, registry='https://registry.npmjs.
 
     >>> expected = 'https://registry.npmjs.org/angular/-/angular-1.6.6.tgz'
     >>> assert npm_download_url(None, 'angular', '1.6.6') == expected
-    """
-    if namespace:
-        ns_name = f'{namespace}/{name}'
 
-    else:
-        ns_name = name
-    return f'{registry}/{ns_name}/-/{name}-{version}.tgz'
+    >>> assert not npm_download_url(None, None, None)
+    """
+    if name and version:
+        if namespace:
+            ns_name = f'{namespace}/{name}'
+        else:
+            ns_name = name
+        return f'{registry}/{ns_name}/-/{name}-{version}.tgz'
 
 
 def npm_api_url(namespace, name, version=None, registry='https://registry.npmjs.org'):
@@ -827,20 +831,25 @@ def npm_api_url(namespace, name, version=None, registry='https://registry.npmjs.
     >>> assert result == 'https://registry.yarnpkg.com/@invisionag%2feslint-config-ivx'
 
     >>> assert npm_api_url(None, 'angular', '1.6.6') == 'https://registry.npmjs.org/angular/1.6.6'
+
+    >>> assert not npm_api_url(None, None, None)
     """
-    version = version or ''
-    if namespace:
-        # this is a legacy wart: older registries used to always encode this /
-        # FIXME: do NOT encode and use plain / instead
-        ns_name = '%2f'.join([namespace, name])
-        # there is no version-specific URL for scoped packages
-        version = ''
-    else:
-        ns_name = name
+    if name:
+        if namespace:
+            # this is a legacy wart: older registries used to always encode this /
+            # FIXME: do NOT encode and use plain / instead
+            ns_name = '%2f'.join([namespace, name])
+            # there is no version-specific URL for scoped packages
+            version = ''
+        else:
+            ns_name = name
+
+        if version:
+            version = f'/{version}'
+        else:
+            version = ''
 
-    if version:
-        version = f'/{version}'
-    return f'{registry}/{ns_name}{version}'
+        return f'{registry}/{ns_name}{version}'
 
 
 def is_scoped_package(name):

tests/packagedcode/data/npm/private-and-yarn/scan.expected.json

@@ -786,7 +786,7 @@
           "parties": [],
           "keywords": [],
           "homepage_url": null,
-          "download_url": "https://registry.npmjs.org/None/-/None-None.tgz",
+          "download_url": null,
           "size": null,
           "sha1": null,
           "md5": null,
@@ -813,9 +813,9 @@
               "resolved_package": {}
             }
           ],
-          "repository_homepage_url": "https://www.npmjs.com/package/None",
-          "repository_download_url": "https://registry.npmjs.org/None/-/None-None.tgz",
-          "api_data_url": "https://registry.npmjs.org/None",
+          "repository_homepage_url": null,
+          "repository_download_url": null,
+          "api_data_url": null,
           "datasource_id": "npm_package_json",
           "purl": null
         }

tests/packagedcode/data/npm/private/package.json

@@ -0,0 +1,14 @@
+{
+  "private": true,
+  "scripts": {
+    "prepare": "lerna run prepare",
+    "rebuild:browser": "theia rebuild:browser",
+    "rebuild:electron": "theia rebuild:electron"
+  },
+  "devDependencies": {
+    "lerna": "3.13.0"
+  },
+  "workspaces": [
+    "ballerina-extension", "browser-app", "electron-app"
+  ]
+}

tests/packagedcode/data/npm/private/scan.expected.json

@@ -0,0 +1,74 @@
+{
+  "dependencies": [
+    {
+      "purl": "pkg:npm/lerna",
+      "extracted_requirement": "3.13.0",
+      "scope": "devDependencies",
+      "is_runtime": false,
+      "is_optional": true,
+      "is_resolved": false,
+      "resolved_package": {},
+      "dependency_uid": "pkg:npm/lerna?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "for_package_uid": null,
+      "datafile_path": "package.json",
+      "datasource_id": "npm_package_json"
+    }
+  ],
+  "packages": [],
+  "files": [
+    {
+      "path": "package.json",
+      "type": "file",
+      "package_data": [
+        {
+          "type": "npm",
+          "namespace": null,
+          "name": null,
+          "version": null,
+          "qualifiers": {},
+          "subpath": null,
+          "primary_language": "JavaScript",
+          "description": null,
+          "release_date": null,
+          "parties": [],
+          "keywords": [],
+          "homepage_url": null,
+          "download_url": null,
+          "size": null,
+          "sha1": null,
+          "md5": null,
+          "sha256": null,
+          "sha512": null,
+          "bug_tracking_url": null,
+          "code_view_url": null,
+          "vcs_url": null,
+          "copyright": null,
+          "license_expression": null,
+          "declared_license": [],
+          "notice_text": null,
+          "source_packages": [],
+          "file_references": [],
+          "extra_data": {},
+          "dependencies": [
+            {
+              "purl": "pkg:npm/lerna",
+              "extracted_requirement": "3.13.0",
+              "scope": "devDependencies",
+              "is_runtime": false,
+              "is_optional": true,
+              "is_resolved": false,
+              "resolved_package": {}
+            }
+          ],
+          "repository_homepage_url": null,
+          "repository_download_url": null,
+          "api_data_url": null,
+          "datasource_id": "npm_package_json",
+          "purl": null
+        }
+      ],
+      "for_packages": [],
+      "scan_errors": []
+    }
+  ]
+}

tests/packagedcode/data/npm/with_name/package.json.expected

@@ -12,7 +12,7 @@
     "parties": [],
     "keywords": [],
     "homepage_url": null,
-    "download_url": "https://registry.npmjs.org/bson/-/bson-None.tgz",
+    "download_url": null,
     "size": null,
     "sha1": null,
     "md5": null,
@@ -35,7 +35,7 @@
     "extra_data": {},
     "dependencies": [],
     "repository_homepage_url": "https://www.npmjs.com/package/bson",
-    "repository_download_url": "https://registry.npmjs.org/bson/-/bson-None.tgz",
+    "repository_download_url": null,
     "api_data_url": "https://registry.npmjs.org/bson",
     "datasource_id": "npm_package_json",
     "purl": "pkg:npm/bson"