Improve license detection

etc/scripts/licenses/buildrules.py

@@ -204,6 +204,10 @@ def cli(licenses_file):
         relevance = rdata.data.get("relevance")
         rdata.data["has_stored_relevance"] = bool(relevance)
 
+        license_expression = rdata.data.get("license_expression")
+        if license_expression:
+            rdata.data["license_expression"] = license_expression.lower().strip()
+
         minimum_coverage = rdata.data.get("minimum_coverage")
         rdata.data["has_stored_minimum_coverage"] = bool(minimum_coverage)
 
@@ -226,7 +230,7 @@ def cli(licenses_file):
         text = rule.text()
 
         existing_rule = rule_exists(text)
-        skinny_text = " ".join(text[:80].split())
+        skinny_text = " ".join(text[:80].split()).replace("{", " ").replace("}", " ")
 
         existing_msg = (
             f"Skipping rule for: {base_name!r}, "

src/licensedcode/data/licenses/cc-by-3.0-at.yml

@@ -10,5 +10,4 @@ text_urls:
     - https://creativecommons.org/licenses/by/3.0/at/legalcode
 faq_url: https://creativecommons.org/licenses/by/3.0/at/
 ignorable_urls:
-  - https://creativecommons.org/
-
+    - https://creativecommons.org/

src/licensedcode/data/licenses/cc-by-sa-3.0-at.yml

@@ -8,6 +8,5 @@ homepage_url: https://creativecommons.org/licenses/by-sa/3.0/at/legalcode
 spdx_license_key: CC-BY-SA-3.0-AT
 faq_url: https://creativecommons.org/licenses/by-sa/3.0/at/
 ignorable_urls:
-  - https://creativecommons.org/
-  - https://creativecommons.org/compatiblelicenses
-
+    - https://creativecommons.org/
+    - https://creativecommons.org/compatiblelicenses

src/licensedcode/data/licenses/etalab-2.0.yml

@@ -1,7 +1,6 @@
 key: etalab-2.0
-language: en
-short_name: Etalab Open License 2.0 English
-name: Etalab Open License 2.0 English
+short_name: Etalab Open License 2.0
+name: Etalab Open License 2.0
 category: Permissive
 owner: DINUM
 homepage_url: https://raw.githubusercontent.com/DISIC/politique-de-contribution-open-source/master/LICENSE

src/licensedcode/data/licenses/gitlab-ee.LICENSE

@@ -0,0 +1,44 @@
+The GitLab Enterprise Edition (EE) license (the “EE License”)
+Copyright (c) 2011-present GitLab B.V.
+
+With regard to the GitLab Software:
+
+This software and associated documentation files (the "Software") may only be
+used in production, if you (and any entity that you represent) have agreed to,
+and are in compliance with, the GitLab Subscription Terms of Service, available
+at https://about.gitlab.com/terms/#subscription (the “EE Terms”), or other
+agreement governing the use of the Software, as agreed by you and GitLab,
+and otherwise have a valid GitLab Enterprise Edition subscription for the
+correct number of user seats. Subject to the foregoing sentence, you are free to
+modify this Software and publish patches to the Software. You agree that GitLab
+and/or its licensors (as applicable) retain all right, title and interest in and
+to all such modifications and/or patches, and all such modifications and/or
+patches may only be used, copied, modified, displayed, distributed, or otherwise
+exploited with a valid GitLab Enterprise Edition subscription for the  correct
+number of user seats.  Notwithstanding the foregoing, you may copy and modify
+the Software for development and testing purposes, without requiring a
+subscription.  You agree that GitLab and/or its licensors (as applicable) retain
+all right, title and interest in and to all such modifications.  You are not
+granted any other rights beyond what is expressly stated herein.  Subject to the
+foregoing, it is forbidden to copy, merge, publish, distribute, sublicense,
+and/or sell the Software.
+
+This EE License applies only to the part of this Software that is not
+distributed as part of GitLab Community Edition (CE). Any part of this Software
+distributed as part of GitLab CE or is served client-side as an image, font,
+cascading stylesheet (CSS), file which produces or is compiled, arranged,
+augmented, or combined into client-side JavaScript, in whole or in part, is
+copyrighted under the MIT Expat license. The full text of this EE License shall
+be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+For all third party components incorporated into the GitLab Software, those
+components are licensed under the original license provided by the owner of the
+applicable component.

src/licensedcode/data/licenses/gitlab-ee.yml

@@ -0,0 +1,14 @@
+key: gitlab-ee
+short_name: GitLab EE
+name: GitLab Enterprise Edition (EE) license
+category: Commercial
+owner: GitLab.org
+homepage_url: https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/LICENSE
+spdx_license_key: LicenseRef-scancode-gitlab-ee
+faq_url: https://docs.gitlab.com/ee/development/licensing.html
+ignorable_copyrights:
+    - Copyright (c) 2011-present GitLab B.V.
+ignorable_holders:
+    - GitLab B.V.
+ignorable_urls:
+    - https://about.gitlab.com/terms/#subscription

src/licensedcode/data/licenses/jdbm-1.00.LICENSE

@@ -0,0 +1,39 @@
+JDBM LICENSE v1.00
+
+Redistribution and use of this software and associated documentation
+("Software"), with or without modification, are permitted provided
+that the following conditions are met:
+
+1. Redistributions of source code must retain copyright
+   statements and notices.  Redistributions must also contain a
+   copy of this document.
+
+2. Redistributions in binary form must reproduce the
+   above copyright notice, this list of conditions and the
+   following disclaimer in the documentation and/or other
+   materials provided with the distribution.
+
+3. The name "JDBM" must not be used to endorse or promote
+   products derived from this Software without prior written
+   permission of Cees de Groot.  For written permission,
+   please contact cg@cdegroot.com.
+
+4. Products derived from this Software may not be called "JDBM"
+   nor may "JDBM" appear in their names without prior written
+   permission of Cees de Groot.
+
+5. Due credit should be given to the JDBM Project
+   (http://jdbm.sourceforge.net/).
+
+THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS
+``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT
+NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL
+CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/licenses/jdbm-1.00.yml

@@ -0,0 +1,11 @@
+key: jdbm-1.00
+short_name: JDBM License v1.00
+name: JDBM License v1.00
+category: Permissive
+owner: JovalCM
+homepage_url: https://github.com/joval/jdbm/blob/master/LICENSE.txt
+spdx_license_key: LicenseRef-scancode-jdbm-1.00
+ignorable_urls:
+    - http://jdbm.sourceforge.net/
+ignorable_emails:
+    - cg@cdegroot.com