Fix RAT check

.github/workflows/dev.yml

@@ -16,15 +16,11 @@
 # under the License.
 
 name: Dev
-
-on:
-  # always trigger
-  push:
-  pull_request:
+on: [push, pull_request]
 
 jobs:
   lint:
-    name: Lint C++, Python, R, Rust, Docker, RAT
+    name: Lint C++, Python, R, Rust, Docker
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Arrow
@@ -42,6 +38,19 @@ jobs:
       - name: Lint
         run: archery lint --rat
 
+  rat:
+    name: Release Audit Tool (RAT)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Audit licenses
+        run: ./dev/release/run-rat.sh .
+
   prettier:
     name: Use prettier to check formatting of documents
     runs-on: ubuntu-latest

dev/release/check-rat-report.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python
+##############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##############################################################################
+import fnmatch
+import re
+import sys
+import xml.etree.ElementTree as ET
+
+if len(sys.argv) != 3:
+    sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" %
+                     sys.argv[0])
+    sys.exit(1)
+
+exclude_globs_filename = sys.argv[1]
+xml_filename = sys.argv[2]
+
+globs = [line.strip() for line in open(exclude_globs_filename, "r")]
+
+tree = ET.parse(xml_filename)
+root = tree.getroot()
+resources = root.findall('resource')
+
+all_ok = True
+for r in resources:
+    approvals = r.findall('license-approval')
+    if not approvals or approvals[0].attrib['name'] == 'true':
+        continue
+    clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
+    excluded = False
+    for g in globs:
+        if fnmatch.fnmatch(clean_name, g):
+            excluded = True
+            break
+    if not excluded:
+        sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % (
+            clean_name, r.attrib['name'], approvals[0].attrib['name']))
+        all_ok = False
+
+if not all_ok:
+    sys.exit(1)
+
+print('OK')
+sys.exit(0)

dev/release/rat_exclude_files.txt

@@ -106,3 +106,19 @@ ballista/rust/scheduler/testdata/*
 ballista/ui/scheduler/yarn.lock
 python/rust-toolchain
 python/requirements*.txt
+**/testdata/*
+benchmarks/queries/*
+benchmarks/data/*
+ci/*
+**/*.svg
+**/*.csv
+**/*.json
+venv/*
+testing/*
+target/*
+**/target/*
+Cargo.lock
+**/Cargo.lock
+.history
+parquet-testing/*
+*rat.txt

dev/release/run-rat.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+RAT_VERSION=0.13
+
+# download apache rat
+if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then
+  curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar > apache-rat-${RAT_VERSION}.jar
+fi
+
+RAT="java -jar apache-rat-${RAT_VERSION}.jar -x "
+
+RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+
+# generate the rat report
+$RAT $1 > rat.txt
+python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt rat.txt > filtered_rat.txt
+cat filtered_rat.txt
+UNAPPROVED=`cat filtered_rat.txt  | grep "NOT APPROVED" | wc -l`
+
+if [ "0" -eq "${UNAPPROVED}" ]; then
+  echo "No unapproved licenses"
+else
+  echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
+  exit 1
+fi