Upgrade Transport for Trino v406 (linkedin#132)

* Upgrade Gradle from 6.7 to 7.5 (linkedin#122) * Upgrade to Gradle 7.5 * Upgrade to Gradle 7.5 * remove a unnecessary classpath * upgrade gradle wrapper in transportable-udfs-examples * address comments * add comment * using api() to avoid adding dependencies * Simplify dependencies further --------- Co-authored-by: Yiqiang Ding <yiqding@linkedin.com> Co-authored-by: Jiefan Li <jiefli@linkedin.com> * Upgrade Transport to compile Trino UDF with Java 17 (linkedin#125) Co-authored-by: Yiqiang Ding <yiqding@linkedin.com> * Upgrade Transport to use API from Trino v406 (linkedin#128) * Upgrade Transport to use API from Trino v406 * Upgrade Transport to use API from Trino v406 * PoC of Trino Connector * PoC of Trino Connector * address comments and remove unreachable artifactory repo * address comments * fix the issue of Maven repo * fixing conjar repo issue in transportable-udfs-examples * address comments * address comments * address comments * address comments * address comments * address comments * address comments * add some comments * remove conjar repo * address comments --------- Co-authored-by: Yiqiang Ding <yiqding@linkedin.com> * Improvements on transport-trino-plugin (linkedin#133) * Improvments on transportable-udfs-trino-plugin * Improvments on transportable-udfs-trino-plugin * fix a type * address comments * address comments --------- Co-authored-by: Yiqiang Ding <yiqding@linkedin.com> --------- Co-authored-by: Yiqiang Ding <yiqding@linkedin.com> Co-authored-by: Jiefan Li <jiefli@linkedin.com>
WeiWan5675 · Apr 28, 2023 · 7478dd5 · 7478dd5
1 parent 37c4cf2
commit 7478dd5
Show file tree

Hide file tree

Showing 67 changed files with 1,430 additions and 431 deletions.
diff --git a/build.gradle b/build.gradle
@@ -12,8 +12,7 @@ buildscript {
   }
   dependencies {
     classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
-    classpath 'org.github.ngbinh.scalastyle:gradle-scalastyle-plugin_2.11:1.0.1'
-    classpath 'gradle.plugin.nl.javadude.gradle.plugins:license-gradle-plugin:0.14.0'
+    classpath 'com.github.alisiikh:gradle-scalastyle-plugin:3.4.1'
     classpath "io.github.gradle-nexus:publish-plugin:1.0.0"
     classpath "org.shipkit:shipkit-auto-version:1.1.1"
     classpath "org.shipkit:shipkit-changelog:1.1.10"
@@ -22,6 +21,7 @@ buildscript {
 
 plugins {
   id "checkstyle"
+  id "com.github.hierynomus.license" version "0.16.1"
 }
 
 apply from: "gradle/shipkit.gradle"
@@ -69,8 +69,8 @@ subprojects {
     project.apply(plugin: 'checkstyle')
 
     dependencies {
-      testCompile 'org.testng:testng:6.11'
-      testCompile 'org.slf4j:slf4j-simple:1.7.25'
+      testImplementation 'org.testng:testng:6.11'
+      testImplementation 'org.slf4j:slf4j-simple:1.7.25'
     }
 
     test {
@@ -86,16 +86,4 @@ subprojects {
       toolVersion '8.23'
     }
   }
-
-  plugins.withType(ScalaPlugin) {
-    project.apply(plugin: 'scalaStyle')
-
-    scalaStyle {
-      configLocation = "${rootDir}/gradle/scalastyle/scalastyle-config.xml"
-      source = sourceSets.main.allScala
-      testSource = sourceSets.test.allScala
-      includeTestSourceDirectory = true
-    }
-    check.dependsOn scalaStyle
-  }
 }
diff --git a/defaultEnvironment.gradle b/defaultEnvironment.gradle
@@ -6,12 +6,9 @@ subprojects {
   repositories {
     mavenCentral()
     jcenter()
-    maven {
-      url "https://conjars.org/repo"
-    }
   }
-  project.ext.setProperty('trino-version', '352')
-  project.ext.setProperty('airlift-slice-version', '0.39')
+  project.ext.setProperty('trino-version', '406')
+  project.ext.setProperty('airlift-slice-version', '0.44')
   project.ext.setProperty('spark-group', 'org.apache.spark')
   project.ext.setProperty('spark2-version', '2.3.0')
   project.ext.setProperty('spark3-version', '3.1.1')

diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
@@ -1,5 +1,21 @@
 #!/usr/bin/env sh
 
+#
+# Copyright 2015 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 ##############################################################################
 ##
 ##  Gradle start up script for UN*X
@@ -28,7 +44,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m"'
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
@@ -66,6 +82,7 @@ esac
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
+
 # Determine the Java command to use to start the JVM.
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
@@ -109,10 +126,11 @@ if $darwin; then
     GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
 fi
 
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
     JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
@@ -138,19 +156,19 @@ if $cygwin ; then
         else
             eval `echo args$i`="\"$arg\""
         fi
-        i=$((i+1))
+        i=`expr $i + 1`
     done
     case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+        0) set -- ;;
+        1) set -- "$args0" ;;
+        2) set -- "$args0" "$args1" ;;
+        3) set -- "$args0" "$args1" "$args2" ;;
+        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
     esac
 fi
 
@@ -159,14 +177,9 @@ save () {
     for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
     echo " "
 }
-APP_ARGS=$(save "$@")
+APP_ARGS=`save "$@"`
 
 # Collect all arguments for the java command, following the shell quoting and substitution rules
 eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
-# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
-  cd "$(dirname "$0")"
-fi
-
 exec "$JAVACMD" "$@"
diff --git a/gradlew.bat b/gradlew.bat
@@ -1,3 +1,19 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
 @if "%DEBUG%" == "" @echo off
 @rem ##########################################################################
 @rem
@@ -13,15 +29,18 @@ if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS="-Xmx64m"
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
 
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
+if "%ERRORLEVEL%" == "0" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -35,7 +54,7 @@ goto fail
 set JAVA_HOME=%JAVA_HOME:"=%
 set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
-if exist "%JAVA_EXE%" goto init
+if exist "%JAVA_EXE%" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
@@ -45,28 +64,14 @@ echo location of your Java installation.
 
 goto fail
 
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
 :execute
 @rem Setup the command line
 
 set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
+
 @rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
 
 :end
 @rem End local scope for the variables with windows NT shell

diff --git a/settings.gradle b/settings.gradle
@@ -15,6 +15,7 @@ def modules = [
     'transportable-udfs-spark_2.11',
     'transportable-udfs-spark_2.12',
     'transportable-udfs-trino',
+    'transportable-udfs-trino-plugin',
     'transportable-udfs-test:transportable-udfs-test-api',
     'transportable-udfs-test:transportable-udfs-test-generic',
     'transportable-udfs-test:transportable-udfs-test-hive',

diff --git a/transportable-udfs-annotation-processor/build.gradle b/transportable-udfs-annotation-processor/build.gradle
@@ -1,12 +1,12 @@
 apply plugin: 'java'
 
 dependencies {
-  compile project(':transportable-udfs-api')
-  compile project(':transportable-udfs-compile-utils')
-  compile('com.google.guava:guava:24.1-jre')
-  compile('com.google.code.gson:gson:2.8.5')
+  implementation project(':transportable-udfs-api')
+  implementation project(':transportable-udfs-compile-utils')
+  implementation('com.google.guava:guava:24.1-jre')
+  implementation('com.google.code.gson:gson:2.8.5')
 
-  testCompile('com.google.testing.compile:compile-testing:0.15')
+  testImplementation('com.google.testing.compile:compile-testing:0.15')
 }
 
 license {

diff --git a/transportable-udfs-avro/build.gradle b/transportable-udfs-avro/build.gradle
@@ -1,10 +1,10 @@
 apply plugin: 'java'
 
 dependencies {
-  compile project(':transportable-udfs-api')
-  compile project(':transportable-udfs-type-system')
-  compile('org.apache.avro:avro:1.10.2')
-  testCompile project(path: ':transportable-udfs-type-system', configuration: 'tests')
+  implementation project(':transportable-udfs-api')
+  implementation project(':transportable-udfs-type-system')
+  implementation('org.apache.avro:avro:1.10.2')
+  testImplementation project(path: ':transportable-udfs-type-system', configuration: 'tests')
 }
 
 task jarTests(type: Jar, dependsOn: testClasses) {
@@ -14,7 +14,7 @@ task jarTests(type: Jar, dependsOn: testClasses) {
 
 configurations {
   tests {
-    extendsFrom testRuntime
+    extendsFrom testRuntimeOnly
   }
 }
 

diff --git a/transportable-udfs-codegen/build.gradle b/transportable-udfs-codegen/build.gradle
@@ -1,13 +1,12 @@
-apply plugin: 'java'
-
+apply plugin: 'java-library'
 
 dependencies {
-  compile project(':transportable-udfs-api')
-  compile project(':transportable-udfs-compile-utils')
-  compile ('com.google.guava:guava:24.1-jre')
-  compile ('org.apache.commons:commons-io:1.3.2')
-  compile ('org.apache.commons:commons-text:1.6')
-  compile ('com.squareup:javapoet:1.11.1')
+  api project(':transportable-udfs-compile-utils')
+  implementation project(':transportable-udfs-api')
+  implementation 'com.google.guava:guava:24.1-jre'
+  implementation 'org.apache.commons:commons-io:1.3.2'
+  implementation 'org.apache.commons:commons-text:1.6'
+  implementation 'com.squareup:javapoet:1.11.1'
 }
 
 licenseTest {

diff --git a/...able-udfs-codegen/src/main/java/com/linkedin/transport/codegen/TrinoWrapperGenerator.java b/...able-udfs-codegen/src/main/java/com/linkedin/transport/codegen/TrinoWrapperGenerator.java
@@ -25,7 +25,7 @@ public class TrinoWrapperGenerator implements WrapperGenerator {
   private static final String GET_STD_UDF_METHOD = "getStdUDF";
   private static final ClassName TRINO_STD_UDF_WRAPPER_CLASS_NAME =
       ClassName.bestGuess("com.linkedin.transport.trino.StdUdfWrapper");
-  private static final String SERVICE_FILE = "META-INF/services/io.trino.metadata.SqlScalarFunction";
+  private static final String SERVICE_FILE = "META-INF/services/com.linkedin.transport.trino.StdUdfWrapper";
 
   @Override
   public void generateWrappers(WrapperGeneratorContext context) {

diff --git a/...vices/io.trino.metadata.SqlScalarFunction → ...om.linkedin.transport.trino.StdUdfWrapper b/...vices/io.trino.metadata.SqlScalarFunction → ...om.linkedin.transport.trino.StdUdfWrapper
diff --git a/transportable-udfs-compile-utils/build.gradle b/transportable-udfs-compile-utils/build.gradle
@@ -2,6 +2,6 @@ apply plugin: 'java'
 
 
 dependencies {
-  compile('com.google.guava:guava:24.1-jre')
-  compile('com.google.code.gson:gson:2.8.5')
+  implementation('com.google.guava:guava:24.1-jre')
+  implementation('com.google.code.gson:gson:2.8.5')
 }
diff --git a/transportable-udfs-examples/build.gradle b/transportable-udfs-examples/build.gradle
@@ -10,10 +10,12 @@ buildscript {
   }
   dependencies {
     classpath 'org.github.ngbinh.scalastyle:gradle-scalastyle-plugin_2.11:1.0.1'
-    classpath 'gradle.plugin.nl.javadude.gradle.plugins:license-gradle-plugin:0.14.0'
   }
 }
 
+plugins {
+  id "com.github.hierynomus.license" version "0.16.1"
+}
 
 allprojects {
   group = 'com.linkedin.transport'
@@ -29,9 +31,6 @@ subprojects {
   }
   repositories {
     mavenCentral()
-    maven {
-      url "https://conjars.org/repo"
-    }
   }
 }
 
@@ -50,7 +49,7 @@ subprojects {
     project.apply(plugin: 'checkstyle')
 
     dependencies {
-      testCompile 'org.testng:testng:6.11'
+      testImplementation 'org.testng:testng:6.11'
     }
 
     test {

diff --git a/transportable-udfs-examples/gradle/wrapper/gradle-wrapper.jar b/transportable-udfs-examples/gradle/wrapper/gradle-wrapper.jar
diff --git a/transportable-udfs-examples/gradle/wrapper/gradle-wrapper.properties b/transportable-udfs-examples/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,5 @@
-#Wed Apr 10 22:28:49 PDT 2019
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-all.zip