libboost_context.so not found

apache · Jun 6, 2024 · f7aa0ba · f7aa0ba
1 parent 4d70a5d
commit f7aa0ba
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 5 deletions.
diff --git a/.github/workflows/gluten.yml b/.github/workflows/gluten.yml
@@ -23,6 +23,7 @@ on:
   pull_request:
     branches:
       - master
+  workflow_dispatch:
 
 env:
   MVN_OPT: -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Pjdbc-shaded,gen-policy -Dmaven.plugin.download.cache.path=/tmp/engine-archives
@@ -60,19 +61,33 @@ jobs:
         uses: actions/cache@v3
         with:
           path: gluten/package/target/
-          key: gluten_package_${{ steps.date.outputs.date }}
+          key: gluten_package_${{ steps.date.outputs.date }}_with_thirdparty_lib
       - name: Build gluten project
         run: |
           if [[ "${{ steps.gluten-cache.outputs.cache-hit }}" != 'true' ]]; then
             git clone https://github.com/oap-project/gluten.git
             cd gluten
             ./dev/buildbundle-veloxbe.sh
+            THIRDPARTY_LIB="./package/target/thirdparty-lib"
+            LINUX_OS=$(. /etc/os-release && echo ${ID})
+            VERSION=$(. /etc/os-release && echo ${VERSION_ID})
+            ARCH=`uname -m`
+            mkdir -p $THIRDPARTY_LIB
+            cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libidn.so.12,\
+          libglog.so.0,libgflags.so.2.2,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,\
+          libcurl.so.4,libxml2.so.2,libgsasl.so.7,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,\
+          libldap-2.5.so.0,liblber-2.5.so.0,libntlm.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,\
+          libicudata.so.70,libthrift-0.16.0.so} $THIRDPARTY_LIB/
+            cp /usr/local/lib/{libprotobuf.so.32,libboost_context.so.1.84.0,\
+          libboost_regex.so.1.84.0} $THIRDPARTY_LIB/
+            cd $THIRDPARTY_LIB/
+            jar cvf gluten-thirdparty-lib-$LINUX_OS-$VERSION-$ARCH.jar ./
           fi
       - uses: actions/cache@v3
         if: steps.gluten-cache.outputs.cache-hit != 'true'
         with:
           path: gluten/package/target/
-          key: gluten_package_${{ steps.date.outputs.date }}
+          key: gluten_package_${{ steps.date.outputs.date }}_with_thirdparty_lib
 
   gluten-it:
     name: Gluten Integration TPC-H/DS Test
@@ -105,7 +120,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: gluten/package/target/
-          key: gluten_package_${{ steps.date.outputs.date }}
+          key: gluten_package_${{ steps.date.outputs.date }}_with_thirdparty_lib
       - name: Setup JDK 8
         uses: actions/setup-java@v4
         with:

diff --git a/integration-tests/kyuubi-gluten-it/pom.xml b/integration-tests/kyuubi-gluten-it/pom.xml
@@ -31,7 +31,8 @@
 
     <properties>
         <gluten.version>1.2.0-SNAPSHOT</gluten.version>
-        <gluten.artifact.name>gluten-velox-bundle-spark${spark.binary.version}_${scala.binary.version}-ubuntu_22.04</gluten.artifact.name>
+        <gluten.artifact.name>gluten-velox-bundle-spark${spark.binary.version}_${scala.binary.version}</gluten.artifact.name>
+        <os-maven-plugin.version>1.7.1</os-maven-plugin.version>
     </properties>
 
     <dependencies>
@@ -40,7 +41,15 @@
             <artifactId>${gluten.artifact.name}</artifactId>
             <version>${gluten.version}</version>
             <scope>system</scope>
-            <systemPath>${project.basedir}/../../gluten/package/target/${gluten.artifact.name}_x86_64-${gluten.version}.jar</systemPath>
+            <systemPath>${project.basedir}/../../gluten/package/target/${gluten.artifact.name}-${os.detected.release}_${os.detected.release.version}_${os.detected.arch}-${gluten.version}.jar</systemPath>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.gluten</groupId>
+            <artifactId>gluten-thirdparty-lib</artifactId>
+            <version>${gluten.version}</version>
+            <scope>system</scope>
+            <systemPath>${project.basedir}/../../gluten/package/target/gluten-thirdparty-lib-${os.detected.release}_${os.detected.release.version}_${os.detected.arch}.jar</systemPath>
         </dependency>
 
         <dependency>
@@ -96,11 +105,26 @@
             <artifactId>spark-hive_${scala.binary.version}</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <!-- javax.servlet-api is required by Spark 3.x -->
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>javax.servlet-api</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
         <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
         <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+        <extensions>
+            <!-- provides os.detected.classifier (i.e. linux-x86_64, osx-x86_64) property -->
+            <extension>
+                <groupId>kr.motd.maven</groupId>
+                <artifactId>os-maven-plugin</artifactId>
+                <version>${os-maven-plugin.version}</version>
+            </extension>
+        </extensions>
     </build>
 
     <profiles>