chore(dist): tar source and binary package for hugegraph with pd-store (

#2594) tar source and binary package for hugegraph with pd-store
apache · Aug 6, 2024 · dc3070f · dc3070f
1 parent 91f5b33
commit dc3070f
Show file tree

Hide file tree

Showing 23 changed files with 185 additions and 88 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -5,8 +5,11 @@
 apache-release.sh export-ignore
 .licenserc.yaml export-ignore
 .editorconfig export-ignore
+hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc.so export-ignore
+hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc_aarch64.so export-ignore
 
 # ignored directory
 .github/ export-ignore
-hugegraph-dist/scripts/ export-ignore
-#assembly/ export-ignore
+.idea/ export-ignore
+install-dist/scripts/ export-ignore
+docker/ export-ignore
diff --git a/.gitignore b/.gitignore
@@ -57,7 +57,7 @@ build/
 *.pyc
 
 # maven ignore
-apache-hugegraph-incubating-*/
+apache-hugegraph-*/
 output/
 *.war
 *.zip

diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile
@@ -30,7 +30,7 @@ RUN mvn package $MAVEN_ARGS -e -B -ntp -DskipTests -Dmaven.javadoc.skip=true &&
 # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13
 FROM openjdk:11-slim
 
-COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-incubating-pd-*/ /hugegraph-pd/
+COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-pd-incubating-*/ /hugegraph-pd/
 LABEL maintainer="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
 
 # TODO: use g1gc or zgc as default

diff --git a/hugegraph-pd/hg-pd-dist/pom.xml b/hugegraph-pd/hg-pd-dist/pom.xml
@@ -31,7 +31,6 @@
 
     <properties>
         <dist.dir>${project.parent.basedir}</dist.dir>
-        <shell-executable>bash</shell-executable>
         <assembly.dir>${project.basedir}/src/assembly</assembly.dir>
         <assembly.descriptor.dir>${assembly.dir}/descriptor</assembly.descriptor.dir>
         <assembly.static.dir>${assembly.dir}/static</assembly.static.dir>

diff --git a/hugegraph-pd/pom.xml b/hugegraph-pd/pom.xml
@@ -43,7 +43,7 @@
 
     <properties>
         <log4j2.version>2.17.0</log4j2.version>
-        <final.name>apache-${release.name}-incubating-pd-${project.version}</final.name>
+        <final.name>apache-${release.name}-pd-incubating-${project.version}</final.name>
     </properties>
 
     <dependencyManagement>

diff --git a/hugegraph-server/hugegraph-dist/pom.xml b/hugegraph-server/hugegraph-dist/pom.xml
@@ -230,45 +230,47 @@
                                 </target>
                             </configuration>
                         </execution>
-                        <execution>
-                            <id>download-keystore</id>
-                            <phase>prepare-package</phase>
-                            <goals>
-                                <goal>run</goal>
-                            </goals>
-                            <configuration>
-                                <target>
-                                    <exec executable="${shell-executable}"
-                                          dir="${project.basedir}"
-                                          failonerror="false">
-                                        <arg line="./download_keystore.sh"/>
-                                    </exec>
-                                </target>
-                            </configuration>
-                        </execution>
-                        <execution>
-                            <id>cp-keystore</id>
-                            <phase>package</phase>
-                            <goals>
-                                <goal>run</goal>
-                            </goals>
-                            <configuration>
-                                <target>
-                                    <exec executable="cp"
-                                          dir="${project.basedir}"
-                                          failonerror="false">
-                                        <arg value="hugegraph-server.keystore"/>
-                                        <arg value="../${final.name}/conf/hugegraph-server.keystore"/>
-                                    </exec>
-                                    <exec executable="rm"
-                                          dir="${project.basedir}"
-                                          failonerror="false">
-                                        <arg value="-rf"/>
-                                        <arg value="hugegraph-server.keystore"/>
-                                    </exec>
-                                </target>
-                            </configuration>
-                        </execution>
+                        <!-- NOTE: Considering that the hugegraph-server.keystore does not declare a license,
+                        it is not proactively downloaded during the build, only downloaded at runtime. -->
+<!--                        <execution>-->
+<!--                            <id>download-keystore</id>-->
+<!--                            <phase>prepare-package</phase>-->
+<!--                            <goals>-->
+<!--                                <goal>run</goal>-->
+<!--                            </goals>-->
+<!--                            <configuration>-->
+<!--                                <target>-->
+<!--                                    <exec executable="${shell-executable}"-->
+<!--                                          dir="${project.basedir}"-->
+<!--                                          failonerror="false">-->
+<!--                                        <arg line="./download_keystore.sh"/>-->
+<!--                                    </exec>-->
+<!--                                </target>-->
+<!--                            </configuration>-->
+<!--                        </execution>-->
+<!--                        <execution>-->
+<!--                            <id>cp-keystore</id>-->
+<!--                            <phase>package</phase>-->
+<!--                            <goals>-->
+<!--                                <goal>run</goal>-->
+<!--                            </goals>-->
+<!--                            <configuration>-->
+<!--                                <target>-->
+<!--                                    <exec executable="cp"-->
+<!--                                          dir="${project.basedir}"-->
+<!--                                          failonerror="false">-->
+<!--                                        <arg value="hugegraph-server.keystore"/>-->
+<!--                                        <arg value="../${final.name}/conf/ "/>-->
+<!--                                    </exec>-->
+<!--                                    <exec executable="rm"-->
+<!--                                          dir="${project.basedir}"-->
+<!--                                          failonerror="false">-->
+<!--                                        <arg value="-rf"/>-->
+<!--                                        <arg value="hugegraph-server.keystore"/>-->
+<!--                                    </exec>-->
+<!--                                </target>-->
+<!--                            </configuration>-->
+<!--                        </execution>-->
                     </executions>
                 </plugin>
             </plugins>

diff --git a/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml b/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml
@@ -46,17 +46,6 @@
                 <include>*.jar</include>
             </includes>
         </fileSet>
-        <fileSet>
-            <directory>${top.level.dir}/hugegraph-dist/release-docs/</directory>
-            <outputDirectory>/</outputDirectory>
-        </fileSet>
-        <fileSet>
-            <directory>${top.level.dir}/../</directory>
-            <outputDirectory>/</outputDirectory>
-            <includes>
-                <include>DISCLAIMER*</include>
-            </includes>
-        </fileSet>
     </fileSets>
 
     <dependencySets>

diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh
@@ -61,14 +61,7 @@ ensure_path_writable "$PLUGINS"
 # The maximum and minimum heap memory that service can use
 MAX_MEM=$((32 * 1024))
 MIN_MEM=$((1 * 512))
-# TODO: upgrade to Java 11 in 1.5.0
-MIN_JAVA_VERSION=8
-
-# Note: Download for HTTPS, could comment out if you don't need it
-# TODO: only download it when we config https (check the conf file)
-if [[ ! -e "${CONF}/hugegraph-server.keystore" ]]; then
-    download "${CONF}" "${GITHUB}/apache/hugegraph-doc/raw/binary-1.0/dist/server/hugegraph-server.keystore"
-fi
+MIN_JAVA_VERSION=11
 
 # Add the slf4j-log4j12 binding
 CP=$(find -L $LIB -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':')

diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh
@@ -80,6 +80,11 @@ REST_SERVER_URL=$(read_property "$CONF/rest-server.properties" "restserver.url")
 check_port "$GREMLIN_SERVER_URL"
 check_port "$REST_SERVER_URL"
 
+# Note: Only download hugegraph-server.keystore when we config https (check the conf file)
+if [[ $REST_SERVER_URL == https* && ! -e "${CONF}/hugegraph-server.keystore" ]]; then
+    download "${CONF}" "https://github.com/apache/hugegraph-doc/raw/binary-1.5/dist/server/hugegraph-server.keystore"
+fi
+
 if [ ! -d "$LOGS" ]; then
     mkdir -p "$LOGS"
 fi

diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh
@@ -23,7 +23,7 @@ REPORT_FILE=$REPORT_DIR/jacoco-api-test.xml
 
 TRAVIS_DIR=$(dirname $0)
 VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
-SERVER_DIR=hugegraph-server/apache-hugegraph-incubating-server-$VERSION
+SERVER_DIR=hugegraph-server/apache-hugegraph-server-incubating-$VERSION
 RAFT1_DIR=hugegraph-raft1
 RAFT2_DIR=hugegraph-raft2
 RAFT3_DIR=hugegraph-raft3

diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh
@@ -23,7 +23,7 @@ REPORT_FILE=$REPORT_DIR/jacoco-api-test-for-raft.xml
 
 TRAVIS_DIR=$(dirname $0)
 VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
-SERVER_DIR=hugegraph-server/apache-hugegraph-incubating-server-$VERSION/
+SERVER_DIR=hugegraph-server/apache-hugegraph-server-incubating-$VERSION/
 CONF=$SERVER_DIR/conf/graphs/hugegraph.properties
 REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties
 GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml

diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh
@@ -18,7 +18,7 @@
 set -ev
 
 HOME_DIR=$(pwd)
-PD_DIR=$HOME_DIR/hugegraph-pd/apache-hugegraph-incubating-pd-1.5.0
+PD_DIR=$HOME_DIR/hugegraph-pd/apache-hugegraph-pd-incubating-1.5.0
 
 pushd $PD_DIR
 . bin/start-hugegraph-pd.sh

diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh
@@ -18,7 +18,7 @@
 set -ev
 
 HOME_DIR=$(pwd)
-STORE_DIR=$HOME_DIR/hugegraph-store/apache-hugegraph-incubating-store-1.5.0
+STORE_DIR=$HOME_DIR/hugegraph-store/apache-hugegraph-store-incubating-1.5.0
 
 pushd $STORE_DIR
 . bin/start-hugegraph-store.sh

diff --git a/hugegraph-server/pom.xml b/hugegraph-server/pom.xml
@@ -37,7 +37,7 @@
 
     <properties>
         <top.level.dir>${project.basedir}/..</top.level.dir>
-        <final.name>apache-${release.name}-incubating-server-${project.version}</final.name>
+        <final.name>apache-${release.name}-server-incubating-${project.version}</final.name>
         <slf4j.version>1.7.5</slf4j.version>
         <log4j.version>1.2.17</log4j.version>
         <log4j2.version>2.17.1</log4j2.version>
@@ -49,7 +49,6 @@
         <jersey.version>3.0.3</jersey.version>
         <metrics.version>4.2.4</metrics.version>
         <javassist.version>3.21.0-GA</javassist.version>
-        <shell-executable>bash</shell-executable>
         <checkstyle.plugin.version>3.1.2</checkstyle.plugin.version>
         <checkstyle.version>8.45</checkstyle.version>
         <grpc.version>1.47.0</grpc.version>

diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile
@@ -30,7 +30,7 @@ RUN mvn package $MAVEN_ARGS -e -B -ntp -DskipTests -Dmaven.javadoc.skip=true &&
 # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13
 FROM openjdk:11-slim
 
-COPY --from=build /pkg/hugegraph-store/apache-hugegraph-incubating-store-*/ /hugegraph-store/
+COPY --from=build /pkg/hugegraph-store/apache-hugegraph-store-incubating-*/ /hugegraph-store/
 LABEL maintainer="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
 
 # TODO: use g1gc or zgc as default

diff --git a/hugegraph-store/hg-store-dist/pom.xml b/hugegraph-store/hg-store-dist/pom.xml
@@ -32,7 +32,6 @@
 
     <properties>
         <dist.dir>${project.parent.basedir}</dist.dir>
-        <shell-executable>bash</shell-executable>
         <assembly.dir>${project.basedir}/src/assembly</assembly.dir>
         <assembly.descriptor.dir>${assembly.dir}/descriptor</assembly.descriptor.dir>
         <assembly.static.dir>${assembly.dir}/static</assembly.static.dir>

diff --git a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml
@@ -32,6 +32,9 @@
             <includes>
                 <include>*</include>
             </includes>
+            <excludes>
+                <exclude>*.so</exclude>
+            </excludes>
             <fileMode>755</fileMode>
         </fileSet>
         <fileSet>

diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh
@@ -36,14 +36,33 @@ LOGS="$TOP/logs"
 OUTPUT=${LOGS}/hugegraph-store-server.log
 GITHUB="https://github.com"
 PID_FILE="$BIN/pid"
-arch=$(arch)
 
-echo "Current arch: ", "${arch}"
-#if [[ $arch =~ "aarch64" ]];then
-#	  export LD_PRELOAD="$TOP/bin/libjemalloc_aarch64.so"
-#else
-export LD_PRELOAD="$TOP/bin/libjemalloc.so"
-#fi
+. "$BIN"/util.sh
+
+arch=$(uname -m)
+echo "Current arch: $arch"
+
+if [[ $arch == "aarch64" || $arch == "arm64" ]]; then
+    lib_file="$TOP/bin/libjemalloc_aarch64.so"
+    download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so"
+    expected_md5="2a631d2f81837f9d5864586761c5e380"
+    if download_and_verify $download_url $lib_file $expected_md5; then
+        export LD_PRELOAD=$lib_file
+    else
+        echo "Failed to verify or download $lib_file, skip it"
+    fi
+elif [[ $arch == "x86_64" ]]; then
+    lib_file="$TOP/bin/libjemalloc.so"
+    download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so"
+    expected_md5="fd61765eec3bfea961b646c269f298df"
+    if download_and_verify $download_url $lib_file $expected_md5; then
+        export LD_PRELOAD=$lib_file
+    else
+        echo "Failed to verify or download $lib_file, skip it"
+    fi
+else
+    echo "Unsupported architecture: $arch"
+fi
 
 ##pd/store max user processes, ulimit -u
 # Reduce the maximum number of processes that can be opened by a normal dev/user
@@ -92,11 +111,6 @@ while getopts "g:j:y:" arg; do
     esac
 done
 
-
-
-
-. "$BIN"/util.sh
-
 ensure_path_writable "$LOGS"
 ensure_path_writable "$PLUGINS"
 

diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh
@@ -282,6 +282,36 @@ function download() {
     fi
 }
 
+download_and_verify() {
+    local url=$1
+    local filepath=$2
+    local expected_md5=$3
+
+    if [[ -f $filepath ]]; then
+        echo "File $filepath exists. Verifying MD5 checksum..."
+        actual_md5=$(md5sum $filepath | awk '{ print $1 }')
+        if [[ $actual_md5 != $expected_md5 ]]; then
+            echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5"
+            echo "Deleting $filepath..."
+            rm -f $filepath
+        else
+            echo "MD5 checksum verification succeeded for $filepath."
+            return 0
+        fi
+    fi
+
+    echo "Downloading $filepath..."
+    curl -L -o $filepath $url
+
+    actual_md5=$(md5sum $filepath | awk '{ print $1 }')
+    if [[ $actual_md5 != $expected_md5 ]]; then
+        echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5"
+        return 1
+    fi
+
+    return 0
+}
+
 function ensure_package_exist() {
     local path=$1
     local dir=$2

diff --git a/hugegraph-store/pom.xml b/hugegraph-store/pom.xml
@@ -45,7 +45,7 @@
 
     <properties>
         <log4j2.version>2.15.0</log4j2.version>
-        <final.name>apache-${release.name}-incubating-store-${project.version}</final.name>
+        <final.name>apache-${release.name}-store-incubating-${project.version}</final.name>
     </properties>
 
     <dependencyManagement>
-Original file line number
+Diff line change
@@ Expand Up / @@ -57,7 +57,7 @@ build/ @@
     *.pyc
     # maven ignore
-    apache-hugegraph-incubating-*/
+    apache-hugegraph-*/
     output/
     *.war
     *.zip
@@ Expand Down @@