From dc3070f45bdeefcda061f1876a0dfcb8aacd71e3 Mon Sep 17 00:00:00 2001 From: V_Galaxy Date: Tue, 6 Aug 2024 21:21:14 +0800 Subject: [PATCH] chore(dist): tar source and binary package for hugegraph with pd-store (#2594) tar source and binary package for hugegraph with pd-store --- .gitattributes | 7 +- .gitignore | 2 +- hugegraph-pd/Dockerfile | 2 +- hugegraph-pd/hg-pd-dist/pom.xml | 1 - hugegraph-pd/pom.xml | 2 +- hugegraph-server/hugegraph-dist/pom.xml | 80 ++++++++++--------- .../src/assembly/descriptor/assembly.xml | 11 --- .../assembly/static/bin/hugegraph-server.sh | 9 +-- .../assembly/static/bin/start-hugegraph.sh | 5 ++ .../assembly/travis/run-api-test-for-raft.sh | 2 +- .../src/assembly/travis/run-api-test.sh | 2 +- .../src/assembly/travis/start-pd.sh | 2 +- .../src/assembly/travis/start-store.sh | 2 +- hugegraph-server/pom.xml | 3 +- hugegraph-store/Dockerfile | 2 +- hugegraph-store/hg-store-dist/pom.xml | 1 - .../assembly/descriptor/server-assembly.xml | 3 + .../static/bin/start-hugegraph-store.sh | 38 ++++++--- .../src/assembly/static/bin/util.sh | 30 +++++++ hugegraph-store/pom.xml | 2 +- install-dist/pom.xml | 62 ++++++++++++++ install-dist/scripts/apache-release.sh | 4 +- pom.xml | 1 + 23 files changed, 185 insertions(+), 88 deletions(-) diff --git a/.gitattributes b/.gitattributes index 636f55aa01..ca5e57db70 100755 --- a/.gitattributes +++ b/.gitattributes @@ -5,8 +5,11 @@ apache-release.sh export-ignore .licenserc.yaml export-ignore .editorconfig export-ignore +hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc.so export-ignore +hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc_aarch64.so export-ignore # ignored directory .github/ export-ignore -hugegraph-dist/scripts/ export-ignore -#assembly/ export-ignore +.idea/ export-ignore +install-dist/scripts/ export-ignore +docker/ export-ignore diff --git a/.gitignore b/.gitignore index bf4993e030..c8c2e1eb55 100644 --- a/.gitignore +++ b/.gitignore @@ -57,7 +57,7 @@ build/ *.pyc # maven ignore -apache-hugegraph-incubating-*/ +apache-hugegraph-*/ output/ *.war *.zip diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index a86b5b9bd4..e507900ade 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -30,7 +30,7 @@ RUN mvn package $MAVEN_ARGS -e -B -ntp -DskipTests -Dmaven.javadoc.skip=true && # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM openjdk:11-slim -COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-incubating-pd-*/ /hugegraph-pd/ +COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-pd-incubating-*/ /hugegraph-pd/ LABEL maintainer="HugeGraph Docker Maintainers " # TODO: use g1gc or zgc as default diff --git a/hugegraph-pd/hg-pd-dist/pom.xml b/hugegraph-pd/hg-pd-dist/pom.xml index a764bdd817..251ec8bcb7 100644 --- a/hugegraph-pd/hg-pd-dist/pom.xml +++ b/hugegraph-pd/hg-pd-dist/pom.xml @@ -31,7 +31,6 @@ ${project.parent.basedir} - bash ${project.basedir}/src/assembly ${assembly.dir}/descriptor ${assembly.dir}/static diff --git a/hugegraph-pd/pom.xml b/hugegraph-pd/pom.xml index 34f25b5d8b..b2547a7dc4 100644 --- a/hugegraph-pd/pom.xml +++ b/hugegraph-pd/pom.xml @@ -43,7 +43,7 @@ 2.17.0 - apache-${release.name}-incubating-pd-${project.version} + apache-${release.name}-pd-incubating-${project.version} diff --git a/hugegraph-server/hugegraph-dist/pom.xml b/hugegraph-server/hugegraph-dist/pom.xml index ce4c00b225..7f4dd454c9 100644 --- a/hugegraph-server/hugegraph-dist/pom.xml +++ b/hugegraph-server/hugegraph-dist/pom.xml @@ -230,45 +230,47 @@ - - download-keystore - prepare-package - - run - - - - - - - - - - - cp-keystore - package - - run - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml b/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml index b1c293dfb1..b7f9ee0487 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml +++ b/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml @@ -46,17 +46,6 @@ *.jar - - ${top.level.dir}/hugegraph-dist/release-docs/ - / - - - ${top.level.dir}/../ - / - - DISCLAIMER* - - diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 8174082d49..8b6a16d5ad 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -61,14 +61,7 @@ ensure_path_writable "$PLUGINS" # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) MIN_MEM=$((1 * 512)) -# TODO: upgrade to Java 11 in 1.5.0 -MIN_JAVA_VERSION=8 - -# Note: Download for HTTPS, could comment out if you don't need it -# TODO: only download it when we config https (check the conf file) -if [[ ! -e "${CONF}/hugegraph-server.keystore" ]]; then - download "${CONF}" "${GITHUB}/apache/hugegraph-doc/raw/binary-1.0/dist/server/hugegraph-server.keystore" -fi +MIN_JAVA_VERSION=11 # Add the slf4j-log4j12 binding CP=$(find -L $LIB -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 5e14397f7e..30c2764989 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -80,6 +80,11 @@ REST_SERVER_URL=$(read_property "$CONF/rest-server.properties" "restserver.url") check_port "$GREMLIN_SERVER_URL" check_port "$REST_SERVER_URL" +# Note: Only download hugegraph-server.keystore when we config https (check the conf file) +if [[ $REST_SERVER_URL == https* && ! -e "${CONF}/hugegraph-server.keystore" ]]; then + download "${CONF}" "https://github.com/apache/hugegraph-doc/raw/binary-1.5/dist/server/hugegraph-server.keystore" +fi + if [ ! -d "$LOGS" ]; then mkdir -p "$LOGS" fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh index 85c45b8c47..529874f58b 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh @@ -23,7 +23,7 @@ REPORT_FILE=$REPORT_DIR/jacoco-api-test.xml TRAVIS_DIR=$(dirname $0) VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -SERVER_DIR=hugegraph-server/apache-hugegraph-incubating-server-$VERSION +SERVER_DIR=hugegraph-server/apache-hugegraph-server-incubating-$VERSION RAFT1_DIR=hugegraph-raft1 RAFT2_DIR=hugegraph-raft2 RAFT3_DIR=hugegraph-raft3 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh index 17d4722dc2..8008f39cdb 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh @@ -23,7 +23,7 @@ REPORT_FILE=$REPORT_DIR/jacoco-api-test-for-raft.xml TRAVIS_DIR=$(dirname $0) VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -SERVER_DIR=hugegraph-server/apache-hugegraph-incubating-server-$VERSION/ +SERVER_DIR=hugegraph-server/apache-hugegraph-server-incubating-$VERSION/ CONF=$SERVER_DIR/conf/graphs/hugegraph.properties REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh index c9200fefb3..9f694d5c3c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh @@ -18,7 +18,7 @@ set -ev HOME_DIR=$(pwd) -PD_DIR=$HOME_DIR/hugegraph-pd/apache-hugegraph-incubating-pd-1.5.0 +PD_DIR=$HOME_DIR/hugegraph-pd/apache-hugegraph-pd-incubating-1.5.0 pushd $PD_DIR . bin/start-hugegraph-pd.sh diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh index c4c11c2c35..23e8f2297c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh @@ -18,7 +18,7 @@ set -ev HOME_DIR=$(pwd) -STORE_DIR=$HOME_DIR/hugegraph-store/apache-hugegraph-incubating-store-1.5.0 +STORE_DIR=$HOME_DIR/hugegraph-store/apache-hugegraph-store-incubating-1.5.0 pushd $STORE_DIR . bin/start-hugegraph-store.sh diff --git a/hugegraph-server/pom.xml b/hugegraph-server/pom.xml index 9c03fdefa5..a4dac32ab5 100644 --- a/hugegraph-server/pom.xml +++ b/hugegraph-server/pom.xml @@ -37,7 +37,7 @@ ${project.basedir}/.. - apache-${release.name}-incubating-server-${project.version} + apache-${release.name}-server-incubating-${project.version} 1.7.5 1.2.17 2.17.1 @@ -49,7 +49,6 @@ 3.0.3 4.2.4 3.21.0-GA - bash 3.1.2 8.45 1.47.0 diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index c7329a8f81..042ceef42f 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -30,7 +30,7 @@ RUN mvn package $MAVEN_ARGS -e -B -ntp -DskipTests -Dmaven.javadoc.skip=true && # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM openjdk:11-slim -COPY --from=build /pkg/hugegraph-store/apache-hugegraph-incubating-store-*/ /hugegraph-store/ +COPY --from=build /pkg/hugegraph-store/apache-hugegraph-store-incubating-*/ /hugegraph-store/ LABEL maintainer="HugeGraph Docker Maintainers " # TODO: use g1gc or zgc as default diff --git a/hugegraph-store/hg-store-dist/pom.xml b/hugegraph-store/hg-store-dist/pom.xml index 13df60f0a5..b0988d8aab 100644 --- a/hugegraph-store/hg-store-dist/pom.xml +++ b/hugegraph-store/hg-store-dist/pom.xml @@ -32,7 +32,6 @@ ${project.parent.basedir} - bash ${project.basedir}/src/assembly ${assembly.dir}/descriptor ${assembly.dir}/static diff --git a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml index 3e7a6aeb55..522a93c917 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml +++ b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml @@ -32,6 +32,9 @@ * + + *.so + 755 diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index b12fb354df..b363b7ab45 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -36,14 +36,33 @@ LOGS="$TOP/logs" OUTPUT=${LOGS}/hugegraph-store-server.log GITHUB="https://github.com" PID_FILE="$BIN/pid" -arch=$(arch) -echo "Current arch: ", "${arch}" -#if [[ $arch =~ "aarch64" ]];then -# export LD_PRELOAD="$TOP/bin/libjemalloc_aarch64.so" -#else -export LD_PRELOAD="$TOP/bin/libjemalloc.so" -#fi +. "$BIN"/util.sh + +arch=$(uname -m) +echo "Current arch: $arch" + +if [[ $arch == "aarch64" || $arch == "arm64" ]]; then + lib_file="$TOP/bin/libjemalloc_aarch64.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" + expected_md5="2a631d2f81837f9d5864586761c5e380" + if download_and_verify $download_url $lib_file $expected_md5; then + export LD_PRELOAD=$lib_file + else + echo "Failed to verify or download $lib_file, skip it" + fi +elif [[ $arch == "x86_64" ]]; then + lib_file="$TOP/bin/libjemalloc.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" + expected_md5="fd61765eec3bfea961b646c269f298df" + if download_and_verify $download_url $lib_file $expected_md5; then + export LD_PRELOAD=$lib_file + else + echo "Failed to verify or download $lib_file, skip it" + fi +else + echo "Unsupported architecture: $arch" +fi ##pd/store max user processes, ulimit -u # Reduce the maximum number of processes that can be opened by a normal dev/user @@ -92,11 +111,6 @@ while getopts "g:j:y:" arg; do esac done - - - -. "$BIN"/util.sh - ensure_path_writable "$LOGS" ensure_path_writable "$PLUGINS" diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index 93eae7b890..93b55311cb 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -282,6 +282,36 @@ function download() { fi } +download_and_verify() { + local url=$1 + local filepath=$2 + local expected_md5=$3 + + if [[ -f $filepath ]]; then + echo "File $filepath exists. Verifying MD5 checksum..." + actual_md5=$(md5sum $filepath | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" + echo "Deleting $filepath..." + rm -f $filepath + else + echo "MD5 checksum verification succeeded for $filepath." + return 0 + fi + fi + + echo "Downloading $filepath..." + curl -L -o $filepath $url + + actual_md5=$(md5sum $filepath | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + return 1 + fi + + return 0 +} + function ensure_package_exist() { local path=$1 local dir=$2 diff --git a/hugegraph-store/pom.xml b/hugegraph-store/pom.xml index ebf5f0d2a3..f9cd0bcfb3 100644 --- a/hugegraph-store/pom.xml +++ b/hugegraph-store/pom.xml @@ -45,7 +45,7 @@ 2.15.0 - apache-${release.name}-incubating-store-${project.version} + apache-${release.name}-store-incubating-${project.version} diff --git a/install-dist/pom.xml b/install-dist/pom.xml index d351fcf908..ff4fb92c2f 100644 --- a/install-dist/pom.xml +++ b/install-dist/pom.xml @@ -27,4 +27,66 @@ install-dist + + + apache-${release.name}-incubating-${project.version} + + + + + + maven-antrun-plugin + + + package + + run + + + + + + root_path=$(cd $(dirname $0)/..; pwd) + cd $root_path || exit + + mkdir -p ${final.name} + cp -r -v $root_path/hugegraph-pd/apache-hugegraph-pd-incubating-${project.version} ${final.name}/ || exit + cp -r -v $root_path/hugegraph-store/apache-hugegraph-store-incubating-${project.version} ${final.name}/ || exit + cp -r -v $root_path/hugegraph-server/apache-hugegraph-server-incubating-${project.version} ${final.name}/ || exit + cp -r -v $root_path/install-dist/release-docs/* $root_path/DISCLAIMER ${final.name}/ || exit + + tar zcvf $root_path/target/${final.name}.tar.gz ./${final.name} || exit 1 + + md5sum $root_path/target/${final.name}.tar.gz + echo -n "hugegraph tar.gz available at: " + echo "$root_path/target/${final.name}.tar.gz" + rm -f ${project.basedir}/dist.sh + + + + + + + + + + + maven-clean-plugin + + + + ${project.basedir}/.. + + ${final.name}.tar.gz + + false + + + ${project.basedir}/../${final.name} + + + + + + diff --git a/install-dist/scripts/apache-release.sh b/install-dist/scripts/apache-release.sh index 859a635620..168217c425 100755 --- a/install-dist/scripts/apache-release.sh +++ b/install-dist/scripts/apache-release.sh @@ -47,9 +47,7 @@ cd - || exit # step2: copy the binary file (Optional) # Note: it's optional for project to generate binary package (skip this step if not need) -cp -v ../../hugegraph-server/apache-${REPO}-incubating-server-"${RELEASE_VERSION}".tar.gz \ - dist/apache-${REPO} || exit -# TODO: pd & store +cp -v ../../target/apache-${REPO}-incubating-"${RELEASE_VERSION}".tar.gz dist/apache-${REPO} || exit # step3: sign + hash ##### 3.1 sign in source & binary package diff --git a/pom.xml b/pom.xml index 0f8bc3e228..44be456375 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ UTF-8 11 11 + bash