update java version to 11 (#170)

coderzc · web-flow · commit 18bedd260fea · 2022-01-18T16:58:50.000+08:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -22,7 +22,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           java-version: '8'
-          distribution: 'adopt'
+          distribution: 'zulu'
 
       - name: Setup Python3
         uses: actions/setup-python@v2
@@ -57,9 +57,6 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: Compile
-        run: mvn clean compile -Dmaven.javadoc.skip=true | grep -v "Downloading\|Downloaded"
-
       - name: Prepare env and service
         env:
           TRAVIS_DIR: computer-dist/src/assembly/travis
@@ -69,6 +66,15 @@ jobs:
           $TRAVIS_DIR/install-hugegraph-from-source.sh $COMMIT_ID | grep -v "Downloading\|Downloaded"
           $TRAVIS_DIR/load-data-into-hugegraph.sh
 
+      - name: Install JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'zulu'
+
+      - name: Compile
+        run: mvn clean compile -Dmaven.javadoc.skip=true | grep -v "Downloading\|Downloaded"
+
       - name: Integrate test
         run: mvn test -P integrate-test
 
diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml
@@ -36,11 +36,11 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Install JDK 8
+      - name: Install JDK 11
         uses: actions/setup-java@v2
         with:
-          java-version: '8'
-          distribution: 'adopt'
+          java-version: '11'
+          distribution: 'zulu'
 
       - name: Cache Maven packages
         uses: actions/cache@v2
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -8,10 +8,10 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-      - name: Install JDK 8
+      - name: Install JDK 11
         uses: actions/setup-java@v2
         with:
-          java-version: '8'
+          java-version: '11'
           distribution: 'adopt'
 
       - name: Cache Maven packages
diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/config/ComputerOptions.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/config/ComputerOptions.java
@@ -638,15 +638,15 @@ public static synchronized ComputerOptions instance() {
                     "transport.server_threads",
                     "The number of transport threads for server.",
                     positiveInt(),
-                    TransportConf.NUMBER_CPU_CORES
+                    TransportConf.DEFAULT_THREADS
             );
 
     public static final ConfigOption<Integer> TRANSPORT_CLIENT_THREADS =
             new ConfigOption<>(
                     "transport.client_threads",
                     "The number of transport threads for client.",
                     positiveInt(),
-                    TransportConf.NUMBER_CPU_CORES
+                    TransportConf.DEFAULT_THREADS
             );
 
     public static final ConfigOption<Class<?>> TRANSPORT_PROVIDER_CLASS =
diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/TransportConf.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/TransportConf.java
@@ -35,10 +35,10 @@ public class TransportConf {
                                "transport-netty-server";
     public static final String CLIENT_THREAD_GROUP_NAME =
                                "transport-netty-client";
-    public static final int NUMBER_CPU_CORES =
-                            Runtime.getRuntime().availableProcessors();
     private final Config config;
 
+    public static final int DEFAULT_THREADS = 4;
+
     public static TransportConf wrapConfig(Config config) {
         return new TransportConf(config);
     }
@@ -72,11 +72,7 @@ public int clientThreads() {
     }
 
     private int maxTransportThreads() {
-        Integer workerCount = this.config
-                                  .get(ComputerOptions.JOB_WORKERS_COUNT);
-        Integer partitions = this.config
-                                 .get(ComputerOptions.JOB_PARTITIONS_COUNT);
-        return partitions / workerCount + 1;
+        return this.config.get(ComputerOptions.JOB_WORKERS_COUNT);
     }
 
     public TransportProvider transportProvider() {
diff --git a/computer-dist/Dockerfile b/computer-dist/Dockerfile
@@ -1,6 +1,7 @@
-FROM openjdk:8-jre
+FROM openjdk:11-jre
 LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
-ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=2 -XshowSettings:vm"
+# use ParallelGC which is more friendly to olap system
+ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseParallelGC -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm"
 COPY . /etc/local/hugegraph-computer
 WORKDIR /etc/local/hugegraph-computer
 RUN apt-get update && apt-get -y install gettext-base && apt-get -y install wget
diff --git a/computer-dist/src/assembly/static/bin/start-computer.sh b/computer-dist/src/assembly/static/bin/start-computer.sh
@@ -164,6 +164,17 @@ else
     JAVA="$JAVA_HOME/bin/java -server"
 fi
 
+# Set up count of cpu if it unspecified from k8s drive,
+# avoid `Runtime.getRuntime().availableProcessors()` always return 1
+if [[ "${DRIVE}" = "${K8S_DRIVE}" && -z "${CPU_LIMIT}" ]]; then
+    PROCESSOR_COUNT="$(cat /proc/cpuinfo | grep "processor" | wc -l)"
+    let MAX_PROCESSOR_COUNT=8
+    if [[ ${PROCESSOR_COUNT} -gt ${MAX_PROCESSOR_COUNT} ]]; then
+      PROCESSOR_COUNT="$MAX_PROCESSOR_COUNT"
+    fi
+    JAVA_OPTS="${JAVA_OPTS} -XX:ActiveProcessorCount=${PROCESSOR_COUNT}"
+fi
+
 if [ ! -a "${CONF_DIR}" ];then
     mkdir -p "${CONF_DIR}"
 fi
@@ -185,9 +196,9 @@ fi
 MAIN_CLASS=com.baidu.hugegraph.computer.dist.HugeGraphComputer
 
 if [ "${LOG4j_CONF}" != "" ]; then
-    exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JVM_OPTIONS} \
+    exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JAVA_OPTS} ${JVM_OPTIONS} \
         -cp "${CP}" ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
 else
-    exec ${JAVA} -Dname="hugegraph-computer" ${JVM_OPTIONS} -cp "${CP}" \
+    exec ${JAVA} -Dname="hugegraph-computer" ${JAVA_OPTS} ${JVM_OPTIONS} -cp "${CP}" \
         ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
 fi
diff --git a/computer-k8s-operator/Dockerfile b/computer-k8s-operator/Dockerfile
@@ -1,7 +1,5 @@
-FROM openjdk:8-jre-alpine
+FROM openjdk:11-jre
 LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
-
-ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=2 -XshowSettings:vm"
 WORKDIR /opt/app
 COPY target/hugegraph-computer-operator-*.jar hugegraph-computer-operator.jar
 ENTRYPOINT ["java", "-jar", "hugegraph-computer-operator.jar"]
diff --git a/computer-k8s-operator/README.md b/computer-k8s-operator/README.md
@@ -52,12 +52,14 @@ spec:
   image: hugegraph/hugegraph-builtin-algorithms:latest # algorithm image url
   jarFile: /opt/jars/hugegraph-builtin-algorithms.jar
   pullPolicy: Always
-  workerInstances: 50
+  workerCpu: "4"
+  workerMemory: "4Gi"
+  workerInstances: 5
   computerConf:
+    job.partitions_count: "20"
     algorithm.params_class: com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams
     hugegraph.url: http://${hugegraph-server-host}:${hugegraph-server-port} # hugegraph server url
-    job.partitions_count: "200"
-    hugegraph.name: "hugegraph"
+    hugegraph.name: hugegraph
 EOF
 ```
 
diff --git a/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java b/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java
@@ -85,7 +85,6 @@ public class ComputerJobDeployer {
     private static final String RPC_PORT_NAME = "rpc-port";
     private static final int DEFAULT_TRANSPORT_PORT = 8099;
     private static final int DEFAULT_RPC_PORT = 8090;
-    private static final int DEFAULT_TRANSPORT_THREADS = 8;
     private static final String COMPUTER_CONFIG_MAP_VOLUME =
             "computer-config-map-volume";
 
@@ -198,19 +197,6 @@ private Set<ContainerPort> handleConfig(ComputerJobSpec spec) {
             config.put(ComputerOptions.RPC_SERVER_PORT.name(), rpcPort);
         }
 
-        /*
-        Set a default number of transport threads,
-        if the number of CPU quantity of the worker is not specified
-         */
-        if (spec.getWorkerCpu() == null) {
-            String defaultThreads = String.valueOf(DEFAULT_TRANSPORT_THREADS);
-
-            config.putIfAbsent(ComputerOptions.TRANSPORT_CLIENT_THREADS.name(),
-                               defaultThreads);
-            config.putIfAbsent(ComputerOptions.TRANSPORT_SERVER_THREADS.name(),
-                               defaultThreads);
-        }
-
         ContainerPort transportContainerPort = new ContainerPortBuilder()
                 .withName(TRANSPORT_PORT_NAME)
                 .withContainerPort(Integer.valueOf(transportPort))
@@ -471,6 +457,22 @@ private Container getContainer(String name, ComputerJobSpec spec,
             memory = spec.getWorkerMemory();
         }
 
+        if (cpu != null) {
+            EnvVar cpuLimit = new EnvVarBuilder()
+                              .withName(Constants.ENV_CPU_LIMIT)
+                              .withValue(cpu.toString())
+                              .build();
+            envVars.add(cpuLimit);
+        }
+
+        if (memory != null) {
+            EnvVar memoryLimit = new EnvVarBuilder()
+                                 .withName(Constants.ENV_MEMORY_LIMIT)
+                                 .withValue(memory.toString())
+                                 .build();
+            envVars.add(memoryLimit);
+        }
+
         List<VolumeMount> volumeMounts = spec.getVolumeMounts();
         if (volumeMounts == null) {
             volumeMounts = new ArrayList<>();
diff --git a/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java b/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java
@@ -42,6 +42,8 @@ public class Constants {
     public static final String ENV_JAR_FILE_PATH = "JAR_FILE_PATH";
     public static final String ENV_JOB_JAR_URI = "JOB_JAR_URI";
     public static final String ENV_JVM_OPTIONS = "JVM_OPTIONS";
+    public static final String ENV_CPU_LIMIT = "CPU_LIMIT";
+    public static final String ENV_MEMORY_LIMIT = "MEMORY_LIMIT";
 
     public static final String CONFIG_DIR =  "/opt/hugegraph-computer/conf";
     public static final String COMPUTER_CONF_FILE = "computer.properties";
diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/value/ListValueTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/value/ListValueTest.java
@@ -130,14 +130,10 @@ public void testGet() {
 
         Assert.assertThrows(IndexOutOfBoundsException.class, () -> {
             value1.get(3);
-        }, e -> {
-            Assert.assertContains("Index: 3, Size: 3", e.getMessage());
         });
 
         Assert.assertThrows(IndexOutOfBoundsException.class, () -> {
             value2.get(3);
-        }, e -> {
-            Assert.assertContains("Index: 3, Size: 2", e.getMessage());
         });
     }
 
diff --git a/pom.xml b/pom.xml
@@ -42,8 +42,8 @@
         <top.level.dir>${project.basedir}/..</top.level.dir>
         <release.name>hugegraph-computer</release.name>
         <final.name>${release.name}-${project.version}</final.name>
-        <compiler.source>1.8</compiler.source>
-        <compiler.target>1.8</compiler.target>
+        <compiler.source>11</compiler.source>
+        <compiler.target>11</compiler.target>
         <shell-executable>bash</shell-executable>
         <etcd.version>0.5.4</etcd.version>
         <hadoop-version>3.1.2</hadoop-version>
