update

crucial-project · Nov 13, 2020 · 3a8ae65 · 3a8ae65
1 parent cc7a1c0
commit 3a8ae65
Show file tree

Hide file tree

Showing 80 changed files with 632 additions and 839 deletions.
diff --git a/client/pom.xml b/client/pom.xml
@@ -4,14 +4,18 @@
     <parent>
         <groupId>org.crucial</groupId>
         <artifactId>dso</artifactId>
-        <version>1.0</version>
+        <version>2.0</version>
         <relativePath>../</relativePath>
     </parent>
 
     <artifactId>dso-client</artifactId>
     <name>dso-client</name>
     <description>Client side of the DSO datastore.</description>
 
+    <properties>
+        <maven.exec.skip>true</maven.exec.skip>
+    </properties>
+
     <dependencies>
 
         <dependency>
@@ -29,7 +33,6 @@
         <dependency>
             <groupId>info.picocli</groupId>
             <artifactId>picocli</artifactId>
-            <version>4.0.4</version>
         </dependency>
 
         <dependency>
@@ -79,30 +82,6 @@
                     </compilerArgs>
                 </configuration>
             </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>aspectj-maven-plugin</artifactId>
-		<version>${version.aspectj-maven}</version>
-                <configuration>
-                    <aspectLibraries>
-                        <aspectLibrary>
-                            <groupId>${project.groupId}</groupId>
-                            <artifactId>dso-core</artifactId>
-                        </aspectLibrary>
-                    </aspectLibraries>
-                    <complianceLevel>1.8</complianceLevel>
-                    <source>1.8</source>
-                    <target>1.8</target>
-                </configuration>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>compile</goal>       <!-- use this goal to weave all your main classes -->
-                            <goal>test-compile</goal>  <!-- use this goal to weave all your test classes -->
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
             <plugin>
                 <artifactId>maven-dependency-plugin</artifactId>
                 <executions>
@@ -129,6 +108,27 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>1.6.0</version>
+                <configuration>
+                    <skip>${maven.exec.skip}</skip>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>Unit test</id>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <configuration>
+                            <executable>bash</executable>
+                            <commandlineArgs>${basedir}/src/test/bin/unit.sh</commandlineArgs>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 

diff --git a/client/src/main/java/org/crucial/dso/client/Client.java b/client/src/main/java/org/crucial/dso/client/Client.java
@@ -3,6 +3,7 @@
 import org.crucial.dso.*;
 
 import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -20,61 +21,74 @@ private Client(String server) {
         factory = Factory.get(server);
     }
 
+    private Client(String server, long seed) { factory = Factory.get(server, seed); }
+
     /**
      * Return a client or create one if not already present.
      * @param server
+     * @param seed
      * @return
      */
-    public synchronized static Client getClient(String server) {
+    public synchronized static Client getClient(String server, long seed) {
         if (client == null) {
-            client = new Client(server== null ? defaultServer : server);
+            client = new Client(server== null ? defaultServer : server, seed);
         }
         return client;
     }
 
-    public Logger getLog(String key) {
-        return factory.getInstanceOf(Logger.class, key);
+    public synchronized static Client getClient(String server) {
+        return getClient(server, 0);
     }
 
-    public <K, V> Map<K, V> getMap(String key) {
-        return factory.getInstanceOf(Map.class, key);
+    public Logger getLog(String key) {
+        return factory.getInstanceOf(Logger.class, key);
     }
 
-    public <T> ArrayList<T> getArrayList(String key) {
-        return factory.getInstanceOf(ArrayList.class, key);
+    public <V> List<V> getAtomicList(String key) {
+        return factory.getInstanceOf(AtomicList.class, key);
     }
 
-    public <T> ArrayList<T> getArrayList(String key, int initialCapacity) {
-        return factory.getInstanceOf(ArrayList.class, key);
+    public <T> Future<T> getFuture(String key, boolean forceNew) {
+        return forceNew ? factory.getInstanceOf(Future.class, key)
+                : factory.getInstanceOf(Future.class, key, false, false, false);
     }
 
-    public <T> Future<T> getCleanFuture(String key, boolean forceNew) {
-        return forceNew ? factory.getInstanceOf(Future.class, key)
-                : factory.getInstanceOf(Future.class, key, false, false, true);
+    public MonitorCyclicBarrier getMonitorCyclicBarrier(String key, int parties) {
+        return factory.getInstanceOf(MonitorCyclicBarrier.class, key, false, false, false, key, parties);
     }
 
-    public MonitorCyclicBarrier getMonitorCyclicBarrier(String key) {
-        return factory.getInstanceOf(MonitorCyclicBarrier.class, key);
+    public CyclicBarrier getCyclicBarrier(String key, int parties) {
+        AtomicCounter counter = client.getAtomicCounter(key+"-counter",0);
+        AtomicCounter generation = client.getAtomicCounter(key+"-generation",0);
+        return new CyclicBarrier(key, parties, counter, generation);
     }
 
-    public MonitorCyclicBarrier getMonitorCyclicBarrier(String key, int parties) {
-        return factory.getInstanceOf(MonitorCyclicBarrier.class, key, false, false, true);
+    public ScalableCyclicBarrier getScalableCyclicBarrier(String key, int parties) {
+        int logParties = (int)(Math.log(parties)/Math.log(2));
+       AtomicBoolean[][] answers = new AtomicBoolean[parties][logParties];
+        for(int p=0; p<parties; p++) {
+            for(int i=0; i<logParties; i++){
+                answers[p][i] = client.getAtomicBoolean(key+"-"+p+"-"+i,false);
+            }
+        }
+        AtomicCounter identity = client.getAtomicCounter(key+"-identity",-1);
+        return new ScalableCyclicBarrier(key, parties, answers, identity);
     }
 
     public Semaphore getSemaphore(String key) {
         return factory.getInstanceOf(Semaphore.class, key);
     }
 
     public Semaphore getSemaphore(String key, int permits) {
-        return factory.getInstanceOf(Semaphore.class, key, false, false, true, permits);
+        return factory.getInstanceOf(Semaphore.class, key, false, false, false, key, permits);
     }
 
     public AtomicInteger getAtomicInt(String key) {
         return factory.getInstanceOf(AtomicInteger.class, key);
     }
 
     public AtomicInteger getAtomicInt(String key, int initialValue) {
-        return factory.getInstanceOf(AtomicInteger.class, key, false, false, true, initialValue);
+        return factory.getInstanceOf(AtomicInteger.class, key, false, false, false, key, initialValue);
     }
 
     public AtomicLong getAtomicLong(String key) {
@@ -85,13 +99,41 @@ public AtomicByteArray getAtomicByteArray(String key) {
         return factory.getInstanceOf(AtomicByteArray.class, key);
     }
 
-    public AtomicBoolean getBoolean(String key, boolean initialValue) {
-        return factory.getInstanceOf(AtomicBoolean.class, key, false, false, true, initialValue);
+    public AtomicBoolean getAtomicBoolean(String key, boolean initialValue) {
+        return factory.getInstanceOf(AtomicBoolean.class, key, false, false, false, key, initialValue);
     }
 
-    public AtomicBoolean getBoolean(String key) {
+    public AtomicBoolean getAtomicBoolean(String key) {
         return factory.getInstanceOf(AtomicBoolean.class, key);
     }
 
-}
+    public AtomicCounter getAtomicCounter(String key, int initialValue) {
+        return factory.getInstanceOf(AtomicCounter.class, key, false, false, false, key, initialValue);
+    }
 
+    public AtomicMap getAtomicMap(String key) {
+        return factory.getInstanceOf(AtomicMap.class, key);
+    }
+
+    public AtomicMatrix getAtomicMatrix(String key, Class clazz, int n, int m) {
+        return factory.getInstanceOf(AtomicMatrix.class, key, false, false, false, key, clazz, n, m);
+    }
+
+    public Blob getBlob(String key) {
+        return factory.getInstanceOf(Blob.class, key, false, false, false);
+    }
+
+    public CountDownLatch getCountDownLatch(String key, int parties) {
+        AtomicCounter counter = client.getAtomicCounter(key, parties);
+        return new CountDownLatch(key, parties, counter);
+    }
+
+    public void clear() {
+        factory.clear();
+    }
+
+    public void close() {
+        factory.close();
+    }
+
+}
diff --git a/client/src/main/java/org/crucial/dso/client/Interpreter.java b/client/src/main/java/org/crucial/dso/client/Interpreter.java
@@ -21,8 +21,8 @@ public class Interpreter implements Callable<Integer> {
     public static void main(String[] args) {
 
         // 1 - parse
-        Interpreter client = new Interpreter();
-        CommandLine commandLine = new CommandLine(client);
+        Interpreter interpreter = new Interpreter();
+        CommandLine commandLine = new CommandLine(interpreter);
         AtomicCounter counter = newInstance(AtomicCounter.class);
         AtomicList list = newInstance(AtomicList.class);
         AtomicMap map = newInstance(AtomicMap.class);
@@ -35,12 +35,12 @@ public static void main(String[] args) {
         commandLine.parseArgs(args);
 
         // 2 - execute
-        Factory.get(client.server);
-        commandLine = new CommandLine(client);
-        commandLine.addSubcommand("counter",new AtomicCounter(counter.name, counter.count));
-        commandLine.addSubcommand("list",new AtomicList<>(counter.name));
-        commandLine.addSubcommand("map",new AtomicMap<>(map.name));
-        commandLine.addSubcommand("barrier",new CyclicBarrier(barrier.name, barrier.parties));
+        Client client = Client.getClient(interpreter.server);
+        commandLine = new CommandLine(interpreter);
+        commandLine.addSubcommand("counter",client.getAtomicCounter(counter.name, counter.count));
+        commandLine.addSubcommand("list",client.getAtomicList(counter.name));
+        commandLine.addSubcommand("map",client.getAtomicMap(map.name));
+        commandLine.addSubcommand("barrier",client.getCyclicBarrier(barrier.name, barrier.parties));
         commandLine.registerConverter(BiFunction.class, s -> new BiFunctionTypeConverter().convert(s));
         commandLine.execute(args);
 

diff --git a/client/src/test/bin/aliases.sh b/client/src/test/bin/aliases.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PROJDIR=${DIR}/../../..
+
+source ${DIR}/utils.sh
+
+export SERVER=$(config crucial.server)
+export CLASSPATH=${PROJDIR}/target/*:${PROJDIR}/target/lib/*
+
+counter(){
+    java org.crucial.dso.client.Interpreter -s ${SERVER} counter $@
+}
+
+list(){
+    java org.crucial.dso.client.Interpreter -s ${SERVER} list $@
+}
+
+map(){
+    java org.crucial.dso.client.Interpreter -s ${SERVER} map $@
+}
+
+barrier(){
+    java org.crucial.dso.client.Interpreter -s ${SERVER} barrier $@
+}
+
+export -f counter
+export -f list
+export -f map
+export -f barrier
diff --git a/client/src/test/bin/config.properties b/client/src/test/bin/config.properties
@@ -0,0 +1 @@
+crucial.server=localhost:11222
diff --git a/client/src/test/bin/config.properties.tmpl b/client/src/test/bin/config.properties.tmpl
@@ -0,0 +1 @@
+crucial.server=IP:PORT
diff --git a/client/src/test/bin/k8s/gcp/bootstrap.sh b/client/src/test/bin/k8s/gcp/bootstrap.sh
@@ -3,10 +3,12 @@
 DIR=$(dirname "${BASH_SOURCE[0]}")
 
 MACHINE_TYPE=n1-standard-4
-NODE_NUMBER=3
 GCP_PROJECT=$(gcloud config list --format='value(core.project)')
 NETWORK="projects/${GCP_PROJECT}/global/networks/default"
 
+NODE_NUMBER=1
+USER=0track@gmail.com
+
 create_cluster() {
     if [ $# -ne 3 ]; then
         echo "usage: create_cluster cluster_name cluster_zone sleep_time"
@@ -19,9 +21,9 @@ create_cluster() {
     sleep ${seconds}
 
     gcloud container clusters create ${name} \
-	   --disk-size 30\
+	   --disk-size 15\
            --zone ${zone} \
-           --num-nodes ${NODE_NUMBER} \
+	   --num-nodes ${NODE_NUMBER} --enable-autoscaling --min-nodes 0 --max-nodes $((2*NODE_NUMBER)) \
            --machine-type ${MACHINE_TYPE} \
 	   --network ${NETWORK} \
            --preemptible \
@@ -71,6 +73,6 @@ create_cluster ${name} ${zone} ${sleep_time}
 fetch_credentials ${name} ${zone}
 
 # RBAC
-kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user 0track@gmail.com
-kubectl apply -f ${DIR}/../templates/service.yaml
+kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user ${USER}
 kubectl apply -f ${DIR}/../templates/role.yaml
+# kubectl apply -f ${DIR}/../templates/service.yaml
diff --git a/client/src/test/bin/k8s/templates/service.yaml b/client/src/test/bin/k8s/templates/service.yaml
@@ -1,14 +1,12 @@
 kind: Service
 apiVersion: v1
 metadata:
-  name: "helloworldservice"
+  name: "dso-server"
 spec:
   selector:
-    app: "hello-world"
+    app: "dso-server"
   ports:
   - protocol: TCP
-    port: 8080
-    targetPort: 80
-    nodePort: 30001
+    port: 11222
+    targetPort: 11222
   type: LoadBalancer
-
diff --git a/client/src/test/bin/k8s/test.sh b/client/src/test/bin/k8s/test.sh
@@ -21,7 +21,7 @@ THREADS=1
 if [[ "$1" == "-create" ]]
 then    
     k8s_rs_create ${TMPLDIR}/replicaset.yaml.tmpl 1 0 "LAUNCHED"
-    k8s_rs_cp ${TMPLDIR}/replicaset.yaml.tmpl ${DIR}/../../../../target/dso-client-1.0.jar/ /tmp
+    # k8s_rs_cp ${TMPLDIR}/replicaset.yaml.tmpl ${DIR}/../../../../target/dso-client-1.0.jar/ /tmp
     kubectl create -f ${TMPLDIR}/service.yaml
     # kubectl create -f ${TMPLDIR}/autoscaler.yaml
     # kubectl autoscale replicaset dso-server --cpu-percent=50 --min=3 --max=8 # FIXME

diff --git a/client/src/test/bin/local/test.sh b/client/src/test/bin/local/test.sh
@@ -30,18 +30,19 @@ then
     for i in `seq 1 ${NSERVERS}`
     do
 	port=$((11221+i))
-	docker run --net host --rm --env EXTRA="-rf 2" --env CLOUD=local --env PORT=${port} ${IMAGE} 2>&1 > ${TARGETDIR}/${i}.log &
+	docker run --net host --rm \
+	  --env JVM_EXTRA="-XX:+UseG1GC -Xms64m -Xmx1024m -Dlog4j.configuration=log4j-debug.properties" \
+	  --env EXTRA="-rf 2" \
+	  --env CLOUD=local \
+	  --env PORT=${port} \
+	  ${IMAGE} 2>&1 > ${TARGETDIR}/${i}.log &
     done
     up=0
     while [ ${up} != ${NSERVERS} ]; do
 	up=$(cat ${TARGETDIR}/*.log | grep "LAUNCHED" | wc -l)
 	echo -n "."
 	sleep 1
     done
-    for container in $(docker ps | awk '{print $1}' | tail -n ${NSERVERS})
-    do
-    docker cp ${PROJDIR}/target/${CLIENT}-${VERSION}.jar ${container}:/tmp
-    done
     echo " up!"
 elif [[ "$1" == "-delete" ]]
   then