Redesign for use as sidecar

.dockerignore

@@ -1,3 +1,8 @@
 *
+# For ./mvnw package -Pnative -Dnative-image.docker-build=true
+#!target/*-runner
+#!target/*-runner.jar
+#!target/lib/*
+# For docker build .
 !src
-!build.gradle
+!pom.xml

.gitignore

@@ -12,3 +12,7 @@ gradlew
 out/
 node_modules
 dist
+.mvn
+target
+.vscode
+kubeconfig-kkv-kind

Dockerfile

@@ -0,0 +1,104 @@
+FROM maven:3.6.1-jdk-8-slim@sha256:dce33cc7a4702cc5f3ea3a6deb4ea840c17001895ffe169d96e1fd9d7041eb15 as maven
+
+FROM openjdk:11.0.3-jdk-slim@sha256:ee1ee5fd0c9cef0ec5ed72999567ed7a6efd5bfdbf49326bfd9423c0dca84ef0 \
+  as dev
+
+COPY --from=maven /usr/share/maven /usr/share/maven
+RUN ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
+ENV MAVEN_HOME=/usr/share/maven
+ENV MAVEN_CONFIG=/root/.m2
+
+WORKDIR /workspace
+RUN mvn io.quarkus:quarkus-maven-plugin:0.15.0:create \
+    -DprojectGroupId=org.acme \
+    -DprojectArtifactId=getting-started \
+    -DclassName="org.acme.quickstart.GreetingResource" \
+    -Dpath="/hello"
+COPY pom.xml .
+RUN mvn package && \
+  rm -r src target mvnw* && \
+  ls -la
+COPY . .
+
+ENTRYPOINT [ "mvn", "compile", "quarkus:dev" ]
+CMD [ "-Dquarkus.http.host=0.0.0.0", "-Dquarkus.http.port=8090" ]
+
+FROM openjdk:11.0.3-jdk-slim@sha256:ee1ee5fd0c9cef0ec5ed72999567ed7a6efd5bfdbf49326bfd9423c0dca84ef0 \
+  as maven-build
+
+COPY --from=maven /usr/share/maven /usr/share/maven
+RUN ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
+ENV MAVEN_HOME=/usr/share/maven
+ENV MAVEN_CONFIG=/root/.m2
+
+COPY --from=dev ${MAVEN_HOME} ${MAVEN_HOME}
+COPY --from=dev ${MAVEN_CONFIG} ${MAVEN_CONFIG}
+
+WORKDIR /workspace
+COPY . .
+# Can't get integration tests to pass on docker hub, and can't get logs from them
+#RUN mvn -o package
+RUN mvn -o package -DskipTests
+
+FROM openjdk:11.0.3-jre-slim@sha256:73c29cc971f328f1456e443f55e4a7ce403638a0429a173549b5be76ef24ab37 \
+  as runtime-plainjava
+ARG SOURCE_COMMIT
+ARG SOURCE_BRANCH
+ARG IMAGE_NAME
+
+WORKDIR /app
+COPY --from=maven-build /workspace/target/lib ./lib
+COPY --from=maven-build /workspace/target/*-runner.jar ./quarkus-kafka.jar
+
+EXPOSE 8090
+ENTRYPOINT [ "java", \
+  "-Dquarkus.http.host=0.0.0.0", \
+  "-Dquarkus.http.port=8090", \
+  "-Djava.util.logging.manager=org.jboss.logmanager.LogManager", \
+  "-cp", "./lib/*", \
+  "-jar", "./quarkus-kafka.jar" ]
+
+ENV SOURCE_COMMIT=${SOURCE_COMMIT} SOURCE_BRANCH=${SOURCE_BRANCH} IMAGE_NAME=${IMAGE_NAME}
+
+# https://github.com/quarkusio/quarkus/issues/2412#issuecomment-494933951
+#FROM oracle/graalvm-ce:19.0.0@sha256:71d4990f47e9b2300c57775e1306af477232019b624376c8f120d910caedb4b4 \
+#  as native-build
+#RUN gu install native-image
+FROM oracle/graalvm-ce:1.0.0-rc16@sha256:aa8b12e0bf15ffec6581f1a9feb42bf4c4f67d7c57d5739cca5fdc6c25fe4c54 \
+  as native-build
+
+WORKDIR /project
+COPY --from=maven-build /workspace/target/lib ./lib
+COPY --from=maven-build /workspace/target/*-runner.jar ./
+
+# from Quarkus' maven plugin mvn package -Pnative -Dnative-image.docker-build=true
+# but CollectionPolicy commented out due to "Error: policy com.oracle.svm.core.genscavenge.CollectionPolicy cannot be instantiated."
+RUN native-image \
+  -J-Djava.util.logging.manager=org.jboss.logmanager.LogManager \
+  #-H:InitialCollectionPolicy=com.oracle.svm.core.genscavenge.CollectionPolicy$BySpaceAndTime \
+  -jar kafka-keyvalue-1.0-SNAPSHOT-runner.jar \
+  -J-Djava.util.concurrent.ForkJoinPool.common.parallelism=1 \
+  -H:FallbackThreshold=0 \
+  -H:+PrintAnalysisCallTree \
+  -H:-AddAllCharsets \
+  -H:EnableURLProtocols=http \
+  -H:-SpawnIsolates \
+  -H:-JNI \
+  --no-server \
+  -H:-UseServiceLoaderFeature \
+  -H:+StackTrace
+
+# The rest should be identical to src/main/docker/Dockerfile which is the recommended quarkus build
+FROM registry.fedoraproject.org/fedora-minimal@sha256:28dcdc19fd1d55598dc308a44c40287f4b00d0bf5a53cd01c39368c16cf85d57
+ARG SOURCE_COMMIT
+ARG SOURCE_BRANCH
+ARG IMAGE_NAME
+
+WORKDIR /work/
+COPY --from=native-build /project/*-runner /work/application
+#RUN chmod 775 /work
+EXPOSE 8090
+ENTRYPOINT ["./application", "-Djava.util.logging.manager=org.jboss.logmanager.LogManager"]
+CMD ["-Dquarkus.http.host=0.0.0.0", "-Dquarkus.http.port=8090"]
+
+ENV SOURCE_COMMIT=${SOURCE_COMMIT} SOURCE_BRANCH=${SOURCE_BRANCH} IMAGE_NAME=${IMAGE_NAME}

README.md

@@ -3,54 +3,25 @@
 
 ## Example usage
 
-See [the Node.js example](./example-nodejs-client/cache-update-flow.spec.js).
+See the `- name: kkv` sidecar in [the example yaml](kontrakt/kkv-example.yaml).
 
-## Design choices
-
- * OnUpdate does not include message value because it could mean that consumers get old values, for example at retries or restarts.
- * Restricted to a single replica per application ID at the moment. Based on [bakdata/kafka-key-value-store](https://medium.com/bakdata/queryable-kafka-topics-with-kafka-streams-8d2cca9de33f) we can add support for patitioned stores later.
+Automated docker builds are available at [docker.io/solsson/kafka-keyvalue](https://hub.docker.com/r/solsson/kafka-keyvalue).
 
 ## Constraints
 
  * Topic keys must be deserializable as [String](https://kafka.apache.org/21/javadoc/org/apache/kafka/common/serialization/Serdes.html#String--) because these strings are used in REST URIs.
 
-## Running
-
-Docker builds are available at [docker.io/kafka/keyvalue](https://hub.docker.com/r/yolean/kafka-keyvalue).
-
-See examples of CLI args in [ArgsToOptionsTest](./src/test/java/se/yolean/kafka/keyvalue/cli/ArgsToOptionsTest.java).
-
-For REST endpoints see `@Path` in [Endpoints](./src/main/java/se/yolean/kafka/keyvalue/Endpoints.java).
-
 ## Development
 
-The [build-contract](https://github.com/Yolean/build-contract/) can be used as dev stack.
+Use [Skaffold](), for example:
 
+```bash
+eval $(minikube docker-env)
+kubectl apply -k github.com/Yolean/kubernetes-kafka/variants/dev-small?ref=v6.0.0
+kubectl apply -f https://github.com/Yolean/kubernetes-kafka/raw/50345f266287861d7964d3339a2c2a28e79db2fe/variants/prometheus-operator-example/k8s-cluster-rbac.yaml
+SKAFFOLD_NO_PRUNE=true skaffold dev
 ```
-alias compose='docker-compose -f build-contracts/docker-compose.yml'
-gradle jibDockerBuild --image=yolean/kafka-keyvalue:dev
-compose up -d cache1
-compose up smoketest
-compose up --build example-nodejs-client
-compose down
-```
-
-Note: `build-contract` (see [build-and-push.sh](./build-and-push.sh)) sometimes fails due to timing issues. Try re-running.
-
-During development of the cache itself or the example nodejs client
-it's more convenient to start only `kafka` and `pixy` through docker.
-
-The main class is `se.yolean.kafka.keyvalue.cli.Main`.
-
-Run the cache service from your IDE with args like: `--port 18081 --streams-props bootstrap.servers=localhost:19092 num.standby.replicas=0 --hostname localhost --topic topic1 --application-id kv-test1-local-001 --onupdate http://127.0.0.1:8081/kafka-keyvalue/v1/updates`
-
-Test manually using for example `echo 'mytest={"t":1}' | kafkacat -b localhost:19092 -P -t topic1 -K '='; curl http://localhost:19081/cache/v1/raw/mytest`.
-
-Run the nodejs locally using: `cd example-nodejs-client; npm ci; ./node_modules/.bin/jest --runInBand --watch `
-(Note that the mock server for unupdate calls only exists during Jest runs)
 
 ## Logging
 
-The distribution bundles log4j2, which we also use as logging API.
-To configure per deployment, provide a file and set `-Dlog4j.configurationFile`.
-For reconfigurability at runtime Kubernets configmaps could be used with [monitorInterval](https://logging.apache.org/log4j/2.x/manual/configuration.html#AutomaticReconfiguration).
+See [Quarkus' logging configuration](https://quarkus.io/guides/logging-guide).

build-and-push.sh

@@ -1,15 +1,11 @@
 #!/bin/sh
 set -e
-# TODO make jib build run the unit tests
 
-# If all tooling is available locally use
-#gradlejibdocker=gradle
+# github.com/Yolean/build-contract local or in-docker
 #contract=build-contract
-gradlejibdocker="docker run --rm -v $(pwd):/workspace -v /var/run/docker.sock:/var/run/docker.sock solsson/gradle-jib-docker@sha256:390f765ba4c8423e30ae1668bfd2e74f026a11b5ec3f0bae23bd36b0ed4c0c75 gradle --no-daemon --no-parallel"
 contract="docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/:/source  --rm --name kafka-keyvalue-build solsson/build-contract@sha256:961624a502c4bf64bdec328e65a911a2096192e7c1a268d7360b9c85ae7a35b8"
 
-$gradlejibdocker --stacktrace test
-$gradlejibdocker --stacktrace jibDockerBuild --image=yolean/kafka-keyvalue:dev -Djib.baseImageCache=build/jib-base-image-cache
+docker build -t yolean/kafka-keyvalue:dev .
 
 $contract test
 

build-contracts/docker-compose.yml

@@ -2,13 +2,13 @@ version: "3.4"
 services:
 
   zookeeper:
-    image: solsson/kafka:2.1.1@sha256:8bc8242c649c395ab79d76cc83b1052e63b4efea7f83547bf11eb3ef5ea6f8e1
+    image: solsson/kafka:2.2.0@sha256:cf048d6211b6b48f1783f97cb41add511386e2f0a5f5c62fa0eee9564dcd3e9a
     entrypoint: ./bin/zookeeper-server-start.sh
     command:
     - ./config/zookeeper.properties
 
   kafka:
-    image: solsson/kafka:2.1.1@sha256:8bc8242c649c395ab79d76cc83b1052e63b4efea7f83547bf11eb3ef5ea6f8e1
+    image: solsson/kafka:2.2.0@sha256:cf048d6211b6b48f1783f97cb41add511386e2f0a5f5c62fa0eee9564dcd3e9a
     depends_on:
     - zookeeper
     entrypoint:
@@ -47,7 +47,7 @@ services:
     -  0.0.0.0:19090
 
   topic1-create:
-    image: solsson/kafka:2.1.1@sha256:8bc8242c649c395ab79d76cc83b1052e63b4efea7f83547bf11eb3ef5ea6f8e1
+    image: solsson/kafka:2.2.0@sha256:cf048d6211b6b48f1783f97cb41add511386e2f0a5f5c62fa0eee9564dcd3e9a
     depends_on:
     - kafka
     entrypoint:
@@ -64,32 +64,34 @@ services:
     - --replication-factor
     -   '1'
 
+  kkv-plainjava:
+    image: yolean/kafka-keyvalue:plainjava
+    build:
+      context: ../
+      target: runtime-plainjava
+    labels:
+    - com.yolean.build-target
+    command:
+    - echo
+    - This was only a build. Exiting.
+
   cache1:
     depends_on:
     - kafka
     - topic1-create
-    image: yolean/kafka-keyvalue:dev
+    image: yolean/kafka-keyvalue:latest
+    build:
+      context: ../
     labels:
     - com.yolean.build-target
     ports:
-    - 19081:19081
-    command:
-    - --standalone
-    - --application-id
-    -   kv-001-
-    - --port
-    -   '19081'
-    - --streams-props
-    -   bootstrap.servers=kafka:9092
-    -   num.standby.replicas=0
-    - --topic
-    -   topic1
-    - --onupdate
-    -   http://example-nodejs-client:8081/kafka-keyvalue/v1/updates
-    -   http://onupdate-logging:8080/
-    -   http://example-nodejs-client:8082/testpost
-    - --starttimeout
-    -   '10'
+    - 19081:8080
+    environment:
+    - kafka_bootstrap=kafka:9092
+    - kafka_group_id=cache1
+    - kafka_offset_reset=latest
+    - topics=topic1
+    #- update_targets=http://example-nodejs-client:8081/kafka-keyvalue/v1/updates,http://onupdate-logging:8080/,http://example-nodejs-client:8082/testpost
 
   onupdate-logging:
     image: solsson/httpstatus-random:access-logging@sha256:5330b2e84457e65ae9552acb00b26e8b7b2a878bcafb3baabddc13c1f840cd90
@@ -139,70 +141,36 @@ services:
     # By design the cache service deals with a single topic, meaning that tests probably can't run concurrently
     - --runInBand
 
-  topic2-create:
-    image: solsson/kafka:2.1.1@sha256:8bc8242c649c395ab79d76cc83b1052e63b4efea7f83547bf11eb3ef5ea6f8e1
-    depends_on:
-    - kafka
-    entrypoint:
-    - ./bin/kafka-topics.sh
+  # compose build && compose up -d k8s-kind && compose logs -f k8s-kind
+  # compose exec k8s-kind cat /root/.kube/kind-config-kkv > ./kubeconfig-kkv-kind
+  # export KUBECONFIG=$(pwd)/kubeconfig-kkv-kind
+  # kubectl apply -f kontrakt/
+  k8s-kind:
+    build:
+      context: ../kontrakt/kind
+    volumes:
+    - /var/run/docker.sock:/var/run/docker.sock:rw
+    network_mode: host
+    environment:
+    - KIND_NAME=kkv
+    - KUBECONFIG=/root/.kube/kind-config-kkv
     command:
-    - --zookeeper
-    -   zookeeper:2181
-    - --topic
-    -   topic2
-    - --create
-    - --if-not-exists
-    - --partitions
-    -   '12'
-    - --replication-factor
-    -   '1'
+    - /bin/sh
+    - -cex
+    - |
+      kind delete cluster --name=$$KIND_NAME || echo "Found no prior cluster"
+      kind create cluster --name=$$KIND_NAME
+      kind get kubeconfig-path --name=$$KIND_NAME
+      kubectl cluster-info
 
-  cache2-01:
-    depends_on:
-    - kafka
-    - topic2-create
-    image: yolean/kafka-keyvalue:dev
-    expose:
-    - '19082'
-    command:
-    - --port
-    -   '19082'
-    - --streams-props
-    -   bootstrap.servers=kafka:9092
-    # Not sure how to do this in docker-compose
-    #-   num.standby.replicas=1
-    - --hostname
-    -   cache2-01
-    - --application-id
-    -   kv-002
-    - --topic
-    -   topic2
-    - --onupdate
-    -   http://onupdate-logging:8080/sharded/01
-    - --starttimeout
-    -   '25'
+      cd kubernetes-kafka
+      kubectl apply -f 00-namespace.yml
+      kubectl apply -f rbac-namespace-default/
+      kubectl apply -k variants/scale-1-ephemeral/
+      kubectl apply -f pixy/
 
-  cache2-02:
-    depends_on:
-    - kafka
-    - topic2-create
-    image: yolean/kafka-keyvalue:dev
-    expose:
-    - '19083'
-    command:
-    - --port
-    -   '19083'
-    - --streams-props
-    -   bootstrap.servers=kafka:9092
-    # Not sure how to do this in docker-compose
-    #-   num.standby.replicas=1
-    - --hostname
-    -   cache2-02
-    - --application-id
-    -   kv-002
-    - --topic
-    -   topic2
-    - --onupdate
-    -   http://onupdate-logging:8080/sharded/02
-    - --starttimeout
-    -   '25'
+      # Load images built by docker-compose (to re-load use compose exec)
+      kind load docker-image yolean/kafka-keyvalue:latest --name=$$KIND_NAME
+      kind load docker-image yolean/kafka-keyvalue:plainjava --name=$$KIND_NAME
+
+      tail -f /dev/null