Rapha/dockerization cli

.env.template

@@ -8,4 +8,4 @@ CHAIN_ID=kaon-1
 RPC=https://rpc-eu-1.kaon.kyve.network
 REST=https://api-eu-1.kaon.kyve.network
 DEBUG=false
-VALACCOUNT=<your 12 or 24 mnemonic>
+VALACCOUNT="<your 12 or 24 mnemonic>"

.gitignore

@@ -147,4 +147,4 @@ dist
 # End of https://www.toptal.com/developers/gitignore/api/node
 
 # Ignore docker compose file
-docker-compose.yml
+./docker-compose.yml

README.md

@@ -126,6 +126,28 @@ reviewed and approved your PR it can be merged and used.
 
 **NOTE**: The usage of [Conventional Commits](https://conventionalcommits.org) is required when creating PRs and committing to this repository
 
+### How to write a KYVE integration
+
+**Prerequisites:**
+- Docker
+- yarn (optional)
+
+#### Bootstrap a new integration
+
+Run the following command to start the integration wizard:
+
+```bash
+# with yarn
+yarn bootstrap:integration
+```
+```bash
+# or without yarn
+sh ./tools/kystrap/start.sh
+```
+
+Follow the instructions to create a new integration.
+The wizard will create a new folder in `integrations` with the integration name you provided.
+
 ## How to release
 
 In order to release new changes which got merged into `main` lerna can be used. Lerna will look into every change and create a new release tag if necessary. After the user has approved the new version tags (bumped according to [Semantic Versioning](https://semver.org/)) lerna will push those new tags to `main`, starting the CI/CD pipeline and creating the releases.

package.json

@@ -27,11 +27,13 @@
     "graph": "yarn nx graph",
     "build": "lerna run build",
     "build:binaries": "lerna run build:binaries --concurrency 1",
-    "generate:protos": "bash proto/proto-gen.sh",
+    "generate:protos": "sh proto/proto-gen.sh",
+    "bootstrap:integration": "sh tools/kystrap/start.sh create",
     "pub": "lerna run build && lerna publish",
     "build:changed": "lerna run --since origin/master --include-dependents build",
     "test": "lerna run test",
     "test:unit": "lerna run test:unit",
+    "test:integration": "sh tools/kystrap/start.sh test",
     "fmt": "prettier --write .",
     "clean": "lerna run clean",
     "lint": "lerna run lint",

proto/Dockerfile

@@ -1,10 +1,35 @@
-FROM bufbuild/buf:latest as BUILDER
-FROM node:20-alpine
+###################################################################################
+# Buf.build image that contains buf cli
+###################################################################################
+FROM bufbuild/buf:latest as BUF_BUILDER
 
+###################################################################################
+# Go image that contains go plugins for protoc
+###################################################################################
+FROM golang:1.21.4-alpine as GO_BUILDER
+
+# Install protoc-gen-go for plugin `go` in buf.gen.yaml
+RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0
+
+# Install protoc-gen-go-grpc for plugin `go-grpc` in buf.gen.yaml
+RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0
+
+###################################################################################
+# Python/Nodejs image that contains python/typescript plugins for protoc
+###################################################################################
+FROM nikolaik/python-nodejs:python3.12-nodejs21
+
+# Install ts-proto for plugin `ts_proto` in buf.gen.yaml
 RUN npm install -g ts-proto
 
+# Install betterproto for plugin `python_betterproto` in buf.gen.yaml
+# Use a beta version of betterproto because 2.0.0 is not released yet
+# -> https://github.com/danielgtaylor/python-betterproto
+RUN python -m pip install "betterproto[compiler]==2.0.0b6"
+
 # Create cache directory
 RUN mkdir /.cache
 RUN chmod 777 /.cache
 
-COPY --from=BUILDER /usr/local/bin /usr/local/bin
+COPY --from=BUF_BUILDER /usr/local/bin /usr/local/bin
+COPY --from=GO_BUILDER /go/bin/* /usr/local/bin/

proto/buf.gen.yaml

@@ -1,10 +1,20 @@
 version: v1
 plugins:
   - name: ts_proto
-    out: out
+    out: out/ts
     opt:
       - esModuleInterop=true
       - forceLong=string
       - useOptionals=messages
       - snakeToCamel=false
       - outputServices=grpc-js
+  - name: go
+    out: out/go
+    opt:
+      - paths=source_relative
+  - name: go-grpc
+    out: out/go
+    opt:
+      - paths=source_relative
+  - name: python_betterproto
+    out: out/python

proto/kyverdk/runtime/v1/runtime.proto

@@ -4,6 +4,8 @@ package kyverdk.runtime.v1;
 
 import "kyve/bundles/v1beta1/tx.proto";
 
+option go_package = "github.com/KYVENetwork/kyvejs/kyverdk/runtime/v1";
+
 /*
  * Interface of Runtime.
  *

proto/proto-gen.sh

@@ -4,6 +4,7 @@ cd proto || exit 1
 
 # Variables
 OUTPUT_FOLDER="out"
+DESCRIPTOR_FILE="protobuf.descriptor.bin"
 
 build_docker_image() {
   printf "🏗️ Building docker image...\n"
@@ -51,25 +52,103 @@ run_protobuf_generator() {
   printf "✅ Completed generating proto files!\n\n"
 }
 
-copy_files() {
-  printf "📄 Copy generated files to folders...\n"
+run_protobuf_builder() {
+  printf "🛠️ Building proto descriptor...\n"
+  docker run --rm \
+    --volume "$(pwd)":/workspace \
+    --workdir /workspace \
+    --user "$(id -u):$(id -g)" \
+    kyve-protocol-proto \
+    buf build \
+    -o "$OUTPUT_FOLDER"/"$DESCRIPTOR_FILE" || exit 1
+}
+
+copy_go_files() {
+  printf "  🇬 Go\n"
+
+  # find all go folders in ./integrations by checking if they have a go.mod
+  folders=$(find ../integrations -maxdepth 2 -name go.mod -exec dirname {} \;)
+
+  # find all go folders in ./tools/kystrap/templates by checking if they have a go.mod
+  folders="$folders,$(find ../tools/kystrap/templates -maxdepth 2 -name go.mod -exec dirname {} \;)"
+
+  # remove all old files in the proto folders
+  for folder in $(echo "$folders" | tr "," "\n"); do
+    printf "    🧹 Cleaning %s\n" "$folder/proto/*"
+    rm -rf "$folder"/proto/*
+  done
+
+  # copy the new files to the proto folders
+  for folder in $(echo "$folders" | tr "," "\n"); do
+    printf "    📁 Copying files to %s\n" "$folder/proto/"
+    mkdir -p "$folder/proto"
+    cp -r "$OUTPUT_FOLDER"/go/* "$folder/proto/"
+  done
+}
+
+copy_typescript_files() {
+  printf "  🇹 Typescript\n"
 
-  # find all folders in ./integrations that have a package.json (go just one level deep)
+  # find all typescript folders in ./integrations by checking if they have a package.json
   folders=$(find ../integrations -maxdepth 2 -name package.json -exec dirname {} \;)
 
+  # find all typescript folders in ./tools/kystrap/templates by checking if they have a package.json
+  folders="$folders,$(find ../tools/kystrap/templates -maxdepth 2 -name package.json -exec dirname {} \;)"
+
   # add the common folder
   folders="$folders,../common/protocol"
 
+  # remove all old files in the proto folders
   for folder in $(echo "$folders" | tr "," "\n"); do
-    printf "  🧹 Cleaning %s\n" "$folder/src/proto/*"
-    rm -rf "$folder/src/proto/*"
+    printf "    🧹 Cleaning %s\n" "$folder/src/proto/*"
+    rm -rf "$folder"/src/proto/*
   done
 
+  # copy the new files to the proto folders
   for folder in $(echo "$folders" | tr "," "\n"); do
-    printf "  📁 Copying files to %s\n" "$folder/src/proto/"
+    printf "    📁 Copying files to %s\n" "$folder/src/proto/"
     mkdir -p "$folder/src/proto"
-    cp -r "$OUTPUT_FOLDER"/* "$folder/src/proto/"
+    cp -r "$OUTPUT_FOLDER"/ts/* "$folder/src/proto/"
+  done
+}
+
+copy_python_files() {
+  printf "  🐍 Python\n"
+
+  # find all python folders in ./integrations by checking if they have a requirements.txt
+  folders=$(find ../integrations -maxdepth 2 -name requirements.txt -exec dirname {} \;)
+
+  # find all python folders in ./tools/kystrap/templates by checking if they have a requirements.txt
+  folders="$folders,$(find ../tools/kystrap/templates -maxdepth 2 -name requirements.txt -exec dirname {} \;)"
+
+  # remove all old files in the proto folders
+  for folder in $(echo "$folders" | tr "," "\n"); do
+    printf "    🧹 Cleaning %s\n" "$folder/proto/*"
+    rm -rf "$folder"/proto/*
   done
+
+  # copy the new files to the proto folders
+  for folder in $(echo "$folders" | tr "," "\n"); do
+    printf "    📁 Copying files to %s\n" "$folder/proto/"
+    mkdir -p "$folder/proto"
+    cp -r "$OUTPUT_FOLDER"/python/* "$folder/proto/"
+  done
+}
+
+copy_descriptor_file() {
+  printf "  📄 Descriptor\n"
+  printf "    📁 Copy descriptor to ../tools/kystrap/%s\n" "${DESCRIPTOR_FILE}"
+  cp "$OUTPUT_FOLDER"/"$DESCRIPTOR_FILE" ../tools/kystrap/"$DESCRIPTOR_FILE"
+}
+
+copy_files() {
+  printf "📄 Copy generated files to folders...\n"
+
+  copy_typescript_files
+  copy_go_files
+  copy_python_files
+  copy_descriptor_file
+
   printf "✅ Completed copying files!\n\n"
 }
 
@@ -81,5 +160,6 @@ build_docker_image
 run_protobuf_formatter
 run_protobuf_linter
 run_protobuf_generator
+run_protobuf_builder
 copy_files
 clean_up

tools/kystrap/.dockerignore

@@ -0,0 +1,6 @@
+.idea
+**/out
+
+# Ignore source files not needed to build released binaries
+start.sh
+README.md

tools/kystrap/Dockerfile

@@ -0,0 +1,35 @@
+# Build stage
+FROM golang:1.21-alpine AS build
+
+# Set work directory
+WORKDIR /app
+
+# Copy go mod/sum files and download dependencies
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy source files
+COPY bootstrap ./bootstrap
+COPY cmd ./cmd
+COPY grpcall ./grpcall
+COPY types ./types
+COPY main.go ./
+
+# Build the Go app
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o kystrap .
+
+# Final stage to run the binary
+FROM alpine:latest
+
+# Set work directory
+WORKDIR /app
+
+# Copy the binary from the build stage
+COPY --from=build /app/kystrap .
+
+# Copy the template files
+COPY ./templates /app/templates
+COPY ./template-strings.yml /app/template-strings.yml
+COPY ./protobuf.descriptor.bin /app/protobuf.descriptor.bin
+
+ENTRYPOINT ["./kystrap"]