Add booster-tester command for load testing piece-store implementations

Makefile

@@ -115,6 +115,11 @@ boostci: $(BUILD_DEPS)
 	$(GOCC) build $(GOFLAGS) -o boostci ./cmd/boostci
 .PHONY: boostci
 
+booster-tester: $(BUILD_DEPS)
+	rm -f booster-tester
+	$(GOCC) build $(GOFLAGS) -o booster-tester ./cmd/booster-tester
+.PHONY: booster-tester
+
 react: check-node-lts
 	npm_config_legacy_peer_deps=yes npm ci --no-audit --prefix react
 	npm run --prefix react build
@@ -244,6 +249,11 @@ docker/booster-bitswap:
 		-t $(docker_user)/booster-bitswap-dev:dev --build-arg BUILD_VERSION=dev \
 		-f docker/devnet/Dockerfile.source --target booster-bitswap-dev .
 .PHONY: docker/booster-bitswap
+docker/booster-tester:
+	DOCKER_BUILDKIT=1 $(docker_build_cmd) \
+		-t $(docker_user)/booster-tester-dev:dev --build-arg BUILD_VERSION=dev \
+		-f docker/devnet/Dockerfile.source --target booster-tester-dev .
+.PHONY: docker/booster-tester
 docker/all: docker/lotus-test docker/boost docker/booster-http docker/booster-bitswap \
-	docker/lotus docker/lotus-miner
+	docker/booster-tester docker/lotus docker/lotus-miner
 .PHONY: docker/all

cmd/booster-tester/generate_cars_cmd.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"sync"
+
+	"github.com/urfave/cli/v2"
+)
+
+var generateCarsCmd = &cli.Command{
+	Name:        "generate-cars",
+	Usage:       "",
+	Description: "Generates CAR files",
+	Flags:       []cli.Flag{},
+	ArgsUsage:   "<file-paths>",
+	Action: func(cctx *cli.Context) error {
+		log.Debugln("args", cctx.Args().Slice())
+
+		if cctx.Args().Len() < 1 {
+			return fmt.Errorf("must provide at least one file path")
+		}
+
+		files := cctx.Args().Slice()
+
+		var wg sync.WaitGroup
+
+		for _, file := range files {
+			wg.Add(1)
+
+			file := file
+
+			go func() {
+				defer wg.Done()
+				runBoostxGenerateCar(file)
+			}()
+		}
+
+		wg.Wait()
+
+		return nil
+	},
+}
+
+func runBoostxGenerateCar(file string) {
+	ext := filepath.Ext(file)
+	car := file[0:len(file)-len(ext)] + ".car"
+
+	cmd := exec.Command("boostx", "generate-car", file, car)
+
+	cmd.Run()
+}

cmd/booster-tester/graphsync_cmd.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"sync"
+	"time"
+
+	"github.com/urfave/cli/v2"
+)
+
+var graphsyncCmd = &cli.Command{
+	Name:        "graphsync",
+	Usage:       "",
+	Description: "Executes numerous graphsync requests to a provider in parallel",
+	Flags:       []cli.Flag{},
+	ArgsUsage:   "<provider> <...cids>",
+	Action: func(cctx *cli.Context) error {
+		log.Debugln("args", cctx.Args().Slice())
+
+		if cctx.Args().Len() < 2 {
+			return fmt.Errorf("must provide a provider ID and at least one CID")
+		}
+
+		args := cctx.Args().Slice()
+		providerId := args[0]
+		cids := args[1:]
+
+		var wg sync.WaitGroup
+
+		for _, cid := range cids {
+			wg.Add(1)
+
+			cid := cid
+
+			go func() {
+				defer wg.Done()
+				runLotusClientRetrieve(providerId, cid)
+			}()
+		}
+
+		wg.Wait()
+
+		return nil
+	},
+}
+
+func runLotusClientRetrieve(providerId string, cid string) {
+	outfile := fmt.Sprintf("load-test-%d", time.Now().Unix())
+	cmd := exec.Command("lotus", "client", "retrieve", "--provider", providerId, cid, outfile)
+	cmd.Run()
+}

cmd/booster-tester/main.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"os"
+	"os/exec"
+
+	"github.com/filecoin-project/boost/build"
+	"github.com/urfave/cli/v2"
+
+	cliutil "github.com/filecoin-project/boost/cli/util"
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var log = logging.Logger("booster-tester")
+
+func main() {
+	app := &cli.App{
+		Name:                 "booster-tester",
+		Usage:                "Various utilities for testing retrievals",
+		EnableBashCompletion: true,
+		Version:              build.UserVersion(),
+		Flags: []cli.Flag{
+			cliutil.FlagVeryVerbose,
+		},
+		Before: before,
+		Commands: []*cli.Command{
+			generateCarsCmd,
+			graphsyncCmd,
+			helloworldCmd,
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func before(cctx *cli.Context) error {
+	_ = logging.SetLogLevel("booster-tester", "INFO")
+
+	if cliutil.IsVeryVerbose {
+		_ = logging.SetLogLevel("booster-tester", "DEBUG")
+	}
+
+	return nil
+}
+
+var helloworldCmd = &cli.Command{
+	Name:        "helloworld",
+	Usage:       "",
+	Description: "Echos \"Hello World!\"",
+	Action: func(cctx *cli.Context) error {
+
+		cmd := StartHelloWorldCommand()
+		cmd.Wait()
+
+		return nil
+	},
+}
+
+func StartHelloWorldCommand() *exec.Cmd {
+	cmd := exec.Command("echo", "Hello World!")
+
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	cmd.Start()
+
+	return cmd
+}

docker/devnet/.env

@@ -5,4 +5,5 @@ BOOST_IMAGE=${DOCKER_USER}/boost-dev:dev
 BOOSTER_HTTP_IMAGE=${DOCKER_USER}/booster-http-dev:dev
 BOOSTER_BITSWAP_IMAGE=${DOCKER_USER}/booster-bitswap-dev:dev
 BOOST_GUI_IMAGE=${DOCKER_USER}/boost-gui:dev
+BOOSTER_TESTER_IMAGE=${DOCKER_USER}/booster-tester-dev:dev
 FIL_PROOFS_PARAMETER_CACHE=${HOME}/.cache/filecoin-proof-parameters

docker/devnet/Dockerfile.source

@@ -44,7 +44,7 @@ COPY --from=react-builder /src/react/build /go/src/react/build
 
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-      make debug booster-http booster-bitswap
+      make debug booster-http booster-bitswap booster-tester
 #########################################################################################
 FROM ubuntu:20.04 as runner
 
@@ -133,3 +133,25 @@ COPY --from=builder /go/src/booster-bitswap /usr/local/bin/
 COPY docker/devnet/booster-bitswap/entrypoint.sh /app/
 
 ENTRYPOINT ["./entrypoint.sh"]
+#########################################################################################
+FROM runner as booster-tester-dev
+
+ARG BUILD_VERSION=0.1
+
+LABEL org.opencontainers.image.version=$BUILD_VERSION \
+      org.opencontainers.image.authors="Boost Dev Team" \
+      name="booster-tester-dev" \
+      maintainer="Boost Dev Team" \
+      vendor="Boost Dev Team" \
+      version=$BUILD_VERSION \
+      release=$BUILD_VERSION \
+      summary="This image is used to test retrievals" \
+      description="This image is used to test retrievals" 
+
+COPY --from=builder /go/src/boost /usr/local/bin/
+COPY --from=builder /go/src/boostx /usr/local/bin/
+COPY --from=builder /go/src/booster-bitswap /usr/local/bin
+COPY --from=builder /go/src/booster-tester /usr/local/bin
+COPY docker/devnet/booster-tester/init.sh /app/
+
+CMD ["/bin/bash"]

docker/devnet/booster-tester/init.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+export FULLNODE_API_INFO=`lotus auth api-info --perm=admin | cut -f2 -d=`
+
+boost init
+
+lotus send --from=$(lotus wallet default) $(boost wallet default) 10
+
+until boostx market-add 1; do printf "\nOops, maybe funds not added yet.\nNeed to wait some time. \n"; sleep 3; done

docker/devnet/docker-compose.yaml

@@ -108,3 +108,18 @@ services:
     logging: *default-logging
     volumes:
       - ./data/sample:/usr/share/nginx/html:ro
+
+  booster-tester:
+    container_name: booster-tester
+    image: ${BOOSTER_TESTER_IMAGE}
+    tty: true
+    environment:
+      - BOOST_PATH=/var/lib/boost
+      - LOTUS_PATH=/var/lib/lotus
+      - LOTUS_MINER_PATH=/var/lib/lotus-miner
+    logging: *default-logging
+    volumes:
+      - ./data/booster-tester:/var/lib/booster-tester
+      - ./data/boost:/var/lib/boost:ro
+      - ./data/lotus:/var/lib/lotus:ro
+      - ./data/lotus-miner:/var/lib/lotus-miner:ro