fix: make devnet work for lid

.gitignore

@@ -1,12 +1,14 @@
 /boost
 /boostx
 /boostd
+/boostd-data
 /devnet
 /booster-http
 /booster-bitswap
 /docgen-md
 /docgen-openrpc
 extern/filecoin-ffi/rust/target
+extern/boostd-data/boostd-data
 **/*.a
 **/*.pc
 /**/*/.DS_STORE

Makefile

@@ -93,6 +93,12 @@ boost: $(BUILD_DEPS)
 .PHONY: boost
 BINS+=boost boostx boostd
 
+boostd-data:
+	$(MAKE) -C ./extern/boostd-data
+	install -C ./extern/boostd-data/boostd-data ./boostd-data
+.PHONY: boostd-data
+BINS+=boostd-data
+
 booster-http: $(BUILD_DEPS)
 	rm -f booster-http
 	$(GOCC) build $(GOFLAGS) -o booster-http ./cmd/booster-http
@@ -141,6 +147,7 @@ install-boost:
 	install -C ./boost /usr/local/bin/boost
 	install -C ./boostd /usr/local/bin/boostd
 	install -C ./boostx /usr/local/bin/boostx
+	install -C ./extern/boostd-data /usr/local/bin/boostd-data
 
 install-devnet:
 	install -C ./devnet /usr/local/bin/devnet

docker/devnet/Dockerfile.source

@@ -72,7 +72,7 @@ COPY --from=react-builder /src/react/build /go/src/react/build
 
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-      make debug booster-http booster-bitswap
+      make debug boostd-data booster-http booster-bitswap
 #########################################################################################
 FROM ubuntu:20.04 as runner
 
@@ -112,6 +112,7 @@ EXPOSE 8080
 COPY --from=builder /go/src/boostd /usr/local/bin/
 COPY --from=builder /go/src/boost /usr/local/bin/
 COPY --from=builder /go/src/boostx /usr/local/bin/
+COPY --from=builder /go/src/boostd-data /usr/local/bin/
 COPY docker/devnet/boost/entrypoint.sh /app/
 COPY docker/devnet/boost/sample/* /app/sample/
 RUN boost -v

docker/devnet/boost/entrypoint.sh

@@ -73,5 +73,10 @@ if [ ! -f $BOOST_PATH/.register.boost ]; then
 	echo Super. DONE! Boostd is now configured and will be started soon
 fi
 
-echo Starting boost in dev mode...
-exec boostd -vv run --nosync=true
+## Override config options
+echo Updating config values
+sed -i 's|ServiceApiInfo = ""|ServiceApiInfo = "http://localhost:8042"|g' $BOOST_PATH/config.toml
+
+echo Starting LID service and boost in dev mode...
+trap 'kill %1' SIGINT
+exec boostd-data run leveldb --addr=0.0.0.0:8042 & boostd -vv run --nosync=true

docker/devnet/booster-bitswap/entrypoint.sh

@@ -3,11 +3,10 @@ set -e
 
 export FULLNODE_API_INFO=`lotus auth api-info --perm=admin | cut -f2 -d=`
 export MINER_API_INFO=`lotus-miner auth api-info --perm=admin | cut -f2 -d=`
-export BOOST_API_INFO=`boostd auth api-info --perm=admin | cut -f2 -d=`
 
 echo $FULLNODE_API_INFO
 echo $MINER_API_INFO
-echo $BOOST_API_INFO
+echo $LID_API_INFO
 echo $BOOSTER_BITSWAP_REPO
 
 if [ ! -f $BOOSTER_BITSWAP_REPO/.init.booster-bitswap ]; then
@@ -21,4 +20,4 @@ fi
 
 echo Starting booster-bitswap...
 export GOLOG_LOG_LEVEL=remote-blockstore=debug,booster=debug,engine=debug,bitswap-server=debug
-exec booster-bitswap --vv run --api-boost=$BOOST_API_INFO --tracing --engine-blockstore-worker-count=64 --engine-task-worker-count=64 --max-outstanding-bytes-per-peer=8388608 --target-message-size=524288 --task-worker-count=64
+exec booster-bitswap --vv run --api-lid=$LID_API_INFO --api-fullnode=$FULLNODE_API_INFO --api-storage=$MINER_API_INFO --tracing --engine-blockstore-worker-count=64 --engine-task-worker-count=64 --max-outstanding-bytes-per-peer=8388608 --target-message-size=524288 --task-worker-count=64

docker/devnet/booster-http/entrypoint.sh

@@ -3,11 +3,10 @@ set -e
 
 export FULLNODE_API_INFO=`lotus auth api-info --perm=admin | cut -f2 -d=`
 export MINER_API_INFO=`lotus-miner auth api-info --perm=admin | cut -f2 -d=`
-export BOOST_API_INFO=`boostd auth api-info --perm=admin | cut -f2 -d=`
 
 echo $FULLNODE_API_INFO
 echo $MINER_API_INFO
-echo $BOOST_API_INFO
+echo $LID_API_INFO
 
 echo Starting booster-http...
-exec booster-http run --api-boost=$BOOST_API_INFO --api-fullnode=$FULLNODE_API_INFO --api-storage=$MINER_API_INFO --tracing
+exec booster-http run --serve-files=true --api-lid=$LID_API_INFO  --api-fullnode=$FULLNODE_API_INFO --api-storage=$MINER_API_INFO --tracing

docker/devnet/docker-compose.yaml

@@ -78,6 +78,7 @@ services:
       - "7777:7777"
     environment:
       - BOOST_PATH=/var/lib/boost
+      - LID_API_INFO=http://boost:8042
       - LOTUS_PATH=/var/lib/lotus
       - LOTUS_MINER_PATH=/var/lib/lotus-miner
     restart: unless-stopped
@@ -98,6 +99,7 @@ services:
     environment:
       - BOOSTER_BITSWAP_REPO=/var/lib/booster-bitswap
       - BOOST_PATH=/var/lib/boost
+      - LID_API_INFO=http://boost:8042
       - LOTUS_PATH=/var/lib/lotus
       - LOTUS_MINER_PATH=/var/lib/lotus-miner
     restart: unless-stopped

extern/boostd-data/cmd/run.go

@@ -3,13 +3,14 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/http"
+
 	"github.com/filecoin-project/boostd-data/couchbase"
 	"github.com/filecoin-project/boostd-data/shared/cliutil"
 	"github.com/filecoin-project/boostd-data/shared/tracing"
 	"github.com/filecoin-project/boostd-data/svc"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
-	"net/http"
 )
 
 var runCmd = &cli.Command{
@@ -21,10 +22,10 @@ var runCmd = &cli.Command{
 }
 
 var runFlags = []cli.Flag{
-	&cli.UintFlag{
-		Name:  "port",
-		Usage: "the port the boostd-data listens on",
-		Value: 8042,
+	&cli.StringFlag{
+		Name:  "addr",
+		Usage: "the address the boostd-data listens on",
+		Value: "localhost:8042",
 	},
 	&cli.BoolFlag{
 		Name:  "pprof",
@@ -130,14 +131,14 @@ func runAction(cctx *cli.Context, dbType string, store *svc.Service) error {
 	}
 
 	// Start the server
-	port := cctx.Int("port")
-	err = store.Start(ctx, port)
+	addr := cctx.String("addr")
+	err = store.Start(ctx, addr)
 	if err != nil {
 		return fmt.Errorf("starting %s store: %w", dbType, err)
 	}
 
-	log.Infof("Started boostd-data %s service on port %d",
-		dbType, port)
+	log.Infof("Started boostd-data %s service on address %s",
+		dbType, addr)
 
 	// Monitor for shutdown.
 	<-ctx.Done()

extern/boostd-data/couchbase/db.go

@@ -478,7 +478,7 @@ func (db *DB) GetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid) (Couc
 // GetOffsetSize gets the offset and size of the multihash in the given piece.
 // Note that recordCount is needed in order to determine which shard the multihash is in.
 func (db *DB) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash, recordCount int) (*model.OffsetSize, error) {
-	ctx, span := tracing.Tracer.Start(ctx, "db.get_offset_size")
+	_, span := tracing.Tracer.Start(ctx, "db.get_offset_size")
 	defer span.End()
 
 	// Get the prefix for the shard that the multihash is in
@@ -1161,7 +1161,7 @@ func (db *DB) RemoveDealForPiece(ctx context.Context, dealId string, pieceCid ci
 		// Remove Metadata if removed deal was last one
 		if len(metadata.Deals) == 0 {
 			if err := db.RemovePieceMetadata(ctx, pieceCid); err != nil {
-				fmt.Errorf("Failed to remove the Metadata after removing the last deal: %w", err)
+				return fmt.Errorf("failed to remove the Metadata after removing the last deal: %w", err)
 			}
 			return nil
 		}

extern/boostd-data/couchbase/service.go

@@ -46,11 +46,11 @@ func (s *Store) Start(ctx context.Context) error {
 func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error {
 	log.Debugw("handle.add-deal-for-piece", "piece-cid", pieceCid)
 
-	ctx, span := tracing.Tracer.Start(context.Background(), "store.add_deal_for_piece")
+	ctx, span := tracing.Tracer.Start(ctx, "store.add_deal_for_piece")
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.add-deal-for-piece", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.add-deal-for-piece", "took", time.Since(now))
 	}(time.Now())
 
 	return s.db.AddDealForPiece(ctx, pieceCid, dealInfo)
@@ -59,11 +59,11 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo
 func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error {
 	log.Debugw("handle.set-car-size", "piece-cid", pieceCid, "size", size)
 
-	ctx, span := tracing.Tracer.Start(context.Background(), "store.set-car-size")
+	ctx, span := tracing.Tracer.Start(ctx, "store.set-car-size")
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.set-car-size", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.set-car-size", "took", time.Since(now))
 	}(time.Now())
 
 	err := s.db.SetCarSize(ctx, pieceCid, size)
@@ -77,7 +77,7 @@ func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr s
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.mark-piece-index-errored", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.mark-piece-index-errored", "took", time.Since(now))
 	}(time.Now())
 
 	err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr))
@@ -95,7 +95,7 @@ func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Mul
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.get-offset-size", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.get-offset-size", "took", time.Since(now))
 	}(time.Now())
 
 	md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid)
@@ -115,7 +115,7 @@ func (s *Store) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.M
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.get-piece-metadata", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.get-piece-metadata", "took", time.Since(now))
 	}(time.Now())
 
 	md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid)
@@ -134,7 +134,7 @@ func (s *Store) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.De
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.get-piece-deals", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.get-piece-deals", "took", time.Since(now))
 	}(time.Now())
 
 	md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid)
@@ -154,7 +154,7 @@ func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) (
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.pieces-containing-mh", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.pieces-containing-mh", "took", time.Since(now))
 	}(time.Now())
 
 	pcids, err := s.db.GetPieceCidsByMultihash(ctx, m)
@@ -168,7 +168,7 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record,
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.get-index", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.get-index", "took", time.Since(now))
 	}(time.Now())
 
 	md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid)
@@ -203,7 +203,7 @@ func (s *Store) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, er
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.is-complete-index", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.is-complete-index", "took", time.Since(now))
 	}(time.Now())
 
 	md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid)
@@ -255,7 +255,7 @@ func (s *Store) IndexedAt(ctx context.Context, pieceCid cid.Cid) (time.Time, err
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.indexed-at", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.indexed-at", "took", time.Since(now))
 	}(time.Now())
 
 	md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid)
@@ -273,7 +273,7 @@ func (s *Store) ListPieces(ctx context.Context) ([]cid.Cid, error) {
 	defer span.End()
 
 	defer func(now time.Time) {
-		log.Debugw("handled.list-pieces", "took", fmt.Sprintf("%s", time.Since(now)))
+		log.Debugw("handled.list-pieces", "took", time.Since(now))
 	}(time.Now())
 
 	return s.db.ListPieces(ctx)

extern/boostd-data/ldb/db.go

@@ -307,7 +307,7 @@ func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) err
 	return db.SetPieceCidToMetadata(ctx, pieceCid, md)
 }
 
-func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err error) error {
+func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, sourceErr error) error {
 	ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored")
 	defer span.End()
 
@@ -322,7 +322,7 @@ func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err error)
 	}
 
 	md.Error = err.Error()
-	md.ErrorType = fmt.Sprintf("%T", err)
+	md.ErrorType = fmt.Sprintf("%T", sourceErr)
 
 	return db.SetPieceCidToMetadata(ctx, pieceCid, md)
 }
@@ -360,7 +360,7 @@ func (db *DB) AllRecords(ctx context.Context, cursor uint64) ([]model.Record, er
 		kcid := cid.NewCidV1(cid.Raw, m)
 
 		offset, n := binary.Uvarint(r.Value)
-		size, n := binary.Uvarint(r.Value[n:])
+		size, _ := binary.Uvarint(r.Value[n:])
 
 		records = append(records, model.Record{
 			Cid: kcid,
@@ -401,7 +401,7 @@ func (db *DB) GetOffsetSize(ctx context.Context, cursorPrefix string, m multihas
 	}
 
 	offset, n := binary.Uvarint(b)
-	size, n := binary.Uvarint(b[n:])
+	size, _ := binary.Uvarint(b[n:])
 	return &model.OffsetSize{
 		Offset: offset,
 		Size:   size,