|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "crypto/rand" |
| 6 | + "fmt" |
| 7 | + "net/http" |
| 8 | + _ "net/http/pprof" |
| 9 | + "sort" |
| 10 | + "sync/atomic" |
| 11 | + "time" |
| 12 | + |
| 13 | + "github.com/filecoin-project/boost/cmd/booster-bitswap/bitswap" |
| 14 | + lcli "github.com/filecoin-project/lotus/cli" |
| 15 | + "github.com/ipfs/go-bitswap/client" |
| 16 | + bsnetwork "github.com/ipfs/go-bitswap/network" |
| 17 | + blocks "github.com/ipfs/go-block-format" |
| 18 | + "github.com/ipfs/go-cid" |
| 19 | + nilrouting "github.com/ipfs/go-ipfs-routing/none" |
| 20 | + ipldlegacy "github.com/ipfs/go-ipld-legacy" |
| 21 | + "github.com/ipld/go-car/v2/blockstore" |
| 22 | + "github.com/libp2p/go-libp2p" |
| 23 | + "github.com/libp2p/go-libp2p/core/crypto" |
| 24 | + "github.com/libp2p/go-libp2p/core/network" |
| 25 | + "github.com/libp2p/go-libp2p/core/peer" |
| 26 | + "github.com/libp2p/go-libp2p/p2p/muxer/mplex" |
| 27 | + "github.com/libp2p/go-libp2p/p2p/muxer/yamux" |
| 28 | + quic "github.com/libp2p/go-libp2p/p2p/transport/quic" |
| 29 | + "github.com/libp2p/go-libp2p/p2p/transport/tcp" |
| 30 | + "github.com/urfave/cli/v2" |
| 31 | + "golang.org/x/sync/errgroup" |
| 32 | +) |
| 33 | + |
| 34 | +var fetchCmd = &cli.Command{ |
| 35 | + Name: "fetch", |
| 36 | + Usage: "fetch <multiaddr> <root cid> <output car path>", |
| 37 | + Description: "Fetch all blocks in the DAG under the given root cid from the bitswap node at multiaddr", |
| 38 | + Before: before, |
| 39 | + Flags: []cli.Flag{ |
| 40 | + &cli.BoolFlag{ |
| 41 | + Name: "pprof", |
| 42 | + Usage: "run pprof web server on localhost:6071", |
| 43 | + }, |
| 44 | + &cli.IntFlag{ |
| 45 | + Name: "concurrency", |
| 46 | + Usage: "concurrent request limit - 0 means unlimited", |
| 47 | + Value: 10, |
| 48 | + }, |
| 49 | + }, |
| 50 | + Action: func(cctx *cli.Context) error { |
| 51 | + if cctx.Bool("pprof") { |
| 52 | + go func() { |
| 53 | + err := http.ListenAndServe("localhost:6071", nil) |
| 54 | + if err != nil { |
| 55 | + log.Error(err) |
| 56 | + } |
| 57 | + }() |
| 58 | + } |
| 59 | + |
| 60 | + if cctx.Args().Len() != 3 { |
| 61 | + return fmt.Errorf("usage: fetch <multiaddr> <root cid> <output car path>") |
| 62 | + } |
| 63 | + |
| 64 | + addrInfoStr := cctx.Args().Get(0) |
| 65 | + serverAddrInfo, err := peer.AddrInfoFromString(addrInfoStr) |
| 66 | + if err != nil { |
| 67 | + return fmt.Errorf("parsing server multiaddr %s: %w", addrInfoStr, err) |
| 68 | + } |
| 69 | + |
| 70 | + rootCidStr := cctx.Args().Get(1) |
| 71 | + rootCid, err := cid.Parse(rootCidStr) |
| 72 | + if err != nil { |
| 73 | + return fmt.Errorf("parsing cid %s: %w", rootCidStr, err) |
| 74 | + } |
| 75 | + |
| 76 | + outputCarPath := cctx.Args().Get(2) |
| 77 | + |
| 78 | + ctx := lcli.ReqContext(cctx) |
| 79 | + |
| 80 | + // setup libp2p host |
| 81 | + log.Infow("generating libp2p key") |
| 82 | + privKey, _, err := crypto.GenerateECDSAKeyPair(rand.Reader) |
| 83 | + if err != nil { |
| 84 | + return err |
| 85 | + } |
| 86 | + |
| 87 | + host, err := libp2p.New( |
| 88 | + libp2p.Transport(tcp.NewTCPTransport), |
| 89 | + libp2p.Transport(quic.NewTransport), |
| 90 | + libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport), |
| 91 | + libp2p.Muxer("/yamux/1.0.0", yamux.DefaultTransport), |
| 92 | + libp2p.Identity(privKey), |
| 93 | + libp2p.ResourceManager(network.NullResourceManager), |
| 94 | + ) |
| 95 | + if err != nil { |
| 96 | + return err |
| 97 | + } |
| 98 | + |
| 99 | + // Create a bitswap client |
| 100 | + nilRouter, err := nilrouting.ConstructNilRouting(ctx, nil, nil, nil) |
| 101 | + if err != nil { |
| 102 | + return err |
| 103 | + } |
| 104 | + net := bsnetwork.NewFromIpfsHost(host, nilRouter) |
| 105 | + bs, err := blockstore.OpenReadWrite(outputCarPath, []cid.Cid{rootCid}, blockstore.UseWholeCIDs(true)) |
| 106 | + if err != nil { |
| 107 | + return fmt.Errorf("creating blockstore at %s: %w", outputCarPath, err) |
| 108 | + } |
| 109 | + |
| 110 | + ctx, cancel := context.WithCancel(ctx) |
| 111 | + defer cancel() |
| 112 | + brn := &blockReceiver{bs: bs, ctx: ctx, cancel: cancel} |
| 113 | + bsClient := client.New(ctx, net, bs, client.WithBlockReceivedNotifier(brn)) |
| 114 | + defer bsClient.Close() |
| 115 | + net.Start(bsClient) |
| 116 | + |
| 117 | + // Connect to host |
| 118 | + connectStart := time.Now() |
| 119 | + log.Infow("connecting to server", "server", serverAddrInfo.String()) |
| 120 | + err = host.Connect(ctx, *serverAddrInfo) |
| 121 | + if err != nil { |
| 122 | + return fmt.Errorf("connecting to %s: %w", serverAddrInfo, err) |
| 123 | + } |
| 124 | + log.Debugw("connected to server", "duration", time.Since(connectStart).String()) |
| 125 | + |
| 126 | + // Check host's libp2p protocols |
| 127 | + protos, err := host.Peerstore().GetProtocols(serverAddrInfo.ID) |
| 128 | + if err != nil { |
| 129 | + return fmt.Errorf("getting protocols from peer store for %s: %w", serverAddrInfo.ID, err) |
| 130 | + } |
| 131 | + sort.Slice(protos, func(i, j int) bool { |
| 132 | + return protos[i] < protos[j] |
| 133 | + }) |
| 134 | + log.Debugw("host libp2p protocols", "protocols", protos) |
| 135 | + p, err := host.Peerstore().FirstSupportedProtocol(serverAddrInfo.ID, bitswap.ProtocolStrings...) |
| 136 | + if err != nil { |
| 137 | + return fmt.Errorf("getting first supported protocol from peer store for %s: %w", serverAddrInfo.ID, err) |
| 138 | + } |
| 139 | + if p == "" { |
| 140 | + return fmt.Errorf("host %s does not support any know bitswap protocols: %s", serverAddrInfo.ID, bitswap.ProtocolStrings) |
| 141 | + } |
| 142 | + |
| 143 | + var throttle chan struct{} |
| 144 | + concurrency := cctx.Int("concurrency") |
| 145 | + if concurrency > 0 { |
| 146 | + throttle = make(chan struct{}, concurrency) |
| 147 | + } |
| 148 | + |
| 149 | + // Fetch all blocks under the root cid |
| 150 | + log.Infow("fetch", "cid", rootCid, "concurrency", concurrency) |
| 151 | + start := time.Now() |
| 152 | + count, size, err := getBlocks(ctx, bsClient, rootCid, throttle) |
| 153 | + if err != nil { |
| 154 | + return fmt.Errorf("getting blocks: %w", err) |
| 155 | + } |
| 156 | + |
| 157 | + log.Infow("fetch complete", "count", count, "size", size, "duration", time.Since(start).String()) |
| 158 | + log.Debug("finalizing") |
| 159 | + finalizeStart := time.Now() |
| 160 | + defer func() { log.Infow("finalize complete", "duration", time.Since(finalizeStart).String()) }() |
| 161 | + return bs.Finalize() |
| 162 | + }, |
| 163 | +} |
| 164 | + |
| 165 | +func getBlocks(ctx context.Context, bsClient *client.Client, c cid.Cid, throttle chan struct{}) (uint64, uint64, error) { |
| 166 | + if throttle != nil { |
| 167 | + throttle <- struct{}{} |
| 168 | + } |
| 169 | + // Get the block |
| 170 | + start := time.Now() |
| 171 | + blk, err := bsClient.GetBlock(ctx, c) |
| 172 | + if throttle != nil { |
| 173 | + <-throttle |
| 174 | + } |
| 175 | + if err != nil { |
| 176 | + return 0, 0, err |
| 177 | + } |
| 178 | + |
| 179 | + var size = uint64(len(blk.RawData())) |
| 180 | + log.Debugw("receive", "cid", c, "size", size, "duration", time.Since(start).String()) |
| 181 | + |
| 182 | + // Read the links from the block to child nodes in the DAG |
| 183 | + var count = uint64(1) |
| 184 | + nd, err := ipldlegacy.DecodeNode(ctx, blk) |
| 185 | + if err != nil { |
| 186 | + return 0, 0, fmt.Errorf("decoding node %s: %w", c, err) |
| 187 | + } |
| 188 | + |
| 189 | + var eg errgroup.Group |
| 190 | + lnks := nd.Links() |
| 191 | + for _, l := range lnks { |
| 192 | + l := l |
| 193 | + // Launch a go routine to fetch the blocks underneath each link |
| 194 | + eg.Go(func() error { |
| 195 | + cnt, sz, err := getBlocks(ctx, bsClient, l.Cid, throttle) |
| 196 | + if err != nil { |
| 197 | + return err |
| 198 | + } |
| 199 | + atomic.AddUint64(&count, cnt) |
| 200 | + atomic.AddUint64(&size, sz) |
| 201 | + return nil |
| 202 | + }) |
| 203 | + } |
| 204 | + |
| 205 | + return count, size, eg.Wait() |
| 206 | +} |
| 207 | + |
| 208 | +type blockReceiver struct { |
| 209 | + bs *blockstore.ReadWrite |
| 210 | + ctx context.Context |
| 211 | + cancel context.CancelFunc |
| 212 | +} |
| 213 | + |
| 214 | +func (b blockReceiver) ReceivedBlocks(id peer.ID, blks []blocks.Block) { |
| 215 | + err := b.bs.PutMany(b.ctx, blks) |
| 216 | + if err != nil { |
| 217 | + log.Errorw("failed to write blocks to blockstore: %s", err) |
| 218 | + b.cancel() |
| 219 | + } |
| 220 | +} |
0 commit comments