From 63d0b17abeb303af70191b056903b84ea8c2acef Mon Sep 17 00:00:00 2001 From: simlecode <69969590+simlecode@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:08:04 +0800 Subject: [PATCH 1/3] feat: auto import direct deal --- cli/direct-deal.go | 18 +-- cmd/droplet-client/direct-deal.go | 200 +++++++++++++++++++++++------- docs/zh/direct-on-boarding.md | 48 ++++--- 3 files changed, 197 insertions(+), 69 deletions(-) diff --git a/cli/direct-deal.go b/cli/direct-deal.go index e46188d0..f3c30d5b 100644 --- a/cli/direct-deal.go +++ b/cli/direct-deal.go @@ -270,11 +270,11 @@ var importDirectDealCmd = &cli.Command{ allocationID := cliCtx.Uint64("allocation-id") - startEpoch, err := getStartEpoch(cliCtx, fapi) + startEpoch, err := GetStartEpoch(cliCtx, fapi) if err != nil { return err } - endEpoch, err := checkAndGetEndEpoch(cliCtx.Context, fapi, client, allocationID, startEpoch) + endEpoch, err := CheckAndGetEndEpoch(cliCtx.Context, fapi, client, allocationID, startEpoch) if err != nil { return err } @@ -306,14 +306,14 @@ var importDirectDealCmd = &cli.Command{ }, } -func getStartEpoch(cliCtx *cli.Context, fapi v1api.FullNode) (abi.ChainEpoch, error) { +func GetStartEpoch(cliCtx *cli.Context, fapi v1api.FullNode) (abi.ChainEpoch, error) { startEpoch := abi.ChainEpoch(cliCtx.Int("start-epoch")) if startEpoch == 0 { head, err := fapi.ChainHead(cliCtx.Context) if err != nil { return 0, err } - startEpoch = head.Height() + builtin.EpochsInDay*2 + startEpoch = head.Height() + builtin.EpochsInDay*8 } return startEpoch, nil @@ -349,7 +349,7 @@ var importDirectDealsCmd = &cli.Command{ }, &cli.IntFlag{ Name: "start-epoch", - Usage: "start epoch by when the deal should be proved by provider on-chain (default: 2 days from now)", + Usage: "start epoch by when the deal should be proved by provider on-chain (default: 8 days from now)", }, }, Action: func(cliCtx *cli.Context) error { @@ -386,7 +386,7 @@ var importDirectDealsCmd = &cli.Command{ return "", fmt.Errorf("car %s file not found", pieceCID.String()) } - startEpoch, err := getStartEpoch(cliCtx, fapi) + startEpoch, err := GetStartEpoch(cliCtx, fapi) if err != nil { return err } @@ -411,7 +411,7 @@ var importDirectDealsCmd = &cli.Command{ return fmt.Errorf("invalid client: %w", err) } - endEpoch, err := checkAndGetEndEpoch(ctx, fapi, client, allocationID, startEpoch) + endEpoch, err := CheckAndGetEndEpoch(ctx, fapi, client, allocationID, startEpoch) if err != nil { return err } @@ -441,7 +441,7 @@ var importDirectDealsCmd = &cli.Command{ return fmt.Errorf("failed to load allocations: %w", err) } for _, a := range allocations { - endEpoch, err := checkAndGetEndEpoch(ctx, fapi, a.Client, a.AllocationID, startEpoch) + endEpoch, err := CheckAndGetEndEpoch(ctx, fapi, a.Client, a.AllocationID, startEpoch) if err != nil { return err } @@ -483,7 +483,7 @@ var importDirectDealsCmd = &cli.Command{ }, } -func checkAndGetEndEpoch(ctx context.Context, +func CheckAndGetEndEpoch(ctx context.Context, fapi v1api.FullNode, client address.Address, allocationID uint64, diff --git a/cmd/droplet-client/direct-deal.go b/cmd/droplet-client/direct-deal.go index 7e506b92..1e7b704e 100644 --- a/cmd/droplet-client/direct-deal.go +++ b/cmd/droplet-client/direct-deal.go @@ -18,8 +18,12 @@ import ( verifregst "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" "github.com/filecoin-project/venus/venus-shared/actors" "github.com/filecoin-project/venus/venus-shared/actors/builtin/datacap" + "github.com/filecoin-project/venus/venus-shared/api" v1 "github.com/filecoin-project/venus/venus-shared/api/chain/v1" + marketapi "github.com/filecoin-project/venus/venus-shared/api/market/v1" "github.com/filecoin-project/venus/venus-shared/types" + types2 "github.com/filecoin-project/venus/venus-shared/types/market" + "github.com/google/uuid" cli2 "github.com/ipfs-force-community/droplet/v2/cli" "github.com/ipfs-force-community/droplet/v2/cli/tablewriter" "github.com/ipfs/go-cid" @@ -74,8 +78,9 @@ var directDealAllocate = &cli.Command{ termMaxFlag, expirationFlag, &cli.StringSliceFlag{ - Name: "piece-info", - Usage: "data pieceInfo[s] to create the allocation. The format must be pieceCid1=pieceSize1 pieceCid2=pieceSize2", + Name: "piece-info", + Usage: "data pieceInfo[s] to create the allocation. The format must be pieceCid1=pieceSize1 pieceCid2=pieceSize2", + Hidden: true, }, &cli.BoolFlag{ Name: "quiet", @@ -91,6 +96,18 @@ var directDealAllocate = &cli.Command{ Usage: "Output allocation information to a file.", Value: "allocation.csv", }, + &cli.StringFlag{ + Name: "droplet-url", + Usage: "Url of the droplet service", + }, + &cli.StringFlag{ + Name: "droplet-token", + Usage: "Token of the droplet service", + }, + &cli.IntFlag{ + Name: "start-epoch", + Usage: "start epoch by when the deal should be proved by provider on-chain (default: 8 days from now)", + }, }, Action: func(cctx *cli.Context) error { if cctx.IsSet("piece-info") && cctx.IsSet("manifest") { @@ -250,7 +267,27 @@ var directDealAllocate = &cli.Command{ return fmt.Errorf("failed to execute the message with error: %s", res.Receipt.ExitCode.Error()) } - return showAllocations(ctx, fapi, walletAddr, oldAllocations, cctx.Bool("json"), cctx.Bool("quiet"), cctx.String("output-allocation-to-file"), pieceInfos) + newAllocations, err := findNewAllocations(ctx, fapi, walletAddr, oldAllocations) + if err != nil { + return fmt.Errorf("failed to find new allocations: %w", err) + } + + if err := writeAllocationsToFile(cctx.String("output-allocation-to-file"), newAllocations, pieceInfos); err != nil { + fmt.Println("failed to write allocations to file: ", err) + } + + if err := showAllocations(newAllocations, cctx.Bool("json"), cctx.Bool("quiet")); err != nil { + fmt.Println("failed to show allocations: ", err) + } + + if cctx.IsSet("droplet-url") { + fmt.Println("importing deal to droplet") + if err := autoImportDealToDroplet(cctx, newAllocations, pieceInfos); err != nil { + return fmt.Errorf("failed to import deal to droplet: %w", err) + } + } + + return nil }, } @@ -346,18 +383,66 @@ func getAllocationParams(cctx *cli.Context, currentHeight abi.ChainEpoch) (*allo return ¶ms, nil } +func findNewAllocations(ctx context.Context, fapi v1.FullNode, walletAddr address.Address, oldAllocations map[types.AllocationId]types.Allocation) (map[types.AllocationId]types.Allocation, error) { + allAllocations, err := fapi.StateGetAllocations(ctx, walletAddr, types.EmptyTSK) + if err != nil { + return nil, fmt.Errorf("failed to get allocations: %w", err) + } + + newAllocations := make(map[types.AllocationId]types.Allocation, len(allAllocations)-len(oldAllocations)) + for k, v := range allAllocations { + if _, ok := oldAllocations[k]; !ok { + newAllocations[k] = v + } + } + + return newAllocations, nil +} + type partAllocationInfo struct { AllocationID types.AllocationId PieceCID cid.Cid Client address.Address } -func showAllocations(ctx context.Context, fapi v1.FullNode, walletAddr address.Address, oldAllocations map[types.AllocationId]types.Allocation, useJSON bool, quite bool, allocationFile string, pieceInfos []*pieceInfo) error { - newAllocations, err := fapi.StateGetAllocations(ctx, walletAddr, types.EmptyTSK) - if err != nil { - return fmt.Errorf("failed to get allocations: %w", err) +func writeAllocationsToFile(allocationFile string, allocations map[types.AllocationId]types.Allocation, pieceInfo []*pieceInfo) error { + payloadSizes := make(map[cid.Cid]uint64) + for _, info := range pieceInfo { + payloadSizes[info.pieceCID] = info.payloadSize + } + + infos := make([]partAllocationInfo, 0, len(allocations)) + for id, v := range allocations { + clientAddr, _ := address.NewIDAddress(uint64(v.Client)) + infos = append(infos, partAllocationInfo{ + AllocationID: id, + PieceCID: v.Data, + Client: clientAddr, + }) + } + + sort.Slice(infos, func(i, j int) bool { + return infos[i].AllocationID < infos[j].AllocationID + }) + + buf := &bytes.Buffer{} + w := csv.NewWriter(buf) + if err := w.Write([]string{"AllocationID", "PieceCID", "Client", "PayloadSize"}); err != nil { + return err + } + for _, info := range infos { + if err := w.Write([]string{fmt.Sprintf("%d", info.AllocationID), info.PieceCID.String(), info.Client.String(), fmt.Sprintf("%d", payloadSizes[info.PieceCID])}); err != nil { + return err + } } + w.Flush() + + fmt.Println("writing allocations to:", allocationFile) + return os.WriteFile(allocationFile, buf.Bytes(), 0644) +} + +func showAllocations(allocations map[types.AllocationId]types.Allocation, useJSON bool, quite bool) error { // Map Keys. Corresponds to the standard tablewriter output allocationID := "AllocationID" client := "Client" @@ -381,34 +466,20 @@ func showAllocations(ctx context.Context, fapi v1.FullNode, walletAddr address.A } var allocs []map[string]interface{} - var partAllocationInfos []partAllocationInfo - for key, val := range newAllocations { - _, ok := oldAllocations[key] - if !ok { - clientAddr, _ := address.NewIDAddress(uint64(val.Client)) - providerAddr, _ := address.NewIDAddress(uint64(val.Provider)) - alloc := map[string]interface{}{ - allocationID: key, - client: clientAddr, - provider: providerAddr, - pieceCid: val.Data, - pieceSize: val.Size, - tMin: val.TermMin, - tMax: val.TermMax, - expr: val.Expiration, - } - allocs = append(allocs, alloc) - - partAllocationInfos = append(partAllocationInfos, partAllocationInfo{ - AllocationID: key, - PieceCID: val.Data, - Client: walletAddr, - }) + for key, val := range allocations { + clientAddr, _ := address.NewIDAddress(uint64(val.Client)) + providerAddr, _ := address.NewIDAddress(uint64(val.Provider)) + alloc := map[string]interface{}{ + allocationID: key, + client: clientAddr, + provider: providerAddr, + pieceCid: val.Data, + pieceSize: val.Size, + tMin: val.TermMin, + tMax: val.TermMax, + expr: val.Expiration, } - } - - if err := outputAllocationToFile(allocationFile, partAllocationInfos, pieceInfos); err != nil { - fmt.Println("output allocation to file error: ", err) + allocs = append(allocs, alloc) } if quite { @@ -445,26 +516,63 @@ func showAllocations(ctx context.Context, fapi v1.FullNode, walletAddr address.A return tw.Flush(os.Stdout) } -func outputAllocationToFile(allocationFile string, infos []partAllocationInfo, pieceInfo []*pieceInfo) error { +func autoImportDealToDroplet(cliCtx *cli.Context, allocations map[types.AllocationId]types.Allocation, pieceInfos []*pieceInfo) error { + ctx := cliCtx.Context + dropletURL := cliCtx.String("droplet-url") + dropletToken := cliCtx.String("droplet-token") + + apiInfo := api.NewAPIInfo(dropletURL, dropletToken) + addr, err := apiInfo.DialArgs("v0") + if err != nil { + return err + } + + mapi, close, err := marketapi.NewIMarketRPC(ctx, addr, apiInfo.AuthHeader()) + if err != nil { + return err + } + defer close() + + fapi, fclose, err := cli2.NewFullNode(cliCtx, cli2.OldClientRepoPath) + if err != nil { + return err + } + defer fclose() + + params := types2.DirectDealParams{ + SkipCommP: true, + SkipGenerateIndex: true, + NoCopyCarFile: true, + DealParams: make([]types2.DirectDealParam, 0, len(allocations)), + } + payloadSizes := make(map[cid.Cid]uint64) - for _, info := range pieceInfo { + for _, info := range pieceInfos { payloadSizes[info.pieceCID] = info.payloadSize } - sort.Slice(infos, func(i, j int) bool { - return infos[i].AllocationID < infos[j].AllocationID - }) - buf := &bytes.Buffer{} - w := csv.NewWriter(buf) - if err := w.Write([]string{"AllocationID", "PieceCID", "Client", "PayloadSize"}); err != nil { + startEpoch, err := cli2.GetStartEpoch(cliCtx, fapi) + if err != nil { return err } - for _, info := range infos { - if err := w.Write([]string{fmt.Sprintf("%d", info.AllocationID), info.PieceCID.String(), info.Client.String(), fmt.Sprintf("%d", payloadSizes[info.PieceCID])}); err != nil { + + for id, alloc := range allocations { + clientAddr, _ := address.NewIDAddress(uint64(alloc.Client)) + endEpoch, err := cli2.CheckAndGetEndEpoch(ctx, fapi, clientAddr, uint64(id), startEpoch) + if err != nil { return err } + + params.DealParams = append(params.DealParams, types2.DirectDealParam{ + DealUUID: uuid.New(), + AllocationID: uint64(id), + PayloadSize: payloadSizes[alloc.Data], + Client: clientAddr, + PieceCID: alloc.Data, + StartEpoch: startEpoch, + EndEpoch: endEpoch, + }) } - w.Flush() - return os.WriteFile(allocationFile, buf.Bytes(), 0644) + return mapi.ImportDirectDeal(ctx, ¶ms) } diff --git a/docs/zh/direct-on-boarding.md b/docs/zh/direct-on-boarding.md index 3c9e2d26..8e84f243 100644 --- a/docs/zh/direct-on-boarding.md +++ b/docs/zh/direct-on-boarding.md @@ -2,13 +2,13 @@ ### 生成单个订单 -1. 生成 car 文件 +#### 生成 piece 文件 ``` ./droplet-client data generate-car droplet droplet.car ``` -2. 计算 commp +#### 计算 commP ``` ./droplet-client data commP droplet.car @@ -18,7 +18,7 @@ CID: baga6ea4seaqconolebafjmjlqc35z4foyzfipxfuiav25okti22kjof7rbgoipa Piece size: 254 MiB ( 266338304 B ) ``` -3. 生成 allocation +#### 生成订单 ``` ./droplet-client direct-deal allocate --miner t060973 --wallet t3wivhkdivcxj5zp2l4wjkzon232s52smnd5m3na66ujl5nel75jggguhgaa3zbhjo3as4epf5ytxl6ly3qoha --piece-info baga6ea4seaqconolebafjmjlqc35z4foyzfipxfuiav25okti22kjof7rbgoipa=266338304 @@ -32,7 +32,13 @@ AllocationID Client Miner PieceCid 31649 t018678 t060973 baga6ea4seaqconolebafjmjlqc35z4foyzfipxfuiav25okti22kjof7rbgoipa 266338304 518400 777600 1406893 ``` -#### 导入订单 +#### 导入单个订单 + +flag 解释： + +* --skip-commp 跳过计算验证 piece cid，可以减少导入时间 +* --skip-index 不生成索引 +* --no-copy-car-file 不拷贝 piece 到 piece storage ``` ./droplet storage direct-deal import-deal --allocation-id 31649 --client t3wivhkdivcxj5zp2l4wjkzon232s52smnd5m3na66ujl5nel75jggguhgaa3zbhjo3as4epf5ytxl6ly3qoha baga6ea4seaqconolebafjmjlqc35z4foyzfipxfuiav25okti22kjof7rbgoipa droplet.car @@ -40,9 +46,15 @@ AllocationID Client Miner PieceCid ### 生成多个订单 -#### 生成 piece 文件 +#### 使用 [go-graphsplit](https://github.com/filedrive-team/go-graphsplit) 生成 piece。 + +flag 解释： -1. 使用 [go-graphsplit](https://github.com/filedrive-team/go-graphsplit) 生成 piece。 +* --slice-size piece 文件大小 +* --car-dir 用于存储生成的 piece 文件的目录 +* --calc-commp 计算 piece cid +* --rename 用 piece cid 作为文件名，方便后续使用 +* --graph-name 可随意命名，暂无实际作用 ``` ./graphsplit chunk --slice-size 1048576 --car-dir data2 --calc-commp --rename --graph-name node graphsplit @@ -50,7 +62,15 @@ AllocationID Client Miner PieceCid 执行完成后会在 `data2` 生成 piece 文件及 `manifest.csv`，`manifest.csv` 包含 piece 的基本信息，也可以用于批量发布订单。 -2. 批量生成订单 +#### 批量生成订单 + +flag 解释： + +--wallet 发单地址，需要有 DC +--manifest 由 graphsplit 生成 +--output-allocation-to-file 用于保存新生成的 allocation 信息，可以用批量导入订单 +--droplet-url droplet url，如果设置了，则自动导入新生成的订单到 droplet，无需执行后续的 `批量导入订单` +--droplet-token droplet token ``` ./droplet-client direct-deal allocate --miner t060973 --wallet t3wivhkdivcxj5zp2l4wjkzon232s52smnd5m3na66ujl5nel75jggguhgaa3zbhjo3as4epf5ytxl6ly3qoha --manifest ./data2/manifest.csv @@ -67,7 +87,13 @@ AllocationID Client Miner PieceCid 31654 t018678 t060973 baga6ea4seaqgozgsl7ddfjqig6za3l7o5sf6oiw5hd4ggug7tiqfhi5gajwq4ja 2097152 518400 777600 1386797 ``` -3. 批量导入订单 +#### 批量导入订单 + +flag 解释： + +* --skip-commp 跳过计算验证 piece cid，可以减少导入时间 +* --skip-index 不生成索引 +* --no-copy-car-file 不拷贝 piece 到 piece storage ``` ./droplet storage direct-deal import-deals --allocation-file allocation.csv --car-dir ./data2/ @@ -75,9 +101,3 @@ AllocationID Client Miner PieceCid # res import deal success ``` - -导入订单命令提供了几个可选的 flag： - -* --no-copy-car-file 不拷贝 car 文件到 piece storage，也不会计算 commp 和生成索引 -* --skip-commp 不计算 commp，不能保证导入的订单数据是否正确 -* --skip-index 不生成索引 From 70a032f685d6388fb4142c7a04a7dd258c8a5020 Mon Sep 17 00:00:00 2001 From: simlecode <69969590+simlecode@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:41:10 +0800 Subject: [PATCH 2/3] docs: add cli doc --- docs/zh/direct-on-boarding.md | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/zh/direct-on-boarding.md b/docs/zh/direct-on-boarding.md index 8e84f243..d3cf360c 100644 --- a/docs/zh/direct-on-boarding.md +++ b/docs/zh/direct-on-boarding.md @@ -101,3 +101,51 @@ flag 解释： # res import deal success ``` + +### 查询订单信息 + +#### 查询单个订单 + +``` +./droplet storage direct-deal get --allocation-id 32224 +or +./droplet storage direct-deal get --id 07cd5814-02bf-494d-b81c-87df73b3422b + +# res +Creation 2024-04-01T14:30:39+08:00 +PieceCID baga6ea4seaqgzkse45r2tinm4cy7pducjt45c2r77qnu4r6uytxlsiazev6xwpy +PieceSize 2097152 +Client t018678 +Provider t060973 +AllocationID 32224 +State DealAllocated +Message +SectorID 0 +Offset 0 +Length 0 +PayloadSize 1048794 +StartEpoch 1510595 +EndEpoch 2028995 +``` + +#### 列出订单 + +> 该命令默认只会列出 DealAllocated 状态的订单，可以通过 --state flag 指定特定状态的订单 + +``` +./droplet storage direct-deal list + +# res +Creation ID AllocationId PieceCid State Client Provider Size Message +2024-04-01T14:30:40+08:00 07cd5814-02bf-494d-b81c-87df73b3422b 32227 baga6ea4seaqhbpwuqszynr4wmtn2osjwkru3nrp6z6bjte6c4rzlntfm4l5s2ia DealAllocated t018678 t060973 1048576 +2024-04-01T14:30:40+08:00 aeaf18c5-2d92-4376-9370-594b8536190f 32226 baga6ea4seaqeddngzgmtkxa3wqetu27j5ydqie7hwa5rjrl5de6osamz3ldpegi DealAllocated t018678 t060973 2097152 +2024-04-01T14:30:39+08:00 d94f289d-50be-460b-8555-8b9a398e35d6 32223 baga6ea4seaqp6jm4x3pf7llach7tdhbwrwlcetv52dnjicqpcl6lkwib5n76gii DealAllocated t018678 t060973 2097152 +2024-04-01T14:30:39+08:00 ea01b452-109f-4373-ab80-6af76d75b6d6 32224 baga6ea4seaqgzkse45r2tinm4cy7pducjt45c2r77qnu4r6uytxlsiazev6xwpy DealAllocated t018678 t060973 2097152 +2024-04-01T14:30:39+08:00 f82a5335-6036-4bb5-9d6b-d31030cb3272 32225 baga6ea4seaqdddnss5oalozaqsogmhfohxh2hyclw5ewxdgdme53tn3pidzgeeq DealAllocated t018678 t060973 2097152 +``` + +#### 更新订单状态 + +``` +./droplet storage direct-deal update-state --state 1 07cd5814-02bf-494d-b81c-87df73b3422b +``` \ No newline at end of file From 409c9e9a4f03362c73676981b8d8b6e5de41b2f8 Mon Sep 17 00:00:00 2001 From: simlecode <69969590+simlecode@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:42:32 +0800 Subject: [PATCH 3/3] opt: output deal id --- cli/direct-deal.go | 1 + cmd/droplet-client/direct-deal.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cli/direct-deal.go b/cli/direct-deal.go index f3c30d5b..d46820ca 100644 --- a/cli/direct-deal.go +++ b/cli/direct-deal.go @@ -82,6 +82,7 @@ var getDirectDealCmd = &cli.Command{ data := []kv{ {"Creation", time.Unix(int64(deal.CreatedAt), 0).Format(time.RFC3339)}, + {"ID", deal.ID}, {"PieceCID", deal.PieceCID}, {"PieceSize", deal.PieceSize}, {"Client", deal.Client}, diff --git a/cmd/droplet-client/direct-deal.go b/cmd/droplet-client/direct-deal.go index 1e7b704e..beffcae3 100644 --- a/cmd/droplet-client/direct-deal.go +++ b/cmd/droplet-client/direct-deal.go @@ -285,6 +285,7 @@ var directDealAllocate = &cli.Command{ if err := autoImportDealToDroplet(cctx, newAllocations, pieceInfos); err != nil { return fmt.Errorf("failed to import deal to droplet: %w", err) } + fmt.Println("successfully imported deal to droplet") } return nil