diff --git a/README.md b/README.md index 550c9023..111563d7 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ run as venus-pool service run in local ```shell script -./venus-market solo-run --node-url --node-token --wallet-url --wallet-token --payment-addr +./venus-market solo-run --node-url --node-token --signer-url --signer-token --payment-addr ``` set peerid and address @@ -55,7 +55,7 @@ set retrieval ask ## how to setup market client ```shell script -./market-client run --node-url --node-token --wallet-url --wallet-token --addr +./market-client run --node-url --node-token --signer-url --signer-token --addr ``` Note:**please use a seperate address, or maybe nonce confiction** diff --git a/api/clients/gateway_client.go b/api/clients/gateway_client.go deleted file mode 100644 index 3c48560f..00000000 --- a/api/clients/gateway_client.go +++ /dev/null @@ -1,45 +0,0 @@ -package clients - -import ( - "context" - - api "github.com/filecoin-project/venus/venus-shared/api/gateway/v1" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-jsonrpc" - "github.com/filecoin-project/venus-market/v2/config" - "github.com/filecoin-project/venus-market/v2/minermgr" - vCrypto "github.com/filecoin-project/venus/pkg/crypto" - types2 "github.com/filecoin-project/venus/venus-shared/types" - "github.com/ipfs-force-community/venus-common-utils/metrics" -) - -func newGatewayWalletClient(mctx metrics.MetricsCtx, mgr minermgr.IAddrMgr, nodeCfg *config.Signer) (ISinger, jsonrpc.ClientCloser, error) { - client, closer, err := api.DialIGatewayRPC(mctx, nodeCfg.Url, nodeCfg.Token, nil) - return &GatewayClient{ - innerClient: client, - importMgr: mgr, - }, closer, err -} - -type GatewayClient struct { - innerClient api.IWalletClient - importMgr minermgr.IAddrMgr -} - -func (gatewayClient *GatewayClient) WalletHas(ctx context.Context, addr address.Address) (bool, error) { - account, err := gatewayClient.importMgr.GetAccount(ctx, addr) - if err != nil { - return false, err - } - return gatewayClient.innerClient.WalletHas(ctx, account, addr) -} - -func (gatewayClient *GatewayClient) WalletSign(ctx context.Context, addr address.Address, msg []byte, meta types2.MsgMeta) (*vCrypto.Signature, error) { - account, err := gatewayClient.importMgr.GetAccount(ctx, addr) - if err != nil { - return nil, err - } - - return gatewayClient.innerClient.WalletSign(ctx, account, addr, msg, meta) -} diff --git a/api/clients/mix_msgclient.go b/api/clients/mix_msgclient.go index 9be363a5..d456b5cc 100644 --- a/api/clients/mix_msgclient.go +++ b/api/clients/mix_msgclient.go @@ -5,20 +5,22 @@ import ( "fmt" "time" - types2 "github.com/filecoin-project/venus/venus-shared/types/messager" + "go.uber.org/fx" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/venus-market/v2/minermgr" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/venus-market/v2/api/clients/signer" "github.com/filecoin-project/venus-market/v2/utils" + v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" "github.com/filecoin-project/venus/venus-shared/types" - "github.com/ipfs/go-cid" - "go.uber.org/fx" + msgTypes "github.com/filecoin-project/venus/venus-shared/types/messager" ) type IMixMessage interface { GetMessage(ctx context.Context, mid cid.Cid) (*types.Message, error) - PushMessage(ctx context.Context, p1 *types.Message, p2 *types.MessageSendSpec) (cid.Cid, error) + PushMessage(ctx context.Context, msg *types.Message, msgSendSpec *types.MessageSendSpec) (cid.Cid, error) GetMessageChainCid(ctx context.Context, mid cid.Cid) (*cid.Cid, error) WaitMsg(ctx context.Context, mCid cid.Cid, confidence uint64, loopBackLimit abi.ChainEpoch, allowReplaced bool) (*types.MsgLookup, error) SearchMsg(ctx context.Context, from types.TipSetKey, mCid cid.Cid, loopBackLimit abi.ChainEpoch, allowReplaced bool) (*types.MsgLookup, error) @@ -27,49 +29,46 @@ type IMixMessage interface { type MPoolReplaceParams struct { fx.In FullNode v1api.FullNode - Singer ISinger `optional:"true"` - VenusMessager IVenusMessager `optional:"true"` - Mgr minermgr.IAddrMgr `optional:"true"` + Signer signer.ISigner + VenusMessager IVenusMessager `optional:"true"` } type MixMsgClient struct { - full v1api.FullNode - messager IVenusMessager - addrMgr minermgr.IAddrMgr - signer ISinger - nonceAssign *nonceAssigner + full v1api.FullNode + venusMessager IVenusMessager + signer signer.ISigner + nonceAssign *nonceAssigner } func NewMixMsgClient(params MPoolReplaceParams) IMixMessage { return &MixMsgClient{ - full: params.FullNode, - messager: params.VenusMessager, - addrMgr: params.Mgr, - signer: params.Singer, - nonceAssign: newNonceAssign(params.FullNode), + full: params.FullNode, + venusMessager: params.VenusMessager, + nonceAssign: newNonceAssign(params.FullNode), } } -func (msgClient *MixMsgClient) PushMessage(ctx context.Context, p1 *types.Message, p2 *types.MessageSendSpec) (cid.Cid, error) { - if msgClient.messager == nil { +func (msgClient *MixMsgClient) PushMessage(ctx context.Context, msg *types.Message, msgSendSpec *types.MessageSendSpec) (cid.Cid, error) { + if msgClient.venusMessager == nil { var sendSpec *types.MessageSendSpec - if p2 != nil { + if msgSendSpec != nil { sendSpec = &types.MessageSendSpec{ - MaxFee: p2.MaxFee, - GasOverEstimation: p2.GasOverEstimation, + MaxFee: msgSendSpec.MaxFee, + GasOverEstimation: msgSendSpec.GasOverEstimation, } } + var err error - p1.From, err = msgClient.full.StateAccountKey(ctx, p1.From, types.EmptyTSK) + msg.From, err = msgClient.full.StateAccountKey(ctx, msg.From, types.EmptyTSK) if err != nil { return cid.Undef, err } // estimate -> sign -> push - estimatedMsg, err := msgClient.full.GasEstimateMessageGas(ctx, p1, sendSpec, types.EmptyTSK) + estimatedMsg, err := msgClient.full.GasEstimateMessageGas(ctx, msg, sendSpec, types.EmptyTSK) if err != nil { return cid.Undef, err } - estimatedMsg.Nonce, err = msgClient.nonceAssign.AssignNonce(ctx, p1.From) + estimatedMsg.Nonce, err = msgClient.nonceAssign.AssignNonce(ctx, msg.From) if err != nil { return cid.Undef, err } @@ -77,7 +76,8 @@ func (msgClient *MixMsgClient) PushMessage(ctx context.Context, p1 *types.Messag if err != nil { return cid.Undef, err } - sig, err := msgClient.full.WalletSign(ctx, estimatedMsg.From, storageBlock.Cid().Bytes(), types.MsgMeta{ + + sig, err := msgClient.signer.WalletSign(ctx, estimatedMsg.From, storageBlock.Cid().Bytes(), types.MsgMeta{ Type: types.MTChainMsg, Extra: storageBlock.RawData(), }) @@ -91,42 +91,33 @@ func (msgClient *MixMsgClient) PushMessage(ctx context.Context, p1 *types.Messag if err != nil { return cid.Undef, err } + log.Warnf("push message %s to daemon", signedCid.String()) + return signedCid, nil } - msgid, err := utils.NewMId() + + msgID, err := utils.NewMId() if err != nil { return cid.Undef, err } - if msgClient.addrMgr != nil { - fromAddr, err := msgClient.full.StateAccountKey(ctx, p1.From, types.EmptyTSK) - if err != nil { - return cid.Undef, err - } - account, err := msgClient.addrMgr.GetAccount(ctx, fromAddr) - if err != nil { - return cid.Undef, err - } - _, err = msgClient.messager.ForcePushMessageWithId(ctx, account, msgid.String(), p1, nil) - if err != nil { - return cid.Undef, err - } - } else { - // for client account has in token - _, err = msgClient.messager.PushMessageWithId(ctx, msgid.String(), p1, nil) - if err != nil { - return cid.Undef, err - } + + // from-account-signer handling moved to venus-gateway + _, err = msgClient.venusMessager.PushMessageWithId(ctx, msgID.String(), msg, nil) + if err != nil { + return cid.Undef, err } - log.Warnf("push message %s to venus-messager", msgid.String()) - return msgid, nil + log.Warnf("push message %s to venus-messager", msgID.String()) + + return msgID, nil } func (msgClient *MixMsgClient) WaitMsg(ctx context.Context, mCid cid.Cid, confidence uint64, loopbackLimit abi.ChainEpoch, allowReplaced bool) (*types.MsgLookup, error) { - if msgClient.messager == nil || mCid.Prefix() != utils.MidPrefix { + if msgClient.venusMessager == nil || mCid.Prefix() != utils.MidPrefix { return msgClient.full.StateWaitMsg(ctx, mCid, confidence, loopbackLimit, allowReplaced) } + tm := time.NewTicker(time.Second * 30) defer tm.Stop() @@ -136,7 +127,7 @@ func (msgClient *MixMsgClient) WaitMsg(ctx context.Context, mCid cid.Cid, confid for { select { case <-doneCh: - msg, err := msgClient.messager.GetMessageByUid(ctx, mCid.String()) + msg, err := msgClient.venusMessager.GetMessageByUid(ctx, mCid.String()) if err != nil { log.Warnf("get message %s fail while wait %v", mCid, err) time.Sleep(time.Second * 5) @@ -144,17 +135,17 @@ func (msgClient *MixMsgClient) WaitMsg(ctx context.Context, mCid cid.Cid, confid } switch msg.State { - // OffChain - case types2.FillMsg: + //OffChain + case msgTypes.FillMsg: fallthrough - case types2.UnFillMsg: + case msgTypes.UnFillMsg: fallthrough - case types2.UnKnown: + case msgTypes.UnKnown: continue - // OnChain - case types2.ReplacedMsg: + //OnChain + case msgTypes.ReplacedMsg: fallthrough - case types2.OnChainMsg: + case msgTypes.OnChainMsg: if msg.Confidence > int64(confidence) { return &types.MsgLookup{ Message: mCid, @@ -168,8 +159,8 @@ func (msgClient *MixMsgClient) WaitMsg(ctx context.Context, mCid cid.Cid, confid }, nil } continue - // Error - case types2.FailedMsg: + //Error + case msgTypes.FailedMsg: var reason string if msg.Receipt != nil { reason = string(msg.Receipt.Return) @@ -186,10 +177,11 @@ func (msgClient *MixMsgClient) WaitMsg(ctx context.Context, mCid cid.Cid, confid } func (msgClient *MixMsgClient) SearchMsg(ctx context.Context, from types.TipSetKey, mCid cid.Cid, loopbackLimit abi.ChainEpoch, allowReplaced bool) (*types.MsgLookup, error) { - if msgClient.messager == nil || mCid.Prefix() != utils.MidPrefix { + if msgClient.venusMessager == nil || mCid.Prefix() != utils.MidPrefix { return msgClient.full.StateSearchMsg(ctx, from, mCid, loopbackLimit, allowReplaced) } - msg, err := msgClient.messager.GetMessageByUid(ctx, mCid.String()) + + msg, err := msgClient.venusMessager.GetMessageByUid(ctx, mCid.String()) if err != nil { log.Warnw("get message fail while wait %w", err) time.Sleep(time.Second * 5) @@ -197,17 +189,17 @@ func (msgClient *MixMsgClient) SearchMsg(ctx context.Context, from types.TipSetK } switch msg.State { - // OffChain - case types2.FillMsg: + //OffChain + case msgTypes.FillMsg: fallthrough - case types2.UnFillMsg: + case msgTypes.UnFillMsg: fallthrough - case types2.UnKnown: + case msgTypes.UnKnown: return nil, nil - // OnChain - case types2.ReplacedMsg: + //OnChain + case msgTypes.ReplacedMsg: fallthrough - case types2.OnChainMsg: + case msgTypes.OnChainMsg: return &types.MsgLookup{ Message: mCid, Receipt: types.MessageReceipt{ @@ -218,8 +210,8 @@ func (msgClient *MixMsgClient) SearchMsg(ctx context.Context, from types.TipSetK TipSet: msg.TipSetKey, Height: abi.ChainEpoch(msg.Height), }, nil - // Error - case types2.FailedMsg: + //Error + case msgTypes.FailedMsg: var reason string if msg.Receipt != nil { reason = string(msg.Receipt.Return) @@ -231,26 +223,30 @@ func (msgClient *MixMsgClient) SearchMsg(ctx context.Context, from types.TipSetK } func (msgClient *MixMsgClient) GetMessage(ctx context.Context, mCid cid.Cid) (*types.Message, error) { - if msgClient.messager == nil || mCid.Prefix() != utils.MidPrefix { + if msgClient.venusMessager == nil || mCid.Prefix() != utils.MidPrefix { return msgClient.full.ChainGetMessage(ctx, mCid) } - msg, err := msgClient.messager.GetMessageByUid(ctx, mCid.String()) + + msg, err := msgClient.venusMessager.GetMessageByUid(ctx, mCid.String()) if err != nil { return nil, err } + return msg.VMMessage(), nil } func (msgClient *MixMsgClient) GetMessageChainCid(ctx context.Context, mid cid.Cid) (*cid.Cid, error) { if mid.Prefix() == utils.MidPrefix { - if msgClient.messager == nil { + if msgClient.venusMessager == nil { return nil, fmt.Errorf("unable to get message chain cid from messager,no messager configured") } - msg, err := msgClient.messager.GetMessageByUid(ctx, mid.String()) + msg, err := msgClient.venusMessager.GetMessageByUid(ctx, mid.String()) if err != nil { return nil, err } + return msg.SignedCid, nil } + return &mid, nil } diff --git a/api/clients/modules.go b/api/clients/modules.go index 4dd3a678..d0218b8c 100644 --- a/api/clients/modules.go +++ b/api/clients/modules.go @@ -1,64 +1,37 @@ package clients import ( - "context" + logging "github.com/ipfs/go-log/v2" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/venus-market/v2/config" - vCrypto "github.com/filecoin-project/venus/pkg/crypto" - v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" - api "github.com/filecoin-project/venus/venus-shared/api/gateway/v1" - types2 "github.com/filecoin-project/venus/venus-shared/types" + "github.com/ipfs-force-community/metrics" "github.com/ipfs-force-community/venus-common-utils/builder" - "github.com/ipfs-force-community/venus-common-utils/metrics" "github.com/ipfs-force-community/venus-gateway/marketevent" - types3 "github.com/ipfs-force-community/venus-gateway/types" - logging "github.com/ipfs/go-log/v2" -) -var log = logging.Logger("clients") + gwTypes "github.com/ipfs-force-community/venus-gateway/types" -var ReplaceWalletMethod = builder.NextInvoke() - -func ConvertWalletToISinge(fullNode v1api.FullNode, signer ISinger) error { - fullNodeStruct := fullNode.(*v1api.FullNodeStruct) - fullNodeStruct.IWalletStruct.Internal.WalletHas = func(p0 context.Context, p1 address.Address) (bool, error) { - return signer.WalletHas(p0, p1) - } - fullNodeStruct.IWalletStruct.Internal.WalletSign = func(p0 context.Context, p1 address.Address, p2 []byte, p3 types2.MsgMeta) (*vCrypto.Signature, error) { - return signer.WalletSign(p0, p1, p2, p3) - } - return nil -} + "github.com/filecoin-project/venus-market/v2/api/clients/signer" + "github.com/filecoin-project/venus-market/v2/config" -func NewMarketEvent(mctx metrics.MetricsCtx) (*marketevent.MarketEventStream, error) { - stream := marketevent.NewMarketEventStream(mctx, &localMinerValidator{}, types3.DefaultConfig()) - return stream, nil -} + v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" + gwAPI "github.com/filecoin-project/venus/venus-shared/api/gateway/v2" +) -func NewMarketEventAPI(stream *marketevent.MarketEventStream) (api.IMarketServiceProvider, error) { - return stream, nil -} +var log = logging.Logger("clients") -func NewIMarketEvent(stream *marketevent.MarketEventStream) (MarketRequestEvent, error) { +func NewMarketEvent(mctx metrics.MetricsCtx) (gwAPI.IMarketEvent, error) { + stream := marketevent.NewMarketEventStream(mctx, &localMinerValidator{}, gwTypes.DefaultConfig()) return stream, nil } -var ClientsOpts = func(server bool, mode string, mCfg *config.Messager, signerCfg *config.Signer) builder.Option { +var ClientsOpts = func(server bool, mode string, msgCfg *config.Messager, signerCfg *config.Signer) builder.Option { opts := builder.Options( builder.Override(new(IMixMessage), NewMixMsgClient), + builder.Override(new(signer.ISigner), signer.NewISignerClient(server)), builder.ApplyIf( func(s *builder.Settings) bool { - return len(mCfg.Url) > 0 + return len(msgCfg.Url) > 0 }, builder.Override(new(IVenusMessager), MessagerClient)), - builder.ApplyIf( - func(s *builder.Settings) bool { - return len(signerCfg.SignerType) > 0 && len(signerCfg.Url) > 0 - }, - builder.Override(new(ISinger), NewISignerClient(server)), - builder.Override(ReplaceWalletMethod, ConvertWalletToISinge), - ), ) if server { @@ -69,12 +42,11 @@ var ClientsOpts = func(server bool, mode string, mCfg *config.Messager, signerCf func(s *builder.Settings) bool { return mode == "solo" }, - builder.Override(new(*marketevent.MarketEventStream), NewMarketEvent), - builder.Override(new(api.IMarketServiceProvider), NewMarketEventAPI), - builder.Override(new(MarketRequestEvent), NewIMarketEvent), + builder.Override(new(gwAPI.IMarketEvent), NewMarketEvent), ), ) } + return builder.Options(opts, builder.Override(new(v1api.FullNode), NodeClient), ) diff --git a/api/clients/node_client.go b/api/clients/node_client.go index 4525a894..62e7c682 100644 --- a/api/clients/node_client.go +++ b/api/clients/node_client.go @@ -3,19 +3,23 @@ package clients import ( "context" + "go.uber.org/fx" + "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/utils" + + "github.com/filecoin-project/venus/venus-shared/api" v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" + "github.com/ipfs-force-community/metrics" - "github.com/ipfs-force-community/venus-common-utils/apiinfo" - "go.uber.org/fx" ) func NodeClient(mctx metrics.MetricsCtx, lc fx.Lifecycle, nodeCfg *config.Node) (v1api.FullNode, error) { fullNode := v1api.FullNodeStruct{} - aInfo := apiinfo.NewAPIInfo(nodeCfg.Url, nodeCfg.Token) + aInfo := api.NewAPIInfo(nodeCfg.Url, nodeCfg.Token) addr, err := aInfo.DialArgs("v1") if err != nil { return nil, err diff --git a/api/clients/sign_client.go b/api/clients/sign_client.go deleted file mode 100644 index 2c195d4a..00000000 --- a/api/clients/sign_client.go +++ /dev/null @@ -1,62 +0,0 @@ -package clients - -import ( - "context" - "fmt" - - "github.com/filecoin-project/venus/venus-shared/api/wallet" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-jsonrpc" - "github.com/filecoin-project/venus-market/v2/config" - "github.com/filecoin-project/venus-market/v2/minermgr" - vCrypto "github.com/filecoin-project/venus/pkg/crypto" - types2 "github.com/filecoin-project/venus/venus-shared/types" - "github.com/ipfs-force-community/metrics" - "go.uber.org/fx" -) - -type ISinger interface { - WalletHas(ctx context.Context, addr address.Address) (bool, error) - WalletSign(ctx context.Context, k address.Address, msg []byte, meta types2.MsgMeta) (*vCrypto.Signature, error) -} - -type SignerParams struct { - fx.In - SignerCfg *config.Signer - Mgr minermgr.IAddrMgr `optional:"true"` -} - -func NewISignerClient(isServer bool) func(metrics.MetricsCtx, fx.Lifecycle, SignerParams) (ISinger, error) { - return func(mctx metrics.MetricsCtx, lc fx.Lifecycle, params SignerParams) (ISinger, error) { - var ( - cfg = params.SignerCfg - ctx = metrics.LifecycleCtx(mctx, lc) - signer ISinger - closer jsonrpc.ClientCloser - err error - ) - - switch params.SignerCfg.SignerType { - case config.SignerTypeWallet: - signer, closer, err = wallet.DialIFullAPIRPC(ctx, cfg.Url, cfg.Token, nil) - case config.SignerTypeGateway: - if !isServer { - return nil, fmt.Errorf("gateway signer not supported in client mode") - } - signer, closer, err = newGatewayWalletClient(context.Background(), params.Mgr, params.SignerCfg) - default: - return nil, fmt.Errorf("unsupport sign type %s", params.SignerCfg.SignerType) - } - - lc.Append(fx.Hook{ - OnStop: func(_ context.Context) error { - if closer != nil { - closer() - } - return nil - }, - }) - return signer, err - } -} diff --git a/api/clients/signer/gateway_client.go b/api/clients/signer/gateway_client.go new file mode 100644 index 00000000..b99ca1d2 --- /dev/null +++ b/api/clients/signer/gateway_client.go @@ -0,0 +1,82 @@ +package signer + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/venus-auth/jwtclient" + + "github.com/filecoin-project/venus-market/v2/config" + + vCrypto "github.com/filecoin-project/venus/pkg/crypto" + "github.com/filecoin-project/venus/venus-shared/api" + vTypes "github.com/filecoin-project/venus/venus-shared/types" +) + +type GatewayClient struct { + Internal struct { + WalletHas func(context.Context, address.Address, []string) (bool, error) + WalletSign func(context.Context, address.Address, []string, []byte, vTypes.MsgMeta) (*vCrypto.Signature, error) + } +} + +func (lnw *GatewayClient) WalletHas(ctx context.Context, addr address.Address, accounts []string) (bool, error) { + return lnw.Internal.WalletHas(ctx, addr, accounts) +} + +func (lnw *GatewayClient) WalletSign(ctx context.Context, addr address.Address, accounts []string, msg []byte, meta vTypes.MsgMeta) (*vCrypto.Signature, error) { + return lnw.Internal.WalletSign(ctx, addr, accounts, msg, meta) +} + +type WrapperGatewayClient struct { + gatewayClient *GatewayClient + authClient jwtclient.IAuthClient +} + +func (w *WrapperGatewayClient) getAccountsOfSigner(ctx context.Context, addr address.Address) ([]string, error) { + users, err := w.authClient.GetUserBySigner(addr.String()) + if err != nil { + return nil, err + } + + accounts := make([]string, 0, len(users)) + for _, user := range users { + accounts = append(accounts, user.Name) + } + + return accounts, nil +} + +func (w *WrapperGatewayClient) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + accounts, err := w.getAccountsOfSigner(ctx, addr) + if err != nil { + return false, err + } + return w.gatewayClient.WalletHas(ctx, addr, accounts) +} + +func (w *WrapperGatewayClient) WalletSign(ctx context.Context, addr address.Address, msg []byte, meta vTypes.MsgMeta) (*vCrypto.Signature, error) { + accounts, err := w.getAccountsOfSigner(ctx, addr) + if err != nil { + return nil, err + } + return w.gatewayClient.WalletSign(ctx, addr, accounts, msg, meta) +} + +func newGatewayWalletClient(ctx context.Context, nodeCfg *config.Signer, authClient jwtclient.IAuthClient) (ISigner, jsonrpc.ClientCloser, error) { + apiInfo := api.NewAPIInfo(nodeCfg.Url, nodeCfg.Token) + dialAddr, err := apiInfo.DialArgs("v2") + if err != nil { + return nil, nil, err + } + + gatewayClient := GatewayClient{} + closer, err := jsonrpc.NewMergeClient(ctx, dialAddr, "Gateway", api.GetInternalStructs(&gatewayClient), apiInfo.AuthHeader()) + if err != nil { + return nil, nil, err + } + + return &WrapperGatewayClient{gatewayClient: &gatewayClient, authClient: authClient}, closer, err +} diff --git a/api/clients/signer/lotusnode_client.go b/api/clients/signer/lotusnode_client.go new file mode 100644 index 00000000..4e49cb28 --- /dev/null +++ b/api/clients/signer/lotusnode_client.go @@ -0,0 +1,54 @@ +package signer + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/venus-market/v2/config" + + vCrypto "github.com/filecoin-project/venus/pkg/crypto" + "github.com/filecoin-project/venus/venus-shared/api" + vTypes "github.com/filecoin-project/venus/venus-shared/types" +) + +type LotusnodeClient struct { + Internal struct { + WalletHas func(context.Context, address.Address) (bool, error) + WalletSign func(context.Context, address.Address, []byte) (*vCrypto.Signature, error) + } +} + +func (lnw *LotusnodeClient) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + return lnw.Internal.WalletHas(ctx, addr) +} + +func (lnw *LotusnodeClient) WalletSign(ctx context.Context, addr address.Address, msg []byte) (*vCrypto.Signature, error) { + return lnw.Internal.WalletSign(ctx, addr, msg) +} + +type WrapperLotusnodeClient struct { + lotusnodeClient *LotusnodeClient +} + +func (w *WrapperLotusnodeClient) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + return w.lotusnodeClient.WalletHas(ctx, addr) +} + +func (w *WrapperLotusnodeClient) WalletSign(ctx context.Context, addr address.Address, msg []byte, meta vTypes.MsgMeta) (*vCrypto.Signature, error) { + return w.lotusnodeClient.WalletSign(ctx, addr, msg) +} + +func newLotusnodeClient(ctx context.Context, nodeCfg *config.Signer) (ISigner, jsonrpc.ClientCloser, error) { + apiInfo := api.NewAPIInfo(nodeCfg.Url, nodeCfg.Token) + addr, err := apiInfo.DialArgs("v1") + if err != nil { + return nil, nil, err + } + + lotusnodeClient := LotusnodeClient{} + closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", []interface{}{&lotusnodeClient.Internal}, apiInfo.AuthHeader()) + + return &WrapperLotusnodeClient{lotusnodeClient: &lotusnodeClient}, closer, err +} diff --git a/api/clients/signer/sign_client.go b/api/clients/signer/sign_client.go new file mode 100644 index 00000000..509e444b --- /dev/null +++ b/api/clients/signer/sign_client.go @@ -0,0 +1,64 @@ +package signer + +import ( + "context" + "fmt" + + "go.uber.org/fx" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-jsonrpc" + + "github.com/ipfs-force-community/metrics" + + "github.com/filecoin-project/venus-auth/jwtclient" + + "github.com/filecoin-project/venus-market/v2/config" + + vCrypto "github.com/filecoin-project/venus/pkg/crypto" + vTypes "github.com/filecoin-project/venus/venus-shared/types" +) + +type ISigner interface { + WalletHas(ctx context.Context, signerAddr address.Address) (bool, error) + WalletSign(ctx context.Context, signerAddr address.Address, msg []byte, meta vTypes.MsgMeta) (*vCrypto.Signature, error) +} + +func NewISignerClient(isServer bool) func(metrics.MetricsCtx, fx.Lifecycle, *config.Signer, *jwtclient.AuthClient) (ISigner, error) { + + return func(mCtx metrics.MetricsCtx, lc fx.Lifecycle, signerCfg *config.Signer, authClient *jwtclient.AuthClient) (ISigner, error) { + var ( + signer ISigner + closer jsonrpc.ClientCloser + err error + ) + + switch signerCfg.SignerType { + // Sign with lotus node + case config.SignerTypeLotusnode: + signer, closer, err = newLotusnodeClient(mCtx, signerCfg) + // Sign with lotus-wallet/venus-wallet/other wallet + case config.SignerTypeWallet: + signer, closer, err = newWalletClient(mCtx, signerCfg) + // Signing through venus chain-service + case config.SignerTypeGateway: + if !isServer { + return nil, fmt.Errorf("signing through the venus-gateway cannot be used for market-client") + } + signer, closer, err = newGatewayWalletClient(mCtx, signerCfg, authClient) + default: + return nil, fmt.Errorf("unsupport signer type %s", signerCfg.SignerType) + } + + lc.Append(fx.Hook{ + OnStop: func(_ context.Context) error { + if closer != nil { + closer() + } + return nil + }, + }) + return signer, err + } + +} diff --git a/api/clients/signer/wallet_client.go b/api/clients/signer/wallet_client.go new file mode 100644 index 00000000..fde8406d --- /dev/null +++ b/api/clients/signer/wallet_client.go @@ -0,0 +1,42 @@ +package signer + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/venus-market/v2/config" + + vCrypto "github.com/filecoin-project/venus/pkg/crypto" + "github.com/filecoin-project/venus/venus-shared/api" + vTypes "github.com/filecoin-project/venus/venus-shared/types" +) + +type WalletClient struct { + Internal struct { + WalletHas func(context.Context, address.Address) (bool, error) + WalletSign func(context.Context, address.Address, []byte, vTypes.MsgMeta) (*vCrypto.Signature, error) + } +} + +func (walletClient *WalletClient) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + return walletClient.Internal.WalletHas(ctx, addr) +} + +func (walletClient *WalletClient) WalletSign(ctx context.Context, addr address.Address, msg []byte, meta vTypes.MsgMeta) (*vCrypto.Signature, error) { + return walletClient.Internal.WalletSign(ctx, addr, msg, meta) +} + +func newWalletClient(ctx context.Context, nodeCfg *config.Signer) (ISigner, jsonrpc.ClientCloser, error) { + apiInfo := api.NewAPIInfo(nodeCfg.Url, nodeCfg.Token) + addr, err := apiInfo.DialArgs("v0") + if err != nil { + return nil, nil, err + } + + walletClient := WalletClient{} + closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", []interface{}{&walletClient.Internal}, apiInfo.AuthHeader()) + + return &walletClient, closer, err +} diff --git a/api/impl/market.go b/api/impl/market.go index d25e82b3..a408c1fa 100644 --- a/api/impl/market.go +++ b/api/impl/market.go @@ -3,16 +3,17 @@ package impl import ( "context" - "github.com/filecoin-project/venus-market/v2/api/clients" - "github.com/filecoin-project/venus-market/v2/fundmgr" - v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" - "github.com/ipfs/go-cid" "go.uber.org/fx" "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/venus-market/v2/api/clients" + "github.com/filecoin-project/venus-market/v2/fundmgr" + "github.com/filecoin-project/venus/venus-shared/actors" marketactor "github.com/filecoin-project/venus/venus-shared/actors/builtin/market" + v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" "github.com/filecoin-project/venus/venus-shared/types" ) @@ -30,13 +31,15 @@ func (a *FundAPI) MarketAddBalance(ctx context.Context, wallet, addr address.Add return cid.Undef, err } - msgId, aerr := a.MsgClient.PushMessage(ctx, &types.Message{ - To: marketactor.Address, - From: wallet, - Value: amt, - Method: marketactor.Methods.AddBalance, - Params: params, - }, nil) + msgId, aerr := a.MsgClient.PushMessage( + ctx, + &types.Message{ + To: marketactor.Address, + From: wallet, + Value: amt, + Method: marketactor.Methods.AddBalance, + Params: params, + }, nil) if aerr != nil { return cid.Undef, aerr diff --git a/api/impl/market_clent.go b/api/impl/market_client.go similarity index 93% rename from api/impl/market_clent.go rename to api/impl/market_client.go index 5c16d9ba..727340fe 100644 --- a/api/impl/market_clent.go +++ b/api/impl/market_client.go @@ -3,14 +3,15 @@ package impl import ( "context" - clients2 "github.com/filecoin-project/venus-market/v2/api/clients" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/venus-market/v2/api/clients" "github.com/filecoin-project/venus-market/v2/client" "github.com/filecoin-project/venus-market/v2/version" + "github.com/filecoin-project/venus/pkg/constants" clientapi "github.com/filecoin-project/venus/venus-shared/api/market/client" - vTypes "github.com/filecoin-project/venus/venus-shared/types" - "github.com/ipfs/go-cid" ) var _ clientapi.IMarketClient = (*MarketClientNodeImpl)(nil) @@ -18,7 +19,7 @@ var _ clientapi.IMarketClient = (*MarketClientNodeImpl)(nil) type MarketClientNodeImpl struct { client.API FundAPI - Messager clients2.IMixMessage + Messager clients.IMixMessage } func (m *MarketClientNodeImpl) MessagerWaitMessage(ctx context.Context, mid cid.Cid) (*vTypes.MsgLookup, error) { @@ -34,6 +35,7 @@ func (m *MarketClientNodeImpl) MessagerPushMessage(ctx context.Context, msg *vTy GasOverEstimation: meta.GasOverEstimation, } } + return m.Messager.PushMessage(ctx, msg, spec) } diff --git a/api/impl/market_event.go b/api/impl/market_event.go index 95b6acfa..56591766 100644 --- a/api/impl/market_event.go +++ b/api/impl/market_event.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - api "github.com/filecoin-project/venus/venus-shared/api/gateway/v1" - "github.com/filecoin-project/venus/venus-shared/types/gateway" "go.uber.org/fx" + + v2GatewayAPI "github.com/filecoin-project/venus/venus-shared/api/gateway/v2" + "github.com/filecoin-project/venus/venus-shared/types/gateway" ) type MarketEventAPI struct { fx.In - Event api.IMarketServiceProvider `optional:"true"` + Event v2GatewayAPI.IMarketEvent `optional:"true"` } var errNotSupportGateWayMode = fmt.Errorf("MarketEvent api supported only when it runs in 'solo' mode") diff --git a/api/impl/venus_market.go b/api/impl/venus_market.go index 5af33e33..23a642d1 100644 --- a/api/impl/venus_market.go +++ b/api/impl/venus_market.go @@ -9,8 +9,6 @@ import ( "sort" "time" - "github.com/filecoin-project/go-fil-markets/stores" - "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/host" @@ -25,7 +23,12 @@ import ( "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v8/paych" + + "github.com/filecoin-project/venus-auth/jwtclient" clients2 "github.com/filecoin-project/venus-market/v2/api/clients" "github.com/filecoin-project/venus-market/v2/config" @@ -38,7 +41,6 @@ import ( "github.com/filecoin-project/venus-market/v2/storageprovider" "github.com/filecoin-project/venus-market/v2/version" - "github.com/filecoin-project/go-state-types/builtin/v8/paych" "github.com/filecoin-project/venus/pkg/constants" v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" marketapi "github.com/filecoin-project/venus/venus-shared/api/market" @@ -64,12 +66,14 @@ type MarketNodeImpl struct { DealPublisher *storageprovider.DealPublisher DealAssigner storageprovider.DealAssiger + AuthClient *jwtclient.AuthClient + Messager clients2.IMixMessage StorageAsk storageprovider.IStorageAsk DAGStore *dagstore.DAGStore DAGStoreWrapper stores.DAGStoreWrapper PieceStorageMgr *piecestorage.PieceStorageManager - MinerMgr minermgr.IAddrMgr + UserMgr minermgr.IMinerMgr PaychAPI *paychmgr.PaychAPI Repo repo.Repo Config *config.MarketConfig @@ -87,22 +91,39 @@ type MarketNodeImpl struct { SetConsiderVerifiedStorageDealsConfigFunc config.SetConsiderVerifiedStorageDealsConfigFunc ConsiderUnverifiedStorageDealsConfigFunc config.ConsiderUnverifiedStorageDealsConfigFunc SetConsiderUnverifiedStorageDealsConfigFunc config.SetConsiderUnverifiedStorageDealsConfigFunc - /* SetSealingConfigFunc dtypes.SetSealingConfigFunc - GetSealingConfigFunc dtypes.GetSealingConfigFunc */ + GetExpectedSealDurationFunc config.GetExpectedSealDurationFunc SetExpectedSealDurationFunc config.SetExpectedSealDurationFunc + + GetMaxDealStartDelayFunc config.GetMaxDealStartDelayFunc + SetMaxDealStartDelayFunc config.SetMaxDealStartDelayFunc + + TransferPathFunc config.TransferPathFunc + SetTransferPathFunc config.SetTransferPathFunc + + PublishMsgPeriodConfigFunc config.PublishMsgPeriodConfigFunc + SetPublishMsgPeriodConfigFunc config.SetPublishMsgPeriodConfigFunc + MaxDealsPerPublishMsgFunc config.MaxDealsPerPublishMsgFunc + SetMaxDealsPerPublishMsgFunc config.SetMaxDealsPerPublishMsgFunc + MaxProviderCollateralMultiplierFunc config.MaxProviderCollateralMultiplierFunc + SetMaxProviderCollateralMultiplierFunc config.SetMaxProviderCollateralMultiplierFunc + + MaxPublishDealsFeeFunc config.MaxPublishDealsFeeFunc + SetMaxPublishDealsFeeFunc config.SetMaxPublishDealsFeeFunc + MaxMarketBalanceAddFeeFunc config.MaxMarketBalanceAddFeeFunc + SetMaxMarketBalanceAddFeeFunc config.SetMaxMarketBalanceAddFeeFunc } -func (m *MarketNodeImpl) ActorList(ctx context.Context) ([]types.User, error) { - return m.MinerMgr.ActorList(ctx) +func (m MarketNodeImpl) ActorList(ctx context.Context) ([]types.User, error) { + return m.UserMgr.ActorList(ctx) } -func (m *MarketNodeImpl) ActorExist(ctx context.Context, addr address.Address) (bool, error) { - return m.MinerMgr.Has(ctx, addr), nil +func (m MarketNodeImpl) ActorExist(ctx context.Context, addr address.Address) (bool, error) { + return m.UserMgr.Has(ctx, addr), nil } -func (m *MarketNodeImpl) ActorSectorSize(ctx context.Context, addr address.Address) (abi.SectorSize, error) { - if bHas := m.MinerMgr.Has(ctx, addr); bHas { +func (m MarketNodeImpl) ActorSectorSize(ctx context.Context, addr address.Address) (abi.SectorSize, error) { + if bHas := m.UserMgr.Has(ctx, addr); bHas { minerInfo, err := m.FullNode.StateMinerInfo(ctx, addr, vTypes.EmptyTSK) if err != nil { return 0, err @@ -132,7 +153,8 @@ func (m *MarketNodeImpl) MarketListDeals(ctx context.Context, addrs []address.Ad return m.listDeals(ctx, addrs) } -func (m *MarketNodeImpl) MarketListRetrievalDeals(ctx context.Context, mAddr address.Address) ([]types.ProviderDealState, error) { +// 检索订单没法按 `miner address` 过滤 +func (m *MarketNodeImpl) MarketListRetrievalDeals(ctx context.Context) ([]types.ProviderDealState, error) { var out []types.ProviderDealState deals, err := m.RetrievalProvider.ListDeals(ctx) if err != nil { @@ -145,7 +167,6 @@ func (m *MarketNodeImpl) MarketListRetrievalDeals(ctx context.Context, mAddr add deal.ChannelID = nil // don't try to push unparsable peer IDs over jsonrpc } } - // todo: 按miner过滤交易 out = append(out, *deal) } return out, nil @@ -326,68 +347,188 @@ func (m *MarketNodeImpl) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Ci return &ci, nil } -func (m *MarketNodeImpl) DealsConsiderOnlineStorageDeals(ctx context.Context) (bool, error) { - return m.ConsiderOnlineStorageDealsConfigFunc() +func (m *MarketNodeImpl) permissionVerify(ctx context.Context, mAddr address.Address) error { + if bHas := auth.HasPerm(ctx, []auth.Permission{}, "admin"); bHas { + return nil + } + + name, bExist := jwtclient.CtxGetName(ctx) + if !bExist { + return fmt.Errorf("token illegal") + } + + bExist, err := m.AuthClient.MinerExistInUser(name, mAddr.String()) + if err != nil { + return err + } + + if !bExist { + return fmt.Errorf("the token has no permissions to operate this miner") + } + + return nil } -func (m *MarketNodeImpl) DealsSetConsiderOnlineStorageDeals(ctx context.Context, b bool) error { - return m.SetConsiderOnlineStorageDealsConfigFunc(b) +func (m *MarketNodeImpl) DealsConsiderOnlineStorageDeals(ctx context.Context, mAddr address.Address) (bool, error) { + return m.ConsiderOnlineStorageDealsConfigFunc(mAddr) +} + +func (m *MarketNodeImpl) DealsSetConsiderOnlineStorageDeals(ctx context.Context, mAddr address.Address, b bool) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetConsiderOnlineStorageDealsConfigFunc(mAddr, b) } -func (m *MarketNodeImpl) DealsConsiderOnlineRetrievalDeals(ctx context.Context) (bool, error) { - return m.ConsiderOnlineRetrievalDealsConfigFunc() +func (m *MarketNodeImpl) DealsConsiderOnlineRetrievalDeals(ctx context.Context, mAddr address.Address) (bool, error) { + return m.ConsiderOnlineRetrievalDealsConfigFunc(mAddr) } -func (m *MarketNodeImpl) DealsSetConsiderOnlineRetrievalDeals(ctx context.Context, b bool) error { - return m.SetConsiderOnlineRetrievalDealsConfigFunc(b) +func (m *MarketNodeImpl) DealsSetConsiderOnlineRetrievalDeals(ctx context.Context, mAddr address.Address, b bool) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetConsiderOnlineRetrievalDealsConfigFunc(mAddr, b) } -func (m *MarketNodeImpl) DealsPieceCidBlocklist(ctx context.Context) ([]cid.Cid, error) { - return m.StorageDealPieceCidBlocklistConfigFunc() +func (m *MarketNodeImpl) DealsPieceCidBlocklist(ctx context.Context, mAddr address.Address) ([]cid.Cid, error) { + return m.StorageDealPieceCidBlocklistConfigFunc(mAddr) } -func (m *MarketNodeImpl) DealsSetPieceCidBlocklist(ctx context.Context, cids []cid.Cid) error { - return m.SetStorageDealPieceCidBlocklistConfigFunc(cids) +func (m *MarketNodeImpl) DealsSetPieceCidBlocklist(ctx context.Context, mAddr address.Address, cids []cid.Cid) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetStorageDealPieceCidBlocklistConfigFunc(mAddr, cids) } -func (m *MarketNodeImpl) DealsConsiderOfflineStorageDeals(ctx context.Context) (bool, error) { - return m.ConsiderOfflineStorageDealsConfigFunc() +func (m *MarketNodeImpl) DealsConsiderOfflineStorageDeals(ctx context.Context, mAddr address.Address) (bool, error) { + return m.ConsiderOfflineStorageDealsConfigFunc(mAddr) } -func (m *MarketNodeImpl) DealsSetConsiderOfflineStorageDeals(ctx context.Context, b bool) error { - return m.SetConsiderOfflineStorageDealsConfigFunc(b) +func (m *MarketNodeImpl) DealsSetConsiderOfflineStorageDeals(ctx context.Context, mAddr address.Address, b bool) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetConsiderOfflineStorageDealsConfigFunc(mAddr, b) } -func (m *MarketNodeImpl) DealsConsiderOfflineRetrievalDeals(ctx context.Context) (bool, error) { - return m.ConsiderOfflineRetrievalDealsConfigFunc() +func (m *MarketNodeImpl) DealsConsiderOfflineRetrievalDeals(ctx context.Context, mAddr address.Address) (bool, error) { + return m.ConsiderOfflineRetrievalDealsConfigFunc(mAddr) } -func (m *MarketNodeImpl) DealsSetConsiderOfflineRetrievalDeals(ctx context.Context, b bool) error { - return m.SetConsiderOfflineRetrievalDealsConfigFunc(b) +func (m *MarketNodeImpl) DealsSetConsiderOfflineRetrievalDeals(ctx context.Context, mAddr address.Address, b bool) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetConsiderOfflineRetrievalDealsConfigFunc(mAddr, b) } -func (m *MarketNodeImpl) DealsConsiderVerifiedStorageDeals(ctx context.Context) (bool, error) { - return m.ConsiderVerifiedStorageDealsConfigFunc() +func (m *MarketNodeImpl) DealsConsiderVerifiedStorageDeals(ctx context.Context, mAddr address.Address) (bool, error) { + return m.ConsiderVerifiedStorageDealsConfigFunc(mAddr) } -func (m *MarketNodeImpl) DealsSetConsiderVerifiedStorageDeals(ctx context.Context, b bool) error { - return m.SetConsiderVerifiedStorageDealsConfigFunc(b) +func (m *MarketNodeImpl) DealsSetConsiderVerifiedStorageDeals(ctx context.Context, mAddr address.Address, b bool) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetConsiderVerifiedStorageDealsConfigFunc(mAddr, b) } -func (m *MarketNodeImpl) DealsConsiderUnverifiedStorageDeals(ctx context.Context) (bool, error) { - return m.ConsiderUnverifiedStorageDealsConfigFunc() +func (m *MarketNodeImpl) DealsConsiderUnverifiedStorageDeals(ctx context.Context, mAddr address.Address) (bool, error) { + return m.ConsiderUnverifiedStorageDealsConfigFunc(mAddr) } -func (m *MarketNodeImpl) DealsSetConsiderUnverifiedStorageDeals(ctx context.Context, b bool) error { - return m.SetConsiderUnverifiedStorageDealsConfigFunc(b) +func (m *MarketNodeImpl) DealsSetConsiderUnverifiedStorageDeals(ctx context.Context, mAddr address.Address, b bool) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetConsiderUnverifiedStorageDealsConfigFunc(mAddr, b) } -func (m *MarketNodeImpl) SectorGetSealDelay(ctx context.Context) (time.Duration, error) { - return m.GetExpectedSealDurationFunc() +func (m *MarketNodeImpl) SectorGetExpectedSealDuration(ctx context.Context, mAddr address.Address) (time.Duration, error) { + return m.GetExpectedSealDurationFunc(mAddr) } -func (m *MarketNodeImpl) SectorSetExpectedSealDuration(ctx context.Context, duration time.Duration) error { - return m.SetExpectedSealDurationFunc(duration) +func (m *MarketNodeImpl) SectorSetExpectedSealDuration(ctx context.Context, mAddr address.Address, duration time.Duration) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetExpectedSealDurationFunc(mAddr, duration) +} + +func (m *MarketNodeImpl) DealsMaxStartDelay(ctx context.Context, mAddr address.Address) (time.Duration, error) { + return m.GetMaxDealStartDelayFunc(mAddr) +} + +func (m *MarketNodeImpl) DealsSetMaxStartDelay(ctx context.Context, mAddr address.Address, duration time.Duration) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetMaxDealStartDelayFunc(mAddr, duration) +} + +func (m *MarketNodeImpl) DealsPublishMsgPeriod(ctx context.Context, mAddr address.Address) (time.Duration, error) { + return m.PublishMsgPeriodConfigFunc(mAddr) +} + +func (m *MarketNodeImpl) DealsSetPublishMsgPeriod(ctx context.Context, mAddr address.Address, duration time.Duration) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetPublishMsgPeriodConfigFunc(mAddr, duration) +} + +func (m *MarketNodeImpl) MarketDataTransferPath(ctx context.Context, mAddr address.Address) (string, error) { + return m.TransferPathFunc(mAddr) +} + +func (m *MarketNodeImpl) MarketDataSetTransferPath(ctx context.Context, mAddr address.Address, path string) error { + return m.SetTransferPathFunc(mAddr, path) +} + +func (m *MarketNodeImpl) MarketMaxDealsPerPublishMsg(ctx context.Context, mAddr address.Address) (uint64, error) { + return m.MaxDealsPerPublishMsgFunc(mAddr) +} + +func (m *MarketNodeImpl) MarketSetMaxDealsPerPublishMsg(ctx context.Context, mAddr address.Address, num uint64) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetMaxDealsPerPublishMsgFunc(mAddr, num) +} + +func (m *MarketNodeImpl) DealsMaxProviderCollateralMultiplier(ctx context.Context, mAddr address.Address) (uint64, error) { + return m.MaxProviderCollateralMultiplierFunc(mAddr) +} + +func (m *MarketNodeImpl) DealsSetMaxProviderCollateralMultiplier(ctx context.Context, mAddr address.Address, c uint64) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetMaxProviderCollateralMultiplierFunc(mAddr, c) +} + +func (m *MarketNodeImpl) DealsMaxPublishFee(ctx context.Context, mAddr address.Address) (vTypes.FIL, error) { + return m.MaxPublishDealsFeeFunc(mAddr) +} + +func (m *MarketNodeImpl) DealsSetMaxPublishFee(ctx context.Context, mAddr address.Address, fee vTypes.FIL) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetMaxPublishDealsFeeFunc(mAddr, fee) +} + +func (m *MarketNodeImpl) MarketMaxBalanceAddFee(ctx context.Context, mAddr address.Address) (vTypes.FIL, error) { + return m.MaxMarketBalanceAddFeeFunc(mAddr) +} + +func (m *MarketNodeImpl) MarketSetMaxBalanceAddFee(ctx context.Context, mAddr address.Address, fee vTypes.FIL) error { + if err := m.permissionVerify(ctx, mAddr); err != nil { + return err + } + return m.SetMaxMarketBalanceAddFeeFunc(mAddr, fee) } func (m *MarketNodeImpl) MessagerWaitMessage(ctx context.Context, mid cid.Cid) (*vTypes.MsgLookup, error) { @@ -434,7 +575,7 @@ func (m *MarketNodeImpl) listDeals(ctx context.Context, addrs []address.Address) } for _, deal := range allDeals { - if m.MinerMgr.Has(ctx, deal.Proposal.Provider) && has(deal.Proposal.Provider) { + if m.UserMgr.Has(ctx, deal.Proposal.Provider) && has(deal.Proposal.Provider) { out = append(out, deal) } } diff --git a/cli/actor.go b/cli/actor.go index 354216e2..cbc1bfaa 100644 --- a/cli/actor.go +++ b/cli/actor.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin" miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/venus/venus-shared/actors" "github.com/filecoin-project/venus/venus-shared/types" ) diff --git a/cli/market.go b/cli/market.go index a4ad7757..18a60193 100644 --- a/cli/market.go +++ b/cli/market.go @@ -17,28 +17,31 @@ import ( "text/tabwriter" "time" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/venus-market/v2/storageprovider" - "github.com/filecoin-project/venus/venus-shared/types/market" - tm "github.com/buger/goterm" "github.com/docker/go-units" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" "github.com/urfave/cli/v2" + "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/venus-market/v2/storageprovider" + "github.com/filecoin-project/venus/pkg/constants" "github.com/filecoin-project/venus/venus-shared/types" + "github.com/filecoin-project/venus/venus-shared/types/market" ) var storageDealSelectionCmd = &cli.Command{ Name: "selection", Usage: "Configure acceptance criteria for storage deal proposals", + Flags: []cli.Flag{ + minerFlag, + }, Subcommands: []*cli.Command{ storageDealSelectionShowCmd, storageDealSelectionResetCmd, @@ -50,24 +53,41 @@ var storageDealSelectionShowCmd = &cli.Command{ Name: "list", Usage: "List storage deal proposal selection criteria", Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + smapi, closer, err := NewMarketNode(cctx) if err != nil { return err } defer closer() - onlineOk, err := smapi.DealsConsiderOnlineStorageDeals(DaemonContext(cctx)) + onlineOk, err := smapi.DealsConsiderOnlineStorageDeals(DaemonContext(cctx), mAddr) + if err != nil { + return err + } + + offlineOk, err := smapi.DealsConsiderOfflineStorageDeals(DaemonContext(cctx), mAddr) + if err != nil { + return err + } + + verifiedOk, err := smapi.DealsConsiderVerifiedStorageDeals(DaemonContext(cctx), mAddr) if err != nil { return err } - offlineOk, err := smapi.DealsConsiderOfflineStorageDeals(DaemonContext(cctx)) + unverifiedOk, err := smapi.DealsConsiderUnverifiedStorageDeals(DaemonContext(cctx), mAddr) if err != nil { return err } fmt.Printf("considering online storage deals: %t\n", onlineOk) fmt.Printf("considering offline storage deals: %t\n", offlineOk) + fmt.Printf("considering verified storage deals: %t\n", verifiedOk) + fmt.Printf("considering unverified storage deals: %t\n", unverifiedOk) return nil }, @@ -77,28 +97,33 @@ var storageDealSelectionResetCmd = &cli.Command{ Name: "reset", Usage: "Reset storage deal proposal selection criteria to default values", Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + smapi, closer, err := NewMarketNode(cctx) if err != nil { return err } defer closer() - err = smapi.DealsSetConsiderOnlineStorageDeals(DaemonContext(cctx), true) + err = smapi.DealsSetConsiderOnlineStorageDeals(DaemonContext(cctx), mAddr, true) if err != nil { return err } - err = smapi.DealsSetConsiderOfflineStorageDeals(DaemonContext(cctx), true) + err = smapi.DealsSetConsiderOfflineStorageDeals(DaemonContext(cctx), mAddr, true) if err != nil { return err } - err = smapi.DealsSetConsiderVerifiedStorageDeals(DaemonContext(cctx), true) + err = smapi.DealsSetConsiderVerifiedStorageDeals(DaemonContext(cctx), mAddr, true) if err != nil { return err } - err = smapi.DealsSetConsiderUnverifiedStorageDeals(DaemonContext(cctx), true) + err = smapi.DealsSetConsiderUnverifiedStorageDeals(DaemonContext(cctx), mAddr, true) if err != nil { return err } @@ -125,6 +150,11 @@ var storageDealSelectionRejectCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + smapi, closer, err := NewMarketNode(cctx) if err != nil { return err @@ -132,28 +162,28 @@ var storageDealSelectionRejectCmd = &cli.Command{ defer closer() if cctx.Bool("online") { - err = smapi.DealsSetConsiderOnlineStorageDeals(DaemonContext(cctx), false) + err = smapi.DealsSetConsiderOnlineStorageDeals(DaemonContext(cctx), mAddr, false) if err != nil { return err } } if cctx.Bool("offline") { - err = smapi.DealsSetConsiderOfflineStorageDeals(DaemonContext(cctx), false) + err = smapi.DealsSetConsiderOfflineStorageDeals(DaemonContext(cctx), mAddr, false) if err != nil { return err } } if cctx.Bool("verified") { - err = smapi.DealsSetConsiderVerifiedStorageDeals(DaemonContext(cctx), false) + err = smapi.DealsSetConsiderVerifiedStorageDeals(DaemonContext(cctx), mAddr, false) if err != nil { return err } } if cctx.Bool("unverified") { - err = smapi.DealsSetConsiderUnverifiedStorageDeals(DaemonContext(cctx), false) + err = smapi.DealsSetConsiderUnverifiedStorageDeals(DaemonContext(cctx), mAddr, false) if err != nil { return err } @@ -334,7 +364,9 @@ var StorageDealsCmd = &cli.Command{ setBlocklistCmd, getBlocklistCmd, resetBlocklistCmd, - setSealDurationCmd, + expectedSealDurationCmd, + maxDealStartDelayCmd, + dealsPublishMsgPeriodCmd, dealsPendingPublish, }, } @@ -657,15 +689,21 @@ var getBlocklistCmd = &cli.Command{ Usage: "List the contents of the miner's piece CID blocklist", Flags: []cli.Flag{ &CidBaseFlag, + minerFlag, }, Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + api, closer, err := NewMarketNode(cctx) if err != nil { return err } defer closer() - blocklist, err := api.DealsPieceCidBlocklist(DaemonContext(cctx)) + blocklist, err := api.DealsPieceCidBlocklist(DaemonContext(cctx), mAddr) if err != nil { return err } @@ -687,8 +725,15 @@ var setBlocklistCmd = &cli.Command{ Name: "set-blocklist", Usage: "Set the miner's list of blocklisted piece CIDs", ArgsUsage: "[ (optional, will read from stdin if omitted)]", - Flags: []cli.Flag{}, + Flags: []cli.Flag{ + minerFlag, + }, Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + api, closer, err := NewMarketNode(cctx) if err != nil { return err @@ -726,30 +771,146 @@ var setBlocklistCmd = &cli.Command{ return err } - return api.DealsSetPieceCidBlocklist(DaemonContext(cctx), blocklist) + return api.DealsSetPieceCidBlocklist(DaemonContext(cctx), mAddr, blocklist) }, } var resetBlocklistCmd = &cli.Command{ Name: "reset-blocklist", Usage: "Remove all entries from the miner's piece CID blocklist", - Flags: []cli.Flag{}, + Flags: []cli.Flag{ + minerFlag, + }, Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + api, closer, err := NewMarketNode(cctx) if err != nil { return err } defer closer() - return api.DealsSetPieceCidBlocklist(DaemonContext(cctx), []cid.Cid{}) + return api.DealsSetPieceCidBlocklist(DaemonContext(cctx), mAddr, []cid.Cid{}) }, } -var setSealDurationCmd = &cli.Command{ +var expectedSealDurationCmd = &cli.Command{ + Name: "seal-duration", + Usage: "Configure the expected time, that you expect sealing sectors to take. Deals that start before this duration will be rejected.", + Flags: []cli.Flag{ + minerFlag, + }, + Subcommands: []*cli.Command{ + expectedSealDurationGetCmd, + expectedSealDurationSetCmd, + }, +} + +var expectedSealDurationGetCmd = &cli.Command{ + Name: "get", + Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + + marketApi, closer, err := NewMarketNode(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + t, err := marketApi.SectorGetExpectedSealDuration(ctx, mAddr) + if err != nil { + return err + } + fmt.Println("seal-duration: ", t.String()) + return nil + }, +} + +var expectedSealDurationSetCmd = &cli.Command{ Name: "set-seal-duration", - Usage: "Set the expected time, in minutes, that you expect sealing sectors to take. Deals that start before this duration will be rejected.", + Usage: "eg. '1m','30s',...", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + + marketApi, closer, err := NewMarketNode(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + if cctx.Args().Len() != 1 { + return fmt.Errorf("must pass duration") + } + + d, err := time.ParseDuration(cctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("could not parse duration: %w", err) + } + + return marketApi.SectorSetExpectedSealDuration(ctx, mAddr, d) + }, +} + +var maxDealStartDelayCmd = &cli.Command{ + Name: "max-start-delay", + Usage: "Configure the maximum amount of time proposed deal StartEpoch can be in future.", + Flags: []cli.Flag{ + minerFlag, + }, + Subcommands: []*cli.Command{ + maxDealStartDelayGetCmd, + maxDealStartDelaySetCmd, + }, +} + +var maxDealStartDelayGetCmd = &cli.Command{ + Name: "get", + Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + + marketApi, closer, err := NewMarketNode(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + t, err := marketApi.DealsMaxStartDelay(ctx, mAddr) + if err != nil { + return err + } + fmt.Println("max start delay: ", t.String()) + return nil + }, +} + +var maxDealStartDelaySetCmd = &cli.Command{ + Name: "set", + Usage: "eg. '1m','30s',...", ArgsUsage: "", + Flags: []cli.Flag{ + minerFlag, + }, Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + marketApi, closer, err := NewMarketNode(cctx) if err != nil { return err @@ -757,17 +918,82 @@ var setSealDurationCmd = &cli.Command{ defer closer() ctx := ReqContext(cctx) if cctx.Args().Len() != 1 { - return fmt.Errorf("must pass duration in minutes") + return fmt.Errorf("must pass duration") } - hs, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64) + delay, err := time.ParseDuration(cctx.Args().Get(0)) if err != nil { return fmt.Errorf("could not parse duration: %w", err) } - delay := hs * uint64(time.Minute) + return marketApi.DealsSetMaxStartDelay(ctx, mAddr, delay) + }, +} + +var dealsPublishMsgPeriodCmd = &cli.Command{ + Name: "max-start-delay", + Usage: "Configure the the amount of time to wait for more deals to be ready to publish before publishing them all as a batch.", + Flags: []cli.Flag{ + minerFlag, + }, + Subcommands: []*cli.Command{ + dealsPublishMsgPeriodGetCmd, + dealsPublishMsgPeriodSetCmd, + }, +} + +var dealsPublishMsgPeriodGetCmd = &cli.Command{ + Name: "get", + Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + + marketApi, closer, err := NewMarketNode(cctx) + if err != nil { + return err + } + defer closer() - return marketApi.SectorSetExpectedSealDuration(ctx, time.Duration(delay)) + ctx := ReqContext(cctx) + t, err := marketApi.DealsPublishMsgPeriod(ctx, mAddr) + if err != nil { + return err + } + fmt.Println("publish msg period: ", t.String()) + return nil + }, +} + +var dealsPublishMsgPeriodSetCmd = &cli.Command{ + Name: "set", + Usage: "eg. '1m','30s',...", + ArgsUsage: "", + Flags: []cli.Flag{ + minerFlag, + }, + Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + + marketApi, closer, err := NewMarketNode(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + if cctx.Args().Len() != 1 { + return fmt.Errorf("must pass duration") + } + + period, err := time.ParseDuration(cctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("could not parse duration: %w", err) + } + return marketApi.DealsSetPublishMsgPeriod(ctx, mAddr, period) }, } diff --git a/cli/retrieval-deals.go b/cli/retrieval-deals.go index 42c9676d..96cd45c9 100644 --- a/cli/retrieval-deals.go +++ b/cli/retrieval-deals.go @@ -39,19 +39,27 @@ var retrievalDealSelectionCmd = &cli.Command{ var retrievalDealSelectionShowCmd = &cli.Command{ Name: "list", Usage: "List retrieval deal proposal selection criteria", + Flags: []cli.Flag{ + minerFlag, + }, Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + smapi, closer, err := NewMarketNode(cctx) if err != nil { return err } defer closer() - onlineOk, err := smapi.DealsConsiderOnlineRetrievalDeals(DaemonContext(cctx)) + onlineOk, err := smapi.DealsConsiderOnlineRetrievalDeals(DaemonContext(cctx), mAddr) if err != nil { return err } - offlineOk, err := smapi.DealsConsiderOfflineRetrievalDeals(DaemonContext(cctx)) + offlineOk, err := smapi.DealsConsiderOfflineRetrievalDeals(DaemonContext(cctx), mAddr) if err != nil { return err } @@ -66,19 +74,27 @@ var retrievalDealSelectionShowCmd = &cli.Command{ var retrievalDealSelectionResetCmd = &cli.Command{ Name: "reset", Usage: "Reset retrieval deal proposal selection criteria to default values", + Flags: []cli.Flag{ + minerFlag, + }, Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + smapi, closer, err := NewMarketNode(cctx) if err != nil { return err } defer closer() - err = smapi.DealsSetConsiderOnlineRetrievalDeals(DaemonContext(cctx), true) + err = smapi.DealsSetConsiderOnlineRetrievalDeals(DaemonContext(cctx), mAddr, true) if err != nil { return err } - err = smapi.DealsSetConsiderOfflineRetrievalDeals(DaemonContext(cctx), true) + err = smapi.DealsSetConsiderOfflineRetrievalDeals(DaemonContext(cctx), mAddr, true) if err != nil { return err } @@ -97,8 +113,14 @@ var retrievalDealSelectionRejectCmd = &cli.Command{ &cli.BoolFlag{ Name: "offline", }, + minerFlag, }, Action: func(cctx *cli.Context) error { + mAddr, err := shouldAddress(cctx.String("miner"), false, false) + if err != nil { + return fmt.Errorf("invalid miner address: %w", err) + } + smapi, closer, err := NewMarketNode(cctx) if err != nil { return err @@ -106,14 +128,14 @@ var retrievalDealSelectionRejectCmd = &cli.Command{ defer closer() if cctx.Bool("online") { - err = smapi.DealsSetConsiderOnlineRetrievalDeals(DaemonContext(cctx), false) + err = smapi.DealsSetConsiderOnlineRetrievalDeals(DaemonContext(cctx), mAddr, false) if err != nil { return err } } if cctx.Bool("offline") { - err = smapi.DealsSetConsiderOfflineRetrievalDeals(DaemonContext(cctx), false) + err = smapi.DealsSetConsiderOfflineRetrievalDeals(DaemonContext(cctx), mAddr, false) if err != nil { return err } @@ -126,9 +148,6 @@ var retrievalDealSelectionRejectCmd = &cli.Command{ var retrievalDealsListCmd = &cli.Command{ Name: "list", Usage: "List all active retrieval deals for this miner", - Flags: []cli.Flag{ - minerFlag, - }, Action: func(cctx *cli.Context) error { api, closer, err := NewMarketNode(cctx) if err != nil { @@ -136,15 +155,7 @@ var retrievalDealsListCmd = &cli.Command{ } defer closer() - var mAddr address.Address - if cctx.IsSet("miner") { - mAddr, err = address.NewFromString(cctx.String("miner")) - if err != nil { - return err - } - } - - deals, err := api.MarketListRetrievalDeals(DaemonContext(cctx), mAddr) + deals, err := api.MarketListRetrievalDeals(DaemonContext(cctx)) if err != nil { return err } diff --git a/cli/util.go b/cli/util.go index d30c74ba..84d0f16a 100644 --- a/cli/util.go +++ b/cli/util.go @@ -9,6 +9,7 @@ import ( "os/signal" "path" "sort" + "strconv" "strings" "syscall" @@ -20,15 +21,18 @@ import ( "github.com/multiformats/go-multibase" "github.com/urfave/cli/v2" + "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/venus-market/v2/cli/tablewriter" "github.com/filecoin-project/venus-market/v2/config" + + "github.com/filecoin-project/venus/venus-shared/api" v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" marketapi "github.com/filecoin-project/venus/venus-shared/api/market" clientapi "github.com/filecoin-project/venus/venus-shared/api/market/client" types "github.com/filecoin-project/venus/venus-shared/types/market" - "github.com/ipfs-force-community/venus-common-utils/apiinfo" ) var CidBaseFlag = cli.StringFlag{ @@ -86,7 +90,7 @@ func NewMarketNode(cctx *cli.Context) (marketapi.IMarket, jsonrpc.ClientCloser, if err != nil { return nil, nil, err } - apiInfo := apiinfo.NewAPIInfo(string(apiUrl), string(token)) + apiInfo := api.NewAPIInfo(string(apiUrl), string(token)) addr, err := apiInfo.DialArgs("v0") if err != nil { return nil, nil, err @@ -109,7 +113,7 @@ func NewMarketClientNode(cctx *cli.Context) (clientapi.IMarketClient, jsonrpc.Cl if err != nil { return nil, nil, err } - apiInfo := apiinfo.NewAPIInfo(string(apiUrl), string(token)) + apiInfo := api.NewAPIInfo(string(apiUrl), string(token)) addr, err := apiInfo.DialArgs("v0") if err != nil { return nil, nil, err @@ -125,7 +129,7 @@ func NewFullNode(cctx *cli.Context) (v1api.FullNode, jsonrpc.ClientCloser, error if err != nil { return nil, nil, err } - apiInfo := apiinfo.NewAPIInfo(marketCfg.Node.Url, marketCfg.Node.Token) + apiInfo := api.NewAPIInfo(marketCfg.Node.Url, marketCfg.Node.Token) addr, err := apiInfo.DialArgs("v1") if err != nil { return nil, nil, err @@ -328,3 +332,18 @@ func (a *AppFmt) GetScret(prompt string, isMasked bool) (string, error) { } return string(pw), nil } + +func shouldAddress(s string, checkEmpty bool, allowActor bool) (address.Address, error) { + if checkEmpty && s == "" { + return address.Undef, fmt.Errorf("empty address string") + } + + if allowActor { + id, err := strconv.ParseUint(s, 10, 64) + if err == nil { + return address.NewIDAddress(id) + } + } + + return address.NewFromString(s) +} diff --git a/client/client.go b/client/client.go index bb9ff843..290379cf 100644 --- a/client/client.go +++ b/client/client.go @@ -12,37 +12,36 @@ import ( "strings" "time" - "github.com/filecoin-project/venus-auth/log" - format "github.com/ipfs/go-ipld-format" - "github.com/ipld/go-car/util" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/datamodel" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/traversal" - selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" - - bstore "github.com/ipfs/go-ipfs-blockstore" - unixfile "github.com/ipfs/go-unixfs/file" - "github.com/ipld/go-car" - carv2 "github.com/ipld/go-car/v2" - "github.com/ipld/go-car/v2/blockstore" + "go.uber.org/fx" - "github.com/filecoin-project/go-padreader" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/dline" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" + bstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" files "github.com/ipfs/go-ipfs-files" + format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + unixfile "github.com/ipfs/go-unixfs/file" + + "github.com/ipld/go-car" + "github.com/ipld/go-car/util" + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/blockstore" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + textselector "github.com/ipld/go-ipld-selector-text-lite" + "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multibase" mh "github.com/multiformats/go-multihash" - "go.uber.org/fx" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" @@ -54,22 +53,27 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/builtin/v9/market" - textselector "github.com/ipld/go-ipld-selector-text-lite" + "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/venus-market/v2/api/clients/signer" "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/imports" + marketNetwork "github.com/filecoin-project/venus-market/v2/network" "github.com/filecoin-project/venus-market/v2/retrievalprovider" "github.com/filecoin-project/venus-market/v2/storageprovider" - types2 "github.com/filecoin-project/venus/venus-shared/types/market" - - marketNetwork "github.com/filecoin-project/venus-market/v2/network" "github.com/filecoin-project/venus-market/v2/utils" + + "github.com/filecoin-project/venus-auth/log" + "github.com/filecoin-project/venus/pkg/constants" "github.com/filecoin-project/venus/venus-shared/actors/builtin/miner" v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" vTypes "github.com/filecoin-project/venus/venus-shared/types" + types2 "github.com/filecoin-project/venus/venus-shared/types/market" types "github.com/filecoin-project/venus/venus-shared/types/market/client" ) @@ -95,6 +99,8 @@ type API struct { DataTransfer marketNetwork.ClientDataTransfer Host host.Host Cfg *config.MarketClientConfig + + Signer signer.ISigner } func calcDealExpiration(minDuration uint64, md *dline.Info, startEpoch abi.ChainEpoch) abi.ChainEpoch { @@ -150,7 +156,7 @@ func (a *API) dealStarter(ctx context.Context, params *types.StartDealParams, is return nil, fmt.Errorf("failed resolving params.Wallet addr (%s): %w", params.Wallet, err) } - exist, err := a.Full.WalletHas(ctx, walletKey) + exist, err := a.Signer.WalletHas(ctx, walletKey) if err != nil { return nil, fmt.Errorf("failed getting addr from signer (%s): %w", params.Wallet, err) } @@ -252,10 +258,11 @@ func (a *API) dealStarter(ctx context.Context, params *types.StartDealParams, is return nil, fmt.Errorf("failed to serialize deal proposal: %w", err) } - dealProposalSig, err := a.Full.WalletSign(ctx, walletKey, dealProposalSerialized, vTypes.MsgMeta{ - Type: vTypes.MTDealProposal, - Extra: dealProposalSerialized, - }) + dealProposalSig, err := a.Signer.WalletSign(ctx, walletKey, dealProposalSerialized, + vTypes.MsgMeta{ + Type: vTypes.MTDealProposal, + Extra: dealProposalSerialized, + }) if err != nil { return nil, fmt.Errorf("failed to sign proposal : %w", err) } diff --git a/cmd/load_builtin_actor.go b/cmd/load_builtin_actor.go index 40d1bfab..7a7a853e 100644 --- a/cmd/load_builtin_actor.go +++ b/cmd/load_builtin_actor.go @@ -4,13 +4,14 @@ import ( "context" "github.com/filecoin-project/venus-market/v2/config" + + "github.com/filecoin-project/venus/venus-shared/api" v1 "github.com/filecoin-project/venus/venus-shared/api/chain/v1" "github.com/filecoin-project/venus/venus-shared/utils" - "github.com/ipfs-force-community/venus-common-utils/apiinfo" ) func FetchAndLoadBundles(ctx context.Context, nodeCfg config.Node) error { - apiInfo := apiinfo.NewAPIInfo(nodeCfg.Url, nodeCfg.Token) + apiInfo := api.NewAPIInfo(nodeCfg.Url, nodeCfg.Token) addr, err := apiInfo.DialArgs("v1") if err != nil { return err diff --git a/cmd/market-client/main.go b/cmd/market-client/main.go index 4b0ac517..ba3b6290 100644 --- a/cmd/market-client/main.go +++ b/cmd/market-client/main.go @@ -5,21 +5,19 @@ import ( "fmt" "os" - clients2 "github.com/filecoin-project/venus-market/v2/api/clients" - "github.com/filecoin-project/venus-market/v2/cmd" - clientapi "github.com/filecoin-project/venus/venus-shared/api/market/client" "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" - metrics2 "github.com/ipfs/go-metrics-interface" "github.com/urfave/cli/v2" "go.uber.org/fx" "github.com/filecoin-project/go-address" + clients2 "github.com/filecoin-project/venus-market/v2/api/clients" "github.com/filecoin-project/venus-market/v2/api/impl" cli2 "github.com/filecoin-project/venus-market/v2/cli" "github.com/filecoin-project/venus-market/v2/client" + "github.com/filecoin-project/venus-market/v2/cmd" "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/fundmgr" "github.com/filecoin-project/venus-market/v2/models" @@ -29,15 +27,17 @@ import ( "github.com/filecoin-project/venus-market/v2/storageprovider" types2 "github.com/filecoin-project/venus-market/v2/types" "github.com/filecoin-project/venus-market/v2/utils" - "github.com/filecoin-project/venus/venus-shared/api/permission" - "github.com/filecoin-project/venus-market/v2/version" + _ "github.com/filecoin-project/venus/pkg/crypto/bls" _ "github.com/filecoin-project/venus/pkg/crypto/secp" "github.com/ipfs-force-community/metrics" "github.com/ipfs-force-community/venus-common-utils/builder" "github.com/ipfs-force-community/venus-common-utils/journal" + + clientapi "github.com/filecoin-project/venus/venus-shared/api/market/client" + "github.com/filecoin-project/venus/venus-shared/api/permission" ) var ( @@ -76,12 +76,18 @@ var ( Usage: "token used to connect venus chain service components, eg. venus-meassger, venus", } + SignerTypeFlag = &cli.StringFlag{ + Name: "signer-type", + Usage: "signer service type(lotusnode, wallet)", + Value: config.SignerTypeWallet, + Hidden: false, + } SignerUrlFlag = &cli.StringFlag{ - Name: "wallet-url", + Name: "signer-url", Usage: "used to connect wallet service for sign", } SignerTokenFlag = &cli.StringFlag{ - Name: "wallet-token", + Name: "signer-token", Usage: "wallet token for connect signer service", } @@ -121,6 +127,7 @@ func main() { MessagerUrlFlag, MessagerTokenFlag, AuthTokenFlag, + SignerTypeFlag, SignerUrlFlag, SignerTokenFlag, DefaultAddressFlag, @@ -136,6 +143,7 @@ func main() { func flagData(cctx *cli.Context, cfg *config.MarketClientConfig) error { if cctx.IsSet(NodeUrlFlag.Name) { cfg.Node.Url = cctx.String(NodeUrlFlag.Name) + cfg.Signer.Url = cctx.String(NodeUrlFlag.Name) } if cctx.IsSet(MessagerUrlFlag.Name) { @@ -153,21 +161,37 @@ func flagData(cctx *cli.Context, cfg *config.MarketClientConfig) error { if cctx.IsSet(NodeTokenFlag.Name) { cfg.Node.Token = cctx.String(NodeTokenFlag.Name) + cfg.Signer.Token = cctx.String(NodeTokenFlag.Name) } if cctx.IsSet(MessagerTokenFlag.Name) { cfg.Messager.Token = cctx.String(MessagerTokenFlag.Name) } - if cctx.IsSet(SignerUrlFlag.Name) { - if !cctx.IsSet(SignerTokenFlag.Name) { - return fmt.Errorf("signer-url is set, but signer-token is not set") + signerType := cctx.String(SignerTypeFlag.Name) + switch signerType { + case config.SignerTypeWallet: + { + if cctx.IsSet(SignerUrlFlag.Name) { + cfg.Signer.Url = cctx.String(SignerUrlFlag.Name) + } + if cctx.IsSet(SignerTokenFlag.Name) { + cfg.Signer.Token = cctx.String(SignerTokenFlag.Name) + } } - - cfg.Signer.SignerType = "wallet" - cfg.Signer.Url = cctx.String(SignerUrlFlag.Name) - cfg.Signer.Token = cctx.String(SignerTokenFlag.Name) + case config.SignerTypeLotusnode: + { + if cctx.IsSet(NodeUrlFlag.Name) { + cfg.Signer.Url = cctx.String(NodeUrlFlag.Name) + } + if cctx.IsSet(NodeTokenFlag.Name) { + cfg.Signer.Token = cctx.String(NodeTokenFlag.Name) + } + } + default: + return fmt.Errorf("unsupport signer type %s", signerType) } + cfg.Signer.SignerType = signerType if cctx.IsSet(DefaultAddressFlag.Name) { addr, err := address.NewFromString(cctx.String(DefaultAddressFlag.Name)) @@ -224,6 +248,12 @@ func marketClient(cctx *cli.Context) error { if err != nil { return err } + + // Configuration sanity check + if len(cfg.Signer.Url) == 0 { + return fmt.Errorf("the signer node must be configured") + } + if err := cmd.FetchAndLoadBundles(cctx.Context, cfg.Node); err != nil { return err } diff --git a/cmd/market-client/storage.go b/cmd/market-client/storage.go index f26e99d1..23b6f760 100644 --- a/cmd/market-client/storage.go +++ b/cmd/market-client/storage.go @@ -32,7 +32,6 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin" "github.com/filecoin-project/venus/venus-shared/actors/builtin" "github.com/filecoin-project/venus/venus-shared/actors/builtin/market" @@ -61,7 +60,7 @@ var storageCmd = &cli.Command{ // nolint var MinDealDuration, MaxDealDuration = policy.DealDurationBounds(0) -const BlockDelaySecs = uint64(builtin7.EpochDurationSeconds) +const BlockDelaySecs = uint64(builtin.EpochDurationSeconds) type QueriedAsk struct { Ask *storagemarket.StorageAsk diff --git a/cmd/venus-market/main.go b/cmd/venus-market/main.go index 82cf7d68..dc4e1393 100644 --- a/cmd/venus-market/main.go +++ b/cmd/venus-market/main.go @@ -1,22 +1,15 @@ package main import ( - "fmt" "log" "os" - "strings" - - "github.com/filecoin-project/venus-market/v2/cmd" logging "github.com/ipfs/go-log/v2" "github.com/urfave/cli/v2" - "github.com/filecoin-project/go-address" - "github.com/ipfs-force-community/venus-common-utils/builder" cli2 "github.com/filecoin-project/venus-market/v2/cli" - "github.com/filecoin-project/venus-market/v2/config" _ "github.com/filecoin-project/venus-market/v2/network" "github.com/filecoin-project/venus-market/v2/version" @@ -29,8 +22,7 @@ var mainLog = logging.Logger("main") // Invokes are called in the order they are defined. // nolint:golint var ( - InitJournalKey = builder.NextInvoke() //nolint - ExtractApiKey = builder.NextInvoke() + ExtractApiKey = builder.NextInvoke() ) var ( @@ -69,17 +61,10 @@ var ( } SignerTypeFlag = &cli.StringFlag{ - Name: "signer-type", - Usage: "signer service type（wallet, gateway）", - DefaultText: "wallet", - } - HidenSignerTypeFlag = &cli.StringFlag{ - Name: "signer-type", - Usage: "signer service type（wallet, gateway）", - DefaultText: "wallet", - Hidden: true, + Name: "signer-type", + Usage: "signer service type(lotusnode, wallet, gateway)", + Hidden: false, } - SignerUrlFlag = &cli.StringFlag{ Name: "signer-url", Usage: "used to connect signer service for sign", @@ -90,25 +75,12 @@ var ( } GatewayUrlFlag = &cli.StringFlag{ - Name: "gateway-url", - Aliases: []string{"signer-url"}, - Usage: "used to connect gateway service for sign", + Name: "gateway-url", + Usage: "used to connect gateway service for sign", } GatewayTokenFlag = &cli.StringFlag{ - Name: "gateway-token", - Aliases: []string{"signer-token"}, - Usage: "used to connect gateway service for sign", - } - - WalletUrlFlag = &cli.StringFlag{ - Name: "wallet-url", - Aliases: []string{"signer-url"}, - Usage: "used to connect signer wallet for sign", - } - WalletTokenFlag = &cli.StringFlag{ - Name: "wallet-token", - Aliases: []string{"signer-token"}, - Usage: "auth token for connect wallet service", + Name: "gateway-token", + Usage: "used to connect gateway service for sign", } MysqlDsnFlag = &cli.StringFlag{ @@ -116,13 +88,9 @@ var ( Usage: "mysql connection string", } - MinerListFlag = &cli.StringSliceFlag{ - Name: "miner", - Usage: "support miner(f01000:jimmy)", - } - PaymentAddressFlag = &cli.StringFlag{ + RetrievalPaymentAddress = &cli.StringFlag{ Name: "payment-addr", - Usage: "payment address for receive retrieval address", + Usage: "payment address for retrieval, eg. f01000", } ) @@ -156,142 +124,3 @@ func main() { log.Fatal(err) } } - -func prepare(cctx *cli.Context, defSignerType config.SignerType) (*config.MarketConfig, error) { - if !cctx.IsSet(HidenSignerTypeFlag.Name) { - if err := cctx.Set(HidenSignerTypeFlag.Name, defSignerType); err != nil { - return nil, fmt.Errorf("set %s with wallet failed %v", HidenSignerTypeFlag.Name, err) - } - } - cfg := config.DefaultMarketConfig - cfg.HomeDir = cctx.String(RepoFlag.Name) - cfgPath, err := cfg.ConfigPath() - if err != nil { - return nil, err - } - mainLog.Info("load config from path ", cfgPath) - if _, err := os.Stat(cfgPath); os.IsNotExist(err) { - // create - err = flagData(cctx, cfg) - if err != nil { - return nil, fmt.Errorf("parser data from flag %w", err) - } - - err = config.SaveConfig(cfg) - if err != nil { - return nil, fmt.Errorf("save config to %s %w", cfgPath, err) - } - } else if err == nil { - // loadConfig - err = config.LoadConfig(cfgPath, cfg) - if err != nil { - return nil, err - } - - err = flagData(cctx, cfg) - if err != nil { - return nil, fmt.Errorf("parser data from flag %w", err) - } - } else { - return nil, err - } - return cfg, cmd.FetchAndLoadBundles(cctx.Context, cfg.Node) -} - -func flagData(cctx *cli.Context, cfg *config.MarketConfig) error { - if cctx.IsSet(NodeUrlFlag.Name) { - cfg.Node.Url = cctx.String(NodeUrlFlag.Name) - } - - if cctx.IsSet(MessagerUrlFlag.Name) { - cfg.Messager.Url = cctx.String(MessagerUrlFlag.Name) - } - - if cctx.IsSet(AuthUrlFlag.Name) { - cfg.AuthNode.Url = cctx.String(AuthUrlFlag.Name) - } - - if cctx.IsSet(SignerTypeFlag.Name) { - cfg.Signer.SignerType = cctx.String(SignerTypeFlag.Name) - } - - if cctx.IsSet(SignerUrlFlag.Name) { - cfg.Signer.Url = cctx.String(SignerUrlFlag.Name) - } - - if cctx.IsSet(AuthTokeFlag.Name) { - cfg.Node.Token = cctx.String(AuthTokeFlag.Name) - - if len(cfg.AuthNode.Url) > 0 { - cfg.AuthNode.Token = cctx.String(AuthTokeFlag.Name) - } - - if len(cfg.Messager.Url) > 0 { - cfg.Messager.Token = cctx.String(AuthTokeFlag.Name) - } - - if cfg.Signer.SignerType == "gateway" { - cfg.Signer.Token = cctx.String(AuthTokeFlag.Name) - } - } - - if cctx.IsSet(NodeTokenFlag.Name) { - cfg.Node.Token = cctx.String(NodeTokenFlag.Name) - } - if cctx.IsSet(MessagerTokenFlag.Name) { - cfg.Messager.Token = cctx.String(MessagerTokenFlag.Name) - } - if cctx.IsSet(SignerTokenFlag.Name) { - cfg.Signer.Token = cctx.String(SignerTokenFlag.Name) - } - - if cctx.IsSet(MysqlDsnFlag.Name) { - cfg.Mysql.ConnectionString = cctx.String(MysqlDsnFlag.Name) - } - - if cctx.IsSet(MinerListFlag.Name) { - storageMiners := make(map[config.Address]struct{}, 0) - for _, storageMiner := range cfg.StorageMiners { - if _, ok := storageMiners[storageMiner.Addr]; !ok { - storageMiners[storageMiner.Addr] = struct{}{} - } - } - - addrStrs := cctx.StringSlice(MinerListFlag.Name) - for _, miners := range addrStrs { - addrStr := strings.Split(miners, ":") - addr, err := address.NewFromString(addrStr[0]) - if err != nil { - return fmt.Errorf("flag provide a wrong address %s %w", addrStr, err) - } - account := "" - if len(addrStr) >= 2 { - account = addrStr[1] - } - - if _, ok := storageMiners[config.Address(addr)]; !ok { - cfg.StorageMiners = append(cfg.StorageMiners, config.User{ - Addr: config.Address(addr), - Account: account, - }) - } - } - } - - if cctx.IsSet(PaymentAddressFlag.Name) { - addrStr := strings.Split(cctx.String(PaymentAddressFlag.Name), ":") - addr, err := address.NewFromString(addrStr[0]) - if err != nil { - return fmt.Errorf("flag provide a wrong address %s %w", addrStr, err) - } - account := "" - if len(addrStr) >= 2 { - account = addrStr[1] - } - cfg.RetrievalPaymentAddress = config.User{ - Addr: config.Address(addr), - Account: account, - } - } - return nil -} diff --git a/cmd/venus-market/pool-run.go b/cmd/venus-market/pool-run.go index ac742dae..ed7deff8 100644 --- a/cmd/venus-market/pool-run.go +++ b/cmd/venus-market/pool-run.go @@ -3,7 +3,15 @@ package main import ( "fmt" + "github.com/gorilla/mux" + "github.com/urfave/cli/v2" + "go.uber.org/fx" + + "github.com/ipfs-force-community/venus-common-utils/builder" + "github.com/ipfs-force-community/venus-common-utils/journal" + "github.com/filecoin-project/venus-auth/jwtclient" + "github.com/filecoin-project/venus-market/v2/api/clients" "github.com/filecoin-project/venus-market/v2/api/impl" cli2 "github.com/filecoin-project/venus-market/v2/cli" @@ -21,19 +29,14 @@ import ( "github.com/filecoin-project/venus-market/v2/storageprovider" types2 "github.com/filecoin-project/venus-market/v2/types" "github.com/filecoin-project/venus-market/v2/utils" + marketapi "github.com/filecoin-project/venus/venus-shared/api/market" "github.com/filecoin-project/venus/venus-shared/api/permission" - "github.com/gorilla/mux" - "github.com/ipfs-force-community/venus-common-utils/builder" - "github.com/ipfs-force-community/venus-common-utils/journal" - "github.com/urfave/cli/v2" - "go.uber.org/fx" ) var poolRunCmd = &cli.Command{ - Name: "pool-run", - Usage: "Run the market daemon in pool mode", - ArgsUsage: "[minerAddress]", + Name: "pool-run", + Usage: "Run the market daemon in pool mode", Flags: []cli.Flag{ NodeUrlFlag, NodeTokenFlag, @@ -41,12 +44,13 @@ var poolRunCmd = &cli.Command{ AuthTokeFlag, MessagerUrlFlag, MessagerTokenFlag, - HidenSignerTypeFlag, GatewayUrlFlag, GatewayTokenFlag, + SignerTypeFlag, + SignerUrlFlag, + SignerTokenFlag, MysqlDsnFlag, - MinerListFlag, - PaymentAddressFlag, + RetrievalPaymentAddress, }, Action: poolDaemon, } @@ -58,9 +62,13 @@ func poolDaemon(cctx *cli.Context) error { return fmt.Errorf("prepare pool run failed:%w", err) } - // venus-auth is must in 'pool' mode + // Configuration sanity check if len(cfg.AuthNode.Url) == 0 { - return fmt.Errorf("auth-url is required in 'pool' mode") + return fmt.Errorf("pool mode have to configure auth node") + } + + if len(cfg.Signer.Url) == 0 { + return fmt.Errorf("the signer node must be configured") } ctx := cctx.Context @@ -82,11 +90,12 @@ func poolDaemon(cctx *cli.Context) error { // override marketconfig builder.Override(new(config.MarketConfig), cfg), builder.Override(new(types2.ShutdownChan), shutdownChan), - // config + + //config config.ConfigServerOpts(cfg), - // miner manager - minermgr.MinerMgrOpts(cfg), + // user manager + minermgr.MinerMgrOpts(), // clients clients.ClientsOpts(true, "pool", &cfg.Messager, &cfg.Signer), @@ -114,13 +123,13 @@ func poolDaemon(cctx *cli.Context) error { defer closeFunc(ctx) //nolint finishCh := utils.MonitorShutdown(shutdownChan) - mux := mux.NewRouter() - if err = mux.Handle("/resource", rpc.NewPieceStorageServer(resAPI.PieceStorageMgr)).GetError(); err != nil { + router := mux.NewRouter() + if err = router.Handle("/resource", rpc.NewPieceStorageServer(resAPI.PieceStorageMgr)).GetError(); err != nil { return fmt.Errorf("handle 'resource' failed: %w", err) } var fullAPI marketapi.IMarketStruct permission.PermissionProxy(marketapi.IMarket(resAPI), &fullAPI) - return rpc.ServeRPC(ctx, cfg, &cfg.API, mux, 1000, cli2.API_NAMESPACE_VENUS_MARKET, authClient, &fullAPI, finishCh) + return rpc.ServeRPC(ctx, cfg, &cfg.API, router, 1000, cli2.API_NAMESPACE_VENUS_MARKET, authClient, &fullAPI, finishCh) } diff --git a/cmd/venus-market/solo-run.go b/cmd/venus-market/solo-run.go index c5e4ca4f..a7909e84 100644 --- a/cmd/venus-market/solo-run.go +++ b/cmd/venus-market/solo-run.go @@ -3,15 +3,21 @@ package main import ( "fmt" - "github.com/filecoin-project/venus-market/v2/metrics" + "go.uber.org/fx" + + "github.com/gorilla/mux" + "github.com/urfave/cli/v2" + + "github.com/ipfs-force-community/venus-common-utils/builder" + "github.com/ipfs-force-community/venus-common-utils/journal" - "github.com/filecoin-project/venus-auth/jwtclient" "github.com/filecoin-project/venus-market/v2/api/clients" "github.com/filecoin-project/venus-market/v2/api/impl" cli2 "github.com/filecoin-project/venus-market/v2/cli" "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/dagstore" "github.com/filecoin-project/venus-market/v2/fundmgr" + "github.com/filecoin-project/venus-market/v2/metrics" "github.com/filecoin-project/venus-market/v2/minermgr" "github.com/filecoin-project/venus-market/v2/models" "github.com/filecoin-project/venus-market/v2/network" @@ -22,28 +28,24 @@ import ( "github.com/filecoin-project/venus-market/v2/storageprovider" types2 "github.com/filecoin-project/venus-market/v2/types" "github.com/filecoin-project/venus-market/v2/utils" + + "github.com/filecoin-project/venus-auth/jwtclient" + marketapi "github.com/filecoin-project/venus/venus-shared/api/market" "github.com/filecoin-project/venus/venus-shared/api/permission" - "github.com/gorilla/mux" - "github.com/ipfs-force-community/venus-common-utils/builder" - "github.com/ipfs-force-community/venus-common-utils/journal" - "github.com/urfave/cli/v2" - "go.uber.org/fx" ) var soloRunCmd = &cli.Command{ - Name: "solo-run", - Usage: "Run the market daemon in solo mode", - ArgsUsage: "[minerAddress]", + Name: "solo-run", + Usage: "Run the market daemon in solo mode", Flags: []cli.Flag{ NodeUrlFlag, NodeTokenFlag, - HidenSignerTypeFlag, - WalletUrlFlag, - WalletTokenFlag, + SignerTypeFlag, + SignerUrlFlag, + SignerTokenFlag, MysqlDsnFlag, - MinerListFlag, - PaymentAddressFlag, + RetrievalPaymentAddress, }, Action: soloDaemon, } @@ -53,17 +55,23 @@ func soloDaemon(cctx *cli.Context) error { cfg, err := prepare(cctx, config.SignerTypeWallet) if err != nil { - return fmt.Errorf("prepare solo run failed:%w", err) + return fmt.Errorf("prepare solo run failed: %w", err) + } + + // Configuration sanity check + if len(cfg.AuthNode.Url) > 0 { + return fmt.Errorf("solo mode does not need to configure auth node") + } + + if len(cfg.Signer.Url) == 0 { + return fmt.Errorf("the signer node must be configured") } + ctx := cctx.Context resAPI := &impl.MarketNodeImpl{} shutdownChan := make(chan struct{}) closeFunc, err := builder.New(ctx, - // defaults - // 'solo' mode doesn't needs a 'AuthClient' of venus-auth, - // provide a nil 'AuthClient', just for making 'NeAddrMgrImpl' happy - builder.Override(new(*jwtclient.AuthClient), func() *jwtclient.AuthClient { return nil }), builder.Override(new(journal.DisabledEvents), journal.EnvDisabledEvents), builder.Override(new(journal.Journal), func(lc fx.Lifecycle, home config.IHome, disabled journal.DisabledEvents) (journal.Journal, error) { return journal.OpenFilesystemJournal(lc, home.MustHomePath(), "venus-market", disabled) @@ -78,8 +86,12 @@ func soloDaemon(cctx *cli.Context) error { // config config.ConfigServerOpts(cfg), - // miner manager - minermgr.MinerMgrOpts(cfg), + // 'solo' mode doesn't needs a 'AuthClient' of venus-auth, + // provide a nil 'AuthClient', just for making 'NewUserMgrImpl' happy + builder.Override(new(*jwtclient.AuthClient), func() *jwtclient.AuthClient { return nil }), + + // user manager + minermgr.MinerMgrOpts(), // clients clients.ClientsOpts(true, "solo", &cfg.Messager, &cfg.Signer), @@ -108,11 +120,11 @@ func soloDaemon(cctx *cli.Context) error { finishCh := utils.MonitorShutdown(shutdownChan) - mux := mux.NewRouter() - mux.Handle("resource", rpc.NewPieceStorageServer(resAPI.PieceStorageMgr)) + router := mux.NewRouter() + router.Handle("resource", rpc.NewPieceStorageServer(resAPI.PieceStorageMgr)) var fullAPI marketapi.IMarketStruct permission.PermissionProxy(marketapi.IMarket(resAPI), &fullAPI) - return rpc.ServeRPC(ctx, cfg, &cfg.API, mux, 1000, cli2.API_NAMESPACE_VENUS_MARKET, nil, &fullAPI, finishCh) + return rpc.ServeRPC(ctx, cfg, &cfg.API, router, 1000, cli2.API_NAMESPACE_VENUS_MARKET, nil, &fullAPI, finishCh) } diff --git a/cmd/venus-market/utils.go b/cmd/venus-market/utils.go new file mode 100644 index 00000000..a5cbfcee --- /dev/null +++ b/cmd/venus-market/utils.go @@ -0,0 +1,142 @@ +package main + +import ( + "fmt" + "os" + + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/venus-market/v2/cmd" + "github.com/filecoin-project/venus-market/v2/config" +) + +func flagData(cctx *cli.Context, cfg *config.MarketConfig) error { + if cctx.IsSet(NodeUrlFlag.Name) { + cfg.Node.Url = cctx.String(NodeUrlFlag.Name) + } + + if cctx.IsSet(MessagerUrlFlag.Name) { + cfg.Messager.Url = cctx.String(MessagerUrlFlag.Name) + } + + if cctx.IsSet(AuthUrlFlag.Name) { + cfg.AuthNode.Url = cctx.String(AuthUrlFlag.Name) + } + + signerType := cctx.String(SignerTypeFlag.Name) + switch signerType { + case config.SignerTypeGateway: + { + if cctx.IsSet(GatewayUrlFlag.Name) { + cfg.Signer.Url = cctx.String(GatewayUrlFlag.Name) + } + if cctx.IsSet(GatewayTokenFlag.Name) { + cfg.Signer.Token = cctx.String(GatewayTokenFlag.Name) + } + } + case config.SignerTypeWallet: + { + if cctx.IsSet(SignerUrlFlag.Name) { + cfg.Signer.Url = cctx.String(SignerUrlFlag.Name) + } + if cctx.IsSet(SignerTokenFlag.Name) { + cfg.Signer.Token = cctx.String(SignerTokenFlag.Name) + } + } + case config.SignerTypeLotusnode: + { + if cctx.IsSet(NodeUrlFlag.Name) { + cfg.Signer.Url = cctx.String(NodeUrlFlag.Name) + } + if cctx.IsSet(NodeTokenFlag.Name) { + cfg.Signer.Token = cctx.String(NodeTokenFlag.Name) + } + } + default: + return fmt.Errorf("unsupport signer type %s", signerType) + } + cfg.Signer.SignerType = signerType + + if cctx.IsSet(AuthTokeFlag.Name) { + cfg.Node.Token = cctx.String(AuthTokeFlag.Name) + + if len(cfg.AuthNode.Url) > 0 { + cfg.AuthNode.Token = cctx.String(AuthTokeFlag.Name) + } + + if len(cfg.Messager.Url) > 0 { + cfg.Messager.Token = cctx.String(AuthTokeFlag.Name) + } + + if len(cfg.Signer.Url) > 0 { + cfg.Signer.Token = cctx.String(AuthTokeFlag.Name) + } + } + + if cctx.IsSet(NodeTokenFlag.Name) { + cfg.Node.Token = cctx.String(NodeTokenFlag.Name) + } + if cctx.IsSet(MessagerTokenFlag.Name) { + cfg.Messager.Token = cctx.String(MessagerTokenFlag.Name) + } + + if cctx.IsSet(MysqlDsnFlag.Name) { + cfg.Mysql.ConnectionString = cctx.String(MysqlDsnFlag.Name) + } + + if cctx.IsSet(RetrievalPaymentAddress.Name) { + addrStr := cctx.String(RetrievalPaymentAddress.Name) + addr, err := address.NewFromString(addrStr) + if err != nil { + return fmt.Errorf("flag provide a wrong address %s %w", addrStr, err) + } + + cfg.RetrievalPaymentAddress = config.Address(addr) + } + return nil +} + +func prepare(cctx *cli.Context, defSignerType config.SignerType) (*config.MarketConfig, error) { + if !cctx.IsSet(SignerTypeFlag.Name) { + if err := cctx.Set(SignerTypeFlag.Name, defSignerType); err != nil { + return nil, fmt.Errorf("set `%s` with wallet failed %w", SignerTypeFlag.Name, err) + } + } + cfg := config.DefaultMarketConfig + cfg.HomeDir = cctx.String(RepoFlag.Name) + cfgPath, err := cfg.ConfigPath() + if err != nil { + return nil, err + } + + mainLog.Info("load config from path ", cfgPath) + if _, err := os.Stat(cfgPath); os.IsNotExist(err) { + //create + err = flagData(cctx, cfg) + if err != nil { + return nil, fmt.Errorf("parser data from flag %w", err) + } + + err = config.SaveConfig(cfg) + if err != nil { + return nil, fmt.Errorf("save config to %s %w", cfgPath, err) + } + } else if err == nil { + //loadConfig + err = config.LoadConfig(cfgPath, cfg) + if err != nil { + return nil, err + } + + err = flagData(cctx, cfg) + if err != nil { + return nil, fmt.Errorf("parser data from flag %w", err) + } + } else { + return nil, err + } + + return cfg, cmd.FetchAndLoadBundles(cctx.Context, cfg.Node) +} diff --git a/config/client_config.go b/config/client_config.go new file mode 100644 index 00000000..d16d4157 --- /dev/null +++ b/config/client_config.go @@ -0,0 +1,15 @@ +package config + +type MarketClientConfig struct { + Home `toml:"-"` + Common + + Node Node + Messager Messager + Signer Signer + + // The maximum number of parallel online data transfers (piecestorage+retrieval) + SimultaneousTransfersForRetrieval uint64 + SimultaneousTransfersForStorage uint64 + DefaultMarketAddress Address +} diff --git a/config/common.go b/config/common.go new file mode 100644 index 00000000..0510b8ac --- /dev/null +++ b/config/common.go @@ -0,0 +1,155 @@ +package config + +import ( + "time" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/venus/venus-shared/types" +) + +// API contains configs for API endpoint +type API struct { + ListenAddress string + RemoteListenAddress string + Timeout Duration +} + +// Libp2p contains configs for libp2p +type Libp2p struct { + // Binding address for the libp2p host - 0 means random port. + // Format: multiaddress; see https://multiformats.io/multiaddr/ + ListenAddresses []string + // Addresses to explicitally announce to other peers. If not specified, + // all interface addresses are announced + // Format: multiaddress + AnnounceAddresses []string + // Addresses to not announce + // Format: multiaddress + NoAnnounceAddresses []string + ProtectedPeers []string + + PrivateKey string +} + +type Common struct { + API API + Libp2p Libp2p +} + +// chain-service connect config +type ConnectConfig struct { + Url string + Token string +} + +type ( + Node ConnectConfig + Messager ConnectConfig + Market ConnectConfig + AuthNode ConnectConfig +) + +// signer config +type SignerType = string + +const ( + SignerTypeLotusnode = "lotusnode" + SignerTypeWallet = "wallet" + SignerTypeGateway = "gateway" +) + +type Signer struct { + SignerType SignerType `toml:"Type"` // wallet/gateway + Url string + Token string +} + +// common config for provider +type ProviderConfig struct { + // When enabled, the miner can accept online deals + ConsiderOnlineStorageDeals bool + // When enabled, the miner can accept offline deals + ConsiderOfflineStorageDeals bool + // When enabled, the miner can accept retrieval deals + ConsiderOnlineRetrievalDeals bool + // When enabled, the miner can accept offline retrieval deals + ConsiderOfflineRetrievalDeals bool + // When enabled, the miner can accept verified deals + ConsiderVerifiedStorageDeals bool + // When enabled, the miner can accept unverified deals + ConsiderUnverifiedStorageDeals bool + // A list of Data CIDs to reject when making deals + PieceCidBlocklist []cid.Cid + // Maximum expected amount of time getting the deal into a sealed sector will take + // This includes the time the deal will need to get transferred and published + // before being assigned to a sector + ExpectedSealDuration Duration + // Maximum amount of time proposed deal StartEpoch can be in future + MaxDealStartDelay Duration + // todo 以上参数在目前实现中没有起到实际的作用??? + + // todo 以下参数缺少配置API??? + // When a deal is ready to publish, the amount of time to wait for more + // deals to be ready to publish before publishing them all as a batch + PublishMsgPeriod Duration + // The maximum number of deals to include in a single PublishStorageDeals + // message + MaxDealsPerPublishMsg uint64 + + // The maximum collateral that the provider will put up against a deal, + // as a multiplier of the minimum collateral bound + MaxProviderCollateralMultiplier uint64 + + // A command used for fine-grained evaluation of piecestorage deals + // see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details + Filter string + // A command used for fine-grained evaluation of retrieval deals + // see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details + RetrievalFilter string + + TransferPath string + + RetrievalPricing *RetrievalPricing // todo reserve + + MaxPublishDealsFee types.FIL + MaxMarketBalanceAddFee types.FIL + + AddressConfig AddressConfig // todo 当前版本无效 +} + +func defaultProviderConfig() *ProviderConfig { + return &ProviderConfig{ + ConsiderOnlineStorageDeals: true, + ConsiderOfflineStorageDeals: true, + ConsiderOnlineRetrievalDeals: true, + ConsiderOfflineRetrievalDeals: true, + ConsiderVerifiedStorageDeals: true, + ConsiderUnverifiedStorageDeals: true, + PieceCidBlocklist: []cid.Cid{}, + // TODO: It'd be nice to set this based on sector size + MaxDealStartDelay: Duration(time.Hour * 24 * 14), + ExpectedSealDuration: Duration(time.Hour * 24), + PublishMsgPeriod: Duration(time.Hour), + MaxDealsPerPublishMsg: 8, + MaxProviderCollateralMultiplier: 2, + + Filter: "", + RetrievalFilter: "", + + TransferPath: "", + + RetrievalPricing: &RetrievalPricing{ + Strategy: RetrievalPricingDefaultMode, + Default: &RetrievalPricingDefault{ + VerifiedDealsFreeTransfer: true, + }, + External: &RetrievalPricingExternal{ + Path: "", + }, + }, + + MaxPublishDealsFee: types.FIL(types.NewInt(0)), + MaxMarketBalanceAddFee: types.FIL(types.NewInt(0)), + } +} diff --git a/config/config.go b/config/config.go index 2d085d7a..167f55ab 100644 --- a/config/config.go +++ b/config/config.go @@ -1,84 +1,13 @@ package config import ( - "encoding" "fmt" - "time" "github.com/ipfs-force-community/metrics" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/venus/venus-shared/types" - "github.com/ipfs/go-cid" ) -// API contains configs for API endpoint -type API struct { - // Binding address for the Lotus API - ListenAddress string - RemoteListenAddress string - Timeout Duration -} - -// Libp2p contains configs for libp2p -type Libp2p struct { - // Binding address for the libp2p host - 0 means random port. - // Format: multiaddress; see https://multiformats.io/multiaddr/ - ListenAddresses []string - // Addresses to explicitally announce to other peers. If not specified, - // all interface addresses are announced - // Format: multiaddress - AnnounceAddresses []string - // Addresses to not announce - // Format: multiaddress - NoAnnounceAddresses []string - ProtectedPeers []string - - PrivateKey string -} - -type ConnectConfig struct { - Url string - Token string -} - -type ( - Node ConnectConfig - Messager ConnectConfig - Market ConnectConfig - AuthNode ConnectConfig -) - -type Common struct { - API API - Libp2p Libp2p -} - -type SignerType = string - -const ( - SignerTypeWallet = "wallet" - SignerTypeGateway = "gateway" -) - -type Signer struct { - SignerType SignerType `toml:"Type"` // wallet/gateway - Url string - Token string -} - -type Mysql struct { - ConnectionString string - MaxOpenConn int - MaxIdleConn int - ConnMaxLifeTime string - Debug bool -} - -type Journal struct { - Path string -} - const ( // RetrievalPricingDefault configures the node to use the default retrieval pricing policy. RetrievalPricingDefaultMode = "default" @@ -109,7 +38,7 @@ type RetrievalPricingDefault struct { } type AddressConfig struct { - DealPublishControl []User + DealPublishControl []Address // DisableWorkerFallback disables usage of the worker address for messages // sent automatically, if control addresses are configured. @@ -120,12 +49,16 @@ type AddressConfig struct { func (ac AddressConfig) Address() []address.Address { addrs := make([]address.Address, len(ac.DealPublishControl)) - for index, miner := range ac.DealPublishControl { - addrs[index] = address.Address(miner.Addr) + for index, mAddr := range ac.DealPublishControl { + addrs[index] = address.Address(mAddr) } return addrs } +type Journal struct { + Path string +} + type DAGStoreConfig struct { // Path to the dagstore root directory. This directory contains three // subdirectories, which can be symlinked to alternative locations if @@ -180,13 +113,11 @@ type PieceStorage struct { Fs []*FsPieceStorage S3 []*S3PieceStorage } - type FsPieceStorage struct { Name string ReadOnly bool Path string } - type S3PieceStorage struct { Name string ReadOnly bool @@ -199,62 +130,26 @@ type S3PieceStorage struct { Token string } -type User struct { +type Mysql struct { + ConnectionString string + MaxOpenConn int + MaxIdleConn int + ConnMaxLifeTime string + Debug bool +} + +type MinerConfig struct { Addr Address - Account string + Account string // todo 在合并run模式后才真正起作用 + + *ProviderConfig } -// StorageMiner is a miner config type MarketConfig struct { Home `toml:"-"` Common - Node Node - Messager Messager - Signer Signer - AuthNode AuthNode - - Mysql Mysql - - PieceStorage PieceStorage - Journal Journal - AddressConfig AddressConfig - DAGStore DAGStoreConfig - - StorageMiners []User - RetrievalPaymentAddress User - - // When enabled, the miner can accept online deals - ConsiderOnlineStorageDeals bool - // When enabled, the miner can accept offline deals - ConsiderOfflineStorageDeals bool - // When enabled, the miner can accept retrieval deals - ConsiderOnlineRetrievalDeals bool - // When enabled, the miner can accept offline retrieval deals - ConsiderOfflineRetrievalDeals bool - // When enabled, the miner can accept verified deals - ConsiderVerifiedStorageDeals bool - // When enabled, the miner can accept unverified deals - ConsiderUnverifiedStorageDeals bool - // A list of Data CIDs to reject when making deals - PieceCidBlocklist []cid.Cid - // Maximum expected amount of time getting the deal into a sealed sector will take - // This includes the time the deal will need to get transferred and published - // before being assigned to a sector - ExpectedSealDuration Duration - // Maximum amount of time proposed deal StartEpoch can be in future - MaxDealStartDelay Duration - // When a deal is ready to publish, the amount of time to wait for more - // deals to be ready to publish before publishing them all as a batch - PublishMsgPeriod Duration - // The maximum number of deals to include in a single PublishStorageDeals - // message - MaxDealsPerPublishMsg uint64 - // The maximum collateral that the provider will put up against a deal, - // as a multiplier of the minimum collateral bound - MaxProviderCollateralMultiplier uint64 - // The maximum number of parallel online data transfers for storage deals SimultaneousTransfersForStorage uint64 // The maximum number of simultaneous data transfers from any single client @@ -267,20 +162,22 @@ type MarketConfig struct { // The maximum number of parallel online data transfers for retrieval deals SimultaneousTransfersForRetrieval uint64 - // A command used for fine-grained evaluation of piecestorage deals - // see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details - Filter string - // A command used for fine-grained evaluation of retrieval deals - // see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details - RetrievalFilter string + Node Node + Messager Messager + Signer Signer + AuthNode AuthNode + + Mysql Mysql - TransfePath string + PieceStorage PieceStorage + DAGStore DAGStoreConfig - RetrievalPricing *RetrievalPricing + RetrievalPaymentAddress Address // todo 也需要每个矿工可以单独设置 - MaxPublishDealsFee types.FIL - MaxMarketBalanceAddFee types.FIL + CommonProviderConfig *ProviderConfig + Miners []*MinerConfig + Journal Journal Metrics metrics.MetricsConfig } @@ -310,69 +207,35 @@ func (m *MarketConfig) AddS3PieceStorage(fsps *S3PieceStorage) error { return SaveConfig(m) } -type MarketClientConfig struct { - Home `toml:"-"` - Common - - Node Node - Messager Messager - Signer Signer - - // The maximum number of parallel online data transfers (piecestorage+retrieval) - SimultaneousTransfersForRetrieval uint64 - SimultaneousTransfersForStorage uint64 - DefaultMarketAddress Address -} - -var ( - _ encoding.TextMarshaler = (*Duration)(nil) - _ encoding.TextUnmarshaler = (*Duration)(nil) -) - -// Duration is a wrapper type for Duration -// for decoding and encoding from/to TOML -type Duration time.Duration - -// UnmarshalText implements interface for TOML decoding -func (dur *Duration) UnmarshalText(text []byte) error { - d, err := time.ParseDuration(string(text)) - if err != nil { - return err +func (m *MarketConfig) MinerProviderConfig(mAddr address.Address, useCommon bool) *ProviderConfig { + for i := range m.Miners { + if m.Miners[i].Addr == Address(mAddr) { + return m.Miners[i].ProviderConfig + } } - *dur = Duration(d) - return err -} - -func (dur Duration) MarshalText() ([]byte, error) { - d := time.Duration(dur) - return []byte(d.String()), nil -} - -// Address is a wrapper type for Address -// for decoding and encoding from/to TOML -type Address address.Address -// UnmarshalText implements interface for TOML decoding -func (addr *Address) UnmarshalText(text []byte) error { - d, err := address.NewFromString(string(text)) - if err != nil { - return err + if useCommon { + return m.CommonProviderConfig } - *addr = Address(d) - return err -} -func (addr Address) MarshalText() ([]byte, error) { - if address.Address(addr) == address.Undef { - return []byte{}, nil - } - return []byte(address.Address(addr).String()), nil + return nil } -func ConvertConfigAddress(addrs []Address) []address.Address { - addrs2 := make([]address.Address, len(addrs)) - for index, addr := range addrs { - addrs2[index] = address.Address(addr) +func (m *MarketConfig) SetMinerProviderConfig(mAddr address.Address, pCfg *ProviderConfig) { + if mAddr == address.Undef { + m.CommonProviderConfig = pCfg + } else { + for i := range m.Miners { + if m.Miners[i].Addr == Address(mAddr) { + m.Miners[i].ProviderConfig = pCfg + return + } + } + + // create + m.Miners = append(m.Miners, &MinerConfig{ + Addr: Address(mAddr), + ProviderConfig: pCfg, + }) } - return addrs2 } diff --git a/config/def_config.go b/config/def_config.go index cfbbe87c..fe087248 100644 --- a/config/def_config.go +++ b/config/def_config.go @@ -6,8 +6,6 @@ import ( "github.com/ipfs-force-community/metrics" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/venus/venus-shared/types" - "github.com/ipfs/go-cid" ) const ( @@ -44,6 +42,10 @@ var DefaultMarketConfig = &MarketConfig{ Url: "", // /ip4//tcp/5678 Token: "", }, + AuthNode: AuthNode{ + Url: "", // "http://:8989", + Token: "", + }, Mysql: Mysql{ ConnectionString: "", MaxOpenConn: 100, @@ -51,51 +53,23 @@ var DefaultMarketConfig = &MarketConfig{ ConnMaxLifeTime: "1m", Debug: false, }, - AuthNode: AuthNode{ - Url: "", // "http://:8989", - Token: "", + PieceStorage: PieceStorage{ + Fs: []*FsPieceStorage{}, }, DAGStore: DAGStoreConfig{ MaxConcurrentIndex: 5, MaxConcurrencyStorageCalls: 100, GCInterval: Duration(1 * time.Minute), }, - Journal: Journal{Path: "journal"}, - PieceStorage: PieceStorage{ - Fs: []*FsPieceStorage{}, - }, - ConsiderOnlineStorageDeals: true, - ConsiderOfflineStorageDeals: true, - ConsiderOnlineRetrievalDeals: true, - ConsiderOfflineRetrievalDeals: true, - ConsiderVerifiedStorageDeals: true, - ConsiderUnverifiedStorageDeals: true, - PieceCidBlocklist: []cid.Cid{}, - // TODO: It'd be nice to set this based on sector size - MaxDealStartDelay: Duration(time.Hour * 24 * 14), - ExpectedSealDuration: Duration(time.Hour * 24), - PublishMsgPeriod: Duration(time.Hour), - - MaxDealsPerPublishMsg: 8, - MaxProviderCollateralMultiplier: 2, SimultaneousTransfersForRetrieval: DefaultSimultaneousTransfers, SimultaneousTransfersForStoragePerClient: DefaultSimultaneousTransfers, SimultaneousTransfersForStorage: DefaultSimultaneousTransfers, - RetrievalPricing: &RetrievalPricing{ - Strategy: RetrievalPricingDefaultMode, - Default: &RetrievalPricingDefault{ - VerifiedDealsFreeTransfer: true, - }, - External: &RetrievalPricingExternal{ - Path: "", - }, - }, - - MaxPublishDealsFee: types.FIL(types.NewInt(0)), - MaxMarketBalanceAddFee: types.FIL(types.NewInt(0)), - Metrics: *metrics.DefaultMetricsConfig(), + CommonProviderConfig: defaultProviderConfig(), + Miners: make([]*MinerConfig, 0), + Journal: Journal{Path: "journal"}, + Metrics: *metrics.DefaultMetricsConfig(), } var DefaultMarketClientConfig = &MarketClientConfig{ diff --git a/config/modules.go b/config/modules.go index d50d0fcc..b17dfbc8 100644 --- a/config/modules.go +++ b/config/modules.go @@ -4,124 +4,320 @@ import ( "time" "github.com/ipfs/go-cid" + "github.com/mitchellh/go-homedir" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/filestore" "github.com/ipfs-force-community/venus-common-utils/builder" + + vsTypes "github.com/filecoin-project/venus/venus-shared/types" ) func NewConsiderOnlineStorageDealsConfigFunc(cfg *MarketConfig) (ConsiderOnlineStorageDealsConfigFunc, error) { - return func() (out bool, err error) { - return cfg.ConsiderOnlineStorageDeals, nil + return func(mAddr address.Address) (bool, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.ConsiderOnlineStorageDeals, nil }, nil } func NewSetConsideringOnlineStorageDealsFunc(cfg *MarketConfig) (SetConsiderOnlineStorageDealsConfigFunc, error) { - return func(b bool) (err error) { - cfg.ConsiderOnlineStorageDeals = b + return func(mAddr address.Address, b bool) error { + // mAddr==Undef,update global; otherwise, if exist, update, else create with global + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.ConsiderOnlineStorageDeals = b + cfg.SetMinerProviderConfig(mAddr, pCfg) return SaveConfig(cfg) }, nil } func NewConsiderOnlineRetrievalDealsConfigFunc(cfg *MarketConfig) (ConsiderOnlineRetrievalDealsConfigFunc, error) { - return func() (out bool, err error) { - return cfg.ConsiderOnlineRetrievalDeals, nil + return func(mAddr address.Address) (bool, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.ConsiderOnlineRetrievalDeals, nil }, nil } func NewSetConsiderOnlineRetrievalDealsConfigFunc(cfg *MarketConfig) (SetConsiderOnlineRetrievalDealsConfigFunc, error) { - return func(b bool) (err error) { - cfg.ConsiderOnlineRetrievalDeals = b + return func(mAddr address.Address, b bool) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.ConsiderOnlineRetrievalDeals = b + cfg.SetMinerProviderConfig(mAddr, pCfg) return SaveConfig(cfg) }, nil } func NewStorageDealPieceCidBlocklistConfigFunc(cfg *MarketConfig) (StorageDealPieceCidBlocklistConfigFunc, error) { - return func() (out []cid.Cid, err error) { - return cfg.PieceCidBlocklist, nil + return func(mAddr address.Address) ([]cid.Cid, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.PieceCidBlocklist, nil }, nil } func NewSetStorageDealPieceCidBlocklistConfigFunc(cfg *MarketConfig) (SetStorageDealPieceCidBlocklistConfigFunc, error) { - return func(blocklist []cid.Cid) (err error) { - cfg.PieceCidBlocklist = blocklist + return func(mAddr address.Address, blocklist []cid.Cid) (err error) { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.PieceCidBlocklist = blocklist + cfg.SetMinerProviderConfig(mAddr, pCfg) return SaveConfig(cfg) }, nil } func NewConsiderOfflineStorageDealsConfigFunc(cfg *MarketConfig) (ConsiderOfflineStorageDealsConfigFunc, error) { - return func() (out bool, err error) { - return cfg.ConsiderOfflineStorageDeals, nil + return func(mAddr address.Address) (bool, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.ConsiderOfflineStorageDeals, nil }, nil } func NewSetConsideringOfflineStorageDealsFunc(cfg *MarketConfig) (SetConsiderOfflineStorageDealsConfigFunc, error) { - return func(b bool) (err error) { - cfg.ConsiderOfflineStorageDeals = b + return func(mAddr address.Address, b bool) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.ConsiderOfflineStorageDeals = b + cfg.SetMinerProviderConfig(mAddr, pCfg) return SaveConfig(cfg) }, nil } func NewConsiderOfflineRetrievalDealsConfigFunc(cfg *MarketConfig) (ConsiderOfflineRetrievalDealsConfigFunc, error) { - return func() (out bool, err error) { - return cfg.ConsiderOfflineRetrievalDeals, nil + return func(mAddr address.Address) (bool, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.ConsiderOfflineRetrievalDeals, nil }, nil } func NewSetConsiderOfflineRetrievalDealsConfigFunc(cfg *MarketConfig) (SetConsiderOfflineRetrievalDealsConfigFunc, error) { - return func(b bool) (err error) { - cfg.ConsiderOfflineRetrievalDeals = b + return func(mAddr address.Address, b bool) (err error) { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.ConsiderOfflineRetrievalDeals = b + cfg.SetMinerProviderConfig(mAddr, pCfg) return SaveConfig(cfg) }, nil } func NewConsiderVerifiedStorageDealsConfigFunc(cfg *MarketConfig) (ConsiderVerifiedStorageDealsConfigFunc, error) { - return func() (out bool, err error) { - return cfg.ConsiderVerifiedStorageDeals, nil + return func(mAddr address.Address) (bool, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.ConsiderVerifiedStorageDeals, nil }, nil } func NewSetConsideringVerifiedStorageDealsFunc(cfg *MarketConfig) (SetConsiderVerifiedStorageDealsConfigFunc, error) { - return func(b bool) (err error) { - cfg.ConsiderVerifiedStorageDeals = b + return func(mAddr address.Address, b bool) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.ConsiderVerifiedStorageDeals = b + cfg.SetMinerProviderConfig(mAddr, pCfg) return SaveConfig(cfg) }, nil } func NewConsiderUnverifiedStorageDealsConfigFunc(cfg *MarketConfig) (ConsiderUnverifiedStorageDealsConfigFunc, error) { - return func() (out bool, err error) { - return cfg.ConsiderUnverifiedStorageDeals, nil + return func(mAddr address.Address) (bool, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.ConsiderUnverifiedStorageDeals, nil }, nil } func NewSetConsideringUnverifiedStorageDealsFunc(cfg *MarketConfig) (SetConsiderUnverifiedStorageDealsConfigFunc, error) { - return func(b bool) (err error) { - cfg.ConsiderUnverifiedStorageDeals = b + return func(mAddr address.Address, b bool) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.ConsiderUnverifiedStorageDeals = b + cfg.SetMinerProviderConfig(mAddr, pCfg) return SaveConfig(cfg) }, nil } -func NewSetExpectedSealDurationFunc(cfg *MarketConfig) (SetExpectedSealDurationFunc, error) { - return func(delay time.Duration) (err error) { - cfg.ExpectedSealDuration = Duration(delay) +func NewGetMaxDealStartDelayFunc(cfg *MarketConfig) (GetMaxDealStartDelayFunc, error) { + return func(mAddr address.Address) (time.Duration, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return time.Duration(pCfg.MaxDealStartDelay), nil + }, nil +} + +func NewSetMaxDealStartDelayFunc(cfg *MarketConfig) (SetMaxDealStartDelayFunc, error) { + return func(mAddr address.Address, delay time.Duration) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.MaxDealStartDelay = Duration(delay) + cfg.SetMinerProviderConfig(mAddr, pCfg) return SaveConfig(cfg) }, nil } func NewGetExpectedSealDurationFunc(cfg *MarketConfig) (GetExpectedSealDurationFunc, error) { - return func() (out time.Duration, err error) { - return time.Duration(cfg.ExpectedSealDuration), nil + return func(mAddr address.Address) (time.Duration, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return time.Duration(pCfg.ExpectedSealDuration), nil }, nil } -func NewSetMaxDealStartDelayFunc(cfg *MarketConfig) (SetMaxDealStartDelayFunc, error) { - return func(delay time.Duration) (err error) { - cfg.MaxDealStartDelay = Duration(delay) +func NewSetExpectedSealDurationFunc(cfg *MarketConfig) (SetExpectedSealDurationFunc, error) { + return func(mAddr address.Address, delay time.Duration) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.ExpectedSealDuration = Duration(delay) + cfg.SetMinerProviderConfig(mAddr, pCfg) return SaveConfig(cfg) }, nil } -func NewGetMaxDealStartDelayFunc(cfg *MarketConfig) (GetMaxDealStartDelayFunc, error) { - return func() (out time.Duration, err error) { - return time.Duration(cfg.MaxDealStartDelay), nil +func NewTransferPathFunc(cfg *MarketConfig) (TransferPathFunc, error) { + return func(mAddr address.Address) (string, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.TransferPath, nil + }, nil +} + +func NewSetTransferPathFunc(cfg *MarketConfig) (SetTransferPathFunc, error) { + return func(mAddr address.Address, path string) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.TransferPath = path + cfg.SetMinerProviderConfig(mAddr, pCfg) + return SaveConfig(cfg) + }, nil +} + +func NewTransferFileStoreConfigFunc(cfg *MarketConfig) (TransferFileStoreConfigFunc, error) { + return func(mAddr address.Address) (store filestore.FileStore, err error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + transferPath := pCfg.TransferPath + if len(transferPath) == 0 { + transferPath = cfg.MustHomePath() + } else { + transferPath, err = homedir.Expand(transferPath) + if err != nil { + return nil, err + } + } + + store, err = filestore.NewLocalFileStore(filestore.OsPath(transferPath)) + if err != nil { + return nil, err + } + return store, nil + }, nil +} + +func NewPublishMsgPeriodConfigFunc(cfg *MarketConfig) (PublishMsgPeriodConfigFunc, error) { + return func(mAddr address.Address) (time.Duration, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return time.Duration(pCfg.PublishMsgPeriod), nil + }, nil +} + +func NewSetPublishMsgPeriodConfigFunc(cfg *MarketConfig) (SetPublishMsgPeriodConfigFunc, error) { + return func(mAddr address.Address, d time.Duration) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.PublishMsgPeriod = Duration(d) + cfg.SetMinerProviderConfig(mAddr, pCfg) + return SaveConfig(cfg) + }, nil +} + +func NewMaxDealsPerPublishMsgFunc(cfg *MarketConfig) (MaxDealsPerPublishMsgFunc, error) { + return func(mAddr address.Address) (uint64, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.MaxDealsPerPublishMsg, nil + }, nil +} + +func NewSetMaxDealsPerPublishMsgFunc(cfg *MarketConfig) (SetMaxDealsPerPublishMsgFunc, error) { + return func(mAddr address.Address, nums uint64) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.MaxDealsPerPublishMsg = nums + cfg.SetMinerProviderConfig(mAddr, pCfg) + return SaveConfig(cfg) + }, nil +} + +func NewMaxProviderCollateralMultiplierFunc(cfg *MarketConfig) (MaxProviderCollateralMultiplierFunc, error) { + return func(mAddr address.Address) (uint64, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.MaxProviderCollateralMultiplier, nil + }, nil +} + +func NewSetMaxProviderCollateralMultiplierFunc(cfg *MarketConfig) (SetMaxProviderCollateralMultiplierFunc, error) { + return func(mAddr address.Address, c uint64) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.MaxProviderCollateralMultiplier = c + cfg.SetMinerProviderConfig(mAddr, pCfg) + return SaveConfig(cfg) + }, nil +} + +func NewMaxPublishDealsFeeFunc(cfg *MarketConfig) (MaxPublishDealsFeeFunc, error) { + return func(mAddr address.Address) (vsTypes.FIL, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.MaxPublishDealsFee, nil + }, nil +} + +func NewSetMaxPublishDealsFeeFunc(cfg *MarketConfig) (SetMaxPublishDealsFeeFunc, error) { + return func(mAddr address.Address, f vsTypes.FIL) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.MaxPublishDealsFee = f + cfg.SetMinerProviderConfig(mAddr, pCfg) + return SaveConfig(cfg) + }, nil +} + +func NewMaxMarketBalanceAddFeeFunc(cfg *MarketConfig) (MaxMarketBalanceAddFeeFunc, error) { + return func(mAddr address.Address) (vsTypes.FIL, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + return pCfg.MaxMarketBalanceAddFee, nil + }, nil +} + +func NewSetMaxMarketBalanceAddFeeFunc(cfg *MarketConfig) (SetMaxMarketBalanceAddFeeFunc, error) { + return func(mAddr address.Address, f vsTypes.FIL) error { + pCfg := cfg.MinerProviderConfig(mAddr, false) + if pCfg == nil { + pCfg = defaultProviderConfig() + } + pCfg.MaxMarketBalanceAddFee = f + cfg.SetMinerProviderConfig(mAddr, pCfg) + return SaveConfig(cfg) }, nil } @@ -157,6 +353,22 @@ var ConfigServerOpts = func(cfg *MarketConfig) builder.Option { builder.Override(new(GetExpectedSealDurationFunc), NewGetExpectedSealDurationFunc), builder.Override(new(SetMaxDealStartDelayFunc), NewSetMaxDealStartDelayFunc), builder.Override(new(GetMaxDealStartDelayFunc), NewGetMaxDealStartDelayFunc), + + builder.Override(new(TransferPathFunc), NewTransferPathFunc), + builder.Override(new(SetTransferPathFunc), NewSetTransferPathFunc), + builder.Override(new(TransferFileStoreConfigFunc), NewTransferFileStoreConfigFunc), + + builder.Override(new(PublishMsgPeriodConfigFunc), NewPublishMsgPeriodConfigFunc), + builder.Override(new(SetPublishMsgPeriodConfigFunc), NewSetPublishMsgPeriodConfigFunc), + builder.Override(new(MaxDealsPerPublishMsgFunc), NewMaxDealsPerPublishMsgFunc), + builder.Override(new(SetMaxDealsPerPublishMsgFunc), NewSetMaxDealsPerPublishMsgFunc), + builder.Override(new(MaxProviderCollateralMultiplierFunc), NewMaxProviderCollateralMultiplierFunc), + builder.Override(new(SetMaxProviderCollateralMultiplierFunc), NewSetMaxProviderCollateralMultiplierFunc), + + builder.Override(new(MaxPublishDealsFeeFunc), NewMaxPublishDealsFeeFunc), + builder.Override(new(SetMaxPublishDealsFeeFunc), NewSetMaxPublishDealsFeeFunc), + builder.Override(new(MaxMarketBalanceAddFeeFunc), NewMaxMarketBalanceAddFeeFunc), + builder.Override(new(SetMaxMarketBalanceAddFeeFunc), NewSetMaxMarketBalanceAddFeeFunc), ) } diff --git a/config/setter_getter.go b/config/setter_getter.go index a74bca28..4b911cdc 100644 --- a/config/setter_getter.go +++ b/config/setter_getter.go @@ -4,82 +4,122 @@ import ( "context" "time" - "github.com/filecoin-project/go-fil-markets/storagemarket" - types "github.com/filecoin-project/venus/venus-shared/types/market" "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/filestore" + vsTypes "github.com/filecoin-project/venus/venus-shared/types" + types "github.com/filecoin-project/venus/venus-shared/types/market" ) // ConsiderOnlineStorageDealsConfigFunc is a function which reads from miner -// config to determine if the user has disabled piecestorage deals (or not). -type ConsiderOnlineStorageDealsConfigFunc func() (bool, error) +// config to determine if the miner has disabled storage deals (or not). +type ConsiderOnlineStorageDealsConfigFunc func(address.Address) (bool, error) // SetConsiderOnlineStorageDealsConfigFunc is a function which is used to // disable or enable piecestorage deal acceptance. -type SetConsiderOnlineStorageDealsConfigFunc func(bool) error +type SetConsiderOnlineStorageDealsConfigFunc func(address.Address, bool) error // ConsiderOnlineRetrievalDealsConfigFunc is a function which reads from miner // config to determine if the user has disabled retrieval acceptance (or not). -type ConsiderOnlineRetrievalDealsConfigFunc func() (bool, error) +type ConsiderOnlineRetrievalDealsConfigFunc func(address.Address) (bool, error) // SetConsiderOnlineRetrievalDealsConfigFunc is a function which is used to // disable or enable retrieval deal acceptance. -type SetConsiderOnlineRetrievalDealsConfigFunc func(bool) error +type SetConsiderOnlineRetrievalDealsConfigFunc func(address.Address, bool) error // StorageDealPieceCidBlocklistConfigFunc is a function which reads from miner // config to obtain a list of CIDs for which the miner will not accept // piecestorage proposals. -type StorageDealPieceCidBlocklistConfigFunc func() ([]cid.Cid, error) +type StorageDealPieceCidBlocklistConfigFunc func(address.Address) ([]cid.Cid, error) // SetStorageDealPieceCidBlocklistConfigFunc is a function which is used to set a // list of CIDs for which the miner will reject deal proposals. -type SetStorageDealPieceCidBlocklistConfigFunc func([]cid.Cid) error +type SetStorageDealPieceCidBlocklistConfigFunc func(address.Address, []cid.Cid) error // ConsiderOfflineStorageDealsConfigFunc is a function which reads from miner -// config to determine if the user has disabled piecestorage deals (or not). -type ConsiderOfflineStorageDealsConfigFunc func() (bool, error) +// config to determine if the miner has disabled storage deals (or not). +type ConsiderOfflineStorageDealsConfigFunc func(address.Address) (bool, error) // SetConsiderOfflineStorageDealsConfigFunc is a function which is used to // disable or enable piecestorage deal acceptance. -type SetConsiderOfflineStorageDealsConfigFunc func(bool) error +type SetConsiderOfflineStorageDealsConfigFunc func(address.Address, bool) error // ConsiderOfflineRetrievalDealsConfigFunc is a function which reads from miner // config to determine if the user has disabled retrieval acceptance (or not). -type ConsiderOfflineRetrievalDealsConfigFunc func() (bool, error) +type ConsiderOfflineRetrievalDealsConfigFunc func(address.Address) (bool, error) // SetConsiderOfflineRetrievalDealsConfigFunc is a function which is used to // disable or enable retrieval deal acceptance. -type SetConsiderOfflineRetrievalDealsConfigFunc func(bool) error +type SetConsiderOfflineRetrievalDealsConfigFunc func(address.Address, bool) error // ConsiderVerifiedStorageDealsConfigFunc is a function which reads from miner // config to determine if the user has disabled verified piecestorage deals (or not). -type ConsiderVerifiedStorageDealsConfigFunc func() (bool, error) +type ConsiderVerifiedStorageDealsConfigFunc func(address.Address) (bool, error) // SetConsiderVerifiedStorageDealsConfigFunc is a function which is used to // disable or enable verified piecestorage deal acceptance. -type SetConsiderVerifiedStorageDealsConfigFunc func(bool) error +type SetConsiderVerifiedStorageDealsConfigFunc func(address.Address, bool) error // ConsiderUnverifiedStorageDealsConfigFunc is a function which reads from miner // config to determine if the user has disabled unverified piecestorage deals (or not). -type ConsiderUnverifiedStorageDealsConfigFunc func() (bool, error) +type ConsiderUnverifiedStorageDealsConfigFunc func(address.Address) (bool, error) // SetConsiderUnverifiedStorageDealsConfigFunc is a function which is used to // disable or enable unverified piecestorage deal acceptance. -type SetConsiderUnverifiedStorageDealsConfigFunc func(bool) error +type SetConsiderUnverifiedStorageDealsConfigFunc func(address.Address, bool) error type ( - SetMaxDealStartDelayFunc func(time.Duration) error - GetMaxDealStartDelayFunc func() (time.Duration, error) + SetMaxDealStartDelayFunc func(address.Address, time.Duration) error + GetMaxDealStartDelayFunc func(address.Address) (time.Duration, error) ) // SetExpectedSealDurationFunc is a function which is used to set how long sealing is expected to take. // Deals that would need to start earlier than this duration will be rejected. -type SetExpectedSealDurationFunc func(time.Duration) error +type SetExpectedSealDurationFunc func(address.Address, time.Duration) error // GetExpectedSealDurationFunc is a function which reads from miner // too determine how long sealing is expected to take -type GetExpectedSealDurationFunc func() (time.Duration, error) +type GetExpectedSealDurationFunc func(address.Address) (time.Duration, error) type ( - StorageDealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) - RetrievalDealFilter func(ctx context.Context, deal types.ProviderDealState) (bool, string, error) + StorageDealFilter func(ctx context.Context, mAddr address.Address, deal *types.MinerDeal) (bool, string, error) + RetrievalDealFilter func(ctx context.Context, mAddr address.Address, deal types.ProviderDealState) (bool, string, error) ) + +// TransferFileStoreConfigFunc is a function which reads transfer-path from miner config creates FileStore object. +type TransferFileStoreConfigFunc func(address.Address) (filestore.FileStore, error) + +// PublishMsgPeriodConfigFunc is a function which reads the publish message period from miner config. +type PublishMsgPeriodConfigFunc func(address.Address) (time.Duration, error) + +// SetPublishMsgPeriodConfigFunc is a function which is used to set the publish message period for miner. +type SetPublishMsgPeriodConfigFunc func(address.Address, time.Duration) error + +// MaxDealsPerPublishMsgFunc is a function which reads the maximum number of deals to include +// in a single PublishStorageDeals message from miner config. +type MaxDealsPerPublishMsgFunc func(address.Address) (uint64, error) + +// SetMaxDealsPerPublishMsgFunc is a function which is used to set the maximum number of deals +// to include in a single PublishStorageDeals message for miner. +type SetMaxDealsPerPublishMsgFunc func(address.Address, uint64) error + +// MaxProviderCollateralMultiplierFunc is a function which reads the maximum collateral that the provider +// will put up against a deal from miner config. +type MaxProviderCollateralMultiplierFunc func(address.Address) (uint64, error) + +// SetMaxProviderCollateralMultiplierFunc is a function which is used to set the maximum collateral +// that the provider will put up against a deal for miner. +type SetMaxProviderCollateralMultiplierFunc func(address.Address, uint64) error + +// TransferPathFunc is a function which reads the transfer-path from miner config. +type TransferPathFunc func(address.Address) (string, error) + +// SetTransferPathFunc is a function which is used to set the transfer-path for miner. +type SetTransferPathFunc func(address.Address, string) error + +type MaxPublishDealsFeeFunc func(address.Address) (vsTypes.FIL, error) +type SetMaxPublishDealsFeeFunc func(address.Address, vsTypes.FIL) error + +type MaxMarketBalanceAddFeeFunc func(address.Address) (vsTypes.FIL, error) +type SetMaxMarketBalanceAddFeeFunc func(address.Address, vsTypes.FIL) error diff --git a/config/util.go b/config/util.go new file mode 100644 index 00000000..aaec30af --- /dev/null +++ b/config/util.go @@ -0,0 +1,53 @@ +package config + +import ( + "encoding" + "time" + + "github.com/filecoin-project/go-address" +) + +// Duration is a wrapper type for Duration +// for decoding and encoding from/to TOML +type Duration time.Duration + +// UnmarshalText implements interface for TOML decoding +func (dur *Duration) UnmarshalText(text []byte) error { + d, err := time.ParseDuration(string(text)) + if err != nil { + return err + } + *dur = Duration(d) + return err +} + +func (dur Duration) MarshalText() ([]byte, error) { + d := time.Duration(dur) + return []byte(d.String()), nil +} + +var ( + _ encoding.TextMarshaler = (*Duration)(nil) + _ encoding.TextUnmarshaler = (*Duration)(nil) +) + +// Address is a wrapper type for Address +// for decoding and encoding from/to TOML +type Address address.Address + +// UnmarshalText implements interface for TOML decoding +func (addr *Address) UnmarshalText(text []byte) error { + d, err := address.NewFromString(string(text)) + if err != nil { + return err + } + *addr = Address(d) + return err +} + +func (addr Address) MarshalText() ([]byte, error) { + if address.Address(addr) == address.Undef { + return []byte{}, nil + } + return []byte(address.Address(addr).String()), nil +} diff --git a/dealfilter/cli.go b/dealfilter/cli.go index 194dd5f6..4d043ace 100644 --- a/dealfilter/cli.go +++ b/dealfilter/cli.go @@ -6,26 +6,37 @@ import ( "encoding/json" "os/exec" - "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/venus-market/v2/config" types "github.com/filecoin-project/venus/venus-shared/types/market" ) -func CliStorageDealFilter(cmd string) config.StorageDealFilter { - return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { +func CliStorageDealFilter(cfg *config.MarketConfig) config.StorageDealFilter { + return func(ctx context.Context, mAddr address.Address, deal *types.MinerDeal) (bool, string, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + if pCfg == nil || len(pCfg.Filter) == 0 { + return true, "", nil + } + d := struct { - storagemarket.MinerDeal + *types.MinerDeal DealType string }{ MinerDeal: deal, DealType: "piecestorage", } - return runDealFilter(ctx, cmd, d) + return runDealFilter(ctx, pCfg.Filter, d) } } -func CliRetrievalDealFilter(cmd string) config.RetrievalDealFilter { - return func(ctx context.Context, deal types.ProviderDealState) (bool, string, error) { +func CliRetrievalDealFilter(cfg *config.MarketConfig) config.RetrievalDealFilter { + return func(ctx context.Context, mAddr address.Address, deal types.ProviderDealState) (bool, string, error) { + pCfg := cfg.MinerProviderConfig(mAddr, true) + if pCfg == nil || len(pCfg.Filter) == 0 { + return true, "", nil + } + d := struct { types.ProviderDealState DealType string @@ -33,7 +44,7 @@ func CliRetrievalDealFilter(cmd string) config.RetrievalDealFilter { ProviderDealState: deal, DealType: "retrieval", } - return runDealFilter(ctx, cmd, d) + return runDealFilter(ctx, pCfg.RetrievalFilter, d) } } diff --git "a/docs/zh/market-client\351\205\215\347\275\256\350\247\243\351\207\212.md" "b/docs/zh/market-client\351\205\215\347\275\256\350\247\243\351\207\212.md" index aa98896a..0f9c5a94 100644 --- "a/docs/zh/market-client\351\205\215\347\275\256\350\247\243\351\207\212.md" +++ "b/docs/zh/market-client\351\205\215\347\275\256\350\247\243\351\207\212.md" @@ -6,7 +6,10 @@ market-client 的一份典型的配置如下 SimultaneousTransfersForRetrieval = 20 SimultaneousTransfersForStorage = 20 -DefaultMarketAddress = "t3qkgm5h7nmusacfggd744w7fdj45rn6iyl7n6s6lr34t4qlfebiphmm3vxtwc4a4acqi4nv3pqk6h7ddqqz5q" + +[DefaultMarketAddress] + Addr = "t3qkgm5h7nmusacfggd744w7fdj45rn6iyl7n6s6lr34t4qlfebiphmm3vxtwc4a4acqi4nv3pqk6h7ddqqz5q" + Account = "" [API] ListenAddress = "/ip4/127.0.0.1/tcp/41231/ws" @@ -152,5 +155,5 @@ SimultaneousTransfersForStorage = 20 # 当前market-client的默认地址 # 字符串类型 可选 （也可以直接通过命令行的 --addr flag 进行配置） -DefaultMarketAddress = "t3qkgm5h7nmusacfggd744w7fdj45rn6iyl7n6s6lr34t4qlfebiphmm3vxtwc4a4acqi4nv3pqk6h7ddqqz5q" +DefaultMarketAddress = "t3qkgm5h7nmusacfggd744w7fdj45rn6iyl7n6s6lr34t4qlfebiphmm3vxtwc4a4acqi4nv3pqk6h7ddqqz5q:username" ``` diff --git "a/docs/zh/venus-market\351\205\215\347\275\256\350\247\243\351\207\212.md" "b/docs/zh/venus-market\351\205\215\347\275\256\350\247\243\351\207\212.md" index b8d929dc..d4cb7756 100644 --- "a/docs/zh/venus-market\351\205\215\347\275\256\350\247\243\351\207\212.md" +++ "b/docs/zh/venus-market\351\205\215\347\275\256\350\247\243\351\207\212.md" @@ -1,63 +1,107 @@ # venus market 的配置解释 -一份典型的venus market 的配置是这样的: +一份典型的 `venus-market` 的配置是这样的: ``` -# ****** 基础参数配置 ******** -ConsiderOnlineStorageDeals = true -ConsiderOfflineStorageDeals = true -ConsiderOnlineRetrievalDeals = true -ConsiderOfflineRetrievalDeals = true -ConsiderVerifiedStorageDeals = true -ConsiderUnverifiedStorageDeals = true -PieceCidBlocklist = [] -ExpectedSealDuration = "24h0m0s" -MaxDealStartDelay = "336h0m0s" -PublishMsgPeriod = "5m0s" -MaxDealsPerPublishMsg = 8 -MaxProviderCollateralMultiplier = 2 +# ****** 数据传输参数配置 ******** SimultaneousTransfersForStorage = 20 SimultaneousTransfersForStoragePerClient = 20 SimultaneousTransfersForRetrieval = 20 -Filter = "" -RetrievalFilter = "" -TransfePath = "" -MaxPublishDealsFee = "0 FIL" -MaxMarketBalanceAddFee = "0 FIL" -# ****** venus market 网络配置 ******** +# ****** 全局基础参数配置 ******** +[CommonProviderConfig] + ConsiderOnlineStorageDeals = true + ConsiderOfflineStorageDeals = true + ConsiderOnlineRetrievalDeals = true + ConsiderOfflineRetrievalDeals = true + ConsiderVerifiedStorageDeals = true + ConsiderUnverifiedStorageDeals = true + PieceCidBlocklist = [] + ExpectedSealDuration = "24h0m0s" + MaxDealStartDelay = "336h0m0s" + PublishMsgPeriod = "1h0m0s" + MaxDealsPerPublishMsg = 8 + MaxProviderCollateralMultiplier = 2 + Filter = "" + RetrievalFilter = "" + TransferPath = "" + MaxPublishDealsFee = "0 FIL" + MaxMarketBalanceAddFee = "0 FIL" + [CommonProviderConfig.RetrievalPricing] + Strategy = "default" + [CommonProviderConfig.RetrievalPricing.Default] + VerifiedDealsFreeTransfer = true + [CommonProviderConfig.RetrievalPricing.External] + Path = "" + [CommonProviderConfig.AddressConfig] + DisableWorkerFallback = false + + +每个矿工可以有独立的基础参数，没有配置时使用全局配置，配置方式如下： + +# ****** miner基础参数配置 ******** +[[Miners]] + Addr = "f01000" + Account = "testuser01" + + ConsiderOnlineStorageDeals = true + ConsiderOfflineStorageDeals = true + ConsiderOnlineRetrievalDeals = true + ConsiderOfflineRetrievalDeals = true + ConsiderVerifiedStorageDeals = true + ConsiderUnverifiedStorageDeals = true + PieceCidBlocklist = [] + ExpectedSealDuration = "24h0m0s" + MaxDealStartDelay = "336h0m0s" + PublishMsgPeriod = "1h0m0s" + MaxDealsPerPublishMsg = 8 + MaxProviderCollateralMultiplier = 2 + Filter = "" + RetrievalFilter = "" + TransferPath = "" + MaxPublishDealsFee = "0 FIL" + MaxMarketBalanceAddFee = "0 FIL" + [CommonProviderConfig.RetrievalPricing] + Strategy = "default" + [CommonProviderConfig.RetrievalPricing.Default] + VerifiedDealsFreeTransfer = true + [CommonProviderConfig.RetrievalPricing.External] + Path = "" + [CommonProviderConfig.AddressConfig] + DisableWorkerFallback = false +# ****** venus market 网络配置 ******** [API] -ListenAddress = "/ip4/0.0.0.0/tcp/41235" -RemoteListenAddress = "" -Secret = "878f9c1f88c6f68ee7be17e5f0848c9312897b5d22ff7d89ca386ed0a583da3c" -Timeout = "30s" + ListenAddress = "/ip4/127.0.0.1/tcp/41235" + RemoteListenAddress = "" + Secret = "e647ee23cf95424162b974cd641b6a6479cbc7cb1209cc755f762c8248d50ba4" + Timeout = "30s" [Libp2p] -ListenAddresses = ["/ip4/0.0.0.0/tcp/58418", "/ip6/::/tcp/0"] -AnnounceAddresses = [] -NoAnnounceAddresses = [] -PrivateKey = "08011240ae580daabbe087007d2b4db4e880af10d582215d2272669a94c49c854f36f99c35c38130ac8731dedae9cc885c644554d3e4ca9203ffeeeb9ee7a689a3e52a21" + ListenAddresses = ["/ip4/0.0.0.0/tcp/58418", "/ip6/::/tcp/0"] + AnnounceAddresses = [] + NoAnnounceAddresses = [] + PrivateKey = "08011240d47934b6fccf8b79786335a55ccc04bdb9c92866cae2c0cea2fdefe0f2e7c18650dfbde5dd126c2a23a0d1c60686d3dedd064b67ba97c6161dd8007f0675e1a9" # ****** venus 组件服务配置 ******** [Node] -Url = "/ip4/192.168.200.128/tcp/3453" -Token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiZm9yY2VuZXQtbnYxNiIsInBlcm0iOiJhZG1pbiIsImV4dCI6IiJ9.PuzEy1TlAjjNiSUu_tbHi2XPUritDLm9Xf5UW3MHRe8" + Url = "/ip4/192.168.200.151/tcp/3453" + Token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoidGVzdC11c2VyMDEiLCJwZXJtIjoic2lnbiIsImV4dCI6IiJ9.ETjNy3HMDS3ScZ3cax9xYb6AopNWYp4y71lZGCvYxMg" [Messager] -Url = "/ip4/192.168.200.128/tcp/39812/" -Token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiZm9yY2VuZXQtbnYxNiIsInBlcm0iOiJhZG1pbiIsImV4dCI6IiJ9.PuzEy1TlAjjNiSUu_tbHi2XPUritDLm9Xf5UW3MHRe8" + Url = "/ip4/127.0.0.1/tcp/39812" + Token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoidGVzdC11c2VyMDEiLCJwZXJtIjoic2lnbiIsImV4dCI6IiJ9.ETjNy3HMDS3ScZ3cax9xYb6AopNWYp4y71lZGCvYxMg" [Signer] -Type = "gateway" -Url = "/ip4/192.168.200.128/tcp/45132/" -Token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiZm9yY2VuZXQtbnYxNiIsInBlcm0iOiJhZG1pbiIsImV4dCI6IiJ9.PuzEy1TlAjjNiSUu_tbHi2XPUritDLm9Xf5UW3MHRe8" + Type = "gateway" + Url = "/ip4/127.0.0.1/tcp/45132" + Token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoidGVzdC11c2VyMDEiLCJwZXJtIjoic2lnbiIsImV4dCI6IiJ9.ETjNy3HMDS3ScZ3cax9xYb6AopNWYp4y71lZGCvYxMg" [AuthNode] -Url = "http://192.168.200.128:8989" -Token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiZm9yY2VuZXQtbnYxNiIsInBlcm0iOiJhZG1pbiIsImV4dCI6IiJ9.PuzEy1TlAjjNiSUu_tbHi2XPUritDLm9Xf5UW3MHRe8" + Url = "http://127.0.0.1:8989" + Token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoidGVzdC11c2VyMDEiLCJwZXJtIjoic2lnbiIsImV4dCI6IiJ9.ETjNy3HMDS3ScZ3cax9xYb6AopNWYp4y71lZGCvYxMg" @@ -85,11 +129,6 @@ Path = "./.vscode/test" Path = "journal" -# ******** 消息发送地址的配置 ******** -[AddressConfig] -DisableWorkerFallback = false - - # ******** DAG存储设置 ******** [DAGStore] @@ -105,16 +144,7 @@ UseTransient = false # ******** 数据检索配置 ******** -[RetrievalPaymentAddress] -Addr = "" -Account = "" - -[RetrievalPricing] -Strategy = "default" -[RetrievalPricing.Default] -VerifiedDealsFreeTransfer = true -[RetrievalPricing.External] -Path = "" +RetrievalPaymentAddress = "" @@ -139,6 +169,21 @@ Path = "" 接下来，将这个配置分成基础参数，网络配置，Venus组件配置等多个部分进行讲解 +## 数据传输参数配置 +``` +# 存储订单的最大同时传输数目 +# 整数类型 默认为：20 +SimultaneousTransfersForStorage = 20 + +# 针对每一个客户端的存储订单最大同时传输数目 +# 整数类型 默认为：20 +SimultaneousTransfersForStoragePerClient = 20 + +# 获取数据最大同时传输数目 +# 整数类型 默认为：20 +SimultaneousTransfersForRetrieval = 20 +``` + ## 基础参数配置 这部分的配置主要是决定了了market在进行工作时的偏好，满足定制化的需求，其中各项配置的作用如下： @@ -194,27 +239,15 @@ MaxDealsPerPublishMsg = 8 # 整数类型 默认为：2 MaxProviderCollateralMultiplier = 2 -# 存储订单的最大同时传输数目 -# 整数类型 默认为：20 -SimultaneousTransfersForStorage = 20 - -# 针对每一个客户端的存储订单最大同时传输数目 -# 整数类型 默认为：20 -SimultaneousTransfersForStoragePerClient = 20 - -# 获取数据最大同时传输数目 -# 整数类型 默认为：20 -SimultaneousTransfersForRetrieval = 20 - -# 保留字段 +# 通过外部执行器来筛选存储订单,是可执行的程序或脚本 Filter = "" -# 保留字段 +# 通过外部执行器来筛选检索订单,是可执行的程序或脚本 RetrievalFilter = "" # 订单传输数据的存储位置 # 字符串类型 可选 为空值时默认使用`MARKET_REPO`的路径 -TransfePath = "" +TransferPath = "" # 发送订单消息的最大费用 # FIL类型 默认为："0 FIL" @@ -224,6 +257,43 @@ MaxPublishDealsFee = "0 FIL" # 发送增加抵押消息时花费的最大费用 # FIL类型 默认为："0 FIL" MaxMarketBalanceAddFee = "0 FIL" + +# 保留字段，当前配置无效 +[RetrievalPricing] + +# 使用的策略类型 +# 字符串类型 可以选择"default"和"external" 默认为:"default" +# 前者使用内置的默认策略，后者使用外部提供的脚本自定义的策略 +Strategy = "default" + +[RetrievalPricing.Default] + +# 对于经过认证的订单数据，是否定价为0 +# 布尔值 默认为 "true" +# 只有Strategy = "default" 才会生效 +VerifiedDealsFreeTransfer = true + +[RetrievalPricing.External] +# 定义外部策略的脚本的路径 +# 字符串类型 如果选择external策略时，必选 +Path = "" + +# 该设置为保留字段，当前无效 +[AddressConfig] + +# 是否降低使用woker地址发布消息的优先级，如果是，则只有在其他可选地址没有的情况下才会使用woker的地址发消息 +# 布尔值 默认为 false +DisableWorkerFallback = false + +[[AddressConfig.DealPublishControl]] + +# 发布订单消息的地址 +# 字符串类型 必选 +Addr = "" + +# 持有相应地址的账户 +# 字符串类型 必选 +Account ="" ``` ## venus market 网络配置 @@ -314,7 +384,7 @@ venus 提供签名服务的组件，它可以由两种类型：由venus-wallet ``` [Signer] # 签名服务组件的类型 -# 字符串类型 枚举："gateway"，"wallet" +# 字符串类型 枚举："gateway"，"wallet"，"lotusnode" Type = "gateway" # 签名服务入口 @@ -347,7 +417,7 @@ Token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiZm9yY2VuZXQtbnYxNiIsIn 预置矿工信息 ``` -[[StorageMiners]] +[[Miners]] # 矿工的地址 # 字符串类型 必选 Addr ="" @@ -355,6 +425,8 @@ Addr ="" # 账户名 # 字符串类型 必选 Account = "" + +# 基础参数，见上文 ``` @@ -464,30 +536,6 @@ Token = "" Path = "journal" ``` -## 消息发送地址的配置 - -该设置为保留字段，当前无效 - -``` -[AddressConfig] - -# 是否降低使用woker地址发布消息的优先级，如果是，则只有在其他可选地址没有的情况下才会使用woker的地址发消息 -# 布尔值 默认为 false -DisableWorkerFallback = false - - -[[DealPublishControl]] - -# 发布订单消息的地址 -# 字符串类型 必选 -Addr = "" - -# 持有相应地址的账户 -# 字符串类型 必选 -Account ="" - -``` - ## DAG存储设置 @@ -537,40 +585,11 @@ UseTransient = false 获取订单中存储的扇区数据时的相关配置 ### [RetrievalPaymentAddress] -获取订单扇区数据时，使用的支付地址 +获取订单扇区数据时，使用的收款地址 ``` -[RetrievalPaymentAddress] -Addr = "" -Account = "" - +RetrievalPaymentAddress = "" ``` -### [RetrievalPricing] - -保留字段，当前配置无效 - -``` -[RetrievalPricing] - -# 使用的策略类型 -# 字符串类型 可以选择"default"和"external" 默认为:"default" -# 前者使用内置的默认策略，后者使用外部提供的脚本自定义的策略 -Strategy = "default" - -[RetrievalPricing.Default] - -# 对于经过认证的订单数据，是否定价为0 -# 布尔值 默认为 "true" -# 只有Strategy = "default" 才会生效 -VerifiedDealsFreeTransfer = true - -[RetrievalPricing.External] -# 定义外部策略的脚本的路径 -# 字符串类型 如果选择external策略时，必选 -Path = "" -``` - - ## Metric 配置 配置 Metric 相关的参数 diff --git "a/docs/zh/\345\277\253\351\200\237\345\220\257\347\224\250.md" "b/docs/zh/\345\277\253\351\200\237\345\220\257\347\224\250.md" index c69b3b56..1093813b 100644 --- "a/docs/zh/\345\277\253\351\200\237\345\220\257\347\224\250.md" +++ "b/docs/zh/\345\277\253\351\200\237\345\220\257\347\224\250.md" @@ -271,19 +271,29 @@ Price per GiB/Epoch Verified Min. Piece Size (padded) Max. Piece Size (padded --node-url=/ip4//tcp/ \ --messager-url=/ip4//tcp/ \ --auth-token= \ ---wallet-url=/ip4//tcp/ \ ---wallet-token= \ +--signer-type=wallet \ +--signer-url=/ip4//tcp/ \ +--signer-toke= \ --addr= \ ``` > `venus-wallet` 生成具有签名权限的 `token` 请参考上文. + +- 连接 `lotus fullnode` 启动 +```shell +./market-client run \ +--node-url=/ip4//tcp/ \ +--node-token= \ +--signer-type=lotusnode \ +--addr= \ +``` + - 连接 `venus fullnode` 启动 ```shell ./market-client run \ --node-url=/ip4//tcp/ \ --node-token= \ ---wallet-url=/ip4//tcp/ \ ---wallet-token= \ +--signer-type=wallet \ --addr= \ ``` diff --git "a/docs/zh/\347\255\276\345\220\215\351\207\215\346\236\204.md" "b/docs/zh/\347\255\276\345\220\215\351\207\215\346\236\204.md" new file mode 100644 index 00000000..8a9cf479 --- /dev/null +++ "b/docs/zh/\347\255\276\345\220\215\351\207\215\346\236\204.md" @@ -0,0 +1,138 @@ +# venus 链服务组件间签名机制 + +## 背景介绍 + +`venus` 的 `account` 体系中，各角色的关系如下： + +- `venus-auth` 为指定的用户（`account`）生成所需权限的 `token`， `SP` 用户用此 `token` 访问链服务接口； +- `venus-auth` 保存 `account-minerIDs`，即一个用户可以拥有对个 `minerID`； +- 链服务组件之间的接口访问没有用到用户的 `token`, 是系统内置的共享 `token` + +### 用户请求的签名机制 + +以 `venus-cluster` 推送消息的过程为例： + +- 以 `token` 请求 `venus-messager` 的接口: + +``` +PushMessage(ctx context.Context, msg *shared.Message, meta *types.SendSpec) (string, error) +``` + +- `venus-messager` 处理 `rpcapi` 请求，从 `token` 中解析获得在 `venus-auth` 中生成的 `account`, 然后调用 `venus-gateway` 中的签名接口: + +``` +WalletSign(ctx context.Context, account string, addr address.Address, toSign []byte, meta venusTypes.MsgMeta) (*crypto.Signature, error) +``` + +- `venus-gateway` 根据 `account` 找到对应的 `venus-wallet` 服务进行签名。 + +### 链服务内部的签名机制 + + 链服务组件之间的接口访问用的是内置的 `share token`，故无法通过 `token` 解析出 `account`， 这样就需要依赖 `venus-auth` 的 保存 `account-minerIDs` 体系。以 `venus-miner` 请求 `venus-gateway` 为例进行说明。 + +- `venus-miner` 从 `venus-auth` 获取 `account-miners` 列表； +- 先从`account-miners` 列表中查找 `minerID` 对应的 `account`， 然后调用 `venus-gateway` 的 `WalletSign` 接口。 + +### 问题分析 + +本问题以接入 `venus` 链服务的场景进行分析 + +- 链服务中没有 `signer address` 与 `account` 的关系表，而依赖链服务的签名接口需要 `account` 参数，`venus-market` 需要额外扩展 `signer-account` 关系，如下： + +假设 `f0128788` 的可签名地址有: `f3wylwd6pclppme4qmbgwled5xpsbgwgqbn2alxa7yahg2gnbfkipsdv6m764xm5coizujmwdmkxeugplmorha`, `f3r47fkdzfmtex5ic3jnwlzc7bkpbj7s4d6limyt4f57t3cuqq5nuvhvwv2cu2a6iga2s64vjqcxjqiezyjooq`。 + +初始 `miner-account`, `venus-auth` 中实现的关系： +``` +f0128788-test +``` + +扩展后，`venus-market` 中的扩展: +``` +f0128788-test +f3wylwd6pclppme4qmbgwled5xpsbgwgqbn2alxa7yahg2gnbfkipsdv6m764xm5coizujmwdmkxeugplmorha-test +f3r47fkdzfmtex5ic3jnwlzc7bkpbj7s4d6limyt4f57t3cuqq5nuvhvwv2cu2a6iga2s64vjqcxjqiezyjooq-test +``` + +- 存储市场跟踪订单或扇区状态产生了不必要的工作量，会把非 `miner Actor` 的地址也进行遍历，如下： +``` +func (dealTracker *DealTracker) scanDeal(ctx metrics.MetricsCtx) { + addrs, err := dealTracker.minerMgr.ActorAddress(ctx) + if err != nil { + log.Errorf("get miners list %w", err) + } + head, err := dealTracker.fullNode.ChainHead(ctx) + if err != nil { + log.Errorf("get chain head %w", err) + } + + for _, addr := range addrs { + dealTracker.checkSlash(ctx, addr, head.Key()) + dealTracker.checkPreCommitAndCommit(ctx, addr, head.Key()) + } +} +``` + + +## Specification (feature Spec) + +本次重构涉及组件包括：venus-auth，venus-gateway，venus-messager， venus-miner， venus-market等，接下来将一一介绍修改内容。 + +### venus-auth + +`venus-auth` 提供接口将签名地址和账号进行绑定，并提供查询接口，保障链服务可以安全地支持多用户相互签名，接口如下： +``` +type IAuthClient interface { + GetUserBySigner(signer string) (auth.ListUsersResponse, error) + RegisterSigners(user string, addrs []string) error + UnregisterSigners(user string, addrs []string) error +} +``` + +### venus-gateway + +签名接口重构: +``` +type IWalletClient interface { + WalletHas(ctx context.Context, addr address.Address, accounts []string) (bool, error) //perm:admin + WalletSign(ctx context.Context, addr address.Address, accounts []string, toSign []byte, meta types.MsgMeta) (*crypto.Signature, error) //perm:admin +} +``` + +- `venus-gateway` 签名接口只开放对具有 `admin` 权限的 `token` 访问,也就是只能被链服务内部访问，如：`venus-miner`,`venus-messager`； +- 在调用 `venus-gateway` 的签名接口前需要完成权限的检查，这样做为了避免构造消息，调用别人账号下的私钥签名； +- 在调用 `venus-gateway` 的签名接口时需要带上密钥绑定的账号，这样做是为了多个账号的`venus-wallet`可以相互签名； + +自动注册/注销签名地址： + +- 在`venus-wallet` 建立连接时将其保存的私钥和支持的账号进行校验，通过后调用接口保存到 `venus-auth`。如 某个`venus-wallet`终端管理的私钥有： w1，w2；配置的支持账号有：user01，user02；那么绑定的关系有： user01-w1，user01-w2，user02-w1，user02-w2； +- 在`venus-wallet` 连接端口或通过接口删除某个地址时自动注销签名地址和账号的绑定关系。 + +### venus-messager + +删除强制推送消息的接口，SP统一消息推送接口。 +``` +type IMessager interface { + - ForcePushMessage(ctx context.Context, account string, msg *types.Message, meta *mtypes.SendSpec) (string, error) //perm:admin + - ForcePushMessageWithId(ctx context.Context, id string, account string, msg *types.Message, meta *mtypes.SendSpec) (string, error) //perm:write +} +``` + +推送消息流程修改: +- SP组件调用链服务通常是 [`sign`,`read`,`write`] 权限的 `token`，`venus-messager` 首先要检查接口带的 `token` 权限是否满足： + - `token` 对应的的账号与消息中的签名地址 `msg.From` 是绑定的； + - `token` 具有 `sign` 权限(待定)； +- 给签名接口提供签名地址绑定的所有账号，这样只要有一个连接到 `venus-gateway` 的签名终端(`venus-wallet`)支持对此密钥签名则可以签名此条消息，也就是多个账号相互帮助签名； + +### venus-miner + +无逻辑变化，只需匹配新的接口调用即可。 + +### venus-market + +- 取消额外扩展 `signer-account` 关系的逻辑； +- 消息签名前先检查请求 `token` 的权限，通过后调用 `venus-gateway`的签名接口。 + +这里要注意的是 `market-client` 组件，其实现和 `venus-market` 在同一项目，但是其本质是SP类组件，过程中的签名不能通过`venus-gateway`, 其签名逻辑： +- 对于deals的签名需要额外的签名终端(`venus-wallet`或`lotus/venus`)； +- 交易过程中的消息推送到 `venus-messager`, 和 `venus-cluster`等推送消息一致； +> 所以 `market-client` 组件代码独立为一个项目比较好？从 `venus-market` 中剥离？ diff --git a/fundmgr/fundmanager.go b/fundmgr/fundmanager.go index 6181a5ce..dc4a1ef3 100644 --- a/fundmgr/fundmanager.go +++ b/fundmgr/fundmanager.go @@ -5,23 +5,23 @@ import ( "fmt" "sync" - "github.com/filecoin-project/venus-market/v2/api/clients" - "github.com/filecoin-project/venus/venus-shared/actors" - - "github.com/filecoin-project/venus-market/v2/models/repo" - - "github.com/filecoin-project/venus/pkg/constants" - v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" - types "github.com/filecoin-project/venus/venus-shared/types/market" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "go.uber.org/fx" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/go-state-types/builtin/v9/market" + + "github.com/filecoin-project/venus-market/v2/api/clients" + "github.com/filecoin-project/venus-market/v2/models/repo" + + "github.com/filecoin-project/venus/pkg/constants" + "github.com/filecoin-project/venus/venus-shared/actors" + v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" types2 "github.com/filecoin-project/venus/venus-shared/types" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - "go.uber.org/fx" + types "github.com/filecoin-project/venus/venus-shared/types/market" ) var log = logging.Logger("market_adapter") @@ -689,13 +689,15 @@ func (env *fundManagerEnvironment) AddFunds( return cid.Undef, err } - msgId, aerr := env.api.PushMessage(ctx, &types2.Message{ - To: builtin.StorageMarketActorAddr, - From: wallet, - Value: amt, - Method: builtin.MethodsMarket.AddBalance, - Params: params, - }, nil) + msgId, aerr := env.api.PushMessage( + ctx, + &types2.Message{ + To: builtin.StorageMarketActorAddr, + From: wallet, + Value: amt, + Method: builtin.MethodsMarket.AddBalance, + Params: params, + }, nil) if aerr != nil { return cid.Undef, aerr @@ -718,13 +720,15 @@ func (env *fundManagerEnvironment) WithdrawFunds( return cid.Undef, fmt.Errorf("serializing params: %w", err) } - msgId, aerr := env.api.PushMessage(ctx, &types2.Message{ - To: builtin.StorageMarketActorAddr, - From: wallet, - Value: types2.NewInt(0), - Method: builtin.MethodsMarket.WithdrawBalance, - Params: params, - }, nil) + msgId, aerr := env.api.PushMessage( + ctx, + &types2.Message{ + To: builtin.StorageMarketActorAddr, + From: wallet, + Value: types2.NewInt(0), + Method: builtin.MethodsMarket.WithdrawBalance, + Params: params, + }, nil) if aerr != nil { return cid.Undef, aerr diff --git a/fundmgr/fundmanager_test.go b/fundmgr/fundmanager_test.go index 82802379..1a51e80d 100644 --- a/fundmgr/fundmanager_test.go +++ b/fundmgr/fundmanager_test.go @@ -7,21 +7,24 @@ import ( "testing" "time" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/require" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/go-state-types/builtin/v9/market" tutils "github.com/filecoin-project/specs-actors/v2/support/testing" + "github.com/filecoin-project/venus-market/v2/models/badger" "github.com/filecoin-project/venus-market/v2/models/repo" + "github.com/filecoin-project/venus/pkg/config" _ "github.com/filecoin-project/venus/pkg/crypto/secp" "github.com/filecoin-project/venus/pkg/wallet" "github.com/filecoin-project/venus/venus-shared/types" - "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" - "github.com/stretchr/testify/require" ) // TestFundManagerBasic verifies that the basic fund manager operations work diff --git a/go.mod b/go.mod index f670edb5..76d3c8d7 100644 --- a/go.mod +++ b/go.mod @@ -25,15 +25,15 @@ require ( github.com/filecoin-project/go-fil-markets v1.24.0-v17 github.com/filecoin-project/go-jsonrpc v0.1.5 github.com/filecoin-project/go-padreader v0.0.1 - github.com/filecoin-project/go-state-types v0.9.1 + github.com/filecoin-project/go-state-types v0.9.8 github.com/filecoin-project/go-statemachine v1.0.2 github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/specs-actors/v2 v2.3.6 github.com/filecoin-project/specs-actors/v7 v7.0.1 github.com/filecoin-project/specs-storage v0.4.1 - github.com/filecoin-project/venus v1.8.0-rc3 - github.com/filecoin-project/venus-auth v1.8.0-rc1 - github.com/filecoin-project/venus-messager v1.8.0-rc2 + github.com/filecoin-project/venus v1.7.1-0.20221116064917-aab91a46434e + github.com/filecoin-project/venus-auth v1.8.0-rc1.0.20221101071442-c2ed29a74e9e + github.com/filecoin-project/venus-messager v1.8.0-rc2.0.20221103053206-08c153719d87 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 @@ -41,7 +41,7 @@ require ( github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef github.com/ipfs-force-community/metrics v1.0.1-0.20220824061112-ac916bacf2ea github.com/ipfs-force-community/venus-common-utils v0.0.0-20220217030526-e5e4c6bc14f7 - github.com/ipfs-force-community/venus-gateway v1.8.0-rc1 + github.com/ipfs-force-community/venus-gateway v1.8.0-rc1.0.20221102034912-cf73a722a26a github.com/ipfs/go-block-format v0.0.3 github.com/ipfs/go-blockservice v0.4.0 github.com/ipfs/go-cid v0.2.0 @@ -103,7 +103,6 @@ require ( github.com/DataDog/zstd v1.4.1 // indirect github.com/Stebalien/go-bitfield v0.0.1 // indirect github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249 // indirect - github.com/ahmetb/go-linq/v3 v3.2.0 // indirect github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect github.com/awnumar/memcall v0.0.0-20191004114545-73db50fd9f80 // indirect github.com/awnumar/memguard v0.22.2 // indirect @@ -141,12 +140,10 @@ require ( github.com/filecoin-project/specs-actors/v4 v4.0.2 // indirect github.com/filecoin-project/specs-actors/v5 v5.0.6 // indirect github.com/filecoin-project/specs-actors/v6 v6.0.2 // indirect - github.com/filecoin-project/specs-actors/v8 v8.0.1 // indirect github.com/filecoin-project/storetheindex v0.4.17 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/gbrlsnchs/jwt/v3 v3.0.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.8.1 // indirect @@ -282,7 +279,6 @@ require ( github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/go-logging v0.0.1 // indirect github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect - github.com/x448/float16 v0.8.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.0.2 // indirect github.com/xdg-go/stringprep v1.0.2 // indirect diff --git a/go.sum b/go.sum index 04911129..249e88d8 100644 --- a/go.sum +++ b/go.sum @@ -102,7 +102,6 @@ github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249 h1:fMi9ZZ/it4or github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249/go.mod h1:iU1PxQMQwoHZZWmMKrMkrNlY+3+p9vxIjpZOVyxWa0g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/ahmetb/go-linq/v3 v3.2.0 h1:BEuMfp+b59io8g5wYzNoFe9pWPalRklhlhbiU3hYZDE= github.com/ahmetb/go-linq/v3 v3.2.0/go.mod h1:haQ3JfOeWK8HpVxMtHHEMPVgBKiYyQ+f1/kLZh/cj9U= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= @@ -423,9 +422,9 @@ github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/ github.com/filecoin-project/go-state-types v0.1.8/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.12/go.mod h1:n/kujdC9JphvYTrmaD1+vJpvDPy/DwzckoMzP0nBKWI= -github.com/filecoin-project/go-state-types v0.9.0/go.mod h1:+HCZifUV+e8TlQkgll22Ucuiq8OrVJkK+4Kh4u75iiw= -github.com/filecoin-project/go-state-types v0.9.1 h1:Dr++Hy+tm8RC5jLQBPfFIvThPCa6uEJ4NwlCWh3V85Q= -github.com/filecoin-project/go-state-types v0.9.1/go.mod h1:+HCZifUV+e8TlQkgll22Ucuiq8OrVJkK+4Kh4u75iiw= +github.com/filecoin-project/go-state-types v0.9.7/go.mod h1:+HCZifUV+e8TlQkgll22Ucuiq8OrVJkK+4Kh4u75iiw= +github.com/filecoin-project/go-state-types v0.9.8 h1:xkdITiR7h691z1tWOhNCJxHI+cq+Mq7ATkpHQ7f1gu8= +github.com/filecoin-project/go-state-types v0.9.8/go.mod h1:+HCZifUV+e8TlQkgll22Ucuiq8OrVJkK+4Kh4u75iiw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v1.0.1/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= github.com/filecoin-project/go-statemachine v1.0.2-0.20220322104818-27f8fbb86dfd/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= @@ -476,14 +475,15 @@ github.com/filecoin-project/storetheindex v0.4.17 h1:w0dVc954TGPukoVbidlYvn9Xt+w github.com/filecoin-project/storetheindex v0.4.17/go.mod h1:y2dL8C5D3PXi183hdxgGtM8vVYOZ1lg515tpl/D3tN8= github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/filecoin-project/venus v1.2.4/go.mod h1:hJULXHGAnWuq5S5KRtPkwbT8DqgM9II7NwyNU7t59D0= -github.com/filecoin-project/venus v1.8.0-rc1/go.mod h1:nLm+ry+z4lVpIpyYlh5xvCrYZ53CQE4aKng1rPv+5Vg= -github.com/filecoin-project/venus v1.8.0-rc3 h1:qQJGhVS9IBAy9V3ZLnxm8YNQDVAR4iptrT7PYVCNkhQ= -github.com/filecoin-project/venus v1.8.0-rc3/go.mod h1:D49YBoTKkIHd2O022uE6BBrmu92K7vZDN9+hhZ9/kI4= +github.com/filecoin-project/venus v1.7.1-0.20221102021930-ac0f60ee4c31/go.mod h1:+QxMl7tEDvNLPt2ZnsGQOKWdmNoWF4ZxrHP+Z4phv/0= +github.com/filecoin-project/venus v1.7.1-0.20221116064917-aab91a46434e h1:bHSvy9208RQc1d3/LsRjZd191slW3PEI55L/bsetPdQ= +github.com/filecoin-project/venus v1.7.1-0.20221116064917-aab91a46434e/go.mod h1:+QxMl7tEDvNLPt2ZnsGQOKWdmNoWF4ZxrHP+Z4phv/0= github.com/filecoin-project/venus-auth v1.3.2/go.mod h1:m5Jog2GYxztwP7w3m/iJdv/V1/bTcAVU9rm/CbhxRQU= -github.com/filecoin-project/venus-auth v1.8.0-rc1 h1:BvaPt+wfsXmIGNiitv7y5XHXOcQlfXZEznGKIAyR6j0= github.com/filecoin-project/venus-auth v1.8.0-rc1/go.mod h1:DR/RdqOycbMyQ4/oKGYMGlh8qwpLw8A40eL1Qbj7KpQ= -github.com/filecoin-project/venus-messager v1.8.0-rc2 h1:gEPM7663ZBgGtQ31XQyROWM6bWtUcosvjBmr94DYzLM= -github.com/filecoin-project/venus-messager v1.8.0-rc2/go.mod h1:3+sYW+9gCZEFiJt9e1QKJ95djU9pX3A8HDhxJxlhG3o= +github.com/filecoin-project/venus-auth v1.8.0-rc1.0.20221101071442-c2ed29a74e9e h1:KsM5XFZrnoN4JmeRBMwSlpUuexJaNQuRJMjTbZzPcQc= +github.com/filecoin-project/venus-auth v1.8.0-rc1.0.20221101071442-c2ed29a74e9e/go.mod h1:DR/RdqOycbMyQ4/oKGYMGlh8qwpLw8A40eL1Qbj7KpQ= +github.com/filecoin-project/venus-messager v1.8.0-rc2.0.20221103053206-08c153719d87 h1:WBR3DxQkavOPPA3qdy/XRLB1rTW+HdZjQ4aIFzLQ+PI= +github.com/filecoin-project/venus-messager v1.8.0-rc2.0.20221103053206-08c153719d87/go.mod h1:5OasDzCbO9ZbH95SrNG+XWDB5FBaVc41tgMmCjo8nvw= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= @@ -504,7 +504,6 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= github.com/gammazero/keymutex v0.0.2/go.mod h1:qtzWCCLMisQUmVa4dvqHVgwfh4BP2YB7JxNDGXnsKrs= @@ -855,8 +854,8 @@ github.com/ipfs-force-community/metrics v1.0.1-0.20220824061112-ac916bacf2ea/go. github.com/ipfs-force-community/venus-common-utils v0.0.0-20210924063144-1d3a5b30de87/go.mod h1:RTVEOzM+hkpqmcEWpyLDkx1oGO5r9ZWCgYxG/CsXzJQ= github.com/ipfs-force-community/venus-common-utils v0.0.0-20220217030526-e5e4c6bc14f7 h1:v/1/INcqm3kHLauWQYB63MwWJRWGz+3WEuUPp0jzIl8= github.com/ipfs-force-community/venus-common-utils v0.0.0-20220217030526-e5e4c6bc14f7/go.mod h1:sSTUXgIu95tPHvgcYhdLuELmgPJWCP/pNMFtsrVtOyA= -github.com/ipfs-force-community/venus-gateway v1.8.0-rc1 h1:eagL5hyTfaLnavoR3joIC5AQs6WZhKcqYWbvq5c4Xdw= -github.com/ipfs-force-community/venus-gateway v1.8.0-rc1/go.mod h1:GKHndOiawZ/z72S1EQ4F+GQiiViw8P5BCEvwYks9gyA= +github.com/ipfs-force-community/venus-gateway v1.8.0-rc1.0.20221102034912-cf73a722a26a h1:UKfKybn7vfPaLNp0aBYYp4OWc3OxhhcGY9/+P4s4Q9k= +github.com/ipfs-force-community/venus-gateway v1.8.0-rc1.0.20221102034912-cf73a722a26a/go.mod h1:e6f4ykbl1NG9b2dSWlWJPYHuU7ai799+Mab4CptlzKA= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= @@ -2221,7 +2220,6 @@ github.com/whyrusleeping/pubsub v0.0.0-20190708150250-92bcb0691325/go.mod h1:g7c github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= diff --git a/metrics/modules.go b/metrics/modules.go index e3def9d2..bf66ee48 100644 --- a/metrics/modules.go +++ b/metrics/modules.go @@ -3,10 +3,11 @@ package metrics import ( "context" + "go.uber.org/fx" + "github.com/ipfs-force-community/metrics" "github.com/ipfs-force-community/venus-common-utils/builder" metrics2 "github.com/ipfs/go-metrics-interface" - "go.uber.org/fx" ) var startMetricsKey = builder.NextInvoke() diff --git a/minermgr/miner_mgr.go b/minermgr/miner_mgr.go new file mode 100644 index 00000000..4fcd9f73 --- /dev/null +++ b/minermgr/miner_mgr.go @@ -0,0 +1,160 @@ +package minermgr + +import ( + "context" + "sync" + "time" + + logging "github.com/ipfs/go-log/v2" + + "github.com/ipfs-force-community/metrics" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/venus-auth/auth" + "github.com/filecoin-project/venus-auth/core" + "github.com/filecoin-project/venus-auth/jwtclient" + + "github.com/filecoin-project/venus-market/v2/config" + + "github.com/filecoin-project/venus/venus-shared/types/market" +) + +const CoMinersLimit = 500 + +var log = logging.Logger("user-manager") + +type MinerMgrImpl struct { + authClient *jwtclient.AuthClient + authNode config.AuthNode + + miners map[address.Address]*market.User + lk sync.Mutex + + localMiners map[address.Address]*market.User +} + +var _ IMinerMgr = (*MinerMgrImpl)(nil) + +func NewMinerMgrImpl(ctx metrics.MetricsCtx, authClient *jwtclient.AuthClient, cfg *config.MarketConfig) (IMinerMgr, error) { + m := &MinerMgrImpl{ + authClient: authClient, + authNode: cfg.AuthNode, + + miners: make(map[address.Address]*market.User), + localMiners: make(map[address.Address]*market.User), + } + + // storage miner + for _, miner := range cfg.Miners { + m.localMiners[address.Address(miner.Addr)] = &market.User{Addr: address.Address(miner.Addr), Account: miner.Account} + } + + if authClient != nil { + go m.refreshUsers(ctx) + } + + return m, nil +} + +func (m *MinerMgrImpl) MinerList(context.Context) ([]address.Address, error) { + m.lk.Lock() + defer m.lk.Unlock() + + miners := make([]address.Address, len(m.miners)) + for miner := range m.miners { + miners = append(miners, miner) + } + + for miner := range m.localMiners { + miners = append(miners, miner) + } + + return miners, nil +} + +func (m *MinerMgrImpl) ActorList(ctx context.Context) ([]market.User, error) { + m.lk.Lock() + defer m.lk.Unlock() + + users := make([]market.User, 0) + for _, user := range m.miners { + users = append(users, market.User{Addr: user.Addr, Account: user.Account}) + } + + for _, user := range m.localMiners { + users = append(users, market.User{Addr: user.Addr, Account: user.Account}) + } + + return users, nil +} + +func (m *MinerMgrImpl) Has(ctx context.Context, mAddr address.Address) bool { + m.lk.Lock() + defer m.lk.Unlock() + + _, ok := m.miners[mAddr] + if ok { + return ok + } + + _, ok = m.localMiners[mAddr] + return ok +} + +func (m *MinerMgrImpl) getMinerFromVenusAuth(ctx context.Context, skip, limit int64) error { + m.lk.Lock() + defer m.lk.Unlock() + + if m.authClient == nil { + return nil + } + + if limit == 0 { + limit = CoMinersLimit + } + + usersWithMiners, err := m.authClient.ListUsersWithMiners(&auth.ListUsersRequest{ + Page: &core.Page{Skip: skip, Limit: limit}, + State: int(core.UserStateEnabled), + }) + if err != nil { + return err + } + + // To avoid some deleted or disabled ones that cannot be removed, re-add them here + m.miners = make(map[address.Address]*market.User) + + for _, u := range usersWithMiners { + for _, miner := range u.Miners { + addr, err := address.NewFromString(miner.Miner) + if err != nil { + log.Warnf("invalid miner: %s in user: %s", miner.Miner, u.Name) + continue + } + + if _, ok := m.miners[addr]; !ok { + m.miners[addr] = &market.User{Addr: addr, Account: u.Name} + } + } + } + + return nil +} + +func (m *MinerMgrImpl) refreshUsers(ctx context.Context) { + if err := m.getMinerFromVenusAuth(ctx, 0, 0); err != nil { + log.Errorf("first sync users from venus-auth(%s) failed: %s", m.authNode.Url, err) + } + + tm := time.NewTicker(time.Minute) + defer tm.Stop() + + for range tm.C { + log.Infof("sync users from venus-auth, url: %s\n", m.authNode.Url) + + if err := m.getMinerFromVenusAuth(ctx, 0, 0); err != nil { + log.Errorf("users from venus-auth(%s) failed:%s", m.authNode.Url, err.Error()) + } + } +} diff --git a/minermgr/minermgr.go b/minermgr/minermgr.go deleted file mode 100644 index f807ab35..00000000 --- a/minermgr/minermgr.go +++ /dev/null @@ -1,292 +0,0 @@ -package minermgr - -import ( - "bytes" - "context" - "fmt" - "sync" - "time" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/venus-auth/auth" - "github.com/filecoin-project/venus-auth/core" - "github.com/filecoin-project/venus-auth/jwtclient" - "github.com/filecoin-project/venus-market/v2/config" - "github.com/filecoin-project/venus/venus-shared/actors/builtin" - v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" - vTypes "github.com/filecoin-project/venus/venus-shared/types" - types "github.com/filecoin-project/venus/venus-shared/types/market" - "github.com/ipfs-force-community/metrics" - logging "github.com/ipfs/go-log/v2" - "github.com/pkg/errors" -) - -const CoMinersLimit = 200 - -var log = logging.Logger("address-manager") - -type IAddrMgr interface { - ActorAddress(ctx context.Context) ([]address.Address, error) - ActorList(ctx context.Context) ([]types.User, error) - Has(ctx context.Context, addr address.Address) bool - GetMiners(ctx context.Context) ([]types.User, error) - GetAccount(ctx context.Context, addr address.Address) (string, error) -} - -type UserMgrImpl struct { - fullNode v1api.FullNode - - authClient *jwtclient.AuthClient - authNode config.AuthNode - - miners []types.User - lk sync.Mutex -} - -var _ IAddrMgr = (*UserMgrImpl)(nil) - -func NeAddrMgrImpl(ctx metrics.MetricsCtx, fullNode v1api.FullNode, authClient *jwtclient.AuthClient, cfg *config.MarketConfig) (IAddrMgr, error) { - m := &UserMgrImpl{fullNode: fullNode, authClient: authClient, authNode: cfg.AuthNode} - - err := m.addUser(ctx, convertConfigAddress(cfg.StorageMiners)...) - if err != nil { - return nil, err - } - - err = m.addUser(ctx, types.User{ - Addr: address.Address(cfg.RetrievalPaymentAddress.Addr), - Account: cfg.RetrievalPaymentAddress.Account, - }) - if err != nil { - return nil, err - } - - err = m.addUser(ctx, convertConfigAddress(cfg.AddressConfig.DealPublishControl)...) - if err != nil { - return nil, err - } - - if m.authClient != nil { - if err := m.refreshOnce(ctx); err != nil { - return nil, fmt.Errorf("first refresh users from venus-auth(%s) failed: %w", m.authNode.Url, err) - } - go m.refreshUsers(ctx) - } - return m, nil -} - -func (m *UserMgrImpl) ActorAddress(ctx context.Context) ([]address.Address, error) { - m.lk.Lock() - defer m.lk.Unlock() - addrs := make([]address.Address, len(m.miners)) - for index, user := range m.miners { - addrs[index] = user.Addr - } - return addrs, nil -} - -func (m *UserMgrImpl) ActorList(ctx context.Context) ([]types.User, error) { - m.lk.Lock() - defer m.lk.Unlock() - users := make([]types.User, len(m.miners)) - for index, user := range m.miners { - users[index] = types.User{ - Addr: user.Addr, - Account: user.Account, - } - } - return users, nil -} - -func (m *UserMgrImpl) GetMiners(ctx context.Context) ([]types.User, error) { - m.lk.Lock() - defer m.lk.Unlock() - return m.miners, nil -} - -func (m *UserMgrImpl) Has(ctx context.Context, addr address.Address) bool { - m.lk.Lock() - defer m.lk.Unlock() - - for _, miner := range m.miners { - if bytes.Equal(miner.Addr.Bytes(), addr.Bytes()) { - return true - } - } - - return false -} - -func (m *UserMgrImpl) GetAccount(ctx context.Context, addr address.Address) (string, error) { - m.lk.Lock() - defer m.lk.Unlock() - - var account string - for _, miner := range m.miners { - if bytes.Equal(miner.Addr.Bytes(), addr.Bytes()) { - account = miner.Account - } - } - - if len(account) == 0 { - return "", errors.Errorf("unable find account of address %s", addr) - } - - return account, nil -} - -func (m *UserMgrImpl) getMinerFromVenusAuth(ctx context.Context, skip, limit int64) ([]types.User, error) { - if m.authClient == nil { - return nil, nil - } - if limit == 0 { - limit = CoMinersLimit - } - - usersWithMiners, err := m.authClient.ListUsersWithMiners(&auth.ListUsersRequest{ - Page: &core.Page{Skip: skip, Limit: limit}, - }) - if err != nil { - return nil, err - } - - var users []types.User - for _, u := range usersWithMiners { - if u.State != core.UserStateEnabled { - log.Warnf("user:%s state is: %s, won't list its mienrs", u.Name, u.State.String()) - continue - } - for _, m := range u.Miners { - addr, err := address.NewFromString(m.Miner) - if err != nil { - log.Warnf("invalid miner:%s in user:%s", m.Miner, u.Name) - continue - } - users = append(users, types.User{Addr: addr, Account: u.Name}) - } - } - - return users, nil -} - -func (m *UserMgrImpl) addUser(ctx context.Context, usrs ...types.User) error { - for _, usr := range usrs { - if usr.Addr == address.Undef { - continue - } - - actor, err := m.fullNode.StateGetActor(ctx, usr.Addr, vTypes.EmptyTSK) - if err != nil { - return err - } - - if err = m.appendAddress(ctx, usr.Account, usr.Addr); err != nil { - return err - } - if builtin.IsStorageMinerActor(actor.Code) { - // add owner/worker/controladdress for this miner - minerInfo, err := m.fullNode.StateMinerInfo(ctx, usr.Addr, vTypes.EmptyTSK) - if err != nil { - return err - } - - // Notice `multisig` address is not sign-able. we should ignore the `owner`, if it is a `multisig` - if err = m.appendAddress(ctx, usr.Account, minerInfo.Owner); err != nil { - return err - } - - if err = m.appendAddress(ctx, usr.Account, minerInfo.Worker); err != nil { - return err - } - - for _, ctlAddr := range minerInfo.ControlAddresses { - if err = m.appendAddress(ctx, usr.Account, ctlAddr); err != nil { - return err - } - } - } - - } - return nil -} - -func (m *UserMgrImpl) appendAddress(ctx context.Context, account string, addr address.Address) error { - m.lk.Lock() - defer m.lk.Unlock() - filter := make(map[address.Address]struct{}) - for _, miner := range m.miners { - filter[miner.Addr] = struct{}{} - } - // since `multisig` address is not sign-able. - // we should ignore the `owner`, if it is a `multisig` - actor, err := m.fullNode.StateGetActor(ctx, addr, vTypes.EmptyTSK) - if err != nil { - return err - } - - if builtin.IsAccountActor(actor.Code) { - accountKey, err := m.fullNode.StateAccountKey(ctx, addr, vTypes.EmptyTSK) - if err != nil { - return err - } - if _, ok := filter[accountKey]; !ok { - filter[accountKey] = struct{}{} - m.miners = append(m.miners, types.User{ - Addr: accountKey, - Account: account, - }) - } - } else if builtin.IsStorageMinerActor(actor.Code) { - if _, ok := filter[addr]; !ok { - filter[addr] = struct{}{} - m.miners = append(m.miners, types.User{ - Addr: addr, - Account: account, - }) - } - } - return nil -} - -// todo: looks like refreshUsers only add miners, -// -// considering venus-auth may delete/disable user(very few, but occurs), -// the correct way is syncing 'miners' with venus-auth. -func (m *UserMgrImpl) refreshOnce(ctx context.Context) error { - if m.authClient == nil { - return errors.Errorf("authClient is nil") - } - - log.Infof("refresh miners from venus-auth, url: %s\n", m.authNode.Url) - miners, err := m.getMinerFromVenusAuth(ctx, 0, 0) - if err != nil { - return err - } - return m.addUser(ctx, miners...) -} - -func (m *UserMgrImpl) refreshUsers(ctx context.Context) { - if m.authClient == nil { - log.Warnf("auth client is nil, won't refresh users from venus-auth") - return - } - tm := time.NewTicker(time.Minute) - defer tm.Stop() - - for range tm.C { - if err := m.refreshOnce(ctx); err != nil { - log.Errorf("refresh user from auth(%s) failed:%s", m.authNode.Url, err.Error()) - } - } -} - -func convertConfigAddress(addrs []config.User) []types.User { - addrs2 := make([]types.User, len(addrs)) - for index, miner := range addrs { - addrs2[index] = types.User{ - Addr: address.Address(miner.Addr), - Account: miner.Account, - } - } - return addrs2 -} diff --git a/minermgr/module.go b/minermgr/module.go index 1c4815c1..5b156521 100644 --- a/minermgr/module.go +++ b/minermgr/module.go @@ -1,12 +1,23 @@ package minermgr import ( - "github.com/filecoin-project/venus-market/v2/config" + "context" + + "github.com/filecoin-project/go-address" + "github.com/ipfs-force-community/venus-common-utils/builder" + + marketTypes "github.com/filecoin-project/venus/venus-shared/types/market" ) -var MinerMgrOpts = func(cfg *config.MarketConfig) builder.Option { +type IMinerMgr interface { + Has(context.Context, address.Address) bool + MinerList(context.Context) ([]address.Address, error) + ActorList(ctx context.Context) ([]marketTypes.User, error) +} + +var MinerMgrOpts = func() builder.Option { return builder.Options( - builder.Override(new(IAddrMgr), NeAddrMgrImpl), + builder.Override(new(IMinerMgr), NewMinerMgrImpl), ) } diff --git a/paychmgr/manager.go b/paychmgr/manager.go index b4eb582d..414c5dba 100644 --- a/paychmgr/manager.go +++ b/paychmgr/manager.go @@ -6,17 +6,22 @@ import ( "fmt" "sync" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/builtin/v8/paych" + "github.com/filecoin-project/venus-market/v2/api/clients" + "github.com/filecoin-project/venus-market/v2/api/clients/signer" "github.com/filecoin-project/venus-market/v2/models/repo" + v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" types2 "github.com/filecoin-project/venus/venus-shared/types" types "github.com/filecoin-project/venus/venus-shared/types/market" + "github.com/ipfs-force-community/metrics" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("paych") @@ -88,11 +93,11 @@ type ManagerParams struct { SM IStateManager } -func NewManager(mctx metrics.MetricsCtx, repo repo.Repo, msgClient clients.IMixMessage, fullNode v1api.FullNode) (*Manager, error) { +func NewManager(mctx metrics.MetricsCtx, repo repo.Repo, msgClient clients.IMixMessage, signer signer.ISigner, fullNode v1api.FullNode) (*Manager, error) { ctx, shutdown := context.WithCancel(mctx) impl := &managerAPIImpl{ IStateManager: newStateMgrAdapter(fullNode), - paychDependencyAPI: newPaychDependencyAPI(msgClient, fullNode, fullNode), + paychDependencyAPI: newPaychDependencyAPI(msgClient, fullNode, signer), } pm := &Manager{ ctx: ctx, @@ -336,6 +341,7 @@ func (pm *Manager) trackInboundChannel(ctx context.Context, ch address.Address) if err != nil { return nil, err } + has, err := pm.pchapi.WalletHas(ctx, toKey) if err != nil { return nil, err diff --git a/paychmgr/paych.go b/paychmgr/paych.go index 43803a40..f7d2175a 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -13,11 +13,12 @@ import ( "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/builtin/v8/paych" + + "github.com/filecoin-project/venus-market/v2/models/repo" + "github.com/filecoin-project/venus/pkg/crypto" lpaych "github.com/filecoin-project/venus/venus-shared/actors/builtin/paych" types2 "github.com/filecoin-project/venus/venus-shared/types" - - "github.com/filecoin-project/venus-market/v2/models/repo" types "github.com/filecoin-project/venus/venus-shared/types/market" ) @@ -622,6 +623,7 @@ func (ca *channelAccessor) settle(ctx context.Context, ch address.Address) (cid. if err != nil { return cid.Undef, err } + msgId, err := ca.api.PushMessage(ctx, msg, nil) if err != nil { return cid.Undef, err diff --git a/paychmgr/simple.go b/paychmgr/simple.go index ba9926cc..c01a63c6 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -7,14 +7,15 @@ import ( "fmt" "sync" - "github.com/filecoin-project/venus-market/v2/models/repo" - "github.com/ipfs/go-cid" "golang.org/x/sync/errgroup" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" - init7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/init" + + init_ "github.com/filecoin-project/go-state-types/builtin/v9/init" + + "github.com/filecoin-project/venus-market/v2/models/repo" types2 "github.com/filecoin-project/venus/venus-shared/types" types "github.com/filecoin-project/venus/venus-shared/types/market" ) @@ -439,7 +440,7 @@ func (ca *channelAccessor) waitPaychCreateMsg(ctx context.Context, channelID str // TODO: ActorUpgrade abstract over this. // This "works" because it hasn't changed from v0 to v2, but we still // need an abstraction here. - var decodedReturn init7.ExecReturn + var decodedReturn init_.ExecReturn err = decodedReturn.UnmarshalCBOR(bytes.NewReader(mwait.Receipt.Return)) if err != nil { log.Error(err) diff --git a/retrievalprovider/modules.go b/retrievalprovider/modules.go index 6377f9f5..e7d4d7d6 100644 --- a/retrievalprovider/modules.go +++ b/retrievalprovider/modules.go @@ -3,17 +3,21 @@ package retrievalprovider import ( "context" - types "github.com/filecoin-project/venus/venus-shared/types/market" + "go.uber.org/fx" - rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" "github.com/libp2p/go-libp2p/core/host" - "go.uber.org/fx" + + "github.com/filecoin-project/go-address" + rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" + + "github.com/ipfs-force-community/venus-common-utils/builder" + "github.com/ipfs-force-community/venus-common-utils/journal" "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/dealfilter" _ "github.com/filecoin-project/venus-market/v2/network" - "github.com/ipfs-force-community/venus-common-utils/builder" - "github.com/ipfs-force-community/venus-common-utils/journal" + + types "github.com/filecoin-project/venus/venus-shared/types/market" ) var HandleRetrievalKey = builder.NextInvoke() @@ -23,8 +27,8 @@ func RetrievalDealFilter(userFilter config.RetrievalDealFilter) func(onlineOk co return func(onlineOk config.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk config.ConsiderOfflineRetrievalDealsConfigFunc, ) config.RetrievalDealFilter { - return func(ctx context.Context, state types.ProviderDealState) (bool, string, error) { - b, err := onlineOk() + return func(ctx context.Context, mAddr address.Address, state types.ProviderDealState) (bool, string, error) { + b, err := onlineOk(mAddr) if err != nil { return false, "miner error", err } @@ -34,20 +38,16 @@ func RetrievalDealFilter(userFilter config.RetrievalDealFilter) func(onlineOk co return false, "miner is not accepting online retrieval deals", nil } - b, err = offlineOk() + b, err = offlineOk(mAddr) if err != nil { return false, "miner error", err } - if !b { log.Info("offline retrieval has not been implemented yet") } - if userFilter != nil { - return userFilter(ctx, state) - } - - return true, "", nil + // user never will be nil? + return userFilter(ctx, mAddr, state) } } } @@ -76,10 +76,7 @@ var RetrievalProviderOpts = func(cfg *config.MarketConfig) builder.Option { // Markets (retrieval) builder.Override(new(rmnet.RetrievalMarketNetwork), RetrievalNetwork), builder.Override(new(IRetrievalProvider), NewProvider), // save to metadata /retrievals/provider - builder.Override(new(config.RetrievalDealFilter), RetrievalDealFilter(nil)), builder.Override(HandleRetrievalKey, HandleRetrieval), - builder.If(cfg.RetrievalFilter != "", - builder.Override(new(config.RetrievalDealFilter), RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.RetrievalFilter))), - ), + builder.Override(new(config.RetrievalDealFilter), RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg))), ) } diff --git a/retrievalprovider/provider.go b/retrievalprovider/provider.go index 2bcabf42..17863b7b 100644 --- a/retrievalprovider/provider.go +++ b/retrievalprovider/provider.go @@ -5,20 +5,22 @@ import ( "math" "time" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/venus-market/v2/config" - "github.com/filecoin-project/venus-market/v2/paychmgr" - v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" - types "github.com/filecoin-project/venus/venus-shared/types/market" + logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/dtutils" "github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" "github.com/filecoin-project/go-fil-markets/stores" + + "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/models/repo" "github.com/filecoin-project/venus-market/v2/network" - logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/venus-market/v2/paychmgr" + + v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" + types "github.com/filecoin-project/venus/venus-shared/types/market" ) var queryTimeout = 5 * time.Second @@ -57,6 +59,7 @@ func NewProvider(network rmnet.RetrievalMarketNetwork, payAPI *paychmgr.PaychAPI, repo repo.Repo, cfg *config.MarketConfig, + rdf config.RetrievalDealFilter, ) (*RetrievalProvider, error) { storageDealsRepo := repo.StorageDealRepo() retrievalDealRepo := repo.RetrievalDealRepo() @@ -70,11 +73,11 @@ func NewProvider(network rmnet.RetrievalMarketNetwork, retrievalDealRepo: retrievalDealRepo, storageDealRepo: storageDealsRepo, stores: stores.NewReadOnlyBlockstores(), - retrievalStreamHandler: NewRetrievalStreamHandler(retrievalAskRepo, retrievalDealRepo, storageDealsRepo, pieceInfo, address.Address(cfg.RetrievalPaymentAddress.Addr)), + retrievalStreamHandler: NewRetrievalStreamHandler(retrievalAskRepo, retrievalDealRepo, storageDealsRepo, pieceInfo, address.Address(cfg.RetrievalPaymentAddress)), } retrievalHandler := NewRetrievalDealHandler(&providerDealEnvironment{p}, retrievalDealRepo, storageDealsRepo) - p.requestValidator = NewProviderRequestValidator(address.Address(cfg.RetrievalPaymentAddress.Addr), storageDealsRepo, retrievalDealRepo, retrievalAskRepo, pieceInfo) + p.requestValidator = NewProviderRequestValidator(address.Address(cfg.RetrievalPaymentAddress), storageDealsRepo, retrievalDealRepo, retrievalAskRepo, pieceInfo, rdf) transportConfigurer := dtutils.TransportConfigurer(network.ID(), &providerStoreGetter{retrievalDealRepo, p.stores}) p.reValidator = NewProviderRevalidator(fullNode, payAPI, retrievalDealRepo, retrievalHandler) diff --git a/retrievalprovider/requestvalidation.go b/retrievalprovider/requestvalidation.go index 17cc5a81..10b370e9 100644 --- a/retrievalprovider/requestvalidation.go +++ b/retrievalprovider/requestvalidation.go @@ -7,21 +7,21 @@ import ( "time" "github.com/filecoin-project/go-address" + datatransfer "github.com/filecoin-project/go-data-transfer" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/venus-market/v2/models/repo" - types "github.com/filecoin-project/venus/venus-shared/types/market" - + "github.com/filecoin-project/go-state-types/big" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" peer "github.com/libp2p/go-libp2p/core/peer" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/venus-market/v2/config" + "github.com/filecoin-project/venus-market/v2/models/repo" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations" + types "github.com/filecoin-project/venus/venus-shared/types/market" ) var allSelectorBytes []byte @@ -41,11 +41,26 @@ type ProviderRequestValidator struct { pieceInfo *PieceInfo retrievalDeal repo.IRetrievalDealRepo retrievalAsk repo.IRetrievalAskRepo + rdf config.RetrievalDealFilter } // NewProviderRequestValidator returns a new instance of the ProviderRequestValidator -func NewProviderRequestValidator(paymentAddr address.Address, storageDeals repo.StorageDealRepo, retrievalDeal repo.IRetrievalDealRepo, retrievalAsk repo.IRetrievalAskRepo, pieceInfo *PieceInfo) *ProviderRequestValidator { - return &ProviderRequestValidator{paymentAddr: paymentAddr, storageDeals: storageDeals, retrievalDeal: retrievalDeal, retrievalAsk: retrievalAsk, pieceInfo: pieceInfo} +func NewProviderRequestValidator( + paymentAddr address.Address, + storageDeals repo.StorageDealRepo, + retrievalDeal repo.IRetrievalDealRepo, + retrievalAsk repo.IRetrievalAskRepo, + pieceInfo *PieceInfo, + rdf config.RetrievalDealFilter, +) *ProviderRequestValidator { + return &ProviderRequestValidator{ + paymentAddr: paymentAddr, + storageDeals: storageDeals, + retrievalDeal: retrievalDeal, + retrievalAsk: retrievalAsk, + pieceInfo: pieceInfo, + rdf: rdf, + } } // ValidatePush validates a push request received from the peer that will send data @@ -160,6 +175,13 @@ func (rv *ProviderRequestValidator) validatePull(ctx context.Context, isRestart return &response, datatransfer.ErrPause } +func (rv *ProviderRequestValidator) runDealDecisionLogic(ctx context.Context, deal *types.ProviderDealState) (bool, string, error) { + if rv.rdf == nil { + return true, "", nil + } + return rv.rdf(ctx, address.Undef, *deal) +} + func (rv *ProviderRequestValidator) acceptDeal(ctx context.Context, deal *types.ProviderDealState) (retrievalmarket.DealStatus, error) { minerdeals, err := rv.pieceInfo.GetPieceInfoFromCid(ctx, deal.PayloadCID, deal.PieceCID) if err != nil { @@ -172,7 +194,6 @@ func (rv *ProviderRequestValidator) acceptDeal(ctx context.Context, deal *types. ctx, cancel := context.WithTimeout(ctx, askTimeout) defer cancel() - // todo how to select deal deal.SelStorageProposalCid = minerdeals[0].ProposalCid ask, err := rv.retrievalAsk.GetAsk(ctx, rv.paymentAddr) if err != nil { @@ -186,6 +207,15 @@ func (rv *ProviderRequestValidator) acceptDeal(ctx context.Context, deal *types. return retrievalmarket.DealStatusRejected, err } + // todo: 检索订单的 `miner` 从哪里来? + accepted, reason, err := rv.runDealDecisionLogic(ctx, deal) + if err != nil { + return retrievalmarket.DealStatusErrored, err + } + if !accepted { + return retrievalmarket.DealStatusRejected, errors.New(reason) + } + if deal.UnsealPrice.GreaterThan(big.Zero()) { return retrievalmarket.DealStatusFundsNeededUnseal, nil } diff --git a/rpc/rpc.go b/rpc/rpc.go index 362f4884..31a6a9ef 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -8,13 +8,14 @@ import ( "path" "github.com/filecoin-project/go-jsonrpc" + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" "github.com/filecoin-project/venus-auth/jwtclient" + "github.com/filecoin-project/venus-market/v2/config" - "github.com/gorilla/mux" - logging "github.com/ipfs/go-log/v2" - manet "github.com/multiformats/go-multiaddr/net" ) var log = logging.Logger("modules") diff --git a/storageprovider/address_select.go b/storageprovider/address_select.go index bd5d62f9..b58adca4 100644 --- a/storageprovider/address_select.go +++ b/storageprovider/address_select.go @@ -14,6 +14,7 @@ import ( ) type addrSelectAPI interface { + // search on the chain WalletBalance(context.Context, address.Address) (types.BigInt, error) WalletHas(context.Context, address.Address) (bool, error) diff --git a/storageprovider/client.go b/storageprovider/client.go index 1a64ca6d..698e03b7 100644 --- a/storageprovider/client.go +++ b/storageprovider/client.go @@ -8,8 +8,6 @@ import ( "errors" "fmt" - "github.com/filecoin-project/venus-market/v2/api/clients" - "github.com/ipfs/go-cid" "go.uber.org/fx" @@ -19,16 +17,19 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/builtin" + "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/venus-market/v2/api/clients" + "github.com/filecoin-project/venus-market/v2/api/clients/signer" "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/fundmgr" "github.com/filecoin-project/venus-market/v2/utils" + "github.com/ipfs-force-community/metrics" - "github.com/filecoin-project/go-state-types/builtin" - "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/filecoin-project/venus/pkg/constants" vcrypto "github.com/filecoin-project/venus/pkg/crypto" "github.com/filecoin-project/venus/pkg/events" @@ -47,13 +48,15 @@ type ClientNodeAdapter struct { dsMatcher *dealStateMatcher scMgr *SectorCommittedManager cfg *config.MarketClientConfig + + signer signer.ISigner } type clientAPI struct { full v1api.FullNode } -func NewClientNodeAdapter(mctx metrics.MetricsCtx, lc fx.Lifecycle, fullNode v1api.FullNode, msgClient clients.IMixMessage, fundmgr *fundmgr.FundManager, cfg *config.MarketClientConfig) storagemarket.StorageClientNode { +func NewClientNodeAdapter(mctx metrics.MetricsCtx, lc fx.Lifecycle, fullNode v1api.FullNode, msgClient clients.IMixMessage, fundmgr *fundmgr.FundManager, cfg *config.MarketClientConfig, signer signer.ISigner) storagemarket.StorageClientNode { capi := &clientAPI{fullNode} ctx := metrics.LifecycleCtx(mctx, lc) @@ -70,6 +73,7 @@ func NewClientNodeAdapter(mctx metrics.MetricsCtx, lc fx.Lifecycle, fullNode v1a ev: ev, cfg: cfg, dsMatcher: newDealStateMatcher(state.NewStatePredicates(state.WrapFastAPI(capi.full))), + signer: signer, } a.scMgr = NewSectorCommittedManager(ev, struct { @@ -384,12 +388,12 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add return nil, err } - signer, err = c.full.StateAccountKey(ctx, signer, types.EmptyTSK) + signerAddr, err := c.full.StateAccountKey(ctx, signer, types.EmptyTSK) if err != nil { return nil, err } - sig, err := c.full.WalletSign(ctx, signer, buf, types.MsgMeta{ + sig, err := c.signer.WalletSign(ctx, signerAddr, buf, types.MsgMeta{ Type: types.MTDealProposal, }) if err != nil { @@ -438,12 +442,12 @@ func (c *ClientNodeAdapter) GetMinerInfo(ctx context.Context, addr address.Addre } func (c *ClientNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*crypto.Signature, error) { - signer, err := c.full.StateAccountKey(ctx, signer, types.EmptyTSK) + signerAddr, err := c.full.StateAccountKey(ctx, signer, types.EmptyTSK) if err != nil { return nil, err } - localSignature, err := c.full.WalletSign(ctx, signer, b, types.MsgMeta{ + localSignature, err := c.signer.WalletSign(ctx, signerAddr, b, types.MsgMeta{ Type: types.MTUnknown, // TODO: pass type here }) if err != nil { diff --git a/storageprovider/deal_handler.go b/storageprovider/deal_handler.go index c11be28e..277fc03b 100644 --- a/storageprovider/deal_handler.go +++ b/storageprovider/deal_handler.go @@ -6,12 +6,11 @@ import ( "io" "os" - "github.com/ipfs-force-community/metrics" - - marketMetrics "github.com/filecoin-project/venus-market/v2/metrics" "go.opencensus.io/stats" "go.opencensus.io/tag" + "github.com/ipfs-force-community/metrics" + "github.com/ipfs/go-cid" carv2 "github.com/ipld/go-car/v2" @@ -29,14 +28,17 @@ import ( "github.com/filecoin-project/go-fil-markets/stores" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/builtin/v9/market" + "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/v7/actors/builtin/market" - "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" - minermgr2 "github.com/filecoin-project/venus-market/v2/minermgr" + "github.com/filecoin-project/venus-market/v2/config" + marketMetrics "github.com/filecoin-project/venus-market/v2/metrics" + "github.com/filecoin-project/venus-market/v2/minermgr" "github.com/filecoin-project/venus-market/v2/models/repo" network2 "github.com/filecoin-project/venus-market/v2/network" "github.com/filecoin-project/venus-market/v2/piecestorage" + vTypes "github.com/filecoin-project/venus/venus-shared/types" types "github.com/filecoin-project/venus/venus-shared/types/market" ) @@ -60,12 +62,14 @@ type StorageDealProcessImpl struct { spn StorageProviderNode deals repo.StorageDealRepo ask IStorageAsk - fs filestore.FileStore + tf config.TransferFileStoreConfigFunc stores *stores.ReadWriteBlockstores - dagStore stores.DAGStoreWrapper // TODO:检查是否遗漏 + dagStore stores.DAGStoreWrapper - minerMgr minermgr2.IAddrMgr + minerMgr minermgr.IMinerMgr pieceStorageMgr *piecestorage.PieceStorageManager + + sdf config.StorageDealFilter } // NewStorageDealProcessImpl returns a new deal process instance @@ -76,21 +80,20 @@ func NewStorageDealProcessImpl( spn StorageProviderNode, deals repo.StorageDealRepo, ask IStorageAsk, - fs filestore.FileStore, - minerMgr minermgr2.IAddrMgr, - repo repo.Repo, + tf config.TransferFileStoreConfigFunc, + minerMgr minermgr.IMinerMgr, pieceStorageMgr *piecestorage.PieceStorageManager, dataTransfer network2.ProviderDataTransfer, dagStore stores.DAGStoreWrapper, + sdf config.StorageDealFilter, ) (StorageDealHandler, error) { - stores := stores.NewReadWriteBlockstores() - err := dataTransfer.RegisterVoucherType(&requestvalidation.StorageDataTransferVoucher{}, requestvalidation.NewUnifiedRequestValidator(&providerPushDeals{deals}, nil)) if err != nil { return nil, err } - err = dataTransfer.RegisterTransportConfigurer(&requestvalidation.StorageDataTransferVoucher{}, dtutils.TransportConfigurer(newProviderStoreGetter(deals, stores))) + blockstores := stores.NewReadWriteBlockstores() + err = dataTransfer.RegisterTransportConfigurer(&requestvalidation.StorageDataTransferVoucher{}, dtutils.TransportConfigurer(newProviderStoreGetter(deals, blockstores))) if err != nil { return nil, err } @@ -102,20 +105,28 @@ func NewStorageDealProcessImpl( spn: spn, deals: deals, ask: ask, - fs: fs, - stores: stores, + tf: tf, + stores: blockstores, minerMgr: minerMgr, pieceStorageMgr: pieceStorageMgr, dagStore: dagStore, + + sdf: sdf, }, nil } +func (storageDealPorcess *StorageDealProcessImpl) runDealDecisionLogic(ctx context.Context, minerDeal *types.MinerDeal) (bool, string, error) { + if storageDealPorcess.sdf == nil { + return true, "", nil + } + return storageDealPorcess.sdf(ctx, minerDeal.Proposal.Provider, minerDeal) +} + // StorageDealUnknown->StorageDealValidating(ValidateDealProposal)->StorageDealAcceptWait(DecideOnProposal)->StorageDealWaitingForData func (storageDealPorcess *StorageDealProcessImpl) AcceptDeal(ctx context.Context, minerDeal *types.MinerDeal) error { storageDealPorcess.peerTagger.TagPeer(minerDeal.Client, minerDeal.ProposalCid.String()) - tok, curEpoch, err := storageDealPorcess.spn.GetChainHead(ctx) if err != nil { return storageDealPorcess.HandleReject(ctx, minerDeal, storagemarket.StorageDealRejecting, fmt.Errorf("node error getting most recent state id: %w", err)) @@ -127,7 +138,6 @@ func (storageDealPorcess *StorageDealProcessImpl) AcceptDeal(ctx context.Context proposal := minerDeal.Proposal - // TODO: 判断 proposal.Provider 在本矿池中 if !storageDealPorcess.minerMgr.Has(ctx, proposal.Provider) { return storageDealPorcess.HandleReject(ctx, minerDeal, storagemarket.StorageDealRejecting, fmt.Errorf("incorrect provider for deal")) } @@ -169,7 +179,7 @@ func (storageDealPorcess *StorageDealProcessImpl) AcceptDeal(ctx context.Context return storageDealPorcess.HandleReject(ctx, minerDeal, storagemarket.StorageDealRejecting, fmt.Errorf("invalid deal end epoch %d: cannot be more than %d past current epoch %d", proposal.EndEpoch, miner.MaxSectorExpirationExtension, curEpoch)) } - pcMin, pcMax, err := storageDealPorcess.spn.DealProviderCollateralBounds(ctx, proposal.PieceSize, proposal.VerifiedDeal) + pcMin, pcMax, err := storageDealPorcess.spn.DealProviderCollateralBounds(ctx, proposal.Provider, proposal.PieceSize, proposal.VerifiedDeal) if err != nil { return storageDealPorcess.HandleReject(ctx, minerDeal, storagemarket.StorageDealRejecting, fmt.Errorf("node error getting collateral bounds: %w", err)) } @@ -235,6 +245,15 @@ func (storageDealPorcess *StorageDealProcessImpl) AcceptDeal(ctx context.Context } } + accept, reason, err := storageDealPorcess.runDealDecisionLogic(ctx, minerDeal) + if err != nil { + return storageDealPorcess.HandleReject(ctx, minerDeal, storagemarket.StorageDealRejecting, fmt.Errorf("custom deal decision logic failed: %w", err)) + } + + if !accept { + return storageDealPorcess.HandleReject(ctx, minerDeal, storagemarket.StorageDealRejecting, fmt.Errorf(reason)) + } + err = storageDealPorcess.SendSignedResponse(ctx, proposal.Provider, &network.Response{ State: storagemarket.StorageDealWaitingForData, Proposal: minerDeal.ProposalCid, @@ -402,7 +421,11 @@ func (storageDealPorcess *StorageDealProcessImpl) HandleOff(ctx context.Context, if deal.PiecePath != "" { // Data for offline deals is stored on disk, so if PiecePath is set, // create a Reader from the file path - file, err := storageDealPorcess.fs.Open(deal.PiecePath) + fs, err := storageDealPorcess.tf(deal.Proposal.Provider) + if err != nil { + return storageDealPorcess.HandleError(ctx, deal, fmt.Errorf("get temp file store for %s: %w", deal.Proposal.Provider, err)) + } + file, err := fs.Open(deal.PiecePath) if err != nil { return storageDealPorcess.HandleError(ctx, deal, fmt.Errorf("reading piece at path %s: %w", deal.PiecePath, err)) } @@ -521,7 +544,6 @@ func (storageDealPorcess *StorageDealProcessImpl) SendSignedResponse(ctx context Signature: sig, } - // TODO: review ??? err = s.WriteDealResponse(signedResponse, storageDealPorcess.spn.SignWithGivenMiner(mAddr)) if err != nil { // Assume client disconnected @@ -546,7 +568,6 @@ func (storageDealPorcess *StorageDealProcessImpl) HandleReject(ctx context.Conte log.Errorf("failed response for reject: %s", err.Error()) } - // 断开连接 if err = storageDealPorcess.conns.Disconnect(deal.ProposalCid); err != nil { log.Warnf("closing client connection: %+v", err) } @@ -565,15 +586,25 @@ func (storageDealPorcess *StorageDealProcessImpl) HandleError(ctx context.Contex storageDealPorcess.peerTagger.UntagPeer(deal.Client, deal.ProposalCid.String()) if deal.PiecePath != filestore.Path("") { - err := storageDealPorcess.fs.Delete(deal.PiecePath) + fs, err := storageDealPorcess.tf(deal.Proposal.Provider) if err != nil { - log.Warnf("deleting piece at path %s: %w", deal.PiecePath, err) + log.Warnf("get temp file store for %s: %w", deal.Proposal.Provider, err) + } else { + err = fs.Delete(deal.PiecePath) + if err != nil { + log.Warnf("deleting piece at path %s: %w", deal.PiecePath, err) + } } } if deal.MetadataPath != filestore.Path("") { - err := storageDealPorcess.fs.Delete(deal.MetadataPath) + fs, err := storageDealPorcess.tf(deal.Proposal.Provider) if err != nil { - log.Warnf("deleting piece at path %s: %w", deal.MetadataPath, err) + log.Warnf("get temp file store for %s: %w", deal.Proposal.Provider, err) + } else { + err = fs.Delete(deal.MetadataPath) + if err != nil { + log.Warnf("deleting piece at path %s: %w", deal.MetadataPath, err) + } } } diff --git a/storageprovider/dealpublisher.go b/storageprovider/deal_publisher.go similarity index 91% rename from storageprovider/dealpublisher.go rename to storageprovider/deal_publisher.go index ca536bec..ce068a30 100644 --- a/storageprovider/dealpublisher.go +++ b/storageprovider/deal_publisher.go @@ -7,21 +7,24 @@ import ( "sync" "time" + "github.com/ipfs/go-cid" + "go.uber.org/fx" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/go-state-types/builtin/v9/market" + "github.com/filecoin-project/venus-market/v2/api/clients" "github.com/filecoin-project/venus-market/v2/config" types2 "github.com/filecoin-project/venus-market/v2/types" + "github.com/filecoin-project/venus/venus-shared/actors" marketactor "github.com/filecoin-project/venus/venus-shared/actors/builtin/market" v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" "github.com/filecoin-project/venus/venus-shared/types" marketTypes "github.com/filecoin-project/venus/venus-shared/types/market" - "github.com/ipfs/go-cid" - "go.uber.org/fx" ) type dealPublisherAPI interface { @@ -38,10 +41,8 @@ type dealPublisherAPI interface { type DealPublisher struct { api dealPublisherAPI - as *AddressSelector - publishSpec *types.MessageSendSpec - publishMsgCfg PublishMsgConfig + cfg *config.MarketConfig lk sync.Mutex publishers map[address.Address]*singleDealPublisher @@ -49,21 +50,17 @@ type DealPublisher struct { func NewDealPublisherWrapper( cfg *config.MarketConfig, -) func(lc fx.Lifecycle, full v1api.FullNode, msgClient clients.IMixMessage, as *AddressSelector) *DealPublisher { - return func(lc fx.Lifecycle, full v1api.FullNode, msgClient clients.IMixMessage, as *AddressSelector) *DealPublisher { +) func(lc fx.Lifecycle, full v1api.FullNode, msgClient clients.IMixMessage) *DealPublisher { + return func(lc fx.Lifecycle, full v1api.FullNode, msgClient clients.IMixMessage) *DealPublisher { dp := &DealPublisher{ api: struct { v1api.FullNode clients.IMixMessage }{full, msgClient}, - as: as, - publishSpec: &types.MessageSendSpec{MaxFee: abi.TokenAmount(cfg.MaxPublishDealsFee)}, - publishMsgCfg: PublishMsgConfig{ - Period: time.Duration(cfg.PublishMsgPeriod), - MaxDealsPerMsg: cfg.MaxDealsPerPublishMsg, - }, + cfg: cfg, publishers: map[address.Address]*singleDealPublisher{}, } + lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { dp.lk.Lock() @@ -110,7 +107,13 @@ func (p *DealPublisher) Publish(ctx context.Context, deal market.ClientDealPropo providerAddr := deal.Proposal.Provider publisher, ok := p.publishers[providerAddr] if !ok { - publisher = newDealPublisher(p.api, p.as, p.publishMsgCfg, p.publishSpec) + pCfg := p.cfg.MinerProviderConfig(providerAddr, true) + publisher = newDealPublisher( + p.api, + &AddressSelector{AddressConfig: pCfg.AddressConfig}, + pCfg.MaxDealsPerPublishMsg, + time.Duration(pCfg.PublishMsgPeriod), + &types.MessageSendSpec{MaxFee: abi.TokenAmount(pCfg.MaxPublishDealsFee)}) p.publishers[providerAddr] = publisher } publisher.processNewDeal(pdeal) @@ -170,19 +173,11 @@ func newPendingDeal(ctx context.Context, deal market.ClientDealProposal) *pendin } } -type PublishMsgConfig struct { - // The amount of time to wait for more deals to arrive before - // publishing - Period time.Duration - // The maximum number of deals to include in a single PublishStorageDeals - // message - MaxDealsPerMsg uint64 -} - func newDealPublisher( dpapi dealPublisherAPI, as *AddressSelector, - publishMsgCfg PublishMsgConfig, + maxDealsPerPublishMsg uint64, + publishPeriod time.Duration, publishSpec *types.MessageSendSpec, ) *singleDealPublisher { ctx, cancel := context.WithCancel(context.Background()) @@ -191,8 +186,8 @@ func newDealPublisher( as: as, ctx: ctx, Shutdown: cancel, - maxDealsPerPublishMsg: publishMsgCfg.MaxDealsPerMsg, - publishPeriod: publishMsgCfg.Period, + maxDealsPerPublishMsg: maxDealsPerPublishMsg, + publishPeriod: publishPeriod, publishSpec: publishSpec, } } @@ -430,13 +425,16 @@ func (p *singleDealPublisher) publishDealProposals(deals []market.ClientDealProp return cid.Undef, fmt.Errorf("selecting address for publishing deals: %w", err) } - msgId, err := p.api.PushMessage(p.ctx, &types.Message{ - To: marketactor.Address, - From: addr, - Value: types.NewInt(0), - Method: builtin.MethodsMarket.PublishStorageDeals, - Params: params, - }, p.publishSpec) + msgId, err := p.api.PushMessage( + p.ctx, + &types.Message{ + To: marketactor.Address, + From: addr, + Value: types.NewInt(0), + Method: builtin.MethodsMarket.PublishStorageDeals, + Params: params, + }, p.publishSpec) + if err != nil { return cid.Undef, err } diff --git a/storageprovider/deal_tracker.go b/storageprovider/deal_tracker.go index 318e9301..8ff65c86 100644 --- a/storageprovider/deal_tracker.go +++ b/storageprovider/deal_tracker.go @@ -6,14 +6,15 @@ import ( "strings" "time" + "go.uber.org/fx" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" - "go.uber.org/fx" "github.com/filecoin-project/venus-market/v2/minermgr" "github.com/filecoin-project/venus-market/v2/models/repo" - v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" + v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" vTypes "github.com/filecoin-project/venus/venus-shared/types" "github.com/filecoin-project/venus/venus-shared/types/market" @@ -23,13 +24,13 @@ import ( type DealTracker struct { period time.Duration // TODO: Preferably configurable? storageRepo repo.StorageDealRepo - minerMgr minermgr.IAddrMgr + minerMgr minermgr.IMinerMgr fullNode v1api.FullNode } var ReadyRetrievalDealStatus = []storagemarket.StorageDealStatus{storagemarket.StorageDealAwaitingPreCommit, storagemarket.StorageDealSealing, storagemarket.StorageDealActive} -func NewDealTracker(lc fx.Lifecycle, r repo.Repo, minerMgr minermgr.IAddrMgr, fullNode v1api.FullNode) *DealTracker { +func NewDealTracker(lc fx.Lifecycle, r repo.Repo, minerMgr minermgr.IMinerMgr, fullNode v1api.FullNode) *DealTracker { tracker := &DealTracker{period: time.Minute, storageRepo: r.StorageDealRepo(), minerMgr: minerMgr, fullNode: fullNode} lc.Append(fx.Hook{ @@ -56,7 +57,7 @@ func (dealTracker *DealTracker) Start(ctx metrics.MetricsCtx) { } func (dealTracker *DealTracker) scanDeal(ctx metrics.MetricsCtx) { - addrs, err := dealTracker.minerMgr.ActorAddress(ctx) + addrs, err := dealTracker.minerMgr.MinerList(ctx) if err != nil { log.Errorf("get miners list %w", err) } diff --git a/storageprovider/modules.go b/storageprovider/modules.go index 5c573c2c..7fc3995a 100644 --- a/storageprovider/modules.go +++ b/storageprovider/modules.go @@ -8,23 +8,25 @@ import ( "github.com/libp2p/go-libp2p/core/host" "go.uber.org/fx" + "github.com/filecoin-project/go-address" dtimpl "github.com/filecoin-project/go-data-transfer/impl" dtnet "github.com/filecoin-project/go-data-transfer/network" dtgstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs-force-community/metrics" + "github.com/ipfs-force-community/venus-common-utils/builder" + "github.com/ipfs-force-community/venus-common-utils/journal" + "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/dealfilter" "github.com/filecoin-project/venus-market/v2/models/badger" "github.com/filecoin-project/venus-market/v2/network" "github.com/filecoin-project/venus-market/v2/utils" - "github.com/ipfs-force-community/metrics" - "github.com/ipfs-force-community/venus-common-utils/builder" - "github.com/ipfs-force-community/venus-common-utils/journal" - "github.com/filecoin-project/venus/pkg/constants" + types "github.com/filecoin-project/venus/venus-shared/types/market" ) var ( @@ -85,8 +87,8 @@ func BasicDealFilter(user config.StorageDealFilter) func(onlineOk config.Conside startDelay config.GetMaxDealStartDelayFunc, spn storagemarket.StorageProviderNode, ) config.StorageDealFilter { - return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { - b, err := onlineOk() + return func(ctx context.Context, mAddr address.Address, deal *types.MinerDeal) (bool, string, error) { + b, err := onlineOk(mAddr) if err != nil { return false, "miner error", err } @@ -96,7 +98,7 @@ func BasicDealFilter(user config.StorageDealFilter) func(onlineOk config.Conside return false, "miner is not considering online piecestorage deals", nil } - b, err = offlineOk() + b, err = offlineOk(mAddr) if err != nil { return false, "miner error", err } @@ -106,7 +108,7 @@ func BasicDealFilter(user config.StorageDealFilter) func(onlineOk config.Conside return false, "miner is not accepting offline piecestorage deals", nil } - b, err = verifiedOk() + b, err = verifiedOk(mAddr) if err != nil { return false, "miner error", err } @@ -116,7 +118,7 @@ func BasicDealFilter(user config.StorageDealFilter) func(onlineOk config.Conside return false, "miner is not accepting verified piecestorage deals", nil } - b, err = unverifiedOk() + b, err = unverifiedOk(mAddr) if err != nil { return false, "miner error", err } @@ -126,7 +128,7 @@ func BasicDealFilter(user config.StorageDealFilter) func(onlineOk config.Conside return false, "miner is not accepting unverified piecestorage deals", nil } - blocklist, err := blocklistFunc() + blocklist, err := blocklistFunc(mAddr) if err != nil { return false, "miner error", err } @@ -138,7 +140,7 @@ func BasicDealFilter(user config.StorageDealFilter) func(onlineOk config.Conside } } - sealDuration, err := expectedSealTimeFunc() + sealDuration, err := expectedSealTimeFunc(mAddr) if err != nil { return false, "miner error", err } @@ -154,7 +156,7 @@ func BasicDealFilter(user config.StorageDealFilter) func(onlineOk config.Conside return false, fmt.Sprintf("cannot seal a sector before %s", deal.Proposal.StartEpoch), nil } - sd, err := startDelay() + sd, err := startDelay(mAddr) if err != nil { return false, "miner error", err } @@ -166,34 +168,21 @@ func BasicDealFilter(user config.StorageDealFilter) func(onlineOk config.Conside return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil } - if user != nil { - return user(ctx, deal) - } - - return true, "", nil + // user never will be nil? + return user(ctx, mAddr, deal) } } } -func NewAddressSelector(cfg *config.MarketConfig) (*AddressSelector, error) { - return &AddressSelector{ - AddressConfig: cfg.AddressConfig, - }, nil -} - var StorageProviderOpts = func(cfg *config.MarketConfig) builder.Option { return builder.Options( - builder.Override(new(*AddressSelector), NewAddressSelector), builder.Override(new(IStorageAsk), NewStorageAsk), builder.Override(new(network.ProviderDataTransfer), NewProviderDAGServiceDataTransfer), // save to metadata /datatransfer/provider/transfers // save to metadata /deals/provider/piecestorage-ask/latest - builder.Override(new(config.StorageDealFilter), BasicDealFilter(nil)), builder.Override(new(StorageProvider), NewStorageProvider), builder.Override(new(*DealPublisher), NewDealPublisherWrapper(cfg)), builder.Override(HandleDealsKey, HandleDeals), - builder.If(cfg.Filter != "", - builder.Override(new(config.StorageDealFilter), BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Filter))), - ), + builder.Override(new(config.StorageDealFilter), BasicDealFilter(dealfilter.CliStorageDealFilter(cfg))), builder.Override(new(StorageProviderNode), NewProviderNodeAdapter(cfg)), builder.Override(new(DealAssiger), NewDealAssigner), builder.Override(StartDealTracker, NewDealTracker), diff --git a/storageprovider/ondealsectorcommitted.go b/storageprovider/ondealsectorcommitted.go index f793bb4c..dad76011 100644 --- a/storageprovider/ondealsectorcommitted.go +++ b/storageprovider/ondealsectorcommitted.go @@ -12,8 +12,6 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" - miner7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" - "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/venus/pkg/constants" @@ -329,7 +327,7 @@ func dealSectorInPreCommitMsg(msg *types.Message, res CurrentDealInfo) (*abi.Sec } } case builtin.MethodsMiner.PreCommitSectorBatch: - var params miner7.PreCommitSectorBatchParams + var params miner.PreCommitSectorBatchParams if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return nil, fmt.Errorf("unmarshal pre commit: %w", err) } @@ -362,7 +360,7 @@ func sectorInCommitMsg(msg *types.Message, sectorNumber abi.SectorNumber) (bool, return params.SectorNumber == sectorNumber, nil case builtin.MethodsMiner.ProveCommitAggregate: - var params miner7.ProveCommitAggregateParams + var params miner.ProveCommitAggregateParams if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return false, fmt.Errorf("failed to unmarshal prove commit sector params: %w", err) } diff --git a/storageprovider/provider.go b/storageprovider/provider.go index cd090e1c..f80e897e 100644 --- a/storageprovider/provider.go +++ b/storageprovider/provider.go @@ -6,13 +6,8 @@ import ( "context" "fmt" - "github.com/filecoin-project/go-state-types/builtin" - "github.com/filecoin-project/go-state-types/builtin/v9/market" - "github.com/filecoin-project/venus-market/v2/api/clients" - "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - "go.uber.org/fx" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" @@ -20,9 +15,17 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin" + "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/venus-market/v2/api/clients" + "github.com/filecoin-project/venus-market/v2/api/clients/signer" + "github.com/filecoin-project/venus-market/v2/config" + "github.com/filecoin-project/venus-market/v2/fundmgr" + "github.com/filecoin-project/venus-market/v2/utils" + "github.com/filecoin-project/venus/pkg/constants" vCrypto "github.com/filecoin-project/venus/pkg/crypto" "github.com/filecoin-project/venus/pkg/events/state" @@ -30,49 +33,54 @@ import ( "github.com/filecoin-project/venus/venus-shared/actors/builtin/miner" v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" "github.com/filecoin-project/venus/venus-shared/types" - - "github.com/filecoin-project/venus-market/v2/config" - "github.com/filecoin-project/venus-market/v2/fundmgr" - "github.com/filecoin-project/venus-market/v2/utils" types2 "github.com/filecoin-project/venus/venus-shared/types/market" - "github.com/ipfs-force-community/metrics" ) -var ( - defaultMaxProviderCollateralMultiplier = uint64(2) - log = logging.Logger("storageadapter") -) +var log = logging.Logger("storageadapter") type ProviderNodeAdapter struct { v1api.FullNode + cfg *config.MarketConfig + fundMgr *fundmgr.FundManager msgClient clients.IMixMessage dealPublisher *DealPublisher - extendPieceMeta DealAssiger - addBalanceSpec *types.MessageSendSpec - maxDealCollateralMultiplier uint64 - dsMatcher *dealStateMatcher - dealInfo *CurrentDealInfoManager + extendPieceMeta DealAssiger + dsMatcher *dealStateMatcher + dealInfo *CurrentDealInfoManager + + signer signer.ISigner } -func NewProviderNodeAdapter(fc *config.MarketConfig) func(mctx metrics.MetricsCtx, lc fx.Lifecycle, node v1api.FullNode, msgClient clients.IMixMessage, dealPublisher *DealPublisher, fundMgr *fundmgr.FundManager, extendPieceMeta DealAssiger) StorageProviderNode { - return func(mctx metrics.MetricsCtx, lc fx.Lifecycle, full v1api.FullNode, msgClient clients.IMixMessage, dealPublisher *DealPublisher, fundMgr *fundmgr.FundManager, extendPieceMeta DealAssiger) StorageProviderNode { +func NewProviderNodeAdapter(cfg *config.MarketConfig) func( + dealPublisher *DealPublisher, + extendPieceMeta DealAssiger, + fundMgr *fundmgr.FundManager, + fullNode v1api.FullNode, + msgClient clients.IMixMessage, + signer signer.ISigner, +) StorageProviderNode { + return func( + dealPublisher *DealPublisher, + extendPieceMeta DealAssiger, + fundMgr *fundmgr.FundManager, + fullNode v1api.FullNode, + msgClient clients.IMixMessage, + signer signer.ISigner, + ) StorageProviderNode { na := &ProviderNodeAdapter{ - FullNode: full, + FullNode: fullNode, msgClient: msgClient, dealPublisher: dealPublisher, - dsMatcher: newDealStateMatcher(state.NewStatePredicates(state.WrapFastAPI(full))), + dsMatcher: newDealStateMatcher(state.NewStatePredicates(state.WrapFastAPI(fullNode))), extendPieceMeta: extendPieceMeta, fundMgr: fundMgr, + cfg: cfg, + signer: signer, } - if fc != nil { - na.addBalanceSpec = &types.MessageSendSpec{MaxFee: abi.TokenAmount(fc.MaxMarketBalanceAddFee)} - na.maxDealCollateralMultiplier = fc.MaxProviderCollateralMultiplier - } - na.maxDealCollateralMultiplier = defaultMaxProviderCollateralMultiplier na.dealInfo = &CurrentDealInfoManager{ CDAPI: &CurrentDealInfoAPIAdapter{CurrentDealInfoTskAPI: na}, } @@ -80,12 +88,12 @@ func NewProviderNodeAdapter(fc *config.MarketConfig) func(mctx metrics.MetricsCt } } -func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal types2.MinerDeal) (cid.Cid, error) { - return n.dealPublisher.Publish(ctx, deal.ClientDealProposal) +func (pna *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal types2.MinerDeal) (cid.Cid, error) { + return pna.dealPublisher.Publish(ctx, deal.ClientDealProposal) } -func (n *ProviderNodeAdapter) VerifySignature(ctx context.Context, sig crypto.Signature, addr address.Address, input []byte, _ shared.TipSetToken) (bool, error) { - addr, err := n.StateAccountKey(ctx, addr, types.EmptyTSK) +func (pna *ProviderNodeAdapter) VerifySignature(ctx context.Context, sig crypto.Signature, addr address.Address, input []byte, _ shared.TipSetToken) (bool, error) { + addr, err := pna.StateAccountKey(ctx, addr, types.EmptyTSK) if err != nil { return false, err } @@ -94,31 +102,31 @@ func (n *ProviderNodeAdapter) VerifySignature(ctx context.Context, sig crypto.Si return err == nil, err } -func (n *ProviderNodeAdapter) GetMinerWorkerAddress(ctx context.Context, maddr address.Address, tok shared.TipSetToken) (address.Address, error) { +func (pna *ProviderNodeAdapter) GetMinerWorkerAddress(ctx context.Context, maddr address.Address, tok shared.TipSetToken) (address.Address, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return address.Undef, err } - mi, err := n.StateMinerInfo(ctx, maddr, tsk) + mi, err := pna.StateMinerInfo(ctx, maddr, tsk) if err != nil { return address.Address{}, err } return mi.Worker, nil } -func (n *ProviderNodeAdapter) GetProofType(ctx context.Context, maddr address.Address, tok shared.TipSetToken) (abi.RegisteredSealProof, error) { +func (pna *ProviderNodeAdapter) GetProofType(ctx context.Context, maddr address.Address, tok shared.TipSetToken) (abi.RegisteredSealProof, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return 0, err } - mi, err := n.StateMinerInfo(ctx, maddr, tsk) + mi, err := pna.StateMinerInfo(ctx, maddr, tsk) if err != nil { return 0, err } - nver, err := n.StateNetworkVersion(ctx, tsk) + nver, err := pna.StateNetworkVersion(ctx, tsk) if err != nil { return 0, err } @@ -126,12 +134,7 @@ func (n *ProviderNodeAdapter) GetProofType(ctx context.Context, maddr address.Ad return miner.PreferredSealProofTypeFromWindowPoStType(nver, mi.WindowPoStProofType) } -func (n *ProviderNodeAdapter) Sign(ctx context.Context, data interface{}) (*crypto.Signature, error) { - tok, _, err := n.GetChainHead(ctx) - if err != nil { - return nil, fmt.Errorf("couldn't get chain head: %w", err) - } - +func (pna *ProviderNodeAdapter) Sign(ctx context.Context, data interface{}) (*crypto.Signature, error) { switch data.(type) { case *types2.SignInfo: @@ -145,46 +148,49 @@ func (n *ProviderNodeAdapter) Sign(ctx context.Context, data interface{}) (*cryp return nil, fmt.Errorf("serializing: %w", err) } - worker, err := n.GetMinerWorkerAddress(ctx, info.Addr, tok) + tok, _, err := pna.GetChainHead(ctx) + if err != nil { + return nil, fmt.Errorf("couldn't get chain head: %w", err) + } + + worker, err := pna.GetMinerWorkerAddress(ctx, info.Addr, tok) if err != nil { return nil, err } - signer, err := n.StateAccountKey(ctx, worker, types.EmptyTSK) + signerAddr, err := pna.StateAccountKey(ctx, worker, types.EmptyTSK) if err != nil { return nil, err } - localSignature, err := n.WalletSign(ctx, signer, msgBytes, types.MsgMeta{ - Type: info.Type, - }) + localSignature, err := pna.signer.WalletSign(ctx, signerAddr, msgBytes, types.MsgMeta{Type: info.Type}) if err != nil { return nil, err } return localSignature, nil } -func (n *ProviderNodeAdapter) SignWithGivenMiner(mAddr address.Address) network.ResigningFunc { +func (pna *ProviderNodeAdapter) SignWithGivenMiner(mAddr address.Address) network.ResigningFunc { return func(ctx context.Context, data interface{}) (*crypto.Signature, error) { - tok, _, err := n.GetChainHead(ctx) + msgBytes, err := cborutil.Dump(data) if err != nil { - return nil, fmt.Errorf("couldn't get chain head: %w", err) + return nil, fmt.Errorf("serializing: %w", err) } - msgBytes, err := cborutil.Dump(data) + tok, _, err := pna.GetChainHead(ctx) if err != nil { - return nil, fmt.Errorf("serializing: %w", err) + return nil, fmt.Errorf("couldn't get chain head: %w", err) } - worker, err := n.GetMinerWorkerAddress(ctx, mAddr, tok) + worker, err := pna.GetMinerWorkerAddress(ctx, mAddr, tok) if err != nil { return nil, err } - signer, err := n.StateAccountKey(ctx, worker, types.EmptyTSK) + signerAddr, err := pna.StateAccountKey(ctx, worker, types.EmptyTSK) if err != nil { return nil, err } - localSignature, err := n.WalletSign(ctx, signer, msgBytes, types.MsgMeta{ + localSignature, err := pna.signer.WalletSign(ctx, signerAddr, msgBytes, types.MsgMeta{ Type: types.MTUnknown, }) if err != nil { @@ -194,23 +200,27 @@ func (n *ProviderNodeAdapter) SignWithGivenMiner(mAddr address.Address) network. } } -func (n *ProviderNodeAdapter) ReserveFunds(ctx context.Context, wallet, addr address.Address, amt abi.TokenAmount) (cid.Cid, error) { - return n.fundMgr.Reserve(ctx, wallet, addr, amt) +func (pna *ProviderNodeAdapter) ReserveFunds(ctx context.Context, wallet, addr address.Address, amt abi.TokenAmount) (cid.Cid, error) { + return pna.fundMgr.Reserve(ctx, wallet, addr, amt) } -func (n *ProviderNodeAdapter) ReleaseFunds(ctx context.Context, addr address.Address, amt abi.TokenAmount) error { - return n.fundMgr.Release(addr, amt) +func (pna *ProviderNodeAdapter) ReleaseFunds(ctx context.Context, addr address.Address, amt abi.TokenAmount) error { + return pna.fundMgr.Release(addr, amt) } // Adds funds with the StorageMinerActor for a piecestorage participant. Used by both providers and clients. -func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) (cid.Cid, error) { +func (pna *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) (cid.Cid, error) { // (Provider Node API) - msgId, err := n.msgClient.PushMessage(ctx, &types.Message{ - To: marketactor.Address, - From: addr, - Value: amount, - Method: builtin.MethodsMarket.AddBalance, - }, n.addBalanceSpec) + pCfg := pna.cfg.MinerProviderConfig(addr, true) + msgId, err := pna.msgClient.PushMessage(ctx, + &types.Message{ + To: marketactor.Address, + From: addr, + Value: amount, + Method: builtin.MethodsMarket.AddBalance, + }, + &types.MessageSendSpec{MaxFee: abi.TokenAmount(pCfg.MaxMarketBalanceAddFee)}, + ) if err != nil { return cid.Undef, err } @@ -218,13 +228,13 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address return msgId, nil } -func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (storagemarket.Balance, error) { +func (pna *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (storagemarket.Balance, error) { tsk, err := types.TipSetKeyFromBytes(encodedTs) if err != nil { return storagemarket.Balance{}, err } - bal, err := n.StateMarketBalance(ctx, addr, tsk) + bal, err := pna.StateMarketBalance(ctx, addr, tsk) if err != nil { return storagemarket.Balance{}, err } @@ -232,21 +242,22 @@ func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Addre return utils.ToSharedBalance(bal), nil } -func (n *ProviderNodeAdapter) DealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, isVerified bool) (abi.TokenAmount, abi.TokenAmount, error) { - bounds, err := n.StateDealProviderCollateralBounds(ctx, size, isVerified, types.EmptyTSK) +func (pna *ProviderNodeAdapter) DealProviderCollateralBounds(ctx context.Context, provider address.Address, size abi.PaddedPieceSize, isVerified bool) (abi.TokenAmount, abi.TokenAmount, error) { + bounds, err := pna.StateDealProviderCollateralBounds(ctx, size, isVerified, types.EmptyTSK) if err != nil { return abi.TokenAmount{}, abi.TokenAmount{}, err } // The maximum amount of collateral that the provider will put into escrow // for a deal is calculated as a multiple of the minimum bounded amount - max := types.BigMul(bounds.Min, types.NewInt(n.maxDealCollateralMultiplier)) + pCfg := pna.cfg.MinerProviderConfig(provider, true) + max := types.BigMul(bounds.Min, types.NewInt(pCfg.MaxProviderCollateralMultiplier)) return bounds.Min, max, nil } -func (n *ProviderNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetToken, abi.ChainEpoch, error) { - head, err := n.ChainHead(ctx) +func (pna *ProviderNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetToken, abi.ChainEpoch, error) { + head, err := pna.ChainHead(ctx) if err != nil { return nil, 0, err } @@ -254,8 +265,8 @@ func (n *ProviderNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetTo return head.Key().Bytes(), head.Height(), nil } -func (n *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, finalCid cid.Cid, err error) error) error { - receipt, err := n.msgClient.WaitMsg(ctx, mcid, 2*constants.MessageConfidence, constants.LookbackNoLimit, true) +func (pna *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, finalCid cid.Cid, err error) error) error { + receipt, err := pna.msgClient.WaitMsg(ctx, mcid, 2*constants.MessageConfidence, constants.LookbackNoLimit, true) if err != nil { return cb(0, nil, cid.Undef, err) } @@ -263,9 +274,9 @@ func (n *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, return cb(receipt.Receipt.ExitCode, receipt.Receipt.Return, receipt.Message, nil) } -func (n *ProviderNodeAdapter) WaitForPublishDeals(ctx context.Context, publishCid cid.Cid, proposal market.DealProposal) (*storagemarket.PublishDealsWaitResult, error) { +func (pna *ProviderNodeAdapter) WaitForPublishDeals(ctx context.Context, publishCid cid.Cid, proposal market.DealProposal) (*storagemarket.PublishDealsWaitResult, error) { // Wait for deal to be published (plus additional time for confidence) - receipt, err := n.msgClient.WaitMsg(ctx, publishCid, 2*constants.MessageConfidence, constants.LookbackNoLimit, true) + receipt, err := pna.msgClient.WaitMsg(ctx, publishCid, 2*constants.MessageConfidence, constants.LookbackNoLimit, true) if err != nil { return nil, fmt.Errorf("WaitForPublishDeals errored: %w", err) } @@ -275,12 +286,12 @@ func (n *ProviderNodeAdapter) WaitForPublishDeals(ctx context.Context, publishCi // The deal ID may have changed since publish if there was a reorg, so // get the current deal ID - head, err := n.ChainHead(ctx) + head, err := pna.ChainHead(ctx) if err != nil { return nil, fmt.Errorf("WaitForPublishDeals failed to get chain head: %w", err) } - res, err := n.dealInfo.GetCurrentDealInfo(ctx, head.Key(), &proposal, publishCid) + res, err := pna.dealInfo.GetCurrentDealInfo(ctx, head.Key(), &proposal, publishCid) if err != nil { return nil, fmt.Errorf("WaitForPublishDeals getting deal info errored: %w", err) } @@ -288,22 +299,22 @@ func (n *ProviderNodeAdapter) WaitForPublishDeals(ctx context.Context, publishCi return &storagemarket.PublishDealsWaitResult{DealID: res.DealID, FinalCid: receipt.Message}, nil } -func (n *ProviderNodeAdapter) GetDataCap(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (*abi.StoragePower, error) { +func (pna *ProviderNodeAdapter) GetDataCap(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (*abi.StoragePower, error) { tsk, err := types.TipSetKeyFromBytes(encodedTs) if err != nil { return nil, err } - sp, err := n.StateVerifiedClientStatus(ctx, addr, tsk) + sp, err := pna.StateVerifiedClientStatus(ctx, addr, tsk) return sp, err } -func (n *ProviderNodeAdapter) SearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*types.MsgLookup, error) { - return n.msgClient.WaitMsg(ctx, msg, constants.MessageConfidence, limit, allowReplaced) +func (pna *ProviderNodeAdapter) SearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*types.MsgLookup, error) { + return pna.msgClient.WaitMsg(ctx, msg, constants.MessageConfidence, limit, allowReplaced) } -func (n *ProviderNodeAdapter) GetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) { - return n.msgClient.GetMessage(ctx, mc) +func (pna *ProviderNodeAdapter) GetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) { + return pna.msgClient.GetMessage(ctx, mc) } // StorageProviderNode are common interfaces provided by a filecoin Node to both StorageClient and StorageProvider @@ -337,7 +348,7 @@ type StorageProviderNode interface { WaitForMessage(ctx context.Context, mcid cid.Cid, onCompletion func(exitcode.ExitCode, []byte, cid.Cid, error) error) error // DealProviderCollateralBounds returns the min and max collateral a storage provider can issue. - DealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, isVerified bool) (abi.TokenAmount, abi.TokenAmount, error) + DealProviderCollateralBounds(ctx context.Context, provider address.Address, size abi.PaddedPieceSize, isVerified bool) (abi.TokenAmount, abi.TokenAmount, error) // PublishDeals publishes a deal on chain, returns the message cid, but does not wait for message to appear PublishDeals(ctx context.Context, deal types2.MinerDeal) (cid.Cid, error) diff --git a/storageprovider/storage_ask.go b/storageprovider/storage_ask.go index 1c1aafeb..e1af98ce 100644 --- a/storageprovider/storage_ask.go +++ b/storageprovider/storage_ask.go @@ -9,30 +9,37 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/venus-market/v2/api/clients/signer" "github.com/filecoin-project/venus-market/v2/models/repo" + v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" "github.com/filecoin-project/venus/venus-shared/types" types2 "github.com/filecoin-project/venus/venus-shared/types/market" - "github.com/ipfs-force-community/metrics" ) type IStorageAsk interface { ListAsk(ctx context.Context) ([]*types2.SignedStorageAsk, error) - GetAsk(mctx context.Context, Addr address.Address) (*types2.SignedStorageAsk, error) + GetAsk(ctx context.Context, Addr address.Address) (*types2.SignedStorageAsk, error) SetAsk(ctx context.Context, mAddr address.Address, price abi.TokenAmount, verifiedPrice abi.TokenAmount, duration abi.ChainEpoch, options ...storagemarket.StorageAskOption) error } func NewStorageAsk( - ctx metrics.MetricsCtx, + fullNode v1api.FullNode, repo repo.Repo, - fullnode v1api.FullNode, + signer signer.ISigner, ) (IStorageAsk, error) { - return &StorageAsk{repo: repo.StorageAskRepo(), fullNode: fullnode}, nil + return &StorageAsk{ + fullNode: fullNode, + repo: repo.StorageAskRepo(), + signer: signer, + }, nil } type StorageAsk struct { repo repo.IStorageAskRepo fullNode v1api.FullNode + signer signer.ISigner } func (storageAsk *StorageAsk) ListAsk(ctx context.Context) ([]*types2.SignedStorageAsk, error) { @@ -77,21 +84,20 @@ func (storageAsk *StorageAsk) SetAsk(ctx context.Context, miner address.Address, var signedAsk *types2.SignedStorageAsk - if signedAsk, err = storageAsk.signAsk(ask); err != nil { + if signedAsk, err = storageAsk.signAsk(ctx, ask); err != nil { return fmt.Errorf("miner %s sign data failed: %v", miner.String(), err) } return storageAsk.repo.SetAsk(ctx, signedAsk) } -func (storageAsk *StorageAsk) signAsk(ask *storagemarket.StorageAsk) (*types2.SignedStorageAsk, error) { +func (storageAsk *StorageAsk) signAsk(ctx context.Context, ask *storagemarket.StorageAsk) (*types2.SignedStorageAsk, error) { askBytes, err := cborutil.Dump(ask) if err != nil { return nil, err } // get worker address for miner - ctx := context.TODO() tok, err := storageAsk.fullNode.ChainHead(ctx) if err != nil { return nil, err @@ -107,9 +113,7 @@ func (storageAsk *StorageAsk) signAsk(ask *storagemarket.StorageAsk) (*types2.Si return nil, err } - sig, err := storageAsk.fullNode.WalletSign(ctx, addr, askBytes, types.MsgMeta{ - Type: types.MTStorageAsk, - }) + sig, err := storageAsk.signer.WalletSign(ctx, addr, askBytes, types.MsgMeta{Type: types.MTStorageAsk}) if err != nil { return nil, err } diff --git a/storageprovider/storage_ask_test.go b/storageprovider/storage_ask_test.go index 2b0c7556..6ad0b964 100644 --- a/storageprovider/storage_ask_test.go +++ b/storageprovider/storage_ask_test.go @@ -17,14 +17,12 @@ import ( // go test -v ./storageadapter -test.run TestStorageAsk -mysql='root:ko2005@tcp(127.0.0.1:3306)/storage_market?charset=utf8mb4&parseTime=True&loc=Local&timeout=10s' func TestStorageAsk(t *testing.T) { - ctx := context.Background() t.Run("mysql", func(t *testing.T) { - mysqlAsk, _ := NewStorageAsk(ctx, models.MysqlDB(t), &test_helper.MockFullnode{T: t}) - + mysqlAsk, _ := NewStorageAsk(&test_helper.MockFullnode{T: t}, models.MysqlDB(t), &test_helper.MockFullnode{T: t}) testStorageAsk(t, mysqlAsk) }) t.Run("badger", func(t *testing.T) { - badgerAsk, _ := NewStorageAsk(ctx, badger.NewBadgerRepo(badger.BadgerDSParams{AskDS: models.BadgerDB(t)}), + badgerAsk, _ := NewStorageAsk(&test_helper.MockFullnode{T: t}, badger.NewBadgerRepo(badger.BadgerDSParams{AskDS: models.BadgerDB(t)}), &test_helper.MockFullnode{T: t}) testStorageAsk(t, badgerAsk) }) diff --git a/storageprovider/storageprovider.go b/storageprovider/storage_provider.go similarity index 95% rename from storageprovider/storageprovider.go rename to storageprovider/storage_provider.go index c9d65880..3b462b2d 100644 --- a/storageprovider/storageprovider.go +++ b/storageprovider/storage_provider.go @@ -9,11 +9,9 @@ import ( "github.com/ipfs-force-community/metrics" - "github.com/filecoin-project/go-fil-markets/storagemarket/impl/providerutils" "github.com/hannahhoward/go-pubsub" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/host" - "github.com/mitchellh/go-homedir" cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/go-address" @@ -28,9 +26,12 @@ import ( "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/connmanager" + "github.com/filecoin-project/go-fil-markets/storagemarket/impl/providerutils" smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/go-state-types/builtin/v9/market" + "github.com/filecoin-project/venus-market/v2/api/clients" "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/minermgr" @@ -39,8 +40,6 @@ import ( "github.com/filecoin-project/venus-market/v2/piecestorage" "github.com/filecoin-project/venus-market/v2/utils" - "github.com/filecoin-project/go-state-types/builtin/v9/market" - types2 "github.com/filecoin-project/venus/venus-shared/types" types "github.com/filecoin-project/venus/venus-shared/types/market" ) @@ -78,8 +77,9 @@ type StorageProvider interface { type StorageProviderImpl struct { net smnet.StorageMarketNetwork + tf config.TransferFileStoreConfigFunc + spn StorageProviderNode - fs filestore.FileStore conns *connmanager.ConnManager storedAsk IStorageAsk @@ -91,7 +91,7 @@ type StorageProviderImpl struct { dealProcess StorageDealHandler transferProcess IDatatransferHandler storageReceiver smnet.StorageReceiver - minerMgr minermgr.IAddrMgr + minerMgr minermgr.IMinerMgr } type internalProviderEvent struct { @@ -117,37 +117,24 @@ func NewStorageProvider( mCtx metrics.MetricsCtx, storedAsk IStorageAsk, h host.Host, - cfg *config.MarketConfig, - homeDir *config.HomeDir, + tf config.TransferFileStoreConfigFunc, pieceStorageMgr *piecestorage.PieceStorageManager, dataTransfer network.ProviderDataTransfer, spn StorageProviderNode, dagStore stores.DAGStoreWrapper, repo repo.Repo, - minerMgr minermgr.IAddrMgr, + minerMgr minermgr.IMinerMgr, mixMsgClient clients.IMixMessage, + sdf config.StorageDealFilter, ) (StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) - var err error - transferPath := cfg.TransfePath - if len(transferPath) == 0 { - transferPath = string(*homeDir) - } - transferPath, err = homedir.Expand(transferPath) - if err != nil { - return nil, err - } - store, err := filestore.NewLocalFileStore(filestore.OsPath(transferPath)) - if err != nil { - return nil, err - } - spV2 := &StorageProviderImpl{ net: net, + tf: tf, + spn: spn, - fs: store, conns: connmanager.NewConnManager(), storedAsk: storedAsk, @@ -158,7 +145,7 @@ func NewStorageProvider( minerMgr: minerMgr, } - dealProcess, err := NewStorageDealProcessImpl(mCtx, spV2.conns, newPeerTagger(spV2.net), spV2.spn, spV2.dealStore, spV2.storedAsk, spV2.fs, minerMgr, repo, pieceStorageMgr, dataTransfer, dagStore) + dealProcess, err := NewStorageDealProcessImpl(mCtx, spV2.conns, newPeerTagger(spV2.net), spV2.spn, spV2.dealStore, spV2.storedAsk, tf, minerMgr, pieceStorageMgr, dataTransfer, dagStore, sdf) if err != nil { return nil, err } @@ -168,7 +155,7 @@ func NewStorageProvider( // register a data transfer event handler -- this will send events to the state machines based on DT events spV2.unsubDataTransfer = dataTransfer.SubscribeToEvents(ProviderDataTransferSubscriber(spV2.transferProcess)) // fsm.Group - storageReceiver, err := NewStorageDealStream(spV2.conns, spV2.storedAsk, spV2.spn, spV2.dealStore, spV2.net, spV2.fs, dealProcess, mixMsgClient) + storageReceiver, err := NewStorageDealStream(spV2.conns, spV2.storedAsk, spV2.spn, spV2.dealStore, spV2.net, tf, dealProcess, mixMsgClient) if err != nil { return nil, err } @@ -251,7 +238,6 @@ func (p *StorageProviderImpl) ImportDataForDeal(ctx context.Context, propCid cid return fmt.Errorf("failed getting deal %s: %w", propCid, err) } - // TODO: Check the deal status if isTerminateState(d) { return fmt.Errorf("deal %s is terminate state", propCid) } @@ -260,7 +246,12 @@ func (p *StorageProviderImpl) ImportDataForDeal(ctx context.Context, propCid cid return fmt.Errorf("deal %s does not support offline data", propCid) } - tempfi, err := p.fs.CreateTemp() + fs, err := p.tf(d.Proposal.Provider) + if err != nil { + return fmt.Errorf("failed to create temp filestore for provider %s: %w", d.Proposal.Provider.String(), err) + } + + tempfi, err := fs.CreateTemp() if err != nil { return fmt.Errorf("failed to create temp file for data import: %w", err) } @@ -271,7 +262,7 @@ func (p *StorageProviderImpl) ImportDataForDeal(ctx context.Context, propCid cid }() cleanup := func() { _ = tempfi.Close() - _ = p.fs.Delete(tempfi.Path()) + _ = fs.Delete(tempfi.Path()) } log.Debugw("will copy imported file to local file", "propCid", propCid) diff --git a/storageprovider/storageprovider_test.go b/storageprovider/storageprovider_test.go index b6e34e39..e80675df 100644 --- a/storageprovider/storageprovider_test.go +++ b/storageprovider/storageprovider_test.go @@ -8,7 +8,11 @@ import ( "sync" "testing" + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/assert" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" @@ -16,18 +20,19 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/venus-market/v2/config" + "github.com/filecoin-project/venus-market/v2/minermgr" "github.com/filecoin-project/venus-market/v2/models" network2 "github.com/filecoin-project/venus-market/v2/network" "github.com/filecoin-project/venus-market/v2/piecestorage" + vCrypto "github.com/filecoin-project/venus/pkg/crypto" _ "github.com/filecoin-project/venus/pkg/crypto/bls" _ "github.com/filecoin-project/venus/pkg/crypto/secp" "github.com/filecoin-project/venus/venus-shared/api/chain/v1/mock" "github.com/filecoin-project/venus/venus-shared/types" marketypes "github.com/filecoin-project/venus/venus-shared/types/market" - "github.com/ipfs/go-cid" - "github.com/stretchr/testify/assert" ) //go:embed testdata/import_deal.json @@ -124,7 +129,7 @@ func setup(t *testing.T) StorageProvider { } r := models.NewInMemoryRepo(t) - ask, _ := NewStorageAsk(ctx, r, spn) + ask, _ := NewStorageAsk(spn, r, spn) h, err := network2.MockHost(ctx) if err != nil { t.Error(err) @@ -134,14 +139,21 @@ func setup(t *testing.T) StorageProvider { t.Error(err) } - homeDir := config.HomeDir("") psManager, err := piecestorage.NewPieceStorageManager(&config.PieceStorage{}) assert.Nil(t, err) psManager.AddMemPieceStorage(piecestorage.NewMemPieceStore("", nil)) addrMgr := mockAddrMgr{} // todo how to mock dagstore - provider, err := NewStorageProvider(ctx, ask, h, config.DefaultMarketConfig, &homeDir, psManager, dt, spn, nil, r, addrMgr, nil) + tf := func(address.Address) (filestore.FileStore, error) { + transferPath := config.DefaultMarketConfig.MustHomePath() + store, err := filestore.NewLocalFileStore(filestore.OsPath(transferPath)) + if err != nil { + return nil, err + } + return store, nil + } + provider, err := NewStorageProvider(ctx, ask, h, tf, psManager, dt, spn, nil, r, addrMgr, nil, nil) if err != nil { t.Error(err) } @@ -150,27 +162,24 @@ func setup(t *testing.T) StorageProvider { type mockAddrMgr struct{} -func (m mockAddrMgr) Has(ctx context.Context, addr address.Address) bool { - return addr.String() == "t01043" || addr.String() == "t010938" -} +var _ minermgr.IMinerMgr = (*mockAddrMgr)(nil) -func (m mockAddrMgr) ActorAddress(ctx context.Context) ([]address.Address, error) { - // TODO implement me +func (m mockAddrMgr) MinerList(ctx context.Context) ([]address.Address, error) { + //TODO implement me panic("implement me") } -func (m mockAddrMgr) ActorList(ctx context.Context) ([]marketypes.User, error) { - // TODO implement me - panic("implement me") +func (m mockAddrMgr) Has(ctx context.Context, addr address.Address) bool { + return addr.String() == "t01043" || addr.String() == "t010938" } -func (m mockAddrMgr) GetMiners(ctx context.Context) ([]marketypes.User, error) { - // TODO implement me +func (m mockAddrMgr) ActorList(ctx context.Context) ([]marketypes.User, error) { + //TODO implement me panic("implement me") } -func (m mockAddrMgr) GetAccount(ctx context.Context, addr address.Address) (string, error) { - // TODO implement me +func (m mockAddrMgr) GetMiners(ctx context.Context) ([]marketypes.User, error) { + //TODO implement me panic("implement me") } @@ -181,6 +190,8 @@ type mockProviderNode struct { head *types.TipSet } +var _ StorageProviderNode = (*mockProviderNode)(nil) + func newMockProviderNode() *mockProviderNode { return &mockProviderNode{ MockFullNode: mock.MockFullNode{}, @@ -258,7 +269,7 @@ func (m *mockProviderNode) WaitForMessage(ctx context.Context, mcid cid.Cid, onC panic("implement me") } -func (m *mockProviderNode) DealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, isVerified bool) (abi.TokenAmount, abi.TokenAmount, error) { +func (m *mockProviderNode) DealProviderCollateralBounds(ctx context.Context, provider address.Address, size abi.PaddedPieceSize, isVerified bool) (abi.TokenAmount, abi.TokenAmount, error) { // TODO implement me panic("implement me") } diff --git a/storageprovider/stream.go b/storageprovider/stream.go index ce88a0c9..3ddbe90c 100644 --- a/storageprovider/stream.go +++ b/storageprovider/stream.go @@ -5,20 +5,20 @@ import ( "fmt" "os" - "github.com/filecoin-project/venus-market/v2/api/clients" - "github.com/filecoin-project/venus-market/v2/utils" - vTypes "github.com/filecoin-project/venus/venus-shared/types" - "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/connmanager" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/providerutils" "github.com/filecoin-project/go-fil-markets/storagemarket/network" + "github.com/filecoin-project/venus-market/v2/api/clients" + "github.com/filecoin-project/venus-market/v2/config" "github.com/filecoin-project/venus-market/v2/models/repo" + "github.com/filecoin-project/venus-market/v2/utils" + + vTypes "github.com/filecoin-project/venus/venus-shared/types" types "github.com/filecoin-project/venus/venus-shared/types/market" ) @@ -30,7 +30,7 @@ type StorageDealStream struct { spn StorageProviderNode deals repo.StorageDealRepo net network.StorageMarketNetwork - fs filestore.FileStore + tf config.TransferFileStoreConfigFunc dealProcess StorageDealHandler mixMsgClient clients.IMixMessage } @@ -42,7 +42,7 @@ func NewStorageDealStream( spn StorageProviderNode, deals repo.StorageDealRepo, net network.StorageMarketNetwork, - fs filestore.FileStore, + tf config.TransferFileStoreConfigFunc, dealProcess StorageDealHandler, mixMsgClient clients.IMixMessage, ) (network.StorageReceiver, error) { @@ -52,7 +52,7 @@ func NewStorageDealStream( spn: spn, deals: deals, net: net, - fs: fs, + tf: tf, dealProcess: dealProcess, mixMsgClient: mixMsgClient, }, nil @@ -134,7 +134,12 @@ func (storageDealStream *StorageDealStream) HandleDealStream(s network.StorageDe var path string // create an empty CARv2 file at a temp location that Graphysnc will write the incoming blocks to via a CARv2 ReadWrite blockstore wrapper. if proposal.Piece.TransferType != storagemarket.TTManual { - tmp, err := storageDealStream.fs.CreateTemp() + fs, err := storageDealStream.tf(proposal.DealProposal.Proposal.Provider) + if err != nil { + log.Errorf("failed to create temp file store for provider %s: %w", proposal.DealProposal.Proposal.Provider.String(), err) + return + } + tmp, err := fs.CreateTemp() if err != nil { log.Errorf("failed to create an empty temp CARv2 file: %w", err) return