diff --git a/go.mod b/go.mod index 681196346..81c83ce29 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,6 @@ require ( github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/lotus v1.20.3-boost github.com/filecoin-project/specs-actors v0.9.15 - github.com/filecoin-project/specs-storage v0.4.1 github.com/gbrlsnchs/jwt/v3 v3.0.1 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 @@ -56,7 +55,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 - github.com/ipfs/go-ipfs-ds-help v1.1.0 + github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipfs-exchange-offline v0.3.0 github.com/ipfs/go-ipfs-files v0.1.1 diff --git a/go.sum b/go.sum index c9ca83daa..75185a63e 100644 --- a/go.sum +++ b/go.sum @@ -392,7 +392,6 @@ github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-state-types v0.1.4/go.mod h1:xCA/WfKlC2zcn3fUmDv4IrzznwS98X5XW/irUP3Lhxg= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.8/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= @@ -428,8 +427,6 @@ github.com/filecoin-project/specs-actors/v7 v7.0.1 h1:w72xCxijK7xs1qzmJiw+WYJaVt github.com/filecoin-project/specs-actors/v7 v7.0.1/go.mod h1:tPLEYXoXhcpyLh69Ccq91SOuLXsPWjHiY27CzawjUEk= github.com/filecoin-project/specs-actors/v8 v8.0.1 h1:4u0tIRJeT5G7F05lwLRIsDnsrN+bJ5Ixj6h49Q7uE2Y= github.com/filecoin-project/specs-actors/v8 v8.0.1/go.mod h1:UYIPg65iPWoFw5NEftREdJwv9b/5yaLKdCgTvNI/2FA= -github.com/filecoin-project/specs-storage v0.4.1 h1:yvLEaLZj8f+uByhNC4mFOtCUyL2wQku+NGBp6hjTe9M= -github.com/filecoin-project/specs-storage v0.4.1/go.mod h1:Z2eK6uMwAOSLjek6+sy0jNV2DSsMEENziMUz0GHRFBw= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= diff --git a/storage/sectorblocks/sectorblocks.go b/storage/sectorblocks/sectorblocks.go deleted file mode 100644 index 414e325a0..000000000 --- a/storage/sectorblocks/sectorblocks.go +++ /dev/null @@ -1,176 +0,0 @@ -package sectorblocks - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "fmt" - "io" - "sync" - - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - "github.com/ipfs/go-datastore/query" - dshelp "github.com/ipfs/go-ipfs-ds-help" - - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-storage/storage" - - "github.com/filecoin-project/lotus/api" - lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -type SealSerialization uint8 - -const ( - SerializationUnixfs0 SealSerialization = 'u' -) - -var dsPrefix = datastore.NewKey("/sealedblocks") - -var ErrNotFound = errors.New("not found") - -func DealIDToDsKey(dealID abi.DealID) datastore.Key { - buf := make([]byte, binary.MaxVarintLen64) - size := binary.PutUvarint(buf, uint64(dealID)) - return dshelp.NewKeyFromBinary(buf[:size]) -} - -func DsKeyToDealID(key datastore.Key) (uint64, error) { - buf, err := dshelp.BinaryFromDsKey(key) - if err != nil { - return 0, err - } - dealID, _ := binary.Uvarint(buf) - return dealID, nil -} - -type SectorBuilder interface { - SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d api.PieceDealInfo) (api.SectorOffset, error) - SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) -} - -type SectorBlocks struct { - SectorBuilder - - keys datastore.Batching - keyLk sync.Mutex -} - -func NewSectorBlocks(sb SectorBuilder, ds lotus_dtypes.MetadataDS) *SectorBlocks { - sbc := &SectorBlocks{ - SectorBuilder: sb, - keys: namespace.Wrap(ds, dsPrefix), - } - - return sbc -} - -func (st *SectorBlocks) writeRef(ctx context.Context, dealID abi.DealID, sectorID abi.SectorNumber, offset abi.PaddedPieceSize, size abi.UnpaddedPieceSize) error { - st.keyLk.Lock() // TODO: make this multithreaded - defer st.keyLk.Unlock() - - v, err := st.keys.Get(ctx, DealIDToDsKey(dealID)) - if err == datastore.ErrNotFound { - err = nil - } - if err != nil { - return fmt.Errorf("getting existing refs: %w", err) - } - - var refs api.SealedRefs - if len(v) > 0 { - if err := cborutil.ReadCborRPC(bytes.NewReader(v), &refs); err != nil { - return fmt.Errorf("decoding existing refs: %w", err) - } - } - - refs.Refs = append(refs.Refs, api.SealedRef{ - SectorID: sectorID, - Offset: offset, - Size: size, - }) - - newRef, err := cborutil.Dump(&refs) - if err != nil { - return fmt.Errorf("serializing refs: %w", err) - } - return st.keys.Put(ctx, DealIDToDsKey(dealID), newRef) // TODO: batch somehow -} - -func (st *SectorBlocks) AddPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d api.PieceDealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { - so, err := st.SectorBuilder.SectorAddPieceToAny(ctx, size, r, d) - if err != nil { - return 0, 0, err - } - - // TODO: DealID has very low finality here - err = st.writeRef(ctx, d.DealID, so.Sector, so.Offset, size) - if err != nil { - return 0, 0, fmt.Errorf("writeRef: %w", err) - } - - return so.Sector, so.Offset, nil -} - -func (st *SectorBlocks) List(ctx context.Context) (map[uint64][]api.SealedRef, error) { - res, err := st.keys.Query(ctx, query.Query{}) - if err != nil { - return nil, err - } - - ents, err := res.Rest() - if err != nil { - return nil, err - } - - out := map[uint64][]api.SealedRef{} - for _, ent := range ents { - dealID, err := DsKeyToDealID(datastore.RawKey(ent.Key)) - if err != nil { - return nil, err - } - - var refs api.SealedRefs - if err := cborutil.ReadCborRPC(bytes.NewReader(ent.Value), &refs); err != nil { - return nil, err - } - - out[dealID] = refs.Refs - } - - return out, nil -} - -func (st *SectorBlocks) GetRefs(ctx context.Context, dealID abi.DealID) ([]api.SealedRef, error) { // TODO: track local sectors - ent, err := st.keys.Get(ctx, DealIDToDsKey(dealID)) - if err == datastore.ErrNotFound { - err = ErrNotFound - } - if err != nil { - return nil, err - } - - var refs api.SealedRefs - if err := cborutil.ReadCborRPC(bytes.NewReader(ent), &refs); err != nil { - return nil, err - } - - return refs.Refs, nil -} - -func (st *SectorBlocks) GetSize(ctx context.Context, dealID abi.DealID) (uint64, error) { - refs, err := st.GetRefs(ctx, dealID) - if err != nil { - return 0, err - } - - return uint64(refs[0].Size), nil -} - -func (st *SectorBlocks) Has(ctx context.Context, dealID abi.DealID) (bool, error) { - // TODO: ensure sector is still there - return st.keys.Has(ctx, DealIDToDsKey(dealID)) -} diff --git a/storagemarket/types/types.go b/storagemarket/types/types.go index df2c9aee3..1c503f160 100644 --- a/storagemarket/types/types.go +++ b/storagemarket/types/types.go @@ -16,7 +16,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/storage/sealer/storiface" "github.com/google/uuid" "github.com/ipfs/go-cid" ) @@ -146,7 +146,7 @@ type PieceAdder interface { } type CommpCalculator interface { - ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) + ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (abi.PieceInfo, error) } type DealPublisher interface {