Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: update go-libp2p to v0.16.0 #8522

Closed
wants to merge 25 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
378b416
WIP: update go-libp2p to v0.16.0
marten-seemann Oct 21, 2021
61d93d9
WIP relay config
marten-seemann Oct 19, 2021
3aa2cfd
refactor: apply config changes from review
lidel Oct 26, 2021
af01301
refactor: RelayTransport → Relay
lidel Oct 26, 2021
5a73fff
refactor: use *OptionalInteger in config
lidel Oct 27, 2021
c517a12
refactor: flatten RelayService limits
lidel Oct 27, 2021
023ffa7
docs(config): circuit relay v2
lidel Oct 28, 2021
2577549
docs(config): user relay before service
lidel Oct 28, 2021
8b2367a
docs(config): fix links and headers
lidel Oct 28, 2021
9294163
feat(config): Internal.Libp2pForceReachability
lidel Oct 28, 2021
7d05741
simplify processing of relay service opts
marten-seemann Oct 29, 2021
05744cc
use the ForceReachability option, update sharness test
marten-seemann Oct 29, 2021
f5cc4a6
refactor: switch to real flag in sharness
lidel Oct 29, 2021
93330d5
fix ForceReachability dependency injection
marten-seemann Oct 30, 2021
0e7e5e1
use configuration option for static relays
marten-seemann Nov 13, 2021
dea5469
update sharness
marten-seemann Nov 14, 2021
4142b46
plumb through pseudoversions for context changes
guseggert Nov 15, 2021
925f7d7
fix StaticRelay config option
marten-seemann Nov 16, 2021
97993d8
update go-libp2p
marten-seemann Nov 16, 2021
5358b0d
plumb through pseudoversions for context changes
guseggert Nov 15, 2021
b526b28
update go-ipfs-config to v0.17.0, fix sharness
marten-seemann Nov 17, 2021
3267832
add docs for Swarm.RelayClient configuration
marten-seemann Nov 19, 2021
e9c5347
use LIBP2P_TCP_REUSEPORT instead of IPFS_REUSEPORT
marten-seemann Nov 23, 2021
f8e80e2
chore: go-ipfs-config v0.18.0
lidel Nov 23, 2021
d510aa3
feat: circuit v1 migration prompt when Swarm.EnableRelayHop is set (#…
marten-seemann Nov 23, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .circleci/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ executors:
environment:
<<: *default_environment
NO_SANDBOX: true
IPFS_REUSEPORT: false
LIBP2P_TCP_REUSEPORT: false
LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
E2E_IPFSD_TYPE: go
dockerizer:
Expand Down Expand Up @@ -227,7 +227,7 @@ jobs:
npx ipfs-interop -- -t node -f $(sed -n -e "s|^require('\(.*\)')$|test/\1|p" node_modules/ipfs-interop/test/node.js | circleci tests split) -- --reporter mocha-circleci-reporter
working_directory: ~/ipfs/go-ipfs/interop
environment:
IPFS_REUSEPORT: false
LIBP2P_TCP_REUSEPORT: false
LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs
- store_test_results:
Expand Down
2 changes: 1 addition & 1 deletion Rules.mk
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ include mk/golang.mk
ifeq ($(TEST_NO_FUSE),1)
GOTAGS += nofuse
endif
export IPFS_REUSEPORT=false
export LIBP2P_TCP_REUSEPORT=false

# -------------------- #
# sub-files #
Expand Down
10 changes: 5 additions & 5 deletions blocks/blockstoreutil/remove.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (

cid "github.com/ipfs/go-cid"
bs "github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-pinner"
pin "github.com/ipfs/go-ipfs-pinner"
)

// RemovedBlock is used to represent the result of removing a block.
Expand Down Expand Up @@ -40,15 +40,15 @@ func RmBlocks(ctx context.Context, blocks bs.GCBlockstore, pins pin.Pinner, cids
go func() {
defer close(out)

unlocker := blocks.GCLock()
defer unlocker.Unlock()
unlocker := blocks.GCLock(ctx)
defer unlocker.Unlock(ctx)

stillOkay := FilterPinned(ctx, pins, out, cids)

for _, c := range stillOkay {
// Kept for backwards compatibility. We may want to
// remove this sometime in the future.
has, err := blocks.Has(c)
has, err := blocks.Has(ctx, c)
if err != nil {
out <- &RemovedBlock{Hash: c.String(), Error: err.Error()}
continue
Expand All @@ -58,7 +58,7 @@ func RmBlocks(ctx context.Context, blocks bs.GCBlockstore, pins pin.Pinner, cids
continue
}

err = blocks.DeleteBlock(c)
err = blocks.DeleteBlock(ctx, c)
if err != nil {
out <- &RemovedBlock{Hash: c.String(), Error: err.Error()}
} else if !opts.Quiet {
Expand Down
6 changes: 3 additions & 3 deletions core/commands/dag/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
// This is especially important for use cases like dagger:
// ipfs dag import $( ... | ipfs-dagger --stdout=carfifos )
//
unlocker := node.Blockstore.PinLock()
defer unlocker.Unlock()
unlocker := node.Blockstore.PinLock(req.Context)
defer unlocker.Unlock(req.Context)

doPinRoots, _ := req.Options[pinRootsOptionName].(bool)

Expand Down Expand Up @@ -87,7 +87,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment

ret := RootMeta{Cid: c}

if block, err := node.Blockstore.Get(c); err != nil {
if block, err := node.Blockstore.Get(req.Context, c); err != nil {
ret.PinErrorMsg = err.Error()
} else if nd, err := ipld.Decode(block); err != nil {
ret.PinErrorMsg = err.Error()
Expand Down
2 changes: 1 addition & 1 deletion core/commands/dht.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ var provideRefDhtCmd = &cmds.Command{
return err
}

has, err := nd.Blockstore.Has(c)
has, err := nd.Blockstore.Has(req.Context, c)
if err != nil {
return err
}
Expand Down
13 changes: 7 additions & 6 deletions core/commands/filestore.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
package commands

import (
"context"
"fmt"
"io"
"os"

filestore "github.com/ipfs/go-filestore"
cmds "github.com/ipfs/go-ipfs-cmds"
core "github.com/ipfs/go-ipfs/core"
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
e "github.com/ipfs/go-ipfs/core/commands/e"

"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-cmds"
)

var FileStoreCmd = &cmds.Command{
Expand Down Expand Up @@ -56,7 +57,7 @@ The output is:
}
args := req.Arguments
if len(args) > 0 {
return listByArgs(res, fs, args)
return listByArgs(req.Context, res, fs, args)
}

fileOrder, _ := req.Options[fileOrderOptionName].(bool)
Expand Down Expand Up @@ -133,7 +134,7 @@ For ERROR entries the error will also be printed to stderr.
}
args := req.Arguments
if len(args) > 0 {
return listByArgs(res, fs, args)
return listByArgs(req.Context, res, fs, args)
}

fileOrder, _ := req.Options[fileOrderOptionName].(bool)
Expand Down Expand Up @@ -206,7 +207,7 @@ var dupsFileStore = &cmds.Command{
}

for cid := range ch {
have, err := fs.MainBlockstore().Has(cid)
have, err := fs.MainBlockstore().Has(req.Context, cid)
if err != nil {
return res.Emit(&RefWrapper{Err: err.Error()})
}
Expand Down Expand Up @@ -235,7 +236,7 @@ func getFilestore(env cmds.Environment) (*core.IpfsNode, *filestore.Filestore, e
return n, fs, err
}

func listByArgs(res cmds.ResponseEmitter, fs *filestore.Filestore, args []string) error {
func listByArgs(ctx context.Context, res cmds.ResponseEmitter, fs *filestore.Filestore, args []string) error {
for _, arg := range args {
c, err := cid.Decode(arg)
if err != nil {
Expand All @@ -248,7 +249,7 @@ func listByArgs(res cmds.ResponseEmitter, fs *filestore.Filestore, args []string
}
continue
}
r := filestore.Verify(fs, c)
r := filestore.Verify(ctx, fs, c)
if err := res.Emit(r); err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion core/commands/pin/remotepin.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ NOTE: a comma-separated notation is supported in CLI for convenience:
return err
}

isInBlockstore, err := node.Blockstore.Has(rp.Cid())
isInBlockstore, err := node.Blockstore.Has(req.Context, rp.Cid())
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions core/commands/repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Ci
defer wg.Done()

for k := range keys {
_, err := bs.Get(k)
_, err := bs.Get(ctx, k)
if err != nil {
select {
case results <- fmt.Sprintf("block %s was corrupt (%s)", k, err):
Expand Down Expand Up @@ -288,7 +288,7 @@ var repoVerifyCmd = &cmds.Command{
}

bs := bstore.NewBlockstore(nd.Repo.Datastore())
bs.HashOnRead(true)
bs.HashOnRead(req.Context, true)

keys, err := bs.AllKeysChan(req.Context)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions core/coreapi/block.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
}

if settings.Pin {
defer api.blockstore.PinLock().Unlock()
defer api.blockstore.PinLock(ctx).Unlock(ctx)
}

err = api.blocks.AddBlock(b)
err = api.blocks.AddBlock(ctx, b)
if err != nil {
return nil, err
}
Expand Down
6 changes: 3 additions & 3 deletions core/coreapi/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"

cid "github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-pinner"
pin "github.com/ipfs/go-ipfs-pinner"
ipld "github.com/ipfs/go-ipld-format"
dag "github.com/ipfs/go-merkledag"
)
Expand All @@ -18,7 +18,7 @@ type dagAPI struct {
type pinningAdder CoreAPI

func (adder *pinningAdder) Add(ctx context.Context, nd ipld.Node) error {
defer adder.blockstore.PinLock().Unlock()
defer adder.blockstore.PinLock(ctx).Unlock(ctx)

if err := adder.dag.Add(ctx, nd); err != nil {
return err
Expand All @@ -30,7 +30,7 @@ func (adder *pinningAdder) Add(ctx context.Context, nd ipld.Node) error {
}

func (adder *pinningAdder) AddMany(ctx context.Context, nds []ipld.Node) error {
defer adder.blockstore.PinLock().Unlock()
defer adder.blockstore.PinLock(ctx).Unlock(ctx)

if err := adder.dag.AddMany(ctx, nds); err != nil {
return err
Expand Down
2 changes: 1 addition & 1 deletion core/coreapi/dht.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func (api *DhtAPI) Provide(ctx context.Context, path path.Path, opts ...caopts.D

c := rp.Cid()

has, err := api.blockstore.Has(c)
has, err := api.blockstore.Has(ctx, c)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions core/coreapi/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"io/ioutil"

cid "github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-pinner"
pin "github.com/ipfs/go-ipfs-pinner"
ipld "github.com/ipfs/go-ipld-format"
dag "github.com/ipfs/go-merkledag"
"github.com/ipfs/go-merkledag/dagutils"
Expand Down Expand Up @@ -110,7 +110,7 @@ func (api *ObjectAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Obj
}

if options.Pin {
defer api.blockstore.PinLock().Unlock()
defer api.blockstore.PinLock(ctx).Unlock(ctx)
}

err = api.dag.Add(ctx, dagnode)
Expand Down
6 changes: 3 additions & 3 deletions core/coreapi/pin.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp
return err
}

defer api.blockstore.PinLock().Unlock()
defer api.blockstore.PinLock(ctx).Unlock(ctx)

err = api.pinning.Pin(ctx, dagNode, settings.Recursive)
if err != nil {
Expand Down Expand Up @@ -89,7 +89,7 @@ func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOpti

// Note: after unpin the pin sets are flushed to the blockstore, so we need
// to take a lock to prevent a concurrent garbage collection
defer api.blockstore.PinLock().Unlock()
defer api.blockstore.PinLock(ctx).Unlock(ctx)

if err = api.pinning.Unpin(ctx, rp.Cid(), settings.Recursive); err != nil {
return err
Expand All @@ -114,7 +114,7 @@ func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opt
return err
}

defer api.blockstore.PinLock().Unlock()
defer api.blockstore.PinLock(ctx).Unlock(ctx)

err = api.pinning.Update(ctx, fp.Cid(), tp.Cid(), settings.Unpin)
if err != nil {
Expand Down
15 changes: 7 additions & 8 deletions core/coreapi/test/path_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/ipld/go-ipld-prime"
)


func TestPathUnixFSHAMTPartial(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -27,16 +26,16 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
a := apis[0]

// Setting this after instantiating the swarm so that it's not clobbered by loading the go-ipfs config
prevVal := uio.UseHAMTSharding
uio.UseHAMTSharding = true
defer func() {
uio.UseHAMTSharding = prevVal
}()
// prevVal := uio.UseHAMTSharding
// uio.UseHAMTSharding = true
// defer func() {
// uio.UseHAMTSharding = prevVal
// }()

// Create and add a sharded directory
dir := make(map[string]files.Node)
// Make sure we have at least two levels of sharding
for i := 0; i < uio.DefaultShardWidth + 1; i++ {
for i := 0; i < uio.DefaultShardWidth+1; i++ {
dir[strconv.Itoa(i)] = files.NewBytesFile([]byte(strconv.Itoa(i)))
}

Expand Down Expand Up @@ -67,7 +66,7 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
for k := range dir {
// The node will go out to the (non-existent) network looking for the missing block. Make sure we're erroring
// because we exceeded the timeout on our query
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second * 1)
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second*1)
_, err := a.ResolveNode(timeoutCtx, path.Join(r, k))
if err != nil {
if timeoutCtx.Err() == nil {
Expand Down
6 changes: 3 additions & 3 deletions core/coreapi/unixfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
DAGService: dserv,
syncFn: func() error {
ds := api.repo.Datastore()
if err := ds.Sync(bstore.BlockPrefix); err != nil {
if err := ds.Sync(ctx, bstore.BlockPrefix); err != nil {
return err
}
return ds.Sync(filestore.FilestorePrefix)
return ds.Sync(ctx, filestore.FilestorePrefix)
},
}
}
Expand Down Expand Up @@ -164,7 +164,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
fileAdder.SetMfsRoot(mr)
}

nd, err := fileAdder.AddAllAndPin(files)
nd, err := fileAdder.AddAllAndPin(ctx, files)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion core/corehttp/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func TestPeersTotal(t *testing.T) {
hosts := make([]*bhost.BasicHost, 4)
for i := 0; i < 4; i++ {
var err error
hosts[i], err = bhost.NewHost(ctx, swarmt.GenSwarm(t, ctx), nil)
hosts[i], err = bhost.NewHost(swarmt.GenSwarm(t), nil)
if err != nil {
t.Fatal(err)
}
Expand Down
2 changes: 1 addition & 1 deletion core/corerepo/gc.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func ConditionalGC(ctx context.Context, node *core.IpfsNode, offset uint64) erro
}

func (gc *GC) maybeGC(ctx context.Context, offset uint64) error {
storage, err := gc.Repo.GetStorageUsage()
storage, err := gc.Repo.GetStorageUsage(ctx)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion core/corerepo/stat.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func RepoSize(ctx context.Context, n *core.IpfsNode) (SizeStat, error) {
return SizeStat{}, err
}

usage, err := r.GetStorageUsage()
usage, err := r.GetStorageUsage(ctx)
if err != nil {
return SizeStat{}, err
}
Expand Down
Loading