From 41597c2856d6ac7328baca1340c3e36ab0edd382 Mon Sep 17 00:00:00 2001 From: holisticode Date: Thu, 7 Feb 2019 09:49:19 -0500 Subject: [PATCH] swarm: Debug API and HasChunks() API endpoint (#18980) --- cmd/swarm/main.go | 2 +- swarm/api/inspector.go | 58 +++++++++++++++++++++++++++++ swarm/api/testapi.go | 34 ----------------- swarm/network/stream/common_test.go | 5 +++ swarm/storage/common_test.go | 9 +++++ swarm/storage/ldbstore.go | 12 ++++++ swarm/storage/localstore.go | 7 ++++ swarm/storage/localstore_test.go | 33 ++++++++++++++++ swarm/storage/memstore.go | 5 +++ swarm/storage/netstore.go | 7 ++++ swarm/storage/types.go | 8 +++- swarm/swarm.go | 2 +- 12 files changed, 145 insertions(+), 37 deletions(-) create mode 100644 swarm/api/inspector.go delete mode 100644 swarm/api/testapi.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 385afad99233..af00503ea3e8 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -468,5 +468,5 @@ func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) { } cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node) } - log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes)) + } diff --git a/swarm/api/inspector.go b/swarm/api/inspector.go new file mode 100644 index 000000000000..6706b32e6529 --- /dev/null +++ b/swarm/api/inspector.go @@ -0,0 +1,58 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "context" + + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +type Inspector struct { + api *API + hive *network.Hive + netStore *storage.NetStore +} + +func NewInspector(api *API, hive *network.Hive, netStore *storage.NetStore) *Inspector { + return &Inspector{api, hive, netStore} +} + +// Hive prints the kademlia table +func (inspector *Inspector) Hive() string { + return inspector.hive.String() +} + +type HasInfo struct { + Addr string `json:"address"` + Has bool `json:"has"` +} + +// Has checks whether each chunk address is present in the underlying datastore, +// the bool in the returned structs indicates if the underlying datastore has +// the chunk stored with the given address (true), or not (false) +func (inspector *Inspector) Has(chunkAddresses []storage.Address) []HasInfo { + results := make([]HasInfo, 0) + for _, addr := range chunkAddresses { + res := HasInfo{} + res.Addr = addr.String() + res.Has = inspector.netStore.Has(context.Background(), addr) + results = append(results, res) + } + return results +} diff --git a/swarm/api/testapi.go b/swarm/api/testapi.go deleted file mode 100644 index 6fec55f559c9..000000000000 --- a/swarm/api/testapi.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package api - -import ( - "github.com/ethereum/go-ethereum/swarm/network" -) - -type Control struct { - api *API - hive *network.Hive -} - -func NewControl(api *API, hive *network.Hive) *Control { - return &Control{api, hive} -} - -func (c *Control) Hive() string { - return c.hive.String() -} diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 3b6e4a9465df..8a7d851fb7ef 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -214,6 +214,11 @@ func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { } } +// not used in this context, only to fulfill ChunkStore interface +func (rrs *roundRobinStore) Has(ctx context.Context, addr storage.Address) bool { + panic("RoundRobinStor doesn't support HasChunk") +} + func (rrs *roundRobinStore) Get(ctx context.Context, addr storage.Address) (storage.Chunk, error) { return nil, errors.New("get not well defined on round robin store") } diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 6c737af44ebf..23d43beccfcf 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -267,6 +267,15 @@ func (m *MapChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { return chunk, nil } +// Need to implement Has from SyncChunkStore +func (m *MapChunkStore) Has(ctx context.Context, ref Address) bool { + m.mu.RLock() + defer m.mu.RUnlock() + + _, has := m.chunks[ref.Hex()] + return has +} + func (m *MapChunkStore) Close() { } diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 635d33429720..a2f24eff06bf 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -969,6 +969,18 @@ func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) return s.get(addr) } +// Has queries the underlying DB if a chunk with the given address is stored +// Returns true if the chunk is found, false if not +func (s *LDBStore) Has(_ context.Context, addr Address) bool { + s.lock.RLock() + defer s.lock.RUnlock() + + ikey := getIndexKey(addr) + _, err := s.db.Get(ikey) + + return err == nil +} + // TODO: To conform with other private methods of this object indices should not be updated func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { if s.closed { diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 956560902b8c..eefb7565a59d 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -132,6 +132,13 @@ func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error { return err } +// Has queries the underlying DbStore if a chunk with the given address +// is being stored there. +// Returns true if it is stored, false if not +func (ls *LocalStore) Has(ctx context.Context, addr Address) bool { + return ls.DbStore.Has(ctx, addr) +} + // Get(chunk *Chunk) looks up a chunk in the local stores // This method is blocking until the chunk is retrieved // so additional timeout may be needed to wrap this call if diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 7a4162a475a4..ec69951c4fa1 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -209,3 +209,36 @@ func setupLocalStore(t *testing.T, ldbCap int) (ls *LocalStore, cleanup func()) return store, cleanup } + +func TestHas(t *testing.T) { + ldbCap := defaultGCRatio + store, cleanup := setupLocalStore(t, ldbCap) + defer cleanup() + + nonStoredAddr := GenerateRandomChunk(128).Address() + + has := store.Has(context.Background(), nonStoredAddr) + if has { + t.Fatal("Expected Has() to return false, but returned true!") + } + + storeChunks := GenerateRandomChunks(128, 3) + for _, ch := range storeChunks { + err := store.Put(context.Background(), ch) + if err != nil { + t.Fatalf("Expected store to store chunk, but it failed: %v", err) + } + + has := store.Has(context.Background(), ch.Address()) + if !has { + t.Fatal("Expected Has() to return true, but returned false!") + } + } + + //let's be paranoic and test again that the non-existent chunk returns false + has = store.Has(context.Background(), nonStoredAddr) + if has { + t.Fatal("Expected Has() to return false, but returned true!") + } + +} diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 86e5813d1b91..611ac3bc51da 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -48,6 +48,11 @@ func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore) { } } +// Has needed to implement SyncChunkStore +func (m *MemStore) Has(_ context.Context, addr Address) bool { + return m.cache.Contains(addr) +} + func (m *MemStore) Get(_ context.Context, addr Address) (Chunk, error) { if m.disabled { return nil, ErrChunkNotFound diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 16bc48a9a2cd..b24d08bc2d1f 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -158,6 +158,13 @@ func (n *NetStore) get(ctx context.Context, ref Address) (Chunk, func(context.Co return chunk, nil, nil } +// Has is the storage layer entry point to query the underlying +// database to return if it has a chunk or not. +// Called from the DebugAPI +func (n *NetStore) Has(ctx context.Context, ref Address) bool { + return n.store.Has(ctx, ref) +} + // getOrCreateFetcher attempts at retrieving an existing fetchers // if none exists, creates one and saves it in the fetchers cache // caller must hold the lock diff --git a/swarm/storage/types.go b/swarm/storage/types.go index d792352252e3..7ec21328e2bb 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -292,6 +292,7 @@ func (v *ContentAddressValidator) Validate(chunk Chunk) bool { type ChunkStore interface { Put(ctx context.Context, ch Chunk) (err error) Get(rctx context.Context, ref Address) (ch Chunk, err error) + Has(rctx context.Context, ref Address) bool Close() } @@ -314,7 +315,12 @@ func (f *FakeChunkStore) Put(_ context.Context, ch Chunk) error { return nil } -// Gut doesn't store anything it is just here to implement ChunkStore +// Has doesn't do anything it is just here to implement ChunkStore +func (f *FakeChunkStore) Has(_ context.Context, ref Address) bool { + panic("FakeChunkStore doesn't support HasChunk") +} + +// Get doesn't store anything it is just here to implement ChunkStore func (f *FakeChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { panic("FakeChunkStore doesn't support Get") } diff --git a/swarm/swarm.go b/swarm/swarm.go index 8274a168d6a9..705fc43975cb 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -485,7 +485,7 @@ func (self *Swarm) APIs() []rpc.API { { Namespace: "bzz", Version: "3.0", - Service: api.NewControl(self.api, self.bzz.Hive), + Service: api.NewInspector(self.api, self.bzz.Hive, self.netStore), Public: false, }, {