filestore util: Add 'filestore rm' command.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

core/commands/filestore.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"io"
 
+	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
+	butil "github.com/ipfs/go-ipfs/blocks/blockstore/util"
 	cmds "github.com/ipfs/go-ipfs/commands"
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/filestore"
@@ -20,6 +22,7 @@ var FileStoreCmd = &cmds.Command{
 		"ls":     lsFileStore,
 		"verify": verifyFileStore,
 		"dups":   dupsFileStore,
+		"rm":     rmFileStore,
 	},
 }
 
@@ -193,6 +196,59 @@ var dupsFileStore = &cmds.Command{
 	Type:       RefWrapper{},
 }
 
+var rmFileStore = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Remove IPFS block(s) from just the filestore or blockstore.",
+		ShortDescription: `
+Remove blocks from either the filestore or the main blockstore.
+`,
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("hash", true, true, "CID's of block(s) to remove."),
+	},
+	Options: []cmds.Option{
+		cmds.BoolOption("force", "f", "Ignore nonexistent blocks."),
+		cmds.BoolOption("quiet", "q", "Write minimal output."),
+		cmds.BoolOption("non-filestore", "Remove non-filestore blocks"),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		n, fs, err := getFilestore(req)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		hashes := req.Arguments()
+		force, _, _ := req.Option("force").Bool()
+		quiet, _, _ := req.Option("quiet").Bool()
+		nonFilestore, _, _ := req.Option("non-filestore").Bool()
+		prefix := filestore.FilestorePrefix.String()
+		if nonFilestore {
+			prefix = bs.BlockPrefix.String()
+		}
+		cids := make([]*cid.Cid, 0, len(hashes))
+		for _, hash := range hashes {
+			c, err := cid.Decode(hash)
+			if err != nil {
+				res.SetError(fmt.Errorf("invalid content id: %s (%s)", hash, err), cmds.ErrNormal)
+				return
+			}
+
+			cids = append(cids, c)
+		}
+		ch, err := filestore.RmBlocks(fs, n.Blockstore, n.Pinning, cids, butil.RmBlocksOpts{
+			Prefix: prefix,
+			Quiet: quiet,
+			Force: force,
+		})
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		res.SetOutput(ch)
+	},
+	PostRun: blockRmCmd.PostRun,
+	Type: butil.RemovedBlock{},
+}
 func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error) {
 	n, err := req.InvocContext().GetNode()
 	if err != nil {

filestore/remove.go

@@ -0,0 +1,89 @@
+package filestore
+
+import (
+	"fmt"
+
+	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
+	u "github.com/ipfs/go-ipfs/blocks/blockstore/util"
+	"github.com/ipfs/go-ipfs/pin"
+	ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore"
+	cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
+)
+
+// Note: Like util/remove.go but allows removal of pinned block from
+//   one store if it is also in the other
+
+type Deleter interface {
+	DeleteBlock(c *cid.Cid) error
+}
+
+func RmBlocks(fs *Filestore, lock bs.GCLocker, pins pin.Pinner, cids []*cid.Cid, opts u.RmBlocksOpts) (<-chan interface{}, error) {
+	// make the channel large enough to hold any result to avoid
+	// blocking while holding the GCLock
+	out := make(chan interface{}, len(cids))
+
+	var blocks Deleter
+	switch opts.Prefix {
+	case FilestorePrefix.String():
+		blocks = fs.fm
+	case bs.BlockPrefix.String():
+		blocks = fs.bs
+	default:
+		return nil, fmt.Errorf("Unknown prefix: %s\n", opts.Prefix)
+	}
+
+	go func() {
+		defer close(out)
+
+		unlocker := lock.GCLock()
+		defer unlocker.Unlock()
+
+		stillOkay := FilterPinned(fs, pins, out, cids, blocks)
+
+		for _, c := range stillOkay {
+			err := blocks.DeleteBlock(c)
+			if err != nil && opts.Force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
+				// ignore non-existent blocks
+			} else if err != nil {
+				out <- &u.RemovedBlock{Hash: c.String(), Error: err.Error()}
+			} else if !opts.Quiet {
+				out <- &u.RemovedBlock{Hash: c.String()}
+			}
+		}
+	}()
+	return out, nil
+}
+
+func FilterPinned(fs *Filestore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, foundIn Deleter) []*cid.Cid {
+	stillOkay := make([]*cid.Cid, 0, len(cids))
+	res, err := pins.CheckIfPinned(cids...)
+	if err != nil {
+		out <- &u.RemovedBlock{Error: fmt.Sprintf("pin check failed: %s", err)}
+		return nil
+	}
+	for _, r := range res {
+		if !r.Pinned() || AvailableElsewhere(fs, foundIn, r.Key) {
+			stillOkay = append(stillOkay, r.Key)
+		} else {
+			out <- &u.RemovedBlock{
+				Hash:  r.Key.String(),
+				Error: r.String(),
+			}
+		}
+	}
+	return stillOkay
+}
+
+func AvailableElsewhere(fs *Filestore, foundIn Deleter, c *cid.Cid) bool {
+	switch {
+	case fs.fm == foundIn:
+		have, _ := fs.bs.Has(c)
+		return have
+	case fs.bs == foundIn:
+		have, _ := fs.fm.Has(c)
+		return have
+	default:
+		// programmer error
+		panic("invalid pointer for foundIn")
+	}
+}

test/sharness/t0271-filestore-utils.sh

@@ -139,7 +139,7 @@ test_filestore_verify() {
         test_init_dataset
 }
 
-test_filestore_dups() {
+test_filestore_dups_and_rm() {
 	# make sure the filestore is in a clean state
 	test_filestore_state
 
@@ -149,6 +149,28 @@ test_filestore_dups() {
 		echo "$FILE1_HASH" > dups_expect
 		test_cmp dups_expect dups_actual
 	'
+
+	test_expect_success "remove non-filestore block of dup ok" '
+		ipfs filestore rm --non-filestore $FILE1_HASH &&
+		ipfs filestore dups > dups_actual &&
+		test_cmp /dev/null dups_actual
+	'
+
+	test_expect_success "block still in filestore" '
+		ipfs filestore ls $FILE1_HASH | grep -q file1
+	'
+
+	test_expect_success "remove non-duplicate pinned block not ok" '
+		test_must_fail ipfs filestore rm $FILE1_HASH 2>&1 | tee rm_err &&
+		grep -q pinned rm_err
+	'
+
+	test_expect_success "remove filestore block of dup ok" '
+		ipfs add --raw-leaves somedir/file1 &&
+		ipfs filestore rm $FILE1_HASH &&
+		ipfs filestore dups > dups_actual &&
+		test_cmp /dev/null dups_actual
+	'
 }
 
 #
@@ -161,7 +183,7 @@ test_filestore_adds
 
 test_filestore_verify
 
-test_filestore_dups
+test_filestore_dups_and_rm
 
 #
 # With daemon
@@ -177,7 +199,7 @@ test_filestore_adds
 
 test_filestore_verify
 
-test_filestore_dups
+test_filestore_dups_and_rm
 
 test_kill_ipfs_daemon