Skip to content

Commit 9dcf004

Browse files
authored
Merge pull request #150 from ipfs/fix/11-to-12-querying
11-to-12: Re-work logic to use the backup file for both migrate and revert
2 parents 25fed4c + fa319e8 commit 9dcf004

File tree

7 files changed

+140
-106
lines changed

7 files changed

+140
-106
lines changed

fs-repo-11-to-12/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ require (
77
github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea
88
github.com/ipfs/go-cid v0.0.7
99
github.com/ipfs/go-datastore v0.4.5
10-
github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5 // indirect
10+
github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612 // indirect
1111
github.com/ipfs/go-filestore v1.0.0
1212
github.com/ipfs/go-ipfs v0.8.0
1313
github.com/ipfs/go-ipfs-ds-help v1.0.0

fs-repo-11-to-12/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -295,8 +295,8 @@ github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9
295295
github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE=
296296
github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk=
297297
github.com/ipfs/go-ds-badger v0.2.6/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA=
298-
github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5 h1:ovdpQk2ZVK6eQLzMCZy1z2tJae7yvE8xaPUw4Pr1RqI=
299-
github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA=
298+
github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612 h1:Uvp2/ZNlR3YmH04XJGv4YsPabhPzRPyeroLltVKBSr8=
299+
github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA=
300300
github.com/ipfs/go-ds-flatfs v0.4.5 h1:4QceuKEbH+HVZ2ZommstJMi3o3II+dWS3IhLaD7IGHs=
301301
github.com/ipfs/go-ds-flatfs v0.4.5/go.mod h1:e4TesLyZoA8k1gV/yCuBTnt2PJtypn4XUlB5n8KQMZY=
302302
github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=

fs-repo-11-to-12/migration/migration.go

Lines changed: 62 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,6 @@ func (m *Migration) Apply(opts migrate.Options) error {
9292
log.Error(err)
9393
return err
9494
}
95-
defer f.Close()
9695
buf := bufio.NewWriter(f)
9796

9897
swapCh := make(chan Swap, 1000)
@@ -119,7 +118,7 @@ func (m *Migration) Apply(opts migrate.Options) error {
119118
for _, prefix := range migrationPrefixes {
120119
log.VLog(" - Adding keys in prefix %s to backup file", prefix)
121120
cidSwapper := CidSwapper{Prefix: prefix, Store: m.dstore, SwapCh: swapCh}
122-
total, err := cidSwapper.Run(true) // DRY RUN
121+
total, err := cidSwapper.Prepare() // DRY RUN
123122
if err != nil {
124123
close(swapCh)
125124
log.Error(err)
@@ -131,17 +130,12 @@ func (m *Migration) Apply(opts migrate.Options) error {
131130
// Wait for our writing to finish before doing the flushing.
132131
<-writingDone
133132
buf.Flush()
133+
f.Close()
134134

135-
// MIGRATION: Run the real migration.
136-
for _, prefix := range migrationPrefixes {
137-
log.VLog(" - Migrating keys in prefix %s", prefix)
138-
cidSwapper := CidSwapper{Prefix: prefix, Store: m.dstore}
139-
total, err := cidSwapper.Run(false) // NOT a Dry Run
140-
if err != nil {
141-
log.Error(err)
142-
return err
143-
}
144-
log.Log("%d CIDv1 keys in %s have been migrated", total, prefix)
135+
err = m.scanAndSwap(filepath.Join(opts.Path, backupFile), false) // revert=false
136+
if err != nil {
137+
log.Error(err)
138+
return err
145139
}
146140

147141
// Wrap up, we are now in repo-version 12.
@@ -187,21 +181,48 @@ func (m *Migration) Revert(opts migrate.Options) error {
187181

188182
// Open revert path for reading
189183
backupPath := filepath.Join(opts.Path, backupFile)
184+
err = m.scanAndSwap(backupPath, true) // revert = true
185+
if err != nil {
186+
log.Error(err)
187+
return err
188+
}
189+
190+
// Wrap up the Revert. We are back at version 11.
191+
if err := repo.WriteVersion("11"); err != nil {
192+
log.Error("failed to write version file")
193+
return err
194+
}
195+
196+
log.Log("reverted version file to version 11")
197+
198+
// Move the backup file out of the way.
199+
err = os.Rename(backupPath, backupPath+".reverted")
200+
if err != nil {
201+
log.Error("could not rename the backup file, but migration worked: %s", err)
202+
return err
203+
}
204+
return nil
205+
}
206+
207+
// Receives a backup file which contains all the things that need to be
208+
// migrated and reads every line, performing swaps in the needed direction.
209+
func (m *Migration) scanAndSwap(backupPath string, revert bool) error {
190210
f, err := getBackupFile(backupPath)
191211
if err != nil {
192212
log.Error(err)
193213
return err
194214
}
215+
defer f.Close()
195216

196-
unswapCh := make(chan Swap, 1000)
217+
swapCh := make(chan Swap, 1000)
197218
scanner := bufio.NewScanner(f)
198219
var scannerErr error
199220

200-
// This will send swap objects to the Unswapper on unswapCh as they
221+
// This will send swap objects to the swapping channel as they
201222
// are read from the backup file on disk. It will also send MFS and
202-
// pinset pins for reversal.
223+
// pinset pins for reversal when doing a revert.
203224
go func() {
204-
defer close(unswapCh)
225+
defer close(swapCh)
205226

206227
// Process backup file first.
207228
for scanner.Scan() {
@@ -222,60 +243,53 @@ func (m *Migration) Revert(opts migrate.Options) error {
222243
break
223244
}
224245
mhashPath := prefix.Child(dshelp.MultihashToDsKey(cid.Hash()))
225-
// This is the original swap object which is what we
226-
// wanted to rebuild. Old is the old path and new is
227-
// the new path and the unswapper will revert this.
246+
247+
// The swapper will move cidPath to mhashPath, and the unswapper
248+
// will do the opposite.
228249
sw := Swap{Old: cidPath, New: mhashPath}
229-
unswapCh <- sw
250+
swapCh <- sw
230251
}
231252
if err := scanner.Err(); err != nil {
232253
log.Error(err)
233254
return
234255
}
235256

236-
// Process MFS/pinset. We have to do this in cases the user
237-
// has been running with the migration for some time and made changes to
238-
// the pinset or the MFS root.
239-
if err := walkPinsAndMFS(unswapCh, m.dstore); err != nil {
240-
log.Error(err)
241-
return
257+
if revert {
258+
// Process MFS/pinset. We have to do this in cases the
259+
// user has been running with the migration for some
260+
// time and made changes to the pinset or the MFS
261+
// root.
262+
if err := walkPinsAndMFS(swapCh, m.dstore); err != nil {
263+
log.Error(err)
264+
return
265+
}
242266
}
243-
244267
}()
245268

246269
// The backup file contains prefixed keys, so we do not need to set
247270
// Prefix in the CidSwapper.
248271
cidSwapper := CidSwapper{Store: m.dstore}
249-
total, err := cidSwapper.Revert(unswapCh)
272+
var total uint64
273+
if revert {
274+
total, err = cidSwapper.Revert(swapCh)
275+
} else {
276+
total, err = cidSwapper.Run(swapCh)
277+
}
250278
if err != nil {
251279
log.Error(err)
252280
return err
253281
}
254-
// Revert will only return after unswapCh is closed, so we know
282+
283+
// The swapper will only return after swapCh is closed, so we know
255284
// scannerErr is safe to read at this point.
256285
if scannerErr != nil {
257286
return err
258287
}
259288

260-
// Wrap up the Revert. We are back at version 11.
261-
log.Log("%d multihashes reverted to CidV1s", total)
262-
if err := repo.WriteVersion("11"); err != nil {
263-
log.Error("failed to write version file")
264-
return err
265-
}
266-
267-
log.Log("reverted version file to version 11")
268-
err = f.Close()
269-
if err != nil {
270-
log.Error("could not close backup file")
271-
return err
272-
}
273-
274-
// Move the backup file out of the way.
275-
err = os.Rename(backupPath, backupPath+".reverted")
276-
if err != nil {
277-
log.Error("could not rename the backup file, but migration worked: %s", err)
278-
return err
289+
if revert {
290+
log.Log("%d multihashes swapped to CidV1s", total)
291+
} else {
292+
log.Log("%d CidV1s swapped to multihashes", total)
279293
}
280294
return nil
281295
}

0 commit comments

Comments
 (0)