Forward-looking erase-state CRCs

.github/workflows/test.yml

@@ -473,6 +473,42 @@ jobs:
           path: status
           retention-days: 1
 
+  # run compatibility tests using the current master as the previous version
+  test-compat:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+        if: ${{github.event_name == 'pull_request'}}
+      # checkout the current pr target into lfsp
+      - uses: actions/checkout@v2
+        if: ${{github.event_name == 'pull_request'}}
+        with:
+          ref: ${{github.event.pull_request.base.ref}}
+          path: lfsp
+      - name: install
+        if: ${{github.event_name == 'pull_request'}}
+        run: |
+          # need a few things
+          sudo apt-get update -qq
+          sudo apt-get install -qq gcc python3 python3-pip
+          pip3 install toml
+          gcc --version
+          python3 --version
+      # adjust prefix of lfsp
+      - name: changeprefix
+        if: ${{github.event_name == 'pull_request'}}
+        run: |
+          ./scripts/changeprefix.py lfs lfsp lfsp/*.h lfsp/*.c
+      - name: test-compat
+        if: ${{github.event_name == 'pull_request'}}
+        run: |
+          TESTS=tests/test_compat.toml \
+            SRC="$(find . lfsp -name '*.c' -maxdepth 1 \
+                -and -not -name '*.t.*' \
+                -and -not -name '*.b.*')" \
+            CFLAGS="-DLFSP=lfsp/lfsp.h" \
+            make test
+
   # self-host with littlefs-fuse for a fuzz-like test
   fuse:
     runs-on: ubuntu-22.04

Makefile

@@ -1,15 +1,5 @@
-ifdef BUILDDIR
-# bit of a hack, but we want to make sure BUILDDIR directory structure
-# is correct before any commands
-$(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \
-	$(BUILDDIR)/ \
-	$(BUILDDIR)/bd \
-	$(BUILDDIR)/runners \
-	$(BUILDDIR)/tests \
-	$(BUILDDIR)/benches))
-endif
+# overrideable build dir, default is in-place
 BUILDDIR ?= .
-
 # overridable target/src/tools/flags/etc
 ifneq ($(wildcard test.c main.c),)
 TARGET ?= $(BUILDDIR)/lfs
@@ -163,6 +153,18 @@ TESTFLAGS  += --perf-path="$(PERF)"
 BENCHFLAGS += --perf-path="$(PERF)"
 endif
 
+# this is a bit of a hack, but we want to make sure the BUILDDIR
+# directory structure is correct before we run any commands
+ifneq ($(BUILDDIR),.)
+$(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \
+	$(addprefix $(BUILDDIR)/,$(dir \
+		$(SRC) \
+		$(TESTS) \
+		$(TEST_SRC) \
+		$(BENCHES) \
+		$(BENCH_SRC)))))
+endif
+
 
 # commands
 
@@ -514,6 +516,9 @@ $(BUILDDIR)/runners/bench_runner: $(BENCH_OBJ)
 $(BUILDDIR)/%.o $(BUILDDIR)/%.ci: %.c
 	$(CC) -c -MMD $(CFLAGS) $< -o $(BUILDDIR)/$*.o
 
+$(BUILDDIR)/%.o $(BUILDDIR)/%.ci: $(BUILDDIR)/%.c
+	$(CC) -c -MMD $(CFLAGS) $< -o $(BUILDDIR)/$*.o
+
 $(BUILDDIR)/%.s: %.c
 	$(CC) -S $(CFLAGS) $< -o $@
 

SPEC.md

@@ -1,10 +1,10 @@
 ## littlefs technical specification
 
-This is the technical specification of the little filesystem. This document
-covers the technical details of how the littlefs is stored on disk for
-introspection and tooling. This document assumes you are familiar with the
-design of the littlefs, for more info on how littlefs works check
-out [DESIGN.md](DESIGN.md).
+This is the technical specification of the little filesystem with on-disk
+version lfs2.1. This document covers the technical details of how the littlefs
+is stored on disk for introspection and tooling. This document assumes you are
+familiar with the design of the littlefs, for more info on how littlefs works
+check out [DESIGN.md](DESIGN.md).
 
 ```
    | | |     .---._____
@@ -133,12 +133,6 @@ tags XORed together, starting with `0xffffffff`.
 '-------------------'                    '-------------------'
 ```
 
-One last thing to note before we get into the details around tag encoding. Each
-tag contains a valid bit used to indicate if the tag and containing commit is
-valid. This valid bit is the first bit found in the tag and the commit and can
-be used to tell if we've attempted to write to the remaining space in the
-block.
-
 Here's a more complete example of metadata block containing 4 entries:
 
 ```
@@ -191,6 +185,53 @@ Here's a more complete example of metadata block containing 4 entries:
                                              '---- most recent D
 ```
 
+Two things to note before we get into the details around tag encoding:
+
+1. Each tag contains a valid bit used to indicate if the tag and containing
+   commit is valid. After XORing, this bit should always be zero.
+
+   At the end of each commit, the valid bit of the previous tag is XORed
+   with the lowest bit in the type field of the CRC tag. This allows
+   the CRC tag to force the next commit to fail the valid bit test if it
+   has not yet been written to.
+
+2. The valid bit alone is not enough info to know if the next commit has been
+   erased. We don't know the order bits will be programmed in a program block,
+   so it's possible that the next commit had an attempted program that left the
+   valid bit unchanged.
+
+   To ensure we only ever program erased bytes, each commit can contain an
+   optional forward-CRC (FCRC). An FCRC contains a checksum of some amount of
+   bytes in the next commit at the time it was erased.
+
+   ```
+   .-------------------. \      \
+   |  revision count   | |      |
+   |-------------------| |      |
+   |     metadata      | |      |
+   |                   | +---.  +-- current commit
+   |                   | |   |  |
+   |-------------------| |   |  |
+   |       FCRC       ---|-. |  |
+   |-------------------| / | |  |
+   |        CRC       -----|-'  /
+   |-------------------|   |
+   |      padding      |   |        padding (does't need CRC)
+   |                   |   |
+   |-------------------| \ |    \
+   |      erased?      | +-'    |
+   |         |         | |      +-- next commit
+   |         v         | /      |
+   |                   |        /
+   |                   |
+   '-------------------'
+   ```
+
+   If the FCRC is missing or the checksum does not match, we must assume a
+   commit was attempted but failed due to power-loss.
+
+   Note that end-of-block commits do not need an FCRC.
+
 ## Metadata tags
 
 So in littlefs, 32-bit tags describe every type of metadata. And this means
@@ -785,3 +826,41 @@ CRC fields:
    are made about the contents.
 
 ---
+#### `0x5ff` LFS_TYPE_FCRC
+
+Added in lfs2.1, the optional FCRC tag contains a checksum of some amount of
+bytes in the next commit at the time it was erased. This allows us to ensure
+that we only ever program erased bytes, even if a previous commit failed due
+to power-loss.
+
+When programming a commit, the FCRC size must be at least as large as the
+program block size. However, the program block is not saved on disk, and can
+change between mounts, so the FCRC size on disk may be different than the
+current program block size.
+
+If the FCRC is missing or the checksum does not match, we must assume a
+commit was attempted but failed due to power-loss.
+
+Layout of the FCRC tag:
+
+```
+        tag                          data
+[--      32      --][--      32      --|--      32      --]
+[1|- 11 -| 10 | 10 ][--      32      --|--      32      --]
+ ^    ^     ^    ^            ^- fcrc size       ^- fcrc
+ |    |     |    '- size (8)
+ |    |     '------ id (0x3ff)
+ |    '------------ type (0x5ff)
+ '----------------- valid bit
+```
+
+FCRC fields:
+
+1. **FCRC size (32-bits)** - Number of bytes after this commit's CRC tag's
+   padding to include in the FCRC.
+
+2. **FCRC (32-bits)** - CRC of the bytes after this commit's CRC tag's padding
+   when erased. Like the CRC tag, this uses a CRC-32 with a polynomial of
+   `0x04c11db7` initialized with `0xffffffff`.
+
+---

bd/lfs_emubd.c

@@ -584,13 +584,14 @@ lfs_emubd_swear_t lfs_emubd_wear(const struct lfs_config *cfg,
         wear = 0;
     }
 
-    LFS_EMUBD_TRACE("lfs_emubd_wear -> %"PRIu32, wear);
+    LFS_EMUBD_TRACE("lfs_emubd_wear -> %"PRIi32, wear);
     return wear;
 }
 
 int lfs_emubd_setwear(const struct lfs_config *cfg,
         lfs_block_t block, lfs_emubd_wear_t wear) {
-    LFS_EMUBD_TRACE("lfs_emubd_setwear(%p, %"PRIu32")", (void*)cfg, block);
+    LFS_EMUBD_TRACE("lfs_emubd_setwear(%p, %"PRIu32", %"PRIi32")",
+            (void*)cfg, block, wear);
     lfs_emubd_t *bd = cfg->context;
 
     // check if block is valid
@@ -599,12 +600,12 @@ int lfs_emubd_setwear(const struct lfs_config *cfg,
     // set the wear
     lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]);
     if (!b) {
-        LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, LFS_ERR_NOMEM);
+        LFS_EMUBD_TRACE("lfs_emubd_setwear -> %d", LFS_ERR_NOMEM);
         return LFS_ERR_NOMEM;
     }
     b->wear = wear;
 
-    LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, 0);
+    LFS_EMUBD_TRACE("lfs_emubd_setwear -> %d", 0);
     return 0;
 }