Skip to content

Commit b413a7a

Browse files
qkaiserrxpha3l
qkaiser
authored and
rxpha3l
committed
feat(handler): add erofs filesystem & decompression handler
EROFS is a high-performance read-only filesystem originally developed by Huawei and now used extensively in AOSP. It offers features like compression (zstd, lz4, lz4hc, lzma, deflate, libdeflate), inline data, and reduced storage overhead, making it ideal for read-only system images. Create EROFS image: > mkfs.erofs -z[compression,level (optional)] foo.erofs.img foo/ Extract EROFS image: > fsck.erofs --no-preserve --extract=foo/ foo.erofs.img Key notes: - EROFS creates images from directories - The header is located at offset 0x400, min 128 bytes - There is no official file ending, but the documentation uses ".erofs.img" - Checksum calculation differ from version to version Header structure : > 4 bytes : Magic > 4 bytes : Checksum > 4 bytes : Feature compact > 1 byte : Block size in bit shift > 1 byte : Extended super block (128 + sb_extslots * 16) > 2 bytes : Root NID > 8 bytes : Total valid inodes > 8 bytes : Build time as timestamp > 4 bytes : Built time in nsec > 4 bytes : Block count > 4 bytes : Start block of metadata area > 4 bytes : Start block of shared xattr area > 16 bytes : 128-bit uuid of volume > 16 bytes : Volume name > 4 bytes : Feature incompact > 2 bytes : Union of compression alogrithm & LZ4 max distance > 2 bytes : Extra device > 2 bytes : Devt start offset > 1 byte : Directory block size > 1 byte : Number of long xattr name prefixes > 4 bytes : Start of long attr prefixes > 8 bytes : NID of the special packed inode > 1 byte : Reserved for xattr name filter > 23 bytes : Rerved (most likely NULL padding) Unblob struct the header until the feature_incompact. The rest is labeled as "reserved". Unblob parser the end offset by multiplying the block size in bit shift with the block count. Since the checksum is not consistent troughout the version, unblob matched on block count (at least 1), printable volume name and if there is a build time. [Sources] https://www.kernel.org/doc/html/latest/filesystems/erofs.html https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/fs/erofs/erofs_fs.h https://erofs.docs.kernel.org/en/latest/core_ondisk.html https://elixir.bootlin.com/linux/v6.14-rc6/source/fs/erofs/erofs_fs.h https://github.com/srlabs/extractor/blob/main/erofs_tool.py
1 parent 8502308 commit b413a7a

File tree

9 files changed

+212
-40
lines changed

9 files changed

+212
-40
lines changed

‎install-deps.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ apt-get update
55
apt-get install --no-install-recommends -y \
66
android-sdk-libsparse-utils \
77
curl \
8+
erofs-utils \
89
lz4 \
910
lziprecover \
1011
lzop \

‎overlay.nix‎

Lines changed: 1 addition & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -8,42 +8,5 @@ final: prev:
88
nativeCheckInputs = (super.nativeCheckInputs or [ ]) ++ [ final.which ];
99
});
1010

11-
unblob =
12-
let
13-
pyproject_toml = (builtins.fromTOML (builtins.readFile ./pyproject.toml));
14-
version = pyproject_toml.project.version;
15-
in
16-
(prev.unblob.override { e2fsprogs = final.e2fsprogs-nofortify; }).overridePythonAttrs (super: rec {
17-
inherit version;
18-
19-
src = final.nix-filter {
20-
root = ./.;
21-
include = [
22-
"Cargo.lock"
23-
"Cargo.toml"
24-
"pyproject.toml"
25-
"python"
26-
"rust"
27-
"tests"
28-
"README.md"
29-
];
30-
};
31-
32-
dependencies = (super.dependencies or []) ++ [ prev.python3.pkgs.pyzstd ];
33-
34-
# remove this when packaging changes are upstreamed
35-
cargoDeps = final.rustPlatform.importCargoLock {
36-
lockFile = ./Cargo.lock;
37-
};
38-
39-
nativeBuildInputs = with final.rustPlatform; [
40-
cargoSetupHook
41-
maturinBuildHook
42-
];
43-
44-
# override disabling of 'test_all_handlers[filesystem.extfs]' from upstream
45-
pytestFlagsArray = [
46-
"--no-cov"
47-
];
48-
});
11+
unblob = final.callPackage ./package.nix { };
4912
}

‎package.nix‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
{
2+
lib,
3+
python3,
4+
fetchFromGitHub,
5+
makeWrapper,
6+
e2fsprogs-nofortify,
7+
erofs-utils,
8+
jefferson,
9+
lz4,
10+
lziprecover,
11+
lzop,
12+
p7zip,
13+
nix-filter,
14+
sasquatch,
15+
sasquatch-v4be,
16+
simg2img,
17+
ubi_reader,
18+
unar,
19+
zstd,
20+
versionCheckHook,
21+
rustPlatform,
22+
}:
23+
24+
let
25+
# These dependencies are only added to PATH
26+
runtimeDeps = [
27+
e2fsprogs-nofortify
28+
erofs-utils
29+
jefferson
30+
lziprecover
31+
lzop
32+
p7zip
33+
sasquatch
34+
sasquatch-v4be
35+
ubi_reader
36+
simg2img
37+
unar
38+
zstd
39+
lz4
40+
];
41+
pyproject_toml = (builtins.fromTOML (builtins.readFile ./pyproject.toml));
42+
version = pyproject_toml.project.version;
43+
in
44+
python3.pkgs.buildPythonApplication rec {
45+
pname = "unblob";
46+
pyproject = true;
47+
disabled = python3.pkgs.pythonOlder "3.9";
48+
inherit version;
49+
src = nix-filter {
50+
root = ./.;
51+
include = [
52+
"Cargo.lock"
53+
"Cargo.toml"
54+
"pyproject.toml"
55+
"python"
56+
"rust"
57+
"tests"
58+
"README.md"
59+
];
60+
};
61+
62+
strictDeps = true;
63+
64+
build-system = with python3.pkgs; [ poetry-core ];
65+
66+
dependencies = with python3.pkgs; [
67+
arpy
68+
attrs
69+
click
70+
cryptography
71+
dissect-cstruct
72+
lark
73+
lief.py
74+
python3.pkgs.lz4 # shadowed by pkgs.lz4
75+
plotext
76+
pluggy
77+
pyfatfs
78+
pyperscan
79+
python-magic
80+
pyzstd
81+
rarfile
82+
rich
83+
structlog
84+
treelib
85+
unblob-native
86+
];
87+
88+
cargoDeps = rustPlatform.importCargoLock {
89+
lockFile = ./Cargo.lock;
90+
};
91+
92+
nativeBuildInputs = with rustPlatform; [
93+
cargoSetupHook
94+
maturinBuildHook
95+
makeWrapper
96+
];
97+
98+
# These are runtime-only CLI dependencies, which are used through
99+
# their CLI interface
100+
pythonRemoveDeps = [
101+
"jefferson"
102+
"ubi-reader"
103+
];
104+
105+
pythonImportsCheck = [ "unblob" ];
106+
107+
makeWrapperArgs = [
108+
"--prefix PATH : ${lib.makeBinPath runtimeDeps}"
109+
];
110+
111+
nativeCheckInputs =
112+
with python3.pkgs;
113+
[
114+
pytestCheckHook
115+
pytest-cov
116+
versionCheckHook
117+
]
118+
++ runtimeDeps;
119+
120+
versionCheckProgramArg = "--version";
121+
122+
pytestFlagsArray = [
123+
"--no-cov"
124+
];
125+
126+
passthru = {
127+
# helpful to easily add these to a nix-shell environment
128+
inherit runtimeDeps;
129+
};
130+
131+
}

‎python/unblob/handlers/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
ubi,
5050
yaffs,
5151
)
52-
from .filesystem.android import sparse
52+
from .filesystem.android import erofs, sparse
5353

5454
BUILTIN_HANDLERS: Handlers = (
5555
cramfs.CramFSHandler,
@@ -118,6 +118,7 @@
118118
engenius.EngeniusHandler,
119119
ecc.AutelECCHandler,
120120
uzip.UZIPHandler,
121+
erofs.EROFSHandler,
121122
)
122123

123124
BUILTIN_DIR_HANDLERS: DirectoryHandlers = (

‎python/unblob/handlers/filesystem/android/erofs.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import io
2+
from typing import Optional
3+
4+
from unblob.extractors import Command
5+
from unblob.file_utils import (
6+
Endian,
7+
InvalidInputFormat,
8+
)
9+
from unblob.models import (
10+
File,
11+
HexString,
12+
StructHandler,
13+
ValidChunk,
14+
)
15+
16+
C_DEFINITIONS = r"""
17+
typedef struct erofs_handler{
18+
uint32_t magic;
19+
uint32_t crc32c;
20+
uint32_t feature_compact;
21+
uint8_t block_size_bs;
22+
uint8_t sb_extslots;
23+
uint16_t root_nid;
24+
uint64_t inos;
25+
uint64_t build_time;
26+
uint32_t build_time_nsec;
27+
uint32_t block_count;
28+
uint32_t meta_blkaddr;
29+
uint32_t xattr_blkaddr;
30+
uint8_t uuid[16];
31+
char volume_name[16];
32+
uint32_t feature_incompact;
33+
char reserved[44];
34+
} erofs_handler_t;
35+
"""
36+
37+
SUPERBLOCK_OFFSET = 0x400
38+
39+
40+
class EROFSHandler(StructHandler):
41+
NAME = "erofs"
42+
PATTERNS = [HexString("e2 e1 f5 e0")] # Magic in little endian
43+
HEADER_STRUCT = "erofs_handler_t"
44+
C_DEFINITIONS = C_DEFINITIONS
45+
EXTRACTOR = Command(
46+
"fsck.erofs",
47+
"--no-preserve",
48+
"--extract={outdir}",
49+
"{inpath}",
50+
)
51+
PATTERN_MATCH_OFFSET = -SUPERBLOCK_OFFSET
52+
53+
def is_valid_header(self, header) -> bool:
54+
return (
55+
header.block_count >= 1
56+
and header.build_time > 0
57+
and str(header.volume_name).isprintable()
58+
)
59+
60+
def calculate_chunk(self, file: File, start_offset: int) -> Optional[ValidChunk]:
61+
file.seek(SUPERBLOCK_OFFSET, io.SEEK_CUR)
62+
header = self.parse_header(file, Endian.LITTLE)
63+
if not self.is_valid_header(header):
64+
raise InvalidInputFormat("Invalid erofs header.")
65+
66+
end_offset = (1 << header.block_size_bs) * header.block_count
67+
return ValidChunk(
68+
start_offset=start_offset,
69+
end_offset=end_offset,
70+
)

‎shell.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ let
44
./flake.lock
55
./flake.nix
66
./overlay.nix
7-
./nix
7+
./package.nix
88
];
99

1010
lock = builtins.fromJSON (builtins.readFile ./flake.lock);

‎tests/integration/filesystem/android/erofs/__input__/foo.erofs.img‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:7f400d3497501c6e68324868d88207aa9398888902414045f4ea498ff26bcdcd
3+
size 4096

‎tests/integration/filesystem/android/erofs/__output__/foo.erofs.img_extract/bar.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:0bfc3200f0152ab9e91e662afd75add3306131f670a1d71539f680c7acdb0a9f
3+
size 13

‎tests/integration/filesystem/android/erofs/__output__/foo.erofs.img_extract/notes/apple.pdf‎

127 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)