Skip to content

Commit 690aa66

Browse files
andreypfauandreypfau
committed
BoC decode options
Signed-off-by: andreypfau <andreypfau@ton.org>
1 parent cacb98c commit 690aa66

File tree

8 files changed

+320
-139
lines changed

8 files changed

+320
-139
lines changed

‎bitstring/src/BitString.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public class BitString internal constructor(
3535
size = if (data.isEmpty()) 0 else data.size * 8 - data.last().countTrailingZeroBits() - 1
3636
)
3737

38-
public constructor(data: ByteArray, size: Int) : this(data.copyOf(), size, 0)
38+
public constructor(data: ByteArray, size: Int) : this(data.copyOf((size + 7) ushr 3), size, 0)
3939

4040
override fun equals(other: Any?): Boolean {
4141
if (this === other) return true

‎cell/src/boc/BagOfCells.kt‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,53 @@
11
package org.ton.sdk.cell.boc
22

33
import org.ton.sdk.cell.Cell
4+
import kotlin.jvm.JvmOverloads
5+
import kotlin.jvm.JvmStatic
46

57
public abstract class BagOfCells {
68
public abstract fun getRootCell(index: Int): Cell
9+
10+
public class DecodeOptions internal constructor(
11+
public val checkHashes: Boolean,
12+
public val lazyLoad: Boolean,
13+
public val checkCrc32c: Boolean,
14+
) {
15+
public class Builder {
16+
public var checkHashes: Boolean = true
17+
18+
public var lazyLoad: Boolean = false
19+
20+
public var checkCrc32c: Boolean = true
21+
22+
public fun build(): DecodeOptions {
23+
return DecodeOptions(
24+
checkHashes = checkHashes,
25+
lazyLoad = lazyLoad,
26+
checkCrc32c = checkCrc32c,
27+
)
28+
}
29+
}
30+
31+
public companion object {
32+
public val Default: DecodeOptions = DecodeOptions(
33+
checkHashes = true,
34+
lazyLoad = false,
35+
checkCrc32c = true
36+
)
37+
}
38+
}
39+
40+
public companion object {
41+
@JvmStatic
42+
@JvmOverloads
43+
public fun decodeFromByteArray(
44+
byteArray: ByteArray,
45+
options: DecodeOptions = DecodeOptions.Default
46+
): Array<Cell> {
47+
val boc = StaticBagOfCells(byteArray, options)
48+
return Array(boc.header.rootCount) {
49+
boc.getRootCell(it)
50+
}
51+
}
52+
}
753
}

‎cell/src/boc/MemoryBagOfCells.kt‎

Lines changed: 94 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -3,29 +3,29 @@ package org.ton.sdk.cell.boc
33
import kotlinx.atomicfu.locks.reentrantLock
44
import kotlinx.atomicfu.locks.withLock
55
import kotlinx.io.*
6-
import kotlinx.io.bytestring.ByteString
7-
import org.ton.sdk.bitstring.BitString
8-
import org.ton.sdk.bitstring.unsafe.UnsafeBitStringOperations
6+
import kotlinx.io.unsafe.UnsafeBufferOperations
7+
import kotlinx.io.unsafe.withData
98
import org.ton.sdk.cell.*
10-
import org.ton.sdk.cell.boc.internal.ByteArrayRandomAccessStorage
11-
import org.ton.sdk.cell.boc.internal.ByteStringRandomAccessSource
12-
import org.ton.sdk.cell.boc.internal.RandomAccessSource
13-
import org.ton.sdk.cell.boc.internal.readLong
9+
import org.ton.sdk.cell.boc.internal.*
1410
import org.ton.sdk.cell.internal.DataCell
15-
import org.ton.sdk.crypto.HashBytes
11+
import org.ton.sdk.cell.internal.ExtCell
12+
import org.ton.sdk.crypto.CRC32C
13+
import kotlin.math.min
1614

1715
private const val HASH_BYTES = 32
1816
private const val DEPTH_BYTES = 2
1917

2018
public class StaticBagOfCells internal constructor(
21-
private val source: RandomAccessSource,
22-
private val checkHashes: Boolean = true
19+
private val source: SeekableRawSource,
20+
private val options: DecodeOptions
2321
) : BagOfCells(), CellContext {
24-
public constructor(byteArray: ByteArray) : this(ByteArrayRandomAccessStorage(byteArray))
25-
public constructor(byteString: ByteString) : this(ByteStringRandomAccessSource(byteString))
22+
public constructor(byteArray: ByteArray, decodeOptions: DecodeOptions) : this(
23+
ByteArraySeekableRawSource(byteArray),
24+
decodeOptions
25+
)
2626

2727
public val header: BagOfCellsHeader by lazy {
28-
readHeader()
28+
loadHeader()
2929
}
3030
private val cachedCells = CellCache()
3131
private val cachedLocations = ArrayList<CellLocation>()
@@ -39,42 +39,14 @@ public class StaticBagOfCells internal constructor(
3939
return RootCell(this, index, dataCell)
4040
}
4141

42-
private fun loadSerializedCell(index: Int): Cell {
43-
val cellLocation = cachedLocations.getOrElse(index) {
44-
getCellLocation(index)
45-
}
46-
val buffer = Buffer()
47-
48-
var descriptor = cellLocation.descriptor
49-
if (descriptor == null) {
50-
source.position = cellLocation.start
51-
source.readAtMostTo(buffer, 2)
52-
val d1 = buffer.readByte()
53-
val d2 = buffer.readByte()
54-
descriptor = CellDescriptor(d1, d2)
55-
}
56-
source.position = cellLocation.start + 2
42+
private fun deserializeCell(index: Int, source: Source, descriptor: CellDescriptor, shouldCache: Boolean): Cell {
5743
require(descriptor.referenceCount in 0..4) {
5844
"Invalid BOC cell #$index has invalid reference count: ${descriptor.referenceCount}"
5945
}
6046

61-
fun loadBits(): BitString {
62-
val dataLength = descriptor.byteLength
63-
source.readAtMostTo(buffer, dataLength.toLong())
64-
val data = buffer.readByteArray(dataLength)
65-
val bitLength = if (descriptor.isAligned) {
66-
data.size * 8
67-
} else {
68-
data.size * 8 - data.last().countTrailingZeroBits() - 1
69-
}
70-
@Suppress("OPT_IN_USAGE")
71-
return UnsafeBitStringOperations.wrapUnsafe(data, bitLength)
72-
}
73-
7447
fun loadRefs(): List<Cell> {
7548
return List(descriptor.referenceCount) {
76-
source.readAtMostTo(buffer, header.refByteSize.toLong())
77-
val refIndex = buffer.readLong(header.refByteSize).toInt()
49+
val refIndex = source.readLong(header.refByteSize).toInt()
7850
check(refIndex < header.cellCount) {
7951
"Invalid BOC cell #$index refers ($it) to cell #$refIndex which too big, cellCount=${header.cellCount}"
8052
}
@@ -86,34 +58,22 @@ public class StaticBagOfCells internal constructor(
8658
}
8759

8860
val hashHashes = descriptor.hasHashes
89-
val hashes: Array<HashBytes>
90-
val depths: IntArray
9161
val cell = if (hashHashes) {
92-
val hashCount = descriptor.levelMask.hashCount
93-
source.readAtMostTo(buffer, (HASH_BYTES + 2L) * hashCount)
94-
hashes = Array(hashCount) {
95-
HashBytes(buffer.readByteString(HASH_BYTES))
96-
}
97-
depths = IntArray(hashCount) {
98-
buffer.readUShort().toInt()
99-
}
100-
101-
val bits = loadBits()
62+
val hashes = source.readHashes(descriptor.levelMask)
63+
val depths = source.readDepths(descriptor.levelMask)
64+
val bits = source.readBits(descriptor)
10265
val refs = loadRefs()
103-
10466
val cell = DataCell(descriptor, bits, refs, hashes, depths)
105-
if (checkHashes) {
67+
if (options.checkHashes) {
10668
val expectedCell = with(CellBuilder()) {
10769
store(bits)
10870
refs.forEach {
10971
storeReference(it)
11072
}
11173
build(descriptor.isExotic)
11274
}
113-
check(expectedCell.descriptor == cell.descriptor) {
114-
"Invalid BOC cell #$index descriptor: expected=${expectedCell.descriptor}, found=${cell.descriptor}"
115-
}
11675
for (level in 0 until descriptor.levelMask.level) {
76+
if (level !in descriptor.levelMask) continue
11777
val expectedDepth = expectedCell.depth(level)
11878
val actualDepth = cell.depth(level)
11979
check(expectedDepth == actualDepth) {
@@ -126,12 +86,10 @@ public class StaticBagOfCells internal constructor(
12686
"Invalid BOC cell #$index hash at level $level: expected=$expectedHash, found=$actualHash"
12787
}
12888
}
129-
expectedCell
130-
} else {
131-
cell
13289
}
90+
cell
13391
} else {
134-
val bits = loadBits()
92+
val bits = source.readBits(descriptor)
13593
val refs = loadRefs()
13694
with(CellBuilder()) {
13795
store(bits)
@@ -142,20 +100,57 @@ public class StaticBagOfCells internal constructor(
142100
}
143101
}
144102

145-
if (cellLocation.shouldCache) {
103+
if (shouldCache) {
146104
cachedCells[index] = cell
147105
}
148106
return cell
149107
}
150108

109+
private fun deserializeAnyCell(index: Int, source: Source, descriptor: CellDescriptor, shouldCache: Boolean): Cell {
110+
if (options.lazyLoad && descriptor.hasHashes) {
111+
return ExtCell(descriptor, loader = { loadSerializedCell(index) })
112+
}
113+
return deserializeCell(index, source, descriptor, shouldCache)
114+
}
115+
116+
private fun loadSerializedCell(index: Int): Cell {
117+
cachedCells[index]?.let { return it }
118+
val cellLocation = getCellLocation(index)
119+
source.position = cellLocation.start
120+
val buffer = source.buffer(cellLocation.end - cellLocation.start)
121+
122+
var descriptor = cellLocation.descriptor
123+
if (descriptor == null) {
124+
val d1 = buffer.readByte()
125+
val d2 = buffer.readByte()
126+
descriptor = CellDescriptor(d1, d2)
127+
} else {
128+
buffer.skip(2)
129+
}
130+
return deserializeCell(index, buffer, descriptor, cellLocation.shouldCache)
131+
}
132+
151133
private fun loadAnyCell(index: Int): Cell {
152-
return loadSerializedCell(index)
134+
cachedCells[index]?.let { return it }
135+
val cellLocation = getCellLocation(index)
136+
source.position = cellLocation.start
137+
val buffer = source.buffer(cellLocation.end - cellLocation.start)
138+
var descriptor = cellLocation.descriptor
139+
if (descriptor == null) {
140+
val d1 = buffer.readByte()
141+
val d2 = buffer.readByte()
142+
descriptor = CellDescriptor(d1, d2)
143+
} else {
144+
buffer.skip(2)
145+
}
146+
return deserializeAnyCell(index, buffer, descriptor, cellLocation.shouldCache)
153147
}
154148

155149
private fun getCellLocation(index: Int): CellLocation {
156150
require(index in 0 until header.cellCount) {
157151
"Invalid cell index: $index, cellCount=${header.cellCount}"
158152
}
153+
cachedLocations.getOrNull(index)?.let { return it }
159154
if (!header.hasIndex) {
160155
println("Warning: BOC does not have index, using linear search for cell #$index")
161156
if (index < cachedLocations.size) {
@@ -178,6 +173,7 @@ public class StaticBagOfCells internal constructor(
178173
}
179174

180175
fun Source.loadIndexOffset(index: Int): Long {
176+
if (index < 0) return 0
181177
source.position = header.indexOffset + (index.toLong() * header.offsetByteSize)
182178
return readLong(header.offsetByteSize)
183179
}
@@ -210,10 +206,37 @@ public class StaticBagOfCells internal constructor(
210206
return buffer.readLong(header.refByteSize).toInt()
211207
}
212208

213-
@OptIn(ExperimentalStdlibApi::class)
214-
private fun readHeader(): BagOfCellsHeader {
209+
@OptIn(InternalIoApi::class, UnsafeIoApi::class)
210+
private fun loadHeader(): BagOfCellsHeader {
215211
source.position = 0
216-
return BagOfCellsHeader.parse(source.buffered())
212+
val buffer = source.buffered()
213+
if (!options.checkCrc32c) {
214+
return BagOfCellsHeader.parse(buffer)
215+
}
216+
val header = BagOfCellsHeader.parse(buffer.peek())
217+
if (header.hasCrc32c) {
218+
val crc32c = CRC32C()
219+
val dataSize = header.totalSize - 4
220+
var remaining = dataSize
221+
UnsafeBufferOperations.forEachSegment(buffer.buffer) { ctx, segment ->
222+
ctx.withData(segment) { data, startIndex, endIndex ->
223+
val byteCount = endIndex - startIndex
224+
if (remaining <= 0) return@withData
225+
val toRead = min(remaining, byteCount.toLong()).toInt()
226+
crc32c.update(data, startIndex, startIndex + toRead)
227+
remaining -= toRead
228+
}
229+
}
230+
buffer.skip(dataSize)
231+
val expectedCrc32c = buffer.readIntLe()
232+
val actualCrc32c = crc32c.intDigest()
233+
check(expectedCrc32c == actualCrc32c) {
234+
"Invalid BOC CRC32C: expected=${expectedCrc32c.toUInt().toString(16)}, found=${
235+
actualCrc32c.toUInt().toString(16)
236+
}"
237+
}
238+
}
239+
return header
217240
}
218241

219242
override fun toString(): String = "StaticBagOfCells(header=$header)"
@@ -225,12 +248,13 @@ public class StaticBagOfCells internal constructor(
225248
is LoadedCell -> cell
226249
is RootCell -> toLoadedCell(cell.cell)
227250
is BocCell -> toLoadedCell(cell.cell)
251+
is ExtCell -> toLoadedCell(cell.cell)
228252
else -> throw IllegalArgumentException("Can't load ${cell::class} $cell")
229253
}
230254
}
231255

232256
override fun finalizeCell(builder: CellBuilder): Cell {
233-
TODO("Not yet implemented")
257+
return CellContext.EMPTY.finalizeCell(builder)
234258
}
235259

236260
public class RootCell(

0 commit comments

Comments
 (0)