Skip to content

Commit 7342b02

Browse files
jasonpaulosjasonpaulos
committed
Add supportObjectNumberKeys decoding option, and split options for decoding raw string keys/values
1 parent a1addc5 commit 7342b02

File tree

3 files changed

+158
-24
lines changed

3 files changed

+158
-24
lines changed

‎src/Decoder.ts

Lines changed: 53 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,25 @@ export type DecoderOptions<ContextType = undefined> = Readonly<
3333
*
3434
* This is useful if the strings may contain invalid UTF-8 sequences.
3535
*
36-
* Note that this option only applies to string values, not map keys. Additionally, when
37-
* enabled, raw string length is limited by the maxBinLength option.
36+
* When enabled, raw string length is limited by the maxBinLength option.
37+
*
38+
* Note that this option only applies to string values, not map keys. See `rawBinaryStringKeys`
39+
* for map keys.
40+
*/
41+
rawBinaryStringValues: boolean;
42+
43+
/**
44+
* By default, map keys will be decoded as UTF-8 strings. However, if this option is true, map
45+
* keys will be returned as Uint8Arrays without additional decoding.
46+
*
47+
* Requires `useMap` to be true, since plain objects do not support binary keys.
48+
*
49+
* When enabled, raw string length is limited by the maxBinLength option.
50+
*
51+
* Note that this option only applies to map keys, not string values. See `rawBinaryStringValues`
52+
* for string values.
3853
*/
39-
useRawBinaryStrings: boolean;
54+
rawBinaryStringKeys: boolean;
4055

4156
/**
4257
* If true, the decoder will use the Map object to store map values. If false, it will use plain
@@ -48,6 +63,20 @@ export type DecoderOptions<ContextType = undefined> = Readonly<
4863
*/
4964
useMap: boolean;
5065

66+
/**
67+
* If true, the decoder will support decoding numbers as map keys on plain objects. Defaults to
68+
* false.
69+
*
70+
* Note that any numbers used as object keys will be converted to strings, so there is a risk of
71+
* key collision as well as the inability to re-encode the object to the same representation.
72+
*
73+
* This option is ignored if `useMap` is true.
74+
*
75+
* This is useful for backwards compatibility before `useMap` was introduced. Consider instead
76+
* using `useMap` for new code.
77+
*/
78+
supportObjectNumberKeys: boolean;
79+
5180
/**
5281
* Maximum string length.
5382
*
@@ -94,12 +123,12 @@ const STATE_MAP_VALUE = "map_value";
94123

95124
type MapKeyType = string | number | bigint | Uint8Array;
96125

97-
function isValidMapKeyType(key: unknown, useMap: boolean): key is MapKeyType {
126+
function isValidMapKeyType(key: unknown, useMap: boolean, supportObjectNumberKeys: boolean): key is MapKeyType {
98127
if (useMap) {
99128
return typeof key === "string" || typeof key === "number" || typeof key === "bigint" || key instanceof Uint8Array;
100129
}
101130
// Plain objects support a more limited set of key types
102-
return typeof key === "string";
131+
return typeof key === "string" || (supportObjectNumberKeys && typeof key === "number");
103132
}
104133

105134
type StackMapState = {
@@ -229,8 +258,10 @@ export class Decoder<ContextType = undefined> {
229258
private readonly extensionCodec: ExtensionCodecType<ContextType>;
230259
private readonly context: ContextType;
231260
private readonly intMode: IntMode;
232-
private readonly useRawBinaryStrings: boolean;
261+
private readonly rawBinaryStringValues: boolean;
262+
private readonly rawBinaryStringKeys: boolean;
233263
private readonly useMap: boolean;
264+
private readonly supportObjectNumberKeys: boolean;
234265
private readonly maxStrLength: number;
235266
private readonly maxBinLength: number;
236267
private readonly maxArrayLength: number;
@@ -251,15 +282,21 @@ export class Decoder<ContextType = undefined> {
251282
this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined
252283

253284
this.intMode = options?.intMode ?? (options?.useBigInt64 ? IntMode.AS_ENCODED : IntMode.UNSAFE_NUMBER);
254-
this.useRawBinaryStrings = options?.useRawBinaryStrings ?? false;
285+
this.rawBinaryStringValues = options?.rawBinaryStringValues ?? false;
286+
this.rawBinaryStringKeys = options?.rawBinaryStringKeys ?? false;
255287
this.useMap = options?.useMap ?? false;
288+
this.supportObjectNumberKeys = options?.supportObjectNumberKeys ?? false;
256289
this.maxStrLength = options?.maxStrLength ?? UINT32_MAX;
257290
this.maxBinLength = options?.maxBinLength ?? UINT32_MAX;
258291
this.maxArrayLength = options?.maxArrayLength ?? UINT32_MAX;
259292
this.maxMapLength = options?.maxMapLength ?? UINT32_MAX;
260293
this.maxExtLength = options?.maxExtLength ?? UINT32_MAX;
261294
this.keyDecoder = options?.keyDecoder !== undefined ? options.keyDecoder : sharedCachedKeyDecoder;
262295

296+
if (this.rawBinaryStringKeys && !this.useMap) {
297+
throw new Error("rawBinaryStringKeys is only supported when useMap is true");
298+
}
299+
263300
this.stack = new StackPool(this.useMap);
264301
}
265302

@@ -591,8 +628,12 @@ export class Decoder<ContextType = undefined> {
591628
continue DECODE;
592629
}
593630
} else if (state.type === STATE_MAP_KEY) {
594-
if (!isValidMapKeyType(object, this.useMap)) {
595-
const acceptableTypes = this.useMap ? "string, number, bigint, or Uint8Array" : "string";
631+
if (!isValidMapKeyType(object, this.useMap, this.supportObjectNumberKeys)) {
632+
const acceptableTypes = this.useMap
633+
? "string, number, bigint, or Uint8Array"
634+
: this.supportObjectNumberKeys
635+
? "string or number"
636+
: "string";
596637
throw new DecodeError(`The type of key must be ${acceptableTypes} but got ${typeof object}`);
597638
}
598639
if (!this.useMap && object === "__proto__") {
@@ -675,10 +716,10 @@ export class Decoder<ContextType = undefined> {
675716
}
676717

677718
private decodeString(byteLength: number, headerOffset: number): string | Uint8Array {
678-
if (!this.useRawBinaryStrings || (!this.useMap && this.stateIsMapKey())) {
679-
return this.decodeUtf8String(byteLength, headerOffset);
719+
if (this.stateIsMapKey() ? this.rawBinaryStringKeys : this.rawBinaryStringValues) {
720+
return this.decodeBinary(byteLength, headerOffset);
680721
}
681-
return this.decodeBinary(byteLength, headerOffset);
722+
return this.decodeUtf8String(byteLength, headerOffset);
682723
}
683724

684725
private decodeUtf8String(byteLength: number, headerOffset: number): string {

‎test/decode-raw-strings.test.ts

Lines changed: 88 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@ import assert from "assert";
22
import { encode, decode } from "../src";
33
import type { DecoderOptions } from "../src";
44

5-
describe("decode with useRawBinaryStrings specified", () => {
6-
const options = { useRawBinaryStrings: true } satisfies DecoderOptions;
5+
describe("decode with rawBinaryStringValues specified", () => {
6+
const options = { rawBinaryStringValues: true } satisfies DecoderOptions;
77

8-
it("decodes string as binary", () => {
8+
it("decodes string values as binary", () => {
99
const actual = decode(encode("foo"), options);
1010
const expected = Uint8Array.from([0x66, 0x6f, 0x6f]);
1111
assert.deepStrictEqual(actual, expected);
1212
});
1313

14-
it("decodes invalid UTF-8 string as binary", () => {
14+
it("decodes invalid UTF-8 string values as binary", () => {
1515
const invalidUtf8String = Uint8Array.from([
1616
61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176,
1717
184, 221, 66, 188, 171, 36, 135, 121,
@@ -25,18 +25,12 @@ describe("decode with useRawBinaryStrings specified", () => {
2525
assert.deepStrictEqual(actual, invalidUtf8String);
2626
});
2727

28-
it("decodes object keys as strings", () => {
28+
it("decodes map string keys as strings", () => {
2929
const actual = decode(encode({ key: "foo" }), options);
3030
const expected = { key: Uint8Array.from([0x66, 0x6f, 0x6f]) };
3131
assert.deepStrictEqual(actual, expected);
3232
});
3333

34-
it("decodes map keys as binary when useMap is enabled", () => {
35-
const actual = decode(encode({ key: "foo" }), { ...options, useMap: true });
36-
const expected = new Map([[Uint8Array.from([0x6b, 0x65, 0x79]), Uint8Array.from([0x66, 0x6f, 0x6f])]]);
37-
assert.deepStrictEqual(actual, expected);
38-
});
39-
4034
it("ignores maxStrLength", () => {
4135
const lengthLimitedOptions = { ...options, maxStrLength: 1 } satisfies DecoderOptions;
4236

@@ -53,3 +47,86 @@ describe("decode with useRawBinaryStrings specified", () => {
5347
}, /max length exceeded/i);
5448
});
5549
});
50+
51+
describe("decode with rawBinaryStringKeys specified", () => {
52+
const options = { rawBinaryStringKeys: true, useMap: true } satisfies DecoderOptions;
53+
54+
it("errors if useMap is not enabled", () => {
55+
assert.throws(() => {
56+
decode(encode({ key: "foo" }), { rawBinaryStringKeys: true });
57+
}, new Error("rawBinaryStringKeys is only supported when useMap is true"));
58+
});
59+
60+
it("decodes map string keys as binary", () => {
61+
const actual = decode(encode({ key: "foo" }), options);
62+
const expected = new Map([[Uint8Array.from([0x6b, 0x65, 0x79]), "foo"]]);
63+
assert.deepStrictEqual(actual, expected);
64+
});
65+
66+
it("decodes invalid UTF-8 string keys as binary", () => {
67+
const invalidUtf8String = Uint8Array.from([
68+
61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176,
69+
184, 221, 66, 188, 171, 36, 135, 121,
70+
]);
71+
const encodedMap = Uint8Array.from([
72+
129, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247,
73+
19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, 163, 97, 98, 99,
74+
]);
75+
const actual = decode(encodedMap, options);
76+
const expected = new Map([[invalidUtf8String, "abc"]]);
77+
assert.deepStrictEqual(actual, expected);
78+
});
79+
80+
it("decodes string values as strings", () => {
81+
const actual = decode(encode("foo"), options);
82+
const expected = "foo";
83+
assert.deepStrictEqual(actual, expected);
84+
});
85+
86+
it("ignores maxStrLength", () => {
87+
const lengthLimitedOptions = { ...options, maxStrLength: 1 } satisfies DecoderOptions;
88+
89+
const actual = decode(encode({ foo: 1 }), lengthLimitedOptions);
90+
const expected = new Map([[Uint8Array.from([0x66, 0x6f, 0x6f]), 1]]);
91+
assert.deepStrictEqual(actual, expected);
92+
});
93+
94+
it("respects maxBinLength", () => {
95+
const lengthLimitedOptions = { ...options, maxBinLength: 1 } satisfies DecoderOptions;
96+
97+
assert.throws(() => {
98+
decode(encode({ foo: 1 }), lengthLimitedOptions);
99+
}, /max length exceeded/i);
100+
});
101+
});
102+
103+
describe("decode with rawBinaryStringKeys and rawBinaryStringValues", () => {
104+
const options = { rawBinaryStringValues: true, rawBinaryStringKeys: true, useMap: true } satisfies DecoderOptions;
105+
106+
it("errors if useMap is not enabled", () => {
107+
assert.throws(() => {
108+
decode(encode({ key: "foo" }), { rawBinaryStringKeys: true, rawBinaryStringValues: true });
109+
}, new Error("rawBinaryStringKeys is only supported when useMap is true"));
110+
});
111+
112+
it("decodes map string keys and values as binary", () => {
113+
const actual = decode(encode({ key: "foo" }), options);
114+
const expected = new Map([[Uint8Array.from([0x6b, 0x65, 0x79]), Uint8Array.from([0x66, 0x6f, 0x6f])]]);
115+
assert.deepStrictEqual(actual, expected);
116+
});
117+
118+
it("decodes invalid UTF-8 string keys and values as binary", () => {
119+
const invalidUtf8String = Uint8Array.from([
120+
61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176,
121+
184, 221, 66, 188, 171, 36, 135, 121,
122+
]);
123+
const encodedMap = Uint8Array.from([
124+
129, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247,
125+
19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84,
126+
121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121,
127+
]);
128+
const actual = decode(encodedMap, options);
129+
const expected = new Map([[invalidUtf8String, invalidUtf8String]]);
130+
assert.deepStrictEqual(actual, expected);
131+
});
132+
});

‎test/edge-cases.test.ts

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// kind of hand-written fuzzing data
22
// any errors should not break Encoder/Decoder instance states
33
import assert from "assert";
4-
import { encode, decodeAsync, decode, Encoder, Decoder, decodeMulti, decodeMultiStream } from "../src";
4+
import { encode, decodeAsync, decode, Encoder, Decoder, decodeMulti, decodeMultiStream, DecodeError } from "../src";
55
import { DataViewIndexOutOfBoundsError } from "../src/Decoder";
66

77
function testEncoder(encoder: Encoder): void {
@@ -55,6 +55,22 @@ describe("edge cases", () => {
5555
});
5656
});
5757

58+
context("numeric map keys", () => {
59+
const input = encode(new Map([[0, 1]]));
60+
61+
it("throws error by default", () => {
62+
assert.throws(() => decode(input), new DecodeError("The type of key must be string but got number"));
63+
});
64+
65+
it("succeeds with supportObjectNumberKeys", () => {
66+
// note: useMap is the preferred way to decode maps with non-string keys.
67+
// supportObjectNumberKeys is only for backward compatibility
68+
const actual = decode(input, { supportObjectNumberKeys: true });
69+
const expected = { "0": 1 };
70+
assert.deepStrictEqual(actual, expected);
71+
});
72+
});
73+
5874
context("try to decode a map with non-string keys (asynchronous)", () => {
5975
it("throws errors", async () => {
6076
const decoder = new Decoder();

0 commit comments

Comments
 (0)