Skip to content

Commit

Permalink
feat(JavaScript): implement xlang protocol (apache#1487)
Browse files Browse the repository at this point in the history
<!--
**Thanks for contributing to Fury.**

**If this is your first time opening a PR on fury, you can refer to
[CONTRIBUTING.md](https://github.com/apache/incubator-fury/blob/main/CONTRIBUTING.md).**

Contribution Checklist

- The **Apache Fury (incubating)** community has restrictions on the
naming of pr titles. You can also find instructions in
[CONTRIBUTING.md](https://github.com/apache/incubator-fury/blob/main/CONTRIBUTING.md).

- Fury has a strong focus on performance. If the PR you submit will have
an impact on performance, please benchmark it first and provide the
benchmark result here.
-->

## What does this PR do?
1. Remove unsigned types which are removed in xlang protocol.
2. Rename `long int double` to `int64 int32 float64` to ensure
consistency with other languages
3. Implements the float16 writer and float16 reader.
## Related issues


## Does this PR introduce any user-facing change?
The JavaScript implementation has not been released yet, so there are no
user-facing changes.

## Benchmark
There are no fluctuations in the benchmark.

| (index)  | serialize | deserialize |
|----------|-----------|-------------|
| fury     | 116       | 118         |
| protobuf | 25        | 33          |
| json     | 20        | 23          |

---------

Co-authored-by: wangweipeng <weipeng>
theweipeng authored Apr 11, 2024
1 parent 730c713 commit 417f064
Showing 30 changed files with 447 additions and 609 deletions.
120 changes: 96 additions & 24 deletions javascript/packages/fury/lib/classResolver.ts
Original file line number Diff line number Diff line change
@@ -84,35 +84,35 @@ export default class SerializerResolver {
private writeStringIndex: number[] = [];

private registerSerializer(fury: Fury, description: TypeDescription) {
return fury.classResolver.registerSerializerById(description.type, generateSerializer(fury, description));
return fury.classResolver.registerSerializerById(SerializerResolver.getTypeIdByInternalSerializerType(description.type), generateSerializer(fury, description));
}

private initInternalSerializer(fury: Fury) {
this.registerSerializer(fury, Type.string());
this.registerSerializer(fury, Type.array(Type.any()));
this.registerSerializer(fury, Type.map(Type.any(), Type.any()));
this.registerSerializer(fury, Type.bool());
this.registerSerializer(fury, Type.uint8());
this.registerSerializer(fury, Type.int8());
this.registerSerializer(fury, Type.uint16());
this.registerSerializer(fury, Type.int16());
this.registerSerializer(fury, Type.uint32());
this.registerSerializer(fury, Type.int32());
this.registerSerializer(fury, Type.uint64());
this.registerSerializer(fury, Type.varInt32());
this.registerSerializer(fury, Type.int64());
this.registerSerializer(fury, Type.float());
this.registerSerializer(fury, Type.double());
this.registerSerializer(fury, Type.sliInt64());
this.registerSerializer(fury, Type.float16());
this.registerSerializer(fury, Type.float32());
this.registerSerializer(fury, Type.float64());
this.registerSerializer(fury, Type.timestamp());
this.registerSerializer(fury, Type.date());
this.registerSerializer(fury, Type.duration());
this.registerSerializer(fury, Type.set(Type.any()));
this.registerSerializer(fury, Type.binary());
this.registerSerializer(fury, Type.stringTypedArray());
this.registerSerializer(fury, Type.boolTypedArray());
this.registerSerializer(fury, Type.shortTypedArray());
this.registerSerializer(fury, Type.intTypedArray());
this.registerSerializer(fury, Type.longTypedArray());
this.registerSerializer(fury, Type.floatTypedArray());
this.registerSerializer(fury, Type.doubleTypedArray());
this.registerSerializer(fury, Type.boolArray());
this.registerSerializer(fury, Type.int8Array());
this.registerSerializer(fury, Type.int16Array());
this.registerSerializer(fury, Type.int32Array());
this.registerSerializer(fury, Type.int64Array());
this.registerSerializer(fury, Type.float16Array());
this.registerSerializer(fury, Type.float32Array());
this.registerSerializer(fury, Type.float64Array());
}

init(fury: Fury) {
@@ -124,7 +124,11 @@ export default class SerializerResolver {
this.writeStringIndex.fill(-1);
}

getSerializerById(id: InternalSerializerType) {
getSerializerByType(type: InternalSerializerType) {
return this.internalSerializer[SerializerResolver.getTypeIdByInternalSerializerType(type)];
}

getSerializerById(id: number) {
return this.internalSerializer[id];
}

@@ -213,37 +217,105 @@ export default class SerializerResolver {
}

if (typeof v === "number") {
return this.getSerializerById(InternalSerializerType.DOUBLE);
return this.getSerializerByType(InternalSerializerType.FLOAT64);
}

if (typeof v === "bigint") {
return this.getSerializerById(InternalSerializerType.INT64);
return this.getSerializerByType(InternalSerializerType.INT64);
}

if (typeof v === "boolean") {
return this.getSerializerById(InternalSerializerType.BOOL);
return this.getSerializerByType(InternalSerializerType.BOOL);
}

if (v instanceof Date) {
return this.getSerializerById(InternalSerializerType.TIMESTAMP);
return this.getSerializerByType(InternalSerializerType.TIMESTAMP);
}

if (typeof v === "string") {
return this.getSerializerById(InternalSerializerType.STRING);
return this.getSerializerByType(InternalSerializerType.STRING);
}

if (v instanceof Map) {
return this.getSerializerById(InternalSerializerType.MAP);
return this.getSerializerByType(InternalSerializerType.MAP);
}

if (v instanceof Set) {
return this.getSerializerById(InternalSerializerType.FURY_SET);
return this.getSerializerByType(InternalSerializerType.SET);
}

if (Array.isArray(v)) {
return this.getSerializerById(InternalSerializerType.ARRAY);
return this.getSerializerByType(InternalSerializerType.ARRAY);
}

throw new Error(`Failed to detect the Fury type from JavaScript type: ${typeof v}`);
}

static getTypeIdByInternalSerializerType(type: InternalSerializerType) {
switch (type) {
case InternalSerializerType.BOOL:
return 1;
case InternalSerializerType.INT8:
return 2;
case InternalSerializerType.INT16:
return 3;
case InternalSerializerType.INT32:
return 4;
case InternalSerializerType.VAR_INT32:
return 5;
case InternalSerializerType.INT64:
return 6;
case InternalSerializerType.VAR_INT64:
return 7;
case InternalSerializerType.SLI_INT64:
return 8;
case InternalSerializerType.FLOAT16:
return 9;
case InternalSerializerType.FLOAT32:
return 10;
case InternalSerializerType.FLOAT64:
return 11;
case InternalSerializerType.STRING:
return 12;
case InternalSerializerType.ENUM:
return 13;
case InternalSerializerType.LIST:
return 14;
case InternalSerializerType.SET:
return 15;
case InternalSerializerType.MAP:
return 16;
case InternalSerializerType.DURATION:
return 17;
case InternalSerializerType.TIMESTAMP:
return 18;
case InternalSerializerType.DECIMAL:
return 19;
case InternalSerializerType.BINARY:
return 20;
case InternalSerializerType.TUPLE:
case InternalSerializerType.ARRAY:
return 21;
case InternalSerializerType.BOOL_ARRAY:
return 22;
case InternalSerializerType.INT8_ARRAY:
return 23;
case InternalSerializerType.INT16_ARRAY:
return 24;
case InternalSerializerType.INT32_ARRAY:
return 25;
case InternalSerializerType.INT64_ARRAY:
return 26;
case InternalSerializerType.FLOAT16_ARRAY:
return 27;
case InternalSerializerType.FLOAT32_ARRAY:
return 28;
case InternalSerializerType.FLOAT64_ARRAY:
return 29;
case InternalSerializerType.OBJECT: // todo
return 256;
default:
throw new Error(`typeId is not assigned to type ${InternalSerializerType[type]}`);
}
}
}
Loading

0 comments on commit 417f064

Please sign in to comment.