Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: decimal map key may be truncated and populated unexpectedly #382

Merged
merged 7 commits into from
Mar 14, 2023
Merged

fix: decimal map key may be truncated and populated unexpectedly #382

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
70188c0
fix:(StreamDecoder) Integer map key may be truncated and populated un…
AsterDY Mar 13, 2023
e5edb29
feat: expose option `DefaultBufferSize`
AsterDY Mar 13, 2023
e426c58
test adjust
AsterDY Mar 13, 2023
41faa8e
test adjust
AsterDY Mar 13, 2023
c942d1b
Merge branch 'fix/stream2' of https://github.com/bytedance/sonic into…
AsterDY Mar 14, 2023
9e610bb
feat: add options to make default buffer size adjustable
AsterDY Mar 14, 2023
253ad48
Merge branch 'main' into fix/stream2
AsterDY Mar 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix:(StreamDecoder) Integer map key may be truncated and populated un… 
…expectedly
  • Loading branch information
@AsterDY
AsterDY committed Mar 13, 2023
commit 70188c00c93de4acc7c9b78dc956ddf32d197c83
20 changes: 17 additions & 3 deletions decoder/assembler_amd64_go116.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1487,18 +1487,21 @@ func (self *_Assembler) _asm_OP_map_init(_ *_Instr) {
func (self *_Assembler) _asm_OP_map_key_i8(p *_Instr) {
self.parse_signed(int8Type, "", p.vi()) // PARSE int8
self.range_signed(_I_int8, _T_int8, math.MinInt8, math.MaxInt8) // RANGE int8
self.match_char(`"`)
self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int8, mapassign, st.Iv
}

func (self *_Assembler) _asm_OP_map_key_i16(p *_Instr) {
self.parse_signed(int16Type, "", p.vi()) // PARSE int16
self.range_signed(_I_int16, _T_int16, math.MinInt16, math.MaxInt16) // RANGE int16
self.match_char(`"`)
self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int16, mapassign, st.Iv
}

func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) {
self.parse_signed(int32Type, "", p.vi()) // PARSE int32
self.range_signed(_I_int32, _T_int32, math.MinInt32, math.MaxInt32) // RANGE int32
self.match_char(`"`)
if vt := p.vt(); !mapfast(vt) {
self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int32, mapassign, st.Iv
} else {
Expand All @@ -1508,6 +1511,7 @@ func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) {

func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) {
self.parse_signed(int64Type, "", p.vi()) // PARSE int64
self.match_char(`"`)
if vt := p.vt(); !mapfast(vt) {
self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int64, mapassign, st.Iv
} else {
Expand All @@ -1519,18 +1523,21 @@ func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) {
func (self *_Assembler) _asm_OP_map_key_u8(p *_Instr) {
self.parse_unsigned(uint8Type, "", p.vi()) // PARSE uint8
self.range_unsigned(_I_uint8, _T_uint8, math.MaxUint8) // RANGE uint8
self.match_char(`"`)
self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint8, vt.Iv
}

func (self *_Assembler) _asm_OP_map_key_u16(p *_Instr) {
self.parse_unsigned(uint16Type, "", p.vi()) // PARSE uint16
self.range_unsigned(_I_uint16, _T_uint16, math.MaxUint16) // RANGE uint16
self.match_char(`"`)
self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint16, vt.Iv
}

func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) {
self.parse_unsigned(uint32Type, "", p.vi()) // PARSE uint32
self.range_unsigned(_I_uint32, _T_uint32, math.MaxUint32) // RANGE uint32
self.match_char(`"`)
if vt := p.vt(); !mapfast(vt) {
self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint32, vt.Iv
} else {
Expand All @@ -1540,6 +1547,7 @@ func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) {

func (self *_Assembler) _asm_OP_map_key_u64(p *_Instr) {
self.parse_unsigned(uint64Type, "", p.vi()) // PARSE uint64
self.match_char(`"`)
if vt := p.vt(); !mapfast(vt) {
self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint64, vt.Iv
} else {
Expand All @@ -1552,11 +1560,13 @@ func (self *_Assembler) _asm_OP_map_key_f32(p *_Instr) {
self.parse_number(float32Type, "", p.vi()) // PARSE NUMBER
self.range_single() // RANGE float32
self.Emit("MOVSS", _X0, _VAR_st_Dv) // MOVSS X0, st.Dv
self.match_char(`"`)
self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv
}

func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) {
self.parse_number(float64Type, "", p.vi()) // PARSE NUMBER
self.match_char(`"`)
self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv
}

Expand Down Expand Up @@ -1787,10 +1797,14 @@ func (self *_Assembler) lspace(subfix string) {
}

func (self *_Assembler) _asm_OP_match_char(p *_Instr) {
self.match_char(p.vb())
}

func (self *_Assembler) match_char(char byte) {
self.check_eof(1)
self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()}
self.Sjmp("JNE" , _LB_char_0_error) // JNE _char_0_error
self.Emit("ADDQ", jit.Imm(1), _IC) // ADDQ $1, IC
self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(char))) // CMPB (IP)(IC), ${p.vb()}
self.Sjmp("JNE" , _LB_char_0_error) // JNE _char_0_error
self.Emit("ADDQ", jit.Imm(1), _IC) // ADDQ $1, IC
}

func (self *_Assembler) _asm_OP_check_char(p *_Instr) {
Expand Down
18 changes: 16 additions & 2 deletions decoder/assembler_amd64_go117.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1482,18 +1482,21 @@ func (self *_Assembler) _asm_OP_map_init(_ *_Instr) {
func (self *_Assembler) _asm_OP_map_key_i8(p *_Instr) {
self.parse_signed(int8Type, "", p.vi()) // PARSE int8
self.range_signed_CX(_I_int8, _T_int8, math.MinInt8, math.MaxInt8) // RANGE int8
self.match_char('"')
self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int8, mapassign, st.Iv
}

func (self *_Assembler) _asm_OP_map_key_i16(p *_Instr) {
self.parse_signed(int16Type, "", p.vi()) // PARSE int16
self.range_signed_CX(_I_int16, _T_int16, math.MinInt16, math.MaxInt16) // RANGE int16
self.match_char('"')
self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int16, mapassign, st.Iv
}

func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) {
self.parse_signed(int32Type, "", p.vi()) // PARSE int32
self.range_signed_CX(_I_int32, _T_int32, math.MinInt32, math.MaxInt32) // RANGE int32
self.match_char('"')
if vt := p.vt(); !mapfast(vt) {
self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int32, mapassign, st.Iv
} else {
Expand All @@ -1504,6 +1507,7 @@ func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) {

func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) {
self.parse_signed(int64Type, "", p.vi()) // PARSE int64
self.match_char('"')
if vt := p.vt(); !mapfast(vt) {
self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int64, mapassign, st.Iv
} else {
Expand All @@ -1515,18 +1519,21 @@ func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) {
func (self *_Assembler) _asm_OP_map_key_u8(p *_Instr) {
self.parse_unsigned(uint8Type, "", p.vi()) // PARSE uint8
self.range_unsigned_CX(_I_uint8, _T_uint8, math.MaxUint8) // RANGE uint8
self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint8, vt.Iv
self.match_char('"')
self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint8, vt.Iv
}

func (self *_Assembler) _asm_OP_map_key_u16(p *_Instr) {
self.parse_unsigned(uint16Type, "", p.vi()) // PARSE uint16
self.range_unsigned_CX(_I_uint16, _T_uint16, math.MaxUint16) // RANGE uint16
self.match_char('"')
self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint16, vt.Iv
}

func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) {
self.parse_unsigned(uint32Type, "", p.vi()) // PARSE uint32
self.range_unsigned_CX(_I_uint32, _T_uint32, math.MaxUint32) // RANGE uint32
self.match_char('"')
if vt := p.vt(); !mapfast(vt) {
self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint32, vt.Iv
} else {
Expand All @@ -1537,6 +1544,7 @@ func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) {

func (self *_Assembler) _asm_OP_map_key_u64(p *_Instr) {
self.parse_unsigned(uint64Type, "", p.vi()) // PARSE uint64
self.match_char('"')
if vt := p.vt(); !mapfast(vt) {
self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint64, vt.Iv
} else {
Expand All @@ -1549,11 +1557,13 @@ func (self *_Assembler) _asm_OP_map_key_f32(p *_Instr) {
self.parse_number(float32Type, "", p.vi()) // PARSE NUMBER
self.range_single_X0() // RANGE float32
self.Emit("MOVSS", _X0, _VAR_st_Dv) // MOVSS X0, st.Dv
self.match_char('"')
self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv
}

func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) {
self.parse_number(float64Type, "", p.vi()) // PARSE NUMBER
self.match_char('"')
self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv
}

Expand Down Expand Up @@ -1767,8 +1777,12 @@ func (self *_Assembler) lspace(subfix string) {
}

func (self *_Assembler) _asm_OP_match_char(p *_Instr) {
self.match_char(p.vb())
}

func (self *_Assembler) match_char(char byte) {
self.check_eof(1)
self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()}
self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(char))) // CMPB (IP)(IC), ${p.vb()}
self.Sjmp("JNE" , _LB_char_0_error) // JNE _char_0_error
self.Emit("ADDQ", jit.Imm(1), _IC) // ADDQ $1, IC
}
Expand Down
10 changes: 0 additions & 10 deletions decoder/compiler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -639,11 +639,6 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op
skip2 := p.pc()
p.rtt(op, vt)

/* match the closing quote if needed */
if op != _OP_map_key_str && op != _OP_map_key_utext && op != _OP_map_key_utext_p {
p.chr(_OP_match_char, '"')
}

/* match the value separator */
p.add(_OP_lspace)
p.chr(_OP_match_char, ':')
Expand All @@ -660,11 +655,6 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op
skip3 := p.pc()
p.rtt(op, vt)

/* match the closing quote if needed */
if op != _OP_map_key_str && op != _OP_map_key_utext && op != _OP_map_key_utext_p {
p.chr(_OP_match_char, '"')
}

/* match the value separator */
p.add(_OP_lspace)
p.chr(_OP_match_char, ':')
Expand Down
51 changes: 51 additions & 0 deletions issue_test/issue381_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/**
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package issue_test

import (
`bytes`
`encoding/json`
`strings`
`testing`

`github.com/bytedance/sonic`
_ `github.com/davecgh/go-spew/spew`
`github.com/stretchr/testify/require`

_ `github.com/stretchr/testify/assert`
)

var decoderBufferSize = 4096

func testSonicUnmarshal(t *testing.T) {
val := strings.Repeat(" ", decoderBufferSize-3) + `{"123":{}}`

res := make(map[int64]map[string]interface{})
res2 := make(map[int64]map[string]interface{})

dec := sonic.ConfigDefault.NewDecoder(bytes.NewBufferString(val))
err := dec.Decode(&res)
require.NoError(t, err)

err2 := json.Unmarshal([]byte(val), &res2)
require.NoError(t, err2)
require.Equal(t, res2, res)
}

func TestStreamxx(t *testing.T) {
testSonicUnmarshal(t)
}