Skip to content

Commit 48ba40b

Browse files
bimoadityarCopilot
bimoadityar
and
Copilot
authored
fix: make validate skip behave more similarly like encoding/json (bytedance#815)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent d55adc4 commit 48ba40b

38 files changed

Lines changed: 63999 additions & 56856 deletions

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ junit.xml
4747
ast/test.out
4848
ast/bench.sh
4949

50-
!testdata/*.json.gz
50+
!testdata/**/*.json.gz
5151
fuzz/testdata
5252
*__debug_bin*
5353
*pprof

‎compat_test.go‎

Lines changed: 197 additions & 158 deletions
Original file line numberDiff line numberDiff line change
@@ -18,189 +18,228 @@ package sonic
1818

1919
import (
2020
"bytes"
21+
"compress/gzip"
2122
"encoding/json"
23+
"io"
24+
"os"
2225
"reflect"
2326
"testing"
2427

28+
"github.com/bytedance/sonic/internal/envs"
2529
"github.com/bytedance/sonic/option"
30+
"github.com/stretchr/testify/assert"
2631
"github.com/stretchr/testify/require"
2732
)
2833

2934
func TestCompatUnmarshalStd(t *testing.T) {
30-
var sobj = map[string]interface{}{}
31-
var jobj = map[string]interface{}{}
32-
var data = []byte(`{"a":1.00000001E-10}`)
33-
var str = string(data)
34-
serr := ConfigStd.UnmarshalFromString(str, &sobj)
35-
jerr := json.Unmarshal(data, &jobj)
36-
require.Equal(t, jerr, serr)
37-
require.Equal(t, jobj, sobj)
38-
data[2] = '0'
39-
require.Equal(t, jobj, sobj)
40-
41-
sobj = map[string]interface{}{}
42-
jobj = map[string]interface{}{}
43-
data = []byte(`{"a":1}`)
44-
cfg := Config{
45-
UseNumber: true,
46-
}.Froze()
47-
serr = cfg.Unmarshal(data, &sobj)
48-
dec := json.NewDecoder(bytes.NewBuffer(data))
49-
dec.UseNumber()
50-
jerr = dec.Decode(&jobj)
51-
require.Equal(t, jerr, serr)
52-
require.Equal(t, jobj, sobj)
53-
54-
x := struct{
55-
A json.Number
56-
B json.Number
57-
}{}
58-
y := struct{
59-
A json.Number
60-
B json.Number
61-
}{}
62-
data = []byte(`{"A":"1", "C":-1, "B":1}`)
63-
cfg = Config{
64-
DisallowUnknownFields: true,
65-
}.Froze()
66-
serr = cfg.Unmarshal(data, &x)
67-
dec = json.NewDecoder(bytes.NewBuffer(data))
68-
dec.UseNumber()
69-
dec.DisallowUnknownFields()
70-
jerr = dec.Decode(&y)
71-
require.Equal(t, jerr, serr)
72-
// require.Equal(t, y, x)
35+
var sobj = map[string]interface{}{}
36+
var jobj = map[string]interface{}{}
37+
var data = []byte(`{"a":1.00000001E-10}`)
38+
var str = string(data)
39+
serr := ConfigStd.UnmarshalFromString(str, &sobj)
40+
jerr := json.Unmarshal(data, &jobj)
41+
require.Equal(t, jerr, serr)
42+
require.Equal(t, jobj, sobj)
43+
data[2] = '0'
44+
require.Equal(t, jobj, sobj)
45+
46+
sobj = map[string]interface{}{}
47+
jobj = map[string]interface{}{}
48+
data = []byte(`{"a":1}`)
49+
cfg := Config{
50+
UseNumber: true,
51+
}.Froze()
52+
serr = cfg.Unmarshal(data, &sobj)
53+
dec := json.NewDecoder(bytes.NewBuffer(data))
54+
dec.UseNumber()
55+
jerr = dec.Decode(&jobj)
56+
require.Equal(t, jerr, serr)
57+
require.Equal(t, jobj, sobj)
58+
59+
x := struct {
60+
A json.Number
61+
B json.Number
62+
}{}
63+
y := struct {
64+
A json.Number
65+
B json.Number
66+
}{}
67+
data = []byte(`{"A":"1", "C":-1, "B":1}`)
68+
cfg = Config{
69+
DisallowUnknownFields: true,
70+
}.Froze()
71+
serr = cfg.Unmarshal(data, &x)
72+
dec = json.NewDecoder(bytes.NewBuffer(data))
73+
dec.UseNumber()
74+
dec.DisallowUnknownFields()
75+
jerr = dec.Decode(&y)
76+
require.Equal(t, jerr, serr)
77+
// require.Equal(t, y, x)
7378
}
7479

7580
func TestCompatMarshalStd(t *testing.T) {
76-
t.Parallel()
77-
var obj = map[string]interface{}{
78-
"c": json.RawMessage(" [ \"<&>\" ] "),
79-
"b": json.RawMessage(" [ ] "),
80-
}
81-
sout, serr := ConfigStd.Marshal(obj)
82-
jout, jerr := json.Marshal(obj)
83-
require.Equal(t, jerr, serr)
84-
require.Equal(t, string(jout), string(sout))
85-
86-
obj = map[string]interface{}{
87-
"a": json.RawMessage(" [} "),
88-
}
89-
sout, serr = ConfigStd.Marshal(obj)
90-
jout, jerr = json.Marshal(obj)
91-
require.NotNil(t, jerr)
92-
require.NotNil(t, serr)
93-
require.Equal(t, string(jout), string(sout))
94-
95-
obj = map[string]interface{}{
96-
"a": json.RawMessage("1"),
97-
}
98-
sout, serr = ConfigStd.MarshalIndent(obj, "xxxx", " ")
99-
jout, jerr = json.MarshalIndent(obj, "xxxx", " ")
100-
require.Equal(t, jerr, serr)
101-
require.Equal(t, string(jout), string(sout))
81+
t.Parallel()
82+
var obj = map[string]interface{}{
83+
"c": json.RawMessage(" [ \"<&>\" ] "),
84+
"b": json.RawMessage(" [ ] "),
85+
}
86+
sout, serr := ConfigStd.Marshal(obj)
87+
jout, jerr := json.Marshal(obj)
88+
require.Equal(t, jerr, serr)
89+
require.Equal(t, string(jout), string(sout))
90+
91+
obj = map[string]interface{}{
92+
"a": json.RawMessage(" [} "),
93+
}
94+
sout, serr = ConfigStd.Marshal(obj)
95+
jout, jerr = json.Marshal(obj)
96+
require.NotNil(t, jerr)
97+
require.NotNil(t, serr)
98+
require.Equal(t, string(jout), string(sout))
99+
100+
obj = map[string]interface{}{
101+
"a": json.RawMessage("1"),
102+
}
103+
sout, serr = ConfigStd.MarshalIndent(obj, "xxxx", " ")
104+
jout, jerr = json.MarshalIndent(obj, "xxxx", " ")
105+
require.Equal(t, jerr, serr)
106+
require.Equal(t, string(jout), string(sout))
102107
}
103108

104109
func TestCompatEncoderStd(t *testing.T) {
105-
var o = map[string]interface{}{
106-
"a": "<>",
107-
"b": json.RawMessage(" [ ] "),
108-
}
109-
var w1 = bytes.NewBuffer(nil)
110-
var w2 = bytes.NewBuffer(nil)
111-
var enc1 = json.NewEncoder(w1)
112-
var enc2 = ConfigStd.NewEncoder(w2)
113-
114-
require.Nil(t, enc1.Encode(o))
115-
require.Nil(t, enc2.Encode(o))
116-
require.Equal(t, w1.String(), w2.String())
117-
118-
enc1.SetEscapeHTML(true)
119-
enc2.SetEscapeHTML(true)
120-
enc1.SetIndent("\n", " ")
121-
enc2.SetIndent("\n", " ")
122-
require.Nil(t, enc1.Encode(o))
123-
require.Nil(t, enc2.Encode(o))
124-
require.Equal(t, w1.String(), w2.String())
125-
126-
enc1.SetEscapeHTML(false)
127-
enc2.SetEscapeHTML(false)
128-
enc1.SetIndent("", "")
129-
enc2.SetIndent("", "")
130-
require.Nil(t, enc1.Encode(o))
131-
require.Nil(t, enc2.Encode(o))
132-
require.Equal(t, w1.String(), w2.String())
110+
var o = map[string]interface{}{
111+
"a": "<>",
112+
"b": json.RawMessage(" [ ] "),
113+
}
114+
var w1 = bytes.NewBuffer(nil)
115+
var w2 = bytes.NewBuffer(nil)
116+
var enc1 = json.NewEncoder(w1)
117+
var enc2 = ConfigStd.NewEncoder(w2)
118+
119+
require.Nil(t, enc1.Encode(o))
120+
require.Nil(t, enc2.Encode(o))
121+
require.Equal(t, w1.String(), w2.String())
122+
123+
enc1.SetEscapeHTML(true)
124+
enc2.SetEscapeHTML(true)
125+
enc1.SetIndent("\n", " ")
126+
enc2.SetIndent("\n", " ")
127+
require.Nil(t, enc1.Encode(o))
128+
require.Nil(t, enc2.Encode(o))
129+
require.Equal(t, w1.String(), w2.String())
130+
131+
enc1.SetEscapeHTML(false)
132+
enc2.SetEscapeHTML(false)
133+
enc1.SetIndent("", "")
134+
enc2.SetIndent("", "")
135+
require.Nil(t, enc1.Encode(o))
136+
require.Nil(t, enc2.Encode(o))
137+
require.Equal(t, w1.String(), w2.String())
133138
}
134139

135140
func TestCompatDecoderStd(t *testing.T) {
136-
var o1 = map[string]interface{}{}
137-
var o2 = map[string]interface{}{}
138-
var s = `{"a":"b"} {"1":"2"} a {}`
139-
var w1 = bytes.NewBuffer([]byte(s))
140-
var w2 = bytes.NewBuffer([]byte(s))
141-
var enc1 = json.NewDecoder(w1)
142-
var enc2 = ConfigStd.NewDecoder(w2)
143-
144-
require.Equal(t, enc1.More(), enc2.More())
145-
require.Nil(t, enc1.Decode(&o1))
146-
require.Nil(t, enc2.Decode(&o2))
147-
require.Equal(t, w1.String(), w2.String())
148-
149-
require.Equal(t, enc1.More(), enc2.More())
150-
require.Nil(t, enc1.Decode(&o1))
151-
require.Nil(t, enc2.Decode(&o2))
152-
require.Equal(t, w1.String(), w2.String())
153-
154-
require.Equal(t, enc1.More(), enc2.More())
155-
require.NotNil(t, enc1.Decode(&o1))
156-
require.NotNil(t, enc2.Decode(&o2))
157-
require.Equal(t, w1.String(), w2.String())
141+
var o1 = map[string]interface{}{}
142+
var o2 = map[string]interface{}{}
143+
var s = `{"a":"b"} {"1":"2"} a {}`
144+
var w1 = bytes.NewBuffer([]byte(s))
145+
var w2 = bytes.NewBuffer([]byte(s))
146+
var enc1 = json.NewDecoder(w1)
147+
var enc2 = ConfigStd.NewDecoder(w2)
148+
149+
require.Equal(t, enc1.More(), enc2.More())
150+
require.Nil(t, enc1.Decode(&o1))
151+
require.Nil(t, enc2.Decode(&o2))
152+
require.Equal(t, w1.String(), w2.String())
153+
154+
require.Equal(t, enc1.More(), enc2.More())
155+
require.Nil(t, enc1.Decode(&o1))
156+
require.Nil(t, enc2.Decode(&o2))
157+
require.Equal(t, w1.String(), w2.String())
158+
159+
require.Equal(t, enc1.More(), enc2.More())
160+
require.NotNil(t, enc1.Decode(&o1))
161+
require.NotNil(t, enc2.Decode(&o2))
162+
require.Equal(t, w1.String(), w2.String())
158163
}
159164

160165
func TestPretouch(t *testing.T) {
161-
var v map[string]interface{}
162-
if err := Pretouch(reflect.TypeOf(v)); err != nil {
163-
t.Errorf("err:%v", err)
164-
}
165-
166-
if err := Pretouch(reflect.TypeOf(v),
167-
option.WithCompileRecursiveDepth(1),
168-
option.WithCompileMaxInlineDepth(2),
169-
); err != nil {
170-
t.Errorf("err:%v", err)
171-
}
166+
var v map[string]interface{}
167+
if err := Pretouch(reflect.TypeOf(v)); err != nil {
168+
t.Errorf("err:%v", err)
169+
}
170+
171+
if err := Pretouch(reflect.TypeOf(v),
172+
option.WithCompileRecursiveDepth(1),
173+
option.WithCompileMaxInlineDepth(2),
174+
); err != nil {
175+
t.Errorf("err:%v", err)
176+
}
172177
}
173178

174179
func TestGet(t *testing.T) {
175-
var data = `{"a":"b"}`
176-
r, err := GetFromString(data, "a")
177-
if err != nil {
178-
t.Fatal(err)
179-
}
180-
v, err := r.String()
181-
if err != nil {
182-
t.Fatal(err)
183-
}
184-
if v != "b" {
185-
t.Fatal(v)
186-
}
180+
var data = `{"a":"b"}`
181+
r, err := GetFromString(data, "a")
182+
if err != nil {
183+
t.Fatal(err)
184+
}
185+
v, err := r.String()
186+
if err != nil {
187+
t.Fatal(err)
188+
}
189+
if v != "b" {
190+
t.Fatal(v)
191+
}
187192
}
188193

189194
func TestUnmarshalWithTrailingChars(t *testing.T) {
190-
for i, str := range([]string {
191-
"123",
192-
"123 ",
193-
"{} [] ",
194-
"{} [ ",
195-
"[],",
196-
"[]null",
197-
"false null",
198-
}) {
199-
var msg1, msg2 json.RawMessage
200-
err1 := json.Unmarshal([]byte(str), &msg1)
201-
err2 := ConfigStd.UnmarshalFromString(str, &msg2)
202-
require.Equal(t, err1 == nil, err2 == nil, i)
203-
// sonic will not clear the unmarshaled value here, but encoding/json will
204-
// require.Equal(t, msg1, msg2)
205-
}
195+
for i, str := range []string{
196+
"123",
197+
"123 ",
198+
"{} [] ",
199+
"{} [ ",
200+
"[],",
201+
"[]null",
202+
"false null",
203+
} {
204+
var msg1, msg2 json.RawMessage
205+
err1 := json.Unmarshal([]byte(str), &msg1)
206+
err2 := ConfigStd.UnmarshalFromString(str, &msg2)
207+
require.Equal(t, err1 == nil, err2 == nil, i)
208+
// sonic will not clear the unmarshaled value here, but encoding/json will
209+
// require.Equal(t, msg1, msg2)
210+
}
211+
}
212+
213+
func TestUnmarshalJSONSuite(t *testing.T) {
214+
if envs.UseOptDec {
215+
t.Skip("this test still fails in OPTDEC") // FIXME: fix the optdec issues
216+
}
217+
218+
gzFile, err := os.Open("testdata/JSONTestSuite/testdata.json.gz")
219+
require.NoError(t, err)
220+
defer gzFile.Close()
221+
222+
gzReader, err := gzip.NewReader(gzFile)
223+
require.NoError(t, err)
224+
defer gzReader.Close()
225+
226+
data, err := io.ReadAll(gzReader)
227+
require.NoError(t, err)
228+
229+
var tests map[string]string
230+
err = json.Unmarshal(data, &tests)
231+
require.NoError(t, err)
232+
233+
for name, tt := range tests {
234+
b := []byte(tt)
235+
t.Run(name, func(t *testing.T) {
236+
serr := ConfigStd.Unmarshal(b, new(json.RawMessage))
237+
jerr := json.Unmarshal(b, new(json.RawMessage))
238+
assert.Equal(t, jerr != nil, serr != nil, "json: %v, sonic: %v", jerr, serr)
239+
240+
serr = ConfigStd.Unmarshal(b, new(interface{}))
241+
jerr = json.Unmarshal(b, new(interface{}))
242+
assert.Equal(t, jerr != nil, serr != nil, "json: %v, sonic: %v", jerr, serr)
243+
})
244+
}
206245
}

0 commit comments

Comments
 (0)