fix: support decode json-encoded base64 in JIT (#329)

Co-authored-by: liuqiang <liuqiang.06@bytedance.com>

decoder/assembler_amd64_go116.go

@@ -89,6 +89,10 @@ const (
     _BM_space = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n')
 )
 
+const (
+    _MODE_JSON = 1 << 3 // base64 mode
+)
+
 const (
     _LB_error           = "_error"
     _LB_im_error        = "_im_error"
@@ -1231,7 +1235,7 @@ func (self *_Assembler) _asm_OP_bin(_ *_Instr) {
     self.malloc(_SI, _SI)                               // MALLOC SI, SI
 
     // TODO: due to base64x's bug, only use AVX mode now
-    self.Emit("XORL", _CX, _CX)                     // XORL CX, CX
+    self.Emit("MOVL", jit.Imm(_MODE_JSON), _CX)          //  MOVL $_MODE_JSON, CX
 
     /* call the decoder */
     self.Emit("XORL" , _DX, _DX)                // XORL  DX, DX

decoder/assembler_amd64_go117.go

@@ -89,6 +89,10 @@ const (
     _BM_space = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n')
 )
 
+const (
+    _MODE_JSON = 1 << 3 // base64 mode
+)
+
 const (
     _LB_error           = "_error"
     _LB_im_error        = "_im_error"
@@ -1225,7 +1229,7 @@ func (self *_Assembler) _asm_OP_bin(_ *_Instr) {
     self.malloc_AX(_SI, _SI)                               // MALLOC SI, SI
 
     // TODO: due to base64x's bug, only use AVX mode now
-    self.Emit("XORL", _CX, _CX)                     // XORL CX, CX
+    self.Emit("MOVL", jit.Imm(_MODE_JSON), _CX)          //  MOVL $_MODE_JSON, CX
 
     /* call the decoder */
     self.Emit("XORL" , _DX, _DX)                // XORL  DX, DX

go.mod

@@ -3,10 +3,10 @@ module github.com/bytedance/sonic
 go 1.15
 
 require (
-	github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311
 	github.com/davecgh/go-spew v1.1.1
 	github.com/klauspost/cpuid/v2 v2.0.9
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.8.1
 	github.com/twitchyliquid64/golang-asm v0.15.1
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670
 )

go.sum

@@ -1,5 +1,7 @@
-github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06 h1:1sDoSuDPWzhkdzNVxCxtIaKiAe96ESVPv8coGwc1gZ4=
+github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
+github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
+github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -8,14 +10,20 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

issue_test/issue119_test.go

@@ -0,0 +1,61 @@
+
+/*
+ * Copyright 2022 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package issue_test
+
+ import (
+	 `testing`
+ 
+	 `encoding/json`
+	 . `github.com/bytedance/sonic`
+	 `github.com/stretchr/testify/require`
+ )
+
+ func TestIssue_UnmarshalBase64(t *testing.T) {
+	var obj, stdobj []byte
+	tests := []string {
+		`"xy\r\nzu"`,
+		`"xy\/\/"`,
+		`"\/\/=="`,
+		`"\/\/\u003d\u003d"`,
+		`"\u0030\u0030\u0030\u003d"`,
+	}
+	for _, data := range(tests) {
+		stderr := json.Unmarshal([]byte(data), &stdobj)
+		err := Unmarshal([]byte(data), &obj)
+		require.NoError(t, stderr, data)
+		require.NoError(t, err, data)
+		require.Equal(t, stdobj, obj, data)
+	}
+}
+
+func TestIssue_UnmarshalBase64Error(t *testing.T) {
+	var obj, stdobj []byte
+	tests := []string {
+		`"xy\r\nzu0==="`,
+		`"xy\/\/`,
+		`"\/\/==`,
+		`"\/\/\u003d0\u003d"`,
+		`"\u0030\u0030\u0030\u003d`,
+	}
+	for _, data := range(tests) {
+		stderr := json.Unmarshal([]byte(data), &stdobj)
+		err := Unmarshal([]byte(data), &obj)
+		require.Equal(t, stderr != nil, err != nil)
+	}
+	var _, _ = obj, stdobj
+}