feat (encoder): add encoder option NoNullSliceOrMap (#218)

* feat (encoder): add encoder option `NoNullSliceOrMap`

* feat: add option on `sonic.Config`

* build: specify `self-host` to x64 machine

.github/workflows/license-check.yml

@@ -4,11 +4,11 @@ on: push
 
 jobs:
   build:
-    runs-on: self-hosted
+    runs-on: [self-hosted, X64]
     steps:
       - uses: actions/checkout@v2
 
       - name: Check License Header
-        uses: apache/skywalking-eyes@main
+        uses: apache/skywalking-eyes/header@main
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

api.go

@@ -31,6 +31,7 @@ import (
     UseUnicodeErrors              bool
     DisallowUnknownFields         bool
     CopyString                    bool
+    NoNullSliceOrMap              bool
  }
  
  var (

encode_test.go

@@ -19,13 +19,13 @@
 package sonic
 
 import (
-    `os`
     `bytes`
     `encoding`
     `encoding/json`
     `fmt`
     `log`
     `math`
+    `os`
     `reflect`
     `regexp`
     `runtime`
@@ -36,6 +36,7 @@ import (
     `unsafe`
 
     `github.com/bytedance/sonic/encoder`
+    `github.com/stretchr/testify/assert`
 )
 
 var (
@@ -1153,3 +1154,18 @@ func TestMarshalerError(t *testing.T) {
         }
     }
 }
+
+func TestMarshalNullNil(t *testing.T) {
+    var v = struct {
+        A []int
+        B map[string]int
+    }{}
+    o, e := Marshal(v)
+    assert.Nil(t, e)
+    assert.Equal(t, `{"A":null,"B":null}`, string(o))
+    o, e = Config{
+        NoNullSliceOrMap: true,
+    }.Froze().Marshal(v)
+    assert.Nil(t, e)
+    assert.Equal(t, `{"A":[],"B":{}}`, string(o))
+}

encoder/assembler_amd64_go116.go

@@ -89,11 +89,13 @@ const (
 )
 
 const (
-    _IM_null = 0x6c6c756e           // 'null'
+    _IM_null   = 0x6c6c756e           // 'null'
-    _IM_true = 0x65757274           // 'true'
+    _IM_true   = 0x65757274           // 'true'
-    _IM_fals = 0x736c6166           // 'fals' ('false' without the 'e')
+    _IM_fals   = 0x736c6166           // 'fals' ('false' without the 'e')
-    _IM_open = 0x00225c22           // '"\"∅'
+    _IM_open   = 0x00225c22           // '"\"∅'
-    _IM_mulv = -0x5555555555555555
+    _IM_array  = 0x5d5b               // '[]'
+    _IM_object = 0x7d7b               // '{}'
+    _IM_mulv   = -0x5555555555555555
 )
 
 const (
@@ -204,6 +206,8 @@ func (self *_Assembler) compile() {
 
 var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
     _OP_null           : (*_Assembler)._asm_OP_null,
+    _OP_empty_arr      : (*_Assembler)._asm_OP_empty_arr,
+    _OP_empty_obj      : (*_Assembler)._asm_OP_empty_obj, 
     _OP_bool           : (*_Assembler)._asm_OP_bool,
     _OP_i8             : (*_Assembler)._asm_OP_i8,
     _OP_i16            : (*_Assembler)._asm_OP_i16,
@@ -767,6 +771,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) {
     self.Emit("ADDQ", jit.Imm(4), _RL)                             // ADDQ $4, RL
 }
 
+func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) {
+    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
+    self.Sjmp("JC", "_empty_arr_{n}")
+    self._asm_OP_null(nil)
+    self.Sjmp("JMP", "_empty_arr_end_{n}")
+    self.Link("_empty_arr_{n}")
+    self.check_size(2)
+    self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0)) 
+    self.Emit("ADDQ", jit.Imm(2), _RL)    
+    self.Link("_empty_arr_end_{n}")                  
+}
+
+func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) {
+    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
+    self.Sjmp("JC", "_empty_obj_{n}")
+    self._asm_OP_null(nil)
+    self.Sjmp("JMP", "_empty_obj_end_{n}")
+    self.Link("_empty_obj_{n}")
+    self.check_size(2)
+    self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))  
+    self.Emit("ADDQ", jit.Imm(2), _RL) 
+    self.Link("_empty_obj_end_{n}")                                             
+}
+
 func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
     self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0))                // CMPB (SP.p), $0
     self.Sjmp("JE"  , "_false_{n}")                                 // JE   _false_{n}

encoder/assembler_amd64_go117.go

@@ -91,11 +91,13 @@ const (
 )
 
 const (
-    _IM_null = 0x6c6c756e           // 'null'
+    _IM_null   = 0x6c6c756e           // 'null'
-    _IM_true = 0x65757274           // 'true'
+    _IM_true   = 0x65757274           // 'true'
-    _IM_fals = 0x736c6166           // 'fals' ('false' without the 'e')
+    _IM_fals   = 0x736c6166           // 'fals' ('false' without the 'e')
-    _IM_open = 0x00225c22           // '"\"∅'
+    _IM_open   = 0x00225c22           // '"\"∅'
-    _IM_mulv = -0x5555555555555555
+    _IM_array  = 0x5d5b               // '[]'
+    _IM_object = 0x7d7b               // '{}'
+    _IM_mulv   = -0x5555555555555555
 )
 
 const (
@@ -130,16 +132,16 @@ var (
 )
 
 var (
-	_ST = jit.Reg("R15")     // can't use R14 since it's always scratched by Go...
+    _ST = jit.Reg("R15")     // can't use R14 since it's always scratched by Go...
-	_RP = jit.Reg("DI")
+    _RP = jit.Reg("DI")
-	_RL = jit.Reg("SI")
+    _RL = jit.Reg("SI")
-	_RC = jit.Reg("DX")
+    _RC = jit.Reg("DX")
 )
 
 var (
     _LR = jit.Reg("R9")
-	_ET = jit.Reg("AX")
+    _ET = jit.Reg("AX")
-	_EP = jit.Reg("BX")
+    _EP = jit.Reg("BX")
 )
 
 var (
@@ -209,6 +211,8 @@ func (self *_Assembler) compile() {
 
 var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
     _OP_null           : (*_Assembler)._asm_OP_null,
+    _OP_empty_arr      : (*_Assembler)._asm_OP_empty_arr,
+    _OP_empty_obj      : (*_Assembler)._asm_OP_empty_obj, 
     _OP_bool           : (*_Assembler)._asm_OP_bool,
     _OP_i8             : (*_Assembler)._asm_OP_i8,
     _OP_i16            : (*_Assembler)._asm_OP_i16,
@@ -780,6 +784,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) {
     self.Emit("ADDQ", jit.Imm(4), _RL)                             // ADDQ $4, RL
 }
 
+func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) {
+    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
+    self.Sjmp("JC", "_empty_arr_{n}")
+    self._asm_OP_null(nil)
+    self.Sjmp("JMP", "_empty_arr_end_{n}")
+    self.Link("_empty_arr_{n}")
+    self.check_size(2)
+    self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0)) 
+    self.Emit("ADDQ", jit.Imm(2), _RL)    
+    self.Link("_empty_arr_end_{n}")                  
+}
+
+func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) {
+    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
+    self.Sjmp("JC", "_empty_obj_{n}")
+    self._asm_OP_null(nil)
+    self.Sjmp("JMP", "_empty_obj_end_{n}")
+    self.Link("_empty_obj_{n}")
+    self.check_size(2)
+    self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))  
+    self.Emit("ADDQ", jit.Imm(2), _RL) 
+    self.Link("_empty_obj_end_{n}")                                             
+}
+
 func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
     self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0))                // CMPB (SP.p), $0
     self.Sjmp("JE"  , "_false_{n}")                                 // JE   _false_{n}

encoder/compiler.go

@@ -32,6 +32,8 @@ type _Op uint8
 
 const (
     _OP_null _Op = iota + 1
+    _OP_empty_arr
+    _OP_empty_obj
     _OP_bool
     _OP_i8
     _OP_i16
@@ -93,6 +95,8 @@ const (
 
 var _OpNames = [256]string {
     _OP_null           : "null",
+    _OP_empty_arr      : "empty_arr",
+    _OP_empty_obj      : "empty_obj",
     _OP_bool           : "bool",
     _OP_i8             : "i8",
     _OP_i16            : "i16",
@@ -481,19 +485,19 @@ func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) {
     }
 }
 
-func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, fn func(*_Program, int, reflect.Type)) {
+func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, nil_op _Op, fn func(*_Program, int, reflect.Type)) {
     x := p.pc()
     p.add(_OP_is_nil)
     fn(p, sp, vt)
     e := p.pc()
     p.add(_OP_goto)
     p.pin(x)
-    p.add(_OP_null)
+    p.add(nil_op)
     p.pin(e)
 }
 
 func (self *_Compiler) compilePtr(p *_Program, sp int, vt reflect.Type) {
-    self.compileNil(p, sp, vt, self.compilePtrBody)
+    self.compileNil(p, sp, vt, _OP_null, self.compilePtrBody)
 }
 
 func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) {
@@ -505,7 +509,7 @@ func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) {
 }
 
 func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) {
-    self.compileNil(p, sp, vt, self.compileMapBody)
+    self.compileNil(p, sp, vt, _OP_empty_obj, self.compileMapBody)
 }
 
 func (self *_Compiler) compileMapBody(p *_Program, sp int, vt reflect.Type) {
@@ -591,7 +595,7 @@ func (self *_Compiler) compileMapBodyUtextPtr(p *_Program, vk reflect.Type) {
 }
 
 func (self *_Compiler) compileSlice(p *_Program, sp int, vt reflect.Type) {
-    self.compileNil(p, sp, vt, self.compileSliceBody)
+    self.compileNil(p, sp, vt, _OP_empty_arr, self.compileSliceBody)
 }
 
 func (self *_Compiler) compileSliceBody(p *_Program, sp int, vt reflect.Type) {

encoder/encoder.go

@@ -37,6 +37,7 @@ const (
     bitEscapeHTML          
     bitCompactMarshaler
     bitNoQuoteTextMarshaler
+    bitNoNullSliceOrMap
 )
 
 const (
@@ -52,12 +53,17 @@ const (
 
     // CompactMarshaler indicates that the output JSON from json.Marshaler 
     // is always compact and needs no validation 
-    CompactMarshaler   Options = 1 << bitCompactMarshaler
+    CompactMarshaler     Options = 1 << bitCompactMarshaler
 
     // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler 
     // is always escaped string and needs no quoting
     NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler
 
+    // NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',
+    // instead of 'null'
+    NoNullSliceOrMap     Options = 1 << bitNoNullSliceOrMap
+  
+    // CompatibleWithStd is used to be compatible with std encoder.
     CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler
 )
 

encoder/encoder_test.go

@@ -76,6 +76,56 @@ func TestGC(t *testing.T) {
     wg.Wait()
 }
 
+type sample struct {
+    M map[string]interface{}
+    S []interface{}
+    A [0]interface{}
+    MP *map[string]interface{}
+    SP *[]interface{}
+    AP *[0]interface{}
+}
+
+func TestOptionSliceOrMapNoNull(t *testing.T) {
+    obj := sample{}
+    out, err := Encode(obj, NoNullSliceOrMap)
+    if err != nil {
+        t.Fatal(err)
+    }
+    require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
+   
+    obj2 := sample{}
+    out, err = Encode(obj2, 0)
+    if err != nil {
+        t.Fatal(err)
+    }
+    require.Equal(t, `{"M":null,"S":null,"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
+}
+
+func BenchmarkOptionSliceOrMapNoNull(b *testing.B) {
+    b.Run("true", func (b *testing.B) {
+        obj := sample{}
+        _, err := Encode(obj, NoNullSliceOrMap)
+        if err != nil {
+            b.Fatal(err)
+        }
+        b.ResetTimer()
+        for i:=0;i<b.N;i++{
+            _, _ = Encode(obj, NoNullSliceOrMap)
+        }
+    })
+   
+    b.Run("false", func (b *testing.B) {
+        obj2 := sample{}
+        _, err := Encode(obj2, 0)
+        if err != nil {
+            b.Fatal(err)
+        }
+        for i:=0;i<b.N;i++{
+            _, _ = Encode(obj2, 0)
+        }
+    })
+}
+
 func runEncoderTest(t *testing.T, fn func(string)string, exp string, arg string) {
     require.Equal(t, exp, fn(arg))
 }

sonic.go

@@ -85,6 +85,9 @@ func (cfg Config) Froze() API {
     if cfg.CopyString {
         api.decoderOpts |= decoder.OptionCopyString
     }
+    if cfg.NoNullSliceOrMap {
+        api.encoderOpts |= encoder.NoNullSliceOrMap
+    }
     return api
 }