XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-24 02:16:45 +08:00
Code Issues Projects Releases Wiki Activity

feat: ast supports Node.Exists()/MapUseNode()/ArrayUseNode() (#19)

Browse source 
* feat: ast supports Node.Exists()/MapUseNode()/ArrayUseNode()

* jump table for swithcRawType

Co-authored-by: duanyi.aster <duanyi.aster@bytedance.com>
This commit is contained in:
Yi Duan 2021-06-24 16:37:32 +08:00 committed by GitHub
parent 955ab75cf5
commit e9b6634cac
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 656 additions and 224 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -43,3 +43,5 @@ coverage.xml
junit.xml junit.xml
*.profile *.profile
*.svg *.svg
*.out
ast/test.out

53
ast/iterator_test.go
View file

@ -22,7 +22,7 @@ import (
`testing` `testing`
) )
func TestIterator(t *testing.T) { func getTestIteratorSample() (string, int) {
var data []int var data []int
var v1 = "" var v1 = ""
var v2 = "" var v2 = ""
@ -36,7 +36,56 @@ func TestIterator(t *testing.T) {
v2+=`,` v2+=`,`
} }
} }
var str = `{"array":[`+v1+`], "object":{`+v2+`}}` return `{"array":[`+v1+`], "object":{`+v2+`}}`, loop
}
func TestRawIterator(t *testing.T) {
str, loop := getTestIteratorSample()
fmt.Println(str)
root, err := NewSearcher(str).GetByPath("array")
if err != nil {
t.Fatal(err)
}
ai := root.Values()
i := int64(0)
for ai.HasNext() {
v := &Node{}
if !ai.Next(v) {
t.Fatalf("no next")
}
if i < int64(loop) && v.Int64() != i {
t.Fatalf("exp:%v, got:%v", i, v)
}
if i != int64(ai.Pos())-1 || i >= int64(ai.Len()) {
t.Fatal(i)
}
i++
}
root, err = NewSearcher(str).GetByPath("object")
if err != nil {
t.Fatal(err)
}
mi := root.Properties()
i = int64(0)
for mi.HasNext() {
v := &Pair{}
if !mi.Next(v) {
t.Fatalf("no next")
}
if i < int64(loop) &&( v.Value.Int64() != i ||v.Key != fmt.Sprintf("k%d", i)) {
t.Fatalf("exp:%v, got:%v", i, v.Value.Interface())
}
if i != int64(mi.Pos())-1 || i >= int64(mi.Len()) {
t.Fatal(i)
}
i++
}
}
func TestIterator(t *testing.T) {
str, loop := getTestIteratorSample()
fmt.Println(str) fmt.Println(str)
root, err := NewParser(str).Parse() root, err := NewParser(str).Parse()

488
ast/node.go
View file

@ -18,7 +18,6 @@ package ast
import ( import (
`encoding/json` `encoding/json`
`fmt`
`unsafe` `unsafe`
`github.com/bytedance/sonic/internal/native/types` `github.com/bytedance/sonic/internal/native/types`
@ -27,7 +26,6 @@ import (
) )
const ( const (
_MAP_THRESHOLD = 5
_CAP_BITS = 32 _CAP_BITS = 32
_LEN_MASK = 1 << _CAP_BITS - 1 _LEN_MASK = 1 << _CAP_BITS - 1
_APPEND_EXTRA_SIZE = 5 _APPEND_EXTRA_SIZE = 5
@ -37,25 +35,66 @@ const (
) )
const ( const (
V_NODE_BASE types.ValueType = 32768 _V_NONE types.ValueType = 0
V_RAW types.ValueType = 1 << 16 _V_NODE_BASE types.ValueType = 1<<5
V_NUMBER = V_NODE_BASE + 0 _V_LAZY types.ValueType = 1 << 7
V_ARRAY_RAW = V_RAW | types.V_ARRAY _V_RAW types.ValueType = 1 << 8
V_OBJECT_RAW = V_RAW | types.V_OBJECT _V_NUMBER = _V_NODE_BASE + 1
MASK_RAW = V_RAW - 1 _V_ARRAY_LAZY = _V_LAZY | types.V_ARRAY
_V_OBJECT_LAZY = _V_LAZY | types.V_OBJECT
_MASK_LAZY = _V_LAZY - 1
_MASK_RAW = _V_RAW - 1
)
const (
V_NONE = 0
V_NULL = 2
V_TRUE = 3
V_FALSE = 4
V_ARRAY = 5
V_OBJECT = 6
V_STRING = 7
V_NUMBER = int(_V_NUMBER)
) )
type Node struct { type Node struct {
v int64 v int64
t types.ValueType t types.ValueType
p unsafe.Pointer p unsafe.Pointer
m map[string]unsafe.Pointer
} }
/** Node Type Accessor **/ /** Node Type Accessor **/
func (self *Node) Type() types.ValueType { // Type returns json type represented by the node
return self.t & MASK_RAW // It will be one of belows:
// V_NONE = 0
// V_NULL = 2
// V_TRUE = 3
// V_FALSE = 4
// V_ARRAY = 5
// V_OBJECT = 6
// V_STRING = 7
// V_NUMBER = 33
func (self *Node) Type() int {
return int(self.t & _MASK_LAZY & _MASK_RAW)
}
func (self *Node) itype() types.ValueType {
return self.t & _MASK_LAZY & _MASK_RAW
}
// Exists returns false only if the node is nil or got by invalid path
func (self *Node) Exists() bool {
return self != nil && self.t != _V_NONE
}
// IsRaw returns true if node's underlying value is raw json
func (self *Node) IsRaw() bool {
return self.t&_V_RAW != 0
}
func (self *Node) isLazy() bool {
return self.t&_V_LAZY != 0
} }
/** Simple Value Methods **/ /** Simple Value Methods **/
@ -63,94 +102,82 @@ func (self *Node) Type() types.ValueType {
// Raw returns underlying json string of an raw node, // Raw returns underlying json string of an raw node,
// which usually created by Search() api // which usually created by Search() api
func (self *Node) Raw() string { func (self *Node) Raw() string {
if self.t != V_RAW { if !self.IsRaw() {
panic("value cannot be represented as raw json") panic("value cannot be represented as raw json")
} }
return addr2str(self.p, self.v) return addr2str(self.p, self.v)
} }
func (self *Node) checkRaw() {
if !self.IsRaw() {
return
}
*self = self.parseRaw()
}
// Bool returns bool value represented by this node // Bool returns bool value represented by this node
// //
// If node type is not types.V_TRUE or types.V_FALSE, or V_RAW (must be a bool json value) // If node type is not types.V_TRUE or types.V_FALSE, or V_RAW (must be a bool json value)
// it will panic // it will panic
func (self *Node) Bool() bool { func (self *Node) Bool() bool {
self.checkRaw()
switch self.t { switch self.t {
case types.V_TRUE : return true case types.V_TRUE : return true
case types.V_FALSE : return false case types.V_FALSE : return false
case V_RAW :
n := self.parseRaw()
return n.Bool()
default : panic("value cannot be represented as a boolean") default : panic("value cannot be represented as a boolean")
} }
} }
// Int64 as above. // Int64 as above.
func (self *Node) Int64() int64 { func (self *Node) Int64() int64 {
self.checkRaw()
switch self.t { switch self.t {
case V_NUMBER : return numberToInt64(self) case _V_NUMBER : return numberToInt64(self)
case types.V_TRUE : return 1 case types.V_TRUE : return 1
case types.V_FALSE : return 0 case types.V_FALSE : return 0
case V_RAW :
n := self.parseRaw()
return n.Int64()
default : panic("value cannot be represented as an integer") default : panic("value cannot be represented as an integer")
} }
} }
// Number as above. // Number as above.
func (self *Node) Number() json.Number { func (self *Node) Number() json.Number {
self.checkRaw()
switch self.t { switch self.t {
case V_RAW : case _V_NUMBER : return toNumber(self)
n := self.parseRaw()
return n.Number()
case V_NUMBER : return toNumber(self)
default : panic("value cannot be represented as a json.Number") default : panic("value cannot be represented as a json.Number")
} }
} }
// String as above. // String as above.
func (self *Node) String() string { func (self *Node) String() string {
self.checkRaw()
switch self.t { switch self.t {
case V_NUMBER : return toNumber(self).String() case _V_NUMBER : return toNumber(self).String()
case types.V_NULL : return "null" case types.V_NULL : return "null"
case types.V_TRUE : return "true" case types.V_TRUE : return "true"
case types.V_FALSE : return "false" case types.V_FALSE : return "false"
case types.V_STRING : return addr2str(self.p, self.v) case types.V_STRING : return addr2str(self.p, self.v)
case V_RAW :
n := self.parseRaw()
return n.String()
default : panic("value cannot be represented as a simple string") default : panic("value cannot be represented as a simple string")
} }
} }
// Float64 as above. // Float64 as above.
func (self *Node) Float64() float64 { func (self *Node) Float64() float64 {
self.checkRaw()
switch self.t { switch self.t {
case V_NUMBER : return numberToFloat64(self) case _V_NUMBER : return numberToFloat64(self)
case types.V_TRUE : return 1.0 case types.V_TRUE : return 1.0
case types.V_FALSE : return 0.0 case types.V_FALSE : return 0.0
case V_RAW :
n := self.parseRaw()
return n.Float64()
default : panic("value cannot be represented as an integer") default : panic("value cannot be represented as an integer")
} }
} }
// IsRaw returns true if the node is type of below three:
//
// 1. V_RAW (never parsed)
// 2. types.V_Object_RAW (partially parsed)
// 3. types.V_Array_RAW (partially parsed)
func (self *Node) IsRaw() bool {
return self.t&V_RAW != 0
}
/** Sequencial Value Methods **/ /** Sequencial Value Methods **/
// Len returns children count of a array|object|string node // Len returns children count of a array|object|string node
// For partially loaded node, it also works but only counts the parsed children // For partially loaded node, it also works but only counts the parsed children
func (self *Node) Len() int { func (self *Node) Len() int {
if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == V_ARRAY_RAW || self.t == V_OBJECT_RAW { if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == _V_ARRAY_LAZY || self.t == _V_OBJECT_LAZY {
return int(self.v & _LEN_MASK) return int(self.v & _LEN_MASK)
} else if self.t == types.V_STRING { } else if self.t == types.V_STRING {
return int(self.v) return int(self.v)
@ -161,7 +188,7 @@ func (self *Node) Len() int {
// Cap returns malloc capacity of a array|object node for children // Cap returns malloc capacity of a array|object node for children
func (self *Node) Cap() int { func (self *Node) Cap() int {
if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == V_ARRAY_RAW || self.t == V_OBJECT_RAW { if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == _V_ARRAY_LAZY || self.t == _V_OBJECT_LAZY {
return int(self.v >> _CAP_BITS) return int(self.v >> _CAP_BITS)
} else { } else {
panic("value does not have a capacity") panic("value does not have a capacity")
@ -171,22 +198,19 @@ func (self *Node) Cap() int {
// Set sets the given node for the key under object node // Set sets the given node for the key under object node
func (self *Node) Set(key string, node Node) { func (self *Node) Set(key string, node Node) {
p := self.Get(key) p := self.Get(key)
if p == nil { if !p.Exists() {
l := self.Len() l := self.Len()
c := self.Cap() c := self.Cap()
if l == c { if l == c {
// TODO: maybe change append_extra_size in future // TODO: maybe change append_extra_size in future
c += _APPEND_EXTRA_SIZE c += _APPEND_EXTRA_SIZE
mem := unsafe_NewArray(_PAIR_TYPE, c) mem := unsafe_NewArray(_PAIR_TYPE, c)
memmove(mem, self.p, _PAIR_SIZE*uintptr(l)) memmove(mem, self.p, _PAIR_SIZE * uintptr(l))
self.p = mem self.p = mem
} }
v := self.pairAt(l) v := self.pairAt(l)
v.Key = key v.Key = key
v.Value = node v.Value = node
if self.m != nil {
self.m[key] = unsafe.Pointer(&v.Value)
}
self.setCapAndLen(c, l+1) self.setCapAndLen(c, l+1)
} else { } else {
*p = node *p = node
@ -198,7 +222,7 @@ func (self *Node) Set(key string, node Node) {
// The index must within parent array's range // The index must within parent array's range
func (self *Node) SetByIndex(index int, node Node) { func (self *Node) SetByIndex(index int, node Node) {
p := self.Index(index) p := self.Index(index)
if p == nil { if !p.Exists() {
panic("index to nil node") panic("index to nil node")
} else { } else {
*p = node *p = node
@ -215,7 +239,7 @@ func (self *Node) Add(node Node) {
// TODO: maybe change append_extra_size in future // TODO: maybe change append_extra_size in future
c += _APPEND_EXTRA_SIZE c += _APPEND_EXTRA_SIZE
mem := unsafe_NewArray(_NODE_TYPE, c) mem := unsafe_NewArray(_NODE_TYPE, c)
memmove(mem, self.p, _NODE_SIZE*uintptr(l)) memmove(mem, self.p, _NODE_SIZE * uintptr(l))
self.p = mem self.p = mem
} }
v := self.nodeAt(l) v := self.nodeAt(l)
@ -231,8 +255,14 @@ func (self *Node) GetByPath(path ...interface{}) *Node {
switch p.(type) { switch p.(type) {
case int: case int:
s = s.Index(p.(int)) s = s.Index(p.(int))
if !s.Exists() {
return s
}
case string: case string:
s = s.Get(p.(string)) s = s.Get(p.(string))
if !s.Exists() {
return s
}
default: default:
panic("path must be either int or string") panic("path must be either int or string")
} }
@ -255,14 +285,14 @@ func (self *Node) Index(idx int) *Node {
// Values returns iterator for array's children traversal // Values returns iterator for array's children traversal
func (self *Node) Values() ListIterator { func (self *Node) Values() ListIterator {
self.must(types.V_ARRAY, "an array") self.must(types.V_ARRAY, "an array")
self.loadAllIndex() self.skipAllIndex()
return ListIterator{Iterator{p: self}} return ListIterator{Iterator{p: self}}
} }
// Properties returns iterator for object's children traversal // Properties returns iterator for object's children traversal
func (self *Node) Properties() ObjectIterator { func (self *Node) Properties() ObjectIterator {
self.must(types.V_OBJECT, "an object") self.must(types.V_OBJECT, "an object")
self.loadAllKey() self.skipAllKey()
return ObjectIterator{Iterator{p: self}} return ObjectIterator{Iterator{p: self}}
} }
@ -282,6 +312,23 @@ func (self *Node) MapUseNumber() map[string]interface{} {
return self.toGenericObjectUseNumber() return self.toGenericObjectUseNumber()
} }
// MapUseNode scans both parsed and non-parsed chidren nodes,
// and map them by their keys
func (self *Node) MapUseNode() map[string]Node {
self.must(types.V_OBJECT, "an object")
self.skipAllKey()
return self.toGenericObjectUseNode()
}
// MapUnsafe exports the underlying pointer to its children map
// WARN: don't use it unless you know what you are doing
func (self *Node) UnsafeMap() []Pair {
self.must(types.V_OBJECT, "an object")
self.skipAllKey()
s := ptr2slice(self.p, self.Len(), self.Cap())
return *(*[]Pair)(s)
}
// Array loads all indexes of an array node // Array loads all indexes of an array node
func (self *Node) Array() []interface{} { func (self *Node) Array() []interface{} {
self.must(types.V_ARRAY, "an array") self.must(types.V_ARRAY, "an array")
@ -296,10 +343,28 @@ func (self *Node) ArrayUseNumber() []interface{} {
return self.toGenericArrayUseNumber() return self.toGenericArrayUseNumber()
} }
// ArrayUseNode copys both parsed and non-parsed chidren nodes,
// and indexes them by original order
func (self *Node) ArrayUseNode() []Node {
self.must(types.V_ARRAY, "an array")
self.skipAllIndex()
return self.toGenericArrayUseNode()
}
// ArrayUnsafe exports the underlying pointer to its children array
// WARN: don't use it unless you know what you are doing
func (self *Node) UnsafeArray() []Node {
self.must(types.V_ARRAY, "an array")
self.skipAllIndex()
s := ptr2slice(self.p, self.Len(), self.Cap())
return *(*[]Node)(s)
}
// Interface loads all children under all pathes from this node, // Interface loads all children under all pathes from this node,
// and converts itself as generic go type // and converts itself as generic go type
// all numberic nodes are casted to float64 // all numberic nodes are casted to float64
func (self *Node) Interface() interface{} { func (self *Node) Interface() interface{} {
self.checkRaw()
switch self.t { switch self.t {
case types.V_EOF : panic("invalid value") case types.V_EOF : panic("invalid value")
case types.V_NULL : return nil case types.V_NULL : return nil
@ -308,17 +373,13 @@ func (self *Node) Interface() interface{} {
case types.V_ARRAY : return self.toGenericArray() case types.V_ARRAY : return self.toGenericArray()
case types.V_OBJECT : return self.toGenericObject() case types.V_OBJECT : return self.toGenericObject()
case types.V_STRING : return addr2str(self.p, self.v) case types.V_STRING : return addr2str(self.p, self.v)
case V_NUMBER : case _V_NUMBER : return numberToFloat64(self)
return numberToFloat64(self) case _V_ARRAY_LAZY:
case V_ARRAY_RAW:
self.loadAllIndex() self.loadAllIndex()
return self.toGenericArray() return self.toGenericArray()
case V_OBJECT_RAW: case _V_OBJECT_LAZY :
self.loadAllKey() self.loadAllKey()
return self.toGenericObject() return self.toGenericObject()
case V_RAW :
n := self.parseRaw()
return n.Interface()
default : panic("not gonna happen") default : panic("not gonna happen")
} }
} }
@ -326,6 +387,7 @@ func (self *Node) Interface() interface{} {
// InterfaceUseNumber works same with Interface() // InterfaceUseNumber works same with Interface()
// except numberic nodes are casted to json.Number // except numberic nodes are casted to json.Number
func (self *Node) InterfaceUseNumber() interface{} { func (self *Node) InterfaceUseNumber() interface{} {
self.checkRaw()
switch self.t { switch self.t {
case types.V_EOF : panic("invalid value") case types.V_EOF : panic("invalid value")
case types.V_NULL : return nil case types.V_NULL : return nil
@ -334,21 +396,34 @@ func (self *Node) InterfaceUseNumber() interface{} {
case types.V_ARRAY : return self.toGenericArrayUseNumber() case types.V_ARRAY : return self.toGenericArrayUseNumber()
case types.V_OBJECT : return self.toGenericObjectUseNumber() case types.V_OBJECT : return self.toGenericObjectUseNumber()
case types.V_STRING : return addr2str(self.p, self.v) case types.V_STRING : return addr2str(self.p, self.v)
case V_NUMBER : case _V_NUMBER : return toNumber(self)
return toNumber(self) case _V_ARRAY_LAZY :
case V_ARRAY_RAW:
self.loadAllIndex() self.loadAllIndex()
return self.toGenericArrayUseNumber() return self.toGenericArrayUseNumber()
case V_OBJECT_RAW: case _V_OBJECT_LAZY :
self.loadAllKey() self.loadAllKey()
return self.toGenericObjectUseNumber() return self.toGenericObjectUseNumber()
case V_RAW :
n := self.parseRaw()
return n.InterfaceUseNumber()
default : panic("not gonna happen") default : panic("not gonna happen")
} }
} }
// InterfaceUseNode clone itself as a new node,
// or its children as map[string]Node (or []Node)
func (self *Node) InterfaceUseNode() interface{} {
self.checkRaw()
switch self.t {
case types.V_ARRAY : return self.toGenericArrayUseNode()
case types.V_OBJECT : return self.toGenericObjectUseNode()
case _V_ARRAY_LAZY:
self.skipAllIndex()
return self.toGenericArrayUseNode()
case _V_OBJECT_LAZY:
self.skipAllKey()
return self.toGenericObjectUseNode()
default : return *self
}
}
/** Internal Helper Methods **/ /** Internal Helper Methods **/
var ( var (
@ -357,7 +432,7 @@ var (
) )
func (self *Node) setCapAndLen(cap int, len int) { func (self *Node) setCapAndLen(cap int, len int) {
if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == V_ARRAY_RAW || self.t == V_OBJECT_RAW { if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == _V_ARRAY_LAZY || self.t == _V_OBJECT_LAZY {
self.v = int64(len&_LEN_MASK | cap<<_CAP_BITS) self.v = int64(len&_LEN_MASK | cap<<_CAP_BITS)
} else { } else {
panic("value does not have a length") panic("value does not have a length")
@ -373,10 +448,8 @@ func (self *Pair) unsafe_next() *Pair {
} }
func (self *Node) must(t types.ValueType, s string) { func (self *Node) must(t types.ValueType, s string) {
if self.t == V_RAW { self.checkRaw()
*self = self.parseRaw() if self.itype() != t {
}
if self.t&MASK_RAW != t {
panic("value cannot be represented as " + s) panic("value cannot be represented as " + s)
} }
} }
@ -396,21 +469,13 @@ func (self *Node) pairAt(i int) *Pair {
} }
func (self *Node) findKey(key string) *Node { func (self *Node) findKey(key string) *Node {
if self.mapped() {
if n, ok := (self.m)[key]; !ok {
return nil
} else {
return (*Node)(n)
}
}
nb := self.Len() nb := self.Len()
if nb <= 0 { if nb <= 0 {
return nil return nil
} }
var p *Pair var p *Pair
if !self.IsRaw() { if !self.isLazy() {
p = (*Pair)(self.p) p = (*Pair)(self.p)
} else { } else {
s := (*parseObjectStack)(self.p) s := (*parseObjectStack)(self.p)
@ -431,128 +496,139 @@ func (self *Node) findKey(key string) *Node {
return nil return nil
} }
func (self *Node) mapped() bool { func (self *Node) getParserAndArrayStack() (*Parser, *parseArrayStack) {
if self.m != nil && len(self.m) == self.Len() { stack := (*parseArrayStack)(self.p)
return true return &stack.parser, stack
}
if self.Len() > _MAP_THRESHOLD {
self.appendMap()
return true
}
return false
} }
func (self *Node) appendMap() { func (self *Node) getParserAndObjectStack() (*Parser, *parseObjectStack) {
nb := self.Len() stack := (*parseObjectStack)(self.p)
nm := len(self.m) return &stack.parser, stack
if nb <= nm {
return
}
if self.m == nil {
self.m = make(map[string]unsafe.Pointer, nb)
}
var p *Pair
if !self.IsRaw() {
p = self.pairAt(nm)
} else {
s := (*parseObjectStack)(self.p)
p = &s.v[nm]
}
(self.m)[p.Key] = unsafe.Pointer(&p.Value)
for i := nm + 1; i < nb; i++ {
p = p.unsafe_next()
(self.m)[p.Key] = unsafe.Pointer(&p.Value)
}
} }
func (self *Node) loadAllIndex() { func (self *Node) loadAllIndex() {
if !self.IsRaw() { if !self.isLazy() {
return return
} }
var err types.ParsingError var err types.ParsingError
stack := (*parseArrayStack)(self.p) parser, stack := self.getParserAndArrayStack()
parser := &stack.parser
old := parser.noLazy old := parser.noLazy
parser.noLazy = true parser.noLazy = true
start := parser.p
*self, err = parser.decodeArray(stack.v) *self, err = parser.decodeArray(stack.v)
if err != 0 { if err != 0 {
panic(fmt.Sprintf("%s at position %d", err, stack.parser.Pos())) panic(parser.ExportError(err, start))
} }
parser.noLazy = old parser.noLazy = old
} }
func (self *Node) skipAllIndex() {
if !self.isLazy() {
return
}
var err types.ParsingError
parser, stack := self.getParserAndArrayStack()
olds := parser.skipValue
parser.skipValue = true
oldl := parser.noLazy
parser.noLazy = true
start := parser.p
*self, err = parser.decodeArray(stack.v)
if err != 0 {
panic(parser.ExportError(err, start))
}
parser.skipValue = olds
parser.noLazy = oldl
}
func (self *Node) loadAllKey() { func (self *Node) loadAllKey() {
if !self.IsRaw() { if !self.isLazy() {
return return
} }
var err types.ParsingError var err types.ParsingError
stack := (*parseObjectStack)(self.p) parser, stack := self.getParserAndObjectStack()
parser := &stack.parser
old := parser.noLazy old := parser.noLazy
parser.noLazy = true parser.noLazy = true
start := parser.p
*self, err = parser.decodeObject(stack.v) *self, err = parser.decodeObject(stack.v)
if err != 0 { if err != 0 {
panic(fmt.Sprintf("%s at position %d", err, stack.parser.Pos())) panic(parser.ExportError(err, start))
} }
parser.noLazy = old parser.noLazy = old
} }
func (self *Node) loadIndex(index int) *Node { func (self *Node) skipAllKey() {
if !self.IsRaw() { if !self.isLazy() {
self.bound(index) return
return self.nodeAt(index)
} }
var err types.ParsingError
parser, stack := self.getParserAndObjectStack()
olds := parser.skipValue
parser.skipValue = true
oldl := parser.noLazy
parser.noLazy = true
start := parser.p
*self, err = parser.decodeObject(stack.v)
if err != 0 {
panic(parser.ExportError(err, start))
}
parser.skipValue = olds
parser.noLazy = oldl
}
func (self *Node) loadIndex(index int) *Node {
nb := self.Len() nb := self.Len()
if nb > index { if nb > index {
return self.nodeAt(index) return self.nodeAt(index)
} }
if !self.isLazy() {
return &Node{}
}
// lazy load // lazy load
for last := self.loadNextNode(); last != nil; last = self.loadNextNode(){
for last, err := self.loadNextNode(); last != nil; {
if err != 0 {
panic(fmt.Sprintf("%s at index %d", err, nb-1))
}
if self.Len() > index { if self.Len() > index {
return last return last
} }
last, err = self.loadNextNode() if !self.isLazy() {
break
}
} }
return nil return &Node{}
} }
func (self *Node) loadNextNode() (*Node, types.ParsingError) { func (self *Node) loadNextNode() *Node {
stack := (*parseArrayStack)(self.p) stack := (*parseArrayStack)(self.p)
ret := stack.v ret := stack.v
parser := &stack.parser parser := &stack.parser
sp := parser.p sp := parser.p
ns := len(parser.s) ns := len(parser.s)
start := sp
/* check for EOF */ /* check for EOF */
if parser.p = parser.lspace(sp); parser.p >= ns { if parser.p = parser.lspace(sp); parser.p >= ns {
return nil, types.ERR_EOF panic(parser.ExportError(types.ERR_EOF, start))
} }
/* check for empty array */ /* check for empty array */
if parser.s[parser.p] == ']' { if parser.s[parser.p] == ']' {
parser.p++ parser.p++
self.setArray(ret) self.setArray(ret)
return nil, 0 return nil
} }
var val Node var val Node
var err types.ParsingError var err types.ParsingError
/* decode the value */ /* decode the value */
old := parser.noLazy
parser.noLazy = true parser.noLazy = true
start = parser.p
if val, err = parser.Parse(); err != 0 { if val, err = parser.Parse(); err != 0 {
return nil, err panic(parser.ExportError(err, start))
} }
parser.noLazy = false parser.noLazy = old
/* add the value to result */ /* add the value to result */
ret = append(ret, val) ret = append(ret, val)
@ -560,61 +636,64 @@ func (self *Node) loadNextNode() (*Node, types.ParsingError) {
/* check for EOF */ /* check for EOF */
if parser.p >= ns { if parser.p >= ns {
return &ret[len(ret)-1], types.ERR_EOF panic(parser.ExportError(types.ERR_EOF, start))
} }
/* check for the next character */ /* check for the next character */
switch parser.s[parser.p] { switch parser.s[parser.p] {
case ',': case ',':
parser.p++ parser.p++
self.setRawArray(parser, ret) self.setLazyArray(parser, ret)
return &ret[len(ret)-1], 0 return &ret[len(ret)-1]
case ']': case ']':
parser.p++ parser.p++
self.setArray(ret) self.setArray(ret)
return &ret[len(ret)-1], 0 return &ret[len(ret)-1]
default: default:
return &ret[len(ret)-1], types.ERR_INVALID_CHAR panic(parser.ExportError(types.ERR_INVALID_CHAR, start))
} }
} }
func (self *Node) loadKey(key string) *Node { func (self *Node) loadKey(key string) *Node {
node := self.findKey(key) node := self.findKey(key)
if node != nil || !self.IsRaw() { if node != nil {
return node return node
} }
if !self.isLazy() {
return &Node{}
}
// lazy load // lazy load
for last, err := self.loadNextPair(); last != nil; { for last := self.loadNextPair(); last != nil; last = self.loadNextPair() {
if err != 0 {
panic(fmt.Sprintf("%s at key %s", err, last.Key))
}
if last.Key == key { if last.Key == key {
return &last.Value return &last.Value
} }
last, err = self.loadNextPair() if !self.isLazy() {
break
}
} }
return nil return &Node{}
} }
func (self *Node) loadNextPair() (*Pair, types.ParsingError) { func (self *Node) loadNextPair() (*Pair) {
stack := (*parseObjectStack)(self.p) stack := (*parseObjectStack)(self.p)
ret := stack.v ret := stack.v
parser := &stack.parser parser := &stack.parser
sp := parser.p sp := parser.p
ns := len(parser.s) ns := len(parser.s)
start := sp
/* check for EOF */ /* check for EOF */
if parser.p = parser.lspace(sp); parser.p >= ns { if parser.p = parser.lspace(sp); parser.p >= ns {
return nil, types.ERR_EOF panic(parser.ExportError(types.ERR_EOF, start))
} }
/* check for empty object */ /* check for empty object */
if parser.s[parser.p] == '}' { if parser.s[parser.p] == '}' {
parser.p++ parser.p++
self.setObject(ret) self.setObject(ret)
return nil, 0 return nil
} }
/* decode one pair */ /* decode one pair */
@ -623,8 +702,9 @@ func (self *Node) loadNextPair() (*Pair, types.ParsingError) {
var err types.ParsingError var err types.ParsingError
/* decode the key */ /* decode the key */
start = parser.p
if njs = parser.decodeValue(); njs.Vt != types.V_STRING { if njs = parser.decodeValue(); njs.Vt != types.V_STRING {
return nil, types.ERR_INVALID_CHAR panic(parser.ExportError(types.ERR_INVALID_CHAR, start))
} }
/* extract the key */ /* extract the key */
@ -634,21 +714,24 @@ func (self *Node) loadNextPair() (*Pair, types.ParsingError) {
/* check for escape sequence */ /* check for escape sequence */
if njs.Ep != -1 { if njs.Ep != -1 {
if key, err = unquote.String(key); err != 0 { if key, err = unquote.String(key); err != 0 {
return nil, err panic(parser.ExportError(err, start))
} }
} }
/* expect a ':' delimiter */ /* expect a ':' delimiter */
start = parser.p
if err = parser.delim(); err != 0 { if err = parser.delim(); err != 0 {
return nil, err panic(parser.ExportError(err, start))
} }
/* decode the value */ /* decode the value */
old := parser.noLazy
parser.noLazy = true parser.noLazy = true
start = parser.p
if val, err = parser.Parse(); err != 0 { if val, err = parser.Parse(); err != 0 {
return nil, err panic(parser.ExportError(err, start))
} }
parser.noLazy = false parser.noLazy = old
/* add the value to result */ /* add the value to result */
ret = append(ret, Pair{Key: key, Value: val}) ret = append(ret, Pair{Key: key, Value: val})
@ -656,21 +739,21 @@ func (self *Node) loadNextPair() (*Pair, types.ParsingError) {
/* check for EOF */ /* check for EOF */
if parser.p >= ns { if parser.p >= ns {
return &ret[len(ret)-1], types.ERR_EOF panic(parser.ExportError(types.ERR_EOF, start))
} }
/* check for the next character */ /* check for the next character */
switch parser.s[parser.p] { switch parser.s[parser.p] {
case ',': case ',':
parser.p++ parser.p++
self.setRawObject(parser, ret) self.setLazyObject(parser, ret)
return &ret[len(ret)-1], 0 return &ret[len(ret)-1]
case '}': case '}':
parser.p++ parser.p++
self.setObject(ret) self.setObject(ret)
return &ret[len(ret)-1], 0 return &ret[len(ret)-1]
default: default:
return &ret[len(ret)-1], types.ERR_INVALID_CHAR panic(parser.ExportError(types.ERR_INVALID_CHAR, start))
} }
} }
@ -712,6 +795,23 @@ func (self *Node) toGenericArrayUseNumber() []interface{} {
return ret return ret
} }
func (self *Node) toGenericArrayUseNode() []Node {
var nb = self.Len()
var out = make([]Node, nb)
if nb == 0 {
return out
}
var p = (*Node)(self.p)
out[0] = *p
for i := 1; i < nb; i++ {
p = p.unsafe_next()
out[i] = *p
}
return out
}
func (self *Node) toGenericObject() map[string]interface{} { func (self *Node) toGenericObject() map[string]interface{} {
nb := self.Len() nb := self.Len()
ret := make(map[string]interface{}, nb) ret := make(map[string]interface{}, nb)
@ -751,6 +851,24 @@ func (self *Node) toGenericObjectUseNumber() map[string]interface{} {
return ret return ret
} }
func (self *Node) toGenericObjectUseNode() map[string]Node {
var nb = self.Len()
var out = make(map[string]Node, nb)
if nb == 0 {
return out
}
var p = (*Pair)(self.p)
out[p.Key] = p.Value
for i := 1; i < nb; i++ {
p = p.unsafe_next()
out[p.Key] = p.Value
}
/* all done */
return out
}
/** Internal Factory Methods **/ /** Internal Factory Methods **/
var ( var (
@ -766,7 +884,7 @@ func newNumber(v string) Node {
return Node{ return Node{
v: int64(len(v)), v: int64(len(v)),
p: str2ptr(v), p: str2ptr(v),
t: V_NUMBER, t: _V_NUMBER,
} }
} }
@ -844,49 +962,49 @@ type parseArrayStack struct {
v []Node v []Node
} }
func newRawArray(p *Parser, v []Node) Node { func newLazyArray(p *Parser, v []Node) Node {
s := new(parseArrayStack) s := new(parseArrayStack)
s.parser = *p s.parser = *p
s.v = v s.v = v
return Node{ return Node{
t: V_ARRAY_RAW, t: _V_ARRAY_LAZY,
v: int64(len(v)&_LEN_MASK | cap(v)<<_CAP_BITS), v: int64(len(v)&_LEN_MASK | cap(v)<<_CAP_BITS),
p: unsafe.Pointer(s), p: unsafe.Pointer(s),
} }
} }
func (self *Node) setRawArray(p *Parser, v []Node) { func (self *Node) setLazyArray(p *Parser, v []Node) {
s := new(parseArrayStack) s := new(parseArrayStack)
s.parser = *p s.parser = *p
s.v = v s.v = v
self.t = V_ARRAY_RAW self.t = _V_ARRAY_LAZY
self.setCapAndLen(cap(v), len(v)) self.setCapAndLen(cap(v), len(v))
self.p = (unsafe.Pointer)(s) self.p = (unsafe.Pointer)(s)
} }
func newRawObject(p *Parser, v []Pair) Node { func newLazyObject(p *Parser, v []Pair) Node {
s := new(parseObjectStack) s := new(parseObjectStack)
s.parser = *p s.parser = *p
s.v = v s.v = v
return Node{ return Node{
t: V_OBJECT_RAW, t: _V_OBJECT_LAZY,
v: int64(len(v)&_LEN_MASK | cap(v)<<_CAP_BITS), v: int64(len(v)&_LEN_MASK | cap(v)<<_CAP_BITS),
p: unsafe.Pointer(s), p: unsafe.Pointer(s),
} }
} }
func (self *Node) setRawObject(p *Parser, v []Pair) { func (self *Node) setLazyObject(p *Parser, v []Pair) {
s := new(parseObjectStack) s := new(parseObjectStack)
s.parser = *p s.parser = *p
s.v = v s.v = v
self.t = V_OBJECT_RAW self.t = _V_OBJECT_LAZY
self.setCapAndLen(cap(v), len(v)) self.setCapAndLen(cap(v), len(v))
self.p = (unsafe.Pointer)(s) self.p = (unsafe.Pointer)(s)
} }
func newRawNode(str string) Node { func newRawNode(str string, typ types.ValueType) Node {
return Node{ return Node{
t: V_RAW, t: _V_RAW | typ,
p: str2ptr(str), p: str2ptr(str),
v: int64(len(str)), v: int64(len(str)),
} }
@ -894,9 +1012,31 @@ func newRawNode(str string) Node {
func (self *Node) parseRaw() Node { func (self *Node) parseRaw() Node {
raw := addr2str(self.p, self.v) raw := addr2str(self.p, self.v)
n, e := NewParser(raw).Parse() parser := NewParser(raw)
n, e := parser.Parse()
if e != 0 { if e != 0 {
panic(fmt.Sprintf("%v, raw json: %s", e.Error(), raw)) panic(parser.ExportError(e, 0))
} }
return n return n
} }
func switchRawType(c byte) types.ValueType {
var t types.ValueType = _V_NONE
switch c {
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' :
t = _V_NUMBER
case '"' :
t = types.V_STRING
case 'n' :
t = types.V_NULL
case 't' :
t = types.V_TRUE
case 'f' :
t = types.V_FALSE
case '[' :
t = types.V_ARRAY
case '{' :
t = types.V_OBJECT
}
return t
}

113
ast/node_test.go
View file

@ -27,6 +27,115 @@ import (
var parallelism = 4 var parallelism = 4
func TestUnsafeNode(t *testing.T) {
str, loop := getTestIteratorSample()
root, err := NewSearcher(str).GetByPath("array")
if err != nil {
t.Fatal(err)
}
a := root.UnsafeArray()
if len(a) != loop {
t.Fatalf("exp:%v, got:%v", loop, len(a))
}
for i := int64(0); i<int64(loop); i++{
in := a[i]
if in.Int64() != i {
t.Fatalf("exp:%v, got:%v", i, in.Int64())
}
}
root, err = NewSearcher(str).GetByPath("object")
if err != nil {
t.Fatal(err)
}
b := root.UnsafeMap()
if len(b) != loop {
t.Fatalf("exp:%v, got:%v", loop, len(b))
}
for i := int64(0); i<int64(loop); i++ {
k := `k`+strconv.Itoa(int(i))
if k != b[i].Key {
t.Fatalf("unexpected element: %#v", b[i])
}
if b[i].Value.Int64() != i {
t.Fatalf("exp:%v, got:%v", i, b[i].Value.Int64())
}
}
}
func TestUseNode(t *testing.T) {
str, loop := getTestIteratorSample()
root, err := NewSearcher(str).GetByPath("array")
if err != nil {
t.Fatal(err)
}
a := root.ArrayUseNode()
if len(a) != loop {
t.Fatalf("exp:%v, got:%v", loop, len(a))
}
for i := int64(0); i<int64(loop); i++{
in := a[i]
if in.Int64() != i {
t.Fatalf("exp:%v, got:%v", i, in.Int64())
}
}
root, err = NewSearcher(str).GetByPath("array")
if err != nil {
t.Fatal(err)
}
a = root.InterfaceUseNode().([]Node)
if len(a) != loop {
t.Fatalf("exp:%v, got:%v", loop, len(a))
}
for i := int64(0); i<int64(loop); i++{
in := a[i]
if in.Int64() != i {
t.Fatalf("exp:%v, got:%v", i, in.Int64())
}
}
root, err = NewSearcher(str).GetByPath("object")
if err != nil {
t.Fatal(err)
}
b := root.MapUseNode()
if len(b) != loop {
t.Fatalf("exp:%v, got:%v", loop, len(b))
}
for i := int64(0); i<int64(loop); i++ {
k := `k`+strconv.Itoa(int(i))
xn, ok := b[k]
if !ok {
t.Fatalf("unexpected element: %#v", xn)
}
if xn.Int64() != i {
t.Fatalf("exp:%v, got:%v", i, xn.Int64())
}
}
root, err = NewSearcher(str).GetByPath("object")
if err != nil {
t.Fatal(err)
}
b = root.InterfaceUseNode().(map[string]Node)
if len(b) != loop {
t.Fatalf("exp:%v, got:%v", loop, len(b))
}
for i := int64(0); i<int64(loop); i++ {
k := `k`+strconv.Itoa(int(i))
xn, ok := b[k]
if !ok {
t.Fatalf("unexpected element: %#v", xn)
}
if xn.Int64() != i {
t.Fatalf("exp:%v, got:%v", i, xn.Int64())
}
}
}
func TestUseNumber(t *testing.T) { func TestUseNumber(t *testing.T) {
node, err := NewParser("1061346755812312312313").Parse() node, err := NewParser("1061346755812312312313").Parse()
if err != 0 { if err != 0 {
@ -350,7 +459,7 @@ func BenchmarkNodeGet(b *testing.B) {
node.Set("test5", newNumber("5")) node.Set("test5", newNumber("5"))
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
node.Get("text").Interface() node.Get("text")
} }
} }
@ -424,6 +533,6 @@ func BenchmarkSliceAdd(b *testing.B) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
root, _ := NewParser(data).Parse() root, _ := NewParser(data).Parse()
node := root.Get("statuses").Array() node := root.Get("statuses").Array()
_ = append(node, map[string]interface{}{"test": 1}) node = append(node, map[string]interface{}{"test": 1})
} }
} }

76
ast/parser.go
View file

@ -17,6 +17,7 @@
package ast package ast
import ( import (
`fmt`
`sync` `sync`
`unsafe` `unsafe`
@ -26,12 +27,15 @@ import (
`github.com/bytedance/sonic/unquote` `github.com/bytedance/sonic/unquote`
) )
const _DEFAULT_NODE_CAP = 16 const _DEFAULT_NODE_CAP int = 16
const _ERR_NOT_FOUND types.ParsingError = 33
type Parser struct { type Parser struct {
p int p int
s string s string
noLazy bool noLazy bool
skipValue bool
} }
var stackPool = sync.Pool{ var stackPool = sync.Pool{
@ -100,8 +104,10 @@ func (self *Parser) array() types.ParsingError {
} }
func (self *Parser) lspace(sp int) int { func (self *Parser) lspace(sp int) int {
sv := (*rt.GoString)(unsafe.Pointer(&self.s)) ns := len(self.s)
return native.Lspace(sv.Ptr, sv.Len, sp) for ; sp<ns && isSpace(self.s[sp]); sp+=1 {}
return sp
} }
func (self *Parser) decodeValue() (val types.JsonState) { func (self *Parser) decodeValue() (val types.JsonState) {
@ -130,10 +136,26 @@ func (self *Parser) decodeArray(ret []Node) (Node, types.ParsingError) {
var val Node var val Node
var err types.ParsingError var err types.ParsingError
if self.skipValue {
/* skip the value */
var start int
if start, err = self.skip(); err != 0 {
return Node{}, err
}
if self.p > ns {
return Node{}, types.ERR_EOF
}
t := switchRawType(self.s[start])
if t == _V_NONE {
return Node{}, types.ERR_INVALID_CHAR
}
val = newRawNode(self.s[start:self.p], t)
}else{
/* decode the value */ /* decode the value */
if val, err = self.Parse(); err != 0 { if val, err = self.Parse(); err != 0 {
return Node{}, err return Node{}, err
} }
}
/* add the value to result */ /* add the value to result */
ret = append(ret, val) ret = append(ret, val)
@ -149,8 +171,8 @@ func (self *Parser) decodeArray(ret []Node) (Node, types.ParsingError) {
case ',' : self.p++ case ',' : self.p++
case ']' : self.p++; return newArray(ret), 0 case ']' : self.p++; return newArray(ret), 0
default: default:
if val.IsRaw() { if val.isLazy() {
return newRawArray(self, ret), 0 return newLazyArray(self, ret), 0
} }
return Node{}, types.ERR_INVALID_CHAR return Node{}, types.ERR_INVALID_CHAR
} }
@ -199,10 +221,27 @@ func (self *Parser) decodeObject(ret []Pair) (Node, types.ParsingError) {
return Node{}, err return Node{}, err
} }
if self.skipValue {
/* skip the value */
var start int
if start, err = self.skip(); err != 0 {
return Node{}, err
}
if self.p > ns {
return Node{}, types.ERR_EOF
}
t := switchRawType(self.s[start])
if t == _V_NONE {
return Node{}, types.ERR_INVALID_CHAR
}
val = newRawNode(self.s[start:self.p], t)
}else{
/* decode the value */ /* decode the value */
if val, err = self.Parse(); err != 0 { if val, err = self.Parse(); err != 0 {
return Node{}, err return Node{}, err
} }
}
/* add the value to result */ /* add the value to result */
ret = append(ret, Pair{Key: key, Value: val}) ret = append(ret, Pair{Key: key, Value: val})
@ -218,8 +257,8 @@ func (self *Parser) decodeObject(ret []Pair) (Node, types.ParsingError) {
case ',' : self.p++ case ',' : self.p++
case '}' : self.p++; return newObject(ret), 0 case '}' : self.p++; return newObject(ret), 0
default: default:
if val.IsRaw() { if val.isLazy() {
return newRawObject(self, ret), 0 return newLazyObject(self, ret), 0
} }
return Node{}, types.ERR_INVALID_CHAR return Node{}, types.ERR_INVALID_CHAR
} }
@ -264,12 +303,12 @@ func (self *Parser) Parse() (Node, types.ParsingError) {
if self.noLazy { if self.noLazy {
return self.decodeArray(make([]Node, 0, _DEFAULT_NODE_CAP)) return self.decodeArray(make([]Node, 0, _DEFAULT_NODE_CAP))
} }
return newRawArray(self, make([]Node, 0, _DEFAULT_NODE_CAP)), 0 return newLazyArray(self, make([]Node, 0, _DEFAULT_NODE_CAP)), 0
case types.V_OBJECT: case types.V_OBJECT:
if self.noLazy { if self.noLazy {
return self.decodeObject(make([]Pair, 0, _DEFAULT_NODE_CAP)) return self.decodeObject(make([]Pair, 0, _DEFAULT_NODE_CAP))
} }
return newRawObject(self, make([]Pair, 0, _DEFAULT_NODE_CAP)), 0 return newLazyObject(self, make([]Pair, 0, _DEFAULT_NODE_CAP)), 0
case types.V_DOUBLE : return newNumber(self.s[val.Ep:self.p]), 0 case types.V_DOUBLE : return newNumber(self.s[val.Ep:self.p]), 0
case types.V_INTEGER : return newNumber(self.s[val.Ep:self.p]), 0 case types.V_INTEGER : return newNumber(self.s[val.Ep:self.p]), 0
default : return Node{}, types.ParsingError(-val.Vt) default : return Node{}, types.ParsingError(-val.Vt)
@ -318,3 +357,22 @@ func LoadsUseNumber(src string) (int, interface{}, types.ParsingError) {
func NewParser(src string) *Parser { func NewParser(src string) *Parser {
return &Parser{s: src} return &Parser{s: src}
} }
// ExportError converts types.ParsingError to std Error
func (self *Parser) ExportError(err types.ParsingError, start int) error {
if err == _ERR_NOT_FOUND {
return fmt.Errorf("node not exists")
}
return fmt.Errorf("%v at %d, near '%s'", err, self.p, self.printNear(start))
}
func (self *Parser) printNear(start int) string {
if start < 0 {
start = 0
}
end := self.p + 10
if end > len(self.s) {
end = len(self.s)
}
return self.s[start:end]
}

23
ast/parser_test.go
View file

@ -167,6 +167,29 @@ func TestLoads(t *testing.T) {
assert.Equal(t, map[string]interface{}{"a": "123", "b": true, "c": false, "d": nil, "e": json.Number("2.4"), "f": json.Number("1.2e15"), "g": json.Number("1")}, i) assert.Equal(t, map[string]interface{}{"a": "123", "b": true, "c": false, "d": nil, "e": json.Number("2.4"), "f": json.Number("1.2e15"), "g": json.Number("1")}, i)
} }
func TestParsehNotExist(t *testing.T) {
s,err := NewParser(` { "xx" : [ 0, "" ] ,"yy" :{ "2": "" } } `).Parse()
if err != 0 {
t.Fatal(err)
}
node := s.GetByPath("xx", 2)
if node.Exists() {
t.Fatalf("node: %v", node)
}
node = s.GetByPath("xx", 1)
if !node.Exists() {
t.Fatalf("node: %v", nil)
}
node = s.GetByPath("yy", "3")
if node.Exists() {
t.Fatalf("node: %v", node)
}
node = s.GetByPath("yy", "2")
if !node.Exists() {
t.Fatalf("node: %v", nil)
}
}
func BenchmarkParser_StdLib(b *testing.B) { func BenchmarkParser_StdLib(b *testing.B) {
var bv = []byte(_TwitterJson) var bv = []byte(_TwitterJson)
b.SetBytes(int64(len(_TwitterJson))) b.SetBytes(int64(len(_TwitterJson)))

35
ast/search.go
View file

@ -36,18 +36,6 @@ func NewSearcher(str string) *Searcher {
} }
} }
func (self *Parser) printNear(start int) string {
start -= 10
if start < 0 {
start = 0
}
end := self.p + 10
if end > len(self.s) {
end = len(self.s)
}
return self.s[start:end]
}
func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
self.parser.p = 0 self.parser.p = 0
@ -57,12 +45,12 @@ func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
case int: case int:
start := self.parser.p start := self.parser.p
if err = self.parser.searchIndex(p.(int)); err != 0 { if err = self.parser.searchIndex(p.(int)); err != 0 {
return Node{}, fmt.Errorf("%v at %d, near '%s'", err, start, self.parser.printNear(start)) return Node{}, self.parser.ExportError(err, start)
} }
case string: case string:
start := self.parser.p start := self.parser.p
if err = self.parser.searchKey(p.(string)); err != 0 { if err = self.parser.searchKey(p.(string)); err != 0 {
return Node{}, fmt.Errorf("%v at %d, near '%s'", err, start, self.parser.printNear(start)) return Node{}, self.parser.ExportError(err, start)
} }
default: default:
panic("path must be either int or string") panic("path must be either int or string")
@ -71,14 +59,19 @@ func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
var start int var start int
if start, err = self.parser.skip(); err != 0 { if start, err = self.parser.skip(); err != 0 {
return Node{}, fmt.Errorf("%v at %d, near '%s'", err, self.parser.p, self.parser.printNear(start)) return Node{}, self.parser.ExportError(err, start)
} }
ns := len(self.parser.s) ns := len(self.parser.s)
if self.parser.p > ns || start > ns { if self.parser.p > ns || start >= ns {
return Node{}, fmt.Errorf("skip %d char out of json boundary", start) return Node{}, fmt.Errorf("skip %d char out of json boundary", start)
} }
return newRawNode(self.parser.s[start:self.parser.p]), nil t := switchRawType(self.parser.s[start])
if t == _V_NONE {
return Node{}, self.parser.ExportError(err, start)
}
return newRawNode(self.parser.s[start:self.parser.p], t), nil
} }
func (self *Parser) searchKey(match string) types.ParsingError { func (self *Parser) searchKey(match string) types.ParsingError {
@ -95,7 +88,7 @@ func (self *Parser) searchKey(match string) types.ParsingError {
/* check for empty object */ /* check for empty object */
if self.s[self.p] == '}' { if self.s[self.p] == '}' {
self.p++ self.p++
return types.ERR_EOF return _ERR_NOT_FOUND
} }
var njs types.JsonState var njs types.JsonState
@ -145,7 +138,7 @@ func (self *Parser) searchKey(match string) types.ParsingError {
self.p++ self.p++
case '}': case '}':
self.p++ self.p++
return types.ERR_EOF return _ERR_NOT_FOUND
default: default:
return types.ERR_INVALID_CHAR return types.ERR_INVALID_CHAR
} }
@ -166,7 +159,7 @@ func (self *Parser) searchIndex(idx int) types.ParsingError {
/* check for empty array */ /* check for empty array */
if self.s[self.p] == ']' { if self.s[self.p] == ']' {
self.p++ self.p++
return types.ERR_EOF return _ERR_NOT_FOUND
} }
var err types.ParsingError var err types.ParsingError
@ -190,7 +183,7 @@ func (self *Parser) searchIndex(idx int) types.ParsingError {
self.p++ self.p++
case ']': case ']':
self.p++ self.p++
return types.ERR_EOF return _ERR_NOT_FOUND
default: default:
return types.ERR_INVALID_CHAR return types.ERR_INVALID_CHAR
} }

20
ast/search_test.go
View file

@ -102,6 +102,26 @@ func TestLoadIndex(t *testing.T) {
}) })
} }
func TestSearchNotExist(t *testing.T) {
s := NewSearcher(` { "xx" : [ 0, "" ] ,"yy" :{ "2": "" } } `)
node, e := s.GetByPath("xx", 2)
if node.Exists() {
t.Fatalf("node: %v, err: %v", node, e)
}
node, e = s.GetByPath("xx", 1)
if e != nil || !node.Exists() {
t.Fatalf("node: %v, err: %v", node, e)
}
node, e = s.GetByPath("yy", "3")
if node.Exists() {
t.Fatalf("node: %v, err: %v", node, e)
}
node, e = s.GetByPath("yy", "2")
if e != nil || !node.Exists() {
t.Fatalf("node: %v, err: %v", node, e)
}
}
func BenchmarkSearchOne_Gjson(b *testing.B) { func BenchmarkSearchOne_Gjson(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson))) b.SetBytes(int64(len(_TwitterJson)))
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {

15
ast/utils.go
View file

@ -27,6 +27,16 @@ func mem2ptr(s []byte) unsafe.Pointer {
return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr
} }
//go:nosplit
func ptr2slice(s unsafe.Pointer, l int, c int) unsafe.Pointer {
slice := &rt.GoSlice{
Ptr: s,
Len: l,
Cap: c,
}
return unsafe.Pointer(slice)
}
//go:nosplit //go:nosplit
func str2ptr(s string) unsafe.Pointer { func str2ptr(s string) unsafe.Pointer {
return (*rt.GoString)(unsafe.Pointer(&s)).Ptr return (*rt.GoString)(unsafe.Pointer(&s)).Ptr
@ -39,3 +49,8 @@ func addr2str(p unsafe.Pointer, n int64) (s string) {
return return
} }
const _SPACE_CHAR_MASK = (1<<' ')|(1<<'\t')|(1<<'\r')|(1<<'\n')
func isSpace(c byte) bool {
return (int(1<<c) & _SPACE_CHAR_MASK) != 0
}

23
search_test.go
View file

@ -594,3 +594,26 @@ func TestGetMany4(t *testing.T) {
} }
} }
} }
func TestGetNotExist(t *testing.T) {
var dataStr = `{"m1":{"m2":3}}`
ret, err := GetFromString(dataStr, "not_exist", "m3")
if err == nil || ret.Exists() {
t.Fatal("Get exist!")
}
if ret.Type() != ast.V_NONE {
t.Fatal(ret.Type())
}
ret, err = GetFromString(dataStr)
if !ret.IsRaw() || ret.Type() != ast.V_OBJECT {
t.Fatal(ret.Type())
}
v11 := ret.Get("not_exist")
if v11.Exists() {
t.Fatal()
}
v2 := ret.GetByPath("m1", "m2")
if !v2.Exists() || v2.IsRaw() || v2.Type() != ast.V_NUMBER {
t.Fatal(ret.Type())
}
}