gvsafronov/tile38
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Immutable Object type

Browse Source
This commit is contained in:
tidwall 2022-09-20 14:20:53 -07:00
parent ba9a767988
commit 2c643996e7
22 changed files with 819 additions and 912 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -27,7 +27,7 @@ require (
github.com/tidwall/redbench v0.1.0 github.com/tidwall/redbench v0.1.0
github.com/tidwall/redcon v1.4.4 github.com/tidwall/redcon v1.4.4
github.com/tidwall/resp v0.1.1 github.com/tidwall/resp v0.1.1
github.com/tidwall/rtree v1.9.1 github.com/tidwall/rtree v1.9.2
github.com/tidwall/sjson v1.2.4 github.com/tidwall/sjson v1.2.4
github.com/xdg/scram v1.0.5 github.com/xdg/scram v1.0.5
github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da

4
go.sum
View File

@ -383,8 +383,8 @@ github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYg
github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
github.com/tidwall/rtree v1.3.1/go.mod h1:S+JSsqPTI8LfWA4xHBo5eXzie8WJLVFeppAutSegl6M= github.com/tidwall/rtree v1.3.1/go.mod h1:S+JSsqPTI8LfWA4xHBo5eXzie8WJLVFeppAutSegl6M=
github.com/tidwall/rtree v1.9.1 h1:UIPtvE09nLKZRnMNEwRZxu9jRAkzROAZDR+NPS/9IRs= github.com/tidwall/rtree v1.9.2 h1:6HiSU/bf4a7l2smEC+fEum/WloHMFCIQKWHjahm0Do8=
github.com/tidwall/rtree v1.9.1/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ= github.com/tidwall/rtree v1.9.2/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ=
github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=

423
internal/collection/collection.go
View File

@ -5,11 +5,11 @@ import (
"github.com/tidwall/btree" "github.com/tidwall/btree"
"github.com/tidwall/geojson" "github.com/tidwall/geojson"
"github.com/tidwall/geojson/geo"
"github.com/tidwall/geojson/geometry" "github.com/tidwall/geojson/geometry"
"github.com/tidwall/rtree" "github.com/tidwall/rtree"
"github.com/tidwall/tile38/internal/deadline" "github.com/tidwall/tile38/internal/deadline"
"github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/object"
) )
// yieldStep forces the iterator to yield goroutine every 256 steps. // yieldStep forces the iterator to yield goroutine every 256 steps.
@ -21,20 +21,13 @@ type Cursor interface {
Step(count uint64) Step(count uint64)
} }
type itemT struct { func byID(a, b *object.Object) bool {
id string return a.ID() < b.ID()
obj geojson.Object
expires int64 // unix nano expiration
fields field.List
} }
func byID(a, b *itemT) bool { func byValue(a, b *object.Object) bool {
return a.id < b.id value1 := a.String()
} value2 := b.String()
func byValue(a, b *itemT) bool {
value1 := a.obj.String()
value2 := b.obj.String()
if value1 < value2 { if value1 < value2 {
return true return true
} }
@ -45,30 +38,23 @@ func byValue(a, b *itemT) bool {
return byID(a, b) return byID(a, b)
} }
func byExpires(a, b *itemT) bool { func byExpires(a, b *object.Object) bool {
if a.expires < b.expires { if a.Expires() < b.Expires() {
return true return true
} }
if a.expires > b.expires { if a.Expires() > b.Expires() {
return false return false
} }
// the values match so we'll compare IDs, which are always unique. // the values match so we'll compare IDs, which are always unique.
return byID(a, b) return byID(a, b)
} }
func (item *itemT) Rect() geometry.Rect {
if item.obj != nil {
return item.obj.Rect()
}
return geometry.Rect{}
}
// Collection represents a collection of geojson objects. // Collection represents a collection of geojson objects.
type Collection struct { type Collection struct {
items *btree.BTreeG[*itemT] // items sorted by id items *btree.BTreeG[*object.Object] // sorted by id
spatial *rtree.RTreeGN[float32, *itemT] // items geospatially indexed spatial *rtree.RTreeGN[float32, *object.Object] // geospatially indexed
values *btree.BTreeG[*itemT] // items sorted by value+id values *btree.BTreeG[*object.Object] // sorted by value+id
expires *btree.BTreeG[*itemT] // items sorted by ex+id expires *btree.BTreeG[*object.Object] // sorted by ex+id
weight int weight int
points int points int
objects int // geometry count objects int // geometry count
@ -83,7 +69,7 @@ func New() *Collection {
items: btree.NewBTreeGOptions(byID, optsNoLock), items: btree.NewBTreeGOptions(byID, optsNoLock),
values: btree.NewBTreeGOptions(byValue, optsNoLock), values: btree.NewBTreeGOptions(byValue, optsNoLock),
expires: btree.NewBTreeGOptions(byExpires, optsNoLock), expires: btree.NewBTreeGOptions(byExpires, optsNoLock),
spatial: &rtree.RTreeGN[float32, *itemT]{}, spatial: &rtree.RTreeGN[float32, *object.Object]{},
} }
return col return col
} }
@ -121,31 +107,14 @@ func (c *Collection) Bounds() (minX, minY, maxX, maxY float64) {
right.Rect().Max.X, top.Rect().Max.Y right.Rect().Max.X, top.Rect().Max.Y
} }
func objIsSpatial(obj geojson.Object) bool { func (c *Collection) indexDelete(item *object.Object) {
_, ok := obj.(geojson.Spatial) if !item.Geo().Empty() {
return ok
}
func (c *Collection) objWeight(item *itemT) int {
var weight int
weight += len(item.id)
if objIsSpatial(item.obj) {
weight += item.obj.NumPoints() * 16
} else {
weight += len(item.obj.String())
}
weight += item.fields.Weight()
return weight
}
func (c *Collection) indexDelete(item *itemT) {
if !item.obj.Empty() {
c.spatial.Delete(rtreeItem(item)) c.spatial.Delete(rtreeItem(item))
} }
} }
func (c *Collection) indexInsert(item *itemT) { func (c *Collection) indexInsert(item *object.Object) {
if !item.obj.Empty() { if !item.Geo().Empty() {
c.spatial.Insert(rtreeItem(item)) c.spatial.Insert(rtreeItem(item))
} }
} }
@ -176,7 +145,7 @@ func rtreeValueUp(d float64) float32 {
return f return f
} }
func rtreeItem(item *itemT) (min, max [2]float32, data *itemT) { func rtreeItem(item *object.Object) (min, max [2]float32, data *object.Object) {
min, max = rtreeRect(item.Rect()) min, max = rtreeRect(item.Rect())
return min, max, item return min, max, item
} }
@ -193,105 +162,68 @@ func rtreeRect(rect geometry.Rect) (min, max [2]float32) {
// Set adds or replaces an object in the collection and returns the fields // Set adds or replaces an object in the collection and returns the fields
// array. // array.
func (c *Collection) Set(id string, obj geojson.Object, fields field.List, ex int64) ( func (c *Collection) Set(obj *object.Object) (prev *object.Object) {
oldObject geojson.Object, oldFields, newFields field.List, prev, _ = c.items.Set(obj)
) { if prev != nil {
newItem := &itemT{ if prev.IsSpatial() {
id: id, c.indexDelete(prev)
obj: obj,
expires: ex,
fields: fields,
}
// add the new item to main btree and remove the old one if needed
oldItem, ok := c.items.Set(newItem)
if ok {
// the old item was removed, now let's remove it from the rtree/btree.
if objIsSpatial(oldItem.obj) {
c.indexDelete(oldItem)
c.objects-- c.objects--
} else { } else {
c.values.Delete(oldItem) c.values.Delete(prev)
c.nobjects-- c.nobjects--
} }
// delete old item from the expires queue if prev.Expires() != 0 {
if oldItem.expires != 0 { c.expires.Delete(prev)
c.expires.Delete(oldItem)
} }
c.points -= prev.Geo().NumPoints()
// decrement the point count c.weight -= prev.Weight()
c.points -= oldItem.obj.NumPoints()
// decrement the weights
c.weight -= c.objWeight(oldItem)
} }
if obj.IsSpatial() {
// insert the new item into the rtree or strings tree. c.indexInsert(obj)
if objIsSpatial(newItem.obj) {
c.indexInsert(newItem)
c.objects++ c.objects++
} else { } else {
c.values.Set(newItem) c.values.Set(obj)
c.nobjects++ c.nobjects++
} }
// insert item into expires queue. if obj.Expires() != 0 {
if newItem.expires != 0 { c.expires.Set(obj)
c.expires.Set(newItem)
} }
c.points += obj.Geo().NumPoints()
// increment the point count c.weight += obj.Weight()
c.points += newItem.obj.NumPoints() return prev
// add the new weights
c.weight += c.objWeight(newItem)
if oldItem != nil {
return oldItem.obj, oldItem.fields, newItem.fields
}
return nil, field.List{}, newItem.fields
} }
// Delete removes an object and returns it. // Delete removes an object and returns it.
// If the object does not exist then the 'ok' return value will be false. // If the object does not exist then the 'ok' return value will be false.
func (c *Collection) Delete(id string) ( func (c *Collection) Delete(id string) (prev *object.Object) {
obj geojson.Object, fields field.List, ok bool, key := object.New(id, nil, 0, 0, field.List{})
) { prev, _ = c.items.Delete(key)
oldItem, ok := c.items.Delete(&itemT{id: id}) if prev == nil {
if !ok { return nil
return nil, field.List{}, false
} }
if objIsSpatial(oldItem.obj) { if prev.IsSpatial() {
if !oldItem.obj.Empty() { if !prev.Geo().Empty() {
c.indexDelete(oldItem) c.indexDelete(prev)
} }
c.objects-- c.objects--
} else { } else {
c.values.Delete(oldItem) c.values.Delete(prev)
c.nobjects-- c.nobjects--
} }
// delete old item from expires queue if prev.Expires() != 0 {
if oldItem.expires != 0 { c.expires.Delete(prev)
c.expires.Delete(oldItem)
} }
c.weight -= c.objWeight(oldItem) c.points -= prev.Geo().NumPoints()
c.points -= oldItem.obj.NumPoints() c.weight -= prev.Weight()
return prev
return oldItem.obj, oldItem.fields, true
} }
// Get returns an object. // Get returns an object.
// If the object does not exist then the 'ok' return value will be false. // If the object does not exist then the 'ok' return value will be false.
func (c *Collection) Get(id string) ( func (c *Collection) Get(id string) *object.Object {
obj geojson.Object, key := object.New(id, nil, 0, 0, field.List{})
fields field.List, obj, _ := c.items.Get(key)
ex int64, return obj
ok bool,
) {
item, ok := c.items.Get(&itemT{id: id})
if !ok {
return nil, field.List{}, 0, false
}
return item.obj, item.fields, item.expires, true
} }
// Scan iterates though the collection ids. // Scan iterates though the collection ids.
@ -299,7 +231,7 @@ func (c *Collection) Scan(
desc bool, desc bool,
cursor Cursor, cursor Cursor,
deadline *deadline.Deadline, deadline *deadline.Deadline,
iterator func(id string, obj geojson.Object, fields field.List) bool, iterator func(obj *object.Object) bool,
) bool { ) bool {
var keepon = true var keepon = true
var count uint64 var count uint64
@ -308,13 +240,13 @@ func (c *Collection) Scan(
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(item *itemT) bool { iter := func(obj *object.Object) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
} }
nextStep(count, cursor, deadline) nextStep(count, cursor, deadline)
keepon = iterator(item.id, item.obj, item.fields) keepon = iterator(obj)
return keepon return keepon
} }
if desc { if desc {
@ -331,7 +263,7 @@ func (c *Collection) ScanRange(
desc bool, desc bool,
cursor Cursor, cursor Cursor,
deadline *deadline.Deadline, deadline *deadline.Deadline,
iterator func(id string, obj geojson.Object, fields field.List) bool, iterator func(o *object.Object) bool,
) bool { ) bool {
var keepon = true var keepon = true
var count uint64 var count uint64
@ -340,29 +272,30 @@ func (c *Collection) ScanRange(
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(item *itemT) bool { iter := func(o *object.Object) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
} }
nextStep(count, cursor, deadline) nextStep(count, cursor, deadline)
if !desc { if !desc {
if item.id >= end { if o.ID() >= end {
return false return false
} }
} else { } else {
if item.id <= end { if o.ID() <= end {
return false return false
} }
} }
keepon = iterator(item.id, item.obj, item.fields) keepon = iterator(o)
return keepon return keepon
} }
pstart := object.New(start, nil, 0, 0, field.List{})
if desc { if desc {
c.items.Descend(&itemT{id: start}, iter) c.items.Descend(pstart, iter)
} else { } else {
c.items.Ascend(&itemT{id: start}, iter) c.items.Ascend(pstart, iter)
} }
return keepon return keepon
} }
@ -372,7 +305,7 @@ func (c *Collection) SearchValues(
desc bool, desc bool,
cursor Cursor, cursor Cursor,
deadline *deadline.Deadline, deadline *deadline.Deadline,
iterator func(id string, obj geojson.Object, fields field.List) bool, iterator func(o *object.Object) bool,
) bool { ) bool {
var keepon = true var keepon = true
var count uint64 var count uint64
@ -381,13 +314,13 @@ func (c *Collection) SearchValues(
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(item *itemT) bool { iter := func(o *object.Object) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
} }
nextStep(count, cursor, deadline) nextStep(count, cursor, deadline)
keepon = iterator(item.id, item.obj, item.fields) keepon = iterator(o)
return keepon return keepon
} }
if desc { if desc {
@ -402,7 +335,7 @@ func (c *Collection) SearchValues(
func (c *Collection) SearchValuesRange(start, end string, desc bool, func (c *Collection) SearchValuesRange(start, end string, desc bool,
cursor Cursor, cursor Cursor,
deadline *deadline.Deadline, deadline *deadline.Deadline,
iterator func(id string, obj geojson.Object, fields field.List) bool, iterator func(o *object.Object) bool,
) bool { ) bool {
var keepon = true var keepon = true
var count uint64 var count uint64
@ -411,38 +344,39 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool,
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(item *itemT) bool { iter := func(o *object.Object) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
} }
nextStep(count, cursor, deadline) nextStep(count, cursor, deadline)
keepon = iterator(item.id, item.obj, item.fields) keepon = iterator(o)
return keepon return keepon
} }
pstart := &itemT{obj: String(start)}
pend := &itemT{obj: String(end)} pstart := object.New("", String(start), 0, 0, field.List{})
pend := object.New("", String(end), 0, 0, field.List{})
if desc { if desc {
// descend range // descend range
c.values.Descend(pstart, func(item *itemT) bool { c.values.Descend(pstart, func(item *object.Object) bool {
return bGT(c.values, item, pend) && iter(item) return bGT(c.values, item, pend) && iter(item)
}) })
} else { } else {
c.values.Ascend(pstart, func(item *itemT) bool { c.values.Ascend(pstart, func(item *object.Object) bool {
return bLT(c.values, item, pend) && iter(item) return bLT(c.values, item, pend) && iter(item)
}) })
} }
return keepon return keepon
} }
func bLT(tr *btree.BTreeG[*itemT], a, b *itemT) bool { return tr.Less(a, b) } func bLT(tr *btree.BTreeG[*object.Object], a, b *object.Object) bool { return tr.Less(a, b) }
func bGT(tr *btree.BTreeG[*itemT], a, b *itemT) bool { return tr.Less(b, a) } func bGT(tr *btree.BTreeG[*object.Object], a, b *object.Object) bool { return tr.Less(b, a) }
// ScanGreaterOrEqual iterates though the collection starting with specified id. // ScanGreaterOrEqual iterates though the collection starting with specified id.
func (c *Collection) ScanGreaterOrEqual(id string, desc bool, func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
cursor Cursor, cursor Cursor,
deadline *deadline.Deadline, deadline *deadline.Deadline,
iterator func(id string, obj geojson.Object, fields field.List, ex int64) bool, iterator func(o *object.Object) bool,
) bool { ) bool {
var keepon = true var keepon = true
var count uint64 var count uint64
@ -451,33 +385,34 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(item *itemT) bool { iter := func(o *object.Object) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
} }
nextStep(count, cursor, deadline) nextStep(count, cursor, deadline)
keepon = iterator(item.id, item.obj, item.fields, item.expires) keepon = iterator(o)
return keepon return keepon
} }
pstart := object.New(id, nil, 0, 0, field.List{})
if desc { if desc {
c.items.Descend(&itemT{id: id}, iter) c.items.Descend(pstart, iter)
} else { } else {
c.items.Ascend(&itemT{id: id}, iter) c.items.Ascend(pstart, iter)
} }
return keepon return keepon
} }
func (c *Collection) geoSearch( func (c *Collection) geoSearch(
rect geometry.Rect, rect geometry.Rect,
iter func(id string, obj geojson.Object, fields field.List) bool, iter func(o *object.Object) bool,
) bool { ) bool {
alive := true alive := true
min, max := rtreeRect(rect) min, max := rtreeRect(rect)
c.spatial.Search( c.spatial.Search(
min, max, min, max,
func(_, _ [2]float32, item *itemT) bool { func(_, _ [2]float32, o *object.Object) bool {
alive = iter(item.id, item.obj, item.fields) alive = iter(o)
return alive return alive
}, },
) )
@ -486,29 +421,25 @@ func (c *Collection) geoSearch(
func (c *Collection) geoSparse( func (c *Collection) geoSparse(
obj geojson.Object, sparse uint8, obj geojson.Object, sparse uint8,
iter func(id string, obj geojson.Object, fields field.List) (match, ok bool), iter func(o *object.Object) (match, ok bool),
) bool { ) bool {
matches := make(map[string]bool) matches := make(map[string]bool)
alive := true alive := true
c.geoSparseInner(obj.Rect(), sparse, c.geoSparseInner(obj.Rect(), sparse, func(o *object.Object) (match, ok bool) {
func(id string, o geojson.Object, fields field.List) ( ok = true
match, ok bool, if !matches[o.ID()] {
) { match, ok = iter(o)
ok = true if match {
if !matches[id] { matches[o.ID()] = true
match, ok = iter(id, o, fields)
if match {
matches[id] = true
}
} }
return match, ok }
}, return match, ok
) })
return alive return alive
} }
func (c *Collection) geoSparseInner( func (c *Collection) geoSparseInner(
rect geometry.Rect, sparse uint8, rect geometry.Rect, sparse uint8,
iter func(id string, obj geojson.Object, fields field.List) (match, ok bool), iter func(o *object.Object) (match, ok bool),
) bool { ) bool {
if sparse > 0 { if sparse > 0 {
w := rect.Max.X - rect.Min.X w := rect.Max.X - rect.Min.X
@ -539,16 +470,14 @@ func (c *Collection) geoSparseInner(
return true return true
} }
alive := true alive := true
c.geoSearch(rect, c.geoSearch(rect, func(o *object.Object) bool {
func(id string, obj geojson.Object, fields field.List) bool { match, ok := iter(o)
match, ok := iter(id, obj, fields) if !ok {
if !ok { alive = false
alive = false return false
return false }
} return !match
return !match })
},
)
return alive return alive
} }
@ -559,7 +488,7 @@ func (c *Collection) Within(
sparse uint8, sparse uint8,
cursor Cursor, cursor Cursor,
deadline *deadline.Deadline, deadline *deadline.Deadline,
iter func(id string, obj geojson.Object, fields field.List) bool, iter func(o *object.Object) bool,
) bool { ) bool {
var count uint64 var count uint64
var offset uint64 var offset uint64
@ -568,45 +497,39 @@ func (c *Collection) Within(
cursor.Step(offset) cursor.Step(offset)
} }
if sparse > 0 { if sparse > 0 {
return c.geoSparse(obj, sparse, return c.geoSparse(obj, sparse, func(o *object.Object) (match, ok bool) {
func(id string, o geojson.Object, fields field.List) (
match, ok bool,
) {
count++
if count <= offset {
return false, true
}
nextStep(count, cursor, deadline)
if match = o.Within(obj); match {
ok = iter(id, o, fields)
}
return match, ok
},
)
}
return c.geoSearch(obj.Rect(),
func(id string, o geojson.Object, fields field.List) bool {
count++ count++
if count <= offset { if count <= offset {
return true return false, true
} }
nextStep(count, cursor, deadline) nextStep(count, cursor, deadline)
if o.Within(obj) { if match = o.Geo().Within(obj); match {
return iter(id, o, fields) ok = iter(o)
} }
return match, ok
})
}
return c.geoSearch(obj.Rect(), func(o *object.Object) bool {
count++
if count <= offset {
return true return true
}, }
) nextStep(count, cursor, deadline)
if o.Geo().Within(obj) {
return iter(o)
}
return true
})
} }
// Intersects returns all object that are intersect an object or bounding box. // Intersects returns all object that are intersect an object or bounding box.
// Set obj to nil in order to use the bounding box. // Set obj to nil in order to use the bounding box.
func (c *Collection) Intersects( func (c *Collection) Intersects(
obj geojson.Object, gobj geojson.Object,
sparse uint8, sparse uint8,
cursor Cursor, cursor Cursor,
deadline *deadline.Deadline, deadline *deadline.Deadline,
iter func(id string, obj geojson.Object, fields field.List) bool, iter func(o *object.Object) bool,
) bool { ) bool {
var count uint64 var count uint64
var offset uint64 var offset uint64
@ -615,34 +538,29 @@ func (c *Collection) Intersects(
cursor.Step(offset) cursor.Step(offset)
} }
if sparse > 0 { if sparse > 0 {
return c.geoSparse(obj, sparse, return c.geoSparse(gobj, sparse, func(o *object.Object) (match, ok bool) {
func(id string, o geojson.Object, fields field.List) (
match, ok bool,
) {
count++
if count <= offset {
return false, true
}
nextStep(count, cursor, deadline)
if match = o.Intersects(obj); match {
ok = iter(id, o, fields)
}
return match, ok
},
)
}
return c.geoSearch(obj.Rect(),
func(id string, o geojson.Object, fields field.List) bool {
count++ count++
if count <= offset { if count <= offset {
return true return false, true
} }
nextStep(count, cursor, deadline) nextStep(count, cursor, deadline)
if o.Intersects(obj) { if match = o.Geo().Intersects(gobj); match {
return iter(id, o, fields) ok = iter(o)
} }
return match, ok
})
}
return c.geoSearch(gobj.Rect(), func(o *object.Object) bool {
count++
if count <= offset {
return true return true
}, }
nextStep(count, cursor, deadline)
if o.Geo().Intersects(gobj) {
return iter(o)
}
return true
},
) )
} }
@ -651,41 +569,8 @@ func (c *Collection) Nearby(
target geojson.Object, target geojson.Object,
cursor Cursor, cursor Cursor,
deadline *deadline.Deadline, deadline *deadline.Deadline,
iter func(id string, obj geojson.Object, fields field.List, dist float64) bool, iter func(o *object.Object, dist float64) bool,
) bool { ) bool {
// First look to see if there's at least one candidate in the circle's
// outer rectangle. This is a fast-fail operation.
if circle, ok := target.(*geojson.Circle); ok {
meters := circle.Meters()
if meters > 0 {
center := circle.Center()
minLat, minLon, maxLat, maxLon :=
geo.RectFromCenter(center.Y, center.X, meters)
var exists bool
min, max := rtreeRect(geometry.Rect{
Min: geometry.Point{
X: minLon,
Y: minLat,
},
Max: geometry.Point{
X: maxLon,
Y: maxLat,
},
})
c.spatial.Search(
min, max,
func(_, _ [2]float32, item *itemT) bool {
exists = true
return false
},
)
if !exists {
// no candidates
return true
}
}
}
// do the kNN operation
alive := true alive := true
center := target.Center() center := target.Center()
var count uint64 var count uint64
@ -694,22 +579,22 @@ func (c *Collection) Nearby(
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
distFn := geodeticDistAlgo[*itemT]([2]float64{center.X, center.Y}) distFn := geodeticDistAlgo([2]float64{center.X, center.Y})
c.spatial.Nearby( c.spatial.Nearby(
func(min, max [2]float32, data *itemT, item bool) float32 { func(min, max [2]float32, data *object.Object, item bool) float64 {
return float32(distFn( return distFn(
[2]float64{float64(min[0]), float64(min[1])}, [2]float64{float64(min[0]), float64(min[1])},
[2]float64{float64(max[0]), float64(max[1])}, [2]float64{float64(max[0]), float64(max[1])},
data, item, data, item,
)) )
}, },
func(_, _ [2]float32, item *itemT, dist float32) bool { func(_, _ [2]float32, o *object.Object, dist float64) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
} }
nextStep(count, cursor, deadline) nextStep(count, cursor, deadline)
alive = iter(item.id, item.obj, item.fields, float64(dist)) alive = iter(o, dist)
return alive return alive
}, },
) )
@ -727,8 +612,6 @@ func nextStep(step uint64, cursor Cursor, deadline *deadline.Deadline) {
} }
// ScanExpires returns a list of all objects that have expired. // ScanExpires returns a list of all objects that have expired.
func (c *Collection) ScanExpires(iter func(id string, expires int64) bool) { func (c *Collection) ScanExpires(iter func(o *object.Object) bool) {
c.expires.Scan(func(item *itemT) bool { c.expires.Scan(iter)
return iter(item.id, item.expires)
})
} }

443
internal/collection/collection_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/tidwall/geojson/geometry" "github.com/tidwall/geojson/geometry"
"github.com/tidwall/gjson" "github.com/tidwall/gjson"
"github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/object"
) )
func PO(x, y float64) *geojson.Point { func PO(x, y float64) *geojson.Point {
@ -47,14 +48,14 @@ func TestCollectionNewCollection(t *testing.T) {
id := strconv.FormatInt(int64(i), 10) id := strconv.FormatInt(int64(i), 10)
obj := PO(rand.Float64()*360-180, rand.Float64()*180-90) obj := PO(rand.Float64()*360-180, rand.Float64()*180-90)
objs[id] = obj objs[id] = obj
c.Set(id, obj, field.List{}, 0) c.Set(object.New(id, obj, 0, 0, field.List{}))
} }
count := 0 count := 0
bbox := geometry.Rect{ bbox := geometry.Rect{
Min: geometry.Point{X: -180, Y: -90}, Min: geometry.Point{X: -180, Y: -90},
Max: geometry.Point{X: 180, Y: 90}, Max: geometry.Point{X: 180, Y: 90},
} }
c.geoSearch(bbox, func(id string, obj geojson.Object, _ field.List) bool { c.geoSearch(bbox, func(o *object.Object) bool {
count++ count++
return true return true
}) })
@ -80,44 +81,32 @@ func TestCollectionSet(t *testing.T) {
t.Run("AddString", func(t *testing.T) { t.Run("AddString", func(t *testing.T) {
c := New() c := New()
str1 := String("hello") str1 := String("hello")
oldObject, oldFields, newFields := c.Set("str", str1, field.List{}, 0) old := c.Set(object.New("str", str1, 0, 0, field.List{}))
expect(t, oldObject == nil) expect(t, old == nil)
expect(t, oldFields.Len() == 0)
expect(t, newFields.Len() == 0)
}) })
t.Run("UpdateString", func(t *testing.T) { t.Run("UpdateString", func(t *testing.T) {
c := New() c := New()
str1 := String("hello") str1 := String("hello")
str2 := String("world") str2 := String("world")
oldObject, oldFields, newFields := c.Set("str", str1, field.List{}, 0) old := c.Set(object.New("str", str1, 0, 0, field.List{}))
expect(t, oldObject == nil) expect(t, old == nil)
expect(t, oldFields.Len() == 0) old = c.Set(object.New("str", str2, 0, 0, field.List{}))
expect(t, newFields.Len() == 0) expect(t, old.Geo() == str1)
oldObject, oldFields, newFields = c.Set("str", str2, field.List{}, 0)
expect(t, oldObject == str1)
expect(t, oldFields.Len() == 0)
expect(t, newFields.Len() == 0)
}) })
t.Run("AddPoint", func(t *testing.T) { t.Run("AddPoint", func(t *testing.T) {
c := New() c := New()
point1 := PO(-112.1, 33.1) point1 := PO(-112.1, 33.1)
oldObject, oldFields, newFields := c.Set("point", point1, field.List{}, 0) old := c.Set(object.New("point", point1, 0, 0, field.List{}))
expect(t, oldObject == nil) expect(t, old == nil)
expect(t, oldFields.Len() == 0)
expect(t, newFields.Len() == 0)
}) })
t.Run("UpdatePoint", func(t *testing.T) { t.Run("UpdatePoint", func(t *testing.T) {
c := New() c := New()
point1 := PO(-112.1, 33.1) point1 := PO(-112.1, 33.1)
point2 := PO(-112.2, 33.2) point2 := PO(-112.2, 33.2)
oldObject, oldFields, newFields := c.Set("point", point1, field.List{}, 0) old := c.Set(object.New("point", point1, 0, 0, field.List{}))
expect(t, oldObject == nil) expect(t, old == nil)
expect(t, oldFields.Len() == 0) old = c.Set(object.New("point", point2, 0, 0, field.List{}))
expect(t, newFields.Len() == 0) expect(t, old.Geo().Center() == point1.Base())
oldObject, oldFields, newFields = c.Set("point", point2, field.List{}, 0)
expect(t, oldObject == point1)
expect(t, oldFields.Len() == 0)
expect(t, newFields.Len() == 0)
}) })
t.Run("Fields", func(t *testing.T) { t.Run("Fields", func(t *testing.T) {
c := New() c := New()
@ -126,11 +115,8 @@ func TestCollectionSet(t *testing.T) {
fNames := []string{"a", "b", "c"} fNames := []string{"a", "b", "c"}
fValues := []string{"1", "2", "3"} fValues := []string{"1", "2", "3"}
fields1 := toFields(fNames, fValues) fields1 := toFields(fNames, fValues)
oldObj, oldFlds, newFlds := c.Set("str", str1, fields1, 0) old := c.Set(object.New("str", str1, 0, 0, fields1))
expect(t, old == nil)
expect(t, oldObj == nil)
expect(t, oldFlds.Len() == 0)
expect(t, reflect.DeepEqual(newFlds, fields1))
str2 := String("hello") str2 := String("hello")
@ -138,25 +124,23 @@ func TestCollectionSet(t *testing.T) {
fValues = []string{"4", "5", "6"} fValues = []string{"4", "5", "6"}
fields2 := toFields(fNames, fValues) fields2 := toFields(fNames, fValues)
oldObj, oldFlds, newFlds = c.Set("str", str2, fields2, 0) old = c.Set(object.New("str", str2, 0, 0, fields2))
expect(t, oldObj == str1) expect(t, old.Geo() == str1)
expect(t, reflect.DeepEqual(oldFlds, fields1)) expect(t, reflect.DeepEqual(old.Fields(), fields1))
expect(t, reflect.DeepEqual(newFlds, fields2))
fNames = []string{"a", "b", "c", "d", "e", "f"} fNames = []string{"a", "b", "c", "d", "e", "f"}
fValues = []string{"7", "8", "9", "10", "11", "12"} fValues = []string{"7", "8", "9", "10", "11", "12"}
fields3 := toFields(fNames, fValues) fields3 := toFields(fNames, fValues)
oldObj, oldFlds, newFlds = c.Set("str", str1, fields3, 0) old = c.Set(object.New("str", str1, 0, 0, fields3))
expect(t, oldObj == str2) expect(t, old.Geo() == str2)
expect(t, reflect.DeepEqual(oldFlds, fields2)) expect(t, reflect.DeepEqual(old.Fields(), fields2))
expect(t, reflect.DeepEqual(newFlds, fields3))
}) })
t.Run("Delete", func(t *testing.T) { t.Run("Delete", func(t *testing.T) {
c := New() c := New()
c.Set("1", String("1"), field.List{}, 0) c.Set(object.New("1", String("1"), 0, 0, field.List{}))
c.Set("2", String("2"), field.List{}, 0) c.Set(object.New("2", String("2"), 0, 0, field.List{}))
c.Set("3", PO(1, 2), field.List{}, 0) c.Set(object.New("3", PO(1, 2), 0, 0, field.List{}))
expect(t, c.Count() == 3) expect(t, c.Count() == 3)
expect(t, c.StringCount() == 2) expect(t, c.StringCount() == 2)
@ -164,78 +148,30 @@ func TestCollectionSet(t *testing.T) {
expect(t, bounds(c) == geometry.Rect{ expect(t, bounds(c) == geometry.Rect{
Min: geometry.Point{X: 1, Y: 2}, Min: geometry.Point{X: 1, Y: 2},
Max: geometry.Point{X: 1, Y: 2}}) Max: geometry.Point{X: 1, Y: 2}})
var v geojson.Object var prev *object.Object
var ok bool
// var flds []float64
// var updated bool
// var updateCount int
v, _, ok = c.Delete("2") prev = c.Delete("2")
expect(t, v.String() == "2") expect(t, prev.Geo().String() == "2")
expect(t, ok)
expect(t, c.Count() == 2) expect(t, c.Count() == 2)
expect(t, c.StringCount() == 1) expect(t, c.StringCount() == 1)
expect(t, c.PointCount() == 1) expect(t, c.PointCount() == 1)
v, _, ok = c.Delete("1") prev = c.Delete("1")
expect(t, v.String() == "1") expect(t, prev.Geo().String() == "1")
expect(t, ok)
expect(t, c.Count() == 1) expect(t, c.Count() == 1)
expect(t, c.StringCount() == 0) expect(t, c.StringCount() == 0)
expect(t, c.PointCount() == 1) expect(t, c.PointCount() == 1)
// expect(t, len(c.FieldMap()) == 0) prev = c.Delete("3")
expect(t, prev.Geo().String() == `{"type":"Point","coordinates":[1,2]}`)
// _, flds, updated, ok = c.SetField("3", "hello", 123)
// expect(t, ok)
// expect(t, reflect.DeepEqual(flds, []float64{123}))
// expect(t, updated)
// expect(t, c.FieldMap()["hello"] == 0)
// _, flds, updated, ok = c.SetField("3", "hello", 1234)
// expect(t, ok)
// expect(t, reflect.DeepEqual(flds, []float64{1234}))
// expect(t, updated)
// _, flds, updated, ok = c.SetField("3", "hello", 1234)
// expect(t, ok)
// expect(t, reflect.DeepEqual(flds, []float64{1234}))
// expect(t, !updated)
// _, flds, updateCount, ok = c.SetFields("3",
// []string{"planet", "world"}, []float64{55, 66})
// expect(t, ok)
// expect(t, reflect.DeepEqual(flds, []float64{1234, 55, 66}))
// expect(t, updateCount == 2)
// expect(t, c.FieldMap()["hello"] == 0)
// expect(t, c.FieldMap()["planet"] == 1)
// expect(t, c.FieldMap()["world"] == 2)
v, _, ok = c.Delete("3")
expect(t, v.String() == `{"type":"Point","coordinates":[1,2]}`)
expect(t, ok)
expect(t, c.Count() == 0) expect(t, c.Count() == 0)
expect(t, c.StringCount() == 0) expect(t, c.StringCount() == 0)
expect(t, c.PointCount() == 0) expect(t, c.PointCount() == 0)
v, _, ok = c.Delete("3") prev = c.Delete("3")
expect(t, v == nil) expect(t, prev == nil)
expect(t, !ok)
expect(t, c.Count() == 0) expect(t, c.Count() == 0)
expect(t, bounds(c) == geometry.Rect{}) expect(t, bounds(c) == geometry.Rect{})
v, _, _, ok = c.Get("3") expect(t, c.Get("3") == nil)
expect(t, v == nil)
expect(t, !ok)
// _, _, _, ok = c.SetField("3", "hello", 123)
// expect(t, !ok)
// _, _, _, ok = c.SetFields("3", []string{"hello"}, []float64{123})
// expect(t, !ok)
// expect(t, c.TotalWeight() == 0)
// expect(t, c.FieldMap()["hello"] == 0)
// expect(t, c.FieldMap()["planet"] == 1)
// expect(t, c.FieldMap()["world"] == 2)
// expect(t, reflect.DeepEqual(
// c.FieldArr(), []string{"hello", "planet", "world"}),
// )
}) })
} }
@ -260,82 +196,82 @@ func TestCollectionScan(t *testing.T) {
c := New() c := New()
for _, i := range rand.Perm(N) { for _, i := range rand.Perm(N) {
id := fmt.Sprintf("%04d", i) id := fmt.Sprintf("%04d", i)
c.Set(id, String(id), makeFields( c.Set(object.New(id, String(id), 0, 0, makeFields(
field.Make("ex", id), field.Make("ex", id),
), 0) )))
} }
var n int var n int
var prevID string var prevID string
c.Scan(false, nil, nil, func(id string, obj geojson.Object, fields field.List) bool { c.Scan(false, nil, nil, func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, id > prevID) expect(t, o.ID() > prevID)
} }
expect(t, id == fieldValueAt(fields, 0)) expect(t, o.ID() == fieldValueAt(o.Fields(), 0))
n++ n++
prevID = id prevID = o.ID()
return true return true
}) })
expect(t, n == c.Count()) expect(t, n == c.Count())
n = 0 n = 0
c.Scan(true, nil, nil, func(id string, obj geojson.Object, fields field.List) bool { c.Scan(true, nil, nil, func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, id < prevID) expect(t, o.ID() < prevID)
} }
expect(t, id == fieldValueAt(fields, 0)) expect(t, o.ID() == fieldValueAt(o.Fields(), 0))
n++ n++
prevID = id prevID = o.ID()
return true return true
}) })
expect(t, n == c.Count()) expect(t, n == c.Count())
n = 0 n = 0
c.ScanRange("0060", "0070", false, nil, nil, c.ScanRange("0060", "0070", false, nil, nil,
func(id string, obj geojson.Object, fields field.List) bool { func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, id > prevID) expect(t, o.ID() > prevID)
} }
expect(t, id == fieldValueAt(fields, 0)) expect(t, o.ID() == fieldValueAt(o.Fields(), 0))
n++ n++
prevID = id prevID = o.ID()
return true return true
}) })
expect(t, n == 10) expect(t, n == 10)
n = 0 n = 0
c.ScanRange("0070", "0060", true, nil, nil, c.ScanRange("0070", "0060", true, nil, nil,
func(id string, obj geojson.Object, fields field.List) bool { func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, id < prevID) expect(t, o.ID() < prevID)
} }
expect(t, id == fieldValueAt(fields, 0)) expect(t, o.ID() == fieldValueAt(o.Fields(), 0))
n++ n++
prevID = id prevID = o.ID()
return true return true
}) })
expect(t, n == 10) expect(t, n == 10)
n = 0 n = 0
c.ScanGreaterOrEqual("0070", true, nil, nil, c.ScanGreaterOrEqual("0070", true, nil, nil,
func(id string, obj geojson.Object, fields field.List, ex int64) bool { func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, id < prevID) expect(t, o.ID() < prevID)
} }
expect(t, id == fieldValueAt(fields, 0)) expect(t, o.ID() == fieldValueAt(o.Fields(), 0))
n++ n++
prevID = id prevID = o.ID()
return true return true
}) })
expect(t, n == 71) expect(t, n == 71)
n = 0 n = 0
c.ScanGreaterOrEqual("0070", false, nil, nil, c.ScanGreaterOrEqual("0070", false, nil, nil,
func(id string, obj geojson.Object, fields field.List, ex int64) bool { func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, id > prevID) expect(t, o.ID() > prevID)
} }
expect(t, id == fieldValueAt(fields, 0)) expect(t, o.ID() == fieldValueAt(o.Fields(), 0))
n++ n++
prevID = id prevID = o.ID()
return true return true
}) })
expect(t, n == c.Count()-70) expect(t, n == c.Count()-70)
@ -356,58 +292,59 @@ func TestCollectionSearch(t *testing.T) {
for i, j := range rand.Perm(N) { for i, j := range rand.Perm(N) {
id := fmt.Sprintf("%04d", j) id := fmt.Sprintf("%04d", j)
ex := fmt.Sprintf("%04d", i) ex := fmt.Sprintf("%04d", i)
c.Set(id, String(ex), c.Set(object.New(id, String(ex),
0, 0,
makeFields( makeFields(
field.Make("i", ex), field.Make("i", ex),
field.Make("j", id), field.Make("j", id),
), 0) )))
} }
var n int var n int
var prevValue string var prevValue string
c.SearchValues(false, nil, nil, func(id string, obj geojson.Object, fields field.List) bool { c.SearchValues(false, nil, nil, func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, obj.String() > prevValue) expect(t, o.Geo().String() > prevValue)
} }
expect(t, id == fieldValueAt(fields, 1)) expect(t, o.ID() == fieldValueAt(o.Fields(), 1))
n++ n++
prevValue = obj.String() prevValue = o.Geo().String()
return true return true
}) })
expect(t, n == c.Count()) expect(t, n == c.Count())
n = 0 n = 0
c.SearchValues(true, nil, nil, func(id string, obj geojson.Object, fields field.List) bool { c.SearchValues(true, nil, nil, func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, obj.String() < prevValue) expect(t, o.Geo().String() < prevValue)
} }
expect(t, id == fieldValueAt(fields, 1)) expect(t, o.ID() == fieldValueAt(o.Fields(), 1))
n++ n++
prevValue = obj.String() prevValue = o.Geo().String()
return true return true
}) })
expect(t, n == c.Count()) expect(t, n == c.Count())
n = 0 n = 0
c.SearchValuesRange("0060", "0070", false, nil, nil, c.SearchValuesRange("0060", "0070", false, nil, nil,
func(id string, obj geojson.Object, fields field.List) bool { func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, obj.String() > prevValue) expect(t, o.Geo().String() > prevValue)
} }
expect(t, id == fieldValueAt(fields, 1)) expect(t, o.ID() == fieldValueAt(o.Fields(), 1))
n++ n++
prevValue = obj.String() prevValue = o.Geo().String()
return true return true
}) })
expect(t, n == 10) expect(t, n == 10)
n = 0 n = 0
c.SearchValuesRange("0070", "0060", true, nil, nil, c.SearchValuesRange("0070", "0060", true, nil, nil,
func(id string, obj geojson.Object, fields field.List) bool { func(o *object.Object) bool {
if n > 0 { if n > 0 {
expect(t, obj.String() < prevValue) expect(t, o.Geo().String() < prevValue)
} }
expect(t, id == fieldValueAt(fields, 1)) expect(t, o.ID() == fieldValueAt(o.Fields(), 1))
n++ n++
prevValue = obj.String() prevValue = o.Geo().String()
return true return true
}) })
expect(t, n == 10) expect(t, n == 10)
@ -415,41 +352,37 @@ func TestCollectionSearch(t *testing.T) {
func TestCollectionWeight(t *testing.T) { func TestCollectionWeight(t *testing.T) {
c := New() c := New()
c.Set("1", String("1"), field.List{}, 0) c.Set(object.New("1", String("1"), 0, 0, field.List{}))
expect(t, c.TotalWeight() > 0) expect(t, c.TotalWeight() > 0)
c.Delete("1") c.Delete("1")
expect(t, c.TotalWeight() == 0) expect(t, c.TotalWeight() == 0)
c.Set("1", String("1"), c.Set(object.New("1", String("1"), 0, 0,
toFields( toFields(
[]string{"a", "b", "c"}, []string{"a", "b", "c"},
[]string{"1", "2", "3"}, []string{"1", "2", "3"},
), ),
0, ))
)
expect(t, c.TotalWeight() > 0) expect(t, c.TotalWeight() > 0)
c.Delete("1") c.Delete("1")
expect(t, c.TotalWeight() == 0) expect(t, c.TotalWeight() == 0)
c.Set("1", String("1"), c.Set(object.New("1", String("1"), 0, 0,
toFields( toFields(
[]string{"a", "b", "c"}, []string{"a", "b", "c"},
[]string{"1", "2", "3"}, []string{"1", "2", "3"},
), ),
0, ))
) c.Set(object.New("2", String("2"), 0, 0,
c.Set("2", String("2"),
toFields( toFields(
[]string{"d", "e", "f"}, []string{"d", "e", "f"},
[]string{"4", "5", "6"}, []string{"4", "5", "6"},
), ),
0, ))
) c.Set(object.New("1", String("1"), 0, 0,
c.Set("1", String("1"),
toFields( toFields(
[]string{"d", "e", "f"}, []string{"d", "e", "f"},
[]string{"4", "5", "6"}, []string{"4", "5", "6"},
), ),
0, ))
)
c.Delete("1") c.Delete("1")
c.Delete("2") c.Delete("2")
expect(t, c.TotalWeight() == 0) expect(t, c.TotalWeight() == 0)
@ -484,77 +417,63 @@ func TestSpatialSearch(t *testing.T) {
q4, _ := geojson.Parse(gjson.Get(json, `features.#[id=="q4"]`).Raw, nil) q4, _ := geojson.Parse(gjson.Get(json, `features.#[id=="q4"]`).Raw, nil)
c := New() c := New()
c.Set("p1", p1, field.List{}, 0) c.Set(object.New("p1", p1, 0, 0, field.List{}))
c.Set("p2", p2, field.List{}, 0) c.Set(object.New("p2", p2, 0, 0, field.List{}))
c.Set("p3", p3, field.List{}, 0) c.Set(object.New("p3", p3, 0, 0, field.List{}))
c.Set("p4", p4, field.List{}, 0) c.Set(object.New("p4", p4, 0, 0, field.List{}))
c.Set("r1", r1, field.List{}, 0) c.Set(object.New("r1", r1, 0, 0, field.List{}))
c.Set("r2", r2, field.List{}, 0) c.Set(object.New("r2", r2, 0, 0, field.List{}))
c.Set("r3", r3, field.List{}, 0) c.Set(object.New("r3", r3, 0, 0, field.List{}))
var n int var n int
n = 0 n = 0
c.Within(q1, 0, nil, nil, c.Within(q1, 0, nil, nil, func(o *object.Object) bool {
func(id string, obj geojson.Object, _ field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 3) expect(t, n == 3)
n = 0 n = 0
c.Within(q2, 0, nil, nil, c.Within(q2, 0, nil, nil, func(o *object.Object) bool {
func(id string, obj geojson.Object, _ field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 7) expect(t, n == 7)
n = 0 n = 0
c.Within(q3, 0, nil, nil, c.Within(q3, 0, nil, nil, func(o *object.Object) bool {
func(id string, obj geojson.Object, _ field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 4) expect(t, n == 4)
n = 0 n = 0
c.Intersects(q1, 0, nil, nil, c.Intersects(q1, 0, nil, nil, func(o *object.Object) bool {
func(_ string, _ geojson.Object, _ field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 4) expect(t, n == 4)
n = 0 n = 0
c.Intersects(q2, 0, nil, nil, c.Intersects(q2, 0, nil, nil, func(o *object.Object) bool {
func(_ string, _ geojson.Object, _ field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 7) expect(t, n == 7)
n = 0 n = 0
c.Intersects(q3, 0, nil, nil, c.Intersects(q3, 0, nil, nil, func(o *object.Object) bool {
func(_ string, _ geojson.Object, _ field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 5) expect(t, n == 5)
n = 0 n = 0
c.Intersects(q3, 0, nil, nil, c.Intersects(q3, 0, nil, nil, func(o *object.Object) bool {
func(_ string, _ geojson.Object, _ field.List) bool { n++
n++ return n <= 1
return n <= 1 })
},
)
expect(t, n == 2) expect(t, n == 2)
var items []geojson.Object var items []geojson.Object
@ -564,15 +483,13 @@ func TestSpatialSearch(t *testing.T) {
lastDist := float64(-1) lastDist := float64(-1)
distsMonotonic := true distsMonotonic := true
c.Nearby(q4, nil, nil, c.Nearby(q4, nil, nil, func(o *object.Object, dist float64) bool {
func(id string, obj geojson.Object, fields field.List, dist float64) bool { if dist < lastDist {
if dist < lastDist { distsMonotonic = false
distsMonotonic = false }
} items = append(items, o.Geo())
items = append(items, obj) return true
return true })
},
)
expect(t, len(items) == 7) expect(t, len(items) == 7)
expect(t, distsMonotonic) expect(t, distsMonotonic)
expect(t, reflect.DeepEqual(items, exitems)) expect(t, reflect.DeepEqual(items, exitems))
@ -590,72 +507,60 @@ func TestCollectionSparse(t *testing.T) {
x := (r.Max.X-r.Min.X)*rand.Float64() + r.Min.X x := (r.Max.X-r.Min.X)*rand.Float64() + r.Min.X
y := (r.Max.Y-r.Min.Y)*rand.Float64() + r.Min.Y y := (r.Max.Y-r.Min.Y)*rand.Float64() + r.Min.Y
point := PO(x, y) point := PO(x, y)
c.Set(fmt.Sprintf("%d", i), point, field.List{}, 0) c.Set(object.New(fmt.Sprintf("%d", i), point, 0, 0, field.List{}))
} }
var n int var n int
n = 0 n = 0
c.Within(rect, 1, nil, nil, c.Within(rect, 1, nil, nil, func(o *object.Object) bool {
func(id string, obj geojson.Object, fields field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 4) expect(t, n == 4)
n = 0 n = 0
c.Within(rect, 2, nil, nil, c.Within(rect, 2, nil, nil, func(o *object.Object) bool {
func(id string, obj geojson.Object, fields field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 16) expect(t, n == 16)
n = 0 n = 0
c.Within(rect, 3, nil, nil, c.Within(rect, 3, nil, nil, func(o *object.Object) bool {
func(id string, obj geojson.Object, fields field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 64) expect(t, n == 64)
n = 0 n = 0
c.Within(rect, 3, nil, nil, c.Within(rect, 3, nil, nil, func(o *object.Object) bool {
func(id string, obj geojson.Object, fields field.List) bool { n++
n++ return n <= 30
return n <= 30 })
},
)
expect(t, n == 31) expect(t, n == 31)
n = 0 n = 0
c.Intersects(rect, 3, nil, nil, c.Intersects(rect, 3, nil, nil, func(o *object.Object) bool {
func(id string, _ geojson.Object, _ field.List) bool { n++
n++ return true
return true })
},
)
expect(t, n == 64) expect(t, n == 64)
n = 0 n = 0
c.Intersects(rect, 3, nil, nil, c.Intersects(rect, 3, nil, nil, func(o *object.Object) bool {
func(id string, _ geojson.Object, _ field.List) bool { n++
n++ return n <= 30
return n <= 30 })
},
)
expect(t, n == 31) expect(t, n == 31)
} }
func testCollectionVerifyContents(t *testing.T, c *Collection, objs map[string]geojson.Object) { func testCollectionVerifyContents(t *testing.T, c *Collection, objs map[string]geojson.Object) {
for id, o2 := range objs { for id, o2 := range objs {
o1, _, _, ok := c.Get(id) o := c.Get(id)
if !ok { if o == nil {
t.Fatalf("ok[%s] = false, expect true", id) t.Fatalf("ok[%s] = false, expect true", id)
} }
j1 := string(o1.AppendJSON(nil)) j1 := string(o.Geo().AppendJSON(nil))
j2 := string(o2.AppendJSON(nil)) j2 := string(o2.AppendJSON(nil))
if j1 != j2 { if j1 != j2 {
t.Fatalf("j1 == %s, expect %s", j1, j2) t.Fatalf("j1 == %s, expect %s", j1, j2)
@ -682,7 +587,7 @@ func TestManyCollections(t *testing.T) {
col = New() col = New()
colsM[key] = col colsM[key] = col
} }
col.Set(id, obj, field.List{}, 0) col.Set(object.New(id, obj, 0, 0, field.List{}))
k++ k++
} }
} }
@ -693,7 +598,7 @@ func TestManyCollections(t *testing.T) {
Min: geometry.Point{X: -180, Y: 30}, Min: geometry.Point{X: -180, Y: 30},
Max: geometry.Point{X: 34, Y: 100}, Max: geometry.Point{X: 34, Y: 100},
} }
col.geoSearch(bbox, func(id string, obj geojson.Object, fields field.List) bool { col.geoSearch(bbox, func(o *object.Object) bool {
//println(id) //println(id)
return true return true
}) })
@ -736,7 +641,7 @@ func benchmarkInsert(t *testing.B, nFields int) {
col := New() col := New()
t.ResetTimer() t.ResetTimer()
for i := 0; i < t.N; i++ { for i := 0; i < t.N; i++ {
col.Set(items[i].id, items[i].object, items[i].fields, 0) col.Set(object.New(items[i].id, items[i].object, 0, 0, items[i].fields))
} }
} }
@ -760,12 +665,12 @@ func benchmarkReplace(t *testing.B, nFields int) {
} }
col := New() col := New()
for i := 0; i < t.N; i++ { for i := 0; i < t.N; i++ {
col.Set(items[i].id, items[i].object, items[i].fields, 0) col.Set(object.New(items[i].id, items[i].object, 0, 0, items[i].fields))
} }
t.ResetTimer() t.ResetTimer()
for _, i := range rand.Perm(t.N) { for _, i := range rand.Perm(t.N) {
o, _, _ := col.Set(items[i].id, items[i].object, field.List{}, 0) o := col.Set(object.New(items[i].id, items[i].object, 0, 0, field.List{}))
if o != items[i].object { if o.Geo() != items[i].object {
t.Fatal("shoot!") t.Fatal("shoot!")
} }
} }
@ -791,12 +696,12 @@ func benchmarkGet(t *testing.B, nFields int) {
} }
col := New() col := New()
for i := 0; i < t.N; i++ { for i := 0; i < t.N; i++ {
col.Set(items[i].id, items[i].object, items[i].fields, 0) col.Set(object.New(items[i].id, items[i].object, 0, 0, items[i].fields))
} }
t.ResetTimer() t.ResetTimer()
for _, i := range rand.Perm(t.N) { for _, i := range rand.Perm(t.N) {
o, _, _, _ := col.Get(items[i].id) o := col.Get(items[i].id)
if o != items[i].object { if o.Geo() != items[i].object {
t.Fatal("shoot!") t.Fatal("shoot!")
} }
} }
@ -822,12 +727,12 @@ func benchmarkRemove(t *testing.B, nFields int) {
} }
col := New() col := New()
for i := 0; i < t.N; i++ { for i := 0; i < t.N; i++ {
col.Set(items[i].id, items[i].object, items[i].fields, 0) col.Set(object.New(items[i].id, items[i].object, 0, 0, items[i].fields))
} }
t.ResetTimer() t.ResetTimer()
for _, i := range rand.Perm(t.N) { for _, i := range rand.Perm(t.N) {
o, _, _ := col.Delete(items[i].id) prev := col.Delete(items[i].id)
if o != items[i].object { if prev.Geo() != items[i].object {
t.Fatal("shoot!") t.Fatal("shoot!")
} }
} }
@ -853,12 +758,12 @@ func benchmarkScan(t *testing.B, nFields int) {
} }
col := New() col := New()
for i := 0; i < t.N; i++ { for i := 0; i < t.N; i++ {
col.Set(items[i].id, items[i].object, items[i].fields, 0) col.Set(object.New(items[i].id, items[i].object, 0, 0, items[i].fields))
} }
t.ResetTimer() t.ResetTimer()
for i := 0; i < t.N; i++ { for i := 0; i < t.N; i++ {
var scanIteration int var scanIteration int
col.Scan(true, nil, nil, func(id string, obj geojson.Object, fields field.List) bool { col.Scan(true, nil, nil, func(o *object.Object) bool {
scanIteration++ scanIteration++
return scanIteration <= 500 return scanIteration <= 500
}) })

19
internal/collection/geodesic.go
View File

@ -1,12 +1,23 @@
package collection package collection
import "math" import (
"math"
func geodeticDistAlgo[T any](center [2]float64) ( "github.com/tidwall/tile38/internal/object"
algo func(min, max [2]float64, data T, item bool) (dist float64), )
func geodeticDistAlgo(center [2]float64) (
algo func(min, max [2]float64, obj *object.Object, item bool) (dist float64),
) { ) {
const earthRadius = 6371e3 const earthRadius = 6371e3
return func(min, max [2]float64, data T, item bool) (dist float64) { return func(min, max [2]float64, obj *object.Object, item bool) (dist float64) {
if item {
r := obj.Rect()
min[0] = r.Min.X
min[1] = r.Min.Y
max[0] = r.Max.X
max[1] = r.Max.Y
}
return earthRadius * pointRectDistGeodeticDeg( return earthRadius * pointRectDistGeodeticDeg(
center[1], center[0], center[1], center[0],
min[1], min[0], min[1], min[0],

12
internal/field/list_binary.go
View File

@ -151,6 +151,9 @@ func (fields List) Set(field Field) List {
func delfield(b []byte, s, e int) *byte { func delfield(b []byte, s, e int) *byte {
totallen := s + (len(b) - e) totallen := s + (len(b) - e)
if totallen == 0 {
return nil
}
var psz [10]byte var psz [10]byte
pn := binary.PutUvarint(psz[:], uint64(totallen)) pn := binary.PutUvarint(psz[:], uint64(totallen))
plen := pn + totallen plen := pn + totallen
@ -347,3 +350,12 @@ func (fields List) Weight() int {
x, n := uvarint(*(*[]byte)(unsafe.Pointer(&bytes{fields.p, 10, 10}))) x, n := uvarint(*(*[]byte)(unsafe.Pointer(&bytes{fields.p, 10, 10})))
return x + n return x + n
} }
// Bytes returns the raw bytes (including the header)
func (fields List) Bytes() []byte {
if fields.p == nil {
return nil
}
x, n := uvarint(*(*[]byte)(unsafe.Pointer(&bytes{fields.p, 10, 10})))
return (*(*[]byte)(unsafe.Pointer(&bytes{fields.p, 0, n + x})))
}

0
internal/field/list_array.go → internal/field/list_struct.go
View File

96
internal/object/object.go Normal file
View File

@ -0,0 +1,96 @@
package object
import (
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
"github.com/tidwall/tile38/internal/field"
)
type Object struct {
id string
geo geojson.Object
created int64 // unix nano created
expires int64 // unix nano expiration
fields field.List
}
func (o *Object) ID() string {
if o == nil {
return ""
}
return o.id
}
func (o *Object) Fields() field.List {
if o == nil {
return field.List{}
}
return o.fields
}
func (o *Object) Created() int64 {
if o == nil {
return 0
}
return o.created
}
func (o *Object) Expires() int64 {
if o == nil {
return 0
}
return o.expires
}
func (o *Object) Rect() geometry.Rect {
if o == nil || o.geo == nil {
return geometry.Rect{}
}
return o.geo.Rect()
}
func (o *Object) Geo() geojson.Object {
if o == nil || o.geo == nil {
return nil
}
return o.geo
}
func (o *Object) String() string {
if o == nil || o.geo == nil {
return ""
}
return o.geo.String()
}
func (o *Object) IsSpatial() bool {
_, ok := o.geo.(geojson.Spatial)
return ok
}
func (o *Object) Weight() int {
if o == nil {
return 0
}
var weight int
weight += len(o.ID())
if o.IsSpatial() {
weight += o.Geo().NumPoints() * 16
} else {
weight += len(o.Geo().String())
}
weight += o.Fields().Weight()
return weight
}
func New(id string, geo geojson.Object, created, expires int64,
fields field.List,
) *Object {
return &Object{
id: id,
geo: geo,
created: created,
expires: expires,
fields: fields,
}
}

18
internal/object/object_test.go Normal file
View File

@ -0,0 +1,18 @@
package object
import (
"testing"
"github.com/tidwall/assert"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
"github.com/tidwall/tile38/internal/field"
)
func P(x, y float64) geojson.Object {
return geojson.NewSimplePoint(geometry.Point{X: 10, Y: 20})
}
func TestObject(t *testing.T) {
o := New("hello", P(10, 20), 0, 99, field.List{})
assert.Assert(o.ID() == "hello")
}

8
internal/server/aof.go
View File

@ -226,8 +226,8 @@ func (s *Server) getQueueCandidates(d *commandDetails) []*Hook {
return true return true
}) })
// look for candidates that might "cross" geofences // look for candidates that might "cross" geofences
if d.oldObj != nil && d.obj != nil && s.hookCross.Len() > 0 { if d.old != nil && d.obj != nil && s.hookCross.Len() > 0 {
r1, r2 := d.oldObj.Rect(), d.obj.Rect() r1, r2 := d.old.Rect(), d.obj.Rect()
s.hookCross.Search( s.hookCross.Search(
[2]float64{ [2]float64{
math.Min(r1.Min.X, r2.Min.X), math.Min(r1.Min.X, r2.Min.X),
@ -246,8 +246,8 @@ func (s *Server) getQueueCandidates(d *commandDetails) []*Hook {
}) })
} }
// look for candidates that overlap the old object // look for candidates that overlap the old object
if d.oldObj != nil { if d.old != nil {
r1 := d.oldObj.Rect() r1 := d.old.Rect()
s.hookTree.Search( s.hookTree.Search(
[2]float64{r1.Min.X, r1.Min.Y}, [2]float64{r1.Min.X, r1.Min.Y},
[2]float64{r1.Max.X, r1.Max.Y}, [2]float64{r1.Max.X, r1.Max.Y},

20
internal/server/aofshrink.go
View File

@ -8,11 +8,11 @@ import (
"time" "time"
"github.com/tidwall/btree" "github.com/tidwall/btree"
"github.com/tidwall/geojson"
"github.com/tidwall/tile38/core" "github.com/tidwall/tile38/core"
"github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/log"
"github.com/tidwall/tile38/internal/object"
) )
const maxkeys = 8 const maxkeys = 8
@ -97,10 +97,10 @@ func (s *Server) aofshrink() {
var now = time.Now().UnixNano() // used for expiration var now = time.Now().UnixNano() // used for expiration
var count = 0 // the object count var count = 0 // the object count
col.ScanGreaterOrEqual(nextid, false, nil, nil, col.ScanGreaterOrEqual(nextid, false, nil, nil,
func(id string, obj geojson.Object, fields field.List, ex int64) bool { func(o *object.Object) bool {
if count == maxids { if count == maxids {
// we reached the max number of ids for one batch // we reached the max number of ids for one batch
nextid = id nextid = o.ID()
idsdone = false idsdone = false
return false return false
} }
@ -108,8 +108,8 @@ func (s *Server) aofshrink() {
values = values[:0] values = values[:0]
values = append(values, "set") values = append(values, "set")
values = append(values, keys[0]) values = append(values, keys[0])
values = append(values, id) values = append(values, o.ID())
fields.Scan(func(f field.Field) bool { o.Fields().Scan(func(f field.Field) bool {
if !f.Value().IsZero() { if !f.Value().IsZero() {
values = append(values, "field") values = append(values, "field")
values = append(values, f.Name()) values = append(values, f.Name())
@ -117,8 +117,8 @@ func (s *Server) aofshrink() {
} }
return true return true
}) })
if ex != 0 { if o.Expires() != 0 {
ttl := math.Floor(float64(ex-now)/float64(time.Second)*10) / 10 ttl := math.Floor(float64(o.Expires()-now)/float64(time.Second)*10) / 10
if ttl < 0.1 { if ttl < 0.1 {
// always leave a little bit of ttl. // always leave a little bit of ttl.
ttl = 0.1 ttl = 0.1
@ -126,12 +126,12 @@ func (s *Server) aofshrink() {
values = append(values, "ex") values = append(values, "ex")
values = append(values, strconv.FormatFloat(ttl, 'f', -1, 64)) values = append(values, strconv.FormatFloat(ttl, 'f', -1, 64))
} }
if objIsSpatial(obj) { if objIsSpatial(o.Geo()) {
values = append(values, "object") values = append(values, "object")
values = append(values, string(obj.AppendJSON(nil))) values = append(values, string(o.Geo().AppendJSON(nil)))
} else { } else {
values = append(values, "string") values = append(values, "string")
values = append(values, obj.String()) values = append(values, o.Geo().String())
} }
// append the values to the aof buffer // append the values to the aof buffer

232
internal/server/crud.go
View File

@ -14,30 +14,9 @@ import (
"github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/glob" "github.com/tidwall/tile38/internal/glob"
"github.com/tidwall/tile38/internal/object"
) )
// type fvt struct {
// field string
// value float64
// }
// func orderFields(fmap map[string]int, farr []string, fields []float64) []fvt {
// var fv fvt
// var idx int
// fvs := make([]fvt, 0, len(fmap))
// for _, field := range farr {
// idx = fmap[field]
// if idx < len(fields) {
// fv.field = field
// fv.value = fields[idx]
// if fv.value != 0 {
// fvs = append(fvs, fv)
// }
// }
// }
// return fvs
// }
func (s *Server) cmdBounds(msg *Message) (resp.Value, error) { func (s *Server) cmdBounds(msg *Message) (resp.Value, error) {
start := time.Now() start := time.Now()
vs := msg.Args[1:] vs := msg.Args[1:]
@ -150,7 +129,8 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
} }
return NOMessage, errKeyNotFound return NOMessage, errKeyNotFound
} }
o, fields, _, ok := col.Get(id) o := col.Get(id)
ok = o != nil
if !ok { if !ok {
if msg.OutputType == RESP { if msg.OutputType == RESP {
return resp.NullValue(), nil return resp.NullValue(), nil
@ -174,17 +154,17 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
case "object": case "object":
if msg.OutputType == JSON { if msg.OutputType == JSON {
buf.WriteString(`,"object":`) buf.WriteString(`,"object":`)
buf.WriteString(string(o.AppendJSON(nil))) buf.WriteString(string(o.Geo().AppendJSON(nil)))
} else { } else {
vals = append(vals, resp.StringValue(o.String())) vals = append(vals, resp.StringValue(o.Geo().String()))
} }
case "point": case "point":
if msg.OutputType == JSON { if msg.OutputType == JSON {
buf.WriteString(`,"point":`) buf.WriteString(`,"point":`)
buf.Write(appendJSONSimplePoint(nil, o)) buf.Write(appendJSONSimplePoint(nil, o.Geo()))
} else { } else {
point := o.Center() point := o.Geo().Center()
z := extractZCoordinate(o) z := extractZCoordinate(o.Geo())
if z != 0 { if z != 0 {
vals = append(vals, resp.ArrayValue([]resp.Value{ vals = append(vals, resp.ArrayValue([]resp.Value{
resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)), resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)),
@ -209,7 +189,7 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
if err != nil || precision < 1 || precision > 12 { if err != nil || precision < 1 || precision > 12 {
return NOMessage, errInvalidArgument(sprecision) return NOMessage, errInvalidArgument(sprecision)
} }
center := o.Center() center := o.Geo().Center()
p := geohash.EncodeWithPrecision(center.Y, center.X, uint(precision)) p := geohash.EncodeWithPrecision(center.Y, center.X, uint(precision))
if msg.OutputType == JSON { if msg.OutputType == JSON {
buf.WriteString(`"` + p + `"`) buf.WriteString(`"` + p + `"`)
@ -219,7 +199,7 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
case "bounds": case "bounds":
if msg.OutputType == JSON { if msg.OutputType == JSON {
buf.WriteString(`,"bounds":`) buf.WriteString(`,"bounds":`)
buf.Write(appendJSONSimpleBounds(nil, o)) buf.Write(appendJSONSimpleBounds(nil, o.Geo()))
} else { } else {
bbox := o.Rect() bbox := o.Rect()
vals = append(vals, resp.ArrayValue([]resp.Value{ vals = append(vals, resp.ArrayValue([]resp.Value{
@ -239,14 +219,14 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
return NOMessage, errInvalidNumberOfArguments return NOMessage, errInvalidNumberOfArguments
} }
if withfields { if withfields {
nfields := fields.Len() nfields := o.Fields().Len()
if nfields > 0 { if nfields > 0 {
fvals := make([]resp.Value, 0, nfields*2) fvals := make([]resp.Value, 0, nfields*2)
if msg.OutputType == JSON { if msg.OutputType == JSON {
buf.WriteString(`,"fields":{`) buf.WriteString(`,"fields":{`)
} }
var i int var i int
fields.Scan(func(f field.Field) bool { o.Fields().Scan(func(f field.Field) bool {
if msg.OutputType == JSON { if msg.OutputType == JSON {
if i > 0 { if i > 0 {
buf.WriteString(`,`) buf.WriteString(`,`)
@ -282,57 +262,64 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
return NOMessage, nil return NOMessage, nil
} }
func (s *Server) cmdDel(msg *Message) (res resp.Value, d commandDetails, err error) { // DEL key id [ERRON404]
func (s *Server) cmdDel(msg *Message) (resp.Value, commandDetails, error) {
start := time.Now() start := time.Now()
vs := msg.Args[1:]
var ok bool // >> Args
if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
err = errInvalidNumberOfArguments args := msg.Args
return if len(args) < 3 {
} return retwerr(errInvalidNumberOfArguments)
if vs, d.id, ok = tokenval(vs); !ok || d.id == "" {
err = errInvalidNumberOfArguments
return
} }
key := args[1]
id := args[2]
erron404 := false erron404 := false
if len(vs) > 0 { for i := 3; i < len(args); i++ {
_, arg, ok := tokenval(vs) switch strings.ToLower(args[i]) {
if ok && strings.ToLower(arg) == "erron404" { case "erron404":
erron404 = true erron404 = true
vs = vs[1:] default:
} else { return retwerr(errInvalidArgument(args[i]))
err = errInvalidArgument(arg)
return
} }
} }
if len(vs) != 0 {
err = errInvalidNumberOfArguments // >> Operation
return
} updated := false
found := false var old *object.Object
col, _ := s.cols.Get(d.key) col, _ := s.cols.Get(key)
if col != nil { if col != nil {
d.obj, d.fields, ok = col.Delete(d.id) old = col.Delete(id)
if ok { if old != nil {
if col.Count() == 0 { if col.Count() == 0 {
s.cols.Delete(d.key) s.cols.Delete(key)
} }
found = true updated = true
} else if erron404 { } else if erron404 {
err = errIDNotFound return retwerr(errIDNotFound)
return
} }
} else if erron404 { } else if erron404 {
err = errKeyNotFound return retwerr(errKeyNotFound)
return
} }
s.groupDisconnectObject(d.key, d.id) s.groupDisconnectObject(key, id)
// >> Response
var d commandDetails
d.command = "del" d.command = "del"
d.updated = found d.key = key
d.obj = old
d.updated = updated
d.timestamp = time.Now() d.timestamp = time.Now()
var res resp.Value
switch msg.OutputType { switch msg.OutputType {
case JSON: case JSON:
res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") res = resp.StringValue(`{"ok":true,"elapsed":"` +
time.Since(start).String() + "\"}")
case RESP: case RESP:
if d.updated { if d.updated {
res = resp.IntegerValue(1) res = resp.IntegerValue(1)
@ -340,7 +327,7 @@ func (s *Server) cmdDel(msg *Message) (res resp.Value, d commandDetails, err err
res = resp.IntegerValue(0) res = resp.IntegerValue(0)
} }
} }
return return res, d, nil
} }
func (s *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, err error) { func (s *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, err error) {
@ -360,14 +347,14 @@ func (s *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, err er
return return
} }
now := time.Now() now := time.Now()
iter := func(id string, o geojson.Object, fields field.List) bool { iter := func(o *object.Object) bool {
if match, _ := glob.Match(d.pattern, id); match { if match, _ := glob.Match(d.pattern, o.ID()); match {
d.children = append(d.children, &commandDetails{ d.children = append(d.children, &commandDetails{
command: "del", command: "del",
updated: true, updated: true,
timestamp: now, timestamp: now,
key: d.key, key: d.key,
id: id, obj: o,
}) })
} }
return true return true
@ -384,14 +371,15 @@ func (s *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, err er
} }
var atLeastOneNotDeleted bool var atLeastOneNotDeleted bool
for i, dc := range d.children { for i, dc := range d.children {
dc.obj, dc.fields, ok = col.Delete(dc.id) old := col.Delete(dc.obj.ID())
if !ok { if old == nil {
d.children[i].command = "?" d.children[i].command = "?"
atLeastOneNotDeleted = true atLeastOneNotDeleted = true
} else { } else {
dc.obj = old
d.children[i] = dc d.children[i] = dc
} }
s.groupDisconnectObject(dc.key, dc.id) s.groupDisconnectObject(dc.key, dc.obj.ID())
} }
if atLeastOneNotDeleted { if atLeastOneNotDeleted {
var nchildren []*commandDetails var nchildren []*commandDetails
@ -565,7 +553,7 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
var ex int64 var ex int64
var xx bool var xx bool
var nx bool var nx bool
var obj geojson.Object var oobj geojson.Object
args := msg.Args args := msg.Args
if len(args) < 3 { if len(args) < 3 {
@ -614,7 +602,7 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
} }
str := args[i+1] str := args[i+1]
i += 1 i += 1
obj = collection.String(str) oobj = collection.String(str)
case "point": case "point":
if i+2 >= len(args) { if i+2 >= len(args) {
return retwerr(errInvalidNumberOfArguments) return retwerr(errInvalidNumberOfArguments)
@ -642,9 +630,9 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
return retwerr(errInvalidArgument(slon)) return retwerr(errInvalidArgument(slon))
} }
if !hasZ { if !hasZ {
obj = geojson.NewPoint(geometry.Point{X: x, Y: y}) oobj = geojson.NewPoint(geometry.Point{X: x, Y: y})
} else { } else {
obj = geojson.NewPointZ(geometry.Point{X: x, Y: y}, z) oobj = geojson.NewPointZ(geometry.Point{X: x, Y: y}, z)
} }
case "bounds": case "bounds":
if i+4 >= len(args) { if i+4 >= len(args) {
@ -659,7 +647,7 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
} }
} }
i += 4 i += 4
obj = geojson.NewRect(geometry.Rect{ oobj = geojson.NewRect(geometry.Rect{
Min: geometry.Point{X: vals[1], Y: vals[0]}, Min: geometry.Point{X: vals[1], Y: vals[0]},
Max: geometry.Point{X: vals[3], Y: vals[2]}, Max: geometry.Point{X: vals[3], Y: vals[2]},
}) })
@ -670,7 +658,7 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
shash := args[i+1] shash := args[i+1]
i += 1 i += 1
lat, lon := geohash.Decode(shash) lat, lon := geohash.Decode(shash)
obj = geojson.NewPoint(geometry.Point{X: lon, Y: lat}) oobj = geojson.NewPoint(geometry.Point{X: lon, Y: lat})
case "object": case "object":
if i+1 >= len(args) { if i+1 >= len(args) {
return retwerr(errInvalidNumberOfArguments) return retwerr(errInvalidNumberOfArguments)
@ -678,7 +666,7 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
json := args[i+1] json := args[i+1]
i += 1 i += 1
var err error var err error
obj, err = geojson.Parse(json, &s.geomParseOpts) oobj, err = geojson.Parse(json, &s.geomParseOpts)
if err != nil { if err != nil {
return retwerr(err) return retwerr(err)
} }
@ -702,7 +690,10 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
var ofields field.List var ofields field.List
if !nada { if !nada {
_, ofields, _, ok = col.Get(id) o := col.Get(id)
if o != nil {
ofields = o.Fields()
}
if xx || nx { if xx || nx {
if (nx && ok) || (xx && !ok) { if (nx && ok) || (xx && !ok) {
nada = true nada = true
@ -730,18 +721,16 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
ofields = ofields.Set(f) ofields = ofields.Set(f)
} }
oldObj, oldFields, newFields := col.Set(id, obj, ofields, ex) obj := object.New(id, oobj, 0, ex, ofields)
old := col.Set(obj)
// >> Response // >> Response
var d commandDetails var d commandDetails
d.command = "set" d.command = "set"
d.key = key d.key = key
d.id = id
d.obj = obj d.obj = obj
d.oldObj = oldObj d.old = old
d.oldFields = oldFields
d.fields = newFields
d.updated = true // perhaps we should do a diff on the previous object? d.updated = true // perhaps we should do a diff on the previous object?
d.timestamp = time.Now() d.timestamp = time.Now()
@ -811,12 +800,14 @@ func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) {
if !ok { if !ok {
return retwerr(errKeyNotFound) return retwerr(errKeyNotFound)
} }
obj, ofields, ex, ok := col.Get(id) o := col.Get(id)
ok = o != nil
if !(ok || xx) { if !(ok || xx) {
return retwerr(errIDNotFound) return retwerr(errIDNotFound)
} }
if ok { if ok {
ofields := o.Fields()
for _, f := range fields { for _, f := range fields {
prev := ofields.Get(f.Name()) prev := ofields.Get(f.Name())
if !prev.Value().Equals(f.Value()) { if !prev.Value().Equals(f.Value()) {
@ -824,11 +815,11 @@ func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) {
updateCount++ updateCount++
} }
} }
col.Set(id, obj, ofields, ex) obj := object.New(id, o.Geo(), 0, o.Expires(), ofields)
d.obj = obj col.Set(obj)
d.command = "fset" d.command = "fset"
d.key = key d.key = key
d.id = id d.obj = obj
d.timestamp = time.Now() d.timestamp = time.Now()
d.updated = updateCount > 0 d.updated = updateCount > 0
} }
@ -861,21 +852,22 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) {
return retwerr(errInvalidArgument(svalue)) return retwerr(errInvalidArgument(svalue))
} }
var ok bool var ok bool
var obj *object.Object
col, _ := s.cols.Get(key) col, _ := s.cols.Get(key)
if col != nil { if col != nil {
// replace the expiration by getting the old objec // replace the expiration by getting the old objec
ex := time.Now().Add(time.Duration(float64(time.Second) * value)).UnixNano() ex := time.Now().Add(time.Duration(float64(time.Second) * value)).UnixNano()
var obj geojson.Object o := col.Get(id)
var fields field.List ok = o != nil
obj, fields, _, ok = col.Get(id)
if ok { if ok {
col.Set(id, obj, fields, ex) obj = object.New(id, o.Geo(), 0, ex, o.Fields())
col.Set(obj)
} }
} }
var d commandDetails var d commandDetails
if ok { if ok {
d.key = key d.key = key
d.id = id d.obj = obj
d.command = "expire" d.command = "expire"
d.updated = true d.updated = true
d.timestamp = time.Now() d.timestamp = time.Now()
@ -909,41 +901,35 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) {
return retwerr(errInvalidNumberOfArguments) return retwerr(errInvalidNumberOfArguments)
} }
key, id := args[1], args[2] key, id := args[1], args[2]
var cleared bool
var ok bool
col, _ := s.cols.Get(key) col, _ := s.cols.Get(key)
if col != nil { if col == nil {
var ex int64
_, _, ex, ok = col.Get(id)
if ok && ex != 0 {
var obj geojson.Object
var fields field.List
obj, fields, _, ok = col.Get(id)
if ok {
col.Set(id, obj, fields, 0)
}
if ok {
cleared = true
}
}
}
if !ok {
if msg.OutputType == RESP { if msg.OutputType == RESP {
return resp.IntegerValue(0), commandDetails{}, nil return resp.IntegerValue(0), commandDetails{}, nil
} }
if col == nil { return retwerr(errKeyNotFound)
return retwerr(errKeyNotFound) }
o := col.Get(id)
if o == nil {
if msg.OutputType == RESP {
return resp.IntegerValue(0), commandDetails{}, nil
} }
return retwerr(errIDNotFound) return retwerr(errIDNotFound)
} }
var obj *object.Object
var cleared bool
if o.Expires() != 0 {
obj = object.New(id, o.Geo(), 0, 0, o.Fields())
col.Set(obj)
cleared = true
}
var res resp.Value var res resp.Value
var d commandDetails var d commandDetails
d.key = key
d.id = id
d.command = "persist" d.command = "persist"
d.key = key
d.obj = obj
d.updated = cleared d.updated = cleared
d.timestamp = time.Now() d.timestamp = time.Now()
@ -973,15 +959,15 @@ func (s *Server) cmdTTL(msg *Message) (resp.Value, error) {
var ok2 bool var ok2 bool
col, _ := s.cols.Get(key) col, _ := s.cols.Get(key)
if col != nil { if col != nil {
var ex int64 o := col.Get(id)
_, _, ex, ok = col.Get(id) ok = o != nil
if ok { if ok {
if ex != 0 { if o.Expires() != 0 {
now := start.UnixNano() now := start.UnixNano()
if now > ex { if now > o.Expires() {
ok2 = false ok2 = false
} else { } else {
v = float64(ex-now) / float64(time.Second) v = float64(o.Expires()-now) / float64(time.Second)
if v < 0 { if v < 0 {
v = 0 v = 0
} }

7
internal/server/expire.go
View File

@ -5,6 +5,7 @@ import (
"github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/log"
"github.com/tidwall/tile38/internal/object"
) )
const bgExpireDelay = time.Second / 10 const bgExpireDelay = time.Second / 10
@ -31,11 +32,11 @@ func (s *Server) backgroundExpireObjects(now time.Time) {
nano := now.UnixNano() nano := now.UnixNano()
var msgs []*Message var msgs []*Message
s.cols.Scan(func(key string, col *collection.Collection) bool { s.cols.Scan(func(key string, col *collection.Collection) bool {
col.ScanExpires(func(id string, expires int64) bool { col.ScanExpires(func(o *object.Object) bool {
if nano < expires { if nano < o.Expires() {
return false return false
} }
msgs = append(msgs, &Message{Args: []string{"del", key, id}}) msgs = append(msgs, &Message{Args: []string{"del", key, o.ID()}})
return true return true
}) })
return true return true

143
internal/server/fence.go
View File

@ -12,6 +12,7 @@ import (
"github.com/tidwall/gjson" "github.com/tidwall/gjson"
"github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/glob" "github.com/tidwall/tile38/internal/glob"
"github.com/tidwall/tile38/internal/object"
) )
// FenceMatch executes a fence match returns back json messages for fence detection. // FenceMatch executes a fence match returns back json messages for fence detection.
@ -81,10 +82,13 @@ func fenceMatch(
`,"time":` + jsonTimeFormat(details.timestamp) + `}`, `,"time":` + jsonTimeFormat(details.timestamp) + `}`,
} }
} }
if !multiGlobMatch(fence.globs, details.id) { if details.obj == nil {
return nil return nil
} }
if details.obj == nil || !objIsSpatial(details.obj) { if !multiGlobMatch(fence.globs, details.obj.ID()) {
return nil
}
if !objIsSpatial(details.obj.Geo()) {
return nil return nil
} }
if details.command == "fset" { if details.command == "fset" {
@ -97,7 +101,7 @@ func fenceMatch(
return []string{ return []string{
`{"command":"del"` + hookJSONString(hookName, metas) + `{"command":"del"` + hookJSONString(hookName, metas) +
`,"key":` + jsonString(details.key) + `,"key":` + jsonString(details.key) +
`,"id":` + jsonString(details.id) + `,"id":` + jsonString(details.obj.ID()) +
`,"time":` + jsonTimeFormat(details.timestamp) + `}`, `,"time":` + jsonTimeFormat(details.timestamp) + `}`,
} }
} }
@ -107,8 +111,7 @@ func fenceMatch(
if fence.roam.on { if fence.roam.on {
if details.command == "set" { if details.command == "set" {
roamNearbys, roamFaraways = roamNearbys, roamFaraways =
fenceMatchRoam(sw.s, fence, details.id, fenceMatchRoam(sw.s, fence, details.obj, details.old)
details.oldObj, details.obj)
if len(roamNearbys) == 0 && len(roamFaraways) == 0 { if len(roamNearbys) == 0 && len(roamFaraways) == 0 {
return nil return nil
} }
@ -117,14 +120,14 @@ func fenceMatch(
} else { } else {
var nocross bool var nocross bool
// not using roaming // not using roaming
match1 := fenceMatchObject(fence, details.oldObj) match1 := fenceMatchObject(fence, details.old)
if match1 { if match1 {
match1, _, _ = sw.testObject(details.id, details.oldObj, details.oldFields) match1, _, _ = sw.testObject(details.old)
nocross = !match1 nocross = !match1
} }
match2 := fenceMatchObject(fence, details.obj) match2 := fenceMatchObject(fence, details.obj)
if match2 { if match2 {
match2, _, _ = sw.testObject(details.id, details.obj, details.fields) match2, _, _ = sw.testObject(details.obj)
nocross = !match2 nocross = !match2
} }
if match1 && match2 { if match1 && match2 {
@ -140,11 +143,11 @@ func fenceMatch(
if details.command != "fset" { if details.command != "fset" {
// Maybe the old object and new object create a line that crosses the fence. // Maybe the old object and new object create a line that crosses the fence.
// Must detect for that possibility. // Must detect for that possibility.
if !nocross && details.oldObj != nil { if !nocross && details.old != nil {
ls := geojson.NewLineString(geometry.NewLine( ls := geojson.NewLineString(geometry.NewLine(
[]geometry.Point{ []geometry.Point{
details.oldObj.Center(), details.old.Geo().Center(),
details.obj.Center(), details.obj.Geo().Center(),
}, nil)) }, nil))
temp := false temp := false
if fence.cmd == "within" { if fence.cmd == "within" {
@ -153,7 +156,8 @@ func fenceMatch(
fence.cmd = "intersects" fence.cmd = "intersects"
temp = true temp = true
} }
if fenceMatchObject(fence, ls) { lso := object.New("", ls, 0, 0, field.List{})
if fenceMatchObject(fence, lso) {
detect = "cross" detect = "cross"
} }
if temp { if temp {
@ -185,18 +189,15 @@ func fenceMatch(
} }
var distance float64 var distance float64
if fence.distance && fence.obj != nil { if fence.distance && fence.obj != nil {
distance = details.obj.Distance(fence.obj) distance = details.obj.Geo().Distance(fence.obj)
} }
// TODO: fields
// sw.fmap = details.fmap
sw.fullFields = true sw.fullFields = true
sw.msg.OutputType = JSON sw.msg.OutputType = JSON
sw.writeObject(ScanWriterParams{ sw.writeObject(ScanWriterParams{
id: details.id, obj: details.obj,
o: details.obj,
fields: details.fields,
noTest: true, noTest: true,
distance: distance, dist: distance,
distOutput: fence.distance, distOutput: fence.distance,
}) })
@ -215,14 +216,14 @@ func fenceMatch(
var group string var group string
if detect == "enter" { if detect == "enter" {
group = sw.s.groupConnect(hookName, details.key, details.id) group = sw.s.groupConnect(hookName, details.key, details.obj.ID())
} else if detect == "cross" { } else if detect == "cross" {
sw.s.groupDisconnect(hookName, details.key, details.id) sw.s.groupDisconnect(hookName, details.key, details.obj.ID())
group = sw.s.groupConnect(hookName, details.key, details.id) group = sw.s.groupConnect(hookName, details.key, details.obj.ID())
} else { } else {
group = sw.s.groupGet(hookName, details.key, details.id) group = sw.s.groupGet(hookName, details.key, details.obj.ID())
if group == "" { if group == "" {
group = sw.s.groupConnect(hookName, details.key, details.id) group = sw.s.groupConnect(hookName, details.key, details.obj.ID())
} }
} }
var msgs []string var msgs []string
@ -287,26 +288,24 @@ func extendRoamMessage(
nmsg = append(nmsg, `,"scan":[`...) nmsg = append(nmsg, `,"scan":[`...)
col, _ := sw.s.cols.Get(fence.roam.key) col, _ := sw.s.cols.Get(fence.roam.key)
if col != nil { if col != nil {
obj, _, _, ok := col.Get(match.id) o := col.Get(match.id)
if ok { if o != nil {
nmsg = append(nmsg, `{"id":`...) nmsg = append(nmsg, `{"id":`...)
nmsg = appendJSONString(nmsg, match.id) nmsg = appendJSONString(nmsg, match.id)
nmsg = append(nmsg, `,"self":true,"object":`...) nmsg = append(nmsg, `,"self":true,"object":`...)
nmsg = obj.AppendJSON(nmsg) nmsg = o.Geo().AppendJSON(nmsg)
nmsg = append(nmsg, '}') nmsg = append(nmsg, '}')
} }
pattern := match.id + fence.roam.scan pattern := match.id + fence.roam.scan
iterator := func( iterator := func(o *object.Object) bool {
oid string, o geojson.Object, fields field.List, if o.ID() == match.id {
) bool {
if oid == match.id {
return true return true
} }
if matched, _ := glob.Match(pattern, oid); matched { if matched, _ := glob.Match(pattern, o.ID()); matched {
nmsg = append(nmsg, `,{"id":`...) nmsg = append(nmsg, `,{"id":`...)
nmsg = appendJSONString(nmsg, oid) nmsg = appendJSONString(nmsg, o.ID())
nmsg = append(nmsg, `,"object":`...) nmsg = append(nmsg, `,"object":`...)
nmsg = o.AppendJSON(nmsg) nmsg = o.Geo().AppendJSON(nmsg)
nmsg = append(nmsg, '}') nmsg = append(nmsg, '}')
} }
return true return true
@ -345,8 +344,8 @@ func makemsg(
return string(buf) return string(buf)
} }
func fenceMatchObject(fence *liveFenceSwitches, obj geojson.Object) bool { func fenceMatchObject(fence *liveFenceSwitches, o *object.Object) bool {
if obj == nil { if o == nil {
return false return false
} }
if fence.roam.on { if fence.roam.on {
@ -356,18 +355,18 @@ func fenceMatchObject(fence *liveFenceSwitches, obj geojson.Object) bool {
switch fence.cmd { switch fence.cmd {
case "nearby": case "nearby":
// nearby is an INTERSECT on a Circle // nearby is an INTERSECT on a Circle
return obj.Intersects(fence.obj) return o.Geo().Intersects(fence.obj)
case "within": case "within":
return obj.Within(fence.obj) return o.Geo().Within(fence.obj)
case "intersects": case "intersects":
return obj.Intersects(fence.obj) return o.Geo().Intersects(fence.obj)
} }
return false return false
} }
func fenceMatchNearbys( func fenceMatchNearbys(
s *Server, fence *liveFenceSwitches, s *Server, fence *liveFenceSwitches,
id string, obj geojson.Object, obj *object.Object,
) (nearbys []roamMatch) { ) (nearbys []roamMatch) {
if obj == nil { if obj == nil {
return nil return nil
@ -376,49 +375,49 @@ func fenceMatchNearbys(
if col == nil { if col == nil {
return nil return nil
} }
center := obj.Center() center := obj.Geo().Center()
minLat, minLon, maxLat, maxLon := minLat, minLon, maxLat, maxLon :=
geo.RectFromCenter(center.Y, center.X, fence.roam.meters) geo.RectFromCenter(center.Y, center.X, fence.roam.meters)
rect := geometry.Rect{ rect := geometry.Rect{
Min: geometry.Point{X: minLon, Y: minLat}, Min: geometry.Point{X: minLon, Y: minLat},
Max: geometry.Point{X: maxLon, Y: maxLat}, Max: geometry.Point{X: maxLon, Y: maxLat},
} }
col.Intersects(geojson.NewRect(rect), 0, nil, nil, func( col.Intersects(geojson.NewRect(rect), 0, nil, nil,
id2 string, obj2 geojson.Object, fields field.List, func(o *object.Object) bool {
) bool { var idMatch bool
var idMatch bool if o.ID() == obj.ID() {
if id2 == id { return true // skip self
return true // skip self }
} meters := o.Geo().Distance(o.Geo())
meters := obj.Distance(obj2) if meters > fence.roam.meters {
if meters > fence.roam.meters { return true // skip outside radius
return true // skip outside radius }
} if fence.roam.pattern {
if fence.roam.pattern { idMatch, _ = glob.Match(fence.roam.id, o.ID())
idMatch, _ = glob.Match(fence.roam.id, id2) } else {
} else { idMatch = fence.roam.id == o.ID()
idMatch = fence.roam.id == id2 }
} if !idMatch {
if !idMatch { return true // skip non-id match
return true // skip non-id match }
} match := roamMatch{
match := roamMatch{ id: o.ID(),
id: id2, obj: o.Geo(),
obj: obj2, meters: obj.Geo().Distance(o.Geo()),
meters: obj.Distance(obj2), }
} nearbys = append(nearbys, match)
nearbys = append(nearbys, match) return true
return true },
}) )
return nearbys return nearbys
} }
func fenceMatchRoam( func fenceMatchRoam(
s *Server, fence *liveFenceSwitches, s *Server, fence *liveFenceSwitches,
id string, old, obj geojson.Object, obj, old *object.Object,
) (nearbys, faraways []roamMatch) { ) (nearbys, faraways []roamMatch) {
oldNearbys := fenceMatchNearbys(s, fence, id, old) oldNearbys := fenceMatchNearbys(s, fence, old)
newNearbys := fenceMatchNearbys(s, fence, id, obj) newNearbys := fenceMatchNearbys(s, fence, obj)
// Go through all matching objects in new-nearbys and old-nearbys. // Go through all matching objects in new-nearbys and old-nearbys.
for i := 0; i < len(oldNearbys); i++ { for i := 0; i < len(oldNearbys); i++ {
var match bool var match bool
@ -444,7 +443,7 @@ func fenceMatchRoam(
faraways, nearbys = oldNearbys, newNearbys faraways, nearbys = oldNearbys, newNearbys
// ensure the faraways distances are to the new object // ensure the faraways distances are to the new object
for i := 0; i < len(faraways); i++ { for i := 0; i < len(faraways); i++ {
faraways[i].meters = faraways[i].obj.Distance(obj) faraways[i].meters = faraways[i].obj.Distance(obj.Geo())
} }
sortRoamMatches(faraways) sortRoamMatches(faraways)
sortRoamMatches(nearbys) sortRoamMatches(nearbys)

45
internal/server/json.go
View File

@ -12,6 +12,8 @@ import (
"github.com/tidwall/resp" "github.com/tidwall/resp"
"github.com/tidwall/sjson" "github.com/tidwall/sjson"
"github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/object"
) )
func appendJSONString(b []byte, s string) []byte { func appendJSONString(b []byte, s string) []byte {
@ -191,8 +193,8 @@ func (s *Server) cmdJget(msg *Message) (resp.Value, error) {
} }
return NOMessage, errKeyNotFound return NOMessage, errKeyNotFound
} }
o, _, _, ok := col.Get(id) o := col.Get(id)
if !ok { if o == nil {
if msg.OutputType == RESP { if msg.OutputType == RESP {
return resp.NullValue(), nil return resp.NullValue(), nil
} }
@ -200,9 +202,9 @@ func (s *Server) cmdJget(msg *Message) (resp.Value, error) {
} }
var res gjson.Result var res gjson.Result
if doget { if doget {
res = gjson.Get(o.String(), path) res = gjson.Get(o.Geo().String(), path)
} else { } else {
res = gjson.Parse(o.String()) res = gjson.Parse(o.Geo().String())
} }
var val string var val string
if raw { if raw {
@ -270,10 +272,12 @@ func (s *Server) cmdJset(msg *Message) (res resp.Value, d commandDetails, err er
} }
var json string var json string
var geoobj bool var geoobj bool
o, fields, _, ok := col.Get(id) var fields field.List
if ok { o := col.Get(id)
geoobj = objIsSpatial(o) if o != nil {
json = o.String() geoobj = objIsSpatial(o.Geo())
json = o.Geo().String()
fields = o.Fields()
} }
if raw { if raw {
// set as raw block // set as raw block
@ -295,14 +299,15 @@ func (s *Server) cmdJset(msg *Message) (res resp.Value, d commandDetails, err er
if createcol { if createcol {
s.cols.Set(key, col) s.cols.Set(key, col)
} }
var oobj geojson.Object = collection.String(json)
obj := object.New(id, oobj, 0, 0, fields)
col.Set(obj)
d.key = key d.key = key
d.id = id d.obj = obj
d.obj = collection.String(json)
d.timestamp = time.Now() d.timestamp = time.Now()
d.updated = true d.updated = true
col.Set(d.id, d.obj, fields, 0)
switch msg.OutputType { switch msg.OutputType {
case JSON: case JSON:
var buf bytes.Buffer var buf bytes.Buffer
@ -335,10 +340,12 @@ func (s *Server) cmdJdel(msg *Message) (res resp.Value, d commandDetails, err er
var json string var json string
var geoobj bool var geoobj bool
o, fields, _, ok := col.Get(id) var fields field.List
if ok { o := col.Get(id)
geoobj = objIsSpatial(o) if o != nil {
json = o.String() geoobj = objIsSpatial(o.Geo())
json = o.Geo().String()
fields = o.Fields()
} }
njson, err := sjson.Delete(json, path) njson, err := sjson.Delete(json, path)
if err != nil { if err != nil {
@ -361,12 +368,14 @@ func (s *Server) cmdJdel(msg *Message) (res resp.Value, d commandDetails, err er
return s.cmdSET(&nmsg) return s.cmdSET(&nmsg)
} }
var oobj geojson.Object = collection.String(json)
obj := object.New(id, oobj, 0, 0, fields)
col.Set(obj)
d.key = key d.key = key
d.id = id d.obj = obj
d.obj = collection.String(json)
d.timestamp = time.Now() d.timestamp = time.Now()
d.updated = true d.updated = true
col.Set(d.id, d.obj, fields, 0)
switch msg.OutputType { switch msg.OutputType {
case JSON: case JSON:
var buf bytes.Buffer var buf bytes.Buffer

15
internal/server/scan.go
View File

@ -5,9 +5,8 @@ import (
"errors" "errors"
"time" "time"
"github.com/tidwall/geojson"
"github.com/tidwall/resp" "github.com/tidwall/resp"
"github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/object"
) )
func (s *Server) cmdScanArgs(vs []string) ( func (s *Server) cmdScanArgs(vs []string) (
@ -70,11 +69,9 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) {
if limits[0] == "" && limits[1] == "" { if limits[0] == "" && limits[1] == "" {
sw.col.Scan(args.desc, sw, sw.col.Scan(args.desc, sw,
msg.Deadline, msg.Deadline,
func(id string, o geojson.Object, fields field.List) bool { func(o *object.Object) bool {
keepGoing, err := sw.pushObject(ScanWriterParams{ keepGoing, err := sw.pushObject(ScanWriterParams{
id: id, obj: o,
o: o,
fields: fields,
}) })
if err != nil { if err != nil {
ierr = err ierr = err
@ -86,11 +83,9 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) {
} else { } else {
sw.col.ScanRange(limits[0], limits[1], args.desc, sw, sw.col.ScanRange(limits[0], limits[1], args.desc, sw,
msg.Deadline, msg.Deadline,
func(id string, o geojson.Object, fields field.List) bool { func(o *object.Object) bool {
keepGoing, err := sw.pushObject(ScanWriterParams{ keepGoing, err := sw.pushObject(ScanWriterParams{
id: id, obj: o,
o: o,
fields: fields,
}) })
if err != nil { if err != nil {
ierr = err ierr = err

126
internal/server/scanner.go
View File

@ -14,6 +14,7 @@ import (
"github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/glob" "github.com/tidwall/tile38/internal/glob"
"github.com/tidwall/tile38/internal/object"
) )
const limitItems = 100 const limitItems = 100
@ -60,10 +61,8 @@ type scanWriter struct {
} }
type ScanWriterParams struct { type ScanWriterParams struct {
id string obj *object.Object
o geojson.Object dist float64
fields field.List
distance float64
distOutput bool // query or fence requested distance output distOutput bool // query or fence requested distance output
noTest bool noTest bool
ignoreGlobMatch bool ignoreGlobMatch bool
@ -201,34 +200,37 @@ func extractZCoordinate(o geojson.Object) float64 {
} }
} }
func getFieldValue(o geojson.Object, fields field.List, name string) field.Value { func getFieldValue(o *object.Object, name string) field.Value {
if name == "z" { if name == "z" {
return field.ValueOf(strconv.FormatFloat(extractZCoordinate(o), 'f', -1, 64)) z := extractZCoordinate(o.Geo())
return field.ValueOf(strconv.FormatFloat(z, 'f', -1, 64))
} }
f := fields.Get(name) return o.Fields().Get(name).Value()
return f.Value()
} }
func (sw *scanWriter) fieldMatch(o geojson.Object, fields field.List) (bool, error) { func (sw *scanWriter) fieldMatch(o *object.Object) (bool, error) {
for _, where := range sw.wheres { for _, where := range sw.wheres {
if !where.match(getFieldValue(o, fields, where.name)) { if !where.match(getFieldValue(o, where.name)) {
return false, nil return false, nil
} }
} }
for _, wherein := range sw.whereins { for _, wherein := range sw.whereins {
if !wherein.match(getFieldValue(o, fields, wherein.name)) { if !wherein.match(getFieldValue(o, wherein.name)) {
return false, nil return false, nil
} }
} }
if len(sw.whereevals) > 0 { if len(sw.whereevals) > 0 {
fieldsWithNames := make(map[string]field.Value) fieldNames := make(map[string]field.Value)
fieldsWithNames["z"] = field.ValueOf(strconv.FormatFloat(extractZCoordinate(o), 'f', -1, 64)) if objIsSpatial(o.Geo()) {
fields.Scan(func(f field.Field) bool { z := extractZCoordinate(o.Geo())
fieldsWithNames[f.Name()] = f.Value() fieldNames["z"] = field.ValueOf(strconv.FormatFloat(z, 'f', -1, 64))
}
o.Fields().Scan(func(f field.Field) bool {
fieldNames[f.Name()] = f.Value()
return true return true
}) })
for _, whereval := range sw.whereevals { for _, whereval := range sw.whereevals {
match, err := whereval.match(fieldsWithNames) match, err := whereval.match(fieldNames)
if err != nil { if err != nil {
return false, err return false, err
} }
@ -240,7 +242,7 @@ func (sw *scanWriter) fieldMatch(o geojson.Object, fields field.List) (bool, err
return true, nil return true, nil
} }
func (sw *scanWriter) globMatch(id string, o geojson.Object) (ok, keepGoing bool) { func (sw *scanWriter) globMatch(o *object.Object) (ok, keepGoing bool) {
if sw.globEverything { if sw.globEverything {
return true, true return true, true
} }
@ -248,7 +250,7 @@ func (sw *scanWriter) globMatch(id string, o geojson.Object) (ok, keepGoing bool
if sw.matchValues { if sw.matchValues {
val = o.String() val = o.String()
} else { } else {
val = id val = o.ID()
} }
for _, pattern := range sw.globs { for _, pattern := range sw.globs {
ok, _ := glob.Match(pattern, val) ok, _ := glob.Match(pattern, val)
@ -270,13 +272,13 @@ func (sw *scanWriter) Step(n uint64) {
// ok is whether the object passes the test and should be written // ok is whether the object passes the test and should be written
// keepGoing is whether there could be more objects to test // keepGoing is whether there could be more objects to test
func (sw *scanWriter) testObject(id string, o geojson.Object, fields field.List, func (sw *scanWriter) testObject(o *object.Object,
) (ok, keepGoing bool, err error) { ) (ok, keepGoing bool, err error) {
match, kg := sw.globMatch(id, o) match, kg := sw.globMatch(o)
if !match { if !match {
return false, kg, nil return false, kg, nil
} }
ok, err = sw.fieldMatch(o, fields) ok, err = sw.fieldMatch(o)
if err != nil { if err != nil {
return false, false, err return false, false, err
} }
@ -288,7 +290,7 @@ func (sw *scanWriter) pushObject(opts ScanWriterParams) (keepGoing bool, err err
if !opts.noTest { if !opts.noTest {
var ok bool var ok bool
var err error var err error
ok, keepGoing, err = sw.testObject(opts.id, opts.o, opts.fields) ok, keepGoing, err = sw.testObject(opts.obj)
if err != nil { if err != nil {
return false, err return false, err
} }
@ -301,10 +303,17 @@ func (sw *scanWriter) pushObject(opts ScanWriterParams) (keepGoing bool, err err
return sw.count < sw.limit, nil return sw.count < sw.limit, nil
} }
if opts.clip != nil { if opts.clip != nil {
opts.o = clip.Clip(opts.o, opts.clip, &sw.s.geomIndexOpts) // create a newly clipped object
opts.obj = object.New(
opts.obj.ID(),
clip.Clip(opts.obj.Geo(), opts.clip, &sw.s.geomIndexOpts),
0, opts.obj.Expires(),
opts.obj.Fields(),
)
} }
if !sw.fullFields { if !sw.fullFields {
opts.fields.Scan(func(f field.Field) bool { opts.obj.Fields().Scan(func(f field.Field) bool {
sw.fkeys.Insert(f.Name()) sw.fkeys.Insert(f.Name())
return true return true
}) })
@ -339,10 +348,10 @@ func (sw *scanWriter) writeFilled(opts ScanWriterParams) {
} }
fieldsOutput := sw.hasFieldsOutput() fieldsOutput := sw.hasFieldsOutput()
if fieldsOutput && sw.fullFields { if fieldsOutput && sw.fullFields {
if opts.fields.Len() > 0 { if opts.obj.Fields().Len() > 0 {
jsfields = `,"fields":{` jsfields = `,"fields":{`
var i int var i int
opts.fields.Scan(func(f field.Field) bool { opts.obj.Fields().Scan(func(f field.Field) bool {
if !f.Value().IsZero() { if !f.Value().IsZero() {
if i > 0 { if i > 0 {
jsfields += `,` jsfields += `,`
@ -361,7 +370,7 @@ func (sw *scanWriter) writeFilled(opts ScanWriterParams) {
if i > 0 { if i > 0 {
jsfields += `,` jsfields += `,`
} }
f := opts.fields.Get(name) f := opts.obj.Fields().Get(name)
jsfields += f.Value().JSON() jsfields += f.Value().JSON()
i++ i++
return true return true
@ -369,29 +378,29 @@ func (sw *scanWriter) writeFilled(opts ScanWriterParams) {
jsfields += `]` jsfields += `]`
} }
if sw.output == outputIDs { if sw.output == outputIDs {
if opts.distOutput || opts.distance > 0 { if opts.distOutput || opts.dist > 0 {
wr.WriteString(`{"id":` + jsonString(opts.id) + wr.WriteString(`{"id":` + jsonString(opts.obj.ID()) +
`,"distance":` + strconv.FormatFloat(opts.distance, 'f', -1, 64) + "}") `,"distance":` + strconv.FormatFloat(opts.dist, 'f', -1, 64) + "}")
} else { } else {
wr.WriteString(jsonString(opts.id)) wr.WriteString(jsonString(opts.obj.ID()))
} }
} else { } else {
wr.WriteString(`{"id":` + jsonString(opts.id)) wr.WriteString(`{"id":` + jsonString(opts.obj.ID()))
switch sw.output { switch sw.output {
case outputObjects: case outputObjects:
wr.WriteString(`,"object":` + string(opts.o.AppendJSON(nil))) wr.WriteString(`,"object":` + string(opts.obj.Geo().AppendJSON(nil)))
case outputPoints: case outputPoints:
wr.WriteString(`,"point":` + string(appendJSONSimplePoint(nil, opts.o))) wr.WriteString(`,"point":` + string(appendJSONSimplePoint(nil, opts.obj.Geo())))
case outputHashes: case outputHashes:
center := opts.o.Center() center := opts.obj.Geo().Center()
p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision)) p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision))
wr.WriteString(`,"hash":"` + p + `"`) wr.WriteString(`,"hash":"` + p + `"`)
case outputBounds: case outputBounds:
wr.WriteString(`,"bounds":` + string(appendJSONSimpleBounds(nil, opts.o))) wr.WriteString(`,"bounds":` + string(appendJSONSimpleBounds(nil, opts.obj.Geo())))
} }
wr.WriteString(jsfields) wr.WriteString(jsfields)
if opts.distOutput || opts.distance > 0 { if opts.distOutput || opts.dist > 0 {
wr.WriteString(`,"distance":` + strconv.FormatFloat(opts.distance, 'f', -1, 64)) wr.WriteString(`,"distance":` + strconv.FormatFloat(opts.dist, 'f', -1, 64))
} }
wr.WriteString(`}`) wr.WriteString(`}`)
@ -399,10 +408,10 @@ func (sw *scanWriter) writeFilled(opts ScanWriterParams) {
sw.wr.Write(wr.Bytes()) sw.wr.Write(wr.Bytes())
case RESP: case RESP:
vals := make([]resp.Value, 1, 3) vals := make([]resp.Value, 1, 3)
vals[0] = resp.StringValue(opts.id) vals[0] = resp.StringValue(opts.obj.ID())
if sw.output == outputIDs { if sw.output == outputIDs {
if opts.distOutput || opts.distance > 0 { if opts.distOutput || opts.dist > 0 {
vals = append(vals, resp.FloatValue(opts.distance)) vals = append(vals, resp.FloatValue(opts.dist))
sw.values = append(sw.values, resp.ArrayValue(vals)) sw.values = append(sw.values, resp.ArrayValue(vals))
} else { } else {
sw.values = append(sw.values, vals[0]) sw.values = append(sw.values, vals[0])
@ -410,10 +419,10 @@ func (sw *scanWriter) writeFilled(opts ScanWriterParams) {
} else { } else {
switch sw.output { switch sw.output {
case outputObjects: case outputObjects:
vals = append(vals, resp.StringValue(opts.o.String())) vals = append(vals, resp.StringValue(opts.obj.String()))
case outputPoints: case outputPoints:
point := opts.o.Center() point := opts.obj.Geo().Center()
z := extractZCoordinate(opts.o) z := extractZCoordinate(opts.obj.Geo())
if z != 0 { if z != 0 {
vals = append(vals, resp.ArrayValue([]resp.Value{ vals = append(vals, resp.ArrayValue([]resp.Value{
resp.FloatValue(point.Y), resp.FloatValue(point.Y),
@ -427,11 +436,11 @@ func (sw *scanWriter) writeFilled(opts ScanWriterParams) {
})) }))
} }
case outputHashes: case outputHashes:
center := opts.o.Center() center := opts.obj.Geo().Center()
p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision)) p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision))
vals = append(vals, resp.StringValue(p)) vals = append(vals, resp.StringValue(p))
case outputBounds: case outputBounds:
bbox := opts.o.Rect() bbox := opts.obj.Rect()
vals = append(vals, resp.ArrayValue([]resp.Value{ vals = append(vals, resp.ArrayValue([]resp.Value{
resp.ArrayValue([]resp.Value{ resp.ArrayValue([]resp.Value{
resp.FloatValue(bbox.Min.Y), resp.FloatValue(bbox.Min.Y),
@ -444,23 +453,22 @@ func (sw *scanWriter) writeFilled(opts ScanWriterParams) {
})) }))
} }
if sw.hasFieldsOutput() { if sw.hasFieldsOutput() {
if opts.fields.Len() > 0 { var fvals []resp.Value
var fvals []resp.Value var i int
var i int opts.obj.Fields().Scan(func(f field.Field) bool {
opts.fields.Scan(func(f field.Field) bool { if !f.Value().IsZero() {
if !f.Value().IsZero() { fvals = append(fvals, resp.StringValue(f.Name()), resp.StringValue(f.Value().Data()))
fvals = append(fvals, resp.StringValue(f.Name()), resp.StringValue(f.Value().Data())) i++
i++ }
} return true
return true })
}) if len(fvals) > 0 {
vals = append(vals, resp.ArrayValue(fvals)) vals = append(vals, resp.ArrayValue(fvals))
} }
} }
if opts.distOutput || opts.distance > 0 { if opts.distOutput || opts.dist > 0 {
vals = append(vals, resp.FloatValue(opts.distance)) vals = append(vals, resp.FloatValue(opts.dist))
} }
sw.values = append(sw.values, resp.ArrayValue(vals)) sw.values = append(sw.values, resp.ArrayValue(vals))
} }
} }

3
internal/server/scanner_test.go
View File

@ -10,6 +10,7 @@ import (
"github.com/tidwall/geojson" "github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry" "github.com/tidwall/geojson/geometry"
"github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/object"
) )
type testPointItem struct { type testPointItem struct {
@ -47,7 +48,7 @@ func BenchmarkFieldMatch(t *testing.B) {
for i := 0; i < t.N; i++ { for i := 0; i < t.N; i++ {
// one call is super fast, measurements are not reliable, let's do 100 // one call is super fast, measurements are not reliable, let's do 100
for ix := 0; ix < 100; ix++ { for ix := 0; ix < 100; ix++ {
sw.fieldMatch(items[i].object, items[i].fields) sw.fieldMatch(object.New("", items[i].object, 0, 0, items[i].fields))
} }
} }
} }

97
internal/server/search.go
View File

@ -16,8 +16,8 @@ import (
"github.com/tidwall/tile38/internal/bing" "github.com/tidwall/tile38/internal/bing"
"github.com/tidwall/tile38/internal/buffer" "github.com/tidwall/tile38/internal/buffer"
"github.com/tidwall/tile38/internal/clip" "github.com/tidwall/tile38/internal/clip"
"github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/glob" "github.com/tidwall/tile38/internal/glob"
"github.com/tidwall/tile38/internal/object"
) )
const defaultCircleSteps = 64 const defaultCircleSteps = 64
@ -376,11 +376,12 @@ func (s *Server) cmdSearchArgs(
err = errKeyNotFound err = errKeyNotFound
return return
} }
lfs.obj, _, _, ok = col.Get(id) o := col.Get(id)
if !ok { if o == nil {
err = errIDNotFound err = errIDNotFound
return return
} }
lfs.obj = o.Geo()
case "roam": case "roam":
lfs.roam.on = true lfs.roam.on = true
if vs, lfs.roam.key, ok = tokenval(vs); !ok || lfs.roam.key == "" { if vs, lfs.roam.key, ok = tokenval(vs); !ok || lfs.roam.key == "" {
@ -499,12 +500,10 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
} }
var ierr error var ierr error
if sw.col != nil { if sw.col != nil {
iterStep := func(id string, o geojson.Object, fields field.List, meters float64) bool { iterStep := func(o *object.Object, dist float64) bool {
keepGoing, err := sw.pushObject(ScanWriterParams{ keepGoing, err := sw.pushObject(ScanWriterParams{
id: id, obj: o,
o: o, dist: dist,
fields: fields,
distance: meters,
distOutput: sargs.distance, distOutput: sargs.distance,
ignoreGlobMatch: true, ignoreGlobMatch: true,
skipTesting: true, skipTesting: true,
@ -523,16 +522,16 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
errors.New("cannot use SPARSE without a point distance") errors.New("cannot use SPARSE without a point distance")
} }
// An intersects operation is required for SPARSE // An intersects operation is required for SPARSE
iter := func(id string, o geojson.Object, fields field.List) bool { iter := func(o *object.Object) bool {
var meters float64 var dist float64
if sargs.distance { if sargs.distance {
meters = o.Distance(sargs.obj) dist = o.Geo().Distance(sargs.obj)
} }
return iterStep(id, o, fields, meters) return iterStep(o, dist)
} }
sw.col.Intersects(sargs.obj, sargs.sparse, sw, msg.Deadline, iter) sw.col.Intersects(sargs.obj, sargs.sparse, sw, msg.Deadline, iter)
} else { } else {
iter := func(id string, o geojson.Object, fields field.List, dist float64) bool { iter := func(o *object.Object, dist float64) bool {
if maxDist > 0 && dist > maxDist { if maxDist > 0 && dist > maxDist {
return false return false
} }
@ -540,7 +539,7 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
if sargs.distance { if sargs.distance {
meters = dist meters = dist
} }
return iterStep(id, o, fields, meters) return iterStep(o, meters)
} }
sw.col.Nearby(sargs.obj, sw, msg.Deadline, iter) sw.col.Nearby(sargs.obj, sw, msg.Deadline, iter)
} }
@ -599,41 +598,31 @@ func (s *Server) cmdWITHINorINTERSECTS(cmd string, msg *Message) (res resp.Value
var ierr error var ierr error
if sw.col != nil { if sw.col != nil {
if cmd == "within" { if cmd == "within" {
sw.col.Within(sargs.obj, sargs.sparse, sw, msg.Deadline, func( sw.col.Within(sargs.obj, sargs.sparse, sw, msg.Deadline,
id string, o geojson.Object, fields field.List, func(o *object.Object) bool {
) bool { keepGoing, err := sw.pushObject(ScanWriterParams{obj: o})
keepGoing, err := sw.pushObject(ScanWriterParams{ if err != nil {
id: id, ierr = err
o: o, return false
fields: fields, }
}) return keepGoing
if err != nil { },
ierr = err )
return false
}
return keepGoing
})
} else if cmd == "intersects" { } else if cmd == "intersects" {
sw.col.Intersects(sargs.obj, sargs.sparse, sw, msg.Deadline, func( sw.col.Intersects(sargs.obj, sargs.sparse, sw, msg.Deadline,
id string, func(o *object.Object) bool {
o geojson.Object, params := ScanWriterParams{obj: o}
fields field.List, if sargs.clip {
) bool { params.clip = sargs.obj
params := ScanWriterParams{ }
id: id, keepGoing, err := sw.pushObject(params)
o: o, if err != nil {
fields: fields, ierr = err
} return false
if sargs.clip { }
params.clip = sargs.obj return keepGoing
} },
keepGoing, err := sw.pushObject(params) )
if err != nil {
ierr = err
return false
}
return keepGoing
})
} }
} }
if ierr != nil { if ierr != nil {
@ -732,11 +721,9 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
limits := multiGlobParse(sw.globs, sargs.desc) limits := multiGlobParse(sw.globs, sargs.desc)
if limits[0] == "" && limits[1] == "" { if limits[0] == "" && limits[1] == "" {
sw.col.SearchValues(sargs.desc, sw, msg.Deadline, sw.col.SearchValues(sargs.desc, sw, msg.Deadline,
func(id string, o geojson.Object, fields field.List) bool { func(o *object.Object) bool {
keepGoing, err := sw.pushObject(ScanWriterParams{ keepGoing, err := sw.pushObject(ScanWriterParams{
id: id, obj: o,
o: o,
fields: fields,
}) })
if err != nil { if err != nil {
ierr = err ierr = err
@ -750,11 +737,9 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
// globSingle is only for ID matches, not values. // globSingle is only for ID matches, not values.
sw.col.SearchValuesRange(limits[0], limits[1], sargs.desc, sw, sw.col.SearchValuesRange(limits[0], limits[1], sargs.desc, sw,
msg.Deadline, msg.Deadline,
func(id string, o geojson.Object, fields field.List) bool { func(o *object.Object) bool {
keepGoing, err := sw.pushObject(ScanWriterParams{ keepGoing, err := sw.pushObject(ScanWriterParams{
id: id, obj: o,
o: o,
fields: fields,
}) })
if err != nil { if err != nil {
ierr = err ierr = err

11
internal/server/server.go
View File

@ -36,8 +36,8 @@ import (
"github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/deadline" "github.com/tidwall/tile38/internal/deadline"
"github.com/tidwall/tile38/internal/endpoint" "github.com/tidwall/tile38/internal/endpoint"
"github.com/tidwall/tile38/internal/field"
"github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/log"
"github.com/tidwall/tile38/internal/object"
) )
var errOOM = errors.New("OOM command not allowed when used memory > 'maxmemory'") var errOOM = errors.New("OOM command not allowed when used memory > 'maxmemory'")
@ -55,14 +55,11 @@ const (
// for geofence formulas. // for geofence formulas.
type commandDetails struct { type commandDetails struct {
command string // client command, like "SET" or "DEL" command string // client command, like "SET" or "DEL"
key, id string // collection key and object id of object key string // collection key
newKey string // new key, for RENAME command newKey string // new key, for RENAME command
obj geojson.Object // new object obj *object.Object // target object
fields field.List // array of field values old *object.Object // previous object, if any
oldObj geojson.Object // previous object, if any
oldFields field.List // previous object field values
updated bool // object was updated updated bool // object was updated
timestamp time.Time // timestamp when the update occured timestamp time.Time // timestamp when the update occured

5
internal/server/test.go
View File

@ -278,11 +278,12 @@ func (s *Server) parseArea(ovs []string, doClip bool) (vs []string, o geojson.Ob
err = errKeyNotFound err = errKeyNotFound
return return
} }
o, _, _, ok = col.Get(id) obj := col.Get(id)
if !ok { if obj == nil {
err = errIDNotFound err = errIDNotFound
return return
} }
o = obj.Geo()
} }
return return
} }

2
tests/keys_search_test.go
View File

@ -46,7 +46,7 @@ func keys_KNN_basic_test(mc *mockServer) error {
{"NEARBY", "mykey", "LIMIT", 10, "POINTS", "POINT", 20, 20}, { {"NEARBY", "mykey", "LIMIT", 10, "POINTS", "POINT", 20, 20}, {
"[0 [[2 [19 19]] [3 [12 19]] [5 [33 21]] [1 [5 5]] [4 [-5 5]] [6 [52 13]]]]"}, "[0 [[2 [19 19]] [3 [12 19]] [5 [33 21]] [1 [5 5]] [4 [-5 5]] [6 [52 13]]]]"},
{"NEARBY", "mykey", "LIMIT", 10, "IDS", "POINT", 20, 20, 4000000}, {"[0 [2 3 5 1 4 6]]"}, {"NEARBY", "mykey", "LIMIT", 10, "IDS", "POINT", 20, 20, 4000000}, {"[0 [2 3 5 1 4 6]]"},
{"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "IDS", "POINT", 20, 20, 1500000}, {"[0 [[2 152808.671875] [3 895945.125] [5 1448929.625]]]"}, {"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "IDS", "POINT", 20, 20, 1500000}, {"[0 [[2 152808.67164037024] [3 895945.1409106688] [5 1448929.5916252395]]]"},
{"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "POINT", 52, 13, 100}, {`[0 [[6 {"type":"Point","coordinates":[13,52]} 0]]]`}, {"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "POINT", 52, 13, 100}, {`[0 [[6 {"type":"Point","coordinates":[13,52]} 0]]]`},
{"NEARBY", "mykey", "LIMIT", 10, "POINT", 52.1, 13.1, 100000}, {`[0 [[6 {"type":"Point","coordinates":[13,52]}]]]`}, {"NEARBY", "mykey", "LIMIT", 10, "POINT", 52.1, 13.1, 100000}, {`[0 [[6 {"type":"Point","coordinates":[13,52]}]]]`},
{"OUTPUT", "json"}, {func(res string) bool { return gjson.Get(res, "ok").Bool() }}, {"OUTPUT", "json"}, {func(res string) bool { return gjson.Get(res, "ok").Bool() }},