diff --git a/internal/collection/collection.go b/internal/collection/collection.go index 78fe6236..1063f37b 100644 --- a/internal/collection/collection.go +++ b/internal/collection/collection.go @@ -3,11 +3,17 @@ package collection import ( "github.com/tidwall/boxtree/d2" "github.com/tidwall/btree" - "github.com/tidwall/tile38/internal/ds" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" + "github.com/tidwall/tile38/internal/ds" ) +// Cursor allows for quickly paging through Scan, Within, Intersects, and Nearby +type Cursor interface { + Offset() uint64 + Step(count uint64) +} + type itemT struct { id string obj geojson.Object @@ -308,20 +314,24 @@ func (c *Collection) FieldArr() []string { } // Scan iterates though the collection ids. -func (c *Collection) Scan(desc bool, - offset uint64, - inc func(n uint64), +func (c *Collection) Scan(desc bool, cursor Cursor, iterator func(id string, obj geojson.Object, fields []float64) bool, ) bool { var keepon = true var count uint64 - inc(offset) + var offset uint64 + if cursor != nil { + offset = cursor.Offset() + cursor.Step(offset) + } iter := func(key string, value interface{}) bool { count++ if count <= offset { return true } - inc(1) + if cursor != nil { + cursor.Step(1) + } iitm := value.(*itemT) keepon = iterator(iitm.id, iitm.obj, c.getFieldValues(iitm.id)) return keepon @@ -335,20 +345,24 @@ func (c *Collection) Scan(desc bool, } // ScanRange iterates though the collection starting with specified id. -func (c *Collection) ScanRange(start, end string, desc bool, - offset uint64, - inc func(n uint64), +func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor, iterator func(id string, obj geojson.Object, fields []float64) bool, ) bool { var keepon = true var count uint64 - inc(offset) + var offset uint64 + if cursor != nil { + offset = cursor.Offset() + cursor.Step(offset) + } iter := func(key string, value interface{}) bool { count++ if count <= offset { return true } - inc(1) + if cursor != nil { + cursor.Step(1) + } if !desc { if key >= end { return false @@ -372,20 +386,24 @@ func (c *Collection) ScanRange(start, end string, desc bool, } // SearchValues iterates though the collection values. -func (c *Collection) SearchValues(desc bool, - offset uint64, - inc func(n uint64), +func (c *Collection) SearchValues(desc bool, cursor Cursor, iterator func(id string, obj geojson.Object, fields []float64) bool, ) bool { var keepon = true var count uint64 - inc(offset) + var offset uint64 + if cursor != nil { + offset = cursor.Offset() + cursor.Step(offset) + } iter := func(item btree.Item) bool { count++ if count <= offset { return true } - inc(1) + if cursor != nil { + cursor.Step(1) + } iitm := item.(*itemT) keepon = iterator(iitm.id, iitm.obj, c.getFieldValues(iitm.id)) return keepon @@ -400,19 +418,24 @@ func (c *Collection) SearchValues(desc bool, // SearchValuesRange iterates though the collection values. func (c *Collection) SearchValuesRange(start, end string, desc bool, - offset uint64, - inc func(n uint64), + cursor Cursor, iterator func(id string, obj geojson.Object, fields []float64) bool, ) bool { var keepon = true var count uint64 - inc(offset) + var offset uint64 + if cursor != nil { + offset = cursor.Offset() + cursor.Step(offset) + } iter := func(item btree.Item) bool { count++ if count <= offset { return true } - inc(1) + if cursor != nil { + cursor.Step(1) + } iitm := item.(*itemT) keepon = iterator(iitm.id, iitm.obj, c.getFieldValues(iitm.id)) return keepon @@ -429,10 +452,24 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool, // ScanGreaterOrEqual iterates though the collection starting with specified id. func (c *Collection) ScanGreaterOrEqual(id string, desc bool, + cursor Cursor, iterator func(id string, obj geojson.Object, fields []float64) bool, ) bool { var keepon = true + var count uint64 + var offset uint64 + if cursor != nil { + offset = cursor.Offset() + cursor.Step(offset) + } iter := func(key string, value interface{}) bool { + count++ + if count <= offset { + return true + } + if cursor != nil { + cursor.Step(1) + } iitm := value.(*itemT) keepon = iterator(iitm.id, iitm.obj, c.getFieldValues(iitm.id)) return keepon @@ -534,13 +571,16 @@ func (c *Collection) geoSparseInner( // bounding box. Set obj to nil in order to use the bounding box. func (c *Collection) Within( obj geojson.Object, - offset uint64, sparse uint8, - inc func(n uint64), + cursor Cursor, iter func(id string, obj geojson.Object, fields []float64) bool, ) bool { var count uint64 - inc(offset) + var offset uint64 + if cursor != nil { + offset = cursor.Offset() + cursor.Step(offset) + } if sparse > 0 { return c.geoSparse(obj, sparse, func(id string, o geojson.Object, fields []float64) ( @@ -550,7 +590,9 @@ func (c *Collection) Within( if count <= offset { return false, true } - inc(1) + if cursor != nil { + cursor.Step(1) + } if match = o.Within(obj); match { ok = iter(id, o, fields) } @@ -564,7 +606,9 @@ func (c *Collection) Within( if count <= offset { return true } - inc(1) + if cursor != nil { + cursor.Step(1) + } if o.Within(obj) { return iter(id, o, fields) } @@ -577,13 +621,16 @@ func (c *Collection) Within( // Set obj to nil in order to use the bounding box. func (c *Collection) Intersects( obj geojson.Object, - offset uint64, sparse uint8, - inc func(n uint64), + cursor Cursor, iter func(id string, obj geojson.Object, fields []float64) bool, ) bool { var count uint64 - inc(offset) + var offset uint64 + if cursor != nil { + offset = cursor.Offset() + cursor.Step(offset) + } if sparse > 0 { return c.geoSparse(obj, sparse, func(id string, o geojson.Object, fields []float64) ( @@ -593,7 +640,9 @@ func (c *Collection) Intersects( if count <= offset { return false, true } - inc(1) + if cursor != nil { + cursor.Step(1) + } if match = o.Intersects(obj); match { ok = iter(id, o, fields) } @@ -607,7 +656,9 @@ func (c *Collection) Intersects( if count <= offset { return true } - inc(1) + if cursor != nil { + cursor.Step(1) + } if o.Intersects(obj) { return iter(id, o, fields) } @@ -619,14 +670,17 @@ func (c *Collection) Intersects( // Nearby returns the nearest neighbors func (c *Collection) Nearby( target geojson.Object, - offset uint64, - inc func(n uint64), + cursor Cursor, iter func(id string, obj geojson.Object, fields []float64) bool, ) bool { alive := true center := target.Center() var count uint64 - inc(offset) + var offset uint64 + if cursor != nil { + offset = cursor.Offset() + cursor.Step(offset) + } c.index.Nearby( []float64{center.X, center.Y}, []float64{center.X, center.Y}, @@ -635,7 +689,9 @@ func (c *Collection) Nearby( if count <= offset { return true } - inc(1) + if cursor != nil { + cursor.Step(1) + } item := itemv.(*itemT) alive = iter(item.id, item.obj, c.getFieldValues(item.id)) return alive diff --git a/internal/collection/collection_test.go b/internal/collection/collection_test.go index 0eab0175..21325676 100644 --- a/internal/collection/collection_test.go +++ b/internal/collection/collection_test.go @@ -230,7 +230,7 @@ func TestCollectionScan(t *testing.T) { } var n int var prevID string - c.Scan(false, 0, func(n uint64) {}, func(id string, obj geojson.Object, fields []float64) bool { + c.Scan(false, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, id > prevID) } @@ -241,7 +241,7 @@ func TestCollectionScan(t *testing.T) { }) expect(t, n == c.Count()) n = 0 - c.Scan(true, 0, func(n uint64) {}, func(id string, obj geojson.Object, fields []float64) bool { + c.Scan(true, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, id < prevID) } @@ -253,7 +253,7 @@ func TestCollectionScan(t *testing.T) { expect(t, n == c.Count()) n = 0 - c.ScanRange("0060", "0070", false, 0, func(n uint64) {}, + c.ScanRange("0060", "0070", false, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, id > prevID) @@ -266,7 +266,7 @@ func TestCollectionScan(t *testing.T) { expect(t, n == 10) n = 0 - c.ScanRange("0070", "0060", true, 0, func(n uint64) {}, + c.ScanRange("0070", "0060", true, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, id < prevID) @@ -279,7 +279,7 @@ func TestCollectionScan(t *testing.T) { expect(t, n == 10) n = 0 - c.ScanGreaterOrEqual("0070", true, + c.ScanGreaterOrEqual("0070", true, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, id < prevID) @@ -292,7 +292,7 @@ func TestCollectionScan(t *testing.T) { expect(t, n == 71) n = 0 - c.ScanGreaterOrEqual("0070", false, + c.ScanGreaterOrEqual("0070", false, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, id > prevID) @@ -317,7 +317,7 @@ func TestCollectionSearch(t *testing.T) { } var n int var prevValue string - c.SearchValues(false, 0, func(n uint64) {}, func(id string, obj geojson.Object, fields []float64) bool { + c.SearchValues(false, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, obj.String() > prevValue) } @@ -328,7 +328,7 @@ func TestCollectionSearch(t *testing.T) { }) expect(t, n == c.Count()) n = 0 - c.SearchValues(true, 0, func(n uint64) {}, func(id string, obj geojson.Object, fields []float64) bool { + c.SearchValues(true, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, obj.String() < prevValue) } @@ -340,7 +340,7 @@ func TestCollectionSearch(t *testing.T) { expect(t, n == c.Count()) n = 0 - c.SearchValuesRange("0060", "0070", false, 0, func(n uint64) {}, + c.SearchValuesRange("0060", "0070", false, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, obj.String() > prevValue) @@ -353,7 +353,7 @@ func TestCollectionSearch(t *testing.T) { expect(t, n == 10) n = 0 - c.SearchValuesRange("0070", "0060", true, 0, func(n uint64) {}, + c.SearchValuesRange("0070", "0060", true, nil, func(id string, obj geojson.Object, fields []float64) bool { if n > 0 { expect(t, obj.String() < prevValue) @@ -436,7 +436,7 @@ func TestSpatialSearch(t *testing.T) { var n int n = 0 - c.Within(q1, 0, 0, func(n uint64) {}, + c.Within(q1, 0, nil, func(id string, obj geojson.Object, fields []float64) bool { n++ return true @@ -445,7 +445,7 @@ func TestSpatialSearch(t *testing.T) { expect(t, n == 3) n = 0 - c.Within(q2, 0, 0, func(n uint64) {}, + c.Within(q2, 0, nil, func(id string, obj geojson.Object, fields []float64) bool { n++ return true @@ -454,7 +454,7 @@ func TestSpatialSearch(t *testing.T) { expect(t, n == 7) n = 0 - c.Within(q3, 0, 0, func(n uint64) {}, + c.Within(q3, 0, nil, func(id string, obj geojson.Object, fields []float64) bool { n++ return true @@ -463,7 +463,7 @@ func TestSpatialSearch(t *testing.T) { expect(t, n == 4) n = 0 - c.Intersects(q1, 0, 0, func(n uint64) {}, + c.Intersects(q1, 0, nil, func(_ string, _ geojson.Object, _ []float64) bool { n++ return true @@ -472,7 +472,7 @@ func TestSpatialSearch(t *testing.T) { expect(t, n == 4) n = 0 - c.Intersects(q2, 0, 0, func(n uint64) {}, + c.Intersects(q2, 0, nil, func(_ string, _ geojson.Object, _ []float64) bool { n++ return true @@ -481,7 +481,7 @@ func TestSpatialSearch(t *testing.T) { expect(t, n == 7) n = 0 - c.Intersects(q3, 0, 0, func(n uint64) {}, + c.Intersects(q3, 0, nil, func(_ string, _ geojson.Object, _ []float64) bool { n++ return true @@ -490,7 +490,7 @@ func TestSpatialSearch(t *testing.T) { expect(t, n == 5) n = 0 - c.Intersects(q3, 0, 0, func(n uint64) {}, + c.Intersects(q3, 0, nil, func(_ string, _ geojson.Object, _ []float64) bool { n++ return n <= 1 @@ -502,7 +502,7 @@ func TestSpatialSearch(t *testing.T) { exitems := []geojson.Object{ r2, p1, p4, r1, p3, r3, p2, } - c.Nearby(q4, 0, func(n uint64) {}, + c.Nearby(q4, nil, func(id string, obj geojson.Object, fields []float64) bool { items = append(items, obj) return true @@ -528,7 +528,7 @@ func TestCollectionSparse(t *testing.T) { } var n int n = 0 - c.Within(rect, 0,1, func(n uint64) {}, + c.Within(rect, 1, nil, func(id string, obj geojson.Object, fields []float64) bool { n++ return true @@ -537,7 +537,7 @@ func TestCollectionSparse(t *testing.T) { expect(t, n == 4) n = 0 - c.Within(rect, 0, 2, func(n uint64) {}, + c.Within(rect, 2, nil, func(id string, obj geojson.Object, fields []float64) bool { n++ return true @@ -546,7 +546,7 @@ func TestCollectionSparse(t *testing.T) { expect(t, n == 16) n = 0 - c.Within(rect, 0, 3, func(n uint64) {}, + c.Within(rect, 3, nil, func(id string, obj geojson.Object, fields []float64) bool { n++ return true @@ -555,7 +555,7 @@ func TestCollectionSparse(t *testing.T) { expect(t, n == 64) n = 0 - c.Within(rect, 0, 3, func(n uint64) {}, + c.Within(rect, 3, nil, func(id string, obj geojson.Object, fields []float64) bool { n++ return n <= 30 @@ -564,7 +564,7 @@ func TestCollectionSparse(t *testing.T) { expect(t, n == 31) n = 0 - c.Intersects(rect, 0, 3, func(n uint64) {}, + c.Intersects(rect, 3, nil, func(id string, _ geojson.Object, _ []float64) bool { n++ return true @@ -573,7 +573,7 @@ func TestCollectionSparse(t *testing.T) { expect(t, n == 64) n = 0 - c.Intersects(rect, 0, 3, func(n uint64) {}, + c.Intersects(rect, 3, nil, func(id string, _ geojson.Object, _ []float64) bool { n++ return n <= 30 diff --git a/internal/server/aofshrink.go b/internal/server/aofshrink.go index 999b26a4..efc1c1c1 100644 --- a/internal/server/aofshrink.go +++ b/internal/server/aofshrink.go @@ -96,7 +96,7 @@ func (server *Server) aofshrink() { var exm = server.expires[keys[0]] // the expiration map var now = time.Now() // used for expiration var count = 0 // the object count - col.ScanGreaterOrEqual(nextid, false, + col.ScanGreaterOrEqual(nextid, false, nil, func(id string, obj geojson.Object, fields []float64) bool { if count == maxids { // we reached the max number of ids for one batch diff --git a/internal/server/crud.go b/internal/server/crud.go index 12648aea..089a25d3 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -372,9 +372,9 @@ func (server *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetailsT, if col != nil { g := glob.Parse(d.pattern, false) if g.Limits[0] == "" && g.Limits[1] == "" { - col.Scan(false, 0, func(n uint64) {}, iter) + col.Scan(false, nil, iter) } else { - col.ScanRange(g.Limits[0], g.Limits[1], false, 0, func(n uint64) {}, iter) + col.ScanRange(g.Limits[0], g.Limits[1], false, nil, iter) } var atLeastOneNotDeleted bool for i, dc := range d.children { diff --git a/internal/server/fence.go b/internal/server/fence.go index 6a45c700..4fe67b19 100644 --- a/internal/server/fence.go +++ b/internal/server/fence.go @@ -299,10 +299,10 @@ func extendRoamMessage( } g := glob.Parse(pattern, false) if g.Limits[0] == "" && g.Limits[1] == "" { - col.Scan(false, 0, func(n uint64) {}, iterator) + col.Scan(false, nil, iterator) } else { col.ScanRange(g.Limits[0], g.Limits[1], - false, 0, func(n uint64) {}, iterator) + false, nil, iterator) } } nmsg = append(nmsg, ']') @@ -364,7 +364,7 @@ func fenceMatchRoam( prevNearbys := fence.roam.nearbys[tid] var newNearbys map[string]bool - col.Intersects(obj, 0, 0, func(n uint64) {}, func( + col.Intersects(obj, 0, nil, func( id string, obj2 geojson.Object, fields []float64, ) bool { if c.hasExpired(fence.roam.key, id) { diff --git a/internal/server/scan.go b/internal/server/scan.go index 1f36185c..c19894b2 100644 --- a/internal/server/scan.go +++ b/internal/server/scan.go @@ -66,7 +66,7 @@ func (c *Server) cmdScan(msg *Message) (res resp.Value, err error) { } else { g := glob.Parse(sw.globPattern, s.desc) if g.Limits[0] == "" && g.Limits[1] == "" { - sw.col.Scan(s.desc, s.cursor, sw.IncCursor, + sw.col.Scan(s.desc, sw, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ id: id, @@ -76,7 +76,7 @@ func (c *Server) cmdScan(msg *Message) (res resp.Value, err error) { }, ) } else { - sw.col.ScanRange(g.Limits[0], g.Limits[1], s.desc, s.cursor, sw.IncCursor, + sw.col.ScanRange(g.Limits[0], g.Limits[1], s.desc, sw, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ id: id, @@ -95,3 +95,20 @@ func (c *Server) cmdScan(msg *Message) (res resp.Value, err error) { } return sw.respOut, nil } + +// type tCursor struct { +// offset func() uint64 +// iter func(n uint64) +// } + +// func (cursor *tCursor) Offset() uint64 { +// return cursor.offset() +// } + +// func (cursor *tCursor) Step(n uint64) { +// cursor.iter(n) +// } + +// func newCursor(offset func() uint64, iter func(n uint64)) *tCursor { +// return &tCursor{offset, iter} +// } diff --git a/internal/server/scanner.go b/internal/server/scanner.go index d612058a..c1580440 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -327,7 +327,11 @@ func (sw *scanWriter) globMatch(id string, o geojson.Object) (ok, keepGoing bool } // Increment cursor -func (sw *scanWriter) IncCursor(n uint64) { +func (sw *scanWriter) Offset() uint64 { + return sw.cursor +} + +func (sw *scanWriter) Step(n uint64) { sw.numberIters += n } @@ -342,7 +346,7 @@ func (sw *scanWriter) testObject(id string, o geojson.Object, fields []float64, } } nf, ok := sw.fieldMatch(fields, o) - return ok,true, nf + return ok, true, nf } //id string, o geojson.Object, fields []float64, noLock bool diff --git a/internal/server/search.go b/internal/server/search.go index 0da73149..d0a96ed2 100644 --- a/internal/server/search.go +++ b/internal/server/search.go @@ -409,11 +409,11 @@ func (server *Server) nearestNeighbors( maxDist := target.Haversine() limit := int(sw.limit) var items []iterItem - sw.col.Nearby(target, sw.cursor, sw.IncCursor, func(id string, o geojson.Object, fields []float64) bool { + sw.col.Nearby(target, sw, func(id string, o geojson.Object, fields []float64) bool { if server.hasExpired(s.key, id) { return true } - ok, keepGoing, _ := sw.testObject(id, o, fields,true) + ok, keepGoing, _ := sw.testObject(id, o, fields, true) if !ok { return true } @@ -480,7 +480,7 @@ func (server *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp. sw.writeHead() if sw.col != nil { if cmd == "within" { - sw.col.Within(s.obj, s.cursor, s.sparse, sw.IncCursor, func( + sw.col.Within(s.obj, s.sparse, sw, func( id string, o geojson.Object, fields []float64, ) bool { if server.hasExpired(s.key, id) { @@ -494,7 +494,7 @@ func (server *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp. }) }) } else if cmd == "intersects" { - sw.col.Intersects(s.obj, s.cursor, s.sparse, sw.IncCursor, func( + sw.col.Intersects(s.obj, s.sparse, sw, func( id string, o geojson.Object, fields []float64, @@ -578,7 +578,7 @@ func (server *Server) cmdSearch(msg *Message) (res resp.Value, err error) { } else { g := glob.Parse(sw.globPattern, s.desc) if g.Limits[0] == "" && g.Limits[1] == "" { - sw.col.SearchValues(s.desc, s.cursor, sw.IncCursor, + sw.col.SearchValues(s.desc, sw, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ id: id, @@ -592,7 +592,7 @@ func (server *Server) cmdSearch(msg *Message) (res resp.Value, err error) { // must disable globSingle for string value type matching because // globSingle is only for ID matches, not values. sw.globSingle = false - sw.col.SearchValuesRange(g.Limits[0], g.Limits[1], s.desc, s.cursor, sw.IncCursor, + sw.col.SearchValuesRange(g.Limits[0], g.Limits[1], s.desc, sw, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ id: id,