From 1b20a4c590d239888f1ae881e790e1187f2b423e Mon Sep 17 00:00:00 2001 From: w1n2k Date: Tue, 10 Jan 2017 19:49:48 +0300 Subject: [PATCH 1/5] Added distance to NEARBY command --- controller/fence.go | 8 +++++- controller/scan.go | 12 +++++++-- controller/scanner.go | 61 ++++++++++++++++++++++++++++--------------- controller/search.go | 37 ++++++++++++++++++++++---- controller/token.go | 9 +++++++ 5 files changed, 98 insertions(+), 29 deletions(-) diff --git a/controller/fence.go b/controller/fence.go index e9c3f795..92ebfc78 100644 --- a/controller/fence.go +++ b/controller/fence.go @@ -149,7 +149,13 @@ func fenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, metas sw.fmap = details.fmap sw.fullFields = true sw.msg.OutputType = server.JSON - sw.writeObject(details.id, details.obj, details.fields, true) + sw.writeObject(ScanWriterParams{ + id:details.id, + o: details.obj, + fields: details.fields, + noLock: true, + }) + if sw.wr.Len() == 0 { sw.mu.Unlock() return nil diff --git a/controller/scan.go b/controller/scan.go index 1ef61aff..ddfe407e 100644 --- a/controller/scan.go +++ b/controller/scan.go @@ -50,14 +50,22 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) { if g.Limits[0] == "" && g.Limits[1] == "" { s.cursor = sw.col.Scan(s.cursor, s.desc, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) + return sw.writeObject(ScanWriterParams{ + id: id, + o: o, + fields: fields, + }) }, ) } else { s.cursor = sw.col.ScanRange( s.cursor, g.Limits[0], g.Limits[1], s.desc, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) + return sw.writeObject(ScanWriterParams{ + id: id, + o: o, + fields: fields, + }) }, ) } diff --git a/controller/scanner.go b/controller/scanner.go index 5cdc99ef..9ddc5a42 100644 --- a/controller/scanner.go +++ b/controller/scanner.go @@ -54,6 +54,14 @@ type scanWriter struct { matchValues bool } +type ScanWriterParams struct { + id string + o geojson.Object + fields []float64 + distance float64 + noLock bool +} + func (c *Controller) newScanWriter( wr *bytes.Buffer, msg *server.Message, key string, output outputT, precision uint64, globPattern string, matchValues bool, @@ -233,24 +241,25 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) ([]float64, return sw.fvals, true } -func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, noLock bool) bool { - if !noLock { +//id string, o geojson.Object, fields []float64, noLock bool +func (sw *scanWriter) writeObject(opts ScanWriterParams) bool { + if !opts.noLock { sw.mu.Lock() defer sw.mu.Unlock() } keepGoing := true if !sw.globEverything { if sw.globSingle { - if sw.globPattern != id { + if sw.globPattern != opts.id { return true } keepGoing = false // return current object and stop iterating } else { var val string if sw.matchValues { - val = o.String() + val = opts.o.String() } else { - val = id + val = opts.id } ok, _ := glob.Match(sw.globPattern, val) if !ok { @@ -258,7 +267,7 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, } } } - nfields, ok := sw.fieldMatch(fields, o) + nfields, ok := sw.fieldMatch(opts.fields, opts.o) if !ok { return true } @@ -282,12 +291,12 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, jsfields = `,"fields":{` var i int for field, idx := range sw.fmap { - if len(fields) > idx { - if fields[idx] != 0 { + if len(opts.fields) > idx { + if opts.fields[idx] != 0 { if i > 0 { jsfields += `,` } - jsfields += jsonString(field) + ":" + strconv.FormatFloat(fields[idx], 'f', -1, 64) + jsfields += jsonString(field) + ":" + strconv.FormatFloat(opts.fields[idx], 'f', -1, 64) i++ } } @@ -307,38 +316,44 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, } } if sw.output == outputIDs { - wr.WriteString(jsonString(id)) + wr.WriteString(jsonString(opts.id)) } else { - wr.WriteString(`{"id":` + jsonString(id)) + wr.WriteString(`{"id":` + jsonString(opts.id)) switch sw.output { case outputObjects: - wr.WriteString(`,"object":` + o.JSON()) + wr.WriteString(`,"object":` + opts.o.JSON()) case outputPoints: - wr.WriteString(`,"point":` + o.CalculatedPoint().ExternalJSON()) + wr.WriteString(`,"point":` + opts.o.CalculatedPoint().ExternalJSON()) case outputHashes: - p, err := o.Geohash(int(sw.precision)) + p, err := opts.o.Geohash(int(sw.precision)) if err != nil { p = "" } wr.WriteString(`,"hash":"` + p + `"`) case outputBounds: - wr.WriteString(`,"bounds":` + o.CalculatedBBox().ExternalJSON()) + wr.WriteString(`,"bounds":` + opts.o.CalculatedBBox().ExternalJSON()) } + wr.WriteString(jsfields) + + if opts.distance > 0 { + wr.WriteString(`,"distance":` + strconv.FormatFloat(opts.distance, 'f', 2, 64)) + } + wr.WriteString(`}`) } sw.wr.Write(wr.Bytes()) case server.RESP: vals := make([]resp.Value, 1, 3) - vals[0] = resp.StringValue(id) + vals[0] = resp.StringValue(opts.id) if sw.output == outputIDs { sw.values = append(sw.values, vals[0]) } else { switch sw.output { case outputObjects: - vals = append(vals, resp.StringValue(o.String())) + vals = append(vals, resp.StringValue(opts.o.String())) case outputPoints: - point := o.CalculatedPoint() + point := opts.o.CalculatedPoint() if point.Z != 0 { vals = append(vals, resp.ArrayValue([]resp.Value{ resp.FloatValue(point.Y), @@ -352,13 +367,13 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, })) } case outputHashes: - p, err := o.Geohash(int(sw.precision)) + p, err := opts.o.Geohash(int(sw.precision)) if err != nil { p = "" } vals = append(vals, resp.StringValue(p)) case outputBounds: - bbox := o.CalculatedBBox() + bbox := opts.o.CalculatedBBox() vals = append(vals, resp.ArrayValue([]resp.Value{ resp.ArrayValue([]resp.Value{ resp.FloatValue(bbox.Min.Y), @@ -371,7 +386,7 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, })) } - fvs := orderFields(sw.fmap, fields) + fvs := orderFields(sw.fmap, opts.fields) if len(fvs) > 0 { fvals := make([]resp.Value, 0, len(fvs)*2) for i, fv := range fvs { @@ -381,6 +396,10 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, vals = append(vals, resp.ArrayValue(fvals)) } + if opts.distance > 0 { + vals = append(vals, resp.FloatValue(opts.distance)) + } + sw.values = append(sw.values, resp.ArrayValue(vals)) } } diff --git a/controller/search.go b/controller/search.go index e31c6a2e..a256bff0 100644 --- a/controller/search.go +++ b/controller/search.go @@ -291,7 +291,18 @@ func (c *Controller) cmdNearby(msg *server.Message) (res string, err error) { sw.writeHead() if sw.col != nil { s.cursor = sw.col.Nearby(s.cursor, s.sparse, s.lat, s.lon, s.meters, minZ, maxZ, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) + // Calculate distance if we need to + distance := 0.0 + if s.distance { + distance = o.CalculatedPoint().DistanceTo(geojson.Position{X: s.lon, Y: s.lat, Z: 0}) + } + + return sw.writeObject(ScanWriterParams{ + id: id, + o: o, + fields: fields, + distance: distance, + }) }) } sw.writeFoot(s.cursor) @@ -337,13 +348,21 @@ func (c *Controller) cmdWithinOrIntersects(cmd string, msg *server.Message) (res if cmd == "within" { s.cursor = sw.col.Within(s.cursor, s.sparse, s.o, s.minLat, s.minLon, s.maxLat, s.maxLon, minZ, maxZ, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) + return sw.writeObject(ScanWriterParams{ + id: id, + o: o, + fields: fields, + }) }, ) } else if cmd == "intersects" { s.cursor = sw.col.Intersects(s.cursor, s.sparse, s.o, s.minLat, s.minLon, s.maxLat, s.maxLon, minZ, maxZ, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) + return sw.writeObject(ScanWriterParams{ + id: id, + o: o, + fields: fields, + }) }, ) } @@ -394,7 +413,11 @@ func (c *Controller) cmdSearch(msg *server.Message) (res string, err error) { if g.Limits[0] == "" && g.Limits[1] == "" { s.cursor = sw.col.SearchValues(s.cursor, s.desc, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) + return sw.writeObject(ScanWriterParams{ + id: id, + o: o, + fields: fields, + }) }, ) } else { @@ -404,7 +427,11 @@ func (c *Controller) cmdSearch(msg *server.Message) (res string, err error) { s.cursor = sw.col.SearchValuesRange( s.cursor, g.Limits[0], g.Limits[1], s.desc, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) + return sw.writeObject(ScanWriterParams{ + id: id, + o: o, + fields: fields, + }) }, ) } diff --git a/controller/token.go b/controller/token.go index d9d9aaaa..0f0c2fb5 100644 --- a/controller/token.go +++ b/controller/token.go @@ -163,6 +163,7 @@ type searchScanBaseTokens struct { precision uint64 lineout string fence bool + distance bool detect map[string]bool accept map[string]bool glob string @@ -304,6 +305,14 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value, t.accept = nil } continue + } else if (wtok[0] == 'D' || wtok[0] == 'd') && strings.ToLower(wtok) == "distance" { + vs = nvs + if t.distance { + err = errDuplicateArgument(strings.ToUpper(wtok)) + return + } + t.distance = true + continue } else if (wtok[0] == 'D' || wtok[0] == 'd') && strings.ToLower(wtok) == "detect" { vs = nvs if t.detect != nil { From 44a596b12b14c1209a47c5e6a41778707ffb957e Mon Sep 17 00:00:00 2001 From: w1n2k Date: Tue, 10 Jan 2017 21:03:09 +0300 Subject: [PATCH 2/5] Added DISTANCE to command generation --- core/commands.json | 70 ++++++++++++++++++++++++-------------------- core/commands_gen.go | 70 ++++++++++++++++++++++++-------------------- 2 files changed, 76 insertions(+), 64 deletions(-) diff --git a/core/commands.json b/core/commands.json index 04c5e530..49f0bee6 100644 --- a/core/commands.json +++ b/core/commands.json @@ -45,7 +45,7 @@ "arguments":[ { "name": "geojson", - "type": "geojson" + "type": "geojson" } ] }, @@ -54,11 +54,11 @@ "arguments":[ { "name": "lat", - "type": "double" + "type": "double" }, { "name": "lon", - "type": "double" + "type": "double" }, { "name": "z", @@ -72,19 +72,19 @@ "arguments":[ { "name": "minlat", - "type": "double" + "type": "double" }, { "name": "minlon", - "type": "double" + "type": "double" }, { "name": "maxlat", - "type": "double" + "type": "double" }, { "name": "maxlon", - "type": "double" + "type": "double" } ] }, @@ -93,7 +93,7 @@ "arguments":[ { "name": "geohash", - "type": "geohash" + "type": "geohash" } ] }, @@ -102,7 +102,7 @@ "arguments":[ { "name": "value", - "type": "string" + "type": "string" } ] } @@ -483,6 +483,12 @@ "type": "pattern", "optional": true }, + { + "command": "DISTANCE", + "name": [], + "type": [], + "optional": true + }, { "command": "WHERE", "name": ["field","min","max"], @@ -569,15 +575,15 @@ "arguments":[ { "name": "key", - "type": "string" + "type": "string" }, { "name": "pattern", - "type": "pattern" + "type": "pattern" }, { "name": "meters", - "type": "double" + "type": "double" } ] } @@ -701,19 +707,19 @@ "arguments":[ { "name": "minlat", - "type": "double" + "type": "double" }, { "name": "minlon", - "type": "double" + "type": "double" }, { "name": "maxlat", - "type": "double" + "type": "double" }, { "name": "maxlon", - "type": "double" + "type": "double" } ] }, @@ -722,7 +728,7 @@ "arguments":[ { "name": "geojson", - "type": "geojson" + "type": "geojson" } ] }, @@ -731,15 +737,15 @@ "arguments":[ { "name": "x", - "type": "double" + "type": "double" }, { "name": "y", - "type": "double" + "type": "double" }, { "name": "z", - "type": "double" + "type": "double" } ] }, @@ -748,7 +754,7 @@ "arguments":[ { "name": "quadkey", - "type": "string" + "type": "string" } ] }, @@ -881,19 +887,19 @@ "arguments":[ { "name": "minlat", - "type": "double" + "type": "double" }, { "name": "minlon", - "type": "double" + "type": "double" }, { "name": "maxlat", - "type": "double" + "type": "double" }, { "name": "maxlon", - "type": "double" + "type": "double" } ] }, @@ -902,7 +908,7 @@ "arguments":[ { "name": "geojson", - "type": "geojson" + "type": "geojson" } ] }, @@ -911,15 +917,15 @@ "arguments":[ { "name": "x", - "type": "double" + "type": "double" }, { "name": "y", - "type": "double" + "type": "double" }, { "name": "z", - "type": "double" + "type": "double" } ] }, @@ -928,7 +934,7 @@ "arguments":[ { "name": "quadkey", - "type": "string" + "type": "string" } ] }, @@ -952,7 +958,7 @@ "arguments":[ { "name": "parameter", - "type": "string" + "type": "string" } ], "group": "server" @@ -962,7 +968,7 @@ "arguments":[ { "name": "parameter", - "type": "string" + "type": "string" }, { "name": "value", diff --git a/core/commands_gen.go b/core/commands_gen.go index 4cbe4414..528ff97f 100644 --- a/core/commands_gen.go +++ b/core/commands_gen.go @@ -207,7 +207,7 @@ var commandsJSON = `{ "arguments":[ { "name": "geojson", - "type": "geojson" + "type": "geojson" } ] }, @@ -216,11 +216,11 @@ var commandsJSON = `{ "arguments":[ { "name": "lat", - "type": "double" + "type": "double" }, { "name": "lon", - "type": "double" + "type": "double" }, { "name": "z", @@ -234,19 +234,19 @@ var commandsJSON = `{ "arguments":[ { "name": "minlat", - "type": "double" + "type": "double" }, { "name": "minlon", - "type": "double" + "type": "double" }, { "name": "maxlat", - "type": "double" + "type": "double" }, { "name": "maxlon", - "type": "double" + "type": "double" } ] }, @@ -255,7 +255,7 @@ var commandsJSON = `{ "arguments":[ { "name": "geohash", - "type": "geohash" + "type": "geohash" } ] }, @@ -264,7 +264,7 @@ var commandsJSON = `{ "arguments":[ { "name": "value", - "type": "string" + "type": "string" } ] } @@ -645,6 +645,12 @@ var commandsJSON = `{ "type": "pattern", "optional": true }, + { + "command": "DISTANCE", + "name": [], + "type": [], + "optional": true + }, { "command": "WHERE", "name": ["field","min","max"], @@ -731,15 +737,15 @@ var commandsJSON = `{ "arguments":[ { "name": "key", - "type": "string" + "type": "string" }, { "name": "pattern", - "type": "pattern" + "type": "pattern" }, { "name": "meters", - "type": "double" + "type": "double" } ] } @@ -863,19 +869,19 @@ var commandsJSON = `{ "arguments":[ { "name": "minlat", - "type": "double" + "type": "double" }, { "name": "minlon", - "type": "double" + "type": "double" }, { "name": "maxlat", - "type": "double" + "type": "double" }, { "name": "maxlon", - "type": "double" + "type": "double" } ] }, @@ -884,7 +890,7 @@ var commandsJSON = `{ "arguments":[ { "name": "geojson", - "type": "geojson" + "type": "geojson" } ] }, @@ -893,15 +899,15 @@ var commandsJSON = `{ "arguments":[ { "name": "x", - "type": "double" + "type": "double" }, { "name": "y", - "type": "double" + "type": "double" }, { "name": "z", - "type": "double" + "type": "double" } ] }, @@ -910,7 +916,7 @@ var commandsJSON = `{ "arguments":[ { "name": "quadkey", - "type": "string" + "type": "string" } ] }, @@ -1043,19 +1049,19 @@ var commandsJSON = `{ "arguments":[ { "name": "minlat", - "type": "double" + "type": "double" }, { "name": "minlon", - "type": "double" + "type": "double" }, { "name": "maxlat", - "type": "double" + "type": "double" }, { "name": "maxlon", - "type": "double" + "type": "double" } ] }, @@ -1064,7 +1070,7 @@ var commandsJSON = `{ "arguments":[ { "name": "geojson", - "type": "geojson" + "type": "geojson" } ] }, @@ -1073,15 +1079,15 @@ var commandsJSON = `{ "arguments":[ { "name": "x", - "type": "double" + "type": "double" }, { "name": "y", - "type": "double" + "type": "double" }, { "name": "z", - "type": "double" + "type": "double" } ] }, @@ -1090,7 +1096,7 @@ var commandsJSON = `{ "arguments":[ { "name": "quadkey", - "type": "string" + "type": "string" } ] }, @@ -1114,7 +1120,7 @@ var commandsJSON = `{ "arguments":[ { "name": "parameter", - "type": "string" + "type": "string" } ], "group": "server" @@ -1124,7 +1130,7 @@ var commandsJSON = `{ "arguments":[ { "name": "parameter", - "type": "string" + "type": "string" }, { "name": "value", From b55721a6c095699a6e4fb2896ea018e70acf443c Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Tue, 10 Jan 2017 12:16:55 -0700 Subject: [PATCH 3/5] Gitter badge Created a new Gitter room at https://gitter.im/tile38/tile38 closes #120 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 73694c16..199febb7 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ width="200" height="200" border="0" alt="Tile38">

+Gitter Version Build Status Docker Ready From 872f4c95b26b08b7de6ef2e848eadab5c187ec07 Mon Sep 17 00:00:00 2001 From: w1n2k Date: Wed, 11 Jan 2017 09:06:19 +0300 Subject: [PATCH 4/5] Fixed disque typo timeout handling --- controller/endpoint/disque.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/endpoint/disque.go b/controller/endpoint/disque.go index cd315fc2..b47ec998 100644 --- a/controller/endpoint/disque.go +++ b/controller/endpoint/disque.go @@ -35,7 +35,7 @@ func (conn *DisqueEndpointConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { - if time.Now().Sub(conn.t) > httpExpiresAfter { + if time.Now().Sub(conn.t) > disqueExpiresAfter { if conn.conn != nil { conn.close() } From 4300feccd6b993b407fc34fa8e4c199412b8284e Mon Sep 17 00:00:00 2001 From: Pavel Makarenko Date: Wed, 11 Jan 2017 14:41:58 +0300 Subject: [PATCH 5/5] Fixed #124: Added Content-Type json to HTTP headers (#125) * Fixed #124: Added Content-Type json to HTTP headers * Moved setting header after creation of request --- controller/endpoint/http.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/controller/endpoint/http.go b/controller/endpoint/http.go index c15a07bc..34c3cd53 100644 --- a/controller/endpoint/http.go +++ b/controller/endpoint/http.go @@ -63,6 +63,8 @@ func (conn *HTTPEndpointConn) Send(msg string) error { if err != nil { return err } + + req.Header.Set("Content-Type", "application/json") resp, err := conn.client.Do(req) if err != nil { return err