Merge pull request #658 from tidwall/better-tests

Better integration tests and various

cmd/tile38-server/main.go

@@ -141,12 +141,33 @@ Developer Options:
 	}
 
 	var (
-		devMode             bool
 		nohup               bool
 		showEvioDisabled    bool
 		showThreadsDisabled bool
 	)
 
+	var (
+		// use to be in core/options.go
+
+		// DevMode puts application in to dev mode
+		devMode = false
+
+		// ShowDebugMessages allows for log.Debug to print to console.
+		showDebugMessages = false
+
+		// ProtectedMode forces Tile38 to default in protected mode.
+		protectedMode = "no"
+
+		// AppendOnly allows for disabling the appendonly file.
+		appendOnly = true
+
+		// AppendFileName allows for custom appendonly file path
+		appendFileName = ""
+
+		// QueueFileName allows for custom queue.db file path
+		queueFileName = ""
+	)
+
 	// parse non standard args.
 	nargs := []string{os.Args[0]}
 	for i := 1; i < len(os.Args); i++ {
@@ -163,10 +184,10 @@ Developer Options:
 			if i < len(os.Args) {
 				switch strings.ToLower(os.Args[i]) {
 				case "no":
-					core.ProtectedMode = "no"
+					protectedMode = "no"
 					continue
 				case "yes":
-					core.ProtectedMode = "yes"
+					protectedMode = "yes"
 					continue
 				}
 			}
@@ -183,10 +204,10 @@ Developer Options:
 			if i < len(os.Args) {
 				switch strings.ToLower(os.Args[i]) {
 				case "no":
-					core.AppendOnly = false
+					appendOnly = false
 					continue
 				case "yes":
-					core.AppendOnly = true
+					appendOnly = true
 					continue
 				}
 			}
@@ -198,14 +219,14 @@ Developer Options:
 				fmt.Fprintf(os.Stderr, "appendfilename must have a value\n")
 				os.Exit(1)
 			}
-			core.AppendFileName = os.Args[i]
+			appendFileName = os.Args[i]
 		case "--queuefilename", "-queuefilename":
 			i++
 			if i == len(os.Args) || os.Args[i] == "" {
 				fmt.Fprintf(os.Stderr, "queuefilename must have a value\n")
 				os.Exit(1)
 			}
-			core.QueueFileName = os.Args[i]
+			queueFileName = os.Args[i]
 		case "--http-transport", "-http-transport":
 			i++
 			if i < len(os.Args) {
@@ -281,7 +302,7 @@ Developer Options:
 	flag.Parse()
 
 	if logEncoding == "json" {
-		log.LogJSON = true
+		log.SetLogJSON(true)
 		data, _ := os.ReadFile(filepath.Join(dir, "config"))
 		if gjson.GetBytes(data, "logconfig.encoding").String() == "json" {
 			c := gjson.GetBytes(data, "logconfig").String()
@@ -299,17 +320,16 @@ Developer Options:
 	log.SetOutput(logw)
 
 	if quiet {
-		log.Level = 0
+		log.SetLevel(0)
 	} else if veryVerbose {
-		log.Level = 3
+		log.SetLevel(3)
 	} else if verbose {
-		log.Level = 2
+		log.SetLevel(2)
 	} else {
-		log.Level = 1
+		log.SetLevel(1)
 	}
 
-	core.DevMode = devMode
+	showDebugMessages = veryVerbose
-	core.ShowDebugMessages = veryVerbose
 
 	hostd := ""
 	if host != "" {
@@ -425,7 +445,7 @@ Developer Options:
 		saddr = fmt.Sprintf("Port: %d", port)
 	}
 
-	if log.LogJSON {
+	if log.LogJSON() {
 		log.Printf(`Tile38 %s%s %d bit (%s/%s) %s%s, PID: %d. Visit tile38.com/sponsor to support the project`,
 			core.Version, gitsha, strconv.IntSize, runtime.GOARCH, runtime.GOOS, hostd, saddr, os.Getpid())
 	} else {
@@ -449,12 +469,18 @@ Developer Options:
 		log.Warnf("thread flag is deprecated use GOMAXPROCS to set number of threads instead")
 	}
 	opts := server.Options{
-		Host:           host,
+		Host:              host,
-		Port:           port,
+		Port:              port,
-		Dir:            dir,
+		Dir:               dir,
-		UseHTTP:        httpTransport,
+		UseHTTP:           httpTransport,
-		MetricsAddr:    *metricsAddr,
+		MetricsAddr:       *metricsAddr,
-		UnixSocketPath: unixSocket,
+		UnixSocketPath:    unixSocket,
+		DevMode:           devMode,
+		ShowDebugMessages: showDebugMessages,
+		ProtectedMode:     protectedMode,
+		AppendOnly:        appendOnly,
+		AppendFileName:    appendFileName,
+		QueueFileName:     queueFileName,
 	}
 	if err := server.Serve(opts); err != nil {
 		log.Fatal(err)

core/options.go

@@ -1,19 +0,0 @@
-package core
-
-// DevMode puts application in to dev mode
-var DevMode = false
-
-// ShowDebugMessages allows for log.Debug to print to console.
-var ShowDebugMessages = false
-
-// ProtectedMode forces Tile38 to default in protected mode.
-var ProtectedMode = "no"
-
-// AppendOnly allows for disabling the appendonly file.
-var AppendOnly = true
-
-// AppendFileName allows for custom appendonly file path
-var AppendFileName = ""
-
-// QueueFileName allows for custom queue.db file path
-var QueueFileName = ""

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/tidwall/geojson v1.3.6
 	github.com/tidwall/gjson v1.14.3
 	github.com/tidwall/hashmap v1.6.1
+	github.com/tidwall/limiter v0.4.0
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0
 	github.com/tidwall/redbench v0.1.0
@@ -31,6 +32,7 @@ require (
 	github.com/tidwall/sjson v1.2.4
 	github.com/xdg/scram v1.0.5
 	github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da
+	go.uber.org/atomic v1.5.0
 	go.uber.org/zap v1.13.0
 	golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
@@ -95,7 +97,6 @@ require (
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
 	github.com/xdg/stringprep v1.0.3 // indirect
 	go.opencensus.io v0.22.4 // indirect
-	go.uber.org/atomic v1.5.0 // indirect
 	go.uber.org/multierr v1.3.0 // indirect
 	go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect

go.sum

@@ -368,6 +368,8 @@ github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
 github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
 github.com/tidwall/hashmap v1.6.1 h1:FIAHjKwcyOo1Y3/orsQO08floKhInbEX2VQv7CQRNuw=
 github.com/tidwall/hashmap v1.6.1/go.mod h1:hX452N3VtFD8okD3/6q/yOquJvJmYxmZ1H0nLtwkaxM=
+github.com/tidwall/limiter v0.4.0 h1:nj+7mS6aMDRzp15QTVDrgkun0def5/PfB4ogs5NlIVQ=
+github.com/tidwall/limiter v0.4.0/go.mod h1:n+qBGuSOgAvgcq1xUvo+mXWg8oBLQC8wkkheN9KZou0=
 github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
 github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=

internal/endpoint/amqp.go

@@ -27,13 +27,21 @@ func (conn *AMQPConn) Expired() bool {
 	defer conn.mu.Unlock()
 	if !conn.ex {
 		if time.Since(conn.t) > amqpExpiresAfter {
-			conn.ex = true
 			conn.close()
+			conn.ex = true
 		}
 	}
 	return conn.ex
 }
 
+// ExpireNow forces the connection to expire
+func (conn *AMQPConn) ExpireNow() {
+	conn.mu.Lock()
+	defer conn.mu.Unlock()
+	conn.close()
+	conn.ex = true
+}
+
 func (conn *AMQPConn) close() {
 	if conn.conn != nil {
 		conn.conn.Close()

internal/endpoint/disque.go

@@ -33,15 +33,21 @@ func (conn *DisqueConn) Expired() bool {
 	defer conn.mu.Unlock()
 	if !conn.ex {
 		if time.Since(conn.t) > disqueExpiresAfter {
-			if conn.conn != nil {
+			conn.close()
-				conn.close()
-			}
 			conn.ex = true
 		}
 	}
 	return conn.ex
 }
 
+// ExpireNow forces the connection to expire
+func (conn *DisqueConn) ExpireNow() {
+	conn.mu.Lock()
+	defer conn.mu.Unlock()
+	conn.close()
+	conn.ex = true
+}
+
 func (conn *DisqueConn) close() {
 	if conn.conn != nil {
 		conn.conn.Close()

internal/endpoint/endpoint.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/streadway/amqp"
@@ -136,6 +137,7 @@ type Endpoint struct {
 
 // Conn is an endpoint connection
 type Conn interface {
+	ExpireNow()
 	Expired() bool
 	Send(val string) error
 }
@@ -145,6 +147,8 @@ type Manager struct {
 	mu        sync.RWMutex
 	conns     map[string]Conn
 	publisher LocalPublisher
+	shutdown  int32          // atomic bool
+	wg        sync.WaitGroup // run wait group
 }
 
 // NewManager returns a new manager
@@ -153,13 +157,29 @@ func NewManager(publisher LocalPublisher) *Manager {
 		conns:     make(map[string]Conn),
 		publisher: publisher,
 	}
-	go epc.Run()
+	epc.wg.Add(1)
+	go epc.run()
 	return epc
 }
 
+func (epc *Manager) Shutdown() {
+	defer epc.wg.Wait()
+	atomic.StoreInt32(&epc.shutdown, 1)
+	// expire the connections
+	epc.mu.Lock()
+	defer epc.mu.Unlock()
+	for _, conn := range epc.conns {
+		conn.ExpireNow()
+	}
+}
+
 // Run starts the managing of endpoints
-func (epc *Manager) Run() {
+func (epc *Manager) run() {
+	defer epc.wg.Done()
 	for {
+		if atomic.LoadInt32(&epc.shutdown) != 0 {
+			return
+		}
 		time.Sleep(time.Second)
 		func() {
 			epc.mu.Lock()

internal/endpoint/eventHub.go

@@ -28,6 +28,10 @@ func (conn *EvenHubConn) Expired() bool {
 	return false
 }
 
+// ExpireNow forces the connection to expire
+func (conn *EvenHubConn) ExpireNow() {
+}
+
 // Send sends a message
 func (conn *EvenHubConn) Send(msg string) error {
 	hub, err := eventhub.NewHubFromConnectionString(conn.ep.EventHub.ConnectionString)

internal/endpoint/grpc.go

@@ -36,14 +36,21 @@ func (conn *GRPCConn) Expired() bool {
 	defer conn.mu.Unlock()
 	if !conn.ex {
 		if time.Since(conn.t) > grpcExpiresAfter {
-			if conn.conn != nil {
+			conn.close()
-				conn.close()
-			}
 			conn.ex = true
 		}
 	}
 	return conn.ex
 }
+
+// ExpireNow forces the connection to expire
+func (conn *GRPCConn) ExpireNow() {
+	conn.mu.Lock()
+	defer conn.mu.Unlock()
+	conn.close()
+	conn.ex = true
+}
+
 func (conn *GRPCConn) close() {
 	if conn.conn != nil {
 		conn.conn.Close()

internal/endpoint/http.go

@@ -38,6 +38,10 @@ func (conn *HTTPConn) Expired() bool {
 	return false
 }
 
+// ExpireNow forces the connection to expire
+func (conn *HTTPConn) ExpireNow() {
+}
+
 // Send sends a message
 func (conn *HTTPConn) Send(msg string) error {
 	req, err := http.NewRequest("POST", conn.ep.Original, bytes.NewBufferString(msg))

internal/endpoint/kafka.go

@@ -34,15 +34,21 @@ func (conn *KafkaConn) Expired() bool {
 	defer conn.mu.Unlock()
 	if !conn.ex {
 		if time.Since(conn.t) > kafkaExpiresAfter {
-			if conn.conn != nil {
+			conn.close()
-				conn.close()
-			}
 			conn.ex = true
 		}
 	}
 	return conn.ex
 }
 
+// ExpireNow forces the connection to expire
+func (conn *KafkaConn) ExpireNow() {
+	conn.mu.Lock()
+	defer conn.mu.Unlock()
+	conn.close()
+	conn.ex = true
+}
+
 func (conn *KafkaConn) close() {
 	if conn.conn != nil {
 		conn.conn.Close()
@@ -62,7 +68,7 @@ func (conn *KafkaConn) Send(msg string) error {
 	}
 	conn.t = time.Now()
 
-	if log.Level > 2 {
+	if log.Level() > 2 {
 		sarama.Logger = lg.New(log.Output(), "[sarama] ", 0)
 	}
 

internal/endpoint/local.go

@@ -23,6 +23,10 @@ func (conn *LocalConn) Expired() bool {
 	return false
 }
 
+// ExpireNow forces the connection to expire
+func (conn *LocalConn) ExpireNow() {
+}
+
 // Send sends a message
 func (conn *LocalConn) Send(msg string) error {
 	conn.publisher.Publish(conn.ep.Local.Channel, msg)

internal/endpoint/mqtt.go

@@ -40,12 +40,19 @@ func (conn *MQTTConn) Expired() bool {
 	return conn.ex
 }
 
+// ExpireNow forces the connection to expire
+func (conn *MQTTConn) ExpireNow() {
+	conn.mu.Lock()
+	defer conn.mu.Unlock()
+	conn.close()
+	conn.ex = true
+}
+
 func (conn *MQTTConn) close() {
 	if conn.conn != nil {
 		if conn.conn.IsConnected() {
 			conn.conn.Disconnect(250)
 		}
-
 		conn.conn = nil
 	}
 }

internal/endpoint/nats.go

@@ -32,15 +32,21 @@ func (conn *NATSConn) Expired() bool {
 	defer conn.mu.Unlock()
 	if !conn.ex {
 		if time.Since(conn.t) > natsExpiresAfter {
-			if conn.conn != nil {
+			conn.close()
-				conn.close()
-			}
 			conn.ex = true
 		}
 	}
 	return conn.ex
 }
 
+// ExpireNow forces the connection to expire
+func (conn *NATSConn) ExpireNow() {
+	conn.mu.Lock()
+	defer conn.mu.Unlock()
+	conn.close()
+	conn.ex = true
+}
+
 func (conn *NATSConn) close() {
 	if conn.conn != nil {
 		conn.conn.Close()

internal/endpoint/pubsub.go

@@ -83,13 +83,21 @@ func (conn *PubSubConn) Expired() bool {
 	defer conn.mu.Unlock()
 	if !conn.ex {
 		if time.Since(conn.t) > pubsubExpiresAfter {
-			conn.ex = true
 			conn.close()
+			conn.ex = true
 		}
 	}
 	return conn.ex
 }
 
+// ExpireNow forces the connection to expire
+func (conn *PubSubConn) ExpireNow() {
+	conn.mu.Lock()
+	defer conn.mu.Unlock()
+	conn.close()
+	conn.ex = true
+}
+
 func newPubSubConn(ep Endpoint) *PubSubConn {
 	return &PubSubConn{
 		ep: ep,

internal/endpoint/redis.go

@@ -32,15 +32,21 @@ func (conn *RedisConn) Expired() bool {
 	defer conn.mu.Unlock()
 	if !conn.ex {
 		if time.Since(conn.t) > redisExpiresAfter {
-			if conn.conn != nil {
+			conn.close()
-				conn.close()
-			}
 			conn.ex = true
 		}
 	}
 	return conn.ex
 }
 
+// ExpireNow forces the connection to expire
+func (conn *RedisConn) ExpireNow() {
+	conn.mu.Lock()
+	defer conn.mu.Unlock()
+	conn.close()
+	conn.ex = true
+}
+
 func (conn *RedisConn) close() {
 	if conn.conn != nil {
 		conn.conn.Close()

internal/endpoint/sqs.go

@@ -39,13 +39,21 @@ func (conn *SQSConn) Expired() bool {
 	defer conn.mu.Unlock()
 	if !conn.ex {
 		if time.Since(conn.t) > sqsExpiresAfter {
-			conn.ex = true
 			conn.close()
+			conn.ex = true
 		}
 	}
 	return conn.ex
 }
 
+// ExpireNow forces the connection to expire
+func (conn *SQSConn) ExpireNow() {
+	conn.mu.Lock()
+	defer conn.mu.Unlock()
+	conn.close()
+	conn.ex = true
+}
+
 func (conn *SQSConn) close() {
 	if conn.svc != nil {
 		conn.svc = nil
@@ -82,7 +90,7 @@ func (conn *SQSConn) Send(msg string) error {
 		sess := session.Must(session.NewSession(&aws.Config{
 			Region:                        &region,
 			Credentials:                   creds,
-			CredentialsChainVerboseErrors: aws.Bool(log.Level >= 3),
+			CredentialsChainVerboseErrors: aws.Bool(log.Level() >= 3),
 			MaxRetries:                    aws.Int(5),
 		}))
 		svc := sqs.New(sess)

internal/log/log.go

@@ -6,91 +6,120 @@ import (
 	"io"
 	"os"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"go.uber.org/zap"
 	"golang.org/x/term"
 )
 
-var mu sync.Mutex
+var wmu sync.Mutex
 var wr io.Writer
-var tty bool
+
-var LogJSON = false
+var zmu sync.Mutex
-var logger *zap.SugaredLogger
+var zlogger *zap.SugaredLogger
+
+var tty atomic.Bool
+var ljson atomic.Bool
+var llevel atomic.Int32
+
+func init() {
+	SetOutput(os.Stderr)
+	SetLevel(1)
+}
 
 // Level is the log level
 // 0: silent  - do not log
 // 1: normal  - show everything except debug and warn
 // 2: verbose - show everything except debug
 // 3: very verbose - show everything
-var Level = 1
+func SetLevel(level int) {
+	if level < 0 {
+		level = 0
+	} else if level > 3 {
+		level = 3
+	}
+	llevel.Store(int32(level))
+}
+
+// Level returns the log level
+func Level() int {
+	return int(llevel.Load())
+}
+
+func SetLogJSON(logJSON bool) {
+	ljson.Store(logJSON)
+}
+
+func LogJSON() bool {
+	return ljson.Load()
+}
 
 // SetOutput sets the output of the logger
 func SetOutput(w io.Writer) {
 	f, ok := w.(*os.File)
-	tty = ok && term.IsTerminal(int(f.Fd()))
+	tty.Store(ok && term.IsTerminal(int(f.Fd())))
+	wmu.Lock()
 	wr = w
+	wmu.Unlock()
 }
 
 // Build a zap logger from default or custom config
 func Build(c string) error {
+	var zcfg zap.Config
 	if c == "" {
-		zcfg := zap.NewProductionConfig()
+		zcfg = zap.NewProductionConfig()
 
 		// to be able to filter with Tile38 levels
 		zcfg.Level.SetLevel(zap.DebugLevel)
 		// disable caller because caller is always log.go
 		zcfg.DisableCaller = true
 
-		core, err := zcfg.Build()
-		if err != nil {
-			return err
-		}
-		defer core.Sync()
-		logger = core.Sugar()
 	} else {
-		var zcfg zap.Config
 		err := json.Unmarshal([]byte(c), &zcfg)
 		if err != nil {
 			return err
 		}
-
 		// to be able to filter with Tile38 levels
 		zcfg.Level.SetLevel(zap.DebugLevel)
 		// disable caller because caller is always log.go
 		zcfg.DisableCaller = true
-
-		core, err := zcfg.Build()
-		if err != nil {
-			return err
-		}
-		defer core.Sync()
-		logger = core.Sugar()
 	}
+	core, err := zcfg.Build()
+	if err != nil {
+		return err
+	}
+	defer core.Sync()
+	zmu.Lock()
+	zlogger = core.Sugar()
+	zmu.Unlock()
 	return nil
 }
 
 // Set a zap logger
 func Set(sl *zap.SugaredLogger) {
-	logger = sl
+	zmu.Lock()
+	zlogger = sl
+	zmu.Unlock()
 }
 
 // Get a zap logger
 func Get() *zap.SugaredLogger {
-	return logger
+	zmu.Lock()
+	sl := zlogger
+	zmu.Unlock()
+	return sl
 }
 
-func init() {
+// Output returns the output writer
-	SetOutput(os.Stderr)
-}
-
-// Output retuns the output writer
 func Output() io.Writer {
+	wmu.Lock()
+	defer wmu.Unlock()
 	return wr
 }
 
 func log(level int, tag, color string, formatted bool, format string, args ...interface{}) {
-	if Level < level {
+	if llevel.Load() < int32(level) {
 		return
 	}
 	var msg string
@@ -99,30 +128,32 @@ func log(level int, tag, color string, formatted bool, format string, args ...in
 	} else {
 		msg = fmt.Sprint(args...)
 	}
-	if LogJSON {
+	if ljson.Load() {
+		zmu.Lock()
+		defer zmu.Unlock()
 		switch tag {
 		case "ERRO":
-			logger.Error(msg)
+			zlogger.Error(msg)
 		case "FATA":
-			logger.Fatal(msg)
+			zlogger.Fatal(msg)
 		case "WARN":
-			logger.Warn(msg)
+			zlogger.Warn(msg)
 		case "DEBU":
-			logger.Debug(msg)
+			zlogger.Debug(msg)
 		default:
-			logger.Info(msg)
+			zlogger.Info(msg)
 		}
 		return
 	}
 	s := []byte(time.Now().Format("2006/01/02 15:04:05"))
 	s = append(s, ' ')
-	if tty {
+	if tty.Load() {
 		s = append(s, color...)
 	}
 	s = append(s, '[')
 	s = append(s, tag...)
 	s = append(s, ']')
-	if tty {
+	if tty.Load() {
 		s = append(s, "\x1b[0m"...)
 	}
 	s = append(s, ' ')
@@ -130,79 +161,79 @@ func log(level int, tag, color string, formatted bool, format string, args ...in
 	if s[len(s)-1] != '\n' {
 		s = append(s, '\n')
 	}
-	mu.Lock()
+	wmu.Lock()
 	wr.Write(s)
-	mu.Unlock()
+	wmu.Unlock()
 }
 
 var emptyFormat string
 
 // Infof ...
 func Infof(format string, args ...interface{}) {
-	if Level >= 1 {
+	if llevel.Load() >= 1 {
 		log(1, "INFO", "\x1b[36m", true, format, args...)
 	}
 }
 
 // Info ...
 func Info(args ...interface{}) {
-	if Level >= 1 {
+	if llevel.Load() >= 1 {
 		log(1, "INFO", "\x1b[36m", false, emptyFormat, args...)
 	}
 }
 
 // HTTPf ...
 func HTTPf(format string, args ...interface{}) {
-	if Level >= 1 {
+	if llevel.Load() >= 1 {
 		log(1, "HTTP", "\x1b[1m\x1b[30m", true, format, args...)
 	}
 }
 
 // HTTP ...
 func HTTP(args ...interface{}) {
-	if Level >= 1 {
+	if llevel.Load() >= 1 {
 		log(1, "HTTP", "\x1b[1m\x1b[30m", false, emptyFormat, args...)
 	}
 }
 
 // Errorf ...
 func Errorf(format string, args ...interface{}) {
-	if Level >= 1 {
+	if llevel.Load() >= 1 {
 		log(1, "ERRO", "\x1b[1m\x1b[31m", true, format, args...)
 	}
 }
 
 // Error ..
 func Error(args ...interface{}) {
-	if Level >= 1 {
+	if llevel.Load() >= 1 {
 		log(1, "ERRO", "\x1b[1m\x1b[31m", false, emptyFormat, args...)
 	}
 }
 
 // Warnf ...
 func Warnf(format string, args ...interface{}) {
-	if Level >= 1 {
+	if llevel.Load() >= 1 {
 		log(2, "WARN", "\x1b[33m", true, format, args...)
 	}
 }
 
 // Warn ...
 func Warn(args ...interface{}) {
-	if Level >= 1 {
+	if llevel.Load() >= 1 {
 		log(2, "WARN", "\x1b[33m", false, emptyFormat, args...)
 	}
 }
 
 // Debugf ...
 func Debugf(format string, args ...interface{}) {
-	if Level >= 3 {
+	if llevel.Load() >= 3 {
 		log(3, "DEBU", "\x1b[35m", true, format, args...)
 	}
 }
 
 // Debug ...
 func Debug(args ...interface{}) {
-	if Level >= 3 {
+	if llevel.Load() >= 3 {
 		log(3, "DEBU", "\x1b[35m", false, emptyFormat, args...)
 	}
 }

internal/log/log_test.go

@@ -13,7 +13,7 @@ import (
 
 func TestLog(t *testing.T) {
 	f := &bytes.Buffer{}
-	LogJSON = false
+	SetLogJSON(false)
 	SetOutput(f)
 	Printf("hello %v", "everyone")
 	if !strings.HasSuffix(f.String(), "hello everyone\n") {
@@ -23,7 +23,7 @@ func TestLog(t *testing.T) {
 
 func TestLogJSON(t *testing.T) {
 
-	LogJSON = true
+	SetLogJSON(true)
 	Build("")
 
 	type tcase struct {
@@ -40,7 +40,7 @@ func TestLogJSON(t *testing.T) {
 		return func(t *testing.T) {
 			observedZapCore, observedLogs := observer.New(zap.DebugLevel)
 			Set(zap.New(observedZapCore).Sugar())
-			Level = tc.level
+			SetLevel(tc.level)
 
 			if tc.format != "" {
 				tc.fops(tc.format, tc.args)
@@ -187,8 +187,8 @@ func TestLogJSON(t *testing.T) {
 }
 
 func BenchmarkLogPrintf(t *testing.B) {
-	LogJSON = false
+	SetLogJSON(false)
-	Level = 1
+	SetLevel(1)
 	SetOutput(io.Discard)
 	t.ResetTimer()
 	for i := 0; i < t.N; i++ {
@@ -197,8 +197,8 @@ func BenchmarkLogPrintf(t *testing.B) {
 }
 
 func BenchmarkLogJSONPrintf(t *testing.B) {
-	LogJSON = true
+	SetLogJSON(true)
-	Level = 1
+	SetLevel(1)
 
 	ec := zap.NewProductionEncoderConfig()
 	ec.EncodeDuration = zapcore.NanosDurationEncoder

internal/server/aof.go

@@ -41,10 +41,7 @@ func (s *Server) loadAOF() (err error) {
 		ps := float64(count) / (float64(d) / float64(time.Second))
 		suf := []string{"bytes/s", "KB/s", "MB/s", "GB/s", "TB/s"}
 		bps := float64(fi.Size()) / (float64(d) / float64(time.Second))
-		for i := 0; bps > 1024; i++ {
+		for i := 0; bps > 1024 && len(suf) > 1; i++ {
-			if len(suf) == 1 {
-				break
-			}
 			bps /= 1024
 			suf = suf[1:]
 		}
@@ -123,11 +120,7 @@ func commandErrIsFatal(err error) bool {
 	// FSET (and other writable commands) may return errors that we need
 	// to ignore during the loading process. These errors may occur (though unlikely)
 	// due to the aof rewrite operation.
-	switch err {
+	return !(err == errKeyNotFound || err == errIDNotFound)
-	case errKeyNotFound, errIDNotFound:
-		return false
-	}
-	return true
 }
 
 // flushAOF flushes all aof buffer data to disk. Set sync to true to sync the
@@ -406,73 +399,80 @@ func (s liveAOFSwitches) Error() string {
 	return goingLive
 }
 
-func (s *Server) cmdAOFMD5(msg *Message) (res resp.Value, err error) {
+// AOFMD5 pos size
+func (s *Server) cmdAOFMD5(msg *Message) (resp.Value, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
-	var ok bool
-	var spos, ssize string
 
-	if vs, spos, ok = tokenval(vs); !ok || spos == "" {
+	// >> Args
-		return NOMessage, errInvalidNumberOfArguments
+
+	args := msg.Args
+	if len(args) != 3 {
+		return retrerr(errInvalidNumberOfArguments)
 	}
-	if vs, ssize, ok = tokenval(vs); !ok || ssize == "" {
+	pos, err := strconv.ParseInt(args[1], 10, 64)
-		return NOMessage, errInvalidNumberOfArguments
-	}
-	if len(vs) != 0 {
-		return NOMessage, errInvalidNumberOfArguments
-	}
-	pos, err := strconv.ParseInt(spos, 10, 64)
 	if err != nil || pos < 0 {
-		return NOMessage, errInvalidArgument(spos)
+		return retrerr(errInvalidArgument(args[1]))
 	}
-	size, err := strconv.ParseInt(ssize, 10, 64)
+	size, err := strconv.ParseInt(args[2], 10, 64)
 	if err != nil || size < 0 {
-		return NOMessage, errInvalidArgument(ssize)
+		return retrerr(errInvalidArgument(args[2]))
 	}
+
+	// >> Operation
+
 	sum, err := s.checksum(pos, size)
 	if err != nil {
-		return NOMessage, err
+		return retrerr(err)
 	}
-	switch msg.OutputType {
+
-	case JSON:
+	// >> Response
-		res = resp.StringValue(
+
-			fmt.Sprintf(`{"ok":true,"md5":"%s","elapsed":"%s"}`, sum, time.Since(start)))
+	if msg.OutputType == JSON {
-	case RESP:
+		return resp.StringValue(fmt.Sprintf(
-		res = resp.SimpleStringValue(sum)
+			`{"ok":true,"md5":"%s","elapsed":"%s"}`,
+			sum, time.Since(start))), nil
 	}
-	return res, nil
+	return resp.SimpleStringValue(sum), nil
 }
 
-func (s *Server) cmdAOF(msg *Message) (res resp.Value, err error) {
+// AOF pos
+func (s *Server) cmdAOF(msg *Message) (resp.Value, error) {
 	if s.aof == nil {
-		return NOMessage, errors.New("aof disabled")
+		return retrerr(errors.New("aof disabled"))
 	}
-	vs := msg.Args[1:]
 
-	var ok bool
+	// >> Args
-	var spos string
+
-	if vs, spos, ok = tokenval(vs); !ok || spos == "" {
+	args := msg.Args
-		return NOMessage, errInvalidNumberOfArguments
+	if len(args) != 2 {
+		return retrerr(errInvalidNumberOfArguments)
 	}
-	if len(vs) != 0 {
+
-		return NOMessage, errInvalidNumberOfArguments
+	pos, err := strconv.ParseInt(args[1], 10, 64)
-	}
-	pos, err := strconv.ParseInt(spos, 10, 64)
 	if err != nil || pos < 0 {
-		return NOMessage, errInvalidArgument(spos)
+		return retrerr(errInvalidArgument(args[1]))
 	}
+
+	// >> Operation
+
 	f, err := os.Open(s.aof.Name())
 	if err != nil {
-		return NOMessage, err
+		return retrerr(err)
 	}
 	defer f.Close()
+
 	n, err := f.Seek(0, 2)
 	if err != nil {
-		return NOMessage, err
+		return retrerr(err)
 	}
+
 	if n < pos {
-		return NOMessage, errors.New("pos is too big, must be less that the aof_size of leader")
+		return retrerr(errors.New(
+			"pos is too big, must be less that the aof_size of leader"))
 	}
+
+	// >> Response
+
 	var ls liveAOFSwitches
 	ls.pos = pos
 	return NOMessage, ls
@@ -485,8 +485,6 @@ func (s *Server) liveAOF(pos int64, conn net.Conn, rd *PipelineReader, msg *Mess
 	if err != nil {
 		return err
 	}
-	defer f.Close()
-
 	s.mu.Lock()
 	s.aofconnM[conn] = f
 	s.mu.Unlock()
@@ -495,91 +493,42 @@ func (s *Server) liveAOF(pos int64, conn net.Conn, rd *PipelineReader, msg *Mess
 		delete(s.aofconnM, conn)
 		s.mu.Unlock()
 		conn.Close()
+		f.Close()
 	}()
 
 	if _, err := conn.Write([]byte("+OK\r\n")); err != nil {
 		return err
 	}
-
 	if _, err := f.Seek(pos, 0); err != nil {
 		return err
 	}
-	cond := sync.NewCond(&sync.Mutex{})
+	var wg sync.WaitGroup
-	var mustQuit bool
+	wg.Add(1)
 	go func() {
 		defer func() {
-			cond.L.Lock()
+			f.Close()
-			mustQuit = true
+			conn.Close()
-			cond.Broadcast()
+			wg.Done()
-			cond.L.Unlock()
 		}()
-		for {
+		// Any incoming message should end the connection
-			vs, err := rd.ReadMessages()
+		rd.ReadMessages()
-			if err != nil {
-				if err != io.EOF {
-					log.Error(err)
-				}
-				return
-			}
-			for _, v := range vs {
-				switch v.Command() {
-				default:
-					log.Error("received a live command that was not QUIT")
-					return
-				case "quit", "":
-					return
-				}
-			}
-		}
 	}()
-	go func() {
+	_, err = io.Copy(conn, f)
-		defer func() {
+	if err != nil {
-			cond.L.Lock()
+		return err
-			mustQuit = true
+	}
-			cond.Broadcast()
+	b := make([]byte, 4096*2)
-			cond.L.Unlock()
+	for {
-		}()
+		n, err := f.Read(b)
-		err := func() error {
+		if n > 0 {
-			_, err := io.Copy(conn, f)
+			if _, err := conn.Write(b[:n]); err != nil {
-			if err != nil {
 				return err
 			}
-
-			b := make([]byte, 4096)
-			// The reader needs to be OK with the eof not
-			for {
-				n, err := f.Read(b)
-				if n > 0 {
-					if _, err := conn.Write(b[:n]); err != nil {
-						return err
-					}
-				}
-				if err != io.EOF {
-					if err != nil {
-						return err
-					}
-					continue
-				}
-				s.fcond.L.Lock()
-				s.fcond.Wait()
-				s.fcond.L.Unlock()
-			}
-		}()
-		if err != nil {
-			if !strings.Contains(err.Error(), "use of closed network connection") &&
-				!strings.Contains(err.Error(), "bad file descriptor") {
-				log.Error(err)
-			}
-			return
 		}
-	}()
+		if err == io.EOF {
-	for {
+			time.Sleep(time.Second / 4)
-		cond.L.Lock()
+		} else if err != nil {
-		if mustQuit {
+			return err
-			cond.L.Unlock()
-			return nil
 		}
-		cond.Wait()
-		cond.L.Unlock()
 	}
 }

internal/server/aofshrink.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/tidwall/btree"
-	"github.com/tidwall/tile38/core"
 	"github.com/tidwall/tile38/internal/collection"
 	"github.com/tidwall/tile38/internal/field"
 	"github.com/tidwall/tile38/internal/log"
@@ -20,12 +19,9 @@ const maxids = 32
 const maxchunk = 4 * 1024 * 1024
 
 func (s *Server) aofshrink() {
-	if s.aof == nil {
-		return
-	}
 	start := time.Now()
 	s.mu.Lock()
-	if s.shrinking {
+	if s.aof == nil || s.shrinking {
 		s.mu.Unlock()
 		return
 	}
@@ -42,7 +38,7 @@ func (s *Server) aofshrink() {
 	}()
 
 	err := func() error {
-		f, err := os.Create(core.AppendFileName + "-shrink")
+		f, err := os.Create(s.opts.AppendFileName + "-shrink")
 		if err != nil {
 			return err
 		}
@@ -279,13 +275,13 @@ func (s *Server) aofshrink() {
 			if err := f.Close(); err != nil {
 				log.Fatalf("shrink new aof close fatal operation: %v", err)
 			}
-			if err := os.Rename(core.AppendFileName, core.AppendFileName+"-bak"); err != nil {
+			if err := os.Rename(s.opts.AppendFileName, s.opts.AppendFileName+"-bak"); err != nil {
 				log.Fatalf("shrink backup fatal operation: %v", err)
 			}
-			if err := os.Rename(core.AppendFileName+"-shrink", core.AppendFileName); err != nil {
+			if err := os.Rename(s.opts.AppendFileName+"-shrink", s.opts.AppendFileName); err != nil {
 				log.Fatalf("shrink rename fatal operation: %v", err)
 			}
-			s.aof, err = os.OpenFile(core.AppendFileName, os.O_CREATE|os.O_RDWR, 0600)
+			s.aof, err = os.OpenFile(s.opts.AppendFileName, os.O_CREATE|os.O_RDWR, 0600)
 			if err != nil {
 				log.Fatalf("shrink openfile fatal operation: %v", err)
 			}
@@ -296,7 +292,7 @@ func (s *Server) aofshrink() {
 			}
 			s.aofsz = int(n)
 
-			os.Remove(core.AppendFileName + "-bak") // ignore error
+			os.Remove(s.opts.AppendFileName + "-bak") // ignore error
 
 			return nil
 		}()

internal/server/atomic.go

@@ -1,32 +0,0 @@
-package server
-
-import (
-	"sync/atomic"
-)
-
-type aint struct{ v uintptr }
-
-func (a *aint) add(d int) int {
-	if d < 0 {
-		return int(atomic.AddUintptr(&a.v, ^uintptr((d*-1)-1)))
-	}
-	return int(atomic.AddUintptr(&a.v, uintptr(d)))
-}
-func (a *aint) get() int {
-	return int(atomic.LoadUintptr(&a.v))
-}
-func (a *aint) set(i int) int {
-	return int(atomic.SwapUintptr(&a.v, uintptr(i)))
-}
-
-type abool struct{ v uint32 }
-
-func (a *abool) on() bool {
-	return atomic.LoadUint32(&a.v) != 0
-}
-func (a *abool) set(t bool) bool {
-	if t {
-		return atomic.SwapUint32(&a.v, 1) != 0
-	}
-	return atomic.SwapUint32(&a.v, 0) != 0
-}

internal/server/atomic_test.go

@@ -1,19 +0,0 @@
-package server
-
-import "testing"
-
-func TestAtomicInt(t *testing.T) {
-	var x aint
-	x.set(10)
-	if x.get() != 10 {
-		t.Fatalf("expected %v, got %v", 10, x.get())
-	}
-	x.add(-9)
-	if x.get() != 1 {
-		t.Fatalf("expected %v, got %v", 1, x.get())
-	}
-	x.add(-1)
-	if x.get() != 0 {
-		t.Fatalf("expected %v, got %v", 0, x.get())
-	}
-}

internal/server/bson.go

@@ -5,7 +5,6 @@ import (
 	"crypto/rand"
 	"encoding/binary"
 	"encoding/hex"
-	"io"
 	"os"
 	"sync/atomic"
 	"time"
@@ -23,23 +22,17 @@ func bsonID() string {
 var (
 	bsonProcess = uint16(os.Getpid())
 	bsonMachine = func() []byte {
-		host, err := os.Hostname()
+		host, _ := os.Hostname()
-		if err != nil {
+		b := make([]byte, 3)
-			b := make([]byte, 3)
+		Must(rand.Read(b))
-			if _, err := io.ReadFull(rand.Reader, b); err != nil {
+		host = Default(host, string(b))
-				panic("random error: " + err.Error())
-			}
-			return b
-		}
 		hw := md5.New()
 		hw.Write([]byte(host))
 		return hw.Sum(nil)[:3]
 	}()
 	bsonCounter = func() uint32 {
 		b := make([]byte, 4)
-		if _, err := io.ReadFull(rand.Reader, b); err != nil {
+		Must(rand.Read(b))
-			panic("random error: " + err.Error())
-		}
 		return binary.BigEndian.Uint32(b)
 	}()
 )

internal/server/bson_test.go

@@ -0,0 +1,10 @@
+package server
+
+import "testing"
+
+func TestBSON(t *testing.T) {
+	id := bsonID()
+	if len(id) != 24 {
+		t.Fail()
+	}
+}

internal/server/checksum.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/tidwall/resp"
-	"github.com/tidwall/tile38/core"
 	"github.com/tidwall/tile38/internal/log"
 )
 
@@ -140,12 +139,12 @@ func getEndOfLastValuePositionInFile(fname string, startPos int64) (int64, error
 // We will do some various checksums on the leader until we find the correct position to start at.
 func (s *Server) followCheckSome(addr string, followc int, auth string,
 ) (pos int64, err error) {
-	if core.ShowDebugMessages {
+	if s.opts.ShowDebugMessages {
 		log.Debug("follow:", addr, ":check some")
 	}
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	if s.followc.get() != followc {
+	if int(s.followc.Load()) != followc {
 		return 0, errNoLongerFollowing
 	}
 	if s.aofsz < checksumsz {
@@ -211,7 +210,7 @@ func (s *Server) followCheckSome(addr string, followc int, auth string,
 		return 0, err
 	}
 	if pos == fullpos {
-		if core.ShowDebugMessages {
+		if s.opts.ShowDebugMessages {
 			log.Debug("follow: aof fully intact")
 		}
 		return pos, nil

internal/server/client.go

@@ -2,7 +2,6 @@ package server
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"sort"
@@ -32,6 +31,8 @@ type Client struct {
 	name   string             // optional defined name
 	opened time.Time          // when the client was created/opened, unix nano
 	last   time.Time          // last client request/response, unix nano
+
+	closer io.Closer // used to close the connection
 }
 
 // Write ...
@@ -40,32 +41,19 @@ func (client *Client) Write(b []byte) (n int, err error) {
 	return len(b), nil
 }
 
-type byID []*Client
+// CLIENT (LIST | KILL | GETNAME | SETNAME)
-
+func (s *Server) cmdCLIENT(msg *Message, client *Client) (resp.Value, error) {
-func (arr byID) Len() int {
-	return len(arr)
-}
-func (arr byID) Less(a, b int) bool {
-	return arr[a].id < arr[b].id
-}
-func (arr byID) Swap(a, b int) {
-	arr[a], arr[b] = arr[b], arr[a]
-}
-
-func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) {
 	start := time.Now()
 
-	if len(msg.Args) == 1 {
+	args := msg.Args
-		return NOMessage, errInvalidNumberOfArguments
+	if len(args) == 1 {
+		return retrerr(errInvalidNumberOfArguments)
 	}
-	switch strings.ToLower(msg.Args[1]) {
+
-	default:
+	switch strings.ToLower(args[1]) {
-		return NOMessage, clientErrorf(
-			"Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)",
-		)
 	case "list":
-		if len(msg.Args) != 2 {
+		if len(args) != 2 {
-			return NOMessage, errInvalidNumberOfArguments
+			return retrerr(errInvalidNumberOfArguments)
 		}
 		var list []*Client
 		s.connsmu.RLock()
@@ -73,7 +61,9 @@ func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) {
 			list = append(list, cc)
 		}
 		s.connsmu.RUnlock()
-		sort.Sort(byID(list))
+		sort.Slice(list, func(i, j int) bool {
+			return list[i].id < list[j].id
+		})
 		now := time.Now()
 		var buf []byte
 		for _, client := range list {
@@ -89,8 +79,7 @@ func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) {
 			)
 			client.mu.Unlock()
 		}
-		switch msg.OutputType {
+		if msg.OutputType == JSON {
-		case JSON:
 			// Create a map of all key/value info fields
 			var cmap []map[string]interface{}
 			clients := strings.Split(string(buf), "\n")
@@ -110,124 +99,113 @@ func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) {
 				}
 			}
 
-			// Marshal the map and use the output in the JSON response
+			data, _ := json.Marshal(cmap)
-			data, err := json.Marshal(cmap)
+			return resp.StringValue(`{"ok":true,"list":` + string(data) +
-			if err != nil {
-				return NOMessage, err
-			}
-			return resp.StringValue(`{"ok":true,"list":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil
-		case RESP:
-			return resp.BytesValue(buf), nil
-		}
-		return NOMessage, nil
-	case "getname":
-		if len(msg.Args) != 2 {
-			return NOMessage, errInvalidNumberOfArguments
-		}
-		name := ""
-		switch msg.OutputType {
-		case JSON:
-			client.mu.Lock()
-			name := client.name
-			client.mu.Unlock()
-			return resp.StringValue(`{"ok":true,"name":` +
-				jsonString(name) +
 				`,"elapsed":"` + time.Since(start).String() + "\"}"), nil
-		case RESP:
-			return resp.StringValue(name), nil
 		}
+		return resp.BytesValue(buf), nil
+	case "getname":
+		if len(args) != 2 {
+			return retrerr(errInvalidNumberOfArguments)
+		}
+		client.mu.Lock()
+		name := client.name
+		client.mu.Unlock()
+		if msg.OutputType == JSON {
+			return resp.StringValue(`{"ok":true,"name":` + jsonString(name) +
+				`,"elapsed":"` + time.Since(start).String() + "\"}"), nil
+		}
+		return resp.StringValue(name), nil
 	case "setname":
-		if len(msg.Args) != 3 {
+		if len(args) != 3 {
-			return NOMessage, errInvalidNumberOfArguments
+			return retrerr(errInvalidNumberOfArguments)
 		}
 		name := msg.Args[2]
 		for i := 0; i < len(name); i++ {
 			if name[i] < '!' || name[i] > '~' {
-				return NOMessage, clientErrorf(
+				return retrerr(clientErrorf(
 					"Client names cannot contain spaces, newlines or special characters.",
-				)
+				))
 			}
 		}
 		client.mu.Lock()
 		client.name = name
 		client.mu.Unlock()
-		switch msg.OutputType {
+		if msg.OutputType == JSON {
-		case JSON:
+			return resp.StringValue(`{"ok":true,"elapsed":"` +
-			return resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}"), nil
+				time.Since(start).String() + "\"}"), nil
-		case RESP:
-			return resp.SimpleStringValue("OK"), nil
 		}
+		return resp.SimpleStringValue("OK"), nil
 	case "kill":
-		if len(msg.Args) < 3 {
+		if len(args) < 3 {
-			return NOMessage, errInvalidNumberOfArguments
+			return retrerr(errInvalidNumberOfArguments)
 		}
 		var useAddr bool
 		var addr string
 		var useID bool
 		var id string
-		for i := 2; i < len(msg.Args); i++ {
+		for i := 2; i < len(args); i++ {
-			arg := msg.Args[i]
+			if useAddr || useID {
+				return retrerr(errInvalidNumberOfArguments)
+			}
+			arg := args[i]
 			if strings.Contains(arg, ":") {
 				addr = arg
 				useAddr = true
-				break
+			} else {
-			}
+				switch strings.ToLower(arg) {
-			switch strings.ToLower(arg) {
+				case "addr":
-			default:
+					i++
-				return NOMessage, clientErrorf("No such client")
+					if i == len(args) {
-			case "addr":
+						return retrerr(errInvalidNumberOfArguments)
-				i++
+					}
-				if i == len(msg.Args) {
+					addr = args[i]
-					return NOMessage, errors.New("syntax error")
+					useAddr = true
+				case "id":
+					i++
+					if i == len(args) {
+						return retrerr(errInvalidNumberOfArguments)
+					}
+					id = args[i]
+					useID = true
+				default:
+					return retrerr(clientErrorf("No such client"))
 				}
-				addr = msg.Args[i]
-				useAddr = true
-			case "id":
-				i++
-				if i == len(msg.Args) {
-					return NOMessage, errors.New("syntax error")
-				}
-				id = msg.Args[i]
-				useID = true
 			}
 		}
-		var cclose *Client
+		var closing []io.Closer
 		s.connsmu.RLock()
 		for _, cc := range s.conns {
 			if useID && fmt.Sprintf("%d", cc.id) == id {
-				cclose = cc
+				if cc.closer != nil {
-				break
+					closing = append(closing, cc.closer)
-			} else if useAddr && client.remoteAddr == addr {
+				}
-				cclose = cc
+			} else if useAddr {
-				break
+				if cc.remoteAddr == addr {
+					if cc.closer != nil {
+						closing = append(closing, cc.closer)
+					}
+				}
 			}
 		}
 		s.connsmu.RUnlock()
-		if cclose == nil {
+		if len(closing) == 0 {
-			return NOMessage, clientErrorf("No such client")
+			return retrerr(clientErrorf("No such client"))
 		}
-
+		// go func() {
-		var res resp.Value
+		// close the connections behind the scene
-		switch msg.OutputType {
+		for _, closer := range closing {
-		case JSON:
+			closer.Close()
-			res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
-		case RESP:
-			res = resp.SimpleStringValue("OK")
 		}
-
+		// }()
-		client.conn.Close()
+		if msg.OutputType == JSON {
-		// closing self, return response now
+			return resp.StringValue(`{"ok":true,"elapsed":"` +
-		// NOTE: This is the only exception where we do convert response to a string
+				time.Since(start).String() + "\"}"), nil
-		var outBytes []byte
-		switch msg.OutputType {
-		case JSON:
-			outBytes = res.Bytes()
-		case RESP:
-			outBytes, _ = res.MarshalRESP()
 		}
-		cclose.conn.Write(outBytes)
+		return resp.SimpleStringValue("OK"), nil
-		cclose.conn.Close()
+	default:
-		return res, nil
+		return retrerr(clientErrorf(
+			"Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)",
+		))
 	}
-	return NOMessage, errors.New("invalid output type")
 }

internal/server/config.go

@@ -427,6 +427,9 @@ func (s *Server) cmdConfigSet(msg *Message) (res resp.Value, err error) {
 	if err := s.config.setProperty(name, value, false); err != nil {
 		return NOMessage, err
 	}
+	if name == MaxMemory {
+		s.checkOutOfMemory()
+	}
 	return OKMessage(msg, start), nil
 }
 func (s *Server) cmdConfigRewrite(msg *Message) (res resp.Value, err error) {

internal/server/crud.go

@@ -2,7 +2,7 @@ package server
 
 import (
 	"bytes"
-	"errors"
+	"math"
 	"strconv"
 	"strings"
 	"time"
@@ -17,27 +17,30 @@ import (
 	"github.com/tidwall/tile38/internal/object"
 )
 
-func (s *Server) cmdBounds(msg *Message) (resp.Value, error) {
+// BOUNDS key
+func (s *Server) cmdBOUNDS(msg *Message) (resp.Value, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
 
-	var ok bool
+	// >> Args
-	var key string
+
-	if vs, key, ok = tokenval(vs); !ok || key == "" {
+	args := msg.Args
-		return NOMessage, errInvalidNumberOfArguments
+	if len(args) != 2 {
-	}
+		return retrerr(errInvalidNumberOfArguments)
-	if len(vs) != 0 {
-		return NOMessage, errInvalidNumberOfArguments
 	}
+	key := args[1]
+
+	// >> Operation
 
 	col, _ := s.cols.Get(key)
 	if col == nil {
 		if msg.OutputType == RESP {
 			return resp.NullValue(), nil
 		}
-		return NOMessage, errKeyNotFound
+		return retrerr(errKeyNotFound)
 	}
 
+	// >> Response
+
 	vals := make([]resp.Value, 0, 2)
 	var buf bytes.Buffer
 	if msg.OutputType == JSON {
@@ -52,105 +55,125 @@ func (s *Server) cmdBounds(msg *Message) (resp.Value, error) {
 	if msg.OutputType == JSON {
 		buf.WriteString(`,"bounds":`)
 		buf.WriteString(string(bbox.AppendJSON(nil)))
-	} else {
-		vals = append(vals, resp.ArrayValue([]resp.Value{
-			resp.ArrayValue([]resp.Value{
-				resp.FloatValue(minX),
-				resp.FloatValue(minY),
-			}),
-			resp.ArrayValue([]resp.Value{
-				resp.FloatValue(maxX),
-				resp.FloatValue(maxY),
-			}),
-		}))
-	}
-	switch msg.OutputType {
-	case JSON:
 		buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}")
 		return resp.StringValue(buf.String()), nil
-	case RESP:
-		return vals[0], nil
 	}
-	return NOMessage, nil
+
+	// RESP
+	vals = append(vals, resp.ArrayValue([]resp.Value{
+		resp.ArrayValue([]resp.Value{
+			resp.FloatValue(minX),
+			resp.FloatValue(minY),
+		}),
+		resp.ArrayValue([]resp.Value{
+			resp.FloatValue(maxX),
+			resp.FloatValue(maxY),
+		}),
+	}))
+	return vals[0], nil
 }
 
-func (s *Server) cmdType(msg *Message) (resp.Value, error) {
+// TYPE key
+// undocumented return "none" or "hash"
+func (s *Server) cmdTYPE(msg *Message) (resp.Value, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
 
-	var ok bool
+	// >> Args
-	var key string
+
-	if _, key, ok = tokenval(vs); !ok || key == "" {
+	args := msg.Args
-		return NOMessage, errInvalidNumberOfArguments
+	if len(args) != 2 {
+		return retrerr(errInvalidNumberOfArguments)
 	}
+	key := args[1]
+
+	// >> Operation
 
 	col, _ := s.cols.Get(key)
 	if col == nil {
 		if msg.OutputType == RESP {
 			return resp.SimpleStringValue("none"), nil
 		}
-		return NOMessage, errKeyNotFound
+		return retrerr(errKeyNotFound)
 	}
 
+	// >> Response
+
 	typ := "hash"
 
-	switch msg.OutputType {
+	if msg.OutputType == JSON {
-	case JSON:
+		return resp.StringValue(`{"ok":true,"type":` + jsonString(typ) +
-		return resp.StringValue(`{"ok":true,"type":` + string(typ) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil
+			`,"elapsed":"` + time.Since(start).String() + "\"}"), nil
-	case RESP:
-		return resp.SimpleStringValue(typ), nil
 	}
-	return NOMessage, nil
+	return resp.SimpleStringValue(typ), nil
 }
 
-func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
+// GET key id [WITHFIELDS] [OBJECT|POINT|BOUNDS|(HASH geohash)]
+func (s *Server) cmdGET(msg *Message) (resp.Value, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
 
-	var ok bool
+	// >> Args
-	var key, id, typ, sprecision string
+
-	if vs, key, ok = tokenval(vs); !ok || key == "" {
+	args := msg.Args
-		return NOMessage, errInvalidNumberOfArguments
+
-	}
+	if len(args) < 3 {
-	if vs, id, ok = tokenval(vs); !ok || id == "" {
+		return retrerr(errInvalidNumberOfArguments)
-		return NOMessage, errInvalidNumberOfArguments
 	}
+	key, id := args[1], args[2]
 
 	withfields := false
-	if _, peek, ok := tokenval(vs); ok && strings.ToLower(peek) == "withfields" {
+	kind := "object"
-		withfields = true
+	var precision int64
-		vs = vs[1:]
+	for i := 3; i < len(args); i++ {
+		switch strings.ToLower(args[i]) {
+		case "withfields":
+			withfields = true
+		case "object":
+			kind = "object"
+		case "point":
+			kind = "point"
+		case "bounds":
+			kind = "bounds"
+		case "hash":
+			kind = "hash"
+			i++
+			if i == len(args) {
+				return retrerr(errInvalidNumberOfArguments)
+			}
+			var err error
+			precision, err = strconv.ParseInt(args[i], 10, 64)
+			if err != nil || precision < 1 || precision > 12 {
+				return retrerr(errInvalidArgument(args[i]))
+			}
+		default:
+			return retrerr(errInvalidNumberOfArguments)
+		}
 	}
 
+	// >> Operation
+
 	col, _ := s.cols.Get(key)
 	if col == nil {
 		if msg.OutputType == RESP {
 			return resp.NullValue(), nil
 		}
-		return NOMessage, errKeyNotFound
+		return retrerr(errKeyNotFound)
 	}
 	o := col.Get(id)
-	ok = o != nil
+	if o == nil {
-	if !ok {
 		if msg.OutputType == RESP {
 			return resp.NullValue(), nil
 		}
-		return NOMessage, errIDNotFound
+		return retrerr(errIDNotFound)
 	}
 
+	// >> Response
+
 	vals := make([]resp.Value, 0, 2)
 	var buf bytes.Buffer
 	if msg.OutputType == JSON {
 		buf.WriteString(`{"ok":true`)
 	}
-	vs, typ, ok = tokenval(vs)
+	switch kind {
-	typ = strings.ToLower(typ)
-	if !ok {
-		typ = "object"
-	}
-	switch typ {
-	default:
-		return NOMessage, errInvalidArgument(typ)
 	case "object":
 		if msg.OutputType == JSON {
 			buf.WriteString(`,"object":`)
@@ -179,16 +202,9 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
 			}
 		}
 	case "hash":
-		if vs, sprecision, ok = tokenval(vs); !ok || sprecision == "" {
-			return NOMessage, errInvalidNumberOfArguments
-		}
 		if msg.OutputType == JSON {
 			buf.WriteString(`,"hash":`)
 		}
-		precision, err := strconv.ParseInt(sprecision, 10, 64)
-		if err != nil || precision < 1 || precision > 12 {
-			return NOMessage, errInvalidArgument(sprecision)
-		}
 		center := o.Geo().Center()
 		p := geohash.EncodeWithPrecision(center.Y, center.X, uint(precision))
 		if msg.OutputType == JSON {
@@ -215,9 +231,6 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
 		}
 	}
 
-	if len(vs) != 0 {
-		return NOMessage, errInvalidNumberOfArguments
-	}
 	if withfields {
 		nfields := o.Fields().Len()
 		if nfields > 0 {
@@ -231,10 +244,11 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
 					if i > 0 {
 						buf.WriteString(`,`)
 					}
-					buf.WriteString(jsonString(f.Name()) + ":" + f.Value().JSON())
+					buf.WriteString(jsonString(f.Name()) + ":" +
+						f.Value().JSON())
 				} else {
-					fvals = append(fvals,
+					fvals = append(fvals, resp.StringValue(f.Name()),
-						resp.StringValue(f.Name()), resp.StringValue(f.Value().Data()))
+						resp.StringValue(f.Value().Data()))
 				}
 				i++
 				return true
@@ -246,24 +260,21 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
 			}
 		}
 	}
-	switch msg.OutputType {
+	if msg.OutputType == JSON {
-	case JSON:
 		buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}")
 		return resp.StringValue(buf.String()), nil
-	case RESP:
-		var oval resp.Value
-		if withfields {
-			oval = resp.ArrayValue(vals)
-		} else {
-			oval = vals[0]
-		}
-		return oval, nil
 	}
-	return NOMessage, nil
+	var oval resp.Value
+	if withfields {
+		oval = resp.ArrayValue(vals)
+	} else {
+		oval = vals[0]
+	}
+	return oval, nil
 }
 
 // DEL key id [ERRON404]
-func (s *Server) cmdDel(msg *Message) (resp.Value, commandDetails, error) {
+func (s *Server) cmdDEL(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
 
 	// >> Args
@@ -330,113 +341,109 @@ func (s *Server) cmdDel(msg *Message) (resp.Value, commandDetails, error) {
 	return res, d, nil
 }
 
-func (s *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, err error) {
+// PDEL key pattern
+func (s *Server) cmdPDEL(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
-	var ok bool
-	if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	if vs, d.pattern, ok = tokenval(vs); !ok || d.pattern == "" {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	if len(vs) != 0 {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	now := time.Now()
-	iter := func(o *object.Object) bool {
-		if match, _ := glob.Match(d.pattern, o.ID()); match {
-			d.children = append(d.children, &commandDetails{
-				command:   "del",
-				updated:   true,
-				timestamp: now,
-				key:       d.key,
-				obj:       o,
-			})
-		}
-		return true
-	}
 
-	var expired int
+	// >> Args
-	col, _ := s.cols.Get(d.key)
+
+	args := msg.Args
+	if len(args) != 3 {
+		return retwerr(errInvalidNumberOfArguments)
+	}
+	key := args[1]
+	pattern := args[2]
+
+	// >> Operation
+
+	now := time.Now()
+	var children []*commandDetails
+	col, _ := s.cols.Get(key)
 	if col != nil {
-		g := glob.Parse(d.pattern, false)
+		g := glob.Parse(pattern, false)
+		var ids []string
+		iter := func(o *object.Object) bool {
+			if match, _ := glob.Match(pattern, o.ID()); match {
+				ids = append(ids, o.ID())
+			}
+			return true
+		}
 		if g.Limits[0] == "" && g.Limits[1] == "" {
 			col.Scan(false, nil, msg.Deadline, iter)
 		} else {
-			col.ScanRange(g.Limits[0], g.Limits[1], false, nil, msg.Deadline, iter)
+			col.ScanRange(g.Limits[0], g.Limits[1],
+				false, nil, msg.Deadline, iter)
 		}
-		var atLeastOneNotDeleted bool
+		for _, id := range ids {
-		for i, dc := range d.children {
+			obj := col.Delete(id)
-			old := col.Delete(dc.obj.ID())
+			children = append(children, &commandDetails{
-			if old == nil {
+				command:   "del",
-				d.children[i].command = "?"
+				updated:   true,
-				atLeastOneNotDeleted = true
+				timestamp: now,
-			} else {
+				key:       key,
-				dc.obj = old
+				obj:       obj,
-				d.children[i] = dc
+			})
-			}
+			s.groupDisconnectObject(key, id)
-			s.groupDisconnectObject(dc.key, dc.obj.ID())
-		}
-		if atLeastOneNotDeleted {
-			var nchildren []*commandDetails
-			for _, dc := range d.children {
-				if dc.command == "del" {
-					nchildren = append(nchildren, dc)
-				}
-			}
-			d.children = nchildren
 		}
 		if col.Count() == 0 {
-			s.cols.Delete(d.key)
+			s.cols.Delete(key)
 		}
 	}
+
+	// >> Response
+
+	var d commandDetails
+	var res resp.Value
+
 	d.command = "pdel"
+	d.children = children
+	d.key = key
+	d.pattern = pattern
 	d.updated = len(d.children) > 0
 	d.timestamp = now
 	d.parent = true
 	switch msg.OutputType {
 	case JSON:
-		res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
+		res = resp.StringValue(`{"ok":true,"elapsed":"` +
+			time.Since(start).String() + "\"}")
 	case RESP:
-		total := len(d.children) - expired
+		res = resp.IntegerValue(len(d.children))
-		if total < 0 {
-			total = 0
-		}
-		res = resp.IntegerValue(total)
 	}
-	return
+	return res, d, nil
 }
 
-func (s *Server) cmdDrop(msg *Message) (res resp.Value, d commandDetails, err error) {
+// DROP key
+func (s *Server) cmdDROP(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
+
-	var ok bool
+	// >> Args
-	if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
+
-		err = errInvalidNumberOfArguments
+	args := msg.Args
-		return
+	if len(args) != 2 {
+		return retwerr(errInvalidNumberOfArguments)
 	}
-	if len(vs) != 0 {
+	key := args[1]
-		err = errInvalidNumberOfArguments
+
-		return
+	// >> Operation
-	}
+
-	col, _ := s.cols.Get(d.key)
+	col, _ := s.cols.Get(key)
 	if col != nil {
-		s.cols.Delete(d.key)
+		s.cols.Delete(key)
-		d.updated = true
-	} else {
-		d.key = "" // ignore the details
-		d.updated = false
 	}
-	s.groupDisconnectCollection(d.key)
+	s.groupDisconnectCollection(key)
+
+	// >> Response
+
+	var res resp.Value
+	var d commandDetails
+	d.key = key
+	d.updated = col != nil
 	d.command = "drop"
 	d.timestamp = time.Now()
 	switch msg.OutputType {
 	case JSON:
-		res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
+		res = resp.StringValue(`{"ok":true,"elapsed":"` +
+			time.Since(start).String() + "\"}")
 	case RESP:
 		if d.updated {
 			res = resp.IntegerValue(1)
@@ -444,57 +451,70 @@ func (s *Server) cmdDrop(msg *Message) (res resp.Value, d commandDetails, err er
 			res = resp.IntegerValue(0)
 		}
 	}
-	return
+	return res, d, nil
 }
 
-func (s *Server) cmdRename(msg *Message) (res resp.Value, d commandDetails, err error) {
+// RENAME key newkey
-	nx := msg.Command() == "renamenx"
+// RENAMENX key newkey
+func (s *Server) cmdRENAME(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
+
-	var ok bool
+	// >> Args
-	if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
+
-		err = errInvalidNumberOfArguments
+	args := msg.Args
-		return
+	if len(args) != 3 {
+		return retwerr(errInvalidNumberOfArguments)
 	}
-	if vs, d.newKey, ok = tokenval(vs); !ok || d.newKey == "" {
+	nx := strings.ToLower(args[0]) == "renamenx"
-		err = errInvalidNumberOfArguments
+	key := args[1]
-		return
+	newKey := args[2]
-	}
+
-	if len(vs) != 0 {
+	// >> Operation
-		err = errInvalidNumberOfArguments
+
-		return
+	col, _ := s.cols.Get(key)
-	}
-	col, _ := s.cols.Get(d.key)
 	if col == nil {
-		err = errKeyNotFound
+		return retwerr(errKeyNotFound)
-		return
 	}
+	var ierr error
 	s.hooks.Ascend(nil, func(v interface{}) bool {
 		h := v.(*Hook)
-		if h.Key == d.key || h.Key == d.newKey {
+		if h.Key == key || h.Key == newKey {
-			err = errKeyHasHooksSet
+			ierr = errKeyHasHooksSet
 			return false
 		}
 		return true
 	})
-	d.command = "rename"
+	if ierr != nil {
-	newCol, _ := s.cols.Get(d.newKey)
+		return retwerr(ierr)
+	}
+	var updated bool
+	newCol, _ := s.cols.Get(newKey)
 	if newCol == nil {
-		d.updated = true
+		updated = true
-	} else if nx {
+	} else if !nx {
-		d.updated = false
+		s.cols.Delete(newKey)
-	} else {
+		updated = true
-		s.cols.Delete(d.newKey)
-		d.updated = true
 	}
-	if d.updated {
+	if updated {
-		s.cols.Delete(d.key)
+		s.cols.Delete(key)
-		s.cols.Set(d.newKey, col)
+		s.cols.Set(newKey, col)
 	}
+
+	// >> Response
+
+	var d commandDetails
+	var res resp.Value
+
+	d.command = "rename"
+	d.key = key
+	d.newKey = newKey
+	d.updated = updated
 	d.timestamp = time.Now()
+
 	switch msg.OutputType {
 	case JSON:
-		res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
+		res = resp.StringValue(`{"ok":true,"elapsed":"` +
+			time.Since(start).String() + "\"}")
 	case RESP:
 		if !nx {
 			res = resp.SimpleStringValue("OK")
@@ -504,17 +524,23 @@ func (s *Server) cmdRename(msg *Message) (res resp.Value, d commandDetails, err
 			res = resp.IntegerValue(0)
 		}
 	}
-	return
+	return res, d, nil
 }
 
-func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err error) {
+// FLUSHDB
+func (s *Server) cmdFLUSHDB(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
+
-	if len(vs) != 0 {
+	// >> Args
-		err = errInvalidNumberOfArguments
+
-		return
+	args := msg.Args
+
+	if len(args) != 1 {
+		return retwerr(errInvalidNumberOfArguments)
 	}
 
+	// >> Operation
+
 	// clear the entire database
 	s.cols.Clear()
 	s.groupHooks.Clear()
@@ -525,23 +551,29 @@ func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err
 	s.hookTree.Clear()
 	s.hookCross.Clear()
 
+	// >> Response
+
+	var d commandDetails
 	d.command = "flushdb"
 	d.updated = true
 	d.timestamp = time.Now()
-	switch msg.OutputType {
+
-	case JSON:
+	var res resp.Value
-		res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
+	if msg.OutputType == JSON {
-	case RESP:
+		res = resp.StringValue(`{"ok":true,"elapsed":"` +
+			time.Since(start).String() + "\"}")
+	} else {
 		res = resp.SimpleStringValue("OK")
 	}
-	return
+	return res, d, nil
 }
 
 // SET key id [FIELD name value ...] [EX seconds] [NX|XX]
-// (OBJECT geojson)|(POINT lat lon z)|(BOUNDS minlat minlon maxlat maxlon)|(HASH geohash)|(STRING value)
+// (OBJECT geojson)|(POINT lat lon z)|(BOUNDS minlat minlon maxlat maxlon)|
+// (HASH geohash)|(STRING value)
 func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
-	if s.config.maxMemory() > 0 && s.outOfMemory.on() {
+	if s.config.maxMemory() > 0 && s.outOfMemory.Load() {
 		return retwerr(errOOM)
 	}
 
@@ -674,23 +706,22 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
 			return retwerr(errInvalidArgument(args[i]))
 		}
 	}
+	if oobj == nil {
+		return retwerr(errInvalidNumberOfArguments)
+	}
 
 	// >> Operation
 
 	nada := func() (resp.Value, commandDetails, error) {
 		// exclude operation due to 'xx' or 'nx' match
-		switch msg.OutputType {
+		if msg.OutputType == JSON {
-		default:
-		case JSON:
 			if nx {
 				return retwerr(errIDAlreadyExists)
 			} else {
 				return retwerr(errIDNotFound)
 			}
-		case RESP:
-			return resp.NullValue(), commandDetails{}, nil
 		}
-		return retwerr(errors.New("nada unknown output"))
+		return resp.NullValue(), commandDetails{}, nil
 	}
 
 	col, ok := s.cols.Get(key)
@@ -749,7 +780,7 @@ func retrerr(err error) (resp.Value, error) {
 // FSET key id [XX] field value [field value...]
 func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
-	if s.config.maxMemory() > 0 && s.outOfMemory.on() {
+	if s.config.maxMemory() > 0 && s.outOfMemory.Load() {
 		return retwerr(errOOM)
 	}
 
@@ -835,6 +866,9 @@ func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) {
 // EXPIRE key id seconds
 func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
+
+	// >> Args
+
 	args := msg.Args
 	if len(args) != 4 {
 		return retwerr(errInvalidNumberOfArguments)
@@ -844,12 +878,16 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) {
 	if err != nil {
 		return retwerr(errInvalidArgument(svalue))
 	}
+
+	// >> Operation
+
 	var ok bool
 	var obj *object.Object
 	col, _ := s.cols.Get(key)
 	if col != nil {
-		// replace the expiration by getting the old objec
+		// replace the expiration by getting the old object
-		ex := time.Now().Add(time.Duration(float64(time.Second) * value)).UnixNano()
+		ex := time.Now().Add(
+			time.Duration(float64(time.Second) * value)).UnixNano()
 		o := col.Get(id)
 		ok = o != nil
 		if ok {
@@ -857,6 +895,9 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) {
 			col.Set(obj)
 		}
 	}
+
+	// >> Response
+
 	var d commandDetails
 	if ok {
 		d.key = key
@@ -889,11 +930,17 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) {
 // PERSIST key id
 func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
+
+	// >> Args
+
 	args := msg.Args
 	if len(args) != 3 {
 		return retwerr(errInvalidNumberOfArguments)
 	}
 	key, id := args[1], args[2]
+
+	// >> Operation
+
 	col, _ := s.cols.Get(key)
 	if col == nil {
 		if msg.OutputType == RESP {
@@ -917,6 +964,8 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) {
 		cleared = true
 	}
 
+	// >> Response
+
 	var res resp.Value
 
 	var d commandDetails
@@ -928,7 +977,8 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) {
 
 	switch msg.OutputType {
 	case JSON:
-		res = resp.SimpleStringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
+		res = resp.SimpleStringValue(`{"ok":true,"elapsed":"` +
+			time.Since(start).String() + "\"}")
 	case RESP:
 		if cleared {
 			res = resp.IntegerValue(1)
@@ -942,62 +992,47 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) {
 // TTL key id
 func (s *Server) cmdTTL(msg *Message) (resp.Value, error) {
 	start := time.Now()
+
+	// >> Args
+
 	args := msg.Args
 	if len(args) != 3 {
 		return retrerr(errInvalidNumberOfArguments)
 	}
 	key, id := args[1], args[2]
-	var v float64
+
-	var ok bool
+	// >> Operation
-	var ok2 bool
+
 	col, _ := s.cols.Get(key)
-	if col != nil {
+	if col == nil {
-		o := col.Get(id)
+		if msg.OutputType == JSON {
-		ok = o != nil
+			return retrerr(errKeyNotFound)
-		if ok {
-			if o.Expires() != 0 {
-				now := start.UnixNano()
-				if now > o.Expires() {
-					ok2 = false
-				} else {
-					v = float64(o.Expires()-now) / float64(time.Second)
-					if v < 0 {
-						v = 0
-					}
-					ok2 = true
-				}
-			}
 		}
+		return resp.IntegerValue(-2), nil
 	}
-	var res resp.Value
+
-	switch msg.OutputType {
+	o := col.Get(id)
-	case JSON:
+	if o == nil {
-		if ok {
+		if msg.OutputType == JSON {
-			var ttl string
-			if ok2 {
-				ttl = strconv.FormatFloat(v, 'f', -1, 64)
-			} else {
-				ttl = "-1"
-			}
-			res = resp.SimpleStringValue(
-				`{"ok":true,"ttl":` + ttl + `,"elapsed":"` +
-					time.Since(start).String() + "\"}")
-		} else {
-			if col == nil {
-				return retrerr(errKeyNotFound)
-			}
 			return retrerr(errIDNotFound)
 		}
-	case RESP:
+		return resp.IntegerValue(-2), nil
-		if ok {
-			if ok2 {
-				res = resp.IntegerValue(int(v))
-			} else {
-				res = resp.IntegerValue(-1)
-			}
-		} else {
-			res = resp.IntegerValue(-2)
-		}
 	}
-	return res, nil
+
+	var ttl float64
+	if o.Expires() == 0 {
+		ttl = -1
+	} else {
+		now := start.UnixNano()
+		ttl = math.Max(float64(o.Expires()-now)/float64(time.Second), 0)
+	}
+
+	// >> Response
+
+	if msg.OutputType == JSON {
+		return resp.SimpleStringValue(
+			`{"ok":true,"ttl":` + strconv.Itoa(int(ttl)) + `,"elapsed":"` +
+				time.Since(start).String() + "\"}"), nil
+	}
+	return resp.IntegerValue(int(ttl)), nil
 }

internal/server/expire.go

@@ -1,6 +1,7 @@
 package server
 
 import (
+	"sync"
 	"time"
 
 	"github.com/tidwall/tile38/internal/collection"
@@ -12,20 +13,15 @@ const bgExpireDelay = time.Second / 10
 
 // backgroundExpiring deletes expired items from the database.
 // It's executes every 1/10 of a second.
-func (s *Server) backgroundExpiring() {
+func (s *Server) backgroundExpiring(wg *sync.WaitGroup) {
-	for {
+	defer wg.Done()
-		if s.stopServer.on() {
+	s.loopUntilServerStops(bgExpireDelay, func() {
-			return
+		s.mu.Lock()
-		}
+		defer s.mu.Unlock()
-		func() {
+		now := time.Now()
-			s.mu.Lock()
+		s.backgroundExpireObjects(now)
-			defer s.mu.Unlock()
+		s.backgroundExpireHooks(now)
-			now := time.Now()
+	})
-			s.backgroundExpireObjects(now)
-			s.backgroundExpireHooks(now)
-		}()
-		time.Sleep(bgExpireDelay)
-	}
 }
 
 func (s *Server) backgroundExpireObjects(now time.Time) {
@@ -42,7 +38,7 @@ func (s *Server) backgroundExpireObjects(now time.Time) {
 		return true
 	})
 	for _, msg := range msgs {
-		_, d, err := s.cmdDel(msg)
+		_, d, err := s.cmdDEL(msg)
 		if err != nil {
 			log.Fatal(err)
 		}

internal/server/follow.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/tidwall/resp"
-	"github.com/tidwall/tile38/core"
 	"github.com/tidwall/tile38/internal/log"
 )
 
@@ -84,10 +83,11 @@ func (s *Server) cmdFollow(msg *Message) (res resp.Value, err error) {
 	}
 	s.config.write(false)
 	if update {
-		s.followc.add(1)
+		s.followc.Add(1)
 		if s.config.followHost() != "" {
 			log.Infof("following new host '%s' '%s'.", host, sport)
-			go s.follow(s.config.followHost(), s.config.followPort(), s.followc.get())
+			go s.follow(s.config.followHost(), s.config.followPort(),
+				int(s.followc.Load()))
 		} else {
 			log.Infof("following no one")
 		}
@@ -153,7 +153,7 @@ func doServer(conn *RESPConn) (map[string]string, error) {
 func (s *Server) followHandleCommand(args []string, followc int, w io.Writer) (int, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	if s.followc.get() != followc {
+	if int(s.followc.Load()) != followc {
 		return s.aofsz, errNoLongerFollowing
 	}
 	msg := &Message{Args: args}
@@ -188,7 +188,7 @@ func (s *Server) followDoLeaderAuth(conn *RESPConn, auth string) error {
 }
 
 func (s *Server) followStep(host string, port int, followc int) error {
-	if s.followc.get() != followc {
+	if int(s.followc.Load()) != followc {
 		return errNoLongerFollowing
 	}
 	s.mu.Lock()
@@ -240,7 +240,7 @@ func (s *Server) followStep(host string, port int, followc int) error {
 	if v.String() != "OK" {
 		return errors.New("invalid response to replconf request")
 	}
-	if core.ShowDebugMessages {
+	if s.opts.ShowDebugMessages {
 		log.Debug("follow:", addr, ":replconf")
 	}
 
@@ -254,7 +254,7 @@ func (s *Server) followStep(host string, port int, followc int) error {
 	if v.String() != "OK" {
 		return errors.New("invalid response to aof live request")
 	}
-	if core.ShowDebugMessages {
+	if s.opts.ShowDebugMessages {
 		log.Debug("follow:", addr, ":read aof")
 	}
 

internal/server/hooks.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/tidwall/buntdb"
@@ -501,7 +502,7 @@ type Hook struct {
 	query      string
 	epm        *endpoint.Manager
 	expires    time.Time
-	counter    *aint // counter that grows when a message was sent
+	counter    *atomic.Int64 // counter that grows when a message was sent
 	sig        int
 }
 
@@ -701,7 +702,7 @@ func (h *Hook) proc() (ok bool) {
 			}
 			log.Debugf("Endpoint send ok: %v: %v: %v", idx, endpoint, err)
 			sent = true
-			h.counter.add(1)
+			h.counter.Add(1)
 			break
 		}
 		if !sent {

internal/server/keys.go

@@ -1,8 +1,7 @@
 package server
 
 import (
-	"bytes"
+	"encoding/json"
-	"strings"
 	"time"
 
 	"github.com/tidwall/resp"
@@ -10,86 +9,59 @@ import (
 	"github.com/tidwall/tile38/internal/glob"
 )
 
-func (s *Server) cmdKeys(msg *Message) (res resp.Value, err error) {
+// KEYS pattern
+func (s *Server) cmdKEYS(msg *Message) (resp.Value, error) {
 	var start = time.Now()
-	vs := msg.Args[1:]
 
-	var pattern string
+	// >> Args
-	var ok bool
-	if vs, pattern, ok = tokenval(vs); !ok || pattern == "" {
-		return NOMessage, errInvalidNumberOfArguments
-	}
-	if len(vs) != 0 {
-		return NOMessage, errInvalidNumberOfArguments
-	}
 
-	var wr = &bytes.Buffer{}
+	args := msg.Args
-	var once bool
+	if len(args) != 2 {
-	if msg.OutputType == JSON {
+		return retrerr(errInvalidNumberOfArguments)
-		wr.WriteString(`{"ok":true,"keys":[`)
 	}
-	var wild bool
+	pattern := args[1]
-	if strings.Contains(pattern, "*") {
-		wild = true
-	}
-	var everything bool
-	var greater bool
-	var greaterPivot string
-	var vals []resp.Value
 
-	iter := func(key string, col *collection.Collection) bool {
+	// >> Operation
-		var match bool
+
-		if everything {
+	keys := []string{}
-			match = true
+	g := glob.Parse(pattern, false)
-		} else if greater {
+	everything := g.Limits[0] == "" && g.Limits[1] == ""
-			if !strings.HasPrefix(key, greaterPivot) {
+	if everything {
-				return false
+		s.cols.Scan(
-			}
+			func(key string, _ *collection.Collection) bool {
-			match = true
+				match, _ := glob.Match(pattern, key)
-		} else {
+				if match {
-			match, _ = glob.Match(pattern, key)
+					keys = append(keys, key)
-		}
-		if match {
-			if once {
-				if msg.OutputType == JSON {
-					wr.WriteByte(',')
 				}
-			} else {
+				return true
-				once = true
+			},
-			}
+		)
-			switch msg.OutputType {
-			case JSON:
-				wr.WriteString(jsonString(key))
-			case RESP:
-				vals = append(vals, resp.StringValue(key))
-			}
-
-			// If no more than one match is expected, stop searching
-			if !wild {
-				return false
-			}
-		}
-		return true
-	}
-
-	// TODO: This can be further optimized by using glob.Parse and limits
-	if pattern == "*" {
-		everything = true
-		s.cols.Scan(iter)
-	} else if strings.HasSuffix(pattern, "*") {
-		greaterPivot = pattern[:len(pattern)-1]
-		if glob.IsGlob(greaterPivot) {
-			s.cols.Scan(iter)
-		} else {
-			greater = true
-			s.cols.Ascend(greaterPivot, iter)
-		}
 	} else {
-		s.cols.Scan(iter)
+		s.cols.Ascend(g.Limits[0],
+			func(key string, _ *collection.Collection) bool {
+				if key > g.Limits[1] {
+					return false
+				}
+				match, _ := glob.Match(pattern, key)
+				if match {
+					keys = append(keys, key)
+				}
+				return true
+			},
+		)
 	}
+
+	// >> Response
+
 	if msg.OutputType == JSON {
-		wr.WriteString(`],"elapsed":"` + time.Since(start).String() + "\"}")
+		data, _ := json.Marshal(keys)
-		return resp.StringValue(wr.String()), nil
+		return resp.StringValue(`{"ok":true,"keys":` + string(data) +
+			`,"elapsed":"` + time.Since(start).String() + `"}`), nil
+	}
+
+	var vals []resp.Value
+	for _, key := range keys {
+		vals = append(vals, resp.StringValue(key))
 	}
 	return resp.ArrayValue(vals), nil
 }

internal/server/live.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/tidwall/redcon"
 	"github.com/tidwall/tile38/internal/log"
+	"go.uber.org/atomic"
 )
 
 type liveBuffer struct {
@@ -21,10 +22,16 @@ type liveBuffer struct {
 	cond    *sync.Cond
 }
 
-func (s *Server) processLives() {
+func (s *Server) processLives(wg *sync.WaitGroup) {
-	defer s.lwait.Done()
+	defer wg.Done()
+	var done atomic.Bool
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		for {
+			if done.Load() {
+				break
+			}
 			s.lcond.Broadcast()
 			time.Sleep(time.Second / 4)
 		}
@@ -32,7 +39,8 @@ func (s *Server) processLives() {
 	s.lcond.L.Lock()
 	defer s.lcond.L.Unlock()
 	for {
-		if s.stopServer.on() {
+		if s.stopServer.Load() {
+			done.Store(true)
 			return
 		}
 		for len(s.lstack) > 0 {
@@ -204,7 +212,7 @@ func (s *Server) goLive(
 					return nil // nil return is fine here
 				}
 			}
-			s.statsTotalMsgsSent.add(len(msgs))
+			s.statsTotalMsgsSent.Add(int64(len(msgs)))
 			lb.cond.L.Lock()
 
 		}

internal/server/metrics.go

@@ -92,9 +92,14 @@ func (s *Server) Collect(ch chan<- prometheus.Metric) {
 	s.extStats(m)
 
 	for metric, descr := range metricDescriptions {
-		if val, ok := m[metric].(int); ok {
+		val, ok := m[metric].(float64)
-			ch <- prometheus.MustNewConstMetric(descr, prometheus.GaugeValue, float64(val))
+		if !ok {
-		} else if val, ok := m[metric].(float64); ok {
+			val2, ok2 := m[metric].(int)
+			if ok2 {
+				val, ok = float64(val2), true
+			}
+		}
+		if ok {
 			ch <- prometheus.MustNewConstMetric(descr, prometheus.GaugeValue, val)
 		}
 	}

internal/server/must.go

@@ -0,0 +1,16 @@
+package server
+
+func Must[T any](a T, err error) T {
+	if err != nil {
+		panic(err)
+	}
+	return a
+}
+
+func Default[T comparable](a, b T) T {
+	var c T
+	if a == c {
+		return b
+	}
+	return a
+}

internal/server/must_test.go

@@ -0,0 +1,38 @@
+package server
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestMust(t *testing.T) {
+	if Must(1, nil) != 1 {
+		t.Fail()
+	}
+	func() {
+		var ended bool
+		defer func() {
+			if ended {
+				t.Fail()
+			}
+			err, ok := recover().(error)
+			if !ok {
+				t.Fail()
+			}
+			if err.Error() != "ok" {
+				t.Fail()
+			}
+		}()
+		Must(1, errors.New("ok"))
+		ended = true
+	}()
+}
+
+func TestDefault(t *testing.T) {
+	if Default("", "2") != "2" {
+		t.Fail()
+	}
+	if Default("1", "2") != "1" {
+		t.Fail()
+	}
+}

internal/server/output.go

@@ -7,35 +7,31 @@ import (
 	"github.com/tidwall/resp"
 )
 
-func (s *Server) cmdOutput(msg *Message) (res resp.Value, err error) {
+// OUTPUT [resp|json]
+func (s *Server) cmdOUTPUT(msg *Message) (resp.Value, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
-	var arg string
-	var ok bool
 
-	if len(vs) != 0 {
+	args := msg.Args
-		if _, arg, ok = tokenval(vs); !ok || arg == "" {
+	switch len(args) {
-			return NOMessage, errInvalidNumberOfArguments
+	case 1:
+		if msg.OutputType == JSON {
+			return resp.StringValue(`{"ok":true,"output":"json","elapsed":` +
+				time.Since(start).String() + `}`), nil
 		}
+		return resp.StringValue("resp"), nil
+	case 2:
 		// Setting the original message output type will be picked up by the
 		// server prior to the next command being executed.
-		switch strings.ToLower(arg) {
+		switch strings.ToLower(args[1]) {
 		default:
-			return NOMessage, errInvalidArgument(arg)
+			return retrerr(errInvalidArgument(args[1]))
 		case "json":
 			msg.OutputType = JSON
 		case "resp":
 			msg.OutputType = RESP
 		}
 		return OKMessage(msg, start), nil
-	}
-	// return the output
-	switch msg.OutputType {
 	default:
-		return NOMessage, nil
+		return retrerr(errInvalidNumberOfArguments)
-	case JSON:
-		return resp.StringValue(`{"ok":true,"output":"json","elapsed":` + time.Since(start).String() + `}`), nil
-	case RESP:
-		return resp.StringValue("resp"), nil
 	}
 }

internal/server/pubsub.go

@@ -280,7 +280,7 @@ func (s *Server) liveSubscription(
 				write(b)
 			}
 		}
-		s.statsTotalMsgsSent.add(1)
+		s.statsTotalMsgsSent.Add(1)
 	}
 
 	m := [2]map[string]bool{

internal/server/readonly.go

@@ -1,44 +1,50 @@
 package server
 
 import (
-	"strings"
 	"time"
 
 	"github.com/tidwall/resp"
 	"github.com/tidwall/tile38/internal/log"
 )
 
-func (s *Server) cmdReadOnly(msg *Message) (res resp.Value, err error) {
+// READONLY yes|no
+func (s *Server) cmdREADONLY(msg *Message) (resp.Value, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
-	var arg string
-	var ok bool
 
-	if vs, arg, ok = tokenval(vs); !ok || arg == "" {
+	// >> Args
-		return NOMessage, errInvalidNumberOfArguments
+
+	args := msg.Args
+	if len(args) != 2 {
+		return retrerr(errInvalidNumberOfArguments)
 	}
-	if len(vs) != 0 {
+
-		return NOMessage, errInvalidNumberOfArguments
+	switch args[1] {
-	}
+	case "yes", "no":
-	update := false
-	switch strings.ToLower(arg) {
 	default:
-		return NOMessage, errInvalidArgument(arg)
+		return retrerr(errInvalidArgument(args[1]))
-	case "yes":
+	}
+
+	// >> Operation
+
+	var updated bool
+	if args[1] == "yes" {
 		if !s.config.readOnly() {
-			update = true
+			updated = true
 			s.config.setReadOnly(true)
 			log.Info("read only")
 		}
-	case "no":
+	} else {
 		if s.config.readOnly() {
-			update = true
+			updated = true
 			s.config.setReadOnly(false)
 			log.Info("read write")
 		}
 	}
-	if update {
+	if updated {
 		s.config.write(false)
 	}
+
+	// >> Response
+
 	return OKMessage(msg, start), nil
 }

internal/server/scripts.go

@@ -596,23 +596,23 @@ func (s *Server) commandInScript(msg *Message) (
 	case "fset":
 		res, d, err = s.cmdFSET(msg)
 	case "del":
-		res, d, err = s.cmdDel(msg)
+		res, d, err = s.cmdDEL(msg)
 	case "pdel":
-		res, d, err = s.cmdPdel(msg)
+		res, d, err = s.cmdPDEL(msg)
 	case "drop":
-		res, d, err = s.cmdDrop(msg)
+		res, d, err = s.cmdDROP(msg)
 	case "expire":
 		res, d, err = s.cmdEXPIRE(msg)
 	case "rename":
-		res, d, err = s.cmdRename(msg)
+		res, d, err = s.cmdRENAME(msg)
 	case "renamenx":
-		res, d, err = s.cmdRename(msg)
+		res, d, err = s.cmdRENAME(msg)
 	case "persist":
 		res, d, err = s.cmdPERSIST(msg)
 	case "ttl":
 		res, err = s.cmdTTL(msg)
 	case "stats":
-		res, err = s.cmdStats(msg)
+		res, err = s.cmdSTATS(msg)
 	case "scan":
 		res, err = s.cmdScan(msg)
 	case "nearby":
@@ -624,9 +624,9 @@ func (s *Server) commandInScript(msg *Message) (
 	case "search":
 		res, err = s.cmdSearch(msg)
 	case "bounds":
-		res, err = s.cmdBounds(msg)
+		res, err = s.cmdBOUNDS(msg)
 	case "get":
-		res, err = s.cmdGet(msg)
+		res, err = s.cmdGET(msg)
 	case "jget":
 		res, err = s.cmdJget(msg)
 	case "jset":
@@ -634,13 +634,13 @@ func (s *Server) commandInScript(msg *Message) (
 	case "jdel":
 		res, d, err = s.cmdJdel(msg)
 	case "type":
-		res, err = s.cmdType(msg)
+		res, err = s.cmdTYPE(msg)
 	case "keys":
-		res, err = s.cmdKeys(msg)
+		res, err = s.cmdKEYS(msg)
 	case "test":
-		res, err = s.cmdTest(msg)
+		res, err = s.cmdTEST(msg)
 	case "server":
-		res, err = s.cmdServer(msg)
+		res, err = s.cmdSERVER(msg)
 	}
 	s.sendMonitor(err, msg, nil, true)
 	return

internal/server/server.go

@@ -80,21 +80,24 @@ type Server struct {
 	config  *Config
 	epc     *endpoint.Manager
 
+	lnmu sync.Mutex
+	ln   net.Listener // server listener
+
 	// env opts
 	geomParseOpts geojson.ParseOptions
 	geomIndexOpts geometry.IndexOptions
 	http500Errors bool
 
 	// atomics
-	followc            aint // counter increases when follow property changes
+	followc            atomic.Int64 // counter when follow property changes
-	statsTotalConns    aint // counter for total connections
+	statsTotalConns    atomic.Int64 // counter for total connections
-	statsTotalCommands aint // counter for total commands
+	statsTotalCommands atomic.Int64 // counter for total commands
-	statsTotalMsgsSent aint // counter for total sent webhook messages
+	statsTotalMsgsSent atomic.Int64 // counter for total sent webhook messages
-	statsExpired       aint // item expiration counter
+	statsExpired       atomic.Int64 // item expiration counter
-	lastShrinkDuration aint
+	lastShrinkDuration atomic.Int64
-	stopServer         abool
+	stopServer         atomic.Bool
-	outOfMemory        abool
+	outOfMemory        atomic.Bool
-	loadedAndReady     abool // server is loaded and ready for commands
+	loadedAndReady     atomic.Bool // server is loaded and ready for commands
 
 	connsmu sync.RWMutex
 	conns   map[int]*Client
@@ -114,7 +117,6 @@ type Server struct {
 	lstack       []*commandDetails
 	lives        map[*liveBuffer]bool
 	lcond        *sync.Cond
-	lwait        sync.WaitGroup
 	fcup         bool         // follow caught up
 	fcuponce     bool         // follow caught up once
 	shrinking    bool         // aof shrinking flag
@@ -135,6 +137,8 @@ type Server struct {
 
 	monconnsMu sync.RWMutex
 	monconns   map[net.Conn]bool // monitor connections
+
+	opts Options
 }
 
 // Options for Serve()
@@ -145,18 +149,51 @@ type Options struct {
 	UseHTTP        bool
 	MetricsAddr    string
 	UnixSocketPath string // path for unix socket
+
+	// DevMode puts application in to dev mode
+	DevMode bool
+
+	// ShowDebugMessages allows for log.Debug to print to console.
+	ShowDebugMessages bool
+
+	// ProtectedMode forces Tile38 to default in protected mode.
+	ProtectedMode string
+
+	// AppendOnly allows for disabling the appendonly file.
+	AppendOnly bool
+
+	// AppendFileName allows for custom appendonly file path
+	AppendFileName string
+
+	// QueueFileName allows for custom queue.db file path
+	QueueFileName string
+
+	// Shutdown allows for shutting down the server.
+	Shutdown <-chan bool
 }
 
 // Serve starts a new tile38 server
 func Serve(opts Options) error {
-	if core.AppendFileName == "" {
+	if opts.AppendFileName == "" {
-		core.AppendFileName = path.Join(opts.Dir, "appendonly.aof")
+		opts.AppendFileName = path.Join(opts.Dir, "appendonly.aof")
 	}
-	if core.QueueFileName == "" {
+	if opts.QueueFileName == "" {
-		core.QueueFileName = path.Join(opts.Dir, "queue.db")
+		opts.QueueFileName = path.Join(opts.Dir, "queue.db")
+	}
+	if opts.ProtectedMode == "" {
+		opts.ProtectedMode = "no"
 	}
 
 	log.Infof("Server started, Tile38 version %s, git %s", core.Version, core.GitSHA)
+	defer func() {
+		log.Warn("Server has shutdown, bye now")
+		if false {
+			// prints the stack, looking for running goroutines.
+			buf := make([]byte, 10000)
+			n := runtime.Stack(buf, true)
+			println(string(buf[:n]))
+		}
+	}()
 
 	// Initialize the s
 	s := &Server{
@@ -183,9 +220,11 @@ func Serve(opts Options) error {
 		groupHooks:   btree.NewNonConcurrent(byGroupHook),
 		groupObjects: btree.NewNonConcurrent(byGroupObject),
 		hookExpires:  btree.NewNonConcurrent(byHookExpires),
+		opts:         opts,
 	}
 
 	s.epc = endpoint.NewManager(s)
+	defer s.epc.Shutdown()
 	s.luascripts = s.newScriptMap()
 	s.luapool = s.newPool()
 	defer s.luapool.Shutdown()
@@ -255,8 +294,25 @@ func Serve(opts Options) error {
 		nerr <- s.netServe()
 	}()
 
+	go func() {
+		<-opts.Shutdown
+		s.stopServer.Store(true)
+		log.Warnf("Shutting down...")
+		s.lnmu.Lock()
+		ln := s.ln
+		s.ln = nil
+		s.lnmu.Unlock()
+		if ln != nil {
+			ln.Close()
+		}
+		for conn, f := range s.aofconnM {
+			conn.Close()
+			f.Close()
+		}
+	}()
+
 	// Load the queue before the aof
-	qdb, err := buntdb.Open(core.QueueFileName)
+	qdb, err := buntdb.Open(opts.QueueFileName)
 	if err != nil {
 		return err
 	}
@@ -284,8 +340,8 @@ func Serve(opts Options) error {
 	if err := s.migrateAOF(); err != nil {
 		return err
 	}
-	if core.AppendOnly {
+	if opts.AppendOnly {
-		f, err := os.OpenFile(core.AppendFileName, os.O_CREATE|os.O_RDWR, 0600)
+		f, err := os.OpenFile(opts.AppendFileName, os.O_CREATE|os.O_RDWR, 0600)
 		if err != nil {
 			return err
 		}
@@ -300,41 +356,69 @@ func Serve(opts Options) error {
 	}
 
 	// Start background routines
-	if s.config.followHost() != "" {
+	var bgwg sync.WaitGroup
-		go s.follow(s.config.followHost(), s.config.followPort(),
-			s.followc.get())
-	}
 
-	if opts.MetricsAddr != "" {
+	if s.config.followHost() != "" {
-		log.Infof("Listening for metrics at: %s", opts.MetricsAddr)
+		bgwg.Add(1)
 		go func() {
-			http.HandleFunc("/", s.MetricsIndexHandler)
+			defer bgwg.Done()
-			http.HandleFunc("/metrics", s.MetricsHandler)
+			s.follow(s.config.followHost(), s.config.followPort(),
-			log.Fatal(http.ListenAndServe(opts.MetricsAddr, nil))
+				int(s.followc.Load()))
 		}()
 	}
 
-	s.lwait.Add(1)
+	var mln net.Listener
-	go s.processLives()
+	if opts.MetricsAddr != "" {
-	go s.watchOutOfMemory()
+		log.Infof("Listening for metrics at: %s", opts.MetricsAddr)
-	go s.watchLuaStatePool()
+		mln, err = net.Listen("tcp", opts.MetricsAddr)
-	go s.watchAutoGC()
+		if err != nil {
-	go s.backgroundExpiring()
+			return err
-	go s.backgroundSyncAOF()
+		}
+		bgwg.Add(1)
+		go func() {
+			defer bgwg.Done()
+			smux := http.NewServeMux()
+			smux.HandleFunc("/", s.MetricsIndexHandler)
+			smux.HandleFunc("/metrics", s.MetricsHandler)
+			err := http.Serve(mln, smux)
+			if err != nil {
+				if !s.stopServer.Load() {
+					log.Fatalf("metrics server: %s", err)
+				}
+			}
+		}()
+	}
+
+	bgwg.Add(1)
+	go s.processLives(&bgwg)
+	bgwg.Add(1)
+	go s.watchOutOfMemory(&bgwg)
+	bgwg.Add(1)
+	go s.watchLuaStatePool(&bgwg)
+	bgwg.Add(1)
+	go s.watchAutoGC(&bgwg)
+	bgwg.Add(1)
+	go s.backgroundExpiring(&bgwg)
+	bgwg.Add(1)
+	go s.backgroundSyncAOF(&bgwg)
 	defer func() {
+		log.Debug("Stopping background routines")
 		// Stop background routines
-		s.followc.add(1) // this will force any follow communication to die
+		s.followc.Add(1) // this will force any follow communication to die
-		s.stopServer.set(true)
+		s.stopServer.Store(true)
-		s.lwait.Wait()
+		if mln != nil {
+			mln.Close() // Stop the metrics server
+		}
+		bgwg.Wait()
 	}()
 
 	// Server is now loaded and ready. Wait for network error messages.
-	s.loadedAndReady.set(true)
+	s.loadedAndReady.Store(true)
 	return <-nerr
 }
 
 func (s *Server) isProtected() bool {
-	if core.ProtectedMode == "no" {
+	if s.opts.ProtectedMode == "no" {
 		// --protected-mode no
 		return false
 	}
@@ -360,28 +444,52 @@ func (s *Server) netServe() error {
 	if err != nil {
 		return err
 	}
-	defer ln.Close()
+	s.lnmu.Lock()
+	s.ln = ln
+	s.lnmu.Unlock()
+
+	var wg sync.WaitGroup
+	defer func() {
+		log.Debug("Closing client connections...")
+		s.connsmu.RLock()
+		for _, c := range s.conns {
+			c.closer.Close()
+		}
+		s.connsmu.RUnlock()
+		wg.Wait()
+		ln.Close()
+		log.Debug("Client connection closed")
+	}()
+
 	log.Infof("Ready to accept connections at %s", ln.Addr())
 	var clientID int64
 	for {
 		conn, err := ln.Accept()
 		if err != nil {
-			return err
+			if s.stopServer.Load() {
+				return nil
+			}
+			log.Warn(err)
+			time.Sleep(time.Second / 5)
+			continue
 		}
-
+		wg.Add(1)
 		go func(conn net.Conn) {
+			defer wg.Done()
+
 			// open connection
 			// create the client
 			client := new(Client)
 			client.id = int(atomic.AddInt64(&clientID, 1))
 			client.opened = time.Now()
 			client.remoteAddr = conn.RemoteAddr().String()
+			client.closer = conn
 
 			// add client to server map
 			s.connsmu.Lock()
 			s.conns[client.id] = client
 			s.connsmu.Unlock()
-			s.statsTotalConns.add(1)
+			s.statsTotalConns.Add(1)
 
 			// set the client keep-alive, if needed
 			if s.config.keepAlive() > 0 {
@@ -460,7 +568,7 @@ func (s *Server) netServe() error {
 						client.mu.Unlock()
 
 						// update total command count
-						s.statsTotalCommands.add(1)
+						s.statsTotalCommands.Add(1)
 
 						// handle the command
 						err := s.handleInputCommand(client, msg)
@@ -592,20 +700,16 @@ func (conn *liveConn) SetWriteDeadline(deadline time.Time) error {
 	panic("not supported")
 }
 
-func (s *Server) watchAutoGC() {
+func (s *Server) watchAutoGC(wg *sync.WaitGroup) {
-	t := time.NewTicker(time.Second)
+	defer wg.Done()
-	defer t.Stop()
 	start := time.Now()
-	for range t.C {
+	s.loopUntilServerStops(time.Second, func() {
-		if s.stopServer.on() {
-			return
-		}
 		autoGC := s.config.autoGC()
 		if autoGC == 0 {
-			continue
+			return
 		}
 		if time.Since(start) < time.Second*time.Duration(autoGC) {
-			continue
+			return
 		}
 		var mem1, mem2 runtime.MemStats
 		runtime.ReadMemStats(&mem1)
@@ -620,58 +724,65 @@ func (s *Server) watchAutoGC() {
 			"alloc: %v, heap_alloc: %v, heap_released: %v",
 			mem2.Alloc, mem2.HeapAlloc, mem2.HeapReleased)
 		start = time.Now()
-	}
+	})
 }
 
-func (s *Server) watchOutOfMemory() {
+func (s *Server) checkOutOfMemory() {
-	t := time.NewTicker(time.Second * 2)
+	if s.stopServer.Load() {
-	defer t.Stop()
+		return
+	}
+	oom := s.outOfMemory.Load()
 	var mem runtime.MemStats
-	for range t.C {
+	if s.config.maxMemory() == 0 {
-		func() {
+		if oom {
-			if s.stopServer.on() {
+			s.outOfMemory.Store(false)
-				return
+		}
-			}
+		return
-			oom := s.outOfMemory.on()
+	}
-			if s.config.maxMemory() == 0 {
+	if oom {
-				if oom {
+		runtime.GC()
-					s.outOfMemory.set(false)
+	}
-				}
+	runtime.ReadMemStats(&mem)
-				return
+	s.outOfMemory.Store(int(mem.HeapAlloc) > s.config.maxMemory())
-			}
+}
-			if oom {
+
-				runtime.GC()
+func (s *Server) loopUntilServerStops(dur time.Duration, op func()) {
-			}
+	var last time.Time
-			runtime.ReadMemStats(&mem)
+	for {
-			s.outOfMemory.set(int(mem.HeapAlloc) > s.config.maxMemory())
+		if s.stopServer.Load() {
-		}()
+			return
+		}
+		now := time.Now()
+		if now.Sub(last) > dur {
+			op()
+			last = now
+		}
+		time.Sleep(time.Second / 5)
 	}
 }
 
-func (s *Server) watchLuaStatePool() {
+func (s *Server) watchOutOfMemory(wg *sync.WaitGroup) {
-	t := time.NewTicker(time.Second * 10)
+	defer wg.Done()
-	defer t.Stop()
+	s.loopUntilServerStops(time.Second*4, func() {
-	for range t.C {
+		s.checkOutOfMemory()
-		func() {
+	})
-			s.luapool.Prune()
+}
-		}()
+
-	}
+func (s *Server) watchLuaStatePool(wg *sync.WaitGroup) {
+	defer wg.Done()
+	s.loopUntilServerStops(time.Second*10, func() {
+		s.luapool.Prune()
+	})
 }
 
 // backgroundSyncAOF ensures that the aof buffer is does not grow too big.
-func (s *Server) backgroundSyncAOF() {
+func (s *Server) backgroundSyncAOF(wg *sync.WaitGroup) {
-	t := time.NewTicker(time.Second)
+	defer wg.Done()
-	defer t.Stop()
+	s.loopUntilServerStops(time.Second, func() {
-	for range t.C {
+		s.mu.Lock()
-		if s.stopServer.on() {
+		defer s.mu.Unlock()
-			return
+		s.flushAOF(true)
-		}
+	})
-		func() {
-			s.mu.Lock()
-			defer s.mu.Unlock()
-			s.flushAOF(true)
-		}()
-	}
 }
 
 func isReservedFieldName(field string) bool {
@@ -812,7 +923,7 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error {
 		return nil
 	}
 
-	if !s.loadedAndReady.on() {
+	if !s.loadedAndReady.Load() {
 		switch msg.Command() {
 		case "output", "ping", "echo":
 		default:
@@ -1014,17 +1125,17 @@ func (s *Server) command(msg *Message, client *Client) (
 	case "fset":
 		res, d, err = s.cmdFSET(msg)
 	case "del":
-		res, d, err = s.cmdDel(msg)
+		res, d, err = s.cmdDEL(msg)
 	case "pdel":
-		res, d, err = s.cmdPdel(msg)
+		res, d, err = s.cmdPDEL(msg)
 	case "drop":
-		res, d, err = s.cmdDrop(msg)
+		res, d, err = s.cmdDROP(msg)
 	case "flushdb":
 		res, d, err = s.cmdFLUSHDB(msg)
 	case "rename":
-		res, d, err = s.cmdRename(msg)
+		res, d, err = s.cmdRENAME(msg)
 	case "renamenx":
-		res, d, err = s.cmdRename(msg)
+		res, d, err = s.cmdRENAME(msg)
 	case "sethook":
 		res, d, err = s.cmdSetHook(msg)
 	case "delhook":
@@ -1048,19 +1159,19 @@ func (s *Server) command(msg *Message, client *Client) (
 	case "ttl":
 		res, err = s.cmdTTL(msg)
 	case "shutdown":
-		if !core.DevMode {
+		if !s.opts.DevMode {
 			err = fmt.Errorf("unknown command '%s'", msg.Args[0])
 			return
 		}
 		log.Fatal("shutdown requested by developer")
 	case "massinsert":
-		if !core.DevMode {
+		if !s.opts.DevMode {
 			err = fmt.Errorf("unknown command '%s'", msg.Args[0])
 			return
 		}
 		res, err = s.cmdMassInsert(msg)
 	case "sleep":
-		if !core.DevMode {
+		if !s.opts.DevMode {
 			err = fmt.Errorf("unknown command '%s'", msg.Args[0])
 			return
 		}
@@ -1070,15 +1181,15 @@ func (s *Server) command(msg *Message, client *Client) (
 	case "replconf":
 		res, err = s.cmdReplConf(msg, client)
 	case "readonly":
-		res, err = s.cmdReadOnly(msg)
+		res, err = s.cmdREADONLY(msg)
 	case "stats":
-		res, err = s.cmdStats(msg)
+		res, err = s.cmdSTATS(msg)
 	case "server":
-		res, err = s.cmdServer(msg)
+		res, err = s.cmdSERVER(msg)
 	case "healthz":
-		res, err = s.cmdHealthz(msg)
+		res, err = s.cmdHEALTHZ(msg)
 	case "info":
-		res, err = s.cmdInfo(msg)
+		res, err = s.cmdINFO(msg)
 	case "scan":
 		res, err = s.cmdScan(msg)
 	case "nearby":
@@ -1090,9 +1201,9 @@ func (s *Server) command(msg *Message, client *Client) (
 	case "search":
 		res, err = s.cmdSearch(msg)
 	case "bounds":
-		res, err = s.cmdBounds(msg)
+		res, err = s.cmdBOUNDS(msg)
 	case "get":
-		res, err = s.cmdGet(msg)
+		res, err = s.cmdGET(msg)
 	case "jget":
 		res, err = s.cmdJget(msg)
 	case "jset":
@@ -1100,11 +1211,11 @@ func (s *Server) command(msg *Message, client *Client) (
 	case "jdel":
 		res, d, err = s.cmdJdel(msg)
 	case "type":
-		res, err = s.cmdType(msg)
+		res, err = s.cmdTYPE(msg)
 	case "keys":
-		res, err = s.cmdKeys(msg)
+		res, err = s.cmdKEYS(msg)
 	case "output":
-		res, err = s.cmdOutput(msg)
+		res, err = s.cmdOUTPUT(msg)
 	case "aof":
 		res, err = s.cmdAOF(msg)
 	case "aofmd5":
@@ -1132,7 +1243,7 @@ func (s *Server) command(msg *Message, client *Client) (
 			return s.command(msg, client)
 		}
 	case "client":
-		res, err = s.cmdClient(msg, client)
+		res, err = s.cmdCLIENT(msg, client)
 	case "eval", "evalro", "evalna":
 		res, err = s.cmdEvalUnified(false, msg)
 	case "evalsha", "evalrosha", "evalnasha":
@@ -1150,10 +1261,11 @@ func (s *Server) command(msg *Message, client *Client) (
 	case "publish":
 		res, err = s.cmdPublish(msg)
 	case "test":
-		res, err = s.cmdTest(msg)
+		res, err = s.cmdTEST(msg)
 	case "monitor":
 		res, err = s.cmdMonitor(msg)
 	}
+
 	s.sendMonitor(err, msg, client, false)
 	return
 }

internal/server/stats.go

@@ -45,22 +45,23 @@ func readMemStats() runtime.MemStats {
 	return ms
 }
 
-func (s *Server) cmdStats(msg *Message) (res resp.Value, err error) {
+// STATS key [key...]
+func (s *Server) cmdSTATS(msg *Message) (resp.Value, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
-	var ms = []map[string]interface{}{}
 
-	if len(vs) == 0 {
+	// >> Args
-		return NOMessage, errInvalidNumberOfArguments
+
+	args := msg.Args
+	if len(args) < 2 {
+		return retrerr(errInvalidNumberOfArguments)
 	}
+
+	// >> Operation
+
 	var vals []resp.Value
-	var key string
+	var ms = []map[string]interface{}{}
-	var ok bool
+	for i := 1; i < len(args); i++ {
-	for {
+		key := args[i]
-		vs, key, ok = tokenval(vs)
-		if !ok {
-			break
-		}
 		col, _ := s.cols.Get(key)
 		if col != nil {
 			m := make(map[string]interface{})
@@ -83,67 +84,82 @@ func (s *Server) cmdStats(msg *Message) (res resp.Value, err error) {
 			}
 		}
 	}
-	switch msg.OutputType {
-	case JSON:
 
-		data, err := json.Marshal(ms)
+	// >> Response
-		if err != nil {
+
-			return NOMessage, err
+	if msg.OutputType == JSON {
-		}
+		data, _ := json.Marshal(ms)
-		res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}")
+		return resp.StringValue(`{"ok":true,"stats":` + string(data) +
-	case RESP:
+			`,"elapsed":"` + time.Since(start).String() + "\"}"), nil
-		res = resp.ArrayValue(vals)
 	}
-	return res, nil
+	return resp.ArrayValue(vals), nil
 }
 
-func (s *Server) cmdHealthz(msg *Message) (res resp.Value, err error) {
+// HEALTHZ
+func (s *Server) cmdHEALTHZ(msg *Message) (resp.Value, error) {
 	start := time.Now()
+
+	// >> Args
+
+	args := msg.Args
+	if len(args) != 1 {
+		return retrerr(errInvalidNumberOfArguments)
+	}
+
+	// >> Operation
+
 	if s.config.followHost() != "" {
 		m := make(map[string]interface{})
 		s.basicStats(m)
 		if fmt.Sprintf("%v", m["caught_up"]) != "true" {
-			return NOMessage, errors.New("not caught up")
+			return retrerr(errors.New("not caught up"))
 		}
 	}
-	switch msg.OutputType {
+
-	case JSON:
+	// >> Response
-		res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
+
-	case RESP:
+	if msg.OutputType == JSON {
-		res = resp.SimpleStringValue("OK")
+		return resp.StringValue(`{"ok":true,"elapsed":"` +
+			time.Since(start).String() + "\"}"), nil
 	}
-	return res, nil
+	return resp.SimpleStringValue("OK"), nil
 }
 
-func (s *Server) cmdServer(msg *Message) (res resp.Value, err error) {
+// SERVER [ext]
+func (s *Server) cmdSERVER(msg *Message) (resp.Value, error) {
 	start := time.Now()
+
+	// >> Args
+
+	args := msg.Args
+	var ext bool
+	for i := 1; i < len(args); i++ {
+		switch strings.ToLower(args[i]) {
+		case "ext":
+			ext = true
+		default:
+			return retrerr(errInvalidArgument(args[i]))
+		}
+	}
+
+	// >> Operation
+
 	m := make(map[string]interface{})
-	args := msg.Args[1:]
 
-	// Switch on the type of stats requested
+	if ext {
-	switch len(args) {
+		s.extStats(m)
-	case 0:
+	} else {
 		s.basicStats(m)
-	case 1:
-		if strings.ToLower(args[0]) == "ext" {
-			s.extStats(m)
-		}
-	default:
-		return NOMessage, errInvalidNumberOfArguments
 	}
 
-	switch msg.OutputType {
+	// >> Response
-	case JSON:
+
-		data, err := json.Marshal(m)
+	if msg.OutputType == JSON {
-		if err != nil {
+		data, _ := json.Marshal(m)
-			return NOMessage, err
+		return resp.StringValue(`{"ok":true,"stats":` + string(data) +
-		}
+			`,"elapsed":"` + time.Since(start).String() + "\"}"), nil
-		res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}")
-	case RESP:
-		vals := respValuesSimpleMap(m)
-		res = resp.ArrayValue(vals)
 	}
-	return res, nil
+	return resp.ArrayValue(respValuesSimpleMap(m)), nil
 }
 
 // basicStats populates the passed map with basic system/go/tile38 statistics
@@ -302,11 +318,11 @@ func (s *Server) extStats(m map[string]interface{}) {
 	// Whether or not a cluster is enabled
 	m["tile38_cluster_enabled"] = false
 	// Whether or not the Tile38 AOF is enabled
-	m["tile38_aof_enabled"] = core.AppendOnly
+	m["tile38_aof_enabled"] = s.opts.AppendOnly
 	// Whether or not an AOF shrink is currently in progress
 	m["tile38_aof_rewrite_in_progress"] = s.shrinking
 	// Length of time the last AOF shrink took
-	m["tile38_aof_last_rewrite_time_sec"] = s.lastShrinkDuration.get() / int(time.Second)
+	m["tile38_aof_last_rewrite_time_sec"] = s.lastShrinkDuration.Load() / int64(time.Second)
 	// Duration of the on-going AOF rewrite operation if any
 	var currentShrinkStart time.Time
 	if currentShrinkStart.IsZero() {
@@ -319,13 +335,13 @@ func (s *Server) extStats(m map[string]interface{}) {
 	// Whether or no the HTTP transport is being served
 	m["tile38_http_transport"] = s.http
 	// Number of connections accepted by the server
-	m["tile38_total_connections_received"] = s.statsTotalConns.get()
+	m["tile38_total_connections_received"] = s.statsTotalConns.Load()
 	// Number of commands processed by the server
-	m["tile38_total_commands_processed"] = s.statsTotalCommands.get()
+	m["tile38_total_commands_processed"] = s.statsTotalCommands.Load()
 	// Number of webhook messages sent by server
-	m["tile38_total_messages_sent"] = s.statsTotalMsgsSent.get()
+	m["tile38_total_messages_sent"] = s.statsTotalMsgsSent.Load()
 	// Number of key expiration events
-	m["tile38_expired_keys"] = s.statsExpired.get()
+	m["tile38_expired_keys"] = s.statsExpired.Load()
 	// Number of connected slaves
 	m["tile38_connected_slaves"] = len(s.aofconnM)
 
@@ -393,9 +409,9 @@ func boolInt(t bool) int {
 	return 0
 }
 func (s *Server) writeInfoPersistence(w *bytes.Buffer) {
-	fmt.Fprintf(w, "aof_enabled:%d\r\n", boolInt(core.AppendOnly))
+	fmt.Fprintf(w, "aof_enabled:%d\r\n", boolInt(s.opts.AppendOnly))
-	fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(s.shrinking))                          // Flag indicating a AOF rewrite operation is on-going
+	fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(s.shrinking))                             // Flag indicating a AOF rewrite operation is on-going
-	fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", s.lastShrinkDuration.get()/int(time.Second)) // Duration of the last AOF rewrite operation in seconds
+	fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", s.lastShrinkDuration.Load()/int64(time.Second)) // Duration of the last AOF rewrite operation in seconds
 
 	var currentShrinkStart time.Time // c.currentShrinkStart.get()
 	if currentShrinkStart.IsZero() {
@@ -406,10 +422,10 @@ func (s *Server) writeInfoPersistence(w *bytes.Buffer) {
 }
 
 func (s *Server) writeInfoStats(w *bytes.Buffer) {
-	fmt.Fprintf(w, "total_connections_received:%d\r\n", s.statsTotalConns.get())  // Total number of connections accepted by the server
+	fmt.Fprintf(w, "total_connections_received:%d\r\n", s.statsTotalConns.Load())  // Total number of connections accepted by the server
-	fmt.Fprintf(w, "total_commands_processed:%d\r\n", s.statsTotalCommands.get()) // Total number of commands processed by the server
+	fmt.Fprintf(w, "total_commands_processed:%d\r\n", s.statsTotalCommands.Load()) // Total number of commands processed by the server
-	fmt.Fprintf(w, "total_messages_sent:%d\r\n", s.statsTotalMsgsSent.get())      // Total number of commands processed by the server
+	fmt.Fprintf(w, "total_messages_sent:%d\r\n", s.statsTotalMsgsSent.Load())      // Total number of commands processed by the server
-	fmt.Fprintf(w, "expired_keys:%d\r\n", s.statsExpired.get())                   // Total number of key expiration events
+	fmt.Fprintf(w, "expired_keys:%d\r\n", s.statsExpired.Load())                   // Total number of key expiration events
 }
 
 // writeInfoReplication writes all replication data to the 'info' response
@@ -441,27 +457,52 @@ func (s *Server) writeInfoCluster(w *bytes.Buffer) {
 	fmt.Fprintf(w, "cluster_enabled:0\r\n")
 }
 
-func (s *Server) cmdInfo(msg *Message) (res resp.Value, err error) {
+// INFO [section ...]
+func (s *Server) cmdINFO(msg *Message) (res resp.Value, err error) {
 	start := time.Now()
 
-	sections := []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "cluster", "keyspace"}
+	// >> Args
-	switch len(msg.Args) {
+
-	default:
+	args := msg.Args
-		return NOMessage, errInvalidNumberOfArguments
+
-	case 1:
+	msects := make(map[string]bool)
-	case 2:
+	allsects := []string{
-		section := strings.ToLower(msg.Args[1])
+		"server", "clients", "memory", "persistence", "stats",
+		"replication", "cpu", "cluster", "keyspace",
+	}
+
+	if len(args) == 1 {
+		for _, s := range allsects {
+			msects[s] = true
+		}
+	}
+	for i := 1; i < len(args); i++ {
+		section := strings.ToLower(args[i])
 		switch section {
+		case "all", "default":
+			for _, s := range allsects {
+				msects[s] = true
+			}
 		default:
-			sections = []string{section}
+			for _, s := range allsects {
-		case "all":
+				if s == section {
-			sections = []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "commandstats", "cluster", "keyspace"}
+					msects[section] = true
-		case "default":
+				}
+			}
+		}
+	}
+
+	// >> Operation
+
+	var sects []string
+	for _, s := range allsects {
+		if msects[s] {
+			sects = append(sects, s)
 		}
 	}
 
 	w := &bytes.Buffer{}
-	for i, section := range sections {
+	for i, section := range sects {
 		if i > 0 {
 			w.WriteString("\r\n")
 		}
@@ -495,8 +536,9 @@ func (s *Server) cmdInfo(msg *Message) (res resp.Value, err error) {
 		}
 	}
 
-	switch msg.OutputType {
+	// >> Response
-	case JSON:
+
+	if msg.OutputType == JSON {
 		// Create a map of all key/value info fields
 		m := make(map[string]interface{})
 		for _, kv := range strings.Split(w.String(), "\r\n") {
@@ -509,15 +551,11 @@ func (s *Server) cmdInfo(msg *Message) (res resp.Value, err error) {
 		}
 
 		// Marshal the map and use the output in the JSON response
-		data, err := json.Marshal(m)
+		data, _ := json.Marshal(m)
-		if err != nil {
+		return resp.StringValue(`{"ok":true,"info":` + string(data) +
-			return NOMessage, err
+			`,"elapsed":"` + time.Since(start).String() + "\"}"), nil
-		}
-		res = resp.StringValue(`{"ok":true,"info":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}")
-	case RESP:
-		res = resp.BytesValue(w.Bytes())
 	}
-	return res, nil
+	return resp.BytesValue(w.Bytes()), nil
 }
 
 // tryParseType attempts to parse the passed string as an integer, float64 and

internal/server/test.go

@@ -50,7 +50,7 @@ func (s *Server) parseArea(ovs []string, doClip bool) (vs []string, o geojson.Ob
 		o = geojson.NewPoint(geometry.Point{X: lon, Y: lat})
 	case "sector":
 		if doClip {
-			err = errInvalidArgument("cannot clip with " + ltyp)
+			err = fmt.Errorf("invalid clip type '%s'", typ)
 			return
 		}
 		var slat, slon, smeters, sb1, sb2 string
@@ -288,8 +288,15 @@ func (s *Server) parseArea(ovs []string, doClip bool) (vs []string, o geojson.Ob
 	return
 }
 
-func (s *Server) cmdTest(msg *Message) (res resp.Value, err error) {
+// TEST (POINT lat lon)|(GET key id)|(BOUNDS minlat minlon maxlat maxlon)|
+// (OBJECT geojson)|(CIRCLE lat lon meters)|(TILE x y z)|(QUADKEY quadkey)|
+// (HASH geohash) INTERSECTS|WITHIN [CLIP] (POINT lat lon)|(GET key id)|
+// (BOUNDS minlat minlon maxlat maxlon)|(OBJECT geojson)|
+// (CIRCLE lat lon meters)|(TILE x y z)|(QUADKEY quadkey)|(HASH geohash)|
+// (SECTOR lat lon meters bearing1 bearing2)
+func (s *Server) cmdTEST(msg *Message) (res resp.Value, err error) {
 	start := time.Now()
+
 	vs := msg.Args[1:]
 
 	var ok bool
@@ -348,8 +355,7 @@ func (s *Server) cmdTest(msg *Message) (res resp.Value, err error) {
 			}
 		}
 	}
-	switch msg.OutputType {
+	if msg.OutputType == JSON {
-	case JSON:
 		var buf bytes.Buffer
 		buf.WriteString(`{"ok":true`)
 		if result != 0 {
@@ -362,13 +368,11 @@ func (s *Server) cmdTest(msg *Message) (res resp.Value, err error) {
 		}
 		buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}")
 		return resp.StringValue(buf.String()), nil
-	case RESP:
-		if clipped != nil {
-			return resp.ArrayValue([]resp.Value{
-				resp.IntegerValue(result),
-				resp.StringValue(clipped.JSON())}), nil
-		}
-		return resp.IntegerValue(result), nil
 	}
-	return NOMessage, nil
+	if clipped != nil {
+		return resp.ArrayValue([]resp.Value{
+			resp.IntegerValue(result),
+			resp.StringValue(clipped.JSON())}), nil
+	}
+	return resp.IntegerValue(result), nil
 }

scripts/test.sh

@@ -5,10 +5,17 @@ cd $(dirname "${BASH_SOURCE[0]}")/..
 
 export CGO_ENABLED=0
 
-# if [ "$NOMODULES" != "1" ]; then
+cd tests
-# 	export GO111MODULE=on
+go test -coverpkg=../internal/server -coverprofile=/tmp/coverage.out $GOTEST
-# 	export GOFLAGS=-mod=vendor
-# fi
 
-cd tests && go test && cd ..
+
-go test $(go list ./... | grep -v /vendor/ | grep -v /tests)
+# go test -coverpkg=../internal/... -coverprofile=/tmp/coverage.out \
+#     -v ./... $GOTEST
+
+go tool cover -html=/tmp/coverage.out -o /tmp/coverage.html
+echo "details: file:///tmp/coverage.html"
+cd ..
+
+if [[ "$GOTEST" == "" ]]; then
+    go test $(go list ./... | grep -v /vendor/ | grep -v /tests)
+fi

tests/aof_test.go

@@ -0,0 +1,347 @@
+package tests
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"net"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	"github.com/gomodule/redigo/redis"
+
+	_ "embed"
+)
+
+func subTestAOF(g *testGroup) {
+	g.regSubTest("loading", aof_loading_test)
+	g.regSubTest("migrate", aof_migrate_test)
+	g.regSubTest("AOF", aof_AOF_test)
+	g.regSubTest("AOFMD5", aof_AOFMD5_test)
+	g.regSubTest("AOFSHRINK", aof_AOFSHRINK_test)
+	g.regSubTest("READONLY", aof_READONLY_test)
+}
+
+func loadAOFAndClose(aof any) error {
+	mc, err := loadAOF(aof)
+	if mc != nil {
+		mc.Close()
+	}
+	return err
+}
+
+func loadAOF(aof any) (*mockServer, error) {
+	var aofb []byte
+	switch aof := aof.(type) {
+	case []byte:
+		aofb = []byte(aof)
+	case string:
+		aofb = []byte(aof)
+	default:
+		return nil, errors.New("aof is not string or bytes")
+	}
+	return mockOpenServer(MockServerOptions{
+		Silent:  true,
+		Metrics: false,
+		AOFData: aofb,
+	})
+}
+
+func aof_loading_test(mc *mockServer) error {
+
+	var err error
+	// invalid command
+	err = loadAOFAndClose("asdfasdf\r\n")
+	if err == nil || err.Error() != "unknown command 'asdfasdf'" {
+		return fmt.Errorf("expected '%v', got '%v'",
+			"unknown command 'asdfasdf'", err)
+	}
+
+	// incomplete command
+	err = loadAOFAndClose("set fleet truck point 10 10\r\nasdfasdf")
+	if err != nil {
+		return err
+	}
+
+	// big aof file
+	var aof string
+	for i := 0; i < 10000; i++ {
+		aof += fmt.Sprintf("SET fleet truck%d POINT 10 10\r\n", i)
+	}
+	err = loadAOFAndClose(aof)
+	if err != nil {
+		return err
+	}
+
+	// extra zeros at various places
+	aof = ""
+	for i := 0; i < 1000; i++ {
+		if i%10 == 0 {
+			aof += string(bytes.Repeat([]byte{0}, 100))
+		}
+		aof += fmt.Sprintf("SET fleet truck%d POINT 10 10\r\n", i)
+	}
+	aof += string(bytes.Repeat([]byte{0}, 5000))
+	err = loadAOFAndClose(aof)
+	if err != nil {
+		return err
+	}
+
+	// bad protocol
+	aof = "*2\r\n$1\r\nh\r\n+OK\r\n"
+	err = loadAOFAndClose(aof)
+	if fmt.Sprintf("%v", err) != "Protocol error: expected '$', got '+'" {
+		return fmt.Errorf("expected '%v', got '%v'",
+			"Protocol error: expected '$', got '+'", err)
+	}
+	return nil
+}
+
+func aof_AOFMD5_test(mc *mockServer) error {
+	for i := 0; i < 10000; i++ {
+		_, err := mc.Do("SET", "fleet", rand.Int(),
+			"POINT", rand.Float64()*180-90, rand.Float64()*360-180)
+		if err != nil {
+			return err
+		}
+	}
+	aof, err := mc.readAOF()
+	if err != nil {
+		return err
+	}
+	check := func(start, size int) func(s string) error {
+		return func(s string) error {
+			sum := md5.Sum(aof[start : start+size])
+			val := hex.EncodeToString(sum[:])
+			if s != val {
+				return fmt.Errorf("expected '%s', got '%s'", val, s)
+			}
+			return nil
+		}
+	}
+	return mc.DoBatch(
+		Do("AOFMD5").Err("wrong number of arguments for 'aofmd5' command"),
+		Do("AOFMD5", 0).Err("wrong number of arguments for 'aofmd5' command"),
+		Do("AOFMD5", 0, 0, 1).Err("wrong number of arguments for 'aofmd5' command"),
+		Do("AOFMD5", -1, 0).Err("invalid argument '-1'"),
+		Do("AOFMD5", 1, -1).Err("invalid argument '-1'"),
+		Do("AOFMD5", 0, 100000000000).Err("EOF"),
+		Do("AOFMD5", 0, 0).Str("d41d8cd98f00b204e9800998ecf8427e"),
+		Do("AOFMD5", 0, 0).JSON().Str(`{"ok":true,"md5":"d41d8cd98f00b204e9800998ecf8427e"}`),
+		Do("AOFMD5", 0, 0).Func(check(0, 0)),
+		Do("AOFMD5", 0, 1).Func(check(0, 1)),
+		Do("AOFMD5", 0, 100).Func(check(0, 100)),
+		Do("AOFMD5", 1002, 4321).Func(check(1002, 4321)),
+	)
+}
+
+func openFollower(mc *mockServer) (conn redis.Conn, err error) {
+	conn, err = redis.Dial("tcp", fmt.Sprintf(":%d", mc.port),
+		redis.DialReadTimeout(time.Second))
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			conn.Close()
+			conn = nil
+		}
+	}()
+	if err := conn.Send("AOF", 0); err != nil {
+		return nil, err
+	}
+	if err := conn.Flush(); err != nil {
+		return nil, err
+	}
+	str, err := redis.String(conn.Receive())
+	if err != nil {
+		return nil, err
+	}
+	if str != "OK" {
+		return nil, fmt.Errorf("expected '%s', got '%s'", "OK", str)
+	}
+	return conn, nil
+}
+
+func aof_AOF_test(mc *mockServer) error {
+	var argss [][]interface{}
+	for i := 0; i < 10000; i++ {
+		args := []interface{}{"SET", "fleet", fmt.Sprint(rand.Int()),
+			"POINT", fmt.Sprint(rand.Float64()*180 - 90),
+			fmt.Sprint(rand.Float64()*360 - 180)}
+		argss = append(argss, args)
+		_, err := mc.Do(fmt.Sprint(args[0]), args[1:]...)
+		if err != nil {
+			return err
+		}
+	}
+	readAll := func() (conn redis.Conn, err error) {
+		conn, err = openFollower(mc)
+		if err != nil {
+			return
+		}
+		defer func() {
+			if err != nil {
+				conn.Close()
+				conn = nil
+			}
+		}()
+		var t bool
+		for i := 0; i < len(argss); i++ {
+			args, err := redis.Values(conn.Receive())
+			if err != nil {
+				return nil, err
+			}
+			if t || (len(args) == len(argss[0]) &&
+				fmt.Sprintf("%s", args[2]) == fmt.Sprintf("%s", argss[0][2])) {
+				t = true
+				if fmt.Sprintf("%s", args[2]) !=
+					fmt.Sprintf("%s", argss[i][2]) {
+					return nil, fmt.Errorf("expected '%s', got '%s'",
+						argss[i][2], args[2])
+				}
+			} else {
+				i--
+			}
+		}
+		return conn, nil
+	}
+
+	conn, err := readAll()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	_, err = conn.Do("fancy") // non-existent error
+	if err == nil || err.Error() != "EOF" {
+		return fmt.Errorf("expected '%v', got '%v'", "EOF", err)
+	}
+
+	conn, err = readAll()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	_, err = conn.Do("quit")
+	if err == nil || err.Error() != "EOF" {
+		return fmt.Errorf("expected '%v', got '%v'", "EOF", err)
+	}
+
+	return mc.DoBatch(
+		Do("AOF").Err("wrong number of arguments for 'aof' command"),
+		Do("AOF", 0, 0).Err("wrong number of arguments for 'aof' command"),
+		Do("AOF", -1).Err("invalid argument '-1'"),
+		Do("AOF", 1000000000000).Err("pos is too big, must be less that the aof_size of leader"),
+	)
+}
+
+func aof_AOFSHRINK_test(mc *mockServer) error {
+	var err error
+	haddr := fmt.Sprintf("localhost:%d", getNextPort())
+	ln, err := net.Listen("tcp", haddr)
+	if err != nil {
+		return err
+	}
+	defer ln.Close()
+	var msgs atomic.Int32
+	go func() {
+		mux := http.NewServeMux()
+		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+			msgs.Add(1)
+			// println(r.URL.Path)
+		})
+		http.Serve(ln, mux)
+	}()
+	err = mc.DoBatch(
+		Do("SETCHAN", "mychan", "INTERSECTS", "mi:0", "BOUNDS", -10, -10, 10, 10).Str("1"),
+		Do("SETHOOK", "myhook", "http://"+haddr, "INTERSECTS", "mi:0", "BOUNDS", -10, -10, 10, 10).Str("1"),
+		Do("MASSINSERT", 5, 10000).OK(),
+	)
+	if err != nil {
+		return err
+	}
+	err = mc.DoBatch(
+		Do("AOFSHRINK").OK(),
+		Do("MASSINSERT", 5, 10000).OK(),
+	)
+	if err != nil {
+		return err
+	}
+	nmsgs := msgs.Load()
+	if nmsgs == 0 {
+		return fmt.Errorf("expected > 0, got %d", nmsgs)
+	}
+	return err
+}
+
+func aof_READONLY_test(mc *mockServer) error {
+	return mc.DoBatch(
+		Do("SET", "mykey", "myid", "POINT", "10", "10").OK(),
+		Do("READONLY", "yes").OK(),
+		Do("SET", "mykey", "myid", "POINT", "10", "10").Err("read only"),
+		Do("READONLY", "no").OK(),
+		Do("SET", "mykey", "myid", "POINT", "10", "10").OK(),
+		Do("READONLY").Err("wrong number of arguments for 'readonly' command"),
+		Do("READONLY", "maybe").Err("invalid argument 'maybe'"),
+	)
+}
+
+//go:embed aof_legacy
+var aofLegacy []byte
+
+func aof_migrate_test(mc *mockServer) error {
+	var aof []byte
+	for i := 0; i < 10000; i++ {
+		aof = append(aof, aofLegacy...)
+	}
+	var mc2 *mockServer
+	var err error
+	defer func() {
+		mc2.Close()
+	}()
+	mc2, err = mockOpenServer(MockServerOptions{
+		AOFFileName: "aof",
+		AOFData:     aof,
+		Silent:      true,
+		Metrics:     true,
+	})
+	if err != nil {
+		return err
+	}
+	err = mc2.DoBatch(
+		Do("GET", "1", "2").Str(`{"type":"Point","coordinates":[20,10]}`),
+	)
+	if err != nil {
+		return err
+	}
+	mc2.Close()
+
+	mc2, err = mockOpenServer(MockServerOptions{
+		AOFFileName: "aof",
+		AOFData:     aofLegacy[:len(aofLegacy)-1],
+		Silent:      true,
+		Metrics:     true,
+	})
+	if err != io.ErrUnexpectedEOF {
+		return fmt.Errorf("expected '%v', got '%v'", io.ErrUnexpectedEOF, err)
+	}
+	mc2.Close()
+
+	mc2, err = mockOpenServer(MockServerOptions{
+		AOFFileName: "aof",
+		AOFData:     aofLegacy[1:],
+		Silent:      true,
+		Metrics:     true,
+	})
+	if err != io.ErrUnexpectedEOF {
+		return fmt.Errorf("expected '%v', got '%v'", io.ErrUnexpectedEOF, err)
+	}
+	mc2.Close()
+
+	return nil
+}

tests/client_test.go

@@ -3,25 +3,33 @@ package tests
 import (
 	"errors"
 	"fmt"
-	"testing"
+	"strings"
 
 	"github.com/gomodule/redigo/redis"
 	"github.com/tidwall/gjson"
+	"github.com/tidwall/pretty"
 )
 
-func subTestClient(t *testing.T, mc *mockServer) {
+func subTestClient(g *testGroup) {
-	runStep(t, mc, "valid json", client_valid_json_test)
+	g.regSubTest("OUTPUT", client_OUTPUT_test)
-	runStep(t, mc, "valid client count", info_valid_client_count_test)
+	g.regSubTest("CLIENT", client_CLIENT_test)
 }
 
-func client_valid_json_test(mc *mockServer) error {
+func client_OUTPUT_test(mc *mockServer) error {
-	if err := mc.DoBatch([][]interface{}{
+	if err := mc.DoBatch(
 		// tests removal of "elapsed" member.
-		{"OUTPUT", "json"}, {`{"ok":true}`},
+		Do("OUTPUT", "json", "yaml").Err(`wrong number of arguments for 'output' command`),
-		{"OUTPUT", "resp"}, {`OK`},
+		Do("OUTPUT", "json").Str(`{"ok":true}`),
-	}); err != nil {
+		Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`),
+		Do("OUTPUT").Str(`resp`), // this is due to the internal Do test
+		Do("OUTPUT", "resp").OK(),
+		Do("OUTPUT", "yaml").Err(`invalid argument 'yaml'`),
+		Do("OUTPUT").Str(`resp`),
+		Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`),
+	); err != nil {
 		return err
 	}
+
 	// run direct commands
 	if _, err := mc.Do("OUTPUT", "json"); err != nil {
 		return err
@@ -45,9 +53,14 @@ func client_valid_json_test(mc *mockServer) error {
 	return nil
 }
 
-func info_valid_client_count_test(mc *mockServer) error {
+func client_CLIENT_test(mc *mockServer) error {
 	numConns := 20
 	var conns []redis.Conn
+	defer func() {
+		for i := range conns {
+			conns[i].Close()
+		}
+	}()
 	for i := 0; i <= numConns; i++ {
 		conn, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port))
 		if err != nil {
@@ -55,9 +68,16 @@ func info_valid_client_count_test(mc *mockServer) error {
 		}
 		conns = append(conns, conn)
 	}
-	for i := range conns {
+
-		defer conns[i].Close()
+	_, err := conns[1].Do("CLIENT", "setname", "cl1")
+	if err != nil {
+		return err
 	}
+	_, err = conns[2].Do("CLIENT", "setname", "cl2")
+	if err != nil {
+		return err
+	}
+
 	if _, err := mc.Do("OUTPUT", "JSON"); err != nil {
 		return err
 	}
@@ -69,9 +89,53 @@ func info_valid_client_count_test(mc *mockServer) error {
 	if !ok {
 		return errors.New("Failed to type assert CLIENT response")
 	}
-	sres := string(bres)
+	sres := string(pretty.Pretty(bres))
-	if len(gjson.Get(sres, "list").Array()) < numConns {
+	if int(gjson.Get(sres, "list.#").Int()) < numConns {
 		return errors.New("Invalid number of connections")
 	}
-	return nil
+
+	client13ID := gjson.Get(sres, "list.13.id").String()
+	client14Addr := gjson.Get(sres, "list.14.addr").String()
+	client15Addr := gjson.Get(sres, "list.15.addr").String()
+
+	return mc.DoBatch(
+		Do("CLIENT", "list").JSON().Func(func(s string) error {
+			if int(gjson.Get(s, "list.#").Int()) < numConns {
+				return errors.New("Invalid number of connections")
+			}
+			return nil
+		}),
+		Do("CLIENT", "list").Func(func(s string) error {
+			if len(strings.Split(strings.TrimSpace(s), "\n")) < numConns {
+				return errors.New("Invalid number of connections")
+			}
+			return nil
+		}),
+		Do("CLIENT").Err(`wrong number of arguments for 'client' command`),
+		Do("CLIENT", "hello").Err(`Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)`),
+		Do("CLIENT", "list", "arg3").Err(`wrong number of arguments for 'client' command`),
+		Do("CLIENT", "getname", "arg3").Err(`wrong number of arguments for 'client' command`),
+		Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":""}`),
+		Do("CLIENT", "getname").Str(``),
+		Do("CLIENT", "setname", "abc").OK(),
+		Do("CLIENT", "getname").Str(`abc`),
+		Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":"abc"}`),
+		Do("CLIENT", "setname", "abc", "efg").Err(`wrong number of arguments for 'client' command`),
+		Do("CLIENT", "setname", " abc ").Err(`Client names cannot contain spaces, newlines or special characters.`),
+		Do("CLIENT", "setname", "abcd").JSON().OK(),
+		Do("CLIENT", "kill", "name", "abcd").Err("No such client"),
+		Do("CLIENT", "getname").Str(`abcd`),
+		Do("CLIENT", "kill").Err(`wrong number of arguments for 'client' command`),
+		Do("CLIENT", "kill", "").Err(`No such client`),
+		Do("CLIENT", "kill", "abcd").Err(`No such client`),
+		Do("CLIENT", "kill", "id", client13ID).OK(),
+		Do("CLIENT", "kill", "id").Err("wrong number of arguments for 'client' command"),
+		Do("CLIENT", "kill", client14Addr).OK(),
+		Do("CLIENT", "kill", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"),
+		Do("CLIENT", "kill", "addr").Err("wrong number of arguments for 'client' command"),
+		Do("CLIENT", "kill", "addr", client15Addr).JSON().OK(),
+		Do("CLIENT", "kill", "addr", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"),
+		Do("CLIENT", "kill", "id", "1000").Err("No such client"),
+	)
+
 }

tests/fence_roaming_test.go

@@ -2,7 +2,7 @@ package tests
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"sync"
@@ -29,7 +29,7 @@ func fence_roaming_webhook_test(mc *mockServer) error {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if err := func() error {
 			// Read the request body
-			body, err := ioutil.ReadAll(r.Body)
+			body, err := io.ReadAll(r.Body)
 			if err != nil {
 				return err
 			}
@@ -115,8 +115,10 @@ func fence_roaming_live_test(mc *mockServer) error {
 	liveReady.Add(1)
 	return goMultiFunc(mc,
 		func() error {
-			sc, err := redis.DialTimeout("tcp", fmt.Sprintf(":%d", mc.port),
+			sc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port),
-				0, time.Second*5, time.Second*5)
+				redis.DialConnectTimeout(0),
+				redis.DialReadTimeout(time.Second*5),
+				redis.DialWriteTimeout(time.Second*5))
 			if err != nil {
 				liveReady.Done()
 				return err

tests/fence_test.go

@@ -12,30 +12,29 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
-	"testing"
 	"time"
 
 	"github.com/gomodule/redigo/redis"
 	"github.com/tidwall/gjson"
 )
 
-func subTestFence(t *testing.T, mc *mockServer) {
+func subTestFence(g *testGroup) {
 
 	// Standard
-	runStep(t, mc, "basic", fence_basic_test)
+	g.regSubTest("basic", fence_basic_test)
-	runStep(t, mc, "channel message order", fence_channel_message_order_test)
+	g.regSubTest("channel message order", fence_channel_message_order_test)
-	runStep(t, mc, "detect inside,outside", fence_detect_inside_test)
+	g.regSubTest("detect inside,outside", fence_detect_inside_test)
 
 	// Roaming
-	runStep(t, mc, "roaming live", fence_roaming_live_test)
+	g.regSubTest("roaming live", fence_roaming_live_test)
-	runStep(t, mc, "roaming channel", fence_roaming_channel_test)
+	g.regSubTest("roaming channel", fence_roaming_channel_test)
-	runStep(t, mc, "roaming webhook", fence_roaming_webhook_test)
+	g.regSubTest("roaming webhook", fence_roaming_webhook_test)
 
 	// channel meta
-	runStep(t, mc, "channel meta", fence_channel_meta_test)
+	g.regSubTest("channel meta", fence_channel_meta_test)
 
 	// various
-	runStep(t, mc, "detect eecio", fence_eecio_test)
+	g.regSubTest("detect eecio", fence_eecio_test)
 }
 
 type fenceReader struct {
@@ -205,7 +204,7 @@ func fence_channel_message_order_test(mc *mockServer) error {
 					break loop
 				}
 			case error:
-				fmt.Printf(err.Error())
+				fmt.Printf("%s\n", err.Error())
 			}
 		}
 
@@ -230,10 +229,10 @@ func fence_channel_message_order_test(mc *mockServer) error {
 	// Fire all setup commands on the base client
 	for _, cmd := range []string{
 		"SET points point POINT 33.412529053733444 -111.93368911743164",
-		fmt.Sprintf(`SETCHAN A WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.95205688476562,33.400491820565236],[-111.92630767822266,33.400491820565236],[-111.92630767822266,33.422272258866045],[-111.95205688476562,33.422272258866045],[-111.95205688476562,33.400491820565236]]]}`),
+		`SETCHAN A WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.95205688476562,33.400491820565236],[-111.92630767822266,33.400491820565236],[-111.92630767822266,33.422272258866045],[-111.95205688476562,33.422272258866045],[-111.95205688476562,33.400491820565236]]]}`,
-		fmt.Sprintf(`SETCHAN B WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.93952560424803,33.403501285221594],[-111.92630767822266,33.403501285221594],[-111.92630767822266,33.41997983836345],[-111.93952560424803,33.41997983836345],[-111.93952560424803,33.403501285221594]]]}`),
+		`SETCHAN B WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.93952560424803,33.403501285221594],[-111.92630767822266,33.403501285221594],[-111.92630767822266,33.41997983836345],[-111.93952560424803,33.41997983836345],[-111.93952560424803,33.403501285221594]]]}`,
-		fmt.Sprintf(`SETCHAN C WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.9255781173706,33.40342963251261],[-111.91201686859131,33.40342963251261],[-111.91201686859131,33.41994401881284],[-111.9255781173706,33.41994401881284],[-111.9255781173706,33.40342963251261]]]}`),
+		`SETCHAN C WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.9255781173706,33.40342963251261],[-111.91201686859131,33.40342963251261],[-111.91201686859131,33.41994401881284],[-111.9255781173706,33.41994401881284],[-111.9255781173706,33.40342963251261]]]}`,
-		fmt.Sprintf(`SETCHAN D WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.92562103271484,33.40063513076968],[-111.90021514892578,33.40063513076968],[-111.90021514892578,33.42212898435788],[-111.92562103271484,33.42212898435788],[-111.92562103271484,33.40063513076968]]]}`),
+		`SETCHAN D WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.92562103271484,33.40063513076968],[-111.90021514892578,33.40063513076968],[-111.90021514892578,33.42212898435788],[-111.92562103271484,33.42212898435788],[-111.92562103271484,33.40063513076968]]]}`,
 		"SET points point POINT 33.412529053733444 -111.91909790039062",
 	} {
 		if _, err := do(bc, cmd); err != nil {

tests/follower_test.go

@@ -0,0 +1,68 @@
+package tests
+
+import "time"
+
+func subTestFollower(g *testGroup) {
+	g.regSubTest("follow", follower_follow_test)
+}
+
+func follower_follow_test(mc *mockServer) error {
+	mc2, err := mockOpenServer(MockServerOptions{
+		Silent: true, Metrics: false,
+	})
+	if err != nil {
+		return err
+	}
+	defer mc2.Close()
+	err = mc.DoBatch(
+		Do("SET", "mykey", "truck1", "POINT", 10, 10).OK(),
+		Do("SET", "mykey", "truck2", "POINT", 10, 10).OK(),
+		Do("SET", "mykey", "truck3", "POINT", 10, 10).OK(),
+		Do("SET", "mykey", "truck4", "POINT", 10, 10).OK(),
+		Do("SET", "mykey", "truck5", "POINT", 10, 10).OK(),
+		Do("SET", "mykey", "truck6", "POINT", 10, 10).OK(),
+		Do("CONFIG", "SET", "requirepass", "1234").OK(),
+		Do("AUTH", "1234").OK(),
+	)
+	if err != nil {
+		return err
+	}
+	err = mc2.DoBatch(
+		Do("SET", "mykey2", "truck1", "POINT", 10, 10).OK(),
+		Do("SET", "mykey2", "truck2", "POINT", 10, 10).OK(),
+		Do("GET", "mykey2", "truck1").Str(`{"type":"Point","coordinates":[10,10]}`),
+		Do("GET", "mykey2", "truck2").Str(`{"type":"Point","coordinates":[10,10]}`),
+
+		Do("CONFIG", "SET", "leaderauth", "1234").OK(),
+		Do("FOLLOW", "localhost", mc.port).OK(),
+		Do("GET", "mykey", "truck1").Err("catching up to leader"),
+		Sleep(time.Second/2),
+
+		Do("GET", "mykey", "truck1").Err(`{"type":"Point","coordinates":[10,10]}`),
+		Do("GET", "mykey", "truck2").Err(`{"type":"Point","coordinates":[10,10]}`),
+	)
+	if err != nil {
+		return err
+	}
+
+	err = mc.DoBatch(
+		Do("SET", "mykey", "truck7", "POINT", 10, 10).OK(),
+		Do("SET", "mykey", "truck8", "POINT", 10, 10).OK(),
+		Do("SET", "mykey", "truck9", "POINT", 10, 10).OK(),
+	)
+	if err != nil {
+		return err
+	}
+
+	err = mc2.DoBatch(
+		Sleep(time.Second/2),
+		Do("GET", "mykey", "truck7").Str(`{"type":"Point","coordinates":[10,10]}`),
+		Do("GET", "mykey", "truck8").Str(`{"type":"Point","coordinates":[10,10]}`),
+		Do("GET", "mykey", "truck9").Str(`{"type":"Point","coordinates":[10,10]}`),
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

tests/json_test.go

@@ -1,11 +1,9 @@
 package tests
 
-import "testing"
+func subTestJSON(g *testGroup) {
-
+	g.regSubTest("basic", json_JSET_basic_test)
-func subTestJSON(t *testing.T, mc *mockServer) {
+	g.regSubTest("geojson", json_JSET_geojson_test)
-	runStep(t, mc, "basic", json_JSET_basic_test)
+	g.regSubTest("number", json_JSET_number_test)
-	runStep(t, mc, "geojson", json_JSET_geojson_test)
-	runStep(t, mc, "number", json_JSET_number_test)
 
 }
 func json_JSET_basic_test(mc *mockServer) error {

tests/keys_search_test.go

@@ -13,26 +13,26 @@ import (
 	"github.com/tidwall/gjson"
 )
 
-func subTestSearch(t *testing.T, mc *mockServer) {
+func subTestSearch(g *testGroup) {
-	runStep(t, mc, "KNN_BASIC", keys_KNN_basic_test)
+	g.regSubTest("KNN_BASIC", keys_KNN_basic_test)
-	runStep(t, mc, "KNN_RANDOM", keys_KNN_random_test)
+	g.regSubTest("KNN_RANDOM", keys_KNN_random_test)
-	runStep(t, mc, "KNN_CURSOR", keys_KNN_cursor_test)
+	g.regSubTest("KNN_CURSOR", keys_KNN_cursor_test)
-	runStep(t, mc, "NEARBY_SPARSE", keys_NEARBY_SPARSE_test)
+	g.regSubTest("NEARBY_SPARSE", keys_NEARBY_SPARSE_test)
-	runStep(t, mc, "WITHIN_CIRCLE", keys_WITHIN_CIRCLE_test)
+	g.regSubTest("WITHIN_CIRCLE", keys_WITHIN_CIRCLE_test)
-	runStep(t, mc, "WITHIN_SECTOR", keys_WITHIN_SECTOR_test)
+	g.regSubTest("WITHIN_SECTOR", keys_WITHIN_SECTOR_test)
-	runStep(t, mc, "INTERSECTS_CIRCLE", keys_INTERSECTS_CIRCLE_test)
+	g.regSubTest("INTERSECTS_CIRCLE", keys_INTERSECTS_CIRCLE_test)
-	runStep(t, mc, "INTERSECTS_SECTOR", keys_INTERSECTS_SECTOR_test)
+	g.regSubTest("INTERSECTS_SECTOR", keys_INTERSECTS_SECTOR_test)
-	runStep(t, mc, "WITHIN", keys_WITHIN_test)
+	g.regSubTest("WITHIN", keys_WITHIN_test)
-	runStep(t, mc, "WITHIN_CURSOR", keys_WITHIN_CURSOR_test)
+	g.regSubTest("WITHIN_CURSOR", keys_WITHIN_CURSOR_test)
-	runStep(t, mc, "WITHIN_CLIPBY", keys_WITHIN_CLIPBY_test)
+	g.regSubTest("WITHIN_CLIPBY", keys_WITHIN_CLIPBY_test)
-	runStep(t, mc, "INTERSECTS", keys_INTERSECTS_test)
+	g.regSubTest("INTERSECTS", keys_INTERSECTS_test)
-	runStep(t, mc, "INTERSECTS_CURSOR", keys_INTERSECTS_CURSOR_test)
+	g.regSubTest("INTERSECTS_CURSOR", keys_INTERSECTS_CURSOR_test)
-	runStep(t, mc, "INTERSECTS_CLIPBY", keys_INTERSECTS_CLIPBY_test)
+	g.regSubTest("INTERSECTS_CLIPBY", keys_INTERSECTS_CLIPBY_test)
-	runStep(t, mc, "SCAN_CURSOR", keys_SCAN_CURSOR_test)
+	g.regSubTest("SCAN_CURSOR", keys_SCAN_CURSOR_test)
-	runStep(t, mc, "SEARCH_CURSOR", keys_SEARCH_CURSOR_test)
+	g.regSubTest("SEARCH_CURSOR", keys_SEARCH_CURSOR_test)
-	runStep(t, mc, "MATCH", keys_MATCH_test)
+	g.regSubTest("MATCH", keys_MATCH_test)
-	runStep(t, mc, "FIELDS", keys_FIELDS_search_test)
+	g.regSubTest("FIELDS", keys_FIELDS_search_test)
-	runStep(t, mc, "BUFFER", keys_BUFFER_search_test)
+	g.regSubTest("BUFFER", keys_BUFFER_search_test)
 }
 
 func keys_KNN_basic_test(mc *mockServer) error {
@@ -122,9 +122,7 @@ func keys_KNN_random_test(mc *mockServer) error {
 	mc.Do("OUTPUT", "json")
 	defer mc.Do("OUTPUT", "resp")
 
-	start := time.Now()
 	res, err := redis.String(mc.Do("NEARBY", "points", "LIMIT", N, "POINT", target[1], target[0]))
-	println(time.Since(start).String())
 	if err != nil {
 		return err
 	}

tests/keys_test.go

@@ -4,314 +4,403 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
-	"os/exec"
-	"strconv"
 	"strings"
-	"testing"
 	"time"
 
 	"github.com/gomodule/redigo/redis"
 	"github.com/tidwall/gjson"
 )
 
-func subTestKeys(t *testing.T, mc *mockServer) {
+func subTestKeys(g *testGroup) {
-	runStep(t, mc, "BOUNDS", keys_BOUNDS_test)
+	g.regSubTest("BOUNDS", keys_BOUNDS_test)
-	runStep(t, mc, "DEL", keys_DEL_test)
+	g.regSubTest("DEL", keys_DEL_test)
-	runStep(t, mc, "DROP", keys_DROP_test)
+	g.regSubTest("DROP", keys_DROP_test)
-	runStep(t, mc, "RENAME", keys_RENAME_test)
+	g.regSubTest("RENAME", keys_RENAME_test)
-	runStep(t, mc, "RENAMENX", keys_RENAMENX_test)
+	g.regSubTest("RENAMENX", keys_RENAMENX_test)
-	runStep(t, mc, "EXPIRE", keys_EXPIRE_test)
+	g.regSubTest("EXPIRE", keys_EXPIRE_test)
-	runStep(t, mc, "FSET", keys_FSET_test)
+	g.regSubTest("FSET", keys_FSET_test)
-	runStep(t, mc, "GET", keys_GET_test)
+	g.regSubTest("GET", keys_GET_test)
-	runStep(t, mc, "KEYS", keys_KEYS_test)
+	g.regSubTest("KEYS", keys_KEYS_test)
-	runStep(t, mc, "PERSIST", keys_PERSIST_test)
+	g.regSubTest("PERSIST", keys_PERSIST_test)
-	runStep(t, mc, "SET", keys_SET_test)
+	g.regSubTest("SET", keys_SET_test)
-	runStep(t, mc, "STATS", keys_STATS_test)
+	g.regSubTest("STATS", keys_STATS_test)
-	runStep(t, mc, "TTL", keys_TTL_test)
+	g.regSubTest("TTL", keys_TTL_test)
-	runStep(t, mc, "SET EX", keys_SET_EX_test)
+	g.regSubTest("SET EX", keys_SET_EX_test)
-	runStep(t, mc, "PDEL", keys_PDEL_test)
+	g.regSubTest("PDEL", keys_PDEL_test)
-	runStep(t, mc, "FIELDS", keys_FIELDS_test)
+	g.regSubTest("FIELDS", keys_FIELDS_test)
-	runStep(t, mc, "WHEREIN", keys_WHEREIN_test)
+	g.regSubTest("WHEREIN", keys_WHEREIN_test)
-	runStep(t, mc, "WHEREEVAL", keys_WHEREEVAL_test)
+	g.regSubTest("WHEREEVAL", keys_WHEREEVAL_test)
+	g.regSubTest("TYPE", keys_TYPE_test)
+	g.regSubTest("FLUSHDB", keys_FLUSHDB_test)
+	g.regSubTest("HEALTHZ", keys_HEALTHZ_test)
+	g.regSubTest("SERVER", keys_SERVER_test)
+	g.regSubTest("INFO", keys_INFO_test)
 }
 
 func keys_BOUNDS_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid1", "POINT", 33, -115}, {"OK"},
+		Do("BOUNDS", "mykey").Str("<nil>"),
-		{"BOUNDS", "mykey"}, {"[[-115 33] [-115 33]]"},
+		Do("BOUNDS", "mykey").JSON().Err("key not found"),
-		{"SET", "mykey", "myid2", "POINT", 34, -112}, {"OK"},
+		Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(),
-		{"BOUNDS", "mykey"}, {"[[-115 33] [-112 34]]"},
+		Do("BOUNDS", "mykey").Str("[[-115 33] [-115 33]]"),
-		{"DEL", "mykey", "myid2"}, {1},
+		Do("BOUNDS", "mykey").JSON().Str(`{"ok":true,"bounds":{"type":"Point","coordinates":[-115,33]}}`),
-		{"BOUNDS", "mykey"}, {"[[-115 33] [-115 33]]"},
+		Do("SET", "mykey", "myid2", "POINT", 34, -112).OK(),
-		{"SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`}, {"OK"},
+		Do("BOUNDS", "mykey").Str("[[-115 33] [-112 34]]"),
-		{"SET", "mykey", "myid4", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`}, {"OK"},
+		Do("DEL", "mykey", "myid2").Str("1"),
-		{"BOUNDS", "mykey"}, {"[[-130 25] [-110 38]]"},
+		Do("BOUNDS", "mykey").Str("[[-115 33] [-115 33]]"),
-	})
+		Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(),
+		Do("SET", "mykey", "myid4", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(),
+		Do("BOUNDS", "mykey").Str("[[-130 25] [-110 38]]"),
+		Do("BOUNDS", "mykey", "hello").Err("wrong number of arguments for 'bounds' command"),
+		Do("BOUNDS", "nada").Str("<nil>"),
+		Do("BOUNDS", "nada").JSON().Err("key not found"),
+		Do("BOUNDS", "").Str("<nil>"),
+		Do("BOUNDS", "mykey").JSON().Str(`{"ok":true,"bounds":{"type":"Polygon","coordinates":[[[-130,25],[-110,25],[-110,38],[-130,38],[-130,25]]]}}`),
+	)
 }
+
 func keys_DEL_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid", "POINT", 33, -115).OK(),
-		{"GET", "mykey", "myid", "POINT"}, {"[33 -115]"},
+		Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"),
-		{"DEL", "mykey", "myid"}, {"1"},
+		Do("DEL", "mykey", "myid2", "ERRON404").Err("id not found"),
-		{"GET", "mykey", "myid"}, {nil},
+		Do("DEL", "mykey", "myid").Str("1"),
-	})
+		Do("DEL", "mykey", "myid").Str("0"),
+		Do("DEL", "mykey").Err("wrong number of arguments for 'del' command"),
+		Do("GET", "mykey", "myid").Str("<nil>"),
+		Do("DEL", "mykey", "myid", "ERRON404").Err("key not found"),
+		Do("DEL", "mykey", "myid", "invalid-arg").Err("invalid argument 'invalid-arg'"),
+		Do("SET", "mykey", "myid", "POINT", 33, -115).OK(),
+		Do("DEL", "mykey", "myid2", "ERRON404").JSON().Err("id not found"),
+		Do("DEL", "mykey", "myid").JSON().OK(),
+		Do("DEL", "mykey", "myid").JSON().OK(),
+		Do("DEL", "mykey", "myid", "ERRON404").JSON().Err("key not found"),
+	)
 }
+
 func keys_DROP_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"},
+		Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(),
-		{"SET", "mykey", "myid2", "HASH", "9my5xp8"}, {"OK"},
+		Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(),
-		{"SCAN", "mykey", "COUNT"}, {2},
+		Do("SCAN", "mykey", "COUNT").Str("2"),
-		{"DROP", "mykey"}, {1},
+		Do("DROP").Err("wrong number of arguments for 'drop' command"),
-		{"SCAN", "mykey", "COUNT"}, {0},
+		Do("DROP", "mykey", "arg3").Err("wrong number of arguments for 'drop' command"),
-		{"DROP", "mykey"}, {0},
+		Do("DROP", "mykey").Str("1"),
-		{"SCAN", "mykey", "COUNT"}, {0},
+		Do("SCAN", "mykey", "COUNT").Str("0"),
-	})
+		Do("DROP", "mykey").Str("0"),
+		Do("SCAN", "mykey", "COUNT").Str("0"),
+		Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(),
+		Do("DROP", "mykey").JSON().OK(),
+		Do("DROP", "mykey").JSON().OK(),
+	)
 }
 func keys_RENAME_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"},
+		Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(),
-		{"SET", "mykey", "myid2", "HASH", "9my5xp8"}, {"OK"},
+		Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(),
-		{"SCAN", "mykey", "COUNT"}, {2},
+		Do("SCAN", "mykey", "COUNT").Str("2"),
-		{"RENAME", "mykey", "mynewkey"}, {"OK"},
+		Do("RENAME", "foo", "mynewkey", "arg3").Err("wrong number of arguments for 'rename' command"),
-		{"SCAN", "mykey", "COUNT"}, {0},
+		Do("RENAME", "mykey", "mynewkey").OK(),
-		{"SCAN", "mynewkey", "COUNT"}, {2},
+		Do("SCAN", "mykey", "COUNT").Str("0"),
-		{"SET", "mykey", "myid3", "HASH", "9my5xp7"}, {"OK"},
+		Do("SCAN", "mynewkey", "COUNT").Str("2"),
-		{"RENAME", "mykey", "mynewkey"}, {"OK"},
+		Do("SET", "mykey", "myid3", "HASH", "9my5xp7").OK(),
-		{"SCAN", "mykey", "COUNT"}, {0},
+		Do("RENAME", "mykey", "mynewkey").OK(),
-		{"SCAN", "mynewkey", "COUNT"}, {1},
+		Do("SCAN", "mykey", "COUNT").Str("0"),
-		{"RENAME", "foo", "mynewkey"}, {"ERR key not found"},
+		Do("SCAN", "mynewkey", "COUNT").Str("1"),
-		{"SCAN", "mynewkey", "COUNT"}, {1},
+		Do("RENAME", "foo", "mynewkey").Err("key not found"),
-	})
+		Do("SCAN", "mynewkey", "COUNT").Str("1"),
+		Do("SETCHAN", "mychan", "INTERSECTS", "mynewkey", "BOUNDS", 10, 10, 20, 20).Str("1"),
+		Do("RENAME", "mynewkey", "foo2").Err("key has hooks set"),
+		Do("RENAMENX", "mynewkey", "foo2").Err("key has hooks set"),
+		Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(),
+		Do("RENAME", "mykey", "foo2").OK(),
+		Do("RENAMENX", "foo2", "foo3").Str("1"),
+		Do("RENAMENX", "foo2", "foo3").Err("key not found"),
+		Do("RENAME", "foo2", "foo3").JSON().Err("key not found"),
+		Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(),
+		Do("RENAMENX", "mykey", "foo3").Str("0"),
+		Do("RENAME", "foo3", "foo4").JSON().OK(),
+	)
 }
 func keys_RENAMENX_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"},
+		Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(),
-		{"SET", "mykey", "myid2", "HASH", "9my5xp8"}, {"OK"},
+		Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(),
-		{"SCAN", "mykey", "COUNT"}, {2},
+		Do("SCAN", "mykey", "COUNT").Str("2"),
-		{"RENAMENX", "mykey", "mynewkey"}, {1},
+		Do("RENAMENX", "mykey", "mynewkey").Str("1"),
-		{"SCAN", "mykey", "COUNT"}, {0},
+		Do("SCAN", "mykey", "COUNT").Str("0"),
-		{"DROP", "mykey"}, {0},
+		Do("DROP", "mykey").Str("0"),
-		{"SCAN", "mykey", "COUNT"}, {0},
+		Do("SCAN", "mykey", "COUNT").Str("0"),
-		{"SCAN", "mynewkey", "COUNT"}, {2},
+		Do("SCAN", "mynewkey", "COUNT").Str("2"),
-		{"SET", "mykey", "myid3", "HASH", "9my5xp7"}, {"OK"},
+		Do("SET", "mykey", "myid3", "HASH", "9my5xp7").OK(),
-		{"RENAMENX", "mykey", "mynewkey"}, {0},
+		Do("RENAMENX", "mykey", "mynewkey").Str("0"),
-		{"SCAN", "mykey", "COUNT"}, {1},
+		Do("SCAN", "mykey", "COUNT").Str("1"),
-		{"SCAN", "mynewkey", "COUNT"}, {2},
+		Do("SCAN", "mynewkey", "COUNT").Str("2"),
-		{"RENAMENX", "foo", "mynewkey"}, {"ERR key not found"},
+		Do("RENAMENX", "foo", "mynewkey").Str("ERR key not found"),
-		{"SCAN", "mynewkey", "COUNT"}, {2},
+		Do("SCAN", "mynewkey", "COUNT").Str("2"),
-	})
+	)
 }
 func keys_EXPIRE_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid", "STRING", "value"}, {"OK"},
+		Do("SET", "mykey", "myid", "STRING", "value").OK(),
-		{"EXPIRE", "mykey", "myid", 1}, {1},
+		Do("EXPIRE", "mykey", "myid").Err("wrong number of arguments for 'expire' command"),
-		{time.Second / 4}, {}, // sleep
+		Do("EXPIRE", "mykey", "myid", "y").Err("invalid argument 'y'"),
-		{"GET", "mykey", "myid"}, {"value"},
+		Do("EXPIRE", "mykey", "myid", 1).Str("1"),
-		{time.Second}, {}, // sleep
+		Do("EXPIRE", "mykey", "myid", 1).JSON().OK(),
-		{"GET", "mykey", "myid"}, {nil},
+		Sleep(time.Second/4),
-	})
+		Do("GET", "mykey", "myid").Str("value"),
+		Sleep(time.Second),
+		Do("GET", "mykey", "myid").Str("<nil>"),
+		Do("EXPIRE", "mykey", "myid", 1).JSON().Err("key not found"),
+		Do("SET", "mykey", "myid1", "STRING", "value1").OK(),
+		Do("SET", "mykey", "myid2", "STRING", "value2").OK(),
+		Do("EXPIRE", "mykey", "myid1", 1).Str("1"),
+		Sleep(time.Second/4),
+		Do("GET", "mykey", "myid1").Str("value1"),
+		Sleep(time.Second),
+		Do("EXPIRE", "mykey", "myid1", 1).Str("0"),
+		Do("EXPIRE", "mykey", "myid1", 1).JSON().Err("id not found"),
+	)
 }
 func keys_FSET_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid", "HASH", "9my5xp7"}, {"OK"},
+		Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(),
-		{"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7]"},
+		Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7]"),
-		{"FSET", "mykey", "myid", "f1", 105.6}, {1},
+		Do("FSET", "mykey", "myid", "f1", 105.6).Str("1"),
-		{"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f1 105.6]]"},
+		Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 105.6]]"),
-		{"FSET", "mykey", "myid", "f1", 1.1, "f2", 2.2}, {2},
+		Do("FSET", "mykey", "myid", "f1", 1.1, "f2", 2.2).Str("2"),
-		{"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f1 1.1 f2 2.2]]"},
+		Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 1.1 f2 2.2]]"),
-		{"FSET", "mykey", "myid", "f1", 1.1, "f2", 22.22}, {1},
+		Do("FSET", "mykey", "myid", "f1", 1.1, "f2", 22.22).Str("1"),
-		{"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f1 1.1 f2 22.22]]"},
+		Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 1.1 f2 22.22]]"),
-		{"FSET", "mykey", "myid", "f1", 0}, {1},
+		Do("FSET", "mykey", "myid", "f1", 0).Str("1"),
-		{"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f2 22.22]]"},
+		Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f2 22.22]]"),
-		{"FSET", "mykey", "myid", "f2", 0}, {1},
+		Do("FSET", "mykey", "myid", "f2", 0).Str("1"),
-		{"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7]"},
+		Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7]"),
-		{"FSET", "mykey", "myid2", "xx", "f1", 1.1, "f2", 2.2}, {0},
+		Do("FSET", "mykey", "myid2", "xx", "f1", 1.1, "f2", 2.2).Str("0"),
-		{"GET", "mykey", "myid2"}, {nil},
+		Do("GET", "mykey", "myid2").Str("<nil>"),
-		{"DEL", "mykey", "myid"}, {"1"},
+		Do("DEL", "mykey", "myid").Str("1"),
-		{"GET", "mykey", "myid"}, {nil},
+		Do("GET", "mykey", "myid").Str("<nil>"),
-	})
+		Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(),
+		Do("CONFIG", "SET", "maxmemory", "1").OK(),
+		Do("FSET", "mykey", "myid", "xx", "f1", 1.1, "f2", 2.2).Err(`OOM command not allowed when used memory > 'maxmemory'`),
+		Do("CONFIG", "SET", "maxmemory", "0").OK(),
+		Do("FSET", "mykey", "myid", "xx").Err("wrong number of arguments for 'fset' command"),
+		Do("FSET", "mykey", "myid", "f1", "a", "f2").Err("wrong number of arguments for 'fset' command"),
+		Do("FSET", "mykey", "myid", "z", "a").Err("invalid argument 'z'"),
+		Do("FSET", "mykey2", "myid", "a", "b").Err("key not found"),
+		Do("FSET", "mykey", "myid2", "a", "b").Err("id not found"),
+		Do("FSET", "mykey", "myid", "f2", 0).JSON().OK(),
+	)
 }
 func keys_GET_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid", "STRING", "value"}, {"OK"},
+		Do("SET", "mykey", "myid", "STRING", "value").OK(),
-		{"GET", "mykey", "myid"}, {"value"},
+		Do("GET", "mykey", "myid").Str("value"),
-		{"SET", "mykey", "myid", "STRING", "value2"}, {"OK"},
+		Do("SET", "mykey", "myid", "STRING", "value2").OK(),
-		{"GET", "mykey", "myid"}, {"value2"},
+		Do("GET", "mykey", "myid").Str("value2"),
-		{"DEL", "mykey", "myid"}, {"1"},
+		Do("DEL", "mykey", "myid").Str("1"),
-		{"GET", "mykey", "myid"}, {nil},
+		Do("GET", "mykey", "myid").Str("<nil>"),
-	})
+		Do("GET", "mykey").Err("wrong number of arguments for 'get' command"),
+		Do("GET", "mykey", "myid", "hash").Err("wrong number of arguments for 'get' command"),
+		Do("GET", "mykey", "myid", "hash", "0").Err("invalid argument '0'"),
+		Do("GET", "mykey", "myid", "hash", "-1").Err("invalid argument '-1'"),
+		Do("GET", "mykey", "myid", "hash", "13").Err("invalid argument '13'"),
+		Do("SET", "mykey", "myid", "field", "hello", "world", "field", "hiya", 55, "point", 33, -112).OK(),
+		Do("GET", "mykey", "myid", "hash", "1").Str("9"),
+		Do("GET", "mykey", "myid", "point").Str("[33 -112]"),
+		Do("GET", "mykey", "myid", "bounds").Str("[[33 -112] [33 -112]]"),
+		Do("GET", "mykey", "myid", "object").Str(`{"type":"Point","coordinates":[-112,33]}`),
+		Do("GET", "mykey", "myid", "object").Str(`{"type":"Point","coordinates":[-112,33]}`),
+		Do("GET", "mykey", "myid", "withfields", "point").Str(`[[33 -112] [hello world hiya 55]]`),
+		Do("GET", "mykey", "myid", "joint").Err("wrong number of arguments for 'get' command"),
+		Do("GET", "mykey2", "myid").Str("<nil>"),
+		Do("GET", "mykey2", "myid").JSON().Err("key not found"),
+		Do("GET", "mykey", "myid2").Str("<nil>"),
+		Do("GET", "mykey", "myid2").JSON().Err("id not found"),
+		Do("GET", "mykey", "myid", "point").JSON().Str(`{"ok":true,"point":{"lat":33,"lon":-112}}`),
+		Do("GET", "mykey", "myid", "object").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[-112,33]}}`),
+		Do("GET", "mykey", "myid", "hash", "1").JSON().Str(`{"ok":true,"hash":"9"}`),
+		Do("GET", "mykey", "myid", "bounds").JSON().Str(`{"ok":true,"bounds":{"sw":{"lat":33,"lon":-112},"ne":{"lat":33,"lon":-112}}}`),
+		Do("SET", "mykey", "myid2", "point", 33, -112, 55).OK(),
+		Do("GET", "mykey", "myid2", "point").Str("[33 -112 55]"),
+		Do("GET", "mykey", "myid2", "point").JSON().Str(`{"ok":true,"point":{"lat":33,"lon":-112,"z":55}}`),
+		Do("GET", "mykey", "myid", "withfields").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[-112,33]},"fields":{"hello":"world","hiya":55}}`),
+	)
 }
 func keys_KEYS_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey11", "myid4", "STRING", "value"}, {"OK"},
+		Do("SET", "mykey11", "myid4", "STRING", "value").OK(),
-		{"SET", "mykey22", "myid2", "HASH", "9my5xp7"}, {"OK"},
+		Do("SET", "mykey22", "myid2", "HASH", "9my5xp7").OK(),
-		{"SET", "mykey22", "myid1", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`}, {"OK"},
+		Do("SET", "mykey22", "myid1", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(),
-		{"SET", "mykey11", "myid3", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`}, {"OK"},
+		Do("SET", "mykey11", "myid3", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(),
-		{"SET", "mykey42", "myid2", "HASH", "9my5xp7"}, {"OK"},
+		Do("SET", "mykey42", "myid2", "HASH", "9my5xp7").OK(),
-		{"SET", "mykey31", "myid4", "STRING", "value"}, {"OK"},
+		Do("SET", "mykey31", "myid4", "STRING", "value").OK(),
-		{"SET", "mykey310", "myid5", "STRING", "value"}, {"OK"},
+		Do("SET", "mykey310", "myid5", "STRING", "value").OK(),
-		{"KEYS", "*"}, {"[mykey11 mykey22 mykey31 mykey310 mykey42]"},
+		Do("KEYS", "*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"),
-		{"KEYS", "*key*"}, {"[mykey11 mykey22 mykey31 mykey310 mykey42]"},
+		Do("KEYS", "*key*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"),
-		{"KEYS", "mykey*"}, {"[mykey11 mykey22 mykey31 mykey310 mykey42]"},
+		Do("KEYS", "mykey*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"),
-		{"KEYS", "mykey4*"}, {"[mykey42]"},
+		Do("KEYS", "mykey4*").Str("[mykey42]"),
-		{"KEYS", "mykey*1"}, {"[mykey11 mykey31]"},
+		Do("KEYS", "mykey*1").Str("[mykey11 mykey31]"),
-		{"KEYS", "mykey*1*"}, {"[mykey11 mykey31 mykey310]"},
+		Do("KEYS", "mykey*1*").Str("[mykey11 mykey31 mykey310]"),
-		{"KEYS", "mykey*10"}, {"[mykey310]"},
+		Do("KEYS", "mykey*10").Str("[mykey310]"),
-		{"KEYS", "mykey*2"}, {"[mykey22 mykey42]"},
+		Do("KEYS", "mykey*2").Str("[mykey22 mykey42]"),
-		{"KEYS", "*2"}, {"[mykey22 mykey42]"},
+		Do("KEYS", "*2").Str("[mykey22 mykey42]"),
-		{"KEYS", "*1*"}, {"[mykey11 mykey31 mykey310]"},
+		Do("KEYS", "*1*").Str("[mykey11 mykey31 mykey310]"),
-		{"KEYS", "mykey"}, {"[]"},
+		Do("KEYS", "mykey").Str("[]"),
-		{"KEYS", "mykey31"}, {"[mykey31]"},
+		Do("KEYS", "mykey31").Str("[mykey31]"),
-		{"KEYS", "mykey[^3]*"}, {"[mykey11 mykey22 mykey42]"},
+		Do("KEYS", "mykey[^3]*").Str("[mykey11 mykey22 mykey42]"),
-	})
+		Do("KEYS").Err("wrong number of arguments for 'keys' command"),
+		Do("KEYS", "*").JSON().Str(`{"ok":true,"keys":["mykey11","mykey22","mykey31","mykey310","mykey42"]}`),
+	)
 }
 func keys_PERSIST_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid", "STRING", "value"}, {"OK"},
+		Do("SET", "mykey", "myid", "STRING", "value").OK(),
-		{"EXPIRE", "mykey", "myid", 2}, {1},
+		Do("EXPIRE", "mykey", "myid", 2).Str("1"),
-		{"PERSIST", "mykey", "myid"}, {1},
+		Do("PERSIST", "mykey", "myid").Str("1"),
-		{"PERSIST", "mykey", "myid"}, {0},
+		Do("PERSIST", "mykey", "myid").Str("0"),
-	})
+		Do("PERSIST", "mykey").Err("wrong number of arguments for 'persist' command"),
+		Do("PERSIST", "mykey2", "myid").Str("0"),
+		Do("PERSIST", "mykey2", "myid").JSON().Err("key not found"),
+		Do("PERSIST", "mykey", "myid2").Str("0"),
+		Do("PERSIST", "mykey", "myid2").JSON().Err("id not found"),
+		Do("EXPIRE", "mykey", "myid", 2).Str("1"),
+		Do("PERSIST", "mykey", "myid").JSON().OK(),
+	)
 }
 func keys_SET_test(mc *mockServer) error {
 	return mc.DoBatch(
-		"point", [][]interface{}{
+		// Section: point
-			{"SET", "mykey", "myid", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid", "POINT", 33, -115).OK(),
-			{"GET", "mykey", "myid", "POINT"}, {"[33 -115]"},
+		Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"),
-			{"GET", "mykey", "myid", "BOUNDS"}, {"[[33 -115] [33 -115]]"},
+		Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"),
-			{"GET", "mykey", "myid", "OBJECT"}, {`{"type":"Point","coordinates":[-115,33]}`},
+		Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`),
-			{"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"},
+		Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"),
-			{"DEL", "mykey", "myid"}, {"1"},
+		Do("DEL", "mykey", "myid").Str("1"),
-			{"GET", "mykey", "myid"}, {nil},
+		Do("GET", "mykey", "myid").Str("<nil>"),
-		},
+		Do("SET", "mykey", "myid", "point", "33", "-112", "99").OK(),
-		"object", [][]interface{}{
+
-			{"SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`}, {"OK"},
+		// Section: object
-			{"GET", "mykey", "myid", "POINT"}, {"[33 -115]"},
+		Do("SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`).OK(),
-			{"GET", "mykey", "myid", "BOUNDS"}, {"[[33 -115] [33 -115]]"},
+		Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"),
-			{"GET", "mykey", "myid", "OBJECT"}, {`{"type":"Point","coordinates":[-115,33]}`},
+		Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"),
-			{"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"},
+		Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`),
-			{"DEL", "mykey", "myid"}, {"1"},
+		Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"),
-			{"GET", "mykey", "myid"}, {nil},
+		Do("DEL", "mykey", "myid").Str("1"),
-		},
+		Do("GET", "mykey", "myid").Str("<nil>"),
-		"bounds", [][]interface{}{
+
-			{"SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115}, {"OK"},
+		// Section: bounds
-			{"GET", "mykey", "myid", "POINT"}, {"[33 -115]"},
+		Do("SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115).OK(),
-			{"GET", "mykey", "myid", "BOUNDS"}, {"[[33 -115] [33 -115]]"},
+		Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"),
-			{"GET", "mykey", "myid", "OBJECT"}, {`{"type":"Point","coordinates":[-115,33]}`},
+		Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"),
-			{"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"},
+		Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`),
-			{"DEL", "mykey", "myid"}, {"1"},
+		Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"),
-			{"GET", "mykey", "myid"}, {nil},
+		Do("DEL", "mykey", "myid").Str("1"),
-		},
+		Do("GET", "mykey", "myid").Str("<nil>"),
-		"hash", [][]interface{}{
+
-			{"SET", "mykey", "myid", "HASH", "9my5xp7"}, {"OK"},
+		// Section: hash
-			{"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"},
+		Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(),
-			{"DEL", "mykey", "myid"}, {"1"},
+		Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"),
-			{"GET", "mykey", "myid"}, {nil},
+		Do("DEL", "mykey", "myid").Str("1"),
-		},
+		Do("GET", "mykey", "myid").Str("<nil>"),
-		"field", [][]interface{}{
+		Do("SET", "mykey", "myid", "HASH", "9my5xp7").JSON().OK(),
-			{"SET", "mykey", "myid", "FIELD", "f1", 33, "FIELD", "a2", 44.5, "HASH", "9my5xp7"}, {"OK"},
+
-			{"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [a2 44.5 f1 33]]"},
+		// Section: field
-			{"FSET", "mykey", "myid", "f1", 0}, {1},
+		Do("SET", "mykey", "myid", "FIELD", "f1", 33, "FIELD", "a2", 44.5, "HASH", "9my5xp7").OK(),
-			{"FSET", "mykey", "myid", "f1", 0}, {0},
+		Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [a2 44.5 f1 33]]"),
-			{"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [a2 44.5]]"},
+		Do("FSET", "mykey", "myid", "f1", 0).Str("1"),
-			{"DEL", "mykey", "myid"}, {"1"},
+		Do("FSET", "mykey", "myid", "f1", 0).Str("0"),
-			{"GET", "mykey", "myid"}, {nil},
+		Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [a2 44.5]]"),
-		},
+		Do("DEL", "mykey", "myid").Str("1"),
-		"string", [][]interface{}{
+		Do("GET", "mykey", "myid").Str("<nil>"),
-			{"SET", "mykey", "myid", "STRING", "value"}, {"OK"},
+
-			{"GET", "mykey", "myid"}, {"value"},
+		// Section: string
-			{"SET", "mykey", "myid", "STRING", "value2"}, {"OK"},
+		Do("SET", "mykey", "myid", "STRING", "value").OK(),
-			{"GET", "mykey", "myid"}, {"value2"},
+		Do("GET", "mykey", "myid").Str("value"),
-			{"DEL", "mykey", "myid"}, {"1"},
+		Do("SET", "mykey", "myid", "STRING", "value2").OK(),
-			{"GET", "mykey", "myid"}, {nil},
+		Do("GET", "mykey", "myid").Str("value2"),
-		},
+		Do("DEL", "mykey", "myid").Str("1"),
+		Do("GET", "mykey", "myid").Str("<nil>"),
+
+		// Test error conditions
+		Do("CONFIG", "SET", "maxmemory", "1").OK(),
+		Do("SET", "mykey", "myid", "STRING", "value2").Err("OOM command not allowed when used memory > 'maxmemory'"),
+		Do("CONFIG", "SET", "maxmemory", "0").OK(),
+		Do("SET").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "FIELD", "f1").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "FIELD", "z", "1").Err("invalid argument 'z'"),
+		Do("SET", "mykey", "myid", "EX").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "EX", "yyy").Err("invalid argument 'yyy'"),
+		Do("SET", "mykey", "myid", "EX", "123").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "nx").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "nx", "xx").Err("invalid argument 'xx'"),
+		Do("SET", "mykey", "myid", "xx", "nx").Err("invalid argument 'nx'"),
+		Do("SET", "mykey", "myid", "string").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "point").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "point", "33").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "point", "33f", "-112").Err("invalid argument '33f'"),
+		Do("SET", "mykey", "myid", "point", "33", "-112f").Err("invalid argument '-112f'"),
+		Do("SET", "mykey", "myid", "point", "33", "-112f", "99").Err("invalid argument '-112f'"),
+		Do("SET", "mykey", "myid", "bounds").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "bounds", "fff", "1", "2", "3").Err("invalid argument 'fff'"),
+		Do("SET", "mykey", "myid", "hash").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "object").Err("wrong number of arguments for 'set' command"),
+		Do("SET", "mykey", "myid", "object", "asd").Err("invalid data"),
+		Do("SET", "mykey", "myid", "joint").Err("invalid argument 'joint'"),
+		Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").Err("<nil>"),
+		Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").JSON().Err("id not found"),
+		Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(),
+		Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").Err("<nil>"),
+		Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").OK(),
+		Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").OK(),
+		Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").Err("<nil>"),
+		Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").JSON().Err("id already exists"),
 	)
 }
 
 func keys_STATS_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"STATS", "mykey"}, {"[nil]"},
+		Do("STATS", "mykey").Str("[nil]"),
-		{"SET", "mykey", "myid", "STRING", "value"}, {"OK"},
+		Do("SET", "mykey", "myid", "STRING", "value").OK(),
-		{"STATS", "mykey"}, {"[[in_memory_size 9 num_objects 1 num_points 0 num_strings 1]]"},
+		Do("STATS", "mykey").Str("[[in_memory_size 9 num_objects 1 num_points 0 num_strings 1]]"),
-		{"SET", "mykey", "myid2", "STRING", "value"}, {"OK"},
+		Do("STATS", "mykey", "hello").JSON().Str(`{"ok":true,"stats":[{"in_memory_size":9,"num_objects":1,"num_points":0,"num_strings":1},null]}`),
-		{"STATS", "mykey"}, {"[[in_memory_size 19 num_objects 2 num_points 0 num_strings 2]]"},
+		Do("SET", "mykey", "myid2", "STRING", "value").OK(),
-		{"SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`}, {"OK"},
+		Do("STATS", "mykey").Str("[[in_memory_size 19 num_objects 2 num_points 0 num_strings 2]]"),
-		{"STATS", "mykey"}, {"[[in_memory_size 40 num_objects 3 num_points 1 num_strings 2]]"},
+		Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`).OK(),
-		{"DEL", "mykey", "myid"}, {1},
+		Do("STATS", "mykey").Str("[[in_memory_size 40 num_objects 3 num_points 1 num_strings 2]]"),
-		{"STATS", "mykey"}, {"[[in_memory_size 31 num_objects 2 num_points 1 num_strings 1]]"},
+		Do("DEL", "mykey", "myid").Str("1"),
-		{"DEL", "mykey", "myid3"}, {1},
+		Do("STATS", "mykey").Str("[[in_memory_size 31 num_objects 2 num_points 1 num_strings 1]]"),
-		{"STATS", "mykey"}, {"[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1]]"},
+		Do("DEL", "mykey", "myid3").Str("1"),
-		{"STATS", "mykey", "mykey2"}, {"[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1] nil]"},
+		Do("STATS", "mykey").Str("[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1]]"),
-		{"DEL", "mykey", "myid2"}, {1},
+		Do("STATS", "mykey", "mykey2").Str("[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1] nil]"),
-		{"STATS", "mykey"}, {"[nil]"},
+		Do("DEL", "mykey", "myid2").Str("1"),
-		{"STATS", "mykey", "mykey2"}, {"[nil nil]"},
+		Do("STATS", "mykey").Str("[nil]"),
-	})
+		Do("STATS", "mykey", "mykey2").Str("[nil nil]"),
+		Do("STATS", "mykey").Str("[nil]"),
+		Do("STATS").Err(`wrong number of arguments for 'stats' command`),
+	)
 }
 func keys_TTL_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid", "STRING", "value"}, {"OK"},
+		Do("SET", "mykey", "myid", "STRING", "value").OK(),
-		{"EXPIRE", "mykey", "myid", 2}, {1},
+		Do("EXPIRE", "mykey", "myid", 2).Str("1"),
-		{time.Second / 4}, {}, // sleep
+		Do("EXPIRE", "mykey", "myid", 2).JSON().OK(),
-		{"TTL", "mykey", "myid"}, {1},
+		Sleep(time.Millisecond*10),
-	})
+		Do("TTL", "mykey", "myid").Str("1"),
+		Do("EXPIRE", "mykey", "myid", 1).Str("1"),
+		Sleep(time.Millisecond*10),
+		Do("TTL", "mykey", "myid").Str("0"),
+		Do("TTL", "mykey", "myid").JSON().Str(`{"ok":true,"ttl":0}`),
+		Do("TTL", "mykey2", "myid").Str("-2"),
+		Do("TTL", "mykey", "myid2").Str("-2"),
+		Do("TTL", "mykey").Err("wrong number of arguments for 'ttl' command"),
+		Do("SET", "mykey", "myid", "STRING", "value").OK(),
+		Do("TTL", "mykey", "myid").Str("-1"),
+		Do("TTL", "mykey2", "myid").JSON().Err("key not found"),
+		Do("TTL", "mykey", "myid2").JSON().Err("id not found"),
+	)
 }
 
-type PSAUX struct {
-	User    string
-	PID     int
-	CPU     float64
-	Mem     float64
-	VSZ     int
-	RSS     int
-	TTY     string
-	Stat    string
-	Start   string
-	Time    string
-	Command string
-}
-
-func atoi(s string) int {
-	n, _ := strconv.ParseInt(s, 10, 64)
-	return int(n)
-}
-func atof(s string) float64 {
-	n, _ := strconv.ParseFloat(s, 64)
-	return float64(n)
-}
-func psaux(pid int) PSAUX {
-	var res []byte
-	res, err := exec.Command("ps", "aux").CombinedOutput()
-	if err != nil {
-		return PSAUX{}
-	}
-	pids := strconv.FormatInt(int64(pid), 10)
-	for _, line := range strings.Split(string(res), "\n") {
-		var words []string
-		for _, word := range strings.Split(line, " ") {
-			if word != "" {
-				words = append(words, word)
-			}
-			if len(words) > 11 {
-				if words[1] == pids {
-					return PSAUX{
-						User:    words[0],
-						PID:     atoi(words[1]),
-						CPU:     atof(words[2]),
-						Mem:     atof(words[3]),
-						VSZ:     atoi(words[4]),
-						RSS:     atoi(words[5]),
-						TTY:     words[6],
-						Stat:    words[7],
-						Start:   words[8],
-						Time:    words[9],
-						Command: words[10],
-					}
-				}
-			}
-		}
-	}
-	return PSAUX{}
-}
 func keys_SET_EX_test(mc *mockServer) (err error) {
 	rand.Seed(time.Now().UnixNano())
 
@@ -364,52 +453,195 @@ func keys_FIELDS_test(mc *mockServer) error {
 }
 
 func keys_PDEL_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid1a", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid1a", "POINT", 33, -115).OK(),
-		{"SET", "mykey", "myid1b", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid1b", "POINT", 33, -115).OK(),
-		{"SET", "mykey", "myid2a", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid2a", "POINT", 33, -115).OK(),
-		{"SET", "mykey", "myid2b", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid2b", "POINT", 33, -115).OK(),
-		{"SET", "mykey", "myid3a", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid3a", "POINT", 33, -115).OK(),
-		{"SET", "mykey", "myid3b", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid3b", "POINT", 33, -115).OK(),
-		{"SET", "mykey", "myid4a", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid4a", "POINT", 33, -115).OK(),
-		{"SET", "mykey", "myid4b", "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid4b", "POINT", 33, -115).OK(),
-		{"PDEL", "mykeyNA", "*"}, {0},
+		Do("PDEL", "mykey").Err("wrong number of arguments for 'pdel' command"),
-		{"PDEL", "mykey", "myid1a"}, {1},
+		Do("PDEL", "mykeyNA", "*").Str("0"),
-		{"PDEL", "mykey", "myid1a"}, {0},
+		Do("PDEL", "mykey", "myid1a").Str("1"),
-		{"PDEL", "mykey", "myid1*"}, {1},
+		Do("PDEL", "mykey", "myid1a").Str("0"),
-		{"PDEL", "mykey", "myid2*"}, {2},
+		Do("PDEL", "mykey", "myid1*").Str("1"),
-		{"PDEL", "mykey", "*b"}, {2},
+		Do("PDEL", "mykey", "myid2*").Str("2"),
-		{"PDEL", "mykey", "*"}, {2},
+		Do("PDEL", "mykey", "*b").Str("2"),
-		{"PDEL", "mykey", "*"}, {0},
+		Do("PDEL", "mykey", "*").Str("2"),
-	})
+		Do("PDEL", "mykey", "*").Str("0"),
+		Do("SET", "mykey", "myid1a", "POINT", 33, -115).OK(),
+		Do("SET", "mykey", "myid1b", "POINT", 33, -115).OK(),
+		Do("SET", "mykey", "myid2a", "POINT", 33, -115).OK(),
+		Do("SET", "mykey", "myid2b", "POINT", 33, -115).OK(),
+		Do("SET", "mykey", "myid3a", "POINT", 33, -115).OK(),
+		Do("PDEL", "mykey", "*").JSON().OK(),
+	)
 }
 
 func keys_WHEREIN_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115).OK(),
-		{"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`},
+		Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`),
-		{"WITHIN", "mykey", "WHEREIN", "a", "a", 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument 'a'"},
+		Do("WITHIN", "mykey", "WHEREIN", "a", "a", 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument 'a'"),
-		{"WITHIN", "mykey", "WHEREIN", "a", 1, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument '1'"},
+		Do("WITHIN", "mykey", "WHEREIN", "a", 1, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument '1'"),
-		{"WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"[0 []]"},
+		Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str("[0 []]"),
-		{"WITHIN", "mykey", "WHEREIN", "a", 4, 0, "a", 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`},
+		Do("WITHIN", "mykey", "WHEREIN", "a", 4, 0, "a", 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`),
-		{"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"},
+		Do("SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115).OK(),
-		{"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"},
+		Do("SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02).OK(),
-		{"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {
+		Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`),
-			`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`},
 		// zero value should not match 1 and 2
-		{"SET", "mykey", "myid_a0", "FIELD", "a", 0, "POINT", 33, -115.02}, {"OK"},
+		Do("SET", "mykey", "myid_a0", "FIELD", "a", 0, "POINT", 33, -115.02).OK(),
-		{"WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`},
+		Do("WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`),
-	})
+	)
 }
 
 func keys_WHEREEVAL_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115}, {"OK"},
+		Do("SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115).OK(),
-		{"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`},
+		Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`),
-		{"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", "a", 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument 'a'"},
+		Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", "a", 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument 'a'"),
-		{"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, 4, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument '4'"},
+		Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, 4, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument '4'"),
-		{"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"},
+		Do("SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115).OK(),
-		{"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"},
+		Do("SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02).OK(),
-		{"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`},
+		Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`),
-	})
+	)
+}
+
+func keys_TYPE_test(mc *mockServer) error {
+	return mc.DoBatch(
+		Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(),
+		Do("TYPE", "mykey").Str("hash"),
+		Do("TYPE", "mykey", "hello").Err("wrong number of arguments for 'type' command"),
+		Do("TYPE", "mykey2").Str("none"),
+		Do("TYPE", "mykey2").JSON().Err("key not found"),
+		Do("TYPE", "mykey").JSON().Str(`{"ok":true,"type":"hash"}`),
+	)
+}
+
+func keys_FLUSHDB_test(mc *mockServer) error {
+	return mc.DoBatch(
+		Do("SET", "mykey1", "myid1", "POINT", 33, -115).OK(),
+		Do("SET", "mykey2", "myid1", "POINT", 33, -115).OK(),
+		Do("SETCHAN", "mychan", "INTERSECTS", "mykey1", "BOUNDS", 10, 10, 10, 10).Str("1"),
+		Do("KEYS", "*").Str("[mykey1 mykey2]"),
+		Do("CHANS", "*").JSON().Func(func(s string) error {
+			if gjson.Get(s, "chans.#").Int() != 1 {
+				return fmt.Errorf("expected '%d', got '%d'", 1, gjson.Get(s, "chans.#").Int())
+			}
+			return nil
+		}),
+		Do("FLUSHDB", "arg2").Err("wrong number of arguments for 'flushdb' command"),
+		Do("FLUSHDB").OK(),
+		Do("KEYS", "*").Str("[]"),
+		Do("CHANS", "*").Str("[]"),
+		Do("SET", "mykey1", "myid1", "POINT", 33, -115).OK(),
+		Do("SET", "mykey2", "myid1", "POINT", 33, -115).OK(),
+		Do("SETCHAN", "mychan", "INTERSECTS", "mykey1", "BOUNDS", 10, 10, 10, 10).Str("1"),
+		Do("FLUSHDB").JSON().OK(),
+	)
+}
+
+func keys_HEALTHZ_test(mc *mockServer) error {
+	return mc.DoBatch(
+		Do("HEALTHZ").OK(),
+		Do("HEALTHZ").JSON().OK(),
+		Do("HEALTHZ", "arg").Err(`wrong number of arguments for 'healthz' command`),
+	)
+}
+
+func keys_SERVER_test(mc *mockServer) error {
+	return mc.DoBatch(
+		Do("SERVER").Func(func(s string) error {
+			valid := strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") &&
+				strings.Contains(s, "cpus") && strings.Contains(s, "mem_alloc")
+			if !valid {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+		Do("SERVER").JSON().Func(func(s string) error {
+			if !gjson.Get(s, "ok").Bool() {
+				return errors.New("not ok")
+			}
+			valid := gjson.Get(s, "stats.cpus").Exists() &&
+				gjson.Get(s, "stats.mem_alloc").Exists()
+			if !valid {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+		Do("SERVER", "ext").Func(func(s string) error {
+			valid := strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") &&
+				strings.Contains(s, "sys_cpus") &&
+				strings.Contains(s, "tile38_connected_clients")
+
+			if !valid {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+		Do("SERVER", "ext").JSON().Func(func(s string) error {
+			if !gjson.Get(s, "ok").Bool() {
+				return errors.New("not ok")
+			}
+			valid := gjson.Get(s, "stats.sys_cpus").Exists() &&
+				gjson.Get(s, "stats.tile38_connected_clients").Exists()
+			if !valid {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+		Do("SERVER", "ett").Err(`invalid argument 'ett'`),
+		Do("SERVER", "ett").JSON().Err(`invalid argument 'ett'`),
+	)
+}
+
+func keys_INFO_test(mc *mockServer) error {
+	return mc.DoBatch(
+		Do("INFO").Func(func(s string) error {
+			if !strings.Contains(s, "# Clients") ||
+				!strings.Contains(s, "# Stats") {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+		Do("INFO", "all").Func(func(s string) error {
+			if !strings.Contains(s, "# Clients") ||
+				!strings.Contains(s, "# Stats") {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+		Do("INFO", "default").Func(func(s string) error {
+			if !strings.Contains(s, "# Clients") ||
+				!strings.Contains(s, "# Stats") {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+		Do("INFO", "cpu").Func(func(s string) error {
+			if !strings.Contains(s, "# CPU") ||
+				strings.Contains(s, "# Clients") ||
+				strings.Contains(s, "# Stats") {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+		Do("INFO", "cpu", "clients").Func(func(s string) error {
+			if !strings.Contains(s, "# CPU") ||
+				!strings.Contains(s, "# Clients") ||
+				strings.Contains(s, "# Stats") {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+		Do("INFO").JSON().Func(func(s string) error {
+			if gjson.Get(s, "info.tile38_version").String() == "" {
+				return errors.New("looks invalid")
+			}
+			return nil
+		}),
+	)
 }

tests/metrics_test.go

@@ -1,42 +1,55 @@
 package tests
 
 import (
-	"io/ioutil"
+	"fmt"
+	"io"
 	"net/http"
 	"strings"
-	"testing"
 )
 
-func downloadURLWithStatusCode(t *testing.T, u string) (int, string) {
+func subTestMetrics(g *testGroup) {
-	resp, err := http.Get(u)
+	g.regSubTest("basic", metrics_basic_test)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return resp.StatusCode, string(body)
 }
 
-func subTestMetrics(t *testing.T, mc *mockServer) {
+func downloadURLWithStatusCode(u string) (int, string, error) {
+	resp, err := http.Get(u)
+	if err != nil {
+		return 0, "", err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return 0, "", err
+	}
+	return resp.StatusCode, string(body), nil
+}
+
+func metrics_basic_test(mc *mockServer) error {
+
+	maddr := fmt.Sprintf("http://127.0.0.1:%d/", mc.metricsPort())
+
 	mc.Do("SET", "metrics_test_1", "1", "FIELD", "foo", 5.5, "POINT", 5, 5)
 	mc.Do("SET", "metrics_test_2", "2", "FIELD", "foo", 19.19, "POINT", 19, 19)
 	mc.Do("SET", "metrics_test_2", "3", "FIELD", "foo", 19.19, "POINT", 19, 19)
 	mc.Do("SET", "metrics_test_2", "truck1:driver", "STRING", "John Denton")
 
-	status, index := downloadURLWithStatusCode(t, "http://127.0.0.1:4321/")
+	status, index, err := downloadURLWithStatusCode(maddr)
+	if err != nil {
+		return err
+	}
 	if status != 200 {
-		t.Fatalf("Expected status code 200, got: %d", status)
+		return fmt.Errorf("Expected status code 200, got: %d", status)
 	}
 	if !strings.Contains(index, "<a href") {
-		t.Fatalf("missing link on index page")
+		return fmt.Errorf("missing link on index page")
 	}
 
-	status, metrics := downloadURLWithStatusCode(t, "http://127.0.0.1:4321/metrics")
+	status, metrics, err := downloadURLWithStatusCode(maddr + "metrics")
+	if err != nil {
+		return err
+	}
 	if status != 200 {
-		t.Fatalf("Expected status code 200, got: %d", status)
+		return fmt.Errorf("Expected status code 200, got: %d", status)
 	}
 	for _, want := range []string{
 		`tile38_connected_clients`,
@@ -50,7 +63,8 @@ func subTestMetrics(t *testing.T, mc *mockServer) {
 		`role="leader"`,
 	} {
 		if !strings.Contains(metrics, want) {
-			t.Fatalf("wanted metric: %s, got: %s", want, metrics)
+			return fmt.Errorf("wanted metric: %s, got: %s", want, metrics)
 		}
 	}
+	return nil
 }

tests/mock_io_test.go

@@ -0,0 +1,150 @@
+package tests
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/tidwall/gjson"
+)
+
+type IO struct {
+	args  []any
+	json  bool
+	out   any
+	sleep bool
+	dur   time.Duration
+	cfile string
+	cln   int
+}
+
+func Do(args ...any) *IO {
+	_, cfile, cln, _ := runtime.Caller(1)
+	return &IO{args: args, cfile: cfile, cln: cln}
+}
+func (cmd *IO) JSON() *IO {
+	cmd.json = true
+	return cmd
+}
+func (cmd *IO) Str(s string) *IO {
+	cmd.out = s
+	return cmd
+}
+func (cmd *IO) Func(fn func(s string) error) *IO {
+	cmd.out = func(s string) error {
+		if cmd.json {
+			if !gjson.Valid(s) {
+				return errors.New("invalid json")
+			}
+		}
+		return fn(s)
+	}
+	return cmd
+}
+
+func (cmd *IO) OK() *IO {
+	return cmd.Func(func(s string) error {
+		if cmd.json {
+			if gjson.Get(s, "ok").Type != gjson.True {
+				return errors.New("not ok")
+			}
+		} else if s != "OK" {
+			return errors.New("not ok")
+		}
+		return nil
+	})
+}
+
+func (cmd *IO) Err(msg string) *IO {
+	return cmd.Func(func(s string) error {
+		if cmd.json {
+			if gjson.Get(s, "ok").Type != gjson.False {
+				return errors.New("ok=true")
+			}
+			if gjson.Get(s, "err").String() != msg {
+				return fmt.Errorf("expected '%s', got '%s'",
+					msg, gjson.Get(s, "err").String())
+			}
+		} else {
+			s = strings.TrimPrefix(s, "ERR ")
+			if s != msg {
+				return fmt.Errorf("expected '%s', got '%s'", msg, s)
+			}
+		}
+		return nil
+	})
+}
+
+func Sleep(duration time.Duration) *IO {
+	return &IO{sleep: true, dur: duration}
+}
+
+func (cmd *IO) deepError(index int, err error) error {
+	frag := "(?)"
+	bdata, _ := os.ReadFile(cmd.cfile)
+	data := string(bdata)
+	ln := 1
+	for i := 0; i < len(data); i++ {
+		if data[i] == '\n' {
+			ln++
+			if ln == cmd.cln {
+				data = data[i+1:]
+				i = strings.IndexByte(data, '(')
+				if i != -1 {
+					j := strings.IndexByte(data[i:], ')')
+					if j != -1 {
+						frag = string(data[i : j+i+1])
+					}
+				}
+				break
+			}
+		}
+	}
+	fsig := fmt.Sprintf("%s:%d", filepath.Base(cmd.cfile), cmd.cln)
+	emsg := err.Error()
+	if strings.HasPrefix(emsg, "expected ") &&
+		strings.Contains(emsg, ", got ") {
+		emsg = "" +
+			"  EXPECTED: " + strings.Split(emsg, ", got ")[0][9:] + "\n" +
+			"       GOT: " +
+			strings.Split(emsg, ", got ")[1]
+	} else {
+		emsg = "" +
+			"     ERROR: " + emsg
+	}
+	return fmt.Errorf("\n%s: entry[%d]\n   COMMAND: %s\n%s",
+		fsig, index+1, frag, emsg)
+}
+
+func (mc *mockServer) doIOTest(index int, cmd *IO) error {
+	if cmd.sleep {
+		time.Sleep(cmd.dur)
+		return nil
+	}
+	// switch json mode if desired
+	if cmd.json {
+		if !mc.ioJSON {
+			if _, err := mc.Do("OUTPUT", "json"); err != nil {
+				return err
+			}
+			mc.ioJSON = true
+		}
+	} else {
+		if mc.ioJSON {
+			if _, err := mc.Do("OUTPUT", "resp"); err != nil {
+				return err
+			}
+			mc.ioJSON = false
+		}
+	}
+
+	err := mc.DoExpect(cmd.out, cmd.args[0].(string), cmd.args[1:]...)
+	if err != nil {
+		return cmd.deepError(index, err)
+	}
+	return nil
+}

tests/mock_test.go

@@ -3,18 +3,18 @@ package tests
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
-	"log"
 	"math/rand"
+	"net"
 	"os"
-	"strconv"
+	"path/filepath"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/gomodule/redigo/redis"
 	"github.com/tidwall/sjson"
-	"github.com/tidwall/tile38/core"
+	"github.com/tidwall/tile38/internal/log"
-	tlog "github.com/tidwall/tile38/internal/log"
 	"github.com/tidwall/tile38/internal/server"
 )
 
@@ -24,7 +24,7 @@ func mockCleanup(silent bool) {
 	if !silent {
 		fmt.Printf("Cleanup: may take some time... ")
 	}
-	files, _ := ioutil.ReadDir(".")
+	files, _ := os.ReadDir(".")
 	for _, file := range files {
 		if strings.HasPrefix(file.Name(), "data-mock-") {
 			os.RemoveAll(file.Name())
@@ -36,51 +36,115 @@ func mockCleanup(silent bool) {
 }
 
 type mockServer struct {
-	port int
+	closed   bool
-	//join string
+	port     int
-	//n    *finn.Node
+	mport    int
-	//m    *Machine
+	conn     redis.Conn
-	conn redis.Conn
+	ioJSON   bool
+	dir      string
+	shutdown chan bool
 }
 
-func mockOpenServer(silent bool) (*mockServer, error) {
+func (mc *mockServer) readAOF() ([]byte, error) {
-	rand.Seed(time.Now().UnixNano())
+	return os.ReadFile(filepath.Join(mc.dir, "appendonly.aof"))
-	port := rand.Int()%20000 + 20000
+}
-	dir := fmt.Sprintf("data-mock-%d", port)
+
-	if !silent {
+func (mc *mockServer) metricsPort() int {
-		fmt.Printf("Starting test server at port %d\n", port)
+	return mc.mport
+}
+
+type MockServerOptions struct {
+	AOFFileName string
+	AOFData     []byte
+	Silent      bool
+	Metrics     bool
+}
+
+var nextPort int32 = 10000
+
+func getNextPort() int {
+	// choose a valid port between 10000-50000
+	for {
+		port := int(atomic.AddInt32(&nextPort, 1))
+		ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+		if err == nil {
+			ln.Close()
+			return port
+		}
 	}
-	logOutput := ioutil.Discard
+}
+
+func mockOpenServer(opts MockServerOptions) (*mockServer, error) {
+
+	logOutput := io.Discard
 	if os.Getenv("PRINTLOG") == "1" {
 		logOutput = os.Stderr
+		log.SetLevel(3)
 	}
-	core.DevMode = true
+	log.SetOutput(logOutput)
-	s := &mockServer{port: port}
+
-	tlog.SetOutput(logOutput)
+	rand.Seed(time.Now().UnixNano())
-	go func() {
+	port := getNextPort()
-		opts := server.Options{
+	dir := fmt.Sprintf("data-mock-%d", port)
-			Host:        "localhost",
+	if !opts.Silent {
-			Port:        port,
+		fmt.Printf("Starting test server at port %d\n", port)
-			Dir:         dir,
+	}
-			UseHTTP:     true,
+	if len(opts.AOFData) > 0 {
-			MetricsAddr: ":4321",
+		if opts.AOFFileName == "" {
+			opts.AOFFileName = "appendonly.aof"
 		}
-		if err := server.Serve(opts); err != nil {
+		if err := os.MkdirAll(dir, 0777); err != nil {
-			log.Fatal(err)
+			return nil, err
+		}
+		err := os.WriteFile(filepath.Join(dir, opts.AOFFileName),
+			opts.AOFData, 0666)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	shutdown := make(chan bool)
+	s := &mockServer{port: port, dir: dir, shutdown: shutdown}
+	if opts.Metrics {
+		s.mport = getNextPort()
+	}
+	var ferrt int32 // atomic flag for when ferr has been set
+	var ferr error  // ferr for when the server fails to start
+	go func() {
+		sopts := server.Options{
+			Host:              "localhost",
+			Port:              port,
+			Dir:               dir,
+			UseHTTP:           true,
+			DevMode:           true,
+			AppendOnly:        true,
+			Shutdown:          shutdown,
+			ShowDebugMessages: true,
+		}
+		if opts.Metrics {
+			sopts.MetricsAddr = fmt.Sprintf(":%d", s.mport)
+		}
+		err := server.Serve(sopts)
+		if err != nil {
+			ferr = err
+			atomic.StoreInt32(&ferrt, 1)
 		}
 	}()
-	if err := s.waitForStartup(); err != nil {
+	if err := s.waitForStartup(&ferr, &ferrt); err != nil {
 		s.Close()
 		return nil, err
 	}
 	return s, nil
 }
 
-func (s *mockServer) waitForStartup() error {
+func (s *mockServer) waitForStartup(ferr *error, ferrt *int32) error {
 	var lerr error
 	start := time.Now()
 	for {
-		if time.Now().Sub(start) > time.Second*5 {
+		if atomic.LoadInt32(ferrt) != 0 {
+			return *ferr
+		}
+		if time.Since(start) > time.Second*5 {
 			if lerr != nil {
 				return lerr
 			}
@@ -106,9 +170,17 @@ func (s *mockServer) waitForStartup() error {
 }
 
 func (mc *mockServer) Close() {
+	if mc == nil || mc.closed {
+		return
+	}
+	mc.closed = true
+	mc.shutdown <- true
 	if mc.conn != nil {
 		mc.conn.Close()
 	}
+	if mc.dir != "" {
+		os.RemoveAll(mc.dir)
+	}
 }
 
 func (mc *mockServer) ResetConn() {
@@ -159,7 +231,28 @@ func (s *mockServer) Do(commandName string, args ...interface{}) (interface{}, e
 	return resps[0], nil
 }
 
-func (mc *mockServer) DoBatch(commands ...interface{}) error { //[][]interface{}) error {
+func (mc *mockServer) DoBatch(commands ...interface{}) error {
+	// Probe for I/O tests
+	if len(commands) > 0 {
+		if _, ok := commands[0].(*IO); ok {
+			var cmds []*IO
+			// If the first is an I/O test then all must be
+			for _, cmd := range commands {
+				if cmd, ok := cmd.(*IO); ok {
+					cmds = append(cmds, cmd)
+				} else {
+					return errors.New("DoBatch cannot mix I/O tests with other kinds")
+				}
+			}
+			for i, cmd := range cmds {
+				if err := mc.doIOTest(i, cmd); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+	}
+
 	var tag string
 	for _, commands := range commands {
 		switch commands := commands.(type) {
@@ -181,6 +274,10 @@ func (mc *mockServer) DoBatch(commands ...interface{}) error { //[][]interface{}
 				}
 			}
 			tag = ""
+		case *IO:
+			return errors.New("DoBatch cannot mix I/O tests with other kinds")
+		default:
+			return fmt.Errorf("Unknown command input")
 		}
 	}
 	return nil
@@ -281,27 +378,3 @@ func (mc *mockServer) DoExpect(expect interface{}, commandName string, args ...i
 	}
 	return nil
 }
-func round(v float64, decimals int) float64 {
-	var pow float64 = 1
-	for i := 0; i < decimals; i++ {
-		pow *= 10
-	}
-	return float64(int((v*pow)+0.5)) / pow
-}
-
-func exfloat(v float64, decimals int) func(v interface{}) (resp, expect interface{}) {
-	ex := round(v, decimals)
-	return func(v interface{}) (resp, expect interface{}) {
-		var s string
-		if b, ok := v.([]uint8); ok {
-			s = string(b)
-		} else {
-			s = fmt.Sprintf("%v", v)
-		}
-		n, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return v, ex
-		}
-		return round(n, decimals), ex
-	}
-}

tests/monitor_test.go

@@ -0,0 +1,77 @@
+package tests
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+func subTestMonitor(g *testGroup) {
+	g.regSubTest("monitor", follower_monitor_test)
+}
+
+func follower_monitor_test(mc *mockServer) error {
+	N := 1000
+	ch := make(chan error)
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		ch <- func() error {
+			conn, err := redis.Dial("tcp", fmt.Sprintf("localhost:%d", mc.port))
+			if err != nil {
+				wg.Done()
+				return err
+			}
+			defer conn.Close()
+			s, err := redis.String(conn.Do("MONITOR"))
+			if err != nil {
+				wg.Done()
+				return err
+			}
+			if s != "OK" {
+				wg.Done()
+				return fmt.Errorf("expected '%s', got '%s'", "OK", s)
+			}
+			wg.Done()
+
+			for i := 0; i < N; i++ {
+				s, err := redis.String(conn.Receive())
+				if err != nil {
+					return err
+				}
+				ex := fmt.Sprintf(`"mykey" "%d"`, i)
+				if !strings.Contains(s, ex) {
+					return fmt.Errorf("expected '%s', got '%s'", ex, s)
+				}
+			}
+			return nil
+		}()
+	}()
+
+	wg.Wait()
+
+	conn, err := redis.Dial("tcp", fmt.Sprintf("localhost:%d", mc.port))
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	for i := 0; i < N; i++ {
+		s, err := redis.String(conn.Do("SET", "mykey", i, "POINT", 10, 10))
+		if err != nil {
+			return err
+		}
+		if s != "OK" {
+			return fmt.Errorf("expected '%s', got '%s'", "OK", s)
+		}
+	}
+
+	err = <-ch
+	if err != nil {
+		err = fmt.Errorf("monitor client: %w", err)
+	}
+
+	return err
+}

tests/scripts_test.go

@@ -3,14 +3,13 @@ package tests
 import (
 	"fmt"
 	"strings"
-	"testing"
 )
 
-func subTestScripts(t *testing.T, mc *mockServer) {
+func subTestScripts(g *testGroup) {
-	runStep(t, mc, "BASIC", scripts_BASIC_test)
+	g.regSubTest("BASIC", scripts_BASIC_test)
-	runStep(t, mc, "ATOMIC", scripts_ATOMIC_test)
+	g.regSubTest("ATOMIC", scripts_ATOMIC_test)
-	runStep(t, mc, "READONLY", scripts_READONLY_test)
+	g.regSubTest("READONLY", scripts_READONLY_test)
-	runStep(t, mc, "NONATOMIC", scripts_NONATOMIC_test)
+	g.regSubTest("NONATOMIC", scripts_NONATOMIC_test)
 }
 
 func scripts_BASIC_test(mc *mockServer) error {

tests/stats_test.go

@@ -2,13 +2,12 @@ package tests
 
 import (
 	"errors"
-	"testing"
 
 	"github.com/tidwall/gjson"
 )
 
-func subTestInfo(t *testing.T, mc *mockServer) {
+func subTestInfo(g *testGroup) {
-	runStep(t, mc, "valid json", info_valid_json_test)
+	g.regSubTest("valid json", info_valid_json_test)
 }
 
 func info_valid_json_test(mc *mockServer) error {

tests/testcmd_test.go

@@ -1,15 +1,11 @@
 package tests
 
-import (
+func subTestTestCmd(g *testGroup) {
-	"testing"
+	g.regSubTest("WITHIN", testcmd_WITHIN_test)
-)
+	g.regSubTest("INTERSECTS", testcmd_INTERSECTS_test)
-
+	g.regSubTest("INTERSECTS_CLIP", testcmd_INTERSECTS_CLIP_test)
-func subTestTestCmd(t *testing.T, mc *mockServer) {
+	g.regSubTest("ExpressionErrors", testcmd_expressionErrors_test)
-	runStep(t, mc, "WITHIN", testcmd_WITHIN_test)
+	g.regSubTest("Expressions", testcmd_expression_test)
-	runStep(t, mc, "INTERSECTS", testcmd_INTERSECTS_test)
-	runStep(t, mc, "INTERSECTS_CLIP", testcmd_INTERSECTS_CLIP_test)
-	runStep(t, mc, "ExpressionErrors", testcmd_expressionErrors_test)
-	runStep(t, mc, "Expressions", testcmd_expression_test)
 }
 
 func testcmd_WITHIN_test(mc *mockServer) error {
@@ -29,30 +25,100 @@ func testcmd_WITHIN_test(mc *mockServer) error {
 	poly9 := `{"type":"Polygon","coordinates":[[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}`
 	poly10 := `{"type":"Polygon","coordinates":[[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}`
 
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"},
+		Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(),
-		{"SET", "mykey", "point2", "POINT", 37.7335, -122.44121}, {"OK"},
+		Do("SET", "mykey", "point2", "POINT", 37.7335, -122.44121).OK(),
-		{"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"},
+		Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(),
-		{"SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"},
+		Do("SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`).OK(),
-		{"SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"},
+		Do("SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`).OK(),
-		{"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"},
+		Do("SET", "mykey", "point6", "POINT", -5, 5).OK(),
-		{"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"},
+		Do("SET", "mykey", "point7", "POINT", 33, 21).OK(),
-		{"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"},
+		Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(),
 
-		{"TEST", "GET", "mykey", "point1", "WITHIN", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"1"},
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").Str("1"),
-		{"TEST", "GET", "mykey", "line3", "WITHIN", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "line3", "WITHIN", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "poly4", "WITHIN", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "poly4", "WITHIN", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "multipoly5", "WITHIN", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "multipoly5", "WITHIN", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "poly8", "WITHIN", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "poly8", "WITHIN", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"1"},
+		Do("TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").Str("1"),
+		Do("TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").JSON().Str(`{"ok":true,"result":true}`),
 
-		{"TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly}, {"0"},
+		Do("TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly).Str("0"),
-		{"TEST", "GET", "mykey", "point7", "WITHIN", "OBJECT", poly}, {"0"},
+		Do("TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly).JSON().Str(`{"ok":true,"result":false}`),
+		Do("TEST", "GET", "mykey", "point7", "WITHIN", "OBJECT", poly).Str("0"),
 
-		{"TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8}, {"1"},
+		Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8).Str("1"),
-		{"TEST", "OBJECT", poly10, "WITHIN", "OBJECT", poly8}, {"0"},
+		Do("TEST", "OBJECT", poly10, "WITHIN", "OBJECT", poly8).Str("0"),
-	})
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "ff").Err("invalid argument 'ff'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "ee", "ff").Err("invalid argument 'ee'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "dd", "ee", "ff").Err("invalid argument 'dd'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "cc", "dd", "ee", "ff").Err("invalid argument 'cc'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "bb", "cc", "dd", "ee", "ff").Err("invalid argument 'bb'"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "1", "1").Err("equal bearings (1 == 1), use CIRCLE instead"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "10000").Str("1"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "10000", "10000").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE").Err("wrong number of arguments for 'test' command"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "cc").Err("invalid argument 'cc'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "bb", "cc").Err("invalid argument 'bb'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "aa", "bb", "cc").Err("invalid argument 'aa'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "-10000").Err("invalid argument '-10000'"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash", "123").Str("0"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash", "123", "asdf").Err("wrong number of arguments for 'test' command"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "123").Str("0"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "pqowie").Err("invalid argument 'pqowie'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "123", "asdf").Err("wrong number of arguments for 'test' command"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2", "3").Str("0"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2", "cc").Err("invalid argument 'cc'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "bb", "cc").Err("invalid argument 'bb'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "aa", "bb", "cc").Err("invalid argument 'aa'"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "2").Str("0"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "point").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "2", "3").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "bb").Err("invalid argument 'bb'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "aa", "bb").Err("invalid argument 'aa'"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN").Err("wrong number of arguments for 'test' command"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "4").Str("0"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "4", "5").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds").Err("wrong number of arguments for 'test' command"),
+
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "dd").Err("invalid argument 'dd'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "cc", "dd").Err("invalid argument 'cc'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "bb", "cc", "dd").Err("invalid argument 'bb'"),
+		Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "aa", "bb", "cc", "dd").Err("invalid argument 'aa'"),
+
+		Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey").Err("wrong number of arguments for 'test' command"),
+		Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey", "point6").Str("1"),
+		Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey__", "point6").Err("key not found"),
+		Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey", "point6__").Err("id not found"),
+	)
 }
 
 func testcmd_INTERSECTS_test(mc *mockServer) error {
@@ -73,30 +139,30 @@ func testcmd_INTERSECTS_test(mc *mockServer) error {
 	poly10 := `{"type": "Polygon","coordinates": [[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}`
 	poly101 := `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}`
 
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"},
+		Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(),
-		{"SET", "mykey", "point2", "POINT", 37.7335, -122.44121}, {"OK"},
+		Do("SET", "mykey", "point2", "POINT", 37.7335, -122.44121).OK(),
-		{"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"},
+		Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(),
-		{"SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"},
+		Do("SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`).OK(),
-		{"SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"},
+		Do("SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`).OK(),
-		{"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"},
+		Do("SET", "mykey", "point6", "POINT", -5, 5).OK(),
-		{"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"},
+		Do("SET", "mykey", "point7", "POINT", 33, 21).OK(),
-		{"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"},
+		Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(),
 
-		{"TEST", "GET", "mykey", "point1", "INTERSECTS", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "point1", "INTERSECTS", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "point2", "INTERSECTS", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "point2", "INTERSECTS", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "line3", "INTERSECTS", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "line3", "INTERSECTS", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "poly4", "INTERSECTS", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "poly4", "INTERSECTS", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "multipoly5", "INTERSECTS", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "multipoly5", "INTERSECTS", "OBJECT", poly).Str("1"),
-		{"TEST", "GET", "mykey", "poly8", "INTERSECTS", "OBJECT", poly}, {"1"},
+		Do("TEST", "GET", "mykey", "poly8", "INTERSECTS", "OBJECT", poly).Str("1"),
 
-		{"TEST", "GET", "mykey", "point6", "INTERSECTS", "OBJECT", poly}, {"0"},
+		Do("TEST", "GET", "mykey", "point6", "INTERSECTS", "OBJECT", poly).Str("0"),
-		{"TEST", "GET", "mykey", "point7", "INTERSECTS", "OBJECT", poly}, {"0"},
+		Do("TEST", "GET", "mykey", "point7", "INTERSECTS", "OBJECT", poly).Str("0"),
 
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8}, {"1"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8).Str("1"),
-		{"TEST", "OBJECT", poly10, "INTERSECTS", "OBJECT", poly8}, {"1"},
+		Do("TEST", "OBJECT", poly10, "INTERSECTS", "OBJECT", poly8).Str("1"),
-		{"TEST", "OBJECT", poly101, "INTERSECTS", "OBJECT", poly8}, {"0"},
+		Do("TEST", "OBJECT", poly101, "INTERSECTS", "OBJECT", poly8).Str("0"),
-	})
+	)
 }
 
 func testcmd_INTERSECTS_CLIP_test(mc *mockServer) error {
@@ -105,53 +171,48 @@ func testcmd_INTERSECTS_CLIP_test(mc *mockServer) error {
 	multipoly5 := `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`
 	poly101 := `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}`
 
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"},
+		Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(),
 
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "OBJECT", "{}"}, {"ERR invalid clip type 'OBJECT'"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "OBJECT", "{}").Err("invalid clip type 'OBJECT'"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "CIRCLE", "1", "2", "3"}, {"ERR invalid clip type 'CIRCLE'"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "CIRCLE", "1", "2", "3").Err("invalid clip type 'CIRCLE'"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "GET", "mykey", "point1"}, {"ERR invalid clip type 'GET'"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "GET", "mykey", "point1").Err("invalid clip type 'GET'"),
-		{"TEST", "OBJECT", poly9, "WITHIN", "CLIP", "BOUNDS", 10, 10, 20, 20}, {"ERR invalid argument 'CLIP'"},
+		Do("TEST", "OBJECT", poly9, "WITHIN", "CLIP", "BOUNDS", 10, 10, 20, 20).Err("invalid argument 'CLIP'"),
 
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135}, {"[1 " + poly9 + "]"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "SECTOR").Err("invalid clip type 'SECTOR'"),
-		{"TEST", "OBJECT", poly8, "INTERSECTS", "CLIP", "BOUNDS", 37.733, -122.4408378, 37.7341129, -122.44}, {"[1 " + poly8 + "]"},
+
-		{"TEST", "OBJECT", multipoly5, "INTERSECTS", "CLIP", "BOUNDS", 37.73227823422744, -122.44120001792908, 37.73319038868677, -122.43955314159392}, {"[1 " + `{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.4408378,37.73319038868677],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.73319038868677],[-122.4408378,37.73319038868677]]]},"properties":{}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.44091033935547,37.73227823422744],[-122.43994474411011,37.73227823422744],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.73227823422744]]]},"properties":{}}]}` + "]"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135).Str("[1 "+poly9+"]"),
-		{"TEST", "OBJECT", poly101, "INTERSECTS", "CLIP", "BOUNDS", 37.73315644825698, -122.44054287672043, 37.73349585185455, -122.44008690118788}, {"0"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135).JSON().Str(`{"ok":true,"result":true,"object":`+poly9+`}`),
-	})
+		Do("TEST", "OBJECT", poly8, "INTERSECTS", "CLIP", "BOUNDS", 37.733, -122.4408378, 37.7341129, -122.44).Str("[1 "+poly8+"]"),
+		Do("TEST", "OBJECT", multipoly5, "INTERSECTS", "CLIP", "BOUNDS", 37.73227823422744, -122.44120001792908, 37.73319038868677, -122.43955314159392).Str("[1 "+`{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.4408378,37.73319038868677],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.73319038868677],[-122.4408378,37.73319038868677]]]},"properties":{}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.44091033935547,37.73227823422744],[-122.43994474411011,37.73227823422744],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.73227823422744]]]},"properties":{}}]}`+"]"),
+		Do("TEST", "OBJECT", poly101, "INTERSECTS", "CLIP", "BOUNDS", 37.73315644825698, -122.44054287672043, 37.73349585185455, -122.44008690118788).Str("0"),
+	)
 }
 
 func testcmd_expressionErrors_test(mc *mockServer) error {
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "foo", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"},
+		Do("SET", "mykey", "foo", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(),
-		{"SET", "mykey", "bar", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"},
+		Do("SET", "mykey", "bar", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(),
-		{"SET", "mykey", "baz", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"},
+		Do("SET", "mykey", "baz", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(),
 
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "(", "GET", "mykey", "bar"}, {
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "(", "GET", "mykey", "bar").Err("wrong number of arguments for 'test' command"),
-			"ERR wrong number of arguments for 'test' command"},
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", ")").Err("invalid argument ')'"),
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", ")"}, {
-			"ERR invalid argument ')'"},
 
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "OR", "GET", "mykey", "bar"}, {
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "OR", "GET", "mykey", "bar").Err("invalid argument 'or'"),
-			"ERR invalid argument 'or'"},
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "AND", "GET", "mykey", "bar").Err("invalid argument 'and'"),
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "AND", "GET", "mykey", "bar"}, {
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"),
-			"ERR invalid argument 'and'"},
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "OR", "GET", "mykey", "baz").Err("invalid argument 'or'"),
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "AND", "GET", "mykey", "baz"}, {
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "OR", "GET", "mykey", "baz").Err("invalid argument 'or'"),
-			"ERR invalid argument 'and'"},
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"),
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "OR", "GET", "mykey", "baz"}, {
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR").Err("wrong number of arguments for 'test' command"),
-			"ERR invalid argument 'or'"},
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND").Err("wrong number of arguments for 'test' command"),
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "OR", "GET", "mykey", "baz"}, {
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT").Err("wrong number of arguments for 'test' command"),
-			"ERR invalid argument 'or'"},
+		Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"),
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "AND", "GET", "mykey", "baz"}, {
+
-			"ERR invalid argument 'and'"},
+		Do("TEST").Err("wrong number of arguments for 'test' command"),
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR"}, {
+		Do("TEST", "GET", "mykey", "foo").Err("wrong number of arguments for 'test' command"),
-			"ERR wrong number of arguments for 'test' command"},
+		Do("TEST", "GET", "mykey", "foo", "jello").Err("invalid argument 'jello'"),
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND"}, {
+	)
-			"ERR wrong number of arguments for 'test' command"},
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT"}, {
-			"ERR wrong number of arguments for 'test' command"},
-		{"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT", "AND", "GET", "mykey", "baz"}, {
-			"ERR invalid argument 'and'"},
-	})
 }
 
 func testcmd_expression_test(mc *mockServer) error {
@@ -170,46 +231,36 @@ func testcmd_expression_test(mc *mockServer) error {
 	poly8 := `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`
 	poly9 := `{"type": "Polygon","coordinates": [[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}`
 
-	return mc.DoBatch([][]interface{}{
+	return mc.DoBatch(
-		{"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"},
+		Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(),
-		{"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"},
+		Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(),
 
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "OBJECT", poly}, {"0"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "OBJECT", poly).Str("0"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "OBJECT", poly}, {"1"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "OBJECT", poly).Str("1"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "NOT", "OBJECT", poly}, {"0"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "NOT", "OBJECT", poly).Str("0"),
 
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "OR", "OBJECT", poly}, {"1"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "OR", "OBJECT", poly).Str("1"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly}, {"1"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "OR", "OBJECT", poly}, {"1"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "OR", "OBJECT", poly).Str("1"),
 
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3"}, {"0"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3").Str("0"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND",
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")").Str("0"),
-			"(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")"}, {"0"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")").Str("1"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND",
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")").Str("1"),
-			"(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")"}, {"1"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "GET", "mykey", "line3").Str("1"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND",
+		Do("TEST", "NOT", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3").Str("1"),
-			"(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")"}, {"1"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "GET", "mykey", "line3"}, {"1"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly, "OR", "GET", "mykey", "line3").Str("1"),
-		{"TEST", "NOT", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3"}, {"1"},
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", "(", "OBJECT", poly8, "AND", "OBJECT", poly, ")").Str("1"),
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3",
+		Do("TEST", "OBJECT", poly9, "INTERSECTS", "(", "GET", "mykey", "line3", "OR", "OBJECT", poly8, ")", "AND", "OBJECT", poly).Str("1"),
-			"OR", "OBJECT", poly8, "AND", "OBJECT", poly}, {"1"},
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly,
-			"OR", "GET", "mykey", "line3"}, {"1"},
-		{"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR",
-			"(", "OBJECT", poly8, "AND", "OBJECT", poly, ")"}, {"1"},
-		{"TEST", "OBJECT", poly9, "INTERSECTS",
-			"(", "GET", "mykey", "line3", "OR", "OBJECT", poly8, ")", "AND", "OBJECT", poly}, {"1"},
 
-		{"TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "OR", "OBJECT", poly}, {"1"},
+		Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "OR", "OBJECT", poly).Str("1"),
-		{"TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "AND", "OBJECT", poly}, {"1"},
+		Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"),
 
-		{"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "line3"}, {"0"},
+		Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "line3").Str("0"),
-		{"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND",
+		Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")").Str("0"),
-			"(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")"}, {"0"},
+		Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")").Str("1"),
-		{"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND",
+		Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")").Str("1"),
-			"(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")"}, {"1"},
+		Do("TEST", "OBJECT", poly9, "WITHIN", "NOT", "GET", "mykey", "line3").Str("1"),
-		{"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND",
+	)
-			"(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")"}, {"1"},
-		{"TEST", "OBJECT", poly9, "WITHIN", "NOT", "GET", "mykey", "line3"}, {"1"},
-	})
 }

tests/tests_test.go

@@ -5,11 +5,16 @@ import (
 	"math/rand"
 	"os"
 	"os/signal"
+	"runtime"
+	"strings"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
 
 	"github.com/gomodule/redigo/redis"
+	"github.com/tidwall/limiter"
+	"go.uber.org/atomic"
 )
 
 const (
@@ -26,11 +31,12 @@ const (
 	white   = "\x1b[37m"
 )
 
-func TestAll(t *testing.T) {
+func TestIntegration(t *testing.T) {
-	mockCleanup(false)
-	defer mockCleanup(false)
 
-	ch := make(chan os.Signal)
+	mockCleanup(true)
+	defer mockCleanup(true)
+
+	ch := make(chan os.Signal, 1)
 	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-ch
@@ -38,62 +44,190 @@ func TestAll(t *testing.T) {
 		os.Exit(1)
 	}()
 
-	mc, err := mockOpenServer(false)
+	regTestGroup("keys", subTestKeys)
-	if err != nil {
+	regTestGroup("json", subTestJSON)
-		t.Fatal(err)
+	regTestGroup("search", subTestSearch)
+	regTestGroup("testcmd", subTestTestCmd)
+	regTestGroup("client", subTestClient)
+	regTestGroup("scripts", subTestScripts)
+	regTestGroup("fence", subTestFence)
+	regTestGroup("info", subTestInfo)
+	regTestGroup("timeouts", subTestTimeout)
+	regTestGroup("metrics", subTestMetrics)
+	regTestGroup("follower", subTestFollower)
+	regTestGroup("aof", subTestAOF)
+	regTestGroup("monitor", subTestMonitor)
+	runTestGroups(t)
+}
+
+var allGroups []*testGroup
+
+func runTestGroups(t *testing.T) {
+	limit := runtime.NumCPU()
+	if limit > 16 {
+		limit = 16
 	}
-	defer mc.Close()
+	l := limiter.New(limit)
-	runSubTest(t, "keys", mc, subTestKeys)
-	runSubTest(t, "json", mc, subTestJSON)
-	runSubTest(t, "search", mc, subTestSearch)
-	runSubTest(t, "testcmd", mc, subTestTestCmd)
-	runSubTest(t, "fence", mc, subTestFence)
-	runSubTest(t, "scripts", mc, subTestScripts)
-	runSubTest(t, "info", mc, subTestInfo)
-	runSubTest(t, "client", mc, subTestClient)
-	runSubTest(t, "timeouts", mc, subTestTimeout)
-	runSubTest(t, "metrics", mc, subTestMetrics)
-}
 
-func runSubTest(t *testing.T, name string, mc *mockServer, test func(t *testing.T, mc *mockServer)) {
+	// Initialize all stores as "skipped", but they'll be unset if the test is
-	t.Run(name, func(t *testing.T) {
+	// not actually skipped.
-		fmt.Printf(bright+"Testing %s\n"+clear, name)
+	for _, g := range allGroups {
-		test(t, mc)
+		for _, s := range g.subs {
-	})
+			s.skipped.Store(true)
-}
-
-func runStep(t *testing.T, mc *mockServer, name string, step func(mc *mockServer) error) {
-	t.Helper()
-	t.Run(name, func(t *testing.T) {
-		t.Helper()
-		if err := func() error {
-			// reset the current server
-			mc.ResetConn()
-			defer mc.ResetConn()
-			// clear the database so the test is consistent
-			if err := mc.DoBatch([][]interface{}{
-				{"OUTPUT", "resp"}, {"OK"},
-				{"FLUSHDB"}, {"OK"},
-			}); err != nil {
-				return err
-			}
-			if err := step(mc); err != nil {
-				return err
-			}
-			return nil
-		}(); err != nil {
-			fmt.Printf("["+red+"fail"+clear+"]: %s\n", name)
-			t.Fatal(err)
 		}
-		fmt.Printf("["+green+"ok"+clear+"]: %s\n", name)
+	}
-	})
+	for _, g := range allGroups {
+		func(g *testGroup) {
+			t.Run(g.name, func(t *testing.T) {
+				for _, s := range g.subs {
+					func(s *testGroupSub) {
+						t.Run(s.name, func(t *testing.T) {
+							s.skipped.Store(false)
+							var wg sync.WaitGroup
+							wg.Add(1)
+							var err error
+							go func() {
+								l.Begin()
+								defer func() {
+									l.End()
+									wg.Done()
+								}()
+								err = s.run()
+							}()
+							if false {
+								t.Parallel()
+								t.Run("bg", func(t *testing.T) {
+									wg.Wait()
+									if err != nil {
+										t.Fatal(err)
+									}
+								})
+							}
+						})
+					}(s)
+				}
+			})
+		}(g)
+	}
+
+	done := make(chan bool)
+	go func() {
+		defer func() { done <- true }()
+		for {
+			finished := true
+			for _, g := range allGroups {
+				skipped := true
+				for _, s := range g.subs {
+					if !s.skipped.Load() {
+						skipped = false
+						break
+					}
+				}
+				if !skipped && !g.printed.Load() {
+					fmt.Printf(bright+"Testing %s\n"+clear, g.name)
+					g.printed.Store(true)
+				}
+				const frtmp = "[" + magenta + "  " + clear + "] %s (running) "
+				for _, s := range g.subs {
+					if !s.skipped.Load() && !s.printedName.Load() {
+						fmt.Printf(frtmp, s.name)
+						s.printedName.Store(true)
+					}
+					if s.done.Load() && !s.printedResult.Load() {
+						fmt.Printf("\r")
+						msg := fmt.Sprintf(frtmp, s.name)
+						fmt.Print(strings.Repeat(" ", len(msg)))
+						fmt.Printf("\r")
+						if s.err != nil {
+							fmt.Printf("["+red+"fail"+clear+"] %s\n", s.name)
+						} else {
+							fmt.Printf("["+green+"ok"+clear+"] %s\n", s.name)
+						}
+						s.printedResult.Store(true)
+					}
+					if !s.skipped.Load() && !s.done.Load() {
+						finished = false
+						break
+					}
+				}
+				if !finished {
+					break
+				}
+			}
+			if finished {
+				break
+			}
+			time.Sleep(time.Second / 4)
+		}
+	}()
+	<-done
+	var fail bool
+	for _, g := range allGroups {
+		for _, s := range g.subs {
+			if s.err != nil {
+				t.Errorf("%s/%s/%s\n%s", t.Name(), g.name, s.name, s.err)
+				fail = true
+			}
+		}
+	}
+	if fail {
+		t.Fail()
+	}
+
+}
+
+type testGroup struct {
+	name    string
+	subs    []*testGroupSub
+	printed atomic.Bool
+}
+
+type testGroupSub struct {
+	g             *testGroup
+	name          string
+	fn            func(mc *mockServer) error
+	err           error
+	skipped       atomic.Bool
+	done          atomic.Bool
+	printedName   atomic.Bool
+	printedResult atomic.Bool
+}
+
+func regTestGroup(name string, fn func(g *testGroup)) {
+	g := &testGroup{name: name}
+	allGroups = append(allGroups, g)
+	fn(g)
+}
+
+func (g *testGroup) regSubTest(name string, fn func(mc *mockServer) error) {
+	s := &testGroupSub{g: g, name: name, fn: fn}
+	g.subs = append(g.subs, s)
+}
+
+func (s *testGroupSub) run() (err error) {
+	// This all happens in a background routine.
+	defer func() {
+		s.err = err
+		s.done.Store(true)
+	}()
+	return func() error {
+		mc, err := mockOpenServer(MockServerOptions{
+			Silent:  true,
+			Metrics: true,
+		})
+		if err != nil {
+			return err
+		}
+		defer mc.Close()
+		return s.fn(mc)
+	}()
 }
 
 func BenchmarkAll(b *testing.B) {
 	mockCleanup(true)
 	defer mockCleanup(true)
 
-	ch := make(chan os.Signal)
+	ch := make(chan os.Signal, 1)
 	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-ch
@@ -101,7 +235,9 @@ func BenchmarkAll(b *testing.B) {
 		os.Exit(1)
 	}()
 
-	mc, err := mockOpenServer(true)
+	mc, err := mockOpenServer(MockServerOptions{
+		Silent: true, Metrics: true,
+	})
 	if err != nil {
 		b.Fatal(err)
 	}

tests/timeout_test.go

@@ -4,19 +4,18 @@ import (
 	"fmt"
 	"math/rand"
 	"strings"
-	"testing"
 	"time"
 
 	"github.com/gomodule/redigo/redis"
 )
 
-func subTestTimeout(t *testing.T, mc *mockServer) {
+func subTestTimeout(g *testGroup) {
-	runStep(t, mc, "spatial", timeout_spatial_test)
+	g.regSubTest("spatial", timeout_spatial_test)
-	runStep(t, mc, "search", timeout_search_test)
+	g.regSubTest("search", timeout_search_test)
-	runStep(t, mc, "scripts", timeout_scripts_test)
+	g.regSubTest("scripts", timeout_scripts_test)
-	runStep(t, mc, "no writes", timeout_no_writes_test)
+	g.regSubTest("no writes", timeout_no_writes_test)
-	runStep(t, mc, "within scripts", timeout_within_scripts_test)
+	g.regSubTest("within scripts", timeout_within_scripts_test)
-	runStep(t, mc, "no writes within scripts", timeout_no_writes_within_scripts_test)
+	g.regSubTest("no writes within scripts", timeout_no_writes_within_scripts_test)
 }
 
 func setup(mc *mockServer, count int, points bool) (err error) {
@@ -54,22 +53,27 @@ func setup(mc *mockServer, count int, points bool) (err error) {
 	return
 }
 
-func timeout_spatial_test(mc *mockServer) (err error) {
+func timeout_spatial_test(mc *mockServer) error {
-	err = setup(mc, 10000, true)
+	err := setup(mc, 10000, true)
+	if err != nil {
+		return err
+	}
+	return mc.DoBatch(
+		Do("SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT").Str("10000"),
+		Do("INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Str("10000"),
+		Do("WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Str("10000"),
 
-	return mc.DoBatch([][]interface{}{
+		Do("TIMEOUT", "0.000001", "SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT").Err("timeout"),
-		{"SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT"}, {"10000"},
+		Do("TIMEOUT", "0.000001", "INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Err("timeout"),
-		{"INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"10000"},
+		Do("TIMEOUT", "0.000001", "WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Err("timeout"),
-		{"WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"10000"},
+	)
-
-		{"TIMEOUT", "0.000001", "SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT"}, {"ERR timeout"},
-		{"TIMEOUT", "0.000001", "INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"ERR timeout"},
-		{"TIMEOUT", "0.000001", "WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"ERR timeout"},
-	})
 }
 
 func timeout_search_test(mc *mockServer) (err error) {
 	err = setup(mc, 10000, false)
+	if err != nil {
+		return err
+	}
 
 	return mc.DoBatch([][]interface{}{
 		{"SEARCH", "mykey", "MATCH", "val:*", "COUNT"}, {"10000"},
@@ -122,6 +126,9 @@ func scriptTimeoutErr(v interface{}) (resp, expect interface{}) {
 
 func timeout_within_scripts_test(mc *mockServer) (err error) {
 	err = setup(mc, 10000, true)
+	if err != nil {
+		return err
+	}
 
 	script1 := "return tile38.call('timeout', 10, 'SCAN', 'mykey', 'WHERE', 'foo', -1, 2, 'COUNT')"
 	script2 := "return tile38.call('timeout', 0.000001, 'SCAN', 'mykey', 'WHERE', 'foo', -1, 2, 'COUNT')"