Merge branch 'unstable' of https://github.com/antirez/redis into unstable

Former-commit-id: 31ff6cffee209ec03b8e7ed851145289c96de0f4

MANIFESTO

@@ -34,7 +34,21 @@ Redis Manifesto
     so that the complexity is obvious and more complex operations can be
     performed as the sum of the basic operations.
 
-4 - Code is like a poem; it's not just something we write to reach some
+4 - We believe in code efficiency. Computers get faster and faster, yet we
+    believe that abusing computing capabilities is not wise: the amount of
+    operations you can do for a given amount of energy remains anyway a
+    significant parameter: it allows to do more with less computers and, at
+    the same time, having a smaller environmental impact. Similarly Redis is
+    able to "scale down" to smaller devices. It is perfectly usable in a
+    Raspberry Pi and other small ARM based computers. Faster code having
+    just the layers of abstractions that are really needed will also result,
+    often, in more predictable performances. We think likewise about memory
+    usage, one of the fundamental goals of the Redis project is to
+    incrementally build more and more memory efficient data structures, so that
+    problems that were not approachable in RAM in the past will be perfectly
+    fine to handle in the future.
+
+5 - Code is like a poem; it's not just something we write to reach some
     practical result. Sometimes people that are far from the Redis philosophy
     suggest using other code written by other authors (frequently in other
     languages) in order to implement something Redis currently lacks. But to us
@@ -45,23 +59,48 @@ Redis Manifesto
     when needed. At the same time, when writing the Redis story we're trying to
     write smaller stories that will fit in to other code.
 
-5 - We're against complexity. We believe designing systems is a fight against
+6 - We're against complexity. We believe designing systems is a fight against
     complexity. We'll accept to fight the complexity when it's worthwhile but
     we'll try hard to recognize when a small feature is not worth 1000s of lines
     of code. Most of the time the best way to fight complexity is by not
-    creating it at all.
+    creating it at all. Complexity is also a form of lock-in: code that is
+    very hard to understand cannot be modified by users in an independent way
+    regardless of the license. One of the main Redis goals is to remain
+    understandable, enough for a single programmer to have a clear idea of how
+    it works in detail just reading the source code for a couple of weeks.
 
-6 - Two levels of API. The Redis API has two levels: 1) a subset of the API fits
+7 - Threading is not a silver bullet. Instead of making Redis threaded we
+    believe on the idea of an efficient (mostly) single threaded Redis core.
+    Multiple of such cores, that may run in the same computer or may run
+    in multiple computers, are abstracted away as a single big system by
+    higher order protocols and features: Redis Cluster and the upcoming
+    Redis Proxy are our main goals. A shared nothing approach is not just
+    much simpler (see the previous point in this document), is also optimal
+    in NUMA systems. In the specific case of Redis it allows for each instance
+    to have a more limited amount of data, making the Redis persist-by-fork
+    approach more sounding. In the future we may explore parallelism only for
+    I/O, which is the low hanging fruit: minimal complexity could provide an
+    improved single process experience.
+
+8 - Two levels of API. The Redis API has two levels: 1) a subset of the API fits
     naturally into a distributed version of Redis and 2) a more complex API that
     supports multi-key operations. Both are useful if used judiciously but
     there's no way to make the more complex multi-keys API distributed in an
     opaque way without violating our other principles. We don't want to provide
     the illusion of something that will work magically when actually it can't in
     all cases. Instead we'll provide commands to quickly migrate keys from one
-    instance to another to perform multi-key operations and expose the tradeoffs
+    instance to another to perform multi-key operations and expose the
-    to the user.
+    trade-offs to the user.
 
-7 - We optimize for joy. We believe writing code is a lot of hard work, and the
+9 - We optimize for joy. We believe writing code is a lot of hard work, and the
     only way it can be worth is by enjoying it. When there is no longer joy in
     writing code, the best thing to do is stop. To prevent this, we'll avoid
     taking paths that will make Redis less of a joy to develop.
+
+10 - All the above points are put together in what we call opportunistic
+     programming: trying to get the most for the user with minimal increases
+     in complexity (hanging fruits). Solve 95% of the problem with 5% of the
+     code when it is acceptable. Avoid a fixed schedule but follow the flow of
+     user requests, inspiration, Redis internal readiness for certain features
+     (sometimes many past changes reach a critical point making a previously
+     complex feature very easy to obtain).

src/hyperloglog.c

@@ -614,10 +614,7 @@ int hllSparseToDense(robj *o) {
         } else {
             runlen = HLL_SPARSE_VAL_LEN(p);
             regval = HLL_SPARSE_VAL_VALUE(p);
-            if ((runlen + idx) > HLL_REGISTERS) {
+            if ((runlen + idx) > HLL_REGISTERS) break; /* Overflow. */
-                sdsfree(dense);
-                return C_ERR;
-            }
             while(runlen--) {
                 HLL_DENSE_SET_REGISTER(hdr->registers,idx,regval);
                 idx++;
@@ -1017,7 +1014,12 @@ uint64_t hllCount(struct hllhdr *hdr, int *invalid) {
     double m = HLL_REGISTERS;
     double E;
     int j;
-    int reghisto[HLL_Q+2] = {0};
+    /* Note that reghisto size could be just HLL_Q+2, becuase HLL_Q+1 is
+     * the maximum frequency of the "000...1" sequence the hash function is
+     * able to return. However it is slow to check for sanity of the
+     * input: instead we history array at a safe size: overflows will
+     * just write data to wrong, but correctly allocated, places. */
+    int reghisto[64] = {0};
 
     /* Compute register histogram */
     if (hdr->encoding == HLL_DENSE) {
@@ -1092,8 +1094,7 @@ int hllMerge(uint8_t *max, size_t cmax, robj *hll) {
             } else {
                 runlen = HLL_SPARSE_VAL_LEN(p);
                 regval = HLL_SPARSE_VAL_VALUE(p);
-                if ((runlen + i) > HLL_REGISTERS)
+                if ((runlen + i) > HLL_REGISTERS) break; /* Overflow. */
-                    return C_ERR;
                 while(runlen--) {
                     if (i < 0 || (size_t)i >= cmax)
                         return C_ERR;

src/redis-check-aof.c

@@ -33,8 +33,8 @@
 
 #define ERROR(...) { \
     char __buf[1024]; \
-    sprintf(__buf, __VA_ARGS__); \
+    snprintf(__buf, sizeof(__buf), __VA_ARGS__); \
-    sprintf(error, "0x%16llx: %s", (long long)epos, __buf); \
+    snprintf(error, sizeof(error), "0x%16llx: %s", (long long)epos, __buf); \
 }
 
 static char error[1024];

src/replication.cpp

@@ -2117,8 +2117,11 @@ void replicaofCommand(client *c) {
         /* Check if we are already attached to the specified slave */
         if (server.masterhost && !strcasecmp(server.masterhost,(const char*)ptrFromObj(c->argv[1]))
             && server.masterport == port) {
-            serverLog(LL_NOTICE,"REPLICAOF would result into synchronization with the master we are already connected with. No operation performed.");
+            serverLog(LL_NOTICE,"REPLICAOF would result into synchronization "
-            addReplySdsAsync(c,sdsnew("+OK Already connected to specified master\r\n"));
+                                "with the master we are already connected "
+                                "with. No operation performed.");
+            addReplySds(c,sdsnew("+OK Already connected to specified "
+                                 "master\r\n"));
             return;
         }
         /* There was no previous master or the user specified a different one,

tests/unit/hyperloglog.tcl

@@ -115,6 +115,34 @@ start_server {tags {"hll"}} {
         set e
     } {*WRONGTYPE*}
 
+    test {Fuzzing dense/sparse encoding: Redis should always detect errors} {
+        for {set j 0} {$j < 1000} {incr j} {
+            r del hll
+            set items {}
+            set numitems [randomInt 3000]
+            for {set i 0} {$i < $numitems} {incr i} {
+                lappend items [expr {rand()}]
+            }
+            r pfadd hll {*}$items
+
+            # Corrupt it in some random way.
+            for {set i 0} {$i < 5} {incr i} {
+                set len [r strlen hll]
+                set pos [randomInt $len]
+                set byte [randstring 1 1 binary]
+                r setrange hll $pos $byte
+                # Don't modify more bytes 50% of times
+                if {rand() < 0.5} break
+            }
+
+            # Use the hyperloglog to check if it crashes
+            # Redis in some way.
+            catch {
+                r pfcount hll
+            }
+        }
+    }
+
     test {PFADD, PFCOUNT, PFMERGE type checking works} {
         r set foo bar
         catch {r pfadd foo 1} e