Normalize NAN to a single nan type, like we do with inf (#11597)

From https://en.wikipedia.org/wiki/NaN#Display, it says
that apart from nan and -nan, we can also get NAN and even
nan(char-sequence) from libc.

In #11482, our conclusion was that we wanna normalize it in
Redis to a single nan type, like we already normalized inf.

For this, we also reverted the assert_match part of the test
added in #11506, using assert_equal to validate the changes.

src/networking.c

@@ -850,6 +850,15 @@ void addReplyDouble(client *c, double d) {
             addReplyProto(c, d > 0 ? ",inf\r\n" : ",-inf\r\n",
                               d > 0 ? 6 : 7);
         }
+    } else if (isnan(d)) {
+        /* Libc in some systems will format nan in a different way,
+         * like nan, -nan, NAN, nan(char-sequence).
+         * So we normalize it and create a single nan form in an explicit way. */
+        if (c->resp == 2) {
+            addReplyBulkCString(c, "nan");
+        } else {
+            addReplyProto(c, ",nan\r\n", 6);
+        }
     } else {
         char dbuf[MAX_LONG_DOUBLE_CHARS+32];
         int dlen = 0;

src/util.c

@@ -779,6 +779,13 @@ int ld2string(char *buf, size_t len, long double value, ld2string_mode mode) {
             memcpy(buf,"-inf",4);
             l = 4;
         }
+    } else if (isnan(value)) {
+        /* Libc in some systems will format nan in a different way,
+         * like nan, -nan, NAN, nan(char-sequence).
+         * So we normalize it and create a single nan form in an explicit way. */
+        if (len < 4) goto err; /* No room. 4 is "nan\0" */
+        memcpy(buf, "nan", 3);
+        l = 3;
     } else {
         switch (mode) {
         case LD_STR_AUTO:
@@ -1253,6 +1260,17 @@ static void test_ll2string(void) {
     assert(!strcmp(buf, "9223372036854775807"));
 }
 
+static void test_ld2string(void) {
+    char buf[32];
+    long double v;
+    int sz;
+
+    v = 0.0 / 0.0;
+    sz = ld2string(buf, sizeof(buf), v, LD_STR_AUTO);
+    assert(sz == 3);
+    assert(!strcmp(buf, "nan"));
+}
+
 static void test_fixedpoint_d2string(void) {
     char buf[32];
     double v;
@@ -1309,6 +1327,7 @@ int utilTest(int argc, char **argv, int flags) {
     test_string2ll();
     test_string2l();
     test_ll2string();
+    test_ld2string();
     test_fixedpoint_d2string();
     return 0;
 }

tests/unit/moduleapi/reply.tcl

@@ -43,16 +43,14 @@ start_server {tags {"modules"}} {
         }
 
         test "RESP$proto: RM_ReplyWithDouble: NaN" {
-            # On some platforms one of these can be -nan but we don't care since they are
-            # synonym, so here we match ignoring the sign
             if {$proto == 2} {
-                assert_match "*nan" [r rw.double 0 0]
+                assert_equal "nan" [r rw.double 0 0]
-                assert_match "*nan" [r rw.double]
+                assert_equal "nan" [r rw.double]
             } else {
                 # TCL won't convert nan into a double, use readraw to verify the protocol
                 r readraw 1
-                assert_match ",*nan" [r rw.double 0 0]
+                assert_equal ",nan" [r rw.double 0 0]
-                assert_match ",*nan" [r rw.double]
+                assert_equal ",nan" [r rw.double]
                 r readraw 0
             }
         }