Move doc metadata from COMMAND to COMMAND DOCS (#10056)

Syntax:
`COMMAND DOCS [<command name> ...]`

Background:
Apparently old version of hiredis (and thus also redis-cli) can't
support more than 7 levels of multi-bulk nesting.

The solution is to move all the doc related metadata from COMMAND to a
new COMMAND DOCS sub-command.

The new DOCS sub-command returns a map of commands (not an array like in COMMAND),
And the same goes for the `subcommands` field inside it (also contains a map)

Besides that, the remaining new fields of COMMAND (hints, key-specs, and
sub-commands), are placed in the outer array rather than a nested map.
this was done mainly for consistency with the old format.

Other changes:
---
* Allow COMMAND INFO with no arguments, which returns all commands, so that we can some day deprecated
  the plain COMMAND (no args)

* Reduce the amount of deferred replies from both COMMAND and COMMAND
  DOCS, especially in the inner loops, since these create many small
  reply objects, which lead to many small write syscalls and many small
  TCP packets.
  To make this easier, when populating the command table, we count the
  history, args, and hints so we later know their size in advance.
  Additionally, the movablekeys flag was moved into the flags register.
* Update generate-commands-json.py to take the data from both command, it
  now executes redis-cli directly, instead of taking input from stdin.
* Sub-commands in both COMMAND (and COMMAND INFO), and also COMMAND DOCS,
  show their full name. i.e. CONFIG 
*   GET will be shown as `config|get` rather than just `get`.
  This will be visible both when asking for `COMMAND INFO config` and COMMAND INFO config|get`, but is
  especially important for the later.
  i.e. imagine someone doing `COMMAND INFO slowlog|get config|get` not being able to distinguish between the two
  items in the array response.

src/commands.c

@@ -3795,6 +3795,20 @@ struct redisCommandArg BGSAVE_Args[] = {
 /* COMMAND COUNT hints */
 #define COMMAND_COUNT_Hints NULL
 
+/********** COMMAND DOCS ********************/
+
+/* COMMAND DOCS history */
+#define COMMAND_DOCS_History NULL
+
+/* COMMAND DOCS hints */
+#define COMMAND_DOCS_Hints NULL
+
+/* COMMAND DOCS argument table */
+struct redisCommandArg COMMAND_DOCS_Args[] = {
+{"command-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE},
+{0}
+};
+
 /********** COMMAND GETKEYS ********************/
 
 /* COMMAND GETKEYS history */
@@ -3814,14 +3828,17 @@ struct redisCommandArg BGSAVE_Args[] = {
 /********** COMMAND INFO ********************/
 
 /* COMMAND INFO history */
-#define COMMAND_INFO_History NULL
+commandHistory COMMAND_INFO_History[] = {
+{"7.0.0","Allowed to be called with no argument to get info on all commands."},
+{0}
+};
 
 /* COMMAND INFO hints */
 #define COMMAND_INFO_Hints NULL
 
 /* COMMAND INFO argument table */
 struct redisCommandArg COMMAND_INFO_Args[] = {
-{"command-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE},
+{"command-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE},
 {0}
 };
 
@@ -3850,9 +3867,10 @@ struct redisCommandArg COMMAND_LIST_Args[] = {
 /* COMMAND command table */
 struct redisCommand COMMAND_Subcommands[] = {
 {"count","Get total number of Redis commands","O(1)","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_COUNT_History,COMMAND_COUNT_Hints,commandCountCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION},
+{"docs","Get array of specific Redis command documentation","O(N) where N is the number of commands to look up","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_DOCS_History,COMMAND_DOCS_Hints,commandDocsCommand,-2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_DOCS_Args},
 {"getkeys","Extract keys given a full Redis command","O(N) where N is the number of arguments to the command","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_GETKEYS_History,COMMAND_GETKEYS_Hints,commandGetKeysCommand,-4,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION},
 {"help","Show helpful text about the different subcommands","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_HELP_History,COMMAND_HELP_Hints,commandHelpCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION},
-{"info","Get array of specific Redis command details","O(N) when N is number of commands to look up","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_INFO_History,COMMAND_INFO_Hints,commandInfoCommand,-3,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_INFO_Args},
+{"info","Get array of specific Redis command details, or all when no argument is given.","O(N) where N is the number of commands to look up","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_INFO_History,COMMAND_INFO_Hints,commandInfoCommand,-2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_INFO_Args},
 {"list","Get an array of Redis command names","O(N) where N is the total number of Redis commands","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_LIST_History,COMMAND_LIST_Hints,commandListCommand,-2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_LIST_Args},
 {0}
 };

src/commands/command-docs.json

@@ -0,0 +1,26 @@
+{
+    "DOCS": {
+        "summary": "Get array of specific Redis command documentation",
+        "complexity": "O(N) where N is the number of commands to look up",
+        "group": "server",
+        "since": "7.0.0",
+        "arity": -2,
+        "container": "COMMAND",
+        "function": "commandDocsCommand",
+        "command_flags": [
+            "LOADING",
+            "STALE"
+        ],
+        "acl_categories": [
+            "CONNECTION"
+        ],
+        "arguments": [
+            {
+                "name": "command-name",
+                "type": "string",
+                "optional": true,
+                "multiple": true
+            }
+        ]
+    }
+}

src/commands/command-info.json

@@ -1,12 +1,18 @@
 {
     "INFO": {
-        "summary": "Get array of specific Redis command details",
+        "summary": "Get array of specific Redis command details, or all when no argument is given.",
-        "complexity": "O(N) when N is number of commands to look up",
+        "complexity": "O(N) where N is the number of commands to look up",
         "group": "server",
         "since": "2.8.13",
-        "arity": -3,
+        "arity": -2,
         "container": "COMMAND",
         "function": "commandInfoCommand",
+        "history": [
+            [
+                "7.0.0",
+                "Allowed to be called with no argument to get info on all commands."
+            ]
+        ],
         "command_flags": [
             "LOADING",
             "STALE"
@@ -18,6 +24,7 @@
             {
                 "name": "command-name",
                 "type": "string",
+                "optional": true,
                 "multiple": true
             }
         ]

src/server.c

@@ -2620,6 +2620,20 @@ void setImplictACLCategories(struct redisCommand *c) {
         c->acl_categories |= ACL_CATEGORY_SLOW;
 }
 
+/* Recursively populate the args structure and return the number of args. */
+int populateArgsStructure(struct redisCommandArg *args) {
+    if (!args)
+        return 0;
+    int count = 0;
+    while (args->name) {
+        args->num_args = populateArgsStructure(args->subargs);
+        count++;
+        args++;
+    }
+    return count;
+}
+
+/* Recursively populate the command stracture. */
 void populateCommandStructure(struct redisCommand *c) {
     /* Redis commands don't need more args than STATIC_KEY_SPECS_NUM (Number of keys
      * specs can be greater than STATIC_KEY_SPECS_NUM only for module commands) */
@@ -2636,6 +2650,13 @@ void populateCommandStructure(struct redisCommand *c) {
         c->key_specs_num++;
     }
 
+    /* Count things so we don't have to use deferred reply in COMMAND reply. */
+    while (c->history && c->history[c->num_history].since)
+        c->num_history++;
+    while (c->hints && c->hints[c->num_hints])
+        c->num_hints++;
+    c->num_args = populateArgsStructure(c->args);
+
     populateCommandLegacyRangeSpec(c);
 
     /* Handle the "movablekeys" flag (must be done after populating all key specs). */
@@ -3302,7 +3323,8 @@ void populateCommandMovableKeys(struct redisCommand *cmd) {
         }
     }
 
-    cmd->movablekeys = movablekeys;
+    if (movablekeys)
+        cmd->flags |= CMD_MOVABLE_KEYS;
 }
 
 /* If this function gets called we already read a whole
@@ -3442,7 +3464,7 @@ int processCommand(client *c) {
         !(c->flags & CLIENT_MASTER) &&
         !(c->flags & CLIENT_SCRIPT &&
           server.script_caller->flags & CLIENT_MASTER) &&
-        !(!c->cmd->movablekeys && c->cmd->key_specs_num == 0 &&
+        !(!(c->cmd->flags&CMD_MOVABLE_KEYS) && c->cmd->key_specs_num == 0 &&
           c->cmd->proc != execCommand))
     {
         int hashslot;
@@ -4022,64 +4044,78 @@ void timeCommand(client *c) {
     addReplyBulkLongLong(c,tv.tv_usec);
 }
 
-/* Helper function for addReplyCommand() to output flags. */
+typedef struct replyFlagNames {
-int addReplyCommandFlag(client *c, uint64_t flags, uint64_t f, char *reply) {
+    uint64_t flag;
-    if (flags & f) {
+    const char *name;
-        addReplyStatus(c, reply);
+} replyFlagNames;
-        return 1;
+
+/* Helper function to output flags. */
+void addReplyCommandFlags(client *c, uint64_t flags, replyFlagNames *replyFlags) {
+    int count = 0, j=0;
+    /* Count them so we don't have to use deferred reply. */
+    while (replyFlags[j].name) {
+        if (flags & replyFlags[j].flag)
+            count++;
+        j++;
+    }
+
+    addReplySetLen(c, count);
+    j = 0;
+    while (replyFlags[j].name) {
+        if (flags & replyFlags[j].flag)
+            addReplyStatus(c, replyFlags[j].name);
+        j++;
     }
-    return 0;
 }
 
 void addReplyFlagsForCommand(client *c, struct redisCommand *cmd) {
-    int flagcount = 0;
+    replyFlagNames flagNames[] = {
-    void *flaglen = addReplyDeferredLen(c);
+        {CMD_WRITE,             "write"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_WRITE, "write");
+        {CMD_READONLY,          "readonly"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_READONLY, "readonly");
+        {CMD_DENYOOM,           "denyoom"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_DENYOOM, "denyoom");
+        {CMD_MODULE,            "module"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_MODULE, "module");
+        {CMD_ADMIN,             "admin"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_ADMIN, "admin");
+        {CMD_PUBSUB,            "pubsub"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_PUBSUB, "pubsub");
+        {CMD_NOSCRIPT,          "noscript"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_NOSCRIPT, "noscript");
+        {CMD_RANDOM,            "random"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_RANDOM, "random");
+        {CMD_SORT_FOR_SCRIPT,   "sort_for_script"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_SORT_FOR_SCRIPT,"sort_for_script");
+        {CMD_LOADING,           "loading"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_LOADING, "loading");
+        {CMD_STALE,             "stale"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_STALE, "stale");
+        {CMD_SKIP_MONITOR,      "skip_monitor"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_SKIP_MONITOR, "skip_monitor");
+        {CMD_SKIP_SLOWLOG,      "skip_slowlog"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_SKIP_SLOWLOG, "skip_slowlog");
+        {CMD_ASKING,            "asking"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_ASKING, "asking");
+        {CMD_FAST,              "fast"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_FAST, "fast");
+        {CMD_NO_AUTH,           "no_auth"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_NO_AUTH, "no_auth");
+        {CMD_MAY_REPLICATE,     "may_replicate"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_MAY_REPLICATE, "may_replicate");
+        {CMD_NO_MANDATORY_KEYS, "no_mandatory_keys"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_NO_MANDATORY_KEYS, "no_mandatory_keys");
+        {CMD_PROTECTED,         "protected"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_PROTECTED, "protected");
+        {CMD_NO_ASYNC_LOADING,  "no_async_loading"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_NO_ASYNC_LOADING, "no_async_loading");
+        {CMD_NO_MULTI,          "no_multi"},
-    flagcount += addReplyCommandFlag(c,cmd->flags,CMD_NO_MULTI, "no_multi");
+        {CMD_MOVABLE_KEYS,      "movablekeys"},
-
+        {0,NULL}
+    };
     /* "sentinel" and "only-sentinel" are hidden on purpose. */
-    if (cmd->movablekeys) {
+    addReplyCommandFlags(c, cmd->flags, flagNames);
-        addReplyStatus(c, "movablekeys");
-        flagcount += 1;
-    }
-    setDeferredSetLen(c, flaglen, flagcount);
 }
 
 void addReplyDocFlagsForCommand(client *c, struct redisCommand *cmd) {
-    int flagcount = 0;
+    replyFlagNames docFlagNames[] = {
-    void *flaglen = addReplyDeferredLen(c);
+        {CMD_DOC_DEPRECATED,         "deprecated"},
-    flagcount += addReplyCommandFlag(c,cmd->doc_flags,CMD_DOC_DEPRECATED, "deprecated");
+        {CMD_DOC_SYSCMD,             "syscmd"},
-    flagcount += addReplyCommandFlag(c,cmd->doc_flags,CMD_DOC_SYSCMD, "syscmd");
+        {0,NULL}
-    setDeferredSetLen(c, flaglen, flagcount);
+    };
+    addReplyCommandFlags(c, cmd->doc_flags, docFlagNames);
 }
 
 void addReplyFlagsForKeyArgs(client *c, uint64_t flags) {
-    int flagcount = 0;
+    replyFlagNames docFlagNames[] = {
-    void *flaglen = addReplyDeferredLen(c);
+        {CMD_KEY_WRITE,              "write"},
-    flagcount += addReplyCommandFlag(c,flags,CMD_KEY_WRITE, "write");
+        {CMD_KEY_READ,               "read"},
-    flagcount += addReplyCommandFlag(c,flags,CMD_KEY_READ, "read");
+        {CMD_KEY_SHARD_CHANNEL,      "shard_channel"},
-    flagcount += addReplyCommandFlag(c,flags,CMD_KEY_SHARD_CHANNEL, "shard_channel");
+        {CMD_KEY_INCOMPLETE,         "incomplete"},
-    flagcount += addReplyCommandFlag(c,flags,CMD_KEY_INCOMPLETE, "incomplete");
+        {0,NULL}
-    setDeferredSetLen(c, flaglen, flagcount);
+    };
+    addReplyCommandFlags(c, flags, docFlagNames);
 }
 
 /* Must match redisCommandArgType */
@@ -4096,60 +4132,60 @@ const char *ARG_TYPE_STR[] = {
 };
 
 void addReplyFlagsForArg(client *c, uint64_t flags) {
-    int flagcount = 0;
+    replyFlagNames argFlagNames[] = {
-    void *flaglen = addReplyDeferredLen(c);
+        {CMD_ARG_OPTIONAL,          "optional"},
-    flagcount += addReplyCommandFlag(c,flags,CMD_ARG_OPTIONAL, "optional");
+        {CMD_ARG_MULTIPLE,          "multiple"},
-    flagcount += addReplyCommandFlag(c,flags,CMD_ARG_MULTIPLE, "multiple");
+        {CMD_ARG_MULTIPLE_TOKEN,    "multiple_token"},
-    flagcount += addReplyCommandFlag(c,flags,CMD_ARG_MULTIPLE_TOKEN, "multiple_token");
+        {0,NULL}
-    setDeferredSetLen(c, flaglen, flagcount);
+    };
+    addReplyCommandFlags(c, flags, argFlagNames);
 }
 
-void addReplyCommandArgList(client *c, struct redisCommandArg *args) {
+void addReplyCommandArgList(client *c, struct redisCommandArg *args, int num_args) {
-    int j;
+    addReplySetLen(c, num_args);
+    for (int j = 0; j<num_args; j++) {
+        /* Count our reply len so we don't have to use deferred reply. */
+        long maplen = 2;
+        if (args[j].type == ARG_TYPE_KEY) maplen++;
+        if (args[j].token) maplen++;
+        if (args[j].summary) maplen++;
+        if (args[j].since) maplen++;
+        if (args[j].flags) maplen++;
+        if (args[j].type == ARG_TYPE_ONEOF || args[j].type == ARG_TYPE_BLOCK)
+            maplen++;
+        addReplyMapLen(c, maplen);
 
-    void *setreply = addReplyDeferredLen(c);
-    for (j = 0; args && args[j].name != NULL; j++) {
-        long maplen = 0;
-        void *mapreply = addReplyDeferredLen(c);
         addReplyBulkCString(c, "name");
         addReplyBulkCString(c, args[j].name);
-        maplen++;
+
         addReplyBulkCString(c, "type");
         addReplyBulkCString(c, ARG_TYPE_STR[args[j].type]);
-        maplen++;
+
         if (args[j].type == ARG_TYPE_KEY) {
             addReplyBulkCString(c, "key_spec_index");
             addReplyLongLong(c, args[j].key_spec_index);
-            maplen++;
         }
         if (args[j].token) {
             addReplyBulkCString(c, "token");
             addReplyBulkCString(c, args[j].token);
-            maplen++;
         }
         if (args[j].summary) {
             addReplyBulkCString(c, "summary");
             addReplyBulkCString(c, args[j].summary);
-            maplen++;
         }
         if (args[j].since) {
             addReplyBulkCString(c, "since");
             addReplyBulkCString(c, args[j].since);
-            maplen++;
         }
         if (args[j].flags) {
             addReplyBulkCString(c, "flags");
             addReplyFlagsForArg(c, args[j].flags);
-            maplen++;
         }
         if (args[j].type == ARG_TYPE_ONEOF || args[j].type == ARG_TYPE_BLOCK) {
             addReplyBulkCString(c, "arguments");
-            addReplyCommandArgList(c, args[j].subargs);
+            addReplyCommandArgList(c, args[j].subargs, args[j].num_args);
-            maplen++;
         }
-        setDeferredMapLen(c, mapreply, maplen);
     }
-    setDeferredSetLen(c, setreply, j);
 }
 
 /* Must match redisCommandRESP2Type */
@@ -4178,25 +4214,19 @@ const char *RESP3_TYPE_STR[] = {
 };
 
 void addReplyCommandHistory(client *c, struct redisCommand *cmd) {
-    int j;
+    addReplySetLen(c, cmd->num_history);
-
+    for (int j = 0; j<cmd->num_history; j++) {
-    void *array = addReplyDeferredLen(c);
-    for (j = 0; cmd->history && cmd->history[j].since != NULL; j++) {
         addReplyArrayLen(c, 2);
         addReplyBulkCString(c, cmd->history[j].since);
         addReplyBulkCString(c, cmd->history[j].changes);
     }
-    setDeferredSetLen(c, array, j);
 }
 
 void addReplyCommandHints(client *c, struct redisCommand *cmd) {
-    int j;
+    addReplySetLen(c, cmd->num_hints);
-
+    for (int j = 0; j<cmd->num_hints; j++) {
-    void *array = addReplyDeferredLen(c);
-    for (j = 0; cmd->hints && cmd->hints[j] != NULL; j++) {
         addReplyBulkCString(c, cmd->hints[j]);
     }
-    setDeferredSetLen(c, array, j);
 }
 
 void addReplyCommandKeySpecs(client *c, struct redisCommand *cmd) {
@@ -4287,20 +4317,24 @@ void addReplyCommandKeySpecs(client *c, struct redisCommand *cmd) {
     }
 }
 
-void addReplyCommand(client *c, struct redisCommand *cmd);
+/* Reply with an array of sub-command using the provided reply callback. */
-
+void addReplyCommandSubCommands(client *c, struct redisCommand *cmd, void (*reply_function)(client*, struct redisCommand*), int use_map) {
-void addReplyCommandSubCommands(client *c, struct redisCommand *cmd) {
     if (!cmd->subcommands_dict) {
         addReplySetLen(c, 0);
         return;
     }
 
+    if (use_map)
+        addReplyMapLen(c, dictSize(cmd->subcommands_dict));
+    else
         addReplyArrayLen(c, dictSize(cmd->subcommands_dict));
     dictEntry *de;
     dictIterator *di = dictGetSafeIterator(cmd->subcommands_dict);
     while((de = dictNext(di)) != NULL) {
         struct redisCommand *sub = (struct redisCommand *)dictGetVal(de);
-        addReplyCommand(c,sub);
+        if (use_map)
+            addReplyBulkSds(c, getFullCommandName(sub));
+        reply_function(c, sub);
     }
     dictReleaseIterator(di);
 }
@@ -4327,8 +4361,8 @@ const char *COMMAND_GROUP_STR[] = {
     "module"
 };
 
-/* Output the representation of a Redis command. Used by the COMMAND command. */
+/* Output the representation of a Redis command. Used by the COMMAND command and COMMAND INFO. */
-void addReplyCommand(client *c, struct redisCommand *cmd) {
+void addReplyCommandInfo(client *c, struct redisCommand *cmd) {
     if (!cmd) {
         addReplyNull(c);
     } else {
@@ -4341,8 +4375,10 @@ void addReplyCommand(client *c, struct redisCommand *cmd) {
             keystep = cmd->legacy_range_key_spec.fk.range.keystep;
         }
 
-        /* We are adding: command name, arg count, flags, first, last, offset, categories, additional information (map) */
+        addReplyArrayLen(c, 10);
-        addReplyArrayLen(c, 8);
+        if (cmd->parent)
+            addReplyBulkSds(c, getFullCommandName(cmd));
+        else
             addReplyBulkCString(c, cmd->name);
         addReplyLongLong(c, cmd->arity);
         addReplyFlagsForCommand(c, cmd);
@@ -4350,63 +4386,61 @@ void addReplyCommand(client *c, struct redisCommand *cmd) {
         addReplyLongLong(c, lastkey);
         addReplyLongLong(c, keystep);
         addReplyCommandCategories(c, cmd);
-        long maplen = 0;
+        addReplyCommandHints(c, cmd);
-        void *mapreply = addReplyDeferredLen(c);
+        addReplyCommandKeySpecs(c, cmd);
+        addReplyCommandSubCommands(c, cmd, addReplyCommandInfo, 0);
+    }
+}
+
+/* Output the representation of a Redis command. Used by the COMMAND DOCS. */
+void addReplyCommandDocs(client *c, struct redisCommand *cmd) {
+    /* Count our reply len so we don't have to use deferred reply. */
+    long maplen = 3;
+    if (cmd->complexity) maplen++;
+    if (cmd->doc_flags) maplen++;
+    if (cmd->deprecated_since) maplen++;
+    if (cmd->replaced_by) maplen++;
+    if (cmd->history) maplen++;
+    if (cmd->args) maplen++;
+    if (cmd->subcommands_dict) maplen++;
+    addReplyMapLen(c, maplen);
+
     addReplyBulkCString(c, "summary");
     addReplyBulkCString(c, cmd->summary);
-        maplen++;
+
     addReplyBulkCString(c, "since");
     addReplyBulkCString(c, cmd->since);
-        maplen++;
+
     addReplyBulkCString(c, "group");
     addReplyBulkCString(c, COMMAND_GROUP_STR[cmd->group]);
-        maplen++;
+
     if (cmd->complexity) {
         addReplyBulkCString(c, "complexity");
         addReplyBulkCString(c, cmd->complexity);
-            maplen++;
     }
     if (cmd->doc_flags) {
         addReplyBulkCString(c, "doc_flags");
         addReplyDocFlagsForCommand(c, cmd);
-            maplen++;
     }
     if (cmd->deprecated_since) {
         addReplyBulkCString(c, "deprecated_since");
         addReplyBulkCString(c, cmd->deprecated_since);
-            maplen++;
     }
     if (cmd->replaced_by) {
         addReplyBulkCString(c, "replaced_by");
         addReplyBulkCString(c, cmd->replaced_by);
-            maplen++;
     }
     if (cmd->history) {
         addReplyBulkCString(c, "history");
         addReplyCommandHistory(c, cmd);
-            maplen++;
-        }
-        if (cmd->hints) {
-            addReplyBulkCString(c, "hints");
-            addReplyCommandHints(c, cmd);
-            maplen++;
     }
     if (cmd->args) {
         addReplyBulkCString(c, "arguments");
-            addReplyCommandArgList(c, cmd->args);
+        addReplyCommandArgList(c, cmd->args, cmd->num_args);
-            maplen++;
-        }
-        if (cmd->key_specs_num) {
-            addReplyBulkCString(c, "key_specs");
-            addReplyCommandKeySpecs(c, cmd);
-            maplen++;
     }
     if (cmd->subcommands_dict) {
         addReplyBulkCString(c, "subcommands");
-            addReplyCommandSubCommands(c, cmd);
+        addReplyCommandSubCommands(c, cmd, addReplyCommandDocs, 1);
-            maplen++;
-        }
-        setDeferredMapLen(c, mapreply, maplen);
     }
 }
 
@@ -4452,7 +4486,7 @@ void commandCommand(client *c) {
     addReplyArrayLen(c, dictSize(server.commands));
     di = dictGetIterator(server.commands);
     while ((de = dictNext(di)) != NULL) {
-        addReplyCommand(c, dictGetVal(de));
+        addReplyCommandInfo(c, dictGetVal(de));
     }
     dictReleaseIterator(di);
 }
@@ -4561,12 +4595,58 @@ void commandListCommand(client *c) {
     dictReleaseIterator(di);
 }
 
-/* COMMAND INFO <command-name> [<command-name> ...] */
+/* COMMAND INFO [<command-name> ...] */
 void commandInfoCommand(client *c) {
     int i;
+
+    if (c->argc == 2) {
+        dictIterator *di;
+        dictEntry *de;
+        addReplyArrayLen(c, dictSize(server.commands));
+        di = dictGetIterator(server.commands);
+        while ((de = dictNext(di)) != NULL) {
+            addReplyCommandInfo(c, dictGetVal(de));
+        }
+        dictReleaseIterator(di);
+    } else {
         addReplyArrayLen(c, c->argc-2);
         for (i = 2; i < c->argc; i++) {
-        addReplyCommand(c, lookupCommandBySds(c->argv[i]->ptr));
+            addReplyCommandInfo(c, lookupCommandBySds(c->argv[i]->ptr));
+        }
+    }
+}
+
+/* COMMAND DOCS [<command-name> ...] */
+void commandDocsCommand(client *c) {
+    int i;
+    if (c->argc == 2) {
+        /* Reply with an array of all commands */
+        dictIterator *di;
+        dictEntry *de;
+        addReplyMapLen(c, dictSize(server.commands));
+        di = dictGetIterator(server.commands);
+        while ((de = dictNext(di)) != NULL) {
+            struct redisCommand *cmd = dictGetVal(de);
+            addReplyBulkCString(c, cmd->name);
+            addReplyCommandDocs(c, cmd);
+        }
+        dictReleaseIterator(di);
+    } else {
+        /* Reply with an array of the requested commands (if we find them) */
+        int numcmds = 0;
+        void *replylen = addReplyDeferredLen(c);
+        for (i = 2; i < c->argc; i++) {
+            struct redisCommand *cmd = lookupCommandBySds(c->argv[i]->ptr);
+            if (!cmd)
+                continue;
+            if (cmd->parent)
+                addReplyBulkSds(c, getFullCommandName(cmd));
+            else
+                addReplyBulkCString(c, cmd->name);
+            addReplyCommandDocs(c, cmd);
+            numcmds++;
+        }
+        setDeferredMapLen(c,replylen,numcmds);
     }
 }
 
@@ -4584,8 +4664,14 @@ void commandHelpCommand(client *c) {
 "    Return the total number of commands in this Redis server.",
 "LIST",
 "    Return a list of all commands in this Redis server.",
-"INFO <command-name> [<command-name> ...]",
+"INFO [<command-name> ...]",
 "    Return details about multiple Redis commands.",
+"    If no command names are given, documentation details for all",
+"    commands are returned.",
+"DOCS [<command-name> ...]",
+"    Return documentation details about multiple Redis commands.",
+"    If no command names are given, documentation details for all",
+"    commands are returned.",
 "GETKEYS <full-command>",
 "    Return the keys from a full Redis command.",
 NULL

src/server.h

@@ -208,6 +208,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define CMD_MODULE_NO_CLUSTER (1ULL<<22) /* Deny on Redis Cluster. */
 #define CMD_NO_ASYNC_LOADING (1ULL<<23)
 #define CMD_NO_MULTI (1ULL<<24)
+#define CMD_MOVABLE_KEYS (1ULL<<25) /* populated by populateCommandMovableKeys */
 
 /* Command flags that describe ACLs categories. */
 #define ACL_CATEGORY_KEYSPACE (1ULL<<0)
@@ -2003,6 +2004,8 @@ typedef struct redisCommandArg {
     const char *since;
     int flags;
     struct redisCommandArg *subargs;
+    /* runtime populated data */
+    int num_args;
 } redisCommandArg;
 
 /* Must be synced with RESP2_TYPE_STR and generate-command-code.py */
@@ -2178,7 +2181,7 @@ struct redisCommand {
     /* Array of arguments (may be NULL) */
     struct redisCommandArg *args;
 
-    /* Runtime data */
+    /* Runtime populated data */
     /* What keys should be loaded in background when calling this command? */
     long long microseconds, calls, rejected_calls, failed_calls;
     int id;     /* Command ID. This is a progressive ID starting from 0 that
@@ -2192,9 +2195,11 @@ struct redisCommand {
                                      * still maintained (if applicable) so that
                                      * we can still support the reply format of
                                      * COMMAND INFO and COMMAND GETKEYS */
+    int num_args;
+    int num_history;
+    int num_hints;
     int key_specs_num;
     int key_specs_max;
-    int movablekeys; /* See populateCommandMovableKeys */
     dict *subcommands_dict;
     struct redisCommand *parent;
 };
@@ -3090,6 +3095,7 @@ void commandListCommand(client *c);
 void commandInfoCommand(client *c);
 void commandGetKeysCommand(client *c);
 void commandHelpCommand(client *c);
+void commandDocsCommand(client *c);
 void setCommand(client *c);
 void setnxCommand(client *c);
 void setexCommand(client *c);

tests/unit/moduleapi/keyspecs.tcl

@@ -4,53 +4,38 @@ start_server {tags {"modules"}} {
     r module load $testmodule
 
     test "Module key specs: Legacy" {
-        set reply [r command info kspec.legacy]
+        set reply [lindex [r command info kspec.legacy] 0]
         # Verify (first, last, step)
-        assert_equal [lindex [lindex $reply 0] 3] 1
+        assert_equal [lindex $reply 3] 1
-        assert_equal [lindex [lindex $reply 0] 4] 2
+        assert_equal [lindex $reply 4] 2
-        assert_equal [lindex [lindex $reply 0] 5] 1
+        assert_equal [lindex $reply 5] 1
-        # create a dict for easy lookup
-        unset -nocomplain mydict
-        foreach {k v} [lindex [lindex $reply 0] 7] {
-            dict append mydict $k $v
-        }
         # Verify key-specs
-        set keyspecs [dict get $mydict key_specs]
+        set keyspecs [lindex $reply 8]
         assert_equal [lindex $keyspecs 0] {flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
         assert_equal [lindex $keyspecs 1] {flags write begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
     }
 
     test "Module key specs: Complex specs, case 1" {
-        set reply [r command info kspec.complex1]
+        set reply [lindex [r command info kspec.complex1] 0]
         # Verify (first, last, step)
-        assert_equal [lindex [lindex $reply 0] 3] 1
+        assert_equal [lindex $reply 3] 1
-        assert_equal [lindex [lindex $reply 0] 4] 1
+        assert_equal [lindex $reply 4] 1
-        assert_equal [lindex [lindex $reply 0] 5] 1
+        assert_equal [lindex $reply 5] 1
-        # create a dict for easy lookup
-        unset -nocomplain mydict
-        foreach {k v} [lindex [lindex $reply 0] 7] {
-            dict append mydict $k $v
-        }
         # Verify key-specs
-        set keyspecs [dict get $mydict key_specs]
+        set keyspecs [lindex $reply 8]
         assert_equal [lindex $keyspecs 0] {flags {} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
         assert_equal [lindex $keyspecs 1] {flags write begin_search {type keyword spec {keyword STORE startfrom 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
         assert_equal [lindex $keyspecs 2] {flags read begin_search {type keyword spec {keyword KEYS startfrom 2}} find_keys {type keynum spec {keynumidx 0 firstkey 1 keystep 1}}}
     }
 
     test "Module key specs: Complex specs, case 2" {
-        set reply [r command info kspec.complex2]
+        set reply [lindex [r command info kspec.complex2] 0]
         # Verify (first, last, step)
-        assert_equal [lindex [lindex $reply 0] 3] 1
+        assert_equal [lindex $reply 3] 1
-        assert_equal [lindex [lindex $reply 0] 4] 2
+        assert_equal [lindex $reply 4] 2
-        assert_equal [lindex [lindex $reply 0] 5] 1
+        assert_equal [lindex $reply 5] 1
-        # create a dict for easy lookup
-        unset -nocomplain mydict
-        foreach {k v} [lindex [lindex $reply 0] 7] {
-            dict append mydict $k $v
-        }
         # Verify key-specs
-        set keyspecs [dict get $mydict key_specs]
+        set keyspecs [lindex $reply 8]
         assert_equal [lindex $keyspecs 0] {flags write begin_search {type keyword spec {keyword STORE startfrom 5}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
         assert_equal [lindex $keyspecs 1] {flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
         assert_equal [lindex $keyspecs 2] {flags read begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}

tests/unit/moduleapi/subcommands.tcl

@@ -5,15 +5,18 @@ start_server {tags {"modules"}} {
 
     test "Module subcommands via COMMAND" {
         # Verify that module subcommands are displayed correctly in COMMAND
-        set reply [r command info subcommands.bitarray]
+        set command_reply [r command info subcommands.bitarray]
-        # create a dict for easy lookup
+        set first_cmd [lindex $command_reply 0]
-        unset -nocomplain mydict
+        set subcmds_in_command [lsort [lindex $first_cmd 9]]
-        foreach {k v} [lindex [lindex $reply 0] 7] {
+        assert_equal [lindex $subcmds_in_command 0] {subcommands.bitarray|get -2 module 1 1 1 {} {} {{flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}} {}}
-            dict append mydict $k $v
+        assert_equal [lindex $subcmds_in_command 1] {subcommands.bitarray|set -2 module 1 1 1 {} {} {{flags write begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}} {}}
-        }
+
-        set subcmds [lsort [dict get $mydict subcommands]]
+        # Verify that module subcommands are displayed correctly in COMMAND DOCS
-        assert_equal [lindex $subcmds 0] {get -2 module 1 1 1 {} {summary {} since {} group module key_specs {{flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}}}}
+        set docs_reply [r command docs subcommands.bitarray]
-        assert_equal [lindex $subcmds 1] {set -2 module 1 1 1 {} {summary {} since {} group module key_specs {{flags write begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}}}}
+        set docs [dict create {*}[lindex $docs_reply 1]]
+        set subcmds_in_cmd_docs [dict create {*}[dict get $docs subcommands]]
+        assert_equal [dict get $subcmds_in_cmd_docs "subcommands.bitarray|get"] {summary {} since {} group module}
+        assert_equal [dict get $subcmds_in_cmd_docs "subcommands.bitarray|set"] {summary {} since {} group module}
     }
 
     test "Module pure-container command fails on arity error" {

tests/unit/type/hash.tcl

@@ -98,10 +98,7 @@ start_server {tags {"hash"}} {
             assert_encoding $type myhash
 
             # create a dict for easy lookup
-            unset -nocomplain mydict
+            set mydict [dict create {*}[r hgetall myhash]]
-            foreach {k v} [r hgetall myhash] {
-                dict append mydict $k $v
-            }
 
             # We'll stress different parts of the code, see the implementation
             # of HRANDFIELD for more information, but basically there are

tests/unit/type/zset.tcl

@@ -2262,10 +2262,7 @@ start_server {tags {"zset"}} {
             assert_encoding $type myzset
 
             # create a dict for easy lookup
-            unset -nocomplain mydict
+            set mydict [dict create {*}[r zrange myzset 0 -1 withscores]]
-            foreach {k v} [r zrange myzset 0 -1 withscores] {
-                dict append mydict $k $v
-            }
 
             # We'll stress different parts of the code, see the implementation
             # of ZRANDMEMBER for more information, but basically there are

utils/generate-commands-json.py

@@ -1,8 +1,11 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import argparse
 import json
+import subprocess
 from collections import OrderedDict
 from sys import argv, stdin
+import os
+
 
 def convert_flags_to_boolean_dict(flags):
     """Return a dict with a key set to `True` per element in the flags list."""
@@ -19,7 +22,7 @@ def convert_argument(arg):
     """Transform an argument."""
     arg.update(convert_flags_to_boolean_dict(arg.pop('flags', [])))
     set_if_not_none_or_empty(arg, 'arguments',
-                            [convert_argument(x) for x in arg.pop('arguments',[])])
+                             [convert_argument(x) for x in arg.pop('arguments', [])])
     return arg
 
 
@@ -29,85 +32,103 @@ def convert_keyspec(spec):
     return spec
 
 
-def convert_entry_to_objects_array(container, cmd):
+def convert_entry_to_objects_array(cmd, docs):
     """Transform the JSON output of `COMMAND` to a friendlier format.
 
-    `COMMAND`'s output per command is a fixed-size (8) list as follows:
+    cmd is the output of `COMMAND` as follows:
     1. Name (lower case, e.g. "lolwut")
     2. Arity
     3. Flags
     4-6. First/last/step key specification (deprecated as of Redis v7.0)
     7. ACL categories
-    8. A dict of meta information (as of Redis 7.0)
+    8. hints (as of Redis 7.0)
+    9. key-specs (as of Redis 7.0)
+    10. subcommands (as of Redis 7.0)
+
+    docs is the output of `COMMAND DOCS`, which holds a map of additional metadata
 
     This returns a list with a dict for the command and per each of its
     subcommands. Each dict contains one key, the command's full name, with a
     value of a dict that's set with the command's properties and meta
     information."""
-    assert len(cmd) >= 8
+    assert len(cmd) >= 9
     obj = {}
     rep = [obj]
     name = cmd[0].upper()
     arity = cmd[1]
     command_flags = cmd[2]
     acl_categories = cmd[6]
-    meta = cmd[7]
+    hints = cmd[7]
-    key = f'{container} {name}' if container else name
+    keyspecs = cmd[8]
+    subcommands = cmd[9] if len(cmd) > 9 else []
+    key = name.replace('|', ' ')
 
-    rep.extend([convert_entry_to_objects_array(name, x)[0] for x in meta.pop('subcommands', [])])
+    subcommand_docs = docs.pop('subcommands', [])
+    rep.extend([convert_entry_to_objects_array(x, subcommand_docs[x[0]])[0] for x in subcommands])
 
     # The command's value is ordered so the interesting stuff that we care about
     # is at the start. Optional `None` and empty list values are filtered out.
     value = OrderedDict()
-    value['summary'] = meta.pop('summary')
+    value['summary'] = docs.pop('summary')
-    value['since'] = meta.pop('since')
+    value['since'] = docs.pop('since')
-    value['group'] = meta.pop('group')
+    value['group'] = docs.pop('group')
-    set_if_not_none_or_empty(value, 'complexity', meta.pop('complexity', None))
+    set_if_not_none_or_empty(value, 'complexity', docs.pop('complexity', None))
-    set_if_not_none_or_empty(value, 'deprecated_since', meta.pop('deprecated_since', None))
+    set_if_not_none_or_empty(value, 'deprecated_since', docs.pop('deprecated_since', None))
-    set_if_not_none_or_empty(value, 'replaced_by', meta.pop('replaced_by', None))
+    set_if_not_none_or_empty(value, 'replaced_by', docs.pop('replaced_by', None))
-    set_if_not_none_or_empty(value, 'history', meta.pop('history', []))
+    set_if_not_none_or_empty(value, 'history', docs.pop('history', []))
     set_if_not_none_or_empty(value, 'acl_categories', acl_categories)
     value['arity'] = arity
     set_if_not_none_or_empty(value, 'key_specs',
-                            [convert_keyspec(x) for x in meta.pop('key_specs',[])])
+                             [convert_keyspec(x) for x in keyspecs])
     set_if_not_none_or_empty(value, 'arguments',
-                            [convert_argument(x) for x in meta.pop('arguments', [])])
+                             [convert_argument(x) for x in docs.pop('arguments', [])])
     set_if_not_none_or_empty(value, 'command_flags', command_flags)
-    set_if_not_none_or_empty(value, 'doc_flags', meta.pop('doc_flags', []))
+    set_if_not_none_or_empty(value, 'doc_flags', docs.pop('doc_flags', []))
-    set_if_not_none_or_empty(value, 'hints', meta.pop('hints', []))
+    set_if_not_none_or_empty(value, 'hints', hints)
 
-    # All remaining meta key-value tuples, if any, are appended to the command
+    # All remaining docs key-value tuples, if any, are appended to the command
     # to be future-proof.
-    while len(meta) > 0:
+    while len(docs) > 0:
-        (k, v) = meta.popitem()
+        (k, v) = docs.popitem()
         value[k] = v
 
     obj[key] = value
     return rep
 
 
+# Figure out where the sources are
+srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src")
+
 # MAIN
 if __name__ == '__main__':
     opts = {
-        'description': 'Transform the output from `redis-cli --json COMMAND` to commands.json format.',
+        'description': 'Transform the output from `redis-cli --json` using COMMAND and COMMAND DOCS to a single commands.json format.',
-        'epilog': f'Usage example: src/redis-cli --json COMMAND | {argv[0]}'
+        'epilog': f'Usage example: {argv[0]} --cli src/redis-cli --port 6379 > commands.json'
     }
     parser = argparse.ArgumentParser(**opts)
-    parser.add_argument('input', help='JSON-formatted input file (default: stdin)',
+    parser.add_argument('--host', type=str, default='localhost')
-                        nargs='?', type=argparse.FileType(), default=stdin)
+    parser.add_argument('--port', type=int, default=6379)
+    parser.add_argument('--cli', type=str, default='%s/redis-cli' % srcdir)
     args = parser.parse_args()
 
     payload = OrderedDict()
-    commands = []
+    cmds = []
-    data = json.load(args.input)
 
-    for entry in data:
+    p = subprocess.Popen([args.cli, '-h', args.host, '-p', str(args.port), '--json', 'command'], stdout=subprocess.PIPE)
-        cmds = convert_entry_to_objects_array(None, entry)
+    stdout, stderr = p.communicate()
-        commands.extend(cmds)
+    commands = json.loads(stdout)
+
+    p = subprocess.Popen([args.cli, '-h', args.host, '-p', str(args.port), '--json', 'command', 'docs'], stdout=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    docs = json.loads(stdout)
+
+    for entry in commands:
+        cmd = convert_entry_to_objects_array(entry, docs[entry[0]])
+        cmds.extend(cmd)
 
     # The final output is a dict of all commands, ordered by name.
-    commands.sort(key=lambda x: list(x.keys())[0])
+    cmds.sort(key=lambda x: list(x.keys())[0])
-    for cmd in commands:
+    for cmd in cmds:
         name = list(cmd.keys())[0]
         payload[name] = cmd[name]