From f810510bb2f96c19b82b59222ae63941eedc9c20 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@garantiadata.com>
Date: Mon, 27 Dec 2021 19:31:13 +0200
Subject: [PATCH] Adds utils/gen-commands-json.py (#9958)
Following #9656, this script generates a "commands.json" file from the output
of the new COMMAND. The output of this script is used in redis/redis-doc#1714
and by redis/redis-io#259. This also converts a couple of rogue dashes (in
'key-specs' and 'multiple-token' flags) to underscores (continues #9959).
---
src/server.c | 4 +-
tests/unit/moduleapi/keyspecs.tcl | 6 +-
tests/unit/moduleapi/subcommands.tcl | 4 +-
utils/generate-commands-json.py | 114 +++++++++++++++++++++++++++
4 files changed, 121 insertions(+), 7 deletions(-)
create mode 100755 utils/generate-commands-json.py
diff --git a/src/server.c b/src/server.c
index e1f7919b7..fa40771ab 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3836,7 +3836,7 @@ void addReplyFlagsForArg(client *c, uint64_t flags) {
void *flaglen = addReplyDeferredLen(c);
flagcount += addReplyCommandFlag(c,flags,CMD_ARG_OPTIONAL, "optional");
flagcount += addReplyCommandFlag(c,flags,CMD_ARG_MULTIPLE, "multiple");
- flagcount += addReplyCommandFlag(c,flags,CMD_ARG_MULTIPLE_TOKEN, "multiple-token");
+ flagcount += addReplyCommandFlag(c,flags,CMD_ARG_MULTIPLE_TOKEN, "multiple_token");
setDeferredSetLen(c, flaglen, flagcount);
}
@@ -4133,7 +4133,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) {
maplen++;
}
if (cmd->key_specs_num) {
- addReplyBulkCString(c, "key-specs");
+ addReplyBulkCString(c, "key_specs");
addReplyCommandKeySpecs(c, cmd);
maplen++;
}
diff --git a/tests/unit/moduleapi/keyspecs.tcl b/tests/unit/moduleapi/keyspecs.tcl
index 265b9e9ee..1358b4f32 100644
--- a/tests/unit/moduleapi/keyspecs.tcl
+++ b/tests/unit/moduleapi/keyspecs.tcl
@@ -15,7 +15,7 @@ start_server {tags {"modules"}} {
dict append mydict $k $v
}
# Verify key-specs
- set keyspecs [dict get $mydict key-specs]
+ set keyspecs [dict get $mydict key_specs]
assert_equal [lindex $keyspecs 0] {flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
assert_equal [lindex $keyspecs 1] {flags write begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
}
@@ -32,7 +32,7 @@ start_server {tags {"modules"}} {
dict append mydict $k $v
}
# Verify key-specs
- set keyspecs [dict get $mydict key-specs]
+ set keyspecs [dict get $mydict key_specs]
assert_equal [lindex $keyspecs 0] {flags {} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
assert_equal [lindex $keyspecs 1] {flags write begin_search {type keyword spec {keyword STORE startfrom 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
assert_equal [lindex $keyspecs 2] {flags read begin_search {type keyword spec {keyword KEYS startfrom 2}} find_keys {type keynum spec {keynumidx 0 firstkey 1 keystep 1}}}
@@ -50,7 +50,7 @@ start_server {tags {"modules"}} {
dict append mydict $k $v
}
# Verify key-specs
- set keyspecs [dict get $mydict key-specs]
+ set keyspecs [dict get $mydict key_specs]
assert_equal [lindex $keyspecs 0] {flags write begin_search {type keyword spec {keyword STORE startfrom 5}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
assert_equal [lindex $keyspecs 1] {flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
assert_equal [lindex $keyspecs 2] {flags read begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}
diff --git a/tests/unit/moduleapi/subcommands.tcl b/tests/unit/moduleapi/subcommands.tcl
index 9be6f5cea..8de4ccbdb 100644
--- a/tests/unit/moduleapi/subcommands.tcl
+++ b/tests/unit/moduleapi/subcommands.tcl
@@ -12,8 +12,8 @@ start_server {tags {"modules"}} {
dict append mydict $k $v
}
set subcmds [lsort [dict get $mydict subcommands]]
- assert_equal [lindex $subcmds 0] {get -2 module 1 1 1 {} {summary {} since {} group module key-specs {{flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}}}}
- assert_equal [lindex $subcmds 1] {set -2 module 1 1 1 {} {summary {} since {} group module key-specs {{flags write begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}}}}
+ assert_equal [lindex $subcmds 0] {get -2 module 1 1 1 {} {summary {} since {} group module key_specs {{flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}}}}
+ assert_equal [lindex $subcmds 1] {set -2 module 1 1 1 {} {summary {} since {} group module key_specs {{flags write begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}}}}
}
test "Module pure-container command fails on arity error" {
diff --git a/utils/generate-commands-json.py b/utils/generate-commands-json.py
new file mode 100755
index 000000000..8e6d915df
--- /dev/null
+++ b/utils/generate-commands-json.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+import argparse
+import json
+from collections import OrderedDict
+from sys import argv, stdin
+
+def convert_flags_to_boolean_dict(flags):
+ """Return a dict with a key set to `True` per element in the flags list."""
+ return {f: True for f in flags}
+
+
+def set_if_not_none_or_empty(dst, key, value):
+ """Set 'key' in 'dst' if 'value' is not `None` or an empty list."""
+ if value is not None and (type(value) is not list or len(value)):
+ dst[key] = value
+
+
+def convert_argument(arg):
+ """Transform an argument."""
+ arg.update(convert_flags_to_boolean_dict(arg.pop('flags', [])))
+ set_if_not_none_or_empty(arg, 'arguments',
+ [convert_argument(x) for x in arg.pop('arguments',[])])
+ return arg
+
+
+def convert_keyspec(spec):
+ """Transform a key spec."""
+ spec.update(convert_flags_to_boolean_dict(spec.pop('flags', [])))
+ return spec
+
+
+def convert_entry_to_objects_array(container, cmd):
+ """Transform the JSON output of `COMMAND` to a friendlier format.
+
+ `COMMAND`'s output per command is a fixed-size (8) list as follows:
+ 1. Name (lower case, e.g. "lolwut")
+ 2. Arity
+ 3. Flags
+ 4-6. First/last/step key specification (deprecated as of Redis v7.0)
+ 7. ACL categories
+ 8. A dict of meta information (as of Redis 7.0)
+
+ This returns a list with a dict for the command and per each of its
+ subcommands. Each dict contains one key, the command's full name, with a
+ value of a dict that's set with the command's properties and meta
+ information."""
+ assert len(cmd) >= 8
+ obj = {}
+ rep = [obj]
+ name = cmd[0].upper()
+ arity = cmd[1]
+ command_flags = cmd[2]
+ acl_categories = cmd[6]
+ meta = cmd[7]
+ key = f'{container} {name}' if container else name
+
+ rep.extend([convert_entry_to_objects_array(name, x)[0] for x in meta.pop('subcommands', [])])
+
+ # The command's value is ordered so the interesting stuff that we care about
+ # is at the start. Optional `None` and empty list values are filtered out.
+ value = OrderedDict()
+ value['summary'] = meta.pop('summary')
+ value['since'] = meta.pop('since')
+ value['group'] = meta.pop('group')
+ set_if_not_none_or_empty(value, 'complexity', meta.pop('complexity', None))
+ set_if_not_none_or_empty(value, 'deprecated_since', meta.pop('deprecated_since', None))
+ set_if_not_none_or_empty(value, 'replaced_by', meta.pop('replaced_by', None))
+ set_if_not_none_or_empty(value, 'history', meta.pop('history', []))
+ set_if_not_none_or_empty(value, 'acl_categories', acl_categories)
+ value['arity'] = arity
+ set_if_not_none_or_empty(value, 'key_specs',
+ [convert_keyspec(x) for x in meta.pop('key_specs',[])])
+ set_if_not_none_or_empty(value, 'arguments',
+ [convert_argument(x) for x in meta.pop('arguments', [])])
+ set_if_not_none_or_empty(value, 'command_flags', command_flags)
+ set_if_not_none_or_empty(value, 'doc_flags', meta.pop('doc_flags', []))
+ set_if_not_none_or_empty(value, 'hints', meta.pop('hints', []))
+
+ # All remaining meta key-value tuples, if any, are appended to the command
+ # to be future-proof.
+ while len(meta) > 0:
+ (k, v) = meta.popitem()
+ value[k] = v
+
+ obj[key] = value
+ return rep
+
+
+# MAIN
+if __name__ == '__main__':
+ opts = {
+ 'description': 'Transform the output from `redis-cli --json COMMAND` to commands.json format.',
+ 'epilog': f'Usage example: src/redis-cli --json COMMAND | {argv[0]}'
+ }
+ parser = argparse.ArgumentParser(**opts)
+ parser.add_argument('input', help='JSON-formatted input file (default: stdin)',
+ nargs='?', type=argparse.FileType(), default=stdin)
+ args = parser.parse_args()
+
+ payload = OrderedDict()
+ commands = []
+ data = json.load(args.input)
+
+ for entry in data:
+ cmds = convert_entry_to_objects_array(None, entry)
+ commands.extend(cmds)
+
+ # The final output is a dict of all commands, ordered by name.
+ commands.sort(key=lambda x: list(x.keys())[0])
+ for cmd in commands:
+ name = list(cmd.keys())[0]
+ payload[name] = cmd[name]
+
+ print(json.dumps(payload, indent=4))