diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..eece8c7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "chrono": "c" + } +} \ No newline at end of file diff --git a/README.md b/README.md index 79b690f..7b348ce 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This module aims to provide simple DML to manipulate the hashes in REDIS for SQL users. It works as simple as you expected. It translates the input statement to a set of pure REDIS commands. It does not need nor generate any intermediate stuffs which occupied your storages. The target data is your hashes only. -## Example +## Usage ```bash $ redis-cli 127.0.0.1:6379> hmset phonebook:0001 name "Peter Nelson" tel "1-456-1246-3421" birth "2019-10-01" pos 3 gender "M" @@ -18,6 +18,71 @@ $ redis-cli 2) "Betty Joan" 3) tel 4) "1-444-9999-1112" +``` + +## Getting started + +### Get the package and build the binary: +```bash +$ git clone https://github.com/cscan/dbx.git +$ cd dbx/src && make +``` + +This plugin library is written in pure C. A file dbx.so is built after successfully compiled. + +### Load the module in redis (3 ways) + +1. Load the module in CLI +```bash +127.0.0.1:6379> module load /path/to/dbx.so +``` + +2. Start the server with loadmodule argument +```bash +$ redis-server --loadmodule /path/to/dbx.so +``` + +3. Adding the following line in the file redis.conf and then restart the server +```bash +loadmodule /path/to/dbx.so +``` + +If you still have problem in loading the module, please visit: https://redis.io/topics/modules-intro + +## More Example + +#### Select Statement +You may specify multiple fields separated by comma +```bash +127.0.0.1:6379> dbx.select name, gender, birth from phonebook +1) 1) name + 2) "Betty Joan" + 3) gender + 4) "F" + 5) birth + 6) "2019-12-01" +2) 1) name + 2) "Mattias Swensson" + 3) gender + 4) "M" + 5) birth + 6) "2017-06-30" +3) 1) name + 2) "Peter Nelson" + 3) gender + 4) "M" + 5) birth + 6) "2019-10-01" +4) 1) name + 2) "Bloody Mary" + 3) gender + 4) "F" + 5) birth + 6) "2018-01-31" +``` + +"*" is support +```bash 127.0.0.1:6379> dbx.select * from phonebook where birth > '2019-11-11' 1) 1) "name" 2) "Betty Joan" @@ -29,6 +94,35 @@ $ redis-cli 8) "1" 9) "gender" 10) "F" +``` + +If you want to show the exact keys, you may try rowid() +```bash +127.0.0.1:6379> dbx.select rowid() from phonebook +1) 1) rowid() + 2) "phonebook:1588299191-764848276" +2) 1) rowid() + 2) "phonebook:1588299202-1052597574" +3) 1) rowid() + 2) "phonebook:1588298418-551514504" +4) 1) rowid() + 2) "phonebook:1588299196-2115347437" +``` + +The above is nearly like REDIS keys command +```bash +127.0.0.1:6379> keys phonebook* +1) "phonebook:1588298418-551514504" +2) "phonebook:1588299196-2115347437" +3) "phonebook:1588299202-1052597574" +4) "phonebook:1588299191-764848276" +``` + +Each record is exactly a hash, you could use raw REDIS commands ``hget, hmget or hgetall`` to retrieve the same content + +#### Where in Select Statement +This module supports =, >, <, >=, <=, <>, != and like conditions. Only single condition is allowed. +```bash 127.0.0.1:6379> dbx.select tel from phonebook where name like Son 1) 1) tel 2) "1-888-3333-1412" @@ -36,34 +130,72 @@ $ redis-cli 2) "1-456-1246-3421" ``` -## Getting started - -### Get the package and build the binary: +#### Order in Select Statement +Ordering can be ascending or descending. All sortings are alpha-sort. ```bash -git clone https://github.com/cscan/dbx.git -cd dbx/src && make +127.0.0.1:6379> dbx.select * from phonebook order by pos asc +... +127.0.0.1:6379> dbx.select * from phonebook order by pos desc +... ``` -This plugin library is written in pure C. A file dbx.so is built after successfully compiled. - -### Load the module in redis - -Load the module in CLI +#### Delete Statement +You may also use Insert and Delete statement to operate the hash ```bash -127.0.0.1:6379> module load /path/to/dbx.so +127.0.0.1:6379> dbx.delete from phonebook where gender = F +(integer) 2 +127.0.0.1:6379> dbx.delete from phonebook +(integer) 2 ``` -Start the server with loadmodule argument +#### Insert Statement ```bash -$ redis-server --loadmodule /path/to/dbx.so +127.0.0.1:6379> dbx.insert into phonebook (name,tel,birth,pos,gender) values ('Peter Nelson' ,1-456-1246-3421, 2019-10-01, 3, M) +"phonebook:1588298418-551514504" +127.0.0.1:6379> dbx.insert into phonebook (name,tel,birth,pos,gender) values ('Betty Joan' ,1-444-9999-1112, 2019-12-01, 1, F) +"phonebook:1588299191-764848276" +127.0.0.1:6379> dbx.insert into phonebook (name,tel,birth,pos,gender) values ('Bloody Mary' ,1-666-1234-9812, 2018-01-31, 2, F) +"phonebook:1588299196-2115347437" +127.0.0.1:6379> dbx.insert into phonebook (name,tel,birth,pos,gender) values ('Mattias Swensson' ,1-888-3333-1412, 2017-06-30, 4, M) +"phonebook:1588299202-1052597574" +``` +Please be noted that Redis requires at least one space after the single and double quoted arguments. +Or you may quote the whole SQL statement as below: +```bash +127.0.0.1:6379> dbx.insert "into phonebook (name,tel,birth,pos,gender) values ('Peter Nelson','1-456-1246-3421','2019-10-01',3, 'M')" ``` -Adding the following line in the file redis.conf and then restart the server +#### Issue command from BASH shell ```bash -loadmodule /path/to/dbx.so +$ redis-cli dbx.select "*" from phonebook where gender = M order by pos desc +1) 1) "name" + 2) "Mattias Swensson" + 3) "tel" + 4) "1-888-3333-1412" + 5) "birth" + 6) "2017-06-30" + 7) "pos" + 8) "4" + 9) "gender" + 10) "M" +2) 1) "name" + 2) "Peter Nelson" + 3) "tel" + 4) "1-456-1246-3421" + 5) "birth" + 6) "2019-10-01" + 7) "pos" + 8) "3" + 9) "gender" + 10) "M" +$ redis-cli dbx.select name from phonebook where tel like 9812 +1) 1) name + 2) "Bloody Mary" +``` +Note that "*" requires double quoted otherwise it will pass all the filename in current directory. Of course you could quote the whole SQL statement. +```bash +$ redis-cli dbx.select "* from phonebook where gender = M order by pos desc" ``` - -If you still have problem in loading the module, please visit: https://redis.io/topics/modules-intro ## Compatibility REDIS v4.0 diff --git a/src/dbx.c b/src/dbx.c index ffba5d9..bea95af 100644 --- a/src/dbx.c +++ b/src/dbx.c @@ -1,6 +1,7 @@ #include #include #include +#include #include "../redismodule.h" #include "../rmutil/util.h" #include "../rmutil/strings.h" @@ -111,18 +112,23 @@ void showRecord(RedisModuleCtx *ctx, RedisModuleString *key, Vector *vSelect) { } RedisModule_FreeCallReply(tags); } + else if (strcmp(field, "rowid()") == 0) { + RedisModule_ReplyWithSimpleString(ctx, field); + RedisModule_ReplyWithString(ctx, key); + n += 2; + } else { // Display the hash name and content + RedisModule_ReplyWithSimpleString(ctx, field); RedisModuleCallReply *tags = RedisModule_Call(ctx, "HGET", "sc", key, field); if (RedisModule_CallReplyLength(tags) > 0) { RedisModuleString *rms = RedisModule_CreateStringFromCallReply(tags); - RedisModule_ReplyWithSimpleString(ctx, field); RedisModule_ReplyWithString(ctx, rms); RedisModule_FreeString(ctx, rms); } else RedisModule_ReplyWithNull(ctx); // If hash is undefined - n+=2; + n += 2; RedisModule_FreeCallReply(tags); } } @@ -186,7 +192,7 @@ int processRecords(RedisModuleCtx *ctx, RedisModuleCallReply *keys, regex_t *r, /* Create temporary set for sorting */ int buildSetByPattern(RedisModuleCtx *ctx, regex_t *r, char *setName, Vector *vWhere) { - RedisModule_Call(ctx, "del", "c", setName); + RedisModule_Call(ctx, "DEL", "c", setName); RedisModuleString *scursor = RedisModule_CreateStringFromLongLong(ctx, 0); long long lcursor; size_t affected = 0; @@ -416,6 +422,278 @@ int SelectCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return REDISMODULE_OK; } +int InsertCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc < 2) + return RedisModule_WrongArity(ctx); + + // Table + RedisModuleString *intoKey; + + // Process the arguments + size_t plen; + char s[1024] = ""; + for (int i=1; i 0) strcat(s, " "); + const char *temp = RedisModule_StringPtrLen(argv[i], &plen); + if (strlen(s) + plen > 1024) { + RedisModule_ReplyWithError(ctx, "arguments are too long"); + return REDISMODULE_ERR; + } + + if (argc > 2) { // argc > 2 means the arguments are not in quoted. i.e. "..." + char *p = (char*)temp; + while (*p++) *p = *p == 32? 7: *p; // Convert all spaces in tabs, then convert back during parsing + } + strcat(s, temp); + } + + int step = 0; + char temp[1024] = ""; + char stmField[1024] = ""; + char stmValue[1024] = ""; + + char *p; + char *token = strtok(s, " "); + while (token != NULL) { + if (token[0] == 39) { + strcpy(temp, &token[1]); + strcat(temp, " "); + strcat(temp, strtok(NULL, "'")); + strcpy(token, temp); + } + switch(step) { + case 0: + if (strcmp("into", token) == 0) + step = -1; + else { + RedisModule_ReplyWithError(ctx, "into keyword is expected"); + return REDISMODULE_ERR; + } + break; + case -1: + // parse into key, assume time+rand always is new key + intoKey = RMUtil_CreateFormattedString(ctx, "%s:%u-%i", token, (unsigned)time(NULL), rand()); + step = -2; + break; + case -2: + if (token[0] == '(') { + strcpy(stmField, &token[1]); + if (token[strlen(token) - 1] == ')') { + stmField[strlen(stmField) - 1] = 0; + step = -4; + } + else + step = -3; + } + else if (strcmp("values", token) == 0) + step = -5; + else { + RedisModule_ReplyWithError(ctx, "values keyword is expected"); + return REDISMODULE_ERR; + } + break; + case -3: + if (token[strlen(token) - 1] == ')') { + token[strlen(token) - 1] = 0; + strcat(stmField, token); + step = -4; + } + break; + case -4: + if (strcmp("values", token) == 0) + step = -5; + else { + RedisModule_ReplyWithError(ctx, "values keyword is expected"); + return REDISMODULE_ERR; + } + break; + case -5: + case -6: + p = token; + while (*p++) *p = *p == 7? 32: *p; + if (token[0] == '(') { + strcpy(stmValue, &token[1]); + if (token[strlen(token) - 1] == ')') { + stmValue[strlen(stmValue) - 1] = 0; + step = 7; + } + else + step = -6; + } + else if (token[strlen(token) - 1] == ')') { + token[strlen(token) - 1] = 0; + strcat(stmValue, token); + step = 7; + } + else + strcat(stmValue, token); + break; + case 7: + RedisModule_ReplyWithError(ctx, "The end of statement is expected"); + return REDISMODULE_ERR; + break; + } + token = strtok(NULL, " "); + } + + if (step < 7) { + RedisModule_ReplyWithError(ctx, "parse error"); + return REDISMODULE_ERR; + } + + Vector *vField = splitStringByChar(stmField, ","); + Vector *vValue = splitStringByChar(stmValue, ","); + + if (Vector_Size(vField) != Vector_Size(vValue)) { + RedisModule_ReplyWithError(ctx, "Number of values does not match"); + return REDISMODULE_ERR; + } + + RedisModule_AutoMemory(ctx); + + for (size_t i=0; i 0) strcat(s, " "); + const char *temp = RedisModule_StringPtrLen(argv[i], &plen); + if (strlen(s) + plen > 1024) { + RedisModule_ReplyWithError(ctx, "arguments are too long"); + return REDISMODULE_ERR; + } + + if (argc > 2) { // argc > 2 means the arguments are not in quoted. i.e. "..." + char *p = (char*)temp; + while (*p++) *p = *p == 32? 7: *p; // Convert all spaces in tabs, then convert back during parsing + } + strcat(s, temp); + } + + int step = 0; + char temp[1024] = ""; + char stmWhere[1024] = ""; + + char *token = strtok(s, " "); + while (token != NULL) { + // If it is beginning in single quote, find the end quote in the following tokens + if (token[0] == 39) { + strcpy(temp, &token[1]); + strcat(temp, " "); + strcat(temp, strtok(NULL, "'")); + strcpy(token, temp); + } + switch(step) { + case 0: + if (strcmp("from", token) == 0) + step = -1; + else { + RedisModule_ReplyWithError(ctx, "from keyword is expected"); + return REDISMODULE_ERR; + } + break; + case -1: + // parse from statement + fromKeys = RMUtil_CreateFormattedString(ctx, token); + step = 2; + break; + case 2: + if (strcmp("where", token) == 0) + step = -3; + else { + RedisModule_ReplyWithError(ctx, "where statement is expected"); + return REDISMODULE_ERR; + } + break; + case -3: + case 4: + if (strlen(stmWhere) + strlen(token) > 512) { + RedisModule_ReplyWithError(ctx, "where arguments are too long"); + return REDISMODULE_ERR; + } + char *p = token; + while (*p++) *p = *p == 7? 32: *p; + strcat(stmWhere, token); + step = 4; + break; + } + token = strtok(NULL, " "); + } + + if (step <= 0) { + RedisModule_ReplyWithError(ctx, "parse error"); + return REDISMODULE_ERR; + } + + Vector *vWhere = splitWhereString(stmWhere); + + RedisModule_AutoMemory(ctx); + + /* Convert key to regex */ + const char *pat = RedisModule_StringPtrLen(fromKeys, &plen); + regex_t regex; + if (regexCompile(ctx, ®ex, pat)) return REDISMODULE_ERR; + + RedisModuleString *scursor = RedisModule_CreateStringFromLongLong(ctx, 0); + long long lcursor; + size_t affected = 0; + do { + RedisModuleCallReply *rep = RedisModule_Call(ctx, "SCAN", "s", scursor); + + /* Get the current cursor. */ + scursor = RedisModule_CreateStringFromCallReply(RedisModule_CallReplyArrayElement(rep, 0)); + RedisModule_StringToLongLong(scursor, &lcursor); + + /* Filter by pattern matching. */ + RedisModuleCallReply *keys = RedisModule_CallReplyArrayElement(rep, 1); + size_t nKeys = RedisModule_CallReplyLength(keys); + for (size_t i = 0; i < nKeys; i++) { + RedisModuleString *key = RedisModule_CreateStringFromCallReply(RedisModule_CallReplyArrayElement(keys, i)); + size_t l; + const char *s = RedisModule_StringPtrLen(key, &l); + if (!regexec(®ex, s, 1, NULL, 0)) { + if (vWhere == NULL || whereRecord(ctx, key, vWhere)) { + RedisModule_Call(ctx, "DEL", "s", key); + affected++; + } + } + RedisModule_FreeString(ctx, key); + } + + RedisModule_FreeCallReply(rep); + } while (lcursor); + + RedisModule_FreeString(ctx, scursor); + + RedisModule_ReplyWithLongLong(ctx, affected); + RedisModule_FreeString(ctx, fromKeys); + Vector_Free(vWhere); + + return REDISMODULE_OK; +} + int RedisModule_OnLoad(RedisModuleCtx *ctx) { // Register the module @@ -426,5 +704,11 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx) { if (RedisModule_CreateCommand(ctx, "dbx.select", SelectCommand, "readonly", 1, 1, 1) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx, "dbx.insert", InsertCommand, "write deny-oom", 1, 1, 1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx, "dbx.delete", DeleteCommand, "write deny-oom", 1, 1, 1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; }