From 7e8fc08b046f3b519faebda3e399c7b67fc62fe6 Mon Sep 17 00:00:00 2001 From: Kenneth Cheng Date: Sun, 3 May 2020 21:06:10 +0800 Subject: [PATCH] add import csv function --- README.md | 70 +++++++++++++++++++++++++++------ src/dbx.c | 114 +++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 151 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 0c59993..e1feea7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Redis Module for maintaining hash by simple SQL +# Redis Module for maintaining hash by simple SQL (Also provide csv import function) -This module aims to provide simple DML to manipulate the hashes in REDIS for SQL users. It works as simple as you expected. It translates the input statement to a set of pure REDIS commands. It does not need nor generate any intermediate stuffs which occupied your storages. The target data is your hashes only. +This module aims to provide simple DML to manipulate the hashes in REDIS for SQL users. It works as simple as you expected. It translates the input statement to a set of pure REDIS commands. It does not need nor generate any intermediate stuffs which occupied your storages. The target data is your hashes only. It also provides the CSV import and export (under construction) function. ## Usage ```sql @@ -51,7 +51,7 @@ If you still have problem in loading the module, please visit: https://redis.io/ ## More Examples -#### Select Statement +### Select statement You may specify multiple fields separated by comma ```sql 127.0.0.1:6379> dbx select name, gender, birth from phonebook @@ -120,7 +120,7 @@ The above is nearly like REDIS keys command Each record is exactly a hash, you could use raw REDIS commands ``hget, hmget or hgetall`` to retrieve the same content -#### Where Clause in Select Statement +#### Where clause in select statement Your could specify =, >, <, >=, <=, <>, != or like conditions in where clause. Now the module only support "and" to join multiple conditions. ```sql 127.0.0.1:6379> dbx select tel from phonebook where name like Son @@ -133,7 +133,7 @@ Your could specify =, >, <, >=, <=, <>, != or like conditions in where clause. N 2) "1-888-3333-1412" ``` -#### Order Clause in Select Statement +#### Order clause in select statement Ordering can be ascending or descending. All sortings are alpha-sort. ```sql 127.0.0.1:6379> dbx select name, pos from phonebook order by pos asc @@ -164,7 +164,7 @@ Ordering can be ascending or descending. All sortings are alpha-sort. 2) "Betty Joan" ``` -#### Top Clause in Select Statement +#### Top clause in select statement ```sql 127.0.0.1:6379> dbx select top 3 name, tel from phonebook order by pos desc 1) 1) name @@ -183,7 +183,7 @@ Ordering can be ascending or descending. All sortings are alpha-sort. (empty list or set) ``` -#### Into Clause in Select Statement +#### Into clause in select statement You could create another table by into clause. ```sql 127.0.0.1:6379> dbx select * into testbook from phonebook @@ -239,7 +239,20 @@ You could create another table by into clause. 10) "F" ``` -#### Delete Statement +#### Into csv clause in select statement +(The feature is under construction) +```sql +127.0.0.1:6379> dbx select * into csv "/tmp/testbook.csv" from phonebook where pos > 2 +(integer) 2 +127.0.0.1:6379> quit +$ cat /tmp/testbook.csv +name,tel,birth,pos,gender +"Kenneth Cheng","123-12134-123","2000-12-31","5","M" +"Kevin Louis","111-2123-1233","2009-12-31","6","F" +$ +``` + +### Delete statement You may also use Insert and Delete statement to operate the hash. If you does not provide the where clause, it will delete all the records of the specified key prefix. (i.e. phonebook) ```sql 127.0.0.1:6379> dbx delete from phonebook where gender = F @@ -248,8 +261,8 @@ You may also use Insert and Delete statement to operate the hash. If you does no (integer) 2 ``` -#### Insert Statement -The module provide simple Insert statement which same as the function of the REDIS command hmset. It will append a random string to your provided key (i.e. phonebook). If operation is successful, it will return the key name. +### Insert statement +The module provides simple Insert statement which same as the function of the REDIS command hmset. It will append a random string to your provided key (i.e. phonebook). If operation is successful, it will return the key name. ```sql 127.0.0.1:6379> dbx insert into phonebook (name,tel,birth,pos,gender) values ('Peter Nelson' ,1-456-1246-3421, 2019-10-01, 3, M) "phonebook:1588298418-551514504" @@ -277,7 +290,42 @@ Note that Redis requires at least one space after the single and double quoted a 127.0.0.1:6379> dbx "insert into phonebook (name,tel,birth,pos,gender) values ('Peter Nelson','1-456-1246-3421','2019-10-01',3, 'M')" ``` -#### Issue command from BASH shell +#### From clause for importing CSV file in insert statement +The module provides simple import function by specifying from clause in Insert statement. It only support comma deliminated. Please make sure that the specified import file can be accessed by Redis server. +```bash +$ cat > /tmp/test.csv << EOF +"Kenneth Cheng","123-12134-123","2000-12-31","5","M" +"Kevin Louis","111-2123-1233","2009-12-31","6","F" +EOF +$ redis-cli +127.0.0.1:6379> dbx insert into phonebook (name, tel, birth, pos, gender) from "/tmp/test.csv" +1) "phonebook:1588509697-1579004777" +2) "phonebook:1588509697-1579004778" +127.0.0.1:6379> dbx select name from phonebook +1) 1) name + 2) "Kenneth Cheng" +2) 1) name + 2) "Kevin Louis" +127.0.0.1:6379> dbx delete from phonebook +(integer) 2 +127.0.0.1:6379> quit +$ cat > /tmp/testheader.csv << EOF +name,tel,birth,pos,gender +"Kenneth Cheng","123-12134-123","2000-12-31","5","M" +"Kevin Louis","111-2123-1233","2009-12-31","6","F" +EOF +$ redis-cli +127.0.0.1:6379> dbx insert into phonebook from "/tmp/testheader.csv" +1) "phonebook:1588509697-1579004779" +2) "phonebook:1588509697-1579004780" +127.0.0.1:6379> dbx select name from phonebook +1) 1) name + 2) "Kenneth Cheng" +2) 1) name + 2) "Kevin Louis" +``` + +### Issue command from BASH shell ```sql $ redis-cli dbx select "*" from phonebook where gender = M order by pos desc 1) 1) "name" diff --git a/src/dbx.c b/src/dbx.c index 0a7036c..9492798 100644 --- a/src/dbx.c +++ b/src/dbx.c @@ -522,12 +522,20 @@ int SelectCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return REDISMODULE_OK; } +char* trim(char* s, char t) { + char* p = s; + if (p[strlen(s)-1] == t) p[strlen(s)-1] = 0; + if (p[0] == t) p++; + return p; +} + int InsertCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { if (argc < 2) return RedisModule_WrongArity(ctx); // Table - RedisModuleString *intoKey; + char intoKey[128] = ""; + RedisModuleString *fromCSV = NULL; // Process the arguments size_t plen; @@ -575,7 +583,7 @@ int InsertCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { break; case -1: // parse into clause, assume time+rand always is new key - intoKey = RMUtil_CreateFormattedString(ctx, "%s:%u-%i", token, (unsigned)time(NULL), rn++); + strcpy(intoKey, token); step = -2; break; case -2: @@ -590,6 +598,8 @@ int InsertCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { } else if (strcmp("values", token) == 0) step = -5; + else if (strcmp("from", token) == 0) + step = -7; else { RedisModule_ReplyWithError(ctx, "values keyword is expected"); return REDISMODULE_ERR; @@ -605,8 +615,10 @@ int InsertCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { case -4: if (strcmp("values", token) == 0) step = -5; + else if (strcmp("from", token) == 0) + step = -7; else { - RedisModule_ReplyWithError(ctx, "values keyword is expected"); + RedisModule_ReplyWithError(ctx, "values or from keyword is expected"); return REDISMODULE_ERR; } break; @@ -618,7 +630,7 @@ int InsertCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { strcpy(stmValue, &token[1]); if (token[strlen(token) - 1] == ')') { stmValue[strlen(stmValue) - 1] = 0; - step = 7; + step = 8; } else step = -6; @@ -626,12 +638,16 @@ int InsertCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { else if (token[strlen(token) - 1] == ')') { token[strlen(token) - 1] = 0; strcat(stmValue, token); - step = 7; + step = 8; } else strcat(stmValue, token); break; - case 7: + case -7: + fromCSV = RedisModule_CreateString(ctx, token, strlen(token)); + step = 8; + break; + case 8: RedisModule_ReplyWithError(ctx, "The end of statement is expected"); return REDISMODULE_ERR; break; @@ -647,25 +663,79 @@ int InsertCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { Vector *vField = splitStringByChar(stmField, ","); Vector *vValue = splitStringByChar(stmValue, ","); - if (Vector_Size(vField) != Vector_Size(vValue)) { - RedisModule_ReplyWithError(ctx, "Number of values does not match"); - return REDISMODULE_ERR; - } - RedisModule_AutoMemory(ctx); - for (size_t i=0; i 0) strcat(stmField, ","); + strcat(stmField, trim(value, '"')); + value = strtok(NULL, ","); + } + if (vField) Vector_Free(vField); + vField = splitStringByChar(stmField, ","); + continue; + } + + RedisModuleString *key = RedisModule_CreateStringPrintf(ctx, "%s:%u-%i", intoKey, (unsigned)time(NULL), rn++); + for (size_t i=0; i