diff --git a/api/librdb-ext-api.h b/api/librdb-ext-api.h index a17b2cc..f687f09 100644 --- a/api/librdb-ext-api.h +++ b/api/librdb-ext-api.h @@ -79,9 +79,13 @@ typedef enum RdbxToJsonEnc { typedef struct RdbxToJsonConf { RdbHandlersLevel level; /* Parsing depth (raw, structures or data-types) */ RdbxToJsonEnc encoding; /* Encoding format for the resulting JSON */ + + /* Additional metadata to include in json output */ + int includeDbInfo; /* Set to include DB and SLOT info in JSON output */ int includeAuxField; /* Set to include auxiliary fields in JSON output */ int includeFunc; /* Set to include functions in JSON output */ int includeStreamMeta; /* Set to include Stream metadata in JSON output */ + int flatten; /* Set to create a flattened JSON structure */ } RdbxToJsonConf; diff --git a/src/cli/rdb-cli.c b/src/cli/rdb-cli.c index eb2738c..00a8173 100644 --- a/src/cli/rdb-cli.c +++ b/src/cli/rdb-cli.c @@ -97,7 +97,8 @@ static void printUsage(int shortUsage) { printf("\t-D, --no-dbnum Exclude DB number\n\n"); printf("FORMAT_OPTIONS ('json'):\n"); - printf("\t-i, --include To include: {aux-val|func|stream-meta}\n"); + printf("\t-i, --include To include: {aux-val|func|stream-meta|db-info}\n"); + printf("\t-m, --meta-prefix To distinct EXTRAS from actual data, Prefix it (Default:\"__\")\n"); printf("\t-f, --flatten Print flatten json, without DBs Parenthesis\n"); printf("\t-o, --output Specify the output file. If not specified, output to stdout\n\n"); @@ -126,20 +127,23 @@ static void printUsage(int shortUsage) { } static RdbRes formatJson(RdbParser *parser, int argc, char **argv) { + extern const char *jsonMetaPrefix; const char *includeArg; const char *output = NULL;/*default:stdout*/ - int includeStreamMeta=0, includeFunc=0, includeAuxField=0, flatten=0; /*without*/ + int includeDbInfo=0, includeStreamMeta=0, includeFunc=0, includeAuxField=0, flatten=0; /*without*/ /* parse specific command options */ for (int at = 1; at < argc; ++at) { char *opt = argv[at]; if (getOptArg(argc, argv, &at, "-o", "--output", NULL, &output)) continue; if (getOptArg(argc, argv, &at, "-f", "--flatten", &flatten, NULL)) continue; + if (getOptArg(argc, argv, &at, "-m", "--meta-prefix", NULL, &jsonMetaPrefix)) continue; if (getOptArg(argc, argv, &at, "-i", "--include", NULL, &includeArg)) { if (strcmp(includeArg, "aux-val") == 0) { includeAuxField = 1; continue; } if (strcmp(includeArg, "func") == 0) { includeFunc = 1; continue; } if (strcmp(includeArg, "stream-meta") == 0) { includeStreamMeta = 1; continue; } + if (strcmp(includeArg, "db-info") == 0) { includeDbInfo = 1; continue; } loggerWrap(RDB_LOG_ERR, "Invalid argument for '--include': %s\n", includeArg); return RDB_ERR_GENERAL; } @@ -156,6 +160,7 @@ static RdbRes formatJson(RdbParser *parser, int argc, char **argv) { .includeAuxField = includeAuxField, .includeFunc = includeFunc, .includeStreamMeta = includeStreamMeta, + .includeDbInfo = includeDbInfo, }; if (RDBX_createHandlersToJson(parser, output, &conf) == NULL) diff --git a/src/ext/handlersToJson.c b/src/ext/handlersToJson.c index c5addf7..f9c3474 100644 --- a/src/ext/handlersToJson.c +++ b/src/ext/handlersToJson.c @@ -13,6 +13,9 @@ struct RdbxToJson; typedef enum { R2J_IDLE = 0, + R2J_AUX_FIELDS, + R2J_FUNCTIONS, + R2J_IN_DB, R2J_IN_KEY, @@ -53,6 +56,8 @@ struct RdbxToJson { unsigned int count_db; }; +const char *jsonMetaPrefix = "__"; /* Distinct meta from data with prefix string. */ + static void outputPlainEscaping(RdbxToJson *ctx, char *p, size_t len) { while(len--) { switch(*p) { @@ -121,6 +126,7 @@ static RdbxToJson *initRdbToJsonCtx(RdbParser *p, const char *outfilename, RdbxT ctx->conf.includeAuxField = 0; ctx->conf.includeFunc = 0; ctx->conf.includeStreamMeta = 0; + ctx->conf.includeDbInfo = 0; /* override configuration if provided */ if (conf) ctx->conf = *conf; @@ -136,15 +142,63 @@ static RdbxToJson *initRdbToJsonCtx(RdbParser *p, const char *outfilename, RdbxT /*** Handling common ***/ +static RdbRes toJsonDbSize(RdbParser *p, void *userData, uint64_t db_size, uint64_t exp_size) { + RdbxToJson *ctx = userData; + UNUSED(p); + + if (ctx->state != R2J_IN_DB) { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, + "toJsonDbSize(): Invalid state value: %d", ctx->state); + return (RdbRes) RDBX_ERR_R2J_INVALID_STATE; + } + + /* output json part */ + fprintf(ctx->outfile, " \"%sdbSize\": {\n", jsonMetaPrefix); + fprintf(ctx->outfile, " \"size\": %lu,\n", db_size); + fprintf(ctx->outfile, " \"expires\": %lu\n", exp_size); + fprintf(ctx->outfile, " }%s\n", (db_size) ? "," : ""); + + return RDB_OK; +} + +static RdbRes toJsonSlotInfo(RdbParser *p, void *userData, RdbSlotInfo *info) { + RdbxToJson *ctx = userData; + UNUSED(p, info); + + if (ctx->state != R2J_IN_DB) { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, + "toJsonSlotInfo(): Invalid state value: %d", ctx->state); + return (RdbRes) RDBX_ERR_R2J_INVALID_STATE; + } + + /* output json part */ + fprintf(ctx->outfile, " \"%sSlotInfo\": {\n", jsonMetaPrefix); + fprintf(ctx->outfile, " \"slotId\": %lu,\n", info->slot_id); + fprintf(ctx->outfile, " \"slotSize\": %lu,\n", info->slot_size); + fprintf(ctx->outfile, " \"slotSExpiresSize\": %lu\n", info->expires_slot_size); + fprintf(ctx->outfile, " },\n"); + return RDB_OK; +} + static RdbRes toJsonAuxField(RdbParser *p, void *userData, RdbBulk auxkey, RdbBulk auxval) { RdbxToJson *ctx = userData; UNUSED(p); + if (ctx->state == R2J_IDLE) { + ctx->state = R2J_AUX_FIELDS; + fprintf(ctx->outfile, "{\n "); /* group aux-fields with { ... } */ + } else if (ctx->state == R2J_AUX_FIELDS) { + fprintf(ctx->outfile, ",\n "); + } else { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, + "toJsonAuxField(): Invalid state value: %d", ctx->state); + return (RdbRes) RDBX_ERR_R2J_INVALID_STATE; + } + /* output json part */ outputQuotedEscaping(ctx, auxkey, RDB_bulkLen(p, auxkey)); fprintf(ctx->outfile, ":"); outputQuotedEscaping(ctx, auxval, RDB_bulkLen(p, auxval)); - fprintf(ctx->outfile, ",\n"); return RDB_OK; } @@ -227,13 +281,17 @@ static RdbRes toJsonNewDb(RdbParser *p, void *userData, int db) { RdbxToJson *ctx = userData; if (ctx->state == R2J_IDLE) { + /* old RDBs might not have aux-fields */ + if (!ctx->conf.flatten) fprintf(ctx->outfile, "{\n"); + } else if (ctx->state == R2J_AUX_FIELDS || ctx->state == R2J_FUNCTIONS) { + fprintf(ctx->outfile, "\n},\n"); if (!ctx->conf.flatten) fprintf(ctx->outfile, "{\n"); } else if (ctx->state == R2J_IN_DB) { /* output json part */ - if (!ctx->conf.flatten) { - fprintf(ctx->outfile, "\n},{\n"); - } else { + if (ctx->conf.flatten) { fprintf(ctx->outfile, ",\n"); + } else { + fprintf(ctx->outfile, "\n},{\n"); } } else { RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, @@ -266,8 +324,10 @@ static RdbRes toJsonNewRdb(RdbParser *p, void *userData, int rdbVersion) { static RdbRes toJsonEndRdb(RdbParser *p, void *userData) { RdbxToJson *ctx = userData; - if (ctx->state == R2J_IDLE) { + if ((ctx->state == R2J_IDLE)) { RDB_log(p, RDB_LOG_WRN, "RDB is empty."); + } else if (ctx->state == R2J_AUX_FIELDS || ctx->state == R2J_FUNCTIONS) { + fprintf(ctx->outfile, "\n},\n"); } else if (ctx->state == R2J_IN_DB) { if (!ctx->conf.flatten) fprintf(ctx->outfile, "\n}"); } else { @@ -433,10 +493,24 @@ static RdbRes toJsonHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk va static RdbRes toJsonFunction(RdbParser *p, void *userData, RdbBulk func) { RdbxToJson *ctx = userData; + + if (ctx->state == R2J_IDLE) { + ctx->state = R2J_FUNCTIONS; + } else if (ctx->state == R2J_AUX_FIELDS) { + fprintf(ctx->outfile, "\n},{\n"); + ctx->state = R2J_FUNCTIONS; + } else if (ctx->state == R2J_FUNCTIONS) { + fprintf(ctx->outfile, ",\n"); + } else { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, + "toJsonFunction(): Invalid state value: %d", ctx->state); + return (RdbRes) RDBX_ERR_R2J_INVALID_STATE; + } + /* output json part */ - fprintf(ctx->outfile, " \"Function_%d\":", ++ctx->count_functions); + fprintf(ctx->outfile, " \"%sFunction_%d\":", jsonMetaPrefix, ++ctx->count_functions); outputQuotedEscaping( (RdbxToJson *) userData, func, RDB_bulkLen(p, func)); - fprintf(ctx->outfile, ",\n"); + ctx->count_functions++; return RDB_OK; } @@ -639,9 +713,9 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo toJsonNewRdb, toJsonEndRdb, toJsonNewDb, - NULL, /* handleResizeDb */ - NULL, - NULL, + NULL, /*handleDbSize*/ + NULL, /*handleSlotInfo*/ + NULL, /*handleAuxField*/ toJsonNewKey, toJsonEndKey, toJsonString, @@ -673,6 +747,11 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo dataCb.handleStreamConsumerPendingEntry = toJsonStreamConsumerPendingEntry; } + if (ctx->conf.includeDbInfo) { + dataCb.handleDbSize = toJsonDbSize; + dataCb.handleSlotInfo = toJsonSlotInfo; + } + RDB_createHandlersData(p, &dataCb, ctx, deleteRdbToJsonCtx); } else if (ctx->conf.level == RDB_LEVEL_STRUCT) { @@ -680,9 +759,9 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo toJsonNewRdb, toJsonEndRdb, toJsonNewDb, - NULL, /* handleResizeDb */ - NULL, - NULL, + NULL, /*handleDbSize*/ + NULL, /*handleSlotInfo*/ + NULL, /*handleAuxField*/ toJsonNewKey, toJsonEndKey, toJsonString, @@ -724,9 +803,9 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo toJsonNewRdb, toJsonEndRdb, toJsonNewDb, - NULL, /* handleResizeDb */ - NULL, - NULL, + NULL, /*handleDbSize*/ + NULL, /*handleSlotInfo*/ + NULL, /*handleAuxField*/ toJsonNewKey, toJsonEndKey, NULL, /*handleBeginModuleAux*/ diff --git a/src/ext/handlersToResp.c b/src/ext/handlersToResp.c index 65c9e55..ceee209 100644 --- a/src/ext/handlersToResp.c +++ b/src/ext/handlersToResp.c @@ -25,7 +25,7 @@ typedef struct RedisToRdbVersion { } RedisToRdbVersion; const RedisToRdbVersion redisToRdbVersion[] = { - {"99.99", VER_VAL(99,99), 12}, // TODO: Update released version + {"7.4", VER_VAL(7,4), 12}, {"7.2", VER_VAL(7,2), 11}, {"7.0", VER_VAL(7,0), 10}, {"5.0", VER_VAL(5,0), 9}, //6 and 6.2 had v9 too diff --git a/src/lib/defines.h b/src/lib/defines.h index 515e1c1..0de1ed8 100644 --- a/src/lib/defines.h +++ b/src/lib/defines.h @@ -90,13 +90,6 @@ #define RDB_ENC_LZF 3 /* string compressed with FASTLZ */ /*#define RDB_ENC_GD 4 string is a gdcompressed entry */ -/* rdbLoad...() functions flags. */ -#define RDB_LOAD_NONE 0 -#define RDB_LOAD_ENC (1<<0) -#define RDB_LOAD_PLAIN (1<<1) -#define RDB_LOAD_SDS (1<<2) - - /* quicklist node container formats */ #define QUICKLIST_NODE_CONTAINER_PLAIN 1 #define QUICKLIST_NODE_CONTAINER_PACKED 2 diff --git a/test/dumps/cluster_slot_info.json b/test/dumps/cluster_slot_info.json new file mode 100644 index 0000000..f40255f --- /dev/null +++ b/test/dumps/cluster_slot_info.json @@ -0,0 +1,22 @@ +[{ + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1713005699", + "used-mem":"2550192", + "repl-stream-db":"0", + "repl-id":"734638bff92ee423e11e46e417b47acbd2d9c896", + "repl-offset":"390", + "aof-base":"0" +}, + { + "__dbSize": { + "size": 1, + "expires": 0 + }, + "__SlotInfo": { + "slotId": 7638, + "slotSize": 1, + "slotSExpiresSize": 0 + }, + "abc":"abc" + }] diff --git a/test/dumps/cluster_slot_info.rdb b/test/dumps/cluster_slot_info.rdb new file mode 100644 index 0000000..e6596de Binary files /dev/null and b/test/dumps/cluster_slot_info.rdb differ diff --git a/test/dumps/function2.json b/test/dumps/function2.json new file mode 100644 index 0000000..256df01 --- /dev/null +++ b/test/dumps/function2.json @@ -0,0 +1,22 @@ +[{ + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1713086666", + "used-mem":"966312", + "aof-base":"0" +},{ + "__Function_1":"#!lua name=mylib2\n\nlocal function my_hset2(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall2(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified2(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset2', my_hset2)\nredis.register_function('my_hgetall2', my_hgetall2)\nredis.register_function('my_hlastmodified2', my_hlastmodified2)\n\n", + "__Function_3":"#!lua name=mylib\n\nlocal function my_hset(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset', my_hset)\nredis.register_function('my_hgetall', my_hgetall)\nredis.register_function('my_hlastmodified', my_hlastmodified)\n\n" +}, + { + "key_97":"value_97", + "key_90":"value_90", + "key_93":"value_93", + "key_99":"value_99", + "key_96":"value_96", + "key_92":"value_92", + "key_95":"value_95", + "key_91":"value_91", + "key_94":"value_94", + "key_98":"value_98" + }] diff --git a/test/dumps/function2.rdb b/test/dumps/function2.rdb new file mode 100644 index 0000000..555ebe3 Binary files /dev/null and b/test/dumps/function2.rdb differ diff --git a/test/dumps/multiple_dbs_data.json b/test/dumps/multiple_dbs_data.json index b0c7835..6e99ff8 100644 --- a/test/dumps/multiple_dbs_data.json +++ b/test/dumps/multiple_dbs_data.json @@ -1,8 +1,14 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1683103535", -"used-mem":"967040", -"aof-base":"0", -"x":"0", -"y":"1", -"z":"2" +[{ + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1683103535", + "used-mem":"967040", + "aof-base":"0" +}, + { + "x":"0" + },{ + "y":"1" +},{ + "z":"2" +}] diff --git a/test/test_common.c b/test/test_common.c index 1fcecf3..7476daa 100644 --- a/test/test_common.c +++ b/test/test_common.c @@ -525,30 +525,14 @@ void *xrealloc(void *ptr, size_t size) { /*** compare json files ***/ -static void sanitize_json_line(char* line) { - size_t length = strlen(line); - - /* remove \n from end of line, if exist */ - if (length > 0 && (line[length - 1] == '\n') ) { - line[length - 1] = '\0'; - length--; - } - - /* remove trailing spaces */ - while ((0 < length) && (line[length-1] == ' ')) --length; - - /* remove comma at the end of line, if exist */ - if (length > 0 && (line[length - 1] == ',') ) { - line[length - 1] = '\0'; - length--; +void sanitizeString(char* str, const char* charSet) { + int j = 0; + for (int i = 0; str[i] != '\0'; i++) { + if (strchr(charSet, str[i]) == NULL) { + str[j++] = str[i]; + } } - - size_t j = 0, i = 0; - /* skip leading spaces */ - while ((i