Skip to content

Commit

Permalink
Support RDB v12
Browse files Browse the repository at this point in the history
  • Loading branch information
@moticless
moticless committed Apr 14, 2024
1 parent c182078 commit 3d473b9
Show file tree
Hide file tree
Showing 14 changed files with 186 additions and 62 deletions.
4 changes: 4 additions & 0 deletions api/librdb-ext-api.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,13 @@ typedef enum RdbxToJsonEnc {
typedef struct RdbxToJsonConf {
RdbHandlersLevel level; /* Parsing depth (raw, structures or data-types) */
RdbxToJsonEnc encoding; /* Encoding format for the resulting JSON */

/* Additional metadata to include in json output */
int includeDbInfo; /* Set to include DB and SLOT info in JSON output */
int includeAuxField; /* Set to include auxiliary fields in JSON output */
int includeFunc; /* Set to include functions in JSON output */
int includeStreamMeta; /* Set to include Stream metadata in JSON output */

int flatten; /* Set to create a flattened JSON structure */
} RdbxToJsonConf;

Expand Down
9 changes: 7 additions & 2 deletions src/cli/rdb-cli.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ static void printUsage(int shortUsage) {
printf("\t-D, --no-dbnum <DBNUM> Exclude DB number\n\n");

printf("FORMAT_OPTIONS ('json'):\n");
printf("\t-i, --include <EXTRAS> To include: {aux-val|func|stream-meta}\n");
printf("\t-i, --include <EXTRAS> To include: {aux-val|func|stream-meta|db-info}\n");
printf("\t-m, --meta-prefix <PREFIX> To distinct EXTRAS from actual data, Prefix it (Default:\"__\")\n");
printf("\t-f, --flatten Print flatten json, without DBs Parenthesis\n");
printf("\t-o, --output <FILE> Specify the output file. If not specified, output to stdout\n\n");

Expand Down Expand Up @@ -126,20 +127,23 @@ static void printUsage(int shortUsage) {
}

static RdbRes formatJson(RdbParser *parser, int argc, char **argv) {
extern const char *jsonMetaPrefix;
const char *includeArg;
const char *output = NULL;/*default:stdout*/
int includeStreamMeta=0, includeFunc=0, includeAuxField=0, flatten=0; /*without*/
int includeDbInfo=0, includeStreamMeta=0, includeFunc=0, includeAuxField=0, flatten=0; /*without*/

/* parse specific command options */
for (int at = 1; at < argc; ++at) {
char *opt = argv[at];
if (getOptArg(argc, argv, &at, "-o", "--output", NULL, &output)) continue;
if (getOptArg(argc, argv, &at, "-f", "--flatten", &flatten, NULL)) continue;
if (getOptArg(argc, argv, &at, "-m", "--meta-prefix", NULL, &jsonMetaPrefix)) continue;

if (getOptArg(argc, argv, &at, "-i", "--include", NULL, &includeArg)) {
if (strcmp(includeArg, "aux-val") == 0) { includeAuxField = 1; continue; }
if (strcmp(includeArg, "func") == 0) { includeFunc = 1; continue; }
if (strcmp(includeArg, "stream-meta") == 0) { includeStreamMeta = 1; continue; }
if (strcmp(includeArg, "db-info") == 0) { includeDbInfo = 1; continue; }
loggerWrap(RDB_LOG_ERR, "Invalid argument for '--include': %s\n", includeArg);
return RDB_ERR_GENERAL;
}
Expand All @@ -156,6 +160,7 @@ static RdbRes formatJson(RdbParser *parser, int argc, char **argv) {
.includeAuxField = includeAuxField,
.includeFunc = includeFunc,
.includeStreamMeta = includeStreamMeta,
.includeDbInfo = includeDbInfo,
};

if (RDBX_createHandlersToJson(parser, output, &conf) == NULL)
Expand Down
111 changes: 95 additions & 16 deletions src/ext/handlersToJson.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ struct RdbxToJson;
typedef enum
{
R2J_IDLE = 0,
R2J_AUX_FIELDS,
R2J_FUNCTIONS,

R2J_IN_DB,
R2J_IN_KEY,

Expand Down Expand Up @@ -53,6 +56,8 @@ struct RdbxToJson {
unsigned int count_db;
};

const char *jsonMetaPrefix = "__"; /* Distinct meta from data with prefix string. */

static void outputPlainEscaping(RdbxToJson *ctx, char *p, size_t len) {
while(len--) {
switch(*p) {
Expand Down Expand Up @@ -121,6 +126,7 @@ static RdbxToJson *initRdbToJsonCtx(RdbParser *p, const char *outfilename, RdbxT
ctx->conf.includeAuxField = 0;
ctx->conf.includeFunc = 0;
ctx->conf.includeStreamMeta = 0;
ctx->conf.includeDbInfo = 0;

/* override configuration if provided */
if (conf) ctx->conf = *conf;
Expand All @@ -136,15 +142,63 @@ static RdbxToJson *initRdbToJsonCtx(RdbParser *p, const char *outfilename, RdbxT

/*** Handling common ***/

static RdbRes toJsonDbSize(RdbParser *p, void *userData, uint64_t db_size, uint64_t exp_size) {
RdbxToJson *ctx = userData;
UNUSED(p);

if (ctx->state != R2J_IN_DB) {
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
"toJsonDbSize(): Invalid state value: %d", ctx->state);
return (RdbRes) RDBX_ERR_R2J_INVALID_STATE;
}

/* output json part */
fprintf(ctx->outfile, " \"%sdbSize\": {\n", jsonMetaPrefix);
fprintf(ctx->outfile, " \"size\": %lu,\n", db_size);
fprintf(ctx->outfile, " \"expires\": %lu\n", exp_size);
fprintf(ctx->outfile, " }%s\n", (db_size) ? "," : "");

return RDB_OK;
}

static RdbRes toJsonSlotInfo(RdbParser *p, void *userData, RdbSlotInfo *info) {
RdbxToJson *ctx = userData;
UNUSED(p, info);

if (ctx->state != R2J_IN_DB) {
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
"toJsonSlotInfo(): Invalid state value: %d", ctx->state);
return (RdbRes) RDBX_ERR_R2J_INVALID_STATE;
}

/* output json part */
fprintf(ctx->outfile, " \"%sSlotInfo\": {\n", jsonMetaPrefix);
fprintf(ctx->outfile, " \"slotId\": %lu,\n", info->slot_id);
fprintf(ctx->outfile, " \"slotSize\": %lu,\n", info->slot_size);
fprintf(ctx->outfile, " \"slotSExpiresSize\": %lu\n", info->expires_slot_size);
fprintf(ctx->outfile, " },\n");
return RDB_OK;
}

static RdbRes toJsonAuxField(RdbParser *p, void *userData, RdbBulk auxkey, RdbBulk auxval) {
RdbxToJson *ctx = userData;
UNUSED(p);

if (ctx->state == R2J_IDLE) {
ctx->state = R2J_AUX_FIELDS;
fprintf(ctx->outfile, "{\n "); /* group aux-fields with { ... } */
} else if (ctx->state == R2J_AUX_FIELDS) {
fprintf(ctx->outfile, ",\n ");
} else {
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
"toJsonAuxField(): Invalid state value: %d", ctx->state);
return (RdbRes) RDBX_ERR_R2J_INVALID_STATE;
}

/* output json part */
outputQuotedEscaping(ctx, auxkey, RDB_bulkLen(p, auxkey));
fprintf(ctx->outfile, ":");
outputQuotedEscaping(ctx, auxval, RDB_bulkLen(p, auxval));
fprintf(ctx->outfile, ",\n");

return RDB_OK;
}
Expand Down Expand Up @@ -227,13 +281,17 @@ static RdbRes toJsonNewDb(RdbParser *p, void *userData, int db) {
RdbxToJson *ctx = userData;

if (ctx->state == R2J_IDLE) {
/* old RDBs might not have aux-fields */
if (!ctx->conf.flatten) fprintf(ctx->outfile, "{\n");
} else if (ctx->state == R2J_AUX_FIELDS || ctx->state == R2J_FUNCTIONS) {
fprintf(ctx->outfile, "\n},\n");
if (!ctx->conf.flatten) fprintf(ctx->outfile, "{\n");
} else if (ctx->state == R2J_IN_DB) {
/* output json part */
if (!ctx->conf.flatten) {
fprintf(ctx->outfile, "\n},{\n");
} else {
if (ctx->conf.flatten) {
fprintf(ctx->outfile, ",\n");
} else {
fprintf(ctx->outfile, "\n},{\n");
}
} else {
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
Expand Down Expand Up @@ -266,8 +324,10 @@ static RdbRes toJsonNewRdb(RdbParser *p, void *userData, int rdbVersion) {
static RdbRes toJsonEndRdb(RdbParser *p, void *userData) {
RdbxToJson *ctx = userData;

if (ctx->state == R2J_IDLE) {
if ((ctx->state == R2J_IDLE)) {
RDB_log(p, RDB_LOG_WRN, "RDB is empty.");
} else if (ctx->state == R2J_AUX_FIELDS || ctx->state == R2J_FUNCTIONS) {
fprintf(ctx->outfile, "\n},\n");
} else if (ctx->state == R2J_IN_DB) {
if (!ctx->conf.flatten) fprintf(ctx->outfile, "\n}");
} else {
Expand Down Expand Up @@ -433,10 +493,24 @@ static RdbRes toJsonHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk va

static RdbRes toJsonFunction(RdbParser *p, void *userData, RdbBulk func) {
RdbxToJson *ctx = userData;

if (ctx->state == R2J_IDLE) {
ctx->state = R2J_FUNCTIONS;
} else if (ctx->state == R2J_AUX_FIELDS) {
fprintf(ctx->outfile, "\n},{\n");
ctx->state = R2J_FUNCTIONS;
} else if (ctx->state == R2J_FUNCTIONS) {
fprintf(ctx->outfile, ",\n");
} else {
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
"toJsonFunction(): Invalid state value: %d", ctx->state);
return (RdbRes) RDBX_ERR_R2J_INVALID_STATE;
}

/* output json part */
fprintf(ctx->outfile, " \"Function_%d\":", ++ctx->count_functions);
fprintf(ctx->outfile, " \"%sFunction_%d\":", jsonMetaPrefix, ++ctx->count_functions);
outputQuotedEscaping( (RdbxToJson *) userData, func, RDB_bulkLen(p, func));
fprintf(ctx->outfile, ",\n");
ctx->count_functions++;
return RDB_OK;
}

Expand Down Expand Up @@ -639,9 +713,9 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo
toJsonNewRdb,
toJsonEndRdb,
toJsonNewDb,
NULL, /* handleResizeDb */
NULL,
NULL,
NULL, /*handleDbSize*/
NULL, /*handleSlotInfo*/
NULL, /*handleAuxField*/
toJsonNewKey,
toJsonEndKey,
toJsonString,
Expand Down Expand Up @@ -673,16 +747,21 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo
dataCb.handleStreamConsumerPendingEntry = toJsonStreamConsumerPendingEntry;
}

if (ctx->conf.includeDbInfo) {
dataCb.handleDbSize = toJsonDbSize;
dataCb.handleSlotInfo = toJsonSlotInfo;
}

RDB_createHandlersData(p, &dataCb, ctx, deleteRdbToJsonCtx);

} else if (ctx->conf.level == RDB_LEVEL_STRUCT) {
RdbHandlersStructCallbacks structCb = {
toJsonNewRdb,
toJsonEndRdb,
toJsonNewDb,
NULL, /* handleResizeDb */
NULL,
NULL,
NULL, /*handleDbSize*/
NULL, /*handleSlotInfo*/
NULL, /*handleAuxField*/
toJsonNewKey,
toJsonEndKey,
toJsonString,
Expand Down Expand Up @@ -724,9 +803,9 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo
toJsonNewRdb,
toJsonEndRdb,
toJsonNewDb,
NULL, /* handleResizeDb */
NULL,
NULL,
NULL, /*handleDbSize*/
NULL, /*handleSlotInfo*/
NULL, /*handleAuxField*/
toJsonNewKey,
toJsonEndKey,
NULL, /*handleBeginModuleAux*/
Expand Down
2 changes: 1 addition & 1 deletion src/ext/handlersToResp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ typedef struct RedisToRdbVersion {
} RedisToRdbVersion;

const RedisToRdbVersion redisToRdbVersion[] = {
{"99.99", VER_VAL(99,99), 12}, // TODO: Update released version
{"7.4", VER_VAL(7,4), 12},
{"7.2", VER_VAL(7,2), 11},
{"7.0", VER_VAL(7,0), 10},
{"5.0", VER_VAL(5,0), 9}, //6 and 6.2 had v9 too
Expand Down
7 changes: 0 additions & 7 deletions src/lib/defines.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,6 @@
#define RDB_ENC_LZF 3 /* string compressed with FASTLZ */
/*#define RDB_ENC_GD 4 string is a gdcompressed entry */

/* rdbLoad...() functions flags. */
#define RDB_LOAD_NONE 0
#define RDB_LOAD_ENC (1<<0)
#define RDB_LOAD_PLAIN (1<<1)
#define RDB_LOAD_SDS (1<<2)


/* quicklist node container formats */
#define QUICKLIST_NODE_CONTAINER_PLAIN 1
#define QUICKLIST_NODE_CONTAINER_PACKED 2
Expand Down
22 changes: 22 additions & 0 deletions test/dumps/cluster_slot_info.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[{
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1713005699",
"used-mem":"2550192",
"repl-stream-db":"0",
"repl-id":"734638bff92ee423e11e46e417b47acbd2d9c896",
"repl-offset":"390",
"aof-base":"0"
},
{
"__dbSize": {
"size": 1,
"expires": 0
},
"__SlotInfo": {
"slotId": 7638,
"slotSize": 1,
"slotSExpiresSize": 0
},
"abc":"abc"
}]
Binary file added test/dumps/cluster_slot_info.rdb
View file
Binary file not shown.
22 changes: 22 additions & 0 deletions test/dumps/function2.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[{
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1713086666",
"used-mem":"966312",
"aof-base":"0"
},{
"__Function_1":"#!lua name=mylib2\n\nlocal function my_hset2(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall2(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified2(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset2', my_hset2)\nredis.register_function('my_hgetall2', my_hgetall2)\nredis.register_function('my_hlastmodified2', my_hlastmodified2)\n\n",
"__Function_3":"#!lua name=mylib\n\nlocal function my_hset(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset', my_hset)\nredis.register_function('my_hgetall', my_hgetall)\nredis.register_function('my_hlastmodified', my_hlastmodified)\n\n"
},
{
"key_97":"value_97",
"key_90":"value_90",
"key_93":"value_93",
"key_99":"value_99",
"key_96":"value_96",
"key_92":"value_92",
"key_95":"value_95",
"key_91":"value_91",
"key_94":"value_94",
"key_98":"value_98"
}]
Binary file added test/dumps/function2.rdb
View file
Binary file not shown.
22 changes: 14 additions & 8 deletions test/dumps/multiple_dbs_data.json
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1683103535",
"used-mem":"967040",
"aof-base":"0",
"x":"0",
"y":"1",
"z":"2"
[{
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1683103535",
"used-mem":"967040",
"aof-base":"0"
},
{
"x":"0"
},{
"y":"1"
},{
"z":"2"
}]
Loading

0 comments on commit 3d473b9

Please sign in to comment.