From c444094c69a2423ad061479082c9bb0d4e376d35 Mon Sep 17 00:00:00 2001 From: Moti Cohen Date: Wed, 12 Jun 2024 18:01:25 +0300 Subject: [PATCH] integrate python json parser service --- README.md | 2 +- src/ext/handlersToJson.c | 23 ++- src/lib/defines.h | 24 +-- test/Makefile | 2 +- test/dumps/cluster_slot_info.json | 27 ++-- test/dumps/function.json | 6 - test/dumps/function2.json | 29 ++-- test/dumps/hash_lp_v11_data.json | 13 +- test/dumps/hash_lp_v11_raw.json | 13 +- test/dumps/hash_lp_v11_struct.json | 13 +- test/dumps/hash_zl_v6_data.json | 13 +- test/dumps/hash_zl_v6_raw.json | 13 +- test/dumps/hash_zl_v6_struct.json | 13 +- test/dumps/multiple_dbs_data.json | 11 +- test/dumps/multiple_lists_strings_data.json | 13 +- test/dumps/multiple_lists_strings_raw.json | 13 +- test/dumps/multiple_lists_strings_struct.json | 13 +- test/dumps/plain_hash_data.json | 11 +- test/dumps/plain_hash_raw.json | 11 +- test/dumps/plain_hash_struct.json | 11 +- test/dumps/plain_zset_2_v11_data.json | 15 +- test/dumps/plain_zset_2_v11_raw.json | 13 +- test/dumps/plain_zset_2_v11_struct.json | 15 +- test/dumps/plain_zset_v6_data.json | 28 +++- test/dumps/plain_zset_v6_raw.json | 2 +- test/dumps/plain_zset_v6_struct.json | 27 +++- test/dumps/quicklist_data.json | 13 +- test/dumps/quicklist_raw.json | 13 +- test/dumps/quicklist_struct.json | 13 +- test/dumps/set_is_v11_data.json | 13 +- test/dumps/set_is_v11_raw.json | 13 +- test/dumps/set_is_v11_struct.json | 13 +- test/dumps/set_lp_v11_data.json | 13 +- test/dumps/set_lp_v11_raw.json | 13 +- test/dumps/set_lp_v11_struct.json | 13 +- test/dumps/single_key_data.json | 19 ++- test/dumps/single_key_raw.json | 19 ++- test/dumps/single_key_struct.json | 19 ++- test/dumps/single_list_data.json | 19 ++- test/dumps/single_list_raw.json | 19 ++- test/dumps/single_list_struct.json | 19 ++- test/dumps/zset_lp_v11_data.json | 13 +- test/dumps/zset_lp_v11_raw.json | 13 +- test/dumps/zset_lp_v11_struct.json | 13 +- test/dumps/zset_zl_v6_data.json | 2 +- test/json_signature_generator.py | 83 ++++++++++ test/test_common.c | 146 +++++++++++------- test/test_common.h | 3 +- test/test_main.c | 2 + test/test_rdb_to_json.c | 2 + 50 files changed, 580 insertions(+), 302 deletions(-) delete mode 100644 test/dumps/function.json create mode 100644 test/json_signature_generator.py diff --git a/README.md b/README.md index 20b0986..3bfea67 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ To see cool internal state printouts of the parser, set env-var `LIBRDB_DEBUG_DA % LIBRDB_DEBUG_DATA=1 make example -To build and run tests, you need to have cmocka unit testing framework installed: +To build and run tests, you need to have cmocka unit testing and python3 installed: % make test diff --git a/src/ext/handlersToJson.c b/src/ext/handlersToJson.c index e9b4f73..525bdf4 100644 --- a/src/ext/handlersToJson.c +++ b/src/ext/handlersToJson.c @@ -153,7 +153,7 @@ static RdbRes toJsonDbSize(RdbParser *p, void *userData, uint64_t db_size, uint6 } /* output json part */ - fprintf(ctx->outfile, " \"%sdbSize\": {\n", jsonMetaPrefix); + fprintf(ctx->outfile, " \"__dbsize__\": {\n"); fprintf(ctx->outfile, " \"size\": %" PRIu64 ",\n", db_size); fprintf(ctx->outfile, " \"expires\": %" PRIu64 "\n", exp_size); fprintf(ctx->outfile, " }%s\n", (db_size) ? "," : ""); @@ -171,7 +171,7 @@ static RdbRes toJsonSlotInfo(RdbParser *p, void *userData, RdbSlotInfo *info) { } /* output json part */ - fprintf(ctx->outfile, " \"%sSlotInfo\": {\n", jsonMetaPrefix); + fprintf(ctx->outfile, " \"__slotinfo__\": {\n"); fprintf(ctx->outfile, " \"slotId\": %lu,\n", info->slot_id); fprintf(ctx->outfile, " \"slotSize\": %lu,\n", info->slot_size); fprintf(ctx->outfile, " \"slotSExpiresSize\": %lu\n", info->expires_slot_size); @@ -184,7 +184,7 @@ static RdbRes toJsonAuxField(RdbParser *p, void *userData, RdbBulk auxkey, RdbBu if (ctx->state == R2J_IDLE) { ctx->state = R2J_AUX_FIELDS; - fprintf(ctx->outfile, "{\n "); /* group aux-fields with { ... } */ + fprintf(ctx->outfile, "\"__aux__\" : {\n "); /* group aux-fields with { ... } */ } else if (ctx->state == R2J_AUX_FIELDS) { fprintf(ctx->outfile, ",\n "); } else { @@ -432,14 +432,20 @@ static RdbRes toJsonSet(RdbParser *p, void *userData, RdbBulk member) { static RdbRes toJsonZset(RdbParser *p, void *userData, RdbBulk member, double score) { RdbxToJson *ctx = userData; - char score_str[MAX_D2STRING_CHARS]; - int len = d2string(score_str, sizeof(score_str), score); + char scoreStr[MAX_D2STRING_CHARS]; + int len = d2string(scoreStr, sizeof(scoreStr), score); + + /* -0 is a valid double, but we want to output it as 0 */ + if ((len == 2) && (scoreStr[0] == '-') && (scoreStr[1] == '0')) { + scoreStr[0] = '0'; + scoreStr[1] = '\0'; + } if (ctx->state == R2J_IN_KEY) { /* output json part */ fprintf(ctx->outfile, "{"); outputQuotedEscaping(ctx, member, RDB_bulkLen(p, member)); - fprintf(ctx->outfile, ":\"%.*s\"", len, score_str); + fprintf(ctx->outfile, ":\"%.*s\"", len, scoreStr); /* update new state */ ctx->state = R2J_IN_ZSET; @@ -448,7 +454,7 @@ static RdbRes toJsonZset(RdbParser *p, void *userData, RdbBulk member, double sc /* output json part */ fprintf(ctx->outfile, ","); outputQuotedEscaping(ctx, member, RDB_bulkLen(p, member)); - fprintf(ctx->outfile, ":\"%.*s\"", len, score_str); + fprintf(ctx->outfile, ":\"%.*s\"", len, scoreStr); /* state unchanged */ @@ -494,8 +500,9 @@ static RdbRes toJsonFunction(RdbParser *p, void *userData, RdbBulk func) { if (ctx->state == R2J_IDLE) { ctx->state = R2J_FUNCTIONS; + fprintf(ctx->outfile, "\"__func__\": {\n"); } else if (ctx->state == R2J_AUX_FIELDS) { - fprintf(ctx->outfile, "\n},{\n"); + fprintf(ctx->outfile, "\n},\n \"__func__\": {\n"); ctx->state = R2J_FUNCTIONS; } else if (ctx->state == R2J_FUNCTIONS) { fprintf(ctx->outfile, ",\n"); diff --git a/src/lib/defines.h b/src/lib/defines.h index 0de1ed8..29bdf5b 100644 --- a/src/lib/defines.h +++ b/src/lib/defines.h @@ -19,20 +19,22 @@ /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* Object types for encoded objects. */ -#define RDB_TYPE_HASH_ZIPMAP 9 -#define RDB_TYPE_LIST_ZIPLIST 10 -#define RDB_TYPE_SET_INTSET 11 -#define RDB_TYPE_ZSET_ZIPLIST 12 -#define RDB_TYPE_HASH_ZIPLIST 13 -#define RDB_TYPE_LIST_QUICKLIST 14 -#define RDB_TYPE_STREAM_LISTPACKS 15 -#define RDB_TYPE_HASH_LISTPACK 16 -#define RDB_TYPE_ZSET_LISTPACK 17 +#define RDB_TYPE_HASH_ZIPMAP 9 +#define RDB_TYPE_LIST_ZIPLIST 10 +#define RDB_TYPE_SET_INTSET 11 +#define RDB_TYPE_ZSET_ZIPLIST 12 +#define RDB_TYPE_HASH_ZIPLIST 13 +#define RDB_TYPE_LIST_QUICKLIST 14 +#define RDB_TYPE_STREAM_LISTPACKS 15 +#define RDB_TYPE_HASH_LISTPACK 16 +#define RDB_TYPE_ZSET_LISTPACK 17 #define RDB_TYPE_LIST_QUICKLIST_2 18 #define RDB_TYPE_STREAM_LISTPACKS_2 19 -#define RDB_TYPE_SET_LISTPACK 20 +#define RDB_TYPE_SET_LISTPACK 20 #define RDB_TYPE_STREAM_LISTPACKS_3 21 -#define RDB_TYPE_MAX 22 +#define RDB_TYPE_HASH_METADATA 22 +#define RDB_TYPE_HASH_LISTPACK_EX 23 +#define RDB_TYPE_MAX 24 /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ diff --git a/test/Makefile b/test/Makefile index 7f72f53..31c9a0b 100644 --- a/test/Makefile +++ b/test/Makefile @@ -17,7 +17,7 @@ STD = -std=c99 STACK = -fstack-protector-all -Wstack-protector WARNS = -Wall -Wextra -pedantic -Werror -Wno-unused-function -OPTIMIZATION?=-O3 +OPTIMIZATION?=-O0 CFLAGS = -fPIC $(OPTIMIZATION) $(STD) $(STACK) $(WARNS) DEBUG = -g3 -DDEBUG=1 diff --git a/test/dumps/cluster_slot_info.json b/test/dumps/cluster_slot_info.json index f40255f..8eb3ff6 100644 --- a/test/dumps/cluster_slot_info.json +++ b/test/dumps/cluster_slot_info.json @@ -1,4 +1,4 @@ -[{ +"__aux__" : { "redis-ver":"255.255.255", "redis-bits":"64", "ctime":"1713005699", @@ -8,15 +8,16 @@ "repl-offset":"390", "aof-base":"0" }, - { - "__dbSize": { - "size": 1, - "expires": 0 - }, - "__SlotInfo": { - "slotId": 7638, - "slotSize": 1, - "slotSExpiresSize": 0 - }, - "abc":"abc" - }] + +"__dbsize__": { + "size": 1, + "expires": 0 +}, + +"__slotinfo__": { + "slotId": 7638, + "slotSize": 1, + "slotSExpiresSize": 0 +}, + +"abc":"abc" diff --git a/test/dumps/function.json b/test/dumps/function.json deleted file mode 100644 index 4866553..0000000 --- a/test/dumps/function.json +++ /dev/null @@ -1,6 +0,0 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1693250800", -"used-mem":"866344", -"aof-base":"0", -"Function_1":"#!lua name=mylib \n redis.register_function('myfunc', function(keys, args) return args[1] end)" diff --git a/test/dumps/function2.json b/test/dumps/function2.json index 256df01..68d7ba0 100644 --- a/test/dumps/function2.json +++ b/test/dumps/function2.json @@ -1,22 +1,23 @@ -[{ +"__aux__" : { "redis-ver":"255.255.255", "redis-bits":"64", "ctime":"1713086666", "used-mem":"966312", "aof-base":"0" -},{ +}, + +"__func__": { "__Function_1":"#!lua name=mylib2\n\nlocal function my_hset2(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall2(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified2(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset2', my_hset2)\nredis.register_function('my_hgetall2', my_hgetall2)\nredis.register_function('my_hlastmodified2', my_hlastmodified2)\n\n", "__Function_3":"#!lua name=mylib\n\nlocal function my_hset(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset', my_hset)\nredis.register_function('my_hgetall', my_hgetall)\nredis.register_function('my_hlastmodified', my_hlastmodified)\n\n" }, - { - "key_97":"value_97", - "key_90":"value_90", - "key_93":"value_93", - "key_99":"value_99", - "key_96":"value_96", - "key_92":"value_92", - "key_95":"value_95", - "key_91":"value_91", - "key_94":"value_94", - "key_98":"value_98" - }] + +"key_97":"value_97", +"key_90":"value_90", +"key_93":"value_93", +"key_99":"value_99", +"key_96":"value_96", +"key_92":"value_92", +"key_95":"value_95", +"key_91":"value_91", +"key_94":"value_94", +"key_98":"value_98" \ No newline at end of file diff --git a/test/dumps/hash_lp_v11_data.json b/test/dumps/hash_lp_v11_data.json index f54235f..23c2a93 100644 --- a/test/dumps/hash_lp_v11_data.json +++ b/test/dumps/hash_lp_v11_data.json @@ -1,7 +1,10 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1695280472", -"used-mem":"1062024", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1695280472", + "used-mem":"1062024", + "aof-base":"0" +}, + "hash2":{"field7":"value7","field8":"value8","9":"value9","field10":"value10","field11":"11","12":"12.0"}, "hash1":{"field2":"2","field3":"3","field4":"value4.0","field5":"5.0","6":"6"} diff --git a/test/dumps/hash_lp_v11_raw.json b/test/dumps/hash_lp_v11_raw.json index cfcdb23..9499c70 100644 --- a/test/dumps/hash_lp_v11_raw.json +++ b/test/dumps/hash_lp_v11_raw.json @@ -1,7 +1,10 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1695280472", -"used-mem":"1062024", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1695280472", + "used-mem":"1062024", + "aof-base":"0" +}, + "hash2":"\u0010\u00c3@C@V\u0013V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value \u0007`\u000f\u00008\u00a0\u000f\u00038\u0007\t\u0001\u0080\u0019\u00029\u0007\u0087`\u0019\u000310\b\u0087`*@\b\u0080\u0011\f1\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff", "hash1":"\u0010\u00c35>\u000f>\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0080\t\u00023\u0007\u0003\u00a0\t\u000b4\u0007\u0088value4.0\t\u0080\u001b\u000b5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff" diff --git a/test/dumps/hash_lp_v11_struct.json b/test/dumps/hash_lp_v11_struct.json index 240cd14..8ec990f 100644 --- a/test/dumps/hash_lp_v11_struct.json +++ b/test/dumps/hash_lp_v11_struct.json @@ -1,7 +1,10 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1695280472", -"used-mem":"1062024", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1695280472", + "used-mem":"1062024", + "aof-base":"0" +}, + "hash2":["V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value7\u0007\u0086field8\u0007\u0086value8\u0007\t\u0001\u0086value9\u0007\u0087field10\b\u0087value10\b\u0087field11\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff"], "hash1":[">\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0086field3\u0007\u0003\u0001\u0086field4\u0007\u0088value4.0\t\u0086field5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff"] diff --git a/test/dumps/hash_zl_v6_data.json b/test/dumps/hash_zl_v6_data.json index 83cd75a..aa41994 100644 --- a/test/dumps/hash_zl_v6_data.json +++ b/test/dumps/hash_zl_v6_data.json @@ -1,6 +1,9 @@ -"redis-ver":"6.0.16", -"redis-bits":"64", -"ctime":"1690533464", -"used-mem":"865216", -"aof-preamble":"0", +"__aux__" : { + "redis-ver":"6.0.16", + "redis-bits":"64", + "ctime":"1690533464", + "used-mem":"865216", + "aof-preamble":"0" +}, + "myhash":{"1":"2","3":"4","5":"6","7.0":"8.0","str1":"str2","str3":"str4"} diff --git a/test/dumps/hash_zl_v6_raw.json b/test/dumps/hash_zl_v6_raw.json index a58ea14..9acb990 100644 --- a/test/dumps/hash_zl_v6_raw.json +++ b/test/dumps/hash_zl_v6_raw.json @@ -1,6 +1,9 @@ -"redis-ver":"6.0.16", -"redis-bits":"64", -"ctime":"1690533464", -"used-mem":"865216", -"aof-preamble":"0", +"__aux__" : { + "redis-ver":"6.0.16", + "redis-bits":"64", + "ctime":"1690533464", + "used-mem":"865216", + "aof-preamble":"0" +}, + "myhash":"\r99\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff" \ No newline at end of file diff --git a/test/dumps/hash_zl_v6_struct.json b/test/dumps/hash_zl_v6_struct.json index 18b29cf..31f674c 100644 --- a/test/dumps/hash_zl_v6_struct.json +++ b/test/dumps/hash_zl_v6_struct.json @@ -1,6 +1,9 @@ -"redis-ver":"6.0.16", -"redis-bits":"64", -"ctime":"1690533464", -"used-mem":"865216", -"aof-preamble":"0", +"__aux__" : { + "redis-ver":"6.0.16", + "redis-bits":"64", + "ctime":"1690533464", + "used-mem":"865216", + "aof-preamble":"0" +}, + "myhash":["9\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff"] diff --git a/test/dumps/multiple_dbs_data.json b/test/dumps/multiple_dbs_data.json index 6e99ff8..2f52988 100644 --- a/test/dumps/multiple_dbs_data.json +++ b/test/dumps/multiple_dbs_data.json @@ -1,13 +1,6 @@ [{ - "redis-ver":"255.255.255", - "redis-bits":"64", - "ctime":"1683103535", - "used-mem":"967040", - "aof-base":"0" -}, - { - "x":"0" - },{ + "x":"0" +},{ "y":"1" },{ "z":"2" diff --git a/test/dumps/multiple_lists_strings_data.json b/test/dumps/multiple_lists_strings_data.json index e4839c3..807d273 100644 --- a/test/dumps/multiple_lists_strings_data.json +++ b/test/dumps/multiple_lists_strings_data.json @@ -1,8 +1,11 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1677580558", -"used-mem":"937464", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1677580558", + "used-mem":"937464", + "aof-base":"0" +}, + "string2":"Hi there!", "mylist1":["v1"], "mylist3":["v3","v2","v1"], diff --git a/test/dumps/multiple_lists_strings_raw.json b/test/dumps/multiple_lists_strings_raw.json index 6a9615a..875f098 100644 --- a/test/dumps/multiple_lists_strings_raw.json +++ b/test/dumps/multiple_lists_strings_raw.json @@ -1,8 +1,11 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1677580558", -"used-mem":"937464", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1677580558", + "used-mem":"937464", + "aof-base":"0" +}, + "string2":"\u0000\tHi there!", "mylist1":"\u0012\u0001\u0002\u000b\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff", "mylist3":"\u0012\u0001\u0002\u0013\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff", diff --git a/test/dumps/multiple_lists_strings_struct.json b/test/dumps/multiple_lists_strings_struct.json index 83ae876..bf4a976 100644 --- a/test/dumps/multiple_lists_strings_struct.json +++ b/test/dumps/multiple_lists_strings_struct.json @@ -1,8 +1,11 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1677580558", -"used-mem":"937464", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1677580558", + "used-mem":"937464", + "aof-base":"0" +}, + "string2":"Hi there!", "mylist1":["\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff"], "mylist3":["\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff"], diff --git a/test/dumps/plain_hash_data.json b/test/dumps/plain_hash_data.json index 4340312..7236adf 100644 --- a/test/dumps/plain_hash_data.json +++ b/test/dumps/plain_hash_data.json @@ -1,5 +1,8 @@ -"redis-ver":"3.2.1", -"redis-bits":"64", -"ctime":"1689605183", -"used-mem":"821904", +"__aux__" : { + "redis-ver":"3.2.1", + "redis-bits":"64", + "ctime":"1689605183", + "used-mem":"821904" +}, + "myhash1":{"5":"5","6":"6","10":"10","1":"1","2":"2","8":"8","4":"4","9":"9","3":"3","7":"7","11":"11"} diff --git a/test/dumps/plain_hash_raw.json b/test/dumps/plain_hash_raw.json index 4b03a40..98352f6 100644 --- a/test/dumps/plain_hash_raw.json +++ b/test/dumps/plain_hash_raw.json @@ -1,5 +1,8 @@ -"redis-ver":"3.2.1", -"redis-bits":"64", -"ctime":"1689605183", -"used-mem":"821904", +"__aux__" : { + "redis-ver": "3.2.1", + "redis-bits": "64", + "ctime": "1689605183", + "used-mem": "821904" +}, + "myhash1":"\u0004\u000b\u00c0\u0005\u00c0\u0005\u00c0\u0006\u00c0\u0006\u00c0\n\u00c0\n\u00c0\u0001\u00c0\u0001\u00c0\u0002\u00c0\u0002\u00c0\b\u00c0\b\u00c0\u0004\u00c0\u0004\u00c0\t\u00c0\t\u00c0\u0003\u00c0\u0003\u00c0\u0007\u00c0\u0007\u00c0\u000b\u00c0\u000b" \ No newline at end of file diff --git a/test/dumps/plain_hash_struct.json b/test/dumps/plain_hash_struct.json index 4340312..7236adf 100644 --- a/test/dumps/plain_hash_struct.json +++ b/test/dumps/plain_hash_struct.json @@ -1,5 +1,8 @@ -"redis-ver":"3.2.1", -"redis-bits":"64", -"ctime":"1689605183", -"used-mem":"821904", +"__aux__" : { + "redis-ver":"3.2.1", + "redis-bits":"64", + "ctime":"1689605183", + "used-mem":"821904" +}, + "myhash1":{"5":"5","6":"6","10":"10","1":"1","2":"2","8":"8","4":"4","9":"9","3":"3","7":"7","11":"11"} diff --git a/test/dumps/plain_zset_2_v11_data.json b/test/dumps/plain_zset_2_v11_data.json index 514363d..71dba99 100644 --- a/test/dumps/plain_zset_2_v11_data.json +++ b/test/dumps/plain_zset_2_v11_data.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1695705520", -"used-mem":"967352", -"aof-base":"0", -"myzset":{"a9":"inf","a8":"inf","a23":"1.000033e+25","a11":"9007199254740992","a21":"1125899906842624","a17":"65536","a15":"255","a13":"8.888888","a7":"2.2","a6":"0","a5":"0","a4":"-0","a3":"-0","a2":"0","a1":"0","a19":"-1","a20":"-1.1","a14":"-9.99999","a16":"-255","a18":"-65536","a22":"-1125899906842624","a12":"-9007199254740992","a24":"-4.329000123123131e+28","a10":"-inf"} +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1695705520", + "used-mem":"967352", + "aof-base":"0" +}, + +"myzset":{"a9":"inf","a8":"inf","a23":"1.000033e+25","a11":"9007199254740992","a21":"1125899906842624","a17":"65536","a15":"255","a13":"8.888888","a7":"2.2","a6":"0","a5":"0","a4":"0","a3":"0","a2":"0","a1":"0","a19":"-1","a20":"-1.1","a14":"-9.99999","a16":"-255","a18":"-65536","a22":"-1125899906842624","a12":"-9007199254740992","a24":"-4.329000123123131e+28","a10":"-inf"} diff --git a/test/dumps/plain_zset_2_v11_raw.json b/test/dumps/plain_zset_2_v11_raw.json index 1b259aa..df0dbb0 100644 --- a/test/dumps/plain_zset_2_v11_raw.json +++ b/test/dumps/plain_zset_2_v11_raw.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1695705520", -"used-mem":"967352", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1695705520", + "used-mem":"967352", + "aof-base":"0" +}, + "myzset":"\u0005\u0018\u0002a9\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u007f\u0002a8\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u007f\u0003a23\u00bc\t\u007f\u00f3M\u008b E\u0003a11\u0000\u0000\u0000\u0000\u0000\u0000@C\u0003a21\u0000\u0000\u0000\u0000\u0000\u0000\u0010C\u0003a17\u0000\u0000\u0000\u0000\u0000\u0000\u00f0@\u0003a15\u0000\u0000\u0000\u0000\u0000\u00e0o@\u0003a13\u001b\u009d\u00f3S\u001c\u00c7!@\u0002a7\u009a\u0099\u0099\u0099\u0099\u0099\u0001@\u0002a6\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0002a5\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0002a4\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0080\u0002a3\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0080\u0002a2\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0002a1\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003a19\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u00bf\u0003a20\u009a\u0099\u0099\u0099\u0099\u0099\u00f1\u00bf\u0003a14r\u00a7t\u00b0\u00fe\u00ff#\u00c0\u0003a16\u0000\u0000\u0000\u0000\u0000\u00e0o\u00c0\u0003a18\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u00c0\u0003a22\u0000\u0000\u0000\u0000\u0000\u0000\u0010\u00c3\u0003a12\u0000\u0000\u0000\u0000\u0000\u0000@\u00c3\u0003a24\u00c0\u00e1\u00a2\u00ca\u0014|\u00e1\u00c5\u0003a10\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u00ff" diff --git a/test/dumps/plain_zset_2_v11_struct.json b/test/dumps/plain_zset_2_v11_struct.json index 514363d..71dba99 100644 --- a/test/dumps/plain_zset_2_v11_struct.json +++ b/test/dumps/plain_zset_2_v11_struct.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1695705520", -"used-mem":"967352", -"aof-base":"0", -"myzset":{"a9":"inf","a8":"inf","a23":"1.000033e+25","a11":"9007199254740992","a21":"1125899906842624","a17":"65536","a15":"255","a13":"8.888888","a7":"2.2","a6":"0","a5":"0","a4":"-0","a3":"-0","a2":"0","a1":"0","a19":"-1","a20":"-1.1","a14":"-9.99999","a16":"-255","a18":"-65536","a22":"-1125899906842624","a12":"-9007199254740992","a24":"-4.329000123123131e+28","a10":"-inf"} +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1695705520", + "used-mem":"967352", + "aof-base":"0" +}, + +"myzset":{"a9":"inf","a8":"inf","a23":"1.000033e+25","a11":"9007199254740992","a21":"1125899906842624","a17":"65536","a15":"255","a13":"8.888888","a7":"2.2","a6":"0","a5":"0","a4":"0","a3":"0","a2":"0","a1":"0","a19":"-1","a20":"-1.1","a14":"-9.99999","a16":"-255","a18":"-65536","a22":"-1125899906842624","a12":"-9007199254740992","a24":"-4.329000123123131e+28","a10":"-inf"} diff --git a/test/dumps/plain_zset_v6_data.json b/test/dumps/plain_zset_v6_data.json index 421336e..90dd846 100644 --- a/test/dumps/plain_zset_v6_data.json +++ b/test/dumps/plain_zset_v6_data.json @@ -1 +1,27 @@ -"myzset":{"a23":"1.000033e+25","a19":"-1","a9":"inf","a24":"-4.329000123123131e+28","a12":"-9007199254740992","a7":"2.2","a13":"8.888888","a17":"65536","a21":"1125899906842624","a20":"-1.1","a22":"-1125899906842624","a3":"-0","a2":"0","a15":"255","a8":"inf","a11":"9007199254740992","a10":"-inf","a5":"0","a14":"-9.99999","a6":"0","a16":"-255","a4":"-0","a1":"0","a18":"-65536"} +"myzset": { + "a23": "1.000033e+25", + "a19": "-1", + "a9": "inf", + "a24": "-4.329000123123131e+28", + "a12": "-9007199254740992", + "a7": "2.2", + "a13": "8.888888", + "a17": "65536", + "a21": "1125899906842624", + "a20": "-1.1", + "a22": "-1125899906842624", + "a3": "0", + "a2": "0", + "a15": "255", + "a8": "inf", + "a11": "9007199254740992", + "a10": "-inf", + "a5": "0", + "a14": "-9.99999", + "a6": "0", + "a16": "-255", + "a4": "0", + "a1": "0", + "a18": "-65536" +} + diff --git a/test/dumps/plain_zset_v6_raw.json b/test/dumps/plain_zset_v6_raw.json index dcbb100..5b81aa5 100644 --- a/test/dumps/plain_zset_v6_raw.json +++ b/test/dumps/plain_zset_v6_raw.json @@ -1 +1 @@ -"myzset":"\u0003\u0018\u0003a23\f1.000033e+25\u0003a19\u0002-1\u0002a9\u00fe\u0003a24\u0017-4.3290001231231309e+28\u0003a12\u0011-9007199254740992\u0002a7\u00122.2000000000000002\u0003a13\u00128.8888879999999997\u0003a17\u000565536\u0003a21\u00101125899906842624\u0003a20\u0013-1.1000000000000001\u0003a22\u0011-1125899906842624\u0002a3\u0002-0\u0002a2\u00010\u0003a15\u0003255\u0002a8\u00fe\u0003a11\u00109007199254740992\u0003a10\u00ff\u0002a5\u00010\u0003a14\u0013-9.9999900000000004\u0002a6\u00010\u0003a16\u0004-255\u0002a4\u0002-0\u0002a1\u00010\u0003a18\u0006-65536" +"myzset": "\u0003\u0018\u0003a23\f1.000033e+25\u0003a19\u0002-1\u0002a9þ\u0003a24\u0017-4.3290001231231309e+28\u0003a12\u0011-9007199254740992\u0002a7\u00122.2000000000000002\u0003a13\u00128.8888879999999997\u0003a17\u000565536\u0003a21\u00101125899906842624\u0003a20\u0013-1.1000000000000001\u0003a22\u0011-1125899906842624\u0002a3\u0002-0\u0002a2\u00010\u0003a15\u0003255\u0002a8þ\u0003a11\u00109007199254740992\u0003a10ÿ\u0002a5\u00010\u0003a14\u0013-9.9999900000000004\u0002a6\u00010\u0003a16\u0004-255\u0002a4\u0002-0\u0002a1\u00010\u0003a18\u0006-65536" diff --git a/test/dumps/plain_zset_v6_struct.json b/test/dumps/plain_zset_v6_struct.json index 421336e..1c3bc64 100644 --- a/test/dumps/plain_zset_v6_struct.json +++ b/test/dumps/plain_zset_v6_struct.json @@ -1 +1,26 @@ -"myzset":{"a23":"1.000033e+25","a19":"-1","a9":"inf","a24":"-4.329000123123131e+28","a12":"-9007199254740992","a7":"2.2","a13":"8.888888","a17":"65536","a21":"1125899906842624","a20":"-1.1","a22":"-1125899906842624","a3":"-0","a2":"0","a15":"255","a8":"inf","a11":"9007199254740992","a10":"-inf","a5":"0","a14":"-9.99999","a6":"0","a16":"-255","a4":"-0","a1":"0","a18":"-65536"} +"myzset": { +"a23": "1.000033e+25", +"a19": "-1", +"a9": "inf", +"a24": "-4.329000123123131e+28", +"a12": "-9007199254740992", +"a7": "2.2", +"a13": "8.888888", +"a17": "65536", +"a21": "1125899906842624", +"a20": "-1.1", +"a22": "-1125899906842624", +"a3": "0", +"a2": "0", +"a15": "255", +"a8": "inf", +"a11": "9007199254740992", +"a10": "-inf", +"a5": "0", +"a14": "-9.99999", +"a6": "0", +"a16": "-255", +"a4": "0", +"a1": "0", +"a18": "-65536" +} diff --git a/test/dumps/quicklist_data.json b/test/dumps/quicklist_data.json index 5824cee..58ce6e9 100644 --- a/test/dumps/quicklist_data.json +++ b/test/dumps/quicklist_data.json @@ -1,7 +1,10 @@ -"redis-ver":"4.0.9", -"redis-bits":"64", -"ctime":"1629882015", -"used-mem":"496256", -"aof-preamble":"0", +"__aux__" : { + "redis-ver":"4.0.9", + "redis-bits":"64", + "ctime":"1629882015", + "used-mem":"496256", + "aof-preamble":"0" +}, + "list":["7"], "x":"7" diff --git a/test/dumps/quicklist_raw.json b/test/dumps/quicklist_raw.json index d214f35..bd2c206 100644 --- a/test/dumps/quicklist_raw.json +++ b/test/dumps/quicklist_raw.json @@ -1,7 +1,10 @@ -"redis-ver":"4.0.9", -"redis-bits":"64", -"ctime":"1629882015", -"used-mem":"496256", -"aof-preamble":"0", +"__aux__" : { + "redis-ver":"4.0.9", + "redis-bits":"64", + "ctime":"1629882015", + "used-mem":"496256", + "aof-preamble":"0" +}, + "list":"\u000e\u0001\r\r\u0000\u0000\u0000\n\u0000\u0000\u0000\u0001\u0000\u0000\u00f8\u00ff", "x":"\u0000\u00c0\u0007" diff --git a/test/dumps/quicklist_struct.json b/test/dumps/quicklist_struct.json index ae1d4e4..de9a145 100644 --- a/test/dumps/quicklist_struct.json +++ b/test/dumps/quicklist_struct.json @@ -1,7 +1,10 @@ -"redis-ver":"4.0.9", -"redis-bits":"64", -"ctime":"1629882015", -"used-mem":"496256", -"aof-preamble":"0", +"__aux__" : { + "redis-ver":"4.0.9", + "redis-bits":"64", + "ctime":"1629882015", + "used-mem":"496256", + "aof-preamble":"0" +}, + "list":["\r\u0000\u0000\u0000\n\u0000\u0000\u0000\u0001\u0000\u0000\u00f8\u00ff"], "x":"7" diff --git a/test/dumps/set_is_v11_data.json b/test/dumps/set_is_v11_data.json index b37df2a..994d3cf 100644 --- a/test/dumps/set_is_v11_data.json +++ b/test/dumps/set_is_v11_data.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1691304897", -"used-mem":"978520", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1691304897", + "used-mem":"978520", + "aof-base":"0" +}, + "myintest":["1","2","3","4","5","1234567890"] \ No newline at end of file diff --git a/test/dumps/set_is_v11_raw.json b/test/dumps/set_is_v11_raw.json index c7cf27d..aa84863 100644 --- a/test/dumps/set_is_v11_raw.json +++ b/test/dumps/set_is_v11_raw.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1691304897", -"used-mem":"978520", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1691304897", + "used-mem":"978520", + "aof-base":"0" +}, + "myintest":["\u0004\u0000\u0000\u0000\u0006\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0004\u0000\u0000\u0000\u0005\u0000\u0000\u0000\u00d2\u0002\u0096I"] diff --git a/test/dumps/set_is_v11_struct.json b/test/dumps/set_is_v11_struct.json index c7cf27d..aa84863 100644 --- a/test/dumps/set_is_v11_struct.json +++ b/test/dumps/set_is_v11_struct.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1691304897", -"used-mem":"978520", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1691304897", + "used-mem":"978520", + "aof-base":"0" +}, + "myintest":["\u0004\u0000\u0000\u0000\u0006\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0004\u0000\u0000\u0000\u0005\u0000\u0000\u0000\u00d2\u0002\u0096I"] diff --git a/test/dumps/set_lp_v11_data.json b/test/dumps/set_lp_v11_data.json index 1efa96c..c9a1a8b 100644 --- a/test/dumps/set_lp_v11_data.json +++ b/test/dumps/set_lp_v11_data.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1691481599", -"used-mem":"866040", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1691481599", + "used-mem":"866040", + "aof-base":"0" +}, + "myset":["1","2","3","1.1","1.2","1.3","a","b","c"] diff --git a/test/dumps/set_lp_v11_raw.json b/test/dumps/set_lp_v11_raw.json index 18a6797..4d906e2 100644 --- a/test/dumps/set_lp_v11_raw.json +++ b/test/dumps/set_lp_v11_raw.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1691481599", -"used-mem":"866040", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1691481599", + "used-mem":"866040", + "aof-base":"0" +}, + "myset":["%\u0000\u0000\u0000\t\u0000\u0001\u0001\u0002\u0001\u0003\u0001\u00831.1\u0004\u00831.2\u0004\u00831.3\u0004\u0081a\u0002\u0081b\u0002\u0081c\u0002\u00ff"] diff --git a/test/dumps/set_lp_v11_struct.json b/test/dumps/set_lp_v11_struct.json index a67d1a0..2c6c1c6 100644 --- a/test/dumps/set_lp_v11_struct.json +++ b/test/dumps/set_lp_v11_struct.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1691481599", -"used-mem":"866040", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1691481599", + "used-mem":"866040", + "aof-base":"0" +}, + "myset":["%\u0000\u0000\u0000\t\u0000\u0001\u0001\u0002\u0001\u0003\u0001\u00831.1\u0004\u00831.2\u0004\u00831.3\u0004\u0081a\u0002\u0081b\u0002\u0081c\u0002\u00ff"] \ No newline at end of file diff --git a/test/dumps/single_key_data.json b/test/dumps/single_key_data.json index 6b65970..7934ad2 100644 --- a/test/dumps/single_key_data.json +++ b/test/dumps/single_key_data.json @@ -1,9 +1,12 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1672087814", -"used-mem":"972952", -"repl-stream-db":"0", -"repl-id":"67ebe8f627f436e2630eef8661a697fa33563a8f", -"repl-offset":"162341903", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1672087814", + "used-mem":"972952", + "repl-stream-db":"0", + "repl-id":"67ebe8f627f436e2630eef8661a697fa33563a8f", + "repl-offset":"162341903", + "aof-base":"0" +}, + "xxx":"111" diff --git a/test/dumps/single_key_raw.json b/test/dumps/single_key_raw.json index 73e14f5..5b3026e 100644 --- a/test/dumps/single_key_raw.json +++ b/test/dumps/single_key_raw.json @@ -1,9 +1,12 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1672087814", -"used-mem":"972952", -"repl-stream-db":"0", -"repl-id":"67ebe8f627f436e2630eef8661a697fa33563a8f", -"repl-offset":"162341903", -"aof-base":"0", +"__aux__" : { + "redis-ver": "255.255.255", + "redis-bits": "64", + "ctime": "1672087814", + "used-mem": "972952", + "repl-stream-db": "0", + "repl-id": "67ebe8f627f436e2630eef8661a697fa33563a8f", + "repl-offset": "162341903", + "aof-base": "0" +}, + "xxx":"\u0000\u00c0o" diff --git a/test/dumps/single_key_struct.json b/test/dumps/single_key_struct.json index 6b65970..1454e88 100644 --- a/test/dumps/single_key_struct.json +++ b/test/dumps/single_key_struct.json @@ -1,9 +1,12 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1672087814", -"used-mem":"972952", -"repl-stream-db":"0", -"repl-id":"67ebe8f627f436e2630eef8661a697fa33563a8f", -"repl-offset":"162341903", -"aof-base":"0", +"__aux__" : { + "redis-ver": "255.255.255", + "redis-bits": "64", + "ctime": "1672087814", + "used-mem": "972952", + "repl-stream-db": "0", + "repl-id": "67ebe8f627f436e2630eef8661a697fa33563a8f", + "repl-offset": "162341903", + "aof-base": "0" +}, + "xxx":"111" diff --git a/test/dumps/single_list_data.json b/test/dumps/single_list_data.json index 8353aaf..87dc6eb 100644 --- a/test/dumps/single_list_data.json +++ b/test/dumps/single_list_data.json @@ -1,9 +1,12 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1677071222", -"used-mem":"982208", -"repl-stream-db":"0", -"repl-id":"f42ea6b158e5926941d08bde6b9ecb6ae88dcd45", -"repl-offset":"162395634", -"aof-base":"0", +"__aux__" : { + "redis-ver": "255.255.255", + "redis-bits": "64", + "ctime": "1677071222", + "used-mem": "982208", + "repl-stream-db": "0", + "repl-id": "f42ea6b158e5926941d08bde6b9ecb6ae88dcd45", + "repl-offset": "162395634", + "aof-base": "0" +}, + "mylist":["val3","val2","val1"] diff --git a/test/dumps/single_list_raw.json b/test/dumps/single_list_raw.json index c654977..985f494 100644 --- a/test/dumps/single_list_raw.json +++ b/test/dumps/single_list_raw.json @@ -1,9 +1,12 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1677071222", -"used-mem":"982208", -"repl-stream-db":"0", -"repl-id":"f42ea6b158e5926941d08bde6b9ecb6ae88dcd45", -"repl-offset":"162395634", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1677071222", + "used-mem":"982208", + "repl-stream-db":"0", + "repl-id":"f42ea6b158e5926941d08bde6b9ecb6ae88dcd45", + "repl-offset":"162395634", + "aof-base":"0" +}, + "mylist":"\u0012\u0001\u0002\u0019\u0019\u0000\u0000\u0000\u0003\u0000\u0084val3\u0005\u0084val2\u0005\u0084val1\u0005\u00ff" \ No newline at end of file diff --git a/test/dumps/single_list_struct.json b/test/dumps/single_list_struct.json index ad3b44d..300d2ce 100644 --- a/test/dumps/single_list_struct.json +++ b/test/dumps/single_list_struct.json @@ -1,9 +1,12 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1677071222", -"used-mem":"982208", -"repl-stream-db":"0", -"repl-id":"f42ea6b158e5926941d08bde6b9ecb6ae88dcd45", -"repl-offset":"162395634", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1677071222", + "used-mem":"982208", + "repl-stream-db":"0", + "repl-id":"f42ea6b158e5926941d08bde6b9ecb6ae88dcd45", + "repl-offset":"162395634", + "aof-base":"0" +}, + "mylist":["\u0019\u0000\u0000\u0000\u0003\u0000\u0084val3\u0005\u0084val2\u0005\u0084val1\u0005\u00ff"] diff --git a/test/dumps/zset_lp_v11_data.json b/test/dumps/zset_lp_v11_data.json index 3b246a3..3b49448 100644 --- a/test/dumps/zset_lp_v11_data.json +++ b/test/dumps/zset_lp_v11_data.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1695704816", -"used-mem":"969032", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1695704816", + "used-mem":"969032", + "aof-base":"0" +}, + "myzset":{"a10":"-inf","a24":"-4.329000123123131e+28","a12":"-9007199254740992","a22":"-1125899906842624","a18":"-65536","a16":"-255","a14":"-9.99999","a20":"-1.1","a19":"-1","a1":"0","a2":"0","a3":"0","a4":"0","a5":"0","a6":"0","a7":"2.2","a13":"8.888888","a15":"255","a17":"65536","a21":"1125899906842624","a11":"9007199254740992","a23":"1.000033e+25","a8":"inf","a9":"inf"} diff --git a/test/dumps/zset_lp_v11_raw.json b/test/dumps/zset_lp_v11_raw.json index d50bb23..81af81d 100644 --- a/test/dumps/zset_lp_v11_raw.json +++ b/test/dumps/zset_lp_v11_raw.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1695704816", -"used-mem":"969032", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1695704816", + "used-mem":"969032", + "aof-base":"0" +}, + "myzset":"\u0011\u00c3@\u00e6A\u0012\u001f\u0012\u0001\u0000\u00000\u0000\u0083a10\u0004\u0084-inf\u0005\u0083a24\u0004\u0096-4.329000\u0002123@\u0002\u000631e+28\u0017 '\u00032\u0004\u00f4\u0000`\u0000\u0002\u00e0\u00ff\t +\u00e0\u0000\u000e\u0000\u00fc@\u000e\u000718\u0004\u00f2\u0000\u0000\u00ff\u0004 \t\u00046\u0004\u00df\u0001\u0002 \u0007\u00064\u0004\u0088-9.9@\u0000@/@f\u00021.1 f\u000e19\u0004\u00df\u00ff\u0002\u0082a1\u0003\u0000\u0001\u0082a2`\u0005\u00003`\u0005\u00004`\u0005\u00005`\u0005\u00006`\u0005\u00057\u0003\u00832.2@V\u00053\u0004\u00888.8`\u0000 N\u000415\u0004\u00c0\u00ff@e\u00007@w\u0000\u0001 \u000121\u00c0\u009f\u0001\u0004\u0000@ \u00e0\u0000\u000e\u0000 @\u000e\u000523\u0004\u008c1. \u00d6\u0002033 \u00d0\u00065\r\u0082a8\u0003\u0083 \u00f6\u0003\u0004\u0082a9`\b\u0001\u0004\u00ff" diff --git a/test/dumps/zset_lp_v11_struct.json b/test/dumps/zset_lp_v11_struct.json index debe4b0..71e9eff 100644 --- a/test/dumps/zset_lp_v11_struct.json +++ b/test/dumps/zset_lp_v11_struct.json @@ -1,6 +1,9 @@ -"redis-ver":"255.255.255", -"redis-bits":"64", -"ctime":"1695704816", -"used-mem":"969032", -"aof-base":"0", +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1695704816", + "used-mem":"969032", + "aof-base":"0" +}, + "myzset":["\u0012\u0001\u0000\u00000\u0000\u0083a10\u0004\u0084-inf\u0005\u0083a24\u0004\u0096-4.329000123123131e+28\u0017\u0083a12\u0004\u00f4\u0000\u0000\u0000\u0000\u0000\u0000\u00e0\u00ff\t\u0083a22\u0004\u00f4\u0000\u0000\u0000\u0000\u0000\u0000\u00fc\u00ff\t\u0083a18\u0004\u00f2\u0000\u0000\u00ff\u0004\u0083a16\u0004\u00df\u0001\u0002\u0083a14\u0004\u0088-9.99999\t\u0083a20\u0004\u0084-1.1\u0005\u0083a19\u0004\u00df\u00ff\u0002\u0082a1\u0003\u0000\u0001\u0082a2\u0003\u0000\u0001\u0082a3\u0003\u0000\u0001\u0082a4\u0003\u0000\u0001\u0082a5\u0003\u0000\u0001\u0082a6\u0003\u0000\u0001\u0082a7\u0003\u00832.2\u0004\u0083a13\u0004\u00888.888888\t\u0083a15\u0004\u00c0\u00ff\u0002\u0083a17\u0004\u00f2\u0000\u0000\u0001\u0004\u0083a21\u0004\u00f4\u0000\u0000\u0000\u0000\u0000\u0000\u0004\u0000\t\u0083a11\u0004\u00f4\u0000\u0000\u0000\u0000\u0000\u0000 \u0000\t\u0083a23\u0004\u008c1.000033e+25\r\u0082a8\u0003\u0083inf\u0004\u0082a9\u0003\u0083inf\u0004\u00ff"] diff --git a/test/dumps/zset_zl_v6_data.json b/test/dumps/zset_zl_v6_data.json index 6e97b2e..bd6f7a8 100644 --- a/test/dumps/zset_zl_v6_data.json +++ b/test/dumps/zset_zl_v6_data.json @@ -1 +1 @@ -"myzset":{"a10":"-inf","a24":"-4.329000123123131e+28","a12":"-9007199254740992","a22":"-1125899906842624","a18":"-65536","a16":"-255","a14":"-9.99999","a20":"-1.1","a19":"-1","a1":"0","a2":"0","a3":"-0","a4":"-0","a5":"0","a6":"0","a7":"2.2","a13":"8.888888","a15":"255","a17":"65536","a21":"1125899906842624","a11":"9007199254740992","a23":"1.000033e+25","a8":"inf","a9":"inf"} +"myzset":{"a10":"-inf","a24":"-4.329000123123131e+28","a12":"-9007199254740992","a22":"-1125899906842624","a18":"-65536","a16":"-255","a14":"-9.99999","a20":"-1.1","a19":"-1","a1":"0","a2":"0","a3":"0","a4":"0","a5":"0","a6":"0","a7":"2.2","a13":"8.888888","a15":"255","a17":"65536","a21":"1125899906842624","a11":"9007199254740992","a23":"1.000033e+25","a8":"inf","a9":"inf"} diff --git a/test/json_signature_generator.py b/test/json_signature_generator.py new file mode 100644 index 0000000..699a8cf --- /dev/null +++ b/test/json_signature_generator.py @@ -0,0 +1,83 @@ +""" +JSON Signature Generator Service + +This script reads a JSON file path from its STDIN, generates its signature, ignoring +elements order, and print result to STDOUT. It run as a service. The file test_common.c +will start this service and will send filenames to it through pipe. + +It is optimized as a service since it is being used extensively in the test suite. +""" +import json +import hashlib +import sys +import logging + +logging.getLogger().disabled = True +# logging.basicConfig(filename='./json_signature_generator.log', level=logging.INFO, +# format='%(asctime)s - %(levelname)s - %(message)s') + +def load_json(file_path): + try: + with open(file_path, 'r') as file: + file_content = file.read() + + file_content = file_content.rstrip(",\r\n") + + # rdb-cli doesn't produce valid JSON. It requires few adaptations + # [ ... ] ==> { "dbs": [ ... ] } + # "a":"b" .... "y":"z" ==> { "a":"b" .... "y":"z" } + if file_content.startswith('['): + file_content = '{ "dbs":' + file_content + '}' + elif not file_content.startswith('{'): + file_content = '{' + file_content + '}' + + data = json.loads(file_content) + return data + except Exception as e: + logging.error(f"Error loading JSON file {file_path}: {str(e)}") + raise + +def sort_json(data): + if isinstance(data, dict): + sorted_dict = {key: sort_json(data[key]) for key in sorted(data)} + return sorted_dict + elif isinstance(data, list): + return sorted((sort_json(item) for item in data), key=lambda x: json.dumps(x)) + else: + return data + +def json_to_string(data): + return json.dumps(data, separators=(',', ':'), ensure_ascii=False) + +def generate_signature(data_string): + return hashlib.sha256(data_string.encode('utf-8')).hexdigest() + +def service(): + logging.info(f"Starting JSON Signature Generator Service") + while True: + try: + # Read file path from standard input + file_path = sys.stdin.readline().strip() + + if not file_path: + break + + # Process the file + json_data = load_json(file_path) + sorted_json_data = sort_json(json_data) + sorted_json_string = json_to_string(sorted_json_data) + signature = generate_signature(sorted_json_string) + + # Return the signature + print(signature) + logging.info(f"Signature generated for file: {file_path}") + sys.stdout.flush() # Flush STDOUT, ensure it's sent immediately + + except Exception as e: + logging.error(f"Error processing file: {file_path}, Error: {str(e)}") + print(f"Error: {str(e)}", file=sys.stderr) + sys.stderr.flush() + continue + +if __name__ == "__main__": + service() diff --git a/test/test_common.c b/test/test_common.c index 7476daa..5754aca 100644 --- a/test/test_common.c +++ b/test/test_common.c @@ -119,7 +119,7 @@ void cleanTmpFolder(void) { char file_path[1024]; snprintf(file_path, sizeof(file_path), "%s/%s", folder_path, entry->d_name); - assert_true (remove(file_path) != -1); + assert_true (remove(file_path) == 0); } closedir(dir); @@ -135,7 +135,7 @@ char *substring(char *str, size_t len, char *substr) { if (sublen > len) return NULL; - for (size_t cmpFromOffest = 0; cmpFromOffest < len - sublen; cmpFromOffest++) { + for (size_t cmpFromOffest = 0; cmpFromOffest <= len - sublen; cmpFromOffest++) { if (strncmp(&str[cmpFromOffest], substr, sublen) == 0) { return &str[cmpFromOffest]; } @@ -418,6 +418,8 @@ void setupRedisServer(const char *extraArgs) { snprintf(redisVer, sizeof(redisVer), MAX_SUPPORTED_REDIS_VERSION); prefixVer = "Unresolved Version. Assumed "; + assert_int_equal(sscanf(MAX_SUPPORTED_REDIS_VERSION, "%d.%d", + &redisVerMajor, &redisVerMinor), 2); } redisVersionInit = 1; @@ -551,65 +553,103 @@ static unsigned char xorstr(const char *str) { return result; } -/* sanitize, sort, and compare */ -#define MAX_LINE_LENGTH 4096 -#define MAX_LINES 4096 -void assert_json_equal(const char* filename1, const char* filename2, int ignoreListOrder) { - char line1[MAX_LINE_LENGTH]; - char line2[MAX_LINE_LENGTH]; - char* lines1[MAX_LINES]; - char* lines2[MAX_LINES]; - int lineCount1 = 0; - int lineCount2 = 0; - int res = -1; - - FILE* file1 = fopen(filename1, "r"); - ASSERT_TRUE(file1, "Failed to open file: %s", filename1); - - FILE* file2 = fopen(filename2, "r"); - ASSERT_TRUE(file2, "Failed to open file: %s", filename2); - - while (fgets(line1, MAX_LINE_LENGTH, file1)) { - sanitizeString(line1, ",{}[] \t\n"); /* roughly sanitize */ - if (strlen(line1) != 0) - lines1[lineCount1++] = strdup(line1); +/* + * Start a python service that reads JSON filename from its STDIN, generates its + * signature, ignoring elements order, and prints SHA256 signature to its STDOUT. + * The reason it is running as a service is to avoid the overhead of starting a + * new process for each evaluation and the intensive use of it. + */ +#define BUFFER_SIZE 1024 +static int pipe_in[2], pipe_out[2]; +static pid_t pid = -1; +void start_json_sign_service() { + if (pipe(pipe_in) == -1 || pipe(pipe_out) == -1) { + perror("pipe failed"); + exit(1); } - while (fgets(line2, MAX_LINE_LENGTH, file2)) { - sanitizeString(line2, ",{}[] \t\n"); /* roughly sanitize */ - if (strlen(line2) != 0) - lines2[lineCount2++] = strdup(line2); + if ( (pid = fork()) == -1) { + perror("fork failed"); + exit(1); } - if (lineCount1 != lineCount2) goto end_cmp; + if (pid == 0) { + /*child*/ + close(pipe_in[1]); /* Close write end of pipe_in */ + close(pipe_out[0]); /* Close read end of pipe_out */ + dup2(pipe_in[0], STDIN_FILENO); /* Redirect child STDIN */ + dup2(pipe_out[1], STDOUT_FILENO); /* Redirect child STDOUT */ + /* now replace child process with python service */ + execlp("python3", "python3", "./test/json_signature_generator.py", NULL); + perror("execlp"); + exit(1); + } else { + /*Parent*/ + close(pipe_in[0]); /* Close read end of pipe_in */ + close(pipe_out[1]); /* Close write end of pipe_out */ + } +} - qsort(lines1, lineCount1, sizeof(char *), compare_json_lines); - qsort(lines2, lineCount2, sizeof(char *), compare_json_lines); +int cmp_json_signatures(const char* filename1, const char* filename2) { + /* Run service if not already running */ + if (pid == -1) + start_json_sign_service(); - for (int i = 0; i < lineCount1; i++) { - /* simplify cmp for ignoreListOrder */ - if ( ((ignoreListOrder) && (xorstr(lines1[i]) != xorstr(lines2[i]))) || - ((!ignoreListOrder) && (strcmp(lines1[i], lines2[i]) != 0)) ) - { - printf("strcmp fail: [%s] [%s]\n", lines1[i], lines2[i]); - goto end_cmp; - } + char buffer[BUFFER_SIZE]; + snprintf(buffer, BUFFER_SIZE, "%s\n", filename1); + write(pipe_in[1], buffer, strlen(buffer)); + + ssize_t bytes_read = read(pipe_out[0], buffer, BUFFER_SIZE - 1); + if (bytes_read <= 0) { + perror("read"); + return 0; } + buffer[bytes_read] = '\0'; /* Null-terminate the string */ + char signature1[BUFFER_SIZE]; + strcpy(signature1, buffer); - res = 0; + snprintf(buffer, BUFFER_SIZE, "%s\n", filename2); + write(pipe_in[1], buffer, strlen(buffer)); -end_cmp: - for (int i = 0; i < lineCount1; i++) - free(lines1[i]); - for (int i = 0; i < lineCount2; i++) - free(lines2[i]); + bytes_read = read(pipe_out[0], buffer, BUFFER_SIZE - 1); - fclose(file1); - fclose(file2); + /* Verify that the signature is of the expected length (SHA256) */ + if (bytes_read != 65) { + printf("strlen(buffer)=%ld %s\n", strlen(buffer), buffer); + perror("read"); + return 0; + } + buffer[bytes_read] = '\0'; /* Null-terminate the string */ + char signature2[BUFFER_SIZE]; + strcpy(signature2, buffer); + + /* Compare signatures */ + if (strcmp(signature1, signature2) == 0) { + return 1; + } else { + return 0; + } +} + +void cleanup_json_sign_service(void) { + if (pid != -1) { + close(pipe_in[1]); + close(pipe_out[0]); + kill(pid, SIGTERM); + waitpid(pid, NULL, 0); + pid = -1; + } +} - if (res == 0) return; +void assert_json_equal(const char* filename1, const char* filename2, int ignoreListOrder) { + UNUSED(ignoreListOrder); + + ASSERT_TRUE(access(filename1, F_OK) != -1, "Failed to open file: %s", filename1); + ASSERT_TRUE(access(filename2, F_OK) != -1, "Failed to open file: %s", filename2); + + if (cmp_json_signatures(filename1, filename2)) + return; - printf("Json files not equal.\n"); printf("---- %s ----\n", filename1); char *f1 = readFile(filename1, NULL, NULL); printf ("%s", f1); @@ -624,7 +664,6 @@ void assert_json_equal(const char* filename1, const char* filename2, int ignoreL assert_true(0); } - /* printHexDump() Generates a formatted hexadecimal and ASCII representation of binary * data. Given a memory address and its length, it produces a human-readable obuf, * displaying byte offsets in hexadecimal and replacing non-printable characters with @@ -640,10 +679,12 @@ void assert_json_equal(const char* filename1, const char* filename2, int ignoreL int printHexDump(const char *input, size_t len, char *obuf, int obuflen) { size_t i; int iout=0, j, llen = 16; /* line len */ - unsigned char buff[llen + 10]; + unsigned char *buff = (unsigned char *)malloc(llen + 10); - if (input == NULL || len <= 0 || obuf == NULL || obuflen < 200 || obuflen > 0xFFFFFF) + if (input == NULL || len <= 0 || obuf == NULL || obuflen < 200 || obuflen > 0xFFFFFF) { + free(buff); return -1; + } for (i = 0, j = 0; (i < len) && (iout + 100 < obuflen) ; i++) { if ((i % llen) == 0) { @@ -677,5 +718,6 @@ int printHexDump(const char *input, size_t len, char *obuf, int obuflen) { iout += snprintf(obuf + iout, obuflen - iout, " %s\n", buff); if (i < len) iout += snprintf(obuf + iout, obuflen - iout, "..."); + free(buff); return iout; } diff --git a/test/test_common.h b/test/test_common.h index a5d486a..a34930b 100644 --- a/test/test_common.h +++ b/test/test_common.h @@ -9,7 +9,7 @@ #include "../api/librdb-api.h" /* RDB library header */ #include "../api/librdb-ext-api.h" /* RDB library extension header */ -#define MAX_SUPPORTED_REDIS_VERSION "7.2" +#define MAX_SUPPORTED_REDIS_VERSION "7.4" #define UNUSED(...) unused( (void *) NULL, __VA_ARGS__); static inline void unused(void *dummy, ...) { (void)(dummy);} @@ -48,6 +48,7 @@ void setValgrind(); void setupRedisServer(const char *extraArgs); const char *getTargetRedisVersion(int *major, int *minor); /* call only after setupRedisServer() */ void teardownRedisServer(void); +void cleanup_json_sign_service(void); int isSetRedisServer(void); char *sendRedisCmd(char *cmd, int expRetType, char *expRsp); int isSupportRestoreModuleAux(void); diff --git a/test/test_main.c b/test/test_main.c index 5c2b280..8bc4bee 100644 --- a/test/test_main.c +++ b/test/test_main.c @@ -309,6 +309,8 @@ int main(int argc, char *argv[]) { /* teardown redis if configured */ teardownRedisServer(); + cleanup_json_sign_service(); + printResPicture(result); return result; diff --git a/test/test_rdb_to_json.c b/test/test_rdb_to_json.c index 8af1e42..a5ffd8e 100644 --- a/test/test_rdb_to_json.c +++ b/test/test_rdb_to_json.c @@ -429,6 +429,8 @@ static void test_r2j_single_string_raw(void **state) { static void test_r2j_multiple_dbs (void **state) { UNUSED(state); RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA); + r2jConf.includeAuxField = 0; + r2jConf.flatten = 0; testRdbToJsonCommon(DUMP_FOLDER("multiple_dbs.rdb"), DUMP_FOLDER("multiple_dbs_data.json"), &r2jConf); }