Integrate python json parser as a service for testing (redis#46)

One of the outputs of librdb is JSON. Until now there was a naive testing code that compared two json files, attempting to ignore JSON elements ordering. This commit replaces it with a python service, run in the background, read JSON filename from its STDIN, generates its signature by ignoring JSON elements order, and prints SHA256 signature to its STDOUT. test_common.c will fork this service. The reason it is running as a service is to avoid the overhead of starting a new process for each evaluation and the intensive use of it.
moticless · Jun 13, 2024 · 46049a1 · 46049a1
1 parent 581221f
commit 46049a1
Show file tree

Hide file tree

Showing 50 changed files with 580 additions and 302 deletions.
diff --git a/README.md b/README.md
@@ -20,7 +20,7 @@ To see cool internal state printouts of the parser, set env-var `LIBRDB_DEBUG_DA
 
     % LIBRDB_DEBUG_DATA=1 make example
 
-To build and run tests, you need to have cmocka unit testing framework installed:
+To build and run tests, you need to have cmocka unit testing and python3 installed:
 
     % make test
 

diff --git a/src/ext/handlersToJson.c b/src/ext/handlersToJson.c
@@ -153,7 +153,7 @@ static RdbRes toJsonDbSize(RdbParser *p, void *userData, uint64_t db_size, uint6
     }
 
     /* output json part */
-    fprintf(ctx->outfile, "    \"%sdbSize\": {\n", jsonMetaPrefix);
+    fprintf(ctx->outfile, "    \"%sdbsize__\": {\n", jsonMetaPrefix); /* group dbsize with {..} */
     fprintf(ctx->outfile, "      \"size\": %" PRIu64 ",\n", db_size);
     fprintf(ctx->outfile, "      \"expires\": %" PRIu64 "\n", exp_size);
     fprintf(ctx->outfile, "    }%s\n", (db_size) ? "," : "");
@@ -171,7 +171,7 @@ static RdbRes toJsonSlotInfo(RdbParser *p, void *userData, RdbSlotInfo *info) {
     }
 
     /* output json part */
-    fprintf(ctx->outfile, "    \"%sSlotInfo\": {\n", jsonMetaPrefix);
+    fprintf(ctx->outfile, "    \"%sslotinfo__\": {\n", jsonMetaPrefix);
     fprintf(ctx->outfile, "      \"slotId\": %lu,\n", info->slot_id);
     fprintf(ctx->outfile, "      \"slotSize\": %lu,\n", info->slot_size);
     fprintf(ctx->outfile, "      \"slotSExpiresSize\": %lu\n", info->expires_slot_size);
@@ -184,7 +184,7 @@ static RdbRes toJsonAuxField(RdbParser *p, void *userData, RdbBulk auxkey, RdbBu
 
     if (ctx->state == R2J_IDLE) {
         ctx->state = R2J_AUX_FIELDS;
-        fprintf(ctx->outfile, "{\n    "); /* group aux-fields with { ... } */
+        fprintf(ctx->outfile, " \"%saux__\": {\n", jsonMetaPrefix); /* group aux-fields with {..} */
     } else if (ctx->state == R2J_AUX_FIELDS) {
         fprintf(ctx->outfile, ",\n    ");
     } else {
@@ -432,14 +432,20 @@ static RdbRes toJsonSet(RdbParser *p, void *userData, RdbBulk member) {
 static RdbRes toJsonZset(RdbParser *p, void *userData, RdbBulk member, double score) {
     RdbxToJson *ctx = userData;
 
-    char score_str[MAX_D2STRING_CHARS];
-    int len = d2string(score_str, sizeof(score_str), score);
+    char scoreStr[MAX_D2STRING_CHARS];
+    int len = d2string(scoreStr, sizeof(scoreStr), score);
+
+    /* -0 is a valid double, but we want to output it as 0 */
+    if ((len == 2) && (scoreStr[0] == '-') && (scoreStr[1] == '0')) {
+        scoreStr[0] = '0';
+        scoreStr[1] = '\0';
+    }
 
     if (ctx->state == R2J_IN_KEY) {
         /* output json part */
         fprintf(ctx->outfile, "{");
         outputQuotedEscaping(ctx, member, RDB_bulkLen(p, member));
-        fprintf(ctx->outfile,  ":\"%.*s\"", len, score_str);
+        fprintf(ctx->outfile, ":\"%.*s\"", len, scoreStr);
 
         /* update new state */
         ctx->state = R2J_IN_ZSET;
@@ -448,7 +454,7 @@ static RdbRes toJsonZset(RdbParser *p, void *userData, RdbBulk member, double sc
         /* output json part */
         fprintf(ctx->outfile, ",");
         outputQuotedEscaping(ctx, member, RDB_bulkLen(p, member));
-        fprintf(ctx->outfile,  ":\"%.*s\"", len, score_str);
+        fprintf(ctx->outfile, ":\"%.*s\"", len, scoreStr);
 
         /* state unchanged */
 
@@ -494,8 +500,9 @@ static RdbRes toJsonFunction(RdbParser *p, void *userData, RdbBulk func) {
 
     if (ctx->state == R2J_IDLE) {
         ctx->state = R2J_FUNCTIONS;
+        fprintf(ctx->outfile, "\"%sfunc__\": {\n", jsonMetaPrefix);
     } else if (ctx->state == R2J_AUX_FIELDS) {
-        fprintf(ctx->outfile, "\n},{\n");
+        fprintf(ctx->outfile, "\n},\n \"%sfunc__\": {\n", jsonMetaPrefix);
         ctx->state = R2J_FUNCTIONS;
     } else if (ctx->state == R2J_FUNCTIONS) {
         fprintf(ctx->outfile, ",\n");

diff --git a/src/lib/defines.h b/src/lib/defines.h
@@ -19,20 +19,22 @@
 /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
 
 /* Object types for encoded objects. */
-#define RDB_TYPE_HASH_ZIPMAP    9
-#define RDB_TYPE_LIST_ZIPLIST  10
-#define RDB_TYPE_SET_INTSET    11
-#define RDB_TYPE_ZSET_ZIPLIST  12
-#define RDB_TYPE_HASH_ZIPLIST  13
-#define RDB_TYPE_LIST_QUICKLIST 14
-#define RDB_TYPE_STREAM_LISTPACKS 15
-#define RDB_TYPE_HASH_LISTPACK 16
-#define RDB_TYPE_ZSET_LISTPACK 17
+#define RDB_TYPE_HASH_ZIPMAP        9
+#define RDB_TYPE_LIST_ZIPLIST       10
+#define RDB_TYPE_SET_INTSET         11
+#define RDB_TYPE_ZSET_ZIPLIST       12
+#define RDB_TYPE_HASH_ZIPLIST       13
+#define RDB_TYPE_LIST_QUICKLIST     14
+#define RDB_TYPE_STREAM_LISTPACKS   15
+#define RDB_TYPE_HASH_LISTPACK      16
+#define RDB_TYPE_ZSET_LISTPACK      17
 #define RDB_TYPE_LIST_QUICKLIST_2   18
 #define RDB_TYPE_STREAM_LISTPACKS_2 19
-#define RDB_TYPE_SET_LISTPACK  20
+#define RDB_TYPE_SET_LISTPACK       20
 #define RDB_TYPE_STREAM_LISTPACKS_3 21
-#define RDB_TYPE_MAX                22
+#define RDB_TYPE_HASH_METADATA      22
+#define RDB_TYPE_HASH_LISTPACK_EX   23
+#define RDB_TYPE_MAX                24
 /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
 
 /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */

diff --git a/test/Makefile b/test/Makefile
@@ -17,7 +17,7 @@ STD         = -std=c99
 STACK       = -fstack-protector-all -Wstack-protector
 WARNS       = -Wall -Wextra -pedantic -Werror -Wno-unused-function
 
-OPTIMIZATION?=-O3
+OPTIMIZATION?=-O0
 
 CFLAGS      = -fPIC $(OPTIMIZATION) $(STD) $(STACK) $(WARNS)
 DEBUG       = -g3 -DDEBUG=1

diff --git a/test/dumps/cluster_slot_info.json b/test/dumps/cluster_slot_info.json
@@ -1,4 +1,4 @@
-[{
+"__aux__" : {
   "redis-ver":"255.255.255",
   "redis-bits":"64",
   "ctime":"1713005699",
@@ -8,15 +8,16 @@
   "repl-offset":"390",
   "aof-base":"0"
 },
-  {
-    "__dbSize": {
-      "size": 1,
-      "expires": 0
-    },
-    "__SlotInfo": {
-      "slotId": 7638,
-      "slotSize": 1,
-      "slotSExpiresSize": 0
-    },
-    "abc":"abc"
-  }]
+
+"__dbsize__": {
+  "size": 1,
+  "expires": 0
+},
+
+"__slotinfo__": {
+  "slotId": 7638,
+  "slotSize": 1,
+  "slotSExpiresSize": 0
+},
+
+"abc":"abc"
diff --git a/test/dumps/function.json b/test/dumps/function.json
diff --git a/test/dumps/function2.json b/test/dumps/function2.json
@@ -1,22 +1,23 @@
-[{
+"__aux__" : {
   "redis-ver":"255.255.255",
   "redis-bits":"64",
   "ctime":"1713086666",
   "used-mem":"966312",
   "aof-base":"0"
-},{
+},
+
+"__func__": {
   "__Function_1":"#!lua name=mylib2\n\nlocal function my_hset2(keys, args)\n  local hash = keys[1]\n  local time = redis.call('TIME')[1]\n  return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall2(keys, args)\n  redis.setresp(3)\n  local hash = keys[1]\n  local res = redis.call('HGETALL', hash)\n  res['map']['_last_modified_'] = nil\n  return res\nend\n\nlocal function my_hlastmodified2(keys, args)\n  local hash = keys[1]\n  return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset2', my_hset2)\nredis.register_function('my_hgetall2', my_hgetall2)\nredis.register_function('my_hlastmodified2', my_hlastmodified2)\n\n",
   "__Function_3":"#!lua name=mylib\n\nlocal function my_hset(keys, args)\n  local hash = keys[1]\n  local time = redis.call('TIME')[1]\n  return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall(keys, args)\n  redis.setresp(3)\n  local hash = keys[1]\n  local res = redis.call('HGETALL', hash)\n  res['map']['_last_modified_'] = nil\n  return res\nend\n\nlocal function my_hlastmodified(keys, args)\n  local hash = keys[1]\n  return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset', my_hset)\nredis.register_function('my_hgetall', my_hgetall)\nredis.register_function('my_hlastmodified', my_hlastmodified)\n\n"
 },
-  {
-    "key_97":"value_97",
-    "key_90":"value_90",
-    "key_93":"value_93",
-    "key_99":"value_99",
-    "key_96":"value_96",
-    "key_92":"value_92",
-    "key_95":"value_95",
-    "key_91":"value_91",
-    "key_94":"value_94",
-    "key_98":"value_98"
-  }]
+
+"key_97":"value_97",
+"key_90":"value_90",
+"key_93":"value_93",
+"key_99":"value_99",
+"key_96":"value_96",
+"key_92":"value_92",
+"key_95":"value_95",
+"key_91":"value_91",
+"key_94":"value_94",
+"key_98":"value_98"
diff --git a/test/dumps/hash_lp_v11_data.json b/test/dumps/hash_lp_v11_data.json
@@ -1,7 +1,10 @@
-"redis-ver":"255.255.255",
-"redis-bits":"64",
-"ctime":"1695280472",
-"used-mem":"1062024",
-"aof-base":"0",
+"__aux__" : {
+  "redis-ver":"255.255.255",
+  "redis-bits":"64",
+  "ctime":"1695280472",
+  "used-mem":"1062024",
+  "aof-base":"0"
+},
+
 "hash2":{"field7":"value7","field8":"value8","9":"value9","field10":"value10","field11":"11","12":"12.0"},
 "hash1":{"field2":"2","field3":"3","field4":"value4.0","field5":"5.0","6":"6"}
diff --git a/test/dumps/hash_lp_v11_raw.json b/test/dumps/hash_lp_v11_raw.json
@@ -1,7 +1,10 @@
-"redis-ver":"255.255.255",
-"redis-bits":"64",
-"ctime":"1695280472",
-"used-mem":"1062024",
-"aof-base":"0",
+"__aux__" : {
+  "redis-ver":"255.255.255",
+  "redis-bits":"64",
+  "ctime":"1695280472",
+  "used-mem":"1062024",
+  "aof-base":"0"
+},
+
 "hash2":"\u0010\u00c3@C@V\u0013V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value \u0007`\u000f\u00008\u00a0\u000f\u00038\u0007\t\u0001\u0080\u0019\u00029\u0007\u0087`\u0019\u000310\b\u0087`*@\b\u0080\u0011\f1\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff",
 "hash1":"\u0010\u00c35>\u000f>\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0080\t\u00023\u0007\u0003\u00a0\t\u000b4\u0007\u0088value4.0\t\u0080\u001b\u000b5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff"
diff --git a/test/dumps/hash_lp_v11_struct.json b/test/dumps/hash_lp_v11_struct.json
@@ -1,7 +1,10 @@
-"redis-ver":"255.255.255",
-"redis-bits":"64",
-"ctime":"1695280472",
-"used-mem":"1062024",
-"aof-base":"0",
+"__aux__" : {
+  "redis-ver":"255.255.255",
+  "redis-bits":"64",
+  "ctime":"1695280472",
+  "used-mem":"1062024",
+  "aof-base":"0"
+},
+
 "hash2":["V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value7\u0007\u0086field8\u0007\u0086value8\u0007\t\u0001\u0086value9\u0007\u0087field10\b\u0087value10\b\u0087field11\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff"],
 "hash1":[">\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0086field3\u0007\u0003\u0001\u0086field4\u0007\u0088value4.0\t\u0086field5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff"]
diff --git a/test/dumps/hash_zl_v6_data.json b/test/dumps/hash_zl_v6_data.json
@@ -1,6 +1,9 @@
-"redis-ver":"6.0.16",
-"redis-bits":"64",
-"ctime":"1690533464",
-"used-mem":"865216",
-"aof-preamble":"0",
+"__aux__" : {
+  "redis-ver":"6.0.16",
+  "redis-bits":"64",
+  "ctime":"1690533464",
+  "used-mem":"865216",
+  "aof-preamble":"0"
+},
+
 "myhash":{"1":"2","3":"4","5":"6","7.0":"8.0","str1":"str2","str3":"str4"}
diff --git a/test/dumps/hash_zl_v6_raw.json b/test/dumps/hash_zl_v6_raw.json
@@ -1,6 +1,9 @@
-"redis-ver":"6.0.16",
-"redis-bits":"64",
-"ctime":"1690533464",
-"used-mem":"865216",
-"aof-preamble":"0",
+"__aux__" : {
+  "redis-ver":"6.0.16",
+  "redis-bits":"64",
+  "ctime":"1690533464",
+  "used-mem":"865216",
+  "aof-preamble":"0"
+},
+
 "myhash":"\r99\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff"
diff --git a/test/dumps/hash_zl_v6_struct.json b/test/dumps/hash_zl_v6_struct.json
@@ -1,6 +1,9 @@
-"redis-ver":"6.0.16",
-"redis-bits":"64",
-"ctime":"1690533464",
-"used-mem":"865216",
-"aof-preamble":"0",
+"__aux__" : {
+  "redis-ver":"6.0.16",
+  "redis-bits":"64",
+  "ctime":"1690533464",
+  "used-mem":"865216",
+  "aof-preamble":"0"
+},
+
 "myhash":["9\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff"]
diff --git a/test/dumps/multiple_dbs_data.json b/test/dumps/multiple_dbs_data.json
@@ -1,13 +1,6 @@
 [{
-  "redis-ver":"255.255.255",
-  "redis-bits":"64",
-  "ctime":"1683103535",
-  "used-mem":"967040",
-  "aof-base":"0"
-},
-  {
-    "x":"0"
-  },{
+  "x":"0"
+},{
   "y":"1"
 },{
   "z":"2"

diff --git a/test/dumps/multiple_lists_strings_data.json b/test/dumps/multiple_lists_strings_data.json
@@ -1,8 +1,11 @@
-"redis-ver":"255.255.255",
-"redis-bits":"64",
-"ctime":"1677580558",
-"used-mem":"937464",
-"aof-base":"0",
+"__aux__" : {
+  "redis-ver":"255.255.255",
+  "redis-bits":"64",
+  "ctime":"1677580558",
+  "used-mem":"937464",
+  "aof-base":"0"
+},
+
 "string2":"Hi there!",
 "mylist1":["v1"],
 "mylist3":["v3","v2","v1"],

diff --git a/test/dumps/multiple_lists_strings_raw.json b/test/dumps/multiple_lists_strings_raw.json
@@ -1,8 +1,11 @@
-"redis-ver":"255.255.255",
-"redis-bits":"64",
-"ctime":"1677580558",
-"used-mem":"937464",
-"aof-base":"0",
+"__aux__" : {
+  "redis-ver":"255.255.255",
+  "redis-bits":"64",
+  "ctime":"1677580558",
+  "used-mem":"937464",
+  "aof-base":"0"
+},
+
 "string2":"\u0000\tHi there!",
 "mylist1":"\u0012\u0001\u0002\u000b\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff",
 "mylist3":"\u0012\u0001\u0002\u0013\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff",

diff --git a/test/dumps/multiple_lists_strings_struct.json b/test/dumps/multiple_lists_strings_struct.json
@@ -1,8 +1,11 @@
-"redis-ver":"255.255.255",
-"redis-bits":"64",
-"ctime":"1677580558",
-"used-mem":"937464",
-"aof-base":"0",
+"__aux__" : {
+  "redis-ver":"255.255.255",
+  "redis-bits":"64",
+  "ctime":"1677580558",
+  "used-mem":"937464",
+  "aof-base":"0"
+},
+
 "string2":"Hi there!",
 "mylist1":["\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff"],
 "mylist3":["\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff"],

diff --git a/test/dumps/plain_hash_data.json b/test/dumps/plain_hash_data.json
@@ -1,5 +1,8 @@
-"redis-ver":"3.2.1",
-"redis-bits":"64",
-"ctime":"1689605183",
-"used-mem":"821904",
+"__aux__" : {
+  "redis-ver":"3.2.1",
+  "redis-bits":"64",
+  "ctime":"1689605183",
+  "used-mem":"821904"
+},
+
 "myhash1":{"5":"5","6":"6","10":"10","1":"1","2":"2","8":"8","4":"4","9":"9","3":"3","7":"7","11":"11"}
diff --git a/test/dumps/plain_hash_raw.json b/test/dumps/plain_hash_raw.json
@@ -1,5 +1,8 @@
-"redis-ver":"3.2.1",
-"redis-bits":"64",
-"ctime":"1689605183",
-"used-mem":"821904",
+"__aux__" : {
+  "redis-ver": "3.2.1",
+  "redis-bits": "64",
+  "ctime": "1689605183",
+  "used-mem": "821904"
+},
+
 "myhash1":"\u0004\u000b\u00c0\u0005\u00c0\u0005\u00c0\u0006\u00c0\u0006\u00c0\n\u00c0\n\u00c0\u0001\u00c0\u0001\u00c0\u0002\u00c0\u0002\u00c0\b\u00c0\b\u00c0\u0004\u00c0\u0004\u00c0\t\u00c0\t\u00c0\u0003\u00c0\u0003\u00c0\u0007\u00c0\u0007\u00c0\u000b\u00c0\u000b"