hexpire - listpack

api/librdb-api.h

@@ -263,6 +263,8 @@ typedef struct RdbHandlersStructCallbacks {
     RdbRes (*handleHashZL)(RdbParser *p, void *userData, RdbBulk ziplist);
     /* Callback to handle a listpack-based hash value */
     RdbRes (*handleHashLP)(RdbParser *p, void *userData, RdbBulk listpack);
+    /* Callback to handle a listpackex-based hash (with expiry on fields) */
+    RdbRes (*handleHashLPEx)(RdbParser *p, void *userData, RdbBulk listpackEx);
     /* Callback to handle a zipmap-based hash value */
     RdbRes (*handleHashZM)(RdbParser *p, void *userData, RdbBulk zipmap);
 

src/ext/handlersFilter.c

@@ -178,6 +178,11 @@ static RdbRes filterHashLP(RdbParser *p, void *userData, RdbBulk listpack) {
     return ((RdbxFilter *) userData)->cbReturnValue;
 }
 
+static RdbRes filterHashLPEx(RdbParser *p, void *userData, RdbBulk listpackEx) {
+    UNUSED(p, listpackEx);
+    return ((RdbxFilter *) userData)->cbReturnValue;
+}
+
 static RdbRes filterHashZM(RdbParser *p, void *userData, RdbBulk zipmap) {
     UNUSED(p, zipmap);
     return ((RdbxFilter *) userData)->cbReturnValue;
@@ -311,6 +316,7 @@ static void defaultFilterStructCb(RdbHandlersStructCallbacks *structCb) {
         filterHashPlain,                    /*handleHashPlain*/
         filterHashZL,                       /*handleHashZL*/
         filterHashLP,                       /*handleHashLP*/
+        filterHashLPEx,                     /*handleHashLPEx*/
         filterHashZM,                       /*handleHashZM*/
         filterSetPlain,                     /*handleSetPlain*/
         filterSetIS,                        /*handleSetIS*/

src/ext/handlersToJson.c

@@ -781,6 +781,7 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo
                 toJsonHash,   /*handleHashPlain*/
                 toJsonStruct, /*handleHashZL*/
                 toJsonStruct, /*handleHashLP*/
+                toJsonStruct, /*handleHashLPEx*/
                 toJsonStruct, /*handleHashZM*/
                 /*set*/
                 toJsonSet,

src/lib/parser.c

@@ -65,6 +65,7 @@ struct ParsingElementInfo peInfo[PE_MAX] = {
         [PE_HASH_META]        = {elementHash, "elementHashMeta", "Parsing Hash with expiry on fields"},
         [PE_HASH_ZL]          = {elementHashZL, "elementHashZL", "Parsing hash Ziplist"},
         [PE_HASH_LP]          = {elementHashLP, "elementHashLP", "Parsing hash Listpack"},
+        [PE_HASH_LP_EX]       = {elementHashLPEx, "elementHashLPEx", "Parsing hash ListpackEx (with expiry)"},
         [PE_HASH_ZM]          = {elementHashZM, "elementHashZM", "Parsing hash Zipmap"},
         /* set */
         [PE_SET]              = {elementSet, "elementSet", "Parsing set"},
@@ -99,6 +100,7 @@ struct ParsingElementInfo peInfo[PE_MAX] = {
         [PE_RAW_HASH_META]    = {elementRawHash, "elementRawHashMeta", "Parsing raw Hash with expiry"},
         [PE_RAW_HASH_ZL]      = {elementRawHashZL, "elementRawHashZL", "Parsing raw hash Ziplist"},
         [PE_RAW_HASH_LP]      = {elementRawHashLP, "elementRawHashLP", "Parsing raw hash Listpack"},
+        [PE_RAW_HASH_LP_EX]   = {elementRawHashLPEx, "elementRawHashLPEx", "Parsing raw hash ListpackEx"},
         [PE_RAW_HASH_ZM]      = {elementRawHashZM, "elementRawHashZM", "Parsing raw hash Zipmap"},
         /* set */
         [PE_RAW_SET]          = {elementRawSet, "elementRawSet", "Parsing raw set"},
@@ -563,6 +565,7 @@ _LIBRDB_API int RDB_handleByLevel(RdbParser *p, RdbDataType type, RdbHandlersLev
             p->handleTypeObjByLevel[RDB_TYPE_HASH_ZIPMAP] = lvl;
             p->handleTypeObjByLevel[RDB_TYPE_HASH_ZIPLIST] = lvl;
             p->handleTypeObjByLevel[RDB_TYPE_HASH_LISTPACK] = lvl;
+            p->handleTypeObjByLevel[RDB_TYPE_HASH_LISTPACK_EX] = lvl;
             break;
         case RDB_DATA_TYPE_MODULE:
             p->handleTypeObjByLevel[RDB_TYPE_MODULE_2] = lvl;
@@ -1050,7 +1053,7 @@ static RdbStatus hashZiplist(RdbParser *p, BulkInfo *ziplistBulk) {
     return RDB_STATUS_OK;
 }
 
-static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) {
+static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk, int withExpiry) {
     size_t items = 0;
 
     if (unlikely(0 == lpValidateIntegrity(lpBulk->ref, lpBulk->len, p->deepIntegCheck, counterCallback, &items))) {
@@ -1059,21 +1062,27 @@ static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) {
         return RDB_STATUS_ERROR;
     }
 
-    if (unlikely((items & 1))) {
+    /* [field][value][expiry] vs [field][value] */
+    int tupleSize = (withExpiry) ? 3 : 2;
+    if (unlikely((items % tupleSize) != 0)) {
         RDB_reportError(p, RDB_ERR_HASH_LP_INTEG_CHECK,
-                        "hashListPack(): Listpack integrity check failed. Uneven number of items.");
+                        "hashListPack(): listpack has unexpected number of items.");
         return RDB_STATUS_ERROR;
     }
 
     if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) {
         registerAppBulkForNextCb(p, lpBulk);
-        CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleHashLP, lpBulk->ref);
+        if (withExpiry)
+            CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleHashLPEx, lpBulk->ref);
+        else
+            CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleHashLP, lpBulk->ref);
         return RDB_STATUS_OK;
     }
 
     p->elmCtx.key.numItemsHint = items;
     unsigned char *iterLP = lpFirst(lpBulk->ref);
     while (iterLP) {
+        int64_t expiryVal = -1;
         unsigned char *field, *value;
         unsigned int fieldLen, valueLen;
         long long fieldVal, valueVal;
@@ -1083,6 +1092,16 @@ static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) {
         iterLP = lpNext(lpBulk->ref, iterLP);
         value = lpGetValue(iterLP, &valueLen, &valueVal);
         iterLP = lpNext(lpBulk->ref, iterLP);
+        if (withExpiry) {
+            if (lpGet(iterLP, &expiryVal, NULL) != NULL) {
+                RDB_reportError(p, RDB_ERR_HASH_LP_INTEG_CHECK,
+                                "hashListPack(): integrity check failed. Expiry is a string instead of value.");
+                return RDB_STATUS_ERROR;
+            }
+
+            if (expiryVal == 0) expiryVal = -1; /* If expiry not set */
+            iterLP = lpNext(lpBulk->ref, iterLP);
+        }
 
         if (!allocEmbeddedBulk(p, field, fieldLen, fieldVal, &embBulk1))
             return RDB_STATUS_ERROR;
@@ -1101,7 +1120,7 @@ static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) {
                          rdbData.handleHashField,
                          embBulk1.binfo.ref,
                          embBulk2.binfo.ref,
-                         -1 /*no expiry*/);
+                         expiryVal);
     }
     return RDB_STATUS_OK;
 }
@@ -1451,6 +1470,7 @@ RdbStatus elementNextRdbType(RdbParser *p) {
         case RDB_TYPE_HASH_METADATA:        return nextParsingElementKeyValue(p, PE_RAW_HASH_META, PE_HASH_META);
         case RDB_TYPE_HASH_ZIPLIST:         return nextParsingElementKeyValue(p, PE_RAW_HASH_ZL, PE_HASH_ZL);
         case RDB_TYPE_HASH_LISTPACK:        return nextParsingElementKeyValue(p, PE_RAW_HASH_LP, PE_HASH_LP);
+        case RDB_TYPE_HASH_LISTPACK_EX:     return nextParsingElementKeyValue(p, PE_RAW_HASH_LP_EX, PE_HASH_LP_EX);
         case RDB_TYPE_HASH_ZIPMAP:          return nextParsingElementKeyValue(p, PE_RAW_HASH_ZM, PE_HASH_ZM);
         /* set */
         case RDB_TYPE_SET:                  return nextParsingElementKeyValue(p, PE_RAW_SET, PE_SET);
@@ -1734,14 +1754,27 @@ RdbStatus elementHashZL(RdbParser *p) {
     return nextParsingElement(p, PE_END_KEY);
 }
 
+RdbStatus elementHashLPEx(RdbParser *p) {
+    BulkInfo *listpackBulk;
+
+    IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &listpackBulk));
+
+    /*** ENTER SAFE STATE ***/
+
+    if (RDB_STATUS_ERROR == hashListPack(p, listpackBulk, 1 /*withExpiry*/))
+        return RDB_STATUS_ERROR;
+
+    return nextParsingElement(p, PE_END_KEY);
+}
+
 RdbStatus elementHashLP(RdbParser *p) {
     BulkInfo *listpackBulk;
 
     IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &listpackBulk));
 
     /*** ENTER SAFE STATE ***/
 
-    if (RDB_STATUS_ERROR == hashListPack(p, listpackBulk))
+    if (RDB_STATUS_ERROR == hashListPack(p, listpackBulk, 0 /*withExpiry*/))
         return RDB_STATUS_ERROR;
 
     return nextParsingElement(p, PE_END_KEY);

src/lib/parser.h

@@ -143,6 +143,7 @@ typedef enum ParsingElementType {
     PE_HASH_META,
     PE_HASH_ZL,
     PE_HASH_LP,
+    PE_HASH_LP_EX,
     PE_HASH_ZM,
     PE_SET,
     PE_SET_IS,
@@ -167,6 +168,7 @@ typedef enum ParsingElementType {
     PE_RAW_HASH_META,
     PE_RAW_HASH_ZL,
     PE_RAW_HASH_LP,
+    PE_RAW_HASH_LP_EX,
     PE_RAW_HASH_ZM,
     PE_RAW_SET,
     PE_RAW_SET_IS,
@@ -534,6 +536,7 @@ RdbStatus elementListZL(RdbParser *p);
 RdbStatus elementHash(RdbParser *p);
 RdbStatus elementHashZL(RdbParser *p);
 RdbStatus elementHashLP(RdbParser *p);
+RdbStatus elementHashLPEx(RdbParser *p);
 RdbStatus elementHashZM(RdbParser *p);
 RdbStatus elementSet(RdbParser *p);
 RdbStatus elementSetIS(RdbParser *p);
@@ -555,6 +558,7 @@ RdbStatus elementRawListZL(RdbParser *p);
 RdbStatus elementRawHash(RdbParser *p);
 RdbStatus elementRawHashZL(RdbParser *p);
 RdbStatus elementRawHashLP(RdbParser *p);
+RdbStatus elementRawHashLPEx(RdbParser *p);
 RdbStatus elementRawHashZM(RdbParser *p);
 RdbStatus elementRawSet(RdbParser *p);
 RdbStatus elementRawSetIS(RdbParser *p);

src/lib/parserRaw.c

@@ -525,6 +525,10 @@ RdbStatus elementRawHashLP(RdbParser *p) {
     return singleStringTypeHandling(p, listpackValidateIntegrityCb, "elementRawHashLP");
 }
 
+RdbStatus elementRawHashLPEx(RdbParser *p) {
+    return singleStringTypeHandling(p, listpackValidateIntegrityCb, "elementRawHashLPEx");
+}
+
 RdbStatus elementRawHashZM(RdbParser *p) {
     return singleStringTypeHandling(p, zipmapValidateIntegrityCb, "elementRawHashZM");
 }

test/dumps/hash_ex_v12_data.json

@@ -0,0 +1,8 @@
+"__aux__" : {
+"redis-ver":"255.255.255",
+"redis-bits":"64",
+"ctime":"1718101998",
+"used-mem":"1168904",
+"aof-base":"0"
+},
+"myhash":{"field1":"value1","field3":"value3","field2":"value2"}

test/dumps/hash_lp_ex_v12_data.json

@@ -0,0 +1,9 @@
+"__aux__" : {
+  "redis-ver":"255.255.255",
+  "redis-bits":"64",
+  "ctime":"1718036120",
+  "used-mem":"1191512",
+  "aof-base":"0"
+},
+
+"myhash":{"field2":"value2","field1":"value1","field3":"value3"}

test/test_common.c

@@ -180,7 +180,7 @@ void assert_file_payload(const char *filename, char *expData, int expLen, MatchT
         printf("%s\n---- file [%s] ----\n", errMsg, filename);
         printHexDump(filedata, filelen, buf, (int) sizeof(buf));
         printf("%s", buf);
-        printf("\n---- Expected %s %s ----\n", matchTypeName, (expMatch) ? "" : "not to match");
+        printf("\n---- Expected %s %s ----\n", matchTypeName, (expMatch) ? "" : "NOT to match");
         printHexDump(expData, expLen, buf, (int) sizeof(buf));
         printf("%s", buf);
         printf("\n------------\n");

test/test_rdb_to_json.c

@@ -156,22 +156,29 @@ static void test_r2j_plain_list_raw (void **state) {
     testRdbToJsonCommon(DUMP_FOLDER("plain_list_v6.rdb"), DUMP_FOLDER("plain_list_v6_raw.json"), &r2jConf);
 }
 
-static void test_r2j_plain_hash_data(void **state) {
+static void test_r2j_hash_data(void **state) {
     UNUSED(state);
     RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA);
-    testRdbToJsonCommon(DUMP_FOLDER("plain_hash_v3.rdb"), DUMP_FOLDER("plain_hash_data.json"), &r2jConf);
+    testRdbToJsonCommon(DUMP_FOLDER("hash_v3.rdb"), DUMP_FOLDER("hash_data.json"), &r2jConf);
 }
 
-static void test_r2j_plain_hash_struct(void **state) {
+static void test_r2j_hash_struct(void **state) {
     UNUSED(state);
     RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_STRUCT);
-    testRdbToJsonCommon(DUMP_FOLDER("plain_hash_v3.rdb"), DUMP_FOLDER("plain_hash_struct.json"), &r2jConf);
+    testRdbToJsonCommon(DUMP_FOLDER("hash_v3.rdb"), DUMP_FOLDER("hash_struct.json"), &r2jConf);
 }
 
-static void test_r2j_plain_hash_raw (void **state) {
+static void test_r2j_hash_raw (void **state) {
     UNUSED(state);
     RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_RAW);
-    testRdbToJsonCommon(DUMP_FOLDER("plain_hash_v3.rdb"), DUMP_FOLDER("plain_hash_raw.json"), &r2jConf);
+    testRdbToJsonCommon(DUMP_FOLDER("hash_v3.rdb"), DUMP_FOLDER("hash_raw.json"), &r2jConf);
+}
+
+/* hash with expiry on fields */
+static void test_r2j_hash_ex_data(void **state) {
+    UNUSED(state);
+    RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA);
+    testRdbToJsonCommon(DUMP_FOLDER("hash_with_expire_v12.rdb"), DUMP_FOLDER("hash_ex_v12_data.json"), &r2jConf);
 }
 
 static void test_r2j_hash_zl_data(void **state) {
@@ -198,6 +205,13 @@ static void test_r2j_hash_lp_data(void **state) {
     testRdbToJsonCommon(DUMP_FOLDER("hash_lp_v11.rdb"), DUMP_FOLDER("hash_lp_v11_data.json"), &r2jConf);
 }
 
+/* lp with expiry on fields */
+static void test_r2j_hash_lp_ex_data(void **state) {
+    UNUSED(state);
+    RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA);
+    testRdbToJsonCommon(DUMP_FOLDER("hash_lp_with_hexpire_v12.rdb"), DUMP_FOLDER("hash_lp_ex_v12_data.json"), &r2jConf);
+}
+
 static void test_r2j_hash_lp_struct(void **state) {
     UNUSED(state);
     RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_STRUCT);
@@ -511,9 +525,11 @@ int group_rdb_to_json(void) {
         cmocka_unit_test(test_r2j_plain_list_raw),
 
         /* hash */
-        cmocka_unit_test(test_r2j_plain_hash_data),
-        cmocka_unit_test(test_r2j_plain_hash_struct),
-        cmocka_unit_test(test_r2j_plain_hash_raw),
+        cmocka_unit_test(test_r2j_hash_data),
+        cmocka_unit_test(test_r2j_hash_struct),
+        cmocka_unit_test(test_r2j_hash_raw),
+
+        cmocka_unit_test(test_r2j_hash_ex_data),
 
         cmocka_unit_test(test_r2j_hash_zl_data),
         cmocka_unit_test(test_r2j_hash_zl_struct),
@@ -523,6 +539,8 @@ int group_rdb_to_json(void) {
         cmocka_unit_test(test_r2j_hash_lp_struct),
         cmocka_unit_test(test_r2j_hash_lp_raw),
 
+        cmocka_unit_test(test_r2j_hash_lp_ex_data),
+
         cmocka_unit_test(test_r2j_hash_zm_data),
         cmocka_unit_test(test_r2j_hash_zm_struct),
         cmocka_unit_test(test_r2j_hash_zm_raw),

test/test_rdb_to_redis.c

@@ -67,21 +67,19 @@ static void rdb_to_json(const char *rdbfile, const char *outfile) {
     RDB_deleteParser(parser);
 }
 
-/* Save & Reload RDB file
+/* Is saving RDB, and librdb reload generates same digest
  *
  * isDigest - if set, compare DB digest before and after reload
  * isRestore - if set, use RESTORE command after reload. Otherwise, plain commands
  */
-static void rdb_save_and_tcp_reload(int isDigest, int isRestore) {
+static void rdb_save_librdb_reload_eq(int isRestore) {
     char *res;
-    const char *rdbfile = DUMP_FOLDER("reload.rdb");
+    const char *rdbfile = TMP_FOLDER("reload.rdb");
     char expectedSha[100];
 
     /* Calculate DB isDigest */
-    if (isDigest) {
-        res = sendRedisCmd("DEBUG DIGEST", REDIS_REPLY_STATUS, NULL);
-        memcpy(expectedSha, res, strlen(res) + 1);
-    }
+    res = sendRedisCmd("DEBUG DIGEST", REDIS_REPLY_STATUS, NULL);
+    memcpy(expectedSha, res, strlen(res) + 1);
 
     /* Keep aside rdb file */
     sendRedisCmd("SAVE", REDIS_REPLY_STATUS, NULL);
@@ -94,7 +92,7 @@ static void rdb_save_and_tcp_reload(int isDigest, int isRestore) {
     /* Reload the RDB file */
     rdb_to_tcp(rdbfile, 1, isRestore, NULL);
 
-    if (isDigest) sendRedisCmd("DEBUG DIGEST", REDIS_REPLY_STATUS, expectedSha);
+    sendRedisCmd("DEBUG DIGEST", REDIS_REPLY_STATUS, expectedSha);
 }
 
 /*
@@ -186,30 +184,35 @@ static void test_rdb_to_redis_single_ziplist(void **state) {
 
 static void test_rdb_to_redis_hash(void **state) {
     UNUSED(state);
-    test_rdb_to_redis_common(DUMP_FOLDER("plain_hash_v3.rdb"), 0, "$4\r\nHSET", NULL);
+    test_rdb_to_redis_common(DUMP_FOLDER("hash_v3.rdb"), 0, "$4\r\nHSET", NULL);
 }
 
-/* - Create hash with expiry on fields via HIREDIS.
- * - Then rdb_save_and_tcp_reload and verify digest before and after
- */
 static void test_rdb_to_redis_hash_with_expire(void **state) {
     UNUSED(state);
 
     /* hash-field-expiration available since 7.4 */
     if ((serverMajorVer<7) || ((serverMajorVer==7) && (serverMinorVer<4)))
         skip();
 
-    /* Test listpack - create RDB file with HFEs */
-
-    /* Test HT - create RDB file with HFEs & calculate DB digest */
+    /* dict (max-lp-entries=0) */
     sendRedisCmd("FLUSHALL", REDIS_REPLY_STATUS, NULL);
     sendRedisCmd("CONFIG SET HASH-MAX-LISTPACK-ENTRIES 0", REDIS_REPLY_STATUS, NULL);
     sendRedisCmd("HSET myhash f1 v1 f2 v2 f3 v3", REDIS_REPLY_INTEGER, "3");
     sendRedisCmd("HPEXPIREAT myhash 70368744177663 FIELDS 2 f1 f2", REDIS_REPLY_ARRAY, "1 1");
-    rdb_save_and_tcp_reload(1 /*digest */, 0 /*restore*/);
-    rdb_save_and_tcp_reload(1 /*digest */, 1 /*restore*/);
+    rdb_save_librdb_reload_eq(0 /*restore*/);
+    rdb_save_librdb_reload_eq(1 /*restore*/);
     sendRedisCmd("HPEXPIRETIME myhash FIELDS 3 f1 f2 f3", REDIS_REPLY_ARRAY,
                  "70368744177663 70368744177663 -1"); /* verify expected output */
+
+    /* listpack */
+    sendRedisCmd("FLUSHALL", REDIS_REPLY_STATUS, NULL);
+    sendRedisCmd("CONFIG SET HASH-MAX-LISTPACK-ENTRIES 512", REDIS_REPLY_STATUS, NULL);
+    sendRedisCmd("HSET myhash f4 v1 f5 v2 f6 v3", REDIS_REPLY_INTEGER, "3");
+    sendRedisCmd("HPEXPIREAT myhash 70368744177663 FIELDS 2 f4 f5", REDIS_REPLY_ARRAY, "1 1");
+    rdb_save_librdb_reload_eq(0 /*restore*/);
+    rdb_save_librdb_reload_eq(1 /*restore*/);
+    sendRedisCmd("HPEXPIRETIME myhash FIELDS 3 f4 f5 f6", REDIS_REPLY_ARRAY,
+                 "70368744177663 70368744177663 -1"); /* verify expected output */
 }
 
 static void test_rdb_to_redis_hash_zl(void **state) {