From ca4ed48db662cbd568df4775003faabf7491043b Mon Sep 17 00:00:00 2001 From: Ozan Tezcan Date: Wed, 8 May 2024 23:11:32 +0300 Subject: [PATCH] Add listpack support, hgetf and hsetf commands (#13209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Changes:** - Adds listpack support to hash field expiration - Implements hgetf/hsetf commands **Listpack support for hash field expiration** We keep field name and value pairs in listpack for the hash type. With this PR, if one of hash field expiration command is called on the key for the first time, it converts listpack layout to triplets to hold field name, value and ttl per field. If a field does not have a TTL, we store zero as the ttl value. Zero is encoded as two bytes in the listpack. So, once we convert listpack to hold triplets, for the fields that don't have a TTL, it will be consuming those extra 2 bytes per item. Fields are ordered by ttl in the listpack to find the field with minimum expiry time efficiently. **New command implementations as part of this PR:** - HGETF command For each specified field get its value and optionally set the field's expiration time in sec/msec /unix-sec/unix-msec: ``` HGETF key [NX | XX | GT | LT] [EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT unix-time-milliseconds | PERSIST] ``` - HSETF command For each specified field value pair: set field to value and optionally set the field's expiration time in sec/msec /unix-sec/unix-msec: ``` HSETF key [DC] [DCF | DOF] [NX | XX | GT | LT] [GETNEW | GETOLD] [EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT unix-time-milliseconds | KEEPTTL] ``` Todo: - Performance improvement. - rdb load/save - aof - defrag --- src/aof.c | 2 +- src/commands.def | 115 ++ src/commands/hgetf.json | 136 ++ src/commands/hsetf.json | 216 +++ src/db.c | 38 +- src/debug.c | 8 +- src/ebuckets.c | 2 +- src/listpack.c | 211 ++- src/listpack.h | 11 +- src/module.c | 2 +- src/object.c | 16 +- src/rdb.c | 12 +- src/server.h | 31 +- src/t_hash.c | 2088 ++++++++++++++++++++++--- src/t_set.c | 8 +- src/t_zset.c | 6 +- tests/unit/type/hash-field-expire.tcl | 1597 +++++++++++++------ 17 files changed, 3693 insertions(+), 806 deletions(-) create mode 100644 src/commands/hgetf.json create mode 100644 src/commands/hsetf.json diff --git a/src/aof.c b/src/aof.c index 610a5c3f4..9632d9c5b 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1944,7 +1944,7 @@ static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) { unsigned int vlen = UINT_MAX; long long vll = LLONG_MAX; - hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll); + hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll, NULL); if (vstr) return rioWriteBulkString(r, (char*)vstr, vlen); else diff --git a/src/commands.def b/src/commands.def index b9416812a..97040f1c5 100644 --- a/src/commands.def +++ b/src/commands.def @@ -3452,6 +3452,52 @@ struct COMMAND_ARG HGETALL_Args[] = { {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; +/********** HGETF ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HGETF history */ +#define HGETF_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HGETF tips */ +#define HGETF_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HGETF key specs */ +keySpec HGETF_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HGETF condition argument table */ +struct COMMAND_ARG HGETF_condition_Subargs[] = { +{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HGETF expiration argument table */ +struct COMMAND_ARG HGETF_expiration_Subargs[] = { +{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("persist",ARG_TYPE_PURE_TOKEN,-1,"PERSIST",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HGETF argument table */ +struct COMMAND_ARG HGETF_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=HGETF_condition_Subargs}, +{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HGETF_expiration_Subargs}, +{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("count",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + /********** HINCRBY ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -3856,6 +3902,73 @@ struct COMMAND_ARG HSET_Args[] = { {MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSET_data_Subargs}, }; +/********** HSETF ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HSETF history */ +#define HSETF_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HSETF tips */ +#define HSETF_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HSETF key specs */ +keySpec HSETF_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HSETF create option argument table */ +struct COMMAND_ARG HSETF_create_option_Subargs[] = { +{MAKE_ARG("dcf",ARG_TYPE_PURE_TOKEN,-1,"DCF",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("dof",ARG_TYPE_PURE_TOKEN,-1,"DOF",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETF return option argument table */ +struct COMMAND_ARG HSETF_return_option_Subargs[] = { +{MAKE_ARG("getnew",ARG_TYPE_PURE_TOKEN,-1,"GETNEW",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("getold",ARG_TYPE_PURE_TOKEN,-1,"GETOLD",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETF condition argument table */ +struct COMMAND_ARG HSETF_condition_Subargs[] = { +{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETF expiration argument table */ +struct COMMAND_ARG HSETF_expiration_Subargs[] = { +{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("keepttl",ARG_TYPE_PURE_TOKEN,-1,"KEEPTTL",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETF data argument table */ +struct COMMAND_ARG HSETF_data_Subargs[] = { +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("value",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETF argument table */ +struct COMMAND_ARG HSETF_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("create key option",ARG_TYPE_PURE_TOKEN,-1,"DC",NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)}, +{MAKE_ARG("create option",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=HSETF_create_option_Subargs}, +{MAKE_ARG("return option",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=HSETF_return_option_Subargs}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=HSETF_condition_Subargs}, +{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HSETF_expiration_Subargs}, +{MAKE_ARG("fvs",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("count",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSETF_data_Subargs}, +}; + /********** HSETNX ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -10989,6 +11102,7 @@ struct COMMAND_STRUCT redisCommandTable[] = { {MAKE_CMD("hexpiretime","Returns the expiration time of a hash field as a Unix timestamp, in seconds.","O(N) where N is the number of arguments to the command","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRETIME_Keyspecs,1,NULL,3),.args=HEXPIRETIME_Args}, {MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HGET_Keyspecs,1,NULL,2),.args=HGET_Args}, {MAKE_CMD("hgetall","Returns all fields and values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETALL_History,0,HGETALL_Tips,1,hgetallCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HGETALL_Keyspecs,1,NULL,1),.args=HGETALL_Args}, +{MAKE_CMD("hgetf","For each specified field, returns its value and optionally set the field's remaining expiration time in seconds / milliseconds","O(N) where N is the number of arguments to the command","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETF_History,0,HGETF_Tips,0,hgetfCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HGETF_Keyspecs,1,NULL,6),.args=HGETF_Args}, {MAKE_CMD("hincrby","Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBY_History,0,HINCRBY_Tips,0,hincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBY_Keyspecs,1,NULL,3),.args=HINCRBY_Args}, {MAKE_CMD("hincrbyfloat","Increments the floating point value of a field by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBYFLOAT_History,0,HINCRBYFLOAT_Tips,0,hincrbyfloatCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBYFLOAT_Keyspecs,1,NULL,3),.args=HINCRBYFLOAT_Args}, {MAKE_CMD("hkeys","Returns all fields in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HKEYS_History,0,HKEYS_Tips,1,hkeysCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HKEYS_Keyspecs,1,NULL,1),.args=HKEYS_Args}, @@ -11003,6 +11117,7 @@ struct COMMAND_STRUCT redisCommandTable[] = { {MAKE_CMD("hrandfield","Returns one or more random fields from a hash.","O(N) where N is the number of fields returned","6.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HRANDFIELD_History,0,HRANDFIELD_Tips,1,hrandfieldCommand,-2,CMD_READONLY,ACL_CATEGORY_HASH,HRANDFIELD_Keyspecs,1,NULL,2),.args=HRANDFIELD_Args}, {MAKE_CMD("hscan","Iterates over fields and values of a hash.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSCAN_History,0,HSCAN_Tips,1,hscanCommand,-3,CMD_READONLY,ACL_CATEGORY_HASH,HSCAN_Keyspecs,1,NULL,5),.args=HSCAN_Args}, {MAKE_CMD("hset","Creates or modifies the value of a field in a hash.","O(1) for each field/value pair added, so O(N) to add N field/value pairs when the command is called with multiple field/value pairs.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSET_History,1,HSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSET_Keyspecs,1,NULL,2),.args=HSET_Args}, +{MAKE_CMD("hsetf","For each specified field, returns its value and optionally set the field's remaining expiration time in seconds / milliseconds","O(N) where N is the number of arguments to the command","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETF_History,0,HSETF_Tips,0,hsetfCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETF_Keyspecs,1,NULL,9),.args=HSETF_Args}, {MAKE_CMD("hsetnx","Sets the value of a field in a hash only when the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETNX_History,0,HSETNX_Tips,0,hsetnxCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETNX_Keyspecs,1,NULL,3),.args=HSETNX_Args}, {MAKE_CMD("hstrlen","Returns the length of the value of a field.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSTRLEN_History,0,HSTRLEN_Tips,0,hstrlenCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HSTRLEN_Keyspecs,1,NULL,2),.args=HSTRLEN_Args}, {MAKE_CMD("httl","Returns the TTL in seconds of a hash field.","O(N) where N is the number of arguments to the command","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,0,httlCommand,-4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HTTL_Keyspecs,1,NULL,3),.args=HTTL_Args}, diff --git a/src/commands/hgetf.json b/src/commands/hgetf.json new file mode 100644 index 000000000..d4668f7ca --- /dev/null +++ b/src/commands/hgetf.json @@ -0,0 +1,136 @@ +{ + "HGETF": { + "summary": "For each specified field, returns its value and optionally set the field's remaining expiration time in seconds / milliseconds", + "complexity": "O(N) where N is the number of arguments to the command", + "group": "hash", + "since": "8.0.0", + "arity": -5, + "function": "hgetfCommand", + "history": [], + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "oneOf": [ + { + "description": "Key does not exist.", + "type": "null" + }, + { + "description": "Array of results", + "type": "array", + "minItems": 1, + "maxItems": 4294967295, + "items": { + "description": "Field value", + "type": "string" + } + } + ] + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "condition", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "nx", + "type": "pure-token", + "token": "NX" + }, + { + "name": "xx", + "type": "pure-token", + "token": "XX" + }, + { + "name": "gt", + "type": "pure-token", + "token": "GT" + }, + { + "name": "lt", + "type": "pure-token", + "token": "LT" + } + ] + }, + { + "name": "expiration", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "seconds", + "type": "integer", + "token": "EX" + }, + { + "name": "milliseconds", + "type": "integer", + "token": "PX" + }, + { + "name": "unix-time-seconds", + "type": "unix-time", + "token": "EXAT" + }, + { + "name": "unix-time-milliseconds", + "type": "unix-time", + "token": "PXAT" + }, + { + "name": "persist", + "type": "pure-token", + "token": "PERSIST" + } + ] + }, + { + "name": "FIELDS", + "type": "string" + }, + { + "name": "count", + "type": "integer" + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } +} diff --git a/src/commands/hsetf.json b/src/commands/hsetf.json new file mode 100644 index 000000000..44f8f4f1b --- /dev/null +++ b/src/commands/hsetf.json @@ -0,0 +1,216 @@ +{ + "HSETF": { + "summary": "For each specified field, returns its value and optionally set the field's remaining expiration time in seconds / milliseconds", + "complexity": "O(N) where N is the number of arguments to the command", + "group": "hash", + "since": "8.0.0", + "arity": -6, + "function": "hsetfCommand", + "history": [], + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "oneOf": [ + { + "description": "Key does not exist and DC condition was given", + "type": "null" + }, + { + "description": "Array of field values when GETNEW/GETOLD is used", + "type": "array", + "minItems": 1, + "maxItems": 4294967295, + "items": { + "oneOf": [ + { + "description": "Field value", + "type": "string" + }, + { + "description": "Field does not exist and couldn't create it (DCF not met)", + "type": "null" + } + ] + } + }, + { + "description": "Array of results", + "type": "array", + "minItems": 1, + "maxItems": 4294967295, + "items": { + "oneOf": [ + { + "description": "Cannot set field value. DOC/DOF condition not met.", + "const": 0 + }, + { + "description": "Set field value without updating TTL", + "const": 1 + }, + { + "description": "Set field value and updated TTL", + "const": 3 + } + ] + } + } + ] + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "create key option", + "optional": true, + "type": "pure-token", + "token": "DC" + }, + { + "name": "create option", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "dcf", + "type": "pure-token", + "token": "DCF" + }, + { + "name": "dof", + "type": "pure-token", + "token": "DOF" + } + ] + }, + { + "name": "return option", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "getnew", + "type": "pure-token", + "token": "GETNEW" + }, + { + "name": "getold", + "type": "pure-token", + "token": "GETOLD" + } + ] + }, + { + "name": "condition", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "nx", + "type": "pure-token", + "token": "NX" + }, + { + "name": "xx", + "type": "pure-token", + "token": "XX" + }, + { + "name": "gt", + "type": "pure-token", + "token": "GT" + }, + { + "name": "lt", + "type": "pure-token", + "token": "LT" + } + ] + }, + { + "name": "expiration", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "seconds", + "type": "integer", + "token": "EX" + }, + { + "name": "milliseconds", + "type": "integer", + "token": "PX" + }, + { + "name": "unix-time-seconds", + "type": "unix-time", + "token": "EXAT" + }, + { + "name": "unix-time-milliseconds", + "type": "unix-time", + "token": "PXAT" + }, + { + "name": "keepttl", + "type": "pure-token", + "token": "KEEPTTL" + } + ] + }, + { + "name": "FVS", + "type": "string" + }, + { + "name": "count", + "type": "integer" + }, + { + "name": "data", + "type": "block", + "multiple": true, + "arguments": [ + { + "name": "field", + "type": "string" + }, + { + "name": "value", + "type": "string" + } + ] + } + ] + } +} diff --git a/src/db.c b/src/db.c index 0ceb54657..2b234e90a 100644 --- a/src/db.c +++ b/src/db.c @@ -1233,6 +1233,40 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) { p = lpNext(o->ptr, p); } cursor = 0; + } else if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_LISTPACK_EX) { + int64_t len; + long long expire_at; + unsigned char *lp = hashTypeListpackGetLp(o); + unsigned char *p = lpFirst(lp); + unsigned char *str, *val; + unsigned char intbuf[LP_INTBUF_SIZE]; + + while (p) { + str = lpGet(p, &len, intbuf); + p = lpNext(lp, p); + val = p; /* Keep pointer to value */ + + p = lpNext(lp, p); + serverAssert(!lpGetValue(p, NULL, &expire_at)); + + if (hashTypeIsExpired(o, expire_at) || + (use_pattern && !stringmatchlen(pat, sdslen(pat), (char *)str, len, 0))) + { + /* jump to the next key/val pair */ + p = lpNext(lp, p); + continue; + } + + /* add key object */ + listAddNodeTail(keys, sdsnewlen(str, len)); + /* add value object */ + if (!no_values) { + str = lpGet(val, &len, intbuf); + listAddNodeTail(keys, sdsnewlen(str, len)); + } + p = lpNext(lp, p); + } + cursor = 0; } else { serverPanic("Not handled encoding in SCAN."); } @@ -1393,7 +1427,7 @@ void renameGenericCommand(client *c, int nx) { /* If hash with expiration on fields then remove it from global HFE DS and * keep next expiration time. Otherwise, dbDelete() will remove it from the * global HFE DS and we will lose the expiration time. */ - if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_HT) + if (o->type == OBJ_HASH) minHashExpireTime = hashTypeRemoveFromExpires(&c->db->hexpires, o); dbDelete(c->db,c->argv[1]); @@ -1472,7 +1506,7 @@ void moveCommand(client *c) { /* If hash with expiration on fields, remove it from global HFE DS and keep * aside registered expiration time. Must be before deletion of the object. * hexpires (ebuckets) embed in stored items its structure. */ - if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_HT) + if (o->type == OBJ_HASH) hashExpireTime = hashTypeRemoveFromExpires(&src->hexpires, o); incrRefCount(o); diff --git a/src/debug.c b/src/debug.c index 6ce1bc71a..84e96aa14 100644 --- a/src/debug.c +++ b/src/debug.c @@ -664,10 +664,14 @@ NULL if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nokeyerr)) == NULL) return; - if (o->encoding != OBJ_ENCODING_LISTPACK) { + if (o->encoding != OBJ_ENCODING_LISTPACK && o->encoding != OBJ_ENCODING_LISTPACK_EX) { addReplyError(c,"Not a listpack encoded object."); } else { - lpRepr(o->ptr); + if (o->encoding == OBJ_ENCODING_LISTPACK) + lpRepr(o->ptr); + else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) + lpRepr(((listpackEx*)o->ptr)->lp); + addReplyStatus(c,"Listpack structure printed on stdout"); } } else if (!strcasecmp(c->argv[1]->ptr,"quicklist") && (c->argc == 3 || c->argc == 4)) { diff --git a/src/ebuckets.c b/src/ebuckets.c index a54030fe2..7f3b900da 100644 --- a/src/ebuckets.c +++ b/src/ebuckets.c @@ -1846,7 +1846,7 @@ ExpireAction expireItemCb(eItem item, void *ctx) { } ExpireAction expireUpdateThirdItemCb(eItem item, void *ctx) { - uint64_t expTime = (uint64_t) ctx; + uint64_t expTime = (uint64_t) (uintptr_t) ctx; static int calls = 0; if ((calls++) == 3) { ebSetMetaExpTime(&(((MyItem *)item)->mexpire), expTime ); diff --git a/src/listpack.c b/src/listpack.c index 0fcdfa9ab..c147b1699 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -1408,15 +1408,20 @@ static inline void lpSaveValue(unsigned char *val, unsigned int len, int64_t lva /* Randomly select a pair of key and value. * total_count is a pre-computed length/2 of the listpack (to avoid calls to lpLength) * 'key' and 'val' are used to store the result key value pair. - * 'val' can be NULL if the value is not needed. */ -void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *key, listpackEntry *val) { + * 'val' can be NULL if the value is not needed. + * 'tuple_len' indicates entry count of a single logical item. It should be 2 + * if listpack was saved as key-value pair or more for key-value-...(n_entries). */ +void lpRandomPair(unsigned char *lp, unsigned long total_count, + listpackEntry *key, listpackEntry *val, int tuple_len) +{ unsigned char *p; + assert(tuple_len >= 2); + /* Avoid div by zero on corrupt listpack */ assert(total_count); - /* Generate even numbers, because listpack saved K-V pair */ - int r = (rand() % total_count) * 2; + int r = (rand() % total_count) * tuple_len; assert((p = lpSeek(lp, r))); key->sval = lpGetValue(p, &(key->slen), &(key->lval)); @@ -1466,26 +1471,31 @@ void lpRandomEntries(unsigned char *lp, unsigned int count, listpackEntry *entri /* Randomly select count of key value pairs and store into 'keys' and * 'vals' args. The order of the picked entries is random, and the selections * are non-unique (repetitions are possible). - * The 'vals' arg can be NULL in which case we skip these. */ -void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals) { + * The 'vals' arg can be NULL in which case we skip these. + * 'tuple_len' indicates entry count of a single logical item. It should be 2 + * if listpack was saved as key-value pair or more for key-value-...(n_entries). */ +void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals, int tuple_len) { unsigned char *p, *key, *value; unsigned int klen = 0, vlen = 0; long long klval = 0, vlval = 0; + assert(tuple_len >= 2); + /* Notice: the index member must be first due to the use in uintCompare */ typedef struct { unsigned int index; unsigned int order; } rand_pick; rand_pick *picks = lp_malloc(sizeof(rand_pick)*count); - unsigned int total_size = lpLength(lp)/2; + unsigned int total_size = lpLength(lp)/tuple_len; /* Avoid div by zero on corrupt listpack */ assert(total_size); /* create a pool of random indexes (some may be duplicate). */ for (unsigned int i = 0; i < count; i++) { - picks[i].index = (rand() % total_size) * 2; /* Generate even indexes */ + /* Generate indexes that key exist at */ + picks[i].index = (rand() % total_size) * tuple_len; /* keep track of the order we picked them */ picks[i].order = i; } @@ -1507,8 +1517,11 @@ void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, l lpSaveValue(value, vlen, vlval, &vals[storeorder]); pickindex++; } - lpindex += 2; - p = lpNext(lp, p); + lpindex += tuple_len; + + for (int i = 0; i < tuple_len - 1; i++) { + p = lpNext(lp, p); + } } lp_free(picks); @@ -1518,13 +1531,20 @@ void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, l * 'vals' args. The selections are unique (no repetitions), and the order of * the picked entries is NOT-random. * The 'vals' arg can be NULL in which case we skip these. + * 'tuple_len' indicates entry count of a single logical item. It should be 2 + * if listpack was saved as key-value pair or more for key-value-...(n_entries). * The return value is the number of items picked which can be lower than the * requested count if the listpack doesn't hold enough pairs. */ -unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals) { +unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, + listpackEntry *keys, listpackEntry *vals, + int tuple_len) +{ + assert(tuple_len >= 2); + unsigned char *p, *key; unsigned int klen = 0; long long klval = 0; - unsigned int total_size = lpLength(lp)/2; + unsigned int total_size = lpLength(lp)/tuple_len; unsigned int index = 0; if (count > total_size) count = total_size; @@ -1532,7 +1552,7 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack p = lpFirst(lp); unsigned int picked = 0, remaining = count; while (picked < count && p) { - assert((p = lpNextRandom(lp, p, &index, remaining, 1))); + assert((p = lpNextRandom(lp, p, &index, remaining, tuple_len))); key = lpGetValue(p, &klen, &klval); lpSaveValue(key, klen, klval, &keys[picked]); assert((p = lpNext(lp, p))); @@ -1554,8 +1574,9 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack * the end of the list. The 'index' needs to be initialized according to the * current zero-based index matching the position of the starting element 'p' * and is updated to match the returned element's zero-based index. If - * 'even_only' is nonzero, an element with an even index is picked, which is - * useful if the listpack represents a key-value pair sequence. + * 'tuple_len' indicates entry count of a single logical item. e.g. This is + * useful if listpack represents key-value pairs. In this case, tuple_len should + * be two and even indexes will be picked. * * Note that this function can return p. In order to skip the previously * returned element, you need to call lpNext() or lpDelete() after each call to @@ -1565,7 +1586,7 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack * p = lpFirst(lp); * i = 0; * while (remaining > 0) { - * p = lpNextRandom(lp, p, &i, remaining--, 0); + * p = lpNextRandom(lp, p, &i, remaining--, 1); * * // ... Do stuff with p ... * @@ -1574,8 +1595,9 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack * } */ unsigned char *lpNextRandom(unsigned char *lp, unsigned char *p, unsigned int *index, - unsigned int remaining, int even_only) + unsigned int remaining, int tuple_len) { + assert(tuple_len > 0); /* To only iterate once, every time we try to pick a member, the probability * we pick it is the quotient of the count left we want to pick and the * count still we haven't visited. This way, we could make every member be @@ -1583,15 +1605,14 @@ unsigned char *lpNextRandom(unsigned char *lp, unsigned char *p, unsigned int *i unsigned int i = *index; unsigned int total_size = lpLength(lp); while (i < total_size && p != NULL) { - if (even_only && i % 2 != 0) { + if (i % tuple_len != 0) { p = lpNext(lp, p); i++; continue; } /* Do we pick this element? */ - unsigned int available = total_size - i; - if (even_only) available /= 2; + unsigned int available = (total_size - i) / tuple_len; double randomDouble = ((double)rand()) / RAND_MAX; double threshold = ((double)remaining) / available; if (randomDouble <= threshold) { @@ -2210,7 +2231,7 @@ int listpackTest(int argc, char *argv[], int flags) { unsigned index = 0; while (remaining > 0) { assert(p != NULL); - p = lpNextRandom(lp, p, &index, remaining--, 0); + p = lpNextRandom(lp, p, &index, remaining--, 1); assert(p != NULL); assert(p != prev); prev = p; @@ -2226,7 +2247,7 @@ int listpackTest(int argc, char *argv[], int flags) { unsigned i = 0; /* Pick from empty listpack returns NULL. */ - assert(lpNextRandom(lp, NULL, &i, 2, 0) == NULL); + assert(lpNextRandom(lp, NULL, &i, 2, 1) == NULL); /* Add some elements and find their pointers within the listpack. */ lp = lpAppend(lp, (unsigned char *)"abc", 3); @@ -2239,19 +2260,19 @@ int listpackTest(int argc, char *argv[], int flags) { assert(lpNext(lp, p2) == NULL); /* Pick zero elements returns NULL. */ - i = 0; assert(lpNextRandom(lp, lpFirst(lp), &i, 0, 0) == NULL); + i = 0; assert(lpNextRandom(lp, lpFirst(lp), &i, 0, 1) == NULL); /* Pick all returns all. */ - i = 0; assert(lpNextRandom(lp, p0, &i, 3, 0) == p0 && i == 0); - i = 1; assert(lpNextRandom(lp, p1, &i, 2, 0) == p1 && i == 1); - i = 2; assert(lpNextRandom(lp, p2, &i, 1, 0) == p2 && i == 2); + i = 0; assert(lpNextRandom(lp, p0, &i, 3, 1) == p0 && i == 0); + i = 1; assert(lpNextRandom(lp, p1, &i, 2, 1) == p1 && i == 1); + i = 2; assert(lpNextRandom(lp, p2, &i, 1, 1) == p2 && i == 2); /* Pick more than one when there's only one left returns the last one. */ - i = 2; assert(lpNextRandom(lp, p2, &i, 42, 0) == p2 && i == 2); + i = 2; assert(lpNextRandom(lp, p2, &i, 42, 1) == p2 && i == 2); /* Pick all even elements returns p0 and p2. */ - i = 0; assert(lpNextRandom(lp, p0, &i, 10, 1) == p0 && i == 0); - i = 1; assert(lpNextRandom(lp, p1, &i, 10, 1) == p2 && i == 2); + i = 0; assert(lpNextRandom(lp, p0, &i, 10, 2) == p0 && i == 0); + i = 1; assert(lpNextRandom(lp, p1, &i, 10, 2) == p2 && i == 2); /* Don't crash even for bad index. */ for (int j = 0; j < 100; j++) { @@ -2264,7 +2285,7 @@ int listpackTest(int argc, char *argv[], int flags) { } i = j % 7; unsigned int remaining = j % 5; - p = lpNextRandom(lp, p, &i, remaining, 0); + p = lpNextRandom(lp, p, &i, remaining, 1); assert(p == p0 || p == p1 || p == p2 || p == NULL); } lpFree(lp); @@ -2275,7 +2296,7 @@ int listpackTest(int argc, char *argv[], int flags) { unsigned char *lp = lpNew(0); lp = lpAppend(lp, (unsigned char*)"abc", 3); lp = lpAppend(lp, (unsigned char*)"123", 3); - lpRandomPair(lp, 1, &key, &val); + lpRandomPair(lp, 1, &key, &val, 2); assert(memcmp(key.sval, "abc", key.slen) == 0); assert(val.lval == 123); lpFree(lp); @@ -2288,7 +2309,7 @@ int listpackTest(int argc, char *argv[], int flags) { lp = lpAppend(lp, (unsigned char*)"123", 3); lp = lpAppend(lp, (unsigned char*)"456", 3); lp = lpAppend(lp, (unsigned char*)"def", 3); - lpRandomPair(lp, 2, &key, &val); + lpRandomPair(lp, 2, &key, &val, 2); if (key.sval) { assert(!memcmp(key.sval, "abc", key.slen)); assert(key.slen == 3); @@ -2301,6 +2322,42 @@ int listpackTest(int argc, char *argv[], int flags) { lpFree(lp); } + TEST("Random pair with tuple_len 3") { + listpackEntry key, val; + unsigned char *lp = lpNew(0); + lp = lpAppend(lp, (unsigned char*)"abc", 3); + lp = lpAppend(lp, (unsigned char*)"123", 3); + lp = lpAppend(lp, (unsigned char*)"xxx", 3); + lp = lpAppend(lp, (unsigned char*)"456", 3); + lp = lpAppend(lp, (unsigned char*)"def", 3); + lp = lpAppend(lp, (unsigned char*)"xxx", 3); + lp = lpAppend(lp, (unsigned char*)"281474976710655", 15); + lp = lpAppend(lp, (unsigned char*)"789", 3); + lp = lpAppend(lp, (unsigned char*)"xxx", 3); + + for (int i = 0; i < 5; i++) { + lpRandomPair(lp, 3, &key, &val, 3); + if (key.sval) { + if (!memcmp(key.sval, "abc", key.slen)) { + assert(key.slen == 3); + assert(val.lval == 123); + } else { + assert(0); + }; + } + if (!key.sval) { + if (key.lval == 456) + assert(!memcmp(val.sval, "def", val.slen)); + else if (key.lval == 281474976710655LL) + assert(val.lval == 789); + else + assert(0); + } + } + + lpFree(lp); + } + TEST("Random pairs with one element") { int count = 5; unsigned char *lp = lpNew(0); @@ -2309,7 +2366,7 @@ int listpackTest(int argc, char *argv[], int flags) { lp = lpAppend(lp, (unsigned char*)"abc", 3); lp = lpAppend(lp, (unsigned char*)"123", 3); - lpRandomPairs(lp, count, keys, vals); + lpRandomPairs(lp, count, keys, vals, 2); assert(memcmp(keys[4].sval, "abc", keys[4].slen) == 0); assert(vals[4].lval == 123); zfree(keys); @@ -2327,7 +2384,7 @@ int listpackTest(int argc, char *argv[], int flags) { lp = lpAppend(lp, (unsigned char*)"123", 3); lp = lpAppend(lp, (unsigned char*)"456", 3); lp = lpAppend(lp, (unsigned char*)"def", 3); - lpRandomPairs(lp, count, keys, vals); + lpRandomPairs(lp, count, keys, vals, 2); for (int i = 0; i < count; i++) { if (keys[i].sval) { assert(!memcmp(keys[i].sval, "abc", keys[i].slen)); @@ -2344,6 +2401,47 @@ int listpackTest(int argc, char *argv[], int flags) { lpFree(lp); } + TEST("Random pairs with many elements and tuple_len 3") { + int count = 5; + lp = lpNew(0); + listpackEntry *keys = zcalloc(sizeof(listpackEntry) * count); + listpackEntry *vals = zcalloc(sizeof(listpackEntry) * count); + + lp = lpAppend(lp, (unsigned char*)"abc", 3); + lp = lpAppend(lp, (unsigned char*)"123", 3); + lp = lpAppend(lp, (unsigned char*)"xxx", 3); + lp = lpAppend(lp, (unsigned char*)"456", 3); + lp = lpAppend(lp, (unsigned char*)"def", 3); + lp = lpAppend(lp, (unsigned char*)"xxx", 3); + lp = lpAppend(lp, (unsigned char*)"281474976710655", 15); + lp = lpAppend(lp, (unsigned char*)"789", 3); + lp = lpAppend(lp, (unsigned char*)"xxx", 3); + + lpRandomPairs(lp, count, keys, vals, 3); + for (int i = 0; i < count; i++) { + if (keys[i].sval) { + if (!memcmp(keys[i].sval, "abc", keys[i].slen)) { + assert(keys[i].slen == 3); + assert(vals[i].lval == 123); + } else { + assert(0); + }; + } + if (!keys[i].sval) { + if (keys[i].lval == 456) + assert(!memcmp(vals[i].sval, "def", vals[i].slen)); + else if (keys[i].lval == 281474976710655LL) + assert(vals[i].lval == 789); + else + assert(0); + } + } + + zfree(keys); + zfree(vals); + lpFree(lp); + } + TEST("Random pairs unique with one element") { unsigned picked; int count = 5; @@ -2353,7 +2451,7 @@ int listpackTest(int argc, char *argv[], int flags) { lp = lpAppend(lp, (unsigned char*)"abc", 3); lp = lpAppend(lp, (unsigned char*)"123", 3); - picked = lpRandomPairsUnique(lp, count, keys, vals); + picked = lpRandomPairsUnique(lp, count, keys, vals, 2); assert(picked == 1); assert(memcmp(keys[0].sval, "abc", keys[0].slen) == 0); assert(vals[0].lval == 123); @@ -2373,7 +2471,7 @@ int listpackTest(int argc, char *argv[], int flags) { lp = lpAppend(lp, (unsigned char*)"123", 3); lp = lpAppend(lp, (unsigned char*)"456", 3); lp = lpAppend(lp, (unsigned char*)"def", 3); - picked = lpRandomPairsUnique(lp, count, keys, vals); + picked = lpRandomPairsUnique(lp, count, keys, vals, 2); assert(picked == 2); for (int i = 0; i < 2; i++) { if (keys[i].sval) { @@ -2391,6 +2489,47 @@ int listpackTest(int argc, char *argv[], int flags) { lpFree(lp); } + TEST("Random pairs unique with many elements and tuple_len 3") { + unsigned picked; + int count = 5; + lp = lpNew(0); + listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count); + listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count); + + lp = lpAppend(lp, (unsigned char*)"abc", 3); + lp = lpAppend(lp, (unsigned char*)"123", 3); + lp = lpAppend(lp, (unsigned char*)"xxx", 3); + lp = lpAppend(lp, (unsigned char*)"456", 3); + lp = lpAppend(lp, (unsigned char*)"def", 3); + lp = lpAppend(lp, (unsigned char*)"xxx", 3); + lp = lpAppend(lp, (unsigned char*)"281474976710655", 15); + lp = lpAppend(lp, (unsigned char*)"789", 3); + lp = lpAppend(lp, (unsigned char*)"xxx", 3); + picked = lpRandomPairsUnique(lp, count, keys, vals, 3); + assert(picked == 3); + for (int i = 0; i < 3; i++) { + if (keys[i].sval) { + if (!memcmp(keys[i].sval, "abc", keys[i].slen)) { + assert(keys[i].slen == 3); + assert(vals[i].lval == 123); + } else { + assert(0); + }; + } + if (!keys[i].sval) { + if (keys[i].lval == 456) + assert(!memcmp(vals[i].sval, "def", vals[i].slen)); + else if (keys[i].lval == 281474976710655LL) + assert(vals[i].lval == 789); + else + assert(0); + } + } + zfree(keys); + zfree(vals); + lpFree(lp); + } + TEST("push various encodings") { lp = lpNew(0); diff --git a/src/listpack.h b/src/listpack.h index 84e1a2a9c..b46939c0e 100644 --- a/src/listpack.h +++ b/src/listpack.h @@ -69,12 +69,15 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep, unsigned char *lpValidateFirst(unsigned char *lp); int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes); unsigned int lpCompare(unsigned char *p, unsigned char *s, uint32_t slen); -void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *key, listpackEntry *val); -void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals); -unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals); +void lpRandomPair(unsigned char *lp, unsigned long total_count, + listpackEntry *key, listpackEntry *val, int tuple_len); +void lpRandomPairs(unsigned char *lp, unsigned int count, + listpackEntry *keys, listpackEntry *vals, int tuple_len); +unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, + listpackEntry *keys, listpackEntry *vals, int tuple_len); void lpRandomEntries(unsigned char *lp, unsigned int count, listpackEntry *entries); unsigned char *lpNextRandom(unsigned char *lp, unsigned char *p, unsigned int *index, - unsigned int remaining, int even_only); + unsigned int remaining, int tuple_len); int lpSafeToAdd(unsigned char* lp, size_t add); void lpRepr(unsigned char *lp); diff --git a/src/module.c b/src/module.c index 11078020f..2f3a81515 100644 --- a/src/module.c +++ b/src/module.c @@ -5295,7 +5295,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { low_flags |= HASH_SET_TAKE_FIELD; robj *argv[2] = {field,value}; - hashTypeTryConversion(key->value,argv,0,1); + hashTypeTryConversion(key->db,key->value,argv,0,1); int updated = hashTypeSet(key->db, key->value, field->ptr, value->ptr, low_flags); count += (flags & REDISMODULE_HASH_COUNT_ALL) ? 1 : updated; diff --git a/src/object.c b/src/object.c index c368dd0df..296c8baae 100644 --- a/src/object.c +++ b/src/object.c @@ -333,17 +333,7 @@ void freeZsetObject(robj *o) { } void freeHashObject(robj *o) { - switch (o->encoding) { - case OBJ_ENCODING_HT: - dictRelease((dict*) o->ptr); - break; - case OBJ_ENCODING_LISTPACK: - lpFree(o->ptr); - break; - default: - serverPanic("Unknown hash encoding type"); - break; - } + hashTypeFree(o); } void freeModuleObject(robj *o) { @@ -939,6 +929,7 @@ char *strEncoding(int encoding) { case OBJ_ENCODING_HT: return "hashtable"; case OBJ_ENCODING_QUICKLIST: return "quicklist"; case OBJ_ENCODING_LISTPACK: return "listpack"; + case OBJ_ENCODING_LISTPACK_EX: return "listpackex"; case OBJ_ENCODING_INTSET: return "intset"; case OBJ_ENCODING_SKIPLIST: return "skiplist"; case OBJ_ENCODING_EMBSTR: return "embstr"; @@ -1051,6 +1042,9 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { } else if (o->type == OBJ_HASH) { if (o->encoding == OBJ_ENCODING_LISTPACK) { asize = sizeof(*o)+zmalloc_size(o->ptr); + } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + listpackEx *lpt = o->ptr; + asize = sizeof(*o) + zmalloc_size(lpt) + zmalloc_size(lpt->lp); } else if (o->encoding == OBJ_ENCODING_HT) { d = o->ptr; di = dictGetIterator(d); diff --git a/src/rdb.c b/src/rdb.c index f190538ad..a6bf7197f 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -694,6 +694,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) { case OBJ_HASH: if (o->encoding == OBJ_ENCODING_LISTPACK) return rdbSaveType(rdb,RDB_TYPE_HASH_LISTPACK); + else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) + return -1; else if (o->encoding == OBJ_ENCODING_HT) return rdbSaveType(rdb,RDB_TYPE_HASH); else @@ -2070,7 +2072,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { /* Too many entries? Use a hash table right from the start. */ if (len > server.hash_max_listpack_entries) - hashTypeConvert(o, OBJ_ENCODING_HT); + hashTypeConvert(o, OBJ_ENCODING_HT, NULL); else if (deep_integrity_validation) { /* In this mode, we need to guarantee that the server won't crash * later when the ziplist is converted to a dict. @@ -2115,7 +2117,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { sdslen(value) > server.hash_max_listpack_value || !lpSafeToAdd(o->ptr, hfieldlen(field) + sdslen(value))) { - hashTypeConvert(o, OBJ_ENCODING_HT); + hashTypeConvert(o, OBJ_ENCODING_HT, NULL); dictUseStoredKeyApi((dict *)o->ptr, 1); ret = dictAdd((dict*)o->ptr, field, value); dictUseStoredKeyApi((dict *)o->ptr, 0); @@ -2331,7 +2333,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { if (hashTypeLength(o, 0) > server.hash_max_listpack_entries || maxlen > server.hash_max_listpack_value) { - hashTypeConvert(o, OBJ_ENCODING_HT); + hashTypeConvert(o, OBJ_ENCODING_HT, NULL); } } break; @@ -2468,7 +2470,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { } if (hashTypeLength(o, 0) > server.hash_max_listpack_entries) - hashTypeConvert(o, OBJ_ENCODING_HT); + hashTypeConvert(o, OBJ_ENCODING_HT, NULL); else o->ptr = lpShrinkToFit(o->ptr); break; @@ -2490,7 +2492,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { } if (hashTypeLength(o, 0) > server.hash_max_listpack_entries) - hashTypeConvert(o, OBJ_ENCODING_HT); + hashTypeConvert(o, OBJ_ENCODING_HT, NULL); break; default: /* totally unreachable */ diff --git a/src/server.h b/src/server.h index bca651ba5..eb6e0405a 100644 --- a/src/server.h +++ b/src/server.h @@ -886,6 +886,7 @@ struct RedisModuleDigest { #define OBJ_ENCODING_QUICKLIST 9 /* Encoded as linked list of listpacks */ #define OBJ_ENCODING_STREAM 10 /* Encoded as a radix tree of listpacks */ #define OBJ_ENCODING_LISTPACK 11 /* Encoded as a listpack */ +#define OBJ_ENCODING_LISTPACK_EX 12 /* Encoded as listpack, extended with metadata */ #define LRU_BITS 24 #define LRU_CLOCK_MAX ((1<lru */ @@ -2432,7 +2433,8 @@ typedef struct { robj *subject; int encoding; - unsigned char *fptr, *vptr; + unsigned char *fptr, *vptr, *tptr; + uint64_t expire_time; /* Only used with OBJ_ENCODING_LISTPACK_EX */ dictIterator *di; dictEntry *de; @@ -3149,13 +3151,27 @@ void setTypeConvert(robj *subject, int enc); int setTypeConvertAndExpand(robj *setobj, int enc, unsigned long cap, int panic); robj *setTypeDup(robj *o); +/* Data structure for OBJ_ENCODING_LISTPACK_EX for hash. It contains listpack + * and metadata fields for hash field expiration.*/ +typedef struct listpackEx { + ExpireMeta meta; /* To be used in order to register the hash in the + global ebuckets (i.e. db->hexpires) with next, + minimum, hash-field to expire. */ + sds key; /* reference to the key, same one that stored in + db->dict. Will be used from active-expiration flow + for notification and deletion of the object, if + needed. */ + void *lp; /* listpack that contains 'key-value-ttl' tuples which + are ordered by ttl. */ +} listpackEx; + /* Hash data type */ #define HASH_SET_TAKE_FIELD (1<<0) #define HASH_SET_TAKE_VALUE (1<<1) #define HASH_SET_COPY 0 -void hashTypeConvert(robj *o, int enc); -void hashTypeTryConversion(robj *subject, robj **argv, int start, int end); +void hashTypeConvert(robj *o, int enc, ebuckets *hexpires); +void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end); int hashTypeExists(robj *o, sds key); int hashTypeDelete(robj *o, sds key); unsigned long hashTypeLength(const robj *o, int subtractExpiredFields); @@ -3165,7 +3181,8 @@ int hashTypeNext(hashTypeIterator *hi, int skipExpiredFields); void hashTypeCurrentFromListpack(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, - long long *vll); + long long *vll, + uint64_t *expireTime); void hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what, char **str, size_t *len, uint64_t *expireTime); void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, @@ -3177,7 +3194,9 @@ int hashTypeSet(redisDb *db, robj *o, sds field, sds value, int flags); robj *hashTypeDup(robj *o, sds newkey, uint64_t *minHashExpire); uint64_t hashTypeRemoveFromExpires(ebuckets *hexpires, robj *o); void hashTypeAddToExpires(redisDb *db, sds key, robj *hashObj, uint64_t expireTime); -int64_t hashTypeGetMinExpire(robj *keyObj); +void hashTypeFree(robj *o); +int hashTypeIsExpired(const robj *o, uint64_t expireAt); +unsigned char *hashTypeListpackGetLp(robj *o); /* Hash-Field data type (of t_hash.c) */ hfield hfieldNew(const void *field, size_t fieldlen, int withExpireMeta); @@ -3637,6 +3656,7 @@ void strlenCommand(client *c); void zrankCommand(client *c); void zrevrankCommand(client *c); void hsetCommand(client *c); +void hsetfCommand(client *c); void hpexpireCommand(client *c); void hexpireCommand(client *c); void hpexpireatCommand(client *c); @@ -3648,6 +3668,7 @@ void hpexpiretimeCommand(client *c); void hpersistCommand(client *c); void hsetnxCommand(client *c); void hgetCommand(client *c); +void hgetfCommand(client *c); void hmgetCommand(client *c); void hdelCommand(client *c); void hlenCommand(client *c); diff --git a/src/t_hash.c b/src/t_hash.c index 88d649125..e2f6f71a6 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -20,8 +20,12 @@ static ExpireMeta* hfieldGetExpireMeta(const eItem field); static ExpireMeta *hashGetExpireMeta(const eItem hash); static void hexpireGenericCommand(client *c, const char *cmd, long long basetime, int unit); static ExpireAction hashTypeActiveExpire(eItem hashObj, void *ctx); -static void hfieldPersist(redisDb *db, robj *hashObj, hfield field); +static void hfieldPersist(robj *hashObj, hfield field); static uint64_t hfieldGetExpireTime(hfield field); +static void updateGlobalHfeDs(redisDb *db, robj *o, uint64_t minExpire, uint64_t minExpireFields); +static uint64_t hashTypeGetNextTimeToExpire(robj *o); +static uint64_t hashTypeGetMinExpire(robj *keyObj); + /* hash dictType funcs */ static int dictHfieldKeyCompare(dict *d, const void *key1, const void *key2); @@ -221,7 +225,6 @@ typedef struct HashTypeSetEx { FieldGet fieldGet; /* [GETNEW | GETOLD] TODO */ /*** metadata ***/ - dictExpireMetadata *dictExpireMeta; /* keep ref to dict's metadata */ uint64_t minExpire; /* if uninit EB_EXPIRE_TIME_INVALID */ redisDb *db; robj *key, *hashObj; @@ -297,6 +300,368 @@ static void hashDictWithExpireOnRelease(dict *d) { ebDestroy(&dictExpireMeta->hfe, &hashFieldExpireBucketsType, NULL); } +/*----------------------------------------------------------------------------- + * listpackEx functions + *----------------------------------------------------------------------------*/ +/* + * If any of hash field expiration command is called on a listpack hash object + * for the first time, we convert it to OBJ_ENCODING_LISTPACK_EX encoding. + * We allocate "struct listpackEx" which holds listpack pointer and metadata to + * register key to the global DS. In the listpack, we append another TTL entry + * for each field-value pair. From now on, listpack will have triplets in it: + * field-value-ttl. If TTL is not set for a field, we store 'zero' as the TTL + * value. 'zero' is encoded as two bytes in the listpack. Memory overhead of a + * non-existing TTL will be two bytes per field. + * + * Fields in the listpack will be ordered by TTL. Field with the smallest expiry + * time will be the first item. Fields without TTL will be at the end of the + * listpack. This way, it is easier/faster to find expired items. + */ + +#define HASH_LP_NO_TTL 0 + +static struct listpackEx *listpackExCreate(void) { + listpackEx *lpt = zcalloc(sizeof(*lpt)); + lpt->meta.trash = 1; + lpt->lp = NULL; + lpt->key = NULL; + return lpt; +} + +static void listpackExFree(listpackEx *lpt) { + lpFree(lpt->lp); + zfree(lpt); +} + +/* Returns number of expired fields. */ +static uint64_t listpackExExpireDryRun(const robj *o) { + serverAssert(o->encoding == OBJ_ENCODING_LISTPACK_EX); + + uint64_t expired = 0; + unsigned char *fptr, *s; + listpackEx *lpt = o->ptr; + + fptr = lpFirst(lpt->lp); + while (fptr != NULL) { + long long val; + + fptr = lpNext(lpt->lp, fptr); + serverAssert(fptr); + fptr = lpNext(lpt->lp, fptr); + serverAssert(fptr); + + s = lpGetValue(fptr, NULL, &val); + serverAssert(!s); + + if (!hashTypeIsExpired(o, val)) + break; + + expired++; + fptr = lpNext(lpt->lp, fptr); + } + + return expired; +} + +/* Returns the expiration time of the item with the nearest expiration. */ +static uint64_t listpackExGetMinExpire(robj *o) { + serverAssert(o->encoding == OBJ_ENCODING_LISTPACK_EX); + + long long expireAt; + unsigned char *fptr, *s; + listpackEx *lpt = o->ptr; + + /* As fields are ordered by expire time, first field will have the smallest + * expiry time. Third element is the expiry time of the first field */ + fptr = lpSeek(lpt->lp, 2); + if (fptr != NULL) { + s = lpGetValue(fptr, NULL, &expireAt); + serverAssert(!s); + + /* Check if this is a non-volatile field. */ + if (expireAt != HASH_LP_NO_TTL) + return expireAt; + } + + return EB_EXPIRE_TIME_INVALID; +} + +/* Walk over fields and delete the expired ones. */ +static void listpackExExpire(robj *o, ExpireInfo *info) { + serverAssert(o->encoding == OBJ_ENCODING_LISTPACK_EX); + uint64_t min = EB_EXPIRE_TIME_INVALID; + unsigned char *ptr, *field, *s; + listpackEx *lpt = o->ptr; + + ptr = lpFirst(lpt->lp); + while (ptr != NULL && (info->itemsExpired < info->maxToExpire)) { + long long val; + + field = ptr; + ptr = lpNext(lpt->lp, ptr); + serverAssert(ptr); + ptr = lpNext(lpt->lp, ptr); + serverAssert(ptr); + + s = lpGetValue(ptr, NULL, &val); + serverAssert(!s); + + /* Fields are ordered by expiry time. If we reached to a non-expired + * field or a non-volatile field, we know rest is not yet expired. */ + if (val == HASH_LP_NO_TTL || (uint64_t) val > info->now) + break; + + server.stat_expired_hash_fields++; + lpt->lp = lpDeleteRangeWithEntry(lpt->lp, &field, 3); + ptr = field; + info->itemsExpired++; + } + + min = hashTypeGetNextTimeToExpire(o); + info->nextExpireTime = (min != EB_EXPIRE_TIME_INVALID) ? min : 0; +} + +/* Remove TTL from the field. */ +static void listpackExPersist(robj *o, sds field, unsigned char *fptr, + unsigned char *vptr) +{ + serverAssert(o->encoding == OBJ_ENCODING_LISTPACK_EX); + + unsigned char tmp[512]; + unsigned int slen; + long long val; + unsigned char *s; + sds p = NULL; + listpackEx *lpt = o->ptr; + + /* To persist a field, we have to delete it first and append to the end as + * we want to maintain order by expiry time. Before deleting it, copy the + * value if it is stored as string. */ + s = lpGetValue(vptr, &slen, &val); + if (s) { + /* Normally, item length in the listpack is limited by + * 'hash-max-listpack-value' config. It is unlikely, but it might be + * larger than sizeof(tmp). */ + if (slen > sizeof(tmp)) + p = sdsnewlen(s, slen); + else + memcpy(tmp, s, slen); + } + + /* Delete field name, value and expiry time. */ + lpt->lp = lpDeleteRangeWithEntry(lpt->lp, &fptr, 3); + + /* Append field to the end as it does not have expiry time. */ + lpt->lp = lpAppend(lpt->lp, (unsigned char*)field, sdslen(field)); + + if (s) + lpt->lp = lpAppend(lpt->lp, p ? (unsigned char*) p : tmp, slen); + else + lpt->lp = lpAppendInteger(lpt->lp, val); + + lpt->lp = lpAppendInteger(lpt->lp, HASH_LP_NO_TTL); + + sdsfree(p); +} + +/* If expiry time is changed, this function will place field into the correct + * position. First, it deletes the field and re-inserts to the listpack ordered + * by expiry time. */ +static void listpackExUpdateExpiry(robj *o, sds field, + unsigned char *fptr, + unsigned char *vptr, + uint64_t expireAt) { + unsigned int slen; + long long val; + unsigned char tmp[512] = {0}; + unsigned char *valstr, *s, *elem; + listpackEx *lpt = o->ptr; + sds tmpval = NULL; + + /* Copy value */ + valstr = lpGetValue(vptr, &slen, &val); + if (valstr) { + /* Normally, item length in the listpack is limited by + * 'hash-max-listpack-value' config. It is unlikely, but it might be + * larger than sizeof(tmp). */ + if (slen > sizeof(tmp)) + tmpval = sdsnewlen(valstr, slen); + else + memcpy(tmp, valstr, slen); + } + + /* Delete field name, value and expiry time */ + lpt->lp = lpDeleteRangeWithEntry(lpt->lp, &fptr, 3); + + /* Insert to the listpack */ + fptr = lpFirst(lpt->lp); + while (fptr) { + long long currExpiry; + + elem = fptr; /* Keep a pointer to field name */ + fptr = lpNext(lpt->lp, fptr); + serverAssert(fptr); + fptr = lpNext(lpt->lp, fptr); + serverAssert(fptr); + + s = lpGetValue(fptr, NULL, &currExpiry); + serverAssert(!s); + + if (currExpiry == HASH_LP_NO_TTL || (uint64_t) currExpiry >= expireAt) { + /* Found a field with no expiry time or with a higher expiry time. + * Insert new field just before it. */ + lpt->lp = lpInsertString(lpt->lp, (unsigned char*) field, + sdslen(field), elem, LP_BEFORE, &fptr); + + /* Insert value after field name */ + if (valstr) { + lpt->lp = lpInsertString(lpt->lp, + tmpval ? (unsigned char*) tmpval : tmp, + slen, fptr, LP_AFTER, &fptr); + } else { + lpt->lp = lpInsertInteger(lpt->lp, val, fptr, LP_AFTER, &fptr); + } + + /* Insert expiry time after value. */ + lpt->lp = lpInsertInteger(lpt->lp, (long long) expireAt, fptr, + LP_AFTER, NULL); + goto out; + } + + fptr = lpNext(lpt->lp, fptr); + } + + /* Listpack is empty, append new item */ + lpt->lp = lpAppend(lpt->lp, (unsigned char*)field, sdslen(field)); + if (valstr) + lpt->lp = lpAppend(lpt->lp, tmpval ? (unsigned char*) tmpval : tmp, slen); + else + lpt->lp = lpAppendInteger(lpt->lp, val); + + lpt->lp = lpAppendInteger(lpt->lp, (long long) expireAt); + +out: + sdsfree(tmpval); +} + +/* Add new field ordered by expire time. */ +static void listpackExAddNew(robj *o, sds field, sds value, uint64_t expireAt) { + unsigned char *fptr, *s, *elem; + listpackEx *lpt = o->ptr; + + /* Shortcut, just append at the end if this is a non-volatile field. */ + if (expireAt == HASH_LP_NO_TTL) { + goto append; + } + + fptr = lpFirst(lpt->lp); + while (fptr) { + long long currExpiry; + + elem = fptr; /* Keep a pointer to field name */ + fptr = lpNext(lpt->lp, fptr); + serverAssert(fptr); + fptr = lpNext(lpt->lp, fptr); + serverAssert(fptr); + + s = lpGetValue(fptr, NULL, &currExpiry); + serverAssert(!s); + + if (currExpiry == HASH_LP_NO_TTL || (uint64_t) currExpiry >= expireAt) { + /* Found a field with no expiry time or with a higher expiry time. + * Insert new field just before it. */ + lpt->lp = lpInsertString(lpt->lp, (unsigned char*) field, + sdslen(field), elem, LP_BEFORE, &fptr); + + lpt->lp = lpInsertString(lpt->lp,(unsigned char*) value, sdslen(value), + fptr, LP_AFTER, &fptr); + + /* Insert expiry time after value. */ + lpt->lp = lpInsertInteger(lpt->lp, (long long) expireAt, fptr, + LP_AFTER, NULL); + return; + } + + fptr = lpNext(lpt->lp, fptr); + } + + /* Either listpack is empty or field expiry time is HASH_LP_NO_TTL */ +append: + lpt->lp = lpAppend(lpt->lp, (unsigned char*)field, sdslen(field)); + lpt->lp = lpAppend(lpt->lp, (unsigned char*)value, sdslen(value)); + lpt->lp = lpAppendInteger(lpt->lp, (long long) expireAt); +} + +/* Update field expire time. */ +SetExRes hashTypeSetExpiryListpack(HashTypeSetEx *ex, sds field, + unsigned char *fptr, unsigned char *vptr, + unsigned char *tptr, uint64_t expireAt) +{ + long long expireTime; + uint64_t prevExpire = EB_EXPIRE_TIME_INVALID; + unsigned char *s; + + s = lpGetValue(tptr, NULL, &expireTime); + serverAssert(!s); + + if (expireTime != HASH_LP_NO_TTL) { + prevExpire = (uint64_t) expireTime; + } + + if (prevExpire == EB_EXPIRE_TIME_INVALID) { + if (ex->expireSetCond & (HFE_XX | HFE_LT | HFE_GT)) + return HSETEX_NO_CONDITION_MET; + } else { + if (((ex->expireSetCond == HFE_GT) && (prevExpire >= expireAt)) || + ((ex->expireSetCond == HFE_LT) && (prevExpire <= expireAt)) || + (ex->expireSetCond == HFE_NX) ) + return HSETEX_NO_CONDITION_MET; + + /* Track of minimum expiration time (only later update global HFE DS) */ + if (ex->minExpireFields > prevExpire) + ex->minExpireFields = prevExpire; + } + + /* if expiration time is in the past */ + if (unlikely(checkAlreadyExpired(expireAt))) { + hashTypeDelete(ex->hashObj, field); + ex->fieldDeleted++; + return HSETEX_DELETED; + } + + if (ex->minExpireFields > expireAt) + ex->minExpireFields = expireAt; + + listpackExUpdateExpiry(ex->hashObj, field, fptr, vptr, expireAt); + ex->fieldUpdated++; + return HSETEX_OK; +} + +/* Returns 1 if expired */ +int hashTypeIsExpired(const robj *o, uint64_t expireAt) { + if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + if (expireAt == HASH_LP_NO_TTL) + return 0; + } else if (o->encoding == OBJ_ENCODING_HT) { + if (expireAt == EB_EXPIRE_TIME_INVALID) + return 0; + } else { + serverPanic("Unknown encoding: %d", o->encoding); + } + + return (mstime_t) expireAt < commandTimeSnapshot(); +} + +/* Returns listpack pointer of the object. */ +unsigned char *hashTypeListpackGetLp(robj *o) { + if (o->encoding == OBJ_ENCODING_LISTPACK) + return o->ptr; + else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) + return ((listpackEx*)o->ptr)->lp; + + serverPanic("Unknown encoding: %d", o->encoding); +} + /*----------------------------------------------------------------------------- * Hash type API *----------------------------------------------------------------------------*/ @@ -304,18 +669,19 @@ static void hashDictWithExpireOnRelease(dict *d) { /* Check the length of a number of objects to see if we need to convert a * listpack to a real hash. Note that we only check string encoded objects * as their string length can be queried in constant time. */ -void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { +void hashTypeTryConversion(redisDb *db, robj *o, robj **argv, int start, int end) { int i; size_t sum = 0; - if (o->encoding != OBJ_ENCODING_LISTPACK) return; + if (o->encoding != OBJ_ENCODING_LISTPACK && o->encoding != OBJ_ENCODING_LISTPACK_EX) + return; /* We guess that most of the values in the input are unique, so * if there are enough arguments we create a pre-sized hash, which * might over allocate memory if there are duplicates. */ size_t new_fields = (end - start + 1) / 2; if (new_fields > server.hash_max_listpack_entries) { - hashTypeConvert(o, OBJ_ENCODING_HT); + hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires); dictExpand(o->ptr, new_fields); return; } @@ -325,13 +691,13 @@ void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { continue; size_t len = sdslen(argv[i]->ptr); if (len > server.hash_max_listpack_value) { - hashTypeConvert(o, OBJ_ENCODING_HT); + hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires); return; } sum += len; } - if (!lpSafeToAdd(o->ptr, sum)) - hashTypeConvert(o, OBJ_ENCODING_HT); + if (!lpSafeToAdd(hashTypeListpackGetLp(o), sum)) + hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires); } /* Get the value from a listpack encoded hash, identified by field. @@ -343,17 +709,41 @@ int hashTypeGetFromListpack(robj *o, sds field, { unsigned char *zl, *fptr = NULL, *vptr = NULL; - serverAssert(o->encoding == OBJ_ENCODING_LISTPACK); - - zl = o->ptr; - fptr = lpFirst(zl); - if (fptr != NULL) { - fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); + if (o->encoding == OBJ_ENCODING_LISTPACK) { + zl = o->ptr; + fptr = lpFirst(zl); if (fptr != NULL) { - /* Grab pointer to the value (fptr points to the field) */ - vptr = lpNext(zl, fptr); - serverAssert(vptr != NULL); + fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); + if (fptr != NULL) { + /* Grab pointer to the value (fptr points to the field) */ + vptr = lpNext(zl, fptr); + serverAssert(vptr != NULL); + } } + } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + long long expire; + unsigned char *h; + listpackEx *lpt = o->ptr; + + fptr = lpFirst(lpt->lp); + if (fptr != NULL) { + fptr = lpFind(lpt->lp, fptr, (unsigned char*)field, sdslen(field), 2); + if (fptr != NULL) { + vptr = lpNext(lpt->lp, fptr); + serverAssert(vptr != NULL); + + h = lpNext(lpt->lp, vptr); + serverAssert(h != NULL); + + h = lpGetValue(h, NULL, &expire); + serverAssert(h == NULL); + + if (hashTypeIsExpired(o, expire)) + return -1; + } + } + } else { + serverPanic("Unknown hash encoding: %d", o->encoding); } if (vptr != NULL) { @@ -392,7 +782,9 @@ sds hashTypeGetFromHashTable(robj *o, sds field) { * can always check the function return by checking the return value * for C_OK and checking if vll (or vstr) is NULL. */ int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, long long *vll) { - if (o->encoding == OBJ_ENCODING_LISTPACK) { + if (o->encoding == OBJ_ENCODING_LISTPACK || + o->encoding == OBJ_ENCODING_LISTPACK_EX) + { *vstr = NULL; if (hashTypeGetFromListpack(o, field, vstr, vlen, vll) == 0) return C_OK; @@ -536,7 +928,8 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict return HSETEX_NO_CONDITION_MET; /* remove old expiry time from hash's private ebuckets */ - ebRemove(&ex->dictExpireMeta->hfe, &hashFieldExpireBucketsType, hfOld); + dictExpireMetadata *dm = (dictExpireMetadata *) dictMetadata(ht); + ebRemove(&dm->hfe, &hashFieldExpireBucketsType, hfOld); /* Track of minimum expiration time (only later update global HFE DS) */ if (ex->minExpireFields > prevExpire) @@ -567,7 +960,8 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict if (ex->minExpireFields > expireAt) ex->minExpireFields = expireAt; - ebAdd(&ex->dictExpireMeta->hfe, &hashFieldExpireBucketsType, hfNew, expireAt); + dictExpireMetadata *dm = (dictExpireMetadata *) dictMetadata(ht); + ebAdd(&dm->hfe, &hashFieldExpireBucketsType, hfNew, expireAt); ex->fieldUpdated++; return HSETEX_OK; } @@ -589,13 +983,17 @@ SetExRes hashTypeSetEx(redisDb *db, robj *o, sds field, HashTypeSet *setKeyVal, /* Check if the field is too long for listpack, and convert before adding the item. * This is needed for HINCRBY* case since in other commands this is handled early by * hashTypeTryConversion, so this check will be a NOP. */ - if (isSetKeyValue && o->encoding == OBJ_ENCODING_LISTPACK) { + if (isSetKeyValue && (o->encoding == OBJ_ENCODING_LISTPACK || + o->encoding == OBJ_ENCODING_LISTPACK_EX)) + { if (sdslen(field) > server.hash_max_listpack_value || sdslen(setKeyVal->value) > server.hash_max_listpack_value) - hashTypeConvert(o, OBJ_ENCODING_HT); + hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires); } - if (o->encoding == OBJ_ENCODING_LISTPACK) { + if (o->encoding == OBJ_ENCODING_LISTPACK || + o->encoding == OBJ_ENCODING_LISTPACK_EX) + { res = hashTypeSetExListpack(db, o, field, setKeyVal, expireAt, exInfo); goto SetExDone; } else if (o->encoding != OBJ_ENCODING_HT) { @@ -625,7 +1023,7 @@ SetExRes hashTypeSetEx(redisDb *db, robj *o, sds field, HashTypeSet *setKeyVal, if (de == NULL) { /* If attached TTL to the old field, then remove it from hash's private ebuckets */ hfield oldField = dictGetKey(existing); - hfieldPersist(db, o, oldField); + hfieldPersist(o, oldField); hfieldFree(oldField); sdsfree(dictGetVal(existing)); @@ -668,7 +1066,6 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm ex->fieldSetCond = fieldSetCond; ex->fieldGet = fieldGet; /* TODO */ ex->expireSetCond = expireSetCond; - ex->dictExpireMeta = NULL; ex->minExpire = EB_EXPIRE_TIME_INVALID; ex->c = c; ex->cmd = cmd; @@ -679,30 +1076,34 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm ex->fieldUpdated = 0; ex->minExpireFields = EB_EXPIRE_TIME_INVALID; - /* Take care dict has HFE metadata */ - if (!isDictWithMetaHFE(ht)) { - /* Realloc (only header of dict) with metadata for hash-field expiration */ - dictTypeAddMeta(&ht, &mstrHashDictTypeWithHFE); - ex->dictExpireMeta = (dictExpireMetadata *) dictMetadata(ht); - ex->hashObj->ptr = ht; + if (ex->hashObj->encoding == OBJ_ENCODING_LISTPACK) { + hashTypeConvert(ex->hashObj, OBJ_ENCODING_LISTPACK_EX, &c->db->hexpires); - /* Find the key in the keyspace. Need to keep reference to the key for - * notifications or even removal of the hash */ - dictEntry *de = dbFind(db, key->ptr); + listpackEx *lpt = ex->hashObj->ptr; + dictEntry *de = dbFind(c->db, key->ptr); serverAssert(de != NULL); + lpt->key = dictGetKey(de); + } else if (ex->hashObj->encoding == OBJ_ENCODING_HT) { + /* Take care dict has HFE metadata */ + if (!isDictWithMetaHFE(ht)) { + /* Realloc (only header of dict) with metadata for hash-field expiration */ + dictTypeAddMeta(&ht, &mstrHashDictTypeWithHFE); + dictExpireMetadata *m = (dictExpireMetadata *) dictMetadata(ht); + ex->hashObj->ptr = ht; - /* Fillup dict HFE metadata */ - ex->dictExpireMeta->key = dictGetKey(de); /* reference key in keyspace */ - ex->dictExpireMeta->hfe = ebCreate(); /* Allocate HFE DS */ - ex->dictExpireMeta->expireMeta.trash = 1; /* mark as trash (as long it wasn't ebAdd()) */ - } else { - ex->dictExpireMeta = (dictExpireMetadata *) dictMetadata(ht); - ExpireMeta *expireMeta = &ex->dictExpireMeta->expireMeta; + /* Find the key in the keyspace. Need to keep reference to the key for + * notifications or even removal of the hash */ + dictEntry *de = dbFind(db, key->ptr); + serverAssert(de != NULL); - /* Keep aside min HFE before update. Verify it is not trash */ - if (expireMeta->trash == 0) - ex->minExpire = ebGetMetaExpTime(&ex->dictExpireMeta->expireMeta); + /* Fillup dict HFE metadata */ + m->key = dictGetKey(de); /* reference key in keyspace */ + m->hfe = ebCreate(); /* Allocate HFE DS */ + m->expireMeta.trash = 1; /* mark as trash (as long it wasn't ebAdd()) */ + } } + + ex->minExpire = hashTypeGetMinExpire(ex->hashObj); return C_OK; } @@ -723,33 +1124,7 @@ void hashTypeSetExDone(HashTypeSetEx *ex) { dbDelete(ex->db,ex->key); if (ex->c) notifyKeyspaceEvent(NOTIFY_GENERIC,"del",ex->key, ex->db->id); } else { - - /* If minimum HFE of the hash is smaller than expiration time of the - * specified fields in the command as well as it is smaller or equal - * than expiration time provided in the command, then the minimum - * HFE of the hash won't change following this command. */ - if (ex->minExpire < ex->minExpireFields) - return; - - /* retrieve new expired time. It might have changed. */ - uint64_t newMinExpire = ebGetNextTimeToExpire(ex->dictExpireMeta->hfe, - &hashFieldExpireBucketsType); - - /* Calculate the diff between old minExpire and newMinExpire. If it is - * only few seconds, then don't have to update global HFE DS. At the worst - * case fields of hash will be active-expired up to few seconds later. - * - * In any case, active-expire operation will know to update global - * HFE DS more efficiently than here for a single item. - */ - uint64_t diff = (ex->minExpire > newMinExpire) ? - (ex->minExpire - newMinExpire) : (newMinExpire - ex->minExpire); - if (diff < HASH_NEW_EXPIRE_DIFF_THRESHOLD) return; - - if (ex->minExpire != EB_EXPIRE_TIME_INVALID) - ebRemove(&ex->db->hexpires, &hashExpireBucketsType, ex->hashObj); - if (newMinExpire != EB_EXPIRE_TIME_INVALID) - ebAdd(&ex->db->hexpires, &hashExpireBucketsType, ex->hashObj, newMinExpire); + updateGlobalHfeDs(ex->db, ex->hashObj, ex->minExpire, ex->minExpireFields); } } } @@ -760,40 +1135,85 @@ void hashTypeSetExDone(HashTypeSetEx *ex) { static SetExRes hashTypeSetExListpack(redisDb *db, robj *o, sds field, HashTypeSet *s, uint64_t expireAt, HashTypeSetEx *ex) { - UNUSED(db); - UNUSED(expireAt); - UNUSED(ex); int res = HSETEX_OK; - unsigned char *zl, *fptr, *vptr; + unsigned char *fptr = NULL, *vptr = NULL, *tptr = NULL; - /* TODO support expiration time for listpack */ - - - zl = o->ptr; - fptr = lpFirst(zl); - if (fptr != NULL) { - fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); + if (o->encoding == OBJ_ENCODING_LISTPACK) { + unsigned char *zl = o->ptr; + fptr = lpFirst(zl); if (fptr != NULL) { - /* Grab pointer to the value (fptr points to the field) */ - vptr = lpNext(zl, fptr); - serverAssert(vptr != NULL); - res = HSET_UPDATE; + fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); + if (fptr != NULL) { + /* Grab pointer to the value (fptr points to the field) */ + vptr = lpNext(zl, fptr); + serverAssert(vptr != NULL); + res = HSET_UPDATE; - /* Replace value */ - zl = lpReplace(zl, &vptr, (unsigned char*)s->value, sdslen(s->value)); + /* Replace value */ + zl = lpReplace(zl, &vptr, (unsigned char *) s->value, sdslen(s->value)); + } + } + + if (res != HSET_UPDATE) { + /* Push new field/value pair onto the tail of the listpack */ + zl = lpAppend(zl, (unsigned char*)field, sdslen(field)); + zl = lpAppend(zl, (unsigned char*)s->value, sdslen(s->value)); + } + o->ptr = zl; + goto out; + } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + listpackEx *lpt = o->ptr; + long long expireTime = HASH_LP_NO_TTL; + + fptr = lpFirst(lpt->lp); + if (fptr != NULL) { + fptr = lpFind(lpt->lp, fptr, (unsigned char*)field, sdslen(field), 2); + if (fptr != NULL) { + unsigned char *p; + /* Grab pointer to the value (fptr points to the field) */ + vptr = lpNext(lpt->lp, fptr); + serverAssert(vptr != NULL); + + if (s) { + /* Replace value */ + lpt->lp = lpReplace(lpt->lp, &vptr, + (unsigned char *) s->value, + sdslen(s->value)); + + fptr = lpPrev(lpt->lp, vptr); + serverAssert(fptr != NULL); + res = HSET_UPDATE; + } + tptr = lpNext(lpt->lp, vptr); + serverAssert(tptr != NULL); + p = lpGetValue(tptr, NULL, &expireTime); + serverAssert(!p); + + if (ex) { + res = hashTypeSetExpiryListpack(ex, field, fptr, vptr, tptr, + expireAt); + if (res != HSETEX_OK) + goto out; + } else if (res == HSET_UPDATE && expireTime != HASH_LP_NO_TTL) { + /* Clear TTL */ + listpackExPersist(o, field, fptr, vptr); + } + } + } + + if (!fptr) { + if (s) { + listpackExAddNew(o, field, s->value, + ex ? expireAt : HASH_LP_NO_TTL); + } else { + res = HSETEX_NO_FIELD; + } } } - - if (res != HSET_UPDATE) { - /* Push new field/value pair onto the tail of the listpack */ - zl = lpAppend(zl, (unsigned char*)field, sdslen(field)); - zl = lpAppend(zl, (unsigned char*)s->value, sdslen(s->value)); - } - o->ptr = zl; - +out: /* Check if the listpack needs to be converted to a hash table */ if (hashTypeLength(o, 0) > server.hash_max_listpack_entries) - hashTypeConvert(o, OBJ_ENCODING_HT); + hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires); return res; } @@ -817,6 +1237,19 @@ int hashTypeDelete(robj *o, sds field) { deleted = 1; } } + } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + unsigned char *fptr; + listpackEx *lpt = o->ptr; + + fptr = lpFirst(lpt->lp); + if (fptr != NULL) { + fptr = lpFind(lpt->lp, fptr, (unsigned char*)field, sdslen(field), 2); + if (fptr != NULL) { + /* Delete field, value and ttl */ + lpt->lp = lpDeleteRangeWithEntry(lpt->lp, &fptr, 3); + deleted = 1; + } + } } else if (o->encoding == OBJ_ENCODING_HT) { /* dictDelete() will call dictHfieldDestructor() */ if (dictDelete((dict*)o->ptr, field) == C_OK) { @@ -838,6 +1271,12 @@ unsigned long hashTypeLength(const robj *o, int subtractExpiredFields) { if (o->encoding == OBJ_ENCODING_LISTPACK) { length = lpLength(o->ptr) / 2; + } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + listpackEx *lpt = o->ptr; + length = lpLength(lpt->lp) / 3; + + if (subtractExpiredFields && lpt->meta.trash == 0) + length -= listpackExExpireDryRun(o); } else if (o->encoding == OBJ_ENCODING_HT) { uint64_t expiredItems = 0; dict *d = (dict*)o->ptr; @@ -861,9 +1300,13 @@ hashTypeIterator *hashTypeInitIterator(robj *subject) { hi->subject = subject; hi->encoding = subject->encoding; - if (hi->encoding == OBJ_ENCODING_LISTPACK) { + if (hi->encoding == OBJ_ENCODING_LISTPACK || + hi->encoding == OBJ_ENCODING_LISTPACK_EX) + { hi->fptr = NULL; hi->vptr = NULL; + hi->tptr = NULL; + hi->expire_time = EB_EXPIRE_TIME_INVALID; } else if (hi->encoding == OBJ_ENCODING_HT) { hi->di = dictGetIterator(subject->ptr); } else { @@ -889,8 +1332,6 @@ int hashTypeNext(hashTypeIterator *hi, int skipExpiredFields) { fptr = hi->fptr; vptr = hi->vptr; - /* TODO-HFE: Handle skipExpiredFields for listpack */ - if (fptr == NULL) { /* Initialize cursor */ serverAssert(vptr == NULL); @@ -909,6 +1350,48 @@ int hashTypeNext(hashTypeIterator *hi, int skipExpiredFields) { /* fptr, vptr now point to the first or next pair */ hi->fptr = fptr; hi->vptr = vptr; + } else if (hi->encoding == OBJ_ENCODING_LISTPACK_EX) { + long long expire_time; + unsigned char *zl = hashTypeListpackGetLp(hi->subject); + unsigned char *fptr, *vptr, *tptr; + + fptr = hi->fptr; + vptr = hi->vptr; + tptr = hi->tptr; + + if (fptr == NULL) { + /* Initialize cursor */ + serverAssert(vptr == NULL); + fptr = lpFirst(zl); + } else { + /* Advance cursor */ + serverAssert(tptr != NULL); + fptr = lpNext(zl, tptr); + } + if (fptr == NULL) return C_ERR; + + while (fptr != NULL) { + /* Grab pointer to the value (fptr points to the field) */ + vptr = lpNext(zl, fptr); + serverAssert(vptr != NULL); + + tptr = lpNext(zl, vptr); + serverAssert(tptr != NULL); + + lpGetValue(tptr, NULL, &expire_time); + + if (!skipExpiredFields || !hashTypeIsExpired(hi->subject, expire_time)) + break; + + fptr = lpNext(zl, tptr); + } + if (fptr == NULL) return C_ERR; + + /* fptr, vptr now point to the first or next pair */ + hi->fptr = fptr; + hi->vptr = vptr; + hi->tptr = tptr; + hi->expire_time = (expire_time != HASH_LP_NO_TTL) ? (uint64_t) expire_time : EB_EXPIRE_TIME_INVALID; } else if (hi->encoding == OBJ_ENCODING_HT) { while ((hi->de = dictNext(hi->di)) != NULL) { if (skipExpiredFields && hfieldIsExpired(dictGetKey(hi->de))) @@ -927,15 +1410,20 @@ int hashTypeNext(hashTypeIterator *hi, int skipExpiredFields) { void hashTypeCurrentFromListpack(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, - long long *vll) + long long *vll, + uint64_t *expireTime) { - serverAssert(hi->encoding == OBJ_ENCODING_LISTPACK); + serverAssert(hi->encoding == OBJ_ENCODING_LISTPACK || + hi->encoding == OBJ_ENCODING_LISTPACK_EX); if (what & OBJ_HASH_KEY) { *vstr = lpGetValue(hi->fptr, vlen, vll); } else { *vstr = lpGetValue(hi->vptr, vlen, vll); } + + if (expireTime) + *expireTime = hi->expire_time; } /* Get the field or value at iterator cursor, for an iterator on a hash value @@ -983,9 +1471,11 @@ void hashTypeCurrentObject(hashTypeIterator *hi, long long *vll, uint64_t *expireTime) { - if (hi->encoding == OBJ_ENCODING_LISTPACK) { + if (hi->encoding == OBJ_ENCODING_LISTPACK || + hi->encoding == OBJ_ENCODING_LISTPACK_EX) + { *vstr = NULL; - hashTypeCurrentFromListpack(hi, what, vstr, vlen, vll); + hashTypeCurrentFromListpack(hi, what, vstr, vlen, vll, expireTime); /* TODO-HFE: Handle expireTime */ } else if (hi->encoding == OBJ_ENCODING_HT) { char *ele; @@ -1016,16 +1506,17 @@ hfield hashTypeCurrentObjectNewHfield(hashTypeIterator *hi) { unsigned char *vstr; unsigned int vlen; long long vll; + uint64_t expireTime; hfield hf; - hashTypeCurrentObject(hi,OBJ_HASH_KEY,&vstr,&vlen,&vll, NULL); + hashTypeCurrentObject(hi,OBJ_HASH_KEY,&vstr,&vlen,&vll, &expireTime); if (!vstr) { vlen = ll2string(buf, sizeof(buf), vll); vstr = (unsigned char *) buf; } - hf = hfieldNew(vstr,vlen, 0); + hf = hfieldNew(vstr,vlen, expireTime != EB_EXPIRE_TIME_INVALID); return hf; } @@ -1047,6 +1538,23 @@ void hashTypeConvertListpack(robj *o, int enc) { if (enc == OBJ_ENCODING_LISTPACK) { /* Nothing to do... */ + } else if (enc == OBJ_ENCODING_LISTPACK_EX) { + unsigned char *p; + + /* Append HASH_LP_NO_TTL to each field name - value pair. */ + p = lpFirst(o->ptr); + while (p != NULL) { + p = lpNext(o->ptr, p); + serverAssert(p); + + o->ptr = lpInsertInteger(o->ptr, HASH_LP_NO_TTL, p, LP_AFTER, &p); + p = lpNext(o->ptr, p); + } + + listpackEx *lpt = listpackExCreate(); + lpt->lp = o->ptr; + o->encoding = OBJ_ENCODING_LISTPACK_EX; + o->ptr = lpt; } else if (enc == OBJ_ENCODING_HT) { hashTypeIterator *hi; dict *dict; @@ -1083,9 +1591,68 @@ void hashTypeConvertListpack(robj *o, int enc) { } } -void hashTypeConvert(robj *o, int enc) { +void hashTypeConvertListpackEx(robj *o, int enc, ebuckets *hexpires) { + serverAssert(o->encoding == OBJ_ENCODING_LISTPACK_EX); + + if (enc == OBJ_ENCODING_LISTPACK_EX) { + return; + } else if (enc == OBJ_ENCODING_HT) { + int ret; + hashTypeIterator *hi; + dict *dict; + dictExpireMetadata *dictExpireMeta; + listpackEx *lpt = o->ptr; + uint64_t minExpire = hashTypeGetMinExpire(o); + + if (hexpires && lpt->meta.trash != 1) + ebRemove(hexpires, &hashExpireBucketsType, o); + + dict = dictCreate(&mstrHashDictTypeWithHFE); + dictExpand(dict,hashTypeLength(o, 0)); + dictExpireMeta = (dictExpireMetadata *) dictMetadata(dict); + + /* Fillup dict HFE metadata */ + dictExpireMeta->key = lpt->key; /* reference key in keyspace */ + dictExpireMeta->hfe = ebCreate(); /* Allocate HFE DS */ + dictExpireMeta->expireMeta.trash = 1; /* mark as trash (as long it wasn't ebAdd()) */ + + hi = hashTypeInitIterator(o); + + while (hashTypeNext(hi, 0) != C_ERR) { + hfield key = hashTypeCurrentObjectNewHfield(hi); + sds value = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE); + dictUseStoredKeyApi(dict, 1); + ret = dictAdd(dict, key, value); + dictUseStoredKeyApi(dict, 0); + if (ret != DICT_OK) { + hfieldFree(key); sdsfree(value); /* Needed for gcc ASAN */ + hashTypeReleaseIterator(hi); /* Needed for gcc ASAN */ + serverLogHexDump(LL_WARNING,"listpack with dup elements dump", + o->ptr,lpBytes(o->ptr)); + serverPanic("Listpack corruption detected"); + } + + if (hi->expire_time != EB_EXPIRE_TIME_INVALID) + ebAdd(&dictExpireMeta->hfe, &hashFieldExpireBucketsType, key, hi->expire_time); + } + hashTypeReleaseIterator(hi); + listpackExFree(lpt); + + o->encoding = OBJ_ENCODING_HT; + o->ptr = dict; + + if (hexpires && minExpire != EB_EXPIRE_TIME_INVALID) + ebAdd(hexpires, &hashExpireBucketsType, o, minExpire); + } else { + serverPanic("Unknown hash encoding: %d", enc); + } +} + +void hashTypeConvert(robj *o, int enc, ebuckets *hexpires) { if (o->encoding == OBJ_ENCODING_LISTPACK) { hashTypeConvertListpack(o, enc); + } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + hashTypeConvertListpackEx(o, enc, hexpires); } else if (o->encoding == OBJ_ENCODING_HT) { serverPanic("Not implemented"); } else { @@ -1111,6 +1678,21 @@ robj *hashTypeDup(robj *o, sds newkey, uint64_t *minHashExpire) { memcpy(new_zl, zl, sz); hobj = createObject(OBJ_HASH, new_zl); hobj->encoding = OBJ_ENCODING_LISTPACK; + } else if(o->encoding == OBJ_ENCODING_LISTPACK_EX) { + listpackEx *lpt = o->ptr; + + if (lpt->meta.trash == 0) + *minHashExpire = ebGetMetaExpTime(&lpt->meta); + + listpackEx *dup = listpackExCreate(); + dup->key = newkey; + + size_t sz = lpBytes(lpt->lp); + dup->lp = lpNew(sz); + memcpy(dup->lp, lpt->lp, sz); + + hobj = createObject(OBJ_HASH, dup); + hobj->encoding = OBJ_ENCODING_LISTPACK_EX; } else if(o->encoding == OBJ_ENCODING_HT) { dictExpireMetadata *dictExpireMetaSrc, *dictExpireMetaDst = NULL; dict *d; @@ -1196,7 +1778,9 @@ void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, listpackEntry val->slen = sdslen(s); } } else if (hashobj->encoding == OBJ_ENCODING_LISTPACK) { - lpRandomPair(hashobj->ptr, hashsize, key, val); + lpRandomPair(hashobj->ptr, hashsize, key, val, 2); + } else if (hashobj->encoding == OBJ_ENCODING_LISTPACK_EX) { + lpRandomPair(hashTypeListpackGetLp(hashobj), hashsize, key, val, 3); } else { serverPanic("Unknown hash encoding"); } @@ -1220,27 +1804,38 @@ void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, listpackEntry static ExpireAction hashTypeActiveExpire(eItem _hashObj, void *ctx) { robj *hashObj = (robj *) _hashObj; ActiveExpireCtx *activeExpireCtx = (ActiveExpireCtx *) ctx; + sds keystr = NULL; + ExpireInfo info = {0}; /* If no more quota left for this callback, stop */ if (activeExpireCtx->fieldsToExpireQuota == 0) return ACT_STOP_ACTIVE_EXP; - if (hashObj->encoding == OBJ_ENCODING_LISTPACK) { - serverPanic("Listpack encoding not supported yet"); + if (hashObj->encoding == OBJ_ENCODING_LISTPACK_EX) { + info = (ExpireInfo){ + .maxToExpire = activeExpireCtx->fieldsToExpireQuota, + .ctx = hashObj, + .now = commandTimeSnapshot(), + .itemsExpired = 0}; + + listpackExExpire(hashObj, &info); + keystr = ((listpackEx*)hashObj->ptr)->key; + } else { + serverAssert(hashObj->encoding == OBJ_ENCODING_HT); + + dict *d = hashObj->ptr; + dictExpireMetadata *dictExpireMeta = (dictExpireMetadata *) dictMetadata(d); + + info = (ExpireInfo){ + .maxToExpire = activeExpireCtx->fieldsToExpireQuota, + .onExpireItem = onFieldExpire, + .ctx = hashObj, + .now = commandTimeSnapshot() + }; + + ebExpire(&dictExpireMeta->hfe, &hashFieldExpireBucketsType, &info); + keystr = dictExpireMeta->key; } - serverAssert(hashObj->encoding == OBJ_ENCODING_HT); - - dict *d = hashObj->ptr; - dictExpireMetadata *dictExpireMeta = (dictExpireMetadata *) dictMetadata(d); - - ExpireInfo info = { - .maxToExpire = activeExpireCtx->fieldsToExpireQuota, - .onExpireItem = onFieldExpire, - .ctx = hashObj, - .now = commandTimeSnapshot() - }; - - ebExpire(&dictExpireMeta->hfe, &hashFieldExpireBucketsType, &info); /* Update quota left */ activeExpireCtx->fieldsToExpireQuota -= info.itemsExpired; @@ -1248,7 +1843,7 @@ static ExpireAction hashTypeActiveExpire(eItem _hashObj, void *ctx) { /* If hash has no more fields to expire, remove it from HFE DB */ if (info.nextExpireTime == 0) { if (hashTypeLength(hashObj, 0) == 0) { - robj *key = createStringObject(dictExpireMeta->key, sdslen(dictExpireMeta->key)); + robj *key = createStringObject(keystr, sdslen(keystr)); dbDelete(activeExpireCtx->db, key); //notifyKeyspaceEvent(NOTIFY_HASH,"xxxxxxxxx",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key, activeExpireCtx->db->id); @@ -1266,20 +1861,46 @@ static ExpireAction hashTypeActiveExpire(eItem _hashObj, void *ctx) { } } +/* Return the next/minimum expiry time of the hash-field. This is useful if a + * field with the minimum expiry is deleted, and you want to get the next + * minimum expiry. Otherwise, consider using hashTypeGetMinExpire() which will + * be faster. If there is no field with expiry, returns EB_EXPIRE_TIME_INVALID */ +uint64_t hashTypeGetNextTimeToExpire(robj *o) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { + return EB_EXPIRE_TIME_INVALID; + } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + return listpackExGetMinExpire(o); + } else { + serverAssert(o->encoding == OBJ_ENCODING_HT); + + dict *d = o->ptr; + if (!isDictWithMetaHFE(d)) + return EB_EXPIRE_TIME_INVALID; + + dictExpireMetadata *expireMeta = (dictExpireMetadata *) dictMetadata(d); + return ebGetNextTimeToExpire(expireMeta->hfe, &hashFieldExpireBucketsType); + } +} + /* Return the next/minimum expiry time of the hash-field. * If not found, return EB_EXPIRE_TIME_INVALID */ -int64_t hashTypeGetMinExpire(robj *o) { +uint64_t hashTypeGetMinExpire(robj *o) { + ExpireMeta *expireMeta = NULL; + if (o->encoding == OBJ_ENCODING_LISTPACK) { - return EB_EXPIRE_TIME_INVALID; /* not supported yet */ - } - - serverAssert(o->encoding == OBJ_ENCODING_HT); - - dict *d = o->ptr; - if (!isDictWithMetaHFE(d)) return EB_EXPIRE_TIME_INVALID; + } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + listpackEx *lpt = o->ptr; + expireMeta = &lpt->meta; + } else { + serverAssert(o->encoding == OBJ_ENCODING_HT); - ExpireMeta *expireMeta = &((dictExpireMetadata *) dictMetadata(d))->expireMeta; + dict *d = o->ptr; + if (!isDictWithMetaHFE(d)) + return EB_EXPIRE_TIME_INVALID; + + expireMeta = &((dictExpireMetadata *) dictMetadata(d))->expireMeta; + } /* Keep aside next hash-field expiry before updating HFE DS. Verify it is not trash */ if (expireMeta->trash == 1) @@ -1289,12 +1910,13 @@ int64_t hashTypeGetMinExpire(robj *o) { } uint64_t hashTypeRemoveFromExpires(ebuckets *hexpires, robj *o) { - if (o->encoding == OBJ_ENCODING_LISTPACK) - return EB_EXPIRE_TIME_INVALID; /* not supported yet */ - - /* If dict doesn't holds HFE metadata */ - if (!isDictWithMetaHFE(o->ptr)) + if (o->encoding == OBJ_ENCODING_LISTPACK) { return EB_EXPIRE_TIME_INVALID; + } else if (o->encoding == OBJ_ENCODING_HT) { + /* If dict doesn't holds HFE metadata */ + if (!isDictWithMetaHFE(o->ptr)) + return EB_EXPIRE_TIME_INVALID; + } uint64_t expireTime = ebGetExpireTime(&hashExpireBucketsType, o); @@ -1313,8 +1935,11 @@ void hashTypeAddToExpires(redisDb *db, sds key, robj *hashObj, uint64_t expireTi if (expireTime == EB_EXPIRE_TIME_INVALID) return; - if (hashObj->encoding == OBJ_ENCODING_LISTPACK) { - return; /* TODO */ + if (hashObj->encoding == OBJ_ENCODING_LISTPACK_EX) { + listpackEx *lpt = hashObj->ptr; + lpt->key = key; + ebAdd(&db->hexpires, &hashExpireBucketsType, hashObj, expireTime); + return; } serverAssert(hashObj->encoding == OBJ_ENCODING_HT); @@ -1357,6 +1982,23 @@ uint64_t hashTypeDbActiveExpire(redisDb *db, uint32_t maxFieldsToExpire) { return maxFieldsToExpire - ctx.fieldsToExpireQuota; } +void hashTypeFree(robj *o) { + switch (o->encoding) { + case OBJ_ENCODING_HT: + dictRelease((dict*) o->ptr); + break; + case OBJ_ENCODING_LISTPACK: + lpFree(o->ptr); + break; + case OBJ_ENCODING_LISTPACK_EX: + listpackExFree(o->ptr); + break; + default: + serverPanic("Unknown hash encoding type"); + break; + } +} + /*----------------------------------------------------------------------------- * Hash type commands *----------------------------------------------------------------------------*/ @@ -1368,7 +2010,7 @@ void hsetnxCommand(client *c) { if (hashTypeExists(o, c->argv[2]->ptr)) { addReply(c, shared.czero); } else { - hashTypeTryConversion(o,c->argv,2,3); + hashTypeTryConversion(c->db, o,c->argv,2,3); hashTypeSet(c->db, o,c->argv[2]->ptr,c->argv[3]->ptr,HASH_SET_COPY); addReply(c, shared.cone); signalModifiedKey(c,c->db,c->argv[1]); @@ -1387,7 +2029,7 @@ void hsetCommand(client *c) { } if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; - hashTypeTryConversion(o,c->argv,2,c->argc-1); + hashTypeTryConversion(c->db,o,c->argv,2,c->argc-1); for (i = 2; i < c->argc; i += 2) created += !hashTypeSet(c->db, o,c->argv[i]->ptr,c->argv[i+1]->ptr,HASH_SET_COPY); @@ -1584,12 +2226,14 @@ void hstrlenCommand(client *c) { } static void addHashIteratorCursorToReply(client *c, hashTypeIterator *hi, int what) { - if (hi->encoding == OBJ_ENCODING_LISTPACK) { + if (hi->encoding == OBJ_ENCODING_LISTPACK || + hi->encoding == OBJ_ENCODING_LISTPACK_EX) + { unsigned char *vstr = NULL; unsigned int vlen = UINT_MAX; long long vll = LLONG_MAX; - hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll); + hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll, NULL); if (vstr) addReplyBulkCBuffer(c, vstr, vlen); else @@ -1751,9 +2395,13 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { if (c->flags & CLIENT_CLOSE_ASAP) break; } - } else if (hash->encoding == OBJ_ENCODING_LISTPACK) { + } else if (hash->encoding == OBJ_ENCODING_LISTPACK || + hash->encoding == OBJ_ENCODING_LISTPACK_EX) + { listpackEntry *keys, *vals = NULL; unsigned long limit, sample_count; + unsigned char *lp = hashTypeListpackGetLp(hash); + int tuple_len = hash->encoding == OBJ_ENCODING_LISTPACK ? 2 : 3; limit = count > HRANDFIELD_RANDOM_SAMPLE_LIMIT ? HRANDFIELD_RANDOM_SAMPLE_LIMIT : count; keys = zmalloc(sizeof(listpackEntry)*limit); @@ -1762,7 +2410,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { while (count) { sample_count = count > limit ? limit : count; count -= sample_count; - lpRandomPairs(hash->ptr, sample_count, keys, vals); + lpRandomPairs(lp, sample_count, keys, vals, tuple_len); hrandfieldReplyWithListpack(c, sample_count, keys, vals); if (c->flags & CLIENT_CLOSE_ASAP) break; @@ -1804,12 +2452,16 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { * * And it is inefficient to repeatedly pick one random element from a * listpack in CASE 4. So we use this instead. */ - if (hash->encoding == OBJ_ENCODING_LISTPACK) { + if (hash->encoding == OBJ_ENCODING_LISTPACK || + hash->encoding == OBJ_ENCODING_LISTPACK_EX) + { + unsigned char *lp = hashTypeListpackGetLp(hash); + int tuple_len = hash->encoding == OBJ_ENCODING_LISTPACK ? 2 : 3; listpackEntry *keys, *vals = NULL; keys = zmalloc(sizeof(listpackEntry)*count); if (withvalues) vals = zmalloc(sizeof(listpackEntry)*count); - serverAssert(lpRandomPairsUnique(hash->ptr, count, keys, vals) == count); + serverAssert(lpRandomPairsUnique(lp, count, keys, vals, tuple_len) == count); hrandfieldReplyWithListpack(c, count, keys, vals); zfree(keys); zfree(vals); @@ -1995,8 +2647,7 @@ static uint64_t hfieldGetExpireTime(hfield field) { } /* Remove TTL from the field. Assumed ExpireMeta is attached and has valid value */ -static void hfieldPersist(redisDb *db, robj *hashObj, hfield field) { - UNUSED(db); +static void hfieldPersist(robj *hashObj, hfield field) { uint64_t fieldExpireTime = hfieldGetExpireTime(field); if (fieldExpireTime == EB_EXPIRE_TIME_INVALID) return; @@ -2042,9 +2693,16 @@ static ExpireAction onFieldExpire(eItem item, void *ctx) { * The caller is responsible for ensuring that it is indeed attached. */ static ExpireMeta *hashGetExpireMeta(const eItem hash) { robj *hashObj = (robj *)hash; - dict *d = hashObj->ptr; - dictExpireMetadata *dictExpireMeta = (dictExpireMetadata *) dictMetadata(d); - return &dictExpireMeta->expireMeta; + if (hashObj->encoding == OBJ_ENCODING_LISTPACK_EX) { + listpackEx *lpt = hashObj->ptr; + return &lpt->meta; + } else if (hashObj->encoding == OBJ_ENCODING_HT) { + dict *d = hashObj->ptr; + dictExpireMetadata *dictExpireMeta = (dictExpireMetadata *) dictMetadata(d); + return &dictExpireMeta->expireMeta; + } else { + serverPanic("Unknown encoding: %d", hashObj->encoding); + } } static void httlGenericCommand(client *c, const char *cmd, long long basetime, int unit) { @@ -2056,14 +2714,6 @@ static void httlGenericCommand(client *c, const char *cmd, long long basetime, i if ((hashObj = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp])) == NULL || checkType(c, hashObj, OBJ_HASH)) return; - /* not supported yet listpack */ - if (hashObj->encoding == OBJ_ENCODING_LISTPACK) { - addReplyError(c,"Hash field expire for listpack not supported yet."); - return; - } - - dict *d = hashObj->ptr; - /* Read number of fields */ if (getRangeLongFromObjectOrReply(c, c->argv[numFieldsAt], 1, LONG_MAX, &numFields, "Parameter `numFileds` should be greater than 0") != C_OK) @@ -2075,31 +2725,93 @@ static void httlGenericCommand(client *c, const char *cmd, long long basetime, i return; } - addReplyArrayLen(c, numFields); - for (int i = 0 ; i < numFields ; i++) { - sds field = c->argv[3+i]->ptr; - dictEntry *de = dictFind(d, field); - if (de == NULL) { - addReplyLongLong(c, HFE_GET_NO_FIELD); - continue; - } + if (hashObj->encoding == OBJ_ENCODING_LISTPACK) { + void *lp = hashObj->ptr; - hfield hf = dictGetKey(de); - uint64_t expire = hfieldGetExpireTime(hf); - if (expire == EB_EXPIRE_TIME_INVALID) { - addReplyLongLong(c, HFE_GET_NO_TTL); /* no ttl */ - continue; - } + addReplyArrayLen(c, numFields); + for (int i = 0 ; i < numFields ; i++) { + sds field = c->argv[3+i]->ptr; + void *fptr = lpFirst(lp); + if (fptr != NULL) + fptr = lpFind(lp, fptr, (unsigned char *) field, sdslen(field), 1); - if ( (long long) expire < commandTimeSnapshot()) { - addReplyLongLong(c, HFE_GET_NO_FIELD); - continue; + if (!fptr) + addReplyLongLong(c, HFE_GET_NO_FIELD); + else + addReplyLongLong(c, HFE_GET_NO_TTL); } + return; + } else if (hashObj->encoding == OBJ_ENCODING_LISTPACK_EX) { + listpackEx *lpt = hashObj->ptr; - if (unit == UNIT_SECONDS) - addReplyLongLong(c, (expire + 999 - basetime) / 1000); - else - addReplyLongLong(c, (expire - basetime)); + addReplyArrayLen(c, numFields); + for (int i = 0 ; i < numFields ; i++) { + long long expire; + sds field = c->argv[3+i]->ptr; + void *fptr = lpFirst(lpt->lp); + if (fptr != NULL) + fptr = lpFind(lpt->lp, fptr, (unsigned char *) field, sdslen(field), 2); + + if (!fptr) { + addReplyLongLong(c, HFE_GET_NO_FIELD); + continue; + } + + fptr = lpNext(lpt->lp, fptr); + serverAssert(fptr); + fptr = lpNext(lpt->lp, fptr); + serverAssert(fptr); + + lpGetValue(fptr, NULL, &expire); + + if (expire == HASH_LP_NO_TTL) { + addReplyLongLong(c, HFE_GET_NO_TTL); + continue; + } + + if (expire <= commandTimeSnapshot()) { + addReplyLongLong(c, HFE_GET_NO_FIELD); + continue; + } + + if (unit == UNIT_SECONDS) + addReplyLongLong(c, (expire + 999 - basetime) / 1000); + else + addReplyLongLong(c, (expire - basetime)); + } + return; + } else if (hashObj->encoding == OBJ_ENCODING_HT) { + dict *d = hashObj->ptr; + + addReplyArrayLen(c, numFields); + for (int i = 0 ; i < numFields ; i++) { + sds field = c->argv[3+i]->ptr; + dictEntry *de = dictFind(d, field); + if (de == NULL) { + addReplyLongLong(c, HFE_GET_NO_FIELD); + continue; + } + + hfield hf = dictGetKey(de); + uint64_t expire = hfieldGetExpireTime(hf); + if (expire == EB_EXPIRE_TIME_INVALID) { + addReplyLongLong(c, HFE_GET_NO_TTL); /* no ttl */ + continue; + } + + if ( (long long) expire < commandTimeSnapshot()) { + addReplyLongLong(c, HFE_GET_NO_FIELD); + continue; + } + + if (unit == UNIT_SECONDS) + addReplyLongLong(c, (expire + 999 - basetime) / 1000); + else + addReplyLongLong(c, (expire - basetime)); + } + return; + } else { + serverPanic("Unknown encoding: %d", hashObj->encoding); } } @@ -2122,12 +2834,6 @@ static void hexpireGenericCommand(client *c, const char *cmd, long long basetime if ((hashObj = lookupKeyWriteOrReply(c, keyArg, shared.null[c->resp])) == NULL || checkType(c, hashObj, OBJ_HASH)) return; - /* not supported yet listpack */ - if (hashObj->encoding == OBJ_ENCODING_LISTPACK) { - addReplyError(c,"Hash field expire for listpack not supported yet."); - return; - } - /* Read the expiry time from command */ if (getLongLongFromObjectOrReply(c, expireArg, &expire, NULL) != C_OK) return; @@ -2190,8 +2896,7 @@ static void hexpireGenericCommand(client *c, const char *cmd, long long basetime addReplyArrayLen(c, numFields); for (int i = 0 ; i < numFields ; i++) { sds field = c->argv[numFieldsAt+i+1]->ptr; - dictEntry *de; - SetExRes res = hashTypeSetExpiry(&exCtx, field, expire, &de); + SetExRes res = hashTypeSetEx(c->db, hashObj, field, NULL, expire, &exCtx); addReplyLongLong(c,res); } hashTypeSetExDone(&exCtx); @@ -2247,14 +2952,6 @@ void hpersistCommand(client *c) { if ((hashObj = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp])) == NULL || checkType(c, hashObj, OBJ_HASH)) return; - /* not supported yet listpack */ - if (hashObj->encoding == OBJ_ENCODING_LISTPACK) { - addReplyError(c,"Hash field expire for listpack not supported yet."); - return; - } - - dict *d = hashObj->ptr; - /* Read number of fields */ if (getRangeLongFromObjectOrReply(c, c->argv[numFieldsAt], 1, LONG_MAX, &numFields, "Parameter `numFileds` should be greater than 0") != C_OK) @@ -2266,29 +2963,984 @@ void hpersistCommand(client *c) { return; } - addReplyArrayLen(c, numFields); - for (int i = 0 ; i < numFields ; i++) { - sds field = c->argv[3+i]->ptr; - dictEntry *de = dictFind(d, field); - if (de == NULL) { - addReplyLongLong(c, HFE_PERSIST_NO_FIELD); - continue; - } + if (hashObj->encoding == OBJ_ENCODING_LISTPACK) { + addReplyArrayLen(c, numFields); + for (int i = 0 ; i < numFields ; i++) { + sds field = c->argv[3 + i]->ptr; + unsigned char *fptr, *zl = hashObj->ptr; - hfield hf = dictGetKey(de); - uint64_t expire = hfieldGetExpireTime(hf); - if (expire == EB_EXPIRE_TIME_INVALID) { - addReplyLongLong(c, HFE_PERSIST_NO_TTL); - continue; - } + fptr = lpFirst(zl); + if (fptr != NULL) + fptr = lpFind(zl, fptr, (unsigned char *) field, sdslen(field), 1); - /* Already expired. Pretend there is no such field */ - if ( (long long) expire < commandTimeSnapshot()) { - addReplyLongLong(c, HFE_PERSIST_NO_FIELD); - continue; + if (!fptr) + addReplyLongLong(c, HFE_PERSIST_NO_FIELD); + else + addReplyLongLong(c, HFE_PERSIST_NO_TTL); } + return; + } else if (hashObj->encoding == OBJ_ENCODING_LISTPACK_EX) { + long long prevExpire; + unsigned char *fptr, *vptr, *tptr, *s; + listpackEx *lpt = hashObj->ptr; - hfieldPersist(c->db, hashObj, hf); - addReplyLongLong(c, HFE_PERSIST_OK); + addReplyArrayLen(c, numFields); + for (int i = 0 ; i < numFields ; i++) { + sds field = c->argv[3 + i]->ptr; + + fptr = lpFirst(lpt->lp); + if (fptr != NULL) + fptr = lpFind(lpt->lp, fptr, (unsigned char*)field, sdslen(field), 2); + + if (!fptr) { + addReplyLongLong(c, HFE_PERSIST_NO_FIELD); + continue; + } + + vptr = lpNext(lpt->lp, fptr); + serverAssert(vptr); + tptr = lpNext(lpt->lp, vptr); + serverAssert(tptr); + + s = lpGetValue(tptr, NULL, &prevExpire); + serverAssert(!s); + + if (prevExpire == HASH_LP_NO_TTL) { + addReplyLongLong(c, HFE_PERSIST_NO_TTL); + continue; + } + + if (prevExpire < commandTimeSnapshot()) { + addReplyLongLong(c, HFE_PERSIST_NO_FIELD); + continue; + } + + listpackExPersist(hashObj, field, fptr, vptr); + addReplyLongLong(c, HFE_PERSIST_OK); + } + return; + } else if (hashObj->encoding == OBJ_ENCODING_HT) { + dict *d = hashObj->ptr; + + addReplyArrayLen(c, numFields); + for (int i = 0 ; i < numFields ; i++) { + sds field = c->argv[3+i]->ptr; + dictEntry *de = dictFind(d, field); + if (de == NULL) { + addReplyLongLong(c, HFE_PERSIST_NO_FIELD); + continue; + } + + hfield hf = dictGetKey(de); + uint64_t expire = hfieldGetExpireTime(hf); + if (expire == EB_EXPIRE_TIME_INVALID) { + addReplyLongLong(c, HFE_PERSIST_NO_TTL); + continue; + } + + /* Already expired. Pretend there is no such field */ + if ( (long long) expire < commandTimeSnapshot()) { + addReplyLongLong(c, HFE_PERSIST_NO_FIELD); + continue; + } + + hfieldPersist(hashObj, hf); + addReplyLongLong(c, HFE_PERSIST_OK); + } + } else { + serverPanic("Unknown encoding: %d", hashObj->encoding); + } +} + +/** + * TODO: Move top of the file + * HGETF - HSETF command arguments + */ +#define HFE_CMD_NX (1<<0) /* If not exist */ +#define HFE_CMD_XX (1<<1) /* If exists */ +#define HFE_CMD_GT (1<<2) /* Greater than */ +#define HFE_CMD_LT (1<<3) /* Less than */ +#define HFE_CMD_COND_MASK (0x0F) + +#define HFE_CMD_PX (1<<4) /* Milliseconds */ +#define HFE_CMD_EX (1<<5) /* Seconds */ +#define HFE_CMD_PXAT (1<<6) /* Unix timestamp milliseconds */ +#define HFE_CMD_EXAT (1<<7) /* Unix timestamp seconds */ +#define HFE_CMD_PERSIST (1<<8) /* Delete TTL */ +#define HFE_CMD_KEEPTTL (1<<9) /* Keep TTL */ +#define HFE_CMD_EXPIRY_MASK (0x3F0) + +#define HFE_CMD_DC (1<<10) /* Don't create key */ +#define HFE_CMD_DCF (1<<11) /* Don't create field */ +#define HFE_CMD_DOF (1<<12) /* Don't overwrite field */ +#define HFE_CMD_GETNEW (1<<13) /* Get new value */ +#define HFE_CMD_GETOLD (1<<14) /* Get old value */ + +#define HSETF_FAIL 0 /* Failed to set value (DCF/DOF not met) */ +#define HSETF_FIELD 1 /* Field value is set without TTL */ +#define HSETF_FIELD_AND_TTL 3 /* Both field value and TTL is set */ + +/* Validate expire time is not more than EB_EXPIRE_TIME_MAX, + * or it does not overflow */ +static int validateExpire(client *c, int unit, robj *o, long long basetime, + uint64_t *expire) +{ + long long val; + /* Read the expiry time from command */ + if (getLongLongFromObjectOrReply(c, o, &val, NULL) != C_OK) + return C_ERR; + + if (val < 0 || val > (long long) EB_EXPIRE_TIME_MAX) { + addReplyErrorExpireTime(c); + return C_ERR; + } + + if (unit == UNIT_SECONDS) { + if (val > (long long) EB_EXPIRE_TIME_MAX / 1000) { + addReplyErrorExpireTime(c); + return C_ERR; + } + val *= 1000; + } else { + if (val > (long long) EB_EXPIRE_TIME_MAX) { + addReplyErrorExpireTime(c); + return C_ERR; + } + } + + if (val > (long long) EB_EXPIRE_TIME_MAX - basetime) { + addReplyErrorExpireTime(c); + return C_ERR; + } + val += basetime; + *expire = val; + return C_OK; +} + +/* Convert listpack to listpackEx encoding or attach hfe meta to dict */ +static void attachHfeMeta(redisDb *db, robj *o, robj *keyArg) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { + hashTypeConvert(o, OBJ_ENCODING_LISTPACK_EX, &db->hexpires); + + listpackEx *lpt = o->ptr; + dictEntry *de = dbFind(db, keyArg->ptr); + serverAssert(de != NULL); + lpt->key = dictGetKey(de); + } else if (o->encoding == OBJ_ENCODING_HT) { + dictExpireMetadata *dictExpireMeta; + dict *d = o->ptr; + + /* If dict doesn't have metadata attached */ + if (!isDictWithMetaHFE(d)) { + /* Realloc (only header of dict) with metadata for hash-field expiration */ + dictTypeAddMeta(&d, &mstrHashDictTypeWithHFE); + dictExpireMeta = (dictExpireMetadata *) dictMetadata(d); + o->ptr = d; + + /* Find the key in the keyspace. Need to keep reference to the key for + * notifications or even removal of the hash */ + dictEntry *de = dbFind(db, keyArg->ptr); + serverAssert(de != NULL); + sds key = dictGetKey(de); + + /* Fillup dict HFE metadata */ + dictExpireMeta->key = key; /* reference key in keyspace */ + dictExpireMeta->hfe = ebCreate(); /* Allocate HFE DS */ + dictExpireMeta->expireMeta.trash = 1; /* mark as trash (as long it wasn't ebAdd()) */ + } + } +} + +/* + * Called after modifying fields to update global hfe DS if necessary + * + * minExpire: minimum expiry time of the key before modification + * minExpireFields: minimum expiry time of the modified fields + */ +static void updateGlobalHfeDs(redisDb *db, robj *o,uint64_t minExpire, uint64_t minExpireFields) +{ + /* If minimum HFE of the hash is smaller than expiration time of the + * specified fields in the command as well as it is smaller or equal + * than expiration time provided in the command, then the minimum + * HFE of the hash won't change following this command. */ + if ((minExpire < minExpireFields)) + return; + + /* retrieve new expired time. It might have changed. */ + uint64_t newMinExpire = hashTypeGetNextTimeToExpire(o); + + /* Calculate the diff between old minExpire and newMinExpire. If it is + * only few seconds, then don't have to update global HFE DS. At the worst + * case fields of hash will be active-expired up to few seconds later. + * + * In any case, active-expire operation will know to update global + * HFE DS more efficiently than here for a single item. + */ + uint64_t diff = (minExpire > newMinExpire) ? + (minExpire - newMinExpire) : (newMinExpire - minExpire); + if (diff < HASH_NEW_EXPIRE_DIFF_THRESHOLD) return; + + if (minExpire != EB_EXPIRE_TIME_INVALID) + ebRemove(&db->hexpires, &hashExpireBucketsType, o); + if (newMinExpire != EB_EXPIRE_TIME_INVALID) + ebAdd(&db->hexpires, &hashExpireBucketsType, o, newMinExpire); +} + +/* Parse hgetf command arguments. */ +static int hgetfParseArgs(client *c, int *flags, uint64_t *expireAt, + int *firstFieldPos, int *fieldCount) +{ + *flags = 0; + *firstFieldPos = -1; + *fieldCount = -1; + + for (int i = 2; i < c->argc; i++) { + if (!strcasecmp(c->argv[i]->ptr, "fields")) { + long val; + + if (*firstFieldPos != -1) { + addReplyErrorFormat(c, "multiple FIELDS argument"); + return C_ERR; + } + + if (i >= c->argc - 2) { + addReplyErrorArity(c); + return C_ERR; + } + + if (getRangeLongFromObjectOrReply(c, c->argv[i + 1], 1, INT_MAX, &val, + "invalid number of fields") != C_OK) + return C_ERR; + + if (val > c->argc - i - 2) { + addReplyErrorArity(c); + return C_ERR; + } + + *firstFieldPos = i + 2; + *fieldCount = (int) val; + i = *firstFieldPos + *fieldCount - 1; + } else if (!strcasecmp(c->argv[i]->ptr, "NX")) { + if (*flags & (HFE_CMD_XX | HFE_CMD_GT | HFE_CMD_LT)) + goto err_condition; + *flags |= HFE_CMD_NX; + } else if (!strcasecmp(c->argv[i]->ptr, "XX")) { + if (*flags & (HFE_CMD_NX | HFE_CMD_GT | HFE_CMD_LT)) + goto err_condition; + *flags |= HFE_CMD_XX; + } else if (!strcasecmp(c->argv[i]->ptr, "GT")) { + if (*flags & (HFE_CMD_NX | HFE_CMD_XX | HFE_CMD_LT)) + goto err_condition; + *flags |= HFE_CMD_GT; + } else if (!strcasecmp(c->argv[i]->ptr, "LT")) { + if (*flags & (HFE_CMD_NX | HFE_CMD_XX | HFE_CMD_GT)) + goto err_condition; + *flags |= HFE_CMD_LT; + } else if (!strcasecmp(c->argv[i]->ptr, "EX")) { + if (*flags & (HFE_CMD_EXAT | HFE_CMD_PX | HFE_CMD_PXAT | HFE_CMD_PERSIST)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_CMD_EX; + i++; + if (validateExpire(c, UNIT_SECONDS, c->argv[i], + commandTimeSnapshot(), expireAt) != C_OK) + return C_ERR; + + } else if (!strcasecmp(c->argv[i]->ptr, "PX")) { + if (*flags & (HFE_CMD_EX | HFE_CMD_EXAT | HFE_CMD_PXAT | HFE_CMD_PERSIST)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_CMD_PX; + i++; + if (validateExpire(c, UNIT_MILLISECONDS, c->argv[i], + commandTimeSnapshot(), expireAt) != C_OK) + return C_ERR; + } else if (!strcasecmp(c->argv[i]->ptr, "EXAT")) { + if (*flags & (HFE_CMD_EX | HFE_CMD_PX | HFE_CMD_PXAT | HFE_CMD_PERSIST)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_CMD_EXAT; + i++; + if (validateExpire(c, UNIT_SECONDS, c->argv[i], 0, expireAt) != C_OK) + return C_ERR; + } else if (!strcasecmp(c->argv[i]->ptr, "PXAT")) { + if (*flags & (HFE_CMD_EX | HFE_CMD_EXAT | HFE_CMD_PX | HFE_CMD_PERSIST)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_CMD_PXAT; + i++; + if (validateExpire(c, UNIT_MILLISECONDS, c->argv[i], 0, expireAt) != C_OK) + return C_ERR; + } else if (!strcasecmp(c->argv[i]->ptr, "PERSIST")) { + if (*flags & (HFE_CMD_EX | HFE_CMD_EXAT | HFE_CMD_PX | HFE_CMD_PXAT)) + goto err_expiration; + *flags |= HFE_CMD_PERSIST; + } else { + addReplyErrorFormat(c, "unknown argument: %s", (char*) c->argv[i]->ptr); + return C_ERR; + } + } + + /* FIELDS argument is mandatory. */ + if (*firstFieldPos < 0) { + addReplyError(c, "missing FIELDS argument"); + return C_ERR; + } + + if (*flags & HFE_CMD_COND_MASK && + (!(*flags & (HFE_CMD_EX | HFE_CMD_EXAT | HFE_CMD_PX | HFE_CMD_PXAT)))) + { + addReplyError(c, "NX, XX, GT, and LT can be specified only when EX, PX, EXAT, or PXAT is specified"); + return C_ERR; + } + + return C_OK; + +err_missing_expire: + addReplyError(c, "missing expire time"); + return C_ERR; +err_condition: + addReplyError(c, "Only one of NX, XX, GT, and LT arguments can be specified"); + return C_ERR; +err_expiration: + addReplyError(c, "Only one of EX, PX, EXAT, PXAT or PERSIST arguments can be specified"); + return C_ERR; +} + +/* Reply with field value and optionally set expire time according to 'flag'. + * Return 1 if expire time is updated. */ +static int hgetfReplyValueAndSetExpiry(client *c, robj *o, sds field, int flag, + uint64_t expireAt, uint64_t *minPrevExp) +{ + unsigned char *fptr = NULL, *vptr = NULL, *tptr, *h; + hfield hf = NULL; + dict *d = NULL; + dictEntry *de = NULL; + uint64_t prevExpire = EB_EXPIRE_TIME_INVALID; + + if (o->encoding == OBJ_ENCODING_HT) { + d = o->ptr; + /* First retrieve the field to check if it exists */ + de = dictFind(d, field); + if (de == NULL) { + addReplyNull(c); + return 0; + } + + hf = dictGetKey(de); + if (hfieldIsExpired(hf)) { + addReplyNull(c); + return 0; + } + prevExpire = hfieldGetExpireTime(hf); + + /* Reply with value */ + sds val = dictGetVal(de); + addReplyBulkCBuffer(c, val, sdslen(val)); + } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + long long expire; + unsigned char *vstr = NULL; + unsigned int vlen = UINT_MAX; + long long vll = LLONG_MAX; + listpackEx *lpt = o->ptr; + + fptr = lpFirst(lpt->lp); + if (fptr != NULL) { + fptr = lpFind(lpt->lp, fptr, (unsigned char *) field, sdslen(field), 2); + if (fptr != NULL) { + vptr = lpNext(lpt->lp, fptr); + serverAssert(vptr != NULL); + + tptr = lpNext(lpt->lp, vptr); + serverAssert(tptr != NULL); + + h = lpGetValue(tptr, NULL, &expire); + serverAssert(h == NULL); + + if (expire != HASH_LP_NO_TTL) + prevExpire = expire; + } + } + + /* Return null if field does not exist */ + if (fptr == NULL || hashTypeIsExpired(o, expire)) { + addReplyNull(c); + return 0; + } + + /* Reply with value */ + vstr = lpGetValue(vptr, &vlen, &vll); + if (vstr) + addReplyBulkCBuffer(c, vstr, vlen); + else + addReplyLongLong(c, vll); + } else { + serverPanic("Unknown encoding: %d", o->encoding); + } + + if (!(flag & HFE_CMD_EXPIRY_MASK) || /* Check if any of EX, EXAT, PX, PXAT, PERSIST flags is set */ + ((flag & HFE_CMD_GT) && (expireAt <= prevExpire)) || + ((flag & HFE_CMD_LT) && (expireAt >= prevExpire)) || + ((flag & HFE_CMD_XX) && (prevExpire == EB_EXPIRE_TIME_INVALID)) || + ((flag & HFE_CMD_NX) && (prevExpire != EB_EXPIRE_TIME_INVALID)) || + ((flag & HFE_CMD_PERSIST) && (prevExpire == EB_EXPIRE_TIME_INVALID))) { + return 0; + } + + if (*minPrevExp > prevExpire) + *minPrevExp = prevExpire; + + /* if expiration time is in the past */ + if (checkAlreadyExpired(expireAt)) { + hashTypeDelete(o, field); + return 1; + } + + if (o->encoding == OBJ_ENCODING_HT) { + if (flag & HFE_CMD_PERSIST) { + hfieldPersist(o, hf); + } else { + if (!hfieldIsExpireAttached(hf)) { + /* allocate new field with expire metadata */ + hfield hfNew = hfieldNew(hf, hfieldlen(hf), 1 /*withExpireMeta*/); + /* Replace the old field with the new one with metadata */ + dictSetKey(d, de, hfNew); + hfieldFree(hf); + hf = hfNew; + } + + dictExpireMetadata *meta = (dictExpireMetadata *) dictMetadata(d); + if (prevExpire != EB_EXPIRE_TIME_INVALID) + ebRemove(&meta->hfe, &hashFieldExpireBucketsType, hf); + + ebAdd(&meta->hfe, &hashFieldExpireBucketsType, hf, expireAt); + } + } else { + if (flag & HFE_CMD_PERSIST) + listpackExPersist(o, field, fptr, vptr); + else + listpackExUpdateExpiry(o, field, fptr, vptr, expireAt); + } + + return 1; +} + +/* + * For each specified field: get its value and optionally set the field's + * remaining time to live. + * + * HGETF key + * [NX | XX | GT | LT] + * [EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT unix-time-milliseconds | PERSIST] + * + **/ +void hgetfCommand(client *c) { + int flags = 0; + robj *hashObj, *keyArg = c->argv[1]; + + int firstFieldPos = 0; + int numFields = 0; + uint64_t expireAt = EB_EXPIRE_TIME_INVALID; + + if (hgetfParseArgs(c, &flags, &expireAt, &firstFieldPos, &numFields) != C_OK) + return; + + /* Read the hash object */ + if ((hashObj = lookupKeyWriteOrReply(c, c->argv[1], shared.null[c->resp])) == NULL || + checkType(c, hashObj, OBJ_HASH)) return; + + attachHfeMeta(c->db, hashObj, keyArg); + uint64_t minExpire = hashTypeGetMinExpire(hashObj); + + /* Figure out from provided set of fields in command, which one has the minimum + * expiration time, before the modification (Will be used for optimization below) */ + uint64_t minExpireFields = EB_EXPIRE_TIME_INVALID; + + int updated = 0; + addReplyArrayLen(c, numFields); + for (int i = 0; i < numFields ; i++) { + sds field = c->argv[firstFieldPos + i]->ptr; + updated += hgetfReplyValueAndSetExpiry(c, hashObj, field, flags, + expireAt, &minExpireFields); + } + + /* Notify keyspace event, update dirty count and update global HFE DS */ + if (updated > 0) { + server.dirty += updated; + signalModifiedKey(c,c->db,keyArg); + notifyKeyspaceEvent(NOTIFY_HASH,"hgetf",keyArg,c->db->id); + if (hashTypeLength(hashObj, 0) == 0) { + dbDelete(c->db,keyArg); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",keyArg, c->db->id); + } else { + updateGlobalHfeDs(c->db, hashObj, minExpire, minExpireFields); + } + } +} + +/* Check hsetf command args and return 1 if TTL will be updated/discarded. */ +static int hsetfCheckTTLCondition(int flag, uint64_t prevExpire, uint64_t expireAt) { + /* When none of EX, PX, EXAT, PXAT, KEEPTTL are specified: + * any previous expiration time associated with field is discarded. */ + if (!(flag & HFE_CMD_EXPIRY_MASK) && prevExpire != EB_EXPIRE_TIME_INVALID) + return 1; + + if ((flag & (HFE_CMD_PX | HFE_CMD_PXAT | HFE_CMD_EX | HFE_CMD_EXAT))) { + if (((flag & HFE_CMD_COND_MASK) == 0) || /* None of NX, PX, GT, LT is set */ + ((flag & HFE_CMD_GT) && (expireAt > prevExpire)) || + ((flag & HFE_CMD_LT) && (expireAt < prevExpire)) || + (flag & HFE_CMD_XX && prevExpire != EB_EXPIRE_TIME_INVALID) || + (flag & HFE_CMD_NX && prevExpire == EB_EXPIRE_TIME_INVALID)) { + return 1; + } + } + return 0; +} + +/* For hsetf command, add reply from listpack item */ +static void hsetfReplyFromListpack(client *c, unsigned char *vptr) { + unsigned int vlen = UINT_MAX; + long long vll = LLONG_MAX; + unsigned char *vstr = NULL; + + if (!vptr) { + addReplyNull(c); + } else { + vstr = lpGetValue(vptr, &vlen, &vll); + if (vstr) + addReplyBulkCBuffer(c, vstr, vlen); + else + addReplyLongLong(c, vll); + } +} + +/* For hsetf command, add reply to client according to flag argument. */ +static void hsetfAddReply(client *c, int flag, sds prevval, sds newval, int ret) { + if (flag & HFE_CMD_GETOLD) { + if (!prevval) { + addReplyNull(c); + } else { + addReplyBulkCBuffer(c, prevval, sdslen(prevval)); + } + } else if (flag & HFE_CMD_GETNEW) { + if (!newval) { + addReplyNull(c); + } else { + addReplyBulkCBuffer(c, newval, sdslen(newval)); + } + } else { + addReplyLongLong(c, ret); + } +} + +/* Set field and expire time according to 'flag'. + * Return 1 if field and/or expire time is updated. */ +static int hsetfSetFieldAndReply(client *c, robj *o, sds field, sds value, + int flag, uint64_t expireAt, uint64_t *minPrevExp) +{ + int ret = HSETF_FAIL; + uint64_t prevExpire = EB_EXPIRE_TIME_INVALID; + + if (o->encoding == OBJ_ENCODING_LISTPACK_EX) { + long long expire; + unsigned char *fptr, *vptr = NULL, *tptr, *h; + listpackEx *lpt = o->ptr; + + fptr = lpFirst(lpt->lp); + if (fptr != NULL) { + fptr = lpFind(lpt->lp, fptr, (unsigned char *) field, sdslen(field), 2); + if (fptr != NULL) { + vptr = lpNext(lpt->lp, fptr); + tptr = lpNext(lpt->lp, vptr); + h = lpGetValue(tptr, NULL, &expire); + serverAssert(!h); + + if (expire != HASH_LP_NO_TTL) + prevExpire = expire; + } + } + + /* Check DCF (don't create fields) and DOF (don't override fields) arg. */ + if ((!fptr && (flag & HFE_CMD_DCF)) || (fptr && (flag & HFE_CMD_DOF))) { + /* When GETNEW or GETOLD is specified, regardless if a set operation + * was actually performed, we return value / old value of field or + * nil if there is no field. One corner case, if GETNEW and DOF + * (don't override fields) arguments are given and field exists, we + * won't override the field and return the existing value. + */ + if (flag & (HFE_CMD_GETNEW | HFE_CMD_GETOLD)) + hsetfReplyFromListpack(c, vptr); + else + addReplyLongLong(c, ret); + + return 0; + } + + /* Field value will be updated. */ + ret = HSETF_FIELD; + + /* Decide if we are going to set TTL */ + if (hsetfCheckTTLCondition(flag, prevExpire, expireAt)) + ret = HSETF_FIELD_AND_TTL; + + if (flag & HFE_CMD_GETOLD) + hsetfReplyFromListpack(c, vptr); + else if (flag & HFE_CMD_GETNEW) + addReplyBulkCBuffer(c, (char*)value, sdslen(value)); + else + addReplyLongLong(c, ret); + + if (!fptr) { + if (ret != HSETF_FIELD_AND_TTL) { + listpackExAddNew(o, field, value, HASH_LP_NO_TTL); + } else { + /* If expiration time is in the past, no need to create the field */ + if (!checkAlreadyExpired(expireAt)) { + if (*minPrevExp > expireAt) + *minPrevExp = expireAt; + + listpackExAddNew(o, field, value, expireAt); + } + } + } else { + lpt->lp = lpReplace(lpt->lp, &vptr, (unsigned char *) value, sdslen(value)); + fptr = lpPrev(lpt->lp, vptr); /* Update fptr as above line invalidates it. */ + serverAssert(fptr != NULL); + + if (ret == HSETF_FIELD_AND_TTL) { + if (*minPrevExp > prevExpire) + *minPrevExp = prevExpire; + + if (!(flag & HFE_CMD_EXPIRY_MASK)) { + /* If none of EX,EXAT,PX,PXAT,KEEPTTL is specified, TTL is + * discarded. */ + listpackExPersist(o, field, fptr, vptr); + } else if (checkAlreadyExpired(expireAt)) { + hashTypeDelete(o, field); + } else { + if (*minPrevExp > expireAt) + *minPrevExp = expireAt; + + listpackExUpdateExpiry(o, field, fptr, vptr, expireAt); + } + } + } + + return 1; + } else if (o->encoding == OBJ_ENCODING_HT) { + hfield hf = NULL; + dictEntry *de = NULL; + dict *d = o->ptr; + dictExpireMetadata *meta = (dictExpireMetadata *) dictMetadata(d); + sds prevVal = NULL; + + /* First retrieve the field to check if it exists */ + de = dictFind(d, field); + if (de) { + hf = dictGetKey(de); + prevExpire = hfieldGetExpireTime(hf); + prevVal = dictGetVal(de); + } + + /* Check DCF (don't create fields) and DOF (don't override fields) arg. */ + if ((!de && (flag & HFE_CMD_DCF)) || (de && (flag & HFE_CMD_DOF))) { + hsetfAddReply(c, flag, prevVal, prevVal, ret); + return 0; + } + + /* Field value will be updated. */ + ret = HSETF_FIELD; + + /* Decide if we are going to set/discard TTL */ + if (hsetfCheckTTLCondition(flag, prevExpire, expireAt)) + ret = HSETF_FIELD_AND_TTL; + + hsetfAddReply(c, flag, prevVal, value, ret); + + if (!hf || !hfieldIsExpireAttached(hf)) { + hfieldFree(hf); + + int withExpireMeta = (ret == HSETF_FIELD_AND_TTL) ? 1 : 0; + hf = hfieldNew(field, sdslen(field), withExpireMeta); + + if (!de) { + dictUseStoredKeyApi(d, 1); + de = dictAddRaw(d, hf, NULL); + dictUseStoredKeyApi(d, 0); + } + dictSetKey(d, de, hf); + } + + dictSetVal(d, de, sdsdup(value)); + sdsfree(prevVal); + + if (ret == HSETF_FIELD_AND_TTL) { + if (*minPrevExp > prevExpire) + *minPrevExp = prevExpire; + + if (!(flag & HFE_CMD_EXPIRY_MASK)) { + /* If none of EX,EXAT,PX,PXAT,KEEPTTL is specified, TTL is + * discarded. */ + hfieldPersist(o, hf); + } else if (checkAlreadyExpired(expireAt)) { + /* if expiration time is in the past */ + hashTypeDelete(o, field); + } else { + if (*minPrevExp > expireAt) + *minPrevExp = expireAt; + + if (prevExpire != EB_EXPIRE_TIME_INVALID) + ebRemove(&meta->hfe, &hashFieldExpireBucketsType, hf); + + ebAdd(&meta->hfe, &hashFieldExpireBucketsType, hf, expireAt); + } + } + + return 1; + } else { + serverPanic("Unknown encoding: %d", o->encoding); + } +} + +/* Parse hsetf command arguments. */ +static int hsetfParseArgs(client *c, int *flags, uint64_t *expireAt, + int *firstFieldPos, int *fieldCount) +{ + long val; + + *flags = 0; + *firstFieldPos = -1; + *fieldCount = -1; + + for (int i = 2; i < c->argc; i++) { + if (!strcasecmp(c->argv[i]->ptr, "fvs")) { + if (*firstFieldPos != -1) { + addReplyErrorFormat(c, "multiple FVS argument"); + return C_ERR; + } + + if (i >= c->argc - 3) { + addReplyErrorArity(c); + return C_ERR; + } + + if (getRangeLongFromObjectOrReply(c, c->argv[i + 1], 1, INT_MAX, &val, + "invalid number of fvs count") != C_OK) + return C_ERR; + + if (val > ((c->argc - i - 2) / 2)) { + addReplyErrorArity(c); + return C_ERR; + } + + *firstFieldPos = i + 2; + *fieldCount = (int) val; + i = *firstFieldPos + (*fieldCount) * 2 - 1; + } else if (!strcasecmp(c->argv[i]->ptr, "NX")) { + if (*flags & (HFE_CMD_XX | HFE_CMD_GT | HFE_CMD_LT)) + goto err_condition; + *flags |= HFE_CMD_NX; + } else if (!strcasecmp(c->argv[i]->ptr, "XX")) { + if (*flags & (HFE_CMD_NX | HFE_CMD_GT | HFE_CMD_LT)) + goto err_condition; + *flags |= HFE_CMD_XX; + } else if (!strcasecmp(c->argv[i]->ptr, "GT")) { + if (*flags & (HFE_CMD_NX | HFE_CMD_XX | HFE_CMD_LT)) + goto err_condition; + *flags |= HFE_CMD_GT; + } else if (!strcasecmp(c->argv[i]->ptr, "LT")) { + if (*flags & (HFE_CMD_NX | HFE_CMD_XX | HFE_CMD_GT)) + goto err_condition; + *flags |= HFE_CMD_LT; + } else if (!strcasecmp(c->argv[i]->ptr, "EX")) { + if (*flags & (HFE_CMD_EXAT | HFE_CMD_PX | HFE_CMD_PXAT | HFE_CMD_KEEPTTL)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_CMD_EX; + i++; + if (validateExpire(c, UNIT_SECONDS, c->argv[i], + commandTimeSnapshot(), expireAt) != C_OK) + return C_ERR; + + } else if (!strcasecmp(c->argv[i]->ptr, "PX")) { + if (*flags & (HFE_CMD_EX | HFE_CMD_EXAT | HFE_CMD_PXAT | HFE_CMD_KEEPTTL)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_CMD_PX; + i++; + if (validateExpire(c, UNIT_MILLISECONDS, c->argv[i], + commandTimeSnapshot(), expireAt) != C_OK) + return C_ERR; + } else if (!strcasecmp(c->argv[i]->ptr, "EXAT")) { + if (*flags & (HFE_CMD_EX | HFE_CMD_PX | HFE_CMD_PXAT | HFE_CMD_KEEPTTL)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_CMD_EXAT; + i++; + if (validateExpire(c, UNIT_SECONDS, c->argv[i], 0, expireAt) != C_OK) + return C_ERR; + } else if (!strcasecmp(c->argv[i]->ptr, "PXAT")) { + if (*flags & (HFE_CMD_EX | HFE_CMD_EXAT | HFE_CMD_PX | HFE_CMD_KEEPTTL)) + goto err_expiration; + + if (i >= c->argc - 1) + goto err_missing_expire; + + *flags |= HFE_CMD_PXAT; + i++; + if (validateExpire(c, UNIT_MILLISECONDS, c->argv[i], 0, expireAt) != C_OK) + return C_ERR; + } else if (!strcasecmp(c->argv[i]->ptr, "KEEPTTL")) { + if (*flags & (HFE_CMD_EX | HFE_CMD_EXAT | HFE_CMD_PX | HFE_CMD_PXAT)) + goto err_expiration; + *flags |= HFE_CMD_KEEPTTL; + } else if (!strcasecmp(c->argv[i]->ptr, "DC")) { + *flags |= HFE_CMD_DC; + } else if (!strcasecmp(c->argv[i]->ptr, "DCF")) { + if (*flags & HFE_CMD_DOF) + goto err_field_condition; + *flags |= HFE_CMD_DCF; + } else if (!strcasecmp(c->argv[i]->ptr, "DOF")) { + if (*flags & HFE_CMD_DCF) + goto err_field_condition; + *flags |= HFE_CMD_DOF; + } else if (!strcasecmp(c->argv[i]->ptr, "GETNEW")) { + if (*flags & HFE_CMD_GETOLD) + goto err_return_condition; + *flags |= HFE_CMD_GETNEW; + } else if (!strcasecmp(c->argv[i]->ptr, "GETOLD")) { + if (*flags & HFE_CMD_GETNEW) + goto err_return_condition; + *flags |= HFE_CMD_GETOLD; + } else { + addReplyErrorFormat(c, "unknown argument: %s", (char*) c->argv[i]->ptr); + return C_ERR; + } + } + + /* FVS argument is mandatory. */ + if (*firstFieldPos <= 0) { + addReplyError(c, "missing FVS argument"); + return C_ERR; + } + + if (*flags & HFE_CMD_COND_MASK && + (!(*flags & (HFE_CMD_EX | HFE_CMD_EXAT | HFE_CMD_PX | HFE_CMD_PXAT)))) + { + addReplyError(c, "NX, XX, GT, and LT can be specified only when EX, PX, EXAT, or PXAT is specified"); + return C_ERR; + } + + return C_OK; + +err_missing_expire: + addReplyError(c, "missing expire time"); + return C_ERR; +err_condition: + addReplyError(c, "Only one of NX, XX, GT, and LT arguments can be specified"); + return C_ERR; +err_expiration: + addReplyError(c, "Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments can be specified"); + return C_ERR; +err_field_condition: + addReplyError(c, "Only one of DCF or DOF arguments can be specified"); + return C_ERR; +err_return_condition: + addReplyError(c, "Only one of GETOLD or GETNEW arguments can be specified"); + return C_ERR; +} + +/* + * Set field value and optionally set the field's remaining time to live. + * Optionally it creates the key/fields. + * + * HSETF key + * [DC] [DCF | DOF] + * [NX | XX | GT | LT] + * [GETNEW | GETOLD] + * [EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT unix-time-milliseconds | KEEPTTL] + * + */ +void hsetfCommand(client *c) { + int flags = 0; + robj *hashObj, *keyArg = c->argv[1]; + + int firstFieldPos = 0; + int numFields = 0; + uint64_t expireAt = EB_EXPIRE_TIME_INVALID; + + if (hsetfParseArgs(c, &flags, &expireAt, &firstFieldPos, &numFields) != C_OK) + return; + + hashObj = lookupKeyWrite(c->db, c->argv[1]); + if (!hashObj) { + /* Don't create the object if command has DC or DCF arguments */ + if (flags & HFE_CMD_DC || flags & HFE_CMD_DCF) { + addReplyOrErrorObject(c, shared.null[c->resp]); + return; + } + + hashObj = createHashObject(); + dbAdd(c->db,c->argv[1],hashObj); + } + + hashTypeTryConversion(c->db,hashObj,c->argv, + firstFieldPos, + firstFieldPos + (numFields * 2) - 1); + + attachHfeMeta(c->db, hashObj, keyArg); + uint64_t minExpire = hashTypeGetMinExpire(hashObj); + + /* Figure out from provided set of fields in command, which one has the minimum + * expiration time, before the modification (Will be used for optimization below) */ + uint64_t minExpireFields = EB_EXPIRE_TIME_INVALID; + + int updated = 0; + addReplyArrayLen(c, numFields); + for (int i = 0; i < numFields ; i++) { + sds field = c->argv[firstFieldPos + (i * 2)]->ptr; + sds value = c->argv[firstFieldPos + (i * 2) + 1]->ptr; + updated += hsetfSetFieldAndReply(c, hashObj, field, value, flags, + expireAt, &minExpireFields); + } + + if (updated == 0) { + /* If we didn't update anything and object is empty, it means we just + * created the object above and leaving it empty. If this is the case, + * we should avoid creating the object in the first place. + * See above DC / DCF flags check when object does not exist. */ + serverAssert(hashTypeLength(hashObj, 0) != 0); + } else { + /* Notify keyspace event, update dirty count and update global HFE DS */ + server.dirty += updated; + signalModifiedKey(c,c->db,keyArg); + notifyKeyspaceEvent(NOTIFY_HASH,"hsetf",keyArg,c->db->id); + if (hashTypeLength(hashObj, 0) == 0) { + dbDelete(c->db,keyArg); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",keyArg, c->db->id); + } else { + updateGlobalHfeDs(c->db, hashObj, minExpire, minExpireFields); + } } } diff --git a/src/t_set.c b/src/t_set.c index 5efbc535c..f16cde818 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -432,7 +432,7 @@ robj *setTypePopRandom(robj *set) { if (set->encoding == OBJ_ENCODING_LISTPACK) { /* Find random and delete it without re-seeking the listpack. */ unsigned int i = 0; - unsigned char *p = lpNextRandom(set->ptr, lpFirst(set->ptr), &i, 1, 0); + unsigned char *p = lpNextRandom(set->ptr, lpFirst(set->ptr), &i, 1, 1); unsigned int len = 0; /* initialize to silence warning */ long long llele = 0; /* initialize to silence warning */ char *str = (char *)lpGetValue(p, &len, &llele); @@ -815,7 +815,7 @@ void spopWithCountCommand(client *c) { unsigned int index = 0; unsigned char **ps = zmalloc(sizeof(char *) * count); for (unsigned long i = 0; i < count; i++) { - p = lpNextRandom(lp, p, &index, count - i, 0); + p = lpNextRandom(lp, p, &index, count - i, 1); unsigned int len; str = (char *)lpGetValue(p, &len, (long long *)&llele); @@ -877,7 +877,7 @@ void spopWithCountCommand(client *c) { unsigned int index = 0; unsigned char **ps = zmalloc(sizeof(char *) * remaining); for (unsigned long i = 0; i < remaining; i++) { - p = lpNextRandom(lp, p, &index, remaining - i, 0); + p = lpNextRandom(lp, p, &index, remaining - i, 1); unsigned int len; str = (char *)lpGetValue(p, &len, (long long *)&llele); setTypeAddAux(newset, str, len, llele, 0); @@ -1103,7 +1103,7 @@ void srandmemberWithCountCommand(client *c) { unsigned int i = 0; addReplyArrayLen(c, count); while (count) { - p = lpNextRandom(lp, p, &i, count--, 0); + p = lpNextRandom(lp, p, &i, count--, 1); unsigned int len; str = (char *)lpGetValue(p, &len, (long long *)&llele); if (str == NULL) { diff --git a/src/t_zset.c b/src/t_zset.c index ec7d5ccaf..8533ff12b 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1754,7 +1754,7 @@ void zsetTypeRandomElement(robj *zsetobj, unsigned long zsetsize, listpackEntry *score = *(double*)dictGetVal(de); } else if (zsetobj->encoding == OBJ_ENCODING_LISTPACK) { listpackEntry val; - lpRandomPair(zsetobj->ptr, zsetsize, key, &val); + lpRandomPair(zsetobj->ptr, zsetsize, key, &val, 2); if (score) { if (val.sval) { *score = zzlStrtod(val.sval,val.slen); @@ -4263,7 +4263,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) { while (count) { sample_count = count > limit ? limit : count; count -= sample_count; - lpRandomPairs(zsetobj->ptr, sample_count, keys, vals); + lpRandomPairs(zsetobj->ptr, sample_count, keys, vals, 2); zrandmemberReplyWithListpack(c, sample_count, keys, vals); if (c->flags & CLIENT_CLOSE_ASAP) break; @@ -4317,7 +4317,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) { keys = zmalloc(sizeof(listpackEntry)*count); if (withscores) vals = zmalloc(sizeof(listpackEntry)*count); - serverAssert(lpRandomPairsUnique(zsetobj->ptr, count, keys, vals) == count); + serverAssert(lpRandomPairsUnique(zsetobj->ptr, count, keys, vals, 2) == count); zrandmemberReplyWithListpack(c, count, keys, vals); zfree(keys); zfree(vals); diff --git a/tests/unit/type/hash-field-expire.tcl b/tests/unit/type/hash-field-expire.tcl index f588d0813..41134ffdc 100644 --- a/tests/unit/type/hash-field-expire.tcl +++ b/tests/unit/type/hash-field-expire.tcl @@ -17,6 +17,11 @@ set P_NO_FIELD -2 set P_NO_EXPIRY -1 set P_OK 1 +######## HSETF +set S_FAIL 0 +set S_FIELD 1 +set S_FIELD_AND_TTL 3 + ############################### AUX FUNCS ###################################### proc create_hash {key entries} { @@ -84,555 +89,1121 @@ proc hrandfieldTest {activeExpireConfig} { ############################### TESTS ######################################### start_server {tags {"external:skip needs:debug"}} { + foreach type {listpack ht} { + if {$type eq "ht"} { + r config set hash-max-listpack-entries 0 + } else { + r config set hash-max-listpack-entries 512 + } - # Currently listpack doesn't support HFE - r config set hash-max-listpack-entries 0 + test "HPEXPIRE(AT) - Test 'NX' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hpexpire myhash 1000 NX 1 field1] [list $E_OK] + assert_equal [r hpexpire myhash 1000 NX 2 field1 field2] [list $E_FAIL $E_OK] - test {HPEXPIRE(AT) - Test 'NX' flag} { - r del myhash - r hset myhash field1 value1 field2 value2 field3 value3 - assert_equal [r hpexpire myhash 1000 NX 1 field1] [list $E_OK] - assert_equal [r hpexpire myhash 1000 NX 2 field1 field2] [list $E_FAIL $E_OK] + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 1 field1] [list $E_OK] + assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 2 field1 field2] [list $E_FAIL $E_OK] + } - r del myhash - r hset myhash field1 value1 field2 value2 field3 value3 - assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 1 field1] [list $E_OK] - assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 2 field1 field2] [list $E_FAIL $E_OK] - } + test "HPEXPIRE(AT) - Test 'XX' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hpexpire myhash 1000 NX 2 field1 field2] [list $E_OK $E_OK] + assert_equal [r hpexpire myhash 1000 XX 2 field1 field3] [list $E_OK $E_FAIL] - test {HPEXPIRE(AT) - Test 'XX' flag} { - r del myhash - r hset myhash field1 value1 field2 value2 field3 value3 - assert_equal [r hpexpire myhash 1000 NX 2 field1 field2] [list $E_OK $E_OK] - assert_equal [r hpexpire myhash 1000 XX 2 field1 field3] [list $E_OK $E_FAIL] + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 2 field1 field2] [list $E_OK $E_OK] + assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] XX 2 field1 field3] [list $E_OK $E_FAIL] + } - r del myhash - r hset myhash field1 value1 field2 value2 field3 value3 - assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 2 field1 field2] [list $E_OK $E_OK] - assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] XX 2 field1 field3] [list $E_OK $E_FAIL] - } + test "HPEXPIRE(AT) - Test 'GT' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 + assert_equal [r hpexpire myhash 1000 NX 1 field1] [list $E_OK] + assert_equal [r hpexpire myhash 2000 NX 1 field2] [list $E_OK] + assert_equal [r hpexpire myhash 1500 GT 2 field1 field2] [list $E_OK $E_FAIL] - test {HPEXPIRE(AT) - Test 'GT' flag} { - r del myhash - r hset myhash field1 value1 field2 value2 - assert_equal [r hpexpire myhash 1000 NX 1 field1] [list $E_OK] - assert_equal [r hpexpire myhash 2000 NX 1 field2] [list $E_OK] - assert_equal [r hpexpire myhash 1500 GT 2 field1 field2] [list $E_OK $E_FAIL] + r del myhash + r hset myhash field1 value1 field2 value2 + assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 1 field1] [list $E_OK] + assert_equal [r hpexpireat myhash [expr {([clock seconds]+2000)*1000}] NX 1 field2] [list $E_OK] + assert_equal [r hpexpireat myhash [expr {([clock seconds]+1500)*1000}] GT 2 field1 field2] [list $E_OK $E_FAIL] + } - r del myhash - r hset myhash field1 value1 field2 value2 - assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 1 field1] [list $E_OK] - assert_equal [r hpexpireat myhash [expr {([clock seconds]+2000)*1000}] NX 1 field2] [list $E_OK] - assert_equal [r hpexpireat myhash [expr {([clock seconds]+1500)*1000}] GT 2 field1 field2] [list $E_OK $E_FAIL] - } + test "HPEXPIRE(AT) - Test 'LT' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 + assert_equal [r hpexpire myhash 1000 NX 1 field1] [list $E_OK] + assert_equal [r hpexpire myhash 2000 NX 1 field2] [list $E_OK] + assert_equal [r hpexpire myhash 1500 LT 2 field1 field2] [list $E_FAIL $E_OK] - test {HPEXPIRE(AT) - Test 'LT' flag} { - r del myhash - r hset myhash field1 value1 field2 value2 - assert_equal [r hpexpire myhash 1000 NX 1 field1] [list $E_OK] - assert_equal [r hpexpire myhash 2000 NX 1 field2] [list $E_OK] - assert_equal [r hpexpire myhash 1500 LT 2 field1 field2] [list $E_FAIL $E_OK] + r del myhash + r hset myhash field1 value1 field2 value2 + assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 1 field1] [list $E_OK] + assert_equal [r hpexpireat myhash [expr {([clock seconds]+2000)*1000}] NX 1 field2] [list $E_OK] + assert_equal [r hpexpireat myhash [expr {([clock seconds]+1500)*1000}] LT 2 field1 field2] [list $E_FAIL $E_OK] + } - r del myhash - r hset myhash field1 value1 field2 value2 - assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX 1 field1] [list $E_OK] - assert_equal [r hpexpireat myhash [expr {([clock seconds]+2000)*1000}] NX 1 field2] [list $E_OK] - assert_equal [r hpexpireat myhash [expr {([clock seconds]+1500)*1000}] LT 2 field1 field2] [list $E_FAIL $E_OK] - } + test "HPEXPIREAT - field not exists or TTL is in the past ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 f4 v4 + r hexpire myhash 1000 NX 1 f4 + assert_equal [r hpexpireat myhash [expr {([clock seconds]-1)*1000}] NX 4 f1 f2 f3 f4] "$E_DELETED $E_DELETED $E_NO_FIELD $E_FAIL" + assert_equal [r hexists myhash field1] 0 + } - test {HPEXPIREAT - field not exists or TTL is in the past} { - r del myhash - r hset myhash f1 v1 f2 v2 f4 v4 - r hexpire myhash 1000 NX 1 f4 - assert_equal [r hpexpireat myhash [expr {([clock seconds]-1)*1000}] NX 4 f1 f2 f3 f4] "$E_DELETED $E_DELETED $E_NO_FIELD $E_FAIL" - assert_equal [r hexists myhash field1] 0 - } + test "HPEXPIRE - wrong number of arguments ($type)" { + r del myhash + r hset myhash f1 v1 + assert_error {*Parameter `numFields` should be greater than 0} {r hpexpire myhash 1000 NX 0 f1 f2 f3} + assert_error {*Parameter `numFileds` is more than number of arguments} {r hpexpire myhash 1000 NX 4 f1 f2 f3} + } - test {HPEXPIRE - wrong number of arguments} { - r del myhash - r hset myhash f1 v1 - assert_error {*Parameter `numFields` should be greater than 0} {r hpexpire myhash 1000 NX 0 f1 f2 f3} - assert_error {*Parameter `numFileds` is more than number of arguments} {r hpexpire myhash 1000 NX 4 f1 f2 f3} - } + test "HPEXPIRE - parameter expire-time near limit of 2^48 ($type)" { + r del myhash + r hset myhash f1 v1 + # below & above + assert_equal [r hpexpire myhash [expr (1<<48) - [clock milliseconds] - 1000 ] 1 f1] [list $E_OK] + assert_error {*invalid expire time*} {r hpexpire myhash [expr (1<<48) - [clock milliseconds] + 100 ] 1 f1} + } - test {HPEXPIRE - parameter expire-time near limit of 2^48} { - r del myhash - r hset myhash f1 v1 - # below & above - assert_equal [r hpexpire myhash [expr (1<<48) - [clock milliseconds] - 1000 ] 1 f1] [list $E_OK] - assert_error {*invalid expire time*} {r hpexpire myhash [expr (1<<48) - [clock milliseconds] + 100 ] 1 f1} - } + test "Lazy - doesn't delete hash that all its fields got expired ($type)" { + r debug set-active-expire 0 + r flushall - test {Lazy - doesn't delete hash that all its fields got expired} { - r debug set-active-expire 0 - r flushall + set hash_sizes {1 15 16 17 31 32 33 40} + foreach h $hash_sizes { + for {set i 1} {$i <= $h} {incr i} { + # random expiration time + r hset hrand$h f$i v$i + r hpexpire hrand$h [expr {50 + int(rand() * 50)}] 1 f$i + assert_equal 1 [r HEXISTS hrand$h f$i] - set hash_sizes {1 15 16 17 31 32 33 40} - foreach h $hash_sizes { - for {set i 1} {$i <= $h} {incr i} { - # random expiration time - r hset hrand$h f$i v$i - r hpexpire hrand$h [expr {50 + int(rand() * 50)}] 1 f$i - assert_equal 1 [r HEXISTS hrand$h f$i] + # same expiration time + r hset same$h f$i v$i + r hpexpire same$h 100 1 f$i + assert_equal 1 [r HEXISTS same$h f$i] - # same expiration time - r hset same$h f$i v$i - r hpexpire same$h 100 1 f$i - assert_equal 1 [r HEXISTS same$h f$i] + # same expiration time + r hset mix$h f$i v$i fieldWithoutExpire$i v$i + r hpexpire mix$h 100 1 f$i + assert_equal 1 [r HEXISTS mix$h f$i] + } + } - # same expiration time - r hset mix$h f$i v$i fieldWithoutExpire$i v$i - r hpexpire mix$h 100 1 f$i - assert_equal 1 [r HEXISTS mix$h f$i] + after 150 + + # Verify that all fields got expired but keys wasn't lazy deleted + foreach h $hash_sizes { + for {set i 1} {$i <= $h} {incr i} { + assert_equal 0 [r HEXISTS mix$h f$i] + } + assert_equal 1 [r EXISTS hrand$h] + assert_equal 1 [r EXISTS same$h] + assert_equal [expr $h * 2] [r HLEN mix$h] + } + # Restore default + r debug set-active-expire 1 + } + + test "Active - deletes hash that all its fields got expired ($type)" { + r flushall + + set hash_sizes {1 15 16 17 31 32 33 40} + foreach h $hash_sizes { + for {set i 1} {$i <= $h} {incr i} { + # random expiration time + r hset hrand$h f$i v$i + r hpexpire hrand$h [expr {50 + int(rand() * 50)}] 1 f$i + assert_equal 1 [r HEXISTS hrand$h f$i] + + # same expiration time + r hset same$h f$i v$i + r hpexpire same$h 100 1 f$i + assert_equal 1 [r HEXISTS same$h f$i] + + # same expiration time + r hset mix$h f$i v$i fieldWithoutExpire$i v$i + r hpexpire mix$h 100 1 f$i + assert_equal 1 [r HEXISTS mix$h f$i] + } + } + + # Wait for active expire + wait_for_condition 50 20 { [r EXISTS same40] == 0 } else { fail "hash `same40` should be expired" } + + # Verify that all fields got expired and keys got deleted + foreach h $hash_sizes { + wait_for_condition 50 20 { + [r HLEN mix$h] == $h + } else { + fail "volatile fields of hash `mix$h` should be expired" + } + + for {set i 1} {$i <= $h} {incr i} { + assert_equal 0 [r HEXISTS mix$h f$i] + } + assert_equal 0 [r EXISTS hrand$h] + assert_equal 0 [r EXISTS same$h] } } - after 150 - - # Verify that all fields got expired but keys wasn't lazy deleted - foreach h $hash_sizes { - for {set i 1} {$i <= $h} {incr i} { - assert_equal 0 [r HEXISTS mix$h f$i] - } - assert_equal 1 [r EXISTS hrand$h] - assert_equal 1 [r EXISTS same$h] - assert_equal [expr $h * 2] [r HLEN mix$h] + test "HPEXPIRE - Flushall deletes all pending expired fields ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 + r hpexpire myhash 10000 NX 1 field1 + r hpexpire myhash 10000 NX 1 field2 + r flushall + r del myhash + r hset myhash field1 value1 field2 value2 + r hpexpire myhash 10000 NX 1 field1 + r hpexpire myhash 10000 NX 1 field2 + r flushall async } - # Restore default - r debug set-active-expire 1 - } - test {Active - deletes hash that all its fields got expired} { - r flushall + test "HTTL/HPTTL - Input validation gets failed on nonexists field or field without expire ($type)" { + r del myhash + r HSET myhash field1 value1 field2 value2 + r HPEXPIRE myhash 1000 NX 1 field1 - set hash_sizes {1 15 16 17 31 32 33 40} - foreach h $hash_sizes { - for {set i 1} {$i <= $h} {incr i} { - # random expiration time - r hset hrand$h f$i v$i - r hpexpire hrand$h [expr {50 + int(rand() * 50)}] 1 f$i - assert_equal 1 [r HEXISTS hrand$h f$i] - - # same expiration time - r hset same$h f$i v$i - r hpexpire same$h 100 1 f$i - assert_equal 1 [r HEXISTS same$h f$i] - - # same expiration time - r hset mix$h f$i v$i fieldWithoutExpire$i v$i - r hpexpire mix$h 100 1 f$i - assert_equal 1 [r HEXISTS mix$h f$i] + foreach cmd {HTTL HPTTL} { + assert_equal [r $cmd non_exists_key 1 f] {} + assert_equal [r $cmd myhash 2 field2 non_exists_field] "$T_NO_EXPIRY $T_NO_FIELD" + # Set numFields less than actual number of fields. Fine. + assert_equal [r $cmd myhash 1 non_exists_field1 non_exists_field2] "$T_NO_FIELD" } } - # Wait for active expire - wait_for_condition 50 20 { [r EXISTS same40] == 0 } else { fail "hash `same40` should be expired" } + test "HTTL/HPTTL - returns time to live in seconds/msillisec ($type)" { + r del myhash + r HSET myhash field1 value1 field2 value2 + r HPEXPIRE myhash 2000 NX 2 field1 field2 + set ttlArray [r HTTL myhash 2 field1 field2] + assert_range [lindex $ttlArray 0] 1 2 + set ttl [r HPTTL myhash 1 field1] + assert_range $ttl 1000 2000 + } - # Verify that all fields got expired and keys got deleted - foreach h $hash_sizes { - for {set i 1} {$i <= $h} {incr i} { - assert_equal 0 [r HEXISTS mix$h f$i] + test "HEXPIRETIME - returns TTL in Unix timestamp ($type)" { + r del myhash + r HSET myhash field1 value1 + r HPEXPIRE myhash 1000 NX 1 field1 + + set lo [expr {[clock seconds] + 1}] + set hi [expr {[clock seconds] + 2}] + assert_range [r HEXPIRETIME myhash 1 field1] $lo $hi + assert_range [r HPEXPIRETIME myhash 1 field1] [expr $lo*1000] [expr $hi*1000] + } + + test "HTTL/HPTTL - Verify TTL progress until expiration ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 + r hpexpire myhash 1000 NX 1 field1 + assert_range [r HPTTL myhash 1 field1] 100 1000 + assert_range [r HTTL myhash 1 field1] 0 1 + after 100 + assert_range [r HPTTL myhash 1 field1] 1 901 + after 910 + assert_equal [r HPTTL myhash 1 field1] $T_NO_FIELD + assert_equal [r HTTL myhash 1 field1] $T_NO_FIELD + } + + test "HPEXPIRE - DEL hash with non expired fields (valgrind test) ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 + r hpexpire myhash 10000 NX 1 field1 + r del myhash + } + + test "HEXPIREAT - Set time in the past ($type)" { + r del myhash + r hset myhash field1 value1 + assert_equal [r hexpireat myhash [expr {[clock seconds] - 1}] NX 1 field1] $E_DELETED + assert_equal [r hexists myhash field1] 0 + } + + test "HEXPIREAT - Set time and then get TTL ($type)" { + r del myhash + r hset myhash field1 value1 + + r hexpireat myhash [expr {[clock seconds] + 2}] NX 1 field1 + assert_range [r hpttl myhash 1 field1] 1000 2000 + assert_range [r httl myhash 1 field1] 1 2 + + r hexpireat myhash [expr {[clock seconds] + 5}] XX 1 field1 + assert_range [r httl myhash 1 field1] 4 5 + } + + test "Lazy expire - delete hash with expired fields ($type)" { + r del myhash + r debug set-active-expire 0 + r hset myhash k v + r hpexpire myhash 1 NX 1 k + after 5 + r del myhash + r debug set-active-expire 1 + } + + # OPEN: To decide if to delete expired fields at start of HRANDFIELD. + # test "Test HRANDFIELD does not return expired fields ($type)" { + # hrandfieldTest 0 + # hrandfieldTest 1 + # } + + test "Test HRANDFIELD can return expired fields ($type)" { + r debug set-active-expire 0 + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + r hpexpire myhash 1 NX 4 f1 f2 f3 f4 + after 5 + set res [cmp_hrandfield_result myhash "f1 f2 f3 f4 f5"] + assert {$res == 1} + r debug set-active-expire 1 + + } + + test "Lazy expire - HLEN does count expired fields ($type)" { + # Enforce only lazy expire + r debug set-active-expire 0 + + r del h1 h4 h18 h20 + r hset h1 k1 v1 + r hpexpire h1 1 NX 1 k1 + + r hset h4 k1 v1 k2 v2 k3 v3 k4 v4 + r hpexpire h4 1 NX 3 k1 k3 k4 + + # beyond 16 fields: HFE DS (ebuckets) converts from list to rax + + r hset h18 k1 v1 k2 v2 k3 v3 k4 v4 k5 v5 k6 v6 k7 v7 k8 v8 k9 v9 k10 v10 k11 v11 k12 v12 k13 v13 k14 v14 k15 v15 k16 v16 k17 v17 k18 v18 + r hpexpire h18 1 NX 18 k1 k2 k3 k4 k5 k6 k7 k8 k9 k10 k11 k12 k13 k14 k15 k16 k17 k18 + + r hset h20 k1 v1 k2 v2 k3 v3 k4 v4 k5 v5 k6 v6 k7 v7 k8 v8 k9 v9 k10 v10 k11 v11 k12 v12 k13 v13 k14 v14 k15 v15 k16 v16 k17 v17 k18 v18 k19 v19 k20 v20 + r hpexpire h20 1 NX 2 k1 k2 + + after 10 + + assert_equal [r hlen h1] 1 + assert_equal [r hlen h4] 4 + assert_equal [r hlen h18] 18 + assert_equal [r hlen h20] 20 + # Restore to support active expire + r debug set-active-expire 1 + } + + test "Lazy expire - HSCAN does not report expired fields ($type)" { + # Enforce only lazy expire + r debug set-active-expire 0 + + r del h1 h20 h4 h18 h20 + r hset h1 01 01 + r hpexpire h1 1 NX 1 01 + + r hset h4 01 01 02 02 03 03 04 04 + r hpexpire h4 1 NX 3 01 03 04 + + # beyond 16 fields hash-field expiration DS (ebuckets) converts from list to rax + + r hset h18 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16 16 17 17 18 18 + r hpexpire h18 1 NX 18 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 + + r hset h20 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16 16 17 17 18 18 19 19 20 20 + r hpexpire h20 1 NX 2 01 02 + + after 10 + + # Verify SCAN does not report expired fields + assert_equal [lsort -unique [lindex [r hscan h1 0 COUNT 10] 1]] "" + assert_equal [lsort -unique [lindex [r hscan h4 0 COUNT 10] 1]] "02" + assert_equal [lsort -unique [lindex [r hscan h18 0 COUNT 10] 1]] "" + assert_equal [lsort -unique [lindex [r hscan h20 0 COUNT 100] 1]] "03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20" + # Restore to support active expire + r debug set-active-expire 1 + } + + test "Test HSCAN with mostly expired fields return empty result ($type)" { + r debug set-active-expire 0 + + # Create hash with 1000 fields and 999 of them will be expired + r del myhash + for {set i 1} {$i <= 1000} {incr i} { + r hset myhash field$i value$i + if {$i > 1} { + r hpexpire myhash 1 NX 1 field$i + } } - assert_equal 0 [r EXISTS hrand$h] - assert_equal 0 [r EXISTS same$h] - assert_equal $h [r HLEN mix$h] - } - } + after 3 - test {HPEXPIRE - Flushall deletes all pending expired fields} { - r del myhash - r hset myhash field1 value1 field2 value2 - r hpexpire myhash 10000 NX 1 field1 - r hpexpire myhash 10000 NX 1 field2 - r flushall - r del myhash - r hset myhash field1 value1 field2 value2 - r hpexpire myhash 10000 NX 1 field1 - r hpexpire myhash 10000 NX 1 field2 - r flushall async - } - - test {HTTL/HPTTL - Input validation gets failed on nonexists field or field without expire} { - r del myhash - r HSET myhash field1 value1 field2 value2 - r HPEXPIRE myhash 1000 NX 1 field1 - - foreach cmd {HTTL HPTTL} { - assert_equal [r $cmd non_exists_key 1 f] {} - assert_equal [r $cmd myhash 2 field2 non_exists_field] "$T_NO_EXPIRY $T_NO_FIELD" - # Set numFields less than actual number of fields. Fine. - assert_equal [r $cmd myhash 1 non_exists_field1 non_exists_field2] "$T_NO_FIELD" - } - } - - test {HTTL/HPTTL - returns time to live in seconds/msillisec} { - r del myhash - r HSET myhash field1 value1 field2 value2 - r HPEXPIRE myhash 2000 NX 2 field1 field2 - set ttlArray [r HTTL myhash 2 field1 field2] - assert_range [lindex $ttlArray 0] 1 2 - set ttl [r HPTTL myhash 1 field1] - assert_range $ttl 1000 2000 - } - - test {HEXPIRETIME - returns TTL in Unix timestamp} { - r del myhash - r HSET myhash field1 value1 - r HPEXPIRE myhash 1000 NX 1 field1 - - set lo [expr {[clock seconds] + 1}] - set hi [expr {[clock seconds] + 2}] - assert_range [r HEXPIRETIME myhash 1 field1] $lo $hi - assert_range [r HPEXPIRETIME myhash 1 field1] [expr $lo*1000] [expr $hi*1000] - } - - test {HTTL/HPTTL - Verify TTL progress until expiration} { - r del myhash - r hset myhash field1 value1 field2 value2 - r hpexpire myhash 200 NX 1 field1 - assert_range [r HPTTL myhash 1 field1] 100 200 - assert_range [r HTTL myhash 1 field1] 0 1 - after 100 - assert_range [r HPTTL myhash 1 field1] 1 101 - after 110 - assert_equal [r HPTTL myhash 1 field1] $T_NO_FIELD - assert_equal [r HTTL myhash 1 field1] $T_NO_FIELD - } - - test {HPEXPIRE - DEL hash with non expired fields (valgrind test)} { - r del myhash - r hset myhash field1 value1 field2 value2 - r hpexpire myhash 10000 NX 1 field1 - r del myhash - } - - test {HEXPIREAT - Set time in the past} { - r del myhash - r hset myhash field1 value1 - assert_equal [r hexpireat myhash [expr {[clock seconds] - 1}] NX 1 field1] $E_DELETED - assert_equal [r hexists myhash field1] 0 - } - - test {HEXPIREAT - Set time and then get TTL} { - r del myhash - r hset myhash field1 value1 - - r hexpireat myhash [expr {[clock seconds] + 2}] NX 1 field1 - assert_range [r hpttl myhash 1 field1] 1000 2000 - assert_range [r httl myhash 1 field1] 1 2 - - r hexpireat myhash [expr {[clock seconds] + 5}] XX 1 field1 - assert_range [r httl myhash 1 field1] 4 5 - } - - test {Lazy expire - delete hash with expired fields} { - r del myhash - r debug set-active-expire 0 - r hset myhash k v - r hpexpire myhash 1 NX 1 k - after 5 - r del myhash - r debug set-active-expire 1 - } - -# OPEN: To decide if to delete expired fields at start of HRANDFIELD. -# test {Test HRANDFIELD does not return expired fields} { -# hrandfieldTest 0 -# hrandfieldTest 1 -# } - - test {Test HRANDFIELD can return expired fields} { - r debug set-active-expire 0 - r del myhash - r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 - r hpexpire myhash 1 NX 4 f1 f2 f3 f4 - after 5 - set res [cmp_hrandfield_result myhash "f1 f2 f3 f4 f5"] - assert {$res == 1} - r debug set-active-expire 1 - - } - - test {Lazy expire - HLEN does count expired fields} { - # Enforce only lazy expire - r debug set-active-expire 0 - - r del h1 h4 h18 h20 - r hset h1 k1 v1 - r hpexpire h1 1 NX 1 k1 - - r hset h4 k1 v1 k2 v2 k3 v3 k4 v4 - r hpexpire h4 1 NX 3 k1 k3 k4 - - # beyond 16 fields: HFE DS (ebuckets) converts from list to rax - - r hset h18 k1 v1 k2 v2 k3 v3 k4 v4 k5 v5 k6 v6 k7 v7 k8 v8 k9 v9 k10 v10 k11 v11 k12 v12 k13 v13 k14 v14 k15 v15 k16 v16 k17 v17 k18 v18 - r hpexpire h18 1 NX 18 k1 k2 k3 k4 k5 k6 k7 k8 k9 k10 k11 k12 k13 k14 k15 k16 k17 k18 - - r hset h20 k1 v1 k2 v2 k3 v3 k4 v4 k5 v5 k6 v6 k7 v7 k8 v8 k9 v9 k10 v10 k11 v11 k12 v12 k13 v13 k14 v14 k15 v15 k16 v16 k17 v17 k18 v18 k19 v19 k20 v20 - r hpexpire h20 1 NX 2 k1 k2 - - after 10 - - assert_equal [r hlen h1] 1 - assert_equal [r hlen h4] 4 - assert_equal [r hlen h18] 18 - assert_equal [r hlen h20] 20 - # Restore to support active expire - r debug set-active-expire 1 - } - - test {Lazy expire - HSCAN does not report expired fields} { - # Enforce only lazy expire - r debug set-active-expire 0 - - r del h1 h20 h4 h18 h20 - r hset h1 01 01 - r hpexpire h1 1 NX 1 01 - - r hset h4 01 01 02 02 03 03 04 04 - r hpexpire h4 1 NX 3 01 03 04 - - # beyond 16 fields hash-field expiration DS (ebuckets) converts from list to rax - - r hset h18 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16 16 17 17 18 18 - r hpexpire h18 1 NX 18 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 - - r hset h20 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16 16 17 17 18 18 19 19 20 20 - r hpexpire h20 1 NX 2 01 02 - - after 10 - - # Verify SCAN does not report expired fields - assert_equal [lsort -unique [lindex [r hscan h1 0 COUNT 10] 1]] "" - assert_equal [lsort -unique [lindex [r hscan h4 0 COUNT 10] 1]] "02" - assert_equal [lsort -unique [lindex [r hscan h18 0 COUNT 10] 1]] "" - assert_equal [lsort -unique [lindex [r hscan h20 0 COUNT 100] 1]] "03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20" - # Restore to support active expire - r debug set-active-expire 1 - } - - test {Test HSCAN with mostly expired fields return empty result} { - r debug set-active-expire 0 - - # Create hash with 1000 fields and 999 of them will be expired - r del myhash - for {set i 1} {$i <= 1000} {incr i} { - r hset myhash field$i value$i - if {$i > 1} { - r hpexpire myhash 1 NX 1 field$i + # Verify iterative HSCAN returns either empty result or only the first field + set countEmptyResult 0 + set cur 0 + while 1 { + set res [r hscan myhash $cur] + set cur [lindex $res 0] + # if the result is not empty, it should contain only the first field + if {[llength [lindex $res 1]] > 0} { + assert_equal [lindex $res 1] "field1 value1" + } else { + incr countEmptyResult + } + if {$cur == 0} break } + assert {$countEmptyResult > 0} + r debug set-active-expire 1 } - after 3 - # Verify iterative HSCAN returns either empty result or only the first field - set countEmptyResult 0 - set cur 0 - while 1 { - set res [r hscan myhash $cur] - set cur [lindex $res 0] - # if the result is not empty, it should contain only the first field - if {[llength [lindex $res 1]] > 0} { - assert_equal [lindex $res 1] "field1 value1" - } else { - incr countEmptyResult + test "Lazy expire - verify various HASH commands handling expired fields ($type)" { + # Enforce only lazy expire + r debug set-active-expire 0 + r del h1 h2 h3 h4 h5 h18 + r hset h1 01 01 + r hset h2 01 01 02 02 + r hset h3 01 01 02 02 03 03 + r hset h4 1 99 2 99 3 99 4 99 + r hset h5 1 1 2 22 3 333 4 4444 5 55555 + r hset h6 01 01 02 02 03 03 04 04 05 05 06 06 + r hset h18 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16 16 17 17 18 18 + r hpexpire h1 100 NX 1 01 + r hpexpire h2 100 NX 1 01 + r hpexpire h2 100 NX 1 02 + r hpexpire h3 100 NX 1 01 + r hpexpire h4 100 NX 1 2 + r hpexpire h5 100 NX 1 3 + r hpexpire h6 100 NX 1 05 + r hpexpire h18 100 NX 17 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 + + after 150 + + # Verify HDEL not ignore expired field. It is too much overhead to check + # if the field is expired before deletion. + assert_equal [r HDEL h1 01] "1" + + # Verify HGET ignore expired field + assert_equal [r HGET h2 01] "" + assert_equal [r HGET h2 02] "" + assert_equal [r HGET h3 01] "" + assert_equal [r HGET h3 02] "02" + assert_equal [r HGET h3 03] "03" + # Verify HINCRBY ignore expired field + assert_equal [r HINCRBY h4 2 1] "1" + assert_equal [r HINCRBY h4 3 1] "100" + # Verify HSTRLEN ignore expired field + assert_equal [r HSTRLEN h5 3] "0" + assert_equal [r HSTRLEN h5 4] "4" + assert_equal [lsort [r HKEYS h6]] "01 02 03 04 06" + # Verify HEXISTS ignore expired field + assert_equal [r HEXISTS h18 07] "0" + assert_equal [r HEXISTS h18 18] "1" + # Verify HVALS ignore expired field + assert_equal [lsort [r HVALS h18]] "18" + # Restore to support active expire + r debug set-active-expire 1 + } + + test "A field with TTL overridden with another value (TTL discarded) ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + r hpexpire myhash 10000 NX 1 field1 + r hpexpire myhash 1 NX 1 field2 + + # field2 TTL will be discarded + r hset myhash field2 value4 + after 5 + # Expected TTL will be discarded + assert_equal [r hget myhash field2] "value4" + assert_equal [r httl myhash 2 field2 field3] "$T_NO_EXPIRY $T_NO_EXPIRY" + assert_not_equal [r httl myhash 1 field1] "$T_NO_EXPIRY" + } + + test "Modify TTL of a field ($type)" { + r del myhash + r hset myhash field1 value1 + r hpexpire myhash 200000 NX 1 field1 + r hpexpire myhash 1000000 XX 1 field1 + after 15 + assert_equal [r hget myhash field1] "value1" + assert_range [r hpttl myhash 1 field1] 900000 1000000 + } + + test "Test return value of set operation ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 + r hexpire myhash 100000 1 f1 + assert_equal [r hset myhash f2 v2] 0 + assert_equal [r hset myhash f3 v3] 1 + assert_equal [r hset myhash f3 v3 f4 v4] 1 + assert_equal [r hset myhash f3 v3 f5 v5 f6 v6] 2 + } + + test "Test HGETALL not return expired fields ($type)" { + # Test with small hash + r debug set-active-expire 0 + r del myhash + r hset myhash1 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 f6 v6 + r hpexpire myhash1 1 NX 3 f2 f4 f6 + after 10 + assert_equal [lsort [r hgetall myhash1]] "f1 f3 f5 v1 v3 v5" + + # Test with large hash + r del myhash + for {set i 1} {$i <= 600} {incr i} { + r hset myhash f$i v$i + if {$i > 3} { r hpexpire myhash 1 NX 1 f$i } } - if {$cur == 0} break + after 10 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 f2 f3 v1 v2 v3"] + r debug set-active-expire 1 } - assert {$countEmptyResult > 0} - r debug set-active-expire 1 - } - test {Lazy expire - verify various HASH commands handling expired fields} { - # Enforce only lazy expire - r debug set-active-expire 0 - r del h1 h2 h3 h4 h5 h18 - r hset h1 01 01 - r hset h2 01 01 02 02 - r hset h3 01 01 02 02 03 03 - r hset h4 1 99 2 99 3 99 4 99 - r hset h5 1 1 2 22 3 333 4 4444 5 55555 - r hset h6 01 01 02 02 03 03 04 04 05 05 06 06 - r hset h18 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16 16 17 17 18 18 - r hpexpire h1 100 NX 1 01 - r hpexpire h2 100 NX 1 01 - r hpexpire h2 100 NX 1 02 - r hpexpire h3 100 NX 1 01 - r hpexpire h4 100 NX 1 2 - r hpexpire h5 100 NX 1 3 - r hpexpire h6 100 NX 1 05 - r hpexpire h18 100 NX 17 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 - - after 150 - - # Verify HDEL not ignore expired field. It is too much overhead to check - # if the field is expired before deletion. - assert_equal [r HDEL h1 01] "1" - - # Verify HGET ignore expired field - assert_equal [r HGET h2 01] "" - assert_equal [r HGET h2 02] "" - assert_equal [r HGET h3 01] "" - assert_equal [r HGET h3 02] "02" - assert_equal [r HGET h3 03] "03" - # Verify HINCRBY ignore expired field - assert_equal [r HINCRBY h4 2 1] "1" - assert_equal [r HINCRBY h4 3 1] "100" - # Verify HSTRLEN ignore expired field - assert_equal [r HSTRLEN h5 3] "0" - assert_equal [r HSTRLEN h5 4] "4" - assert_equal [lsort [r HKEYS h6]] "01 02 03 04 06" - # Verify HEXISTS ignore expired field - assert_equal [r HEXISTS h18 07] "0" - assert_equal [r HEXISTS h18 18] "1" - # Verify HVALS ignore expired field - assert_equal [lsort [r HVALS h18]] "18" - # Restore to support active expire - r debug set-active-expire 1 - } - - test {A field with TTL overridden with another value (TTL discarded)} { - r del myhash - r hset myhash field1 value1 - r hpexpire myhash 1 NX 1 field1 - r hset myhash field1 value2 - after 5 - # Expected TTL will be discarded - assert_equal [r hget myhash field1] "value2" - } - - test {Modify TTL of a field} { - r del myhash - r hset myhash field1 value1 - r hpexpire myhash 200 NX 1 field1 - r hpexpire myhash 1000 XX 1 field1 - after 15 - assert_equal [r hget myhash field1] "value1" - assert_range [r hpttl myhash 1 field1] 900 1000 - } - - test {Test HGETALL not return expired fields} { - # Test with small hash - r debug set-active-expire 0 - r del myhash - r hset myhash1 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 - r hpexpire myhash1 1 NX 2 f2 f4 - after 10 - assert_equal [lsort [r hgetall myhash1]] "f1 f3 f5 v1 v3 v5" - - # Test with large hash - r del myhash - for {set i 1} {$i <= 600} {incr i} { - r hset myhash f$i v$i - if {$i > 3} { r hpexpire myhash 1 NX 1 f$i } + test "Test RENAME hash with fields to be expired ($type)" { + r debug set-active-expire 0 + r del myhash + r hset myhash field1 value1 + r hpexpire myhash 20 NX 1 field1 + r rename myhash myhash2 + assert_equal [r exists myhash] 0 + assert_range [r hpttl myhash2 1 field1] 1 20 + after 25 + # Verify the renamed key exists + assert_equal [r exists myhash2] 1 + r debug set-active-expire 1 + # Only active expire will delete the key + wait_for_condition 30 10 { [r exists myhash2] == 0 } else { fail "`myhash2` should be expired" } } - after 10 - assert_equal [lsort [r hgetall myhash]] [lsort "f1 f2 f3 v1 v2 v3"] - r debug set-active-expire 1 + test "MOVE to another DB hash with fields to be expired ($type)" { + r select 9 + r flushall + r hset myhash field1 value1 + r hpexpire myhash 100 NX 1 field1 + r move myhash 10 + assert_equal [r exists myhash] 0 + assert_equal [r dbsize] 0 + + # Verify the key and its field exists in the target DB + r select 10 + assert_equal [r hget myhash field1] "value1" + assert_equal [r exists myhash] 1 + + # Eventually the field will be expired and the key will be deleted + wait_for_condition 40 10 { [r hget myhash field1] == "" } else { fail "`field1` should be expired" } + wait_for_condition 40 10 { [r exists myhash] == 0 } else { fail "db should be empty" } + } {} {singledb:skip} + + test "Test COPY hash with fields to be expired ($type)" { + r flushall + r hset h1 f1 v1 f2 v2 + r hset h2 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 f6 v6 f7 v7 f8 v8 f9 v9 f10 v10 f11 v11 f12 v12 f13 v13 f14 v14 f15 v15 f16 v16 f17 v17 f18 v18 + r hpexpire h1 100 NX 1 f1 + r hpexpire h2 100 NX 18 f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12 f13 f14 f15 f16 f17 f18 + r COPY h1 h1copy + r COPY h2 h2copy + assert_equal [r hget h1 f1] "v1" + assert_equal [r hget h1copy f1] "v1" + assert_equal [r exists h2] 1 + assert_equal [r exists h2copy] 1 + after 105 + + # Verify lazy expire of field in h1 and its copy + assert_equal [r hget h1 f1] "" + assert_equal [r hget h1copy f1] "" + + # Verify lazy expire of field in h2 and its copy. Verify the key deleted as well. + wait_for_condition 40 10 { [r exists h2] == 0 } else { fail "`h2` should be expired" } + wait_for_condition 40 10 { [r exists h2copy] == 0 } else { fail "`h2copy` should be expired" } + + } {} {singledb:skip} + + test "Test SWAPDB hash-fields to be expired ($type)" { + r select 9 + r flushall + r hset myhash field1 value1 + r hpexpire myhash 50 NX 1 field1 + + r swapdb 9 10 + + # Verify the key and its field doesn't exist in the source DB + assert_equal [r exists myhash] 0 + assert_equal [r dbsize] 0 + + # Verify the key and its field exists in the target DB + r select 10 + assert_equal [r hget myhash field1] "value1" + assert_equal [r dbsize] 1 + + # Eventually the field will be expired and the key will be deleted + wait_for_condition 20 10 { [r exists myhash] == 0 } else { fail "'myhash' should be expired" } + } {} {singledb:skip} + + test "HPERSIST - input validation ($type)" { + # HPERSIST key + r del myhash + r hset myhash f1 v1 f2 v2 + r hexpire myhash 1000 NX 1 f1 + assert_error {*wrong number of arguments*} {r hpersist myhash} + assert_error {*wrong number of arguments*} {r hpersist myhash 1} + assert_equal [r hpersist not-exists-key 1 f1] {} + assert_equal [r hpersist myhash 2 f1 not-exists-field] "$P_OK $P_NO_FIELD" + assert_equal [r hpersist myhash 1 f2] "$P_NO_EXPIRY" + } + + test "HPERSIST - verify fields with TTL are persisted ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 + r hexpire myhash 20 NX 2 f1 f2 + r hpersist myhash 2 f1 f2 + after 25 + assert_equal [r hget myhash f1] "v1" + assert_equal [r hget myhash f2] "v2" + assert_equal [r HTTL myhash 2 f1 f2] "$T_NO_EXPIRY $T_NO_EXPIRY" + } + + test "HTTL/HPERSIST - Test expiry commands with non-volatile hash ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r httl myhash 1 field1] $T_NO_EXPIRY + assert_equal [r httl myhash 1 fieldnonexist] $E_NO_FIELD + + assert_equal [r hpersist myhash 1 field1] $P_NO_EXPIRY + assert_equal [r hpersist myhash 1 fieldnonexist] $P_NO_FIELD + } + + test "HGETF - input validation ($type)" { + assert_error {*wrong number of arguments*} {r hgetf myhash} + assert_error {*wrong number of arguments*} {r hgetf myhash fields} + assert_error {*wrong number of arguments*} {r hgetf myhash fields 1} + assert_error {*wrong number of arguments*} {r hgetf myhash fields 2 a} + assert_error {*wrong number of arguments*} {r hgetf myhash fields 3 a b} + assert_error {*wrong number of arguments*} {r hgetf myhash fields 3 a b} + assert_error {*unknown argument*} {r hgetf myhash fields 1 a unknown} + assert_error {*missing FIELDS argument*} {r hgetf myhash nx ex 100} + assert_error {*multiple FIELDS argument*} {r hgetf myhash fields 1 a fields 1 b} + + r hset myhash f1 v1 f2 v2 f3 v3 + # NX, XX, GT, and LT can be specified only when EX, PX, EXAT, or PXAT is specified + assert_error {*only when EX, PX, EXAT, or PXAT is specified*} {r hgetf myhash nx fields 1 a} + assert_error {*only when EX, PX, EXAT, or PXAT is specified*} {r hgetf myhash xx fields 1 a} + assert_error {*only when EX, PX, EXAT, or PXAT is specified*} {r hgetf myhash gt fields 1 a} + assert_error {*only when EX, PX, EXAT, or PXAT is specified*} {r hgetf myhash lt fields 1 a} + + # Only one of NX, XX, GT, and LT can be specified + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hgetf myhash nx xx EX 100 fields 1 a} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hgetf myhash xx nx EX 100 fields 1 a} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hgetf myhash gt nx EX 100 fields 1 a} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hgetf myhash gt lt EX 100 fields 1 a} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hgetf myhash xx gt EX 100 fields 1 a} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hgetf myhash lt gt EX 100 fields 1 a} + + # Only one of EX, PX, EXAT, PXAT or PERSIST can be specified + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash EX 100 PX 1000 fields 1 a} + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash EX 100 EXAT 100 fields 1 a} + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash EXAT 100 EX 1000 fields 1 a} + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash EXAT 100 PX 1000 fields 1 a} + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash PX 100 EXAT 100 fields 1 a} + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash PX 100 PXAT 100 fields 1 a} + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash PXAT 100 EX 100 fields 1 a} + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash PXAT 100 EXAT 100 fields 1 a} + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash EX 100 PERSIST fields 1 a} + assert_error {*Only one of EX, PX, EXAT, PXAT or PERSIST arguments*} {r hgetf myhash PERSIST EX 100 fields 1 a} + + # missing expire time + assert_error {*not an integer or out of range*} {r hgetf myhash ex fields 1 a} + assert_error {*not an integer or out of range*} {r hgetf myhash px fields 1 a} + assert_error {*not an integer or out of range*} {r hgetf myhash exat fields 1 a} + assert_error {*not an integer or out of range*} {r hgetf myhash pxat fields 1 a} + + # expire time more than 2 ^ 48 + assert_error {*invalid expire time*} {r hgetf myhash EXAT [expr (1<<48)] 1 f1} + assert_error {*invalid expire time*} {r hgetf myhash PXAT [expr (1<<48)] 1 f1} + assert_error {*invalid expire time*} {r hgetf myhash EX [expr (1<<48) - [clock seconds] + 1000 ] 1 f1} + assert_error {*invalid expire time*} {r hgetf myhash PX [expr (1<<48) - [clock milliseconds] + 1000 ] 1 f1} + + # negative expire time + assert_error {*invalid expire time*} {r hgetf myhash EXAT -10 1 f1} + + # negative field value count + assert_error {*invalid number of fields*} {r hgetf myhash fields -1 a} + } + + test "HGETF - Test 'NX' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetf myhash EX 1000 NX FIELDS 1 field1] [list "value1"] + assert_equal [r hgetf myhash EX 10000 NX FIELDS 2 field1 field2] [list "value1" "value2"] + assert_range [r httl myhash 1 field1] 1 1000 + assert_range [r httl myhash 1 field2] 5000 10000 + + # A field with no expiration is treated as an infinite expiration. + # LT should set the expire time if field has no TTL. + r del myhash + r hset myhash field1 value1 + assert_equal [r hgetf myhash EX 1500 LT FIELDS 1 field1] [list "value1"] + assert_not_equal [r httl myhash 1 field1] "$T_NO_EXPIRY" + } + + test "HGETF - Test 'XX' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetf myhash EX 1000 NX FIELDS 1 field1] [list "value1"] + assert_equal [r hgetf myhash EX 10000 XX FIELDS 2 field1 field2] [list "value1" "value2"] + assert_range [r httl myhash 1 field1] 9900 10000 + assert_equal [r httl myhash 1 field2] "$T_NO_EXPIRY" + } + + test "HGETF - Test 'GT' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetf myhash EX 1000 NX FIELDS 1 field1] [list "value1"] + assert_equal [r hgetf myhash EX 2000 NX FIELDS 1 field2] [list "value2"] + assert_equal [r hgetf myhash EX 1500 GT FIELDS 2 field1 field2] [list "value1" "value2"] + assert_range [r httl myhash 1 field1] 1400 1500 + assert_range [r httl myhash 1 field2] 1900 2000 + } + + test "HGETF - Test 'LT' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetf myhash EX 1000 NX FIELDS 1 field1] [list "value1"] + assert_equal [r hgetf myhash EX 2000 NX FIELDS 1 field2] [list "value2"] + assert_equal [r hgetf myhash EX 1500 LT FIELDS 2 field1 field2] [list "value1" "value2"] + assert_range [r httl myhash 1 field1] 1 1000 + assert_range [r httl myhash 1 field2] 1000 1500 + } + + test "HGETF - Test 'EX' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetf myhash EX 1000 FIELDS 1 field3] [list "value3"] + assert_range [r httl myhash 1 field3] 1 1000 + } + + test "HGETF - Test 'EXAT' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetf myhash EXAT 4000000000 FIELDS 1 field3] [list "value3"] + assert_range [expr [r httl myhash 1 field3] + [clock seconds]] 3900000000 4000000000 + } + + test "HGETF - Test 'PX' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetf myhash PX 1000000 FIELDS 1 field3] [list "value3"] + assert_range [r httl myhash 1 field3] 900 1000 + } + + test "HGETF - Test 'PXAT' flag ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + assert_equal [r hgetf myhash PXAT 4000000000000 FIELDS 1 field3] [list "value3"] + assert_range [expr [r httl myhash 1 field3] + [clock seconds]] 3900000000 4000000000 + } + + test "HGETF - Test 'PERSIST' flag ($type)" { + r del myhash + r debug set-active-expire 0 + + r hset myhash f1 v1 f2 v2 f3 v3 + r hgetf myhash PX 5000 FIELDS 3 f1 f2 f3 + assert_not_equal [r httl myhash 1 f1] "$T_NO_EXPIRY" + assert_not_equal [r httl myhash 1 f2] "$T_NO_EXPIRY" + assert_not_equal [r httl myhash 1 f3] "$T_NO_EXPIRY" + + assert_equal [r hgetf myhash PERSIST FIELDS 1 f1] "v1" + assert_equal [r httl myhash 1 f1] "$T_NO_EXPIRY" + + assert_equal [r hgetf myhash PERSIST FIELDS 2 f2 f3] "v2 v3" + assert_equal [r httl myhash 2 f2 f3] "$T_NO_EXPIRY $T_NO_EXPIRY" + } + + test "HGETF - Test setting expired ttl deletes key ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 + + # hgetf without setting ttl + assert_equal [lsort [r hgetf myhash fields 3 f1 f2 f3]] [lsort "v1 v2 v3"] + assert_equal [r httl myhash 3 f1 f2 f3] "$T_NO_EXPIRY $T_NO_EXPIRY $T_NO_EXPIRY" + + # set expired ttl and verify key is deleted + r hgetf myhash PXAT 1 fields 3 f1 f2 f3 + assert_equal [r exists myhash] 0 + } + + test "HGETF - Test active expiry ($type)" { + r del myhash + r debug set-active-expire 0 + + r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + r hgetf myhash PXAT 1 FIELDS 5 f1 f2 f3 f4 f5 + + r debug set-active-expire 1 + wait_for_condition 50 20 { [r EXISTS myhash] == 0 } else { fail "'myhash' should be expired" } + } + + test "HGETF - A field with TTL overridden with another value (TTL discarded) ($type)" { + r del myhash + r hset myhash field1 value1 field2 value2 field3 value3 + r hgetf myhash PX 10000 NX FIELDS 1 field1 + r hgetf myhash EX 100 NX FIELDS 1 field2 + + # field2 TTL will be discarded + r hset myhash field2 value4 + + # Expected TTL will be discarded + assert_equal [r hget myhash field2] "value4" + assert_equal [r httl myhash 2 field2 field3] "$T_NO_EXPIRY $T_NO_EXPIRY" + + # Other field is not affected. + assert_not_equal [r httl myhash 1 field1] "$T_NO_EXPIRY" + } + + test "HSETF - input validation ($type)" { + assert_error {*wrong number of arguments*} {r hsetf myhash} + assert_error {*wrong number of arguments*} {r hsetf myhash fvs} + assert_error {*wrong number of arguments*} {r hsetf myhash fvs 1} + assert_error {*wrong number of arguments*} {r hsetf myhash fvs 2 a b} + assert_error {*wrong number of arguments*} {r hsetf myhash fvs 3 a b c d} + assert_error {*wrong number of arguments*} {r hsetf myhash fvs 3 a b} + assert_error {*unknown argument*} {r hsetf myhash fvs 1 a b unknown} + assert_error {*missing FVS argument*} {r hsetf myhash nx nx ex 100} + assert_error {*multiple FVS argument*} {r hsetf myhash DC fvs 1 a b fvs 1 a b} + + r hset myhash f1 v1 f2 v2 f3 v3 + # NX, XX, GT, and LT can be specified only when EX, PX, EXAT, or PXAT is specified + assert_error {*only when EX, PX, EXAT, or PXAT is specified*} {r hsetf myhash nx fvs 1 a b} + assert_error {*only when EX, PX, EXAT, or PXAT is specified*} {r hsetf myhash xx fvs 1 a b} + assert_error {*only when EX, PX, EXAT, or PXAT is specified*} {r hsetf myhash gt fvs 1 a b} + assert_error {*only when EX, PX, EXAT, or PXAT is specified*} {r hsetf myhash lt fvs 1 a b} + + # Only one of NX, XX, GT, and LT can be specified + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hsetf myhash nx xx EX 100 fvs 1 a b} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hsetf myhash xx nx EX 100 fvs 1 a b} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hsetf myhash gt nx EX 100 fvs 1 a b} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hsetf myhash gt lt EX 100 fvs 1 a b} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hsetf myhash xx gt EX 100 fvs 1 a b} + assert_error {*Only one of NX, XX, GT, and LT arguments*} {r hsetf myhash lt gt EX 100 fvs 1 a b} + + # Only one of EX, PX, EXAT, PXAT or KEEPTTL can be specified + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash EX 100 PX 1000 fvs 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash EX 100 EXAT 100 fvs 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash EXAT 100 EX 1000 fvs 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash EXAT 100 PX 1000 fvs 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash PX 100 EXAT 100 fvs 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash PX 100 PXAT 100 fvs 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash PXAT 100 EX 100 fvs 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash PXAT 100 EXAT 100 fvs 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash EX 100 KEEPTTL fvs 1 a b} + assert_error {*Only one of EX, PX, EXAT, PXAT or KEEPTTL arguments*} {r hsetf myhash KEEPTTL EX 100 fvs 1 a b} + + # Only one of DCF, DOF can be specified + assert_error {*Only one of DCF or DOF arguments can be specified*} {r hsetf myhash DCF DOF fvs 1 a b} + assert_error {*Only one of DCF or DOF arguments can be specified*} {r hsetf myhash DOF DCF fvs 1 a b} + + # Only one of GETNEW, GETOLD can be specified + assert_error {*Only one of GETOLD or GETNEW arguments can be specified*} {r hsetf myhash GETNEW GETOLD fvs 1 a b} + assert_error {*Only one of GETOLD or GETNEW arguments can be specified*} {r hsetf myhash GETOLD GETNEW fvs 1 a b} + + # missing expire time + assert_error {*not an integer or out of range*} {r hsetf myhash ex fvs 1 a b} + assert_error {*not an integer or out of range*} {r hsetf myhash px fvs 1 a b} + assert_error {*not an integer or out of range*} {r hsetf myhash exat fvs 1 a b} + assert_error {*not an integer or out of range*} {r hsetf myhash pxat fvs 1 a b} + + # expire time more than 2 ^ 48 + assert_error {*invalid expire time*} {r hsetf myhash EXAT [expr (1<<48)] 1 a b} + assert_error {*invalid expire time*} {r hsetf myhash PXAT [expr (1<<48)] 1 a b} + assert_error {*invalid expire time*} {r hsetf myhash EX [expr (1<<48) - [clock seconds] + 1000 ] 1 a b} + assert_error {*invalid expire time*} {r hsetf myhash PX [expr (1<<48) - [clock milliseconds] + 1000 ] 1 a b} + + # negative ttl + assert_error {*invalid expire time*} {r hsetf myhash EXAT -1 1 a b} + + # negative field value count + assert_error {*invalid number of fvs count*} {r hsetf myhash fvs -1 a b} + } + + test "HSETF - Test DC flag ($type)" { + r del myhash + # don't create key + assert_equal "" [r hsetf myhash DC fvs 1 a b] + } + + test "HSETF - Test DCF/DOF flag ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 + + # Don't overwrite fields + assert_equal [r hsetf myhash DOF fvs 2 f1 n1 f2 n2] "$S_FAIL $S_FAIL" + assert_equal [r hsetf myhash DOF fvs 3 f1 n1 f2 b2 f4 v4] "$S_FAIL $S_FAIL $S_FIELD" + assert_equal [lsort [r hgetall myhash]] [lsort "f1 v1 f2 v2 f3 v3 f4 v4"] + + # Don't create fields + assert_equal [r hsetf myhash DCF fvs 3 f1 n1 f2 b2 f5 v5] "$S_FIELD $S_FIELD $S_FAIL" + assert_equal [lsort [r hgetall myhash]] [lsort "f1 n1 f2 b2 f3 v3 f4 v4"] + } + + test "HSETF - Test 'NX' flag ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 + assert_equal [r hsetf myhash EX 1000 NX FVS 1 f1 n1] "$S_FIELD_AND_TTL" + assert_equal [r hsetf myhash EX 10000 NX FVS 2 f1 n1 f2 n2] "$S_FIELD $S_FIELD_AND_TTL" + assert_range [r httl myhash 1 f1] 990 1000 + assert_range [r httl myhash 1 f2] 9990 10000 + } + + test "HSETF - Test 'XX' flag ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 + assert_equal [r hsetf myhash EX 1000 NX FVS 1 f1 n1] "$S_FIELD_AND_TTL" + assert_equal [r hsetf myhash EX 10000 XX FVS 2 f1 n1 f2 n2] "$S_FIELD_AND_TTL $S_FIELD" + assert_range [r httl myhash 1 f1] 9900 10000 + assert_equal [r httl myhash 1 f2] "$T_NO_EXPIRY" + } + + test "HSETF - Test 'GT' flag ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 + assert_equal [r hsetf myhash EX 1000 NX FVS 1 f1 n1] "$S_FIELD_AND_TTL" + assert_equal [r hsetf myhash EX 2000 NX FVS 1 f2 n2] "$S_FIELD_AND_TTL" + assert_equal [r hsetf myhash EX 1500 GT FVS 2 f1 n1 f2 n2] "$S_FIELD_AND_TTL $S_FIELD" + assert_range [r httl myhash 1 f1] 1400 1500 + assert_range [r httl myhash 1 f2] 1600 2000 + } + + test "HSETF - Test 'LT' flag ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 + assert_equal [r hsetf myhash EX 1000 NX FVS 1 f1 v1] "$S_FIELD_AND_TTL" + assert_equal [r hsetf myhash EX 2000 NX FVS 1 f2 v2] "$S_FIELD_AND_TTL" + assert_equal [r hsetf myhash EX 1500 LT FVS 2 f1 v1 f2 v2] "$S_FIELD $S_FIELD_AND_TTL" + assert_range [r httl myhash 1 f1] 900 1000 + assert_range [r httl myhash 1 f2] 1400 1500 + + # A field with no expiration is treated as an infinite expiration. + # LT should set the expire time if field has no TTL. + r del myhash + r hset myhash f1 v1 + assert_equal [r hsetf myhash EX 1500 LT FVS 1 f1 v1] "$S_FIELD_AND_TTL" + } + + test "HSETF - Test 'EX' flag ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 + assert_equal [r hsetf myhash EX 1000 FVS 1 f3 v3 ] "$S_FIELD_AND_TTL" + assert_range [r httl myhash 1 f3] 900 1000 + } + + test "HSETF - Test 'EXAT' flag ($type)" { + r del myhash + r hset myhash f1 v1 f2 v2 + assert_equal [r hsetf myhash EXAT 4000000000 FVS 1 f3 v3] "$S_FIELD_AND_TTL" + assert_range [expr [r httl myhash 1 f3] + [clock seconds]] 3900000000 4000000000 + } + + test "HSETF - Test 'PX' flag ($type)" { + r del myhash + assert_equal [r hsetf myhash PX 1000000 FVS 1 f3 v3] "$S_FIELD_AND_TTL" + assert_range [r httl myhash 1 f3] 990 1000 + } + + test "HSETF - Test 'PXAT' flag ($type)" { + r del myhash + r hset myhash f1 v2 f2 v2 f3 v3 + assert_equal [r hsetf myhash PXAT 4000000000000 FVS 1 f2 v2] "$S_FIELD_AND_TTL" + assert_range [expr [r httl myhash 1 f2] + [clock seconds]] 3900000000 4000000000 + } + + test "HSETF - Test 'KEEPTTL' flag ($type)" { + r del myhash + + r hsetf myhash FVS 2 f1 v1 f2 v2 + r hsetf myhash PX 5000 FVS 1 f2 v2 + + # f1 does not have ttl + assert_equal [r httl myhash 1 f1] "$T_NO_EXPIRY" + + # f2 has ttl + assert_not_equal [r httl myhash 1 f2] "$T_NO_EXPIRY" + + # Validate KEEPTTL preserve TTL + assert_equal [r hsetf myhash KEEPTTL FVS 1 f2 n2] "$S_FIELD" + assert_not_equal [r httl myhash 1 f2] "$T_NO_EXPIRY" + assert_equal [r hget myhash f2] "n2" + } + + test "HSETF - Test no expiry flag discards TTL ($type)" { + r del myhash + + r hsetf myhash FVS 1 f1 v1 + r hsetf myhash PX 5000 FVS 1 f2 v2 + + assert_equal [r hsetf myhash FVS 2 f1 v1 f2 v2] "$S_FIELD $S_FIELD_AND_TTL" + assert_not_equal [r httl myhash 1 f1 f2] "$T_NO_EXPIRY $T_NO_EXPIRY" + } + + test "HSETF - Test 'GETNEW/GETOLD' flag ($type)" { + r del myhash + + assert_equal [r hsetf myhash GETOLD fvs 2 f1 v1 f2 v2] "{} {}" + assert_equal [r hsetf myhash GETNEW fvs 2 f1 v1 f2 v2] "v1 v2" + assert_equal [r hsetf myhash GETOLD fvs 2 f1 n1 f2 n2] "v1 v2" + assert_equal [r hsetf myhash GETOLD DOF fvs 2 f1 n1 f2 n2] "n1 n2" + assert_equal [r hsetf myhash GETNEW DOF fvs 2 f1 n1 f2 n2] "n1 n2" + assert_equal [r hsetf myhash GETNEW DCF fvs 2 f1 x1 f2 x2] "x1 x2" + assert_equal [r hsetf myhash GETNEW DCF fvs 2 f4 x4 f5 x5] "{} {}" + + r del myhash + assert_equal [r hsetf myhash GETOLD fvs 2 f1 v1 f2 v2] "{} {}" + + # DOF check will prevent override and GETNEW should return old value + assert_equal [r hsetf myhash DOF GETNEW fvs 2 f1 v12 f2 v22] "v1 v2" + } + + test "HSETF - Test with active expiry" { + r del myhash + r debug set-active-expire 0 + + r hsetf myhash PX 10 FVS 5 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + r debug set-active-expire 1 + wait_for_condition 50 20 { [r EXISTS myhash] == 0 } else { fail "'myhash' should be expired" } + } + + test "HSETF - Set time in the past ($type)" { + r del myhash + assert_equal [r hsetf myhash EXAT [expr {[clock seconds] - 1}] FVS 2 f1 v1 f2 v2] "$S_FIELD_AND_TTL $S_FIELD_AND_TTL" + assert_equal [r hexists myhash field1] 0 + + # Try with override + r hset myhash fvs 2 f1 v1 f2 v2 + assert_equal [r hsetf myhash EXAT [expr {[clock seconds] - 1}] FVS 2 f1 v1 f2 v2] "$S_FIELD_AND_TTL $S_FIELD_AND_TTL" + assert_equal [r hexists myhash field1] 0 + } + + test "HSETF - Test failed hsetf call should not leave empty key ($type)" { + r del myhash + # This should not create the field as DCF flag is given + assert_equal [r hsetf myhash DCF FVS 1 a b] "" + + # Key should not exist + assert_equal [r exists myhash] 0 + + # Try with GETNEW/GETOLD + assert_equal [r hsetf myhash GETNEW DCF FVS 1 a b] "" + assert_equal [r exists myhash] 0 + assert_equal [r hsetf myhash GETOLD DCF FVS 1 a b] "" + assert_equal [r exists myhash] 0 + } } - test {Test RENAME hash with fields to be expired} { - r debug set-active-expire 0 - r del myhash - r hset myhash field1 value1 - r hpexpire myhash 20 NX 1 field1 - r rename myhash myhash2 - assert_equal [r exists myhash] 0 - assert_range [r hpttl myhash2 1 field1] 1 20 - after 25 - # Verify the renamed key exists - assert_equal [r exists myhash2] 1 - r debug set-active-expire 1 - # Only active expire will delete the key - wait_for_condition 30 10 { [r exists myhash2] == 0 } else { fail "`myhash2` should be expired" } - } - - test {MOVE to another DB hash with fields to be expired} { - r select 9 - r flushall - r hset myhash field1 value1 - r hpexpire myhash 100 NX 1 field1 - r move myhash 10 - assert_equal [r exists myhash] 0 - assert_equal [r dbsize] 0 - - # Verify the key and its field exists in the target DB - r select 10 - assert_equal [r hget myhash field1] "value1" - assert_equal [r exists myhash] 1 - - # Eventually the field will be expired and the key will be deleted - wait_for_condition 40 10 { [r hget myhash field1] == "" } else { fail "`field1` should be expired" } - wait_for_condition 40 10 { [r exists myhash] == 0 } else { fail "db should be empty" } - } {} {singledb:skip} - - test {Test COPY hash with fields to be expired} { - r flushall - r hset h1 f1 v1 f2 v2 - r hset h2 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 f6 v6 f7 v7 f8 v8 f9 v9 f10 v10 f11 v11 f12 v12 f13 v13 f14 v14 f15 v15 f16 v16 f17 v17 f18 v18 - r hpexpire h1 100 NX 1 f1 - r hpexpire h2 100 NX 18 f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12 f13 f14 f15 f16 f17 f18 - r COPY h1 h1copy - r COPY h2 h2copy - assert_equal [r hget h1 f1] "v1" - assert_equal [r hget h1copy f1] "v1" - assert_equal [r exists h2] 1 - assert_equal [r exists h2copy] 1 - after 105 - - # Verify lazy expire of field in h1 and its copy - assert_equal [r hget h1 f1] "" - assert_equal [r hget h1copy f1] "" - - # Verify lazy expire of field in h2 and its copy. Verify the key deleted as well. - wait_for_condition 40 10 { [r exists h2] == 0 } else { fail "`h2` should be expired" } - wait_for_condition 40 10 { [r exists h2copy] == 0 } else { fail "`h2copy` should be expired" } - - } {} {singledb:skip} - - test {Test SWAPDB hash-fields to be expired} { - r select 9 - r flushall - r hset myhash field1 value1 - r hpexpire myhash 50 NX 1 field1 - - r swapdb 9 10 - - # Verify the key and its field doesn't exist in the source DB - assert_equal [r exists myhash] 0 - assert_equal [r dbsize] 0 - - # Verify the key and its field exists in the target DB - r select 10 - assert_equal [r hget myhash field1] "value1" - assert_equal [r dbsize] 1 - - # Eventually the field will be expired and the key will be deleted - wait_for_condition 20 10 { [r exists myhash] == 0 } else { fail "'myhash' should be expired" } - } {} {singledb:skip} - - test {HPERSIST - input validation} { - # HPERSIST key - r del myhash - r hset myhash f1 v1 f2 v2 - r hexpire myhash 1000 NX 1 f1 - assert_error {*wrong number of arguments*} {r hpersist myhash} - assert_error {*wrong number of arguments*} {r hpersist myhash 1} - assert_equal [r hpersist not-exists-key 1 f1] {} - assert_equal [r hpersist myhash 2 f1 not-exists-field] "$P_OK $P_NO_FIELD" - assert_equal [r hpersist myhash 1 f2] "$P_NO_EXPIRY" - } - - test {HPERSIST - verify fields with TTL are persisted} { - r del myhash - r hset myhash f1 v1 f2 v2 - r hexpire myhash 20 NX 2 f1 f2 - r hpersist myhash 2 f1 f2 - after 25 - assert_equal [r hget myhash f1] "v1" - assert_equal [r hget myhash f2] "v2" - assert_equal [r HTTL myhash 2 f1 f2] "$T_NO_EXPIRY $T_NO_EXPIRY" - } - r config set hash-max-listpack-entries 1 + r config set hash-max-listpack-entries 512 +} + +start_server {tags {"external:skip needs:debug"}} { + + # Tests that only applies to listpack + + test "Test listpack memory usage" { + r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + r hpexpire myhash 5 2 f2 f4 + + # Just to have code coverage for the new listpack encoding + r memory usage myhash + } + + test "Test listpack object encoding" { + r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + r hpexpire myhash 5 2 f2 f4 + + # Just to have code coverage for the listpackex encoding + assert_equal [r object encoding myhash] "listpackex" + } + + test "Test listpack debug listpack" { + r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + + # Just to have code coverage for the listpackex encoding + r debug listpack myhash + } + + test "Test listpack converts to ht and passive expiry works" { + set prev [lindex [r config get hash-max-listpack-entries] 1] + r config set hash-max-listpack-entries 10 + r debug set-active-expire 0 + + r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + r hpexpire myhash 5 2 f2 f4 + + for {set i 6} {$i < 11} {incr i} { + r hset myhash f$i v$i + } + after 50 + assert_equal [lsort [r hgetall myhash]] [lsort "f1 f3 f5 f6 f7 f8 f9 f10 v1 v3 v5 v6 v7 v8 v9 v10"] + r config set hash-max-listpack-entries $prev + r debug set-active-expire 1 + } + + test "Test listpack converts to ht and active expiry works" { + r del myhash + r debug set-active-expire 0 + + r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 + r hpexpire myhash 10 1 f1 + + for {set i 0} {$i < 2048} {incr i} { + r hset myhash f$i v$i + } + + for {set i 0} {$i < 2048} {incr i} { + r hpexpire myhash 10 1 f$i + } + + r debug set-active-expire 1 + wait_for_condition 50 20 { [r EXISTS myhash] == 0 } else { fail "'myhash' should be expired" } + } + + test "HSETF - Test listpack converts to ht" { + r del myhash + r debug set-active-expire 0 + + # Check expiry works after listpack converts ht by using hsetf + for {set i 0} {$i < 1024} {incr i} { + r hsetf myhash PX 10 FVS 3 a$i b$i c$i d$i e$i f$i + } + + r debug set-active-expire 1 + wait_for_condition 50 20 { [r EXISTS myhash] == 0 } else { fail "'myhash' should be expired" } + } + + test "HPERSIST/HEXPIRE - Test listpack with large values" { + r del myhash + + # Test with larger values to verify we successfully move fields in + # listpack when we are ordering according to TTL. This config change + # will make code to use temporary heap allocation when moving fields. + # See listpackExUpdateExpiry() for details. + r config set hash-max-listpack-value 2048 + + set payload1 [string repeat v3 1024] + set payload2 [string repeat v1 1024] + + # Test with single item list + r hset myhash f1 $payload1 + assert_equal [r hgetf myhash EX 2000 FIELDS 1 f1] $payload1 + r del myhash + + # Test with multiple items + r hset myhash f1 $payload2 f2 v2 f3 $payload1 f4 v4 + r hexpire myhash 100000 1 f3 + r hpersist myhash 1 f3 + assert_equal [r hpersist myhash 1 f3] $P_NO_EXPIRY + + r hpexpire myhash 10 1 f1 + after 20 + assert_equal [lsort [r hgetall myhash]] [lsort "f2 f3 f4 v2 $payload1 v4"] + + r config set hash-max-listpack-value 64 + } }