From ecdcce0d43916e5f33b04d56f0f9bb43002377a5 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 6 Jan 2025 10:46:47 +0200 Subject: [PATCH 01/33] Introduce HASH items expiration Signed-off-by: Ran Shidlansik --- cmake/Modules/SourceFiles.cmake | 2 + src/Makefile | 2 +- src/aof.c | 24 +- src/commands.def | 328 ++++++++++++ src/commands/hexpire.json | 121 +++++ src/commands/hexpireat.json | 120 +++++ src/commands/hexpiretime.json | 83 +++ src/commands/hpersist.json | 84 ++++ src/commands/hpexpire.json | 120 +++++ src/commands/hpexpireat.json | 120 +++++ src/commands/hpexpiretime.json | 83 +++ src/commands/hpttl.json | 83 +++ src/commands/httl.json | 83 +++ src/db.c | 113 +++-- src/expire.c | 15 +- src/hashtable.c | 35 +- src/hashtable.h | 10 + src/module.c | 4 +- src/object.c | 110 ++++ src/rdb.c | 48 +- src/rdb.h | 1 + src/sds.c | 7 +- src/server.c | 26 + src/server.h | 86 +++- src/t_hash.c | 864 +++++++++++++++++++++++++++----- src/t_string.c | 133 +---- src/volatile_set.c | 79 +++ src/volatile_set.h | 41 ++ tests/unit/expire.tcl | 37 +- 29 files changed, 2530 insertions(+), 332 deletions(-) create mode 100644 src/commands/hexpire.json create mode 100644 src/commands/hexpireat.json create mode 100644 src/commands/hexpiretime.json create mode 100644 src/commands/hpersist.json create mode 100644 src/commands/hpexpire.json create mode 100644 src/commands/hpexpireat.json create mode 100644 src/commands/hpexpiretime.json create mode 100644 src/commands/hpttl.json create mode 100644 src/commands/httl.json create mode 100644 src/volatile_set.c create mode 100644 src/volatile_set.h diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake index 298bcaeab2e..aa7158ee475 100644 --- a/cmake/Modules/SourceFiles.cmake +++ b/cmake/Modules/SourceFiles.cmake @@ -110,6 +110,8 @@ set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/unix.c ${CMAKE_SOURCE_DIR}/src/server.c ${CMAKE_SOURCE_DIR}/src/logreqres.c) + ${CMAKE_SOURCE_DIR}/src/volatile_set.c) + # valkey-cli set(VALKEY_CLI_SRCS diff --git a/src/Makefile b/src/Makefile index ac90ee2877d..e6a4058086a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -417,7 +417,7 @@ endif ENGINE_NAME=valkey SERVER_NAME=$(ENGINE_NAME)-server$(PROG_SUFFIX) ENGINE_SENTINEL_NAME=$(ENGINE_NAME)-sentinel$(PROG_SUFFIX) -ENGINE_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o hashtable.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o memory_prefetch.o io_threads.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o cluster_slot_stats.o crc16.o endianconv.o commandlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crccombine.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o valkey-check-rdb.o valkey-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o allocator_defrag.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script.o functions.o commands.o strl.o connection.o unix.o logreqres.o rdma.o scripting_engine.o lua/script_lua.o lua/function_lua.o lua/engine_lua.o lua/debug_lua.o +ENGINE_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o hashtable.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o memory_prefetch.o io_threads.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o cluster_slot_stats.o crc16.o endianconv.o commandlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crccombine.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o valkey-check-rdb.o valkey-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o allocator_defrag.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script.o functions.o commands.o strl.o connection.o unix.o logreqres.o rdma.o scripting_engine.o volatile_set.o lua/script_lua.o lua/function_lua.o lua/engine_lua.o lua/debug_lua.o ENGINE_CLI_NAME=$(ENGINE_NAME)-cli$(PROG_SUFFIX) ENGINE_CLI_OBJ=anet.o adlist.o dict.o valkey-cli.o zmalloc.o release.o ae.o serverassert.o crcspeed.o crccombine.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o cli_commands.o sds.o util.o sha256.o ENGINE_BENCHMARK_NAME=$(ENGINE_NAME)-benchmark$(PROG_SUFFIX) diff --git a/src/aof.c b/src/aof.c index d4c2a523c23..1fb5c2aee35 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1949,7 +1949,7 @@ static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) { * The function returns 0 on error, 1 on success. */ int rewriteHashObject(rio *r, robj *key, robj *o) { hashTypeIterator hi; - long long count = 0, items = hashTypeLength(o); + long long count = 0, volatile_items = hashTypeNumVolatileElements(o), items = hashTypeLength(o) - volatile_items; hashTypeInitIterator(o, &hi); while (hashTypeNext(&hi) != C_ERR) { @@ -1963,6 +1963,9 @@ int rewriteHashObject(rio *r, robj *key, robj *o) { } } + if (volatile_items > 0 && hashTypeEntryHasExpire(hi.next)) + continue; + if (!rioWriteHashIteratorCursor(r, &hi, OBJ_HASH_FIELD) || !rioWriteHashIteratorCursor(r, &hi, OBJ_HASH_VALUE)) { hashTypeResetIterator(&hi); return 0; @@ -1973,6 +1976,25 @@ int rewriteHashObject(rio *r, robj *key, robj *o) { hashTypeResetIterator(&hi); + /* Now serialize volatile items if exist */ + if (hashTypeHasVolatileElements(o)) { + hashTypeInitVolatileIterator(o, &hi); + while (hashTypeNext(&hi) != C_ERR) { + long long expiry = hashTypeEntryGetExpiry(hi.next); + sds field = hashTypeEntryGetField(hi.next); + sds value = hashTypeEntryGetValue(hi.next); + if (rioWriteBulkCount(r, '*', 8) == 0) return 0; + if (rioWriteBulkString(r, "HSETEX", 6) == 0) return 0; + if (rioWriteBulkObject(r, key) == 0) return 0; + if (rioWriteBulkString(r, "PXAT", 4) == 0) return 0; + if (rioWriteBulkLongLong(r, expiry) == 0) return 0; + if (rioWriteBulkString(r, "FIELDS", 6) == 0) return 0; + if (rioWriteBulkLongLong(r, 1) == 0) return 0; + if (rioWriteBulkString(r, field, sdslen(field)) == 0) return 0; + if (rioWriteBulkString(r, value, sdslen(value)) == 0) return 0; + } + hashTypeResetIterator(&hi); + } return 1; } diff --git a/src/commands.def b/src/commands.def index f6066871972..3a416fe93da 100644 --- a/src/commands.def +++ b/src/commands.def @@ -3464,6 +3464,119 @@ struct COMMAND_ARG HEXISTS_Args[] = { {MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; +/********** HEXPIRE ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HEXPIRE history */ +#define HEXPIRE_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HEXPIRE tips */ +#define HEXPIRE_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HEXPIRE key specs */ +keySpec HEXPIRE_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HEXPIRE condition argument table */ +struct COMMAND_ARG HEXPIRE_condition_Subargs[] = { +{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HEXPIRE fields argument table */ +struct COMMAND_ARG HEXPIRE_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HEXPIRE argument table */ +struct COMMAND_ARG HEXPIRE_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("seconds",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,"9.0.0",CMD_ARG_OPTIONAL,4,NULL),.subargs=HEXPIRE_condition_Subargs}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HEXPIRE_fields_Subargs}, +}; + +/********** HEXPIREAT ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HEXPIREAT history */ +#define HEXPIREAT_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HEXPIREAT tips */ +#define HEXPIREAT_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HEXPIREAT key specs */ +keySpec HEXPIREAT_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HEXPIREAT condition argument table */ +struct COMMAND_ARG HEXPIREAT_condition_Subargs[] = { +{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HEXPIREAT fields argument table */ +struct COMMAND_ARG HEXPIREAT_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HEXPIREAT argument table */ +struct COMMAND_ARG HEXPIREAT_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-seconds",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,"9.0.0",CMD_ARG_OPTIONAL,4,NULL),.subargs=HEXPIREAT_condition_Subargs}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HEXPIREAT_fields_Subargs}, +}; + +/********** HEXPIRETIME ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HEXPIRETIME history */ +#define HEXPIRETIME_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HEXPIRETIME tips */ +#define HEXPIRETIME_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HEXPIRETIME key specs */ +keySpec HEXPIRETIME_Keyspecs[1] = { +{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HEXPIRETIME fields argument table */ +struct COMMAND_ARG HEXPIRETIME_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HEXPIRETIME argument table */ +struct COMMAND_ARG HEXPIRETIME_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HEXPIRETIME_fields_Subargs}, +}; + /********** HGET ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -3673,6 +3786,181 @@ struct COMMAND_ARG HMSET_Args[] = { {MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HMSET_data_Subargs}, }; +/********** HPERSIST ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HPERSIST history */ +#define HPERSIST_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HPERSIST tips */ +#define HPERSIST_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HPERSIST key specs */ +keySpec HPERSIST_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HPERSIST fields argument table */ +struct COMMAND_ARG HPERSIST_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HPERSIST argument table */ +struct COMMAND_ARG HPERSIST_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HPERSIST_fields_Subargs}, +}; + +/********** HPEXPIRE ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HPEXPIRE history */ +#define HPEXPIRE_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HPEXPIRE tips */ +#define HPEXPIRE_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HPEXPIRE key specs */ +keySpec HPEXPIRE_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HPEXPIRE condition argument table */ +struct COMMAND_ARG HPEXPIRE_condition_Subargs[] = { +{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HPEXPIRE fields argument table */ +struct COMMAND_ARG HPEXPIRE_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HPEXPIRE argument table */ +struct COMMAND_ARG HPEXPIRE_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,"9.0.0",CMD_ARG_OPTIONAL,4,NULL),.subargs=HPEXPIRE_condition_Subargs}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HPEXPIRE_fields_Subargs}, +}; + +/********** HPEXPIREAT ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HPEXPIREAT history */ +#define HPEXPIREAT_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HPEXPIREAT tips */ +#define HPEXPIREAT_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HPEXPIREAT key specs */ +keySpec HPEXPIREAT_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HPEXPIREAT condition argument table */ +struct COMMAND_ARG HPEXPIREAT_condition_Subargs[] = { +{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HPEXPIREAT fields argument table */ +struct COMMAND_ARG HPEXPIREAT_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HPEXPIREAT argument table */ +struct COMMAND_ARG HPEXPIREAT_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,"9.0.0",CMD_ARG_OPTIONAL,4,NULL),.subargs=HPEXPIREAT_condition_Subargs}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HPEXPIREAT_fields_Subargs}, +}; + +/********** HPEXPIRETIME ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HPEXPIRETIME history */ +#define HPEXPIRETIME_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HPEXPIRETIME tips */ +#define HPEXPIRETIME_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HPEXPIRETIME key specs */ +keySpec HPEXPIRETIME_Keyspecs[1] = { +{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HPEXPIRETIME fields argument table */ +struct COMMAND_ARG HPEXPIRETIME_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HPEXPIRETIME argument table */ +struct COMMAND_ARG HPEXPIRETIME_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HPEXPIRETIME_fields_Subargs}, +}; + +/********** HPTTL ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HPTTL history */ +#define HPTTL_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HPTTL tips */ +#define HPTTL_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HPTTL key specs */ +keySpec HPTTL_Keyspecs[1] = { +{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HPTTL fields argument table */ +struct COMMAND_ARG HPTTL_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HPTTL argument table */ +struct COMMAND_ARG HPTTL_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HPTTL_fields_Subargs}, +}; + /********** HRANDFIELD ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -3820,6 +4108,37 @@ struct COMMAND_ARG HSTRLEN_Args[] = { {MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; +/********** HTTL ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HTTL history */ +#define HTTL_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HTTL tips */ +#define HTTL_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HTTL key specs */ +keySpec HTTL_Keyspecs[1] = { +{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HTTL fields argument table */ +struct COMMAND_ARG HTTL_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HTTL argument table */ +struct COMMAND_ARG HTTL_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HTTL_fields_Subargs}, +}; + /********** HVALS ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -11147,6 +11466,9 @@ struct COMMAND_STRUCT serverCommandTable[] = { /* hash */ {MAKE_CMD("hdel","Deletes one or more fields and their values from a hash. Deletes the hash if no fields remain.","O(N) where N is the number of fields to be removed.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HDEL_History,1,HDEL_Tips,0,hdelCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HDEL_Keyspecs,1,NULL,2),.args=HDEL_Args}, {MAKE_CMD("hexists","Determines whether a field exists in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXISTS_History,0,HEXISTS_Tips,0,hexistsCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXISTS_Keyspecs,1,NULL,2),.args=HEXISTS_Args}, +{MAKE_CMD("hexpire","Set expiry time on hash object.","O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRE_History,0,HEXPIRE_Tips,0,hexpireCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRE_Keyspecs,1,NULL,4),.args=HEXPIRE_Args}, +{MAKE_CMD("hexpireat","Set expiry time on hash object.","O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIREAT_History,0,HEXPIREAT_Tips,0,hexpireAtCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIREAT_Keyspecs,1,NULL,4),.args=HEXPIREAT_Args}, +{MAKE_CMD("hexpiretime","Returns the Unix timestamp in seconds since Unix epoch at which the given key's field(s) will expire","O(1) for each field, so O(N) for N items when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRETIME_Keyspecs,1,NULL,2),.args=HEXPIRETIME_Args}, {MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HGET_Keyspecs,1,NULL,2),.args=HGET_Args}, {MAKE_CMD("hgetall","Returns all fields and values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETALL_History,0,HGETALL_Tips,1,hgetallCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HGETALL_Keyspecs,1,NULL,1),.args=HGETALL_Args}, {MAKE_CMD("hincrby","Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBY_History,0,HINCRBY_Tips,0,hincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBY_Keyspecs,1,NULL,3),.args=HINCRBY_Args}, @@ -11155,11 +11477,17 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("hlen","Returns the number of fields in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HLEN_History,0,HLEN_Tips,0,hlenCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HLEN_Keyspecs,1,NULL,1),.args=HLEN_Args}, {MAKE_CMD("hmget","Returns the values of all fields in a hash.","O(N) where N is the number of fields being requested.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HMGET_History,0,HMGET_Tips,0,hmgetCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HMGET_Keyspecs,1,NULL,2),.args=HMGET_Args}, {MAKE_CMD("hmset","Sets the values of multiple fields.","O(N) where N is the number of fields being set.","2.0.0",CMD_DOC_DEPRECATED,"`HSET` with multiple field-value pairs","4.0.0","hash",COMMAND_GROUP_HASH,HMSET_History,0,HMSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HMSET_Keyspecs,1,NULL,2),.args=HMSET_Args}, +{MAKE_CMD("hpersist","Remove the existing expiration on a hash key's field(s).","O(1) for each field assigned with TTL, so O(N) to persist N items when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPERSIST_History,0,HPERSIST_Tips,0,hpersistCommand,-3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HPERSIST_Keyspecs,1,NULL,2),.args=HPERSIST_Args}, +{MAKE_CMD("hpexpire","Set expiry time on hash object.","O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRE_History,0,HPEXPIRE_Tips,0,hpexpireCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRE_Keyspecs,1,NULL,4),.args=HPEXPIRE_Args}, +{MAKE_CMD("hpexpireat","Set expiry time on hash object.","O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIREAT_History,0,HPEXPIREAT_Tips,0,hpexpireAtCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIREAT_Keyspecs,1,NULL,4),.args=HPEXPIREAT_Args}, +{MAKE_CMD("hpexpiretime","Returns the Unix timestamp in milliseconds since Unix epoch at which the given key's field(s) will expire","O(1) for each field, so O(N) for N items when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRETIME_History,0,HPEXPIRETIME_Tips,0,hpexpiretimeCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRETIME_Keyspecs,1,NULL,2),.args=HPEXPIRETIME_Args}, +{MAKE_CMD("hpttl","Returns the remaining time to live (in milliseconds) of a hash key's field(s) that have an associated expiration.","O(1) for each field assigned with TTL, so O(N) for N items when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPTTL_History,0,HPTTL_Tips,0,hpttlCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HPTTL_Keyspecs,1,NULL,2),.args=HPTTL_Args}, {MAKE_CMD("hrandfield","Returns one or more random fields from a hash.","O(N) where N is the number of fields returned","6.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HRANDFIELD_History,0,HRANDFIELD_Tips,1,hrandfieldCommand,-2,CMD_READONLY,ACL_CATEGORY_HASH,HRANDFIELD_Keyspecs,1,NULL,2),.args=HRANDFIELD_Args}, {MAKE_CMD("hscan","Iterates over fields and values of a hash.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSCAN_History,0,HSCAN_Tips,1,hscanCommand,-3,CMD_READONLY,ACL_CATEGORY_HASH,HSCAN_Keyspecs,1,NULL,5),.args=HSCAN_Args}, {MAKE_CMD("hset","Creates or modifies the value of a field in a hash.","O(1) for each field/value pair added, so O(N) to add N field/value pairs when the command is called with multiple field/value pairs.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSET_History,1,HSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSET_Keyspecs,1,NULL,2),.args=HSET_Args}, {MAKE_CMD("hsetnx","Sets the value of a field in a hash only when the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETNX_History,0,HSETNX_Tips,0,hsetnxCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETNX_Keyspecs,1,NULL,3),.args=HSETNX_Args}, {MAKE_CMD("hstrlen","Returns the length of the value of a field.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSTRLEN_History,0,HSTRLEN_Tips,0,hstrlenCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HSTRLEN_Keyspecs,1,NULL,2),.args=HSTRLEN_Args}, +{MAKE_CMD("httl","Returns the remaining time to live (in seconds) of a hash key's field(s) that have an associated expiration.","O(1) for each field, so O(N) for N items when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,0,httlCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HTTL_Keyspecs,1,NULL,2),.args=HTTL_Args}, {MAKE_CMD("hvals","Returns all values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HVALS_History,0,HVALS_Tips,1,hvalsCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HVALS_Keyspecs,1,NULL,1),.args=HVALS_Args}, /* hyperloglog */ {MAKE_CMD("pfadd","Adds elements to a HyperLogLog key. Creates the key if it doesn't exist.","O(1) to add every element.","2.8.9",CMD_DOC_NONE,NULL,NULL,"hyperloglog",COMMAND_GROUP_HYPERLOGLOG,PFADD_History,0,PFADD_Tips,0,pfaddCommand,-2,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HYPERLOGLOG,PFADD_Keyspecs,1,NULL,2),.args=PFADD_Args}, diff --git a/src/commands/hexpire.json b/src/commands/hexpire.json new file mode 100644 index 00000000000..81fefa4f44f --- /dev/null +++ b/src/commands/hexpire.json @@ -0,0 +1,121 @@ +{ + "HEXPIRE": { + "summary": "Set expiry time on hash object.", + "complexity": "O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.", + "group": "hash", + "since": "9.0.0", + "arity": -5, + "function": "hexpireCommand", + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the result of setting expiry on the specific fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "description": "Field does not exist in the HASH, or HASH is empty.", + "const": -2 + }, + { + "description": "The specified NX | XX | GT | LT condition has not been met.", + "const": 0 + }, + { + "description": "The expiration time was applied.", + "const": 1 + }, + { + "description": "When called with a 0 second", + "const": 2 + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "seconds", + "type": "integer", + "key_spec_index": 0 + }, + { + "name": "condition", + "type": "oneof", + "optional": true, + "since": "9.0.0", + "arguments": [ + { + "name": "nx", + "type": "pure-token", + "token": "NX" + }, + { + "name": "xx", + "type": "pure-token", + "token": "XX" + }, + { + "name": "gt", + "type": "pure-token", + "token": "GT" + }, + { + "name": "lt", + "type": "pure-token", + "token": "LT" + } + ] + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/commands/hexpireat.json b/src/commands/hexpireat.json new file mode 100644 index 00000000000..a65b83711ea --- /dev/null +++ b/src/commands/hexpireat.json @@ -0,0 +1,120 @@ +{ + "HEXPIREAT": { + "summary": "Set expiry time on hash object.", + "complexity": "O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.", + "group": "hash", + "since": "9.0.0", + "arity": -5, + "function": "hexpireAtCommand", + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the result of setting expiry on the specific fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "description": "Field does not exist in the HASH, or HASH is empty.", + "const": -2 + }, + { + "description": "The specified NX | XX | GT | LT condition has not been met.", + "const": 0 + }, + { + "description": "The expiration time was applied.", + "const": 1 + }, + { + "description": "When called with a 0 second or is called with a past Unix time in seconds.", + "const": 2 + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "unix-time-seconds", + "type": "integer" + }, + { + "name": "condition", + "type": "oneof", + "optional": true, + "since": "9.0.0", + "arguments": [ + { + "name": "nx", + "type": "pure-token", + "token": "NX" + }, + { + "name": "xx", + "type": "pure-token", + "token": "XX" + }, + { + "name": "gt", + "type": "pure-token", + "token": "GT" + }, + { + "name": "lt", + "type": "pure-token", + "token": "LT" + } + ] + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/commands/hexpiretime.json b/src/commands/hexpiretime.json new file mode 100644 index 00000000000..99685b21707 --- /dev/null +++ b/src/commands/hexpiretime.json @@ -0,0 +1,83 @@ +{ + "HEXPIRETIME": { + "summary": "Returns the Unix timestamp in seconds since Unix epoch at which the given key's field(s) will expire", + "complexity": "O(1) for each field, so O(N) for N items when the command is called with multiple fields.", + "group": "hash", + "since": "9.0.0", + "arity": -3, + "function": "hexpiretimeCommand", + "command_flags": [ + "READONLY", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RO", + "ACCESS" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the result of getting the absolute expiry timestamp of the specific fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "description": "Field does not exist in the provided hash key, or the hash key is empty", + "const": -2 + }, + { + "description": "Field exists in the provided hash key, but has no expiration associated with it.", + "const": -1 + }, + { + "description": "The expiration time associated with the hash key field, is seconds.", + "type": "integer" + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/commands/hpersist.json b/src/commands/hpersist.json new file mode 100644 index 00000000000..06ea3d5d7e1 --- /dev/null +++ b/src/commands/hpersist.json @@ -0,0 +1,84 @@ +{ + "HPERSIST": { + "summary": "Remove the existing expiration on a hash key's field(s).", + "complexity": "O(1) for each field assigned with TTL, so O(N) to persist N items when the command is called with multiple fields.", + "group": "hash", + "since": "9.0.0", + "arity": -3, + "function": "hpersistCommand", + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the result of setting expiry on the specific fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "description": "Field does not exist in the provided hash key, or the hash key is empty", + "const": -2 + }, + { + "description": "Field exists in the provided hash key, but has no expiration associated with it.", + "const": -1 + }, + { + "description": "The expiration time was removed from the hash key field.", + "const": 1 + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/commands/hpexpire.json b/src/commands/hpexpire.json new file mode 100644 index 00000000000..2ce0635ba87 --- /dev/null +++ b/src/commands/hpexpire.json @@ -0,0 +1,120 @@ +{ + "HPEXPIRE": { + "summary": "Set expiry time on hash object.", + "complexity": "O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.", + "group": "hash", + "since": "9.0.0", + "arity": -5, + "function": "hpexpireCommand", + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the result of setting expiry on the specific fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "description": "Field does not exist in the HASH, or HASH is empty.", + "const": -2 + }, + { + "description": "The specified NX | XX | GT | LT condition has not been met.", + "const": 0 + }, + { + "description": "The expiration time was applied.", + "const": 1 + }, + { + "description": "When called with a 0 millisecond", + "const": 2 + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "milliseconds", + "type": "integer" + }, + { + "name": "condition", + "type": "oneof", + "optional": true, + "since": "9.0.0", + "arguments": [ + { + "name": "nx", + "type": "pure-token", + "token": "NX" + }, + { + "name": "xx", + "type": "pure-token", + "token": "XX" + }, + { + "name": "gt", + "type": "pure-token", + "token": "GT" + }, + { + "name": "lt", + "type": "pure-token", + "token": "LT" + } + ] + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/commands/hpexpireat.json b/src/commands/hpexpireat.json new file mode 100644 index 00000000000..c6e006e4b46 --- /dev/null +++ b/src/commands/hpexpireat.json @@ -0,0 +1,120 @@ +{ + "HPEXPIREAT": { + "summary": "Set expiry time on hash object.", + "complexity": "O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.", + "group": "hash", + "since": "9.0.0", + "arity": -5, + "function": "hpexpireAtCommand", + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the result of setting expiry on the specific fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "description": "Field does not exist in the HASH, or HASH is empty.", + "const": -2 + }, + { + "description": "The specified NX | XX | GT | LT condition has not been met.", + "const": 0 + }, + { + "description": "The expiration time was applied.", + "const": 1 + }, + { + "description": "When called with a 0 second or is called with a past Unix time in milliseconds.", + "const": 2 + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "unix-time-milliseconds", + "type": "integer" + }, + { + "name": "condition", + "type": "oneof", + "optional": true, + "since": "9.0.0", + "arguments": [ + { + "name": "nx", + "type": "pure-token", + "token": "NX" + }, + { + "name": "xx", + "type": "pure-token", + "token": "XX" + }, + { + "name": "gt", + "type": "pure-token", + "token": "GT" + }, + { + "name": "lt", + "type": "pure-token", + "token": "LT" + } + ] + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/commands/hpexpiretime.json b/src/commands/hpexpiretime.json new file mode 100644 index 00000000000..da77e899fe7 --- /dev/null +++ b/src/commands/hpexpiretime.json @@ -0,0 +1,83 @@ +{ + "HPEXPIRETIME": { + "summary": "Returns the Unix timestamp in milliseconds since Unix epoch at which the given key's field(s) will expire", + "complexity": "O(1) for each field, so O(N) for N items when the command is called with multiple fields.", + "group": "hash", + "since": "9.0.0", + "arity": -3, + "function": "hpexpiretimeCommand", + "command_flags": [ + "READONLY", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RO", + "ACCESS" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the result of getting the absolute expiry timestamp of the specific fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "description": "Field does not exist in the provided hash key, or the hash key is empty", + "const": -2 + }, + { + "description": "Field exists in the provided hash key, but has no expiration associated with it.", + "const": -1 + }, + { + "description": "The expiration time associated with the hash key field, is milliseconds.", + "type": "integer" + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/commands/hpttl.json b/src/commands/hpttl.json new file mode 100644 index 00000000000..2ab41e4c099 --- /dev/null +++ b/src/commands/hpttl.json @@ -0,0 +1,83 @@ +{ + "HPTTL": { + "summary": "Returns the remaining time to live (in milliseconds) of a hash key's field(s) that have an associated expiration.", + "complexity": "O(1) for each field assigned with TTL, so O(N) for N items when the command is called with multiple fields.", + "group": "hash", + "since": "9.0.0", + "arity": -3, + "function": "hpttlCommand", + "command_flags": [ + "READONLY", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RO", + "ACCESS" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the result of getting the remaining time-to-live of the specific fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "description": "Field does not exist in the provided hash key, or the hash key is empty", + "const": -2 + }, + { + "description": "Field exists in the provided hash key, but has no expiration associated with it.", + "const": -1 + }, + { + "description": "The expiration time associated with the hash key field, is milliseconds.", + "type": "integer" + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/commands/httl.json b/src/commands/httl.json new file mode 100644 index 00000000000..aa78d6642a3 --- /dev/null +++ b/src/commands/httl.json @@ -0,0 +1,83 @@ +{ + "HTTL": { + "summary": "Returns the remaining time to live (in seconds) of a hash key's field(s) that have an associated expiration.", + "complexity": "O(1) for each field, so O(N) for N items when the command is called with multiple fields.", + "group": "hash", + "since": "9.0.0", + "arity": -3, + "function": "httlCommand", + "command_flags": [ + "READONLY", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RO", + "ACCESS" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the result of getting the remaining time-to-live of the specific fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "description": "Field does not exist in the provided hash key, or the hash key is empty", + "const": -2 + }, + { + "description": "Field exists in the provided hash key, but has no expiration associated with it.", + "const": -1 + }, + { + "description": "The expiration time associated with the hash key field, is seconds.", + "type": "integer" + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/db.c b/src/db.c index dddfe67ed73..8a179d64a48 100644 --- a/src/db.c +++ b/src/db.c @@ -46,13 +46,6 @@ #define EXPIRE_FORCE_DELETE_EXPIRED 1 #define EXPIRE_AVOID_DELETE_EXPIRED 2 -/* Return values for expireIfNeeded */ -typedef enum { - KEY_VALID = 0, /* Could be volatile and not yet expired, non-volatile, or even non-existing key. */ - KEY_EXPIRED, /* Logically expired but not yet deleted. */ - KEY_DELETED /* The key was deleted now. */ -} keyStatus; - static keyStatus expireIfNeededWithDictIndex(serverDb *db, robj *key, robj *val, int flags, int dict_index); static keyStatus expireIfNeeded(serverDb *db, robj *key, robj *val, int flags); static int keyIsExpiredWithDictIndex(serverDb *db, robj *key, int dict_index); @@ -1879,7 +1872,7 @@ void propagateDeletion(serverDb *db, robj *key, int lazy) { } /* Returns 1 if the expire value is expired, 0 otherwise. */ -static int timestampIsExpired(mstime_t when) { +int timestampIsExpired(mstime_t when) { if (when < 0) return 0; /* no expire */ mstime_t now = commandTimeSnapshot(); @@ -1888,6 +1881,62 @@ static int timestampIsExpired(mstime_t when) { return now > when; } +int canExpireWithFlags(int flags, int *delete_expired) { + if (server.lazy_expire_disabled) return 0; + if (server.loading) return 0; + + if (delete_expired) *delete_expired = 0; + + /* If we are running in the context of a replica, instead of + * evicting the expired key from the database, we return ASAP: + * the replica key expiration is controlled by the primary that will + * send us synthesized DEL operations for expired keys. The + * exception is when write operations are performed on writable + * replicas. + * + * Still we try to return the right information to the caller, + * that is, KEY_VALID if we think the key should still be valid, + * KEY_EXPIRED if we think the key is expired but don't want to delete it at this time. + * + * When replicating commands from the primary, keys are never considered + * expired. */ + if (server.primary_host != NULL) { + if (server.current_client && (server.current_client->flag.primary)) return 0; + if (!(flags & EXPIRE_FORCE_DELETE_EXPIRED)) return 1; + } else if (server.import_mode) { + /* If we are running in the import mode on a primary, instead of + * evicting the expired key from the database, we return ASAP: + * the key expiration is controlled by the import source that will + * send us synthesized DEL operations for expired keys. The + * exception is when write operations are performed on this server + * because it's a primary. + * + * Notice: other clients, apart from the import source, should not access + * the data imported by import source. + * + * Still we try to return the right information to the caller, + * that is, KEY_VALID if we think the key should still be valid, + * KEY_EXPIRED if we think the key is expired but don't want to delete it at this time. + * + * When receiving commands from the import source, keys are never considered + * expired. */ + if (server.current_client && (server.current_client->flag.import_source)) return 0; + if (!(flags & EXPIRE_FORCE_DELETE_EXPIRED)) return 1; + } + + /* In some cases we're explicitly instructed to return an indication of a + * missing key without actually deleting it, even on primaries. */ + if (flags & EXPIRE_AVOID_DELETE_EXPIRED) return 1; + + /* If 'expire' action is paused, for whatever reason, then don't expire any key. + * Typically, at the end of the pause we will properly expire the key OR we + * will have failed over and the new primary will send us the expire. */ + if (isPausedActionsWithUpdate(PAUSE_ACTION_EXPIRE)) return 1; + + if (delete_expired) *delete_expired = 1; + return 1; +} + /* Use this instead of keyIsExpired if you already have the value object. */ static int objectIsExpired(robj *val) { /* Don't expire anything while loading. It will be done later. */ @@ -1925,6 +1974,8 @@ int keyIsExpired(serverDb *db, robj *key) { /* val is optional. Pass NULL if val is not yet fetched from the database. */ static keyStatus expireIfNeededWithDictIndex(serverDb *db, robj *key, robj *val, int flags, int dict_index) { + int delete_expired = 0; + if (server.lazy_expire_disabled) return KEY_VALID; if (val != NULL) { if (!objectIsExpired(val)) return KEY_VALID; @@ -1932,51 +1983,9 @@ static keyStatus expireIfNeededWithDictIndex(serverDb *db, robj *key, robj *val, if (!keyIsExpiredWithDictIndexImpl(db, key, dict_index)) return KEY_VALID; } - /* If we are running in the context of a replica, instead of - * evicting the expired key from the database, we return ASAP: - * the replica key expiration is controlled by the primary that will - * send us synthesized DEL operations for expired keys. The - * exception is when write operations are performed on writable - * replicas. - * - * Still we try to return the right information to the caller, - * that is, KEY_VALID if we think the key should still be valid, - * KEY_EXPIRED if we think the key is expired but don't want to delete it at this time. - * - * When replicating commands from the primary, keys are never considered - * expired. */ - if (server.primary_host != NULL) { - if (server.current_client && (server.current_client->flag.primary)) return KEY_VALID; - if (!(flags & EXPIRE_FORCE_DELETE_EXPIRED)) return KEY_EXPIRED; - } else if (server.import_mode) { - /* If we are running in the import mode on a primary, instead of - * evicting the expired key from the database, we return ASAP: - * the key expiration is controlled by the import source that will - * send us synthesized DEL operations for expired keys. The - * exception is when write operations are performed on this server - * because it's a primary. - * - * Notice: other clients, apart from the import source, should not access - * the data imported by import source. - * - * Still we try to return the right information to the caller, - * that is, KEY_VALID if we think the key should still be valid, - * KEY_EXPIRED if we think the key is expired but don't want to delete it at this time. - * - * When receiving commands from the import source, keys are never considered - * expired. */ - if (server.current_client && (server.current_client->flag.import_source)) return KEY_VALID; - if (!(flags & EXPIRE_FORCE_DELETE_EXPIRED)) return KEY_EXPIRED; - } - - /* In some cases we're explicitly instructed to return an indication of a - * missing key without actually deleting it, even on primaries. */ - if (flags & EXPIRE_AVOID_DELETE_EXPIRED) return KEY_EXPIRED; + if (!canExpireWithFlags(flags, &delete_expired)) return KEY_VALID; - /* If 'expire' action is paused, for whatever reason, then don't expire any key. - * Typically, at the end of the pause we will properly expire the key OR we - * will have failed over and the new primary will send us the expire. */ - if (isPausedActionsWithUpdate(PAUSE_ACTION_EXPIRE)) return KEY_EXPIRED; + if (!delete_expired) return KEY_EXPIRED; /* The key needs to be converted from static to heap before deleted */ int static_key = key->refcount == OBJ_STATIC_REFCOUNT; diff --git a/src/expire.c b/src/expire.c index 93b26724e92..0b049c20d3b 100644 --- a/src/expire.c +++ b/src/expire.c @@ -532,23 +532,20 @@ int checkAlreadyExpired(long long when) { return (when <= commandTimeSnapshot() && !server.loading && !server.primary_host && !server.import_mode); } -#define EXPIRE_NX (1 << 0) -#define EXPIRE_XX (1 << 1) -#define EXPIRE_GT (1 << 2) -#define EXPIRE_LT (1 << 3) - -/* Parse additional flags of expire commands +/* Parse additional flags of expire commands up to the specify max_index. + * In case max_index will scan all arguments. * * Supported flags: * - NX: set expiry only when the key has no expiry * - XX: set expiry only when the key has an existing expiry * - GT: set expiry only when the new expiry is greater than current one * - LT: set expiry only when the new expiry is less than current one */ -int parseExtendedExpireArgumentsOrReply(client *c, int *flags) { +int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_index) { int nx = 0, xx = 0, gt = 0, lt = 0; + if (max_index > 0) max_index = c->argc - 1; int j = 3; - while (j < c->argc) { + while (j <= max_index) { char *opt = c->argv[j]->ptr; if (!strcasecmp(opt, "nx")) { *flags |= EXPIRE_NX; @@ -602,7 +599,7 @@ void expireGenericCommand(client *c, long long basetime, int unit) { int flag = 0; /* checking optional flags */ - if (parseExtendedExpireArgumentsOrReply(c, &flag) != C_OK) { + if (parseExtendedExpireArgumentsOrReply(c, &flag, -1) != C_OK) { return; } diff --git a/src/hashtable.c b/src/hashtable.c index 10413f5824d..d9f9ead111e 100644 --- a/src/hashtable.c +++ b/src/hashtable.c @@ -45,6 +45,7 @@ * - The original scan algorithm was designed by Pieter Noordhuis. */ #include "hashtable.h" +#include "server.h" #include "serverassert.h" #include "zmalloc.h" #include "mt19937-64.h" @@ -366,6 +367,9 @@ typedef struct { void **entries; /* Array of sampled entries. */ } scan_samples; +/* --- Access API --- */ +static inline hashtableElementAccessState accessElementIfNeeded(hashtable *ht, void *elem, bucket *b, int pos_in_bucket, int table_index); + /* --- Internal functions --- */ static bucket *findBucketForInsert(hashtable *ht, uint64_t hash, int *pos_in_bucket, int *table_index); @@ -685,6 +689,9 @@ static inline int checkCandidateInBucket(hashtable *ht, bucket *b, int pos, cons if (compareKeys(ht, key, elem_key) == 0) { /* It's a match. */ assert(pos_in_bucket != NULL); + if (accessElementIfNeeded(ht, entry, b, pos, table) != ELEMENT_VALID) { + return 0; + } *pos_in_bucket = pos; if (table_index) *table_index = table; return 1; @@ -876,6 +883,25 @@ static void compactBucketChain(hashtable *ht, size_t bucket_index, int table_ind } } +static inline hashtableElementAccessState accessElementIfNeeded(hashtable *ht, void *elem, bucket *b, int pos_in_bucket, int table_index) { + if (ht->type->accessElement == NULL) return ELEMENT_VALID; + + hashtableElementAccessState element_status = ht->type->accessElement(ht, elem); + serverLog(LL_NOTICE, "hashtable access returned: %d", element_status); + if (element_status == ELEMENT_DELETE) { + b->presence &= ~(1 << pos_in_bucket); + ht->used[table_index]--; + if (b->chained && !hashtableIsRehashingPaused(ht)) { + /* Rehashing is paused while iterating and when a scan callback is + * running. In those cases, we do the compaction in the scan and + * iterator code instead. */ + fillBucketHole(ht, b, pos_in_bucket, table_index); + } + hashtableShrinkIfNeeded(ht); + } + return element_status; +} + /* Find an empty position in the table for inserting an entry with the given hash. */ static bucket *findBucketForInsert(hashtable *ht, uint64_t hash, int *pos_in_bucket, int *table_index) { int table = hashtableIsRehashing(ht) ? 1 : 0; @@ -1765,7 +1791,7 @@ size_t hashtableScanDefrag(hashtable *ht, size_t cursor, hashtableScanFunction f if (b->presence != 0) { int pos; for (pos = 0; pos < ENTRIES_PER_BUCKET; pos++) { - if (isPositionFilled(b, pos)) { + if (isPositionFilled(b, pos) && accessElementIfNeeded(ht, b->entries[pos], b, pos, 0) == ELEMENT_VALID) { void *emit = emit_ref ? &b->entries[pos] : b->entries[pos]; fn(privdata, emit); } @@ -1802,7 +1828,7 @@ size_t hashtableScanDefrag(hashtable *ht, size_t cursor, hashtableScanFunction f do { if (b->presence) { for (int pos = 0; pos < ENTRIES_PER_BUCKET; pos++) { - if (isPositionFilled(b, pos)) { + if (isPositionFilled(b, pos) && accessElementIfNeeded(ht, b->entries[pos], b, pos, 0) == ELEMENT_VALID) { void *emit = emit_ref ? &b->entries[pos] : b->entries[pos]; fn(privdata, emit); } @@ -1832,7 +1858,7 @@ size_t hashtableScanDefrag(hashtable *ht, size_t cursor, hashtableScanFunction f do { if (b->presence) { for (int pos = 0; pos < ENTRIES_PER_BUCKET; pos++) { - if (isPositionFilled(b, pos)) { + if (isPositionFilled(b, pos) && accessElementIfNeeded(ht, b->entries[pos], b, pos, 0) == ELEMENT_VALID) { void *emit = emit_ref ? &b->entries[pos] : b->entries[pos]; fn(privdata, emit); } @@ -2021,6 +2047,9 @@ int hashtableNext(hashtableIterator *iterator, void **elemptr) { /* No entry here. */ continue; } + if (accessElementIfNeeded(iter->hashtable, b->entries[iter->pos_in_bucket], b, iter->pos_in_bucket, iter->table) != ELEMENT_VALID) { + continue; + } /* Return the entry at this position. */ if (elemptr) { *elemptr = b->entries[iter->pos_in_bucket]; diff --git a/src/hashtable.h b/src/hashtable.h index 48a7077b72b..ca67c7da51e 100644 --- a/src/hashtable.h +++ b/src/hashtable.h @@ -42,6 +42,13 @@ typedef uint64_t hashtableIterator[5]; typedef uint64_t hashtablePosition[2]; typedef uint64_t hashtableIncrementalFindState[5]; +typedef enum { + ELEMENT_NONE = 0, + ELEMENT_VALID, + ELEMENT_INVALID, + ELEMENT_DELETE, +} hashtableElementAccessState; + /* --- Non-opaque types --- */ /* The hashtableType is a set of callbacks for a hashtable. All callbacks are @@ -77,6 +84,9 @@ typedef struct { size_t (*getMetadataSize)(void); /* Flag to disable incremental rehashing */ unsigned instant_rehashing : 1; + + hashtableElementAccessState (*accessElement)(hashtable *ht, void *entry); + } hashtableType; typedef enum { diff --git a/src/module.c b/src/module.c index 0607d6b7b7e..034d9b7dd69 100644 --- a/src/module.c +++ b/src/module.c @@ -5345,11 +5345,11 @@ int VM_HashSet(ValkeyModuleKey *key, int flags, ...) { /* If CFIELDS is active, we can pass the ownership of the * SDS object to the low level function that sets the field * to avoid a useless copy. */ - if (flags & VALKEYMODULE_HASH_CFIELDS) low_flags |= HASH_SET_TAKE_FIELD; + if (flags & VALKEYMODULE_HASH_CFIELDS) low_flags |= (HASH_SET_TAKE_FIELD | HASH_SET_KEEP_EXPIRY); robj *argv[2] = {field, value}; hashTypeTryConversion(key->value, argv, 0, 1); - int updated = hashTypeSet(key->value, field->ptr, value->ptr, low_flags); + int updated = hashTypeSet(key->value, field->ptr, value->ptr, EXPIRY_NONE, low_flags); count += (flags & VALKEYMODULE_HASH_COUNT_ALL) ? 1 : updated; /* If CFIELDS is active, SDS string ownership is now of hashTypeSet(), diff --git a/src/object.c b/src/object.c index fccd1f1e5b0..658c9f8fd9c 100644 --- a/src/object.c +++ b/src/object.c @@ -1835,3 +1835,113 @@ void memoryCommand(client *c) { addReplySubcommandSyntaxError(c); } } + +/* + * The parseExtendedStringArgumentsOrReply() function performs the common validation for extended + * string arguments used in SET and GET command. + * + * Get specific commands - PERSIST/DEL + * Set specific commands - XX/NX/GET/IFEQ + * Common commands - EX/EXAT/PX/PXAT/KEEPTTL + * + * Function takes pointers to client, flags, unit, pointer to pointer of expire obj if needed + * to be determined and command_type which can be COMMAND_GET or COMMAND_SET. + * + * If there are any syntax violations C_ERR is returned else C_OK is returned. + * + * Input flags are updated upon parsing the arguments. Unit and expire are updated if there are any + * EX/EXAT/PX/PXAT arguments. Unit is updated to millisecond if PX/PXAT is set. + */ +int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj **expire, robj **compare_val, int command_type) { + int j = command_type == COMMAND_GET ? 2 : 3; + for (; j < c->argc; j++) { + char *opt = c->argv[j]->ptr; + robj *next = (j == c->argc - 1) ? NULL : c->argv[j + 1]; + + /* clang-format off */ + if ((opt[0] == 'n' || opt[0] == 'N') && + (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && + !(*flags & OBJ_SET_XX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET)) + { + *flags |= OBJ_SET_NX; + } else if ((opt[0] == 'x' || opt[0] == 'X') && + (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && + !(*flags & OBJ_SET_NX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET)) + { + *flags |= OBJ_SET_XX; + } else if ((opt[0] == 'i' || opt[0] == 'I') && + (opt[1] == 'f' || opt[1] == 'F') && + (opt[2] == 'e' || opt[2] == 'E') && + (opt[3] == 'q' || opt[3] == 'Q') && opt[4] == '\0' && + next && !(*flags & OBJ_SET_NX || *flags & OBJ_SET_XX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET)) + { + *flags |= OBJ_SET_IFEQ; + *compare_val = next; + j++; + } else if ((opt[0] == 'g' || opt[0] == 'G') && + (opt[1] == 'e' || opt[1] == 'E') && + (opt[2] == 't' || opt[2] == 'T') && opt[3] == '\0' && + (command_type == COMMAND_SET)) + { + *flags |= OBJ_SET_GET; + } else if (!strcasecmp(opt, "KEEPTTL") && !(*flags & OBJ_PERSIST) && + !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && + !(*flags & OBJ_PX) && !(*flags & OBJ_PXAT) && (command_type == COMMAND_SET)) + { + *flags |= OBJ_KEEPTTL; + } else if (!strcasecmp(opt,"PERSIST") && (command_type == COMMAND_GET) && + !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && + !(*flags & OBJ_PX) && !(*flags & OBJ_PXAT) && + !(*flags & OBJ_KEEPTTL)) + { + *flags |= OBJ_PERSIST; + } else if ((opt[0] == 'e' || opt[0] == 'E') && + (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && + !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) && + !(*flags & OBJ_EXAT) && !(*flags & OBJ_PX) && + !(*flags & OBJ_PXAT) && next) + { + *flags |= OBJ_EX; + *expire = next; + j++; + } else if ((opt[0] == 'p' || opt[0] == 'P') && + (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && + !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) && + !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && + !(*flags & OBJ_PXAT) && next) + { + *flags |= OBJ_PX; + *unit = UNIT_MILLISECONDS; + *expire = next; + j++; + } else if ((opt[0] == 'e' || opt[0] == 'E') && + (opt[1] == 'x' || opt[1] == 'X') && + (opt[2] == 'a' || opt[2] == 'A') && + (opt[3] == 't' || opt[3] == 'T') && opt[4] == '\0' && + !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) && + !(*flags & OBJ_EX) && !(*flags & OBJ_PX) && + !(*flags & OBJ_PXAT) && next) + { + *flags |= OBJ_EXAT; + *expire = next; + j++; + } else if ((opt[0] == 'p' || opt[0] == 'P') && + (opt[1] == 'x' || opt[1] == 'X') && + (opt[2] == 'a' || opt[2] == 'A') && + (opt[3] == 't' || opt[3] == 'T') && opt[4] == '\0' && + !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) && + !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && + !(*flags & OBJ_PX) && next) + { + *flags |= OBJ_PXAT; + *unit = UNIT_MILLISECONDS; + *expire = next; + j++; + } else { + addReplyErrorObject(c,shared.syntaxerr); + return C_ERR; + } + /* clang-format on */ + } + return C_OK; +} diff --git a/src/rdb.c b/src/rdb.c index 75821cee9c1..fbfa2101eee 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -717,7 +717,10 @@ int rdbSaveObjectType(rio *rdb, robj *o) { if (o->encoding == OBJ_ENCODING_LISTPACK) return rdbSaveType(rdb, RDB_TYPE_HASH_LISTPACK); else if (o->encoding == OBJ_ENCODING_HASHTABLE) - return rdbSaveType(rdb, RDB_TYPE_HASH); + if (hashTypeHasVolatileElements(o)) + return rdbSaveType(rdb, RDB_TYPE_HASH_2); + else + return rdbSaveType(rdb, RDB_TYPE_HASH); else serverPanic("Unknown hash encoding"); case OBJ_STREAM: return rdbSaveType(rdb, RDB_TYPE_STREAM_LISTPACKS_3); @@ -840,7 +843,6 @@ size_t rdbSaveStreamConsumers(rio *rdb, streamCG *cg) { * Returns -1 on error, number of bytes written on success. */ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { ssize_t n = 0, nwritten = 0; - if (o->type == OBJ_STRING) { /* Save a string value */ if ((n = rdbSaveStringObject(rdb, o)) == -1) return -1; @@ -963,6 +965,9 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { return -1; } nwritten += n; + /* check if need to add expired time for the hash elements */ + int add_expiry = hashTypeHasVolatileElements(o); + setAccessContextWithFlags(o, &server.db[dbid], OBJ_ACCESS_IGNORE_TTL); hashtableIterator iter; hashtableInitIterator(&iter, ht, 0); @@ -981,8 +986,19 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { return -1; } nwritten += n; + if (add_expiry) { + long long expiry = hashTypeEntryGetExpiry(next); + if ((n = rdbSaveMillisecondTime(rdb, expiry) == -1)) { + hashtableResetIterator(&iter); + return -1; + } + serverLog(LL_NOTICE, "save key %s with expiry: %lld", field, expiry); + nwritten += n; + } } hashtableResetIterator(&iter); + resetAccessContext(); + } else { serverPanic("Unknown hash encoding"); } @@ -2069,7 +2085,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { lpSafeToAdd(NULL, totelelen)) { zsetConvert(o, OBJ_ENCODING_LISTPACK); } - } else if (rdbtype == RDB_TYPE_HASH) { + } else if (rdbtype == RDB_TYPE_HASH || rdbtype == RDB_TYPE_HASH_2) { uint64_t len; sds field, value; hashtable *dupSearchHashtable = NULL; @@ -2080,8 +2096,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { o = createHashObject(); - /* Too many entries? Use a hash table right from the start. */ - if (len > server.hash_max_listpack_entries) + /* Too many entries or hash object contains elements with expiry? Use a hash table right from the start. */ + if (len > server.hash_max_listpack_entries || rdbtype == RDB_TYPE_HASH_2) hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); else if (deep_integrity_validation) { /* In this mode, we need to guarantee that the server won't crash @@ -2122,10 +2138,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { } /* Convert to hash table if size threshold is exceeded */ - if (sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value || - !lpSafeToAdd(o->ptr, sdslen(field) + sdslen(value))) { + if (o->encoding != OBJ_ENCODING_HASHTABLE && + (sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value || + !lpSafeToAdd(o->ptr, sdslen(field) + sdslen(value)))) { hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); - hashTypeEntry *entry = hashTypeCreateEntry(field, value); + hashTypeEntry *entry = hashTypeCreateEntry(field, value, -1, NULL, 0); sdsfree(field); if (!hashtableAdd((hashtable *)o->ptr, entry)) { rdbReportCorruptRDB("Duplicate hash fields detected"); @@ -2137,6 +2154,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { break; } + /* Add pair to listpack */ o->ptr = lpAppend(o->ptr, (unsigned char *)field, sdslen(field)); o->ptr = lpAppend(o->ptr, (unsigned char *)value, sdslen(value)); @@ -2174,8 +2192,16 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { return NULL; } + /* Also load the entry expiry */ + long long itemexpiry = -1; + if (rdbtype == RDB_TYPE_HASH_2) { + itemexpiry = rdbLoadMillisecondTime(rdb, RDB_VERSION); + serverLog(LL_NOTICE, "load key %s with expiry: %lld", field, itemexpiry); + if (itemexpiry == LLONG_MAX && rioGetReadError(rdb)) return NULL; + } + /* Add pair to hash table */ - hashTypeEntry *entry = hashTypeCreateEntry(field, value); + hashTypeEntry *entry = hashTypeCreateEntry(field, value, itemexpiry, NULL, 0); sdsfree(field); if (!hashtableAdd((hashtable *)o->ptr, entry)) { rdbReportCorruptRDB("Duplicate hash fields detected"); @@ -2183,6 +2209,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { decrRefCount(o); return NULL; } + + if (rdbtype == RDB_TYPE_HASH_2 && itemexpiry > 0) { + hashTypeTrackEntry(o, entry); + } } /* All pairs should be read by now */ diff --git a/src/rdb.h b/src/rdb.h index 9f19a3a9eca..d2f13546e13 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -112,6 +112,7 @@ static_assert(RDB_VERSION < RDB_FOREIGN_VERSION_MIN || RDB_VERSION > RDB_FOREIGN #define RDB_TYPE_STREAM_LISTPACKS_2 19 #define RDB_TYPE_SET_LISTPACK 20 #define RDB_TYPE_STREAM_LISTPACKS_3 21 +#define RDB_TYPE_HASH_2 22 /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType(), and rdb_type_string[] */ /* Test if a type is an object type. */ diff --git a/src/sds.c b/src/sds.c index 7843363f436..a4014ad16b4 100644 --- a/src/sds.c +++ b/src/sds.c @@ -34,11 +34,16 @@ #include #include #include +#include #include "serverassert.h" #include "sds.h" #include "sdsalloc.h" #include "util.h" +#ifndef min +#define min(a, b) ((a) < (b) ? (a) : (b)) +#endif + const char *SDS_NOINIT = "SDS_NOINIT"; int sdsHdrSize(char type) { @@ -125,7 +130,7 @@ sds _sdsnewlen(const void *init, size_t initlen, int trymalloc) { sds sdswrite(char *buf, size_t bufsize, char type, const char *init, size_t initlen) { assert(bufsize >= sdsReqSize(initlen, type)); int hdrlen = sdsHdrSize(type); - size_t usable = bufsize - hdrlen - 1; + size_t usable = min(bufsize - hdrlen - 1, sdsTypeMaxSize(type)); sds s = buf + hdrlen; unsigned char *fp = ((unsigned char *)s) - 1; /* flags pointer. */ diff --git a/src/server.c b/src/server.c index f7bc6e7660a..86bd1e0d105 100644 --- a/src/server.c +++ b/src/server.c @@ -673,11 +673,19 @@ void hashHashtableTypeDestructor(void *entry) { freeHashTypeEntry(hash_entry); } +size_t hashHashtableTypeMetadataSize(void) { + return sizeof(void *); +} + +extern hashtableElementAccessState hashHashtableTypeAccess(hashtable *ht, void *entry); + hashtableType hashHashtableType = { .hashFunction = dictSdsHash, .entryGetKey = hashHashtableTypeGetKey, .keyCompare = hashtableSdsKeyCompare, .entryDestructor = hashHashtableTypeDestructor, + .getMetadataSize = hashHashtableTypeMetadataSize, + .accessElement = hashHashtableTypeAccess, }; /* Hashtable type without destructor */ @@ -2116,6 +2124,7 @@ void createSharedObjects(void) { shared.multi = createStringObject("MULTI", 5); shared.exec = createStringObject("EXEC", 4); shared.hset = createStringObject("HSET", 4); + shared.hdel = createStringObject("HDEL", 4); shared.srem = createStringObject("SREM", 4); shared.xgroup = createStringObject("XGROUP", 6); shared.xclaim = createStringObject("XCLAIM", 6); @@ -7213,4 +7222,21 @@ __attribute__((weak)) int main(int argc, char **argv) { aeDeleteEventLoop(server.el); return 0; } + +void setAccessContext(robj *o, serverDb *db) { + setAccessContextWithFlags(o, db, OBJ_ACCESS_NONE); +} + +void setAccessContextWithFlags(robj *o, serverDb *db, int flags) { + server.access_context.key = o; + server.access_context.db = db; + server.access_context.flags = flags; +} + +void resetAccessContext(void) { + server.access_context.key = NULL; + server.access_context.db = NULL; + server.access_context.flags = OBJ_ACCESS_NONE; +} + /* The End */ diff --git a/src/server.h b/src/server.h index d6c52de8dbf..ae380a2ee08 100644 --- a/src/server.h +++ b/src/server.h @@ -82,6 +82,7 @@ typedef long long ustime_t; /* microsecond time type. */ #include "rax.h" /* Radix tree */ #include "connection.h" /* Connection abstraction */ #include "memory_prefetch.h" +#include "volatile_set.h" #define dismissMemory zmadvise_dontneed @@ -217,6 +218,10 @@ struct hdr_histogram; extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; +#define COMMAND_GET 0 +#define COMMAND_SET 1 + + /* Command flags. Please check the definition of struct serverCommand in this file * for more information about the meaning of every flag. */ #define CMD_WRITE (1ULL << 0) @@ -313,6 +318,11 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; /* Key flags for when access type is unknown */ #define CMD_KEY_FULL_ACCESS (CMD_KEY_RW | CMD_KEY_ACCESS | CMD_KEY_UPDATE) +#define EXPIRE_NX (1 << 0) +#define EXPIRE_XX (1 << 1) +#define EXPIRE_GT (1 << 2) +#define EXPIRE_LT (1 << 3) + /* Key flags for how key is removed */ #define DB_FLAG_KEY_NONE 0 #define DB_FLAG_KEY_DELETED (1ULL << 0) @@ -595,6 +605,9 @@ typedef enum { #define PAUSE_ACTION_EVICT (1 << 3) #define PAUSE_ACTION_REPLICA (1 << 4) /* pause replica traffic */ +/* Special Expiry values */ +#define EXPIRY_NONE -1 + /* Sets log format */ typedef enum { LOG_FORMAT_LEGACY = 0, LOG_FORMAT_LOGFMT } log_format_type; @@ -700,6 +713,21 @@ typedef enum { * Data types *----------------------------------------------------------------------------*/ + /* Generic set command string object set flags */ +#define OBJ_NO_FLAGS 0 +#define OBJ_SET_NX (1 << 0) /* Set if key not exists. */ +#define OBJ_SET_XX (1 << 1) /* Set if key exists. */ +#define OBJ_EX (1 << 2) /* Set if time in seconds is given */ +#define OBJ_PX (1 << 3) /* Set if time in ms in given */ +#define OBJ_KEEPTTL (1 << 4) /* Set and keep the ttl */ +#define OBJ_SET_GET (1 << 5) /* Set if want to get key before set */ +#define OBJ_EXAT (1 << 6) /* Set if timestamp in second is given */ +#define OBJ_PXAT (1 << 7) /* Set if timestamp in ms is given */ +#define OBJ_PERSIST (1 << 8) /* Set if we need to remove the ttl */ +#define OBJ_SET_IFEQ (1 << 9) /* Set if we need compare and set */ +#define OBJ_ARGV3 (1 << 10) /* Set if the value is at argv[3]; otherwise it's \ + * at argv[2]. */ + /* An Object, that is a type able to hold a string / list / set */ /* The actual Object */ @@ -1320,7 +1348,7 @@ struct sharedObjectsStruct { *loadingerr, *slowevalerr, *slowscripterr, *slowmoduleerr, *bgsaveerr, *primarydownerr, *roreplicaerr, *execaborterr, *noautherr, *noreplicaserr, *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink, *rpop, *lpop, *lpush, - *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *srem, + *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *hdel, *srem, *xgroup, *xclaim, *script, *replconf, *eval, *persist, *set, *pexpireat, *pexpire, *time, *pxat, *absttl, *retrycount, *force, *justid, *entriesread, *lastid, *ping, *setid, *keepttl, *load, *createconsumer, *getack, *special_asterisk, *special_equals, *default_username, *redacted, *ssubscribebulk, *sunsubscribebulk, @@ -1568,6 +1596,24 @@ typedef enum childInfoType { CHILD_INFO_TYPE_MODULE_COW_SIZE } childInfoType; +#define OBJ_ACCESS_NONE 0 /* Will not actively delete expired accessed elements */ +#define OBJ_ACCESS_NORMAL (1 << 0) /* Deleting lazy expired fields. */ +#define OBJ_ACCESS_IGNORE_TTL (1 << 1) /* treat any accessed field as valid regardless of it's TTL */ + +typedef struct keyAccessContext { + int flags; + robj *key; + serverDb *db; +} keyAccessContext; + + +/* Return values for expireIfNeeded */ +typedef enum { + KEY_VALID = 0, /* Could be volatile and not yet expired, non-volatile, or even non-existing key. */ + KEY_EXPIRED, /* Logically expired but not yet deleted. */ + KEY_DELETED /* The key was deleted now. */ +} keyStatus; + struct valkeyServer { /* General */ pid_t pid; /* Main process pid. */ @@ -1642,6 +1688,8 @@ struct valkeyServer { * Value: RDB client object * This structure holds dual-channel sync replicas from the start of their * RDB transfer until their main channel establishes partial synchronization. */ + keyAccessContext access_context; /* The current key access context */ + client *current_client; /* The client that triggered the command execution (External or AOF). */ client *executing_client; /* The client executing the current command (possibly script or module). */ @@ -2550,17 +2598,20 @@ typedef struct { typedef struct { robj *subject; int encoding; - + int volatile_items; unsigned char *fptr, *vptr; hashtableIterator iter; + volatileSetIterator viter; void *next; + } hashTypeIterator; #include "stream.h" /* Stream data type header file. */ #define OBJ_HASH_FIELD 1 #define OBJ_HASH_VALUE 2 +#define OBJ_HASH_EXPIRY /*----------------------------------------------------------------------------- * Extern declarations @@ -2608,6 +2659,7 @@ int validateProcTitleTemplate(const char *template); int serverCommunicateSystemd(const char *sd_notify_msg); void serverSetCpuAffinity(const char *cpulist); void dictVanillaFree(void *val); +int timestampIsExpired(mstime_t when); /* ERROR STATS constants */ @@ -2782,6 +2834,9 @@ void ioThreadWriteToClient(void *data); int canParseCommand(client *c); int processIOThreadsReadDone(void); int processIOThreadsWriteDone(void); +int canExpireWithFlags(int flags, int *can_delete); +int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_index); +int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj **expire, robj **compare_val, int command_type); /* logreqres.c - logging of requests and responses */ void reqresReset(client *c, int free_buf); @@ -3230,6 +3285,9 @@ void *activeDefragAlloc(void *ptr); robj *activeDefragStringOb(robj *ob); void dismissSds(sds s); void dismissMemoryInChild(void); +void setAccessContext(robj *o, serverDb *db); +void setAccessContextWithFlags(robj *o, serverDb *db, int flags); +void resetAccessContext(void); #define RESTART_SERVER_NONE 0 #define RESTART_SERVER_GRACEFULLY (1 << 0) /* Do proper shutdown. */ @@ -3267,12 +3325,16 @@ robj *setTypeDup(robj *o); /* Hash data type */ #define HASH_SET_TAKE_FIELD (1 << 0) #define HASH_SET_TAKE_VALUE (1 << 1) +#define HASH_SET_KEEP_EXPIRY (1 << 2) #define HASH_SET_COPY 0 typedef void hashTypeEntry; -hashTypeEntry *hashTypeCreateEntry(sds field, sds value); +hashTypeEntry *hashTypeCreateEntry(sds field, sds value, long long ttl, void *metadata, size_t metadata_size); sds hashTypeEntryGetField(const hashTypeEntry *entry); sds hashTypeEntryGetValue(const hashTypeEntry *entry); +long long hashTypeEntryGetExpiry(const hashTypeEntry *entry); +int hashTypeEntryHasExpire(const hashTypeEntry *entry); +void hashTypeTrackEntry(robj *o, void *entry); size_t hashTypeEntryMemUsage(hashTypeEntry *entry); hashTypeEntry *hashTypeEntryDefrag(hashTypeEntry *entry, void *(*defragfn)(void *), sds (*sdsdefragfn)(sds)); void dismissHashTypeEntry(hashTypeEntry *entry); @@ -3284,6 +3346,7 @@ int hashTypeExists(robj *o, sds key); int hashTypeDelete(robj *o, sds key); unsigned long hashTypeLength(const robj *o); void hashTypeInitIterator(robj *subject, hashTypeIterator *hi); +void hashTypeInitVolatileIterator(robj *subject, hashTypeIterator *hi); void hashTypeResetIterator(hashTypeIterator *hi); int hashTypeNext(hashTypeIterator *hi); void hashTypeCurrentFromListpack(hashTypeIterator *hi, @@ -3295,8 +3358,10 @@ sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what); sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what); robj *hashTypeLookupWriteOrCreate(client *c, robj *key); robj *hashTypeGetValueObject(robj *o, sds field); -int hashTypeSet(robj *o, sds field, sds value, int flags); +int hashTypeSet(robj *o, sds field, sds value, long long expiry, int flags); robj *hashTypeDup(robj *o); +int hashTypeHasVolatileElements(robj *o); +size_t hashTypeNumVolatileElements(robj *o); /* Pub / Sub */ int pubsubUnsubscribeAllChannels(client *c, int notify); @@ -3778,6 +3843,19 @@ void hgetallCommand(client *c); void hexistsCommand(client *c); void hscanCommand(client *c); void hrandfieldCommand(client *c); +void hexpireCommand(client *c); +void hexpireAtCommand(client *c); +void hpexpireCommand(client *c); +void hpexpireAtCommand(client *c); +void hexpireCommand(client *c); +void hexpireAtCommand(client *c); +void hpexpireCommand(client *c); +void hpexpireAtCommand(client *c); +void httlCommand(client *c); +void hpttlCommand(client *c); +void hexpiretimeCommand(client *c); +void hpexpiretimeCommand(client *c); +void hpersistCommand(client *c); void configSetCommand(client *c); void configGetCommand(client *c); void configResetStatCommand(client *c); diff --git a/src/t_hash.c b/src/t_hash.c index 5a8c17e90c8..296d3380320 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -32,8 +32,30 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include "hashtable.h" +#include "rax.h" +#include "sds.h" +#include "volatile_set.h" #include "server.h" +#include "zmalloc.h" #include +#include + + +sds hashTypeEntryGetField(const hashTypeEntry *entry); +static inline void *hashTypeEntryGetMetadata(const hashTypeEntry *entry); +static inline size_t hashTypeEntryGetMetadataSize(const hashTypeEntry *entry); +static inline hashTypeEntry *hashTypeEntrySetExpiry(hashTypeEntry *entry, long long expiry); +int hashTypeExpireEntry(hashTypeEntry *entry); +int hashTypeExpireRemoveEntry(void *entry); + +volatileEntryType hashvolatileEntryType = { + .entryGetKey = (sds(*)(const void *entry))hashTypeEntryGetField, + .getExpiry = (long long (*)(const void *entry))hashTypeEntryGetExpiry, + .expire = hashTypeExpireRemoveEntry, +}; + + #include /*----------------------------------------------------------------------------- @@ -76,10 +98,24 @@ * instead has an embedded value located after the embedded field. */ #define FIELD_SDS_AUX_BIT_ENTRY_HAS_VALUE_PTR 0 +/* SDS aux flag. If set, it indicates that the entry has TTL metadata set. */ +#define FIELD_SDS_AUX_BIT_ENTRY_HAS_TTL 1 + +/* SDS aux flag. If set, it indicates that the entry has TTL metadata set. */ +#define FIELD_SDS_AUX_BIT_ENTRY_HAS_METADATA 2 + static inline bool entryHasValuePtr(const hashTypeEntry *entry) { return sdsGetAuxBit(entry, FIELD_SDS_AUX_BIT_ENTRY_HAS_VALUE_PTR); } +static inline bool entryHasExpiry(const hashTypeEntry *entry) { + return sdsGetAuxBit(entry, FIELD_SDS_AUX_BIT_ENTRY_HAS_TTL); +} + +static inline bool entryHasMetadata(const hashTypeEntry *entry) { + return sdsGetAuxBit(entry, FIELD_SDS_AUX_BIT_ENTRY_HAS_METADATA); +} + /* Returns the location of a pointer to a separately allocated value. Only for * an entry without an embedded value. */ static sds *hashTypeEntryGetValueRef(const hashTypeEntry *entry) { @@ -90,51 +126,91 @@ static sds *hashTypeEntryGetValueRef(const hashTypeEntry *entry) { } /* takes ownership of value, does not take ownership of field */ -hashTypeEntry *hashTypeCreateEntry(sds field, sds value) { +hashTypeEntry *hashTypeCreateEntry(sds field, sds value, long long expiry, void *metadata, size_t metadata_size) { + serverAssert(metadata_size <= UCHAR_MAX); + sds embedded_field_sds; + size_t expiry_size = (expiry == EXPIRY_NONE) ? 0 : sizeof(long long); + size_t metadata_header_size = metadata_size ? sizeof(char) : 0; size_t field_len = sdslen(field); int field_sds_type = sdsReqType(field_len); + if (field_sds_type == SDS_TYPE_5 && (expiry_size > 0 || metadata_size > 0)) { + field_sds_type = SDS_TYPE_8; + } size_t field_size = sdsReqSize(field_len, field_sds_type); size_t value_len = sdslen(value); size_t value_size = sdsReqSize(value_len, SDS_TYPE_8); - sds embedded_field_sds; - if (field_size + value_size <= EMBED_VALUE_MAX_ALLOC_SIZE) { + size_t alloc_size = field_size + metadata_size + metadata_header_size + expiry_size; + bool embed_value = false; + if (alloc_size + value_size <= EMBED_VALUE_MAX_ALLOC_SIZE) { /* Embed field and value. Value is fixed to SDS_TYPE_8. Unused * allocation space is recorded in the embedded value's SDS header. * - * +--------------+---------------+ - * | field | value | - * | hdr "foo" \0 | hdr8 "bar" \0 | - * +--------------+---------------+ + * +----------+----------+------+--------------+---------------+ + * | metadata | metadata | TTL | field | value | + * | | size | | hdr "foo" \0 | hdr8 "bar" \0 | + * +----------+----------+------+--------------+---------------+ */ - size_t min_size = field_size + value_size; - size_t buf_size; - char *buf = zmalloc_usable(min_size, &buf_size); - embedded_field_sds = sdswrite(buf, field_size, field_sds_type, field, field_len); - sdswrite(buf + field_size, buf_size - field_size, SDS_TYPE_8, value, value_len); - /* Field sds aux bits are zero, which we use for this entry encoding. */ - sdsSetAuxBit(embedded_field_sds, FIELD_SDS_AUX_BIT_ENTRY_HAS_VALUE_PTR, 0); - serverAssert(!entryHasValuePtr(embedded_field_sds)); - sdsfree(value); + embed_value = true; + alloc_size += value_size; } else { /* Embed field, but not value. Field must be >= SDS_TYPE_8 to encode to * indicate this type of entry. * - * +-------+---------------+ - * | value | field | - * | ptr | hdr8 "foo" \0 | - * +-------+---------------+ + * +----------+----------+------+-------+---------------+ + * | metadata | metadata | TTL | value | field | + * | | size | | ptr | hdr8 "foo" \0 | + * +----------+----------+------+-------+---------------+ */ - char field_sds_type = sdsReqType(field_len); - if (field_sds_type == SDS_TYPE_5) field_sds_type = SDS_TYPE_8; - field_size = sdsReqSize(field_len, field_sds_type); - size_t alloc_size = sizeof(sds *) + field_size; - char *buf = zmalloc(alloc_size); + embed_value = false; + alloc_size += sizeof(sds *); + if (field_sds_type == SDS_TYPE_5) { + field_sds_type = SDS_TYPE_8; + alloc_size -= field_size; + field_size = sdsReqSize(field_len, field_sds_type); + alloc_size += field_size; + } + } + + /* allocate the buffer */ + size_t buf_size; + char *buf = zmalloc_usable(alloc_size, &buf_size); + + /* Copy the metadata if exists */ + if (metadata_size) { + memcpy(buf, metadata, metadata_size); + buf += metadata_size; + *buf = (char)(metadata_size & 0xFF); + buf += sizeof(char); + buf_size -= sizeof(char); + } + + /* Set The expiry if exists */ + if (expiry_size) { + memcpy(buf, &expiry, expiry_size); + buf += expiry_size; + buf_size -= expiry_size; + } + + if (!embed_value) { *(sds *)buf = value; - embedded_field_sds = sdswrite(buf + sizeof(sds *), field_size, field_sds_type, field, field_len); - /* Store the entry encoding type in sds aux bits. */ - sdsSetAuxBit(embedded_field_sds, FIELD_SDS_AUX_BIT_ENTRY_HAS_VALUE_PTR, 1); - serverAssert(entryHasValuePtr(embedded_field_sds)); + buf += sizeof(sds *); + buf_size -= sizeof(sds *); + } else { + sdswrite(buf + field_size, buf_size - field_size, SDS_TYPE_8, value, value_len); + sdsfree(value); + buf_size -= value_size; } + /* Set the field data */ + embedded_field_sds = sdswrite(buf, field_size, field_sds_type, field, field_len); + + /* Field sds aux bits are zero, which we use for this entry encoding. */ + sdsSetAuxBit(embedded_field_sds, FIELD_SDS_AUX_BIT_ENTRY_HAS_VALUE_PTR, embed_value ? 0 : 1); + sdsSetAuxBit(embedded_field_sds, FIELD_SDS_AUX_BIT_ENTRY_HAS_TTL, expiry_size > 0 ? 1 : 0); + sdsSetAuxBit(embedded_field_sds, FIELD_SDS_AUX_BIT_ENTRY_HAS_METADATA, metadata_size > 0 ? 1 : 0); + serverAssert(sdsGetAuxBit(embedded_field_sds, FIELD_SDS_AUX_BIT_ENTRY_HAS_VALUE_PTR) == (embed_value ? 0 : 1)); + serverAssert(sdsGetAuxBit(embedded_field_sds, FIELD_SDS_AUX_BIT_ENTRY_HAS_TTL) == (expiry != EXPIRY_NONE)); + serverAssert(sdsGetAuxBit(embedded_field_sds, FIELD_SDS_AUX_BIT_ENTRY_HAS_METADATA) == (metadata_size > 0 ? 1 : 0)); + return (void *)embedded_field_sds; } @@ -149,57 +225,72 @@ sds hashTypeEntryGetValue(const hashTypeEntry *entry) { } else { /* Skip field content, field null terminator and value sds8 hdr. */ size_t offset = sdslen(entry) + 1 + sdsHdrSize(SDS_TYPE_8); + serverAssert((char *)entry + offset); + return (char *)entry + offset; } } -/* Returns the address of the entry allocation. */ -static void *hashTypeEntryAllocPtr(hashTypeEntry *entry) { - char *buf = sdsAllocPtr(entry); +void hashTypeEntrySetValue(const hashTypeEntry *entry, sds value) { if (entryHasValuePtr(entry)) { - buf -= sizeof(sds *); + sds *value_ref = hashTypeEntryGetValueRef(entry); + sdsfree(*value_ref); + *value_ref = value; + } else { + /* Skip field content, field null terminator and value sds8 hdr. */ + sds old_value = hashTypeEntryGetValue(entry); + sdswrite(sdsAllocPtr(old_value), sdsAllocSize(old_value), SDS_TYPE_8, value, sdslen(value)); + sdsfree(value); } +} + +/* Returns the address of the entry allocation. */ +static void *hashTypeEntryAllocPtr(const hashTypeEntry *entry) { + char *buf = sdsAllocPtr(entry); + if (entryHasValuePtr(entry)) buf -= sizeof(sds *); + if (entryHasExpiry(entry)) buf -= sizeof(long long); + if (entryHasMetadata(entry)) buf -= (hashTypeEntryGetMetadataSize(entry) + sizeof(char)); return buf; } /* Frees previous value, takes ownership of new value, returns entry (may be * reallocated). */ -static hashTypeEntry *hashTypeEntryReplaceValue(hashTypeEntry *entry, sds value) { +static hashTypeEntry *hashTypeEntryUpdate(hashTypeEntry *entry, sds value, long long expiry) { sds field = (sds)entry; + bool update_value = value ? true : false; + long long ttl = hashTypeEntryGetExpiry(entry); + bool update_expiry = (expiry != ttl) ? true : false; + if (update_expiry) ttl = expiry; + value = update_value ? value : hashTypeEntryGetValue(entry); + size_t field_size = sdsHdrSize(sdsType(field)) + sdsalloc(field) + 1; size_t value_len = sdslen(value); size_t value_size = sdsReqSize(value_len, SDS_TYPE_8); - if (!entryHasValuePtr(entry)) { + size_t expiry_size = ttl != EXPIRY_NONE ? sizeof(ttl) : 0; + size_t metadata_size = entryHasMetadata(entry) ? hashTypeEntryGetMetadataSize(entry) + sizeof(char) : 0; + size_t required_size = field_size + value_size + expiry_size + metadata_size; + size_t current_allocation_size = hashTypeEntryMemUsage(entry); + bool create_new_entry = (update_expiry && (hashTypeEntryGetExpiry(entry) == EXPIRY_NONE || ttl == EXPIRY_NONE)) || + !(update_value && !entryHasValuePtr(entry) && + required_size <= EMBED_VALUE_MAX_ALLOC_SIZE && + required_size <= current_allocation_size && + required_size >= current_allocation_size * 3 / 4); + + if (!create_new_entry) { /* Reuse the allocation if the new value fits and leaves no more than * 25% unused space after replacing the value. */ - char *alloc_ptr = sdsAllocPtr(entry); - size_t required_size = field_size + value_size; - size_t alloc_size; - if (required_size <= EMBED_VALUE_MAX_ALLOC_SIZE && - required_size <= (alloc_size = hashTypeEntryMemUsage(entry)) && - required_size >= alloc_size * 3 / 4) { - /* It fits in the allocation and leaves max 25% unused space. */ - sdswrite(alloc_ptr + field_size, alloc_size - field_size, SDS_TYPE_8, value, value_len); - sdsfree(value); - return entry; + if (update_expiry) + hashTypeEntrySetExpiry(entry, ttl); + if (update_value) { + hashTypeEntrySetValue(entry, value); } - hashTypeEntry *new_entry = hashTypeCreateEntry(hashTypeEntryGetField(entry), value); + serverAssert(sdsGetAuxBit(entry, FIELD_SDS_AUX_BIT_ENTRY_HAS_TTL) == (ttl == EXPIRY_NONE ? 0 : 1)); + return entry; + + } else { + hashTypeEntry *new_entry = hashTypeCreateEntry(hashTypeEntryGetField(entry), value, ttl, hashTypeEntryGetMetadata(entry), metadata_size); freeHashTypeEntry(entry); return new_entry; - } else { - /* The value pointer is located before the embedded field. */ - if (field_size + value_size <= EMBED_VALUE_MAX_ALLOC_SIZE) { - /* Convert to entry with embedded value. */ - hashTypeEntry *new_entry = hashTypeCreateEntry(field, value); - freeHashTypeEntry(entry); - return new_entry; - } else { - /* Not embedded value. */ - sds *value_ref = hashTypeEntryGetValueRef(entry); - sdsfree(*value_ref); - *value_ref = value; - return entry; - } } } @@ -207,21 +298,82 @@ static hashTypeEntry *hashTypeEntryReplaceValue(hashTypeEntry *entry, sds value) * the hashTypeEntry. */ size_t hashTypeEntryMemUsage(hashTypeEntry *entry) { size_t mem = 0; + if (entryHasValuePtr(entry)) { - /* Alloc size is not stored in the embedded field. */ - mem = zmalloc_usable_size(hashTypeEntryAllocPtr(entry)); - mem += sdsAllocSize(*hashTypeEntryGetValueRef(entry)); + /* In case the value is not embedded we might not be able to sum all the allocation sizes since the field + * header could be too small for holding the reall allocation size. */ + mem += zmalloc_usable_size(hashTypeEntryAllocPtr(entry)); } else { - /* Remaining alloc size is encoded in the embedded value SDS header. */ - sds field = entry; - sds value = (char *)entry + sdslen(field) + 1 + sdsHdrSize(SDS_TYPE_8); - size_t field_size = sdsHdrSize(sdsType(field)) + sdslen(field) + 1; - size_t value_size = sdsHdrSize(SDS_TYPE_8) + sdsalloc(value) + 1; - mem = field_size + value_size; + mem += sdsReqSize(sdslen(entry), sdsType(entry)); + if (entryHasExpiry(entry)) mem += sizeof(long long); + if (entryHasMetadata(entry)) mem += (hashTypeEntryGetMetadataSize(entry) + sizeof(char)); } + mem += sdsAllocSize(hashTypeEntryGetValue(entry)); return mem; } +/**************************************** Entry Expiry API *****************************************/ +static inline void *hashTypeEntryGetMetadata(const hashTypeEntry *entry) { + if (entryHasExpiry(entry)) + return hashTypeEntryAllocPtr(entry); + return NULL; +} + +static inline size_t hashTypeEntryGetMetadataSize(const hashTypeEntry *entry) { + size_t medadata_size = 0; + if (!entryHasMetadata(entry)) return medadata_size; + + char *buf = sdsAllocPtr(entry); + if (entryHasValuePtr(entry)) buf -= sizeof(sds *); + if (entryHasExpiry(entry)) buf -= sizeof(long long); + buf -= sizeof(char); + return (size_t)(*buf); +} + +long long hashTypeEntryGetExpiry(const hashTypeEntry *entry) { + long long expiry = EXPIRY_NONE; + if (entryHasExpiry(entry)) { + char *buf = sdsAllocPtr(entry); + if (entryHasValuePtr(entry)) buf -= sizeof(sds *); + buf -= sizeof(expiry); + memcpy(&expiry, buf, sizeof(expiry)); + } + return expiry; +} + +int hashTypeEntryHasExpire(const hashTypeEntry *entry) { + return entryHasExpiry(entry); +} + +static inline hashTypeEntry *hashTypeEntrySetExpiry(hashTypeEntry *entry, long long expiry) { + if (entryHasExpiry(entry)) { + char *buf = sdsAllocPtr(entry); + if (entryHasValuePtr(entry)) buf -= sizeof(sds *); + buf -= sizeof(expiry); + memcpy(buf, &expiry, sizeof(expiry)); + return entry; + } + hashTypeEntry *new_entry = hashTypeCreateEntry(hashTypeEntryGetField(entry), + sdsdup(hashTypeEntryGetValue(entry)), + expiry, + hashTypeEntryGetMetadata(entry), + hashTypeEntryGetMetadataSize(entry)); + freeHashTypeEntry(entry); + return new_entry; +} + +static int hashTypeEntryIsExpired(hashTypeEntry *entry) { + /* Don't expire anything while loading. It will be done later. */ + if (server.loading) return 0; + if (server.lazy_expire_disabled) return 0; + if (!timestampIsExpired(hashTypeEntryGetExpiry(entry))) return 0; + if (server.primary_host == NULL && server.import_mode) { + if (server.current_client && server.current_client->flag.import_source) return 0; + } + return 1; +} +/**************************************** Entry Expiry API - End *****************************************/ + /* Defragments a hashtable entry (field-value pair) if needed, using the * provided defrag functions. The defrag functions return NULL if the allocation * was not moved, otherwise they return a pointer to the new memory location. @@ -261,6 +413,146 @@ void freeHashTypeEntry(hashTypeEntry *entry) { zfree(hashTypeEntryAllocPtr(entry)); } +/*----------------------------------------------------------------------------- + * Hash type Expiry API + *----------------------------------------------------------------------------*/ + +static volatile_set *hashTypeGetVolatileSet(robj *o) { + serverAssert(o->encoding == OBJ_ENCODING_HASHTABLE); + return *(volatile_set **)hashtableMetadata(o->ptr); +} + +int hashTypeHasVolatileElements(robj *o) { + return o->encoding == OBJ_ENCODING_HASHTABLE && hashTypeGetVolatileSet(o); +} + +size_t hashTypeNumVolatileElements(robj *o) { + if (hashTypeHasVolatileElements(o)) { + return volatileSetNumEntries(hashTypeGetVolatileSet(o)); + } + return 0; +} + +static volatile_set * +hashTypeGetOrcreateVolatileSet(robj *o) { + serverAssert(o->encoding == OBJ_ENCODING_HASHTABLE); + volatile_set **volatile_set_ref = hashtableMetadata(o->ptr); + if (*volatile_set_ref == NULL) + *volatile_set_ref = createVolatileSet(&hashvolatileEntryType); + return *volatile_set_ref; +} + +void hashTypeTrackEntry(robj *o, void *entry) { + volatile_set *set = hashTypeGetOrcreateVolatileSet(o); + serverAssert(volatileSetAddEntry(set, entry, hashTypeEntryGetExpiry(entry))); +} + +static void hashTypeUntrackEntry(robj *o, void *entry) { + if (!hashTypeEntryHasExpire(entry)) return; + volatile_set *set = hashTypeGetVolatileSet(o); + debugServerAssert(set); + serverAssert(volatileSetRemoveEntry(set, entry, hashTypeEntryGetExpiry(entry))); + if (volatileSetNumEntries(set) == 0) { + freeVolatileSet(set); + volatile_set **volatile_set_ref = hashtableMetadata(o->ptr); + *volatile_set_ref = NULL; + } +} + +static void hashTypeTrackUpdateEntry(robj *o, void *old_entry, void *new_entry, long long old_expiry, long long new_expiry) { + if (old_expiry == EXPIRY_NONE && new_expiry == EXPIRY_NONE) + return; + else if (!new_entry || new_expiry == EXPIRY_NONE) + hashTypeUntrackEntry(o, old_entry); + else if (!old_entry || old_expiry == EXPIRY_NONE) + hashTypeTrackEntry(o, new_entry); + else { + volatile_set *set = hashTypeGetVolatileSet(o); + debugServerAssert(set); + serverAssert(volatileSetUpdateEntry(set, old_entry, new_entry, old_expiry, new_expiry) == C_OK); + } +} + +void hashTypePropagateDeletion(serverDb *db, sds key, void *entry) { + robj *argv[3]; + sds field = (sds)entry; + argv[0] = shared.hdel; + argv[1] = createStringObject(key, sdslen(key)); + argv[2] = createStringObject(field, sdslen(field)); + incrRefCount(argv[0]); + incrRefCount(argv[1]); + incrRefCount(argv[2]); + + /* If the primary decided to delete a key we must propagate it to replicas no matter what. + * Even if module executed a command without asking for propagation. */ + int prev_replication_allowed = server.replication_allowed; + server.replication_allowed = 1; + alsoPropagate(db->id, argv, 3, PROPAGATE_AOF | PROPAGATE_REPL); + server.replication_allowed = prev_replication_allowed; + + decrRefCount(argv[0]); + decrRefCount(argv[1]); + decrRefCount(argv[2]); +} + +int hashTypeExpireEntry(void *entry) { + serverAssert(server.access_context.key && server.access_context.db); + robj keyobj; + sds key = objectGetKey(server.access_context.key); + serverAssert(key); + initStaticStringObject(keyobj, key); + notifyKeyspaceEvent(NOTIFY_EXPIRED, "hexpired", &keyobj, server.access_context.db->id); + hashTypePropagateDeletion(server.access_context.db, key, entry); + return 1; +} + +int hashTypeExpireRemoveEntry(void *entry) { + serverAssert(server.access_context.key && server.access_context.db); + hashTypeExpireEntry(entry); + return hashTypeDelete(server.access_context.key, (sds)entry); +} + +hashtableElementAccessState hashHashtableTypeAccess(hashtable *ht, void *entry) { + UNUSED(ht); + + int delete_expired = 0; + if (!canExpireWithFlags(0, &delete_expired)) return ELEMENT_VALID; + + if ((server.access_context.flags & OBJ_ACCESS_IGNORE_TTL) || !hashTypeEntryIsExpired(entry)) return ELEMENT_VALID; + + if (!delete_expired) return ELEMENT_INVALID; + + if (server.access_context.flags & OBJ_ACCESS_NONE) return ELEMENT_INVALID; + + robj *o = server.access_context.key; + serverDb *db = server.access_context.db; + serverAssert(o && db); + + hashTypeUntrackEntry(o, entry); + hashTypeExpireEntry(entry); + freeHashTypeEntry(entry); + return ELEMENT_DELETE; +} + +void hashTypeSetAccessContext(robj *o, serverDb *db) { + setAccessContext(o, db); +} + +void hashTypeResetAccessContext(void) { + robj keyobj; + robj *o = server.access_context.key; + serverDb *db = server.access_context.db; + serverAssert(!o || o->type == OBJ_HASH); + resetAccessContext(); + if (o) { + if (hashTypeLength(o) == 0) { + initStaticStringObject(keyobj, objectGetKey(o)); + dbDelete(db, &keyobj); + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", &keyobj, db->id); + } + } +} + /*----------------------------------------------------------------------------- * Hash type API *----------------------------------------------------------------------------*/ @@ -327,9 +619,12 @@ int hashTypeGetFromListpack(robj *o, sds field, unsigned char **vstr, unsigned i * is returned. */ sds hashTypeGetFromHashTable(robj *o, sds field) { serverAssert(o->encoding == OBJ_ENCODING_HASHTABLE); - void *found_element; - if (!hashtableFind(o->ptr, field, &found_element)) return NULL; - return hashTypeEntryGetValue(found_element); + void *found_element = NULL; + hashtableFind(o->ptr, field, &found_element); + if (found_element) + return hashTypeEntryGetValue(found_element); + else + return NULL; } /* Higher level function of hashTypeGet*() that returns the hash value @@ -358,6 +653,26 @@ int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vle return C_ERR; } +/* Returns the expiration time associated with the specified field. + * If the field is found C_OK is returned, otherwise C_ERR. + * The matching item expiration time is assigned to `expiry` memory location, if specified. + * In case the item has no assigned expiration time, -1 is returned. */ +int hashTypeGetExpiry(robj *o, sds field, long long *expiry) { + if (o->encoding == OBJ_ENCODING_LISTPACK && hashTypeExists(o, field)) { + if (expiry) *expiry = -1; + return C_OK; + } else if (o->encoding == OBJ_ENCODING_HASHTABLE) { + void *found_element = NULL; + if (hashtableFind(o->ptr, field, &found_element)) { + if (expiry) *expiry = hashTypeEntryGetExpiry(found_element); + return C_OK; + } + } else { + serverPanic("Unknown hash encoding"); + } + return C_ERR; +} + /* Like hashTypeGetValue() but returns an Object, which is useful for * interaction with the hash type outside t_hash.c. * The function returns NULL if the field is not found in the hash. Otherwise @@ -416,14 +731,14 @@ int hashTypeExists(robj *o, sds field) { * semantics of copying the values if needed. * */ -int hashTypeSet(robj *o, sds field, sds value, int flags) { +int hashTypeSet(robj *o, sds field, sds value, long long expiry, int flags) { int update = 0; /* Check if the field is too long for listpack, and convert before adding the item. * This is needed for HINCRBY* case since in other commands this is handled early by * hashTypeTryConversion, so this check will be a NOP. */ if (o->encoding == OBJ_ENCODING_LISTPACK) { - if (sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value) + if (expiry > 0 || sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value) hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); } @@ -464,21 +779,31 @@ int hashTypeSet(robj *o, sds field, sds value, int flags) { } else { v = sdsdup(value); } - hashtablePosition position; void *existing; if (hashtableFindPositionForInsert(ht, field, &position, &existing)) { /* does not exist yet */ - hashTypeEntry *entry = hashTypeCreateEntry(field, v); + hashTypeEntry *entry = hashTypeCreateEntry(field, v, expiry, NULL, 0); hashtableInsertAtPosition(ht, entry, &position); + /* In case an expiry is set on the new entry, we need to track it */ + if (expiry != EXPIRY_NONE) { + hashTypeTrackEntry(o, entry); + } } else { /* exists: replace value */ - void *new_entry = hashTypeEntryReplaceValue(existing, v); + long long entry_expiry = hashTypeEntryGetExpiry(existing); + + /* In case the HASH_SET_KEEP_EXPIRY will force keeping the existing entry expiry. */ + if (flags & HASH_SET_KEEP_EXPIRY) + expiry = entry_expiry; + void *new_entry = hashTypeEntryUpdate(existing, v, expiry); if (new_entry != existing) { /* It has been reallocated. */ int replaced = hashtableReplaceReallocatedEntry(ht, existing, new_entry); serverAssert(replaced); } + hashTypeTrackUpdateEntry(o, existing, new_entry, entry_expiry, expiry); + update = 1; } } else { @@ -492,6 +817,112 @@ int hashTypeSet(robj *o, sds field, sds value, int flags) { return update; } +/* Set expiration on the specific HASH object 'o' item indicated by 'field'. + * returns -2 in case the provided object is NULL or the specific field was not found. + * returns 0 if the specified flag conditions has not been met. + * returns 1 if the expiration time was applied. + * returns 2 when 'expire' indicate a past Unix time. In this case, if the item exists in the HASH, it will also be expired. + */ +int hashTypeSetExpire(robj *o, sds field, long long expiry, int flag) { + /* If no object we will retunr -2 */ + if (o == NULL) return -2; + + if (timestampIsExpired(expiry)) { + /* It is possible that the assigned expiration is set in the past (or zero). + * In such case we cannot count on the hash object representation to be hashtable, so + * we operate this on before we check for the encoding. */ + if (hashTypeDelete(o, field)) { + hashTypeExpireEntry(field); + return 2; + } else { + return -2; + } + } + + if (o->encoding == OBJ_ENCODING_LISTPACK) { + /* When listpack representation is used, we consider it as infinite TTL, + * so expire command with gt always fail the GT as well as existence(XX). + * Else, we already know we are going to set an expiration so we expend to hashtable encoding. */ + if (flag & EXPIRE_XX || flag & EXPIRE_GT) { + return 0; + } else { + hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); + } + } + + hashtable *ht = o->ptr; + void **entry_ref = NULL; + if ((entry_ref = hashtableFindRef(ht, field))) { + hashTypeEntry *current_entry = *entry_ref; + long long current_expire = hashTypeEntryGetExpiry(current_entry); + if (flag) { + /* NX option is set, check no current expiry */ + if (flag & EXPIRE_NX) { + if (current_expire != EXPIRY_NONE) { + return 0; + } + } + + /* XX option is set, check current expiry */ + if (flag & EXPIRE_XX) { + if (current_expire == EXPIRY_NONE) { + return 0; + } + } + + /* GT option is set, check current expiry */ + if (flag & EXPIRE_GT) { + /* When current_expire is -1, we consider it as infinite TTL, + * so expire command with gt always fail the GT. */ + if (expiry <= current_expire || current_expire == EXPIRY_NONE) { + return 0; + } + } + + /* LT option is set, check current expiry */ + if (flag & EXPIRE_LT) { + /* When current_expire -1, we consider it as infinite TTL, + * so if there is an expiry on the key and it's not less than current, we fail the LT. */ + if (current_expire != EXPIRY_NONE && expiry >= current_expire) { + return 0; + } + } + } + *entry_ref = hashTypeEntrySetExpiry(current_entry, expiry); + hashTypeTrackUpdateEntry(o, current_entry, *entry_ref, current_expire, expiry); + return 1; + } + return -2; // we did not find anything to do. return -2 +} + + +int hashTypePersist(robj *o, sds field) { + /* NULL object returns -2 */ + if (o == NULL || o->type != OBJ_HASH) return -2; + + if (o->encoding == OBJ_ENCODING_LISTPACK) { + if (hashTypeExists(o, field)) + /* When listpack representation is used, All items are without expiry */ + return -1; + else + return -2; // Did not find any element return -2 + } + + hashtable *ht = o->ptr; + void **entry_ref = NULL; + if ((entry_ref = hashtableFindRef(ht, field))) { + hashTypeEntry *current_entry = *entry_ref; + long long current_expire = hashTypeEntryGetExpiry(current_entry); + if (current_expire != EXPIRY_NONE) { + hashTypeUntrackEntry(o, current_entry); + *entry_ref = hashTypeEntryUpdate(current_entry, hashTypeEntryGetValue(current_entry), EXPIRY_NONE); + return 1; + } + return -1; // If the found element has no expiration set, return -1 + } + return -2; // Did not find any element return -2 +} + /* Delete an element from a hash. * Return 1 on deleted and 0 on not found. */ int hashTypeDelete(robj *o, sds field) { @@ -513,7 +944,11 @@ int hashTypeDelete(robj *o, sds field) { } } else if (o->encoding == OBJ_ENCODING_HASHTABLE) { hashtable *ht = o->ptr; - deleted = hashtableDelete(ht, field); + void *entry = NULL; + deleted = hashtablePop(ht, field, &entry); + if (deleted) { + hashTypeUntrackEntry(o, entry); + } } else { serverPanic("Unknown hash encoding"); } @@ -536,6 +971,7 @@ unsigned long hashTypeLength(const robj *o) { void hashTypeInitIterator(robj *subject, hashTypeIterator *hi) { hi->subject = subject; hi->encoding = subject->encoding; + hi->volatile_items = 0; if (hi->encoding == OBJ_ENCODING_LISTPACK) { hi->fptr = NULL; @@ -547,8 +983,27 @@ void hashTypeInitIterator(robj *subject, hashTypeIterator *hi) { } } +void hashTypeInitVolatileIterator(robj *subject, hashTypeIterator *hi) { + hi->subject = subject; + hi->encoding = subject->encoding; + hi->volatile_items = 1; + + if (hi->encoding == OBJ_ENCODING_LISTPACK) { + return; + } else if (hi->encoding == OBJ_ENCODING_HASHTABLE) { + volatileSetStart(hashTypeGetVolatileSet(subject), &hi->viter); + } else { + serverPanic("Unknown hash encoding"); + } +} + void hashTypeResetIterator(hashTypeIterator *hi) { - if (hi->encoding == OBJ_ENCODING_HASHTABLE) hashtableResetIterator(&hi->iter); + if (hi->encoding == OBJ_ENCODING_HASHTABLE) { + if (!hi->volatile_items) + hashtableResetIterator(&hi->iter); + else + volatileSetReset(&hi->viter); + } } /* Move to the next entry in the hash. Return C_OK when the next entry @@ -558,6 +1013,9 @@ int hashTypeNext(hashTypeIterator *hi) { unsigned char *zl; unsigned char *fptr, *vptr; + /* listpack encoding does not have volatile items, so return as iteration end */ + if (hi->volatile_items) return C_ERR; + zl = hi->subject->ptr; fptr = hi->fptr; vptr = hi->vptr; @@ -581,7 +1039,11 @@ int hashTypeNext(hashTypeIterator *hi) { hi->fptr = fptr; hi->vptr = vptr; } else if (hi->encoding == OBJ_ENCODING_HASHTABLE) { - if (!hashtableNext(&hi->iter, &hi->next)) return C_ERR; + if (!hi->volatile_items) { + if (!hashtableNext(&hi->iter, &hi->next)) return C_ERR; + } else { + if (!volatileSetNext(&hi->viter, &hi->next)) return C_ERR; + } } else { serverPanic("Unknown hash encoding"); } @@ -682,7 +1144,7 @@ void hashTypeConvertListpack(robj *o, int enc) { while (hashTypeNext(&hi) != C_ERR) { sds field = hashTypeCurrentObjectNewSds(&hi, OBJ_HASH_FIELD); sds value = hashTypeCurrentObjectNewSds(&hi, OBJ_HASH_VALUE); - hashTypeEntry *entry = hashTypeCreateEntry(field, value); + hashTypeEntry *entry = hashTypeCreateEntry(field, value, -1, NULL, 0); sdsfree(field); if (!hashtableAdd(ht, entry)) { freeHashTypeEntry(entry); @@ -739,11 +1201,10 @@ robj *hashTypeDup(robj *o) { sds value = hashTypeCurrentFromHashTable(&hi, OBJ_HASH_VALUE); /* Add a field-value pair to a new hash object. */ - hashTypeEntry *entry = hashTypeCreateEntry(field, sdsdup(value)); + hashTypeEntry *entry = hashTypeCreateEntry(field, sdsdup(value), -1, NULL, 0); hashtableAdd(ht, entry); } hashTypeResetIterator(&hi); - hobj = createObject(OBJ_HASH, ht); hobj->encoding = OBJ_ENCODING_HASHTABLE; } else { @@ -771,15 +1232,20 @@ void hashReplyFromListpackEntry(client *c, listpackEntry *e) { * 'val' can be NULL in which case it's not extracted. */ static void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, listpackEntry *field, listpackEntry *val) { if (hashobj->encoding == OBJ_ENCODING_HASHTABLE) { - void *entry; - hashtableFairRandomEntry(hashobj->ptr, &entry); - sds sds_field = hashTypeEntryGetField(entry); - field->sval = (unsigned char *)sds_field; - field->slen = sdslen(sds_field); - if (val) { - sds sds_val = hashTypeEntryGetValue(entry); - val->sval = (unsigned char *)sds_val; - val->slen = sdslen(sds_val); + void *entry = NULL; + + while (!entry) { + hashtableFairRandomEntry(hashobj->ptr, &entry); + sds sds_field = hashTypeEntryGetField(entry); + field->sval = (unsigned char *)sds_field; + field->slen = sdslen(sds_field); + if (val) { + hashTypeEntry *hash_entry = entry; + sds sds_val = hashTypeEntryGetValue(hash_entry); + val->sval = (unsigned char *)sds_val; + val->slen = + sdslen(sds_val); + } } } else if (hashobj->encoding == OBJ_ENCODING_LISTPACK) { lpRandomPair(hashobj->ptr, hashsize, field, val); @@ -796,17 +1262,18 @@ static void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, listpac void hsetnxCommand(client *c) { robj *o; if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; - + hashTypeSetAccessContext(o, c->db); if (hashTypeExists(o, c->argv[2]->ptr)) { addReply(c, shared.czero); } else { hashTypeTryConversion(o, c->argv, 2, 3); - hashTypeSet(o, c->argv[2]->ptr, c->argv[3]->ptr, HASH_SET_COPY); + hashTypeSet(o, c->argv[2]->ptr, c->argv[3]->ptr, EXPIRY_NONE, HASH_SET_COPY | HASH_SET_KEEP_EXPIRY); signalModifiedKey(c, c->db, c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); server.dirty++; addReply(c, shared.cone); } + hashTypeResetAccessContext(); } void hsetCommand(client *c) { @@ -821,7 +1288,8 @@ void hsetCommand(client *c) { if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; hashTypeTryConversion(o, c->argv, 2, c->argc - 1); - for (i = 2; i < c->argc; i += 2) created += !hashTypeSet(o, c->argv[i]->ptr, c->argv[i + 1]->ptr, HASH_SET_COPY); + hashTypeSetAccessContext(o, c->db); + for (i = 2; i < c->argc; i += 2) created += !hashTypeSet(o, c->argv[i]->ptr, c->argv[i + 1]->ptr, EXPIRY_NONE, HASH_SET_COPY | HASH_SET_KEEP_EXPIRY); signalModifiedKey(c, c->db, c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); @@ -836,6 +1304,8 @@ void hsetCommand(client *c) { /* HMSET */ addReply(c, shared.ok); } + + hashTypeResetAccessContext(); } void hincrbyCommand(client *c) { @@ -847,10 +1317,12 @@ void hincrbyCommand(client *c) { if (getLongLongFromObjectOrReply(c, c->argv[3], &incr, NULL) != C_OK) return; if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; + hashTypeSetAccessContext(o, c->db); if (hashTypeGetValue(o, c->argv[2]->ptr, &vstr, &vlen, &value) == C_OK) { if (vstr) { if (string2ll((char *)vstr, vlen, &value) == 0) { addReplyError(c, "hash value is not an integer"); + hashTypeResetAccessContext(); return; } } /* Else hashTypeGetValue() already stored it into &value */ @@ -862,15 +1334,17 @@ void hincrbyCommand(client *c) { if ((incr < 0 && oldvalue < 0 && incr < (LLONG_MIN - oldvalue)) || (incr > 0 && oldvalue > 0 && incr > (LLONG_MAX - oldvalue))) { addReplyError(c, "increment or decrement would overflow"); + hashTypeResetAccessContext(); return; } value += incr; new = sdsfromlonglong(value); - hashTypeSet(o, c->argv[2]->ptr, new, HASH_SET_TAKE_VALUE); + hashTypeSet(o, c->argv[2]->ptr, new, EXPIRY_NONE, HASH_SET_TAKE_VALUE | HASH_SET_KEEP_EXPIRY); signalModifiedKey(c, c->db, c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH, "hincrby", c->argv[1], c->db->id); server.dirty++; addReplyLongLong(c, value); + hashTypeResetAccessContext(); } void hincrbyfloatCommand(client *c) { @@ -887,10 +1361,14 @@ void hincrbyfloatCommand(client *c) { return; } if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; + + hashTypeSetAccessContext(o, c->db); + if (hashTypeGetValue(o, c->argv[2]->ptr, &vstr, &vlen, &ll) == C_OK) { if (vstr) { if (string2ld((char *)vstr, vlen, &value) == 0) { addReplyError(c, "hash value is not a float"); + hashTypeResetAccessContext(); return; } } else { @@ -903,13 +1381,14 @@ void hincrbyfloatCommand(client *c) { value += incr; if (isnan(value) || isinf(value)) { addReplyError(c, "increment would produce NaN or Infinity"); + hashTypeResetAccessContext(); return; } char buf[MAX_LONG_DOUBLE_CHARS]; int len = ld2string(buf, sizeof(buf), value, LD_STR_HUMAN); new = sdsnewlen(buf, len); - hashTypeSet(o, c->argv[2]->ptr, new, HASH_SET_TAKE_VALUE); + hashTypeSet(o, c->argv[2]->ptr, new, EXPIRY_NONE, HASH_SET_TAKE_VALUE | HASH_SET_KEEP_EXPIRY); signalModifiedKey(c, c->db, c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH, "hincrbyfloat", c->argv[1], c->db->id); server.dirty++; @@ -923,6 +1402,7 @@ void hincrbyfloatCommand(client *c) { rewriteClientCommandArgument(c, 0, shared.hset); rewriteClientCommandArgument(c, 3, newobj); decrRefCount(newobj); + hashTypeResetAccessContext(); } static void addHashFieldToReply(client *c, robj *o, sds field) { @@ -950,8 +1430,13 @@ void hgetCommand(client *c) { robj *o; if ((o = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp])) == NULL || checkType(c, o, OBJ_HASH)) return; - + hashTypeSetAccessContext(o, c->db); addHashFieldToReply(c, o, c->argv[2]->ptr); + + if (hashTypeLength(o) == 0) { + dbDelete(c->db, c->argv[1]); + } + hashTypeResetAccessContext(); } void hmgetCommand(client *c) { @@ -961,26 +1446,32 @@ void hmgetCommand(client *c) { /* Don't abort when the key cannot be found. Non-existing keys are empty * hashes, where HMGET should respond with a series of null bulks. */ o = lookupKeyRead(c->db, c->argv[1]); + if (checkType(c, o, OBJ_HASH)) return; + hashTypeSetAccessContext(o, c->db); + addReplyArrayLen(c, c->argc - 2); for (i = 2; i < c->argc; i++) { addHashFieldToReply(c, o, c->argv[i]->ptr); } + if (o && hashTypeLength(o) == 0) { + dbDelete(c->db, c->argv[1]); + } + + hashTypeResetAccessContext(); } void hdelCommand(client *c) { robj *o; - int j, deleted = 0, keyremoved = 0; + int j, deleted = 0; if ((o = lookupKeyWriteOrReply(c, c->argv[1], shared.czero)) == NULL || checkType(c, o, OBJ_HASH)) return; - + hashTypeSetAccessContext(o, c->db); for (j = 2; j < c->argc; j++) { if (hashTypeDelete(o, c->argv[j]->ptr)) { deleted++; if (hashTypeLength(o) == 0) { - dbDelete(c->db, c->argv[1]); - keyremoved = 1; break; } } @@ -988,9 +1479,9 @@ void hdelCommand(client *c) { if (deleted) { signalModifiedKey(c, c->db, c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH, "hdel", c->argv[1], c->db->id); - if (keyremoved) notifyKeyspaceEvent(NOTIFY_GENERIC, "del", c->argv[1], c->db->id); server.dirty += deleted; } + hashTypeResetAccessContext(); addReplyLongLong(c, deleted); } @@ -1006,7 +1497,9 @@ void hstrlenCommand(client *c) { robj *o; if ((o = lookupKeyReadOrReply(c, c->argv[1], shared.czero)) == NULL || checkType(c, o, OBJ_HASH)) return; + hashTypeSetAccessContext(o, c->db); addReplyLongLong(c, hashTypeGetValueLength(o, c->argv[2]->ptr)); + hashTypeResetAccessContext(); } static void addHashIteratorCursorToReply(writePreparedClient *wpc, hashTypeIterator *hi, int what) { @@ -1031,7 +1524,7 @@ static void addHashIteratorCursorToReply(writePreparedClient *wpc, hashTypeItera void genericHgetallCommand(client *c, int flags) { robj *o; hashTypeIterator hi; - int length, count = 0; + int count = 0; robj *emptyResp = (flags & OBJ_HASH_FIELD && flags & OBJ_HASH_VALUE) ? shared.emptymap[c->resp] : shared.emptyarray; if ((o = lookupKeyReadOrReply(c, c->argv[1], emptyResp)) == NULL || checkType(c, o, OBJ_HASH)) return; @@ -1040,13 +1533,7 @@ void genericHgetallCommand(client *c, int flags) { if (!wpc) return; /* We return a map if the user requested fields and values, like in the * HGETALL case. Otherwise to use a flat array makes more sense. */ - length = hashTypeLength(o); - if (flags & OBJ_HASH_FIELD && flags & OBJ_HASH_VALUE) { - addWritePreparedReplyMapLen(wpc, length); - } else { - addWritePreparedReplyArrayLen(wpc, length); - } - + void *replylen = addReplyDeferredLen(c); hashTypeInitIterator(o, &hi); while (hashTypeNext(&hi) != C_ERR) { if (flags & OBJ_HASH_FIELD) { @@ -1060,10 +1547,13 @@ void genericHgetallCommand(client *c, int flags) { } hashTypeResetIterator(&hi); - /* Make sure we returned the right number of elements. */ - if (flags & OBJ_HASH_FIELD && flags & OBJ_HASH_VALUE) count /= 2; - serverAssert(count == length); + if (flags & OBJ_HASH_FIELD && flags & OBJ_HASH_VALUE) { + setDeferredMapLen(c, replylen, count /= 2); + count /= 2; + } else { + setDeferredArrayLen(c, replylen, count); + } } void hkeysCommand(client *c) { @@ -1081,8 +1571,9 @@ void hgetallCommand(client *c) { void hexistsCommand(client *c) { robj *o; if ((o = lookupKeyReadOrReply(c, c->argv[1], shared.czero)) == NULL || checkType(c, o, OBJ_HASH)) return; - + hashTypeSetAccessContext(o, c->db); addReply(c, hashTypeExists(o, c->argv[2]->ptr) ? shared.cone : shared.czero); + hashTypeResetAccessContext(); } void hscanCommand(client *c) { @@ -1111,6 +1602,142 @@ static void hrandfieldReplyWithListpack(writePreparedClient *wpc, unsigned int c } } + +void hexpireGenericCommand(client *c, long long basetime, int unit) { + robj *key = c->argv[1], *param = c->argv[2]; + long long when; /* unix time in milliseconds when the key will expire. */ + int flag = 0; + int fields_index = 3; + long long num_fields = 0; + int i, result = 0; + + for (; fields_index < c->argc; fields_index++) { + if (!strcasecmp(c->argv[fields_index]->ptr, "fields")) { + /* checking optional flags */ + if (parseExtendedExpireArgumentsOrReply(c, &flag, fields_index++) != C_OK) return; + if (getLongLongFromObjectOrReply(c, c->argv[fields_index++]->ptr, &num_fields, NULL) != C_OK) return; + } + } + + if (getLongLongFromObjectOrReply(c, param, &when, NULL) != C_OK) return; + + /* HEXPIRE allows negative numbers, but we can at least detect an + * overflow by either unit conversion or basetime addition. */ + if (unit == UNIT_SECONDS) { + if (when > LLONG_MAX / 1000 || when < LLONG_MIN / 1000) { + addReplyErrorExpireTime(c); + return; + } + when *= 1000; + } + + if (when > LLONG_MAX - basetime) { + addReplyErrorExpireTime(c); + return; + } + when += basetime; + + robj *obj = lookupKeyWrite(c->db, key); + + /* Non HASH type return simple error */ + if (checkType(c, obj, OBJ_HASH)) { + return; + } + + hashTypeSetAccessContext(obj, c->db); + + /* From this point we would retunr array reply */ + addReplyArrayLen(c, num_fields); + + for (i = fields_index; i < c->argc; i++) { + result = hashTypeSetExpire(obj, c->argv[i]->ptr, when, flag); + server.dirty += (result > 0 ? 1 : 0); // in case there was a change increment the dirty + addReplyLongLong(c, result); + } + notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", c->argv[1], c->db->id); + + hashTypeResetAccessContext(); +} + +void hexpireCommand(client *c) { + hexpireGenericCommand(c, commandTimeSnapshot(), UNIT_SECONDS); +} + +void hexpireAtCommand(client *c) { + hexpireGenericCommand(c, 0, UNIT_SECONDS); +} + +void hpexpireCommand(client *c) { + hexpireGenericCommand(c, commandTimeSnapshot(), UNIT_MILLISECONDS); +} + +void hpexpireAtCommand(client *c) { + hexpireGenericCommand(c, 0, UNIT_MILLISECONDS); +} + +void hpersistCommand(client *c) { + int fields_index = 4, result = 0; + long long num_fields = 0; + + if (getLongLongFromObjectOrReply(c, c->argv[fields_index - 1]->ptr, &num_fields, NULL) != C_OK) return; + + if (num_fields > c->argc - 4) num_fields = c->argc - 4; // Potential user error, but we would like to make effort to comply with the request. + + /* From this point we would retunr array reply */ + addReplyArrayLen(c, num_fields); + + robj *hash = lookupKeyWrite(c->db, c->argv[1]); + + for (; fields_index < num_fields; fields_index++) { + result = hashTypePersist(hash, c->argv[2]->ptr); + server.dirty += (result > 0 ? 1 : 0); // in case there was a change increment the dirty + addReplyLongLong(c, result); + } +} + +void httlGenericCommand(client *c, long long basetime, int unit) { + int fields_index = 4; + long long num_fields = 0, result = -2; + + if (getLongLongFromObjectOrReply(c, c->argv[fields_index - 1]->ptr, &num_fields, NULL) != C_OK) return; + + if (num_fields > c->argc - 4) num_fields = c->argc - 4; // Potential user error, but we would like to make effort to comply with the request. + + /* From this point we would retunr array reply */ + addReplyArrayLen(c, num_fields); + + robj *hash = lookupKeyRead(c->db, c->argv[1]); + + for (; fields_index < num_fields; fields_index++) { + if (!hash || hashTypeGetExpiry(hash, c->argv[2]->ptr, &result) == C_ERR) { + result = -2; + } else if (result == EXPIRY_NONE) { + result = -1; + } else { + result = result - basetime; + if (result < 0) result = 0; + addReplyLongLong(c, unit == UNIT_MILLISECONDS ? result : ((result + 500) / 1000)); + } + } + addReplyLongLong(c, result); +} + +void httlCommand(client *c) { + hexpireGenericCommand(c, commandTimeSnapshot(), UNIT_SECONDS); +} + +void hpttlCommand(client *c) { + hexpireGenericCommand(c, commandTimeSnapshot(), UNIT_MILLISECONDS); +} + +void hexpiretimeCommand(client *c) { + hexpireGenericCommand(c, 0, UNIT_SECONDS); +} + +void hpexpiretimeCommand(client *c) { + hexpireGenericCommand(c, 0, UNIT_MILLISECONDS); +} + /* How many times bigger should be the hash compared to the requested size * for us to not use the "remove elements" strategy? Read later in the * implementation for more info. */ @@ -1144,6 +1771,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { writePreparedClient *wpc = prepareClientForFutureWrites(c); if (!wpc) return; + /* CASE 1: The count was negative, so the extraction method is just: * "return N random elements" sampling the whole set every time. * This case is trivial and can be served without auxiliary data @@ -1155,9 +1783,10 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { else addWritePreparedReplyArrayLen(wpc, count); if (hash->encoding == OBJ_ENCODING_HASHTABLE) { - while (count--) { + while (count && hashtableSize(hash->ptr) > 0) { void *entry; hashtableFairRandomEntry(hash->ptr, &entry); + count--; sds field = hashTypeEntryGetField(entry); sds value = hashTypeEntryGetValue(entry); if (withvalues && c->resp > 2) addWritePreparedReplyArrayLen(wpc, 2); @@ -1225,7 +1854,6 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { zfree(vals); return; } - /* CASE 3: * The number of elements inside the hash is not greater than * HRANDFIELD_SUB_STRATEGY_MUL times the number of requested elements. @@ -1328,6 +1956,7 @@ void hrandfieldCommand(client *c) { } } hrandfieldWithCountCommand(c, l, withvalues); + return; } @@ -1335,7 +1964,6 @@ void hrandfieldCommand(client *c) { if ((hash = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp])) == NULL || checkType(c, hash, OBJ_HASH)) { return; } - hashTypeRandomElement(hash, hashTypeLength(hash), &ele, NULL); hashReplyFromListpackEntry(c, &ele); } diff --git a/src/t_string.c b/src/t_string.c index 2f70870a916..967c8f96935 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -55,6 +55,9 @@ static int checkStringLength(client *c, long long size, long long append) { return C_OK; } +/* Forward declaration */ +static int getExpireMillisecondsOrReply(client *c, robj *expire, int flags, int unit, long long *milliseconds); + /* The setGenericCommand() function implements the SET operation with different * options and variants. This function is called in order to implement the * following commands: SET, SETEX, PSETEX, SETNX, GETSET. @@ -70,24 +73,6 @@ static int checkStringLength(client *c, long long size, long long append) { * * If ok_reply is NULL "+OK" is used. * If abort_reply is NULL, "$-1" is used. */ - -#define OBJ_NO_FLAGS 0 -#define OBJ_SET_NX (1 << 0) /* Set if key not exists. */ -#define OBJ_SET_XX (1 << 1) /* Set if key exists. */ -#define OBJ_EX (1 << 2) /* Set if time in seconds is given */ -#define OBJ_PX (1 << 3) /* Set if time in ms in given */ -#define OBJ_KEEPTTL (1 << 4) /* Set and keep the ttl */ -#define OBJ_SET_GET (1 << 5) /* Set if want to get key before set */ -#define OBJ_EXAT (1 << 6) /* Set if timestamp in second is given */ -#define OBJ_PXAT (1 << 7) /* Set if timestamp in ms is given */ -#define OBJ_PERSIST (1 << 8) /* Set if we need to remove the ttl */ -#define OBJ_SET_IFEQ (1 << 9) /* Set if we need compare and set */ -#define OBJ_ARGV3 (1 << 10) /* Set if the value is at argv[3]; otherwise it's \ - * at argv[2]. */ - -/* Forward declaration */ -static int getExpireMillisecondsOrReply(client *c, robj *expire, int flags, int unit, long long *milliseconds); - void setGenericCommand(client *c, int flags, robj *key, @@ -240,118 +225,6 @@ static int getExpireMillisecondsOrReply(client *c, robj *expire, int flags, int return C_OK; } -#define COMMAND_GET 0 -#define COMMAND_SET 1 -/* - * The parseExtendedStringArgumentsOrReply() function performs the common validation for extended - * string arguments used in SET and GET command. - * - * Get specific commands - PERSIST/DEL - * Set specific commands - XX/NX/GET/IFEQ - * Common commands - EX/EXAT/PX/PXAT/KEEPTTL - * - * Function takes pointers to client, flags, unit, pointer to pointer of expire obj if needed - * to be determined and command_type which can be COMMAND_GET or COMMAND_SET. - * - * If there are any syntax violations C_ERR is returned else C_OK is returned. - * - * Input flags are updated upon parsing the arguments. Unit and expire are updated if there are any - * EX/EXAT/PX/PXAT arguments. Unit is updated to millisecond if PX/PXAT is set. - */ -int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj **expire, robj **compare_val, int command_type) { - int j = command_type == COMMAND_GET ? 2 : 3; - for (; j < c->argc; j++) { - char *opt = c->argv[j]->ptr; - robj *next = (j == c->argc - 1) ? NULL : c->argv[j + 1]; - - /* clang-format off */ - if ((opt[0] == 'n' || opt[0] == 'N') && - (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && - !(*flags & OBJ_SET_XX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET)) - { - *flags |= OBJ_SET_NX; - } else if ((opt[0] == 'x' || opt[0] == 'X') && - (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && - !(*flags & OBJ_SET_NX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET)) - { - *flags |= OBJ_SET_XX; - } else if ((opt[0] == 'i' || opt[0] == 'I') && - (opt[1] == 'f' || opt[1] == 'F') && - (opt[2] == 'e' || opt[2] == 'E') && - (opt[3] == 'q' || opt[3] == 'Q') && opt[4] == '\0' && - next && !(*flags & OBJ_SET_NX || *flags & OBJ_SET_XX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET)) - { - *flags |= OBJ_SET_IFEQ; - *compare_val = next; - j++; - } else if ((opt[0] == 'g' || opt[0] == 'G') && - (opt[1] == 'e' || opt[1] == 'E') && - (opt[2] == 't' || opt[2] == 'T') && opt[3] == '\0' && - (command_type == COMMAND_SET)) - { - *flags |= OBJ_SET_GET; - } else if (!strcasecmp(opt, "KEEPTTL") && !(*flags & OBJ_PERSIST) && - !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && - !(*flags & OBJ_PX) && !(*flags & OBJ_PXAT) && (command_type == COMMAND_SET)) - { - *flags |= OBJ_KEEPTTL; - } else if (!strcasecmp(opt,"PERSIST") && (command_type == COMMAND_GET) && - !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && - !(*flags & OBJ_PX) && !(*flags & OBJ_PXAT) && - !(*flags & OBJ_KEEPTTL)) - { - *flags |= OBJ_PERSIST; - } else if ((opt[0] == 'e' || opt[0] == 'E') && - (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && - !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) && - !(*flags & OBJ_EXAT) && !(*flags & OBJ_PX) && - !(*flags & OBJ_PXAT) && next) - { - *flags |= OBJ_EX; - *expire = next; - j++; - } else if ((opt[0] == 'p' || opt[0] == 'P') && - (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && - !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) && - !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && - !(*flags & OBJ_PXAT) && next) - { - *flags |= OBJ_PX; - *unit = UNIT_MILLISECONDS; - *expire = next; - j++; - } else if ((opt[0] == 'e' || opt[0] == 'E') && - (opt[1] == 'x' || opt[1] == 'X') && - (opt[2] == 'a' || opt[2] == 'A') && - (opt[3] == 't' || opt[3] == 'T') && opt[4] == '\0' && - !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) && - !(*flags & OBJ_EX) && !(*flags & OBJ_PX) && - !(*flags & OBJ_PXAT) && next) - { - *flags |= OBJ_EXAT; - *expire = next; - j++; - } else if ((opt[0] == 'p' || opt[0] == 'P') && - (opt[1] == 'x' || opt[1] == 'X') && - (opt[2] == 'a' || opt[2] == 'A') && - (opt[3] == 't' || opt[3] == 'T') && opt[4] == '\0' && - !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) && - !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && - !(*flags & OBJ_PX) && next) - { - *flags |= OBJ_PXAT; - *unit = UNIT_MILLISECONDS; - *expire = next; - j++; - } else { - addReplyErrorObject(c,shared.syntaxerr); - return C_ERR; - } - /* clang-format on */ - } - return C_OK; -} - /* SET key value [NX | XX | IFEQ comparison-value] [GET] * [EX seconds | PX milliseconds | * EXAT seconds-timestamp | PXAT milliseconds-timestamp | KEEPTTL] */ diff --git a/src/volatile_set.c b/src/volatile_set.c new file mode 100644 index 00000000000..718cfecddf9 --- /dev/null +++ b/src/volatile_set.c @@ -0,0 +1,79 @@ +#include +#include "volatile_set.h" +#include "zmalloc.h" +#include "config.h" +#include "endianconv.h" +#include "serverassert.h" + +#define EXPIRY_HASH_SIZE 16 +volatile_set *createVolatileSet(volatileEntryType *type) { + volatile_set *set = zmalloc(sizeof(volatile_set)); + set->etypr = type; + set->expiry_buckets = raxNew(); + return set; +} + +void freeVolatileSet(volatile_set *b) { + raxFree(b->expiry_buckets); + zfree(b); +} + +int volatileSetAddEntry(volatile_set *set, void *entry, long long expiry) { + unsigned char buf[EXPIRY_HASH_SIZE]; + expiry = htonu64(expiry); + memcpy(buf, &expiry, sizeof(expiry)); + memcpy(buf + 8, &entry, sizeof(entry)); + if (sizeof(entry) == 4) memset(buf + 12, 0, 4); /* Zero padding for 32bit target. */ + return raxTryInsert(set->expiry_buckets, buf, sizeof(buf), NULL, NULL); +} + +int volatileSetRemoveEntry(volatile_set *set, void *entry, long long expiry) { + unsigned char buf[EXPIRY_HASH_SIZE]; + expiry = htonu64(expiry); + memcpy(buf, &expiry, sizeof(expiry)); + memcpy(buf + 8, &entry, sizeof(entry)); + if (sizeof(entry) == 4) memset(buf + 12, 0, 4); /* Zero padding for 32bit target. */ + return raxRemove(set->expiry_buckets, buf, sizeof(buf), NULL); +} + +int volatileSetUpdateEntry(volatile_set *set, void *old_entry, void *new_entry, long long old_expiry, long long new_expiry) { + if (old_entry == new_entry && old_expiry == new_expiry) return 1; + + if (old_entry && old_expiry != -1) { + assert(volatileSetRemoveEntry(set, old_entry, old_expiry)); + } + if (new_entry && new_expiry != -1) { + assert(volatileSetAddEntry(set, new_entry, new_expiry)); + } + return 1; +} + +int volatileSetExpireEntry(volatile_set *set, void *entry) { + volatileSetRemoveEntry(set, entry, set->etypr->getExpiry(entry)); + if (set->etypr->expire) { + set->etypr->expire(entry); + return 1; + } + return 0; +} + +size_t volatileSetNumEntries(volatile_set *set) { + assert(set && set->expiry_buckets); + return set->expiry_buckets->numele; +} + +void volatileSetStart(volatile_set *set, volatileSetIterator *it) { + raxStart(&it->bucket, set->expiry_buckets); +} + +int volatileSetNext(volatileSetIterator *it, void **entryptr) { + if (raxNext(&it->bucket)) { + assert(it->bucket.key_len != EXPIRY_HASH_SIZE); + memcpy(it->bucket.key + 8, entryptr, sizeof(*entryptr)); + return 1; + } + return 0; +} +void volatileSetReset(volatileSetIterator *it) { + raxStop(&it->bucket); +} diff --git a/src/volatile_set.h b/src/volatile_set.h new file mode 100644 index 00000000000..ca4d2f75604 --- /dev/null +++ b/src/volatile_set.h @@ -0,0 +1,41 @@ +#ifndef VOLATILESET_H +#define VOLATILESET_H + +#include "fmacros.h" +#include +#include "rax.h" +#include "sds.h" + +typedef struct { + sds (*entryGetKey)(const void *entry); + + long long (*getExpiry)(const void *entry); + + int (*expire)(void *entry); + +} volatileEntryType; + + +typedef struct { + volatileEntryType *etypr; + rax *expiry_buckets; +} volatile_set; + +typedef struct volatileSetIterator { + raxIterator bucket; +} volatileSetIterator; + + +int volatileSetRemoveEntry(volatile_set *set, void *entry, long long expiry); +int volatileSetAddEntry(volatile_set *set, void *entry, long long expiry); +int volatileSetExpireEntry(volatile_set *set, void *entry); +int volatileSetUpdateEntry(volatile_set *set, void *old_entry, void *new_entry, long long old_expiry, long long new_expiry); +size_t volatileSetNumEntries(volatile_set *set); +void volatileSetStart(volatile_set *set, volatileSetIterator *it); +int volatileSetNext(volatileSetIterator *it, void **entryptr); +void volatileSetReset(volatileSetIterator *it); + +void freeVolatileSet(volatile_set *b); +volatile_set *createVolatileSet(volatileEntryType *type); + +#endif diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index 3736538105c..9d4ff8c52ce 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -546,9 +546,8 @@ start_server {tags {"expire"}} { set primary_port [srv -1 port] # Set this inner layer server as replica set replica [srv 0 client] - + $replica replicaof $primary_host $primary_port test {First server should have role slave after REPLICAOF} { - $replica replicaof $primary_host $primary_port wait_for_condition 50 100 { [s 0 role] eq {slave} } else { @@ -615,6 +614,40 @@ start_server {tags {"expire"}} { assert_equal {} [$replica get foo] } } + + test {expired hash field is expired on the replica} { + $primary flushall + $replica config set replica-read-only yes + $primary hset myhash f1 v1 f2 v2 + wait_for_condition 50 100 { + [$primary hlen myhash] eq {2} + } else { + fail "field not added to primary" + } + + wait_for_condition 50 100 { + [$replica hlen myhash] eq {2} + } else { + fail "field not added to replica" + } + + # bp 1 + + $primary hpexpire myhash 1000 fields 1 f2 + + + wait_for_condition 50 100 { + [$primary hget myhash f2] eq {} + } else { + fail "field not removed from primary" + } + + wait_for_condition 50 100 { + [$replica hlen myhash] eq {1} + } else { + fail "field not removed from replica" + } + } } test {SET command will remove expire} { From 89f56b0820f827e9dfa056d01c5735f4d3834e99 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Thu, 15 May 2025 20:38:21 +0300 Subject: [PATCH 02/33] fix new introduced commands Signed-off-by: Ran Shidlansik --- src/commands.def | 8 ++++---- src/commands/hexpire.json | 7 ++----- src/expire.c | 4 ++-- src/t_hash.c | 42 +++++++++++++++++++++++---------------- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/commands.def b/src/commands.def index 3a416fe93da..fdb3a818fe6 100644 --- a/src/commands.def +++ b/src/commands.def @@ -3493,15 +3493,15 @@ struct COMMAND_ARG HEXPIRE_condition_Subargs[] = { /* HEXPIRE fields argument table */ struct COMMAND_ARG HEXPIRE_fields_Subargs[] = { -{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, }; /* HEXPIRE argument table */ struct COMMAND_ARG HEXPIRE_Args[] = { {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, -{MAKE_ARG("seconds",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, -{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,"9.0.0",CMD_ARG_OPTIONAL,4,NULL),.subargs=HEXPIRE_condition_Subargs}, +{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=HEXPIRE_condition_Subargs}, {MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HEXPIRE_fields_Subargs}, }; @@ -11466,7 +11466,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { /* hash */ {MAKE_CMD("hdel","Deletes one or more fields and their values from a hash. Deletes the hash if no fields remain.","O(N) where N is the number of fields to be removed.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HDEL_History,1,HDEL_Tips,0,hdelCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HDEL_Keyspecs,1,NULL,2),.args=HDEL_Args}, {MAKE_CMD("hexists","Determines whether a field exists in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXISTS_History,0,HEXISTS_Tips,0,hexistsCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXISTS_Keyspecs,1,NULL,2),.args=HEXISTS_Args}, -{MAKE_CMD("hexpire","Set expiry time on hash object.","O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRE_History,0,HEXPIRE_Tips,0,hexpireCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRE_Keyspecs,1,NULL,4),.args=HEXPIRE_Args}, +{MAKE_CMD("hexpire","Set expiry time on hash object.","O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.","7.2.4",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRE_History,0,HEXPIRE_Tips,0,hexpireCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRE_Keyspecs,1,NULL,4),.args=HEXPIRE_Args}, {MAKE_CMD("hexpireat","Set expiry time on hash object.","O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIREAT_History,0,HEXPIREAT_Tips,0,hexpireAtCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIREAT_Keyspecs,1,NULL,4),.args=HEXPIREAT_Args}, {MAKE_CMD("hexpiretime","Returns the Unix timestamp in seconds since Unix epoch at which the given key's field(s) will expire","O(1) for each field, so O(N) for N items when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRETIME_Keyspecs,1,NULL,2),.args=HEXPIRETIME_Args}, {MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HGET_Keyspecs,1,NULL,2),.args=HGET_Args}, diff --git a/src/commands/hexpire.json b/src/commands/hexpire.json index 81fefa4f44f..807c32cd10c 100644 --- a/src/commands/hexpire.json +++ b/src/commands/hexpire.json @@ -3,7 +3,7 @@ "summary": "Set expiry time on hash object.", "complexity": "O(1) for each field assigned with TTL, so O(N) to add N TTLs when the command is called with multiple fields.", "group": "hash", - "since": "9.0.0", + "since": "7.2.4", "arity": -5, "function": "hexpireCommand", "command_flags": [ @@ -67,14 +67,12 @@ }, { "name": "seconds", - "type": "integer", - "key_spec_index": 0 + "type": "integer" }, { "name": "condition", "type": "oneof", "optional": true, - "since": "9.0.0", "arguments": [ { "name": "nx", @@ -106,7 +104,6 @@ { "name": "numfields", "type": "integer", - "key_spec_index": 0, "multiple": false }, { diff --git a/src/expire.c b/src/expire.c index 0b049c20d3b..0ebe72f7887 100644 --- a/src/expire.c +++ b/src/expire.c @@ -542,10 +542,10 @@ int checkAlreadyExpired(long long when) { * - LT: set expiry only when the new expiry is less than current one */ int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_index) { int nx = 0, xx = 0, gt = 0, lt = 0; - if (max_index > 0) max_index = c->argc - 1; + if (max_index < 0) max_index = c->argc - 1; int j = 3; - while (j <= max_index) { + while (j < max_index) { char *opt = c->argv[j]->ptr; if (!strcasecmp(opt, "nx")) { *flags |= EXPIRE_NX; diff --git a/src/t_hash.c b/src/t_hash.c index 296d3380320..0dc7a955086 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -522,7 +522,7 @@ hashtableElementAccessState hashHashtableTypeAccess(hashtable *ht, void *entry) if (!delete_expired) return ELEMENT_INVALID; - if (server.access_context.flags & OBJ_ACCESS_NONE) return ELEMENT_INVALID; + if (server.access_context.flags == OBJ_ACCESS_NONE) return ELEMENT_INVALID; robj *o = server.access_context.key; serverDb *db = server.access_context.db; @@ -547,8 +547,8 @@ void hashTypeResetAccessContext(void) { if (o) { if (hashTypeLength(o) == 0) { initStaticStringObject(keyobj, objectGetKey(o)); - dbDelete(db, &keyobj); notifyKeyspaceEvent(NOTIFY_GENERIC, "del", &keyobj, db->id); + dbDelete(db, &keyobj); } } } @@ -1615,7 +1615,8 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { if (!strcasecmp(c->argv[fields_index]->ptr, "fields")) { /* checking optional flags */ if (parseExtendedExpireArgumentsOrReply(c, &flag, fields_index++) != C_OK) return; - if (getLongLongFromObjectOrReply(c, c->argv[fields_index++]->ptr, &num_fields, NULL) != C_OK) return; + if (getLongLongFromObjectOrReply(c, c->argv[fields_index++], &num_fields, NULL) != C_OK) return; + break; } } @@ -1679,7 +1680,7 @@ void hpersistCommand(client *c) { int fields_index = 4, result = 0; long long num_fields = 0; - if (getLongLongFromObjectOrReply(c, c->argv[fields_index - 1]->ptr, &num_fields, NULL) != C_OK) return; + if (getLongLongFromObjectOrReply(c, c->argv[fields_index - 1], &num_fields, NULL) != C_OK) return; if (num_fields > c->argc - 4) num_fields = c->argc - 4; // Potential user error, but we would like to make effort to comply with the request. @@ -1688,54 +1689,61 @@ void hpersistCommand(client *c) { robj *hash = lookupKeyWrite(c->db, c->argv[1]); - for (; fields_index < num_fields; fields_index++) { - result = hashTypePersist(hash, c->argv[2]->ptr); + hashTypeSetAccessContext(hash, c->db); + + for (; fields_index < 4 + num_fields; fields_index++) { + result = hashTypePersist(hash, c->argv[fields_index]->ptr); server.dirty += (result > 0 ? 1 : 0); // in case there was a change increment the dirty addReplyLongLong(c, result); } + + hashTypeResetAccessContext(); } void httlGenericCommand(client *c, long long basetime, int unit) { int fields_index = 4; long long num_fields = 0, result = -2; - if (getLongLongFromObjectOrReply(c, c->argv[fields_index - 1]->ptr, &num_fields, NULL) != C_OK) return; + if (getLongLongFromObjectOrReply(c, c->argv[fields_index - 1], &num_fields, NULL) != C_OK) return; if (num_fields > c->argc - 4) num_fields = c->argc - 4; // Potential user error, but we would like to make effort to comply with the request. + robj *hash = lookupKeyRead(c->db, c->argv[1]); + /* From this point we would retunr array reply */ addReplyArrayLen(c, num_fields); - robj *hash = lookupKeyRead(c->db, c->argv[1]); + hashTypeSetAccessContext(hash, c->db); - for (; fields_index < num_fields; fields_index++) { - if (!hash || hashTypeGetExpiry(hash, c->argv[2]->ptr, &result) == C_ERR) { - result = -2; + for (; fields_index < 4 + num_fields; fields_index++) { + if (!hash || hashTypeGetExpiry(hash, c->argv[fields_index]->ptr, &result) == C_ERR) { + addReplyLongLong(c, -2); } else if (result == EXPIRY_NONE) { - result = -1; + addReplyLongLong(c, -1); } else { result = result - basetime; if (result < 0) result = 0; addReplyLongLong(c, unit == UNIT_MILLISECONDS ? result : ((result + 500) / 1000)); } } - addReplyLongLong(c, result); + + hashTypeResetAccessContext(); } void httlCommand(client *c) { - hexpireGenericCommand(c, commandTimeSnapshot(), UNIT_SECONDS); + httlGenericCommand(c, commandTimeSnapshot(), UNIT_SECONDS); } void hpttlCommand(client *c) { - hexpireGenericCommand(c, commandTimeSnapshot(), UNIT_MILLISECONDS); + httlGenericCommand(c, commandTimeSnapshot(), UNIT_MILLISECONDS); } void hexpiretimeCommand(client *c) { - hexpireGenericCommand(c, 0, UNIT_SECONDS); + httlGenericCommand(c, 0, UNIT_SECONDS); } void hpexpiretimeCommand(client *c) { - hexpireGenericCommand(c, 0, UNIT_MILLISECONDS); + httlGenericCommand(c, 0, UNIT_MILLISECONDS); } /* How many times bigger should be the hash compared to the requested size From 61bd39af78d545d00c2f4138484e78042dec031b Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Thu, 15 May 2025 20:45:18 +0300 Subject: [PATCH 03/33] fix some spelling checks Signed-off-by: Ran Shidlansik --- src/t_hash.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index 0dc7a955086..4d71e26bbc3 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -824,7 +824,7 @@ int hashTypeSet(robj *o, sds field, sds value, long long expiry, int flags) { * returns 2 when 'expire' indicate a past Unix time. In this case, if the item exists in the HASH, it will also be expired. */ int hashTypeSetExpire(robj *o, sds field, long long expiry, int flag) { - /* If no object we will retunr -2 */ + /* If no object we will return -2 */ if (o == NULL) return -2; if (timestampIsExpired(expiry)) { @@ -1647,7 +1647,7 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { hashTypeSetAccessContext(obj, c->db); - /* From this point we would retunr array reply */ + /* From this point we would return array reply */ addReplyArrayLen(c, num_fields); for (i = fields_index; i < c->argc; i++) { @@ -1684,7 +1684,7 @@ void hpersistCommand(client *c) { if (num_fields > c->argc - 4) num_fields = c->argc - 4; // Potential user error, but we would like to make effort to comply with the request. - /* From this point we would retunr array reply */ + /* From this point we would return array reply */ addReplyArrayLen(c, num_fields); robj *hash = lookupKeyWrite(c->db, c->argv[1]); @@ -1710,7 +1710,7 @@ void httlGenericCommand(client *c, long long basetime, int unit) { robj *hash = lookupKeyRead(c->db, c->argv[1]); - /* From this point we would retunr array reply */ + /* From this point we would return array reply */ addReplyArrayLen(c, num_fields); hashTypeSetAccessContext(hash, c->db); From cc7c2a30466f49dc01a813794ac97e49da8266cd Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Thu, 15 May 2025 20:48:12 +0300 Subject: [PATCH 04/33] handle some format check issues Signed-off-by: Ran Shidlansik --- src/server.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/server.h b/src/server.h index ae380a2ee08..68ec96656d1 100644 --- a/src/server.h +++ b/src/server.h @@ -713,7 +713,7 @@ typedef enum { * Data types *----------------------------------------------------------------------------*/ - /* Generic set command string object set flags */ +/* Generic set command string object set flags */ #define OBJ_NO_FLAGS 0 #define OBJ_SET_NX (1 << 0) /* Set if key not exists. */ #define OBJ_SET_XX (1 << 1) /* Set if key exists. */ @@ -1596,8 +1596,8 @@ typedef enum childInfoType { CHILD_INFO_TYPE_MODULE_COW_SIZE } childInfoType; -#define OBJ_ACCESS_NONE 0 /* Will not actively delete expired accessed elements */ -#define OBJ_ACCESS_NORMAL (1 << 0) /* Deleting lazy expired fields. */ +#define OBJ_ACCESS_NONE 0 /* Will not actively delete expired accessed elements */ +#define OBJ_ACCESS_NORMAL (1 << 0) /* Deleting lazy expired fields. */ #define OBJ_ACCESS_IGNORE_TTL (1 << 1) /* treat any accessed field as valid regardless of it's TTL */ typedef struct keyAccessContext { @@ -1688,7 +1688,7 @@ struct valkeyServer { * Value: RDB client object * This structure holds dual-channel sync replicas from the start of their * RDB transfer until their main channel establishes partial synchronization. */ - keyAccessContext access_context; /* The current key access context */ + keyAccessContext access_context; /* The current key access context */ client *current_client; /* The client that triggered the command execution (External or AOF). */ client *executing_client; /* The client executing the current command (possibly script or module). */ @@ -2611,7 +2611,6 @@ typedef struct { #define OBJ_HASH_FIELD 1 #define OBJ_HASH_VALUE 2 -#define OBJ_HASH_EXPIRY /*----------------------------------------------------------------------------- * Extern declarations From fcce92bfe7e3d91f264bcfd8857d1d380a8ef459 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Thu, 15 May 2025 21:28:02 +0300 Subject: [PATCH 05/33] fix expire propagation Signed-off-by: Ran Shidlansik --- src/expire.c | 2 +- src/server.c | 2 +- src/t_hash.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/expire.c b/src/expire.c index 0ebe72f7887..150ebd9012b 100644 --- a/src/expire.c +++ b/src/expire.c @@ -542,7 +542,7 @@ int checkAlreadyExpired(long long when) { * - LT: set expiry only when the new expiry is less than current one */ int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_index) { int nx = 0, xx = 0, gt = 0, lt = 0; - if (max_index < 0) max_index = c->argc - 1; + if (max_index < 0) max_index = c->argc; int j = 3; while (j < max_index) { diff --git a/src/server.c b/src/server.c index 86bd1e0d105..640859cec73 100644 --- a/src/server.c +++ b/src/server.c @@ -7224,7 +7224,7 @@ __attribute__((weak)) int main(int argc, char **argv) { } void setAccessContext(robj *o, serverDb *db) { - setAccessContextWithFlags(o, db, OBJ_ACCESS_NONE); + setAccessContextWithFlags(o, db, OBJ_ACCESS_NORMAL); } void setAccessContextWithFlags(robj *o, serverDb *db, int flags) { diff --git a/src/t_hash.c b/src/t_hash.c index 4d71e26bbc3..1663529fbb8 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -502,6 +502,7 @@ int hashTypeExpireEntry(void *entry) { serverAssert(key); initStaticStringObject(keyobj, key); notifyKeyspaceEvent(NOTIFY_EXPIRED, "hexpired", &keyobj, server.access_context.db->id); + serverLog(LL_NOTICE, "expiring entry %s of key %s", (sds)entry, key); hashTypePropagateDeletion(server.access_context.db, key, entry); return 1; } From 90b7536eb716594af110b1a9cd356490f90a5923 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Thu, 15 May 2025 21:30:46 +0300 Subject: [PATCH 06/33] fix typo Signed-off-by: Ran Shidlansik --- src/t_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_hash.c b/src/t_hash.c index 1663529fbb8..b6107a5679c 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -301,7 +301,7 @@ size_t hashTypeEntryMemUsage(hashTypeEntry *entry) { if (entryHasValuePtr(entry)) { /* In case the value is not embedded we might not be able to sum all the allocation sizes since the field - * header could be too small for holding the reall allocation size. */ + * header could be too small for holding the real allocation size. */ mem += zmalloc_usable_size(hashTypeEntryAllocPtr(entry)); } else { mem += sdsReqSize(sdslen(entry), sdsType(entry)); From 6ee497caa97281ff8ed633c9f38db4130a775a94 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Thu, 15 May 2025 21:32:30 +0300 Subject: [PATCH 07/33] fix some more format issues Signed-off-by: Ran Shidlansik --- src/server.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server.h b/src/server.h index 68ec96656d1..915b56b6762 100644 --- a/src/server.h +++ b/src/server.h @@ -1689,7 +1689,6 @@ struct valkeyServer { * This structure holds dual-channel sync replicas from the start of their * RDB transfer until their main channel establishes partial synchronization. */ keyAccessContext access_context; /* The current key access context */ - client *current_client; /* The client that triggered the command execution (External or AOF). */ client *executing_client; /* The client executing the current command (possibly script or module). */ From 1f0c9339e3344aadd3292adced5393734705afdd Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Sun, 18 May 2025 10:34:27 +0300 Subject: [PATCH 08/33] avoid extra ref count incrementing in hashTypePropagateDeletion Signed-off-by: Ran Shidlansik --- src/t_hash.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index b6107a5679c..bfeb6d863d1 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -480,8 +480,6 @@ void hashTypePropagateDeletion(serverDb *db, sds key, void *entry) { argv[1] = createStringObject(key, sdslen(key)); argv[2] = createStringObject(field, sdslen(field)); incrRefCount(argv[0]); - incrRefCount(argv[1]); - incrRefCount(argv[2]); /* If the primary decided to delete a key we must propagate it to replicas no matter what. * Even if module executed a command without asking for propagation. */ From fce9a437441542d16c3018a2db8e52a9fd49a18e Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Sun, 18 May 2025 10:39:22 +0300 Subject: [PATCH 09/33] fix cmake compilation Signed-off-by: Ran Shidlansik --- cmake/Modules/SourceFiles.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake index aa7158ee475..df89b68f52e 100644 --- a/cmake/Modules/SourceFiles.cmake +++ b/cmake/Modules/SourceFiles.cmake @@ -109,7 +109,7 @@ set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/connection.c ${CMAKE_SOURCE_DIR}/src/unix.c ${CMAKE_SOURCE_DIR}/src/server.c - ${CMAKE_SOURCE_DIR}/src/logreqres.c) + ${CMAKE_SOURCE_DIR}/src/logreqres.c ${CMAKE_SOURCE_DIR}/src/volatile_set.c) From dd62037c2800b47fe06123b26e5a737c75762b56 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Sun, 18 May 2025 11:12:49 +0300 Subject: [PATCH 10/33] remove hashtable redundant log Signed-off-by: Ran Shidlansik --- src/hashtable.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hashtable.c b/src/hashtable.c index d9f9ead111e..cbf7239a0d3 100644 --- a/src/hashtable.c +++ b/src/hashtable.c @@ -887,7 +887,6 @@ static inline hashtableElementAccessState accessElementIfNeeded(hashtable *ht, v if (ht->type->accessElement == NULL) return ELEMENT_VALID; hashtableElementAccessState element_status = ht->type->accessElement(ht, elem); - serverLog(LL_NOTICE, "hashtable access returned: %d", element_status); if (element_status == ELEMENT_DELETE) { b->presence &= ~(1 << pos_in_bucket); ht->used[table_index]--; From 4a09f3d8db7e49760c88574d3e737d32e91b713f Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Sun, 18 May 2025 12:16:44 +0300 Subject: [PATCH 11/33] free entry when calling hashTypeDelete Signed-off-by: Ran Shidlansik --- src/t_hash.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/t_hash.c b/src/t_hash.c index bfeb6d863d1..1780d8b3be2 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -947,6 +947,7 @@ int hashTypeDelete(robj *o, sds field) { deleted = hashtablePop(ht, field, &entry); if (deleted) { hashTypeUntrackEntry(o, entry); + freeHashTypeEntry(entry); } } else { serverPanic("Unknown hash encoding"); From a59f31ae7e5dade35c3800c091707b4e325e3f17 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 14:17:19 +0300 Subject: [PATCH 12/33] Add support for HGETEX and HSETEX Signed-off-by: Ran Shidlansik --- src/commands.def | 104 ++++++++++++ src/expire.c | 7 +- src/object.c | 30 +++- src/server.c | 2 + src/server.h | 14 +- src/t_hash.c | 400 ++++++++++++++++++++++++++++++++++++++--------- src/t_string.c | 4 +- 7 files changed, 469 insertions(+), 92 deletions(-) diff --git a/src/commands.def b/src/commands.def index fdb3a818fe6..1ff48dac80a 100644 --- a/src/commands.def +++ b/src/commands.def @@ -3628,6 +3628,47 @@ struct COMMAND_ARG HGETALL_Args[] = { {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; +/********** HGETEX ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HGETEX history */ +#define HGETEX_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HGETEX tips */ +#define HGETEX_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HGETEX key specs */ +keySpec HGETEX_Keyspecs[1] = { +{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HGETEX expiration argument table */ +struct COMMAND_ARG HGETEX_expiration_Subargs[] = { +{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("persist",ARG_TYPE_PURE_TOKEN,-1,"PERSIST",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HGETEX fields argument table */ +struct COMMAND_ARG HGETEX_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + +/* HGETEX argument table */ +struct COMMAND_ARG HGETEX_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HGETEX_expiration_Subargs}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HGETEX_fields_Subargs}, +}; + /********** HINCRBY ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -4057,6 +4098,67 @@ struct COMMAND_ARG HSET_Args[] = { {MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSET_data_Subargs}, }; +/********** HSETEX ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HSETEX history */ +#define HSETEX_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HSETEX tips */ +#define HSETEX_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HSETEX key specs */ +keySpec HSETEX_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* HSETEX key_condition argument table */ +struct COMMAND_ARG HSETEX_key_condition_Subargs[] = { +{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETEX fields_condition argument table */ +struct COMMAND_ARG HSETEX_fields_condition_Subargs[] = { +{MAKE_ARG("fnx",ARG_TYPE_PURE_TOKEN,-1,"FNX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("fxx",ARG_TYPE_PURE_TOKEN,-1,"FXX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETEX expiration argument table */ +struct COMMAND_ARG HSETEX_expiration_Subargs[] = { +{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("keepttl",ARG_TYPE_PURE_TOKEN,-1,"KEEPTTL",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETEX fields data argument table */ +struct COMMAND_ARG HSETEX_fields_data_Subargs[] = { +{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("value",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* HSETEX fields argument table */ +struct COMMAND_ARG HSETEX_fields_Subargs[] = { +{MAKE_ARG("numfields",ARG_TYPE_INTEGER,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSETEX_fields_data_Subargs}, +}; + +/* HSETEX argument table */ +struct COMMAND_ARG HSETEX_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("key-condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=HSETEX_key_condition_Subargs}, +{MAKE_ARG("fields-condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=HSETEX_fields_condition_Subargs}, +{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HSETEX_expiration_Subargs}, +{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HSETEX_fields_Subargs}, +}; + /********** HSETNX ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -11471,6 +11573,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("hexpiretime","Returns the Unix timestamp in seconds since Unix epoch at which the given key's field(s) will expire","O(1) for each field, so O(N) for N items when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRETIME_Keyspecs,1,NULL,2),.args=HEXPIRETIME_Args}, {MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HGET_Keyspecs,1,NULL,2),.args=HGET_Args}, {MAKE_CMD("hgetall","Returns all fields and values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETALL_History,0,HGETALL_Tips,1,hgetallCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HGETALL_Keyspecs,1,NULL,1),.args=HGETALL_Args}, +{MAKE_CMD("hgetex","Set the value of one or more fields of a given hash key, and optionally set their expiration time.","O(1)","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETEX_History,0,HGETEX_Tips,0,hgetexCommand,-4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HGETEX_Keyspecs,1,NULL,3),.args=HGETEX_Args}, {MAKE_CMD("hincrby","Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBY_History,0,HINCRBY_Tips,0,hincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBY_Keyspecs,1,NULL,3),.args=HINCRBY_Args}, {MAKE_CMD("hincrbyfloat","Increments the floating point value of a field by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBYFLOAT_History,0,HINCRBYFLOAT_Tips,0,hincrbyfloatCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBYFLOAT_Keyspecs,1,NULL,3),.args=HINCRBYFLOAT_Args}, {MAKE_CMD("hkeys","Returns all fields in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HKEYS_History,0,HKEYS_Tips,1,hkeysCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HKEYS_Keyspecs,1,NULL,1),.args=HKEYS_Args}, @@ -11485,6 +11588,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("hrandfield","Returns one or more random fields from a hash.","O(N) where N is the number of fields returned","6.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HRANDFIELD_History,0,HRANDFIELD_Tips,1,hrandfieldCommand,-2,CMD_READONLY,ACL_CATEGORY_HASH,HRANDFIELD_Keyspecs,1,NULL,2),.args=HRANDFIELD_Args}, {MAKE_CMD("hscan","Iterates over fields and values of a hash.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSCAN_History,0,HSCAN_Tips,1,hscanCommand,-3,CMD_READONLY,ACL_CATEGORY_HASH,HSCAN_Keyspecs,1,NULL,5),.args=HSCAN_Args}, {MAKE_CMD("hset","Creates or modifies the value of a field in a hash.","O(1) for each field/value pair added, so O(N) to add N field/value pairs when the command is called with multiple field/value pairs.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSET_History,1,HSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSET_Keyspecs,1,NULL,2),.args=HSET_Args}, +{MAKE_CMD("hsetex","Set the value of one or more fields of a given hash key, and optionally set their expiration time.","O(1)","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETEX_History,0,HSETEX_Tips,0,hsetexCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETEX_Keyspecs,1,NULL,5),.args=HSETEX_Args}, {MAKE_CMD("hsetnx","Sets the value of a field in a hash only when the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETNX_History,0,HSETNX_Tips,0,hsetnxCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETNX_Keyspecs,1,NULL,3),.args=HSETNX_Args}, {MAKE_CMD("hstrlen","Returns the length of the value of a field.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSTRLEN_History,0,HSTRLEN_Tips,0,hstrlenCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HSTRLEN_Keyspecs,1,NULL,2),.args=HSTRLEN_Args}, {MAKE_CMD("httl","Returns the remaining time to live (in seconds) of a hash key's field(s) that have an associated expiration.","O(1) for each field, so O(N) for N items when the command is called with multiple fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,0,httlCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HTTL_Keyspecs,1,NULL,2),.args=HTTL_Args}, diff --git a/src/expire.c b/src/expire.c index 150ebd9012b..a6bf0d3be45 100644 --- a/src/expire.c +++ b/src/expire.c @@ -540,12 +540,11 @@ int checkAlreadyExpired(long long when) { * - XX: set expiry only when the key has an existing expiry * - GT: set expiry only when the new expiry is greater than current one * - LT: set expiry only when the new expiry is less than current one */ -int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_index) { +int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_args) { int nx = 0, xx = 0, gt = 0, lt = 0; - if (max_index < 0) max_index = c->argc; int j = 3; - while (j < max_index) { + while (j < max_args) { char *opt = c->argv[j]->ptr; if (!strcasecmp(opt, "nx")) { *flags |= EXPIRE_NX; @@ -599,7 +598,7 @@ void expireGenericCommand(client *c, long long basetime, int unit) { int flag = 0; /* checking optional flags */ - if (parseExtendedExpireArgumentsOrReply(c, &flag, -1) != C_OK) { + if (parseExtendedExpireArgumentsOrReply(c, &flag, c->argc) != C_OK) { return; } diff --git a/src/object.c b/src/object.c index 1c6d84cf894..f7537703e6b 100644 --- a/src/object.c +++ b/src/object.c @@ -1855,11 +1855,11 @@ void memoryCommand(client *c) { * Input flags are updated upon parsing the arguments. Unit and expire are updated if there are any * EX/EXAT/PX/PXAT arguments. Unit is updated to millisecond if PX/PXAT is set. */ -int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj **expire, robj **compare_val, int command_type) { - int j = command_type == COMMAND_GET ? 2 : 3; - for (; j < c->argc; j++) { +int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj **expire, robj **compare_val, int command_type, int max_args) { + int j = command_type == COMMAND_SET ? 3 : 2; + for (; j < max_args; j++) { char *opt = c->argv[j]->ptr; - robj *next = (j == c->argc - 1) ? NULL : c->argv[j + 1]; + robj *next = (j == max_args - 1) ? NULL : c->argv[j + 1]; /* clang-format off */ if ((opt[0] == 'n' || opt[0] == 'N') && @@ -1872,11 +1872,25 @@ int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj * !(*flags & OBJ_SET_NX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET)) { *flags |= OBJ_SET_XX; + } else if ((opt[0] == 'f' || opt[0] == 'F') && + (opt[1] == 'n' || opt[1] == 'N') && opt[2] == '\0' && + (opt[2] == 'x' || opt[2] == 'X') && opt[3] == '\0' && + !(*flags & OBJ_SET_FXX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_HSET)) + { + *flags |= OBJ_SET_FNX; + } else if ((opt[0] == 'f' || opt[0] == 'F') && + (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && + (opt[2] == 'x' || opt[2] == 'X') && opt[3] == '\0' && + !(*flags & OBJ_SET_FNX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_HSET)) + { + *flags |= OBJ_SET_FXX; } else if ((opt[0] == 'i' || opt[0] == 'I') && (opt[1] == 'f' || opt[1] == 'F') && (opt[2] == 'e' || opt[2] == 'E') && (opt[3] == 'q' || opt[3] == 'Q') && opt[4] == '\0' && - next && !(*flags & OBJ_SET_NX || *flags & OBJ_SET_XX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET)) + next && + !(*flags & OBJ_SET_NX || *flags & OBJ_SET_XX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET) && + !(*flags & OBJ_SET_FNX || *flags & OBJ_SET_FXX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_HSET)) { *flags |= OBJ_SET_IFEQ; *compare_val = next; @@ -1884,15 +1898,15 @@ int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj * } else if ((opt[0] == 'g' || opt[0] == 'G') && (opt[1] == 'e' || opt[1] == 'E') && (opt[2] == 't' || opt[2] == 'T') && opt[3] == '\0' && - (command_type == COMMAND_SET)) + (command_type == COMMAND_SET || command_type == COMMAND_HSET)) { *flags |= OBJ_SET_GET; } else if (!strcasecmp(opt, "KEEPTTL") && !(*flags & OBJ_PERSIST) && !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && - !(*flags & OBJ_PX) && !(*flags & OBJ_PXAT) && (command_type == COMMAND_SET)) + !(*flags & OBJ_PX) && !(*flags & OBJ_PXAT) && (command_type == COMMAND_SET || command_type == COMMAND_HSET)) { *flags |= OBJ_KEEPTTL; - } else if (!strcasecmp(opt,"PERSIST") && (command_type == COMMAND_GET) && + } else if (!strcasecmp(opt,"PERSIST") && (command_type == COMMAND_GET || command_type == COMMAND_HGET) && !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) && !(*flags & OBJ_PX) && !(*flags & OBJ_PXAT) && !(*flags & OBJ_KEEPTTL)) diff --git a/src/server.c b/src/server.c index 640859cec73..95350585d27 100644 --- a/src/server.c +++ b/src/server.c @@ -2125,6 +2125,7 @@ void createSharedObjects(void) { shared.exec = createStringObject("EXEC", 4); shared.hset = createStringObject("HSET", 4); shared.hdel = createStringObject("HDEL", 4); + shared.hpexpireat = createStringObject("HPEXPIREAT", 10); shared.srem = createStringObject("SREM", 4); shared.xgroup = createStringObject("XGROUP", 6); shared.xclaim = createStringObject("XCLAIM", 6); @@ -2157,6 +2158,7 @@ void createSharedObjects(void) { shared.special_asterisk = createStringObject("*", 1); shared.special_equals = createStringObject("=", 1); shared.redacted = makeObjectShared(createStringObject("(redacted)", 10)); + shared.fields = createStringObject("FIELDS", 6); for (j = 0; j < OBJ_SHARED_INTEGERS; j++) { shared.integers[j] = makeObjectShared(createObject(OBJ_STRING, (void *)(long)j)); diff --git a/src/server.h b/src/server.h index 915b56b6762..27e63bd1ad2 100644 --- a/src/server.h +++ b/src/server.h @@ -220,6 +220,8 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; #define COMMAND_GET 0 #define COMMAND_SET 1 +#define COMMAND_HGET 2 +#define COMMAND_HSET 3 /* Command flags. Please check the definition of struct serverCommand in this file @@ -727,6 +729,8 @@ typedef enum { #define OBJ_SET_IFEQ (1 << 9) /* Set if we need compare and set */ #define OBJ_ARGV3 (1 << 10) /* Set if the value is at argv[3]; otherwise it's \ * at argv[2]. */ +#define OBJ_SET_FNX (1 << 11) /* Set if key item not exists. */ +#define OBJ_SET_FXX (1 << 12) /* Set if key item exists. */ /* An Object, that is a type able to hold a string / list / set */ @@ -1348,10 +1352,10 @@ struct sharedObjectsStruct { *loadingerr, *slowevalerr, *slowscripterr, *slowmoduleerr, *bgsaveerr, *primarydownerr, *roreplicaerr, *execaborterr, *noautherr, *noreplicaserr, *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink, *rpop, *lpop, *lpush, - *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *hdel, *srem, + *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *hdel, *hpexpireat, *srem, *xgroup, *xclaim, *script, *replconf, *eval, *persist, *set, *pexpireat, *pexpire, *time, *pxat, *absttl, *retrycount, *force, *justid, *entriesread, *lastid, *ping, *setid, *keepttl, *load, *createconsumer, *getack, - *special_asterisk, *special_equals, *default_username, *redacted, *ssubscribebulk, *sunsubscribebulk, + *special_asterisk, *special_equals, *default_username, *redacted, *ssubscribebulk, *sunsubscribebulk, *fields, *smessagebulk, *select[PROTO_SHARED_SELECT_CMDS], *integers[OBJ_SHARED_INTEGERS], *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*\r\n" */ *bulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "$\r\n" */ @@ -2833,8 +2837,8 @@ int canParseCommand(client *c); int processIOThreadsReadDone(void); int processIOThreadsWriteDone(void); int canExpireWithFlags(int flags, int *can_delete); -int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_index); -int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj **expire, robj **compare_val, int command_type); +int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_args); +int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj **expire, robj **compare_val, int command_type, int max_args); /* logreqres.c - logging of requests and responses */ void reqresReset(client *c, int free_buf); @@ -3820,6 +3824,8 @@ void zrankCommand(client *c); void zrevrankCommand(client *c); void hsetCommand(client *c); void hsetnxCommand(client *c); +void hsetexCommand(client *c); +void hgetexCommand(client *c); void hgetCommand(client *c); void hmgetCommand(client *c); void hdelCommand(client *c); diff --git a/src/t_hash.c b/src/t_hash.c index 1780d8b3be2..dc8b1d40361 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -826,17 +826,7 @@ int hashTypeSetExpire(robj *o, sds field, long long expiry, int flag) { /* If no object we will return -2 */ if (o == NULL) return -2; - if (timestampIsExpired(expiry)) { - /* It is possible that the assigned expiration is set in the past (or zero). - * In such case we cannot count on the hash object representation to be hashtable, so - * we operate this on before we check for the encoding. */ - if (hashTypeDelete(o, field)) { - hashTypeExpireEntry(field); - return 2; - } else { - return -2; - } - } + int expired = timestampIsExpired(expiry); if (o->encoding == OBJ_ENCODING_LISTPACK) { /* When listpack representation is used, we consider it as infinite TTL, @@ -845,6 +835,16 @@ int hashTypeSetExpire(robj *o, sds field, long long expiry, int flag) { if (flag & EXPIRE_XX || flag & EXPIRE_GT) { return 0; } else { + if (expired) { + /* It is possible that the assigned expiration is set in the past (or zero). + * In such case we cannot count on the hash object representation to be hashtable. */ + if (hashTypeDelete(o, field)) { + hashTypeExpireEntry(field); + return 2; + } else { + return -2; + } + } hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); } } @@ -887,6 +887,12 @@ int hashTypeSetExpire(robj *o, sds field, long long expiry, int flag) { } } } + if (expired) { + if (hashTypeDelete(o, field)) { + hashTypeExpireEntry(field); + return 2; + } + } *entry_ref = hashTypeEntrySetExpiry(current_entry, expiry); hashTypeTrackUpdateEntry(o, current_entry, *entry_ref, current_expire, expiry); return 1; @@ -1259,55 +1265,6 @@ static void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, listpac * Hash type commands *----------------------------------------------------------------------------*/ -void hsetnxCommand(client *c) { - robj *o; - if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; - hashTypeSetAccessContext(o, c->db); - if (hashTypeExists(o, c->argv[2]->ptr)) { - addReply(c, shared.czero); - } else { - hashTypeTryConversion(o, c->argv, 2, 3); - hashTypeSet(o, c->argv[2]->ptr, c->argv[3]->ptr, EXPIRY_NONE, HASH_SET_COPY | HASH_SET_KEEP_EXPIRY); - signalModifiedKey(c, c->db, c->argv[1]); - notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); - server.dirty++; - addReply(c, shared.cone); - } - hashTypeResetAccessContext(); -} - -void hsetCommand(client *c) { - int i, created = 0; - robj *o; - - if ((c->argc % 2) == 1) { - addReplyErrorArity(c); - return; - } - - if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; - hashTypeTryConversion(o, c->argv, 2, c->argc - 1); - - hashTypeSetAccessContext(o, c->db); - for (i = 2; i < c->argc; i += 2) created += !hashTypeSet(o, c->argv[i]->ptr, c->argv[i + 1]->ptr, EXPIRY_NONE, HASH_SET_COPY | HASH_SET_KEEP_EXPIRY); - - signalModifiedKey(c, c->db, c->argv[1]); - notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); - server.dirty += (c->argc - 2) / 2; - - /* HMSET (deprecated) and HSET return value is different. */ - char *cmdname = c->argv[0]->ptr; - if (cmdname[1] == 's' || cmdname[1] == 'S') { - /* HSET */ - addReplyLongLong(c, created); - } else { - /* HMSET */ - addReply(c, shared.ok); - } - - hashTypeResetAccessContext(); -} - void hincrbyCommand(client *c) { long long value, incr, oldvalue; robj *o; @@ -1431,11 +1388,9 @@ void hgetCommand(client *c) { if ((o = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp])) == NULL || checkType(c, o, OBJ_HASH)) return; hashTypeSetAccessContext(o, c->db); + addHashFieldToReply(c, o, c->argv[2]->ptr); - if (hashTypeLength(o) == 0) { - dbDelete(c->db, c->argv[1]); - } hashTypeResetAccessContext(); } @@ -1521,6 +1476,281 @@ static void addHashIteratorCursorToReply(writePreparedClient *wpc, hashTypeItera } } +void hsetnxCommand(client *c) { + robj *o; + if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; + hashTypeSetAccessContext(o, c->db); + if (hashTypeExists(o, c->argv[2]->ptr)) { + addReply(c, shared.czero); + } else { + hashTypeTryConversion(o, c->argv, 2, 3); + hashTypeSet(o, c->argv[2]->ptr, c->argv[3]->ptr, EXPIRY_NONE, HASH_SET_COPY | HASH_SET_KEEP_EXPIRY); + signalModifiedKey(c, c->db, c->argv[1]); + notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); + server.dirty++; + addReply(c, shared.cone); + } + hashTypeResetAccessContext(); +} + +void hsetCommand(client *c) { + int i, created = 0; + robj *o; + + if ((c->argc % 2) == 1) { + addReplyErrorArity(c); + return; + } + + if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; + hashTypeTryConversion(o, c->argv, 2, c->argc - 1); + + hashTypeSetAccessContext(o, c->db); + for (i = 2; i < c->argc; i += 2) created += !hashTypeSet(o, c->argv[i]->ptr, c->argv[i + 1]->ptr, EXPIRY_NONE, HASH_SET_COPY | HASH_SET_KEEP_EXPIRY); + + signalModifiedKey(c, c->db, c->argv[1]); + notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); + server.dirty += (c->argc - 2) / 2; + + /* HMSET (deprecated) and HSET return value is different. */ + char *cmdname = c->argv[0]->ptr; + if (cmdname[1] == 's' || cmdname[1] == 'S') { + /* HSET */ + addReplyLongLong(c, created); + } else { + /* HMSET */ + addReply(c, shared.ok); + } + + hashTypeResetAccessContext(); +} + +void hsetexCommand(client *c) { + robj *o; + robj *expire = NULL; + robj *comparison = NULL; + int unit = UNIT_SECONDS; + int flags = OBJ_NO_FLAGS; + int fields_index = 0; + long long num_fields = 0; + long long when = EXPIRY_NONE; + int i = 0; + int set_flags = HASH_SET_COPY, set_expired = 0; + int changes = 0; + + for (; fields_index < c->argc; fields_index++) { + if (!strcasecmp(c->argv[fields_index]->ptr, "fields")) { + /* checking optional flags */ + if (parseExtendedStringArgumentsOrReply(c, &flags, &unit, &expire, &comparison, COMMAND_HSET, fields_index++) != C_OK) return; + if (getLongLongFromObjectOrReply(c, c->argv[fields_index++], &num_fields, NULL) != C_OK) return; + break; + } + } + + if (num_fields > (c->argc - fields_index) / 2) num_fields = (c->argc - fields_index) / 2; // Potential user error, but we would like to make effort to comply with the request. + + o = lookupKeyWrite(c->db, c->argv[1]); + if (checkType(c, o, OBJ_HASH)) + return; + + /* Check for object existence condition */ + if ((flags & OBJ_SET_NX && o) || (flags & OBJ_SET_XX && !o)) { + addReply(c, shared.czero); + return; + } + + if (o == NULL) { + o = createHashObject(); + dbAdd(c->db, c->argv[1], &o); + } + + /* Handle parsing and calculating the expiration time. */ + if (flags & OBJ_KEEPTTL) + set_flags |= HASH_SET_KEEP_EXPIRY; + else if (expire) { + if (getLongLongFromObjectOrReply(c, expire, &when, NULL) != C_OK) return; + if (unit == UNIT_SECONDS) { + if (when > LLONG_MAX / 1000 || when < LLONG_MIN / 1000) { + addReplyErrorExpireTime(c); + return; + } + when *= 1000; + } + if ((flags & (OBJ_EXAT | OBJ_PXAT)) && when > LLONG_MAX - commandTimeSnapshot()) { + addReplyErrorExpireTime(c); + return; + } + when += commandTimeSnapshot(); + + if (((flags & OBJ_PXAT) || (flags & OBJ_EXAT)) && checkAlreadyExpired(when)) { + set_expired = 1; + when = 0; + } + } + + /* Check for all fields condition */ + if (flags & (OBJ_SET_FNX | OBJ_SET_FXX)) { + for (i = fields_index; i < c->argc; i += 2) { + if (((flags & OBJ_SET_FNX) && hashTypeExists(o, c->argv[i]->ptr)) || + ((flags & OBJ_SET_FXX) && !hashTypeExists(o, c->argv[i]->ptr))) { + addReply(c, shared.czero); + return; + } + } + } + + hashTypeSetAccessContext(o, c->db); + + for (i = fields_index; i < c->argc; i += 2) { + if (set_expired) { + changes += hashTypeDelete(o, c->argv[i]->ptr); + } else { + hashTypeSet(o, c->argv[i]->ptr, c->argv[i + 1]->ptr, when, set_flags); + changes++; + } + } + if (expire) { + /* Propagate as HSETEX Key Value PXAT millisecond-timestamp if there is + * EX/PX/EXAT flag. */ + if (!(flags & OBJ_PXAT)) { + for (int i = 2; i < fields_index; i++) { + if (c->argv[i + 1] == expire) { + robj *milliseconds_obj = createStringObjectFromLongLong(when); + rewriteClientCommandArgument(c, i, shared.pxat); + rewriteClientCommandArgument(c, i + 1, milliseconds_obj); + decrRefCount(milliseconds_obj); + break; + } + } + } + notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", o, c->db->id); + if (set_expired && changes) + notifyKeyspaceEvent(NOTIFY_HASH, "hexpired", o, c->db->id); + } + signalModifiedKey(c, c->db, c->argv[1]); + notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); + server.dirty += changes; + addReplyLongLong(c, changes == num_fields ? 1 : 0); + + hashTypeResetAccessContext(); +} + +void hgetexCommand(client *c) { + robj *o; + robj *expire = NULL; + robj *comparison = NULL; + int unit = UNIT_SECONDS; + int flags = OBJ_NO_FLAGS; + int fields_index = 0; + long long num_fields = 0; + long long when = EXPIRY_NONE; + int i = 0; + int set_expiry = 0, set_expired = 0, persist = 0; + int changes = 0; + robj **new_argv = NULL; + robj *milliseconds_obj = NULL, *numitems_obj = NULL; + int new_argc = 0; + int milliseconds_index = -1, numitems_index = -1; + + for (; fields_index < c->argc; fields_index++) { + if (!strcasecmp(c->argv[fields_index]->ptr, "fields")) { + /* checking optional flags */ + if (parseExtendedStringArgumentsOrReply(c, &flags, &unit, &expire, &comparison, COMMAND_HGET, fields_index + 1) != C_OK) return; + fields_index++; + if (getLongLongFromObjectOrReply(c, c->argv[fields_index++], &num_fields, NULL) != C_OK) return; + break; + } + } + + if (num_fields > c->argc - fields_index) num_fields = c->argc - fields_index; // Potential user error, but we would like to make effort to comply with the request. + + o = lookupKeyRead(c->db, c->argv[1]); + if (checkType(c, o, OBJ_HASH)) + return; + + if (o == NULL) { + o = createHashObject(); + dbAdd(c->db, c->argv[1], &o); + } + + /* Handle parsing and calculating the expiration time. */ + if (flags & OBJ_PERSIST) { + persist = 1; + } else if (expire) { + if (getLongLongFromObjectOrReply(c, expire, &when, NULL) != C_OK) return; + if (unit == UNIT_SECONDS) { + if (when > LLONG_MAX / 1000 || when < LLONG_MIN / 1000) { + addReplyErrorExpireTime(c); + return; + } + when *= 1000; + } + if ((flags & (OBJ_EXAT | OBJ_PXAT)) && when > LLONG_MAX - commandTimeSnapshot()) { + addReplyErrorExpireTime(c); + return; + } + when += commandTimeSnapshot(); + if (((flags & OBJ_PXAT) || (flags & OBJ_EXAT)) && checkAlreadyExpired(when)) { + set_expired = 1; + } else { + set_expiry = 1; + } + } + + initDeferredReplyBuffer(c); + + addReplyArrayLen(c, num_fields); + /* This command is never propagated as is. It is either propagated as HPEXPIREAT or PERSIST. + * This why it doesn't need special handling in feedAppendOnlyFile to convert relative expire time to absolute one. */ + if (set_expiry || set_expired || persist) { + /* allocate a new client argv for replicating the command. */ + new_argv = zmalloc(sizeof(robj *) * (num_fields + 5)); + new_argv[new_argc++] = shared.hpexpireat; + new_argv[new_argc++] = c->argv[1]; + if (set_expiry) { + new_argv[new_argc++] = NULL; // placeholder for the expiration time + milliseconds_index = new_argc - 1; + } + new_argv[new_argc++] = shared.fields; + new_argv[new_argc++] = NULL; // placeholder for the number of objects + numitems_index = new_argc - 1; + } + for (i = fields_index; i < c->argc; i++) { + int changed = 0; + addHashFieldToReply(c, o, c->argv[i]->ptr); + if (set_expired) { + changed = hashTypeDelete(o, c->argv[i]->ptr); + } else if (set_expiry) { + changed = (hashTypeSetExpire(o, c->argv[i]->ptr, when, flags) == 1) ? 1 : 0; + } else if (persist) { + changed = hashTypePersist(o, c->argv[i]->ptr); + } + if (changed) { + changes++; + new_argv[new_argc++] = c->argv[i]; + } + } + if (changes) { + if (set_expiry) { + milliseconds_obj = createStringObjectFromLongLong(when); + new_argv[milliseconds_index] = milliseconds_obj; + } + numitems_obj = createStringObjectFromLongLong(changes); + new_argv[numitems_index] = numitems_obj; + replaceClientCommandVector(c, new_argc, new_argv); + server.dirty += changes; + signalModifiedKey(c, c->db, c->argv[1]); + if (set_expired) + notifyKeyspaceEvent(NOTIFY_HASH, "hexpired", c->argv[1], c->db->id); + notifyKeyspaceEvent(NOTIFY_HASH, set_expiry ? "hexpire" : "hpersist", c->argv[1], c->db->id); + if (milliseconds_obj) decrRefCount(milliseconds_obj); + if (numitems_obj) decrRefCount(numitems_obj); + } + if (new_argv) zfree(new_argv); + + commitDeferredReplyBuffer(c, 1); +} + void genericHgetallCommand(client *c, int flags) { robj *o; hashTypeIterator hi; @@ -1609,17 +1839,20 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { int flag = 0; int fields_index = 3; long long num_fields = 0; - int i, result = 0; + int i, result = 0, changes = 0; for (; fields_index < c->argc; fields_index++) { if (!strcasecmp(c->argv[fields_index]->ptr, "fields")) { /* checking optional flags */ - if (parseExtendedExpireArgumentsOrReply(c, &flag, fields_index++) != C_OK) return; + if (parseExtendedExpireArgumentsOrReply(c, &flag, fields_index + 1) != C_OK) return; + fields_index++; if (getLongLongFromObjectOrReply(c, c->argv[fields_index++], &num_fields, NULL) != C_OK) return; break; } } + if (num_fields > c->argc - fields_index) num_fields = c->argc - fields_index; // Potential user error, but we would like to make effort to comply with the request. + if (getLongLongFromObjectOrReply(c, param, &when, NULL) != C_OK) return; /* HEXPIRE allows negative numbers, but we can at least detect an @@ -1650,13 +1883,28 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { /* From this point we would return array reply */ addReplyArrayLen(c, num_fields); - for (i = fields_index; i < c->argc; i++) { + for (i = fields_index; i < num_fields; i++) { result = hashTypeSetExpire(obj, c->argv[i]->ptr, when, flag); server.dirty += (result > 0 ? 1 : 0); // in case there was a change increment the dirty + changes += (result > 0 ? 1 : 0); addReplyLongLong(c, result); } - notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", c->argv[1], c->db->id); + /* Propagate as HPEXPIREAT millisecond-timestamp + * Only rewrite the command arg if not already HPEXPIREAT */ + if (c->cmd->proc != hpexpireAtCommand) { + rewriteClientCommandArgument(c, 0, shared.pexpireat); + } + /* Avoid creating a string object when it's the same as argv[2] parameter */ + if (basetime != 0 || unit == UNIT_SECONDS) { + robj *when_obj = createStringObjectFromLongLong(when); + rewriteClientCommandArgument(c, 2, when_obj); + decrRefCount(when_obj); + } + if (changes) { + notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", c->argv[1], c->db->id); + signalModifiedKey(c, c->db, obj); + } hashTypeResetAccessContext(); } @@ -1677,12 +1925,12 @@ void hpexpireAtCommand(client *c) { } void hpersistCommand(client *c) { - int fields_index = 4, result = 0; + int fields_index = 4, result = 0, changes = 0; long long num_fields = 0; if (getLongLongFromObjectOrReply(c, c->argv[fields_index - 1], &num_fields, NULL) != C_OK) return; - if (num_fields > c->argc - 4) num_fields = c->argc - 4; // Potential user error, but we would like to make effort to comply with the request. + if (num_fields > c->argc - fields_index) num_fields = c->argc - fields_index; // Potential user error, but we would like to make effort to comply with the request. /* From this point we would return array reply */ addReplyArrayLen(c, num_fields); @@ -1691,12 +1939,16 @@ void hpersistCommand(client *c) { hashTypeSetAccessContext(hash, c->db); - for (; fields_index < 4 + num_fields; fields_index++) { + for (; fields_index < num_fields; fields_index++) { result = hashTypePersist(hash, c->argv[fields_index]->ptr); server.dirty += (result > 0 ? 1 : 0); // in case there was a change increment the dirty + changes += (result > 0 ? 1 : 0); addReplyLongLong(c, result); } - + if (changes) { + notifyKeyspaceEvent(NOTIFY_HASH, "hpersist", c->argv[1], c->db->id); + signalModifiedKey(c, c->db, hash); + } hashTypeResetAccessContext(); } @@ -1706,7 +1958,7 @@ void httlGenericCommand(client *c, long long basetime, int unit) { if (getLongLongFromObjectOrReply(c, c->argv[fields_index - 1], &num_fields, NULL) != C_OK) return; - if (num_fields > c->argc - 4) num_fields = c->argc - 4; // Potential user error, but we would like to make effort to comply with the request. + if (num_fields > c->argc - fields_index) num_fields = c->argc - fields_index; // Potential user error, but we would like to make effort to comply with the request. robj *hash = lookupKeyRead(c->db, c->argv[1]); @@ -1715,8 +1967,8 @@ void httlGenericCommand(client *c, long long basetime, int unit) { hashTypeSetAccessContext(hash, c->db); - for (; fields_index < 4 + num_fields; fields_index++) { - if (!hash || hashTypeGetExpiry(hash, c->argv[fields_index]->ptr, &result) == C_ERR) { + for (int i = 0; i < num_fields; i++) { + if (!hash || hashTypeGetExpiry(hash, c->argv[fields_index + i]->ptr, &result) == C_ERR) { addReplyLongLong(c, -2); } else if (result == EXPIRY_NONE) { addReplyLongLong(c, -1); diff --git a/src/t_string.c b/src/t_string.c index 967c8f96935..19a2db49b15 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -234,7 +234,7 @@ void setCommand(client *c) { int unit = UNIT_SECONDS; int flags = OBJ_NO_FLAGS; - if (parseExtendedStringArgumentsOrReply(c, &flags, &unit, &expire, &comparison, COMMAND_SET) != C_OK) { + if (parseExtendedStringArgumentsOrReply(c, &flags, &unit, &expire, &comparison, COMMAND_SET, c->argc) != C_OK) { return; } @@ -300,7 +300,7 @@ void getexCommand(client *c) { int unit = UNIT_SECONDS; int flags = OBJ_NO_FLAGS; - if (parseExtendedStringArgumentsOrReply(c, &flags, &unit, &expire, NULL, COMMAND_GET) != C_OK) { + if (parseExtendedStringArgumentsOrReply(c, &flags, &unit, &expire, NULL, COMMAND_GET, c->argc) != C_OK) { return; } From f62c163c3044e07be985ab1a89ee02f60ef3a157 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 14:20:49 +0300 Subject: [PATCH 13/33] format fixes Signed-off-by: Ran Shidlansik --- src/server.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.h b/src/server.h index 27e63bd1ad2..c0327f1cf06 100644 --- a/src/server.h +++ b/src/server.h @@ -729,8 +729,8 @@ typedef enum { #define OBJ_SET_IFEQ (1 << 9) /* Set if we need compare and set */ #define OBJ_ARGV3 (1 << 10) /* Set if the value is at argv[3]; otherwise it's \ * at argv[2]. */ -#define OBJ_SET_FNX (1 << 11) /* Set if key item not exists. */ -#define OBJ_SET_FXX (1 << 12) /* Set if key item exists. */ +#define OBJ_SET_FNX (1 << 11) /* Set if key item not exists. */ +#define OBJ_SET_FXX (1 << 12) /* Set if key item exists. */ /* An Object, that is a type able to hold a string / list / set */ From 6465314b09cf9dd827c339823e28fad3e185fae9 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 15:00:20 +0300 Subject: [PATCH 14/33] handle negative ttl correctly Signed-off-by: Ran Shidlansik --- src/t_hash.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index dc8b1d40361..56a3d54e0ad 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -1569,6 +1569,8 @@ void hsetexCommand(client *c) { set_flags |= HASH_SET_KEEP_EXPIRY; else if (expire) { if (getLongLongFromObjectOrReply(c, expire, &when, NULL) != C_OK) return; + long long basetime = (flags & (OBJ_EXAT | OBJ_PXAT)) ? 0 : commandTimeSnapshot(); + if (unit == UNIT_SECONDS) { if (when > LLONG_MAX / 1000 || when < LLONG_MIN / 1000) { addReplyErrorExpireTime(c); @@ -1576,11 +1578,11 @@ void hsetexCommand(client *c) { } when *= 1000; } - if ((flags & (OBJ_EXAT | OBJ_PXAT)) && when > LLONG_MAX - commandTimeSnapshot()) { + if (when > LLONG_MAX - basetime) { addReplyErrorExpireTime(c); return; } - when += commandTimeSnapshot(); + when += basetime; if (((flags & OBJ_PXAT) || (flags & OBJ_EXAT)) && checkAlreadyExpired(when)) { set_expired = 1; @@ -1678,6 +1680,8 @@ void hgetexCommand(client *c) { persist = 1; } else if (expire) { if (getLongLongFromObjectOrReply(c, expire, &when, NULL) != C_OK) return; + long long basetime = (flags & (OBJ_EXAT | OBJ_PXAT)) ? 0 : commandTimeSnapshot(); + if (unit == UNIT_SECONDS) { if (when > LLONG_MAX / 1000 || when < LLONG_MIN / 1000) { addReplyErrorExpireTime(c); @@ -1685,13 +1689,15 @@ void hgetexCommand(client *c) { } when *= 1000; } - if ((flags & (OBJ_EXAT | OBJ_PXAT)) && when > LLONG_MAX - commandTimeSnapshot()) { + if (when > LLONG_MAX - basetime) { addReplyErrorExpireTime(c); return; } - when += commandTimeSnapshot(); + when += basetime; + if (((flags & OBJ_PXAT) || (flags & OBJ_EXAT)) && checkAlreadyExpired(when)) { set_expired = 1; + when = 0; } else { set_expiry = 1; } From 4301399d28f9f0a2dbab70bc5b26d4872cd8c323 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 15:08:02 +0300 Subject: [PATCH 15/33] fix wrong assert condition on update entry Signed-off-by: Ran Shidlansik --- src/t_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_hash.c b/src/t_hash.c index 56a3d54e0ad..f4fc98b66aa 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -469,7 +469,7 @@ static void hashTypeTrackUpdateEntry(robj *o, void *old_entry, void *new_entry, else { volatile_set *set = hashTypeGetVolatileSet(o); debugServerAssert(set); - serverAssert(volatileSetUpdateEntry(set, old_entry, new_entry, old_expiry, new_expiry) == C_OK); + serverAssert(volatileSetUpdateEntry(set, old_entry, new_entry, old_expiry, new_expiry) == 1); } } From d97e23f84e1c4065e98e97fb842a5a97e2907899 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 15:19:53 +0300 Subject: [PATCH 16/33] fix FNX/FXX logic Signed-off-by: Ran Shidlansik --- src/object.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/object.c b/src/object.c index f7537703e6b..3b867554a2e 100644 --- a/src/object.c +++ b/src/object.c @@ -1873,13 +1873,13 @@ int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj * { *flags |= OBJ_SET_XX; } else if ((opt[0] == 'f' || opt[0] == 'F') && - (opt[1] == 'n' || opt[1] == 'N') && opt[2] == '\0' && + (opt[1] == 'n' || opt[1] == 'N') && (opt[2] == 'x' || opt[2] == 'X') && opt[3] == '\0' && !(*flags & OBJ_SET_FXX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_HSET)) { *flags |= OBJ_SET_FNX; } else if ((opt[0] == 'f' || opt[0] == 'F') && - (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && + (opt[1] == 'x' || opt[1] == 'X') && (opt[2] == 'x' || opt[2] == 'X') && opt[3] == '\0' && !(*flags & OBJ_SET_FNX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_HSET)) { @@ -1889,8 +1889,7 @@ int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj * (opt[2] == 'e' || opt[2] == 'E') && (opt[3] == 'q' || opt[3] == 'Q') && opt[4] == '\0' && next && - !(*flags & OBJ_SET_NX || *flags & OBJ_SET_XX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET) && - !(*flags & OBJ_SET_FNX || *flags & OBJ_SET_FXX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_HSET)) + !(*flags & OBJ_SET_NX || *flags & OBJ_SET_XX || *flags & OBJ_SET_IFEQ) && (command_type == COMMAND_SET)) { *flags |= OBJ_SET_IFEQ; *compare_val = next; @@ -1898,7 +1897,7 @@ int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj * } else if ((opt[0] == 'g' || opt[0] == 'G') && (opt[1] == 'e' || opt[1] == 'E') && (opt[2] == 't' || opt[2] == 'T') && opt[3] == '\0' && - (command_type == COMMAND_SET || command_type == COMMAND_HSET)) + (command_type == COMMAND_SET)) { *flags |= OBJ_SET_GET; } else if (!strcasecmp(opt, "KEEPTTL") && !(*flags & OBJ_PERSIST) && From 0723625343b5a0ea573ae7fc662559e323db82b5 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 15:24:59 +0300 Subject: [PATCH 17/33] Fix HEXPIRE parse limits Signed-off-by: Ran Shidlansik --- src/t_hash.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index f4fc98b66aa..3bb0546dd22 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -1850,8 +1850,7 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { for (; fields_index < c->argc; fields_index++) { if (!strcasecmp(c->argv[fields_index]->ptr, "fields")) { /* checking optional flags */ - if (parseExtendedExpireArgumentsOrReply(c, &flag, fields_index + 1) != C_OK) return; - fields_index++; + if (parseExtendedExpireArgumentsOrReply(c, &flag, fields_index++) != C_OK) return; if (getLongLongFromObjectOrReply(c, c->argv[fields_index++], &num_fields, NULL) != C_OK) return; break; } @@ -1889,8 +1888,8 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { /* From this point we would return array reply */ addReplyArrayLen(c, num_fields); - for (i = fields_index; i < num_fields; i++) { - result = hashTypeSetExpire(obj, c->argv[i]->ptr, when, flag); + for (i = 0; i < num_fields; i++) { + result = hashTypeSetExpire(obj, c->argv[fields_index + i]->ptr, when, flag); server.dirty += (result > 0 ? 1 : 0); // in case there was a change increment the dirty changes += (result > 0 ? 1 : 0); addReplyLongLong(c, result); From b782d44e2caeb3aff452756489652149fcb4b561 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 15:36:22 +0300 Subject: [PATCH 18/33] fix case of hll command issues on non-existing listpack encoded hash Signed-off-by: Ran Shidlansik --- src/t_hash.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index 3bb0546dd22..b6afaaac7f9 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -657,9 +657,11 @@ int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vle * The matching item expiration time is assigned to `expiry` memory location, if specified. * In case the item has no assigned expiration time, -1 is returned. */ int hashTypeGetExpiry(robj *o, sds field, long long *expiry) { - if (o->encoding == OBJ_ENCODING_LISTPACK && hashTypeExists(o, field)) { - if (expiry) *expiry = -1; - return C_OK; + if (o->encoding == OBJ_ENCODING_LISTPACK) { + if (hashTypeExists(o, field)) { + if (expiry) *expiry = -1; + return C_OK; + } } else if (o->encoding == OBJ_ENCODING_HASHTABLE) { void *found_element = NULL; if (hashtableFind(o->ptr, field, &found_element)) { From 31923c5aabd15b11eb753b70cde8d2285007be97 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 15:46:31 +0300 Subject: [PATCH 19/33] make httl functions verify the type Signed-off-by: Ran Shidlansik --- src/t_hash.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/t_hash.c b/src/t_hash.c index b6afaaac7f9..669499fd22c 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -1969,6 +1969,8 @@ void httlGenericCommand(client *c, long long basetime, int unit) { robj *hash = lookupKeyRead(c->db, c->argv[1]); + if (checkType(c, hash, OBJ_HASH)) return; + /* From this point we would return array reply */ addReplyArrayLen(c, num_fields); From 5e19c90117bbdf058a60b7e7135727a868392857 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 17:19:41 +0300 Subject: [PATCH 20/33] fix HGETEX replication handling Signed-off-by: Ran Shidlansik --- src/expire.c | 24 ++++++++++ src/server.c | 1 + src/server.h | 3 +- src/t_hash.c | 125 +++++++++++++++++++-------------------------------- 4 files changed, 73 insertions(+), 80 deletions(-) diff --git a/src/expire.c b/src/expire.c index a6bf0d3be45..8039998df37 100644 --- a/src/expire.c +++ b/src/expire.c @@ -578,6 +578,30 @@ int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_args) { return C_OK; } +int convertExpireArgumentToUnixTime(client *c, robj *arg, long long basetime, int unit, long long *unixtime) { + long long when; + if (getLongLongFromObjectOrReply(c, arg, &when, NULL) != C_OK) return C_ERR; + + if (when < 0) { + addReplyErrorExpireTime(c); + } + + if (unit == UNIT_SECONDS) { + if (when > LLONG_MAX / 1000 || when < LLONG_MIN / 1000) { + addReplyErrorExpireTime(c); + return C_ERR; + } + when *= 1000; + } + if (when > LLONG_MAX - basetime) { + addReplyErrorExpireTime(c); + return C_ERR; + } + when += basetime; + if (unixtime) *unixtime = when; + return C_OK; +} + /*----------------------------------------------------------------------------- * Expires Commands *----------------------------------------------------------------------------*/ diff --git a/src/server.c b/src/server.c index 95350585d27..afc9b965b1b 100644 --- a/src/server.c +++ b/src/server.c @@ -2126,6 +2126,7 @@ void createSharedObjects(void) { shared.hset = createStringObject("HSET", 4); shared.hdel = createStringObject("HDEL", 4); shared.hpexpireat = createStringObject("HPEXPIREAT", 10); + shared.hpersist = createStringObject("HPERSIST", 8); shared.srem = createStringObject("SREM", 4); shared.xgroup = createStringObject("XGROUP", 6); shared.xclaim = createStringObject("XCLAIM", 6); diff --git a/src/server.h b/src/server.h index c0327f1cf06..ab0e836c717 100644 --- a/src/server.h +++ b/src/server.h @@ -1352,7 +1352,7 @@ struct sharedObjectsStruct { *loadingerr, *slowevalerr, *slowscripterr, *slowmoduleerr, *bgsaveerr, *primarydownerr, *roreplicaerr, *execaborterr, *noautherr, *noreplicaserr, *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink, *rpop, *lpop, *lpush, - *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *hdel, *hpexpireat, *srem, + *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *hdel, *hpexpireat, *hpersist, *srem, *xgroup, *xclaim, *script, *replconf, *eval, *persist, *set, *pexpireat, *pexpire, *time, *pxat, *absttl, *retrycount, *force, *justid, *entriesread, *lastid, *ping, *setid, *keepttl, *load, *createconsumer, *getack, *special_asterisk, *special_equals, *default_username, *redacted, *ssubscribebulk, *sunsubscribebulk, *fields, @@ -2839,6 +2839,7 @@ int processIOThreadsWriteDone(void); int canExpireWithFlags(int flags, int *can_delete); int parseExtendedExpireArgumentsOrReply(client *c, int *flags, int max_args); int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj **expire, robj **compare_val, int command_type, int max_args); +int convertExpireArgumentToUnixTime(client *c, robj *arg, long long basetime, int unit, long long *unixtime); /* logreqres.c - logging of requests and responses */ void reqresReset(client *c, int free_buf); diff --git a/src/t_hash.c b/src/t_hash.c index 669499fd22c..d3d5e068676 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -828,8 +828,6 @@ int hashTypeSetExpire(robj *o, sds field, long long expiry, int flag) { /* If no object we will return -2 */ if (o == NULL) return -2; - int expired = timestampIsExpired(expiry); - if (o->encoding == OBJ_ENCODING_LISTPACK) { /* When listpack representation is used, we consider it as infinite TTL, * so expire command with gt always fail the GT as well as existence(XX). @@ -837,16 +835,6 @@ int hashTypeSetExpire(robj *o, sds field, long long expiry, int flag) { if (flag & EXPIRE_XX || flag & EXPIRE_GT) { return 0; } else { - if (expired) { - /* It is possible that the assigned expiration is set in the past (or zero). - * In such case we cannot count on the hash object representation to be hashtable. */ - if (hashTypeDelete(o, field)) { - hashTypeExpireEntry(field); - return 2; - } else { - return -2; - } - } hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); } } @@ -889,12 +877,6 @@ int hashTypeSetExpire(robj *o, sds field, long long expiry, int flag) { } } } - if (expired) { - if (hashTypeDelete(o, field)) { - hashTypeExpireEntry(field); - return 2; - } - } *entry_ref = hashTypeEntrySetExpiry(current_entry, expiry); hashTypeTrackUpdateEntry(o, current_entry, *entry_ref, current_expire, expiry); return 1; @@ -1570,21 +1552,10 @@ void hsetexCommand(client *c) { if (flags & OBJ_KEEPTTL) set_flags |= HASH_SET_KEEP_EXPIRY; else if (expire) { - if (getLongLongFromObjectOrReply(c, expire, &when, NULL) != C_OK) return; long long basetime = (flags & (OBJ_EXAT | OBJ_PXAT)) ? 0 : commandTimeSnapshot(); - if (unit == UNIT_SECONDS) { - if (when > LLONG_MAX / 1000 || when < LLONG_MIN / 1000) { - addReplyErrorExpireTime(c); - return; - } - when *= 1000; - } - if (when > LLONG_MAX - basetime) { - addReplyErrorExpireTime(c); + if (convertExpireArgumentToUnixTime(c, expire, basetime, unit, &when) == C_ERR) return; - } - when += basetime; if (((flags & OBJ_PXAT) || (flags & OBJ_EXAT)) && checkAlreadyExpired(when)) { set_expired = 1; @@ -1659,8 +1630,7 @@ void hgetexCommand(client *c) { for (; fields_index < c->argc; fields_index++) { if (!strcasecmp(c->argv[fields_index]->ptr, "fields")) { /* checking optional flags */ - if (parseExtendedStringArgumentsOrReply(c, &flags, &unit, &expire, &comparison, COMMAND_HGET, fields_index + 1) != C_OK) return; - fields_index++; + if (parseExtendedStringArgumentsOrReply(c, &flags, &unit, &expire, &comparison, COMMAND_HGET, fields_index++) != C_OK) return; if (getLongLongFromObjectOrReply(c, c->argv[fields_index++], &num_fields, NULL) != C_OK) return; break; } @@ -1681,21 +1651,10 @@ void hgetexCommand(client *c) { if (flags & OBJ_PERSIST) { persist = 1; } else if (expire) { - if (getLongLongFromObjectOrReply(c, expire, &when, NULL) != C_OK) return; long long basetime = (flags & (OBJ_EXAT | OBJ_PXAT)) ? 0 : commandTimeSnapshot(); - if (unit == UNIT_SECONDS) { - if (when > LLONG_MAX / 1000 || when < LLONG_MIN / 1000) { - addReplyErrorExpireTime(c); - return; - } - when *= 1000; - } - if (when > LLONG_MAX - basetime) { - addReplyErrorExpireTime(c); + if (convertExpireArgumentToUnixTime(c, expire, basetime, unit, &when) == C_ERR) return; - } - when += basetime; if (((flags & OBJ_PXAT) || (flags & OBJ_EXAT)) && checkAlreadyExpired(when)) { set_expired = 1; @@ -1713,9 +1672,13 @@ void hgetexCommand(client *c) { if (set_expiry || set_expired || persist) { /* allocate a new client argv for replicating the command. */ new_argv = zmalloc(sizeof(robj *) * (num_fields + 5)); - new_argv[new_argc++] = shared.hpexpireat; + if (persist) + new_argv[new_argc++] = shared.hpersist; + else + new_argv[new_argc++] = shared.hpexpireat; + new_argv[new_argc++] = c->argv[1]; - if (set_expiry) { + if (set_expiry || set_expired) { new_argv[new_argc++] = NULL; // placeholder for the expiration time milliseconds_index = new_argc - 1; } @@ -1729,7 +1692,7 @@ void hgetexCommand(client *c) { if (set_expired) { changed = hashTypeDelete(o, c->argv[i]->ptr); } else if (set_expiry) { - changed = (hashTypeSetExpire(o, c->argv[i]->ptr, when, flags) == 1) ? 1 : 0; + changed = (hashTypeSetExpire(o, c->argv[i]->ptr, when, 0) == 1) ? 1 : 0; } else if (persist) { changed = hashTypePersist(o, c->argv[i]->ptr); } @@ -1745,6 +1708,10 @@ void hgetexCommand(client *c) { } numitems_obj = createStringObjectFromLongLong(changes); new_argv[numitems_index] = numitems_obj; + + for (i = 0; i < new_argc; i++) + if (new_argv[i]) + incrRefCount(new_argv[i]); replaceClientCommandVector(c, new_argc, new_argv); server.dirty += changes; signalModifiedKey(c, c->db, c->argv[1]); @@ -1753,8 +1720,9 @@ void hgetexCommand(client *c) { notifyKeyspaceEvent(NOTIFY_HASH, set_expiry ? "hexpire" : "hpersist", c->argv[1], c->db->id); if (milliseconds_obj) decrRefCount(milliseconds_obj); if (numitems_obj) decrRefCount(numitems_obj); + } else { + if (new_argv) zfree(new_argv); } - if (new_argv) zfree(new_argv); commitDeferredReplyBuffer(c, 1); } @@ -1847,7 +1815,7 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { int flag = 0; int fields_index = 3; long long num_fields = 0; - int i, result = 0, changes = 0; + int i, result = 0, expired = 0, updated = 0; for (; fields_index < c->argc; fields_index++) { if (!strcasecmp(c->argv[fields_index]->ptr, "fields")) { @@ -1860,23 +1828,11 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { if (num_fields > c->argc - fields_index) num_fields = c->argc - fields_index; // Potential user error, but we would like to make effort to comply with the request. - if (getLongLongFromObjectOrReply(c, param, &when, NULL) != C_OK) return; - - /* HEXPIRE allows negative numbers, but we can at least detect an - * overflow by either unit conversion or basetime addition. */ - if (unit == UNIT_SECONDS) { - if (when > LLONG_MAX / 1000 || when < LLONG_MIN / 1000) { - addReplyErrorExpireTime(c); - return; - } - when *= 1000; - } - - if (when > LLONG_MAX - basetime) { - addReplyErrorExpireTime(c); + if (convertExpireArgumentToUnixTime(c, param, basetime, unit, &when) == C_ERR) return; - } - when += basetime; + + if (checkAlreadyExpired(when)) + when = 0; robj *obj = lookupKeyWrite(c->db, key); @@ -1891,24 +1847,35 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { addReplyArrayLen(c, num_fields); for (i = 0; i < num_fields; i++) { - result = hashTypeSetExpire(obj, c->argv[fields_index + i]->ptr, when, flag); + if (when == 0) { + result = -2; + if (hashTypeDelete(obj, c->argv[fields_index + i]->ptr)) { + result = 2; + expired++; + } + } else { + result = hashTypeSetExpire(obj, c->argv[fields_index + i]->ptr, when, flag); + updated++; + } server.dirty += (result > 0 ? 1 : 0); // in case there was a change increment the dirty - changes += (result > 0 ? 1 : 0); addReplyLongLong(c, result); } - /* Propagate as HPEXPIREAT millisecond-timestamp - * Only rewrite the command arg if not already HPEXPIREAT */ - if (c->cmd->proc != hpexpireAtCommand) { - rewriteClientCommandArgument(c, 0, shared.pexpireat); - } - /* Avoid creating a string object when it's the same as argv[2] parameter */ - if (basetime != 0 || unit == UNIT_SECONDS) { - robj *when_obj = createStringObjectFromLongLong(when); - rewriteClientCommandArgument(c, 2, when_obj); - decrRefCount(when_obj); - } - if (changes) { + if (expired || updated) { + /* Propagate as HPEXPIREAT millisecond-timestamp + * Only rewrite the command arg if not already HPEXPIREAT */ + if (c->cmd->proc != hpexpireAtCommand) { + rewriteClientCommandArgument(c, 0, shared.pexpireat); + } + + /* Avoid creating a string object when it's the same as argv[2] parameter */ + if (basetime != 0 || unit == UNIT_SECONDS) { + robj *when_obj = createStringObjectFromLongLong(when); + rewriteClientCommandArgument(c, 2, when_obj); + decrRefCount(when_obj); + } + if (expired) + notifyKeyspaceEvent(NOTIFY_HASH, "hexpired", c->argv[1], c->db->id); notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", c->argv[1], c->db->id); signalModifiedKey(c, c->db, obj); } From 20c0d29529a779f40a450ffe8385eec864c0c78e Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 20:13:23 +0300 Subject: [PATCH 21/33] fix hexpire propagation to use hpexpireat Signed-off-by: Ran Shidlansik --- src/module.c | 2 +- src/t_hash.c | 8 ++++---- tests/unit/expire.tcl | 4 +--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/module.c b/src/module.c index 034d9b7dd69..21b5f15e8a7 100644 --- a/src/module.c +++ b/src/module.c @@ -5345,7 +5345,7 @@ int VM_HashSet(ValkeyModuleKey *key, int flags, ...) { /* If CFIELDS is active, we can pass the ownership of the * SDS object to the low level function that sets the field * to avoid a useless copy. */ - if (flags & VALKEYMODULE_HASH_CFIELDS) low_flags |= (HASH_SET_TAKE_FIELD | HASH_SET_KEEP_EXPIRY); + if (flags & VALKEYMODULE_HASH_CFIELDS) low_flags |= (HASH_SET_TAKE_FIELD); robj *argv[2] = {field, value}; hashTypeTryConversion(key->value, argv, 0, 1); diff --git a/src/t_hash.c b/src/t_hash.c index d3d5e068676..ebd13092865 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -1280,7 +1280,7 @@ void hincrbyCommand(client *c) { } value += incr; new = sdsfromlonglong(value); - hashTypeSet(o, c->argv[2]->ptr, new, EXPIRY_NONE, HASH_SET_TAKE_VALUE | HASH_SET_KEEP_EXPIRY); + hashTypeSet(o, c->argv[2]->ptr, new, EXPIRY_NONE, HASH_SET_TAKE_VALUE); signalModifiedKey(c, c->db, c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH, "hincrby", c->argv[1], c->db->id); server.dirty++; @@ -1329,7 +1329,7 @@ void hincrbyfloatCommand(client *c) { char buf[MAX_LONG_DOUBLE_CHARS]; int len = ld2string(buf, sizeof(buf), value, LD_STR_HUMAN); new = sdsnewlen(buf, len); - hashTypeSet(o, c->argv[2]->ptr, new, EXPIRY_NONE, HASH_SET_TAKE_VALUE | HASH_SET_KEEP_EXPIRY); + hashTypeSet(o, c->argv[2]->ptr, new, EXPIRY_NONE, HASH_SET_TAKE_VALUE); signalModifiedKey(c, c->db, c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH, "hincrbyfloat", c->argv[1], c->db->id); server.dirty++; @@ -1490,7 +1490,7 @@ void hsetCommand(client *c) { hashTypeTryConversion(o, c->argv, 2, c->argc - 1); hashTypeSetAccessContext(o, c->db); - for (i = 2; i < c->argc; i += 2) created += !hashTypeSet(o, c->argv[i]->ptr, c->argv[i + 1]->ptr, EXPIRY_NONE, HASH_SET_COPY | HASH_SET_KEEP_EXPIRY); + for (i = 2; i < c->argc; i += 2) created += !hashTypeSet(o, c->argv[i]->ptr, c->argv[i + 1]->ptr, EXPIRY_NONE, HASH_SET_COPY); signalModifiedKey(c, c->db, c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); @@ -1865,7 +1865,7 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { /* Propagate as HPEXPIREAT millisecond-timestamp * Only rewrite the command arg if not already HPEXPIREAT */ if (c->cmd->proc != hpexpireAtCommand) { - rewriteClientCommandArgument(c, 0, shared.pexpireat); + rewriteClientCommandArgument(c, 0, shared.hpexpireat); } /* Avoid creating a string object when it's the same as argv[2] parameter */ diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index 9d4ff8c52ce..21c0b140aad 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -630,9 +630,7 @@ start_server {tags {"expire"}} { } else { fail "field not added to replica" } - - # bp 1 - + $primary hpexpire myhash 1000 fields 1 f2 From a6844ac3fbaccbefd86e73b2dd99d914fa4aaffa Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 20:15:18 +0300 Subject: [PATCH 22/33] add commands json files Signed-off-by: Ran Shidlansik --- src/commands/hgetex.json | 109 ++++++++++++++++++++++++++++ src/commands/hsetex.json | 151 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 src/commands/hgetex.json create mode 100644 src/commands/hsetex.json diff --git a/src/commands/hgetex.json b/src/commands/hgetex.json new file mode 100644 index 00000000000..05137f1e861 --- /dev/null +++ b/src/commands/hgetex.json @@ -0,0 +1,109 @@ +{ + "HGETEX": { + "summary": "Set the value of one or more fields of a given hash key, and optionally set their expiration time.", + "complexity": "O(1)", + "group": "hash", + "since": "9.0.0", + "arity": -4, + "function": "hgetexCommand", + "command_flags": [ + "READONLY", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RO", + "ACCESS" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "description": "List of values associated with the given fields, in the same order as they are requested.", + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "expiration", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "seconds", + "type": "integer", + "token": "EX" + }, + { + "name": "milliseconds", + "type": "integer", + "token": "PX" + }, + { + "name": "unix-time-seconds", + "type": "unix-time", + "token": "EXAT" + }, + { + "name": "unix-time-milliseconds", + "type": "unix-time", + "token": "PXAT" + }, + { + "name": "persist", + "type": "pure-token", + "token": "PERSIST" + } + ] + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "field", + "type": "string", + "multiple": true + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/commands/hsetex.json b/src/commands/hsetex.json new file mode 100644 index 00000000000..06fe7baad32 --- /dev/null +++ b/src/commands/hsetex.json @@ -0,0 +1,151 @@ +{ + "HSETEX": { + "summary": "Set the value of one or more fields of a given hash key, and optionally set their expiration time.", + "complexity": "O(1)", + "group": "hash", + "since": "9.0.0", + "arity": -4, + "function": "hsetexCommand", + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "HASH" + ], + "key_specs": [ + { + "flags": [ + "RW", + "INSERT" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "reply_schema": { + "oneOf": [ + { + "description": "None of the provided fields value and or expiration time was set.", + "const": 0 + }, + { + "description": "All the fields value and or expiration time was set.", + "const": 1 + } + ] + }, + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "key-condition", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "nx", + "type": "pure-token", + "token": "NX" + }, + { + "name": "xx", + "type": "pure-token", + "token": "XX" + } + ] + }, + { + "name": "fields-condition", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "fnx", + "type": "pure-token", + "token": "FNX" + }, + { + "name": "fxx", + "type": "pure-token", + "token": "FXX" + } + ] + }, + { + "name": "expiration", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "seconds", + "type": "integer", + "token": "EX" + }, + { + "name": "milliseconds", + "type": "integer", + "token": "PX" + }, + { + "name": "unix-time-seconds", + "type": "unix-time", + "token": "EXAT" + }, + { + "name": "unix-time-milliseconds", + "type": "unix-time", + "token": "PXAT" + }, + { + "name": "keepttl", + "type": "pure-token", + "token": "KEEPTTL" + } + ] + }, + { + "name": "fields", + "token": "FIELDS", + "type": "block", + "arguments": [ + { + "name": "numfields", + "type": "integer", + "key_spec_index": 0, + "multiple": false + }, + { + "name": "data", + "type": "block", + "multiple": true, + "arguments": [ + { + "name": "field", + "type": "string" + }, + { + "name": "value", + "type": "string" + } + ] + } + ] + } + ] + } +} \ No newline at end of file From 36b7356d84e4a354f2037cd0052904c6f2cdc6ee Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 21:13:15 +0300 Subject: [PATCH 23/33] allow setting the key object in context Signed-off-by: Ran Shidlansik --- src/rdb.c | 2 +- src/server.c | 10 ++++++---- src/server.h | 5 +++-- src/t_hash.c | 56 ++++++++++++++++++++++++++++------------------------ 4 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 8833126cb02..bec05e6667a 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -967,7 +967,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { nwritten += n; /* check if need to add expired time for the hash elements */ int add_expiry = hashTypeHasVolatileElements(o); - setAccessContextWithFlags(o, &server.db[dbid], OBJ_ACCESS_IGNORE_TTL); + setAccessContextWithFlags(key, o, &server.db[dbid], OBJ_ACCESS_IGNORE_TTL); hashtableIterator iter; hashtableInitIterator(&iter, ht, 0); diff --git a/src/server.c b/src/server.c index afc9b965b1b..3738c738e68 100644 --- a/src/server.c +++ b/src/server.c @@ -7226,18 +7226,20 @@ __attribute__((weak)) int main(int argc, char **argv) { return 0; } -void setAccessContext(robj *o, serverDb *db) { - setAccessContextWithFlags(o, db, OBJ_ACCESS_NORMAL); +void setAccessContext(robj *key, robj *val, serverDb *db) { + setAccessContextWithFlags(key, val, db, OBJ_ACCESS_NORMAL); } -void setAccessContextWithFlags(robj *o, serverDb *db, int flags) { - server.access_context.key = o; +void setAccessContextWithFlags(robj *key, robj *val, serverDb *db, int flags) { + server.access_context.key = key; + server.access_context.val = val; server.access_context.db = db; server.access_context.flags = flags; } void resetAccessContext(void) { server.access_context.key = NULL; + server.access_context.val = NULL; server.access_context.db = NULL; server.access_context.flags = OBJ_ACCESS_NONE; } diff --git a/src/server.h b/src/server.h index ab0e836c717..9b188d5fca3 100644 --- a/src/server.h +++ b/src/server.h @@ -1607,6 +1607,7 @@ typedef enum childInfoType { typedef struct keyAccessContext { int flags; robj *key; + robj *val; serverDb *db; } keyAccessContext; @@ -3288,8 +3289,8 @@ void *activeDefragAlloc(void *ptr); robj *activeDefragStringOb(robj *ob); void dismissSds(sds s); void dismissMemoryInChild(void); -void setAccessContext(robj *o, serverDb *db); -void setAccessContextWithFlags(robj *o, serverDb *db, int flags); +void setAccessContext(robj *key, robj *val,serverDb *db); +void setAccessContextWithFlags(robj *key, robj *val, serverDb *db, int flags); void resetAccessContext(void); #define RESTART_SERVER_NONE 0 diff --git a/src/t_hash.c b/src/t_hash.c index ebd13092865..fe3309762c4 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -494,13 +494,14 @@ void hashTypePropagateDeletion(serverDb *db, sds key, void *entry) { } int hashTypeExpireEntry(void *entry) { - serverAssert(server.access_context.key && server.access_context.db); - robj keyobj; - sds key = objectGetKey(server.access_context.key); - serverAssert(key); - initStaticStringObject(keyobj, key); - notifyKeyspaceEvent(NOTIFY_EXPIRED, "hexpired", &keyobj, server.access_context.db->id); - serverLog(LL_NOTICE, "expiring entry %s of key %s", (sds)entry, key); + serverAssert(server.access_context.val && server.access_context.db); + robj *keyobj = server.access_context.key; + robj *o = server.access_context.val; + sds key = objectGetKey(o); + if (!keyobj) { + keyobj = createStringObject(key, sdslen(key)); + } + notifyKeyspaceEvent(NOTIFY_EXPIRED, "hexpired", keyobj, server.access_context.db->id); hashTypePropagateDeletion(server.access_context.db, key, entry); return 1; } @@ -533,20 +534,23 @@ hashtableElementAccessState hashHashtableTypeAccess(hashtable *ht, void *entry) return ELEMENT_DELETE; } -void hashTypeSetAccessContext(robj *o, serverDb *db) { - setAccessContext(o, db); +void hashTypeSetAccessContext(robj *key, robj *val, serverDb *db) { + setAccessContext(key, val, db); } void hashTypeResetAccessContext(void) { - robj keyobj; - robj *o = server.access_context.key; + robj *keyobj = server.access_context.key; + robj *o = server.access_context.val; serverDb *db = server.access_context.db; serverAssert(!o || o->type == OBJ_HASH); resetAccessContext(); if (o) { if (hashTypeLength(o) == 0) { - initStaticStringObject(keyobj, objectGetKey(o)); - notifyKeyspaceEvent(NOTIFY_GENERIC, "del", &keyobj, db->id); + if (!keyobj) { + sds key = objectGetKey(o); + keyobj = createStringObject(key, sdslen(key)); + } + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", keyobj, db->id); dbDelete(db, &keyobj); } } @@ -1258,7 +1262,7 @@ void hincrbyCommand(client *c) { if (getLongLongFromObjectOrReply(c, c->argv[3], &incr, NULL) != C_OK) return; if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); if (hashTypeGetValue(o, c->argv[2]->ptr, &vstr, &vlen, &value) == C_OK) { if (vstr) { if (string2ll((char *)vstr, vlen, &value) == 0) { @@ -1303,7 +1307,7 @@ void hincrbyfloatCommand(client *c) { } if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); if (hashTypeGetValue(o, c->argv[2]->ptr, &vstr, &vlen, &ll) == C_OK) { if (vstr) { @@ -1371,7 +1375,7 @@ void hgetCommand(client *c) { robj *o; if ((o = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp])) == NULL || checkType(c, o, OBJ_HASH)) return; - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); addHashFieldToReply(c, o, c->argv[2]->ptr); @@ -1388,7 +1392,7 @@ void hmgetCommand(client *c) { if (checkType(c, o, OBJ_HASH)) return; - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); addReplyArrayLen(c, c->argc - 2); for (i = 2; i < c->argc; i++) { @@ -1406,7 +1410,7 @@ void hdelCommand(client *c) { int j, deleted = 0; if ((o = lookupKeyWriteOrReply(c, c->argv[1], shared.czero)) == NULL || checkType(c, o, OBJ_HASH)) return; - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); for (j = 2; j < c->argc; j++) { if (hashTypeDelete(o, c->argv[j]->ptr)) { deleted++; @@ -1436,7 +1440,7 @@ void hstrlenCommand(client *c) { robj *o; if ((o = lookupKeyReadOrReply(c, c->argv[1], shared.czero)) == NULL || checkType(c, o, OBJ_HASH)) return; - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); addReplyLongLong(c, hashTypeGetValueLength(o, c->argv[2]->ptr)); hashTypeResetAccessContext(); } @@ -1463,7 +1467,7 @@ static void addHashIteratorCursorToReply(writePreparedClient *wpc, hashTypeItera void hsetnxCommand(client *c) { robj *o; if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); if (hashTypeExists(o, c->argv[2]->ptr)) { addReply(c, shared.czero); } else { @@ -1489,7 +1493,7 @@ void hsetCommand(client *c) { if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) return; hashTypeTryConversion(o, c->argv, 2, c->argc - 1); - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); for (i = 2; i < c->argc; i += 2) created += !hashTypeSet(o, c->argv[i]->ptr, c->argv[i + 1]->ptr, EXPIRY_NONE, HASH_SET_COPY); signalModifiedKey(c, c->db, c->argv[1]); @@ -1574,7 +1578,7 @@ void hsetexCommand(client *c) { } } - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); for (i = fields_index; i < c->argc; i += 2) { if (set_expired) { @@ -1777,7 +1781,7 @@ void hgetallCommand(client *c) { void hexistsCommand(client *c) { robj *o; if ((o = lookupKeyReadOrReply(c, c->argv[1], shared.czero)) == NULL || checkType(c, o, OBJ_HASH)) return; - hashTypeSetAccessContext(o, c->db); + hashTypeSetAccessContext(c->argv[1], o, c->db); addReply(c, hashTypeExists(o, c->argv[2]->ptr) ? shared.cone : shared.czero); hashTypeResetAccessContext(); } @@ -1841,7 +1845,7 @@ void hexpireGenericCommand(client *c, long long basetime, int unit) { return; } - hashTypeSetAccessContext(obj, c->db); + hashTypeSetAccessContext(key, obj, c->db); /* From this point we would return array reply */ addReplyArrayLen(c, num_fields); @@ -1911,7 +1915,7 @@ void hpersistCommand(client *c) { robj *hash = lookupKeyWrite(c->db, c->argv[1]); - hashTypeSetAccessContext(hash, c->db); + hashTypeSetAccessContext(c->argv[1], hash, c->db); for (; fields_index < num_fields; fields_index++) { result = hashTypePersist(hash, c->argv[fields_index]->ptr); @@ -1941,7 +1945,7 @@ void httlGenericCommand(client *c, long long basetime, int unit) { /* From this point we would return array reply */ addReplyArrayLen(c, num_fields); - hashTypeSetAccessContext(hash, c->db); + hashTypeSetAccessContext(c->argv[1], hash, c->db); for (int i = 0; i < num_fields; i++) { if (!hash || hashTypeGetExpiry(hash, c->argv[fields_index + i]->ptr, &result) == C_ERR) { From e72d7a6754c7411ed617ce403319fcc4adb3f434 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 21:25:17 +0300 Subject: [PATCH 24/33] fix build issues Signed-off-by: Ran Shidlansik --- src/t_hash.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index fe3309762c4..3f94031d3f8 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -500,9 +500,12 @@ int hashTypeExpireEntry(void *entry) { sds key = objectGetKey(o); if (!keyobj) { keyobj = createStringObject(key, sdslen(key)); + } else { + incrRefCount(keyobj); } notifyKeyspaceEvent(NOTIFY_EXPIRED, "hexpired", keyobj, server.access_context.db->id); hashTypePropagateDeletion(server.access_context.db, key, entry); + decrRefCount(keyobj); return 1; } @@ -524,7 +527,7 @@ hashtableElementAccessState hashHashtableTypeAccess(hashtable *ht, void *entry) if (server.access_context.flags == OBJ_ACCESS_NONE) return ELEMENT_INVALID; - robj *o = server.access_context.key; + robj *o = server.access_context.val; serverDb *db = server.access_context.db; serverAssert(o && db); @@ -549,9 +552,12 @@ void hashTypeResetAccessContext(void) { if (!keyobj) { sds key = objectGetKey(o); keyobj = createStringObject(key, sdslen(key)); + } else { + incrRefCount(keyobj); } notifyKeyspaceEvent(NOTIFY_GENERIC, "del", keyobj, db->id); - dbDelete(db, &keyobj); + dbDelete(db, keyobj); + decrRefCount(keyobj); } } } From c5b8d76b97b15e02fd2915c77fc43c5a4ce82552 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Mon, 19 May 2025 21:47:42 +0300 Subject: [PATCH 25/33] fix formatting issue Signed-off-by: Ran Shidlansik --- src/server.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.h b/src/server.h index 9b188d5fca3..45d36954e2a 100644 --- a/src/server.h +++ b/src/server.h @@ -3289,7 +3289,7 @@ void *activeDefragAlloc(void *ptr); robj *activeDefragStringOb(robj *ob); void dismissSds(sds s); void dismissMemoryInChild(void); -void setAccessContext(robj *key, robj *val,serverDb *db); +void setAccessContext(robj *key, robj *val, serverDb *db); void setAccessContextWithFlags(robj *key, robj *val, serverDb *db, int flags); void resetAccessContext(void); From 753ba3c67c2307cbc2c084e52963ca2aaae37828 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Tue, 20 May 2025 14:19:34 +0300 Subject: [PATCH 26/33] fix object pass to keyspace notification in HSETEX Signed-off-by: Ran Shidlansik --- src/t_hash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index 3f94031d3f8..26980388e21 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -1608,9 +1608,9 @@ void hsetexCommand(client *c) { } } } - notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", o, c->db->id); + notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", c->argv[1], c->db->id); if (set_expired && changes) - notifyKeyspaceEvent(NOTIFY_HASH, "hexpired", o, c->db->id); + notifyKeyspaceEvent(NOTIFY_HASH, "hexpired", c->argv[1], c->db->id); } signalModifiedKey(c, c->db, c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH, "hset", c->argv[1], c->db->id); From 0b8dc0307e873b5aab7e81234cbd64a0bfaaa7d8 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Wed, 21 May 2025 14:31:44 +0300 Subject: [PATCH 27/33] centralize keyspace and key signal notifications to the reset context Signed-off-by: Ran Shidlansik --- src/server.c | 2 ++ src/server.h | 1 + src/t_hash.c | 22 +++++++++++++++++----- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/server.c b/src/server.c index 3738c738e68..ef170d7b5ec 100644 --- a/src/server.c +++ b/src/server.c @@ -7235,6 +7235,7 @@ void setAccessContextWithFlags(robj *key, robj *val, serverDb *db, int flags) { server.access_context.val = val; server.access_context.db = db; server.access_context.flags = flags; + server.access_context.expired = 0; } void resetAccessContext(void) { @@ -7242,6 +7243,7 @@ void resetAccessContext(void) { server.access_context.val = NULL; server.access_context.db = NULL; server.access_context.flags = OBJ_ACCESS_NONE; + server.access_context.expired = 0; } /* The End */ diff --git a/src/server.h b/src/server.h index 45d36954e2a..ce394843c71 100644 --- a/src/server.h +++ b/src/server.h @@ -1609,6 +1609,7 @@ typedef struct keyAccessContext { robj *key; robj *val; serverDb *db; + uint64_t expired; } keyAccessContext; diff --git a/src/t_hash.c b/src/t_hash.c index 26980388e21..f38eae6c666 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -503,7 +503,6 @@ int hashTypeExpireEntry(void *entry) { } else { incrRefCount(keyobj); } - notifyKeyspaceEvent(NOTIFY_EXPIRED, "hexpired", keyobj, server.access_context.db->id); hashTypePropagateDeletion(server.access_context.db, key, entry); decrRefCount(keyobj); return 1; @@ -527,8 +526,11 @@ hashtableElementAccessState hashHashtableTypeAccess(hashtable *ht, void *entry) if (server.access_context.flags == OBJ_ACCESS_NONE) return ELEMENT_INVALID; + /* From this point we will be deleting the entry */ + server.access_context.expired++; robj *o = server.access_context.val; serverDb *db = server.access_context.db; + serverAssert(o && db); hashTypeUntrackEntry(o, entry); @@ -546,18 +548,28 @@ void hashTypeResetAccessContext(void) { robj *o = server.access_context.val; serverDb *db = server.access_context.db; serverAssert(!o || o->type == OBJ_HASH); + uint64_t num_expired = server.access_context.expired; resetAccessContext(); if (o) { - if (hashTypeLength(o) == 0) { + int is_empty = hashTypeLength(o) == 0; + if (is_empty || num_expired) { + /* We need to report key changes and notifications. for that we need to make sure we have the key object */ if (!keyobj) { sds key = objectGetKey(o); keyobj = createStringObject(key, sdslen(key)); } else { incrRefCount(keyobj); } - notifyKeyspaceEvent(NOTIFY_GENERIC, "del", keyobj, db->id); - dbDelete(db, keyobj); - decrRefCount(keyobj); + /* In case we have some entries which are expired we need to report it */ + if (num_expired) + notifyKeyspaceEvent(NOTIFY_EXPIRED, "hexpired", keyobj, db->id); + /* In casethe object was left empty, we need to make sure to delete it (we do not support zero size hashes) */ + if (is_empty) { + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", keyobj, db->id); + dbDelete(db, keyobj); + decrRefCount(keyobj); + } + signalModifiedKey(server.current_client, db, keyobj); } } } From e604b37882a0fe80e6fa4a6f5fe96e2757b3ec4a Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Wed, 21 May 2025 14:43:49 +0300 Subject: [PATCH 28/33] make hashtable call entry destructor on delete access Signed-off-by: Ran Shidlansik --- src/hashtable.c | 1 + src/server.h | 3 ++- src/t_hash.c | 3 +-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hashtable.c b/src/hashtable.c index cbf7239a0d3..1cb71d0419c 100644 --- a/src/hashtable.c +++ b/src/hashtable.c @@ -897,6 +897,7 @@ static inline hashtableElementAccessState accessElementIfNeeded(hashtable *ht, v fillBucketHole(ht, b, pos_in_bucket, table_index); } hashtableShrinkIfNeeded(ht); + freeEntry(ht, elem); } return element_status; } diff --git a/src/server.h b/src/server.h index ce394843c71..438a9e23619 100644 --- a/src/server.h +++ b/src/server.h @@ -3339,11 +3339,12 @@ sds hashTypeEntryGetField(const hashTypeEntry *entry); sds hashTypeEntryGetValue(const hashTypeEntry *entry); long long hashTypeEntryGetExpiry(const hashTypeEntry *entry); int hashTypeEntryHasExpire(const hashTypeEntry *entry); -void hashTypeTrackEntry(robj *o, void *entry); size_t hashTypeEntryMemUsage(hashTypeEntry *entry); hashTypeEntry *hashTypeEntryDefrag(hashTypeEntry *entry, void *(*defragfn)(void *), sds (*sdsdefragfn)(sds)); void dismissHashTypeEntry(hashTypeEntry *entry); void freeHashTypeEntry(hashTypeEntry *entry); +void hashTypeTrackEntry(robj *o, void *entry); +void hashTypeUntrackEntry(robj *o, void *entry); void hashTypeConvert(robj *o, int enc); void hashTypeTryConversion(robj *subject, robj **argv, int start, int end); diff --git a/src/t_hash.c b/src/t_hash.c index f38eae6c666..c76eeb2a7de 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -447,7 +447,7 @@ void hashTypeTrackEntry(robj *o, void *entry) { serverAssert(volatileSetAddEntry(set, entry, hashTypeEntryGetExpiry(entry))); } -static void hashTypeUntrackEntry(robj *o, void *entry) { +void hashTypeUntrackEntry(robj *o, void *entry) { if (!hashTypeEntryHasExpire(entry)) return; volatile_set *set = hashTypeGetVolatileSet(o); debugServerAssert(set); @@ -535,7 +535,6 @@ hashtableElementAccessState hashHashtableTypeAccess(hashtable *ht, void *entry) hashTypeUntrackEntry(o, entry); hashTypeExpireEntry(entry); - freeHashTypeEntry(entry); return ELEMENT_DELETE; } From d719dcd6cd0757969419aac1c387606ce10b181c Mon Sep 17 00:00:00 2001 From: xbasel <103044017+xbasel@users.noreply.github.com> Date: Wed, 21 May 2025 14:50:18 +0300 Subject: [PATCH 29/33] Hash TTL - add tests (#1) * add tests Signed-off-by: xbasel <103044017+xbasel@users.noreply.github.com> * fix a bug - return on error Signed-off-by: xbasel <103044017+xbasel@users.noreply.github.com> * disable failing tests Signed-off-by: xbasel <103044017+xbasel@users.noreply.github.com> * rmeove redundant test Signed-off-by: xbasel <103044017+xbasel@users.noreply.github.com> * Update tests/unit/hashexpire.tcl --------- Signed-off-by: xbasel <103044017+xbasel@users.noreply.github.com> Co-authored-by: Ran Shidlansik --- src/expire.c | 1 + tests/unit/expire.tcl | 35 +- tests/unit/hashexpire.tcl | 1244 +++++++++++++++++++++++++++++++++++++ 3 files changed, 1247 insertions(+), 33 deletions(-) create mode 100644 tests/unit/hashexpire.tcl diff --git a/src/expire.c b/src/expire.c index 8039998df37..1c288054778 100644 --- a/src/expire.c +++ b/src/expire.c @@ -584,6 +584,7 @@ int convertExpireArgumentToUnixTime(client *c, robj *arg, long long basetime, in if (when < 0) { addReplyErrorExpireTime(c); + return C_ERR; } if (unit == UNIT_SECONDS) { diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index 21c0b140aad..3736538105c 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -546,8 +546,9 @@ start_server {tags {"expire"}} { set primary_port [srv -1 port] # Set this inner layer server as replica set replica [srv 0 client] - $replica replicaof $primary_host $primary_port + test {First server should have role slave after REPLICAOF} { + $replica replicaof $primary_host $primary_port wait_for_condition 50 100 { [s 0 role] eq {slave} } else { @@ -614,38 +615,6 @@ start_server {tags {"expire"}} { assert_equal {} [$replica get foo] } } - - test {expired hash field is expired on the replica} { - $primary flushall - $replica config set replica-read-only yes - $primary hset myhash f1 v1 f2 v2 - wait_for_condition 50 100 { - [$primary hlen myhash] eq {2} - } else { - fail "field not added to primary" - } - - wait_for_condition 50 100 { - [$replica hlen myhash] eq {2} - } else { - fail "field not added to replica" - } - - $primary hpexpire myhash 1000 fields 1 f2 - - - wait_for_condition 50 100 { - [$primary hget myhash f2] eq {} - } else { - fail "field not removed from primary" - } - - wait_for_condition 50 100 { - [$replica hlen myhash] eq {1} - } else { - fail "field not removed from replica" - } - } } test {SET command will remove expire} { diff --git a/tests/unit/hashexpire.tcl b/tests/unit/hashexpire.tcl new file mode 100644 index 00000000000..15e5bba7ae1 --- /dev/null +++ b/tests/unit/hashexpire.tcl @@ -0,0 +1,1244 @@ + +proc info_field {info field} { +foreach line [split $info "\n"] { + if {[string match "$field:*" $line]} { + return [string trim [lindex [split $line ":"] 1]] + } +} +return "" +} + +start_server {tags {"hashexpire"}} { + + test {HSETEX EX - test if fields expire} { + r flushall + # Set TTL and use HSETEX to add field1 with expiry + set ttl 100 + r HSETEX myhash EX $ttl FIELDS 1 field1 val1 + assert_equal $ttl [r HTTL myhash FIELDS 1 field1] + + # Reset hash, set new fields without expiry (to prevent the deletion of the hash on expiry) + r DEL myhash + r HSET myhash field2 "hello" field3 "world" + + # Add field1 again with short expiry + set ttl 1 + r HSETEX myhash EX $ttl FIELDS 1 field1 val1 + + # Wait for TTL to expire + after 1100 + + # field1 should be expired, others should remain + assert_equal {0} [r HEXISTS myhash field1] + assert_equal {2} [r HLEN myhash] + } + + test {HSETEX KEEPTTL - preserves existing TTL of field} { + r FLUSHALL + + # Set a field with a known TTL + r HSETEX myhash PX 1000 FIELDS 1 field1 val1 + set original_pttl [r HPTTL myhash FIELDS 1 field1] + set original_expiretime [r HEXPIRETIME myhash FIELDS 1 field1] + + # Validate TTL is active and expiretime is in the future + assert {$original_pttl > 0} + assert {$original_expiretime > [clock seconds]} + + # Overwrite the field with KEEPTTL + r HSETEX myhash KEEPTTL FIELDS 1 field1 newval + + # Ensure TTL is preserved + set updated_pttl [r HPTTL myhash FIELDS 1 field1] + set updated_expiretime [r HEXPIRETIME myhash FIELDS 1 field1] + assert {$updated_pttl > 0} + assert {$updated_pttl <= $original_pttl} + assert_equal $original_expiretime $updated_expiretime + + # Ensure value was updated + assert_equal newval [r HGET myhash field1] + } + +# fields mismatch + # test {HSETEX EX - FIELDS 0 returns error} { + # r FLUSHALL + # catch {r HSETEX myhash EX 10 FIELDS 0} e + # set e + # } {ERR wrong number of arguments for 'hsetex' command} + + test {HSETEX EX - test negative ttl} { + set ttl -10 + catch {r HSETEX myhash EX $ttl FIELDS 1 field1 val1} e + set e + } {ERR invalid expire time in 'hsetex' command} + + # test {HSETEX EX - test non-numeric ttl} { + # set ttl abc + # catch {r HSETEX myhash EX $ttl FIELDS 1 field1 val1} e + # set e + # } {ERR Syntax error} + + test {HSETEX EX - overwrite field resets TTL} { + r FLUSHALL + r HSETEX myhash EX 100 FIELDS 1 field1 val1 + r HSETEX myhash EX 200 FIELDS 1 field1 newval + assert_equal 200 [r HTTL myhash FIELDS 1 field1] + assert_equal newval [r HGET myhash field1] + } + + test {HSETEX EX - test zero ttl expires immediately} { + r FLUSHALL + r HSETEX myhash EX 0 FIELDS 1 field1 val1 + after 10 + assert_equal 0 [r HEXISTS myhash field1] + } + + test {HSETEX EX - test mix of expiring and persistent fields} { + r FLUSHALL + r HSET myhash field2 "persistent" + r HSETEX myhash EX 1 FIELDS 1 field1 "temp" + after 1100 + assert_equal 0 [r HEXISTS myhash field1] + assert_equal 1 [r HEXISTS myhash field2] + } + + test {HSETEX EX - test missing TTL} { + catch {r HSETEX myhash EX FIELDS 1 field1 val1} e + set e + } {ERR syntax error} + +# fields != actual number of fields is accepted! + # test {HSETEX EX - mismatched field/value count} { + # catch {r HSETEX myhash EX 10 FIELDS 2 field1 val1} e + # set e + # } {ERR wrong number of arguments for 'hsetex' command} + + + +###### PX ####### + + + test {HSETEX PX - test if fields expire} { + r FLUSHALL + # Set TTL in milliseconds and use HSETEX to add field1 with expiry + set ttl 2000 + r HSETEX myhash PX $ttl FIELDS 1 field1 val1 + set reported_ttl [r HPTTL myhash FIELDS 1 field1] + assert { $reported_ttl <= $ttl && $reported_ttl > 0 } + + # Reset hash, set new fields without expiry + r DEL myhash + r HSET myhash field2 "hello" field3 "world" + + # Add field1 again with short expiry + set ttl 10 + r HSETEX myhash PX $ttl FIELDS 1 field1 val1 + + # Wait for TTL to expire + after 20 + + # field1 should be expired, others should remain + assert_equal {0} [r HEXISTS myhash field1] + assert_equal {2} [r HLEN myhash] + } + + test {HSETEX PX - test negative ttl} { + set ttl -50 + catch {r HSETEX myhash PX $ttl FIELDS 1 field1 val1} e + set e + } {ERR invalid expire time in 'hsetex' command} + + test {HSETEX PX - test non-numeric ttl} { + set ttl xyz + catch {r HSETEX myhash PX $ttl FIELDS 1 field1 val1} e + set e + } {ERR value is not an integer or out of range} + + test {HSETEX PX - overwrite field resets TTL} { + r FLUSHALL + r HSETEX myhash PX 10000 FIELDS 1 field1 val1 + r HSETEX myhash PX 20000 FIELDS 1 field1 newval + set ttl [r HPTTL myhash FIELDS 1 field1] + assert {$ttl >= 19000 && $ttl <= 20000} + assert_equal newval [r HGET myhash field1] + } + + test {HSETEX PX - test zero ttl expires immediately} { + r FLUSHALL + r HSETEX myhash PX 0 FIELDS 1 field1 val1 + after 10 + assert_equal 0 [r HEXISTS myhash field1] + } + + test {HSETEX PX - test mix of expiring and persistent fields} { + r FLUSHALL + r HSET myhash field2 "persistent" + r HSETEX myhash PX 10 FIELDS 1 field1 "temp" + after 20 + assert_equal 0 [r HEXISTS myhash field1] + assert_equal 1 [r HEXISTS myhash field2] + } + + test {HSETEX PX - test missing TTL} { + catch {r HSETEX myhash PX FIELDS 1 field1 val1} e + set e + } {ERR syntax error} + + # test {HSETEX PX - mismatched field/value count} { + # catch {r HSETEX myhash PX 100 FIELDS 2 field1 val1} e + # set e + # } {ERR wrong number of arguments for 'hsetex' command} + + + ## FNX/FXX + +# hsetex throws ERR syntax error, it shouldn't + test {HSETEX EX FNX - set only if none of the fields exist} { + r FLUSHALL + r HSET myhash field1 val1 + set res [r HSETEX myhash EX 10 FNX FIELDS 1 field1 val2] + assert_equal 0 $res + assert_equal val1 [r HGET myhash field1] + + # Now try with all-new fields + set res [r HSETEX myhash EX 10 FNX FIELDS 2 f2 v2 f3 v3] + assert_equal 1 $res + assert_equal v2 [r HGET myhash f2] + assert_equal v3 [r HGET myhash f3] + } + + test {HSETEX EX FXX - set only if all fields exist} { + r FLUSHALL + r HSET myhash field1 val1 field2 val2 + set res [r HSETEX myhash EX 10 FXX FIELDS 2 field1 new1 field2 new2] + assert_equal 1 $res + assert_equal new1 [r HGET myhash field1] + assert_equal new2 [r HGET myhash field2] + + # Now try when one field doesn't exist + set res [r HSETEX myhash EX 10 FXX FIELDS 2 field1 x fieldX y] + assert_equal 0 $res + assert_equal new1 [r HGET myhash field1] + assert_equal 0 [r HEXISTS myhash fieldX] + } + +# Syntax error: HSETEX myhash PX 100 FNX FIELDS 2 x 2 y 3 + test {HSETEX PX FNX - partial conflict returns 0} { + r FLUSHALL + r HSET myhash x 1 + set res [r HSETEX myhash PX 100 FNX FIELDS 2 x 2 y 3] + assert_equal 0 $res + assert_equal 1 [r HEXISTS myhash x] + assert_equal 0 [r HEXISTS myhash y] + } + + test {HSETEX PX FXX - one field missing returns 0} { + r FLUSHALL + r HSET myhash a 1 + set res [r HSETEX myhash PX 100 FXX FIELDS 2 a 2 b 3] + assert_equal 0 $res + assert_equal 1 [r HGET myhash a] + assert_equal 0 [r HEXISTS myhash b] + } + + test {HSETEX EX - FNX and FXX conflict error} { + catch {r HSETEX myhash EX 10 FNX FXX FIELDS 1 x y} e + set e + } {ERR syntax error} + + #################### Lazy Expiry ######################## + + proc test_lazy_expiry {mode ttl desc} { + test "HSETEX $mode - lazy expiry with $desc" { + r FLUSHALL + r debug SET-ACTIVE-EXPIRE no + + if {$mode eq "EX"} { + r HSETEX myhash EX $ttl FIELDS 1 field1 val1 + set wait [expr {$ttl * 1000 + 100}] + } elseif {$mode eq "PX"} { + r HSETEX myhash PX $ttl FIELDS 1 field1 val1 + set wait [expr {$ttl + 100}] + } elseif {$mode eq "EXAT"} { + set now [clock seconds] + r HSETEX myhash EXAT [expr {$now + $ttl}] FIELDS 1 field1 val1 + set wait [expr {$ttl * 1000 + 100}] + } elseif {$mode eq "PXAT"} { + set now [clock milliseconds] + r HSETEX myhash PXAT [expr {$now + $ttl}] FIELDS 1 field1 val1 + set wait [expr {$ttl + 100}] + } + + after $wait + + # Still present due to lazy expiry + assert_equal 1 [r HLEN myhash] + + # Trigger expiry + catch {r HGET myhash field1} + assert_equal 0 [r HLEN myhash] + + r debug SET-ACTIVE-EXPIRE yes + } + } + + test_lazy_expiry EX 1 "relative seconds" + test_lazy_expiry PX 10 "relative milliseconds" + test_lazy_expiry EXAT 1 "absolute seconds" + test_lazy_expiry PXAT 10 "absolute milliseconds" + + test {HGETALL skips expired fields without triggering lazy expiry} { + r FLUSHALL + r DEBUG SET-ACTIVE-EXPIRE no + + # Set two fields: one persistent, one with short TTL + r HSET myhash persistent "val1" + r HSETEX myhash PX 5 FIELDS 1 expiring "val2" + + # Wait for expiry to pass + after 10 + + # HGETALL should skip expired field + set result [r HGETALL myhash] + assert_equal {persistent val1} $result + + # HLEN should still count both fields (expired field not removed) + assert_equal 2 [r HLEN myhash] + + # Re-enable active expiry + r DEBUG SET-ACTIVE-EXPIRE yes + } + + test {HSCAN skips expired fields} { + r FLUSHALL + r DEBUG SET-ACTIVE-EXPIRE no + + # Set multiple fields, one with expiry + r HSET myhash persistent1 "a" persistent2 "b" + r HSETEX myhash PX 5 FIELDS 1 expiring "c" + + # Wait for expiration + after 10 + + # HSCAN must not return the expired field + set cursor 0 + set allfields {} + while {1} { + set res [r HSCAN myhash $cursor] + set cursor [lindex $res 0] + set kvs [lindex $res 1] + lappend allfields {*}$kvs + if {$cursor eq "0"} break + } + + # Extract just the field names + set fieldnames [lmap {k v} $allfields { set k }] + set fieldnames_sorted [lsort $fieldnames] + + # Should only include persistent1 and persistent2 + assert_equal {persistent1 persistent2} $fieldnames_sorted + + # Re-enable active expiry for future tests + r DEBUG SET-ACTIVE-EXPIRE yes + } + + test {MOVE preserves field TTLs} { + r FLUSHALL + r SELECT 0 + r HSETEX myhash PX 50000 FIELDS 1 field1 val1 + + # Capture original TTL + set original_ttl [r HPTTL myhash FIELDS 1 field1] + assert {$original_ttl > 0} + + # Move to DB 1 + assert_equal 1 [r MOVE myhash 1] + + # Switch to target DB + r SELECT 1 + + # Field must exist and TTL must be preserved + set moved_ttl [r HPTTL myhash FIELDS 1 field1] + assert {$moved_ttl > 0 && $moved_ttl <= $original_ttl} + } + + test {HSETEX - lazy expiry with multiple fields, one expired} { + r FLUSHALL + r debug SET-ACTIVE-EXPIRE no + + # This test verifies that lazy expiry is applied at the field level, + # not at the hash key level. Even if one field's TTL expires, + # the key itself should still be accessible, and other fields + # that haven't expired must remain unaffected until explicitly expired or accessed. + + # field1 with short TTL (10ms), field2 is persistent (no TTL) + r HSETEX myhash PX 10 FIELDS 1 field1 shortlived + r HSET myhash field2 persistent + + # Wait for field1 to expire + after 20 + + # Both fields should still be present due to lazy expiry + assert_equal 2 [r HLEN myhash] + + # Accessing field1 triggers its lazy expiry + r HGET myhash field1 + + # field1 should now be gone, but field2 remains + assert_equal 1 [r HLEN myhash] + assert_equal persistent [r HGET myhash field2] + + r debug SET-ACTIVE-EXPIRE yes + } + + +# error + # test {HEXPIRE - extend TTL of expired field before lazy deletion} { + # r FLUSHALL + # r debug SET-ACTIVE-EXPIRE no + + # # This test checks whether a lazily expired field can have its TTL refreshed + # # using HEXPIRE, without accessing or modifying the field's value. + # # If the field is still in memory and hasn't been lazily deleted yet, + # # HEXPIRE should succeed and extend its life. + # # TODO: Is this the desired behavior though? shouldn't the expired field be removed anyway and the command to fail? + + # r HSETEX myhash PX 10 FIELDS 1 field1 val1 + # after 20 + + # # Field should still be present in memory due to lazy expiry + # assert_equal 1 [r HLEN myhash] + + # # Refresh TTL before triggering lazy deletion + # r HEXPIRE myhash 100 FIELDS 1 field1 + + # # Confirm TTL is updated and field is still accessible + # set ttl [r HTTL myhash FIELDS 1 field1] + # # 10 Seconds grace period + # assert {$ttl > 90} + # assert_equal val1 [r HGET myhash field1] + + # r debug SET-ACTIVE-EXPIRE yes + # } + + test {HSET - overwrite lazily expired field without TTL clears expiration} { + r FLUSHALL + r debug SET-ACTIVE-EXPIRE no + + # This test verifies that if a field has expired (but not yet lazily deleted), + # and it is overwritten using a plain HSET (i.e., no TTL), + # Redis treats the field as still existing and updates it, + # effectively clearing the old TTL and making the field persistent. + # TODO: Is this the desired behavior though? shouldn't the expired field be removed anyway and the command to fail? + + r HSETEX myhash PX 10 FIELDS 1 field1 oldval + after 20 + + # Field should still be present in memory due to lazy expiry + assert_equal 1 [r HLEN myhash] + + # Overwrite with HSET (no TTL) before accessing + r HSET myhash field1 newval + + # TTL should now be gone; field becomes persistent + set ttl [r HPTTL myhash FIELDS 1 field1] + assert_equal -1 $ttl + assert_equal newval [r HGET myhash field1] + + r debug SET-ACTIVE-EXPIRE yes + } + + test {HSET - overwrite unexpired field removes TTL} { + r FLUSHALL + r debug SET-ACTIVE-EXPIRE no + + # This test verifies that overwriting a field with HSET, + # even while its TTL is still valid (not expired), + # clears the TTL and makes the field persistent. + # This behavior is consistent with how HSET works for normal keys. + + # Set field with long TTL + r HSETEX myhash PX 1000 FIELDS 1 field1 val1 + + # Confirm TTL is active + set before [r HPTTL myhash FIELDS 1 field1] + assert {$before > 0} + + # Overwrite with HSET before TTL expires + r HSET myhash field1 newval + + # TTL should now be gone + set after [r HPTTL myhash FIELDS 1 field1] + assert_equal -1 $after + assert_equal newval [r HGET myhash field1] + + r debug SET-ACTIVE-EXPIRE yes + } + +test {HDEL - lazily expired field can be deleted directly} { + r FLUSHALL + r debug SET-ACTIVE-EXPIRE no + + # This test ensures that if a field's TTL has expired but hasn't been cleaned up yet, + # calling HDEL removes it without first triggering expiration. This proves that deletion + # takes precedence and doesn't require accessing the value or triggering lazy expiry logic. + + r HSETEX myhash PX 10 FIELDS 1 field1 val1 + after 20 + + # Confirm field is still present in memory (lazy expired) + assert_equal 1 [r HLEN myhash] + + # Delete it directly + r HDEL myhash field1 + + # Confirm field is gone and hash is empty + assert_equal 0 [r HEXISTS myhash field1] + assert_equal 0 [r HLEN myhash] + + r debug SET-ACTIVE-EXPIRE yes +} + + + +test {HDEL - lazily expired field is removed without triggering expiry logic} { + r FLUSHALL + r debug SET-ACTIVE-EXPIRE no + + # This test proves that deleting a lazily expired field with HDEL + # does NOT trigger Redis's expiration mechanism. + # + # The key observation is that Redis tracks how many fields were + # expired via TTL using the `expired_subkeys` counter in INFO stats. + # If HDEL caused expiration to be processed internally, + # this counter would increment. We assert that it remains unchanged. + + # Capture expired_subkeys before + set before_info [r INFO stats] + set before [info_field $before_info expired_subkeys] + + # Create field with short TTL + r HSETEX myhash PX 10 FIELDS 1 field1 val1 + after 20 + + # Field is technically expired, but still in-memory due to lazy expiry + assert_equal 1 [r HLEN myhash] + + # Delete the expired field directly + r HDEL myhash field1 + + # Field should be gone + assert_equal 0 [r HEXISTS myhash field1] + + # Capture expired_subkeys again + set after_info [r INFO stats] + set after [info_field $after_info expired_subkeys] + + # Verify that no expiry occurred internally + assert_equal $before $after + + r debug SET-ACTIVE-EXPIRE yes +} + + +test {EXISTS - key exists before lazy expiry, removed after accessing all expired fields} { + r FLUSHALL + r debug SET-ACTIVE-EXPIRE no + + # This test verifies that Redis considers a key to "exist" even if + # all its fields are expired but haven't yet been lazily deleted. + # + # Redis only removes the hash when lazy expiry is triggered (e.g. via HGET). + # Until then, EXISTS and HLEN report that the key still exists. + # Once a field is accessed and expired, and if all fields are expired, + # the hash is deleted automatically. + + # Set multiple fields with short TTL + r HSETEX myhash PX 10 FIELDS 2 field1 val1 field2 val2 + after 20 + + # The key and both fields should still appear present + assert_equal 1 [r EXISTS myhash] + assert_equal 2 [r HLEN myhash] + + # Trigger lazy expiry on both fields + r HGET myhash field1 + r HGET myhash field2 + + # All fields should now be gone; hash should be deleted + assert_equal 0 [r EXISTS myhash] + assert_equal 0 [r HLEN myhash] + + r debug SET-ACTIVE-EXPIRE yes +} + + +###### Test EXPIRE ############# + + + # Basic Expiry Functionality + test {HEXPIRE - set TTL on existing field} { + r FLUSHALL + r HSET myhash field1 hello + r HEXPIRE myhash 10 FIELDS 1 field1 + set ttl [r HTTL myhash FIELDS 1 field1] + assert {$ttl > 0} + } + +# should return 2 + test {HEXPIRE - TTL 0 deletes field} { + r FLUSHALL + r HSET myhash field1 goodbye + set res [r HEXPIRE myhash 0 FIELDS 1 field1] + assert_equal {2} $res + assert_equal 0 [r HEXISTS myhash field1] + } + + test {HEXPIRE - negative TTL returns error} { + r FLUSHALL + r HSET myhash field1 val + catch {r HEXPIRE myhash -5 FIELDS 1 field1} e + set e + } {ERR invalid expire time in 'hexpire' command} + + test {HEXPIRE - wrong type key returns error} { + r FLUSHALL + r SET myhash notahash + catch {r HEXPIRE myhash 10 FIELDS 1 field1} e + set e + } {WRONGTYPE Operation against a key holding the wrong kind of value} + + # Conditionals: NX + test {HEXPIRE NX - only set when field has no TTL} { + r FLUSHALL + r HSETEX myhash PX 100 FIELDS 1 field1 val + set res [r HEXPIRE myhash 10 NX FIELDS 1 field1] + assert_equal {0} $res + + r HSET myhash field2 val2 + set res2 [r HEXPIRE myhash 10 NX FIELDS 1 field2] + assert_equal {1} $res2 + } + + # Conditionals: XX + test {HEXPIRE XX - only set when field has TTL} { + r FLUSHALL + r HSET myhash field1 val1 field2 val2 + r HEXPIRE myhash 20 FIELDS 1 field1 + set res [r HEXPIRE myhash 30 XX FIELDS 2 field1 field2] + assert_equal {1 0} $res + } + + # Conditionals: GT + test {HEXPIRE GT - only set if new TTL > existing TTL} { + r FLUSHALL + r HSETEX myhash PX 50 FIELDS 1 field1 val1 + after 10 + set res [r HEXPIRE myhash 1 GT FIELDS 1 field1] ;# 1s > ~40ms remaining + assert_equal {1} $res + + # GT should fail if field is persistent + r HSET myhash field2 val2 + set res2 [r HEXPIRE myhash 1 GT FIELDS 1 field2] + assert_equal {0} $res2 + } + + # Conditionals: LT + test {HEXPIRE LT - only set if new TTL < existing TTL} { + r FLUSHALL + r HSETEX myhash PX 10000 FIELDS 1 field1 val1 + set res [r HEXPIRE myhash 1 LT FIELDS 1 field1] + assert_equal {1} $res + + ## TODO this is an expected behavior really? what does non existintg ttl mean? + r HSET myhash field2 val2 + set res2 [r HEXPIRE myhash 1 LT FIELDS 1 field2] + assert_equal {1} $res2 + } + + # TTL Refresh + test {HEXPIRE - refresh TTL with new value} { + r FLUSHALL + r HSET myhash field1 val1 + r HEXPIRE myhash 1 FIELDS 1 field1 + after 500 + r HEXPIRE myhash 3 FIELDS 1 field1 + set ttl [r HTTL myhash FIELDS 1 field1] + assert {$ttl >= 2} + } + +# change error msg + # Error Cases + test {HEXPIRE - conflicting conditions error} { + r FLUSHALL + r HSET myhash field1 val + catch {r HEXPIRE myhash 10 NX XX FIELDS 1 field1} e + set e + } {ERR NX and XX, GT or LT options at the same time are not compatible} + + test {HEXPIRE - missing FIELDS error} { + r FLUSHALL + r HSET myhash field1 val + catch {r HEXPIRE myhash 10} e + set e + } {ERR wrong number of arguments for 'hexpire' command} + +# you allow fields 0 + # test {HEXPIRE - no fields after FIELDS keyword} { + # r FLUSHALL + # r HSET myhash field1 val + # catch {r HEXPIRE myhash 10 FIELDS 0} e + # set e + # } {ERR wrong number of arguments for 'hexpire' command} + + test {HEXPIRE - non-integer TTL error} { + r FLUSHALL + r HSET myhash field1 val + catch {r HEXPIRE myhash abc FIELDS 1 field1} e + set e + } {ERR value is not an integer or out of range} + + test {HEXPIRE - non-existing key returns -2} { + r FLUSHALL + set res [r HEXPIRE nokey 10 FIELDS 1 field1] + assert_equal {-2} $res + } + + test {HEXPIRE EX - set TTL on multiple fields} { + r FLUSHALL + r HSET myhash fieldA valA fieldB valB + set ttl 100 + r HEXPIRE myhash $ttl FIELDS 2 fieldA fieldB + + set ttlA [r HTTL myhash FIELDS 1 fieldA] + set ttlB [r HTTL myhash FIELDS 1 fieldB] + + assert { $ttlA > 0 && $ttlA <= $ttl } + assert { $ttlB > 0 && $ttlB <= $ttl } + } {} + + test {HEXPIRE returns -2 on non-existing key} { + r FLUSHALL + assert_equal {-2 -2} [r HEXPIRE nokey 10 FIELDS 2 field1 field2] + } {} + + + ##### HTTL ##### + test {HTTL - persistent field returns -1} { + r FLUSHALL + r HSET myhash field1 val1 + assert_equal -1 [r HTTL myhash FIELDS 1 field1] + } {} + +# crash: r HTTL myhash FIELDS 1 nofield + # test {HTTL - non-existent field returns -2} { + # r FLUSHALL + # r HSET myhash field1 val1 + # assert_equal -2 [r HTTL myhash FIELDS 1 nofield] + # } {} + + test {HTTL - non-existent key returns -2} { + r FLUSHALL + assert_equal -2 [r HTTL nokey FIELDS 1 field1] + } {} + + ##### EXPIRETIME ###### + + # Basic Expiry Functionality + test {HEXPIREAT - set absolute expiry on field} { + r FLUSHALL + r HSET myhash field1 hello + set now [clock seconds] + set exp [expr {$now + 30}] + r HEXPIREAT myhash $exp FIELDS 1 field1 + set etime [r HEXPIRETIME myhash FIELDS 1 field1] + assert_equal $exp $etime + } + + test {HEXPIREAT - timestamp in past deletes field immediately} { + r FLUSHALL + r HSET myhash field1 gone + set past [expr {[clock seconds] - 1000}] + set res [r HEXPIREAT myhash $past FIELDS 1 field1] + assert_equal {2} $res + assert_equal 0 [r HEXISTS myhash field1] + } + + + test {HEXPIREAT - set TTL on multiple fields (existing + non-existing)} { + r FLUSHALL + r HSET myhash field1 hello field2 world + set exp [expr {[clock seconds] + 10}] + set res [r HEXPIREAT myhash $exp FIELDS 3 field1 field2 fieldX] + assert_equal {1 1 -2} $res + } + + + # Conditionals: NX + test {HEXPIREAT NX - only set when field has no TTL} { + r FLUSHALL + r HSETEX myhash EX 100 FIELDS 1 field1 val + set exp [expr {[clock seconds] + 100}] + set res [r HEXPIREAT myhash $exp NX FIELDS 1 field1] + assert_equal {0} $res + + r HSET myhash field2 val2 + set res2 [r HEXPIREAT myhash $exp NX FIELDS 1 field2] + assert_equal {1} $res2 + } + + # Conditionals: XX + test {HEXPIREAT XX - only set when field has TTL} { + r FLUSHALL + r HSET myhash field1 val1 field2 val2 + set exp1 [expr {[clock seconds] + 20}] + r HEXPIREAT myhash $exp1 FIELDS 1 field1 + set exp2 [expr {[clock seconds] + 30}] + set res [r HEXPIREAT myhash $exp2 XX FIELDS 2 field1 field2] + assert_equal {1 0} $res + } + + # Conditionals: GT + test {HEXPIREAT GT - only set if new expiry > existing} { + r FLUSHALL + r HSETEX myhash PX 5000 FIELDS 1 field1 val1 + after 10 + set now [clock seconds] + set future [expr {$now + 10}] + set res [r HEXPIREAT myhash $future GT FIELDS 1 field1] + assert_equal {1} $res + + r HSET myhash field2 val2 + set res2 [r HEXPIREAT myhash $future GT FIELDS 1 field2] + assert_equal {0} $res2 + } + + + # Conditionals: LT + test {HEXPIREAT LT - only set if new expiry < existing} { + r FLUSHALL + r HSETEX myhash PX 10000 FIELDS 1 field1 val1 + set now [clock seconds] + set earlier [expr {$now + 1}] + set res [r HEXPIREAT myhash $earlier LT FIELDS 1 field1] + assert_equal {1} $res + + r HSET myhash field2 val2 + set res2 [r HEXPIREAT myhash $earlier LT FIELDS 1 field2] + assert_equal {1} $res2 + # TODO is this the expected behavior? if no TTL exist, it should be treated as minimum ttl possible? + } + + test {HEXPIREAT - refresh TTL with new future timestamp} { + r FLUSHALL + r HSET myhash field1 val1 + + # Set initial expiry to very near future + set ts1 [expr {[clock seconds] + 10}] + r HEXPIREAT myhash $ts1 FIELDS 1 field1 + + # Immediately refresh to a further expiry (no sleep needed) + set ts2 [expr {$ts1 + 5}] + r HEXPIREAT myhash $ts2 FIELDS 1 field1 + + # Confirm that expiry was updated + set actual [r HEXPIRETIME myhash FIELDS 1 field1] + assert_equal $ts2 $actual + } + + + # TTL Validations + test {HEXPIREAT - TTL is accurate via HEXPIRETIME} { + r FLUSHALL + r HSET myhash field1 val1 + set ts [expr {[clock seconds] + 50}] + r HEXPIREAT myhash $ts FIELDS 1 field1 + set returned [r HEXPIRETIME myhash FIELDS 1 field1] + assert_equal $ts $returned + } + + # Error Cases + test {HEXPIREAT - conflicting options error} { + r FLUSHALL + r HSET myhash field1 val + set ts [expr {[clock seconds] + 5}] + catch {r HEXPIREAT myhash $ts NX XX FIELDS 1 field1} e + set e + } {ERR NX and XX, GT or LT options at the same time are not compatible} + + + + test {HEXPIREAT - missing FIELDS keyword} { + r FLUSHALL + r HSET myhash field1 val + set ts [expr {[clock seconds] + 5}] + catch {r HEXPIREAT myhash $ts} e + set e + } {ERR wrong number of arguments for 'hexpireat' command} + +# 0 fields + # test {HEXPIREAT - no fields after FIELDS} { + # r FLUSHALL + # r HSET myhash field1 val + # set ts [expr {[clock seconds] + 5}] + # catch {r HEXPIREAT myhash $ts FIELDS 0} e + # set e + # } {ERR wrong number of arguments for 'hexpireat' command} + + test {HEXPIREAT - non-integer timestamp} { + r FLUSHALL + r HSET myhash field1 val + catch {r HEXPIREAT myhash tomorrow FIELDS 1 field1} e + set e + } {ERR value is not an integer or out of range} + + + + test {HEXPIREAT - non-existing key returns -2} { + r FLUSHALL + set ts [expr {[clock seconds] + 5}] + set res [r HEXPIREAT nokey $ts FIELDS 1 field1] + assert_equal {-2} $res + } + + #################### HEXPIRETIME ################## + + # Basic TTL retrieval + test {HEXPIRETIME - returns expiry timestamp for single field with TTL} { + r FLUSHALL + r HSET myhash field1 val + set ts [expr {[clock seconds] + 3}] + r HEXPIREAT myhash $ts FIELDS 1 field1 + set out [r HEXPIRETIME myhash FIELDS 1 field1] + assert_equal $ts $out + } + + + # No expiration set + test {HEXPIRETIME - field has no TTL returns -1} { + r FLUSHALL + r HSET myhash field1 val + set out [r HEXPIRETIME myhash FIELDS 1 field1] + assert_equal -1 $out + } + + # Non-existent field + test {HEXPIRETIME - field does not exist returns -2} { + r FLUSHALL + r HSET myhash field1 val + set out [r HEXPIRETIME myhash FIELDS 1 fieldX] + assert_equal -2 $out + } + + # Non-existent key + test {HEXPIRETIME - key does not exist returns -2} { + r FLUSHALL + set out [r HEXPIRETIME missingkey FIELDS 1 field1] + assert_equal -2 $out + } + + # Multiple fields: mix of TTL, no TTL, and missing + test {HEXPIRETIME - multiple fields mixed cases} { + r FLUSHALL + r HSET myhash f1 a f2 b + set now [clock seconds] + r HEXPIREAT myhash [expr {$now + 100}] FIELDS 1 f1 + set out [r HEXPIRETIME myhash FIELDS 3 f1 f2 f3] + # Should return: expiry for f1, -1 for f2 (no TTL), -2 for f3 (not found) + assert_equal [list [expr {$now + 100}] -1 -2] $out + } + + # Invalid usages + test {HEXPIRETIME - no FIELDS keyword} { + r FLUSHALL + r HSET myhash f1 a + catch {r HEXPIRETIME myhash} e + set e + } {ERR wrong number of arguments for 'hexpiretime' command} + + # why fields 0 is allowed? + # test {HEXPIRETIME - FIELDS 0} { + # r FLUSHALL + # r HSET myhash f1 a + # catch {r HEXPIRETIME myhash FIELDS 0} e + # set e + # } {ERR wrong number of arguments for 'hexpiretime' command} + +# why fields 0 is allowed? + # test {HEXPIRETIME - wrong FIELDS count} { + # r FLUSHALL + # r HSET myhash f1 a + # catch {r HEXPIRETIME myhash FIELDS 1} e + # set e + # } {ERR wrong number of arguments for 'hexpiretime' command} + + test {HEXPIRETIME - wrong type key} { + r FLUSHALL + r SET myhash "not a hash" + catch {r HEXPIRETIME myhash FIELDS 1 f1} e + set e + } {WRONGTYPE Operation against a key holding the wrong kind of value} + + + # Basic expiration in milliseconds + test {HPEXPIREAT - set absolute expiry with ms precision} { + r FLUSHALL + r HSET myhash field1 val + set now [clock milliseconds] + set future [expr {$now + 123456789}] + r HPEXPIREAT myhash $future FIELDS 1 field1 + set t [r HPEXPIRETIME myhash FIELDS 1 field1] + assert_equal $future $t + } + + test {HPEXPIREAT - past timestamp deletes field immediately} { + r FLUSHALL + r HSET myhash field1 val + set past [expr {[clock milliseconds] - 10000}] + set res [r HPEXPIREAT myhash $past FIELDS 1 field1] + assert_equal {2} $res + assert_equal 0 [r HEXISTS myhash field1] + } + + test {HPEXPIREAT - non-existent key returns -2} { + r FLUSHALL + set ts [expr {[clock milliseconds] + 1000}] + set res [r HPEXPIREAT nokey $ts FIELDS 1 field1] + assert_equal {-2} $res + } + + test {HPEXPIREAT - mixed fields} { + r FLUSHALL + r HSET myhash f1 a f2 b + set ts [expr {[clock milliseconds] + 200000}] + set res [r HPEXPIREAT myhash $ts FIELDS 3 f1 f2 fX] + assert_equal {1 1 -2} $res + } + + test {HPEXPIREAT - GT and LT options with success and failure cases} { + r FLUSHALL + r HSET myhash f1 a + + # Setup: assign a baseline expiry time + set now [clock milliseconds] + set ts1 [expr {$now + 10000}] + set ts2 [expr {$now + 20000}] + r HPEXPIREAT myhash $ts1 FIELDS 1 f1 + + # --- GT Case --- + # ts2 > ts1 → should succeed + set res_gt_pass [r HPEXPIREAT myhash $ts2 GT FIELDS 1 f1] + assert_equal {1} $res_gt_pass + + # ts1 < ts2 → now try GT with ts1 again (should fail because ts2 is already set) + set res_gt_fail [r HPEXPIREAT myhash $ts1 GT FIELDS 1 f1] + assert_equal {0} $res_gt_fail + + # --- LT Case --- + # ts1 < ts2 → LT should fail + set res_lt_fail [r HPEXPIREAT myhash $ts2 LT FIELDS 1 f1] + assert_equal {0} $res_lt_fail + + # ts1 < ts2 → try LT with earlier timestamp, should succeed + set ts0 [expr {$now + 5000}] + set res_lt_pass [r HPEXPIREAT myhash $ts0 LT FIELDS 1 f1] + assert_equal {1} $res_lt_pass + } + +# + test {HPEXPIREAT - invalid inputs} { + r FLUSHALL + r HSET myhash f1 a + catch {r HPEXPIREAT myhash abc FIELDS 1 f1} e + assert_match {*not an integer*} $e + + catch {r HPEXPIREAT myhash 12345 NX XX FIELDS 1 f1} e2 + assert_match {ERR NX and XX, GT or LT options at the same time are not compatible} $e2 + } + + + test {HPEXPIRETIME - check with multiple fields} { + r FLUSHALL + + # Setup: one expiring field, one persistent, one missing + r HSET myhash f1 v1 f2 v2 + set ts [expr {[clock milliseconds] + 1000}] + r HPEXPIREAT myhash $ts FIELDS 1 f1 + + # Query all 3 fields + set result [r HPEXPIRETIME myhash FIELDS 3 f1 f2 f3] + + # Expect: [timestamp] for f1, -1 for f2, -2 for f3 + assert {[llength $result] == 3} + # f1: has TTL → returns exact timestamp + assert_equal $ts [lindex $result 0] + + # f2: exists, no TTL → returns -1 + assert_equal -1 [lindex $result 1] + + # f3: doesn't exist → returns -2 + assert_equal -2 [lindex $result 2] + + } +} + +####### Test info +# HGETEX doesn't work +start_server {tags {"hash-ttl-info"}} { + test {Hash ttl - check command stats} { + r FLUSHALL + + # Run all relevant hash TTL commands + r HSET myhash f1 v1 f2 v2 + r HEXPIRE myhash 10 FIELDS 1 f1 + r HEXPIREAT myhash [expr {[clock seconds] + 10}] FIELDS 1 f2 + r HEXPIRETIME myhash FIELDS 2 f1 f2 + r HPEXPIRE myhash 1000 FIELDS 1 f1 + r HPEXPIREAT myhash [expr {[clock milliseconds] + 2000}] FIELDS 1 f2 + r HPEXPIRETIME myhash FIELDS 2 f1 f2 + r HGETEX myhash EX 120 FIELDS 1 f1 + r HTTL myhash FIELDS 1 f2 + r HPTTL myhash FIELDS 1 f1 + + # Fetch commandstats + set info [r INFO commandstats] + + # Extract call counts + proc get_calls {info cmd} { + foreach line [split $info "\n"] { + if {[string match "cmdstat_$cmd:*" $line]} { + regexp {calls=(\d+)} $line -> count + return $count + } + } + return -1 + } + + # Assert each command appears with correct call count (1 call each) + assert_equal 1 [get_calls $info hexpire] + assert_equal 1 [get_calls $info hexpireat] + assert_equal 1 [get_calls $info hexpiretime] + assert_equal 1 [get_calls $info hpexpire] + assert_equal 1 [get_calls $info hpexpireat] + assert_equal 1 [get_calls $info hpexpiretime] + assert_equal 1 [get_calls $info hgetex] + assert_equal 1 [get_calls $info httl] + assert_equal 1 [get_calls $info hpttl] + } +} + + + +#### Replication + +start_server {tags {"hashexpire"}} { + # Start another server to test replication of TTLs + start_server {tags {needs:repl external:skip}} { + # Set the outer layer server as primary + set primary [srv -1 client] + set primary_host [srv -1 host] + set primary_port [srv -1 port] + # Set this inner layer server as replica + set replica [srv 0 client] + + test {Setup replica and check field expiry after full sync} { + $primary flushall + + # Set up some TTLs on primary BEFORE replica connects + set now [clock milliseconds] + set f1_exp [expr {$now + 50000}] + set f2_exp [expr {$now + 70000}] + + $primary HSET myhash f1 v1 f2 v2 + $primary HPEXPIREAT myhash $f1_exp FIELDS 1 f1 + $primary HPEXPIREAT myhash $f2_exp FIELDS 1 f2 + + # Now connect replica + $replica replicaof $primary_host $primary_port + + wait_for_condition 100 100 { + [info_field [$replica info replication] master_link_status] eq "up" + } else { + fail "Master <-> Replica didn't finish sync" + } + + + # Wait for full sync + wait_for_ofs_sync $primary $replica + + + # Validate TTLs replicated correctly + set r1 [$replica HPEXPIRETIME myhash FIELDS 1 f1] + set r2 [$replica HPEXPIRETIME myhash FIELDS 1 f2] + + assert_equal $f1_exp $r1 + assert_equal $f2_exp $r2 + } + + + + test {HASH TTL - replicated TTL is absolute and consistent on replica} { + $primary flushall + + set now [clock milliseconds] + set future [expr {$now + 5000}] + set future_sec [expr {$future / 1000}] + + # HPEXPIREAT + $primary HSET myhash f1 v1 + $primary HPEXPIREAT myhash $future FIELDS 1 f1 + + # HSETEX EX + $primary HSETEX myhash EX 5 FIELDS 1 f2 v2 + + # HEXPIRE + $primary HSET myhash f3 v3 + $primary HEXPIRE myhash 5 FIELDS 1 f3 + + wait_for_ofs_sync $primary $replica + + set t1 [$primary HPEXPIRETIME myhash FIELDS 1 f1] + set t1r [$replica HPEXPIRETIME myhash FIELDS 1 f1] + assert_equal $t1 $t1r + + set t2 [$primary HEXPIRETIME myhash FIELDS 1 f2] + set t2r [$replica HEXPIRETIME myhash FIELDS 1 f2] + assert_equal $t2 $t2r + + set t3 [$primary HEXPIRETIME myhash FIELDS 1 f3] + set t3r [$replica HEXPIRETIME myhash FIELDS 1 f3] + assert_equal $t3 $t3r + } + + test {HASH TTL - field expired on master gets deleted on replica} { + $primary flushall + + $primary HSETEX myhash PX 10 FIELDS 1 f1 val1 + after 20 + wait_for_ofs_sync $primary $replica + + + # Trigger lazy expiry + catch {$primary HGET myhash f1} + wait_for_ofs_sync $primary $replica + + + assert_equal 0 [$replica HEXISTS myhash f1] + } + + + test {HASH TTL - replica retains TTL and field before expiration} { + $primary flushall + + $primary HSETEX myhash PX 1000 FIELDS 1 f1 val1 + wait_for_ofs_sync $primary $replica + + set master_ttl [$primary HPTTL myhash FIELDS 1 f1] + set replica_ttl [$replica HPTTL myhash FIELDS 1 f1] + assert {$replica_ttl > 0} + assert {$replica_ttl <= $master_ttl} + + } + + } +} \ No newline at end of file From 6d9551ee4a6365b1abf3eb6f3298e6329ce032d7 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Wed, 21 May 2025 15:16:53 +0300 Subject: [PATCH 30/33] fix bad memory access issue on entry tracking update Signed-off-by: Ran Shidlansik --- src/t_hash.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index c76eeb2a7de..de4cf3b68d8 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -460,17 +460,29 @@ void hashTypeUntrackEntry(robj *o, void *entry) { } static void hashTypeTrackUpdateEntry(robj *o, void *old_entry, void *new_entry, long long old_expiry, long long new_expiry) { - if (old_expiry == EXPIRY_NONE && new_expiry == EXPIRY_NONE) + int old_tracked = old_entry && old_expiry != EXPIRY_NONE; + int new_tracked = new_entry && new_expiry != EXPIRY_NONE; + /* If entry was not tracked before and not going to be tracked now, we can simply return */ + if (!old_tracked && !new_tracked) return; - else if (!new_entry || new_expiry == EXPIRY_NONE) - hashTypeUntrackEntry(o, old_entry); - else if (!old_entry || old_expiry == EXPIRY_NONE) - hashTypeTrackEntry(o, new_entry); + + volatile_set *set = hashTypeGetOrcreateVolatileSet(o); + debugServerAssert(set); + + if (old_tracked && !new_tracked) + serverAssert(volatileSetRemoveEntry(set, old_entry, old_expiry)); + else if (new_tracked && !old_tracked) + serverAssert(volatileSetAddEntry(set, new_entry, new_expiry)); else { volatile_set *set = hashTypeGetVolatileSet(o); debugServerAssert(set); serverAssert(volatileSetUpdateEntry(set, old_entry, new_entry, old_expiry, new_expiry) == 1); } + if (volatileSetNumEntries(set) == 0) { + freeVolatileSet(set); + volatile_set **volatile_set_ref = hashtableMetadata(o->ptr); + *volatile_set_ref = NULL; + } } void hashTypePropagateDeletion(serverDb *db, sds key, void *entry) { From eab6fc4555634d3b287e6438d1ba272756135118 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Wed, 21 May 2025 16:01:56 +0300 Subject: [PATCH 31/33] fix trackUpdate condition Signed-off-by: Ran Shidlansik --- src/t_hash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index de4cf3b68d8..f155883937b 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -460,8 +460,8 @@ void hashTypeUntrackEntry(robj *o, void *entry) { } static void hashTypeTrackUpdateEntry(robj *o, void *old_entry, void *new_entry, long long old_expiry, long long new_expiry) { - int old_tracked = old_entry && old_expiry != EXPIRY_NONE; - int new_tracked = new_entry && new_expiry != EXPIRY_NONE; + int old_tracked = (old_entry && old_expiry != EXPIRY_NONE); + int new_tracked = (new_entry && new_expiry != EXPIRY_NONE); /* If entry was not tracked before and not going to be tracked now, we can simply return */ if (!old_tracked && !new_tracked) return; From 8654080b86fef1c6335169c288726d0f9f171657 Mon Sep 17 00:00:00 2001 From: Ran Shidlansik Date: Wed, 21 May 2025 19:41:59 +0300 Subject: [PATCH 32/33] make sure to remove the volatile set on hash object detructor Signed-off-by: Ran Shidlansik --- src/object.c | 5 ++++- src/server.h | 1 + src/t_hash.c | 8 +++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/object.c b/src/object.c index 3b867554a2e..746435f7e73 100644 --- a/src/object.c +++ b/src/object.c @@ -525,7 +525,10 @@ void freeZsetObject(robj *o) { void freeHashObject(robj *o) { switch (o->encoding) { - case OBJ_ENCODING_HASHTABLE: hashtableRelease((hashtable *)o->ptr); break; + case OBJ_ENCODING_HASHTABLE: + hashTypeFreeVolatileSet(o); + hashtableRelease((hashtable *)o->ptr); + break; case OBJ_ENCODING_LISTPACK: lpFree(o->ptr); break; default: serverPanic("Unknown hash encoding type"); break; } diff --git a/src/server.h b/src/server.h index 438a9e23619..38597400d5a 100644 --- a/src/server.h +++ b/src/server.h @@ -3343,6 +3343,7 @@ size_t hashTypeEntryMemUsage(hashTypeEntry *entry); hashTypeEntry *hashTypeEntryDefrag(hashTypeEntry *entry, void *(*defragfn)(void *), sds (*sdsdefragfn)(sds)); void dismissHashTypeEntry(hashTypeEntry *entry); void freeHashTypeEntry(hashTypeEntry *entry); +void hashTypeFreeVolatileSet(robj *o); void hashTypeTrackEntry(robj *o, void *entry); void hashTypeUntrackEntry(robj *o, void *entry); diff --git a/src/t_hash.c b/src/t_hash.c index f155883937b..542bd0ffcaa 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -422,6 +422,12 @@ static volatile_set *hashTypeGetVolatileSet(robj *o) { return *(volatile_set **)hashtableMetadata(o->ptr); } +void hashTypeFreeVolatileSet(robj *o) { + volatile_set *set = hashTypeGetVolatileSet(o); + if (set) + freeVolatileSet(set); +} + int hashTypeHasVolatileElements(robj *o) { return o->encoding == OBJ_ENCODING_HASHTABLE && hashTypeGetVolatileSet(o); } @@ -578,9 +584,9 @@ void hashTypeResetAccessContext(void) { if (is_empty) { notifyKeyspaceEvent(NOTIFY_GENERIC, "del", keyobj, db->id); dbDelete(db, keyobj); - decrRefCount(keyobj); } signalModifiedKey(server.current_client, db, keyobj); + decrRefCount(keyobj); } } } From cb45000d237bb9756e4d7e5291dce262ba7fdcc6 Mon Sep 17 00:00:00 2001 From: xbasel <103044017+xbasel@users.noreply.github.com> Date: Sun, 25 May 2025 10:25:11 +0300 Subject: [PATCH 33/33] trigger build --- x | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 x diff --git a/x b/x new file mode 100644 index 00000000000..e69de29bb2d