diff --git a/src/bitops.c b/src/bitops.c index 2222c05ea..44312c6af 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -489,22 +489,27 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) { * bits to a string object. The command creates or pad with zeroes the string * so that the 'maxbit' bit can be addressed. The object is finally * returned. Otherwise if the key holds a wrong type NULL is returned and - * an error is sent to the client. */ -robj *lookupStringForBitCommand(client *c, uint64_t maxbit, int *dirty) { + * an error is sent to the client. + * + * (Must provide all the arguments to the function) + */ +static robj *lookupStringForBitCommand(client *c, uint64_t maxbit, + size_t *strOldSize, size_t *strGrowSize) +{ size_t byte = maxbit >> 3; robj *o = lookupKeyWrite(c->db,c->argv[1]); if (checkType(c,o,OBJ_STRING)) return NULL; - if (dirty) *dirty = 0; if (o == NULL) { o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1)); dbAdd(c->db,c->argv[1],o); - if (dirty) *dirty = 1; + *strGrowSize = byte + 1; + *strOldSize = 0; } else { o = dbUnshareStringValue(c->db,c->argv[1],o); - size_t oldlen = sdslen(o->ptr); + *strOldSize = sdslen(o->ptr); o->ptr = sdsgrowzero(o->ptr,byte+1); - if (dirty && oldlen != sdslen(o->ptr)) *dirty = 1; + *strGrowSize = sdslen(o->ptr) - *strOldSize; } return o; } @@ -561,8 +566,9 @@ void setbitCommand(client *c) { return; } - int dirty; - if ((o = lookupStringForBitCommand(c,bitoffset,&dirty)) == NULL) return; + size_t strOldSize, strGrowSize; + if ((o = lookupStringForBitCommand(c,bitoffset,&strOldSize,&strGrowSize)) == NULL) + return; /* Get current values */ byte = bitoffset >> 3; @@ -573,7 +579,7 @@ void setbitCommand(client *c) { /* Either it is newly created, changed length, or the bit changes before and after. * Note that the bitval here is actually a decimal number. * So we need to use `!!` to convert it to 0 or 1 for comparison. */ - if (dirty || (!!bitval != on)) { + if (strGrowSize || (!!bitval != on)) { /* Update byte with new bit value. */ byteval &= ~(1 << bit); byteval |= ((on & 0x1) << bit); @@ -581,6 +587,13 @@ void setbitCommand(client *c) { signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty++; + + /* If this is not a new key (old size not 0) and size changed, then + * update the keysizes histogram. Otherwise, the histogram already + * updated in lookupStringForBitCommand() by calling dbAdd(). */ + if ((strOldSize > 0) && (strGrowSize != 0)) + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING, + strOldSize, strOldSize + strGrowSize); } /* Return original value. */ @@ -1065,7 +1078,8 @@ struct bitfieldOp { void bitfieldGeneric(client *c, int flags) { robj *o; uint64_t bitoffset; - int j, numops = 0, changes = 0, dirty = 0; + int j, numops = 0, changes = 0; + size_t strOldSize, strGrowSize = 0; struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */ int owtype = BFOVERFLOW_WRAP; /* Overflow type. */ int readonly = 1; @@ -1159,7 +1173,7 @@ void bitfieldGeneric(client *c, int flags) { /* Lookup by making room up to the farthest bit reached by * this operation. */ if ((o = lookupStringForBitCommand(c, - highest_write_offset,&dirty)) == NULL) { + highest_write_offset,&strOldSize,&strGrowSize)) == NULL) { zfree(ops); return; } @@ -1209,7 +1223,7 @@ void bitfieldGeneric(client *c, int flags) { setSignedBitfield(o->ptr,thisop->offset, thisop->bits,newval); - if (dirty || (oldval != newval)) + if (strGrowSize || (oldval != newval)) changes++; } else { addReplyNull(c); @@ -1243,7 +1257,7 @@ void bitfieldGeneric(client *c, int flags) { setUnsignedBitfield(o->ptr,thisop->offset, thisop->bits,newval); - if (dirty || (oldval != newval)) + if (strGrowSize || (oldval != newval)) changes++; } else { addReplyNull(c); @@ -1286,6 +1300,14 @@ void bitfieldGeneric(client *c, int flags) { } if (changes) { + + /* If this is not a new key (old size not 0) and size changed, then + * update the keysizes histogram. Otherwise, the histogram already + * updated in lookupStringForBitCommand() by calling dbAdd(). */ + if ((strOldSize > 0) && (strGrowSize != 0)) + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING, + strOldSize, strOldSize + strGrowSize); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty += changes; diff --git a/src/db.c b/src/db.c index 49c374fb1..42248a20f 100644 --- a/src/db.c +++ b/src/db.c @@ -21,6 +21,8 @@ * C-level DB API *----------------------------------------------------------------------------*/ +static_assert(MAX_KEYSIZES_TYPES == OBJ_TYPE_BASIC_MAX, "Must be equal"); + /* Flags for expireIfNeeded */ #define EXPIRE_FORCE_DELETE_EXPIRED 1 #define EXPIRE_AVOID_DELETE_EXPIRED 2 @@ -46,6 +48,48 @@ void updateLFU(robj *val) { val->lru = (LFUGetTimeInMinutes()<<8) | counter; } +/* + * Update histogram of keys-sizes + * + * It is used to track the distribution of key sizes in the dataset. It is updated + * every time key's length is modified. Available to user via INFO command. + * + * The histogram is a base-2 logarithmic histogram, with 64 bins. The i'th bin + * represents the number of keys with a size in the range 2^i and 2^(i+1) + * exclusive. oldLen/newLen must be smaller than 2^48, and if their value + * equals 0, it means that the key is being created/deleted, respectively. Each + * data type has its own histogram and it is per database (In addition, there is + * histogram per slot for future cluster use). + * + * Examples to LEN values and corresponding bins in histogram: + * [1,2)->0 [2,4)->1 [4,8)->2 [8,16)->3 + */ +void updateKeysizesHist(redisDb *db, int didx, uint32_t type, uint64_t oldLen, uint64_t newLen) { + if(unlikely(type >= OBJ_TYPE_BASIC_MAX)) + return; + + kvstoreDictMetadata *dictMeta = kvstoreGetDictMetadata(db->keys, didx); + kvstoreMetadata *kvstoreMeta = kvstoreGetMetadata(db->keys); + + if (oldLen != 0) { + int old_bin = log2ceil(oldLen); + debugServerAssertWithInfo(server.current_client, NULL, old_bin < MAX_KEYSIZES_BINS); + /* If following a key deletion it is last one in slot's dict, then + * slot's dict might get released as well. Verify if metadata is not NULL. */ + if(dictMeta) dictMeta->keysizes_hist[type][old_bin]--; + kvstoreMeta->keysizes_hist[type][old_bin]--; + } + + if (newLen != 0) { + int new_bin = log2ceil(newLen); + debugServerAssertWithInfo(server.current_client, NULL, new_bin < MAX_KEYSIZES_BINS); + /* If following a key deletion it is last one in slot's dict, then + * slot's dict might get released as well. Verify if metadata is not NULL. */ + if(dictMeta) dictMeta->keysizes_hist[type][new_bin]++; + kvstoreMeta->keysizes_hist[type][new_bin]++; + } +} + /* Lookup a key for read or write operations, or return NULL if the key is not * found in the specified DB. This function implements the functionality of * lookupKeyRead(), lookupKeyWrite() and their ...WithFlags() variants. @@ -205,6 +249,7 @@ static dictEntry *dbAddInternal(redisDb *db, robj *key, robj *val, int update_if kvstoreDictSetVal(db->keys, slot, de, val); signalKeyAsReady(db, key, val->type); notifyKeyspaceEvent(NOTIFY_NEW,"new",key,db->id); + updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); /* add hist */ return de; } @@ -250,6 +295,7 @@ int dbAddRDBLoad(redisDb *db, sds key, robj *val) { int slot = getKeySlot(key); dictEntry *de = kvstoreDictAddRaw(db->keys, slot, key, NULL); if (de == NULL) return 0; + updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); /* add hist */ initObjectLRUOrLFU(val); kvstoreDictSetVal(db->keys, slot, de, val); return 1; @@ -273,6 +319,9 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt serverAssertWithInfo(NULL,key,de != NULL); robj *old = dictGetVal(de); + /* Remove old key from keysizes histogram */ + updateKeysizesHist(db, slot, old->type, getObjectLength(old), 0); /* remove hist */ + val->lru = old->lru; if (overwrite) { @@ -291,6 +340,9 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt } kvstoreDictSetVal(db->keys, slot, de, val); + /* Add new key to keysizes histogram */ + updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); + /* if hash with HFEs, take care to remove from global HFE DS */ if (old->type == OBJ_HASH) hashTypeRemoveFromExpires(&db->hexpires, old); @@ -404,6 +456,9 @@ int dbGenericDelete(redisDb *db, robj *key, int async, int flags) { if (de) { robj *val = dictGetVal(de); + /* remove key from histogram */ + updateKeysizesHist(db, slot, val->type, getObjectLength(val), 0); + /* If hash object with expiry on fields, remove it from HFE DS of DB */ if (val->type == OBJ_HASH) hashTypeRemoveFromExpires(&db->hexpires, val); @@ -599,7 +654,8 @@ redisDb *initTempDb(void) { redisDb *tempDb = zcalloc(sizeof(redisDb)*server.dbnum); for (int i=0; itype->userdata; - kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d); + kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d); listAddNodeTail(kvs->rehashing, d); metadata->rehashing_node = listLast(kvs->rehashing); @@ -201,7 +209,7 @@ static void kvstoreDictRehashingStarted(dict *d) { * the old ht size of the dictionary from the total sum of buckets for a DB. */ static void kvstoreDictRehashingCompleted(dict *d) { kvstore *kvs = d->type->userdata; - kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d); + kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d); if (metadata->rehashing_node) { listDelNode(kvs->rehashing, metadata->rehashing_node); metadata->rehashing_node = NULL; @@ -214,10 +222,15 @@ static void kvstoreDictRehashingCompleted(dict *d) { kvs->overhead_hashtable_rehashing -= from; } -/* Returns the size of the DB dict metadata in bytes. */ -static size_t kvstoreDictMetadataSize(dict *d) { +/* Returns the size of the DB dict base metadata in bytes. */ +static size_t kvstoreDictMetaBaseSize(dict *d) { UNUSED(d); - return sizeof(kvstoreDictMetadata); + return sizeof(kvstoreDictMetaBase); +} +/* Returns the size of the DB dict extended metadata in bytes. */ +static size_t kvstoreDictMetadataExtendSize(dict *d) { + UNUSED(d); + return sizeof(kvstoreDictMetaEx); } /**********************************/ @@ -232,7 +245,13 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) { * for the dict cursor, see kvstoreScan */ assert(num_dicts_bits <= 16); - kvstore *kvs = zcalloc(sizeof(*kvs)); + /* Calc kvstore size */ + size_t kvsize = sizeof(kvstore); + /* Conditionally calc also histogram size */ + if (flags & KVSTORE_ALLOC_META_KEYS_HIST) + kvsize += sizeof(kvstoreMetadata); + + kvstore *kvs = zcalloc(kvsize); memcpy(&kvs->dtype, type, sizeof(kvs->dtype)); kvs->flags = flags; @@ -243,7 +262,10 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) { assert(!type->rehashingStarted); assert(!type->rehashingCompleted); kvs->dtype.userdata = kvs; - kvs->dtype.dictMetadataBytes = kvstoreDictMetadataSize; + if (flags & KVSTORE_ALLOC_META_KEYS_HIST) + kvs->dtype.dictMetadataBytes = kvstoreDictMetadataExtendSize; + else + kvs->dtype.dictMetadataBytes = kvstoreDictMetaBaseSize; kvs->dtype.rehashingStarted = kvstoreDictRehashingStarted; kvs->dtype.rehashingCompleted = kvstoreDictRehashingCompleted; @@ -263,7 +285,6 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) { kvs->bucket_count = 0; kvs->overhead_hashtable_lut = 0; kvs->overhead_hashtable_rehashing = 0; - return kvs; } @@ -272,9 +293,13 @@ void kvstoreEmpty(kvstore *kvs, void(callback)(dict*)) { dict *d = kvstoreGetDict(kvs, didx); if (!d) continue; - kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d); + kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d); if (metadata->rehashing_node) metadata->rehashing_node = NULL; + if (kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST) { + kvstoreDictMetaEx *metaExt = (kvstoreDictMetaEx *) metadata; + memset(&metaExt->meta.keysizes_hist, 0, sizeof(metaExt->meta.keysizes_hist)); + } dictEmpty(d, callback); freeDictIfNeeded(kvs, didx); } @@ -296,7 +321,7 @@ void kvstoreRelease(kvstore *kvs) { dict *d = kvstoreGetDict(kvs, didx); if (!d) continue; - kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d); + kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d); if (metadata->rehashing_node) metadata->rehashing_node = NULL; dictRelease(d); @@ -330,11 +355,15 @@ unsigned long kvstoreBuckets(kvstore *kvs) { size_t kvstoreMemUsage(kvstore *kvs) { size_t mem = sizeof(*kvs); + size_t metaSize = sizeof(kvstoreDictMetaBase); + if (kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST) + metaSize = sizeof(kvstoreDictMetaEx); + unsigned long long keys_count = kvstoreSize(kvs); mem += keys_count * dictEntryMemUsage() + kvstoreBuckets(kvs) * sizeof(dictEntry*) + - kvs->allocated_dicts * (sizeof(dict) + kvstoreDictMetadataSize(NULL)); + kvs->allocated_dicts * (sizeof(dict) + metaSize); /* Values are dict* shared with kvs->dicts */ mem += listLength(kvs->rehashing) * sizeof(listNode); @@ -785,7 +814,7 @@ void kvstoreDictLUTDefrag(kvstore *kvs, kvstoreDictLUTDefragFunction *defragfn) /* After defragmenting the dict, update its corresponding * rehashing node in the kvstore's rehashing list. */ - kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(*d); + kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(*d); if (metadata->rehashing_node) metadata->rehashing_node->value = *d; } @@ -856,6 +885,19 @@ int kvstoreDictDelete(kvstore *kvs, int didx, const void *key) { return ret; } +kvstoreDictMetadata *kvstoreGetDictMetadata(kvstore *kvs, int didx) { + dict *d = kvstoreGetDict(kvs, didx); + if ((!d) || (!(kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST))) + return NULL; + + kvstoreDictMetaEx *metadata = (kvstoreDictMetaEx *)dictMetadata(d); + return &(metadata->meta); +} + +kvstoreMetadata *kvstoreGetMetadata(kvstore *kvs) { + return (kvstoreMetadata *) &kvs->metadata; +} + #ifdef REDIS_TEST #include #include "testhelp.h" @@ -1029,7 +1071,8 @@ int kvstoreTest(int argc, char **argv, int flags) { } TEST("Verify non-empty dict count is correctly updated") { - kvstore *kvs = kvstoreCreate(&KvstoreDictTestType, 2, KVSTORE_ALLOCATE_DICTS_ON_DEMAND); + kvstore *kvs = kvstoreCreate(&KvstoreDictTestType, 2, + KVSTORE_ALLOCATE_DICTS_ON_DEMAND | KVSTORE_ALLOC_META_KEYS_HIST); for (int idx = 0; idx < 4; idx++) { for (i = 0; i < 16; i++) { de = kvstoreDictAddRaw(kvs, idx, stringFromInt(i), NULL); diff --git a/src/kvstore.h b/src/kvstore.h index bce45fe4c..3c3f7948c 100644 --- a/src/kvstore.h +++ b/src/kvstore.h @@ -4,6 +4,21 @@ #include "dict.h" #include "adlist.h" +/* maximum number of bins of keysizes histogram */ +#define MAX_KEYSIZES_BINS 48 +#define MAX_KEYSIZES_TYPES 5 /* static_assert at db.c verifies == OBJ_TYPE_BASIC_MAX */ + +/* When creating kvstore with flag `KVSTORE_ALLOC_META_KEYS_HIST`, then kvstore + * alloc and memset struct kvstoreMetadata on init, yet, managed outside kvstore */ +typedef struct { + uint64_t keysizes_hist[MAX_KEYSIZES_TYPES][MAX_KEYSIZES_BINS]; +} kvstoreMetadata; + +/* Like kvstoreMetadata, this one per dict */ +typedef struct { + uint64_t keysizes_hist[MAX_KEYSIZES_TYPES][MAX_KEYSIZES_BINS]; +} kvstoreDictMetadata; + typedef struct _kvstore kvstore; typedef struct _kvstoreIterator kvstoreIterator; typedef struct _kvstoreDictIterator kvstoreDictIterator; @@ -13,6 +28,7 @@ typedef int (kvstoreExpandShouldSkipDictIndex)(int didx); #define KVSTORE_ALLOCATE_DICTS_ON_DEMAND (1<<0) #define KVSTORE_FREE_EMPTY_DICTS (1<<1) +#define KVSTORE_ALLOC_META_KEYS_HIST (1<<2) /* Alloc keysizes histogram */ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags); void kvstoreEmpty(kvstore *kvs, void(callback)(dict*)); void kvstoreRelease(kvstore *kvs); @@ -71,6 +87,8 @@ void kvstoreDictSetVal(kvstore *kvs, int didx, dictEntry *de, void *val); dictEntry *kvstoreDictTwoPhaseUnlinkFind(kvstore *kvs, int didx, const void *key, dictEntry ***plink, int *table_index); void kvstoreDictTwoPhaseUnlinkFree(kvstore *kvs, int didx, dictEntry *he, dictEntry **plink, int table_index); int kvstoreDictDelete(kvstore *kvs, int didx, const void *key); +kvstoreDictMetadata *kvstoreGetDictMetadata(kvstore *kvs, int didx); +kvstoreMetadata *kvstoreGetMetadata(kvstore *kvs); #ifdef REDIS_TEST int kvstoreTest(int argc, char *argv[], int flags); diff --git a/src/lazyfree.c b/src/lazyfree.c index 858751757..c33bc923e 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -207,7 +207,7 @@ void emptyDbAsync(redisDb *db) { } kvstore *oldkeys = db->keys, *oldexpires = db->expires; ebuckets oldHfe = db->hexpires; - db->keys = kvstoreCreate(&dbDictType, slot_count_bits, flags); + db->keys = kvstoreCreate(&dbDictType, slot_count_bits, flags | KVSTORE_ALLOC_META_KEYS_HIST); db->expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags); db->hexpires = ebCreate(); atomicIncr(lazyfree_objects, kvstoreSize(oldkeys)); diff --git a/src/module.c b/src/module.c index 2b6e625f1..032c3557c 100644 --- a/src/module.c +++ b/src/module.c @@ -4171,15 +4171,7 @@ int RM_KeyType(RedisModuleKey *key) { * If the key pointer is NULL or the key is empty, zero is returned. */ size_t RM_ValueLength(RedisModuleKey *key) { if (key == NULL || key->value == NULL) return 0; - switch(key->value->type) { - case OBJ_STRING: return stringObjectLen(key->value); - case OBJ_LIST: return listTypeLength(key->value); - case OBJ_SET: return setTypeSize(key->value); - case OBJ_ZSET: return zsetLength(key->value); - case OBJ_HASH: return hashTypeLength(key->value, 0); /* OPEN: To subtract expired fields? */ - case OBJ_STREAM: return streamLength(key->value); - default: return 0; - } + return getObjectLength(key->value); } /* If the key is open for writing, remove it, and setup the key to diff --git a/src/object.c b/src/object.c index 2b42e7b3e..d065359fa 100644 --- a/src/object.c +++ b/src/object.c @@ -680,6 +680,18 @@ robj *tryObjectEncoding(robj *o) { return tryObjectEncodingEx(o, 1); } +size_t getObjectLength(robj *o) { + switch(o->type) { + case OBJ_STRING: return stringObjectLen(o); + case OBJ_LIST: return listTypeLength(o); + case OBJ_SET: return setTypeSize(o); + case OBJ_ZSET: return zsetLength(o); + case OBJ_HASH: return hashTypeLength(o, 0); + case OBJ_STREAM: return streamLength(o); + default: return 0; + } +} + /* Get a decoded version of an encoded object (returned as a new object). * If the object is already raw-encoded just increment the ref count. */ robj *getDecodedObject(robj *o) { diff --git a/src/server.c b/src/server.c index 72208c7e2..054ad171f 100644 --- a/src/server.c +++ b/src/server.c @@ -2690,7 +2690,7 @@ void initServer(void) { flags |= KVSTORE_FREE_EMPTY_DICTS; } for (j = 0; j < server.dbnum; j++) { - server.db[j].keys = kvstoreCreate(&dbDictType, slot_count_bits, flags); + server.db[j].keys = kvstoreCreate(&dbDictType, slot_count_bits, flags | KVSTORE_ALLOC_META_KEYS_HIST); server.db[j].expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags); server.db[j].hexpires = ebCreate(); server.db[j].expires_cursor = 0; @@ -5521,7 +5521,7 @@ void releaseInfoSectionDict(dict *sec) { dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, int *out_everything) { char *default_sections[] = { "server", "clients", "memory", "persistence", "stats", "replication", - "cpu", "module_list", "errorstats", "cluster", "keyspace", NULL}; + "cpu", "module_list", "errorstats", "cluster", "keyspace", "keysizes", NULL}; if (!defaults) defaults = default_sections; @@ -6149,6 +6149,60 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) { } } + /* keysizes */ + if (all_sections || (dictFind(section_dict,"keysizes") != NULL)) { + if (sections++) info = sdscat(info,"\r\n"); + info = sdscatprintf(info, "# Keysizes\r\n"); + + char *typestr[] = { + [OBJ_STRING] = "distrib_strings_sizes", + [OBJ_LIST] = "distrib_lists_items", + [OBJ_SET] = "distrib_sets_items", + [OBJ_ZSET] = "distrib_zsets_items", + [OBJ_HASH] = "distrib_hashes_items" + }; + serverAssert(sizeof(typestr)/sizeof(typestr[0]) == OBJ_TYPE_BASIC_MAX); + + for (int dbnum = 0; dbnum < server.dbnum; dbnum++) { + char *expSizeLabels[] = { + "1", "2", "4", "8", "16", "32", "64", "128", "256", "512", /* Byte */ + "1K", "2K", "4K", "8K", "16K", "32K", "64K", "128K", "256K", "512K", /* Kilo */ + "1M", "2M", "4M", "8M", "16M", "32M", "64M", "128M", "256M", "512M", /* Mega */ + "1G", "2G", "4G", "8G", "16G", "32G", "64G", "128G", "256G", "512G", /* Giga */ + "1T", "2T", "4T", "8T", "16T", "32T", "64T", "128T", "256T", "512T", /* Tera */ + "1P", "2P", "4P", "8P", "16P", "32P", "64P", "128P", "256P", "512P", /* Peta */ + "1E", "2E", "4E", "8E" /* Exa */ + }; + + if (kvstoreSize(server.db[dbnum].keys) == 0) + continue; + + for (int type = 0; type < OBJ_TYPE_BASIC_MAX; type++) { + uint64_t *kvstoreHist = kvstoreGetMetadata(server.db[dbnum].keys)->keysizes_hist[type]; + char buf[10000]; + int cnt = 0, buflen = 0; + + /* Print histogram to temp buf[]. First bin is garbage */ + buflen += snprintf(buf + buflen, sizeof(buf) - buflen, "db%d_%s:", dbnum, typestr[type]); + + for (int i = 0; i < MAX_KEYSIZES_BINS; i++) { + if (kvstoreHist[i] == 0) + continue; + + int res = snprintf(buf + buflen, sizeof(buf) - buflen, + (cnt == 0) ? "%s=%llu" : ",%s=%llu", + expSizeLabels[i], (unsigned long long) kvstoreHist[i]); + if (res < 0) break; + buflen += res; + cnt += kvstoreHist[i]; + } + + /* Print the temp buf[] to the info string */ + if (cnt) info = sdscatprintf(info, "%s\r\n", buf); + } + } + } + /* Get info from modules. * Returned when the user asked for "everything", "modules", or a specific module section. * We're not aware of the module section names here, and we rather avoid the search when we can. diff --git a/src/server.h b/src/server.h index b650f2699..4f5686192 100644 --- a/src/server.h +++ b/src/server.h @@ -41,10 +41,6 @@ #include #endif -#ifndef static_assert -#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1] -#endif - typedef long long mstime_t; /* millisecond time type. */ typedef long long ustime_t; /* microsecond time type. */ @@ -698,6 +694,7 @@ typedef enum { #define OBJ_SET 2 /* Set object. */ #define OBJ_ZSET 3 /* Sorted set object. */ #define OBJ_HASH 4 /* Hash object. */ +#define OBJ_TYPE_BASIC_MAX 5 /* Max number of basic object types. */ /* The "module" object type is a special one that signals that the object * is one directly managed by a Redis module. In this case the value points @@ -969,7 +966,7 @@ typedef struct replBufBlock { * by integers from 0 (the default database) up to the max configured * database. The database number is the 'id' field in the structure. */ typedef struct redisDb { - kvstore *keys; /* The keyspace for this DB */ + kvstore *keys; /* The keyspace for this DB. As metadata, holds keysizes histogram */ kvstore *expires; /* Timeout of keys with a timeout set */ ebuckets hexpires; /* Hash expiration DS. Single TTL per hash (of next min field to expire) */ dict *blocking_keys; /* Keys with clients waiting for data (BLPOP)*/ @@ -2799,6 +2796,7 @@ int isSdsRepresentableAsLongLong(sds s, long long *llval); int isObjectRepresentableAsLongLong(robj *o, long long *llongval); robj *tryObjectEncoding(robj *o); robj *tryObjectEncodingEx(robj *o, int try_trim); +size_t getObjectLength(robj *o); robj *getDecodedObject(robj *o); size_t stringObjectLen(robj *o); robj *createStringObjectFromLongLong(long long value); @@ -3363,6 +3361,7 @@ long long getModuleNumericConfig(ModuleConfig *module_config); int setModuleNumericConfig(ModuleConfig *config, long long val, const char **err); /* db.c -- Keyspace access API */ +void updateKeysizesHist(redisDb *db, int didx, uint32_t type, uint64_t oldLen, uint64_t newLen); int removeExpire(redisDb *db, robj *key); void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj); void deleteEvictedKeyAndPropagate(redisDb *db, robj *keyobj, long long *key_mem_freed); diff --git a/src/t_hash.c b/src/t_hash.c index 1625513eb..f114fa9b9 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -422,8 +422,13 @@ void listpackExExpire(redisDb *db, robj *o, ExpireInfo *info) { expired++; } - if (expired) + if (expired) { lpt->lp = lpDeleteRange(lpt->lp, 0, expired * 3); + + /* update keysizes */ + unsigned long l = lpLength(lpt->lp) / 3; + updateKeysizesHist(db, getKeySlot(lpt->key), OBJ_HASH, l + expired, l); + } min = hashTypeGetMinExpire(o, 1 /*accurate*/); info->nextExpireTime = min; @@ -546,6 +551,11 @@ SetExRes hashTypeSetExpiryListpack(HashTypeSetEx *ex, sds field, if (unlikely(checkAlreadyExpired(expireAt))) { propagateHashFieldDeletion(ex->db, ex->key->ptr, field, sdslen(field)); hashTypeDelete(ex->hashObj, field, 1); + + /* get listpack length */ + listpackEx *lpt = ((listpackEx *) ex->hashObj->ptr); + unsigned long length = lpLength(lpt->lp) / 3; + updateKeysizesHist(ex->db, getKeySlot(ex->key->ptr), OBJ_HASH, length+1, length); server.stat_expired_subkeys++; ex->fieldDeleted++; return HSETEX_DELETED; @@ -1042,6 +1052,8 @@ SetExRes hashTypeSetExpiryHT(HashTypeSetEx *exInfo, sds field, uint64_t expireAt /* If expired, then delete the field and propagate the deletion. * If replica, continue like the field is valid */ if (unlikely(checkAlreadyExpired(expireAt))) { + unsigned long length = dictSize(ht); + updateKeysizesHist(exInfo->db, getKeySlot(exInfo->key->ptr), OBJ_HASH, length, length-1); /* replicas should not initiate deletion of fields */ propagateHashFieldDeletion(exInfo->db, exInfo->key->ptr, field, sdslen(field)); hashTypeDelete(exInfo->hashObj, field, 1); @@ -2132,6 +2144,7 @@ ebuckets *hashTypeGetDictMetaHFE(dict *d) { *----------------------------------------------------------------------------*/ void hsetnxCommand(client *c) { + unsigned long hlen; int isHashDeleted; robj *o; if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; @@ -2152,6 +2165,8 @@ void hsetnxCommand(client *c) { addReply(c, shared.cone); signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id); + hlen = hashTypeLength(o, 0); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, hlen - 1, hlen); server.dirty++; } @@ -2180,6 +2195,8 @@ void hsetCommand(client *c) { addReply(c, shared.ok); } signalModifiedKey(c,c->db,c->argv[1]); + unsigned long l = hashTypeLength(o, 0); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, l - created, l); notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id); server.dirty += (c->argc - 2)/2; } @@ -2205,11 +2222,14 @@ void hincrbyCommand(client *c) { } /* Else hashTypeGetValue() already stored it into &value */ } else if ((res == GETF_NOT_FOUND) || (res == GETF_EXPIRED)) { value = 0; + unsigned long l = hashTypeLength(o, 0); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, l, l + 1); } else { /* Field expired and in turn hash deleted. Create new one! */ o = createHashObject(); dbAdd(c->db,c->argv[1],o); value = 0; + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, 0, 1); } oldvalue = value; @@ -2254,11 +2274,14 @@ void hincrbyfloatCommand(client *c) { } } else if ((res == GETF_NOT_FOUND) || (res == GETF_EXPIRED)) { value = 0; + unsigned long l = hashTypeLength(o, 0); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, l, l + 1); } else { /* Field expired and in turn hash deleted. Create new one! */ o = createHashObject(); dbAdd(c->db,c->argv[1],o); value = 0; + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, 0, 1); } value += incr; @@ -2356,6 +2379,8 @@ void hdelCommand(client *c) { if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || checkType(c,o,OBJ_HASH)) return; + unsigned long oldLen = hashTypeLength(o, 0); + /* Hash field expiration is optimized to avoid frequent update global HFE DS for * each field deletion. Eventually active-expiration will run and update or remove * the hash from global HFE DS gracefully. Nevertheless, statistic "subexpiry" @@ -2375,6 +2400,8 @@ void hdelCommand(client *c) { } } if (deleted) { + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, oldLen, oldLen - deleted); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH,"hdel",c->argv[1],c->db->id); if (keyremoved) { @@ -2943,6 +2970,11 @@ static ExpireAction onFieldExpire(eItem item, void *ctx) { dict *d = expCtx->hashObj->ptr; dictExpireMetadata *dictExpireMeta = (dictExpireMetadata *) dictMetadata(d); propagateHashFieldDeletion(expCtx->db, dictExpireMeta->key, hf, hfieldlen(hf)); + + /* update keysizes */ + unsigned long l = hashTypeLength(expCtx->hashObj, 0); + updateKeysizesHist(expCtx->db, getKeySlot(dictExpireMeta->key), OBJ_HASH, l, l - 1); + serverAssert(hashTypeDelete(expCtx->hashObj, hf, 0) == 1); server.stat_expired_subkeys++; return ACT_REMOVE_EXP_ITEM; diff --git a/src/t_list.c b/src/t_list.c index 98b180aa1..9263cbd12 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -7,6 +7,7 @@ */ #include "server.h" +#include "util.h" /*----------------------------------------------------------------------------- * List API @@ -462,6 +463,7 @@ void listTypeDelRange(robj *subject, long start, long count) { /* Implements LPUSH/RPUSH/LPUSHX/RPUSHX. * 'xx': push if key exists. */ void pushGenericCommand(client *c, int where, int xx) { + unsigned long llen; int j; robj *lobj = lookupKeyWrite(c->db, c->argv[1]); @@ -482,11 +484,13 @@ void pushGenericCommand(client *c, int where, int xx) { server.dirty++; } - addReplyLongLong(c, listTypeLength(lobj)); + llen = listTypeLength(lobj); + addReplyLongLong(c, llen); char *event = (where == LIST_HEAD) ? "lpush" : "rpush"; signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, llen - (c->argc - 2), llen); } /* LPUSH [ ...] */ @@ -553,6 +557,8 @@ void linsertCommand(client *c) { notifyKeyspaceEvent(NOTIFY_LIST,"linsert", c->argv[1],c->db->id); server.dirty++; + unsigned long ll = listTypeLength(subject); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, ll-1, ll); } else { /* Notify client of a failed insert */ addReplyLongLong(c,-1); @@ -736,9 +742,11 @@ void addListRangeReply(client *c, robj *o, long start, long end, int reverse) { * if the key got deleted by this function. */ void listElementsRemoved(client *c, robj *key, int where, robj *o, long count, int signal, int *deleted) { char *event = (where == LIST_HEAD) ? "lpop" : "rpop"; - + unsigned long llen = listTypeLength(o); + notifyKeyspaceEvent(NOTIFY_LIST, event, key, c->db->id); - if (listTypeLength(o) == 0) { + updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_LIST, llen + count, llen); + if (llen == 0) { if (deleted) *deleted = 1; dbDelete(c->db, key); @@ -870,7 +878,7 @@ void lrangeCommand(client *c) { /* LTRIM */ void ltrimCommand(client *c) { robj *o; - long start, end, llen, ltrim, rtrim; + long start, end, llen, ltrim, rtrim, llenNew;; if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) || (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return; @@ -908,12 +916,13 @@ void ltrimCommand(client *c) { } notifyKeyspaceEvent(NOTIFY_LIST,"ltrim",c->argv[1],c->db->id); - if (listTypeLength(o) == 0) { + if ((llenNew = listTypeLength(o)) == 0) { dbDelete(c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id); } else { listTypeTryConversion(o,LIST_CONV_SHRINKING,NULL,NULL); } + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, llen, llenNew); signalModifiedKey(c,c->db,c->argv[1]); server.dirty += (ltrim + rtrim); addReply(c,shared.ok); @@ -1066,8 +1075,11 @@ void lremCommand(client *c) { listTypeReleaseIterator(li); if (removed) { + long ll = listTypeLength(subject); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, ll + removed, ll); notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id); - if (listTypeLength(subject) == 0) { + + if (ll == 0) { dbDelete(c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id); } else { @@ -1089,6 +1101,10 @@ void lmoveHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value, listTypeTryConversionAppend(dstobj,&value,0,0,NULL,NULL); listTypePush(dstobj,value,where); signalModifiedKey(c,c->db,dstkey); + + long ll = listTypeLength(dstobj); + updateKeysizesHist(c->db, getKeySlot(dstkey->ptr), OBJ_LIST, ll - 1, ll); + notifyKeyspaceEvent(NOTIFY_LIST, where == LIST_HEAD ? "lpush" : "rpush", dstkey, diff --git a/src/t_set.c b/src/t_set.c index 7b62f94b6..37055275c 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -603,6 +603,8 @@ void saddCommand(client *c) { if (setTypeAdd(set,c->argv[j]->ptr)) added++; } if (added) { + unsigned long size = setTypeSize(set); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, size - added, size); signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[1],c->db->id); } @@ -617,6 +619,8 @@ void sremCommand(client *c) { if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || checkType(c,set,OBJ_SET)) return; + unsigned long oldSize = setTypeSize(set); + for (j = 2; j < c->argc; j++) { if (setTypeRemove(set,c->argv[j]->ptr)) { deleted++; @@ -628,6 +632,8 @@ void sremCommand(client *c) { } } if (deleted) { + + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, oldSize, oldSize - deleted); signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id); if (keyremoved) @@ -669,8 +675,12 @@ void smoveCommand(client *c) { } notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id); + /* Update keysizes histogram */ + unsigned long srcLen = setTypeSize(srcset); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, srcLen + 1, srcLen); + /* Remove the src set from the database when empty */ - if (setTypeSize(srcset) == 0) { + if (srcLen == 0) { dbDelete(c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id); } @@ -686,6 +696,8 @@ void smoveCommand(client *c) { /* An extra key has changed when ele was successfully added to dstset */ if (setTypeAdd(dstset,ele->ptr)) { + unsigned long dstLen = setTypeSize(dstset); + updateKeysizesHist(c->db, getKeySlot(c->argv[2]->ptr), OBJ_SET, dstLen - 1, dstLen); server.dirty++; signalModifiedKey(c,c->db,c->argv[2]); notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[2],c->db->id); @@ -743,7 +755,7 @@ void scardCommand(client *c) { void spopWithCountCommand(client *c) { long l; - unsigned long count, size; + unsigned long count, size, toRemove; robj *set; /* Get the count argument */ @@ -763,10 +775,12 @@ void spopWithCountCommand(client *c) { } size = setTypeSize(set); + toRemove = (count >= size) ? size : count; /* Generate an SPOP keyspace notification */ notifyKeyspaceEvent(NOTIFY_SET,"spop",c->argv[1],c->db->id); - server.dirty += (count >= size) ? size : count; + server.dirty += toRemove; + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, size, size - toRemove); /* CASE 1: * The number of requested elements is greater than or equal to @@ -949,6 +963,7 @@ void spopWithCountCommand(client *c) { } void spopCommand(client *c) { + unsigned long size; robj *set, *ele; if (c->argc == 3) { @@ -964,6 +979,9 @@ void spopCommand(client *c) { if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp])) == NULL || checkType(c,set,OBJ_SET)) return; + size = setTypeSize(set); + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, size, size-1); + /* Pop a random element from the set */ ele = setTypePopRandom(set); diff --git a/src/t_string.c b/src/t_string.c index d1d6dce39..c96f5e89e 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -420,6 +420,7 @@ void getsetCommand(client *c) { } void setrangeCommand(client *c) { + size_t oldLen = 0, newLen; robj *o; long offset; sds value = c->argv[3]->ptr; @@ -449,16 +450,14 @@ void setrangeCommand(client *c) { o = createObject(OBJ_STRING,sdsnewlen(NULL, offset+value_len)); dbAdd(c->db,c->argv[1],o); } else { - size_t olen; - /* Key exists, check type */ if (checkType(c,o,OBJ_STRING)) return; /* Return existing string length when setting nothing */ - olen = stringObjectLen(o); + oldLen = stringObjectLen(o); if (value_len == 0) { - addReplyLongLong(c,olen); + addReplyLongLong(c, oldLen); return; } @@ -478,7 +477,10 @@ void setrangeCommand(client *c) { "setrange",c->argv[1],c->db->id); server.dirty++; } - addReplyLongLong(c,sdslen(o->ptr)); + + newLen = sdslen(o->ptr); + updateKeysizesHist(c->db,getKeySlot(c->argv[1]->ptr),OBJ_STRING,oldLen,newLen); + addReplyLongLong(c,newLen); } void getrangeCommand(client *c) { @@ -669,7 +671,7 @@ void incrbyfloatCommand(client *c) { } void appendCommand(client *c) { - size_t totlen; + size_t totlen, append_len; robj *o, *append; dictEntry *de; @@ -679,7 +681,7 @@ void appendCommand(client *c) { c->argv[2] = tryObjectEncoding(c->argv[2]); dbAdd(c->db,c->argv[1],c->argv[2]); incrRefCount(c->argv[2]); - totlen = stringObjectLen(c->argv[2]); + append_len = totlen = stringObjectLen(c->argv[2]); } else { /* Key exists, check type */ if (checkType(c,o,OBJ_STRING)) @@ -687,7 +689,7 @@ void appendCommand(client *c) { /* "append" is an argument, so always an sds */ append = c->argv[2]; - const size_t append_len = sdslen(append->ptr); + append_len = sdslen(append->ptr); if (checkStringLength(c,stringObjectLen(o),append_len) != C_OK) return; @@ -699,6 +701,7 @@ void appendCommand(client *c) { signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"append",c->argv[1],c->db->id); server.dirty++; + updateKeysizesHist(c->db,getKeySlot(c->argv[1]->ptr),OBJ_STRING, totlen - append_len, totlen); addReplyLongLong(c,totlen); } diff --git a/src/t_zset.c b/src/t_zset.c index 590f21f86..7b014f0d0 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1843,6 +1843,7 @@ void zaddGenericCommand(client *c, int flags) { zsetTypeMaybeConvert(zobj, elements); } + unsigned long llen = zsetLength(zobj); for (j = 0; j < elements; j++) { double newscore; score = scores[j]; @@ -1860,6 +1861,7 @@ void zaddGenericCommand(client *c, int flags) { score = newscore; } server.dirty += (added+updated); + updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, llen, llen+added); reply_to_client: if (incr) { /* ZINCRBY or INCR option. */ @@ -1907,8 +1909,13 @@ void zremCommand(client *c) { if (deleted) { notifyKeyspaceEvent(NOTIFY_ZSET,"zrem",key,c->db->id); - if (keyremoved) - notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); + if (keyremoved) { + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, c->db->id); + /* No need updateKeysizesHist(). dbDelete() done it already. */ + } else { + unsigned long len = zsetLength(zobj); + updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, len + deleted, len); + } signalModifiedKey(c,c->db,key); server.dirty += deleted; } @@ -2023,8 +2030,13 @@ void zremrangeGenericCommand(client *c, zrange_type rangetype) { if (deleted) { signalModifiedKey(c,c->db,key); notifyKeyspaceEvent(NOTIFY_ZSET,notify_type,key,c->db->id); - if (keyremoved) - notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); + if (keyremoved) { + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, c->db->id); + /* No need updateKeysizesHist(). dbDelete() done it already. */ + } else { + unsigned long len = zsetLength(zobj); + updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, len + deleted, len); + } } server.dirty += deleted; addReplyLongLong(c,deleted); @@ -4031,6 +4043,9 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey dbDelete(c->db,key); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); + /* No need updateKeysizesHist(). dbDelete() done it already. */ + } else { + updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, llen, llen - result_count); } signalModifiedKey(c,c->db,key); diff --git a/src/util.c b/src/util.c index c0e69af4d..ec0a3fb0e 100644 --- a/src/util.c +++ b/src/util.c @@ -54,6 +54,13 @@ #define UNUSED(x) ((void)(x)) +/* Selectively define static_assert. Attempt to avoid include server.h in this file. */ +#ifndef static_assert +#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1] +#endif + +static_assert(UINTPTR_MAX == 0xffffffffffffffff || UINTPTR_MAX == 0xffffffff, "Unsupported pointer size"); + /* Glob-style pattern matching. */ static int stringmatchlen_impl(const char *pattern, int patternLen, const char *string, int stringLen, int nocase, int *skipLongerMatches, int nesting) diff --git a/src/util.h b/src/util.h index 518342f02..07cfb61dc 100644 --- a/src/util.h +++ b/src/util.h @@ -79,6 +79,19 @@ int snprintf_async_signal_safe(char *to, size_t n, const char *fmt, ...); size_t redis_strlcpy(char *dst, const char *src, size_t dsize); size_t redis_strlcat(char *dst, const char *src, size_t dsize); +/* to keep it opt without conditions Works only for: 0 < x < 2^63 */ +static inline int log2ceil(size_t x) { +#if UINTPTR_MAX == 0xffffffffffffffff + return 63 - __builtin_clzll(x); +#else + return 31 - __builtin_clz(x); +#endif +} + +#ifndef static_assert +#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1] +#endif + #ifdef REDIS_TEST int utilTest(int argc, char **argv, int flags); #endif diff --git a/tests/unit/info-keysizes.tcl b/tests/unit/info-keysizes.tcl new file mode 100644 index 000000000..98d6d4e6f --- /dev/null +++ b/tests/unit/info-keysizes.tcl @@ -0,0 +1,454 @@ +################################################################################ +# Test the "info keysizes" command. +# The command returns a histogram of the sizes of keys in the database. +################################################################################ + +# Query and Strip result of "info keysizes" from header, spaces, and newlines. +proc get_stripped_info {server} { + set infoStripped [string map { + "# Keysizes" "" + " " "" "\n" "" "\r" "" + } [$server info keysizes] ] + return $infoStripped +} + +# Verify output of "info keysizes" command is as expected. +# +# Arguments: +# cmd - A command that should be run before the verification. +# expOutput - This is a string that represents the expected output abbreviated. +# Instead of the output of "strings_len_exp_distrib" write "STR". +# Similarly for LIST, SET, ZSET and HASH. Spaces and newlines are +# ignored. +# waitCond - If set to 1, the function wait_for_condition 50x50msec for the +# expOutput to match the actual output. +# +# (replicaMode) - Global variable that indicates if the test is running in replica +# mode. If so, run the command on leader, verify the output. Then wait +# for the replica to catch up and verify the output on the replica +# as well. Otherwise, just run the command on the leader and verify +# the output. +proc run_cmd_verify_hist {cmd expOutput {waitCond 0} } { + uplevel 1 $cmd + global replicaMode + + # ref the leader with `server` variable + if {$replicaMode eq 1} { set server [srv -1 client] } else { set server [srv 0 client] } + + # Replace all placeholders with the actual values. Remove spaces & newlines. + set expStripped [string map { + "STR" "distrib_strings_sizes" + "LIST" "distrib_lists_items" + "SET" "distrib_sets_items" + "ZSET" "distrib_zsets_items" + "HASH" "distrib_hashes_items" + " " "" "\n" "" "\r" "" + } $expOutput] + + if {$waitCond} { + wait_for_condition 50 50 { + $expStripped eq [get_stripped_info $server] + } else { + fail "Unexpected KEYSIZES. Expected: `$expStripped` \ + but got: `[get_stripped_info $server]`. Failed after command: $cmd" + + } + } else { + set infoStripped [get_stripped_info $server] + if {$expStripped ne $infoStripped} { + fail "Unexpected KEYSIZES. Expected: `$expStripped` \ + but got: `$infoStripped`. Failed after command: $cmd" + } + } + + # If we are testing `replicaMode` then need to wait for the replica to catch up + if {$replicaMode eq 1} { + wait_for_condition 50 50 { + $expStripped eq [get_stripped_info $server] + } else { + fail "Unexpected replica KEYSIZES. Expected: `$expStripped` \ + but got: `[get_stripped_info $server]`. Failed after command: $cmd" + } + } +} + +proc test_all_keysizes { {replMode 0} } { + # If in replica mode then update global var `replicaMode` so function + # `run_cmd_verify_hist` knows to run the command on the leader and then + # wait for the replica to catch up. + global replicaMode + set replicaMode $replMode + # ref the leader with `server` variable + if {$replicaMode eq 1} { + set server [srv -1 client] + set suffixRepl "(replica)" + } else { + set server [srv 0 client] + set suffixRepl "" + } + + test "KEYSIZES - Test i'th bin counts keysizes between (2^i) and (2^(i+1)-1) as expected $suffixRepl" { + set base_string "" + run_cmd_verify_hist {$server FLUSHALL} {} + for {set i 1} {$i <= 10} {incr i} { + append base_string "x" + set log_value [expr {1 << int(log($i) / log(2))}] + #puts "Iteration $i: $base_string (Log base 2 pattern: $log_value)" + run_cmd_verify_hist {$server set mykey $base_string} "db0_STR:$log_value=1" + } + } + + test "KEYSIZES - Histogram of values of Bytes, Kilo and Mega $suffixRepl" { + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server set x 0123456789ABCDEF} {db0_STR:16=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:32=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:64=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:128=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:256=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:512=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:1K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:2K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:4K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:8K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:16K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:32K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:64K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:128K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:256K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:512K=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:1M=1} + run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:2M=1} + } + + test "KEYSIZES - Test List $suffixRepl" { + # FLUSHALL + run_cmd_verify_hist {$server FLUSHALL} {} + # RPUSH + run_cmd_verify_hist {$server RPUSH l1 1 2 3 4 5} {db0_LIST:4=1} + run_cmd_verify_hist {$server RPUSH l1 6 7 8 9} {db0_LIST:8=1} + # Test also LPUSH, RPUSH, LPUSHX, RPUSHX + run_cmd_verify_hist {$server LPUSH l2 1} {db0_LIST:1=1,8=1} + run_cmd_verify_hist {$server LPUSH l2 2} {db0_LIST:2=1,8=1} + run_cmd_verify_hist {$server LPUSHX l2 3} {db0_LIST:2=1,8=1} + run_cmd_verify_hist {$server RPUSHX l2 4} {db0_LIST:4=1,8=1} + # RPOP + run_cmd_verify_hist {$server RPOP l1} {db0_LIST:4=1,8=1} + run_cmd_verify_hist {$server RPOP l1} {db0_LIST:4=2} + # DEL + run_cmd_verify_hist {$server DEL l1} {db0_LIST:4=1} + # LINSERT, LTRIM + run_cmd_verify_hist {$server RPUSH l3 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} {db0_LIST:4=1,8=1} + run_cmd_verify_hist {$server LINSERT l3 AFTER 9 10} {db0_LIST:4=1,16=1} + run_cmd_verify_hist {$server LTRIM l3 0 8} {db0_LIST:4=1,8=1} + # DEL + run_cmd_verify_hist {$server DEL l3} {db0_LIST:4=1} + run_cmd_verify_hist {$server DEL l2} {} + # LMOVE, BLMOVE + run_cmd_verify_hist {$server RPUSH l4 1 2 3 4 5 6 7 8} {db0_LIST:8=1} + run_cmd_verify_hist {$server LMOVE l4 l5 LEFT LEFT} {db0_LIST:1=1,4=1} + run_cmd_verify_hist {$server LMOVE l4 l5 RIGHT RIGHT} {db0_LIST:2=1,4=1} + run_cmd_verify_hist {$server LMOVE l4 l5 LEFT RIGHT} {db0_LIST:2=1,4=1} + run_cmd_verify_hist {$server LMOVE l4 l5 RIGHT LEFT} {db0_LIST:4=2} + run_cmd_verify_hist {$server BLMOVE l4 l5 RIGHT LEFT 0} {db0_LIST:2=1,4=1} + # DEL + run_cmd_verify_hist {$server DEL l4} {db0_LIST:4=1} + run_cmd_verify_hist {$server DEL l5} {} + # LMPOP + run_cmd_verify_hist {$server RPUSH l6 1 2 3 4 5 6 7 8 9 10} {db0_LIST:8=1} + run_cmd_verify_hist {$server LMPOP 1 l6 LEFT COUNT 2} {db0_LIST:8=1} + run_cmd_verify_hist {$server LMPOP 1 l6 LEFT COUNT 1} {db0_LIST:4=1} + run_cmd_verify_hist {$server LMPOP 1 l6 LEFT COUNT 6} {db0_LIST:1=1} + # LPOP + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH l7 1 2 3 4} {db0_LIST:4=1} + run_cmd_verify_hist {$server LPOP l7} {db0_LIST:2=1} + # LREM + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH l8 1 x 3 x 5 x 7 x 9 10} {db0_LIST:8=1} + run_cmd_verify_hist {$server LREM l8 3 x} {db0_LIST:4=1} + # EXPIRE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH l9 1 2 3 4} {db0_LIST:4=1} + run_cmd_verify_hist {$server PEXPIRE l9 50} {db0_LIST:4=1} + run_cmd_verify_hist {} {} 1 + # SET overwrites + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH l9 1 2 3 4} {db0_LIST:4=1} + run_cmd_verify_hist {$server SET l9 1234567} {db0_STR:4=1} + run_cmd_verify_hist {$server DEL l9} {} + } {} {cluster:skip} + + test "KEYSIZES - Test SET $suffixRepl" { + run_cmd_verify_hist {$server FLUSHALL} {} + # SADD + run_cmd_verify_hist {$server SADD s1 1 2 3 4 5} {db0_SET:4=1} + run_cmd_verify_hist {$server SADD s1 6 7 8} {db0_SET:8=1} + # Test also SADD, SREM, SMOVE, SPOP + run_cmd_verify_hist {$server SADD s2 1} {db0_SET:1=1,8=1} + run_cmd_verify_hist {$server SADD s2 2} {db0_SET:2=1,8=1} + run_cmd_verify_hist {$server SREM s2 3} {db0_SET:2=1,8=1} + run_cmd_verify_hist {$server SMOVE s2 s3 2} {db0_SET:1=2,8=1} + run_cmd_verify_hist {$server SPOP s3} {db0_SET:1=1,8=1} + run_cmd_verify_hist {$server SPOP s2} {db0_SET:8=1} + run_cmd_verify_hist {$server SPOP s1} {db0_SET:4=1} + run_cmd_verify_hist {$server del s1} {} + + # SDIFFSTORE + run_cmd_verify_hist {$server flushall} {} + run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1} + run_cmd_verify_hist {$server SADD s2 6 7 8 9 A B C D} {db0_SET:8=2} + run_cmd_verify_hist {$server SDIFFSTORE s3 s1 s2} {db0_SET:4=1,8=2} + #SINTERSTORE + run_cmd_verify_hist {$server flushall} {} + run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1} + run_cmd_verify_hist {$server SADD s2 6 7 8 9 A B C D} {db0_SET:8=2} + run_cmd_verify_hist {$server SINTERSTORE s3 s1 s2} {db0_SET:2=1,8=2} + #SUNIONSTORE + run_cmd_verify_hist {$server flushall} {} + run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1} + run_cmd_verify_hist {$server SADD s2 6 7 8 9 A B C D} {db0_SET:8=2} + run_cmd_verify_hist {$server SUNIONSTORE s3 s1 s2} {db0_SET:8=3} + run_cmd_verify_hist {$server SADD s4 E F G H} {db0_SET:4=1,8=3} + run_cmd_verify_hist {$server SUNIONSTORE s5 s3 s4} {db0_SET:4=1,8=3,16=1} + # DEL + run_cmd_verify_hist {$server flushall} {} + run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1} + run_cmd_verify_hist {$server DEL s1} {} + # EXPIRE + run_cmd_verify_hist {$server flushall} {} + run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1} + run_cmd_verify_hist {$server PEXPIRE s1 50} {db0_SET:8=1} + run_cmd_verify_hist {} {} 1 + # SET overwrites + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1} + run_cmd_verify_hist {$server SET s1 1234567} {db0_STR:4=1} + run_cmd_verify_hist {$server DEL s1} {} + } {} {cluster:skip} + + test "KEYSIZES - Test ZSET $suffixRepl" { + # ZADD, ZREM + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZADD z1 6 f 7 g 8 h 9 i} {db0_ZSET:8=1} + run_cmd_verify_hist {$server ZADD z2 1 a} {db0_ZSET:1=1,8=1} + run_cmd_verify_hist {$server ZREM z1 a} {db0_ZSET:1=1,8=1} + run_cmd_verify_hist {$server ZREM z1 b} {db0_ZSET:1=1,4=1} + # ZREMRANGEBYSCORE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZREMRANGEBYSCORE z1 -inf (2} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZREMRANGEBYSCORE z1 -inf (3} {db0_ZSET:2=1} + # ZREMRANGEBYRANK + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e 6 f} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZREMRANGEBYRANK z1 0 1} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZREMRANGEBYRANK z1 0 0} {db0_ZSET:2=1} + # ZREMRANGEBYLEX + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 0 a 0 b 0 c 0 d 0 e 0 f} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZREMRANGEBYLEX z1 - (d} {db0_ZSET:2=1} + # ZUNIONSTORE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZADD z2 6 f 7 g 8 h 9 i} {db0_ZSET:4=2} + run_cmd_verify_hist {$server ZUNIONSTORE z3 2 z1 z2} {db0_ZSET:4=2,8=1} + # ZINTERSTORE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZADD z2 3 c 4 d 5 e 6 f} {db0_ZSET:4=2} + run_cmd_verify_hist {$server ZINTERSTORE z3 2 z1 z2} {db0_ZSET:2=1,4=2} + # BZPOPMIN, BZPOPMAX + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server BZPOPMIN z1 0} {db0_ZSET:4=1} + run_cmd_verify_hist {$server BZPOPMAX z1 0} {db0_ZSET:2=1} + # ZDIFFSTORE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZADD z2 3 c 4 d 5 e 6 f} {db0_ZSET:4=2} + run_cmd_verify_hist {$server ZDIFFSTORE z3 2 z1 z2} {db0_ZSET:2=1,4=2} + # ZINTERSTORE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server ZADD z2 3 c 4 d 5 e 6 f} {db0_ZSET:4=2} + run_cmd_verify_hist {$server ZINTERSTORE z3 2 z1 z2} {db0_ZSET:2=1,4=2} + # DEL + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server DEL z1} {} + # EXPIRE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server PEXPIRE z1 50} {db0_ZSET:4=1} + run_cmd_verify_hist {} {} 1 + # SET overwrites + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1} + run_cmd_verify_hist {$server SET z1 1234567} {db0_STR:4=1} + run_cmd_verify_hist {$server DEL z1} {} + } {} {cluster:skip} + + test "KEYSIZES - Test STRING $suffixRepl" { + # SETRANGE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server SET s2 1234567890} {db0_STR:8=1} + run_cmd_verify_hist {$server SETRANGE s2 10 123456} {db0_STR:16=1} + # MSET, MSETNX + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server MSET s3 1 s4 2 s5 3} {db0_STR:1=3} + run_cmd_verify_hist {$server MSETNX s6 1 s7 2 s8 3} {db0_STR:1=6} + # DEL + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server SET s9 1234567890} {db0_STR:8=1} + run_cmd_verify_hist {$server DEL s9} {} + #EXPIRE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server SET s10 1234567890} {db0_STR:8=1} + run_cmd_verify_hist {$server PEXPIRE s10 50} {db0_STR:8=1} + run_cmd_verify_hist {} {} 1 + # SET (+overwrite) + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server SET s1 1024} {db0_STR:4=1} + run_cmd_verify_hist {$server SET s1 842} {db0_STR:2=1} + run_cmd_verify_hist {$server SET s1 2} {db0_STR:1=1} + run_cmd_verify_hist {$server SET s1 1234567} {db0_STR:4=1} + } {} {cluster:skip} + + foreach type {listpackex hashtable} { + # Test different implementations of hash tables and listpacks + if {$type eq "hashtable"} { + $server config set hash-max-listpack-entries 0 + } else { + $server config set hash-max-listpack-entries 512 + } + + test "KEYSIZES - Test HASH ($type) $suffixRepl" { + # HSETNX + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETNX h1 1 1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSETNX h1 2 2} {db0_HASH:2=1} + # HSET, HDEL + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSET h2 1 1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSET h2 2 2} {db0_HASH:2=1} + run_cmd_verify_hist {$server HDEL h2 1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HDEL h2 2} {} + # HMSET + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3} {db0_HASH:2=1} + run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3} {db0_HASH:2=1} + run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3 4 4} {db0_HASH:4=1} + + # HINCRBY + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server hincrby h1 f1 10} {db0_HASH:1=1} + run_cmd_verify_hist {$server hincrby h1 f1 10} {db0_HASH:1=1} + run_cmd_verify_hist {$server hincrby h1 f2 20} {db0_HASH:2=1} + # HINCRBYFLOAT + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server hincrbyfloat h1 f1 10.5} {db0_HASH:1=1} + run_cmd_verify_hist {$server hincrbyfloat h1 f1 10.5} {db0_HASH:1=1} + run_cmd_verify_hist {$server hincrbyfloat h1 f2 10.5} {db0_HASH:2=1} + # HEXPIRE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSET h1 f1 1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSET h1 f2 1} {db0_HASH:2=1} + run_cmd_verify_hist {$server HPEXPIREAT h1 1 FIELDS 1 f1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSET h1 f3 1} {db0_HASH:2=1} + run_cmd_verify_hist {$server HPEXPIRE h1 50 FIELDS 1 f2} {db0_HASH:2=1} + run_cmd_verify_hist {} {db0_HASH:1=1} 1 + run_cmd_verify_hist {$server HPEXPIRE h1 50 FIELDS 1 f3} {db0_HASH:1=1} + run_cmd_verify_hist {} {} 1 + } + } + + test "KEYSIZES - Test STRING BITS $suffixRepl" { + # BITOPS + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server SET b1 "x123456789"} {db0_STR:8=1} + run_cmd_verify_hist {$server SET b2 "x12345678"} {db0_STR:8=2} + run_cmd_verify_hist {$server BITOP AND b3 b1 b2} {db0_STR:8=3} + run_cmd_verify_hist {$server BITOP OR b4 b1 b2} {db0_STR:8=4} + run_cmd_verify_hist {$server BITOP XOR b5 b1 b2} {db0_STR:8=5} + # SETBIT + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server setbit b1 71 1} {db0_STR:8=1} + run_cmd_verify_hist {$server setbit b1 72 1} {db0_STR:8=1} + run_cmd_verify_hist {$server setbit b2 72 1} {db0_STR:8=2} + run_cmd_verify_hist {$server setbit b2 640 0} {db0_STR:8=1,64=1} + # BITFIELD + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server bitfield b3 set u8 6 255} {db0_STR:2=1} + run_cmd_verify_hist {$server bitfield b3 set u8 65 255} {db0_STR:8=1} + run_cmd_verify_hist {$server bitfield b4 set u8 1000 255} {db0_STR:8=1,64=1} + } {} {cluster:skip} + + test "KEYSIZES - Test RESTORE $suffixRepl" { + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH l10 1 2 3 4} {db0_LIST:4=1} + set encoded [$server dump l10] + run_cmd_verify_hist {$server del l10} {} + run_cmd_verify_hist {$server restore l11 0 $encoded} {db0_LIST:4=1} + } + + test "KEYSIZES - Test RENAME $suffixRepl" { + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH l12 1 2 3 4} {db0_LIST:4=1} + run_cmd_verify_hist {$server RENAME l12 l13} {db0_LIST:4=1} + } {} {cluster:skip} + + test "KEYSIZES - Test MOVE $suffixRepl" { + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH l1 1 2 3 4} {db0_LIST:4=1} + run_cmd_verify_hist {$server RPUSH l2 1} {db0_LIST:1=1,4=1} + run_cmd_verify_hist {$server MOVE l1 1} {db0_LIST:1=1 db1_LIST:4=1} + } {} {cluster:skip} + + test "KEYSIZES - Test SWAPDB $suffixRepl" { + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH l1 1 2 3 4} {db0_LIST:4=1} + $server select 1 + run_cmd_verify_hist {$server ZADD z1 1 A} {db0_LIST:4=1 db1_ZSET:1=1} + run_cmd_verify_hist {$server SWAPDB 0 1} {db0_ZSET:1=1 db1_LIST:4=1} + $server select 0 + } {OK} {singledb:skip} + + test "KEYSIZES - Test RDB $suffixRepl" { + run_cmd_verify_hist {$server FLUSHALL} {} + # Write list, set and zset to db0 + run_cmd_verify_hist {$server RPUSH l1 1 2 3 4} {db0_LIST:4=1} + run_cmd_verify_hist {$server SADD s1 1 2 3 4 5} {db0_LIST:4=1 db0_SET:4=1} + run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} + run_cmd_verify_hist {$server SAVE} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} + if {$replicaMode eq 1} { + run_cmd_verify_hist {restart_server -1 true false} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} + } else { + run_cmd_verify_hist {restart_server 0 true false} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} + } + } {} {external:skip} +} + +start_server {} { + # Test KEYSIZES on a single server + r select 0 + test_all_keysizes 0 + + # Start another server to test replication of KEYSIZES + start_server {tags {needs:repl external:skip}} { + # Set the outer layer server as primary + set primary [srv -1 client] + set primary_host [srv -1 host] + set primary_port [srv -1 port] + # Set this inner layer server as replica + set replica [srv 0 client] + + # Server should have role replica + $replica replicaof $primary_host $primary_port + wait_for_condition 50 100 { [s 0 role] eq {slave} } else { fail "Replication not started." } + + # Test KEYSIZES on leader and replica + $primary select 0 + test_all_keysizes 1 + } +} diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index f04af799a..130289aff 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -306,7 +306,7 @@ run_solo {defrag} { r set "{bigstream}smallitem" val - set expected_frag 1.7 + set expected_frag 1.5 if {$::accurate} { # scale the hash to 1m fields in order to have a measurable the latency for {set j 10000} {$j < 1000000} {incr j} { @@ -601,7 +601,7 @@ run_solo {defrag} { # create big keys with 10k items set rd [redis_deferring_client] - set expected_frag 1.7 + set expected_frag 1.5 # add a mass of list nodes to two lists (allocations are interlaced) set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation set elements 500000 diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl index 52d7c3e98..7ce70cda1 100644 --- a/tests/unit/other.tcl +++ b/tests/unit/other.tcl @@ -508,18 +508,25 @@ start_server {tags {"other external:skip"}} { test "Redis can resize empty dict" { # Write and then delete 128 keys, creating an empty dict r flushall + + # Add one key to the db just to create the dict and get its initial size + r set x 1 + set initial_size [dict get [r memory stats] db.9 overhead.hashtable.main] + + # Now add 128 keys and then delete them for {set j 1} {$j <= 128} {incr j} { r set $j{b} a } + for {set j 1} {$j <= 128} {incr j} { r del $j{b} } - # The dict containing 128 keys must have expanded, - # its hash table itself takes a lot more than 400 bytes + + # dict must have expanded. Verify it eventually shrinks back to its initial size. wait_for_condition 100 50 { - [dict get [r memory stats] db.9 overhead.hashtable.main] < 400 + [dict get [r memory stats] db.9 overhead.hashtable.main] == $initial_size } else { - fail "dict did not resize in time" - } + fail "dict did not resize in time to its initial size" + } } }