mirrors
/
redis
mirror of https://mirror.osredm.com/root/redis.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add KEYSIZES section to INFO (#13592) 

This PR adds a new section to the `INFO` command output, called
`keysizes`. This section provides detailed statistics on the
distribution of key sizes for each data type (strings, lists, sets,
hashes and zsets) within the dataset. The distribution is tracked using
a base-2 logarithmic histogram.

# Motivation
Currently, Redis lacks a built-in feature to track key sizes and item
sizes per data type at a granular level. Understanding the distribution
of key sizes is critical for monitoring memory usage and optimizing
performance, particularly in large datasets. This enhancement will allow
users to inspect the size distribution of keys directly from the `INFO`
command, assisting with performance analysis and capacity planning.

# Changes
New Section in `INFO` Command: A new section called `keysizes` has been
added to the `INFO` command output. This section reports a per-database,
per-type histogram of key sizes. It provides insights into how many keys
fall into specific size ranges (represented in powers of 2).

**Example output:**
```
127.0.0.1:6379> INFO keysizes
# Keysizes
db0_distrib_strings_sizes:1=19,2=655,512=100899,1K=31,2K=29,4K=23,8K=16,16K=3,32K=2
db0_distrib_lists_items:1=5784492,32=3558,64=1047,128=676,256=533,512=218,4K=1,8K=42
db0_distrib_sets_items:1=735564=50612,8=21462,64=1365,128=974,2K=292,4K=154,8K=89,
db0_distrib_hashes_items:2=1,4=544,32=141169,64=207329,128=4349,256=136226,1K=1
```
## Future Use Cases:
The key size distribution is collected per slot as well, laying the
groundwork for future enhancements related to Redis Cluster.
This commit is contained in:
Moti Cohen 2024-10-29 13:07:26 +02:00 committed by GitHub
parent 611c950293
commit 2ec78d262d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 836 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/bitops.c
View File

@ -489,22 +489,27 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) {
* bits to a string object. The command creates or pad with zeroes the string
* so that the 'maxbit' bit can be addressed. The object is finally
* returned. Otherwise if the key holds a wrong type NULL is returned and
* an error is sent to the client. */
robj *lookupStringForBitCommand(client *c, uint64_t maxbit, int *dirty) {
* an error is sent to the client.
*
* (Must provide all the arguments to the function)
*/
static robj *lookupStringForBitCommand(client *c, uint64_t maxbit,
size_t *strOldSize, size_t *strGrowSize)
{
size_t byte = maxbit >> 3;
robj *o = lookupKeyWrite(c->db,c->argv[1]);
if (checkType(c,o,OBJ_STRING)) return NULL;
if (dirty) *dirty = 0;
if (o == NULL) {
o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1));
dbAdd(c->db,c->argv[1],o);
if (dirty) *dirty = 1;
*strGrowSize = byte + 1;
*strOldSize = 0;
} else {
o = dbUnshareStringValue(c->db,c->argv[1],o);
size_t oldlen = sdslen(o->ptr);
*strOldSize = sdslen(o->ptr);
o->ptr = sdsgrowzero(o->ptr,byte+1);
if (dirty && oldlen != sdslen(o->ptr)) *dirty = 1;
*strGrowSize = sdslen(o->ptr) - *strOldSize;
}
return o;
}
@ -561,8 +566,9 @@ void setbitCommand(client *c) {
return;
}
int dirty;
if ((o = lookupStringForBitCommand(c,bitoffset,&dirty)) == NULL) return;
size_t strOldSize, strGrowSize;
if ((o = lookupStringForBitCommand(c,bitoffset,&strOldSize,&strGrowSize)) == NULL)
return;
/* Get current values */
byte = bitoffset >> 3;
@ -573,7 +579,7 @@ void setbitCommand(client *c) {
/* Either it is newly created, changed length, or the bit changes before and after.
* Note that the bitval here is actually a decimal number.
* So we need to use `!!` to convert it to 0 or 1 for comparison. */
if (dirty || (!!bitval != on)) {
if (strGrowSize || (!!bitval != on)) {
/* Update byte with new bit value. */
byteval &= ~(1 << bit);
byteval |= ((on & 0x1) << bit);
@ -581,6 +587,13 @@ void setbitCommand(client *c) {
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
server.dirty++;
/* If this is not a new key (old size not 0) and size changed, then
* update the keysizes histogram. Otherwise, the histogram already
* updated in lookupStringForBitCommand() by calling dbAdd(). */
if ((strOldSize > 0) && (strGrowSize != 0))
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING,
strOldSize, strOldSize + strGrowSize);
}
/* Return original value. */
@ -1065,7 +1078,8 @@ struct bitfieldOp {
void bitfieldGeneric(client *c, int flags) {
robj *o;
uint64_t bitoffset;
int j, numops = 0, changes = 0, dirty = 0;
int j, numops = 0, changes = 0;
size_t strOldSize, strGrowSize = 0;
struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */
int owtype = BFOVERFLOW_WRAP; /* Overflow type. */
int readonly = 1;
@ -1159,7 +1173,7 @@ void bitfieldGeneric(client *c, int flags) {
/* Lookup by making room up to the farthest bit reached by
* this operation. */
if ((o = lookupStringForBitCommand(c,
highest_write_offset,&dirty)) == NULL) {
highest_write_offset,&strOldSize,&strGrowSize)) == NULL) {
zfree(ops);
return;
}
@ -1209,7 +1223,7 @@ void bitfieldGeneric(client *c, int flags) {
setSignedBitfield(o->ptr,thisop->offset,
thisop->bits,newval);
if (dirty || (oldval != newval))
if (strGrowSize || (oldval != newval))
changes++;
} else {
addReplyNull(c);
@ -1243,7 +1257,7 @@ void bitfieldGeneric(client *c, int flags) {
setUnsignedBitfield(o->ptr,thisop->offset,
thisop->bits,newval);
if (dirty || (oldval != newval))
if (strGrowSize || (oldval != newval))
changes++;
} else {
addReplyNull(c);
@ -1286,6 +1300,14 @@ void bitfieldGeneric(client *c, int flags) {
}
if (changes) {
/* If this is not a new key (old size not 0) and size changed, then
* update the keysizes histogram. Otherwise, the histogram already
* updated in lookupStringForBitCommand() by calling dbAdd(). */
if ((strOldSize > 0) && (strGrowSize != 0))
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING,
strOldSize, strOldSize + strGrowSize);
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
server.dirty += changes;

58
src/db.c
View File

@ -21,6 +21,8 @@
* C-level DB API
*----------------------------------------------------------------------------*/
static_assert(MAX_KEYSIZES_TYPES == OBJ_TYPE_BASIC_MAX, "Must be equal");
/* Flags for expireIfNeeded */
#define EXPIRE_FORCE_DELETE_EXPIRED 1
#define EXPIRE_AVOID_DELETE_EXPIRED 2
@ -46,6 +48,48 @@ void updateLFU(robj *val) {
val->lru = (LFUGetTimeInMinutes()<<8) | counter;
}
/*
* Update histogram of keys-sizes
*
* It is used to track the distribution of key sizes in the dataset. It is updated
* every time key's length is modified. Available to user via INFO command.
*
* The histogram is a base-2 logarithmic histogram, with 64 bins. The i'th bin
* represents the number of keys with a size in the range 2^i and 2^(i+1)
* exclusive. oldLen/newLen must be smaller than 2^48, and if their value
* equals 0, it means that the key is being created/deleted, respectively. Each
* data type has its own histogram and it is per database (In addition, there is
* histogram per slot for future cluster use).
*
* Examples to LEN values and corresponding bins in histogram:
* [1,2)->0 [2,4)->1 [4,8)->2 [8,16)->3
*/
void updateKeysizesHist(redisDb *db, int didx, uint32_t type, uint64_t oldLen, uint64_t newLen) {
if(unlikely(type >= OBJ_TYPE_BASIC_MAX))
return;
kvstoreDictMetadata *dictMeta = kvstoreGetDictMetadata(db->keys, didx);
kvstoreMetadata *kvstoreMeta = kvstoreGetMetadata(db->keys);
if (oldLen != 0) {
int old_bin = log2ceil(oldLen);
debugServerAssertWithInfo(server.current_client, NULL, old_bin < MAX_KEYSIZES_BINS);
/* If following a key deletion it is last one in slot's dict, then
* slot's dict might get released as well. Verify if metadata is not NULL. */
if(dictMeta) dictMeta->keysizes_hist[type][old_bin]--;
kvstoreMeta->keysizes_hist[type][old_bin]--;
}
if (newLen != 0) {
int new_bin = log2ceil(newLen);
debugServerAssertWithInfo(server.current_client, NULL, new_bin < MAX_KEYSIZES_BINS);
/* If following a key deletion it is last one in slot's dict, then
* slot's dict might get released as well. Verify if metadata is not NULL. */
if(dictMeta) dictMeta->keysizes_hist[type][new_bin]++;
kvstoreMeta->keysizes_hist[type][new_bin]++;
}
}
/* Lookup a key for read or write operations, or return NULL if the key is not
* found in the specified DB. This function implements the functionality of
* lookupKeyRead(), lookupKeyWrite() and their ...WithFlags() variants.
@ -205,6 +249,7 @@ static dictEntry *dbAddInternal(redisDb *db, robj *key, robj *val, int update_if
kvstoreDictSetVal(db->keys, slot, de, val);
signalKeyAsReady(db, key, val->type);
notifyKeyspaceEvent(NOTIFY_NEW,"new",key,db->id);
updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); /* add hist */
return de;
}
@ -250,6 +295,7 @@ int dbAddRDBLoad(redisDb *db, sds key, robj *val) {
int slot = getKeySlot(key);
dictEntry *de = kvstoreDictAddRaw(db->keys, slot, key, NULL);
if (de == NULL) return 0;
updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); /* add hist */
initObjectLRUOrLFU(val);
kvstoreDictSetVal(db->keys, slot, de, val);
return 1;
@ -273,6 +319,9 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt
serverAssertWithInfo(NULL,key,de != NULL);
robj *old = dictGetVal(de);
/* Remove old key from keysizes histogram */
updateKeysizesHist(db, slot, old->type, getObjectLength(old), 0); /* remove hist */
val->lru = old->lru;
if (overwrite) {
@ -291,6 +340,9 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt
}
kvstoreDictSetVal(db->keys, slot, de, val);
/* Add new key to keysizes histogram */
updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val));
/* if hash with HFEs, take care to remove from global HFE DS */
if (old->type == OBJ_HASH)
hashTypeRemoveFromExpires(&db->hexpires, old);
@ -404,6 +456,9 @@ int dbGenericDelete(redisDb *db, robj *key, int async, int flags) {
if (de) {
robj *val = dictGetVal(de);
/* remove key from histogram */
updateKeysizesHist(db, slot, val->type, getObjectLength(val), 0);
/* If hash object with expiry on fields, remove it from HFE DS of DB */
if (val->type == OBJ_HASH)
hashTypeRemoveFromExpires(&db->hexpires, val);
@ -599,7 +654,8 @@ redisDb *initTempDb(void) {
redisDb *tempDb = zcalloc(sizeof(redisDb)*server.dbnum);
for (int i=0; i<server.dbnum; i++) {
tempDb[i].id = i;
tempDb[i].keys = kvstoreCreate(&dbDictType, slot_count_bits, flags);
tempDb[i].keys = kvstoreCreate(&dbDictType, slot_count_bits,
flags | KVSTORE_ALLOC_META_KEYS_HIST);
tempDb[i].expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags);
tempDb[i].hexpires = ebCreate();
}

73
src/kvstore.c
View File

@ -42,6 +42,7 @@ struct _kvstore {
unsigned long long *dict_size_index; /* Binary indexed tree (BIT) that describes cumulative key frequencies up until given dict-index. */
size_t overhead_hashtable_lut; /* The overhead of all dictionaries. */
size_t overhead_hashtable_rehashing; /* The overhead of dictionaries rehashing. */
void *metadata[]; /* conditionally allocated based on "flags" */
};
/* Structure for kvstore iterator that allows iterating across multiple dicts. */
@ -59,10 +60,17 @@ struct _kvstoreDictIterator {
dictIterator di;
};
/* Dict metadata for database, used for record the position in rehashing list. */
/* Basic metadata allocated per dict */
typedef struct {
listNode *rehashing_node; /* list node in rehashing list */
} kvstoreDictMetadata;
} kvstoreDictMetaBase;
/* Conditionally metadata allocated per dict (specifically for keysizes histogram) */
typedef struct {
kvstoreDictMetaBase base; /* must be first in struct ! */
/* External metadata */
kvstoreDictMetadata meta;
} kvstoreDictMetaEx;
/**********************************/
/*** Helpers **********************/
@ -184,7 +192,7 @@ static void freeDictIfNeeded(kvstore *kvs, int didx) {
* If there's one dict, bucket count can be retrieved directly from single dict bucket. */
static void kvstoreDictRehashingStarted(dict *d) {
kvstore *kvs = d->type->userdata;
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d);
kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d);
listAddNodeTail(kvs->rehashing, d);
metadata->rehashing_node = listLast(kvs->rehashing);
@ -201,7 +209,7 @@ static void kvstoreDictRehashingStarted(dict *d) {
* the old ht size of the dictionary from the total sum of buckets for a DB. */
static void kvstoreDictRehashingCompleted(dict *d) {
kvstore *kvs = d->type->userdata;
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d);
kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d);
if (metadata->rehashing_node) {
listDelNode(kvs->rehashing, metadata->rehashing_node);
metadata->rehashing_node = NULL;
@ -214,10 +222,15 @@ static void kvstoreDictRehashingCompleted(dict *d) {
kvs->overhead_hashtable_rehashing -= from;
}
/* Returns the size of the DB dict metadata in bytes. */
static size_t kvstoreDictMetadataSize(dict *d) {
/* Returns the size of the DB dict base metadata in bytes. */
static size_t kvstoreDictMetaBaseSize(dict *d) {
UNUSED(d);
return sizeof(kvstoreDictMetadata);
return sizeof(kvstoreDictMetaBase);
}
/* Returns the size of the DB dict extended metadata in bytes. */
static size_t kvstoreDictMetadataExtendSize(dict *d) {
UNUSED(d);
return sizeof(kvstoreDictMetaEx);
}
/**********************************/
@ -232,7 +245,13 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) {
* for the dict cursor, see kvstoreScan */
assert(num_dicts_bits <= 16);
kvstore *kvs = zcalloc(sizeof(*kvs));
/* Calc kvstore size */
size_t kvsize = sizeof(kvstore);
/* Conditionally calc also histogram size */
if (flags & KVSTORE_ALLOC_META_KEYS_HIST)
kvsize += sizeof(kvstoreMetadata);
kvstore *kvs = zcalloc(kvsize);
memcpy(&kvs->dtype, type, sizeof(kvs->dtype));
kvs->flags = flags;
@ -243,7 +262,10 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) {
assert(!type->rehashingStarted);
assert(!type->rehashingCompleted);
kvs->dtype.userdata = kvs;
kvs->dtype.dictMetadataBytes = kvstoreDictMetadataSize;
if (flags & KVSTORE_ALLOC_META_KEYS_HIST)
kvs->dtype.dictMetadataBytes = kvstoreDictMetadataExtendSize;
else
kvs->dtype.dictMetadataBytes = kvstoreDictMetaBaseSize;
kvs->dtype.rehashingStarted = kvstoreDictRehashingStarted;
kvs->dtype.rehashingCompleted = kvstoreDictRehashingCompleted;
@ -263,7 +285,6 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) {
kvs->bucket_count = 0;
kvs->overhead_hashtable_lut = 0;
kvs->overhead_hashtable_rehashing = 0;
return kvs;
}
@ -272,9 +293,13 @@ void kvstoreEmpty(kvstore *kvs, void(callback)(dict*)) {
dict *d = kvstoreGetDict(kvs, didx);
if (!d)
continue;
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d);
kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d);
if (metadata->rehashing_node)
metadata->rehashing_node = NULL;
if (kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST) {
kvstoreDictMetaEx *metaExt = (kvstoreDictMetaEx *) metadata;
memset(&metaExt->meta.keysizes_hist, 0, sizeof(metaExt->meta.keysizes_hist));
}
dictEmpty(d, callback);
freeDictIfNeeded(kvs, didx);
}
@ -296,7 +321,7 @@ void kvstoreRelease(kvstore *kvs) {
dict *d = kvstoreGetDict(kvs, didx);
if (!d)
continue;
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d);
kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d);
if (metadata->rehashing_node)
metadata->rehashing_node = NULL;
dictRelease(d);
@ -330,11 +355,15 @@ unsigned long kvstoreBuckets(kvstore *kvs) {
size_t kvstoreMemUsage(kvstore *kvs) {
size_t mem = sizeof(*kvs);
size_t metaSize = sizeof(kvstoreDictMetaBase);
if (kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST)
metaSize = sizeof(kvstoreDictMetaEx);
unsigned long long keys_count = kvstoreSize(kvs);
mem += keys_count * dictEntryMemUsage() +
kvstoreBuckets(kvs) * sizeof(dictEntry*) +
kvs->allocated_dicts * (sizeof(dict) + kvstoreDictMetadataSize(NULL));
kvs->allocated_dicts * (sizeof(dict) + metaSize);
/* Values are dict* shared with kvs->dicts */
mem += listLength(kvs->rehashing) * sizeof(listNode);
@ -785,7 +814,7 @@ void kvstoreDictLUTDefrag(kvstore *kvs, kvstoreDictLUTDefragFunction *defragfn)
/* After defragmenting the dict, update its corresponding
* rehashing node in the kvstore's rehashing list. */
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(*d);
kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(*d);
if (metadata->rehashing_node)
metadata->rehashing_node->value = *d;
}
@ -856,6 +885,19 @@ int kvstoreDictDelete(kvstore *kvs, int didx, const void *key) {
return ret;
}
kvstoreDictMetadata *kvstoreGetDictMetadata(kvstore *kvs, int didx) {
dict *d = kvstoreGetDict(kvs, didx);
if ((!d) || (!(kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST)))
return NULL;
kvstoreDictMetaEx *metadata = (kvstoreDictMetaEx *)dictMetadata(d);
return &(metadata->meta);
}
kvstoreMetadata *kvstoreGetMetadata(kvstore *kvs) {
return (kvstoreMetadata *) &kvs->metadata;
}
#ifdef REDIS_TEST
#include <stdio.h>
#include "testhelp.h"
@ -1029,7 +1071,8 @@ int kvstoreTest(int argc, char **argv, int flags) {
}
TEST("Verify non-empty dict count is correctly updated") {
kvstore *kvs = kvstoreCreate(&KvstoreDictTestType, 2, KVSTORE_ALLOCATE_DICTS_ON_DEMAND);
kvstore *kvs = kvstoreCreate(&KvstoreDictTestType, 2,
KVSTORE_ALLOCATE_DICTS_ON_DEMAND | KVSTORE_ALLOC_META_KEYS_HIST);
for (int idx = 0; idx < 4; idx++) {
for (i = 0; i < 16; i++) {
de = kvstoreDictAddRaw(kvs, idx, stringFromInt(i), NULL);

18
src/kvstore.h
View File

@ -4,6 +4,21 @@
#include "dict.h"
#include "adlist.h"
/* maximum number of bins of keysizes histogram */
#define MAX_KEYSIZES_BINS 48
#define MAX_KEYSIZES_TYPES 5 /* static_assert at db.c verifies == OBJ_TYPE_BASIC_MAX */
/* When creating kvstore with flag `KVSTORE_ALLOC_META_KEYS_HIST`, then kvstore
* alloc and memset struct kvstoreMetadata on init, yet, managed outside kvstore */
typedef struct {
uint64_t keysizes_hist[MAX_KEYSIZES_TYPES][MAX_KEYSIZES_BINS];
} kvstoreMetadata;
/* Like kvstoreMetadata, this one per dict */
typedef struct {
uint64_t keysizes_hist[MAX_KEYSIZES_TYPES][MAX_KEYSIZES_BINS];
} kvstoreDictMetadata;
typedef struct _kvstore kvstore;
typedef struct _kvstoreIterator kvstoreIterator;
typedef struct _kvstoreDictIterator kvstoreDictIterator;
@ -13,6 +28,7 @@ typedef int (kvstoreExpandShouldSkipDictIndex)(int didx);
#define KVSTORE_ALLOCATE_DICTS_ON_DEMAND (1<<0)
#define KVSTORE_FREE_EMPTY_DICTS (1<<1)
#define KVSTORE_ALLOC_META_KEYS_HIST (1<<2) /* Alloc keysizes histogram */
kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags);
void kvstoreEmpty(kvstore *kvs, void(callback)(dict*));
void kvstoreRelease(kvstore *kvs);
@ -71,6 +87,8 @@ void kvstoreDictSetVal(kvstore *kvs, int didx, dictEntry *de, void *val);
dictEntry *kvstoreDictTwoPhaseUnlinkFind(kvstore *kvs, int didx, const void *key, dictEntry ***plink, int *table_index);
void kvstoreDictTwoPhaseUnlinkFree(kvstore *kvs, int didx, dictEntry *he, dictEntry **plink, int table_index);
int kvstoreDictDelete(kvstore *kvs, int didx, const void *key);
kvstoreDictMetadata *kvstoreGetDictMetadata(kvstore *kvs, int didx);
kvstoreMetadata *kvstoreGetMetadata(kvstore *kvs);
#ifdef REDIS_TEST
int kvstoreTest(int argc, char *argv[], int flags);

2
src/lazyfree.c
View File

@ -207,7 +207,7 @@ void emptyDbAsync(redisDb *db) {
}
kvstore *oldkeys = db->keys, *oldexpires = db->expires;
ebuckets oldHfe = db->hexpires;
db->keys = kvstoreCreate(&dbDictType, slot_count_bits, flags);
db->keys = kvstoreCreate(&dbDictType, slot_count_bits, flags | KVSTORE_ALLOC_META_KEYS_HIST);
db->expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags);
db->hexpires = ebCreate();
atomicIncr(lazyfree_objects, kvstoreSize(oldkeys));

10
src/module.c
View File

@ -4171,15 +4171,7 @@ int RM_KeyType(RedisModuleKey *key) {
* If the key pointer is NULL or the key is empty, zero is returned. */
size_t RM_ValueLength(RedisModuleKey *key) {
if (key == NULL || key->value == NULL) return 0;
switch(key->value->type) {
case OBJ_STRING: return stringObjectLen(key->value);
case OBJ_LIST: return listTypeLength(key->value);
case OBJ_SET: return setTypeSize(key->value);
case OBJ_ZSET: return zsetLength(key->value);
case OBJ_HASH: return hashTypeLength(key->value, 0); /* OPEN: To subtract expired fields? */
case OBJ_STREAM: return streamLength(key->value);
default: return 0;
}
return getObjectLength(key->value);
}
/* If the key is open for writing, remove it, and setup the key to

12
src/object.c
View File

@ -680,6 +680,18 @@ robj *tryObjectEncoding(robj *o) {
return tryObjectEncodingEx(o, 1);
}
size_t getObjectLength(robj *o) {
switch(o->type) {
case OBJ_STRING: return stringObjectLen(o);
case OBJ_LIST: return listTypeLength(o);
case OBJ_SET: return setTypeSize(o);
case OBJ_ZSET: return zsetLength(o);
case OBJ_HASH: return hashTypeLength(o, 0);
case OBJ_STREAM: return streamLength(o);
default: return 0;
}
}
/* Get a decoded version of an encoded object (returned as a new object).
* If the object is already raw-encoded just increment the ref count. */
robj *getDecodedObject(robj *o) {

58
src/server.c
View File

@ -2690,7 +2690,7 @@ void initServer(void) {
flags |= KVSTORE_FREE_EMPTY_DICTS;
}
for (j = 0; j < server.dbnum; j++) {
server.db[j].keys = kvstoreCreate(&dbDictType, slot_count_bits, flags);
server.db[j].keys = kvstoreCreate(&dbDictType, slot_count_bits, flags | KVSTORE_ALLOC_META_KEYS_HIST);
server.db[j].expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags);
server.db[j].hexpires = ebCreate();
server.db[j].expires_cursor = 0;
@ -5521,7 +5521,7 @@ void releaseInfoSectionDict(dict *sec) {
dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, int *out_everything) {
char *default_sections[] = {
"server", "clients", "memory", "persistence", "stats", "replication",
"cpu", "module_list", "errorstats", "cluster", "keyspace", NULL};
"cpu", "module_list", "errorstats", "cluster", "keyspace", "keysizes", NULL};
if (!defaults)
defaults = default_sections;
@ -6149,6 +6149,60 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
}
}
/* keysizes */
if (all_sections || (dictFind(section_dict,"keysizes") != NULL)) {
if (sections++) info = sdscat(info,"\r\n");
info = sdscatprintf(info, "# Keysizes\r\n");
char *typestr[] = {
[OBJ_STRING] = "distrib_strings_sizes",
[OBJ_LIST] = "distrib_lists_items",
[OBJ_SET] = "distrib_sets_items",
[OBJ_ZSET] = "distrib_zsets_items",
[OBJ_HASH] = "distrib_hashes_items"
};
serverAssert(sizeof(typestr)/sizeof(typestr[0]) == OBJ_TYPE_BASIC_MAX);
for (int dbnum = 0; dbnum < server.dbnum; dbnum++) {
char *expSizeLabels[] = {
"1", "2", "4", "8", "16", "32", "64", "128", "256", "512", /* Byte */
"1K", "2K", "4K", "8K", "16K", "32K", "64K", "128K", "256K", "512K", /* Kilo */
"1M", "2M", "4M", "8M", "16M", "32M", "64M", "128M", "256M", "512M", /* Mega */
"1G", "2G", "4G", "8G", "16G", "32G", "64G", "128G", "256G", "512G", /* Giga */
"1T", "2T", "4T", "8T", "16T", "32T", "64T", "128T", "256T", "512T", /* Tera */
"1P", "2P", "4P", "8P", "16P", "32P", "64P", "128P", "256P", "512P", /* Peta */
"1E", "2E", "4E", "8E" /* Exa */
};
if (kvstoreSize(server.db[dbnum].keys) == 0)
continue;
for (int type = 0; type < OBJ_TYPE_BASIC_MAX; type++) {
uint64_t *kvstoreHist = kvstoreGetMetadata(server.db[dbnum].keys)->keysizes_hist[type];
char buf[10000];
int cnt = 0, buflen = 0;
/* Print histogram to temp buf[]. First bin is garbage */
buflen += snprintf(buf + buflen, sizeof(buf) - buflen, "db%d_%s:", dbnum, typestr[type]);
for (int i = 0; i < MAX_KEYSIZES_BINS; i++) {
if (kvstoreHist[i] == 0)
continue;
int res = snprintf(buf + buflen, sizeof(buf) - buflen,
(cnt == 0) ? "%s=%llu" : ",%s=%llu",
expSizeLabels[i], (unsigned long long) kvstoreHist[i]);
if (res < 0) break;
buflen += res;
cnt += kvstoreHist[i];
}
/* Print the temp buf[] to the info string */
if (cnt) info = sdscatprintf(info, "%s\r\n", buf);
}
}
}
/* Get info from modules.
* Returned when the user asked for "everything", "modules", or a specific module section.
* We're not aware of the module section names here, and we rather avoid the search when we can.

9
src/server.h
View File

@ -41,10 +41,6 @@
#include <systemd/sd-daemon.h>
#endif
#ifndef static_assert
#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1]
#endif
typedef long long mstime_t; /* millisecond time type. */
typedef long long ustime_t; /* microsecond time type. */
@ -698,6 +694,7 @@ typedef enum {
#define OBJ_SET 2 /* Set object. */
#define OBJ_ZSET 3 /* Sorted set object. */
#define OBJ_HASH 4 /* Hash object. */
#define OBJ_TYPE_BASIC_MAX 5 /* Max number of basic object types. */
/* The "module" object type is a special one that signals that the object
* is one directly managed by a Redis module. In this case the value points
@ -969,7 +966,7 @@ typedef struct replBufBlock {
* by integers from 0 (the default database) up to the max configured
* database. The database number is the 'id' field in the structure. */
typedef struct redisDb {
kvstore *keys; /* The keyspace for this DB */
kvstore *keys; /* The keyspace for this DB. As metadata, holds keysizes histogram */
kvstore *expires; /* Timeout of keys with a timeout set */
ebuckets hexpires; /* Hash expiration DS. Single TTL per hash (of next min field to expire) */
dict *blocking_keys; /* Keys with clients waiting for data (BLPOP)*/
@ -2799,6 +2796,7 @@ int isSdsRepresentableAsLongLong(sds s, long long *llval);
int isObjectRepresentableAsLongLong(robj *o, long long *llongval);
robj *tryObjectEncoding(robj *o);
robj *tryObjectEncodingEx(robj *o, int try_trim);
size_t getObjectLength(robj *o);
robj *getDecodedObject(robj *o);
size_t stringObjectLen(robj *o);
robj *createStringObjectFromLongLong(long long value);
@ -3363,6 +3361,7 @@ long long getModuleNumericConfig(ModuleConfig *module_config);
int setModuleNumericConfig(ModuleConfig *config, long long val, const char **err);
/* db.c -- Keyspace access API */
void updateKeysizesHist(redisDb *db, int didx, uint32_t type, uint64_t oldLen, uint64_t newLen);
int removeExpire(redisDb *db, robj *key);
void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj);
void deleteEvictedKeyAndPropagate(redisDb *db, robj *keyobj, long long *key_mem_freed);

34
src/t_hash.c
View File

@ -422,8 +422,13 @@ void listpackExExpire(redisDb *db, robj *o, ExpireInfo *info) {
expired++;
}
if (expired)
if (expired) {
lpt->lp = lpDeleteRange(lpt->lp, 0, expired * 3);
/* update keysizes */
unsigned long l = lpLength(lpt->lp) / 3;
updateKeysizesHist(db, getKeySlot(lpt->key), OBJ_HASH, l + expired, l);
}
min = hashTypeGetMinExpire(o, 1 /*accurate*/);
info->nextExpireTime = min;
@ -546,6 +551,11 @@ SetExRes hashTypeSetExpiryListpack(HashTypeSetEx *ex, sds field,
if (unlikely(checkAlreadyExpired(expireAt))) {
propagateHashFieldDeletion(ex->db, ex->key->ptr, field, sdslen(field));
hashTypeDelete(ex->hashObj, field, 1);
/* get listpack length */
listpackEx *lpt = ((listpackEx *) ex->hashObj->ptr);
unsigned long length = lpLength(lpt->lp) / 3;
updateKeysizesHist(ex->db, getKeySlot(ex->key->ptr), OBJ_HASH, length+1, length);
server.stat_expired_subkeys++;
ex->fieldDeleted++;
return HSETEX_DELETED;
@ -1042,6 +1052,8 @@ SetExRes hashTypeSetExpiryHT(HashTypeSetEx *exInfo, sds field, uint64_t expireAt
/* If expired, then delete the field and propagate the deletion.
* If replica, continue like the field is valid */
if (unlikely(checkAlreadyExpired(expireAt))) {
unsigned long length = dictSize(ht);
updateKeysizesHist(exInfo->db, getKeySlot(exInfo->key->ptr), OBJ_HASH, length, length-1);
/* replicas should not initiate deletion of fields */
propagateHashFieldDeletion(exInfo->db, exInfo->key->ptr, field, sdslen(field));
hashTypeDelete(exInfo->hashObj, field, 1);
@ -2132,6 +2144,7 @@ ebuckets *hashTypeGetDictMetaHFE(dict *d) {
*----------------------------------------------------------------------------*/
void hsetnxCommand(client *c) {
unsigned long hlen;
int isHashDeleted;
robj *o;
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
@ -2152,6 +2165,8 @@ void hsetnxCommand(client *c) {
addReply(c, shared.cone);
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
hlen = hashTypeLength(o, 0);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, hlen - 1, hlen);
server.dirty++;
}
@ -2180,6 +2195,8 @@ void hsetCommand(client *c) {
addReply(c, shared.ok);
}
signalModifiedKey(c,c->db,c->argv[1]);
unsigned long l = hashTypeLength(o, 0);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, l - created, l);
notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
server.dirty += (c->argc - 2)/2;
}
@ -2205,11 +2222,14 @@ void hincrbyCommand(client *c) {
} /* Else hashTypeGetValue() already stored it into &value */
} else if ((res == GETF_NOT_FOUND) || (res == GETF_EXPIRED)) {
value = 0;
unsigned long l = hashTypeLength(o, 0);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, l, l + 1);
} else {
/* Field expired and in turn hash deleted. Create new one! */
o = createHashObject();
dbAdd(c->db,c->argv[1],o);
value = 0;
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, 0, 1);
}
oldvalue = value;
@ -2254,11 +2274,14 @@ void hincrbyfloatCommand(client *c) {
}
} else if ((res == GETF_NOT_FOUND) || (res == GETF_EXPIRED)) {
value = 0;
unsigned long l = hashTypeLength(o, 0);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, l, l + 1);
} else {
/* Field expired and in turn hash deleted. Create new one! */
o = createHashObject();
dbAdd(c->db,c->argv[1],o);
value = 0;
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, 0, 1);
}
value += incr;
@ -2356,6 +2379,8 @@ void hdelCommand(client *c) {
if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,o,OBJ_HASH)) return;
unsigned long oldLen = hashTypeLength(o, 0);
/* Hash field expiration is optimized to avoid frequent update global HFE DS for
* each field deletion. Eventually active-expiration will run and update or remove
* the hash from global HFE DS gracefully. Nevertheless, statistic "subexpiry"
@ -2375,6 +2400,8 @@ void hdelCommand(client *c) {
}
}
if (deleted) {
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, oldLen, oldLen - deleted);
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hdel",c->argv[1],c->db->id);
if (keyremoved) {
@ -2943,6 +2970,11 @@ static ExpireAction onFieldExpire(eItem item, void *ctx) {
dict *d = expCtx->hashObj->ptr;
dictExpireMetadata *dictExpireMeta = (dictExpireMetadata *) dictMetadata(d);
propagateHashFieldDeletion(expCtx->db, dictExpireMeta->key, hf, hfieldlen(hf));
/* update keysizes */
unsigned long l = hashTypeLength(expCtx->hashObj, 0);
updateKeysizesHist(expCtx->db, getKeySlot(dictExpireMeta->key), OBJ_HASH, l, l - 1);
serverAssert(hashTypeDelete(expCtx->hashObj, hf, 0) == 1);
server.stat_expired_subkeys++;
return ACT_REMOVE_EXP_ITEM;

28
src/t_list.c
View File

@ -7,6 +7,7 @@
*/
#include "server.h"
#include "util.h"
/*-----------------------------------------------------------------------------
* List API
@ -462,6 +463,7 @@ void listTypeDelRange(robj *subject, long start, long count) {
/* Implements LPUSH/RPUSH/LPUSHX/RPUSHX.
* 'xx': push if key exists. */
void pushGenericCommand(client *c, int where, int xx) {
unsigned long llen;
int j;
robj *lobj = lookupKeyWrite(c->db, c->argv[1]);
@ -482,11 +484,13 @@ void pushGenericCommand(client *c, int where, int xx) {
server.dirty++;
}
addReplyLongLong(c, listTypeLength(lobj));
llen = listTypeLength(lobj);
addReplyLongLong(c, llen);
char *event = (where == LIST_HEAD) ? "lpush" : "rpush";
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, llen - (c->argc - 2), llen);
}
/* LPUSH <key> <element> [<element> ...] */
@ -553,6 +557,8 @@ void linsertCommand(client *c) {
notifyKeyspaceEvent(NOTIFY_LIST,"linsert",
c->argv[1],c->db->id);
server.dirty++;
unsigned long ll = listTypeLength(subject);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, ll-1, ll);
} else {
/* Notify client of a failed insert */
addReplyLongLong(c,-1);
@ -736,9 +742,11 @@ void addListRangeReply(client *c, robj *o, long start, long end, int reverse) {
* if the key got deleted by this function. */
void listElementsRemoved(client *c, robj *key, int where, robj *o, long count, int signal, int *deleted) {
char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
unsigned long llen = listTypeLength(o);
notifyKeyspaceEvent(NOTIFY_LIST, event, key, c->db->id);
if (listTypeLength(o) == 0) {
updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_LIST, llen + count, llen);
if (llen == 0) {
if (deleted) *deleted = 1;
dbDelete(c->db, key);
@ -870,7 +878,7 @@ void lrangeCommand(client *c) {
/* LTRIM <key> <start> <stop> */
void ltrimCommand(client *c) {
robj *o;
long start, end, llen, ltrim, rtrim;
long start, end, llen, ltrim, rtrim, llenNew;;
if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) ||
(getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return;
@ -908,12 +916,13 @@ void ltrimCommand(client *c) {
}
notifyKeyspaceEvent(NOTIFY_LIST,"ltrim",c->argv[1],c->db->id);
if (listTypeLength(o) == 0) {
if ((llenNew = listTypeLength(o)) == 0) {
dbDelete(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
} else {
listTypeTryConversion(o,LIST_CONV_SHRINKING,NULL,NULL);
}
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, llen, llenNew);
signalModifiedKey(c,c->db,c->argv[1]);
server.dirty += (ltrim + rtrim);
addReply(c,shared.ok);
@ -1066,8 +1075,11 @@ void lremCommand(client *c) {
listTypeReleaseIterator(li);
if (removed) {
long ll = listTypeLength(subject);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, ll + removed, ll);
notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id);
if (listTypeLength(subject) == 0) {
if (ll == 0) {
dbDelete(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
} else {
@ -1089,6 +1101,10 @@ void lmoveHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value,
listTypeTryConversionAppend(dstobj,&value,0,0,NULL,NULL);
listTypePush(dstobj,value,where);
signalModifiedKey(c,c->db,dstkey);
long ll = listTypeLength(dstobj);
updateKeysizesHist(c->db, getKeySlot(dstkey->ptr), OBJ_LIST, ll - 1, ll);
notifyKeyspaceEvent(NOTIFY_LIST,
where == LIST_HEAD ? "lpush" : "rpush",
dstkey,

24
src/t_set.c
View File

@ -603,6 +603,8 @@ void saddCommand(client *c) {
if (setTypeAdd(set,c->argv[j]->ptr)) added++;
}
if (added) {
unsigned long size = setTypeSize(set);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, size - added, size);
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[1],c->db->id);
}
@ -617,6 +619,8 @@ void sremCommand(client *c) {
if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,set,OBJ_SET)) return;
unsigned long oldSize = setTypeSize(set);
for (j = 2; j < c->argc; j++) {
if (setTypeRemove(set,c->argv[j]->ptr)) {
deleted++;
@ -628,6 +632,8 @@ void sremCommand(client *c) {
}
}
if (deleted) {
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, oldSize, oldSize - deleted);
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id);
if (keyremoved)
@ -669,8 +675,12 @@ void smoveCommand(client *c) {
}
notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id);
/* Update keysizes histogram */
unsigned long srcLen = setTypeSize(srcset);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, srcLen + 1, srcLen);
/* Remove the src set from the database when empty */
if (setTypeSize(srcset) == 0) {
if (srcLen == 0) {
dbDelete(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
}
@ -686,6 +696,8 @@ void smoveCommand(client *c) {
/* An extra key has changed when ele was successfully added to dstset */
if (setTypeAdd(dstset,ele->ptr)) {
unsigned long dstLen = setTypeSize(dstset);
updateKeysizesHist(c->db, getKeySlot(c->argv[2]->ptr), OBJ_SET, dstLen - 1, dstLen);
server.dirty++;
signalModifiedKey(c,c->db,c->argv[2]);
notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[2],c->db->id);
@ -743,7 +755,7 @@ void scardCommand(client *c) {
void spopWithCountCommand(client *c) {
long l;
unsigned long count, size;
unsigned long count, size, toRemove;
robj *set;
/* Get the count argument */
@ -763,10 +775,12 @@ void spopWithCountCommand(client *c) {
}
size = setTypeSize(set);
toRemove = (count >= size) ? size : count;
/* Generate an SPOP keyspace notification */
notifyKeyspaceEvent(NOTIFY_SET,"spop",c->argv[1],c->db->id);
server.dirty += (count >= size) ? size : count;
server.dirty += toRemove;
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, size, size - toRemove);
/* CASE 1:
* The number of requested elements is greater than or equal to
@ -949,6 +963,7 @@ void spopWithCountCommand(client *c) {
}
void spopCommand(client *c) {
unsigned long size;
robj *set, *ele;
if (c->argc == 3) {
@ -964,6 +979,9 @@ void spopCommand(client *c) {
if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
== NULL || checkType(c,set,OBJ_SET)) return;
size = setTypeSize(set);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, size, size-1);
/* Pop a random element from the set */
ele = setTypePopRandom(set);

19
src/t_string.c
View File

@ -420,6 +420,7 @@ void getsetCommand(client *c) {
}
void setrangeCommand(client *c) {
size_t oldLen = 0, newLen;
robj *o;
long offset;
sds value = c->argv[3]->ptr;
@ -449,16 +450,14 @@ void setrangeCommand(client *c) {
o = createObject(OBJ_STRING,sdsnewlen(NULL, offset+value_len));
dbAdd(c->db,c->argv[1],o);
} else {
size_t olen;
/* Key exists, check type */
if (checkType(c,o,OBJ_STRING))
return;
/* Return existing string length when setting nothing */
olen = stringObjectLen(o);
oldLen = stringObjectLen(o);
if (value_len == 0) {
addReplyLongLong(c,olen);
addReplyLongLong(c, oldLen);
return;
}
@ -478,7 +477,10 @@ void setrangeCommand(client *c) {
"setrange",c->argv[1],c->db->id);
server.dirty++;
}
addReplyLongLong(c,sdslen(o->ptr));
newLen = sdslen(o->ptr);
updateKeysizesHist(c->db,getKeySlot(c->argv[1]->ptr),OBJ_STRING,oldLen,newLen);
addReplyLongLong(c,newLen);
}
void getrangeCommand(client *c) {
@ -669,7 +671,7 @@ void incrbyfloatCommand(client *c) {
}
void appendCommand(client *c) {
size_t totlen;
size_t totlen, append_len;
robj *o, *append;
dictEntry *de;
@ -679,7 +681,7 @@ void appendCommand(client *c) {
c->argv[2] = tryObjectEncoding(c->argv[2]);
dbAdd(c->db,c->argv[1],c->argv[2]);
incrRefCount(c->argv[2]);
totlen = stringObjectLen(c->argv[2]);
append_len = totlen = stringObjectLen(c->argv[2]);
} else {
/* Key exists, check type */
if (checkType(c,o,OBJ_STRING))
@ -687,7 +689,7 @@ void appendCommand(client *c) {
/* "append" is an argument, so always an sds */
append = c->argv[2];
const size_t append_len = sdslen(append->ptr);
append_len = sdslen(append->ptr);
if (checkStringLength(c,stringObjectLen(o),append_len) != C_OK)
return;
@ -699,6 +701,7 @@ void appendCommand(client *c) {
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STRING,"append",c->argv[1],c->db->id);
server.dirty++;
updateKeysizesHist(c->db,getKeySlot(c->argv[1]->ptr),OBJ_STRING, totlen - append_len, totlen);
addReplyLongLong(c,totlen);
}

23
src/t_zset.c
View File

@ -1843,6 +1843,7 @@ void zaddGenericCommand(client *c, int flags) {
zsetTypeMaybeConvert(zobj, elements);
}
unsigned long llen = zsetLength(zobj);
for (j = 0; j < elements; j++) {
double newscore;
score = scores[j];
@ -1860,6 +1861,7 @@ void zaddGenericCommand(client *c, int flags) {
score = newscore;
}
server.dirty += (added+updated);
updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, llen, llen+added);
reply_to_client:
if (incr) { /* ZINCRBY or INCR option. */
@ -1907,8 +1909,13 @@ void zremCommand(client *c) {
if (deleted) {
notifyKeyspaceEvent(NOTIFY_ZSET,"zrem",key,c->db->id);
if (keyremoved)
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
if (keyremoved) {
notifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, c->db->id);
/* No need updateKeysizesHist(). dbDelete() done it already. */
} else {
unsigned long len = zsetLength(zobj);
updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, len + deleted, len);
}
signalModifiedKey(c,c->db,key);
server.dirty += deleted;
}
@ -2023,8 +2030,13 @@ void zremrangeGenericCommand(client *c, zrange_type rangetype) {
if (deleted) {
signalModifiedKey(c,c->db,key);
notifyKeyspaceEvent(NOTIFY_ZSET,notify_type,key,c->db->id);
if (keyremoved)
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
if (keyremoved) {
notifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, c->db->id);
/* No need updateKeysizesHist(). dbDelete() done it already. */
} else {
unsigned long len = zsetLength(zobj);
updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, len + deleted, len);
}
}
server.dirty += deleted;
addReplyLongLong(c,deleted);
@ -4031,6 +4043,9 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
dbDelete(c->db,key);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
/* No need updateKeysizesHist(). dbDelete() done it already. */
} else {
updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, llen, llen - result_count);
}
signalModifiedKey(c,c->db,key);

7
src/util.c
View File

@ -54,6 +54,13 @@
#define UNUSED(x) ((void)(x))
/* Selectively define static_assert. Attempt to avoid include server.h in this file. */
#ifndef static_assert
#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1]
#endif
static_assert(UINTPTR_MAX == 0xffffffffffffffff || UINTPTR_MAX == 0xffffffff, "Unsupported pointer size");
/* Glob-style pattern matching. */
static int stringmatchlen_impl(const char *pattern, int patternLen,
const char *string, int stringLen, int nocase, int *skipLongerMatches, int nesting)

13
src/util.h
View File

@ -79,6 +79,19 @@ int snprintf_async_signal_safe(char *to, size_t n, const char *fmt, ...);
size_t redis_strlcpy(char *dst, const char *src, size_t dsize);
size_t redis_strlcat(char *dst, const char *src, size_t dsize);
/* to keep it opt without conditions Works only for: 0 < x < 2^63 */
static inline int log2ceil(size_t x) {
#if UINTPTR_MAX == 0xffffffffffffffff
return 63 - __builtin_clzll(x);
#else
return 31 - __builtin_clz(x);
#endif
}
#ifndef static_assert
#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1]
#endif
#ifdef REDIS_TEST
int utilTest(int argc, char **argv, int flags);
#endif

454
tests/unit/info-keysizes.tcl Normal file
View File

@ -0,0 +1,454 @@
################################################################################
# Test the "info keysizes" command.
# The command returns a histogram of the sizes of keys in the database.
################################################################################
# Query and Strip result of "info keysizes" from header, spaces, and newlines.
proc get_stripped_info {server} {
set infoStripped [string map {
"# Keysizes" ""
" " "" "\n" "" "\r" ""
} [$server info keysizes] ]
return $infoStripped
}
# Verify output of "info keysizes" command is as expected.
#
# Arguments:
# cmd - A command that should be run before the verification.
# expOutput - This is a string that represents the expected output abbreviated.
# Instead of the output of "strings_len_exp_distrib" write "STR".
# Similarly for LIST, SET, ZSET and HASH. Spaces and newlines are
# ignored.
# waitCond - If set to 1, the function wait_for_condition 50x50msec for the
# expOutput to match the actual output.
#
# (replicaMode) - Global variable that indicates if the test is running in replica
# mode. If so, run the command on leader, verify the output. Then wait
# for the replica to catch up and verify the output on the replica
# as well. Otherwise, just run the command on the leader and verify
# the output.
proc run_cmd_verify_hist {cmd expOutput {waitCond 0} } {
uplevel 1 $cmd
global replicaMode
# ref the leader with `server` variable
if {$replicaMode eq 1} { set server [srv -1 client] } else { set server [srv 0 client] }
# Replace all placeholders with the actual values. Remove spaces & newlines.
set expStripped [string map {
"STR" "distrib_strings_sizes"
"LIST" "distrib_lists_items"
"SET" "distrib_sets_items"
"ZSET" "distrib_zsets_items"
"HASH" "distrib_hashes_items"
" " "" "\n" "" "\r" ""
} $expOutput]
if {$waitCond} {
wait_for_condition 50 50 {
$expStripped eq [get_stripped_info $server]
} else {
fail "Unexpected KEYSIZES. Expected: `$expStripped` \
but got: `[get_stripped_info $server]`. Failed after command: $cmd"
}
} else {
set infoStripped [get_stripped_info $server]
if {$expStripped ne $infoStripped} {
fail "Unexpected KEYSIZES. Expected: `$expStripped` \
but got: `$infoStripped`. Failed after command: $cmd"
}
}
# If we are testing `replicaMode` then need to wait for the replica to catch up
if {$replicaMode eq 1} {
wait_for_condition 50 50 {
$expStripped eq [get_stripped_info $server]
} else {
fail "Unexpected replica KEYSIZES. Expected: `$expStripped` \
but got: `[get_stripped_info $server]`. Failed after command: $cmd"
}
}
}
proc test_all_keysizes { {replMode 0} } {
# If in replica mode then update global var `replicaMode` so function
# `run_cmd_verify_hist` knows to run the command on the leader and then
# wait for the replica to catch up.
global replicaMode
set replicaMode $replMode
# ref the leader with `server` variable
if {$replicaMode eq 1} {
set server [srv -1 client]
set suffixRepl "(replica)"
} else {
set server [srv 0 client]
set suffixRepl ""
}
test "KEYSIZES - Test i'th bin counts keysizes between (2^i) and (2^(i+1)-1) as expected $suffixRepl" {
set base_string ""
run_cmd_verify_hist {$server FLUSHALL} {}
for {set i 1} {$i <= 10} {incr i} {
append base_string "x"
set log_value [expr {1 << int(log($i) / log(2))}]
#puts "Iteration $i: $base_string (Log base 2 pattern: $log_value)"
run_cmd_verify_hist {$server set mykey $base_string} "db0_STR:$log_value=1"
}
}
test "KEYSIZES - Histogram of values of Bytes, Kilo and Mega $suffixRepl" {
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server set x 0123456789ABCDEF} {db0_STR:16=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:32=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:64=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:128=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:256=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:512=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:1K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:2K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:4K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:8K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:16K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:32K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:64K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:128K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:256K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:512K=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:1M=1}
run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:2M=1}
}
test "KEYSIZES - Test List $suffixRepl" {
# FLUSHALL
run_cmd_verify_hist {$server FLUSHALL} {}
# RPUSH
run_cmd_verify_hist {$server RPUSH l1 1 2 3 4 5} {db0_LIST:4=1}
run_cmd_verify_hist {$server RPUSH l1 6 7 8 9} {db0_LIST:8=1}
# Test also LPUSH, RPUSH, LPUSHX, RPUSHX
run_cmd_verify_hist {$server LPUSH l2 1} {db0_LIST:1=1,8=1}
run_cmd_verify_hist {$server LPUSH l2 2} {db0_LIST:2=1,8=1}
run_cmd_verify_hist {$server LPUSHX l2 3} {db0_LIST:2=1,8=1}
run_cmd_verify_hist {$server RPUSHX l2 4} {db0_LIST:4=1,8=1}
# RPOP
run_cmd_verify_hist {$server RPOP l1} {db0_LIST:4=1,8=1}
run_cmd_verify_hist {$server RPOP l1} {db0_LIST:4=2}
# DEL
run_cmd_verify_hist {$server DEL l1} {db0_LIST:4=1}
# LINSERT, LTRIM
run_cmd_verify_hist {$server RPUSH l3 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} {db0_LIST:4=1,8=1}
run_cmd_verify_hist {$server LINSERT l3 AFTER 9 10} {db0_LIST:4=1,16=1}
run_cmd_verify_hist {$server LTRIM l3 0 8} {db0_LIST:4=1,8=1}
# DEL
run_cmd_verify_hist {$server DEL l3} {db0_LIST:4=1}
run_cmd_verify_hist {$server DEL l2} {}
# LMOVE, BLMOVE
run_cmd_verify_hist {$server RPUSH l4 1 2 3 4 5 6 7 8} {db0_LIST:8=1}
run_cmd_verify_hist {$server LMOVE l4 l5 LEFT LEFT} {db0_LIST:1=1,4=1}
run_cmd_verify_hist {$server LMOVE l4 l5 RIGHT RIGHT} {db0_LIST:2=1,4=1}
run_cmd_verify_hist {$server LMOVE l4 l5 LEFT RIGHT} {db0_LIST:2=1,4=1}
run_cmd_verify_hist {$server LMOVE l4 l5 RIGHT LEFT} {db0_LIST:4=2}
run_cmd_verify_hist {$server BLMOVE l4 l5 RIGHT LEFT 0} {db0_LIST:2=1,4=1}
# DEL
run_cmd_verify_hist {$server DEL l4} {db0_LIST:4=1}
run_cmd_verify_hist {$server DEL l5} {}
# LMPOP
run_cmd_verify_hist {$server RPUSH l6 1 2 3 4 5 6 7 8 9 10} {db0_LIST:8=1}
run_cmd_verify_hist {$server LMPOP 1 l6 LEFT COUNT 2} {db0_LIST:8=1}
run_cmd_verify_hist {$server LMPOP 1 l6 LEFT COUNT 1} {db0_LIST:4=1}
run_cmd_verify_hist {$server LMPOP 1 l6 LEFT COUNT 6} {db0_LIST:1=1}
# LPOP
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server RPUSH l7 1 2 3 4} {db0_LIST:4=1}
run_cmd_verify_hist {$server LPOP l7} {db0_LIST:2=1}
# LREM
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server RPUSH l8 1 x 3 x 5 x 7 x 9 10} {db0_LIST:8=1}
run_cmd_verify_hist {$server LREM l8 3 x} {db0_LIST:4=1}
# EXPIRE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server RPUSH l9 1 2 3 4} {db0_LIST:4=1}
run_cmd_verify_hist {$server PEXPIRE l9 50} {db0_LIST:4=1}
run_cmd_verify_hist {} {} 1
# SET overwrites
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server RPUSH l9 1 2 3 4} {db0_LIST:4=1}
run_cmd_verify_hist {$server SET l9 1234567} {db0_STR:4=1}
run_cmd_verify_hist {$server DEL l9} {}
} {} {cluster:skip}
test "KEYSIZES - Test SET $suffixRepl" {
run_cmd_verify_hist {$server FLUSHALL} {}
# SADD
run_cmd_verify_hist {$server SADD s1 1 2 3 4 5} {db0_SET:4=1}
run_cmd_verify_hist {$server SADD s1 6 7 8} {db0_SET:8=1}
# Test also SADD, SREM, SMOVE, SPOP
run_cmd_verify_hist {$server SADD s2 1} {db0_SET:1=1,8=1}
run_cmd_verify_hist {$server SADD s2 2} {db0_SET:2=1,8=1}
run_cmd_verify_hist {$server SREM s2 3} {db0_SET:2=1,8=1}
run_cmd_verify_hist {$server SMOVE s2 s3 2} {db0_SET:1=2,8=1}
run_cmd_verify_hist {$server SPOP s3} {db0_SET:1=1,8=1}
run_cmd_verify_hist {$server SPOP s2} {db0_SET:8=1}
run_cmd_verify_hist {$server SPOP s1} {db0_SET:4=1}
run_cmd_verify_hist {$server del s1} {}
# SDIFFSTORE
run_cmd_verify_hist {$server flushall} {}
run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1}
run_cmd_verify_hist {$server SADD s2 6 7 8 9 A B C D} {db0_SET:8=2}
run_cmd_verify_hist {$server SDIFFSTORE s3 s1 s2} {db0_SET:4=1,8=2}
#SINTERSTORE
run_cmd_verify_hist {$server flushall} {}
run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1}
run_cmd_verify_hist {$server SADD s2 6 7 8 9 A B C D} {db0_SET:8=2}
run_cmd_verify_hist {$server SINTERSTORE s3 s1 s2} {db0_SET:2=1,8=2}
#SUNIONSTORE
run_cmd_verify_hist {$server flushall} {}
run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1}
run_cmd_verify_hist {$server SADD s2 6 7 8 9 A B C D} {db0_SET:8=2}
run_cmd_verify_hist {$server SUNIONSTORE s3 s1 s2} {db0_SET:8=3}
run_cmd_verify_hist {$server SADD s4 E F G H} {db0_SET:4=1,8=3}
run_cmd_verify_hist {$server SUNIONSTORE s5 s3 s4} {db0_SET:4=1,8=3,16=1}
# DEL
run_cmd_verify_hist {$server flushall} {}
run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1}
run_cmd_verify_hist {$server DEL s1} {}
# EXPIRE
run_cmd_verify_hist {$server flushall} {}
run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1}
run_cmd_verify_hist {$server PEXPIRE s1 50} {db0_SET:8=1}
run_cmd_verify_hist {} {} 1
# SET overwrites
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server SADD s1 1 2 3 4 5 6 7 8} {db0_SET:8=1}
run_cmd_verify_hist {$server SET s1 1234567} {db0_STR:4=1}
run_cmd_verify_hist {$server DEL s1} {}
} {} {cluster:skip}
test "KEYSIZES - Test ZSET $suffixRepl" {
# ZADD, ZREM
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZADD z1 6 f 7 g 8 h 9 i} {db0_ZSET:8=1}
run_cmd_verify_hist {$server ZADD z2 1 a} {db0_ZSET:1=1,8=1}
run_cmd_verify_hist {$server ZREM z1 a} {db0_ZSET:1=1,8=1}
run_cmd_verify_hist {$server ZREM z1 b} {db0_ZSET:1=1,4=1}
# ZREMRANGEBYSCORE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZREMRANGEBYSCORE z1 -inf (2} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZREMRANGEBYSCORE z1 -inf (3} {db0_ZSET:2=1}
# ZREMRANGEBYRANK
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e 6 f} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZREMRANGEBYRANK z1 0 1} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZREMRANGEBYRANK z1 0 0} {db0_ZSET:2=1}
# ZREMRANGEBYLEX
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 0 a 0 b 0 c 0 d 0 e 0 f} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZREMRANGEBYLEX z1 - (d} {db0_ZSET:2=1}
# ZUNIONSTORE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZADD z2 6 f 7 g 8 h 9 i} {db0_ZSET:4=2}
run_cmd_verify_hist {$server ZUNIONSTORE z3 2 z1 z2} {db0_ZSET:4=2,8=1}
# ZINTERSTORE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZADD z2 3 c 4 d 5 e 6 f} {db0_ZSET:4=2}
run_cmd_verify_hist {$server ZINTERSTORE z3 2 z1 z2} {db0_ZSET:2=1,4=2}
# BZPOPMIN, BZPOPMAX
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server BZPOPMIN z1 0} {db0_ZSET:4=1}
run_cmd_verify_hist {$server BZPOPMAX z1 0} {db0_ZSET:2=1}
# ZDIFFSTORE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZADD z2 3 c 4 d 5 e 6 f} {db0_ZSET:4=2}
run_cmd_verify_hist {$server ZDIFFSTORE z3 2 z1 z2} {db0_ZSET:2=1,4=2}
# ZINTERSTORE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server ZADD z2 3 c 4 d 5 e 6 f} {db0_ZSET:4=2}
run_cmd_verify_hist {$server ZINTERSTORE z3 2 z1 z2} {db0_ZSET:2=1,4=2}
# DEL
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server DEL z1} {}
# EXPIRE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server PEXPIRE z1 50} {db0_ZSET:4=1}
run_cmd_verify_hist {} {} 1
# SET overwrites
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c 4 d 5 e} {db0_ZSET:4=1}
run_cmd_verify_hist {$server SET z1 1234567} {db0_STR:4=1}
run_cmd_verify_hist {$server DEL z1} {}
} {} {cluster:skip}
test "KEYSIZES - Test STRING $suffixRepl" {
# SETRANGE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server SET s2 1234567890} {db0_STR:8=1}
run_cmd_verify_hist {$server SETRANGE s2 10 123456} {db0_STR:16=1}
# MSET, MSETNX
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server MSET s3 1 s4 2 s5 3} {db0_STR:1=3}
run_cmd_verify_hist {$server MSETNX s6 1 s7 2 s8 3} {db0_STR:1=6}
# DEL
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server SET s9 1234567890} {db0_STR:8=1}
run_cmd_verify_hist {$server DEL s9} {}
#EXPIRE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server SET s10 1234567890} {db0_STR:8=1}
run_cmd_verify_hist {$server PEXPIRE s10 50} {db0_STR:8=1}
run_cmd_verify_hist {} {} 1
# SET (+overwrite)
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server SET s1 1024} {db0_STR:4=1}
run_cmd_verify_hist {$server SET s1 842} {db0_STR:2=1}
run_cmd_verify_hist {$server SET s1 2} {db0_STR:1=1}
run_cmd_verify_hist {$server SET s1 1234567} {db0_STR:4=1}
} {} {cluster:skip}
foreach type {listpackex hashtable} {
# Test different implementations of hash tables and listpacks
if {$type eq "hashtable"} {
$server config set hash-max-listpack-entries 0
} else {
$server config set hash-max-listpack-entries 512
}
test "KEYSIZES - Test HASH ($type) $suffixRepl" {
# HSETNX
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server HSETNX h1 1 1} {db0_HASH:1=1}
run_cmd_verify_hist {$server HSETNX h1 2 2} {db0_HASH:2=1}
# HSET, HDEL
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server HSET h2 1 1} {db0_HASH:1=1}
run_cmd_verify_hist {$server HSET h2 2 2} {db0_HASH:2=1}
run_cmd_verify_hist {$server HDEL h2 1} {db0_HASH:1=1}
run_cmd_verify_hist {$server HDEL h2 2} {}
# HMSET
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3} {db0_HASH:2=1}
run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3} {db0_HASH:2=1}
run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3 4 4} {db0_HASH:4=1}
# HINCRBY
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server hincrby h1 f1 10} {db0_HASH:1=1}
run_cmd_verify_hist {$server hincrby h1 f1 10} {db0_HASH:1=1}
run_cmd_verify_hist {$server hincrby h1 f2 20} {db0_HASH:2=1}
# HINCRBYFLOAT
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server hincrbyfloat h1 f1 10.5} {db0_HASH:1=1}
run_cmd_verify_hist {$server hincrbyfloat h1 f1 10.5} {db0_HASH:1=1}
run_cmd_verify_hist {$server hincrbyfloat h1 f2 10.5} {db0_HASH:2=1}
# HEXPIRE
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server HSET h1 f1 1} {db0_HASH:1=1}
run_cmd_verify_hist {$server HSET h1 f2 1} {db0_HASH:2=1}
run_cmd_verify_hist {$server HPEXPIREAT h1 1 FIELDS 1 f1} {db0_HASH:1=1}
run_cmd_verify_hist {$server HSET h1 f3 1} {db0_HASH:2=1}
run_cmd_verify_hist {$server HPEXPIRE h1 50 FIELDS 1 f2} {db0_HASH:2=1}
run_cmd_verify_hist {} {db0_HASH:1=1} 1
run_cmd_verify_hist {$server HPEXPIRE h1 50 FIELDS 1 f3} {db0_HASH:1=1}
run_cmd_verify_hist {} {} 1
}
}
test "KEYSIZES - Test STRING BITS $suffixRepl" {
# BITOPS
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server SET b1 "x123456789"} {db0_STR:8=1}
run_cmd_verify_hist {$server SET b2 "x12345678"} {db0_STR:8=2}
run_cmd_verify_hist {$server BITOP AND b3 b1 b2} {db0_STR:8=3}
run_cmd_verify_hist {$server BITOP OR b4 b1 b2} {db0_STR:8=4}
run_cmd_verify_hist {$server BITOP XOR b5 b1 b2} {db0_STR:8=5}
# SETBIT
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server setbit b1 71 1} {db0_STR:8=1}
run_cmd_verify_hist {$server setbit b1 72 1} {db0_STR:8=1}
run_cmd_verify_hist {$server setbit b2 72 1} {db0_STR:8=2}
run_cmd_verify_hist {$server setbit b2 640 0} {db0_STR:8=1,64=1}
# BITFIELD
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server bitfield b3 set u8 6 255} {db0_STR:2=1}
run_cmd_verify_hist {$server bitfield b3 set u8 65 255} {db0_STR:8=1}
run_cmd_verify_hist {$server bitfield b4 set u8 1000 255} {db0_STR:8=1,64=1}
} {} {cluster:skip}
test "KEYSIZES - Test RESTORE $suffixRepl" {
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server RPUSH l10 1 2 3 4} {db0_LIST:4=1}
set encoded [$server dump l10]
run_cmd_verify_hist {$server del l10} {}
run_cmd_verify_hist {$server restore l11 0 $encoded} {db0_LIST:4=1}
}
test "KEYSIZES - Test RENAME $suffixRepl" {
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server RPUSH l12 1 2 3 4} {db0_LIST:4=1}
run_cmd_verify_hist {$server RENAME l12 l13} {db0_LIST:4=1}
} {} {cluster:skip}
test "KEYSIZES - Test MOVE $suffixRepl" {
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server RPUSH l1 1 2 3 4} {db0_LIST:4=1}
run_cmd_verify_hist {$server RPUSH l2 1} {db0_LIST:1=1,4=1}
run_cmd_verify_hist {$server MOVE l1 1} {db0_LIST:1=1 db1_LIST:4=1}
} {} {cluster:skip}
test "KEYSIZES - Test SWAPDB $suffixRepl" {
run_cmd_verify_hist {$server FLUSHALL} {}
run_cmd_verify_hist {$server RPUSH l1 1 2 3 4} {db0_LIST:4=1}
$server select 1
run_cmd_verify_hist {$server ZADD z1 1 A} {db0_LIST:4=1 db1_ZSET:1=1}
run_cmd_verify_hist {$server SWAPDB 0 1} {db0_ZSET:1=1 db1_LIST:4=1}
$server select 0
} {OK} {singledb:skip}
test "KEYSIZES - Test RDB $suffixRepl" {
run_cmd_verify_hist {$server FLUSHALL} {}
# Write list, set and zset to db0
run_cmd_verify_hist {$server RPUSH l1 1 2 3 4} {db0_LIST:4=1}
run_cmd_verify_hist {$server SADD s1 1 2 3 4 5} {db0_LIST:4=1 db0_SET:4=1}
run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1}
run_cmd_verify_hist {$server SAVE} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1}
if {$replicaMode eq 1} {
run_cmd_verify_hist {restart_server -1 true false} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1}
} else {
run_cmd_verify_hist {restart_server 0 true false} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1}
}
} {} {external:skip}
}
start_server {} {
# Test KEYSIZES on a single server
r select 0
test_all_keysizes 0
# Start another server to test replication of KEYSIZES
start_server {tags {needs:repl external:skip}} {
# Set the outer layer server as primary
set primary [srv -1 client]
set primary_host [srv -1 host]
set primary_port [srv -1 port]
# Set this inner layer server as replica
set replica [srv 0 client]
# Server should have role replica
$replica replicaof $primary_host $primary_port
wait_for_condition 50 100 { [s 0 role] eq {slave} } else { fail "Replication not started." }
# Test KEYSIZES on leader and replica
$primary select 0
test_all_keysizes 1
}
}

4
tests/unit/memefficiency.tcl
View File

@ -306,7 +306,7 @@ run_solo {defrag} {
r set "{bigstream}smallitem" val
set expected_frag 1.7
set expected_frag 1.5
if {$::accurate} {
# scale the hash to 1m fields in order to have a measurable the latency
for {set j 10000} {$j < 1000000} {incr j} {
@ -601,7 +601,7 @@ run_solo {defrag} {
# create big keys with 10k items
set rd [redis_deferring_client]
set expected_frag 1.7
set expected_frag 1.5
# add a mass of list nodes to two lists (allocations are interlaced)
set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation
set elements 500000

17
tests/unit/other.tcl
View File

@ -508,18 +508,25 @@ start_server {tags {"other external:skip"}} {
test "Redis can resize empty dict" {
# Write and then delete 128 keys, creating an empty dict
r flushall
# Add one key to the db just to create the dict and get its initial size
r set x 1
set initial_size [dict get [r memory stats] db.9 overhead.hashtable.main]
# Now add 128 keys and then delete them
for {set j 1} {$j <= 128} {incr j} {
r set $j{b} a
}
for {set j 1} {$j <= 128} {incr j} {
r del $j{b}
}
# The dict containing 128 keys must have expanded,
# its hash table itself takes a lot more than 400 bytes
# dict must have expanded. Verify it eventually shrinks back to its initial size.
wait_for_condition 100 50 {
[dict get [r memory stats] db.9 overhead.hashtable.main] < 400
[dict get [r memory stats] db.9 overhead.hashtable.main] == $initial_size
} else {
fail "dict did not resize in time"
}
fail "dict did not resize in time to its initial size"
}
}
}