diff --git a/src/listpack.c b/src/listpack.c index 7d2febd01d6..7cfbcf199bf 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -41,6 +41,14 @@ #include "listpack.h" #include "listpack_malloc.h" + +/* lp_last_alloc_size is defined as a file-local static in listpack_malloc.h + * (included above). This getter exposes it to other modules (e.g. quicklist) + * without leaking the variable as an extern global. */ +size_t lpLastAllocSize(void) { + return lp_last_alloc_size; +} + #include "serverassert.h" #include "util.h" #include "config.h" @@ -175,9 +183,13 @@ void lpFreeVoid(void *lp) { /* Shrink the memory to fit. */ unsigned char *lpShrinkToFit(unsigned char *lp) { size_t size = lpGetTotalBytes(lp); - if (size < lp_malloc_size(lp)) { + size_t alloc_size = lp_malloc_size(lp); + if (size < alloc_size) { return lp_realloc(lp, size); } else { + /* Already at minimum allocation — no realloc needed. + * Update lp_last_alloc_size so callers don't see a stale value. */ + lp_last_alloc_size = alloc_size; return lp; } } @@ -777,9 +789,20 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *dst = lp + poff; /* May be updated after reallocation. */ /* Realloc before: we need more room. */ - if (new_listpack_bytes > old_listpack_bytes && new_listpack_bytes > lp_malloc_size(lp)) { - if ((lp = lp_realloc(lp, new_listpack_bytes)) == NULL) return NULL; - dst = lp + poff; + if (new_listpack_bytes > old_listpack_bytes) { + size_t alloc_size = lp_malloc_size(lp); + if (new_listpack_bytes > alloc_size) { + if ((lp = lp_realloc(lp, new_listpack_bytes)) == NULL) return NULL; + dst = lp + poff; + } else { + /* Growth fits within jemalloc slack — no realloc needed. + * Update lp_last_alloc_size so callers see the current value. */ + lp_last_alloc_size = alloc_size; + } + } else if (new_listpack_bytes == old_listpack_bytes) { + /* Same size (e.g. replace with same-length element) — no realloc. + * Update lp_last_alloc_size so callers don't see a stale value. */ + lp_last_alloc_size = lp_malloc_size(lp); } /* Setup the listpack relocating the elements to make the exact room @@ -927,7 +950,10 @@ unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsi unsigned char *first, *tail; first = tail = *p; - if (num == 0) return lp; /* Nothing to delete, return ASAP. */ + if (num == 0) { /* Nothing to delete, return ASAP. */ + lp_last_alloc_size = lp_malloc_size(lp); + return lp; + } /* Find the next entry to the last entry that needs to be deleted. * lpLength may be unreliable due to corrupt data, so we cannot @@ -966,8 +992,14 @@ unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num) { unsigned char *p; uint32_t numele = lpGetNumElements(lp); - if (num == 0) return lp; /* Nothing to delete, return ASAP. */ - if ((p = lpSeek(lp, index)) == NULL) return lp; + if (num == 0) { /* Nothing to delete, return ASAP. */ + lp_last_alloc_size = lp_malloc_size(lp); + return lp; + } + if ((p = lpSeek(lp, index)) == NULL) { + lp_last_alloc_size = lp_malloc_size(lp); + return lp; + } /* If we know we're gonna delete beyond the end of the listpack, we can just move * the EOF marker, and there's no need to iterate through the entries, diff --git a/src/listpack.h b/src/listpack.h index b1437972610..5886b6ea6d9 100644 --- a/src/listpack.h +++ b/src/listpack.h @@ -97,5 +97,6 @@ unsigned char * lpNextRandom(unsigned char *lp, unsigned char *p, unsigned int *index, unsigned int remaining, int even_only); int lpSafeToAdd(unsigned char *lp, size_t add); void lpRepr(unsigned char *lp); +size_t lpLastAllocSize(void); #endif diff --git a/src/listpack_malloc.h b/src/listpack_malloc.h index a75bd318177..352c6948cf9 100644 --- a/src/listpack_malloc.h +++ b/src/listpack_malloc.h @@ -39,11 +39,18 @@ #ifndef LISTPACK_ALLOC_H #define LISTPACK_ALLOC_H #include "zmalloc.h" + +/* File-local variable defined in listpack.c that captures the usable + * allocation size from the last lp_malloc / lp_realloc call. zmalloc_usable + * and zrealloc_usable already compute this internally; we just stop + * discarding it. Exposed to callers via lpLastAllocSize(). */ +static size_t lp_last_alloc_size = 0; + /* We use zmalloc_usable/zrealloc_usable instead of zmalloc/zrealloc * to ensure the safe invocation of 'zmalloc_usable_size(). * See comment in zmalloc_usable_size(). */ -#define lp_malloc(sz) zmalloc_usable(sz, NULL) -#define lp_realloc(ptr, sz) zrealloc_usable(ptr, sz, NULL) +#define lp_malloc(sz) zmalloc_usable(sz, &lp_last_alloc_size) +#define lp_realloc(ptr, sz) zrealloc_usable(ptr, sz, &lp_last_alloc_size) #define lp_free zfree #define lp_malloc_size zmalloc_usable_size #endif diff --git a/src/object.c b/src/object.c index 7db5b97996f..5474001f7af 100644 --- a/src/object.c +++ b/src/object.c @@ -1356,6 +1356,163 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { return asize; } +/* Same as objectComputeSize but uses tracked_size for quicklists instead of + * iterating through nodes. Used for testing tracked_size accuracy. */ +size_t objectComputeSizeWithTrackedSize(robj *key, robj *o, size_t sample_size, int dbid) { + size_t elesize = 0, samples = 0; + size_t asize = zmalloc_size((void *)o); + + if (o->type == OBJ_STRING) { + if (o->encoding == OBJ_ENCODING_RAW) { + asize += sdsAllocSize(objectGetVal(o)); + } else if (o->encoding != OBJ_ENCODING_INT && o->encoding != OBJ_ENCODING_EMBSTR) { + serverPanic("Unknown string encoding"); + } + } else if (o->type == OBJ_LIST) { + if (o->encoding == OBJ_ENCODING_QUICKLIST) { + quicklist *ql = objectGetVal(o); + asize += ql->tracked_size; + } else if (o->encoding == OBJ_ENCODING_LISTPACK) { + asize += zmalloc_size(objectGetVal(o)); + } else { + serverPanic("Unknown list encoding"); + } + } else if (o->type == OBJ_SET) { + if (o->encoding == OBJ_ENCODING_HASHTABLE) { + hashtable *ht = objectGetVal(o); + asize += hashtableMemUsage(ht); + + hashtableIterator iter; + hashtableInitIterator(&iter, ht, 0); + void *next; + while (hashtableNext(&iter, &next) && samples < sample_size) { + sds element = next; + elesize += sdsAllocSize(element); + samples++; + } + hashtableCleanupIterator(&iter); + if (samples) asize += (double)elesize / samples * hashtableSize(ht); + } else if (o->encoding == OBJ_ENCODING_INTSET) { + asize += zmalloc_size(objectGetVal(o)); + } else if (o->encoding == OBJ_ENCODING_LISTPACK) { + asize += zmalloc_size(objectGetVal(o)); + } else { + serverPanic("Unknown set encoding"); + } + } else if (o->type == OBJ_ZSET) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { + asize += zmalloc_size(objectGetVal(o)); + } else if (o->encoding == OBJ_ENCODING_SKIPLIST) { + hashtable *ht = ((zset *)objectGetVal(o))->ht; + zskiplist *zsl = ((zset *)objectGetVal(o))->zsl; + zskiplistNode *zheader = zslGetHeader(zsl); + zskiplistNode *znode = zheader->level[0].forward; + asize += sizeof(zset) + zslGetAllocSize() + hashtableMemUsage(ht); + while (znode != NULL && samples < sample_size) { + elesize += zmalloc_size(znode); + samples++; + znode = znode->level[0].forward; + } + if (samples) asize += (double)elesize / samples * hashtableSize(ht); + } else { + serverPanic("Unknown sorted set encoding"); + } + } else if (o->type == OBJ_HASH) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { + asize += zmalloc_size(objectGetVal(o)); + } else if (o->encoding == OBJ_ENCODING_HASHTABLE) { + hashtable *ht = objectGetVal(o); + hashtableIterator iter; + vset *volatile_fields = hashtableMetadata(ht); + hashtableInitIterator(&iter, ht, 0); + void *next; + + asize += hashtableMemUsage(ht); + while (hashtableNext(&iter, &next) && samples < sample_size) { + elesize += entryMemUsage(next); + samples++; + } + hashtableCleanupIterator(&iter); + if (samples) asize += (double)elesize / samples * hashtableSize(ht); + if (vsetIsValid(volatile_fields)) asize += vsetMemUsage(volatile_fields); + } else { + serverPanic("Unknown hash encoding"); + } + } else if (o->type == OBJ_STREAM) { + stream *s = objectGetVal(o); + asize += sizeof(*s) + raxAllocSize(s->rax); + + /* Now we have to add the listpacks. The last listpack is often non + * complete, so we estimate the size of the first N listpacks, and + * use the average to compute the size of the first N-1 listpacks, and + * finally add the real size of the last node. */ + raxIterator ri; + raxStart(&ri, s->rax); + raxSeek(&ri, "^", NULL, 0); + size_t lpsize = 0; + samples = 0; + while (samples < sample_size && raxNext(&ri)) { + unsigned char *lp = ri.data; + /* Use the allocated size, since we overprovision the node initially. */ + lpsize += zmalloc_size(lp); + samples++; + } + if (s->rax->numele <= samples) { + asize += lpsize; + } else { + if (samples) lpsize /= samples; /* Compute the average. */ + asize += lpsize * (s->rax->numele - 1); + /* No need to seek, we are already at the last element. */ + asize += zmalloc_size(ri.data); + } + raxStop(&ri); + + /* Consumer groups also have a non-trivial memory overhead if there + * are many consumers and many groups, let's count at least the + * overhead of the pending entries in the groups and consumers + * PELs. */ + if (s->cgroups) { + raxStart(&ri, s->cgroups); + raxSeek(&ri, "^", NULL, 0); + samples = 0; + elesize = 0; + while (samples < sample_size && raxNext(&ri)) { + streamCG *cg = ri.data; + elesize += sizeof(*cg); + elesize += raxAllocSize(cg->pel); + elesize += sizeof(streamNACK) * raxSize(cg->pel); + + /* For each consumer we also need to add the basic data + * structures and the PEL memory usage. */ + raxIterator cri; + raxStart(&cri, cg->consumers); + raxSeek(&cri, "^", NULL, 0); + size_t inner_samples = 0; + size_t inner_elesize = 0; + while (inner_samples < sample_size && raxNext(&cri)) { + streamConsumer *consumer = cri.data; + inner_elesize += sizeof(*consumer); + inner_elesize += sdslen(consumer->name); + inner_elesize += raxAllocSize(consumer->pel); + /* Don't count NACKs again, they are shared with the + * consumer group PEL. */ + inner_samples++; + } + raxStop(&cri); + if (inner_samples) elesize += (double)inner_elesize / inner_samples * raxSize(cg->consumers); + samples++; + } + raxStop(&ri); + if (samples) asize += (double)elesize / samples * raxSize(s->cgroups); + } + } else if (o->type == OBJ_MODULE) { + asize += moduleGetMemUsage(key, o, sample_size, dbid); + } else { + serverPanic("Unknown object type"); + } + return asize; +} + /* Release data obtained with getMemoryOverheadData(). */ void freeMemoryOverheadData(struct serverMemOverhead *mh) { zfree(mh->db); diff --git a/src/quicklist.c b/src/quicklist.c index 67ffe4e17bf..3d31f106e52 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -96,7 +96,7 @@ quicklistBookmark *_quicklistBookmarkFindByName(quicklist *ql, const char *name) quicklistBookmark *_quicklistBookmarkFindByNode(quicklist *ql, quicklistNode *node); void _quicklistBookmarkDelete(quicklist *ql, quicklistBookmark *bm); -static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, int after); +static quicklistNode *_quicklistSplitNode(quicklist *quicklist, quicklistNode *node, int offset, int after); static quicklistNode *_quicklistMergeNodes(quicklist *quicklist, quicklistNode *center); /* Simple way to give quicklistEntry structs default values with one call. */ @@ -118,6 +118,15 @@ static quicklistNode *_quicklistMergeNodes(quicklist *quicklist, quicklistNode * (iter)->zi = NULL; \ } while (0) +/* Update node->entry_alloc_sz from lpLastAllocSize() and adjust + * quicklist->tracked_size by the delta. Use after any listpack operation + * that may reallocate node->entry, for nodes already linked in the list. */ +#define quicklistTrackEntryResize(ql, node, old_alloc_sz) \ + do { \ + (node)->entry_alloc_sz = lpLastAllocSize(); \ + if (ql) (ql)->tracked_size += (node)->entry_alloc_sz - (old_alloc_sz); \ + } while (0) + /* Create a new quicklist. * Free with quicklistRelease(). */ quicklist *quicklistCreate(void) { @@ -130,6 +139,7 @@ quicklist *quicklistCreate(void) { quicklist->compress = 0; quicklist->fill = -2; quicklist->bookmark_count = 0; + quicklist->tracked_size = sizeof(*quicklist); return quicklist; } @@ -171,6 +181,7 @@ static quicklistNode *quicklistCreateNode(void) { node->entry = NULL; node->count = 0; node->sz = 0; + node->entry_alloc_sz = 0; node->next = node->prev = NULL; node->encoding = QUICKLIST_NODE_ENCODING_RAW; node->container = QUICKLIST_NODE_CONTAINER_PACKED; @@ -209,7 +220,7 @@ void quicklistRelease(quicklist *quicklist) { /* Compress the listpack in 'node' and update encoding details. * Returns 1 if listpack compressed successfully. * Returns 0 if compression failed or if listpack too small to compress. */ -static int __quicklistCompressNode(quicklistNode *node) { +static int __quicklistCompressNode(quicklist *quicklist, quicklistNode *node) { node->attempted_compress = 1; if (node->dont_compress) return 0; @@ -230,53 +241,61 @@ static int __quicklistCompressNode(quicklistNode *node) { zfree(lzf); return 0; } - lzf = zrealloc(lzf, sizeof(*lzf) + lzf->sz); + size_t new_entry_alloc_sz; + lzf = zrealloc_usable(lzf, sizeof(*lzf) + lzf->sz, &new_entry_alloc_sz); + size_t old_entry_alloc_sz = node->entry_alloc_sz; zfree(node->entry); node->entry = (unsigned char *)lzf; + node->entry_alloc_sz = new_entry_alloc_sz; + if (quicklist) quicklist->tracked_size += new_entry_alloc_sz - old_entry_alloc_sz; node->encoding = QUICKLIST_NODE_ENCODING_LZF; return 1; } /* Compress only uncompressed nodes. */ -#define quicklistCompressNode(_node) \ +#define quicklistCompressNode(_ql, _node) \ do { \ if ((_node) && (_node)->encoding == QUICKLIST_NODE_ENCODING_RAW) { \ - __quicklistCompressNode((_node)); \ + __quicklistCompressNode((_ql), (_node)); \ } \ } while (0) /* Uncompress the listpack in 'node' and update encoding details. * Returns 1 on successful decode, 0 on failure to decode. */ -static int __quicklistDecompressNode(quicklistNode *node) { +static int __quicklistDecompressNode(quicklist *quicklist, quicklistNode *node) { node->attempted_compress = 0; node->recompress = 0; - void *decompressed = zmalloc(node->sz); + size_t new_entry_alloc_sz; + void *decompressed = zmalloc_usable(node->sz, &new_entry_alloc_sz); quicklistLZF *lzf = (quicklistLZF *)node->entry; if (lzf_decompress(lzf->compressed, lzf->sz, decompressed, node->sz) == 0) { /* Someone requested decompress, but we can't decompress. Not good. */ zfree(decompressed); return 0; } + size_t old_entry_alloc_sz = node->entry_alloc_sz; zfree(lzf); node->entry = decompressed; + node->entry_alloc_sz = new_entry_alloc_sz; + if (quicklist) quicklist->tracked_size += new_entry_alloc_sz - old_entry_alloc_sz; node->encoding = QUICKLIST_NODE_ENCODING_RAW; return 1; } /* Decompress only compressed nodes. */ -#define quicklistDecompressNode(_node) \ +#define quicklistDecompressNode(_ql, _node) \ do { \ if ((_node) && (_node)->encoding == QUICKLIST_NODE_ENCODING_LZF) { \ - __quicklistDecompressNode((_node)); \ + __quicklistDecompressNode((_ql), (_node)); \ } \ } while (0) /* Force node to not be immediately re-compressible */ -#define quicklistDecompressNodeForUse(_node) \ +#define quicklistDecompressNodeForUse(_ql, _node) \ do { \ if ((_node) && (_node)->encoding == QUICKLIST_NODE_ENCODING_LZF) { \ - __quicklistDecompressNode((_node)); \ + __quicklistDecompressNode((_ql), (_node)); \ (_node)->recompress = 1; \ } \ } while (0) @@ -296,7 +315,7 @@ size_t quicklistGetLzf(const quicklistNode *node, void **data) { * The only way to guarantee interior nodes get compressed is to iterate * to our "interior" compress depth then compress the next node we find. * If compress depth is larger than the entire list, we return immediately. */ -static void __quicklistCompress(const quicklist *quicklist, quicklistNode *node) { +static void __quicklistCompress(quicklist *quicklist, quicklistNode *node) { if (quicklist->len == 0) return; /* The head and tail should never be compressed (we should not attempt to recompress them) */ @@ -310,26 +329,26 @@ static void __quicklistCompress(const quicklist *quicklist, quicklistNode *node) /* Optimized cases for small depth counts */ if (quicklist->compress == 1) { quicklistNode *h = quicklist->head, *t = quicklist->tail; - quicklistDecompressNode(h); - quicklistDecompressNode(t); + quicklistDecompressNode(quicklist, h); + quicklistDecompressNode(quicklist, t); if (h != node && t != node) - quicklistCompressNode(node); + quicklistCompressNode(quicklist, node); return; } else if (quicklist->compress == 2) { quicklistNode *h = quicklist->head, *hn = h->next, *hnn = hn->next; quicklistNode *t = quicklist->tail, *tp = t->prev, *tpp = tp->prev; - quicklistDecompressNode(h); - quicklistDecompressNode(hn); - quicklistDecompressNode(t); - quicklistDecompressNode(tp); + quicklistDecompressNode(quicklist, h); + quicklistDecompressNode(quicklist, hn); + quicklistDecompressNode(quicklist, t); + quicklistDecompressNode(quicklist, tp); if (h != node && hn != node && t != node && tp != node) { - quicklistCompressNode(node); + quicklistCompressNode(quicklist, node); } if (hnn != t) { - quicklistCompressNode(hnn); + quicklistCompressNode(quicklist, hnn); } if (tpp != h) { - quicklistCompressNode(tpp); + quicklistCompressNode(quicklist, tpp); } return; } @@ -343,8 +362,8 @@ static void __quicklistCompress(const quicklist *quicklist, quicklistNode *node) int depth = 0; int in_depth = 0; while (depth++ < quicklist->compress) { - quicklistDecompressNode(forward); - quicklistDecompressNode(reverse); + quicklistDecompressNode(quicklist, forward); + quicklistDecompressNode(quicklist, reverse); if (forward == node || reverse == node) in_depth = 1; @@ -356,11 +375,11 @@ static void __quicklistCompress(const quicklist *quicklist, quicklistNode *node) reverse = reverse->prev; } - if (!in_depth) quicklistCompressNode(node); + if (!in_depth) quicklistCompressNode(quicklist, node); /* At this point, forward and reverse are one node beyond depth */ - quicklistCompressNode(forward); - quicklistCompressNode(reverse); + quicklistCompressNode(quicklist, forward); + quicklistCompressNode(quicklist, reverse); } /* This macro is used to compress a node. @@ -372,18 +391,18 @@ static void __quicklistCompress(const quicklist *quicklist, quicklistNode *node) * * If the 'recompress' flag of the node is false, we check whether the node is * within the range of compress depth before compressing it. */ -#define quicklistCompress(_ql, _node) \ - do { \ - if ((_node)->recompress) \ - quicklistCompressNode((_node)); \ - else \ - __quicklistCompress((_ql), (_node)); \ +#define quicklistCompress(_ql, _node) \ + do { \ + if ((_node)->recompress) \ + quicklistCompressNode((_ql), (_node)); \ + else \ + __quicklistCompress((_ql), (_node)); \ } while (0) /* If we previously used quicklistDecompressNodeForUse(), just recompress. */ -#define quicklistRecompressOnly(_node) \ - do { \ - if ((_node)->recompress) quicklistCompressNode((_node)); \ +#define quicklistRecompressOnly(_ql, _node) \ + do { \ + if ((_node)->recompress) quicklistCompressNode((_ql), (_node)); \ } while (0) /* Insert 'new_node' after 'old_node' if 'after' is 1. @@ -413,6 +432,11 @@ static void __quicklistInsertNode(quicklist *quicklist, quicklistNode *old_node, quicklist->head = quicklist->tail = new_node; } + /* Track memory: node struct + entry. + * Use sizeof(quicklistNode) to match original objectComputeSize calculation. */ + quicklist->tracked_size += sizeof(quicklistNode); + if (new_node->entry) quicklist->tracked_size += new_node->entry_alloc_sz; + /* Update len first, so in __quicklistCompress we know exactly len */ quicklist->len++; @@ -523,10 +547,11 @@ static quicklistNode *__quicklistCreateNode(int container, void *value, size_t s quicklistNode *new_node = quicklistCreateNode(); new_node->container = container; if (container == QUICKLIST_NODE_CONTAINER_PLAIN) { - new_node->entry = zmalloc(sz); + new_node->entry = zmalloc_usable(sz, &new_node->entry_alloc_sz); memcpy(new_node->entry, value, sz); } else { new_node->entry = lpPrepend(lpNew(0), value, sz); + new_node->entry_alloc_sz = lpLastAllocSize(); } new_node->sz = sz; new_node->count++; @@ -553,11 +578,14 @@ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) { } if (likely(_quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz))) { + size_t old_sz = quicklist->head->entry_alloc_sz; quicklist->head->entry = lpPrepend(quicklist->head->entry, value, sz); + quicklistTrackEntryResize(quicklist, quicklist->head, old_sz); quicklistNodeUpdateSz(quicklist->head); } else { quicklistNode *node = quicklistCreateNode(); node->entry = lpPrepend(lpNew(0), value, sz); + node->entry_alloc_sz = lpLastAllocSize(); quicklistNodeUpdateSz(node); _quicklistInsertNodeBefore(quicklist, quicklist->head, node); @@ -579,11 +607,14 @@ int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) { } if (likely(_quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz))) { + size_t old_sz = quicklist->tail->entry_alloc_sz; quicklist->tail->entry = lpAppend(quicklist->tail->entry, value, sz); + quicklistTrackEntryResize(quicklist, quicklist->tail, old_sz); quicklistNodeUpdateSz(quicklist->tail); } else { quicklistNode *node = quicklistCreateNode(); node->entry = lpAppend(lpNew(0), value, sz); + node->entry_alloc_sz = lpLastAllocSize(); quicklistNodeUpdateSz(node); _quicklistInsertNodeAfter(quicklist, quicklist->tail, node); @@ -602,6 +633,7 @@ void quicklistAppendListpack(quicklist *quicklist, unsigned char *zl) { node->entry = zl; node->count = lpLength(node->entry); node->sz = lpBytes(zl); + node->entry_alloc_sz = zmalloc_size(zl); _quicklistInsertNodeAfter(quicklist, quicklist->tail, node); quicklist->count += node->count; @@ -617,6 +649,7 @@ void quicklistAppendPlainNode(quicklist *quicklist, unsigned char *data, size_t node->entry = data; node->count = 1; node->sz = sz; + node->entry_alloc_sz = zmalloc_size(data); node->container = QUICKLIST_NODE_CONTAINER_PLAIN; _quicklistInsertNodeAfter(quicklist, quicklist->tail, node); @@ -659,6 +692,7 @@ static void __quicklistDelNode(quicklist *quicklist, quicklistNode *node) { * now have compressed nodes needing to be decompressed. */ __quicklistCompress(quicklist, NULL); + quicklist->tracked_size -= node->entry_alloc_sz + sizeof(quicklistNode); zfree(node->entry); zfree(node); } @@ -678,7 +712,9 @@ static int quicklistDelIndex(quicklist *quicklist, quicklistNode *node, unsigned __quicklistDelNode(quicklist, node); return 1; } + size_t old_sz = node->entry_alloc_sz; node->entry = lpDelete(node->entry, *p, p); + quicklistTrackEntryResize(quicklist, node, old_sz); node->count--; if (node->count == 0) { gone = 1; @@ -728,17 +764,22 @@ void quicklistReplaceEntry(quicklistIter *iter, quicklistEntry *entry, void *dat quicklist *quicklist = iter->quicklist; quicklistNode *node = entry->node; unsigned char *newentry; + size_t _old_entry_sz = 0; if (likely(!QL_NODE_IS_PLAIN(entry->node) && !isLargeElement(sz, quicklist->fill) && - (newentry = lpReplace(entry->node->entry, &entry->zi, data, sz)) != NULL)) { + (_old_entry_sz = entry->node->entry_alloc_sz, + newentry = lpReplace(entry->node->entry, &entry->zi, data, sz)) != NULL)) { entry->node->entry = newentry; + quicklistTrackEntryResize(quicklist, entry->node, _old_entry_sz); quicklistNodeUpdateSz(entry->node); /* quicklistNext() and quicklistGetIteratorEntryAtIdx() provide an uncompressed node */ quicklistCompress(quicklist, entry->node); } else if (QL_NODE_IS_PLAIN(entry->node)) { if (isLargeElement(sz, quicklist->fill)) { + quicklist->tracked_size -= entry->node->entry_alloc_sz; zfree(entry->node->entry); - entry->node->entry = zmalloc(sz); + entry->node->entry = zmalloc_usable(sz, &entry->node->entry_alloc_sz); + quicklist->tracked_size += entry->node->entry_alloc_sz; entry->node->sz = sz; memcpy(entry->node->entry, data, sz); quicklistCompress(quicklist, entry->node); @@ -752,7 +793,7 @@ void quicklistReplaceEntry(quicklistIter *iter, quicklistEntry *entry, void *dat /* If the entry is not at the tail, split the node at the entry's offset. */ if (entry->offset != node->count - 1 && entry->offset != -1) - split_node = _quicklistSplitNode(node, entry->offset, 1); + split_node = _quicklistSplitNode(quicklist, node, entry->offset, 1); /* Create a new node and insert it after the original node. * If the original node was split, insert the split node after the new node. */ @@ -818,18 +859,27 @@ int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, size_t static quicklistNode *_quicklistListpackMerge(quicklist *quicklist, quicklistNode *a, quicklistNode *b) { D("Requested merge (a,b) (%u, %u)", a->count, b->count); - quicklistDecompressNode(a); - quicklistDecompressNode(b); + quicklistDecompressNode(quicklist, a); + quicklistDecompressNode(quicklist, b); + size_t a_entry_sz = a->entry_alloc_sz; + size_t b_entry_sz = b->entry_alloc_sz; if ((lpMerge(&a->entry, &b->entry))) { /* We merged listpacks! Now remove the unused quicklistNode. */ quicklistNode *keep = NULL, *nokeep = NULL; if (!a->entry) { nokeep = a; keep = b; + /* Update tracked size: b's entry was reallocated by lpMerge */ + quicklistTrackEntryResize(quicklist, keep, b_entry_sz); } else if (!b->entry) { nokeep = b; keep = a; + /* Update tracked size: a's entry was reallocated by lpMerge */ + quicklistTrackEntryResize(quicklist, keep, a_entry_sz); } + /* nokeep->entry was freed by lpMerge (entry is NULL), but we leave + * nokeep->entry_alloc_sz unchanged so __quicklistDelNode subtracts + * the correct amount (entry_alloc_sz + sizeof(node)). */ keep->count = lpLength(keep->entry); quicklistNodeUpdateSz(keep); keep->recompress = 0; /* Prevent 'keep' from being recompressed if @@ -917,11 +967,11 @@ static quicklistNode *_quicklistMergeNodes(quicklist *quicklist, quicklistNode * * The input node keeps all elements not taken by the returned node. * * Returns newly created node or NULL if split not possible. */ -static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, int after) { +static quicklistNode *_quicklistSplitNode(quicklist *quicklist, quicklistNode *node, int offset, int after) { size_t zl_sz = node->sz; quicklistNode *new_node = quicklistCreateNode(); - new_node->entry = zmalloc(zl_sz); + new_node->entry = zmalloc_usable(zl_sz, &new_node->entry_alloc_sz); /* Copy original listpack so we can split it */ memcpy(new_node->entry, node->entry, zl_sz); @@ -937,11 +987,15 @@ static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, int a D("After %d (%d); ranges: [%d, %d], [%d, %d]", after, offset, orig_start, orig_extent, new_start, new_extent); + /* Track entry change for original node (new_node not in list yet) */ + size_t old_sz = node->entry_alloc_sz; node->entry = lpDeleteRange(node->entry, orig_start, orig_extent); + quicklistTrackEntryResize(quicklist, node, old_sz); node->count = lpLength(node->entry); quicklistNodeUpdateSz(node); new_node->entry = lpDeleteRange(new_node->entry, new_start, new_extent); + new_node->entry_alloc_sz = lpLastAllocSize(); new_node->count = lpLength(new_node->entry); quicklistNodeUpdateSz(new_node); @@ -969,6 +1023,7 @@ static void _quicklistInsert(quicklistIter *iter, quicklistEntry *entry, void *v } new_node = quicklistCreateNode(); new_node->entry = lpPrepend(lpNew(0), value, sz); + new_node->entry_alloc_sz = lpLastAllocSize(); __quicklistInsertNode(quicklist, NULL, new_node, after); new_node->count++; quicklist->count++; @@ -1003,8 +1058,8 @@ static void _quicklistInsert(quicklistIter *iter, quicklistEntry *entry, void *v if (QL_NODE_IS_PLAIN(node) || (at_tail && after) || (at_head && !after)) { __quicklistInsertPlainNode(quicklist, node, value, sz, after); } else { - quicklistDecompressNodeForUse(node); - new_node = _quicklistSplitNode(node, entry->offset, after); + quicklistDecompressNodeForUse(quicklist, node); + new_node = _quicklistSplitNode(quicklist, node, entry->offset, after); quicklistNode *entry_node = __quicklistCreateNode(QUICKLIST_NODE_CONTAINER_PLAIN, value, sz); __quicklistInsertNode(quicklist, node, entry_node, after); __quicklistInsertNode(quicklist, entry_node, new_node, after); @@ -1016,46 +1071,55 @@ static void _quicklistInsert(quicklistIter *iter, quicklistEntry *entry, void *v /* Now determine where and how to insert the new element */ if (!full && after) { D("Not full, inserting after current position."); - quicklistDecompressNodeForUse(node); + quicklistDecompressNodeForUse(quicklist, node); + size_t old_sz = node->entry_alloc_sz; node->entry = lpInsertString(node->entry, value, sz, entry->zi, LP_AFTER, NULL); + quicklistTrackEntryResize(quicklist, node, old_sz); node->count++; quicklistNodeUpdateSz(node); - quicklistRecompressOnly(node); + quicklistRecompressOnly(quicklist, node); } else if (!full && !after) { D("Not full, inserting before current position."); - quicklistDecompressNodeForUse(node); + quicklistDecompressNodeForUse(quicklist, node); + size_t old_sz = node->entry_alloc_sz; node->entry = lpInsertString(node->entry, value, sz, entry->zi, LP_BEFORE, NULL); + quicklistTrackEntryResize(quicklist, node, old_sz); node->count++; quicklistNodeUpdateSz(node); - quicklistRecompressOnly(node); + quicklistRecompressOnly(quicklist, node); } else if (full && at_tail && avail_next && after) { /* If we are: at tail, next has free space, and inserting after: * - insert entry at head of next node. */ D("Full and tail, but next isn't full; inserting next node head"); new_node = node->next; - quicklistDecompressNodeForUse(new_node); + quicklistDecompressNodeForUse(quicklist, new_node); + size_t old_sz = new_node->entry_alloc_sz; new_node->entry = lpPrepend(new_node->entry, value, sz); + quicklistTrackEntryResize(quicklist, new_node, old_sz); new_node->count++; quicklistNodeUpdateSz(new_node); - quicklistRecompressOnly(new_node); - quicklistRecompressOnly(node); + quicklistRecompressOnly(quicklist, new_node); + quicklistRecompressOnly(quicklist, node); } else if (full && at_head && avail_prev && !after) { /* If we are: at head, previous has free space, and inserting before: * - insert entry at tail of previous node. */ D("Full and head, but prev isn't full, inserting prev node tail"); new_node = node->prev; - quicklistDecompressNodeForUse(new_node); + quicklistDecompressNodeForUse(quicklist, new_node); + size_t old_sz = new_node->entry_alloc_sz; new_node->entry = lpAppend(new_node->entry, value, sz); + quicklistTrackEntryResize(quicklist, new_node, old_sz); new_node->count++; quicklistNodeUpdateSz(new_node); - quicklistRecompressOnly(new_node); - quicklistRecompressOnly(node); + quicklistRecompressOnly(quicklist, new_node); + quicklistRecompressOnly(quicklist, node); } else if (full && ((at_tail && !avail_next && after) || (at_head && !avail_prev && !after))) { /* If we are: full, and our prev/next has no available space, then: * - create new node and attach to quicklist */ D("\tprovisioning new node..."); new_node = quicklistCreateNode(); new_node->entry = lpPrepend(lpNew(0), value, sz); + new_node->entry_alloc_sz = lpLastAllocSize(); new_node->count++; quicklistNodeUpdateSz(new_node); __quicklistInsertNode(quicklist, node, new_node, after); @@ -1063,12 +1127,14 @@ static void _quicklistInsert(quicklistIter *iter, quicklistEntry *entry, void *v /* else, node is full we need to split it. */ /* covers both after and !after cases */ D("\tsplitting node..."); - quicklistDecompressNodeForUse(node); - new_node = _quicklistSplitNode(node, entry->offset, after); + quicklistDecompressNodeForUse(quicklist, node); + new_node = _quicklistSplitNode(quicklist, node, entry->offset, after); if (after) new_node->entry = lpPrepend(new_node->entry, value, sz); else new_node->entry = lpAppend(new_node->entry, value, sz); + new_node->entry_alloc_sz = lpLastAllocSize(); + /* Don't track delta here; __quicklistInsertNode below adds the full entry_alloc_sz */ new_node->count++; quicklistNodeUpdateSz(new_node); __quicklistInsertNode(quicklist, node, new_node, after); @@ -1157,13 +1223,15 @@ int quicklistDelRange(quicklist *quicklist, const long start, const long count) if (delete_entire_node || QL_NODE_IS_PLAIN(node)) { __quicklistDelNode(quicklist, node); } else { - quicklistDecompressNodeForUse(node); + quicklistDecompressNodeForUse(quicklist, node); + size_t old_sz = node->entry_alloc_sz; node->entry = lpDeleteRange(node->entry, offset, del); + quicklistTrackEntryResize(quicklist, node, old_sz); quicklistNodeUpdateSz(node); node->count -= del; quicklist->count -= del; quicklistDeleteIfEmpty(quicklist, node); - if (node) quicklistRecompressOnly(node); + if (node) quicklistRecompressOnly(quicklist, node); } extent -= del; @@ -1310,7 +1378,7 @@ int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { int plain = QL_NODE_IS_PLAIN(iter->current); if (!iter->zi) { /* If !zi, use current index. */ - quicklistDecompressNodeForUse(iter->current); + quicklistDecompressNodeForUse(iter->quicklist, iter->current); if (unlikely(plain)) iter->zi = iter->current->entry; else @@ -1386,10 +1454,10 @@ quicklist *quicklistDup(quicklist *orig) { if (current->encoding == QUICKLIST_NODE_ENCODING_LZF) { quicklistLZF *lzf = (quicklistLZF *)current->entry; size_t lzf_sz = sizeof(*lzf) + lzf->sz; - node->entry = zmalloc(lzf_sz); + node->entry = zmalloc_usable(lzf_sz, &node->entry_alloc_sz); memcpy(node->entry, current->entry, lzf_sz); } else if (current->encoding == QUICKLIST_NODE_ENCODING_RAW) { - node->entry = zmalloc(current->sz); + node->entry = zmalloc_usable(current->sz, &node->entry_alloc_sz); memcpy(node->entry, current->entry, current->sz); } @@ -1595,7 +1663,7 @@ void quicklistRepr(unsigned char *ql, int full) { node->attempted_compress); if (full) { - quicklistDecompressNode(node); + quicklistDecompressNode(quicklist, node); if (node->container == QUICKLIST_NODE_CONTAINER_PACKED) { printf("{ listpack:\n"); lpRepr(node->entry); @@ -1605,7 +1673,7 @@ void quicklistRepr(unsigned char *ql, int full) { printf("{ entry : %s }\n", node->entry); } printf("}\n"); - quicklistRecompressOnly(node); + quicklistRecompressOnly(quicklist, node); } node = node->next; } @@ -1704,9 +1772,9 @@ quicklistNode *testOnlyQuicklistCreateNodeWithValue(int container, void *value, } int testOnlyQuicklistCompressNode(quicklistNode *node) { - return __quicklistCompressNode(node); + return __quicklistCompressNode(NULL, node); } int testOnlyQuicklistDecompressNode(quicklistNode *node) { - return __quicklistDecompressNode(node); + return __quicklistDecompressNode(NULL, node); } diff --git a/src/quicklist.h b/src/quicklist.h index 4411f823b0a..403aeac96cb 100644 --- a/src/quicklist.h +++ b/src/quicklist.h @@ -49,6 +49,7 @@ typedef struct quicklistNode { struct quicklistNode *next; unsigned char *entry; size_t sz; /* entry size in bytes */ + size_t entry_alloc_sz; /* usable allocation size of entry */ unsigned int count : 16; /* count of items in listpack */ unsigned int encoding : 2; /* RAW==1 or LZF==2 */ unsigned int container : 2; /* PLAIN==1 or PACKED==2 */ @@ -109,6 +110,7 @@ typedef struct quicklist { quicklistNode *tail; unsigned long count; /* total count of all entries in all listpacks */ unsigned long len; /* number of quicklistNodes */ + size_t tracked_size; /* total memory allocated for this quicklist */ signed int fill : QL_FILL_BITS; /* fill factor for individual nodes */ unsigned int compress : QL_COMP_BITS; /* depth of end nodes not to compress;0=off */ unsigned int bookmark_count : QL_BM_BITS; diff --git a/src/unit/test_quicklist_tracking.cpp b/src/unit/test_quicklist_tracking.cpp new file mode 100644 index 00000000000..830cd8eb82d --- /dev/null +++ b/src/unit/test_quicklist_tracking.cpp @@ -0,0 +1,701 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" + +#include +#include +#include +#include + +extern "C" { +#include "listpack.h" +#include "quicklist.h" +#include "server.h" +#include "zmalloc.h" + +size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid); +size_t objectComputeSizeWithTrackedSize(robj *key, robj *o, size_t sample_size, int dbid); +} + +class QuicklistTrackingTest : public ::testing::Test { +}; + +/* Helper: assert tracked_size matches ground-truth objectComputeSize. + * objectComputeSize crashes on empty quicklists (do-while on NULL head), + * so we guard against that by only comparing when the list is non-empty. + * For empty lists, we just verify tracked_size equals sizeof(quicklist). */ +#define ASSERT_TRACKED_SIZE_CORRECT(list) \ + do { \ + quicklist *_ql = (quicklist *)objectGetVal(list); \ + if (_ql->len > 0) { \ + size_t _computed = objectComputeSize(nullptr, (list), SIZE_MAX, 0); \ + size_t _tracked = objectComputeSizeWithTrackedSize(nullptr, (list), SIZE_MAX, 0); \ + ASSERT_EQ(_computed, _tracked); \ + } else { \ + ASSERT_EQ(_ql->tracked_size, sizeof(quicklist)); \ + } \ + } while (0) + +/* 1. Basic pushHead / pushTail into existing nodes */ +TEST_F(QuicklistTrackingTest, BasicPushHeadTail) { + robj *list = createQuicklistObject(3, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 50; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "element_%d", i); + if (i % 2 == 0) + quicklistPushTail(ql, buf, strlen(buf)); + else + quicklistPushHead(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + decrRefCount(list); +} + +/* 2. pushHead / pushTail that create new nodes (fill=1 forces new node per element) */ +TEST_F(QuicklistTrackingTest, PushCreatesNewNodes) { + robj *list = createQuicklistObject(1, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 20; i++) { + char buf[64]; + snprintf(buf, sizeof(buf), "single_node_element_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_EQ(ql->len, 20ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + decrRefCount(list); +} + +/* 3. Pop from head and tail */ +TEST_F(QuicklistTrackingTest, PopHeadAndTail) { + robj *list = createQuicklistObject(3, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 30; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "pop_test_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + for (int i = 0; i < 8; i++) { + unsigned char *data; + size_t sz; + long long val; + quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &val); + if (data) zfree(data); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + for (int i = 0; i < 8; i++) { + unsigned char *data; + size_t sz; + long long val; + quicklistPop(ql, QUICKLIST_TAIL, &data, &sz, &val); + if (data) zfree(data); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 4. Delete entire node (pop all elements) */ +TEST_F(QuicklistTrackingTest, DeleteEntireNode) { + robj *list = createQuicklistObject(2, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 10; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "del_node_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_EQ(ql->len, 5ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + + while (quicklistCount(ql) > 0) { + unsigned char *data; + size_t sz; + long long val; + quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &val); + if (data) zfree(data); + ASSERT_TRACKED_SIZE_CORRECT(list); + } + decrRefCount(list); +} + +/* 5. Compression and decompression */ +TEST_F(QuicklistTrackingTest, CompressDecompress) { + robj *list = createQuicklistObject(-2, 1); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 500; i++) { + char buf[64]; + snprintf(buf, sizeof(buf), "compress_test_value_%d_padding_data", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_GE(ql->len, 3ul); + bool has_compressed = false; + for (quicklistNode *n = ql->head; n; n = n->next) { + if (quicklistNodeIsCompressed(n)) { + has_compressed = true; + break; + } + } + ASSERT_TRUE(has_compressed); + ASSERT_TRACKED_SIZE_CORRECT(list); + + for (int i = 0; i < 50; i++) { + unsigned char *data; + size_t sz; + long long val; + quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &val); + if (data) zfree(data); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + for (int i = 0; i < 50; i++) { + unsigned char *data; + size_t sz; + long long val; + quicklistPop(ql, QUICKLIST_TAIL, &data, &sz, &val); + if (data) zfree(data); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + for (int i = 0; i < 50; i++) { + char buf[64]; + snprintf(buf, sizeof(buf), "after_compress_push_%d", i); + quicklistPushHead(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 6. InsertBefore / InsertAfter into non-full node */ +TEST_F(QuicklistTrackingTest, InsertNonFullNode) { + robj *list = createQuicklistObject(-2, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 20; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "insert_test_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistEntry entry; + quicklistIter *iter = quicklistGetIteratorEntryAtIdx(ql, 5, &entry); + ASSERT_NE(iter, nullptr); + quicklistInsertAfter(iter, &entry, (void *)"INSERTED_AFTER", 14); + quicklistReleaseIterator(iter); + ASSERT_TRACKED_SIZE_CORRECT(list); + + iter = quicklistGetIteratorEntryAtIdx(ql, 10, &entry); + ASSERT_NE(iter, nullptr); + quicklistInsertBefore(iter, &entry, (void *)"INSERTED_BEFORE", 15); + quicklistReleaseIterator(iter); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 7. Insert that triggers node split */ +TEST_F(QuicklistTrackingTest, InsertTriggersSplit) { + robj *list = createQuicklistObject(3, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 12; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "split_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_EQ(ql->len, 4ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistEntry entry; + quicklistIter *iter = quicklistGetIteratorEntryAtIdx(ql, 4, &entry); + ASSERT_NE(iter, nullptr); + quicklistInsertAfter(iter, &entry, (void *)"SPLIT_INSERT", 12); + quicklistReleaseIterator(iter); + ASSERT_GT(ql->len, 4ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 8a. Insert into next neighbor (full node, at_tail, avail_next) */ +TEST_F(QuicklistTrackingTest, InsertNextNeighbor) { + robj *list = createQuicklistObject(4, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 6; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "neighbor_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_EQ(ql->len, 2ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistEntry entry; + quicklistIter *iter = quicklistGetIteratorEntryAtIdx(ql, 3, &entry); + ASSERT_NE(iter, nullptr); + quicklistInsertAfter(iter, &entry, (void *)"NEIGHBOR_NEXT", 13); + quicklistReleaseIterator(iter); + ASSERT_EQ(ql->len, 2ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 8b. Insert into prev neighbor (full node, at_head, avail_prev) */ +TEST_F(QuicklistTrackingTest, InsertPrevNeighbor) { + robj *list = createQuicklistObject(4, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 8; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "avprev_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_EQ(ql->len, 2ul); + unsigned char *data; + size_t sz; + long long val; + quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &val); + if (data) zfree(data); + ASSERT_EQ(ql->head->count, 3u); + ASSERT_EQ(ql->tail->count, 4u); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistEntry entry; + quicklistIter *iter = quicklistGetIteratorEntryAtIdx(ql, 3, &entry); + ASSERT_NE(iter, nullptr); + quicklistInsertBefore(iter, &entry, (void *)"NEIGHBOR_PREV", 13); + quicklistReleaseIterator(iter); + ASSERT_EQ(ql->len, 2ul); + ASSERT_EQ(ql->head->count, 4u); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 9. ReplaceEntry - listpack path (lpReplace) */ +TEST_F(QuicklistTrackingTest, ReplaceEntryListpack) { + robj *list = createQuicklistObject(-2, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 20; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "replace_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistEntry entry; + quicklistIter *iter = quicklistGetIteratorEntryAtIdx(ql, 5, &entry); + ASSERT_NE(iter, nullptr); + quicklistReplaceEntry(iter, &entry, (void *)"S", 1); + quicklistReleaseIterator(iter); + ASSERT_TRACKED_SIZE_CORRECT(list); + + iter = quicklistGetIteratorEntryAtIdx(ql, 10, &entry); + ASSERT_NE(iter, nullptr); + char big_replace[200]; + memset(big_replace, 'X', sizeof(big_replace)); + quicklistReplaceEntry(iter, &entry, big_replace, sizeof(big_replace)); + quicklistReleaseIterator(iter); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 10. ReplaceEntry - plain node path */ +TEST_F(QuicklistTrackingTest, ReplaceEntryPlainNode) { + quicklistSetPackedThreshold(64); + + robj *list = createQuicklistObject(-2, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + char large[128]; + memset(large, 'A', sizeof(large)); + quicklistPushTail(ql, large, sizeof(large)); + for (int i = 0; i < 10; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "mixed_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistEntry entry; + quicklistIter *iter = quicklistGetIteratorEntryAtIdx(ql, 0, &entry); + ASSERT_NE(iter, nullptr); + ASSERT_TRUE(QL_NODE_IS_PLAIN(entry.node)); + char new_large[256]; + memset(new_large, 'B', sizeof(new_large)); + quicklistReplaceEntry(iter, &entry, new_large, sizeof(new_large)); + quicklistReleaseIterator(iter); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistSetPackedThreshold(0); + decrRefCount(list); +} + +/* 11. DelRange - partial and full node deletion */ +TEST_F(QuicklistTrackingTest, DelRange) { + robj *list = createQuicklistObject(5, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 30; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "delrange_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistDelRange(ql, 3, 2); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistDelRange(ql, 2, 15); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistDelRange(ql, 0, 3); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistDelRange(ql, -3, 3); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 12. Rotate (move tail to head) */ +TEST_F(QuicklistTrackingTest, Rotate) { + robj *list = createQuicklistObject(3, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 15; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "rotate_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + for (int i = 0; i < 10; i++) { + quicklistRotate(ql); + ASSERT_TRACKED_SIZE_CORRECT(list); + } + + decrRefCount(list); +} + +/* 13. Dup (deep copy) */ +TEST_F(QuicklistTrackingTest, Dup) { + robj *list = createQuicklistObject(-2, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 100; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "dup_test_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklist *copy = quicklistDup(ql); + robj *copy_list = createObject(OBJ_LIST, copy); + copy_list->encoding = OBJ_ENCODING_QUICKLIST; + ASSERT_TRACKED_SIZE_CORRECT(copy_list); + + for (int i = 0; i < 20; i++) { + unsigned char *data; + size_t sz; + long long val; + quicklistPop(copy, QUICKLIST_TAIL, &data, &sz, &val); + if (data) zfree(data); + } + ASSERT_TRACKED_SIZE_CORRECT(copy_list); + + decrRefCount(copy_list); + decrRefCount(list); +} + +/* 14. Dup with compression */ +TEST_F(QuicklistTrackingTest, DupWithCompression) { + robj *list = createQuicklistObject(-2, 1); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 500; i++) { + char buf[64]; + snprintf(buf, sizeof(buf), "dup_compress_%d_padding_data_here", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklist *copy = quicklistDup(ql); + robj *copy_list = createObject(OBJ_LIST, copy); + copy_list->encoding = OBJ_ENCODING_QUICKLIST; + ASSERT_TRACKED_SIZE_CORRECT(copy_list); + + decrRefCount(copy_list); + decrRefCount(list); +} + +/* 15. Node merge via _quicklistListpackMerge */ +TEST_F(QuicklistTrackingTest, NodeMerge) { + robj *list = createQuicklistObject(4, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 12; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "merge_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_EQ(ql->len, 3ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistDelRange(ql, 4, 3); + ASSERT_EQ(ql->len, 3ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistDelRange(ql, 5, 3); + ASSERT_EQ(ql->len, 3ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + + unsigned long nodes_before = ql->len; + + quicklistEntry entry; + quicklistIter *iter = quicklistGetIteratorEntryAtIdx(ql, 2, &entry); + ASSERT_NE(iter, nullptr); + quicklistInsertAfter(iter, &entry, (void *)"MERGE_TRIGGER", 13); + quicklistReleaseIterator(iter); + + ASSERT_LE(ql->len, nodes_before + 1); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 16. Plain node insert */ +TEST_F(QuicklistTrackingTest, PlainNodeInsert) { + quicklistSetPackedThreshold(64); + + robj *list = createQuicklistObject(-2, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 5; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "small_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + for (int i = 0; i < 5; i++) { + char large[128]; + memset(large, 'P' + i, sizeof(large)); + quicklistPushTail(ql, large, sizeof(large)); + ASSERT_TRACKED_SIZE_CORRECT(list); + } + + for (int i = 0; i < 3; i++) { + unsigned char *data; + size_t sz; + long long val; + quicklistPop(ql, QUICKLIST_TAIL, &data, &sz, &val); + if (data) zfree(data); + ASSERT_TRACKED_SIZE_CORRECT(list); + } + + quicklistSetPackedThreshold(0); + decrRefCount(list); +} + +/* 17. Insert into empty list (no reference node) */ +TEST_F(QuicklistTrackingTest, InsertEmptyList) { + robj *list = createQuicklistObject(-2, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + quicklistEntry entry; + memset(&entry, 0, sizeof(entry)); + entry.quicklist = ql; + quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); + quicklistInsertAfter(iter, &entry, (void *)"first_ever", 10); + quicklistReleaseIterator(iter); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 18. Compression with operations interleaved */ +TEST_F(QuicklistTrackingTest, InterleavedCompressOps) { + robj *list = createQuicklistObject(-2, 2); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 1000; i++) { + char buf[64]; + snprintf(buf, sizeof(buf), "interleaved_%d_extra_padding_here", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + for (int i = 0; i < 100; i++) { + char buf[64]; + snprintf(buf, sizeof(buf), "interleaved_new_%d", i); + if (i % 2 == 0) + quicklistPushHead(ql, buf, strlen(buf)); + else + quicklistPushTail(ql, buf, strlen(buf)); + + unsigned char *data; + size_t sz; + long long val; + if (i % 3 == 0) + quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &val); + else + quicklistPop(ql, QUICKLIST_TAIL, &data, &sz, &val); + if (data) zfree(data); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + for (int i = 0; i < 10; i++) { + quicklistEntry entry; + quicklistIter *iter = quicklistGetIteratorEntryAtIdx(ql, (long)quicklistCount(ql) / 2, &entry); + if (iter) { + char buf[100]; + memset(buf, 'R', sizeof(buf)); + quicklistReplaceEntry(iter, &entry, buf, sizeof(buf)); + quicklistReleaseIterator(iter); + } + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 19. AppendListpack / AppendPlainNode (RDB load paths) */ +TEST_F(QuicklistTrackingTest, AppendListpackAndPlainNode) { + robj *list = createQuicklistObject(-2, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + unsigned char *lp = lpNew(0); + for (int i = 0; i < 10; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "rdb_lp_%d", i); + lp = lpAppend(lp, (unsigned char *)buf, strlen(buf)); + } + quicklistAppendListpack(ql, lp); + ASSERT_TRACKED_SIZE_CORRECT(list); + + unsigned char *lp2 = lpNew(0); + for (int i = 0; i < 5; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "rdb_lp2_%d", i); + lp2 = lpAppend(lp2, (unsigned char *)buf, strlen(buf)); + } + quicklistAppendListpack(ql, lp2); + ASSERT_TRACKED_SIZE_CORRECT(list); + + size_t plain_sz = 256; + unsigned char *plain_data = (unsigned char *)zmalloc(plain_sz); + memset(plain_data, 'Z', plain_sz); + quicklistAppendPlainNode(ql, plain_data, plain_sz); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 20. DelEntry via iterator */ +TEST_F(QuicklistTrackingTest, DelEntryViaIterator) { + robj *list = createQuicklistObject(5, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 25; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "delentry_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); + quicklistEntry entry; + int count = 0; + while (quicklistNext(iter, &entry)) { + if (count % 2 == 0) { + quicklistDelEntry(iter, &entry); + } + count++; + } + quicklistReleaseIterator(iter); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 21. ReplaceAtIndex */ +TEST_F(QuicklistTrackingTest, ReplaceAtIndex) { + robj *list = createQuicklistObject(-2, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + for (int i = 0; i < 30; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "replaceatidx_%d", i); + quicklistPushTail(ql, buf, strlen(buf)); + } + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistReplaceAtIndex(ql, 0, (void *)"HEAD_REPLACED", 13); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistReplaceAtIndex(ql, 15, (void *)"MID_REPLACED", 12); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistReplaceAtIndex(ql, 29, (void *)"TAIL_REPLACED", 13); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +} + +/* 23. Same-size replacement: ensures lpLastAllocSize is not stale. + * When lpReplace replaces an element with one of the exact same encoded + * size, lpInsert may skip lp_realloc entirely, which used to leave + * lp_last_alloc_size holding a value from a previous (different node) + * operation. This test creates two nodes, mutates node A (changing + * lp_last_alloc_size), then does a same-size replace on node B to + * verify that tracked_size remains accurate. */ +TEST_F(QuicklistTrackingTest, SameSizeReplaceNoStale) { + /* fill=4 to create multiple nodes quickly. */ + robj *list = createQuicklistObject(4, 0); + quicklist *ql = (quicklist *)objectGetVal(list); + + /* Fill two nodes: 4 entries of "aaaa" each. */ + for (int i = 0; i < 8; i++) { + quicklistPushTail(ql, (void *)"aaaa", 4); + } + ASSERT_EQ(ql->len, 2ul); + ASSERT_TRACKED_SIZE_CORRECT(list); + + /* Mutate node A (head) with a differently-sized value to change + * lpLastAllocSize to something different from node B's alloc size. */ + quicklistReplaceAtIndex(ql, 0, (void *)"bbbbbbbbbbbbbbbb", 16); + ASSERT_TRACKED_SIZE_CORRECT(list); + + /* Now do a same-size replace on node B (tail): replace "aaaa" with + * "cccc" (same 4 bytes). lpInsert will compute new == old bytes + * and may skip lp_realloc. If lp_last_alloc_size is stale from + * the head node's operation, tracked_size will become wrong. */ + quicklistReplaceAtIndex(ql, 7, (void *)"cccc", 4); + ASSERT_TRACKED_SIZE_CORRECT(list); + + /* Also test multiple same-size replacements in a row. */ + quicklistReplaceAtIndex(ql, 5, (void *)"dddd", 4); + ASSERT_TRACKED_SIZE_CORRECT(list); + + quicklistReplaceAtIndex(ql, 6, (void *)"eeee", 4); + ASSERT_TRACKED_SIZE_CORRECT(list); + + decrRefCount(list); +}