diff options
Diffstat (limited to 'meta-oe/recipes-extended/redis/redis/CVE-2021-32627-CVE-2021-32628.patch')
| -rw-r--r-- | meta-oe/recipes-extended/redis/redis/CVE-2021-32627-CVE-2021-32628.patch | 873 |
1 files changed, 873 insertions, 0 deletions
diff --git a/meta-oe/recipes-extended/redis/redis/CVE-2021-32627-CVE-2021-32628.patch b/meta-oe/recipes-extended/redis/redis/CVE-2021-32627-CVE-2021-32628.patch new file mode 100644 index 0000000000..3c60a3e678 --- /dev/null +++ b/meta-oe/recipes-extended/redis/redis/CVE-2021-32627-CVE-2021-32628.patch @@ -0,0 +1,873 @@ +From 2775a3526e3e8bb040e72995231632c801977395 Mon Sep 17 00:00:00 2001 +From: Oran Agra <oran@redislabs.com> +Date: Thu, 3 Jun 2021 12:10:02 +0300 +Subject: [PATCH] Fix ziplist and listpack overflows and truncations + (CVE-2021-32627, CVE-2021-32628) + +- fix possible heap corruption in ziplist and listpack resulting by trying to + allocate more than the maximum size of 4GB. +- prevent ziplist (hash and zset) from reaching size of above 1GB, will be + converted to HT encoding, that's not a useful size. +- prevent listpack (stream) from reaching size of above 1GB. +- XADD will start a new listpack if the new record may cause the previous + listpack to grow over 1GB. +- XADD will respond with an error if a single stream record is over 1GB +- List type (ziplist in quicklist) was truncating strings that were over 4GB, + now it'll respond with an error. + +CVE: CVE-2021-32627,CVE-2021-32628 +Upstream-Status: Backport[https://github.com/redis/redis/commit/f6a40570fa63d5afdd596c78083d754081d80ae3] + +Signed-off-by: Changqing Li <changqing.li@windriver.com> + +--- + src/geo.c | 5 +- + src/listpack.c | 2 +- + src/module.c | 6 +- + src/quicklist.c | 16 +++- + src/rdb.c | 45 +++++++---- + src/server.h | 2 +- + src/t_hash.c | 13 +++- + src/t_list.c | 29 +++++++ + src/t_stream.c | 48 +++++++++--- + src/t_zset.c | 62 +++++++++------ + src/ziplist.c | 17 ++++- + src/ziplist.h | 1 + + tests/unit/violations.tcl | 156 ++++++++++++++++++++++++++++++++++++++ + 13 files changed, 341 insertions(+), 61 deletions(-) + create mode 100644 tests/unit/violations.tcl + +diff --git a/src/geo.c b/src/geo.c +index 7c75738a2..893f78a7e 100644 +--- a/src/geo.c ++++ b/src/geo.c +@@ -770,7 +770,7 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) { + robj *zobj; + zset *zs; + int i; +- size_t maxelelen = 0; ++ size_t maxelelen = 0, totelelen = 0; + + if (returned_items) { + zobj = createZsetObject(); +@@ -785,13 +785,14 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) { + size_t elelen = sdslen(gp->member); + + if (maxelelen < elelen) maxelelen = elelen; ++ totelelen += elelen; + znode = zslInsert(zs->zsl,score,gp->member); + serverAssert(dictAdd(zs->dict,gp->member,&znode->score) == DICT_OK); + gp->member = NULL; + } + + if (returned_items) { +- zsetConvertToZiplistIfNeeded(zobj,maxelelen); ++ zsetConvertToZiplistIfNeeded(zobj,maxelelen,totelelen); + setKey(c,c->db,storekey,zobj); + decrRefCount(zobj); + notifyKeyspaceEvent(NOTIFY_ZSET,flags & GEOSEARCH ? "geosearchstore" : "georadiusstore",storekey, +diff --git a/src/listpack.c b/src/listpack.c +index ee256bad3..27622d4a5 100644 +--- a/src/listpack.c ++++ b/src/listpack.c +@@ -313,7 +313,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui + } else { + if (size < 64) *enclen = 1+size; + else if (size < 4096) *enclen = 2+size; +- else *enclen = 5+size; ++ else *enclen = 5+(uint64_t)size; + return LP_ENCODING_STRING; + } + } +diff --git a/src/module.c b/src/module.c +index bf6580a60..adca9dc9c 100644 +--- a/src/module.c ++++ b/src/module.c +@@ -3319,6 +3319,7 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) { + * - EDOM if the given ID was 0-0 or not greater than all other IDs in the + * stream (only if the AUTOID flag is unset) + * - EFBIG if the stream has reached the last possible ID ++ * - ERANGE if the elements are too large to be stored. + */ + int RM_StreamAdd(RedisModuleKey *key, int flags, RedisModuleStreamID *id, RedisModuleString **argv, long numfields) { + /* Validate args */ +@@ -3362,8 +3363,9 @@ int RM_StreamAdd(RedisModuleKey *key, int flags, RedisModuleStreamID *id, RedisM + use_id_ptr = &use_id; + } + if (streamAppendItem(s, argv, numfields, &added_id, use_id_ptr) == C_ERR) { +- /* ID not greater than all existing IDs in the stream */ +- errno = EDOM; ++ /* Either the ID not greater than all existing IDs in the stream, or ++ * the elements are too large to be stored. either way, errno is already ++ * set by streamAppendItem. */ + return REDISMODULE_ERR; + } + /* Postponed signalKeyAsReady(). Done implicitly by moduleCreateEmptyKey() +diff --git a/src/quicklist.c b/src/quicklist.c +index 5a1e41dcc..a9f8b43b1 100644 +--- a/src/quicklist.c ++++ b/src/quicklist.c +@@ -45,11 +45,16 @@ + #define REDIS_STATIC static + #endif + +-/* Optimization levels for size-based filling */ ++/* Optimization levels for size-based filling. ++ * Note that the largest possible limit is 16k, so even if each record takes ++ * just one byte, it still won't overflow the 16 bit count field. */ + static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536}; + + /* Maximum size in bytes of any multi-element ziplist. +- * Larger values will live in their own isolated ziplists. */ ++ * Larger values will live in their own isolated ziplists. ++ * This is used only if we're limited by record count. when we're limited by ++ * size, the maximum limit is bigger, but still safe. ++ * 8k is a recommended / default size limit */ + #define SIZE_SAFETY_LIMIT 8192 + + /* Minimum ziplist size in bytes for attempting compression. */ +@@ -444,6 +449,8 @@ REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node, + unsigned int new_sz = node->sz + sz + ziplist_overhead; + if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill))) + return 1; ++ /* when we return 1 above we know that the limit is a size limit (which is ++ * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */ + else if (!sizeMeetsSafetyLimit(new_sz)) + return 0; + else if ((int)node->count < fill) +@@ -463,6 +470,8 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a, + unsigned int merge_sz = a->sz + b->sz - 11; + if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill))) + return 1; ++ /* when we return 1 above we know that the limit is a size limit (which is ++ * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */ + else if (!sizeMeetsSafetyLimit(merge_sz)) + return 0; + else if ((int)(a->count + b->count) <= fill) +@@ -482,6 +491,7 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a, + * Returns 1 if new head created. */ + int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) { + quicklistNode *orig_head = quicklist->head; ++ assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */ + if (likely( + _quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz))) { + quicklist->head->zl = +@@ -505,6 +515,7 @@ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) { + * Returns 1 if new tail created. */ + int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) { + quicklistNode *orig_tail = quicklist->tail; ++ assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */ + if (likely( + _quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz))) { + quicklist->tail->zl = +@@ -847,6 +858,7 @@ REDIS_STATIC void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry, + int fill = quicklist->fill; + quicklistNode *node = entry->node; + quicklistNode *new_node = NULL; ++ assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */ + + if (!node) { + /* we have no reference node, so let's create only node in the list */ +diff --git a/src/rdb.c b/src/rdb.c +index 53f67a72e..5456c1d80 100644 +--- a/src/rdb.c ++++ b/src/rdb.c +@@ -1625,7 +1625,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { + } else if (rdbtype == RDB_TYPE_ZSET_2 || rdbtype == RDB_TYPE_ZSET) { + /* Read list/set value. */ + uint64_t zsetlen; +- size_t maxelelen = 0; ++ size_t maxelelen = 0, totelelen = 0; + zset *zs; + + if ((zsetlen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; +@@ -1665,6 +1665,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { + + /* Don't care about integer-encoded strings. */ + if (sdslen(sdsele) > maxelelen) maxelelen = sdslen(sdsele); ++ totelelen += sdslen(sdsele); + + znode = zslInsert(zs->zsl,score,sdsele); + if (dictAdd(zs->dict,sdsele,&znode->score) != DICT_OK) { +@@ -1677,8 +1678,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { + + /* Convert *after* loading, since sorted sets are not stored ordered. */ + if (zsetLength(o) <= server.zset_max_ziplist_entries && +- maxelelen <= server.zset_max_ziplist_value) +- zsetConvert(o,OBJ_ENCODING_ZIPLIST); ++ maxelelen <= server.zset_max_ziplist_value && ++ ziplistSafeToAdd(NULL, totelelen)) ++ { ++ zsetConvert(o,OBJ_ENCODING_ZIPLIST); ++ } + } else if (rdbtype == RDB_TYPE_HASH) { + uint64_t len; + int ret; +@@ -1731,21 +1735,30 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { + } + } + +- /* Add pair to ziplist */ +- o->ptr = ziplistPush(o->ptr, (unsigned char*)field, +- sdslen(field), ZIPLIST_TAIL); +- o->ptr = ziplistPush(o->ptr, (unsigned char*)value, +- sdslen(value), ZIPLIST_TAIL); +- + /* Convert to hash table if size threshold is exceeded */ + if (sdslen(field) > server.hash_max_ziplist_value || +- sdslen(value) > server.hash_max_ziplist_value) ++ sdslen(value) > server.hash_max_ziplist_value || ++ !ziplistSafeToAdd(o->ptr, sdslen(field)+sdslen(value))) + { +- sdsfree(field); +- sdsfree(value); + hashTypeConvert(o, OBJ_ENCODING_HT); ++ ret = dictAdd((dict*)o->ptr, field, value); ++ if (ret == DICT_ERR) { ++ rdbReportCorruptRDB("Duplicate hash fields detected"); ++ if (dupSearchDict) dictRelease(dupSearchDict); ++ sdsfree(value); ++ sdsfree(field); ++ decrRefCount(o); ++ return NULL; ++ } + break; + } ++ ++ /* Add pair to ziplist */ ++ o->ptr = ziplistPush(o->ptr, (unsigned char*)field, ++ sdslen(field), ZIPLIST_TAIL); ++ o->ptr = ziplistPush(o->ptr, (unsigned char*)value, ++ sdslen(value), ZIPLIST_TAIL); ++ + sdsfree(field); + sdsfree(value); + } +@@ -1858,12 +1871,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { + while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) { + if (flen > maxlen) maxlen = flen; + if (vlen > maxlen) maxlen = vlen; +- zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL); +- zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL); + + /* search for duplicate records */ + sds field = sdstrynewlen(fstr, flen); +- if (!field || dictAdd(dupSearchDict, field, NULL) != DICT_OK) { ++ if (!field || dictAdd(dupSearchDict, field, NULL) != DICT_OK || ++ !ziplistSafeToAdd(zl, (size_t)flen + vlen)) { + rdbReportCorruptRDB("Hash zipmap with dup elements, or big length (%u)", flen); + dictRelease(dupSearchDict); + sdsfree(field); +@@ -1872,6 +1884,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { + decrRefCount(o); + return NULL; + } ++ ++ zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL); ++ zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL); + } + + dictRelease(dupSearchDict); +diff --git a/src/server.h b/src/server.h +index d9fef9552..07b34c743 100644 +--- a/src/server.h ++++ b/src/server.h +@@ -2173,7 +2173,7 @@ unsigned char *zzlFirstInRange(unsigned char *zl, zrangespec *range); + unsigned char *zzlLastInRange(unsigned char *zl, zrangespec *range); + unsigned long zsetLength(const robj *zobj); + void zsetConvert(robj *zobj, int encoding); +-void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen); ++void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen); + int zsetScore(robj *zobj, sds member, double *score); + unsigned long zslGetRank(zskiplist *zsl, double score, sds o); + int zsetAdd(robj *zobj, double score, sds ele, int in_flags, int *out_flags, double *newscore); +diff --git a/src/t_hash.c b/src/t_hash.c +index ea0606fb0..2720fdbc7 100644 +--- a/src/t_hash.c ++++ b/src/t_hash.c +@@ -39,17 +39,22 @@ + * as their string length can be queried in constant time. */ + void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { + int i; ++ size_t sum = 0; + + if (o->encoding != OBJ_ENCODING_ZIPLIST) return; + + for (i = start; i <= end; i++) { +- if (sdsEncodedObject(argv[i]) && +- sdslen(argv[i]->ptr) > server.hash_max_ziplist_value) +- { ++ if (!sdsEncodedObject(argv[i])) ++ continue; ++ size_t len = sdslen(argv[i]->ptr); ++ if (len > server.hash_max_ziplist_value) { + hashTypeConvert(o, OBJ_ENCODING_HT); +- break; ++ return; + } ++ sum += len; + } ++ if (!ziplistSafeToAdd(o->ptr, sum)) ++ hashTypeConvert(o, OBJ_ENCODING_HT); + } + + /* Get the value from a ziplist encoded hash, identified by field. +diff --git a/src/t_list.c b/src/t_list.c +index f8ca27458..66c9e3c9d 100644 +--- a/src/t_list.c ++++ b/src/t_list.c +@@ -29,6 +29,8 @@ + + #include "server.h" + ++#define LIST_MAX_ITEM_SIZE ((1ull<<32)-1024) ++ + /*----------------------------------------------------------------------------- + * List API + *----------------------------------------------------------------------------*/ +@@ -224,6 +226,13 @@ robj *listTypeDup(robj *o) { + void pushGenericCommand(client *c, int where, int xx) { + int j; + ++ for (j = 2; j < c->argc; j++) { ++ if (sdslen(c->argv[j]->ptr) > LIST_MAX_ITEM_SIZE) { ++ addReplyError(c, "Element too large"); ++ return; ++ } ++ } ++ + robj *lobj = lookupKeyWrite(c->db, c->argv[1]); + if (checkType(c,lobj,OBJ_LIST)) return; + if (!lobj) { +@@ -287,6 +296,11 @@ void linsertCommand(client *c) { + return; + } + ++ if (sdslen(c->argv[4]->ptr) > LIST_MAX_ITEM_SIZE) { ++ addReplyError(c, "Element too large"); ++ return; ++ } ++ + if ((subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || + checkType(c,subject,OBJ_LIST)) return; + +@@ -354,6 +368,11 @@ void lsetCommand(client *c) { + long index; + robj *value = c->argv[3]; + ++ if (sdslen(value->ptr) > LIST_MAX_ITEM_SIZE) { ++ addReplyError(c, "Element too large"); ++ return; ++ } ++ + if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != C_OK)) + return; + +@@ -576,6 +595,11 @@ void lposCommand(client *c) { + int direction = LIST_TAIL; + long rank = 1, count = -1, maxlen = 0; /* Count -1: option not given. */ + ++ if (sdslen(ele->ptr) > LIST_MAX_ITEM_SIZE) { ++ addReplyError(c, "Element too large"); ++ return; ++ } ++ + /* Parse the optional arguments. */ + for (int j = 3; j < c->argc; j++) { + char *opt = c->argv[j]->ptr; +@@ -671,6 +695,11 @@ void lremCommand(client *c) { + long toremove; + long removed = 0; + ++ if (sdslen(obj->ptr) > LIST_MAX_ITEM_SIZE) { ++ addReplyError(c, "Element too large"); ++ return; ++ } ++ + if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != C_OK)) + return; + +diff --git a/src/t_stream.c b/src/t_stream.c +index 2c30faa06..574195ee3 100644 +--- a/src/t_stream.c ++++ b/src/t_stream.c +@@ -47,6 +47,12 @@ + * setting stream_node_max_bytes to a huge number. */ + #define STREAM_LISTPACK_MAX_PRE_ALLOCATE 4096 + ++/* Don't let listpacks grow too big, even if the user config allows it. ++ * doing so can lead to an overflow (trying to store more than 32bit length ++ * into the listpack header), or actually an assertion since lpInsert ++ * will return NULL. */ ++#define STREAM_LISTPACK_MAX_SIZE (1<<30) ++ + void streamFreeCG(streamCG *cg); + void streamFreeNACK(streamNACK *na); + size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start, streamID *end, size_t count, streamConsumer *consumer); +@@ -433,8 +439,11 @@ void streamGetEdgeID(stream *s, int first, streamID *edge_id) + * + * The function returns C_OK if the item was added, this is always true + * if the ID was generated by the function. However the function may return +- * C_ERR if an ID was given via 'use_id', but adding it failed since the +- * current top ID is greater or equal. */ ++ * C_ERR in several cases: ++ * 1. If an ID was given via 'use_id', but adding it failed since the ++ * current top ID is greater or equal. errno will be set to EDOM. ++ * 2. If a size of a single element or the sum of the elements is too big to ++ * be stored into the stream. errno will be set to ERANGE. */ + int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id) { + + /* Generate the new entry ID. */ +@@ -448,7 +457,23 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ + * or return an error. Automatically generated IDs might + * overflow (and wrap-around) when incrementing the sequence + part. */ +- if (streamCompareID(&id,&s->last_id) <= 0) return C_ERR; ++ if (streamCompareID(&id,&s->last_id) <= 0) { ++ errno = EDOM; ++ return C_ERR; ++ } ++ ++ /* Avoid overflow when trying to add an element to the stream (listpack ++ * can only host up to 32bit length sttrings, and also a total listpack size ++ * can't be bigger than 32bit length. */ ++ size_t totelelen = 0; ++ for (int64_t i = 0; i < numfields*2; i++) { ++ sds ele = argv[i]->ptr; ++ totelelen += sdslen(ele); ++ } ++ if (totelelen > STREAM_LISTPACK_MAX_SIZE) { ++ errno = ERANGE; ++ return C_ERR; ++ } + + /* Add the new entry. */ + raxIterator ri; +@@ -507,9 +532,10 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ + * if we need to switch to the next one. 'lp' will be set to NULL if + * the current node is full. */ + if (lp != NULL) { +- if (server.stream_node_max_bytes && +- lp_bytes >= server.stream_node_max_bytes) +- { ++ size_t node_max_bytes = server.stream_node_max_bytes; ++ if (node_max_bytes == 0 || node_max_bytes > STREAM_LISTPACK_MAX_SIZE) ++ node_max_bytes = STREAM_LISTPACK_MAX_SIZE; ++ if (lp_bytes + totelelen >= node_max_bytes) { + lp = NULL; + } else if (server.stream_node_max_entries) { + unsigned char *lp_ele = lpFirst(lp); +@@ -1796,11 +1822,13 @@ void xaddCommand(client *c) { + /* Append using the low level function and return the ID. */ + streamID id; + if (streamAppendItem(s,c->argv+field_pos,(c->argc-field_pos)/2, +- &id, parsed_args.id_given ? &parsed_args.id : NULL) +- == C_ERR) ++ &id, parsed_args.id_given ? &parsed_args.id : NULL) == C_ERR) + { +- addReplyError(c,"The ID specified in XADD is equal or smaller than the " +- "target stream top item"); ++ if (errno == EDOM) ++ addReplyError(c,"The ID specified in XADD is equal or smaller than " ++ "the target stream top item"); ++ else ++ addReplyError(c,"Elements are too large to be stored"); + return; + } + addReplyStreamID(c,&id); +diff --git a/src/t_zset.c b/src/t_zset.c +index 3b9ebd2bd..2abc1b49b 100644 +--- a/src/t_zset.c ++++ b/src/t_zset.c +@@ -1242,15 +1242,18 @@ void zsetConvert(robj *zobj, int encoding) { + } + + /* Convert the sorted set object into a ziplist if it is not already a ziplist +- * and if the number of elements and the maximum element size is within the +- * expected ranges. */ +-void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen) { ++ * and if the number of elements and the maximum element size and total elements size ++ * are within the expected ranges. */ ++void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen) { + if (zobj->encoding == OBJ_ENCODING_ZIPLIST) return; + zset *zset = zobj->ptr; + + if (zset->zsl->length <= server.zset_max_ziplist_entries && +- maxelelen <= server.zset_max_ziplist_value) +- zsetConvert(zobj,OBJ_ENCODING_ZIPLIST); ++ maxelelen <= server.zset_max_ziplist_value && ++ ziplistSafeToAdd(NULL, totelelen)) ++ { ++ zsetConvert(zobj,OBJ_ENCODING_ZIPLIST); ++ } + } + + /* Return (by reference) the score of the specified member of the sorted set +@@ -1370,20 +1373,28 @@ int zsetAdd(robj *zobj, double score, sds ele, int in_flags, int *out_flags, dou + } + return 1; + } else if (!xx) { +- /* Optimize: check if the element is too large or the list ++ /* check if the element is too large or the list + * becomes too long *before* executing zzlInsert. */ +- zobj->ptr = zzlInsert(zobj->ptr,ele,score); +- if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries || +- sdslen(ele) > server.zset_max_ziplist_value) ++ if (zzlLength(zobj->ptr)+1 > server.zset_max_ziplist_entries || ++ sdslen(ele) > server.zset_max_ziplist_value || ++ !ziplistSafeToAdd(zobj->ptr, sdslen(ele))) ++ { + zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); +- if (newscore) *newscore = score; +- *out_flags |= ZADD_OUT_ADDED; +- return 1; ++ } else { ++ zobj->ptr = zzlInsert(zobj->ptr,ele,score); ++ if (newscore) *newscore = score; ++ *out_flags |= ZADD_OUT_ADDED; ++ return 1; ++ } + } else { + *out_flags |= ZADD_OUT_NOP; + return 1; + } +- } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) { ++ } ++ ++ /* Note that the above block handling ziplist would have either returned or ++ * converted the key to skiplist. */ ++ if (zobj->encoding == OBJ_ENCODING_SKIPLIST) { + zset *zs = zobj->ptr; + zskiplistNode *znode; + dictEntry *de; +@@ -2361,7 +2372,7 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat + } + } + +-static int zsetDictGetMaxElementLength(dict *d) { ++static size_t zsetDictGetMaxElementLength(dict *d, size_t *totallen) { + dictIterator *di; + dictEntry *de; + size_t maxelelen = 0; +@@ -2371,6 +2382,8 @@ static int zsetDictGetMaxElementLength(dict *d) { + while((de = dictNext(di)) != NULL) { + sds ele = dictGetKey(de); + if (sdslen(ele) > maxelelen) maxelelen = sdslen(ele); ++ if (totallen) ++ (*totallen) += sdslen(ele); + } + + dictReleaseIterator(di); +@@ -2378,7 +2391,7 @@ static int zsetDictGetMaxElementLength(dict *d) { + return maxelelen; + } + +-static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) { ++static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen, size_t *totelelen) { + /* DIFF Algorithm 1: + * + * We perform the diff by iterating all the elements of the first set, +@@ -2426,13 +2439,14 @@ static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t * + znode = zslInsert(dstzset->zsl,zval.score,tmp); + dictAdd(dstzset->dict,tmp,&znode->score); + if (sdslen(tmp) > *maxelelen) *maxelelen = sdslen(tmp); ++ (*totelelen) += sdslen(tmp); + } + } + zuiClearIterator(&src[0]); + } + + +-static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) { ++static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen, size_t *totelelen) { + /* DIFF Algorithm 2: + * + * Add all the elements of the first set to the auxiliary set. +@@ -2486,7 +2500,7 @@ static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t * + + /* Using this algorithm, we can't calculate the max element as we go, + * we have to iterate through all elements to find the max one after. */ +- *maxelelen = zsetDictGetMaxElementLength(dstzset->dict); ++ *maxelelen = zsetDictGetMaxElementLength(dstzset->dict, totelelen); + } + + static int zsetChooseDiffAlgorithm(zsetopsrc *src, long setnum) { +@@ -2523,14 +2537,14 @@ static int zsetChooseDiffAlgorithm(zsetopsrc *src, long setnum) { + return (algo_one_work <= algo_two_work) ? 1 : 2; + } + +-static void zdiff(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) { ++static void zdiff(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen, size_t *totelelen) { + /* Skip everything if the smallest input is empty. */ + if (zuiLength(&src[0]) > 0) { + int diff_algo = zsetChooseDiffAlgorithm(src, setnum); + if (diff_algo == 1) { +- zdiffAlgorithm1(src, setnum, dstzset, maxelelen); ++ zdiffAlgorithm1(src, setnum, dstzset, maxelelen, totelelen); + } else if (diff_algo == 2) { +- zdiffAlgorithm2(src, setnum, dstzset, maxelelen); ++ zdiffAlgorithm2(src, setnum, dstzset, maxelelen, totelelen); + } else if (diff_algo != 0) { + serverPanic("Unknown algorithm"); + } +@@ -2565,7 +2579,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in + zsetopsrc *src; + zsetopval zval; + sds tmp; +- size_t maxelelen = 0; ++ size_t maxelelen = 0, totelelen = 0; + robj *dstobj; + zset *dstzset; + zskiplistNode *znode; +@@ -2701,6 +2715,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in + tmp = zuiNewSdsFromValue(&zval); + znode = zslInsert(dstzset->zsl,score,tmp); + dictAdd(dstzset->dict,tmp,&znode->score); ++ totelelen += sdslen(tmp); + if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp); + } + } +@@ -2737,6 +2752,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in + /* Remember the longest single element encountered, + * to understand if it's possible to convert to ziplist + * at the end. */ ++ totelelen += sdslen(tmp); + if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp); + /* Update the element with its initial score. */ + dictSetKey(accumulator, de, tmp); +@@ -2771,14 +2787,14 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in + dictReleaseIterator(di); + dictRelease(accumulator); + } else if (op == SET_OP_DIFF) { +- zdiff(src, setnum, dstzset, &maxelelen); ++ zdiff(src, setnum, dstzset, &maxelelen, &totelelen); + } else { + serverPanic("Unknown operator"); + } + + if (dstkey) { + if (dstzset->zsl->length) { +- zsetConvertToZiplistIfNeeded(dstobj, maxelelen); ++ zsetConvertToZiplistIfNeeded(dstobj, maxelelen, totelelen); + setKey(c, c->db, dstkey, dstobj); + addReplyLongLong(c, zsetLength(dstobj)); + notifyKeyspaceEvent(NOTIFY_ZSET, +diff --git a/src/ziplist.c b/src/ziplist.c +index aae86c1f2..fdc1bb9e1 100644 +--- a/src/ziplist.c ++++ b/src/ziplist.c +@@ -267,6 +267,17 @@ + ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \ + } + ++/* Don't let ziplists grow over 1GB in any case, don't wanna risk overflow in ++ * zlbytes*/ ++#define ZIPLIST_MAX_SAFETY_SIZE (1<<30) ++int ziplistSafeToAdd(unsigned char* zl, size_t add) { ++ size_t len = zl? ziplistBlobLen(zl): 0; ++ if (len + add > ZIPLIST_MAX_SAFETY_SIZE) ++ return 0; ++ return 1; ++} ++ ++ + /* We use this function to receive information about a ziplist entry. + * Note that this is not how the data is actually encoded, is just what we + * get filled by a function in order to operate more easily. */ +@@ -709,7 +720,8 @@ unsigned char *ziplistNew(void) { + } + + /* Resize the ziplist. */ +-unsigned char *ziplistResize(unsigned char *zl, unsigned int len) { ++unsigned char *ziplistResize(unsigned char *zl, size_t len) { ++ assert(len < UINT32_MAX); + zl = zrealloc(zl,len); + ZIPLIST_BYTES(zl) = intrev32ifbe(len); + zl[len-1] = ZIP_END; +@@ -1070,6 +1082,9 @@ unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) { + /* Combined zl length should be limited within UINT16_MAX */ + zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX; + ++ /* larger values can't be stored into ZIPLIST_BYTES */ ++ assert(zlbytes < UINT32_MAX); ++ + /* Save offset positions before we start ripping memory apart. */ + size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first)); + size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second)); +diff --git a/src/ziplist.h b/src/ziplist.h +index 9e7997ad8..569e1259d 100644 +--- a/src/ziplist.h ++++ b/src/ziplist.h +@@ -65,6 +65,7 @@ int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep, + void ziplistRandomPair(unsigned char *zl, unsigned long total_count, ziplistEntry *key, ziplistEntry *val); + void ziplistRandomPairs(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals); + unsigned int ziplistRandomPairsUnique(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals); ++int ziplistSafeToAdd(unsigned char* zl, size_t add); + + #ifdef REDIS_TEST + int ziplistTest(int argc, char *argv[], int accurate); +diff --git a/tests/unit/violations.tcl b/tests/unit/violations.tcl +new file mode 100644 +index 000000000..1d3140c52 +--- /dev/null ++++ b/tests/unit/violations.tcl +@@ -0,0 +1,156 @@ ++# These tests consume massive amounts of memory, and are not ++# suitable to be executed as part of the normal test suite ++set ::str500 [string repeat x 500000000] ;# 500mb ++ ++# Utility function to write big argument into redis client connection ++proc write_big_bulk {size} { ++ r write "\$$size\r\n" ++ while {$size >= 500000000} { ++ r write $::str500 ++ incr size -500000000 ++ } ++ if {$size > 0} { ++ r write [string repeat x $size] ++ } ++ r write "\r\n" ++} ++ ++# One XADD with one huge 5GB field ++# Expected to fail resulting in an empty stream ++start_server [list overrides [list save ""] ] { ++ test {XADD one huge field} { ++ r config set proto-max-bulk-len 10000000000 ;#10gb ++ r config set client-query-buffer-limit 10000000000 ;#10gb ++ r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n" ++ r write "\$1\r\nA\r\n" ++ write_big_bulk 5000000000 ;#5gb ++ r flush ++ catch {r read} err ++ assert_match {*too large*} $err ++ r xlen S1 ++ } {0} ++} ++ ++# One XADD with one huge (exactly nearly) 4GB field ++# This uncovers the overflow in lpEncodeGetType ++# Expected to fail resulting in an empty stream ++start_server [list overrides [list save ""] ] { ++ test {XADD one huge field - 1} { ++ r config set proto-max-bulk-len 10000000000 ;#10gb ++ r config set client-query-buffer-limit 10000000000 ;#10gb ++ r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n" ++ r write "\$1\r\nA\r\n" ++ write_big_bulk 4294967295 ;#4gb-1 ++ r flush ++ catch {r read} err ++ assert_match {*too large*} $err ++ r xlen S1 ++ } {0} ++} ++ ++# Gradually add big stream fields using repeated XADD calls ++start_server [list overrides [list save ""] ] { ++ test {several XADD big fields} { ++ r config set stream-node-max-bytes 0 ++ for {set j 0} {$j<10} {incr j} { ++ r xadd stream * 1 $::str500 2 $::str500 ++ } ++ r ping ++ r xlen stream ++ } {10} ++} ++ ++# Add over 4GB to a single stream listpack (one XADD command) ++# Expected to fail resulting in an empty stream ++start_server [list overrides [list save ""] ] { ++ test {single XADD big fields} { ++ r write "*23\r\n\$4\r\nXADD\r\n\$1\r\nS\r\n\$1\r\n*\r\n" ++ for {set j 0} {$j<10} {incr j} { ++ r write "\$1\r\n$j\r\n" ++ write_big_bulk 500000000 ;#500mb ++ } ++ r flush ++ catch {r read} err ++ assert_match {*too large*} $err ++ r xlen S ++ } {0} ++} ++ ++# Gradually add big hash fields using repeated HSET calls ++# This reproduces the overflow in the call to ziplistResize ++# Object will be converted to hashtable encoding ++start_server [list overrides [list save ""] ] { ++ r config set hash-max-ziplist-value 1000000000 ;#1gb ++ test {hash with many big fields} { ++ for {set j 0} {$j<10} {incr j} { ++ r hset h $j $::str500 ++ } ++ r object encoding h ++ } {hashtable} ++} ++ ++# Add over 4GB to a single hash field (one HSET command) ++# Object will be converted to hashtable encoding ++start_server [list overrides [list save ""] ] { ++ test {hash with one huge field} { ++ catch {r config set hash-max-ziplist-value 10000000000} ;#10gb ++ r config set proto-max-bulk-len 10000000000 ;#10gb ++ r config set client-query-buffer-limit 10000000000 ;#10gb ++ r write "*4\r\n\$4\r\nHSET\r\n\$2\r\nH1\r\n" ++ r write "\$1\r\nA\r\n" ++ write_big_bulk 5000000000 ;#5gb ++ r flush ++ r read ++ r object encoding H1 ++ } {hashtable} ++} ++ ++# Add over 4GB to a single list member (one LPUSH command) ++# Currently unsupported, and expected to fail rather than being truncated ++# Expected to fail resulting in a non-existing list ++start_server [list overrides [list save ""] ] { ++ test {list with one huge field} { ++ r config set proto-max-bulk-len 10000000000 ;#10gb ++ r config set client-query-buffer-limit 10000000000 ;#10gb ++ r write "*3\r\n\$5\r\nLPUSH\r\n\$2\r\nL1\r\n" ++ write_big_bulk 5000000000 ;#5gb ++ r flush ++ catch {r read} err ++ assert_match {*too large*} $err ++ r exists L1 ++ } {0} ++} ++ ++# SORT which attempts to store an element larger than 4GB into a list. ++# Currently unsupported and results in an assertion instead of truncation ++start_server [list overrides [list save ""] ] { ++ test {SORT adds huge field to list} { ++ r config set proto-max-bulk-len 10000000000 ;#10gb ++ r config set client-query-buffer-limit 10000000000 ;#10gb ++ r write "*3\r\n\$3\r\nSET\r\n\$2\r\nS1\r\n" ++ write_big_bulk 5000000000 ;#5gb ++ r flush ++ r read ++ assert_equal [r strlen S1] 5000000000 ++ r set S2 asdf ++ r sadd myset 1 2 ++ r mset D1 1 D2 2 ++ catch {r sort myset by D* get S* store mylist} ++ assert_equal [count_log_message 0 "crashed by signal"] 0 ++ assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 ++ } ++} ++ ++# SORT which stores an integer encoded element into a list. ++# Just for coverage, no news here. ++start_server [list overrides [list save ""] ] { ++ test {SORT adds integer field to list} { ++ r set S1 asdf ++ r set S2 123 ;# integer encoded ++ assert_encoding "int" S2 ++ r sadd myset 1 2 ++ r mset D1 1 D2 2 ++ r sort myset by D* get S* store mylist ++ r llen mylist ++ } {2} ++} +-- +2.17.1 + |