1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
|
From 3c7fd2b2729e3ebcf7877e7a32b3bbabf907a38d Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@redhat.com>
Date: Tue, 26 Feb 2019 01:42:39 +0100
Subject: [PATCH] closes bpo-34656: Avoid relying on signed overflow in _pickle
memos. (GH-9261) (#11869)
(cherry picked from commit a4ae828ee416a66d8c7bf5ee71d653c2cc6a26dd)
CVE: CVE-2018-20406
Upstream-Status: Backport
[https://github.com/python/cpython/commit/ef33dd6036aafbd3f06c1d56e2b1a81dae3da63c]
Signed-off-by: Dan Tran <dantran@microsoft.com>
---
Modules/_pickle.c | 63 ++++++++++++++++++++++++-----------------------
1 file changed, 32 insertions(+), 31 deletions(-)
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index 0f62b1c019..fcb9e87899 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -527,9 +527,9 @@ typedef struct {
} PyMemoEntry;
typedef struct {
- Py_ssize_t mt_mask;
- Py_ssize_t mt_used;
- Py_ssize_t mt_allocated;
+ size_t mt_mask;
+ size_t mt_used;
+ size_t mt_allocated;
PyMemoEntry *mt_table;
} PyMemoTable;
@@ -573,8 +573,8 @@ typedef struct UnpicklerObject {
/* The unpickler memo is just an array of PyObject *s. Using a dict
is unnecessary, since the keys are contiguous ints. */
PyObject **memo;
- Py_ssize_t memo_size; /* Capacity of the memo array */
- Py_ssize_t memo_len; /* Number of objects in the memo */
+ size_t memo_size; /* Capacity of the memo array */
+ size_t memo_len; /* Number of objects in the memo */
PyObject *pers_func; /* persistent_load() method, can be NULL. */
@@ -658,7 +658,6 @@ PyMemoTable_New(void)
static PyMemoTable *
PyMemoTable_Copy(PyMemoTable *self)
{
- Py_ssize_t i;
PyMemoTable *new = PyMemoTable_New();
if (new == NULL)
return NULL;
@@ -675,7 +674,7 @@ PyMemoTable_Copy(PyMemoTable *self)
PyErr_NoMemory();
return NULL;
}
- for (i = 0; i < self->mt_allocated; i++) {
+ for (size_t i = 0; i < self->mt_allocated; i++) {
Py_XINCREF(self->mt_table[i].me_key);
}
memcpy(new->mt_table, self->mt_table,
@@ -721,7 +720,7 @@ _PyMemoTable_Lookup(PyMemoTable *self, PyObject *key)
{
size_t i;
size_t perturb;
- size_t mask = (size_t)self->mt_mask;
+ size_t mask = self->mt_mask;
PyMemoEntry *table = self->mt_table;
PyMemoEntry *entry;
Py_hash_t hash = (Py_hash_t)key >> 3;
@@ -743,22 +742,24 @@ _PyMemoTable_Lookup(PyMemoTable *self, PyObject *key)
/* Returns -1 on failure, 0 on success. */
static int
-_PyMemoTable_ResizeTable(PyMemoTable *self, Py_ssize_t min_size)
+_PyMemoTable_ResizeTable(PyMemoTable *self, size_t min_size)
{
PyMemoEntry *oldtable = NULL;
PyMemoEntry *oldentry, *newentry;
- Py_ssize_t new_size = MT_MINSIZE;
- Py_ssize_t to_process;
+ size_t new_size = MT_MINSIZE;
+ size_t to_process;
assert(min_size > 0);
- /* Find the smallest valid table size >= min_size. */
- while (new_size < min_size && new_size > 0)
- new_size <<= 1;
- if (new_size <= 0) {
+ if (min_size > PY_SSIZE_T_MAX) {
PyErr_NoMemory();
return -1;
}
+
+ /* Find the smallest valid table size >= min_size. */
+ while (new_size < min_size) {
+ new_size <<= 1;
+ }
/* new_size needs to be a power of two. */
assert((new_size & (new_size - 1)) == 0);
@@ -808,6 +809,7 @@ static int
PyMemoTable_Set(PyMemoTable *self, PyObject *key, Py_ssize_t value)
{
PyMemoEntry *entry;
+ size_t desired_size;
assert(key != NULL);
@@ -831,10 +833,12 @@ PyMemoTable_Set(PyMemoTable *self, PyObject *key, Py_ssize_t value)
* Very large memo tables (over 50K items) use doubling instead.
* This may help applications with severe memory constraints.
*/
- if (!(self->mt_used * 3 >= (self->mt_mask + 1) * 2))
+ if (SIZE_MAX / 3 >= self->mt_used && self->mt_used * 3 < self->mt_allocated * 2) {
return 0;
- return _PyMemoTable_ResizeTable(self,
- (self->mt_used > 50000 ? 2 : 4) * self->mt_used);
+ }
+ // self->mt_used is always < PY_SSIZE_T_MAX, so this can't overflow.
+ desired_size = (self->mt_used > 50000 ? 2 : 4) * self->mt_used;
+ return _PyMemoTable_ResizeTable(self, desired_size);
}
#undef MT_MINSIZE
@@ -1273,9 +1277,9 @@ _Unpickler_Readline(UnpicklerObject *self, char **result)
/* Returns -1 (with an exception set) on failure, 0 on success. The memo array
will be modified in place. */
static int
-_Unpickler_ResizeMemoList(UnpicklerObject *self, Py_ssize_t new_size)
+_Unpickler_ResizeMemoList(UnpicklerObject *self, size_t new_size)
{
- Py_ssize_t i;
+ size_t i;
assert(new_size > self->memo_size);
@@ -1292,9 +1296,9 @@ _Unpickler_ResizeMemoList(UnpicklerObject *self, Py_ssize_t new_size)
/* Returns NULL if idx is out of bounds. */
static PyObject *
-_Unpickler_MemoGet(UnpicklerObject *self, Py_ssize_t idx)
+_Unpickler_MemoGet(UnpicklerObject *self, size_t idx)
{
- if (idx < 0 || idx >= self->memo_size)
+ if (idx >= self->memo_size)
return NULL;
return self->memo[idx];
@@ -1303,7 +1307,7 @@ _Unpickler_MemoGet(UnpicklerObject *self, Py_ssize_t idx)
/* Returns -1 (with an exception set) on failure, 0 on success.
This takes its own reference to `value`. */
static int
-_Unpickler_MemoPut(UnpicklerObject *self, Py_ssize_t idx, PyObject *value)
+_Unpickler_MemoPut(UnpicklerObject *self, size_t idx, PyObject *value)
{
PyObject *old_item;
@@ -4194,14 +4198,13 @@ static PyObject *
_pickle_PicklerMemoProxy_copy_impl(PicklerMemoProxyObject *self)
/*[clinic end generated code: output=bb83a919d29225ef input=b73043485ac30b36]*/
{
- Py_ssize_t i;
PyMemoTable *memo;
PyObject *new_memo = PyDict_New();
if (new_memo == NULL)
return NULL;
memo = self->pickler->memo;
- for (i = 0; i < memo->mt_allocated; ++i) {
+ for (size_t i = 0; i < memo->mt_allocated; ++i) {
PyMemoEntry entry = memo->mt_table[i];
if (entry.me_key != NULL) {
int status;
@@ -6620,7 +6623,7 @@ static PyObject *
_pickle_UnpicklerMemoProxy_copy_impl(UnpicklerMemoProxyObject *self)
/*[clinic end generated code: output=e12af7e9bc1e4c77 input=97769247ce032c1d]*/
{
- Py_ssize_t i;
+ size_t i;
PyObject *new_memo = PyDict_New();
if (new_memo == NULL)
return NULL;
@@ -6771,8 +6774,7 @@ static int
Unpickler_set_memo(UnpicklerObject *self, PyObject *obj)
{
PyObject **new_memo;
- Py_ssize_t new_memo_size = 0;
- Py_ssize_t i;
+ size_t new_memo_size = 0;
if (obj == NULL) {
PyErr_SetString(PyExc_TypeError,
@@ -6789,7 +6791,7 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj)
if (new_memo == NULL)
return -1;
- for (i = 0; i < new_memo_size; i++) {
+ for (size_t i = 0; i < new_memo_size; i++) {
Py_XINCREF(unpickler->memo[i]);
new_memo[i] = unpickler->memo[i];
}
@@ -6837,8 +6839,7 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj)
error:
if (new_memo_size) {
- i = new_memo_size;
- while (--i >= 0) {
+ for (size_t i = new_memo_size - 1; i != SIZE_MAX; i--) {
Py_XDECREF(new_memo[i]);
}
PyMem_FREE(new_memo);
--
2.22.0.vfs.1.1.57.gbaf16c8
|
