1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
From 4174a87b600c58e8cc00d9d18d0c507c67ca5d41 Mon Sep 17 00:00:00 2001
From: Damien Neil <dneil@google.com>
Date: Thu, 16 Mar 2023 16:56:12 -0700
Subject: [PATCH 2/3] net/textproto, mime/multipart: improve accounting of
non-file data
For requests containing large numbers of small parts,
memory consumption of a parsed form could be about 250%
over the estimated size.
When considering the size of parsed forms, account for the size of
FileHeader structs and increase the estimate of memory consumed by
map entries.
Thanks to Jakob Ackermann (@das7pad) for reporting this issue.
For CVE-2023-24536
For #59153
For #59269
Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/1802454
Run-TryBot: Damien Neil <dneil@google.com>
Reviewed-by: Roland Shoemaker <bracewell@google.com>
Reviewed-by: Julie Qiu <julieqiu@google.com>
Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/1802396
Run-TryBot: Roland Shoemaker <bracewell@google.com>
Reviewed-by: Damien Neil <dneil@google.com>
Change-Id: I31bc50e9346b4eee6fbe51a18c3c57230cc066db
Reviewed-on: https://go-review.googlesource.com/c/go/+/481984
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
Auto-Submit: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
CVE: CVE-2023-24536
Upstream-Status: Backport [7a359a651c7ebdb29e0a1c03102fce793e9f58f0]
Signed-off-by: Sakib Sajal <sakib.sajal@windriver.com>
---
src/mime/multipart/formdata.go | 9 +++--
src/mime/multipart/formdata_test.go | 55 ++++++++++++-----------------
src/net/textproto/reader.go | 8 ++++-
3 files changed, 37 insertions(+), 35 deletions(-)
diff --git a/src/mime/multipart/formdata.go b/src/mime/multipart/formdata.go
index 975dcb6..3f6ff69 100644
--- a/src/mime/multipart/formdata.go
+++ b/src/mime/multipart/formdata.go
@@ -103,8 +103,9 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) {
// Multiple values for the same key (one map entry, longer slice) are cheaper
// than the same number of values for different keys (many map entries), but
// using a consistent per-value cost for overhead is simpler.
+ const mapEntryOverhead = 200
maxMemoryBytes -= int64(len(name))
- maxMemoryBytes -= 100 // map overhead
+ maxMemoryBytes -= mapEntryOverhead
if maxMemoryBytes < 0 {
// We can't actually take this path, since nextPart would already have
// rejected the MIME headers for being too large. Check anyway.
@@ -128,7 +129,10 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) {
}
// file, store in memory or on disk
+ const fileHeaderSize = 100
maxMemoryBytes -= mimeHeaderSize(p.Header)
+ maxMemoryBytes -= mapEntryOverhead
+ maxMemoryBytes -= fileHeaderSize
if maxMemoryBytes < 0 {
return nil, ErrMessageTooLarge
}
@@ -183,9 +187,10 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) {
}
func mimeHeaderSize(h textproto.MIMEHeader) (size int64) {
+ size = 400
for k, vs := range h {
size += int64(len(k))
- size += 100 // map entry overhead
+ size += 200 // map entry overhead
for _, v := range vs {
size += int64(len(v))
}
diff --git a/src/mime/multipart/formdata_test.go b/src/mime/multipart/formdata_test.go
index f5b5608..8ed26e0 100644
--- a/src/mime/multipart/formdata_test.go
+++ b/src/mime/multipart/formdata_test.go
@@ -192,10 +192,10 @@ func (r *failOnReadAfterErrorReader) Read(p []byte) (n int, err error) {
// TestReadForm_NonFileMaxMemory asserts that the ReadForm maxMemory limit is applied
// while processing non-file form data as well as file form data.
func TestReadForm_NonFileMaxMemory(t *testing.T) {
- n := 10<<20 + 25
if testing.Short() {
- n = 10<<10 + 25
+ t.Skip("skipping in -short mode")
}
+ n := 10 << 20
largeTextValue := strings.Repeat("1", n)
message := `--MyBoundary
Content-Disposition: form-data; name="largetext"
@@ -203,38 +203,29 @@ Content-Disposition: form-data; name="largetext"
` + largeTextValue + `
--MyBoundary--
`
-
testBody := strings.ReplaceAll(message, "\n", "\r\n")
- testCases := []struct {
- name string
- maxMemory int64
- err error
- }{
- {"smaller", 50 + int64(len("largetext")) + 100, nil},
- {"exact-fit", 25 + int64(len("largetext")) + 100, nil},
- {"too-large", 0, ErrMessageTooLarge},
- }
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- if tc.maxMemory == 0 && testing.Short() {
- t.Skip("skipping in -short mode")
- }
- b := strings.NewReader(testBody)
- r := NewReader(b, boundary)
- f, err := r.ReadForm(tc.maxMemory)
- if err == nil {
- defer f.RemoveAll()
- }
- if tc.err != err {
- t.Fatalf("ReadForm error - got: %v; expected: %v", err, tc.err)
- }
- if err == nil {
- if g := f.Value["largetext"][0]; g != largeTextValue {
- t.Errorf("largetext mismatch: got size: %v, expected size: %v", len(g), len(largeTextValue))
- }
- }
- })
+ // Try parsing the form with increasing maxMemory values.
+ // Changes in how we account for non-file form data may cause the exact point
+ // where we change from rejecting the form as too large to accepting it to vary,
+ // but we should see both successes and failures.
+ const failWhenMaxMemoryLessThan = 128
+ for maxMemory := int64(0); maxMemory < failWhenMaxMemoryLessThan*2; maxMemory += 16 {
+ b := strings.NewReader(testBody)
+ r := NewReader(b, boundary)
+ f, err := r.ReadForm(maxMemory)
+ if err != nil {
+ continue
+ }
+ if g := f.Value["largetext"][0]; g != largeTextValue {
+ t.Errorf("largetext mismatch: got size: %v, expected size: %v", len(g), len(largeTextValue))
+ }
+ f.RemoveAll()
+ if maxMemory < failWhenMaxMemoryLessThan {
+ t.Errorf("ReadForm(%v): no error, expect to hit memory limit when maxMemory < %v", maxMemory, failWhenMaxMemoryLessThan)
+ }
+ return
}
+ t.Errorf("ReadForm(x) failed for x < 1024, expect success")
}
// TestReadForm_MetadataTooLarge verifies that we account for the size of field names,
diff --git a/src/net/textproto/reader.go b/src/net/textproto/reader.go
index fcbede8..9af4c49 100644
--- a/src/net/textproto/reader.go
+++ b/src/net/textproto/reader.go
@@ -503,6 +503,12 @@ func readMIMEHeader(r *Reader, lim int64) (MIMEHeader, error) {
m := make(MIMEHeader, hint)
+ // Account for 400 bytes of overhead for the MIMEHeader, plus 200 bytes per entry.
+ // Benchmarking map creation as of go1.20, a one-entry MIMEHeader is 416 bytes and large
+ // MIMEHeaders average about 200 bytes per entry.
+ lim -= 400
+ const mapEntryOverhead = 200
+
// The first line cannot start with a leading space.
if buf, err := r.R.Peek(1); err == nil && (buf[0] == ' ' || buf[0] == '\t') {
line, err := r.readLineSlice()
@@ -552,7 +558,7 @@ func readMIMEHeader(r *Reader, lim int64) (MIMEHeader, error) {
vv := m[key]
if vv == nil {
lim -= int64(len(key))
- lim -= 100 // map entry overhead
+ lim -= mapEntryOverhead
}
lim -= int64(len(value))
if lim < 0 {
--
2.35.5
|
