LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // See malloc.h for overview. | 5 // See malloc.h for overview. |
6 // | 6 // |
7 // TODO(rsc): double-check stats. | 7 // TODO(rsc): double-check stats. |
8 | 8 |
9 package runtime | 9 package runtime |
10 #include "runtime.h" | 10 #include "runtime.h" |
11 #include "arch_GOARCH.h" | 11 #include "arch_GOARCH.h" |
12 #include "malloc.h" | 12 #include "malloc.h" |
13 #include "type.h" | 13 #include "type.h" |
14 #include "typekind.h" | 14 #include "typekind.h" |
15 #include "race.h" | 15 #include "race.h" |
16 #include "stack.h" | 16 #include "stack.h" |
17 #include "../../cmd/ld/textflag.h" | 17 #include "../../cmd/ld/textflag.h" |
18 | 18 |
19 // Mark mheap as 'no pointers', it does not contain interesting pointers but occ
upies ~45K. | 19 // Mark mheap as 'no pointers', it does not contain interesting pointers but occ
upies ~45K. |
20 #pragma dataflag NOPTR | 20 #pragma dataflag NOPTR |
21 MHeap runtime·mheap; | 21 MHeap runtime·mheap; |
| 22 MStats mstats; |
22 | 23 |
23 int32 runtime·checking; | 24 int32 runtime·checking; |
24 | 25 |
25 extern MStats mstats; // defined in zruntime_def_$GOOS_$GOARCH.go | 26 extern MStats mstats; // defined in zruntime_def_$GOOS_$GOARCH.go |
26 | 27 |
27 extern volatile intgo runtime·MemProfileRate; | 28 extern volatile intgo runtime·MemProfileRate; |
28 | 29 |
29 static void* largealloc(uint32, uintptr*); | 30 static MSpan* largealloc(uint32, uintptr*); |
| 31 static void profilealloc(void *v, uintptr size, uintptr typ); |
| 32 static void settype(MSpan *s, void *v, uintptr typ); |
30 | 33 |
31 // Allocate an object of at least size bytes. | 34 // Allocate an object of at least size bytes. |
32 // Small objects are allocated from the per-thread cache's free lists. | 35 // Small objects are allocated from the per-thread cache's free lists. |
33 // Large objects (> 32 kB) are allocated straight from the heap. | 36 // Large objects (> 32 kB) are allocated straight from the heap. |
34 // If the block will be freed with runtime·free(), typ must be 0. | 37 // If the block will be freed with runtime·free(), typ must be 0. |
35 void* | 38 void* |
36 runtime·mallocgc(uintptr size, uintptr typ, uint32 flag) | 39 runtime·mallocgc(uintptr size, uintptr typ, uint32 flag) |
37 { | 40 { |
38 int32 sizeclass; | 41 int32 sizeclass; |
39 uintptr tinysize, size1; | 42 uintptr tinysize, size1; |
40 intgo rate; | 43 intgo rate; |
41 MCache *c; | 44 MCache *c; |
42 » MCacheList *l; | 45 » MSpan *s; |
43 » MLink *v; | 46 » MLink *v, *next; |
44 byte *tiny; | 47 byte *tiny; |
45 | 48 |
46 if(size == 0) { | 49 if(size == 0) { |
47 // All 0-length allocations use this pointer. | 50 // All 0-length allocations use this pointer. |
48 // The language does not require the allocations to | 51 // The language does not require the allocations to |
49 // have distinct values. | 52 // have distinct values. |
50 return &runtime·zerobase; | 53 return &runtime·zerobase; |
51 } | 54 } |
52 if(m->mallocing) | 55 if(m->mallocing) |
53 runtime·throw("malloc/free - deadlock"); | 56 runtime·throw("malloc/free - deadlock"); |
54 » // Disable preemption during settype_flush. | 57 » // Disable preemption during settype. |
55 » // We can not use m->mallocing for this, because settype_flush calls mal
locgc. | 58 » // We can not use m->mallocing for this, because settype calls mallocgc. |
56 m->locks++; | 59 m->locks++; |
57 m->mallocing = 1; | 60 m->mallocing = 1; |
58 | 61 |
59 if(DebugTypeAtBlockEnd) | 62 if(DebugTypeAtBlockEnd) |
60 size += sizeof(uintptr); | 63 size += sizeof(uintptr); |
61 | 64 |
62 c = m->mcache; | 65 c = m->mcache; |
63 if(!runtime·debug.efence && size <= MaxSmallSize) { | 66 if(!runtime·debug.efence && size <= MaxSmallSize) { |
64 if((flag&(FlagNoScan|FlagNoGC)) == FlagNoScan && size < TinySize
) { | 67 if((flag&(FlagNoScan|FlagNoGC)) == FlagNoScan && size < TinySize
) { |
65 // Tiny allocator. | 68 // Tiny allocator. |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
109 c->tiny += size1; | 112 c->tiny += size1; |
110 c->tinysize -= size1; | 113 c->tinysize -= size1; |
111 m->mallocing = 0; | 114 m->mallocing = 0; |
112 m->locks--; | 115 m->locks--; |
113 if(m->locks == 0 && g->preempt) // rest
ore the preemption request in case we've cleared it in newstack | 116 if(m->locks == 0 && g->preempt) // rest
ore the preemption request in case we've cleared it in newstack |
114 g->stackguard0 = StackPreempt; | 117 g->stackguard0 = StackPreempt; |
115 return v; | 118 return v; |
116 } | 119 } |
117 } | 120 } |
118 // Allocate a new TinySize block. | 121 // Allocate a new TinySize block. |
119 » » » l = &c->list[TinySizeClass]; | 122 » » » s = c->alloc[TinySizeClass]; |
120 » » » if(l->list == nil) | 123 » » » if(s->freelist == nil) |
121 » » » » runtime·MCache_Refill(c, TinySizeClass); | 124 » » » » s = runtime·MCache_Refill(c, TinySizeClass); |
122 » » » v = l->list; | 125 » » » v = s->freelist; |
123 » » » l->list = v->next; | 126 » » » next = v->next; |
124 » » » l->nlist--; | 127 » » » s->freelist = next; |
| 128 » » » s->ref++; |
| 129 » » » if(next != nil) // prefetching nil leads to a DTLB miss |
| 130 » » » » PREFETCH(next); |
125 ((uint64*)v)[0] = 0; | 131 ((uint64*)v)[0] = 0; |
126 ((uint64*)v)[1] = 0; | 132 ((uint64*)v)[1] = 0; |
127 // See if we need to replace the existing tiny block wit
h the new one | 133 // See if we need to replace the existing tiny block wit
h the new one |
128 // based on amount of remaining free space. | 134 // based on amount of remaining free space. |
129 if(TinySize-size > tinysize) { | 135 if(TinySize-size > tinysize) { |
130 c->tiny = (byte*)v + size; | 136 c->tiny = (byte*)v + size; |
131 c->tinysize = TinySize - size; | 137 c->tinysize = TinySize - size; |
132 } | 138 } |
133 size = TinySize; | 139 size = TinySize; |
134 goto done; | 140 goto done; |
135 } | 141 } |
136 // Allocate from mcache free lists. | 142 // Allocate from mcache free lists. |
137 // Inlined version of SizeToClass(). | 143 // Inlined version of SizeToClass(). |
138 if(size <= 1024-8) | 144 if(size <= 1024-8) |
139 sizeclass = runtime·size_to_class8[(size+7)>>3]; | 145 sizeclass = runtime·size_to_class8[(size+7)>>3]; |
140 else | 146 else |
141 sizeclass = runtime·size_to_class128[(size-1024+127) >>
7]; | 147 sizeclass = runtime·size_to_class128[(size-1024+127) >>
7]; |
142 size = runtime·class_to_size[sizeclass]; | 148 size = runtime·class_to_size[sizeclass]; |
143 » » l = &c->list[sizeclass]; | 149 » » s = c->alloc[sizeclass]; |
144 » » if(l->list == nil) | 150 » » if(s->freelist == nil) |
145 » » » runtime·MCache_Refill(c, sizeclass); | 151 » » » s = runtime·MCache_Refill(c, sizeclass); |
146 » » v = l->list; | 152 » » v = s->freelist; |
147 » » l->list = v->next; | 153 » » next = v->next; |
148 » » l->nlist--; | 154 » » s->freelist = next; |
| 155 » » s->ref++; |
| 156 » » if(next != nil) // prefetching nil leads to a DTLB miss |
| 157 » » » PREFETCH(next); |
149 if(!(flag & FlagNoZero)) { | 158 if(!(flag & FlagNoZero)) { |
150 v->next = nil; | 159 v->next = nil; |
151 // block is zeroed iff second word is zero ... | 160 // block is zeroed iff second word is zero ... |
152 if(size > 2*sizeof(uintptr) && ((uintptr*)v)[1] != 0) | 161 if(size > 2*sizeof(uintptr) && ((uintptr*)v)[1] != 0) |
153 runtime·memclr((byte*)v, size); | 162 runtime·memclr((byte*)v, size); |
154 } | 163 } |
155 done: | 164 done: |
156 c->local_cachealloc += size; | 165 c->local_cachealloc += size; |
157 } else { | 166 } else { |
158 // Allocate directly from heap. | 167 // Allocate directly from heap. |
159 » » v = largealloc(flag, &size); | 168 » » s = largealloc(flag, &size); |
| 169 » » v = (void*)(s->start << PageShift); |
160 } | 170 } |
161 | 171 |
162 if(flag & FlagNoGC) | 172 if(flag & FlagNoGC) |
163 runtime·marknogc(v); | 173 runtime·marknogc(v); |
164 else if(!(flag & FlagNoScan)) | 174 else if(!(flag & FlagNoScan)) |
165 runtime·markscan(v); | 175 runtime·markscan(v); |
166 | 176 |
167 if(DebugTypeAtBlockEnd) | 177 if(DebugTypeAtBlockEnd) |
168 *(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ; | 178 *(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ; |
169 | 179 |
| 180 m->mallocing = 0; |
170 // TODO: save type even if FlagNoScan? Potentially expensive but might
help | 181 // TODO: save type even if FlagNoScan? Potentially expensive but might
help |
171 // heap profiling/tracing. | 182 // heap profiling/tracing. |
172 » if(UseSpanType && !(flag & FlagNoScan) && typ != 0) { | 183 » if(UseSpanType && !(flag & FlagNoScan) && typ != 0) |
173 » » uintptr *buf, i; | 184 » » settype(s, v, typ); |
174 | 185 |
175 » » buf = m->settype_buf; | 186 » if(raceenabled) |
176 » » i = m->settype_bufsize; | 187 » » runtime·racemalloc(v, size); |
177 » » buf[i++] = (uintptr)v; | 188 |
178 » » buf[i++] = typ; | 189 » if(runtime·debug.allocfreetrace) |
179 » » m->settype_bufsize = i; | 190 » » goto profile; |
180 » } | 191 |
181 | 192 » if(!(flag & FlagNoProfiling) && (rate = runtime·MemProfileRate) > 0) { |
182 » m->mallocing = 0; | 193 » » if(size < rate && size < c->next_sample) |
183 » if(UseSpanType && !(flag & FlagNoScan) && typ != 0 && m->settype_bufsize
== nelem(m->settype_buf)) | 194 » » » c->next_sample -= size; |
184 » » runtime·settype_flush(m); | 195 » » else { |
| 196 » » profile: |
| 197 » » » profilealloc(v, size, typ); |
| 198 » » } |
| 199 » } |
| 200 |
185 m->locks--; | 201 m->locks--; |
186 if(m->locks == 0 && g->preempt) // restore the preemption request in ca
se we've cleared it in newstack | 202 if(m->locks == 0 && g->preempt) // restore the preemption request in ca
se we've cleared it in newstack |
187 g->stackguard0 = StackPreempt; | 203 g->stackguard0 = StackPreempt; |
188 | 204 |
189 if(runtime·debug.allocfreetrace) | |
190 goto profile; | |
191 | |
192 if(!(flag & FlagNoProfiling) && (rate = runtime·MemProfileRate) > 0) { | |
193 if(size >= rate) | |
194 goto profile; | |
195 if(m->mcache->next_sample > size) | |
196 m->mcache->next_sample -= size; | |
197 else { | |
198 // pick next profile time | |
199 // If you change this, also change allocmcache. | |
200 if(rate > 0x3fffffff) // make 2*rate not overflow | |
201 rate = 0x3fffffff; | |
202 m->mcache->next_sample = runtime·fastrand1() % (2*rate); | |
203 profile: | |
204 runtime·MProf_Malloc(v, size, typ); | |
205 } | |
206 } | |
207 | |
208 if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc) | 205 if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc) |
209 runtime·gc(0); | 206 runtime·gc(0); |
210 | 207 |
211 if(raceenabled) | |
212 runtime·racemalloc(v, size); | |
213 return v; | 208 return v; |
214 } | 209 } |
215 | 210 |
216 static void* | 211 static MSpan* |
217 largealloc(uint32 flag, uintptr *sizep) | 212 largealloc(uint32 flag, uintptr *sizep) |
218 { | 213 { |
219 uintptr npages, size; | 214 uintptr npages, size; |
220 MSpan *s; | 215 MSpan *s; |
221 void *v; | 216 void *v; |
222 | 217 |
223 // Allocate directly from heap. | 218 // Allocate directly from heap. |
224 size = *sizep; | 219 size = *sizep; |
225 if(size + PageSize < size) | 220 if(size + PageSize < size) |
226 runtime·throw("out of memory"); | 221 runtime·throw("out of memory"); |
227 npages = size >> PageShift; | 222 npages = size >> PageShift; |
228 if((size & PageMask) != 0) | 223 if((size & PageMask) != 0) |
229 npages++; | 224 npages++; |
230 s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, !(flag & FlagNoZer
o)); | 225 s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, !(flag & FlagNoZer
o)); |
231 if(s == nil) | 226 if(s == nil) |
232 runtime·throw("out of memory"); | 227 runtime·throw("out of memory"); |
233 s->limit = (byte*)(s->start<<PageShift) + size; | 228 s->limit = (byte*)(s->start<<PageShift) + size; |
234 *sizep = npages<<PageShift; | 229 *sizep = npages<<PageShift; |
235 v = (void*)(s->start << PageShift); | 230 v = (void*)(s->start << PageShift); |
236 // setup for mark sweep | 231 // setup for mark sweep |
237 runtime·markspan(v, 0, 0, true); | 232 runtime·markspan(v, 0, 0, true); |
238 » return v; | 233 » return s; |
| 234 } |
| 235 |
| 236 static void |
| 237 profilealloc(void *v, uintptr size, uintptr typ) |
| 238 { |
| 239 » uintptr rate; |
| 240 » int32 next; |
| 241 » MCache *c; |
| 242 |
| 243 » c = m->mcache; |
| 244 » rate = runtime·MemProfileRate; |
| 245 » if(size < rate) { |
| 246 » » // pick next profile time |
| 247 » » // If you change this, also change allocmcache. |
| 248 » » if(rate > 0x3fffffff)» // make 2*rate not overflow |
| 249 » » » rate = 0x3fffffff; |
| 250 » » next = runtime·fastrand1() % (2*rate); |
| 251 » » // Subtract the "remainder" of the current allocation. |
| 252 » » // Otherwise objects that are close in size to sampling rate |
| 253 » » // will be under-sampled, because we consistently discard this r
emainder. |
| 254 » » next -= (size - c->next_sample); |
| 255 » » if(next < 0) |
| 256 » » » next = 0; |
| 257 » » c->next_sample = next; |
| 258 » } |
| 259 » runtime·MProf_Malloc(v, size, typ); |
239 } | 260 } |
240 | 261 |
241 void* | 262 void* |
242 runtime·malloc(uintptr size) | 263 runtime·malloc(uintptr size) |
243 { | 264 { |
244 return runtime·mallocgc(size, 0, FlagNoInvokeGC); | 265 return runtime·mallocgc(size, 0, FlagNoInvokeGC); |
245 } | 266 } |
246 | 267 |
247 // Free the object whose base pointer is v. | 268 // Free the object whose base pointer is v. |
248 void | 269 void |
(...skipping 18 matching lines...) Expand all Loading... |
267 runtime·printf("free %p: not an allocated block\n", v); | 288 runtime·printf("free %p: not an allocated block\n", v); |
268 runtime·throw("free runtime·mlookup"); | 289 runtime·throw("free runtime·mlookup"); |
269 } | 290 } |
270 size = s->elemsize; | 291 size = s->elemsize; |
271 sizeclass = s->sizeclass; | 292 sizeclass = s->sizeclass; |
272 // Objects that are smaller than TinySize can be allocated using tiny al
loc, | 293 // Objects that are smaller than TinySize can be allocated using tiny al
loc, |
273 // if then such object is combined with an object with finalizer, we wil
l crash. | 294 // if then such object is combined with an object with finalizer, we wil
l crash. |
274 if(size < TinySize) | 295 if(size < TinySize) |
275 runtime·throw("freeing too small block"); | 296 runtime·throw("freeing too small block"); |
276 | 297 |
| 298 // Ensure that the span is swept. |
| 299 // If we free into an unswept span, we will corrupt GC bitmaps. |
| 300 runtime·MSpan_EnsureSwept(s); |
| 301 |
277 if(s->specials != nil) | 302 if(s->specials != nil) |
278 runtime·freeallspecials(s, v, size); | 303 runtime·freeallspecials(s, v, size); |
279 | 304 |
280 c = m->mcache; | 305 c = m->mcache; |
281 if(sizeclass == 0) { | 306 if(sizeclass == 0) { |
282 // Large object. | 307 // Large object. |
283 » » *(uintptr*)(s->start<<PageShift) = (uintptr)0xfeedfeedfeedfeedll
;» // mark as "needs to be zeroed" | 308 » » s->needzero = 1; |
284 // Must mark v freed before calling unmarkspan and MHeap_Free: | 309 // Must mark v freed before calling unmarkspan and MHeap_Free: |
285 // they might coalesce v into other spans and change the bitmap
further. | 310 // they might coalesce v into other spans and change the bitmap
further. |
286 » » runtime·markfreed(v, size); | 311 » » runtime·markfreed(v); |
287 runtime·unmarkspan(v, 1<<PageShift); | 312 runtime·unmarkspan(v, 1<<PageShift); |
288 if(runtime·debug.efence) | 313 if(runtime·debug.efence) |
289 runtime·SysFree((void*)(s->start<<PageShift), size, &mst
ats.heap_sys); | 314 runtime·SysFree((void*)(s->start<<PageShift), size, &mst
ats.heap_sys); |
290 else | 315 else |
291 runtime·MHeap_Free(&runtime·mheap, s, 1); | 316 runtime·MHeap_Free(&runtime·mheap, s, 1); |
292 c->local_nlargefree++; | 317 c->local_nlargefree++; |
293 c->local_largefree += size; | 318 c->local_largefree += size; |
294 } else { | 319 } else { |
295 // Small object. | 320 // Small object. |
296 if(size > 2*sizeof(uintptr)) | 321 if(size > 2*sizeof(uintptr)) |
297 ((uintptr*)v)[1] = (uintptr)0xfeedfeedfeedfeedll;
// mark as "needs to be zeroed" | 322 ((uintptr*)v)[1] = (uintptr)0xfeedfeedfeedfeedll;
// mark as "needs to be zeroed" |
298 else if(size > sizeof(uintptr)) | 323 else if(size > sizeof(uintptr)) |
299 ((uintptr*)v)[1] = 0; | 324 ((uintptr*)v)[1] = 0; |
300 // Must mark v freed before calling MCache_Free: | 325 // Must mark v freed before calling MCache_Free: |
301 // it might coalesce v and other blocks into a bigger span | 326 // it might coalesce v and other blocks into a bigger span |
302 // and change the bitmap further. | 327 // and change the bitmap further. |
303 runtime·markfreed(v, size); | |
304 c->local_nsmallfree[sizeclass]++; | 328 c->local_nsmallfree[sizeclass]++; |
305 » » runtime·MCache_Free(c, v, sizeclass, size); | 329 » » c->local_cachealloc -= size; |
| 330 » » if(c->alloc[sizeclass] == s) { |
| 331 » » » // We own the span, so we can just add v to the freelist |
| 332 » » » runtime·markfreed(v); |
| 333 » » » ((MLink*)v)->next = s->freelist; |
| 334 » » » s->freelist = v; |
| 335 » » » s->ref--; |
| 336 » » } else { |
| 337 » » » // Someone else owns this span. Add to free queue. |
| 338 » » » runtime·MCache_Free(c, v, sizeclass, size); |
| 339 » » } |
306 } | 340 } |
307 m->mallocing = 0; | 341 m->mallocing = 0; |
308 } | 342 } |
309 | 343 |
310 int32 | 344 int32 |
311 runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp) | 345 runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp) |
312 { | 346 { |
313 uintptr n, i; | 347 uintptr n, i; |
314 byte *p; | 348 byte *p; |
315 MSpan *s; | 349 MSpan *s; |
(...skipping 30 matching lines...) Expand all Loading... |
346 | 380 |
347 n = s->elemsize; | 381 n = s->elemsize; |
348 if(base) { | 382 if(base) { |
349 i = ((byte*)v - p)/n; | 383 i = ((byte*)v - p)/n; |
350 *base = p + i*n; | 384 *base = p + i*n; |
351 } | 385 } |
352 if(size) | 386 if(size) |
353 *size = n; | 387 *size = n; |
354 | 388 |
355 return 1; | 389 return 1; |
356 } | |
357 | |
358 MCache* | |
359 runtime·allocmcache(void) | |
360 { | |
361 intgo rate; | |
362 MCache *c; | |
363 | |
364 runtime·lock(&runtime·mheap); | |
365 c = runtime·FixAlloc_Alloc(&runtime·mheap.cachealloc); | |
366 runtime·unlock(&runtime·mheap); | |
367 runtime·memclr((byte*)c, sizeof(*c)); | |
368 | |
369 // Set first allocation sample size. | |
370 rate = runtime·MemProfileRate; | |
371 if(rate > 0x3fffffff) // make 2*rate not overflow | |
372 rate = 0x3fffffff; | |
373 if(rate != 0) | |
374 c->next_sample = runtime·fastrand1() % (2*rate); | |
375 | |
376 return c; | |
377 } | |
378 | |
379 void | |
380 runtime·freemcache(MCache *c) | |
381 { | |
382 runtime·MCache_ReleaseAll(c); | |
383 runtime·lock(&runtime·mheap); | |
384 runtime·purgecachedstats(c); | |
385 runtime·FixAlloc_Free(&runtime·mheap.cachealloc, c); | |
386 runtime·unlock(&runtime·mheap); | |
387 } | 390 } |
388 | 391 |
389 void | 392 void |
390 runtime·purgecachedstats(MCache *c) | 393 runtime·purgecachedstats(MCache *c) |
391 { | 394 { |
392 MHeap *h; | 395 MHeap *h; |
393 int32 i; | 396 int32 i; |
394 | 397 |
395 // Protected by either heap or GC lock. | 398 // Protected by either heap or GC lock. |
396 h = &runtime·mheap; | 399 h = &runtime·mheap; |
397 mstats.heap_alloc += c->local_cachealloc; | 400 mstats.heap_alloc += c->local_cachealloc; |
398 c->local_cachealloc = 0; | 401 c->local_cachealloc = 0; |
399 mstats.nlookup += c->local_nlookup; | 402 mstats.nlookup += c->local_nlookup; |
400 c->local_nlookup = 0; | 403 c->local_nlookup = 0; |
401 h->largefree += c->local_largefree; | 404 h->largefree += c->local_largefree; |
402 c->local_largefree = 0; | 405 c->local_largefree = 0; |
403 h->nlargefree += c->local_nlargefree; | 406 h->nlargefree += c->local_nlargefree; |
404 c->local_nlargefree = 0; | 407 c->local_nlargefree = 0; |
405 for(i=0; i<nelem(c->local_nsmallfree); i++) { | 408 for(i=0; i<nelem(c->local_nsmallfree); i++) { |
406 h->nsmallfree[i] += c->local_nsmallfree[i]; | 409 h->nsmallfree[i] += c->local_nsmallfree[i]; |
407 c->local_nsmallfree[i] = 0; | 410 c->local_nsmallfree[i] = 0; |
408 } | 411 } |
409 } | 412 } |
410 | 413 |
411 uintptr runtime·sizeof_C_MStats = sizeof(MStats); | 414 // Size of the trailing by_size array differs between Go and C, |
| 415 // NumSizeClasses was changed, but we can not change Go struct because of backwa
rd compatibility. |
| 416 // sizeof_C_MStats is what C thinks about size of Go struct. |
| 417 uintptr runtime·sizeof_C_MStats = sizeof(MStats) - (NumSizeClasses - 61) * sizeo
f(mstats.by_size[0]); |
412 | 418 |
413 #define MaxArena32 (2U<<30) | 419 #define MaxArena32 (2U<<30) |
414 | 420 |
415 void | 421 void |
416 runtime·mallocinit(void) | 422 runtime·mallocinit(void) |
417 { | 423 { |
418 byte *p; | 424 byte *p; |
419 uintptr arena_size, bitmap_size, spans_size; | 425 uintptr arena_size, bitmap_size, spans_size; |
420 extern byte end[]; | 426 extern byte end[]; |
421 byte *want; | |
422 uintptr limit; | 427 uintptr limit; |
423 uint64 i; | 428 uint64 i; |
424 | 429 |
425 p = nil; | 430 p = nil; |
426 arena_size = 0; | 431 arena_size = 0; |
427 bitmap_size = 0; | 432 bitmap_size = 0; |
428 spans_size = 0; | 433 spans_size = 0; |
429 | 434 |
430 // for 64-bit build | 435 // for 64-bit build |
431 USED(p); | 436 USED(p); |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
470 // Actually we reserve 136 GB (because the bitmap ends up being
8 GB) | 475 // Actually we reserve 136 GB (because the bitmap ends up being
8 GB) |
471 // but it hardly matters: e0 00 is not valid UTF-8 either. | 476 // but it hardly matters: e0 00 is not valid UTF-8 either. |
472 // | 477 // |
473 // If this fails we fall back to the 32 bit memory mechanism | 478 // If this fails we fall back to the 32 bit memory mechanism |
474 arena_size = MaxMem; | 479 arena_size = MaxMem; |
475 bitmap_size = arena_size / (sizeof(void*)*8/4); | 480 bitmap_size = arena_size / (sizeof(void*)*8/4); |
476 spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[
0]); | 481 spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[
0]); |
477 spans_size = ROUND(spans_size, PageSize); | 482 spans_size = ROUND(spans_size, PageSize); |
478 for(i = 0; i <= 0x7f; i++) { | 483 for(i = 0; i <= 0x7f; i++) { |
479 p = (void*)(i<<40 | 0x00c0ULL<<32); | 484 p = (void*)(i<<40 | 0x00c0ULL<<32); |
480 » » » p = runtime·SysReserve(p, bitmap_size + spans_size + are
na_size); | 485 » » » p = runtime·SysReserve(p, bitmap_size + spans_size + are
na_size + PageSize); |
481 if(p != nil) | 486 if(p != nil) |
482 break; | 487 break; |
483 } | 488 } |
484 } | 489 } |
485 if (p == nil) { | 490 if (p == nil) { |
486 // On a 32-bit machine, we can't typically get away | 491 // On a 32-bit machine, we can't typically get away |
487 // with a giant virtual address space reservation. | 492 // with a giant virtual address space reservation. |
488 // Instead we map the memory information bitmap | 493 // Instead we map the memory information bitmap |
489 // immediately after the data segment, large enough | 494 // immediately after the data segment, large enough |
490 // to handle another 2GB of mappings (256 MB), | 495 // to handle another 2GB of mappings (256 MB), |
(...skipping 21 matching lines...) Expand all Loading... |
512 | 517 |
513 // SysReserve treats the address we ask for, end, as a hint, | 518 // SysReserve treats the address we ask for, end, as a hint, |
514 // not as an absolute requirement. If we ask for the end | 519 // not as an absolute requirement. If we ask for the end |
515 // of the data segment but the operating system requires | 520 // of the data segment but the operating system requires |
516 // a little more space before we can start allocating, it will | 521 // a little more space before we can start allocating, it will |
517 // give out a slightly higher pointer. Except QEMU, which | 522 // give out a slightly higher pointer. Except QEMU, which |
518 // is buggy, as usual: it won't adjust the pointer upward. | 523 // is buggy, as usual: it won't adjust the pointer upward. |
519 // So adjust it upward a little bit ourselves: 1/4 MB to get | 524 // So adjust it upward a little bit ourselves: 1/4 MB to get |
520 // away from the running binary image and then round up | 525 // away from the running binary image and then round up |
521 // to a MB boundary. | 526 // to a MB boundary. |
522 » » want = (byte*)ROUND((uintptr)end + (1<<18), 1<<20); | 527 » » p = (byte*)ROUND((uintptr)end + (1<<18), 1<<20); |
523 » » p = runtime·SysReserve(want, bitmap_size + spans_size + arena_si
ze); | 528 » » p = runtime·SysReserve(p, bitmap_size + spans_size + arena_size
+ PageSize); |
524 if(p == nil) | 529 if(p == nil) |
525 runtime·throw("runtime: cannot reserve arena virtual add
ress space"); | 530 runtime·throw("runtime: cannot reserve arena virtual add
ress space"); |
526 » » if((uintptr)p & (((uintptr)1<<PageShift)-1)) | 531 » } |
527 » » » runtime·printf("runtime: SysReserve returned unaligned a
ddress %p; asked for %p", p, | 532 |
528 » » » » bitmap_size+spans_size+arena_size); | 533 » // PageSize can be larger than OS definition of page size, |
529 » } | 534 » // so SysReserve can give us a PageSize-unaligned pointer. |
530 » if((uintptr)p & (((uintptr)1<<PageShift)-1)) | 535 » // To overcome this we ask for PageSize more and round up the pointer. |
531 » » runtime·throw("runtime: SysReserve returned unaligned address"); | 536 » p = (byte*)ROUND((uintptr)p, PageSize); |
532 | 537 |
533 runtime·mheap.spans = (MSpan**)p; | 538 runtime·mheap.spans = (MSpan**)p; |
534 runtime·mheap.bitmap = p + spans_size; | 539 runtime·mheap.bitmap = p + spans_size; |
535 runtime·mheap.arena_start = p + spans_size + bitmap_size; | 540 runtime·mheap.arena_start = p + spans_size + bitmap_size; |
536 runtime·mheap.arena_used = runtime·mheap.arena_start; | 541 runtime·mheap.arena_used = runtime·mheap.arena_start; |
537 runtime·mheap.arena_end = runtime·mheap.arena_start + arena_size; | 542 runtime·mheap.arena_end = runtime·mheap.arena_start + arena_size; |
538 | 543 |
539 // Initialize the rest of the allocator.········ | 544 // Initialize the rest of the allocator.········ |
540 runtime·MHeap_Init(&runtime·mheap); | 545 runtime·MHeap_Init(&runtime·mheap); |
541 m->mcache = runtime·allocmcache(); | 546 m->mcache = runtime·allocmcache(); |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
652 persistent.pos += size; | 657 persistent.pos += size; |
653 runtime·unlock(&persistent); | 658 runtime·unlock(&persistent); |
654 if(stat != &mstats.other_sys) { | 659 if(stat != &mstats.other_sys) { |
655 // reaccount the allocation against provided stat | 660 // reaccount the allocation against provided stat |
656 runtime·xadd64(stat, size); | 661 runtime·xadd64(stat, size); |
657 runtime·xadd64(&mstats.other_sys, -(uint64)size); | 662 runtime·xadd64(&mstats.other_sys, -(uint64)size); |
658 } | 663 } |
659 return p; | 664 return p; |
660 } | 665 } |
661 | 666 |
662 static Lock settype_lock; | 667 static void |
663 | 668 settype(MSpan *s, void *v, uintptr typ) |
664 void | 669 { |
665 runtime·settype_flush(M *mp) | |
666 { | |
667 » uintptr *buf, *endbuf; | |
668 uintptr size, ofs, j, t; | 670 uintptr size, ofs, j, t; |
669 uintptr ntypes, nbytes2, nbytes3; | 671 uintptr ntypes, nbytes2, nbytes3; |
670 uintptr *data2; | 672 uintptr *data2; |
671 byte *data3; | 673 byte *data3; |
672 » void *v; | 674 |
673 » uintptr typ, p; | 675 » if(s->sizeclass == 0) { |
674 » MSpan *s; | 676 » » s->types.compression = MTypes_Single; |
675 | 677 » » s->types.data = typ; |
676 » buf = mp->settype_buf; | 678 » » return; |
677 » endbuf = buf + mp->settype_bufsize; | 679 » } |
678 | 680 » size = s->elemsize; |
679 » runtime·lock(&settype_lock); | 681 » ofs = ((uintptr)v - (s->start<<PageShift)) / size; |
680 » while(buf < endbuf) { | 682 |
681 » » v = (void*)*buf; | 683 » switch(s->types.compression) { |
682 » » *buf = 0; | 684 » case MTypes_Empty: |
683 » » buf++; | 685 » » ntypes = (s->npages << PageShift) / size; |
684 » » typ = *buf; | 686 » » nbytes3 = 8*sizeof(uintptr) + 1*ntypes; |
685 » » buf++; | 687 » » data3 = runtime·mallocgc(nbytes3, 0, FlagNoProfiling|FlagNoScan|
FlagNoInvokeGC); |
686 | 688 » » s->types.compression = MTypes_Bytes; |
687 » » // (Manually inlined copy of runtime·MHeap_Lookup) | 689 » » s->types.data = (uintptr)data3; |
688 » » p = (uintptr)v>>PageShift; | 690 » » ((uintptr*)data3)[1] = typ; |
689 » » p -= (uintptr)runtime·mheap.arena_start >> PageShift; | 691 » » data3[8*sizeof(uintptr) + ofs] = 1; |
690 » » s = runtime·mheap.spans[p]; | 692 » » break; |
691 | 693 » »······· |
692 » » if(s->sizeclass == 0) { | 694 » case MTypes_Words: |
693 » » » s->types.compression = MTypes_Single; | 695 » » ((uintptr*)s->types.data)[ofs] = typ; |
694 » » » s->types.data = typ; | 696 » » break; |
695 » » » continue; | 697 » »······· |
696 » » } | 698 » case MTypes_Bytes: |
697 | 699 » » data3 = (byte*)s->types.data; |
698 » » size = s->elemsize; | 700 » » for(j=1; j<8; j++) { |
699 » » ofs = ((uintptr)v - (s->start<<PageShift)) / size; | 701 » » » if(((uintptr*)data3)[j] == typ) { |
700 | 702 » » » » break; |
701 » » switch(s->types.compression) { | 703 » » » } |
702 » » case MTypes_Empty: | 704 » » » if(((uintptr*)data3)[j] == 0) { |
| 705 » » » » ((uintptr*)data3)[j] = typ; |
| 706 » » » » break; |
| 707 » » » } |
| 708 » » } |
| 709 » » if(j < 8) { |
| 710 » » » data3[8*sizeof(uintptr) + ofs] = j; |
| 711 » » } else { |
703 ntypes = (s->npages << PageShift) / size; | 712 ntypes = (s->npages << PageShift) / size; |
704 » » » nbytes3 = 8*sizeof(uintptr) + 1*ntypes; | 713 » » » nbytes2 = ntypes * sizeof(uintptr); |
705 » » » data3 = runtime·mallocgc(nbytes3, 0, FlagNoProfiling|Fla
gNoScan|FlagNoInvokeGC); | 714 » » » data2 = runtime·mallocgc(nbytes2, 0, FlagNoProfiling|Fla
gNoScan|FlagNoInvokeGC); |
706 » » » s->types.compression = MTypes_Bytes; | 715 » » » s->types.compression = MTypes_Words; |
707 » » » s->types.data = (uintptr)data3; | 716 » » » s->types.data = (uintptr)data2; |
708 » » » ((uintptr*)data3)[1] = typ; | 717 » » »······· |
709 » » » data3[8*sizeof(uintptr) + ofs] = 1; | 718 » » » // Move the contents of data3 to data2. Then deallocate
data3. |
710 » » » break; | 719 » » » for(j=0; j<ntypes; j++) { |
711 | 720 » » » » t = data3[8*sizeof(uintptr) + j]; |
712 » » case MTypes_Words: | 721 » » » » t = ((uintptr*)data3)[t]; |
713 » » » ((uintptr*)s->types.data)[ofs] = typ; | 722 » » » » data2[j] = t; |
714 » » » break; | |
715 | |
716 » » case MTypes_Bytes: | |
717 » » » data3 = (byte*)s->types.data; | |
718 » » » for(j=1; j<8; j++) { | |
719 » » » » if(((uintptr*)data3)[j] == typ) { | |
720 » » » » » break; | |
721 » » » » } | |
722 » » » » if(((uintptr*)data3)[j] == 0) { | |
723 » » » » » ((uintptr*)data3)[j] = typ; | |
724 » » » » » break; | |
725 » » » » } | |
726 } | 723 } |
727 » » » if(j < 8) { | 724 » » » data2[ofs] = typ; |
728 » » » » data3[8*sizeof(uintptr) + ofs] = j; | 725 » » } |
729 » » » } else { | 726 » » break; |
730 » » » » ntypes = (s->npages << PageShift) / size; | 727 » } |
731 » » » » nbytes2 = ntypes * sizeof(uintptr); | |
732 » » » » data2 = runtime·mallocgc(nbytes2, 0, FlagNoProfi
ling|FlagNoScan|FlagNoInvokeGC); | |
733 » » » » s->types.compression = MTypes_Words; | |
734 » » » » s->types.data = (uintptr)data2; | |
735 | |
736 » » » » // Move the contents of data3 to data2. Then dea
llocate data3. | |
737 » » » » for(j=0; j<ntypes; j++) { | |
738 » » » » » t = data3[8*sizeof(uintptr) + j]; | |
739 » » » » » t = ((uintptr*)data3)[t]; | |
740 » » » » » data2[j] = t; | |
741 » » » » } | |
742 » » » » data2[ofs] = typ; | |
743 » » » } | |
744 » » » break; | |
745 » » } | |
746 » } | |
747 » runtime·unlock(&settype_lock); | |
748 | |
749 » mp->settype_bufsize = 0; | |
750 } | 728 } |
751 | 729 |
752 uintptr | 730 uintptr |
753 runtime·gettype(void *v) | 731 runtime·gettype(void *v) |
754 { | 732 { |
755 MSpan *s; | 733 MSpan *s; |
756 uintptr t, ofs; | 734 uintptr t, ofs; |
757 byte *data; | 735 byte *data; |
758 | 736 |
759 s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); | 737 s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); |
(...skipping 12 matching lines...) Expand all Loading... |
772 case MTypes_Bytes: | 750 case MTypes_Bytes: |
773 ofs = (uintptr)v - (s->start<<PageShift); | 751 ofs = (uintptr)v - (s->start<<PageShift); |
774 data = (byte*)s->types.data; | 752 data = (byte*)s->types.data; |
775 t = data[8*sizeof(uintptr) + ofs/s->elemsize]; | 753 t = data[8*sizeof(uintptr) + ofs/s->elemsize]; |
776 t = ((uintptr*)data)[t]; | 754 t = ((uintptr*)data)[t]; |
777 break; | 755 break; |
778 default: | 756 default: |
779 runtime·throw("runtime·gettype: invalid compression kind
"); | 757 runtime·throw("runtime·gettype: invalid compression kind
"); |
780 } | 758 } |
781 if(0) { | 759 if(0) { |
782 runtime·lock(&settype_lock); | |
783 runtime·printf("%p -> %d,%X\n", v, (int32)s->types.compr
ession, (int64)t); | 760 runtime·printf("%p -> %d,%X\n", v, (int32)s->types.compr
ession, (int64)t); |
784 runtime·unlock(&settype_lock); | |
785 } | 761 } |
786 return t; | 762 return t; |
787 } | 763 } |
788 return 0; | 764 return 0; |
789 } | 765 } |
790 | 766 |
791 // Runtime stubs. | 767 // Runtime stubs. |
792 | 768 |
793 void* | 769 void* |
794 runtime·mal(uintptr n) | 770 runtime·mal(uintptr n) |
795 { | 771 { |
796 return runtime·mallocgc(n, 0, 0); | 772 return runtime·mallocgc(n, 0, 0); |
797 } | 773 } |
798 | 774 |
799 #pragma textflag NOSPLIT | 775 #pragma textflag NOSPLIT |
800 void | 776 func new(typ *Type) (ret *uint8) { |
801 runtime·new(Type *typ, uint8 *ret) | |
802 { | |
803 ret = runtime·mallocgc(typ->size, (uintptr)typ | TypeInfo_SingleObject,
typ->kind&KindNoPointers ? FlagNoScan : 0); | 777 ret = runtime·mallocgc(typ->size, (uintptr)typ | TypeInfo_SingleObject,
typ->kind&KindNoPointers ? FlagNoScan : 0); |
804 FLUSH(&ret); | |
805 } | 778 } |
806 | 779 |
807 static void* | 780 static void* |
808 cnew(Type *typ, intgo n, int32 objtyp) | 781 cnew(Type *typ, intgo n, int32 objtyp) |
809 { | 782 { |
810 if((objtyp&(PtrSize-1)) != objtyp) | 783 if((objtyp&(PtrSize-1)) != objtyp) |
811 runtime·throw("runtime: invalid objtyp"); | 784 runtime·throw("runtime: invalid objtyp"); |
812 if(n < 0 || (typ->size > 0 && n > MaxMem/typ->size)) | 785 if(n < 0 || (typ->size > 0 && n > MaxMem/typ->size)) |
813 runtime·panicstring("runtime: allocation size out of range"); | 786 runtime·panicstring("runtime: allocation size out of range"); |
814 return runtime·mallocgc(typ->size*n, (uintptr)typ | objtyp, typ->kind&Ki
ndNoPointers ? FlagNoScan : 0); | 787 return runtime·mallocgc(typ->size*n, (uintptr)typ | objtyp, typ->kind&Ki
ndNoPointers ? FlagNoScan : 0); |
815 } | 788 } |
816 | 789 |
817 // same as runtime·new, but callable from C | 790 // same as runtime·new, but callable from C |
818 void* | 791 void* |
819 runtime·cnew(Type *typ) | 792 runtime·cnew(Type *typ) |
820 { | 793 { |
821 return cnew(typ, 1, TypeInfo_SingleObject); | 794 return cnew(typ, 1, TypeInfo_SingleObject); |
822 } | 795 } |
823 | 796 |
824 void* | 797 void* |
825 runtime·cnewarray(Type *typ, intgo n) | 798 runtime·cnewarray(Type *typ, intgo n) |
826 { | 799 { |
827 return cnew(typ, n, TypeInfo_Array); | 800 return cnew(typ, n, TypeInfo_Array); |
828 } | 801 } |
829 | 802 |
830 func GC() { | 803 func GC() { |
| 804 // We assume that the user expects unused memory to have |
| 805 // been freed when GC returns. To ensure this, run gc(1) twice. |
| 806 // The first will do a collection, and the second will force the |
| 807 // first's sweeping to finish before doing a second collection. |
| 808 // The second collection is overkill, but we assume the user |
| 809 // has a good reason for calling runtime.GC and can stand the |
| 810 // expense. At the least, this fixes all the calls to runtime.GC in |
| 811 // tests that expect finalizers to start running when GC returns. |
| 812 runtime·gc(1); |
831 runtime·gc(1); | 813 runtime·gc(1); |
832 } | 814 } |
833 | 815 |
834 func SetFinalizer(obj Eface, finalizer Eface) { | 816 func SetFinalizer(obj Eface, finalizer Eface) { |
835 byte *base; | 817 byte *base; |
836 uintptr size; | 818 uintptr size; |
837 FuncType *ft; | 819 FuncType *ft; |
838 int32 i; | 820 int32 i; |
839 uintptr nret; | 821 uintptr nret; |
840 Type *t; | 822 Type *t; |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
898 // NOTE: asking to remove a finalizer when there currently isn't
one set is OK. | 880 // NOTE: asking to remove a finalizer when there currently isn't
one set is OK. |
899 runtime·removefinalizer(obj.data); | 881 runtime·removefinalizer(obj.data); |
900 } | 882 } |
901 return; | 883 return; |
902 | 884 |
903 badfunc: | 885 badfunc: |
904 runtime·printf("runtime.SetFinalizer: cannot pass %S to finalizer %S\n",
*obj.type->string, *finalizer.type->string); | 886 runtime·printf("runtime.SetFinalizer: cannot pass %S to finalizer %S\n",
*obj.type->string, *finalizer.type->string); |
905 throw: | 887 throw: |
906 runtime·throw("runtime.SetFinalizer"); | 888 runtime·throw("runtime.SetFinalizer"); |
907 } | 889 } |
LEFT | RIGHT |