Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(500)

Delta Between Two Patch Sets: src/pkg/runtime/malloc.goc

Issue 55100044: code review 55100044: runtime: use custom thunks for race calls instead of cgo (Closed)
Left Patch Set: diff -r 09a55add2bb5 https://dvyukov%40google.com@code.google.com/p/go/ Created 10 years, 2 months ago
Right Patch Set: diff -r 340da08f5f54 https://dvyukov%40google.com@code.google.com/p/go/ Created 10 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « src/pkg/runtime/cgocall.c ('k') | src/pkg/runtime/proc.c » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 // Copyright 2009 The Go Authors. All rights reserved. 1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style 2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file. 3 // license that can be found in the LICENSE file.
4 4
5 // See malloc.h for overview. 5 // See malloc.h for overview.
6 // 6 //
7 // TODO(rsc): double-check stats. 7 // TODO(rsc): double-check stats.
8 8
9 package runtime 9 package runtime
10 #include "runtime.h" 10 #include "runtime.h"
11 #include "arch_GOARCH.h" 11 #include "arch_GOARCH.h"
12 #include "malloc.h" 12 #include "malloc.h"
13 #include "type.h" 13 #include "type.h"
14 #include "typekind.h" 14 #include "typekind.h"
15 #include "race.h" 15 #include "race.h"
16 #include "stack.h" 16 #include "stack.h"
17 #include "../../cmd/ld/textflag.h" 17 #include "../../cmd/ld/textflag.h"
18 18
19 // Mark mheap as 'no pointers', it does not contain interesting pointers but occ upies ~45K. 19 // Mark mheap as 'no pointers', it does not contain interesting pointers but occ upies ~45K.
20 #pragma dataflag NOPTR 20 #pragma dataflag NOPTR
21 MHeap runtime·mheap; 21 MHeap runtime·mheap;
22 MStats mstats;
22 23
23 int32 runtime·checking; 24 int32 runtime·checking;
24 25
25 extern MStats mstats; // defined in zruntime_def_$GOOS_$GOARCH.go 26 extern MStats mstats; // defined in zruntime_def_$GOOS_$GOARCH.go
26 27
27 extern volatile intgo runtime·MemProfileRate; 28 extern volatile intgo runtime·MemProfileRate;
28 29
29 static void* largealloc(uint32, uintptr*); 30 static MSpan* largealloc(uint32, uintptr*);
31 static void profilealloc(void *v, uintptr size, uintptr typ);
32 static void settype(MSpan *s, void *v, uintptr typ);
30 33
31 // Allocate an object of at least size bytes. 34 // Allocate an object of at least size bytes.
32 // Small objects are allocated from the per-thread cache's free lists. 35 // Small objects are allocated from the per-thread cache's free lists.
33 // Large objects (> 32 kB) are allocated straight from the heap. 36 // Large objects (> 32 kB) are allocated straight from the heap.
34 // If the block will be freed with runtime·free(), typ must be 0. 37 // If the block will be freed with runtime·free(), typ must be 0.
35 void* 38 void*
36 runtime·mallocgc(uintptr size, uintptr typ, uint32 flag) 39 runtime·mallocgc(uintptr size, uintptr typ, uint32 flag)
37 { 40 {
38 int32 sizeclass; 41 int32 sizeclass;
39 uintptr tinysize, size1; 42 uintptr tinysize, size1;
40 intgo rate; 43 intgo rate;
41 MCache *c; 44 MCache *c;
42 » MCacheList *l; 45 » MSpan *s;
43 » MLink *v; 46 » MLink *v, *next;
44 byte *tiny; 47 byte *tiny;
45 48
46 if(size == 0) { 49 if(size == 0) {
47 // All 0-length allocations use this pointer. 50 // All 0-length allocations use this pointer.
48 // The language does not require the allocations to 51 // The language does not require the allocations to
49 // have distinct values. 52 // have distinct values.
50 return &runtime·zerobase; 53 return &runtime·zerobase;
51 } 54 }
52 if(m->mallocing) 55 if(m->mallocing)
53 runtime·throw("malloc/free - deadlock"); 56 runtime·throw("malloc/free - deadlock");
54 » // Disable preemption during settype_flush. 57 » // Disable preemption during settype.
55 » // We can not use m->mallocing for this, because settype_flush calls mal locgc. 58 » // We can not use m->mallocing for this, because settype calls mallocgc.
56 m->locks++; 59 m->locks++;
57 m->mallocing = 1; 60 m->mallocing = 1;
58 61
59 if(DebugTypeAtBlockEnd) 62 if(DebugTypeAtBlockEnd)
60 size += sizeof(uintptr); 63 size += sizeof(uintptr);
61 64
62 c = m->mcache; 65 c = m->mcache;
63 if(!runtime·debug.efence && size <= MaxSmallSize) { 66 if(!runtime·debug.efence && size <= MaxSmallSize) {
64 if((flag&(FlagNoScan|FlagNoGC)) == FlagNoScan && size < TinySize ) { 67 if((flag&(FlagNoScan|FlagNoGC)) == FlagNoScan && size < TinySize ) {
65 // Tiny allocator. 68 // Tiny allocator.
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
109 c->tiny += size1; 112 c->tiny += size1;
110 c->tinysize -= size1; 113 c->tinysize -= size1;
111 m->mallocing = 0; 114 m->mallocing = 0;
112 m->locks--; 115 m->locks--;
113 if(m->locks == 0 && g->preempt) // rest ore the preemption request in case we've cleared it in newstack 116 if(m->locks == 0 && g->preempt) // rest ore the preemption request in case we've cleared it in newstack
114 g->stackguard0 = StackPreempt; 117 g->stackguard0 = StackPreempt;
115 return v; 118 return v;
116 } 119 }
117 } 120 }
118 // Allocate a new TinySize block. 121 // Allocate a new TinySize block.
119 » » » l = &c->list[TinySizeClass]; 122 » » » s = c->alloc[TinySizeClass];
120 » » » if(l->list == nil) 123 » » » if(s->freelist == nil)
121 » » » » runtime·MCache_Refill(c, TinySizeClass); 124 » » » » s = runtime·MCache_Refill(c, TinySizeClass);
122 » » » v = l->list; 125 » » » v = s->freelist;
123 » » » l->list = v->next; 126 » » » next = v->next;
124 » » » l->nlist--; 127 » » » s->freelist = next;
128 » » » s->ref++;
129 » » » if(next != nil) // prefetching nil leads to a DTLB miss
130 » » » » PREFETCH(next);
125 ((uint64*)v)[0] = 0; 131 ((uint64*)v)[0] = 0;
126 ((uint64*)v)[1] = 0; 132 ((uint64*)v)[1] = 0;
127 // See if we need to replace the existing tiny block wit h the new one 133 // See if we need to replace the existing tiny block wit h the new one
128 // based on amount of remaining free space. 134 // based on amount of remaining free space.
129 if(TinySize-size > tinysize) { 135 if(TinySize-size > tinysize) {
130 c->tiny = (byte*)v + size; 136 c->tiny = (byte*)v + size;
131 c->tinysize = TinySize - size; 137 c->tinysize = TinySize - size;
132 } 138 }
133 size = TinySize; 139 size = TinySize;
134 goto done; 140 goto done;
135 } 141 }
136 // Allocate from mcache free lists. 142 // Allocate from mcache free lists.
137 // Inlined version of SizeToClass(). 143 // Inlined version of SizeToClass().
138 if(size <= 1024-8) 144 if(size <= 1024-8)
139 sizeclass = runtime·size_to_class8[(size+7)>>3]; 145 sizeclass = runtime·size_to_class8[(size+7)>>3];
140 else 146 else
141 sizeclass = runtime·size_to_class128[(size-1024+127) >> 7]; 147 sizeclass = runtime·size_to_class128[(size-1024+127) >> 7];
142 size = runtime·class_to_size[sizeclass]; 148 size = runtime·class_to_size[sizeclass];
143 » » l = &c->list[sizeclass]; 149 » » s = c->alloc[sizeclass];
144 » » if(l->list == nil) 150 » » if(s->freelist == nil)
145 » » » runtime·MCache_Refill(c, sizeclass); 151 » » » s = runtime·MCache_Refill(c, sizeclass);
146 » » v = l->list; 152 » » v = s->freelist;
147 » » l->list = v->next; 153 » » next = v->next;
148 » » l->nlist--; 154 » » s->freelist = next;
155 » » s->ref++;
156 » » if(next != nil) // prefetching nil leads to a DTLB miss
157 » » » PREFETCH(next);
149 if(!(flag & FlagNoZero)) { 158 if(!(flag & FlagNoZero)) {
150 v->next = nil; 159 v->next = nil;
151 // block is zeroed iff second word is zero ... 160 // block is zeroed iff second word is zero ...
152 if(size > 2*sizeof(uintptr) && ((uintptr*)v)[1] != 0) 161 if(size > 2*sizeof(uintptr) && ((uintptr*)v)[1] != 0)
153 runtime·memclr((byte*)v, size); 162 runtime·memclr((byte*)v, size);
154 } 163 }
155 done: 164 done:
156 c->local_cachealloc += size; 165 c->local_cachealloc += size;
157 } else { 166 } else {
158 // Allocate directly from heap. 167 // Allocate directly from heap.
159 » » v = largealloc(flag, &size); 168 » » s = largealloc(flag, &size);
169 » » v = (void*)(s->start << PageShift);
160 } 170 }
161 171
162 if(flag & FlagNoGC) 172 if(flag & FlagNoGC)
163 runtime·marknogc(v); 173 runtime·marknogc(v);
164 else if(!(flag & FlagNoScan)) 174 else if(!(flag & FlagNoScan))
165 runtime·markscan(v); 175 runtime·markscan(v);
166 176
167 if(DebugTypeAtBlockEnd) 177 if(DebugTypeAtBlockEnd)
168 *(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ; 178 *(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ;
169 179
180 m->mallocing = 0;
170 // TODO: save type even if FlagNoScan? Potentially expensive but might help 181 // TODO: save type even if FlagNoScan? Potentially expensive but might help
171 // heap profiling/tracing. 182 // heap profiling/tracing.
172 » if(UseSpanType && !(flag & FlagNoScan) && typ != 0) { 183 » if(UseSpanType && !(flag & FlagNoScan) && typ != 0)
173 » » uintptr *buf, i; 184 » » settype(s, v, typ);
174 185
175 » » buf = m->settype_buf; 186 » if(raceenabled)
176 » » i = m->settype_bufsize; 187 » » runtime·racemalloc(v, size);
177 » » buf[i++] = (uintptr)v; 188
178 » » buf[i++] = typ; 189 » if(runtime·debug.allocfreetrace)
179 » » m->settype_bufsize = i; 190 » » goto profile;
180 » } 191
181 192 » if(!(flag & FlagNoProfiling) && (rate = runtime·MemProfileRate) > 0) {
182 » m->mallocing = 0; 193 » » if(size < rate && size < c->next_sample)
183 » if(UseSpanType && !(flag & FlagNoScan) && typ != 0 && m->settype_bufsize == nelem(m->settype_buf)) 194 » » » c->next_sample -= size;
184 » » runtime·settype_flush(m); 195 » » else {
196 » » profile:
197 » » » profilealloc(v, size, typ);
198 » » }
199 » }
200
185 m->locks--; 201 m->locks--;
186 if(m->locks == 0 && g->preempt) // restore the preemption request in ca se we've cleared it in newstack 202 if(m->locks == 0 && g->preempt) // restore the preemption request in ca se we've cleared it in newstack
187 g->stackguard0 = StackPreempt; 203 g->stackguard0 = StackPreempt;
188 204
189 if(runtime·debug.allocfreetrace)
190 goto profile;
191
192 if(!(flag & FlagNoProfiling) && (rate = runtime·MemProfileRate) > 0) {
193 if(size >= rate)
194 goto profile;
195 if(m->mcache->next_sample > size)
196 m->mcache->next_sample -= size;
197 else {
198 // pick next profile time
199 // If you change this, also change allocmcache.
200 if(rate > 0x3fffffff) // make 2*rate not overflow
201 rate = 0x3fffffff;
202 m->mcache->next_sample = runtime·fastrand1() % (2*rate);
203 profile:
204 runtime·MProf_Malloc(v, size, typ);
205 }
206 }
207
208 if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc) 205 if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc)
209 runtime·gc(0); 206 runtime·gc(0);
210 207
211 if(raceenabled)
212 runtime·racemalloc(v, size);
213 return v; 208 return v;
214 } 209 }
215 210
216 static void* 211 static MSpan*
217 largealloc(uint32 flag, uintptr *sizep) 212 largealloc(uint32 flag, uintptr *sizep)
218 { 213 {
219 uintptr npages, size; 214 uintptr npages, size;
220 MSpan *s; 215 MSpan *s;
221 void *v; 216 void *v;
222 217
223 // Allocate directly from heap. 218 // Allocate directly from heap.
224 size = *sizep; 219 size = *sizep;
225 if(size + PageSize < size) 220 if(size + PageSize < size)
226 runtime·throw("out of memory"); 221 runtime·throw("out of memory");
227 npages = size >> PageShift; 222 npages = size >> PageShift;
228 if((size & PageMask) != 0) 223 if((size & PageMask) != 0)
229 npages++; 224 npages++;
230 s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, !(flag & FlagNoZer o)); 225 s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, !(flag & FlagNoZer o));
231 if(s == nil) 226 if(s == nil)
232 runtime·throw("out of memory"); 227 runtime·throw("out of memory");
233 s->limit = (byte*)(s->start<<PageShift) + size; 228 s->limit = (byte*)(s->start<<PageShift) + size;
234 *sizep = npages<<PageShift; 229 *sizep = npages<<PageShift;
235 v = (void*)(s->start << PageShift); 230 v = (void*)(s->start << PageShift);
236 // setup for mark sweep 231 // setup for mark sweep
237 runtime·markspan(v, 0, 0, true); 232 runtime·markspan(v, 0, 0, true);
238 » return v; 233 » return s;
234 }
235
236 static void
237 profilealloc(void *v, uintptr size, uintptr typ)
238 {
239 » uintptr rate;
240 » int32 next;
241 » MCache *c;
242
243 » c = m->mcache;
244 » rate = runtime·MemProfileRate;
245 » if(size < rate) {
246 » » // pick next profile time
247 » » // If you change this, also change allocmcache.
248 » » if(rate > 0x3fffffff)» // make 2*rate not overflow
249 » » » rate = 0x3fffffff;
250 » » next = runtime·fastrand1() % (2*rate);
251 » » // Subtract the "remainder" of the current allocation.
252 » » // Otherwise objects that are close in size to sampling rate
253 » » // will be under-sampled, because we consistently discard this r emainder.
254 » » next -= (size - c->next_sample);
255 » » if(next < 0)
256 » » » next = 0;
257 » » c->next_sample = next;
258 » }
259 » runtime·MProf_Malloc(v, size, typ);
239 } 260 }
240 261
241 void* 262 void*
242 runtime·malloc(uintptr size) 263 runtime·malloc(uintptr size)
243 { 264 {
244 return runtime·mallocgc(size, 0, FlagNoInvokeGC); 265 return runtime·mallocgc(size, 0, FlagNoInvokeGC);
245 } 266 }
246 267
247 // Free the object whose base pointer is v. 268 // Free the object whose base pointer is v.
248 void 269 void
(...skipping 18 matching lines...) Expand all
267 runtime·printf("free %p: not an allocated block\n", v); 288 runtime·printf("free %p: not an allocated block\n", v);
268 runtime·throw("free runtime·mlookup"); 289 runtime·throw("free runtime·mlookup");
269 } 290 }
270 size = s->elemsize; 291 size = s->elemsize;
271 sizeclass = s->sizeclass; 292 sizeclass = s->sizeclass;
272 // Objects that are smaller than TinySize can be allocated using tiny al loc, 293 // Objects that are smaller than TinySize can be allocated using tiny al loc,
273 // if then such object is combined with an object with finalizer, we wil l crash. 294 // if then such object is combined with an object with finalizer, we wil l crash.
274 if(size < TinySize) 295 if(size < TinySize)
275 runtime·throw("freeing too small block"); 296 runtime·throw("freeing too small block");
276 297
298 // Ensure that the span is swept.
299 // If we free into an unswept span, we will corrupt GC bitmaps.
300 runtime·MSpan_EnsureSwept(s);
301
277 if(s->specials != nil) 302 if(s->specials != nil)
278 runtime·freeallspecials(s, v, size); 303 runtime·freeallspecials(s, v, size);
279 304
280 c = m->mcache; 305 c = m->mcache;
281 if(sizeclass == 0) { 306 if(sizeclass == 0) {
282 // Large object. 307 // Large object.
283 » » *(uintptr*)(s->start<<PageShift) = (uintptr)0xfeedfeedfeedfeedll ;» // mark as "needs to be zeroed" 308 » » s->needzero = 1;
284 // Must mark v freed before calling unmarkspan and MHeap_Free: 309 // Must mark v freed before calling unmarkspan and MHeap_Free:
285 // they might coalesce v into other spans and change the bitmap further. 310 // they might coalesce v into other spans and change the bitmap further.
286 » » runtime·markfreed(v, size); 311 » » runtime·markfreed(v);
287 runtime·unmarkspan(v, 1<<PageShift); 312 runtime·unmarkspan(v, 1<<PageShift);
288 if(runtime·debug.efence) 313 if(runtime·debug.efence)
289 runtime·SysFree((void*)(s->start<<PageShift), size, &mst ats.heap_sys); 314 runtime·SysFree((void*)(s->start<<PageShift), size, &mst ats.heap_sys);
290 else 315 else
291 runtime·MHeap_Free(&runtime·mheap, s, 1); 316 runtime·MHeap_Free(&runtime·mheap, s, 1);
292 c->local_nlargefree++; 317 c->local_nlargefree++;
293 c->local_largefree += size; 318 c->local_largefree += size;
294 } else { 319 } else {
295 // Small object. 320 // Small object.
296 if(size > 2*sizeof(uintptr)) 321 if(size > 2*sizeof(uintptr))
297 ((uintptr*)v)[1] = (uintptr)0xfeedfeedfeedfeedll; // mark as "needs to be zeroed" 322 ((uintptr*)v)[1] = (uintptr)0xfeedfeedfeedfeedll; // mark as "needs to be zeroed"
298 else if(size > sizeof(uintptr)) 323 else if(size > sizeof(uintptr))
299 ((uintptr*)v)[1] = 0; 324 ((uintptr*)v)[1] = 0;
300 // Must mark v freed before calling MCache_Free: 325 // Must mark v freed before calling MCache_Free:
301 // it might coalesce v and other blocks into a bigger span 326 // it might coalesce v and other blocks into a bigger span
302 // and change the bitmap further. 327 // and change the bitmap further.
303 runtime·markfreed(v, size);
304 c->local_nsmallfree[sizeclass]++; 328 c->local_nsmallfree[sizeclass]++;
305 » » runtime·MCache_Free(c, v, sizeclass, size); 329 » » c->local_cachealloc -= size;
330 » » if(c->alloc[sizeclass] == s) {
331 » » » // We own the span, so we can just add v to the freelist
332 » » » runtime·markfreed(v);
333 » » » ((MLink*)v)->next = s->freelist;
334 » » » s->freelist = v;
335 » » » s->ref--;
336 » » } else {
337 » » » // Someone else owns this span. Add to free queue.
338 » » » runtime·MCache_Free(c, v, sizeclass, size);
339 » » }
306 } 340 }
307 m->mallocing = 0; 341 m->mallocing = 0;
308 } 342 }
309 343
310 int32 344 int32
311 runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp) 345 runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
312 { 346 {
313 uintptr n, i; 347 uintptr n, i;
314 byte *p; 348 byte *p;
315 MSpan *s; 349 MSpan *s;
(...skipping 30 matching lines...) Expand all
346 380
347 n = s->elemsize; 381 n = s->elemsize;
348 if(base) { 382 if(base) {
349 i = ((byte*)v - p)/n; 383 i = ((byte*)v - p)/n;
350 *base = p + i*n; 384 *base = p + i*n;
351 } 385 }
352 if(size) 386 if(size)
353 *size = n; 387 *size = n;
354 388
355 return 1; 389 return 1;
356 }
357
358 MCache*
359 runtime·allocmcache(void)
360 {
361 intgo rate;
362 MCache *c;
363
364 runtime·lock(&runtime·mheap);
365 c = runtime·FixAlloc_Alloc(&runtime·mheap.cachealloc);
366 runtime·unlock(&runtime·mheap);
367 runtime·memclr((byte*)c, sizeof(*c));
368
369 // Set first allocation sample size.
370 rate = runtime·MemProfileRate;
371 if(rate > 0x3fffffff) // make 2*rate not overflow
372 rate = 0x3fffffff;
373 if(rate != 0)
374 c->next_sample = runtime·fastrand1() % (2*rate);
375
376 return c;
377 }
378
379 void
380 runtime·freemcache(MCache *c)
381 {
382 runtime·MCache_ReleaseAll(c);
383 runtime·lock(&runtime·mheap);
384 runtime·purgecachedstats(c);
385 runtime·FixAlloc_Free(&runtime·mheap.cachealloc, c);
386 runtime·unlock(&runtime·mheap);
387 } 390 }
388 391
389 void 392 void
390 runtime·purgecachedstats(MCache *c) 393 runtime·purgecachedstats(MCache *c)
391 { 394 {
392 MHeap *h; 395 MHeap *h;
393 int32 i; 396 int32 i;
394 397
395 // Protected by either heap or GC lock. 398 // Protected by either heap or GC lock.
396 h = &runtime·mheap; 399 h = &runtime·mheap;
397 mstats.heap_alloc += c->local_cachealloc; 400 mstats.heap_alloc += c->local_cachealloc;
398 c->local_cachealloc = 0; 401 c->local_cachealloc = 0;
399 mstats.nlookup += c->local_nlookup; 402 mstats.nlookup += c->local_nlookup;
400 c->local_nlookup = 0; 403 c->local_nlookup = 0;
401 h->largefree += c->local_largefree; 404 h->largefree += c->local_largefree;
402 c->local_largefree = 0; 405 c->local_largefree = 0;
403 h->nlargefree += c->local_nlargefree; 406 h->nlargefree += c->local_nlargefree;
404 c->local_nlargefree = 0; 407 c->local_nlargefree = 0;
405 for(i=0; i<nelem(c->local_nsmallfree); i++) { 408 for(i=0; i<nelem(c->local_nsmallfree); i++) {
406 h->nsmallfree[i] += c->local_nsmallfree[i]; 409 h->nsmallfree[i] += c->local_nsmallfree[i];
407 c->local_nsmallfree[i] = 0; 410 c->local_nsmallfree[i] = 0;
408 } 411 }
409 } 412 }
410 413
411 uintptr runtime·sizeof_C_MStats = sizeof(MStats); 414 // Size of the trailing by_size array differs between Go and C,
415 // NumSizeClasses was changed, but we can not change Go struct because of backwa rd compatibility.
416 // sizeof_C_MStats is what C thinks about size of Go struct.
417 uintptr runtime·sizeof_C_MStats = sizeof(MStats) - (NumSizeClasses - 61) * sizeo f(mstats.by_size[0]);
412 418
413 #define MaxArena32 (2U<<30) 419 #define MaxArena32 (2U<<30)
414 420
415 void 421 void
416 runtime·mallocinit(void) 422 runtime·mallocinit(void)
417 { 423 {
418 byte *p; 424 byte *p;
419 uintptr arena_size, bitmap_size, spans_size; 425 uintptr arena_size, bitmap_size, spans_size;
420 extern byte end[]; 426 extern byte end[];
421 byte *want;
422 uintptr limit; 427 uintptr limit;
423 uint64 i; 428 uint64 i;
424 429
425 p = nil; 430 p = nil;
426 arena_size = 0; 431 arena_size = 0;
427 bitmap_size = 0; 432 bitmap_size = 0;
428 spans_size = 0; 433 spans_size = 0;
429 434
430 // for 64-bit build 435 // for 64-bit build
431 USED(p); 436 USED(p);
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after
470 // Actually we reserve 136 GB (because the bitmap ends up being 8 GB) 475 // Actually we reserve 136 GB (because the bitmap ends up being 8 GB)
471 // but it hardly matters: e0 00 is not valid UTF-8 either. 476 // but it hardly matters: e0 00 is not valid UTF-8 either.
472 // 477 //
473 // If this fails we fall back to the 32 bit memory mechanism 478 // If this fails we fall back to the 32 bit memory mechanism
474 arena_size = MaxMem; 479 arena_size = MaxMem;
475 bitmap_size = arena_size / (sizeof(void*)*8/4); 480 bitmap_size = arena_size / (sizeof(void*)*8/4);
476 spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[ 0]); 481 spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[ 0]);
477 spans_size = ROUND(spans_size, PageSize); 482 spans_size = ROUND(spans_size, PageSize);
478 for(i = 0; i <= 0x7f; i++) { 483 for(i = 0; i <= 0x7f; i++) {
479 p = (void*)(i<<40 | 0x00c0ULL<<32); 484 p = (void*)(i<<40 | 0x00c0ULL<<32);
480 » » » p = runtime·SysReserve(p, bitmap_size + spans_size + are na_size); 485 » » » p = runtime·SysReserve(p, bitmap_size + spans_size + are na_size + PageSize);
481 if(p != nil) 486 if(p != nil)
482 break; 487 break;
483 } 488 }
484 } 489 }
485 if (p == nil) { 490 if (p == nil) {
486 // On a 32-bit machine, we can't typically get away 491 // On a 32-bit machine, we can't typically get away
487 // with a giant virtual address space reservation. 492 // with a giant virtual address space reservation.
488 // Instead we map the memory information bitmap 493 // Instead we map the memory information bitmap
489 // immediately after the data segment, large enough 494 // immediately after the data segment, large enough
490 // to handle another 2GB of mappings (256 MB), 495 // to handle another 2GB of mappings (256 MB),
(...skipping 21 matching lines...) Expand all
512 517
513 // SysReserve treats the address we ask for, end, as a hint, 518 // SysReserve treats the address we ask for, end, as a hint,
514 // not as an absolute requirement. If we ask for the end 519 // not as an absolute requirement. If we ask for the end
515 // of the data segment but the operating system requires 520 // of the data segment but the operating system requires
516 // a little more space before we can start allocating, it will 521 // a little more space before we can start allocating, it will
517 // give out a slightly higher pointer. Except QEMU, which 522 // give out a slightly higher pointer. Except QEMU, which
518 // is buggy, as usual: it won't adjust the pointer upward. 523 // is buggy, as usual: it won't adjust the pointer upward.
519 // So adjust it upward a little bit ourselves: 1/4 MB to get 524 // So adjust it upward a little bit ourselves: 1/4 MB to get
520 // away from the running binary image and then round up 525 // away from the running binary image and then round up
521 // to a MB boundary. 526 // to a MB boundary.
522 » » want = (byte*)ROUND((uintptr)end + (1<<18), 1<<20); 527 » » p = (byte*)ROUND((uintptr)end + (1<<18), 1<<20);
523 » » p = runtime·SysReserve(want, bitmap_size + spans_size + arena_si ze); 528 » » p = runtime·SysReserve(p, bitmap_size + spans_size + arena_size + PageSize);
524 if(p == nil) 529 if(p == nil)
525 runtime·throw("runtime: cannot reserve arena virtual add ress space"); 530 runtime·throw("runtime: cannot reserve arena virtual add ress space");
526 » » if((uintptr)p & (((uintptr)1<<PageShift)-1)) 531 » }
527 » » » runtime·printf("runtime: SysReserve returned unaligned a ddress %p; asked for %p", p, 532
528 » » » » bitmap_size+spans_size+arena_size); 533 » // PageSize can be larger than OS definition of page size,
529 » } 534 » // so SysReserve can give us a PageSize-unaligned pointer.
530 » if((uintptr)p & (((uintptr)1<<PageShift)-1)) 535 » // To overcome this we ask for PageSize more and round up the pointer.
531 » » runtime·throw("runtime: SysReserve returned unaligned address"); 536 » p = (byte*)ROUND((uintptr)p, PageSize);
532 537
533 runtime·mheap.spans = (MSpan**)p; 538 runtime·mheap.spans = (MSpan**)p;
534 runtime·mheap.bitmap = p + spans_size; 539 runtime·mheap.bitmap = p + spans_size;
535 runtime·mheap.arena_start = p + spans_size + bitmap_size; 540 runtime·mheap.arena_start = p + spans_size + bitmap_size;
536 runtime·mheap.arena_used = runtime·mheap.arena_start; 541 runtime·mheap.arena_used = runtime·mheap.arena_start;
537 runtime·mheap.arena_end = runtime·mheap.arena_start + arena_size; 542 runtime·mheap.arena_end = runtime·mheap.arena_start + arena_size;
538 543
539 // Initialize the rest of the allocator.········ 544 // Initialize the rest of the allocator.········
540 runtime·MHeap_Init(&runtime·mheap); 545 runtime·MHeap_Init(&runtime·mheap);
541 m->mcache = runtime·allocmcache(); 546 m->mcache = runtime·allocmcache();
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after
652 persistent.pos += size; 657 persistent.pos += size;
653 runtime·unlock(&persistent); 658 runtime·unlock(&persistent);
654 if(stat != &mstats.other_sys) { 659 if(stat != &mstats.other_sys) {
655 // reaccount the allocation against provided stat 660 // reaccount the allocation against provided stat
656 runtime·xadd64(stat, size); 661 runtime·xadd64(stat, size);
657 runtime·xadd64(&mstats.other_sys, -(uint64)size); 662 runtime·xadd64(&mstats.other_sys, -(uint64)size);
658 } 663 }
659 return p; 664 return p;
660 } 665 }
661 666
662 static Lock settype_lock; 667 static void
663 668 settype(MSpan *s, void *v, uintptr typ)
664 void 669 {
665 runtime·settype_flush(M *mp)
666 {
667 » uintptr *buf, *endbuf;
668 uintptr size, ofs, j, t; 670 uintptr size, ofs, j, t;
669 uintptr ntypes, nbytes2, nbytes3; 671 uintptr ntypes, nbytes2, nbytes3;
670 uintptr *data2; 672 uintptr *data2;
671 byte *data3; 673 byte *data3;
672 » void *v; 674
673 » uintptr typ, p; 675 » if(s->sizeclass == 0) {
674 » MSpan *s; 676 » » s->types.compression = MTypes_Single;
675 677 » » s->types.data = typ;
676 » buf = mp->settype_buf; 678 » » return;
677 » endbuf = buf + mp->settype_bufsize; 679 » }
678 680 » size = s->elemsize;
679 » runtime·lock(&settype_lock); 681 » ofs = ((uintptr)v - (s->start<<PageShift)) / size;
680 » while(buf < endbuf) { 682
681 » » v = (void*)*buf; 683 » switch(s->types.compression) {
682 » » *buf = 0; 684 » case MTypes_Empty:
683 » » buf++; 685 » » ntypes = (s->npages << PageShift) / size;
684 » » typ = *buf; 686 » » nbytes3 = 8*sizeof(uintptr) + 1*ntypes;
685 » » buf++; 687 » » data3 = runtime·mallocgc(nbytes3, 0, FlagNoProfiling|FlagNoScan| FlagNoInvokeGC);
686 688 » » s->types.compression = MTypes_Bytes;
687 » » // (Manually inlined copy of runtime·MHeap_Lookup) 689 » » s->types.data = (uintptr)data3;
688 » » p = (uintptr)v>>PageShift; 690 » » ((uintptr*)data3)[1] = typ;
689 » » p -= (uintptr)runtime·mheap.arena_start >> PageShift; 691 » » data3[8*sizeof(uintptr) + ofs] = 1;
690 » » s = runtime·mheap.spans[p]; 692 » » break;
691 693 » »·······
692 » » if(s->sizeclass == 0) { 694 » case MTypes_Words:
693 » » » s->types.compression = MTypes_Single; 695 » » ((uintptr*)s->types.data)[ofs] = typ;
694 » » » s->types.data = typ; 696 » » break;
695 » » » continue; 697 » »·······
696 » » } 698 » case MTypes_Bytes:
697 699 » » data3 = (byte*)s->types.data;
698 » » size = s->elemsize; 700 » » for(j=1; j<8; j++) {
699 » » ofs = ((uintptr)v - (s->start<<PageShift)) / size; 701 » » » if(((uintptr*)data3)[j] == typ) {
700 702 » » » » break;
701 » » switch(s->types.compression) { 703 » » » }
702 » » case MTypes_Empty: 704 » » » if(((uintptr*)data3)[j] == 0) {
705 » » » » ((uintptr*)data3)[j] = typ;
706 » » » » break;
707 » » » }
708 » » }
709 » » if(j < 8) {
710 » » » data3[8*sizeof(uintptr) + ofs] = j;
711 » » } else {
703 ntypes = (s->npages << PageShift) / size; 712 ntypes = (s->npages << PageShift) / size;
704 » » » nbytes3 = 8*sizeof(uintptr) + 1*ntypes; 713 » » » nbytes2 = ntypes * sizeof(uintptr);
705 » » » data3 = runtime·mallocgc(nbytes3, 0, FlagNoProfiling|Fla gNoScan|FlagNoInvokeGC); 714 » » » data2 = runtime·mallocgc(nbytes2, 0, FlagNoProfiling|Fla gNoScan|FlagNoInvokeGC);
706 » » » s->types.compression = MTypes_Bytes; 715 » » » s->types.compression = MTypes_Words;
707 » » » s->types.data = (uintptr)data3; 716 » » » s->types.data = (uintptr)data2;
708 » » » ((uintptr*)data3)[1] = typ; 717 » » »·······
709 » » » data3[8*sizeof(uintptr) + ofs] = 1; 718 » » » // Move the contents of data3 to data2. Then deallocate data3.
710 » » » break; 719 » » » for(j=0; j<ntypes; j++) {
711 720 » » » » t = data3[8*sizeof(uintptr) + j];
712 » » case MTypes_Words: 721 » » » » t = ((uintptr*)data3)[t];
713 » » » ((uintptr*)s->types.data)[ofs] = typ; 722 » » » » data2[j] = t;
714 » » » break;
715
716 » » case MTypes_Bytes:
717 » » » data3 = (byte*)s->types.data;
718 » » » for(j=1; j<8; j++) {
719 » » » » if(((uintptr*)data3)[j] == typ) {
720 » » » » » break;
721 » » » » }
722 » » » » if(((uintptr*)data3)[j] == 0) {
723 » » » » » ((uintptr*)data3)[j] = typ;
724 » » » » » break;
725 » » » » }
726 } 723 }
727 » » » if(j < 8) { 724 » » » data2[ofs] = typ;
728 » » » » data3[8*sizeof(uintptr) + ofs] = j; 725 » » }
729 » » » } else { 726 » » break;
730 » » » » ntypes = (s->npages << PageShift) / size; 727 » }
731 » » » » nbytes2 = ntypes * sizeof(uintptr);
732 » » » » data2 = runtime·mallocgc(nbytes2, 0, FlagNoProfi ling|FlagNoScan|FlagNoInvokeGC);
733 » » » » s->types.compression = MTypes_Words;
734 » » » » s->types.data = (uintptr)data2;
735
736 » » » » // Move the contents of data3 to data2. Then dea llocate data3.
737 » » » » for(j=0; j<ntypes; j++) {
738 » » » » » t = data3[8*sizeof(uintptr) + j];
739 » » » » » t = ((uintptr*)data3)[t];
740 » » » » » data2[j] = t;
741 » » » » }
742 » » » » data2[ofs] = typ;
743 » » » }
744 » » » break;
745 » » }
746 » }
747 » runtime·unlock(&settype_lock);
748
749 » mp->settype_bufsize = 0;
750 } 728 }
751 729
752 uintptr 730 uintptr
753 runtime·gettype(void *v) 731 runtime·gettype(void *v)
754 { 732 {
755 MSpan *s; 733 MSpan *s;
756 uintptr t, ofs; 734 uintptr t, ofs;
757 byte *data; 735 byte *data;
758 736
759 s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); 737 s = runtime·MHeap_LookupMaybe(&runtime·mheap, v);
(...skipping 12 matching lines...) Expand all
772 case MTypes_Bytes: 750 case MTypes_Bytes:
773 ofs = (uintptr)v - (s->start<<PageShift); 751 ofs = (uintptr)v - (s->start<<PageShift);
774 data = (byte*)s->types.data; 752 data = (byte*)s->types.data;
775 t = data[8*sizeof(uintptr) + ofs/s->elemsize]; 753 t = data[8*sizeof(uintptr) + ofs/s->elemsize];
776 t = ((uintptr*)data)[t]; 754 t = ((uintptr*)data)[t];
777 break; 755 break;
778 default: 756 default:
779 runtime·throw("runtime·gettype: invalid compression kind "); 757 runtime·throw("runtime·gettype: invalid compression kind ");
780 } 758 }
781 if(0) { 759 if(0) {
782 runtime·lock(&settype_lock);
783 runtime·printf("%p -> %d,%X\n", v, (int32)s->types.compr ession, (int64)t); 760 runtime·printf("%p -> %d,%X\n", v, (int32)s->types.compr ession, (int64)t);
784 runtime·unlock(&settype_lock);
785 } 761 }
786 return t; 762 return t;
787 } 763 }
788 return 0; 764 return 0;
789 } 765 }
790 766
791 // Runtime stubs. 767 // Runtime stubs.
792 768
793 void* 769 void*
794 runtime·mal(uintptr n) 770 runtime·mal(uintptr n)
795 { 771 {
796 return runtime·mallocgc(n, 0, 0); 772 return runtime·mallocgc(n, 0, 0);
797 } 773 }
798 774
799 #pragma textflag NOSPLIT 775 #pragma textflag NOSPLIT
800 void 776 func new(typ *Type) (ret *uint8) {
801 runtime·new(Type *typ, uint8 *ret)
802 {
803 ret = runtime·mallocgc(typ->size, (uintptr)typ | TypeInfo_SingleObject, typ->kind&KindNoPointers ? FlagNoScan : 0); 777 ret = runtime·mallocgc(typ->size, (uintptr)typ | TypeInfo_SingleObject, typ->kind&KindNoPointers ? FlagNoScan : 0);
804 FLUSH(&ret);
805 } 778 }
806 779
807 static void* 780 static void*
808 cnew(Type *typ, intgo n, int32 objtyp) 781 cnew(Type *typ, intgo n, int32 objtyp)
809 { 782 {
810 if((objtyp&(PtrSize-1)) != objtyp) 783 if((objtyp&(PtrSize-1)) != objtyp)
811 runtime·throw("runtime: invalid objtyp"); 784 runtime·throw("runtime: invalid objtyp");
812 if(n < 0 || (typ->size > 0 && n > MaxMem/typ->size)) 785 if(n < 0 || (typ->size > 0 && n > MaxMem/typ->size))
813 runtime·panicstring("runtime: allocation size out of range"); 786 runtime·panicstring("runtime: allocation size out of range");
814 return runtime·mallocgc(typ->size*n, (uintptr)typ | objtyp, typ->kind&Ki ndNoPointers ? FlagNoScan : 0); 787 return runtime·mallocgc(typ->size*n, (uintptr)typ | objtyp, typ->kind&Ki ndNoPointers ? FlagNoScan : 0);
815 } 788 }
816 789
817 // same as runtime·new, but callable from C 790 // same as runtime·new, but callable from C
818 void* 791 void*
819 runtime·cnew(Type *typ) 792 runtime·cnew(Type *typ)
820 { 793 {
821 return cnew(typ, 1, TypeInfo_SingleObject); 794 return cnew(typ, 1, TypeInfo_SingleObject);
822 } 795 }
823 796
824 void* 797 void*
825 runtime·cnewarray(Type *typ, intgo n) 798 runtime·cnewarray(Type *typ, intgo n)
826 { 799 {
827 return cnew(typ, n, TypeInfo_Array); 800 return cnew(typ, n, TypeInfo_Array);
828 } 801 }
829 802
830 func GC() { 803 func GC() {
804 // We assume that the user expects unused memory to have
805 // been freed when GC returns. To ensure this, run gc(1) twice.
806 // The first will do a collection, and the second will force the
807 // first's sweeping to finish before doing a second collection.
808 // The second collection is overkill, but we assume the user
809 // has a good reason for calling runtime.GC and can stand the
810 // expense. At the least, this fixes all the calls to runtime.GC in
811 // tests that expect finalizers to start running when GC returns.
812 runtime·gc(1);
831 runtime·gc(1); 813 runtime·gc(1);
832 } 814 }
833 815
834 func SetFinalizer(obj Eface, finalizer Eface) { 816 func SetFinalizer(obj Eface, finalizer Eface) {
835 byte *base; 817 byte *base;
836 uintptr size; 818 uintptr size;
837 FuncType *ft; 819 FuncType *ft;
838 int32 i; 820 int32 i;
839 uintptr nret; 821 uintptr nret;
840 Type *t; 822 Type *t;
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after
898 // NOTE: asking to remove a finalizer when there currently isn't one set is OK. 880 // NOTE: asking to remove a finalizer when there currently isn't one set is OK.
899 runtime·removefinalizer(obj.data); 881 runtime·removefinalizer(obj.data);
900 } 882 }
901 return; 883 return;
902 884
903 badfunc: 885 badfunc:
904 runtime·printf("runtime.SetFinalizer: cannot pass %S to finalizer %S\n", *obj.type->string, *finalizer.type->string); 886 runtime·printf("runtime.SetFinalizer: cannot pass %S to finalizer %S\n", *obj.type->string, *finalizer.type->string);
905 throw: 887 throw:
906 runtime·throw("runtime.SetFinalizer"); 888 runtime·throw("runtime.SetFinalizer");
907 } 889 }
LEFTRIGHT

Powered by Google App Engine
RSS Feeds Recent Issues | This issue
This is Rietveld f62528b