Left: | ||
Right: |
OLD | NEW |
---|---|
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Garbage collector (GC). | 5 // Garbage collector (GC). |
6 // | 6 // |
7 // GC is: | 7 // GC is: |
8 // - mark&sweep | 8 // - mark&sweep |
9 // - mostly precise (with the exception of some C-allocated objects, assembly fr ames/arguments, etc) | 9 // - mostly precise (with the exception of some C-allocated objects, assembly fr ames/arguments, etc) |
10 // - parallel (up to MaxGcproc threads) | 10 // - parallel (up to MaxGcproc threads) |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
56 #include "stack.h" | 56 #include "stack.h" |
57 #include "mgc0.h" | 57 #include "mgc0.h" |
58 #include "chan.h" | 58 #include "chan.h" |
59 #include "race.h" | 59 #include "race.h" |
60 #include "type.h" | 60 #include "type.h" |
61 #include "typekind.h" | 61 #include "typekind.h" |
62 #include "funcdata.h" | 62 #include "funcdata.h" |
63 #include "../../cmd/ld/textflag.h" | 63 #include "../../cmd/ld/textflag.h" |
64 | 64 |
65 enum { | 65 enum { |
66 » Debug = 0, | 66 » Debug» » = 0, |
67 » CollectStats = 0, | 67 » ConcurrentSweep»= 1, |
68 » ConcurrentSweep = 1, | 68 » PreciseScan» = 1, |
69 | 69 |
70 » WorkbufSize» = 16*1024, | 70 » WorkbufSize» = 4*1024, |
71 FinBlockSize = 4*1024, | 71 FinBlockSize = 4*1024, |
72 | |
73 handoffThreshold = 4, | |
74 IntermediateBufferCapacity = 64, | |
75 | |
76 // Bits in type information | |
77 PRECISE = 1, | |
78 LOOP = 2, | |
79 PC_BITS = PRECISE | LOOP, | |
80 | |
81 RootData = 0, | 72 RootData = 0, |
82 RootBss = 1, | 73 RootBss = 1, |
83 RootFinalizers = 2, | 74 RootFinalizers = 2, |
84 » RootSpanTypes» = 3, | 75 » RootSpans» = 3, |
85 RootFlushCaches = 4, | 76 RootFlushCaches = 4, |
86 RootCount = 5, | 77 RootCount = 5, |
87 }; | 78 }; |
88 | 79 |
80 #define ScanConservatively ((byte*)1) | |
89 #define GcpercentUnknown (-2) | 81 #define GcpercentUnknown (-2) |
90 | 82 |
91 // Initialized from $GOGC. GOGC=off means no gc. | 83 // Initialized from $GOGC. GOGC=off means no gc. |
92 static int32 gcpercent = GcpercentUnknown; | 84 static int32 gcpercent = GcpercentUnknown; |
93 | 85 |
94 static FuncVal* poolcleanup; | 86 static FuncVal* poolcleanup; |
95 | 87 |
96 void | 88 void |
97 sync·runtime_registerPoolCleanup(FuncVal *f) | 89 sync·runtime_registerPoolCleanup(FuncVal *f) |
98 { | 90 { |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
131 // runtime·stoptheworld(); | 123 // runtime·stoptheworld(); |
132 // | 124 // |
133 // ... do stuff ... | 125 // ... do stuff ... |
134 // | 126 // |
135 // m->gcing = 0; | 127 // m->gcing = 0; |
136 // runtime·semrelease(&runtime·worldsema); | 128 // runtime·semrelease(&runtime·worldsema); |
137 // runtime·starttheworld(); | 129 // runtime·starttheworld(); |
138 // | 130 // |
139 uint32 runtime·worldsema = 1; | 131 uint32 runtime·worldsema = 1; |
140 | 132 |
141 typedef struct Obj Obj; | |
142 struct Obj | |
143 { | |
144 byte *p; // data pointer | |
145 uintptr n; // size of data in bytes | |
146 uintptr ti; // type info | |
147 }; | |
148 | |
149 typedef struct Workbuf Workbuf; | 133 typedef struct Workbuf Workbuf; |
150 struct Workbuf | 134 struct Workbuf |
151 { | 135 { |
152 #define SIZE (WorkbufSize-sizeof(LFNode)-sizeof(uintptr)) | 136 » LFNode» node; // must be first |
153 » LFNode node; // must be first | 137 » uintptr»nobj; |
154 » uintptr nobj; | 138 » byte*» obj[(WorkbufSize-sizeof(LFNode)-sizeof(uintptr))/PtrSize]; |
155 » Obj obj[SIZE/sizeof(Obj) - 1]; | |
156 » uint8 _padding[SIZE%sizeof(Obj) + sizeof(Obj)]; | |
157 #undef SIZE | |
158 }; | 139 }; |
159 | 140 |
160 typedef struct Finalizer Finalizer; | 141 typedef struct Finalizer Finalizer; |
161 struct Finalizer | 142 struct Finalizer |
162 { | 143 { |
163 FuncVal *fn; | 144 FuncVal *fn; |
164 void *arg; | 145 void *arg; |
165 uintptr nret; | 146 uintptr nret; |
166 Type *fint; | 147 Type *fint; |
167 PtrType *ot; | 148 PtrType *ot; |
(...skipping 28 matching lines...) Expand all Loading... | |
196 static G* fing; | 177 static G* fing; |
197 | 178 |
198 static void runfinq(void); | 179 static void runfinq(void); |
199 static void bgsweep(void); | 180 static void bgsweep(void); |
200 static Workbuf* getempty(Workbuf*); | 181 static Workbuf* getempty(Workbuf*); |
201 static Workbuf* getfull(Workbuf*); | 182 static Workbuf* getfull(Workbuf*); |
202 static void putempty(Workbuf*); | 183 static void putempty(Workbuf*); |
203 static Workbuf* handoff(Workbuf*); | 184 static Workbuf* handoff(Workbuf*); |
204 static void gchelperstart(void); | 185 static void gchelperstart(void); |
205 static void flushallmcaches(void); | 186 static void flushallmcaches(void); |
206 static bool» scanframe(Stkframe *frame, void *wbufp); | 187 static bool» scanframe(Stkframe *frame, void *unused); |
207 static void» addstackroots(G *gp, Workbuf **wbufp); | 188 static void» scanstack(G *gp); |
189 static byte*» unrollglobgcprog(byte *prog, uintptr size); | |
208 | 190 |
209 static FuncVal runfinqv = {runfinq}; | 191 static FuncVal runfinqv = {runfinq}; |
210 static FuncVal bgsweepv = {bgsweep}; | 192 static FuncVal bgsweepv = {bgsweep}; |
211 | 193 |
212 static struct { | 194 static struct { |
213 uint64 full; // lock-free list of full blocks | 195 uint64 full; // lock-free list of full blocks |
214 uint64 empty; // lock-free list of empty blocks | 196 uint64 empty; // lock-free list of empty blocks |
215 byte pad0[CacheLineSize]; // prevents false-sharing between full/empt y and nproc/nwait | 197 byte pad0[CacheLineSize]; // prevents false-sharing between full/empt y and nproc/nwait |
216 uint32 nproc; | 198 uint32 nproc; |
217 int64 tstart; | 199 int64 tstart; |
218 volatile uint32 nwait; | 200 volatile uint32 nwait; |
219 volatile uint32 ndone; | 201 volatile uint32 ndone; |
220 Note alldone; | 202 Note alldone; |
221 » ParFor» *markfor; | 203 » ParFor*»markfor; |
204 » byte*» gcdata; | |
205 » byte*» gcbss; | |
222 } work; | 206 } work; |
223 | 207 |
224 enum { | |
225 GC_DEFAULT_PTR = GC_NUM_INSTR, | |
226 GC_CHAN, | |
227 | |
228 GC_NUM_INSTR2 | |
229 }; | |
230 | |
231 static struct { | |
232 struct { | |
233 uint64 sum; | |
234 uint64 cnt; | |
235 } ptr; | |
236 uint64 nbytes; | |
237 struct { | |
238 uint64 sum; | |
239 uint64 cnt; | |
240 uint64 notype; | |
241 uint64 typelookup; | |
242 } obj; | |
243 uint64 rescan; | |
244 uint64 rescanbytes; | |
245 uint64 instr[GC_NUM_INSTR2]; | |
246 uint64 putempty; | |
247 uint64 getfull; | |
248 struct { | |
249 uint64 foundbit; | |
250 uint64 foundword; | |
251 uint64 foundspan; | |
252 } flushptrbuf; | |
253 struct { | |
254 uint64 foundbit; | |
255 uint64 foundword; | |
256 uint64 foundspan; | |
257 } markonly; | |
258 uint32 nbgsweep; | |
259 uint32 npausesweep; | |
260 } gcstats; | |
261 | |
262 // markonly marks an object. It returns true if the object | |
263 // has been marked by this function, false otherwise. | |
264 // This function doesn't append the object to any buffer. | |
265 static bool | |
266 markonly(void *obj) | |
267 { | |
268 byte *p; | |
269 uintptr *bitp, bits, shift, x, xbits, off; | |
270 MSpan *s; | |
271 PageID k; | |
272 | |
273 // Words outside the arena cannot be pointers. | |
274 if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used) | |
275 return false; | |
276 | |
277 // obj may be a pointer to a live object. | |
278 // Try to find the beginning of the object. | |
279 | |
280 // Round down to word boundary. | |
281 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | |
282 | |
283 // Find bits for this word. | |
284 off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | |
285 bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | |
286 shift = off % wordsPerBitmapWord; | |
287 xbits = *bitp; | |
288 bits = xbits >> shift; | |
289 | |
290 // Pointing at the beginning of a block? | |
291 if((bits & (bitAllocated|bitBlockBoundary)) != 0) { | |
292 if(CollectStats) | |
293 runtime·xadd64(&gcstats.markonly.foundbit, 1); | |
294 goto found; | |
295 } | |
296 | |
297 // Otherwise consult span table to find beginning. | |
298 // (Manually inlined copy of MHeap_LookupMaybe.) | |
299 k = (uintptr)obj>>PageShift; | |
300 x = k; | |
301 x -= (uintptr)runtime·mheap.arena_start>>PageShift; | |
302 s = runtime·mheap.spans[x]; | |
303 if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse ) | |
304 return false; | |
305 p = (byte*)((uintptr)s->start<<PageShift); | |
306 if(s->sizeclass == 0) { | |
307 obj = p; | |
308 } else { | |
309 uintptr size = s->elemsize; | |
310 int32 i = ((byte*)obj - p)/size; | |
311 obj = p+i*size; | |
312 } | |
313 | |
314 // Now that we know the object header, reload bits. | |
315 off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | |
316 bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | |
317 shift = off % wordsPerBitmapWord; | |
318 xbits = *bitp; | |
319 bits = xbits >> shift; | |
320 if(CollectStats) | |
321 runtime·xadd64(&gcstats.markonly.foundspan, 1); | |
322 | |
323 found: | |
324 // Now we have bits, bitp, and shift correct for | |
325 // obj pointing at the base of the object. | |
326 // Only care about allocated and not marked. | |
327 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | |
328 return false; | |
329 if(work.nproc == 1) | |
330 *bitp |= bitMarked<<shift; | |
331 else { | |
332 for(;;) { | |
333 x = *bitp; | |
334 if(x & (bitMarked<<shift)) | |
335 return false; | |
336 if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMa rked<<shift)))) | |
337 break; | |
338 } | |
339 } | |
340 | |
341 // The object is now marked | |
342 return true; | |
343 } | |
344 | |
345 // PtrTarget is a structure used by intermediate buffers. | |
346 // The intermediate buffers hold GC data before it | |
347 // is moved/flushed to the work buffer (Workbuf). | |
348 // The size of an intermediate buffer is very small, | |
349 // such as 32 or 64 elements. | |
350 typedef struct PtrTarget PtrTarget; | |
351 struct PtrTarget | |
352 { | |
353 void *p; | |
354 uintptr ti; | |
355 }; | |
356 | |
357 typedef struct Scanbuf Scanbuf; | |
358 struct Scanbuf | |
359 { | |
360 struct { | |
361 PtrTarget *begin; | |
362 PtrTarget *end; | |
363 PtrTarget *pos; | |
364 } ptr; | |
365 struct { | |
366 Obj *begin; | |
367 Obj *end; | |
368 Obj *pos; | |
369 } obj; | |
370 Workbuf *wbuf; | |
371 Obj *wp; | |
372 uintptr nobj; | |
373 }; | |
374 | |
375 typedef struct BufferList BufferList; | |
376 struct BufferList | |
377 { | |
378 PtrTarget ptrtarget[IntermediateBufferCapacity]; | |
379 Obj obj[IntermediateBufferCapacity]; | |
380 uint32 busy; | |
381 byte pad[CacheLineSize]; | |
382 }; | |
383 #pragma dataflag NOPTR | |
384 static BufferList bufferList[MaxGcproc]; | |
385 | |
386 static Type *itabtype; | |
387 | |
388 static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); | |
389 | |
390 // flushptrbuf moves data from the PtrTarget buffer to the work buffer. | |
391 // The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned, | |
392 // while the work buffer contains blocks which have been marked | |
393 // and are prepared to be scanned by the garbage collector. | |
394 // | |
395 // _wp, _wbuf, _nobj are input/output parameters and are specifying the work buf fer. | |
396 // | |
397 // A simplified drawing explaining how the todo-list moves from a structure to a nother: | |
398 // | |
399 // scanblock | |
400 // (find pointers) | |
401 // Obj ------> PtrTarget (pointer targets) | |
402 // ↑ | | |
403 // | | | |
404 // `----------' | |
405 // flushptrbuf | |
406 // (find block start, mark and enqueue) | |
407 static void | |
408 flushptrbuf(Scanbuf *sbuf) | |
409 { | |
410 byte *p, *arena_start, *obj; | |
411 uintptr size, *bitp, bits, shift, x, xbits, off, nobj, ti, n; | |
412 MSpan *s; | |
413 PageID k; | |
414 Obj *wp; | |
415 Workbuf *wbuf; | |
416 PtrTarget *ptrbuf; | |
417 PtrTarget *ptrbuf_end; | |
418 | |
419 arena_start = runtime·mheap.arena_start; | |
420 | |
421 wp = sbuf->wp; | |
422 wbuf = sbuf->wbuf; | |
423 nobj = sbuf->nobj; | |
424 | |
425 ptrbuf = sbuf->ptr.begin; | |
426 ptrbuf_end = sbuf->ptr.pos; | |
427 n = ptrbuf_end - sbuf->ptr.begin; | |
428 sbuf->ptr.pos = sbuf->ptr.begin; | |
429 | |
430 if(CollectStats) { | |
431 runtime·xadd64(&gcstats.ptr.sum, n); | |
432 runtime·xadd64(&gcstats.ptr.cnt, 1); | |
433 } | |
434 | |
435 // If buffer is nearly full, get a new one. | |
436 if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) { | |
437 if(wbuf != nil) | |
438 wbuf->nobj = nobj; | |
439 wbuf = getempty(wbuf); | |
440 wp = wbuf->obj; | |
441 nobj = 0; | |
442 | |
443 if(n >= nelem(wbuf->obj)) | |
444 runtime·throw("ptrbuf has to be smaller than WorkBuf"); | |
445 } | |
446 | |
447 while(ptrbuf < ptrbuf_end) { | |
448 obj = ptrbuf->p; | |
449 ti = ptrbuf->ti; | |
450 ptrbuf++; | |
451 | |
452 // obj belongs to interval [mheap.arena_start, mheap.arena_used) . | |
453 if(Debug > 1) { | |
454 if(obj < runtime·mheap.arena_start || obj >= runtime·mhe ap.arena_used) | |
455 runtime·throw("object is outside of mheap"); | |
456 } | |
457 | |
458 // obj may be a pointer to a live object. | |
459 // Try to find the beginning of the object. | |
460 | |
461 // Round down to word boundary. | |
462 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { | |
463 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | |
464 ti = 0; | |
465 } | |
466 | |
467 // Find bits for this word. | |
468 off = (uintptr*)obj - (uintptr*)arena_start; | |
469 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | |
470 shift = off % wordsPerBitmapWord; | |
471 xbits = *bitp; | |
472 bits = xbits >> shift; | |
473 | |
474 // Pointing at the beginning of a block? | |
475 if((bits & (bitAllocated|bitBlockBoundary)) != 0) { | |
476 if(CollectStats) | |
477 runtime·xadd64(&gcstats.flushptrbuf.foundbit, 1) ; | |
478 goto found; | |
479 } | |
480 | |
481 ti = 0; | |
482 | |
483 // Otherwise consult span table to find beginning. | |
484 // (Manually inlined copy of MHeap_LookupMaybe.) | |
485 k = (uintptr)obj>>PageShift; | |
486 x = k; | |
487 x -= (uintptr)arena_start>>PageShift; | |
488 s = runtime·mheap.spans[x]; | |
489 if(s == nil || k < s->start || obj >= s->limit || s->state != MS panInUse) | |
490 continue; | |
491 p = (byte*)((uintptr)s->start<<PageShift); | |
492 if(s->sizeclass == 0) { | |
493 obj = p; | |
494 } else { | |
495 size = s->elemsize; | |
496 int32 i = ((byte*)obj - p)/size; | |
497 obj = p+i*size; | |
498 } | |
499 | |
500 // Now that we know the object header, reload bits. | |
501 off = (uintptr*)obj - (uintptr*)arena_start; | |
502 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | |
503 shift = off % wordsPerBitmapWord; | |
504 xbits = *bitp; | |
505 bits = xbits >> shift; | |
506 if(CollectStats) | |
507 runtime·xadd64(&gcstats.flushptrbuf.foundspan, 1); | |
508 | |
509 found: | |
510 // Now we have bits, bitp, and shift correct for | |
511 // obj pointing at the base of the object. | |
512 // Only care about allocated and not marked. | |
513 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | |
514 continue; | |
515 if(work.nproc == 1) | |
516 *bitp |= bitMarked<<shift; | |
517 else { | |
518 for(;;) { | |
519 x = *bitp; | |
520 if(x & (bitMarked<<shift)) | |
521 goto continue_obj; | |
522 if(runtime·casp((void**)bitp, (void*)x, (void*)( x|(bitMarked<<shift)))) | |
523 break; | |
524 } | |
525 } | |
526 | |
527 // If object has no pointers, don't need to scan further. | |
528 if((bits & bitScan) == 0) | |
529 continue; | |
530 | |
531 // Ask span about size class. | |
532 // (Manually inlined copy of MHeap_Lookup.) | |
533 x = (uintptr)obj >> PageShift; | |
534 x -= (uintptr)arena_start>>PageShift; | |
535 s = runtime·mheap.spans[x]; | |
536 | |
537 PREFETCH(obj); | |
538 | |
539 *wp = (Obj){obj, s->elemsize, ti}; | |
540 wp++; | |
541 nobj++; | |
542 continue_obj:; | |
543 } | |
544 | |
545 // If another proc wants a pointer, give it some. | |
546 if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { | |
547 wbuf->nobj = nobj; | |
548 wbuf = handoff(wbuf); | |
549 nobj = wbuf->nobj; | |
550 wp = wbuf->obj + nobj; | |
551 } | |
552 | |
553 sbuf->wp = wp; | |
554 sbuf->wbuf = wbuf; | |
555 sbuf->nobj = nobj; | |
556 } | |
557 | |
558 static void | |
559 flushobjbuf(Scanbuf *sbuf) | |
560 { | |
561 uintptr nobj, off; | |
562 Obj *wp, obj; | |
563 Workbuf *wbuf; | |
564 Obj *objbuf; | |
565 Obj *objbuf_end; | |
566 | |
567 wp = sbuf->wp; | |
568 wbuf = sbuf->wbuf; | |
569 nobj = sbuf->nobj; | |
570 | |
571 objbuf = sbuf->obj.begin; | |
572 objbuf_end = sbuf->obj.pos; | |
573 sbuf->obj.pos = sbuf->obj.begin; | |
574 | |
575 while(objbuf < objbuf_end) { | |
576 obj = *objbuf++; | |
577 | |
578 // Align obj.b to a word boundary. | |
579 off = (uintptr)obj.p & (PtrSize-1); | |
580 if(off != 0) { | |
581 obj.p += PtrSize - off; | |
582 obj.n -= PtrSize - off; | |
583 obj.ti = 0; | |
584 } | |
585 | |
586 if(obj.p == nil || obj.n == 0) | |
587 continue; | |
588 | |
589 // If buffer is full, get a new one. | |
590 if(wbuf == nil || nobj >= nelem(wbuf->obj)) { | |
591 if(wbuf != nil) | |
592 wbuf->nobj = nobj; | |
593 wbuf = getempty(wbuf); | |
594 wp = wbuf->obj; | |
595 nobj = 0; | |
596 } | |
597 | |
598 *wp = obj; | |
599 wp++; | |
600 nobj++; | |
601 } | |
602 | |
603 // If another proc wants a pointer, give it some. | |
604 if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { | |
605 wbuf->nobj = nobj; | |
606 wbuf = handoff(wbuf); | |
607 nobj = wbuf->nobj; | |
608 wp = wbuf->obj + nobj; | |
609 } | |
610 | |
611 sbuf->wp = wp; | |
612 sbuf->wbuf = wbuf; | |
613 sbuf->nobj = nobj; | |
614 } | |
615 | |
616 // Program that scans the whole block and treats every block element as a potent ial pointer | |
617 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; | |
618 | |
619 // Hchan program | |
620 static uintptr chanProg[2] = {0, GC_CHAN}; | |
621 | |
622 // Local variables of a program fragment or loop | |
623 typedef struct Frame Frame; | |
624 struct Frame { | |
625 uintptr count, elemsize, b; | |
626 uintptr *loop_or_ret; | |
627 }; | |
628 | |
629 // Sanity check for the derived type info objti. | |
630 static void | |
631 checkptr(void *obj, uintptr objti) | |
632 { | |
633 uintptr *pc1, *pc2, type, tisize, i, j, x; | |
634 byte *objstart; | |
635 Type *t; | |
636 MSpan *s; | |
637 | |
638 if(!Debug) | |
639 runtime·throw("checkptr is debug only"); | |
640 | |
641 if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used) | |
642 return; | |
643 type = runtime·gettype(obj); | |
644 t = (Type*)(type & ~(uintptr)(PtrSize-1)); | |
645 if(t == nil) | |
646 return; | |
647 x = (uintptr)obj >> PageShift; | |
648 x -= (uintptr)(runtime·mheap.arena_start)>>PageShift; | |
649 s = runtime·mheap.spans[x]; | |
650 objstart = (byte*)((uintptr)s->start<<PageShift); | |
651 if(s->sizeclass != 0) { | |
652 i = ((byte*)obj - objstart)/s->elemsize; | |
653 objstart += i*s->elemsize; | |
654 } | |
655 tisize = *(uintptr*)objti; | |
656 // Sanity check for object size: it should fit into the memory block. | |
657 if((byte*)obj + tisize > objstart + s->elemsize) { | |
658 runtime·printf("object of type '%S' at %p/%p does not fit in blo ck %p/%p\n", | |
659 *t->string, obj, tisize, objstart, s->elemsize); | |
660 runtime·throw("invalid gc type info"); | |
661 } | |
662 if(obj != objstart) | |
663 return; | |
664 // If obj points to the beginning of the memory block, | |
665 // check type info as well. | |
666 if(t->string == nil || | |
667 // Gob allocates unsafe pointers for indirection. | |
668 (runtime·strcmp(t->string->str, (byte*)"unsafe.Pointer") && | |
669 // Runtime and gc think differently about closures. | |
670 runtime·strstr(t->string->str, (byte*)"struct { F uintptr") != t ->string->str)) { | |
671 pc1 = (uintptr*)objti; | |
672 pc2 = (uintptr*)t->gc; | |
673 // A simple best-effort check until first GC_END. | |
674 for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) { | |
675 if(pc1[j] != pc2[j]) { | |
676 runtime·printf("invalid gc type info for '%s', t ype info %p [%d]=%p, block info %p [%d]=%p\n", | |
677 t->string ? (int8*)t->string->str : (int8*)"?", pc1, (int32)j, pc1[j], pc2, (int32)j, pc2[j]); | |
678 runtime·throw("invalid gc type info"); | |
679 } | |
680 } | |
681 } | |
682 }······································· | |
683 | |
684 // scanblock scans a block of n bytes starting at pointer b for references | 208 // scanblock scans a block of n bytes starting at pointer b for references |
685 // to other objects, scanning any it finds recursively until there are no | 209 // to other objects, scanning any it finds recursively until there are no |
686 // unscanned objects left. Instead of using an explicit recursion, it keeps | 210 // unscanned objects left. Instead of using an explicit recursion, it keeps |
687 // a work list in the Workbuf* structures and loops in the main function | 211 // a work list in the Workbuf* structures and loops in the main function |
688 // body. Keeping an explicit work list is easier on the stack allocator and | 212 // body. Keeping an explicit work list is easier on the stack allocator and |
689 // more efficient. | 213 // more efficient. |
690 static void | 214 static void |
691 scanblock(Workbuf *wbuf, bool keepworking) | 215 scanblock(byte *b, uintptr n, byte *ptrmask) |
692 { | 216 { |
693 » byte *b, *arena_start, *arena_used; | 217 » byte *obj, *p, *arena_start, *arena_used, **wp, *scanbuf[8]; |
694 » uintptr n, i, end_b, elemsize, size, ti, objti, count, type, nobj; | 218 » uintptr i, nobj, size, idx, *bitp, bits, xbits, shift, x, off, cached, s canbufpos; |
695 » uintptr *pc, precise_type, nominal_size; | 219 » intptr ncached; |
696 » uintptr *chan_ret, chancap; | 220 » Workbuf *wbuf; |
697 » void *obj; | 221 » String *str; |
698 » Type *t, *et; | 222 » Slice *slice; |
699 » Slice *sliceptr; | 223 » Iface *iface; |
700 » String *stringptr; | |
701 » Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; | |
702 » BufferList *scanbuffers; | |
703 » Scanbuf sbuf; | |
704 Eface *eface; | 224 Eface *eface; |
705 » Iface *iface; | 225 » Type *typ; |
706 » Hchan *chan; | 226 » MSpan *s; |
707 » ChanType *chantype; | 227 » PageID k; |
708 » Obj *wp; | 228 » bool keepworking; |
709 | 229 |
710 » if(sizeof(Workbuf) % WorkbufSize != 0) | 230 » // Cache memory arena parameters in local vars. |
711 » » runtime·throw("scanblock: size of Workbuf is suboptimal"); | |
712 | |
713 » // Memory arena parameters. | |
714 arena_start = runtime·mheap.arena_start; | 231 arena_start = runtime·mheap.arena_start; |
715 arena_used = runtime·mheap.arena_used; | 232 arena_used = runtime·mheap.arena_used; |
716 | 233 |
717 » stack_ptr = stack+nelem(stack)-1; | 234 » wbuf = getempty(nil); |
718 | 235 » nobj = wbuf->nobj; |
719 » precise_type = false; | 236 » wp = &wbuf->obj[nobj]; |
720 » nominal_size = 0; | 237 » keepworking = b == nil; |
721 | 238 » scanbufpos = 0; |
722 » if(wbuf) { | 239 » for(i = 0; i < nelem(scanbuf); i++) |
723 » » nobj = wbuf->nobj; | 240 » » scanbuf[i] = nil; |
724 » » wp = &wbuf->obj[nobj]; | 241 |
725 » } else { | 242 » // ptrmask can have 3 possible values: |
726 » » nobj = 0; | 243 » // 1. nil - obtain pointer mask from GC bitmap. |
727 » » wp = nil; | 244 » // 2. ScanConservatively - don't use any mask, scan conservatively. |
245 » // 3. pointer to a compact mask (for stacks and data). | |
246 » if(b != nil) | |
247 » » goto scanobj; | |
248 » for(;;) { | |
249 » » if(nobj == 0) { | |
250 » » » // Out of work in workbuf. | |
251 » » » // First, see is there is any work in scanbuf. | |
252 » » » for(i = 0; i < nelem(scanbuf); i++) { | |
253 » » » » b = scanbuf[scanbufpos]; | |
254 » » » » scanbuf[scanbufpos++] = nil; | |
255 » » » » if(scanbufpos == nelem(scanbuf)) | |
256 » » » » » scanbufpos = 0; | |
257 » » » » if(b != nil) { | |
258 » » » » » n = arena_used - b; // scan until bitBou ndary or BitsDead | |
259 » » » » » ptrmask = nil; // use GC bitmap for poin ter info | |
260 » » » » » goto scanobj; | |
261 » » » » } | |
262 » » » } | |
263 » » » if(!keepworking) { | |
264 » » » » putempty(wbuf); | |
265 » » » » return; | |
266 » » » } | |
267 » » » // Refill workbuf from global queue. | |
268 » » » wbuf = getfull(wbuf); | |
269 » » » if(wbuf == nil) | |
270 » » » » return; | |
271 » » » nobj = wbuf->nobj; | |
272 » » » wp = &wbuf->obj[nobj]; | |
273 » » } | |
274 | |
275 » » // If another proc wants a pointer, give it some. | |
276 » » if(work.nwait > 0 && nobj > 4 && work.full == 0) { | |
277 » » » wbuf->nobj = nobj; | |
278 » » » wbuf = handoff(wbuf); | |
279 » » » nobj = wbuf->nobj; | |
280 » » » wp = &wbuf->obj[nobj]; | |
281 » » } | |
282 | |
283 » » wp--; | |
284 » » nobj--; | |
285 » » b = *wp; | |
286 » » n = arena_used - b; // scan until next bitBoundary or BitsDead | |
287 » » ptrmask = nil; // use GC bitmap for pointer info | |
288 | |
289 » scanobj: | |
290 » » if(!PreciseScan) { | |
291 » » » if(ptrmask == nil) { | |
292 » » » » // Heap obj, obtain real size. | |
293 » » » » if(!runtime·mlookup(b, &p, &n, nil)) | |
294 » » » » » continue; // not an allocated obj | |
295 » » » » if(b != p) | |
296 » » » » » runtime·throw("bad heap object"); | |
297 » » » } | |
298 » » » ptrmask = ScanConservatively; | |
299 » » } | |
300 » » cached = 0; | |
301 » » ncached = 0; | |
302 » » for(i = 0; i < n; i += PtrSize) { | |
303 » » » obj = nil; | |
304 » » » // Find bits for this word. | |
305 » » » if(ptrmask == nil) { | |
306 » » » » // Check is we have reached end of heap. | |
307 » » » » if((((uintptr)b+i)%PageSize) == 0 && | |
308 » » » » » runtime·mheap.spans[(b-arena_start)>>Pag eShift] != runtime·mheap.spans[(b+i-arena_start)>>PageShift]) | |
309 » » » » » break; | |
310 » » » » // Consult GC bitmap. | |
311 » » » » if(ncached <= 0) { | |
312 » » » » » // Refill cache. | |
313 » » » » » off = (uintptr*)(b+i) - (uintptr*)arena_ start; | |
314 » » » » » bitp = (uintptr*)arena_start - off/words PerBitmapWord - 1; | |
315 » » » » » shift = (off % wordsPerBitmapWord) * gcB its; | |
316 » » » » » cached = *bitp >> shift; | |
317 » » » » » ncached = (PtrSize*8 - shift)/gcBits; | |
318 » » » » } | |
319 » » » » bits = cached; | |
320 » » » » cached >>= gcBits; | |
321 » » » » ncached--; | |
322 » » » » if(i != 0 && (bits&bitMask) != bitMiddle) | |
323 » » » » » break; // reached beginning of the next object | |
324 » » » » bits = (bits>>2)&BitsMask; | |
325 » » » » if(bits == BitsDead) | |
326 » » » » » break; // reached no-scan part of the ob ject | |
327 » » » } else if(ptrmask != ScanConservatively) // dense mask ( stack or data) | |
328 » » » » bits = (ptrmask[(i/PtrSize)/4]>>(((i/PtrSize)%4) *BitsPerPointer))&BitsMask; | |
329 » » » else | |
330 » » » » bits = BitsPointer; | |
331 | |
332 » » » if(bits == BitsScalar || bits == BitsDead) | |
333 » » » » continue; | |
334 » » » if(bits == BitsPointer) { | |
335 » » » » obj = *(byte**)(b+i); | |
336 » » » » goto markobj; | |
337 » » » } | |
338 » » » // Find the next pair of bits. | |
339 » » » if(ptrmask == nil) { | |
340 » » » » if(ncached <= 0) { | |
341 » » » » » off = (uintptr*)(b+i+PtrSize) - (uintptr *)arena_start; | |
342 » » » » » bitp = (uintptr*)arena_start - off/words PerBitmapWord - 1; | |
343 » » » » » shift = (off % wordsPerBitmapWord) * gcB its; | |
344 » » » » » cached = *bitp >> shift; | |
345 » » » » » ncached = (PtrSize*8 - shift)/gcBits; | |
346 » » » » } | |
347 » » » » bits = (cached>>2)&BitsMask; | |
348 » » » } else | |
349 » » » » bits = (ptrmask[((i+PtrSize)/PtrSize)/4]>>((((i+ PtrSize)/PtrSize)%4)*BitsPerPointer))&BitsMask; | |
350 | |
351 » » » switch(bits) { | |
352 » » » case BitsString: | |
353 » » » » str = (String*)(b+i); | |
354 » » » » if(str->len > 0) | |
355 » » » » » obj = str->str; | |
356 » » » » break; | |
357 » » » case BitsSlice: | |
358 » » » » slice = (Slice*)(b+i); | |
359 » » » » if(Debug && slice->cap < slice->len) { | |
360 » » » » » g->m->traceback = 2; | |
361 » » » » » runtime·printf("bad slice in object %p: %p/%p/%p\n", | |
362 » » » » » » b, slice->array, slice->len, sli ce->cap); | |
363 » » » » » runtime·throw("bad slice in heap object" ); | |
364 » » » » } | |
365 » » » » if(slice->cap > 0) | |
366 » » » » » obj = slice->array; | |
367 » » » » break; | |
368 » » » case BitsIface: | |
369 » » » » iface = (Iface*)(b+i); | |
370 » » » » if(iface->tab != nil) { | |
371 » » » » » typ = iface->tab->type; | |
372 » » » » » if(typ->size > PtrSize || !(typ->kind&Ki ndNoPointers)) | |
373 » » » » » » obj = iface->data; | |
374 » » » » } | |
375 » » » » break; | |
376 » » » case BitsEface: | |
377 » » » » eface = (Eface*)(b+i); | |
378 » » » » typ = eface->type; | |
379 » » » » if(typ != nil) { | |
380 » » » » » if(typ->size > PtrSize || !(typ->kind&Ki ndNoPointers)) | |
381 » » » » » » obj = eface->data; | |
382 » » » » } | |
383 » » » » break; | |
384 » » » } | |
385 | |
386 » » » if(bits == BitsSlice) { | |
387 » » » » i += 2*PtrSize; | |
388 » » » » cached >>= 2*gcBits; | |
389 » » » » ncached -= 2; | |
390 » » » } else { | |
391 » » » » i += PtrSize; | |
392 » » » » cached >>= gcBits; | |
393 » » » » ncached--; | |
394 » » » } | |
395 | |
396 » » markobj: | |
397 » » » // At this point we have extracted the next potential po inter. | |
398 » » » // Check if it points into heap. | |
399 » » » if(obj == nil || obj < arena_start || obj >= arena_used) | |
400 » » » » continue; | |
401 » » » // Mark the object. | |
402 » » » off = (uintptr*)obj - (uintptr*)arena_start; | |
403 » » » bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | |
404 » » » shift = (off % wordsPerBitmapWord) * gcBits; | |
405 » » » xbits = *bitp; | |
406 » » » bits = (xbits >> shift) & bitMask; | |
407 » » » if(bits == bitMiddle) { | |
408 » » » » // Not a beginning of a block, check if we have block boundary in xbits. | |
409 » » » » while(shift > 0) { | |
410 » » » » » obj -= PtrSize; | |
411 » » » » » shift -= gcBits; | |
412 » » » » » bits = (xbits >> shift) & bitMask; | |
413 » » » » » if(bits != bitMiddle) | |
414 » » » » » » goto havebits; | |
415 » » » » } | |
416 » » » » // Otherwise consult span table to find the bloc k beginning. | |
417 » » » » k = (uintptr)obj>>PageShift; | |
418 » » » » x = k; | |
419 » » » » x -= (uintptr)arena_start>>PageShift; | |
420 » » » » s = runtime·mheap.spans[x]; | |
421 » » » » if(s == nil || k < s->start || obj >= s->limit | | s->state != MSpanInUse) | |
422 » » » » » continue; | |
423 » » » » p = (byte*)((uintptr)s->start<<PageShift); | |
424 » » » » if(s->sizeclass != 0) { | |
425 » » » » » size = s->elemsize; | |
426 » » » » » idx = ((byte*)obj - p)/size; | |
427 » » » » » p = p+idx*size; | |
428 » » » » } | |
429 » » » » if(p == obj) { | |
430 » » » » » runtime·printf("runtime: failed to find block beginning for %p s->limit=%p\n", p, s->limit); | |
431 » » » » » runtime·throw("failed to find block begi nning"); | |
432 » » » » } | |
433 » » » » obj = p; | |
434 » » » » goto markobj; | |
435 » » » } | |
436 | |
437 » » havebits: | |
438 » » » // Now we have bits, bitp, and shift correct for | |
439 » » » // obj pointing at the base of the object. | |
440 » » » // Only care about allocated and not marked. | |
441 » » » if(bits != bitAllocated) | |
442 » » » » continue; | |
443 » » » if(work.nproc == 1) | |
444 » » » » *bitp |= bitMarked<<shift; | |
445 » » » else { | |
446 » » » » for(;;) { | |
447 » » » » » xbits = *bitp; | |
448 » » » » » bits = (xbits>>shift) & bitMask; | |
449 » » » » » if(bits != bitAllocated) | |
450 » » » » » » break; | |
451 » » » » » if(runtime·casp((void**)bitp, (void*)xbi ts, (void*)(xbits|(bitMarked<<shift)))) | |
452 » » » » » » break; | |
453 » » » » } | |
454 » » » » if(bits != bitAllocated) | |
455 » » » » » continue; | |
456 » » » } | |
457 » » » if(((xbits>>(shift+2))&BitsMask) == BitsDead) | |
458 » » » » continue; // noscan object | |
459 | |
460 » » » // Queue the obj for scanning. | |
461 » » » PREFETCH(obj); | |
462 » » » obj = (byte*)((uintptr)obj & ~(PtrSize-1)); | |
463 » » » p = scanbuf[scanbufpos]; | |
464 » » » scanbuf[scanbufpos++] = obj; | |
465 » » » if(scanbufpos == nelem(scanbuf)) | |
466 » » » » scanbufpos = 0; | |
467 » » » if(p == nil) | |
468 » » » » continue; | |
469 | |
470 » » » // If workbuf is full, obtain an empty one. | |
471 » » » if(nobj >= nelem(wbuf->obj)) { | |
472 » » » » wbuf->nobj = nobj; | |
473 » » » » wbuf = getempty(wbuf); | |
474 » » » » nobj = wbuf->nobj; | |
475 » » » » wp = &wbuf->obj[nobj]; | |
476 » » » } | |
477 » » » *wp = p; | |
478 » » » wp++; | |
479 » » » nobj++; | |
480 » » } | |
481 | |
482 » » if(Debug && ptrmask == nil) { | |
483 » » » // For heap objects ensure that we did not overscan. | |
484 » » » n = 0; | |
485 » » » p = nil; | |
486 » » » if(!runtime·mlookup(b, &p, &n, nil) || b != p || i > n) { | |
487 » » » » runtime·printf("runtime: scanned (%p,%p), heap o bject (%p,%p)\n", b, i, p, n); | |
488 » » » » runtime·throw("scanblock: scanned invalid object "); | |
489 » » » } | |
490 » » } | |
728 } | 491 } |
729 | |
730 // Initialize sbuf | |
731 scanbuffers = &bufferList[g->m->helpgc]; | |
732 | |
733 sbuf.ptr.begin = sbuf.ptr.pos = &scanbuffers->ptrtarget[0]; | |
734 sbuf.ptr.end = sbuf.ptr.begin + nelem(scanbuffers->ptrtarget); | |
735 | |
736 sbuf.obj.begin = sbuf.obj.pos = &scanbuffers->obj[0]; | |
737 sbuf.obj.end = sbuf.obj.begin + nelem(scanbuffers->obj); | |
738 | |
739 sbuf.wbuf = wbuf; | |
740 sbuf.wp = wp; | |
741 sbuf.nobj = nobj; | |
742 | |
743 // (Silence the compiler) | |
744 chan = nil; | |
745 chantype = nil; | |
746 chan_ret = nil; | |
747 | |
748 goto next_block; | |
749 | |
750 for(;;) { | |
751 // Each iteration scans the block b of length n, queueing pointe rs in | |
752 // the work buffer. | |
753 | |
754 if(CollectStats) { | |
755 runtime·xadd64(&gcstats.nbytes, n); | |
756 runtime·xadd64(&gcstats.obj.sum, sbuf.nobj); | |
757 runtime·xadd64(&gcstats.obj.cnt, 1); | |
758 } | |
759 | |
760 if(ti != 0) { | |
761 if(Debug > 1) { | |
762 runtime·printf("scanblock %p %D ti %p\n", b, (in t64)n, ti); | |
763 } | |
764 pc = (uintptr*)(ti & ~(uintptr)PC_BITS); | |
765 precise_type = (ti & PRECISE); | |
766 stack_top.elemsize = pc[0]; | |
767 if(!precise_type) | |
768 nominal_size = pc[0]; | |
769 if(ti & LOOP) { | |
770 stack_top.count = 0; // 0 means an infinite n umber of iterations | |
771 stack_top.loop_or_ret = pc+1; | |
772 } else { | |
773 stack_top.count = 1; | |
774 } | |
775 if(Debug) { | |
776 // Simple sanity check for provided type info ti : | |
777 // The declared size of the object must be not l arger than the actual size | |
778 // (it can be smaller due to inferior pointers). | |
779 // It's difficult to make a comprehensive check due to inferior pointers, | |
780 // reflection, gob, etc. | |
781 if(pc[0] > n) { | |
782 runtime·printf("invalid gc type info: ty pe info size %p, block size %p\n", pc[0], n); | |
783 runtime·throw("invalid gc type info"); | |
784 } | |
785 } | |
786 } else if(UseSpanType) { | |
787 if(CollectStats) | |
788 runtime·xadd64(&gcstats.obj.notype, 1); | |
789 | |
790 type = runtime·gettype(b); | |
791 if(type != 0) { | |
792 if(CollectStats) | |
793 runtime·xadd64(&gcstats.obj.typelookup, 1); | |
794 | |
795 t = (Type*)(type & ~(uintptr)(PtrSize-1)); | |
796 switch(type & (PtrSize-1)) { | |
797 case TypeInfo_SingleObject: | |
798 pc = (uintptr*)t->gc; | |
799 precise_type = true; // type informatio n about 'b' is precise | |
800 stack_top.count = 1; | |
801 stack_top.elemsize = pc[0]; | |
802 break; | |
803 case TypeInfo_Array: | |
804 pc = (uintptr*)t->gc; | |
805 if(pc[0] == 0) | |
806 goto next_block; | |
807 precise_type = true; // type informatio n about 'b' is precise | |
808 stack_top.count = 0; // 0 means an infi nite number of iterations | |
809 stack_top.elemsize = pc[0]; | |
810 stack_top.loop_or_ret = pc+1; | |
811 break; | |
812 case TypeInfo_Chan: | |
813 chan = (Hchan*)b; | |
814 chantype = (ChanType*)t; | |
815 chan_ret = nil; | |
816 pc = chanProg; | |
817 break; | |
818 default: | |
819 if(Debug > 1) | |
820 runtime·printf("scanblock %p %D type %p %S\n", b, (int64)n, type, *t->string); | |
821 runtime·throw("scanblock: invalid type") ; | |
822 return; | |
823 } | |
824 if(Debug > 1) | |
825 runtime·printf("scanblock %p %D type %p %S pc=%p\n", b, (int64)n, type, *t->string, pc); | |
826 } else { | |
827 pc = defaultProg; | |
828 if(Debug > 1) | |
829 runtime·printf("scanblock %p %D unknown type\n", b, (int64)n); | |
830 } | |
831 } else { | |
832 pc = defaultProg; | |
833 if(Debug > 1) | |
834 runtime·printf("scanblock %p %D no span types\n" , b, (int64)n); | |
835 } | |
836 | |
837 if(IgnorePreciseGC) | |
838 pc = defaultProg; | |
839 | |
840 pc++; | |
841 stack_top.b = (uintptr)b; | |
842 end_b = (uintptr)b + n - PtrSize; | |
843 | |
844 for(;;) { | |
845 if(CollectStats) | |
846 runtime·xadd64(&gcstats.instr[pc[0]], 1); | |
847 | |
848 obj = nil; | |
849 objti = 0; | |
850 switch(pc[0]) { | |
851 case GC_PTR: | |
852 obj = *(void**)(stack_top.b + pc[1]); | |
853 objti = pc[2]; | |
854 if(Debug > 2) | |
855 runtime·printf("gc_ptr @%p: %p ti=%p\n", stack_t op.b+pc[1], obj, objti); | |
856 pc += 3; | |
857 if(Debug) | |
858 checkptr(obj, objti); | |
859 break; | |
860 | |
861 case GC_SLICE: | |
862 sliceptr = (Slice*)(stack_top.b + pc[1]); | |
863 if(Debug > 2) | |
864 runtime·printf("gc_slice @%p: %p/%D/%D\n", slice ptr, sliceptr->array, (int64)sliceptr->len, (int64)sliceptr->cap); | |
865 if(sliceptr->cap != 0) { | |
866 obj = sliceptr->array; | |
867 // Can't use slice element type for scanning, | |
868 // because if it points to an array embedded | |
869 // in the beginning of a struct, | |
870 // we will scan the whole struct as the slice. | |
871 // So just obtain type info from heap. | |
872 } | |
873 pc += 3; | |
874 break; | |
875 | |
876 case GC_APTR: | |
877 obj = *(void**)(stack_top.b + pc[1]); | |
878 if(Debug > 2) | |
879 runtime·printf("gc_aptr @%p: %p\n", stack_top.b+ pc[1], obj); | |
880 pc += 2; | |
881 break; | |
882 | |
883 case GC_STRING: | |
884 stringptr = (String*)(stack_top.b + pc[1]); | |
885 if(Debug > 2) | |
886 runtime·printf("gc_string @%p: %p/%D\n", stack_t op.b+pc[1], stringptr->str, (int64)stringptr->len); | |
887 if(stringptr->len != 0) | |
888 markonly(stringptr->str); | |
889 pc += 2; | |
890 continue; | |
891 | |
892 case GC_EFACE: | |
893 eface = (Eface*)(stack_top.b + pc[1]); | |
894 pc += 2; | |
895 if(Debug > 2) | |
896 runtime·printf("gc_eface @%p: %p %p\n", stack_to p.b+pc[1], eface->type, eface->data); | |
897 if(eface->type == nil) | |
898 continue; | |
899 | |
900 // eface->type | |
901 t = eface->type; | |
902 if((void*)t >= arena_start && (void*)t < arena_used) { | |
903 *sbuf.ptr.pos++ = (PtrTarget){t, 0}; | |
904 if(sbuf.ptr.pos == sbuf.ptr.end) | |
905 flushptrbuf(&sbuf); | |
906 } | |
907 | |
908 // eface->data | |
909 if(eface->data >= arena_start && eface->data < arena_use d) { | |
910 if(t->size <= sizeof(void*)) { | |
911 if((t->kind & KindNoPointers)) | |
912 continue; | |
913 | |
914 obj = eface->data; | |
915 if((t->kind & ~KindNoPointers) == KindPt r) { | |
916 // Only use type information if it is a pointer-containing type. | |
917 // This matches the GC programs written by cmd/gc/reflect.c's | |
918 // dgcsym1 in case TPTR32/case T PTR64. See rationale there. | |
919 et = ((PtrType*)t)->elem; | |
920 if(!(et->kind & KindNoPointers)) | |
921 objti = (uintptr)((PtrTy pe*)t)->elem->gc; | |
922 } | |
923 } else { | |
924 obj = eface->data; | |
925 objti = (uintptr)t->gc; | |
926 } | |
927 } | |
928 break; | |
929 | |
930 case GC_IFACE: | |
931 iface = (Iface*)(stack_top.b + pc[1]); | |
932 pc += 2; | |
933 if(Debug > 2) | |
934 runtime·printf("gc_iface @%p: %p/%p %p\n", stack _top.b+pc[1], iface->tab, nil, iface->data); | |
935 if(iface->tab == nil) | |
936 continue; | |
937 ························ | |
938 // iface->tab | |
939 if((void*)iface->tab >= arena_start && (void*)iface->tab < arena_used) { | |
940 *sbuf.ptr.pos++ = (PtrTarget){iface->tab, (uintp tr)itabtype->gc}; | |
941 if(sbuf.ptr.pos == sbuf.ptr.end) | |
942 flushptrbuf(&sbuf); | |
943 } | |
944 | |
945 // iface->data | |
946 if(iface->data >= arena_start && iface->data < arena_use d) { | |
947 t = iface->tab->type; | |
948 if(t->size <= sizeof(void*)) { | |
949 if((t->kind & KindNoPointers)) | |
950 continue; | |
951 | |
952 obj = iface->data; | |
953 if((t->kind & ~KindNoPointers) == KindPt r) { | |
954 // Only use type information if it is a pointer-containing type. | |
955 // This matches the GC programs written by cmd/gc/reflect.c's | |
956 // dgcsym1 in case TPTR32/case T PTR64. See rationale there. | |
957 et = ((PtrType*)t)->elem; | |
958 if(!(et->kind & KindNoPointers)) | |
959 objti = (uintptr)((PtrTy pe*)t)->elem->gc; | |
960 } | |
961 } else { | |
962 obj = iface->data; | |
963 objti = (uintptr)t->gc; | |
964 } | |
965 } | |
966 break; | |
967 | |
968 case GC_DEFAULT_PTR: | |
969 while(stack_top.b <= end_b) { | |
970 obj = *(byte**)stack_top.b; | |
971 if(Debug > 2) | |
972 runtime·printf("gc_default_ptr @%p: %p\n ", stack_top.b, obj); | |
973 stack_top.b += PtrSize; | |
974 if(obj >= arena_start && obj < arena_used) { | |
975 *sbuf.ptr.pos++ = (PtrTarget){obj, 0}; | |
976 if(sbuf.ptr.pos == sbuf.ptr.end) | |
977 flushptrbuf(&sbuf); | |
978 } | |
979 } | |
980 goto next_block; | |
981 | |
982 case GC_END: | |
983 if(--stack_top.count != 0) { | |
984 // Next iteration of a loop if possible. | |
985 stack_top.b += stack_top.elemsize; | |
986 if(stack_top.b + stack_top.elemsize <= end_b+Ptr Size) { | |
987 pc = stack_top.loop_or_ret; | |
988 continue; | |
989 } | |
990 i = stack_top.b; | |
991 } else { | |
992 // Stack pop if possible. | |
993 if(stack_ptr+1 < stack+nelem(stack)) { | |
994 pc = stack_top.loop_or_ret; | |
995 stack_top = *(++stack_ptr); | |
996 continue; | |
997 } | |
998 i = (uintptr)b + nominal_size; | |
999 } | |
1000 if(!precise_type) { | |
1001 // Quickly scan [b+i,b+n) for possible pointers. | |
1002 for(; i<=end_b; i+=PtrSize) { | |
1003 if(*(byte**)i != nil) { | |
1004 // Found a value that may be a p ointer. | |
1005 // Do a rescan of the entire blo ck. | |
1006 enqueue((Obj){b, n, 0}, &sbuf.wb uf, &sbuf.wp, &sbuf.nobj); | |
1007 if(CollectStats) { | |
1008 runtime·xadd64(&gcstats. rescan, 1); | |
1009 runtime·xadd64(&gcstats. rescanbytes, n); | |
1010 } | |
1011 break; | |
1012 } | |
1013 } | |
1014 } | |
1015 goto next_block; | |
1016 | |
1017 case GC_ARRAY_START: | |
1018 i = stack_top.b + pc[1]; | |
1019 count = pc[2]; | |
1020 elemsize = pc[3]; | |
1021 pc += 4; | |
1022 | |
1023 // Stack push. | |
1024 *stack_ptr-- = stack_top; | |
1025 stack_top = (Frame){count, elemsize, i, pc}; | |
1026 continue; | |
1027 | |
1028 case GC_ARRAY_NEXT: | |
1029 if(--stack_top.count != 0) { | |
1030 stack_top.b += stack_top.elemsize; | |
1031 pc = stack_top.loop_or_ret; | |
1032 } else { | |
1033 // Stack pop. | |
1034 stack_top = *(++stack_ptr); | |
1035 pc += 1; | |
1036 } | |
1037 continue; | |
1038 | |
1039 case GC_CALL: | |
1040 // Stack push. | |
1041 *stack_ptr-- = stack_top; | |
1042 stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*re turn address*/}; | |
1043 pc = (uintptr*)((byte*)pc + *(int32*)(pc+2)); // target of the CALL instruction | |
1044 continue; | |
1045 | |
1046 case GC_REGION: | |
1047 obj = (void*)(stack_top.b + pc[1]); | |
1048 size = pc[2]; | |
1049 objti = pc[3]; | |
1050 pc += 4; | |
1051 | |
1052 if(Debug > 2) | |
1053 runtime·printf("gc_region @%p: %D %p\n", stack_t op.b+pc[1], (int64)size, objti); | |
1054 *sbuf.obj.pos++ = (Obj){obj, size, objti}; | |
1055 if(sbuf.obj.pos == sbuf.obj.end) | |
1056 flushobjbuf(&sbuf); | |
1057 continue; | |
1058 | |
1059 case GC_CHAN_PTR: | |
1060 chan = *(Hchan**)(stack_top.b + pc[1]); | |
1061 if(Debug > 2 && chan != nil) | |
1062 runtime·printf("gc_chan_ptr @%p: %p/%D/%D %p\n", stack_top.b+pc[1], chan, (int64)chan->qcount, (int64)chan->dataqsiz, pc[2]); | |
1063 if(chan == nil) { | |
1064 pc += 3; | |
1065 continue; | |
1066 } | |
1067 if(markonly(chan)) { | |
1068 chantype = (ChanType*)pc[2]; | |
1069 if(!(chantype->elem->kind & KindNoPointers)) { | |
1070 // Start chanProg. | |
1071 chan_ret = pc+3; | |
1072 pc = chanProg+1; | |
1073 continue; | |
1074 } | |
1075 } | |
1076 pc += 3; | |
1077 continue; | |
1078 | |
1079 case GC_CHAN: | |
1080 // There are no heap pointers in struct Hchan, | |
1081 // so we can ignore the leading sizeof(Hchan) bytes. | |
1082 if(!(chantype->elem->kind & KindNoPointers)) { | |
1083 // Channel's buffer follows Hchan immediately in memory. | |
1084 // Size of buffer (cap(c)) is second int in the chan struct. | |
1085 chancap = ((uintgo*)chan)[1]; | |
1086 if(chancap > 0) { | |
1087 // TODO(atom): split into two chunks so that only the | |
1088 // in-use part of the circular buffer is scanned. | |
1089 // (Channel routines zero the unused par t, so the current | |
1090 // code does not lead to leaks, it's jus t a little inefficient.) | |
1091 *sbuf.obj.pos++ = (Obj){(byte*)chan+runt ime·Hchansize, chancap*chantype->elem->size, | |
1092 (uintptr)chantype->elem->gc | PR ECISE | LOOP}; | |
1093 if(sbuf.obj.pos == sbuf.obj.end) | |
1094 flushobjbuf(&sbuf); | |
1095 } | |
1096 } | |
1097 if(chan_ret == nil) | |
1098 goto next_block; | |
1099 pc = chan_ret; | |
1100 continue; | |
1101 | |
1102 default: | |
1103 runtime·printf("runtime: invalid GC instruction %p at %p \n", pc[0], pc); | |
1104 runtime·throw("scanblock: invalid GC instruction"); | |
1105 return; | |
1106 } | |
1107 | |
1108 if(obj >= arena_start && obj < arena_used) { | |
1109 *sbuf.ptr.pos++ = (PtrTarget){obj, objti}; | |
1110 if(sbuf.ptr.pos == sbuf.ptr.end) | |
1111 flushptrbuf(&sbuf); | |
1112 } | |
1113 } | |
1114 | |
1115 next_block: | |
1116 // Done scanning [b, b+n). Prepare for the next iteration of | |
1117 // the loop by setting b, n, ti to the parameters for the next b lock. | |
1118 | |
1119 if(sbuf.nobj == 0) { | |
1120 flushptrbuf(&sbuf); | |
1121 flushobjbuf(&sbuf); | |
1122 | |
1123 if(sbuf.nobj == 0) { | |
1124 if(!keepworking) { | |
1125 if(sbuf.wbuf) | |
1126 putempty(sbuf.wbuf); | |
1127 return; | |
1128 } | |
1129 // Emptied our buffer: refill. | |
1130 sbuf.wbuf = getfull(sbuf.wbuf); | |
1131 if(sbuf.wbuf == nil) | |
1132 return; | |
1133 sbuf.nobj = sbuf.wbuf->nobj; | |
1134 sbuf.wp = sbuf.wbuf->obj + sbuf.wbuf->nobj; | |
1135 } | |
1136 } | |
1137 | |
1138 // Fetch b from the work buffer. | |
1139 --sbuf.wp; | |
1140 b = sbuf.wp->p; | |
1141 n = sbuf.wp->n; | |
1142 ti = sbuf.wp->ti; | |
1143 sbuf.nobj--; | |
1144 } | |
1145 } | |
1146 | |
1147 // Append obj to the work buffer. | |
1148 // _wbuf, _wp, _nobj are input/output parameters and are specifying the work buf fer. | |
1149 static void | |
1150 enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj) | |
1151 { | |
1152 uintptr nobj, off; | |
1153 Obj *wp; | |
1154 Workbuf *wbuf; | |
1155 | |
1156 if(Debug > 1) | |
1157 runtime·printf("append obj(%p %D %p)\n", obj.p, (int64)obj.n, ob j.ti); | |
1158 | |
1159 // Align obj.b to a word boundary. | |
1160 off = (uintptr)obj.p & (PtrSize-1); | |
1161 if(off != 0) { | |
1162 obj.p += PtrSize - off; | |
1163 obj.n -= PtrSize - off; | |
1164 obj.ti = 0; | |
1165 } | |
1166 | |
1167 if(obj.p == nil || obj.n == 0) | |
1168 return; | |
1169 | |
1170 // Load work buffer state | |
1171 wp = *_wp; | |
1172 wbuf = *_wbuf; | |
1173 nobj = *_nobj; | |
1174 | |
1175 // If another proc wants a pointer, give it some. | |
1176 if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { | |
1177 wbuf->nobj = nobj; | |
1178 wbuf = handoff(wbuf); | |
1179 nobj = wbuf->nobj; | |
1180 wp = wbuf->obj + nobj; | |
1181 } | |
1182 | |
1183 // If buffer is full, get a new one. | |
1184 if(wbuf == nil || nobj >= nelem(wbuf->obj)) { | |
1185 if(wbuf != nil) | |
1186 wbuf->nobj = nobj; | |
1187 wbuf = getempty(wbuf); | |
1188 wp = wbuf->obj; | |
1189 nobj = 0; | |
1190 } | |
1191 | |
1192 *wp = obj; | |
1193 wp++; | |
1194 nobj++; | |
1195 | |
1196 // Save work buffer state | |
1197 *_wp = wp; | |
1198 *_wbuf = wbuf; | |
1199 *_nobj = nobj; | |
1200 } | |
1201 | |
1202 static void | |
1203 enqueue1(Workbuf **wbufp, Obj obj) | |
1204 { | |
1205 Workbuf *wbuf; | |
1206 | |
1207 wbuf = *wbufp; | |
1208 if(wbuf->nobj >= nelem(wbuf->obj)) | |
1209 *wbufp = wbuf = getempty(wbuf); | |
1210 wbuf->obj[wbuf->nobj++] = obj; | |
1211 } | 492 } |
1212 | 493 |
1213 static void | 494 static void |
1214 markroot(ParFor *desc, uint32 i) | 495 markroot(ParFor *desc, uint32 i) |
1215 { | 496 { |
1216 Workbuf *wbuf; | |
1217 FinBlock *fb; | 497 FinBlock *fb; |
1218 MHeap *h; | 498 MHeap *h; |
1219 MSpan **allspans, *s; | 499 MSpan **allspans, *s; |
1220 uint32 spanidx, sg; | 500 uint32 spanidx, sg; |
1221 G *gp; | 501 G *gp; |
1222 void *p; | 502 void *p; |
1223 | 503 |
1224 USED(&desc); | 504 USED(&desc); |
1225 wbuf = getempty(nil); | |
1226 // Note: if you add a case here, please also update heapdump.c:dumproots . | 505 // Note: if you add a case here, please also update heapdump.c:dumproots . |
1227 switch(i) { | 506 switch(i) { |
1228 case RootData: | 507 case RootData: |
1229 » » enqueue1(&wbuf, (Obj){data, edata - data, (uintptr)gcdata}); | 508 » » scanblock(data, edata - data, work.gcdata); |
509 » » //scanblock(data, edata - data, ScanConservatively); | |
1230 break; | 510 break; |
1231 | 511 |
1232 case RootBss: | 512 case RootBss: |
1233 » » enqueue1(&wbuf, (Obj){bss, ebss - bss, (uintptr)gcbss}); | 513 » » scanblock(bss, ebss - bss, work.gcbss); |
514 » » //scanblock(bss, ebss - bss, ScanConservatively); | |
1234 break; | 515 break; |
1235 | 516 |
1236 case RootFinalizers: | 517 case RootFinalizers: |
1237 for(fb=allfin; fb; fb=fb->alllink) | 518 for(fb=allfin; fb; fb=fb->alllink) |
1238 » » » enqueue1(&wbuf, (Obj){(byte*)fb->fin, fb->cnt*sizeof(fb- >fin[0]), 0}); | 519 » » » scanblock((byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), Sc anConservatively); |
1239 break; | 520 break; |
1240 | 521 |
1241 » case RootSpanTypes: | 522 » case RootSpans: |
1242 » » // mark span types and MSpan.specials (to walk spans only once) | 523 » » // mark MSpan.specials |
1243 h = &runtime·mheap; | 524 h = &runtime·mheap; |
1244 sg = h->sweepgen; | 525 sg = h->sweepgen; |
1245 allspans = h->allspans; | 526 allspans = h->allspans; |
1246 for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { | 527 for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { |
1247 Special *sp; | 528 Special *sp; |
1248 SpecialFinalizer *spf; | 529 SpecialFinalizer *spf; |
1249 | 530 |
1250 s = allspans[spanidx]; | 531 s = allspans[spanidx]; |
1251 if(s->state != MSpanInUse) | 532 if(s->state != MSpanInUse) |
1252 continue; | 533 continue; |
1253 if(s->sweepgen != sg) { | 534 if(s->sweepgen != sg) { |
1254 runtime·printf("sweep %d %d\n", s->sweepgen, sg) ; | 535 runtime·printf("sweep %d %d\n", s->sweepgen, sg) ; |
1255 runtime·throw("gc: unswept span"); | 536 runtime·throw("gc: unswept span"); |
1256 } | 537 } |
1257 // The garbage collector ignores type pointers stored in MSpan.types: | |
1258 // - Compiler-generated types are stored outside of hea p. | |
1259 // - The reflect package has runtime-generated types ca ched in its data structures. | |
1260 // The garbage collector relies on finding the refere nces via that cache. | |
1261 if(s->types.compression == MTypes_Words || s->types.comp ression == MTypes_Bytes) | |
1262 markonly((byte*)s->types.data); | |
1263 for(sp = s->specials; sp != nil; sp = sp->next) { | 538 for(sp = s->specials; sp != nil; sp = sp->next) { |
1264 if(sp->kind != KindSpecialFinalizer) | 539 if(sp->kind != KindSpecialFinalizer) |
1265 continue; | 540 continue; |
1266 // don't mark finalized object, but scan it so w e | 541 // don't mark finalized object, but scan it so w e |
1267 // retain everything it points to. | 542 // retain everything it points to. |
1268 spf = (SpecialFinalizer*)sp; | 543 spf = (SpecialFinalizer*)sp; |
1269 // A finalizer can be set for an inner byte of a n object, find object beginning. | 544 // A finalizer can be set for an inner byte of a n object, find object beginning. |
1270 p = (void*)((s->start << PageShift) + spf->offse t/s->elemsize*s->elemsize); | 545 p = (void*)((s->start << PageShift) + spf->offse t/s->elemsize*s->elemsize); |
1271 » » » » enqueue1(&wbuf, (Obj){p, s->elemsize, 0}); | 546 » » » » scanblock(p, s->elemsize, nil); |
1272 » » » » enqueue1(&wbuf, (Obj){(void*)&spf->fn, PtrSize, 0}); | 547 » » » » scanblock((void*)&spf->fn, PtrSize, ScanConserva tively); |
1273 » » » » enqueue1(&wbuf, (Obj){(void*)&spf->fint, PtrSize , 0}); | |
1274 » » » » enqueue1(&wbuf, (Obj){(void*)&spf->ot, PtrSize, 0}); | |
1275 } | 548 } |
1276 } | 549 } |
1277 break; | 550 break; |
1278 | 551 |
1279 case RootFlushCaches: | 552 case RootFlushCaches: |
1280 flushallmcaches(); | 553 flushallmcaches(); |
1281 break; | 554 break; |
1282 | 555 |
1283 default: | 556 default: |
1284 // the rest is scanning goroutine stacks | 557 // the rest is scanning goroutine stacks |
1285 if(i - RootCount >= runtime·allglen) | 558 if(i - RootCount >= runtime·allglen) |
1286 runtime·throw("markroot: bad index"); | 559 runtime·throw("markroot: bad index"); |
1287 gp = runtime·allg[i - RootCount]; | 560 gp = runtime·allg[i - RootCount]; |
1288 // remember when we've first observed the G blocked | 561 // remember when we've first observed the G blocked |
1289 // needed only to output in traceback | 562 // needed only to output in traceback |
1290 if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->wai tsince == 0) | 563 if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->wai tsince == 0) |
1291 gp->waitsince = work.tstart; | 564 gp->waitsince = work.tstart; |
1292 » » addstackroots(gp, &wbuf); | 565 » » scanstack(gp); |
1293 break; | 566 break; |
1294 ················ | 567 ················ |
1295 } | 568 } |
1296 | |
1297 if(wbuf) | |
1298 scanblock(wbuf, false); | |
1299 } | 569 } |
1300 | 570 |
1301 // Get an empty work buffer off the work.empty list, | 571 // Get an empty work buffer off the work.empty list, |
1302 // allocating new buffers as needed. | 572 // allocating new buffers as needed. |
1303 static Workbuf* | 573 static Workbuf* |
1304 getempty(Workbuf *b) | 574 getempty(Workbuf *b) |
1305 { | 575 { |
1306 if(b != nil) | 576 if(b != nil) |
1307 runtime·lfstackpush(&work.full, &b->node); | 577 runtime·lfstackpush(&work.full, &b->node); |
1308 b = (Workbuf*)runtime·lfstackpop(&work.empty); | 578 b = (Workbuf*)runtime·lfstackpop(&work.empty); |
1309 if(b == nil) | 579 if(b == nil) |
1310 b = runtime·persistentalloc(sizeof(*b), CacheLineSize, &mstats.g c_sys); | 580 b = runtime·persistentalloc(sizeof(*b), CacheLineSize, &mstats.g c_sys); |
1311 b->nobj = 0; | 581 b->nobj = 0; |
1312 return b; | 582 return b; |
1313 } | 583 } |
1314 | 584 |
1315 static void | 585 static void |
1316 putempty(Workbuf *b) | 586 putempty(Workbuf *b) |
1317 { | 587 { |
1318 if(CollectStats) | |
1319 runtime·xadd64(&gcstats.putempty, 1); | |
1320 | |
1321 runtime·lfstackpush(&work.empty, &b->node); | 588 runtime·lfstackpush(&work.empty, &b->node); |
1322 } | 589 } |
1323 | 590 |
1324 // Get a full work buffer off the work.full list, or return nil. | 591 // Get a full work buffer off the work.full list, or return nil. |
1325 static Workbuf* | 592 static Workbuf* |
1326 getfull(Workbuf *b) | 593 getfull(Workbuf *b) |
1327 { | 594 { |
1328 int32 i; | 595 int32 i; |
1329 | 596 |
1330 if(CollectStats) | |
1331 runtime·xadd64(&gcstats.getfull, 1); | |
1332 | |
1333 if(b != nil) | 597 if(b != nil) |
1334 runtime·lfstackpush(&work.empty, &b->node); | 598 runtime·lfstackpush(&work.empty, &b->node); |
1335 b = (Workbuf*)runtime·lfstackpop(&work.full); | 599 b = (Workbuf*)runtime·lfstackpop(&work.full); |
1336 if(b != nil || work.nproc == 1) | 600 if(b != nil || work.nproc == 1) |
1337 return b; | 601 return b; |
1338 | 602 |
1339 runtime·xadd(&work.nwait, +1); | 603 runtime·xadd(&work.nwait, +1); |
1340 for(i=0;; i++) { | 604 for(i=0;; i++) { |
1341 if(work.full != 0) { | 605 if(work.full != 0) { |
1342 runtime·xadd(&work.nwait, -1); | 606 runtime·xadd(&work.nwait, -1); |
(...skipping 30 matching lines...) Expand all Loading... | |
1373 b1->nobj = n; | 637 b1->nobj = n; |
1374 runtime·memmove(b1->obj, b->obj+b->nobj, n*sizeof b1->obj[0]); | 638 runtime·memmove(b1->obj, b->obj+b->nobj, n*sizeof b1->obj[0]); |
1375 g->m->gcstats.nhandoff++; | 639 g->m->gcstats.nhandoff++; |
1376 g->m->gcstats.nhandoffcnt += n; | 640 g->m->gcstats.nhandoffcnt += n; |
1377 | 641 |
1378 // Put b on full list - let first half of b get stolen. | 642 // Put b on full list - let first half of b get stolen. |
1379 runtime·lfstackpush(&work.full, &b->node); | 643 runtime·lfstackpush(&work.full, &b->node); |
1380 return b1; | 644 return b1; |
1381 } | 645 } |
1382 | 646 |
1383 extern byte pclntab[]; // base for f->ptrsoff | |
1384 | |
1385 BitVector | 647 BitVector |
1386 runtime·stackmapdata(StackMap *stackmap, int32 n) | 648 runtime·stackmapdata(StackMap *stackmap, int32 n) |
1387 { | 649 { |
1388 if(n < 0 || n >= stackmap->n) | 650 if(n < 0 || n >= stackmap->n) |
1389 runtime·throw("stackmapdata: index out of range"); | 651 runtime·throw("stackmapdata: index out of range"); |
1390 return (BitVector){stackmap->nbit, stackmap->data + n*((stackmap->nbit+3 1)/32)}; | 652 return (BitVector){stackmap->nbit, stackmap->data + n*((stackmap->nbit+3 1)/32)}; |
1391 } | 653 } |
1392 | 654 |
1393 // Scans an interface data value when the interface type indicates | |
1394 // that it is a pointer. | |
1395 static void | |
1396 scaninterfacedata(uintptr bits, byte *scanp, void *wbufp) | |
1397 { | |
1398 Itab *tab; | |
1399 Type *type; | |
1400 | |
1401 if(runtime·precisestack) { | |
1402 if(bits == BitsIface) { | |
1403 tab = *(Itab**)scanp; | |
1404 if(tab->type->size <= sizeof(void*) && (tab->type->kind & KindNoPointers)) | |
1405 return; | |
1406 } else { // bits == BitsEface | |
1407 type = *(Type**)scanp; | |
1408 if(type->size <= sizeof(void*) && (type->kind & KindNoPo inters)) | |
1409 return; | |
1410 } | |
1411 } | |
1412 enqueue1(wbufp, (Obj){scanp+PtrSize, PtrSize, 0}); | |
1413 } | |
1414 | |
1415 // Starting from scanp, scans words corresponding to set bits. | |
1416 static void | |
1417 scanbitvector(Func *f, bool precise, byte *scanp, BitVector *bv, void *wbufp) | |
1418 { | |
1419 uintptr word, bits; | |
1420 uint32 *wordp; | |
1421 int32 i, remptrs; | |
1422 byte *p; | |
1423 | |
1424 wordp = bv->data; | |
1425 for(remptrs = bv->n; remptrs > 0; remptrs -= 32) { | |
1426 word = *wordp++; | |
1427 if(remptrs < 32) | |
1428 i = remptrs; | |
1429 else | |
1430 i = 32; | |
1431 i /= BitsPerPointer; | |
1432 for(; i > 0; i--) { | |
1433 bits = word & 3; | |
1434 switch(bits) { | |
1435 case BitsDead: | |
1436 if(runtime·debug.gcdead) | |
1437 *(uintptr*)scanp = PoisonGC; | |
1438 break; | |
1439 case BitsScalar: | |
1440 break; | |
1441 case BitsPointer: | |
1442 p = *(byte**)scanp; | |
1443 if(p != nil) { | |
1444 if(Debug > 2) | |
1445 runtime·printf("frame %s @%p: pt r %p\n", runtime·funcname(f), scanp, p); | |
1446 if(precise && (p < (byte*)PageSize || (u intptr)p == PoisonGC || (uintptr)p == PoisonStack)) { | |
1447 // Looks like a junk value in a pointer slot. | |
1448 // Liveness analysis wrong? | |
1449 g->m->traceback = 2; | |
1450 runtime·printf("bad pointer in f rame %s at %p: %p\n", runtime·funcname(f), scanp, p); | |
1451 runtime·throw("bad pointer in sc anbitvector"); | |
1452 } | |
1453 enqueue1(wbufp, (Obj){scanp, PtrSize, 0} ); | |
1454 } | |
1455 break; | |
1456 case BitsMultiWord: | |
1457 p = scanp; | |
1458 word >>= BitsPerPointer; | |
1459 scanp += PtrSize; | |
1460 i--; | |
1461 if(i == 0) { | |
1462 // Get next chunk of bits | |
1463 remptrs -= 32; | |
1464 word = *wordp++; | |
1465 if(remptrs < 32) | |
1466 i = remptrs; | |
1467 else | |
1468 i = 32; | |
1469 i /= BitsPerPointer; | |
1470 } | |
1471 switch(word & 3) { | |
1472 case BitsString: | |
1473 if(Debug > 2) | |
1474 runtime·printf("frame %s @%p: st ring %p/%D\n", runtime·funcname(f), p, ((String*)p)->str, (int64)((String*)p)->l en); | |
1475 if(((String*)p)->len != 0) | |
1476 markonly(((String*)p)->str); | |
1477 break; | |
1478 case BitsSlice: | |
1479 word >>= BitsPerPointer; | |
1480 scanp += PtrSize; | |
1481 i--; | |
1482 if(i == 0) { | |
1483 // Get next chunk of bits | |
1484 remptrs -= 32; | |
1485 word = *wordp++; | |
1486 if(remptrs < 32) | |
1487 i = remptrs; | |
1488 else | |
1489 i = 32; | |
1490 i /= BitsPerPointer; | |
1491 } | |
1492 if(Debug > 2) | |
1493 runtime·printf("frame %s @%p: sl ice %p/%D/%D\n", runtime·funcname(f), p, ((Slice*)p)->array, (int64)((Slice*)p)- >len, (int64)((Slice*)p)->cap); | |
1494 if(((Slice*)p)->cap < ((Slice*)p)->len) { | |
1495 g->m->traceback = 2; | |
1496 runtime·printf("bad slice in fra me %s at %p: %p/%p/%p\n", runtime·funcname(f), p, ((byte**)p)[0], ((byte**)p)[1] , ((byte**)p)[2]); | |
1497 runtime·throw("slice capacity sm aller than length"); | |
1498 } | |
1499 if(((Slice*)p)->cap != 0) | |
1500 enqueue1(wbufp, (Obj){p, PtrSize , 0}); | |
1501 break; | |
1502 case BitsIface: | |
1503 case BitsEface: | |
1504 if(*(byte**)p != nil) { | |
1505 if(Debug > 2) { | |
1506 if((word&3) == BitsEface ) | |
1507 runtime·printf(" frame %s @%p: eface %p %p\n", runtime·funcname(f), p, ((uintptr*)p)[0], ((uintpt r*)p)[1]); | |
1508 else | |
1509 runtime·printf(" frame %s @%p: iface %p %p\n", runtime·funcname(f), p, ((uintptr*)p)[0], ((uintpt r*)p)[1]); | |
1510 } | |
1511 scaninterfacedata(word & 3, p, w bufp); | |
1512 } | |
1513 break; | |
1514 } | |
1515 } | |
1516 word >>= BitsPerPointer; | |
1517 scanp += PtrSize; | |
1518 } | |
1519 } | |
1520 } | |
1521 | |
1522 // Scan a stack frame: local variables and function arguments/results. | 655 // Scan a stack frame: local variables and function arguments/results. |
1523 static bool | 656 static bool |
1524 scanframe(Stkframe *frame, void *wbufp) | 657 scanframe(Stkframe *frame, void *unused) |
1525 { | 658 { |
1526 Func *f; | 659 Func *f; |
1527 StackMap *stackmap; | 660 StackMap *stackmap; |
1528 BitVector bv; | 661 BitVector bv; |
1529 uintptr size; | 662 uintptr size; |
1530 uintptr targetpc; | 663 uintptr targetpc; |
1531 int32 pcdata; | 664 int32 pcdata; |
1532 bool precise; | |
1533 | 665 |
666 USED(unused); | |
1534 f = frame->fn; | 667 f = frame->fn; |
1535 targetpc = frame->continpc; | 668 targetpc = frame->continpc; |
1536 if(targetpc == 0) { | 669 if(targetpc == 0) { |
1537 // Frame is dead. | 670 // Frame is dead. |
1538 return true; | 671 return true; |
1539 } | 672 } |
1540 if(targetpc != f->entry) | 673 if(targetpc != f->entry) |
1541 targetpc--; | 674 targetpc--; |
1542 pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc); | 675 pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc); |
1543 if(pcdata == -1) { | 676 if(pcdata == -1) { |
1544 // We do not have a valid pcdata value but there might be a | 677 // We do not have a valid pcdata value but there might be a |
1545 // stackmap for this function. It is likely that we are looking | 678 // stackmap for this function. It is likely that we are looking |
1546 // at the function prologue, assume so and hope for the best. | 679 // at the function prologue, assume so and hope for the best. |
1547 pcdata = 0; | 680 pcdata = 0; |
1548 } | 681 } |
1549 | 682 |
1550 // Scan local variables if stack frame has been allocated. | 683 // Scan local variables if stack frame has been allocated. |
1551 // Use pointer information if known. | 684 // Use pointer information if known. |
1552 precise = false; | |
1553 stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); | 685 stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); |
1554 if(stackmap == nil) { | 686 if(stackmap == nil) { |
1555 // No locals information, scan everything. | 687 // No locals information, scan everything. |
1556 size = frame->varp - (byte*)frame->sp; | 688 size = frame->varp - (byte*)frame->sp; |
1557 if(Debug > 2) | 689 if(Debug > 2) |
1558 runtime·printf("frame %s unsized locals %p+%p\n", runtim e·funcname(f), frame->varp-size, size); | 690 runtime·printf("frame %s unsized locals %p+%p\n", runtim e·funcname(f), frame->varp-size, size); |
1559 » » enqueue1(wbufp, (Obj){frame->varp - size, size, 0}); | 691 » » scanblock(frame->varp - size, size, ScanConservatively); |
1560 } else if(stackmap->n < 0) { | 692 } else if(stackmap->n < 0) { |
1561 // Locals size information, scan just the locals. | 693 // Locals size information, scan just the locals. |
1562 size = -stackmap->n; | 694 size = -stackmap->n; |
1563 if(Debug > 2) | 695 if(Debug > 2) |
1564 runtime·printf("frame %s conservative locals %p+%p\n", r untime·funcname(f), frame->varp-size, size); | 696 runtime·printf("frame %s conservative locals %p+%p\n", r untime·funcname(f), frame->varp-size, size); |
1565 » » enqueue1(wbufp, (Obj){frame->varp - size, size, 0}); | 697 » » scanblock(frame->varp - size, size, ScanConservatively); |
1566 } else if(stackmap->n > 0) { | 698 } else if(stackmap->n > 0) { |
1567 » » // Locals bitmap information, scan just the pointers in | 699 » » // Locals bitmap information, scan just the pointers in locals. |
1568 » » // locals. | |
1569 if(pcdata < 0 || pcdata >= stackmap->n) { | 700 if(pcdata < 0 || pcdata >= stackmap->n) { |
1570 // don't know where we are | 701 // don't know where we are |
1571 runtime·printf("pcdata is %d and %d stack map entries fo r %s (targetpc=%p)\n", | 702 runtime·printf("pcdata is %d and %d stack map entries fo r %s (targetpc=%p)\n", |
1572 pcdata, stackmap->n, runtime·funcname(f), target pc); | 703 pcdata, stackmap->n, runtime·funcname(f), target pc); |
1573 runtime·throw("scanframe: bad symbol table"); | 704 runtime·throw("scanframe: bad symbol table"); |
1574 } | 705 } |
1575 bv = runtime·stackmapdata(stackmap, pcdata); | 706 bv = runtime·stackmapdata(stackmap, pcdata); |
1576 size = (bv.n * PtrSize) / BitsPerPointer; | 707 size = (bv.n * PtrSize) / BitsPerPointer; |
1577 » » precise = true; | 708 » » scanblock(frame->varp - size, bv.n/BitsPerPointer*PtrSize, (byte *)bv.data); |
1578 » » scanbitvector(f, true, frame->varp - size, &bv, wbufp); | |
1579 } | 709 } |
1580 | 710 |
1581 // Scan arguments. | 711 // Scan arguments. |
1582 // Use pointer information if known. | 712 // Use pointer information if known. |
1583 stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); | 713 stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); |
1584 if(stackmap != nil) { | 714 if(stackmap != nil) { |
1585 bv = runtime·stackmapdata(stackmap, pcdata); | 715 bv = runtime·stackmapdata(stackmap, pcdata); |
1586 » » scanbitvector(f, precise, frame->argp, &bv, wbufp); | 716 » » scanblock(frame->argp, bv.n/BitsPerPointer*PtrSize, (byte*)bv.da ta); |
1587 } else { | 717 } else { |
1588 if(Debug > 2) | 718 if(Debug > 2) |
1589 runtime·printf("frame %s conservative args %p+%p\n", run time·funcname(f), frame->argp, (uintptr)frame->arglen); | 719 runtime·printf("frame %s conservative args %p+%p\n", run time·funcname(f), frame->argp, (uintptr)frame->arglen); |
1590 » » enqueue1(wbufp, (Obj){frame->argp, frame->arglen, 0}); | 720 » » scanblock(frame->argp, frame->arglen, ScanConservatively); |
1591 } | 721 } |
1592 return true; | 722 return true; |
1593 } | 723 } |
1594 | 724 |
1595 static void | 725 static void |
1596 addstackroots(G *gp, Workbuf **wbufp) | 726 scanstack(G *gp) |
1597 { | 727 { |
1598 M *mp; | 728 M *mp; |
1599 int32 n; | 729 int32 n; |
1600 Stktop *stk; | 730 Stktop *stk; |
1601 uintptr sp, guard; | 731 uintptr sp, guard; |
1602 | 732 |
1603 switch(gp->status){ | 733 switch(gp->status){ |
1604 default: | 734 default: |
1605 runtime·printf("unexpected G.status %d (goroutine %p %D)\n", gp- >status, gp, gp->goid); | 735 runtime·printf("unexpected G.status %d (goroutine %p %D)\n", gp- >status, gp, gp->goid); |
1606 runtime·throw("mark - bad status"); | 736 runtime·throw("mark - bad status"); |
(...skipping 25 matching lines...) Expand all Loading... | |
1632 // Scanning another goroutine's stack. | 762 // Scanning another goroutine's stack. |
1633 // The goroutine is usually asleep (the world is stopped). | 763 // The goroutine is usually asleep (the world is stopped). |
1634 sp = gp->sched.sp; | 764 sp = gp->sched.sp; |
1635 stk = (Stktop*)gp->stackbase; | 765 stk = (Stktop*)gp->stackbase; |
1636 guard = gp->stackguard; | 766 guard = gp->stackguard; |
1637 } | 767 } |
1638 if(ScanStackByFrames) { | 768 if(ScanStackByFrames) { |
1639 USED(sp); | 769 USED(sp); |
1640 USED(stk); | 770 USED(stk); |
1641 USED(guard); | 771 USED(guard); |
1642 » » runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x 7fffffff, scanframe, wbufp, false); | 772 » » runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x 7fffffff, scanframe, nil, false); |
1643 } else { | 773 } else { |
1644 n = 0; | 774 n = 0; |
1645 while(stk) { | 775 while(stk) { |
1646 if(sp < guard-StackGuard || (uintptr)stk < sp) { | 776 if(sp < guard-StackGuard || (uintptr)stk < sp) { |
1647 runtime·printf("scanstack inconsistent: g%D#%d s p=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); | 777 runtime·printf("scanstack inconsistent: g%D#%d s p=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); |
1648 runtime·throw("scanstack"); | 778 runtime·throw("scanstack"); |
1649 } | 779 } |
1650 if(Debug > 2) | 780 if(Debug > 2) |
1651 runtime·printf("conservative stack %p+%p\n", (by te*)sp, (uintptr)stk-sp); | 781 runtime·printf("conservative stack %p+%p\n", (by te*)sp, (uintptr)stk-sp); |
1652 » » » enqueue1(wbufp, (Obj){(byte*)sp, (uintptr)stk - sp, (uin tptr)defaultProg | PRECISE | LOOP}); | 782 » » » scanblock((byte*)sp, (uintptr)stk - sp, ScanConservative ly); |
1653 sp = stk->gobuf.sp; | 783 sp = stk->gobuf.sp; |
1654 guard = stk->stackguard; | 784 guard = stk->stackguard; |
1655 stk = (Stktop*)stk->stackbase; | 785 stk = (Stktop*)stk->stackbase; |
1656 n++; | 786 n++; |
1657 } | 787 } |
1658 } | 788 } |
1659 } | 789 } |
1660 | 790 |
1661 void | 791 void |
1662 runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType * ot) | 792 runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType * ot) |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1726 runtime·osyield();·· | 856 runtime·osyield();·· |
1727 } | 857 } |
1728 | 858 |
1729 // Sweep frees or collects finalizers for blocks not marked in the mark phase. | 859 // Sweep frees or collects finalizers for blocks not marked in the mark phase. |
1730 // It clears the mark bits in preparation for the next GC round. | 860 // It clears the mark bits in preparation for the next GC round. |
1731 // Returns true if the span was returned to heap. | 861 // Returns true if the span was returned to heap. |
1732 bool | 862 bool |
1733 runtime·MSpan_Sweep(MSpan *s) | 863 runtime·MSpan_Sweep(MSpan *s) |
1734 { | 864 { |
1735 int32 cl, n, npages, nfree; | 865 int32 cl, n, npages, nfree; |
1736 » uintptr size, off, *bitp, shift, bits; | 866 » uintptr size, off, *bitp, shift, xbits, bits; |
1737 uint32 sweepgen; | 867 uint32 sweepgen; |
1738 byte *p; | 868 byte *p; |
1739 MCache *c; | 869 MCache *c; |
1740 byte *arena_start; | 870 byte *arena_start; |
1741 MLink head, *end; | 871 MLink head, *end; |
1742 byte *type_data; | |
1743 byte compression; | |
1744 uintptr type_data_inc; | |
1745 MLink *x; | |
1746 Special *special, **specialp, *y; | 872 Special *special, **specialp, *y; |
1747 bool res, sweepgenset; | 873 bool res, sweepgenset; |
1748 | 874 |
1749 // It's critical that we enter this function with preemption disabled, | 875 // It's critical that we enter this function with preemption disabled, |
1750 // GC must not start while we are in the middle of this function. | 876 // GC must not start while we are in the middle of this function. |
1751 if(g->m->locks == 0 && g->m->mallocing == 0 && g != g->m->g0) | 877 if(g->m->locks == 0 && g->m->mallocing == 0 && g != g->m->g0) |
1752 runtime·throw("MSpan_Sweep: m is not locked"); | 878 runtime·throw("MSpan_Sweep: m is not locked"); |
1753 sweepgen = runtime·mheap.sweepgen; | 879 sweepgen = runtime·mheap.sweepgen; |
1754 if(s->state != MSpanInUse || s->sweepgen != sweepgen-1) { | 880 if(s->state != MSpanInUse || s->sweepgen != sweepgen-1) { |
1755 runtime·printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen =%d\n", | 881 runtime·printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen =%d\n", |
1756 s->state, s->sweepgen, sweepgen); | 882 s->state, s->sweepgen, sweepgen); |
1757 runtime·throw("MSpan_Sweep: bad span state"); | 883 runtime·throw("MSpan_Sweep: bad span state"); |
1758 } | 884 } |
1759 arena_start = runtime·mheap.arena_start; | 885 arena_start = runtime·mheap.arena_start; |
1760 cl = s->sizeclass; | 886 cl = s->sizeclass; |
1761 size = s->elemsize; | 887 size = s->elemsize; |
1762 » if(cl == 0) { | 888 » n = 1; |
rsc
2014/07/23 16:23:20
seems like a gratuitous change
dvyukov
2014/07/23 19:01:45
reverted
| |
1763 » » n = 1; | 889 » if(cl != 0) { |
1764 » } else { | |
1765 // Chunk full of small blocks. | 890 // Chunk full of small blocks. |
1766 npages = runtime·class_to_allocnpages[cl]; | 891 npages = runtime·class_to_allocnpages[cl]; |
1767 n = (npages << PageShift) / size; | 892 n = (npages << PageShift) / size; |
1768 } | 893 } |
1769 res = false; | 894 res = false; |
1770 nfree = 0; | 895 nfree = 0; |
1771 end = &head; | 896 end = &head; |
1772 c = g->m->mcache; | 897 c = g->m->mcache; |
1773 sweepgenset = false; | 898 sweepgenset = false; |
1774 | 899 |
1775 // mark any free objects in this span so we don't collect them | |
1776 for(x = s->freelist; x != nil; x = x->next) { | |
1777 // This is markonly(x) but faster because we don't need | |
1778 // atomic access and we're guaranteed to be pointing at | |
1779 // the head of a valid object. | |
1780 off = (uintptr*)x - (uintptr*)runtime·mheap.arena_start; | |
1781 bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapW ord - 1; | |
1782 shift = off % wordsPerBitmapWord; | |
1783 *bitp |= bitMarked<<shift; | |
1784 } | |
1785 | |
1786 // Unlink & free special records for any objects we're about to free. | 900 // Unlink & free special records for any objects we're about to free. |
1787 specialp = &s->specials; | 901 specialp = &s->specials; |
1788 special = *specialp; | 902 special = *specialp; |
1789 while(special != nil) { | 903 while(special != nil) { |
1790 // A finalizer can be set for an inner byte of an object, find o bject beginning. | 904 // A finalizer can be set for an inner byte of an object, find o bject beginning. |
1791 p = (byte*)(s->start << PageShift) + special->offset/size*size; | 905 p = (byte*)(s->start << PageShift) + special->offset/size*size; |
1792 off = (uintptr*)p - (uintptr*)arena_start; | 906 off = (uintptr*)p - (uintptr*)arena_start; |
1793 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | 907 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
1794 » » shift = off % wordsPerBitmapWord; | 908 » » shift = (off % wordsPerBitmapWord) * gcBits; |
1795 » » bits = *bitp>>shift; | 909 » » bits = (*bitp>>shift) & bitMask; |
1796 » » if((bits & (bitAllocated|bitMarked)) == bitAllocated) { | 910 » » if(bits == bitAllocated) { |
1797 // Find the exact byte for which the special was setup | 911 // Find the exact byte for which the special was setup |
1798 // (as opposed to object beginning). | 912 // (as opposed to object beginning). |
1799 p = (byte*)(s->start << PageShift) + special->offset; | 913 p = (byte*)(s->start << PageShift) + special->offset; |
1800 // about to free object: splice out special record | 914 // about to free object: splice out special record |
1801 y = special; | 915 y = special; |
1802 special = special->next; | 916 special = special->next; |
1803 *specialp = special; | 917 *specialp = special; |
1804 if(!runtime·freespecial(y, p, size, false)) { | 918 if(!runtime·freespecial(y, p, size, false)) { |
1805 // stop freeing of object if it has a finalizer | 919 // stop freeing of object if it has a finalizer |
1806 *bitp |= bitMarked << shift; | 920 *bitp |= bitMarked << shift; |
1807 } | 921 } |
1808 } else { | 922 } else { |
1809 // object is still live: keep special record | 923 // object is still live: keep special record |
924 if(bits != bitMarked) { | |
925 runtime·printf("runtime: bad bits for special ob ject %p: %d\n", p, (int32)bits); | |
926 runtime·throw("runtime: bad bits for special obj ect"); | |
927 } | |
1810 specialp = &special->next; | 928 specialp = &special->next; |
1811 special = *specialp; | 929 special = *specialp; |
1812 } | 930 } |
1813 } | 931 } |
1814 | 932 |
1815 type_data = (byte*)s->types.data; | |
1816 type_data_inc = sizeof(uintptr); | |
1817 compression = s->types.compression; | |
1818 switch(compression) { | |
1819 case MTypes_Bytes: | |
1820 type_data += 8*sizeof(uintptr); | |
1821 type_data_inc = 1; | |
1822 break; | |
1823 } | |
1824 | |
1825 // Sweep through n objects of given size starting at p. | 933 // Sweep through n objects of given size starting at p. |
1826 // This thread owns the span now, so it can manipulate | 934 // This thread owns the span now, so it can manipulate |
1827 // the block bitmap without atomic operations. | 935 // the block bitmap without atomic operations. |
1828 p = (byte*)(s->start << PageShift); | 936 p = (byte*)(s->start << PageShift); |
1829 » for(; n > 0; n--, p += size, type_data+=type_data_inc) { | 937 » for(; n > 0; n--, p += size) { |
1830 off = (uintptr*)p - (uintptr*)arena_start; | 938 off = (uintptr*)p - (uintptr*)arena_start; |
1831 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | 939 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
1832 » » shift = off % wordsPerBitmapWord; | 940 » » shift = (off % wordsPerBitmapWord) * gcBits; |
1833 » » bits = *bitp>>shift; | 941 » » xbits = *bitp; |
942 » » bits = (xbits>>shift) & bitMask; | |
1834 | 943 |
1835 » » if((bits & bitAllocated) == 0) | 944 » » // Non-allocated or FlagNoGC object, ignore. |
945 » » if(bits == bitBoundary) | |
1836 continue; | 946 continue; |
1837 | 947 » » // Allocated and marked object, reset bits to allocated. |
1838 » » if((bits & bitMarked) != 0) { | 948 » » if(bits == bitMarked) { |
1839 » » » *bitp &= ~(bitMarked<<shift); | 949 » » » *bitp = (xbits & ~(bitMarked<<shift)) | (bitAllocated<<s hift); |
1840 continue; | 950 continue; |
1841 } | 951 } |
1842 | 952 » » // Garbage! |
1843 if(runtime·debug.allocfreetrace) | 953 if(runtime·debug.allocfreetrace) |
1844 runtime·tracefree(p, size); | 954 runtime·tracefree(p, size); |
1845 | 955 » » // Reset to boundary. |
1846 » » // Clear mark and scan bits. | 956 » » *bitp = (xbits & ~(bitAllocated<<shift)) | (bitBoundary<<shift); |
1847 » » *bitp &= ~((bitScan|bitMarked)<<shift); | |
1848 | |
1849 if(cl == 0) { | 957 if(cl == 0) { |
1850 // Free large span. | 958 // Free large span. |
1851 » » » runtime·unmarkspan(p, 1<<PageShift); | 959 » » » runtime·unmarkspan(p, s->npages<<PageShift); |
1852 s->needzero = 1; | 960 s->needzero = 1; |
1853 // important to set sweepgen before returning it to heap | 961 // important to set sweepgen before returning it to heap |
1854 runtime·atomicstore(&s->sweepgen, sweepgen); | 962 runtime·atomicstore(&s->sweepgen, sweepgen); |
1855 sweepgenset = true; | 963 sweepgenset = true; |
1856 // See note about SysFault vs SysFree in malloc.goc. | 964 // See note about SysFault vs SysFree in malloc.goc. |
1857 » » » if(runtime·debug.efence) | 965 » » » if(runtime·debug.efence) { |
966 » » » » s->limit = nil;»// prevent mlookup from finding this span | |
1858 runtime·SysFault(p, size); | 967 runtime·SysFault(p, size); |
1859 » » » else | 968 » » » } else |
1860 runtime·MHeap_Free(&runtime·mheap, s, 1); | 969 runtime·MHeap_Free(&runtime·mheap, s, 1); |
1861 c->local_nlargefree++; | 970 c->local_nlargefree++; |
1862 c->local_largefree += size; | 971 c->local_largefree += size; |
1863 runtime·xadd64(&mstats.next_gc, -(uint64)(size * (gcperc ent + 100)/100)); | 972 runtime·xadd64(&mstats.next_gc, -(uint64)(size * (gcperc ent + 100)/100)); |
1864 res = true; | 973 res = true; |
1865 } else { | 974 } else { |
1866 // Free small object. | 975 // Free small object. |
1867 switch(compression) { | |
1868 case MTypes_Words: | |
1869 *(uintptr*)type_data = 0; | |
1870 break; | |
1871 case MTypes_Bytes: | |
1872 *(byte*)type_data = 0; | |
1873 break; | |
1874 } | |
1875 if(size > 2*sizeof(uintptr)) | 976 if(size > 2*sizeof(uintptr)) |
1876 ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll ; // mark as "needs to be zeroed" | 977 ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll ; // mark as "needs to be zeroed" |
1877 else if(size > sizeof(uintptr)) | 978 else if(size > sizeof(uintptr)) |
1878 ((uintptr*)p)[1] = 0; | 979 ((uintptr*)p)[1] = 0; |
1879 | 980 |
1880 end->next = (MLink*)p; | 981 end->next = (MLink*)p; |
1881 end = (MLink*)p; | 982 end = (MLink*)p; |
1882 nfree++; | 983 nfree++; |
1883 } | 984 } |
1884 } | 985 } |
(...skipping 12 matching lines...) Expand all Loading... | |
1897 s->state, s->sweepgen, sweepgen); | 998 s->state, s->sweepgen, sweepgen); |
1898 runtime·throw("MSpan_Sweep: bad span state after sweep") ; | 999 runtime·throw("MSpan_Sweep: bad span state after sweep") ; |
1899 } | 1000 } |
1900 runtime·atomicstore(&s->sweepgen, sweepgen); | 1001 runtime·atomicstore(&s->sweepgen, sweepgen); |
1901 } | 1002 } |
1902 if(nfree > 0) { | 1003 if(nfree > 0) { |
1903 c->local_nsmallfree[cl] += nfree; | 1004 c->local_nsmallfree[cl] += nfree; |
1904 c->local_cachealloc -= nfree * size; | 1005 c->local_cachealloc -= nfree * size; |
1905 runtime·xadd64(&mstats.next_gc, -(uint64)(nfree * size * (gcperc ent + 100)/100)); | 1006 runtime·xadd64(&mstats.next_gc, -(uint64)(nfree * size * (gcperc ent + 100)/100)); |
1906 res = runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, n free, head.next, end); | 1007 res = runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, n free, head.next, end); |
1907 » » //MCentral_FreeSpan updates sweepgen | 1008 » » // MCentral_FreeSpan updates sweepgen |
1908 } | 1009 } |
1909 return res; | 1010 return res; |
1910 } | 1011 } |
1911 | 1012 |
1912 // State of background sweep. | 1013 // State of background sweep. |
1913 // Pretected by gclock. | 1014 // Pretected by gclock. |
1914 static struct | 1015 static struct |
1915 { | 1016 { |
1916 G* g; | 1017 G* g; |
1917 bool parked; | 1018 bool parked; |
1918 | 1019 |
1919 MSpan** spans; | 1020 MSpan** spans; |
1920 uint32 nspan; | 1021 uint32 nspan; |
1921 uint32 spanidx; | 1022 uint32 spanidx; |
1023 | |
1024 uint32 nbgsweep; | |
1025 uint32 npausesweep; | |
1922 } sweep; | 1026 } sweep; |
1923 | 1027 |
1924 // background sweeping goroutine | 1028 // background sweeping goroutine |
1925 static void | 1029 static void |
1926 bgsweep(void) | 1030 bgsweep(void) |
1927 { | 1031 { |
1928 g->issystem = 1; | 1032 g->issystem = 1; |
1929 for(;;) { | 1033 for(;;) { |
1930 while(runtime·sweepone() != -1) { | 1034 while(runtime·sweepone() != -1) { |
1931 » » » gcstats.nbgsweep++; | 1035 » » » sweep.nbgsweep++; |
1932 runtime·gosched(); | 1036 runtime·gosched(); |
1933 } | 1037 } |
1934 runtime·lock(&gclock); | 1038 runtime·lock(&gclock); |
1935 if(!runtime·mheap.sweepdone) { | 1039 if(!runtime·mheap.sweepdone) { |
1936 // It's possible if GC has happened between sweepone has | 1040 // It's possible if GC has happened between sweepone has |
1937 // returned -1 and gclock lock. | 1041 // returned -1 and gclock lock. |
1938 runtime·unlock(&gclock); | 1042 runtime·unlock(&gclock); |
1939 continue; | 1043 continue; |
1940 } | 1044 } |
1941 sweep.parked = true; | 1045 sweep.parked = true; |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1975 if(s->incache) | 1079 if(s->incache) |
1976 runtime·throw("sweep of incache span"); | 1080 runtime·throw("sweep of incache span"); |
1977 npages = s->npages; | 1081 npages = s->npages; |
1978 if(!runtime·MSpan_Sweep(s)) | 1082 if(!runtime·MSpan_Sweep(s)) |
1979 npages = 0; | 1083 npages = 0; |
1980 g->m->locks--; | 1084 g->m->locks--; |
1981 return npages; | 1085 return npages; |
1982 } | 1086 } |
1983 } | 1087 } |
1984 | 1088 |
1985 static void | |
1986 dumpspan(uint32 idx) | |
1987 { | |
1988 int32 sizeclass, n, npages, i, column; | |
1989 uintptr size; | |
1990 byte *p; | |
1991 byte *arena_start; | |
1992 MSpan *s; | |
1993 bool allocated; | |
1994 | |
1995 s = runtime·mheap.allspans[idx]; | |
1996 if(s->state != MSpanInUse) | |
1997 return; | |
1998 arena_start = runtime·mheap.arena_start; | |
1999 p = (byte*)(s->start << PageShift); | |
2000 sizeclass = s->sizeclass; | |
2001 size = s->elemsize; | |
2002 if(sizeclass == 0) { | |
2003 n = 1; | |
2004 } else { | |
2005 npages = runtime·class_to_allocnpages[sizeclass]; | |
2006 n = (npages << PageShift) / size; | |
2007 } | |
2008 ········ | |
2009 runtime·printf("%p .. %p:\n", p, p+n*size); | |
2010 column = 0; | |
2011 for(; n>0; n--, p+=size) { | |
2012 uintptr off, *bitp, shift, bits; | |
2013 | |
2014 off = (uintptr*)p - (uintptr*)arena_start; | |
2015 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | |
2016 shift = off % wordsPerBitmapWord; | |
2017 bits = *bitp>>shift; | |
2018 | |
2019 allocated = ((bits & bitAllocated) != 0); | |
2020 | |
2021 for(i=0; i<size; i+=sizeof(void*)) { | |
2022 if(column == 0) { | |
2023 runtime·printf("\t"); | |
2024 } | |
2025 if(i == 0) { | |
2026 runtime·printf(allocated ? "(" : "["); | |
2027 runtime·printf("%p: ", p+i); | |
2028 } else { | |
2029 runtime·printf(" "); | |
2030 } | |
2031 | |
2032 runtime·printf("%p", *(void**)(p+i)); | |
2033 | |
2034 if(i+sizeof(void*) >= size) { | |
2035 runtime·printf(allocated ? ") " : "] "); | |
2036 } | |
2037 | |
2038 column++; | |
2039 if(column == 8) { | |
2040 runtime·printf("\n"); | |
2041 column = 0; | |
2042 } | |
2043 } | |
2044 } | |
2045 runtime·printf("\n"); | |
2046 } | |
2047 | |
2048 // A debugging function to dump the contents of memory | |
2049 void | |
2050 runtime·memorydump(void) | |
2051 { | |
2052 uint32 spanidx; | |
2053 | |
2054 for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { | |
2055 dumpspan(spanidx); | |
2056 } | |
2057 } | |
2058 | |
2059 void | 1089 void |
2060 runtime·gchelper(void) | 1090 runtime·gchelper(void) |
2061 { | 1091 { |
2062 uint32 nproc; | 1092 uint32 nproc; |
2063 | 1093 |
2064 g->m->traceback = 2; | 1094 g->m->traceback = 2; |
2065 gchelperstart(); | 1095 gchelperstart(); |
2066 | 1096 |
2067 // parallel mark for over gc roots | 1097 // parallel mark for over gc roots |
2068 runtime·parfordo(work.markfor); | 1098 runtime·parfordo(work.markfor); |
2069 | 1099 |
2070 // help other threads scan secondary blocks | 1100 // help other threads scan secondary blocks |
2071 » scanblock(nil, true); | 1101 » scanblock(nil, 0, nil); |
2072 | 1102 |
2073 bufferList[g->m->helpgc].busy = 0; | |
2074 nproc = work.nproc; // work.nproc can change right after we increment w ork.ndone | 1103 nproc = work.nproc; // work.nproc can change right after we increment w ork.ndone |
2075 if(runtime·xadd(&work.ndone, +1) == nproc-1) | 1104 if(runtime·xadd(&work.ndone, +1) == nproc-1) |
2076 runtime·notewakeup(&work.alldone); | 1105 runtime·notewakeup(&work.alldone); |
2077 g->m->traceback = 0; | 1106 g->m->traceback = 0; |
2078 } | 1107 } |
2079 | 1108 |
2080 static void | 1109 static void |
2081 cachestats(void) | 1110 cachestats(void) |
2082 { | 1111 { |
2083 MCache *c; | 1112 MCache *c; |
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2228 struct gc_args a; | 1257 struct gc_args a; |
2229 int32 i; | 1258 int32 i; |
2230 | 1259 |
2231 // The atomic operations are not atomic if the uint64s | 1260 // The atomic operations are not atomic if the uint64s |
2232 // are not aligned on uint64 boundaries. This has been | 1261 // are not aligned on uint64 boundaries. This has been |
2233 // a problem in the past. | 1262 // a problem in the past. |
2234 if((((uintptr)&work.empty) & 7) != 0) | 1263 if((((uintptr)&work.empty) & 7) != 0) |
2235 runtime·throw("runtime: gc work buffer is misaligned"); | 1264 runtime·throw("runtime: gc work buffer is misaligned"); |
2236 if((((uintptr)&work.full) & 7) != 0) | 1265 if((((uintptr)&work.full) & 7) != 0) |
2237 runtime·throw("runtime: gc work buffer is misaligned"); | 1266 runtime·throw("runtime: gc work buffer is misaligned"); |
1267 if(sizeof(Workbuf) != WorkbufSize) | |
1268 runtime·throw("runtime: size of Workbuf is suboptimal"); | |
2238 | 1269 |
2239 // The gc is turned off (via enablegc) until | 1270 // The gc is turned off (via enablegc) until |
2240 // the bootstrap has completed. | 1271 // the bootstrap has completed. |
2241 // Also, malloc gets called in the guts | 1272 // Also, malloc gets called in the guts |
2242 // of a number of libraries that might be | 1273 // of a number of libraries that might be |
2243 // holding locks. To avoid priority inversion | 1274 // holding locks. To avoid priority inversion |
2244 // problems, don't bother trying to run gc | 1275 // problems, don't bother trying to run gc |
2245 // while holding a lock. The next mallocgc | 1276 // while holding a lock. The next mallocgc |
2246 // without a lock will do the gc instead. | 1277 // without a lock will do the gc instead. |
2247 if(!mstats.enablegc || g == g->m->g0 || g->m->locks > 0 || runtime·panic king) | 1278 if(!mstats.enablegc || g == g->m->g0 || g->m->locks > 0 || runtime·panic king) |
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2307 gc(gp->param); | 1338 gc(gp->param); |
2308 gp->param = nil; | 1339 gp->param = nil; |
2309 gp->status = Grunning; | 1340 gp->status = Grunning; |
2310 runtime·gogo(&gp->sched); | 1341 runtime·gogo(&gp->sched); |
2311 } | 1342 } |
2312 | 1343 |
2313 static void | 1344 static void |
2314 gc(struct gc_args *args) | 1345 gc(struct gc_args *args) |
2315 { | 1346 { |
2316 int64 t0, t1, t2, t3, t4; | 1347 int64 t0, t1, t2, t3, t4; |
2317 » uint64 heap0, heap1, obj, ninstr; | 1348 » uint64 heap0, heap1, obj; |
2318 GCStats stats; | 1349 GCStats stats; |
2319 uint32 i; | 1350 uint32 i; |
2320 Eface eface; | |
2321 | 1351 |
2322 if(runtime·debug.allocfreetrace) | 1352 if(runtime·debug.allocfreetrace) |
2323 runtime·tracegc(); | 1353 runtime·tracegc(); |
2324 | 1354 |
1355 // This is required while we explicitly free objects and have imprecise GC. | |
rsc
2014/07/23 16:23:20
I thought that once the select change was in you c
dvyukov
2014/07/23 19:01:45
I still want to do this. But I am waiting for this
| |
1356 // If we don't do this, then scanblock can queue an object for scanning; | |
1357 // then another thread frees this object during RootFlushCaches; | |
1358 // then the first thread scans the object; then debug check in scanblock | |
1359 // finds this object already freed and throws. | |
1360 if(Debug) | |
1361 flushallmcaches(); | |
1362 | |
2325 g->m->traceback = 2; | 1363 g->m->traceback = 2; |
2326 t0 = args->start_time; | 1364 t0 = args->start_time; |
2327 work.tstart = args->start_time;· | 1365 work.tstart = args->start_time;· |
2328 | 1366 |
2329 » if(CollectStats) | 1367 » if(work.gcdata == nil) { |
2330 » » runtime·memclr((byte*)&gcstats, sizeof(gcstats)); | 1368 » » work.gcdata = unrollglobgcprog(gcdata, edata - data); |
1369 » » work.gcbss = unrollglobgcprog(gcbss, ebss - bss); | |
1370 » } | |
2331 | 1371 |
2332 g->m->locks++; // disable gc during mallocs in parforalloc | |
2333 if(work.markfor == nil) | 1372 if(work.markfor == nil) |
2334 work.markfor = runtime·parforalloc(MaxGcproc); | 1373 work.markfor = runtime·parforalloc(MaxGcproc); |
2335 g->m->locks--; | |
2336 | |
2337 if(itabtype == nil) { | |
2338 // get C pointer to the Go type "itab" | |
2339 runtime·gc_itab_ptr(&eface); | |
2340 itabtype = ((PtrType*)eface.type)->elem; | |
2341 } | |
2342 | 1374 |
2343 t1 = 0; | 1375 t1 = 0; |
2344 if(runtime·debug.gctrace) | 1376 if(runtime·debug.gctrace) |
2345 t1 = runtime·nanotime(); | 1377 t1 = runtime·nanotime(); |
2346 | 1378 |
2347 // Sweep what is not sweeped by bgsweep. | 1379 // Sweep what is not sweeped by bgsweep. |
2348 while(runtime·sweepone() != -1) | 1380 while(runtime·sweepone() != -1) |
2349 » » gcstats.npausesweep++; | 1381 » » sweep.npausesweep++; |
2350 | 1382 |
2351 work.nwait = 0; | 1383 work.nwait = 0; |
2352 work.ndone = 0; | 1384 work.ndone = 0; |
2353 work.nproc = runtime·gcprocs(); | 1385 work.nproc = runtime·gcprocs(); |
2354 runtime·parforsetup(work.markfor, work.nproc, RootCount + runtime·allgle n, nil, false, markroot); | 1386 runtime·parforsetup(work.markfor, work.nproc, RootCount + runtime·allgle n, nil, false, markroot); |
2355 if(work.nproc > 1) { | 1387 if(work.nproc > 1) { |
2356 runtime·noteclear(&work.alldone); | 1388 runtime·noteclear(&work.alldone); |
2357 runtime·helpgc(work.nproc); | 1389 runtime·helpgc(work.nproc); |
2358 } | 1390 } |
2359 | 1391 |
2360 t2 = 0; | 1392 t2 = 0; |
2361 if(runtime·debug.gctrace) | 1393 if(runtime·debug.gctrace) |
2362 t2 = runtime·nanotime(); | 1394 t2 = runtime·nanotime(); |
2363 | 1395 |
2364 gchelperstart(); | 1396 gchelperstart(); |
2365 runtime·parfordo(work.markfor); | 1397 runtime·parfordo(work.markfor); |
2366 » scanblock(nil, true); | 1398 » scanblock(nil, 0, nil); |
2367 | 1399 |
2368 t3 = 0; | 1400 t3 = 0; |
2369 if(runtime·debug.gctrace) | 1401 if(runtime·debug.gctrace) |
2370 t3 = runtime·nanotime(); | 1402 t3 = runtime·nanotime(); |
2371 | 1403 |
2372 bufferList[g->m->helpgc].busy = 0; | |
2373 if(work.nproc > 1) | 1404 if(work.nproc > 1) |
2374 runtime·notesleep(&work.alldone); | 1405 runtime·notesleep(&work.alldone); |
2375 | 1406 |
2376 cachestats(); | 1407 cachestats(); |
2377 // next_gc calculation is tricky with concurrent sweep since we don't kn ow size of live heap | 1408 // next_gc calculation is tricky with concurrent sweep since we don't kn ow size of live heap |
2378 // estimate what was live heap size after previous GC (for tracing only) | 1409 // estimate what was live heap size after previous GC (for tracing only) |
2379 heap0 = mstats.next_gc*100/(gcpercent+100); | 1410 heap0 = mstats.next_gc*100/(gcpercent+100); |
2380 // conservatively set next_gc to high value assuming that everything is live | 1411 // conservatively set next_gc to high value assuming that everything is live |
2381 // concurrent/lazy sweep will reduce this number while discovering new g arbage | 1412 // concurrent/lazy sweep will reduce this number while discovering new g arbage |
2382 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; | 1413 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; |
(...skipping 18 matching lines...) Expand all Loading... | |
2401 stats.nprocyield += work.markfor->nprocyield; | 1432 stats.nprocyield += work.markfor->nprocyield; |
2402 stats.nosyield += work.markfor->nosyield; | 1433 stats.nosyield += work.markfor->nosyield; |
2403 stats.nsleep += work.markfor->nsleep; | 1434 stats.nsleep += work.markfor->nsleep; |
2404 | 1435 |
2405 runtime·printf("gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D ) objects," | 1436 runtime·printf("gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D ) objects," |
2406 " %d/%d/%d sweeps," | 1437 " %d/%d/%d sweeps," |
2407 " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\ n", | 1438 " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\ n", |
2408 mstats.numgc, work.nproc, (t1-t0)/1000, (t2-t1)/1000, (t 3-t2)/1000, (t4-t3)/1000, | 1439 mstats.numgc, work.nproc, (t1-t0)/1000, (t2-t1)/1000, (t 3-t2)/1000, (t4-t3)/1000, |
2409 heap0>>20, heap1>>20, obj, | 1440 heap0>>20, heap1>>20, obj, |
2410 mstats.nmalloc, mstats.nfree, | 1441 mstats.nmalloc, mstats.nfree, |
2411 » » » sweep.nspan, gcstats.nbgsweep, gcstats.npausesweep, | 1442 » » » sweep.nspan, sweep.nbgsweep, sweep.npausesweep, |
2412 stats.nhandoff, stats.nhandoffcnt, | 1443 stats.nhandoff, stats.nhandoffcnt, |
2413 work.markfor->nsteal, work.markfor->nstealcnt, | 1444 work.markfor->nsteal, work.markfor->nstealcnt, |
2414 stats.nprocyield, stats.nosyield, stats.nsleep); | 1445 stats.nprocyield, stats.nosyield, stats.nsleep); |
2415 » » gcstats.nbgsweep = gcstats.npausesweep = 0; | 1446 » » sweep.nbgsweep = sweep.npausesweep = 0; |
2416 » » if(CollectStats) { | |
2417 » » » runtime·printf("scan: %D bytes, %D objects, %D untyped, %D types from MSpan\n", | |
2418 » » » » gcstats.nbytes, gcstats.obj.cnt, gcstats.obj.not ype, gcstats.obj.typelookup); | |
2419 » » » if(gcstats.ptr.cnt != 0) | |
2420 » » » » runtime·printf("avg ptrbufsize: %D (%D/%D)\n", | |
2421 » » » » » gcstats.ptr.sum/gcstats.ptr.cnt, gcstats .ptr.sum, gcstats.ptr.cnt); | |
2422 » » » if(gcstats.obj.cnt != 0) | |
2423 » » » » runtime·printf("avg nobj: %D (%D/%D)\n", | |
2424 » » » » » gcstats.obj.sum/gcstats.obj.cnt, gcstats .obj.sum, gcstats.obj.cnt); | |
2425 » » » runtime·printf("rescans: %D, %D bytes\n", gcstats.rescan , gcstats.rescanbytes); | |
2426 | |
2427 » » » runtime·printf("instruction counts:\n"); | |
2428 » » » ninstr = 0; | |
2429 » » » for(i=0; i<nelem(gcstats.instr); i++) { | |
2430 » » » » runtime·printf("\t%d:\t%D\n", i, gcstats.instr[i ]); | |
2431 » » » » ninstr += gcstats.instr[i]; | |
2432 » » » } | |
2433 » » » runtime·printf("\ttotal:\t%D\n", ninstr); | |
2434 | |
2435 » » » runtime·printf("putempty: %D, getfull: %D\n", gcstats.pu tempty, gcstats.getfull); | |
2436 | |
2437 » » » runtime·printf("markonly base lookup: bit %D word %D spa n %D\n", gcstats.markonly.foundbit, gcstats.markonly.foundword, gcstats.markonly .foundspan); | |
2438 » » » runtime·printf("flushptrbuf base lookup: bit %D word %D span %D\n", gcstats.flushptrbuf.foundbit, gcstats.flushptrbuf.foundword, gcstats .flushptrbuf.foundspan); | |
2439 » » } | |
2440 } | 1447 } |
2441 | 1448 |
2442 // We cache current runtime·mheap.allspans array in sweep.spans, | 1449 // We cache current runtime·mheap.allspans array in sweep.spans, |
2443 // because the former can be resized and freed. | 1450 // because the former can be resized and freed. |
2444 // Otherwise we would need to take heap lock every time | 1451 // Otherwise we would need to take heap lock every time |
2445 // we want to convert span index to span pointer. | 1452 // we want to convert span index to span pointer. |
2446 | 1453 |
2447 // Free the old cached array if necessary. | 1454 // Free the old cached array if necessary. |
2448 if(sweep.spans && sweep.spans != runtime·mheap.allspans) | 1455 if(sweep.spans && sweep.spans != runtime·mheap.allspans) |
2449 runtime·SysFree(sweep.spans, sweep.nspan*sizeof(sweep.spans[0]), &mstats.other_sys); | 1456 runtime·SysFree(sweep.spans, sweep.nspan*sizeof(sweep.spans[0]), &mstats.other_sys); |
(...skipping 11 matching lines...) Expand all Loading... | |
2461 if(sweep.g == nil) | 1468 if(sweep.g == nil) |
2462 sweep.g = runtime·newproc1(&bgsweepv, nil, 0, 0, runtime ·gc); | 1469 sweep.g = runtime·newproc1(&bgsweepv, nil, 0, 0, runtime ·gc); |
2463 else if(sweep.parked) { | 1470 else if(sweep.parked) { |
2464 sweep.parked = false; | 1471 sweep.parked = false; |
2465 runtime·ready(sweep.g); | 1472 runtime·ready(sweep.g); |
2466 } | 1473 } |
2467 runtime·unlock(&gclock); | 1474 runtime·unlock(&gclock); |
2468 } else { | 1475 } else { |
2469 // Sweep all spans eagerly. | 1476 // Sweep all spans eagerly. |
2470 while(runtime·sweepone() != -1) | 1477 while(runtime·sweepone() != -1) |
2471 » » » gcstats.npausesweep++; | 1478 » » » sweep.npausesweep++; |
2472 } | 1479 } |
2473 | 1480 |
2474 // Shrink a stack if not much of it is being used. | 1481 // Shrink a stack if not much of it is being used. |
2475 // TODO: do in a parfor | 1482 // TODO: do in a parfor |
2476 for(i = 0; i < runtime·allglen; i++) | 1483 for(i = 0; i < runtime·allglen; i++) |
2477 runtime·shrinkstack(runtime·allg[i]); | 1484 runtime·shrinkstack(runtime·allg[i]); |
2478 | 1485 |
2479 runtime·MProf_GC(); | 1486 runtime·MProf_GC(); |
2480 g->m->traceback = 0; | 1487 g->m->traceback = 0; |
2481 } | 1488 } |
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2553 gcpercent = in; | 1560 gcpercent = in; |
2554 runtime·unlock(&runtime·mheap); | 1561 runtime·unlock(&runtime·mheap); |
2555 return out; | 1562 return out; |
2556 } | 1563 } |
2557 | 1564 |
2558 static void | 1565 static void |
2559 gchelperstart(void) | 1566 gchelperstart(void) |
2560 { | 1567 { |
2561 if(g->m->helpgc < 0 || g->m->helpgc >= MaxGcproc) | 1568 if(g->m->helpgc < 0 || g->m->helpgc >= MaxGcproc) |
2562 runtime·throw("gchelperstart: bad m->helpgc"); | 1569 runtime·throw("gchelperstart: bad m->helpgc"); |
2563 if(runtime·xchg(&bufferList[g->m->helpgc].busy, 1)) | |
2564 runtime·throw("gchelperstart: already busy"); | |
2565 if(g != g->m->g0) | 1570 if(g != g->m->g0) |
2566 runtime·throw("gchelper not running on g0 stack"); | 1571 runtime·throw("gchelper not running on g0 stack"); |
2567 } | 1572 } |
2568 | 1573 |
2569 static void | 1574 static void |
2570 runfinq(void) | 1575 runfinq(void) |
2571 { | 1576 { |
2572 Finalizer *f; | 1577 Finalizer *f; |
2573 FinBlock *fb, *next; | 1578 FinBlock *fb, *next; |
2574 byte *frame; | 1579 byte *frame; |
(...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2691 runtime·lock(&finlock); | 1696 runtime·lock(&finlock); |
2692 if(runtime·fingwait && runtime·fingwake) { | 1697 if(runtime·fingwait && runtime·fingwake) { |
2693 runtime·fingwait = false; | 1698 runtime·fingwait = false; |
2694 runtime·fingwake = false; | 1699 runtime·fingwake = false; |
2695 res = fing; | 1700 res = fing; |
2696 } | 1701 } |
2697 runtime·unlock(&finlock); | 1702 runtime·unlock(&finlock); |
2698 return res; | 1703 return res; |
2699 } | 1704 } |
2700 | 1705 |
1706 // Recursively GC program in prog. | |
1707 // mask is where to store the result. | |
1708 // ppos is a pointer to position in mask, in bits. | |
1709 // sparse says to generate 4-bits per word mask for heap (2-bits for data/bss ot herwise). | |
1710 static byte* | |
1711 unrollgcprog1(byte *mask, byte *prog, uintptr *ppos, bool inplace, bool sparse) | |
1712 { | |
1713 uintptr *b, off, shift, pos, siz, i; | |
1714 byte *arena_start, *prog1, v; | |
1715 | |
1716 arena_start = runtime·mheap.arena_start; | |
1717 pos = *ppos; | |
1718 for(;;) { | |
1719 switch(prog[0]) { | |
1720 case insData: | |
1721 prog++; | |
1722 siz = prog[0]; | |
1723 prog++; | |
1724 for(i = 0; i < siz; i++) { | |
1725 v = prog[i/PointersPerByte]; | |
1726 v >>= (i%PointersPerByte)*BitsPerPointer; | |
1727 v &= BitsMask; | |
1728 if(inplace) { | |
1729 // Store directly into GC bitmap. | |
1730 off = (uintptr*)(mask+pos) - (uintptr*)a rena_start; | |
1731 b = (uintptr*)arena_start - off/wordsPer BitmapWord - 1; | |
1732 shift = (off % wordsPerBitmapWord) * gcB its; | |
1733 if((shift%8)==0) | |
1734 ((byte*)b)[shift/8] = 0; | |
1735 ((byte*)b)[shift/8] |= v<<((shift%8)+2); | |
1736 pos += PtrSize; | |
1737 } else if(sparse) { | |
1738 // 4-bits per word | |
1739 v <<= (pos%8)+2; | |
1740 mask[pos/8] |= v; | |
1741 pos += gcBits; | |
1742 } else { | |
1743 // 2-bits per word | |
1744 v <<= pos%8; | |
1745 mask[pos/8] |= v; | |
1746 pos += BitsPerPointer; | |
1747 } | |
1748 } | |
1749 prog += ROUND(siz*BitsPerPointer, 8)/8; | |
1750 break; | |
1751 case insArray: | |
1752 prog++; | |
1753 siz = 0; | |
1754 for(i = 0; i < PtrSize; i++) | |
1755 siz = (siz<<8) + prog[PtrSize-i-1]; | |
1756 prog += PtrSize; | |
1757 prog1 = nil; | |
1758 for(i = 0; i < siz; i++) | |
1759 prog1 = unrollgcprog1(mask, prog, &pos, inplace, sparse); | |
1760 if(prog1[0] != insArrayEnd) | |
1761 runtime·throw("unrollgcprog: array does not end with insArrayEnd"); | |
1762 prog = prog1+1; | |
1763 break; | |
1764 case insArrayEnd: | |
1765 case insEnd: | |
1766 *ppos = pos; | |
1767 return prog; | |
1768 default: | |
1769 runtime·throw("unrollgcprog: unknown instruction"); | |
1770 } | |
1771 } | |
1772 } | |
1773 | |
1774 // Unrolls GC program prog for data/bss, returns dense GC mask. | |
1775 static byte* | |
1776 unrollglobgcprog(byte *prog, uintptr size) | |
1777 { | |
1778 byte *mask; | |
1779 uintptr pos, masksize; | |
1780 | |
1781 masksize = ROUND(ROUND(size, PtrSize)/PtrSize*BitsPerPointer, 8)/8; | |
1782 mask = runtime·persistentalloc(masksize+1, 0, &mstats.gc_sys); | |
1783 mask[masksize] = 0xa1; | |
1784 pos = 0; | |
1785 prog = unrollgcprog1(mask, prog, &pos, false, false); | |
1786 if(pos != size/PtrSize*BitsPerPointer) { | |
1787 runtime·printf("unrollglobgcprog: bad program size, got %D, expe ct %D\n", | |
1788 (uint64)pos, (uint64)size/PtrSize*BitsPerPointer); | |
1789 runtime·throw("unrollglobgcprog: bad program size"); | |
1790 } | |
1791 if(prog[0] != insEnd) | |
1792 runtime·throw("unrollglobgcprog: program does not end with insEn d"); | |
1793 if(mask[masksize] != 0xa1) | |
1794 runtime·throw("unrollglobgcprog: overflow"); | |
1795 return mask; | |
1796 } | |
1797 | |
1798 static void | |
1799 unrollgcproginplace(void *v, uintptr size, uintptr size0, Type *typ) | |
1800 { | |
1801 uintptr *b, off, shift, pos; | |
1802 byte *arena_start, *prog; | |
1803 | |
1804 pos = 0; | |
1805 prog = (byte*)typ->gc[1]; | |
1806 while(pos != size0) | |
1807 unrollgcprog1(v, prog, &pos, true, true); | |
1808 // Mark first word as bitAllocated. | |
1809 arena_start = runtime·mheap.arena_start; | |
1810 off = (uintptr*)v - (uintptr*)arena_start; | |
1811 b = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | |
1812 shift = (off % wordsPerBitmapWord) * gcBits; | |
1813 *b |= bitAllocated<<shift; | |
1814 // Mark word after last as bitAllocated. | |
1815 if(size0 < size) { | |
1816 off = (uintptr*)((byte*)v + size0) - (uintptr*)arena_start; | |
1817 b = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | |
1818 shift = (off % wordsPerBitmapWord) * gcBits; | |
1819 *b &= ~(bitPtrMask<<shift); | |
1820 } | |
1821 } | |
1822 | |
1823 // Unrolls GC program in typ->gc[1] into typ->gc[0] | |
1824 static void | |
1825 unrollgcprog(Type *typ) | |
1826 { | |
1827 static Lock lock; | |
1828 byte *mask, *prog; | |
1829 uintptr pos; | |
1830 uint32 x; | |
1831 | |
1832 runtime·lock(&lock); | |
1833 mask = (byte*)typ->gc[0]; | |
1834 if(mask[0] == 0) { | |
1835 pos = 8; // skip the unroll flag | |
1836 prog = (byte*)typ->gc[1]; | |
1837 prog = unrollgcprog1(mask, prog, &pos, false, true); | |
1838 if(prog[0] != insEnd) | |
1839 runtime·throw("unrollgcprog: program does not end with i nsEnd"); | |
1840 if(((typ->size/PtrSize)%2) != 0) { | |
1841 // repeat the program twice | |
1842 prog = (byte*)typ->gc[1]; | |
1843 unrollgcprog1(mask, prog, &pos, false, true); | |
1844 } | |
1845 // atomic way to say mask[0] = 1 | |
1846 x = ((uint32*)mask)[0]; | |
1847 runtime·atomicstore((uint32*)mask, x|1); | |
1848 } | |
1849 runtime·unlock(&lock); | |
1850 } | |
1851 | |
2701 void | 1852 void |
2702 runtime·marknogc(void *v) | 1853 runtime·markallocated(void *v, uintptr size, uintptr size0, Type *typ, bool scan ) |
2703 { | 1854 { |
2704 » uintptr *b, off, shift; | 1855 » uintptr *b, off, shift, i, ti, te, nptr, masksize; |
2705 | 1856 » byte *arena_start, x; |
2706 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 1857 » bool *ptrmask; |
2707 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 1858 |
2708 » shift = off % wordsPerBitmapWord; | 1859 » arena_start = runtime·mheap.arena_start; |
2709 » *b = (*b & ~(bitAllocated<<shift)) | bitBlockBoundary<<shift; | 1860 » off = (uintptr*)v - (uintptr*)arena_start; |
2710 } | 1861 » b = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
2711 | 1862 » shift = (off % wordsPerBitmapWord) * gcBits; |
2712 void | 1863 » if(Debug && (((*b)>>shift)&bitMask) != bitBoundary) { |
2713 runtime·markscan(void *v) | 1864 » » runtime·printf("runtime: bad bits in markallocated (%p) b=%p[%p] \n", v, b, *b); |
2714 { | 1865 » » runtime·throw("bad bits in markallocated"); |
2715 » uintptr *b, off, shift; | 1866 » } |
2716 | 1867 |
2717 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 1868 » if(!scan) { |
2718 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 1869 » » // BitsDead in the first quadruple means don't scan. |
2719 » shift = off % wordsPerBitmapWord; | 1870 » » if(size == PtrSize) |
2720 » *b |= bitScan<<shift; | 1871 » » » *b = (*b & ~((bitBoundary|bitPtrMask)<<shift)) | (bitAll ocated<<shift); |
1872 » » else | |
1873 » » » ((byte*)b)[shift/8] = bitAllocated; | |
1874 » » return; | |
1875 » } | |
1876 » if(size == PtrSize) { | |
1877 » » // It's one word and it has pointers, it must be a pointer. | |
1878 » » *b = (*b & ~((bitBoundary|bitPtrMask)<<shift)) | ((bitAllocated | (BitsPointer<<2))<<shift); | |
1879 » » return; | |
1880 » } | |
1881 » ti = te = 0; | |
1882 » ptrmask = nil; | |
1883 » if(typ != nil && (typ->gc[0]|typ->gc[1]) != 0 && typ->size > PtrSize) { | |
1884 » » if(typ->kind&KindGCProg) { | |
1885 » » » nptr = ROUND(typ->size, PtrSize)/PtrSize; | |
1886 » » » masksize = nptr; | |
1887 » » » if(masksize%2) | |
1888 » » » » masksize *= 2;» // repeated twice | |
1889 » » » masksize = masksize*PointersPerByte/8;» // 4 bits per wo rd | |
1890 » » » masksize++;» // unroll flag in the beginning | |
1891 » » » if(masksize > MaxGCMask) { | |
1892 » » » » // If the mask is too large, unroll the program directly | |
1893 » » » » // into the GC bitmap. It's 7 times slower than copying | |
1894 » » » » // from the pre-unrolled mask, but saves 1/16 of type size | |
1895 » » » » // memory for the mask. | |
1896 » » » » unrollgcproginplace(v, size, size0, typ); | |
1897 » » » » return; | |
1898 » » » } | |
1899 » » » ptrmask = (byte*)typ->gc[0]; | |
1900 » » » // check whether the program is already unrolled | |
1901 » » » if((runtime·atomicload((uint32*)ptrmask)&0xff) == 0) | |
1902 » » » » unrollgcprog(typ); | |
1903 » » » ptrmask++; // skip the unroll flag byte | |
1904 » » } else | |
1905 » » » ptrmask = (byte*)&typ->gc[0]; // embed mask | |
1906 » » if(size == 2*PtrSize) { | |
1907 » » » ((byte*)b)[shift/8] = ptrmask[0] | bitAllocated; | |
1908 » » » return; | |
1909 » » } | |
1910 » » te = typ->size/PtrSize; | |
1911 » » // if the type occupies odd number of words, its mask is repeate d twice | |
1912 » » if((te%2) == 0) | |
1913 » » » te /= 2; | |
1914 » } | |
1915 » if(size == 2*PtrSize) { | |
1916 » » ((byte*)b)[shift/8] = (BitsPointer<<2) | (BitsPointer<<6) | bitA llocated; | |
1917 » » return; | |
1918 » } | |
1919 » // Copy pointer bitmask into the bitmap. | |
1920 » for(i=0; i<size0; i+=2*PtrSize) { | |
1921 » » x = (BitsPointer<<2) | (BitsPointer<<6); | |
1922 » » if(ptrmask != nil) { | |
1923 » » » x = ptrmask[ti++]; | |
1924 » » » if(ti == te) | |
1925 » » » » ti = 0; | |
1926 » » } | |
1927 » » off = (uintptr*)((byte*)v + i) - (uintptr*)arena_start; | |
1928 » » b = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | |
1929 » » shift = (off % wordsPerBitmapWord) * gcBits; | |
1930 » » if(i == 0) | |
1931 » » » x |= bitAllocated; | |
1932 » » if(i+PtrSize == size0) | |
1933 » » » x &= ~(bitPtrMask<<4); | |
1934 » » ((byte*)b)[shift/8] = x; | |
1935 » } | |
1936 » if(size0 == i && size0 < size) { | |
1937 » » // mark the word after last object's word as BitsDead | |
1938 » » off = (uintptr*)((byte*)v + size0) - (uintptr*)arena_start; | |
1939 » » b = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | |
1940 » » shift = (off % wordsPerBitmapWord) * gcBits; | |
1941 » » ((byte*)b)[shift/8] = 0; | |
1942 » } | |
2721 } | 1943 } |
2722 | 1944 |
2723 // mark the block at v as freed. | 1945 // mark the block at v as freed. |
2724 void | 1946 void |
2725 runtime·markfreed(void *v) | 1947 runtime·markfreed(void *v) |
2726 { | 1948 { |
2727 » uintptr *b, off, shift, xbits; | 1949 » uintptr *b, off, shift, xbits, bits; |
2728 | |
2729 » if(0) | |
2730 » » runtime·printf("markfreed %p\n", v); | |
2731 | 1950 |
2732 if((byte*)v > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mhea p.arena_start) | 1951 if((byte*)v > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mhea p.arena_start) |
2733 runtime·throw("markfreed: bad pointer"); | 1952 runtime·throw("markfreed: bad pointer"); |
2734 | 1953 |
2735 off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 1954 off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset |
2736 b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 1955 b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2737 » shift = off % wordsPerBitmapWord; | 1956 » shift = (off % wordsPerBitmapWord) * gcBits; |
1957 » xbits = *b; | |
1958 » bits = (xbits>>shift) & bitMask; | |
1959 | |
1960 » if(bits == bitMiddle) | |
1961 » » runtime·throw("bad bits in markfreed"); | |
1962 » if(bits == bitBoundary) | |
1963 » » return; // FlagNoGC object | |
2738 if(!g->m->gcing || work.nproc == 1) { | 1964 if(!g->m->gcing || work.nproc == 1) { |
2739 // During normal operation (not GC), the span bitmap is not upda ted concurrently, | 1965 // During normal operation (not GC), the span bitmap is not upda ted concurrently, |
2740 // because either the span is cached or accesses are protected w ith MCentral lock. | 1966 // because either the span is cached or accesses are protected w ith MCentral lock. |
2741 » » *b = (*b & ~(bitMask<<shift)) | (bitAllocated<<shift); | 1967 » » *b = (xbits & ~(bitMask<<shift)) | (bitBoundary<<shift); |
2742 } else { | 1968 } else { |
2743 // During GC other threads concurrently mark heap. | 1969 // During GC other threads concurrently mark heap. |
2744 for(;;) { | 1970 for(;;) { |
2745 xbits = *b; | 1971 xbits = *b; |
2746 » » » if(runtime·casp((void**)b, (void*)xbits, (void*)((xbits & ~(bitMask<<shift)) | (bitAllocated<<shift)))) | 1972 » » » if(runtime·casp((void**)b, (void*)xbits, (void*)((xbits & ~(bitMask<<shift)) | (bitBoundary<<shift)))) |
2747 break; | 1973 break; |
2748 } | 1974 } |
2749 } | 1975 } |
2750 } | 1976 } |
2751 | 1977 |
2752 // check that the block at v of size n is marked freed. | |
2753 void | |
2754 runtime·checkfreed(void *v, uintptr n) | |
2755 { | |
2756 uintptr *b, bits, off, shift; | |
2757 | |
2758 if(!runtime·checking) | |
2759 return; | |
2760 | |
2761 if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) | |
2762 return; // not allocated, so okay | |
2763 | |
2764 off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | |
2765 b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | |
2766 shift = off % wordsPerBitmapWord; | |
2767 | |
2768 bits = *b>>shift; | |
2769 if((bits & bitAllocated) != 0) { | |
2770 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", | |
2771 v, n, off, bits & bitMask); | |
2772 runtime·throw("checkfreed: not freed"); | |
2773 } | |
2774 } | |
2775 | |
2776 // mark the span of memory at v as having n blocks of the given size. | 1978 // mark the span of memory at v as having n blocks of the given size. |
2777 // if leftover is true, there is left over space at the end of the span. | 1979 // if leftover is true, there is left over space at the end of the span. |
2778 void | 1980 void |
2779 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) | 1981 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) |
2780 { | 1982 { |
2781 » uintptr *b, *b0, off, shift, i, x; | 1983 » uintptr *b, *b0, off, shift, x; |
2782 byte *p; | 1984 byte *p; |
2783 | 1985 |
2784 if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runti me·mheap.arena_start) | 1986 if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runti me·mheap.arena_start) |
2785 runtime·throw("markspan: bad pointer"); | 1987 runtime·throw("markspan: bad pointer"); |
2786 | 1988 |
2787 if(runtime·checking) { | |
2788 // bits should be all zero at the start | |
2789 off = (byte*)v + size - runtime·mheap.arena_start; | |
2790 b = (uintptr*)(runtime·mheap.arena_start - off/wordsPerBitmapWor d); | |
2791 for(i = 0; i < size/PtrSize/wordsPerBitmapWord; i++) { | |
2792 if(b[i] != 0) | |
2793 runtime·throw("markspan: span bits not zero"); | |
2794 } | |
2795 } | |
2796 | |
2797 p = v; | 1989 p = v; |
2798 if(leftover) // mark a boundary just past end of last block too | 1990 if(leftover) // mark a boundary just past end of last block too |
2799 n++; | 1991 n++; |
2800 | 1992 |
2801 b0 = nil; | 1993 b0 = nil; |
2802 x = 0; | 1994 x = 0; |
2803 for(; n-- > 0; p += size) { | 1995 for(; n-- > 0; p += size) { |
2804 // Okay to use non-atomic ops here, because we control | 1996 // Okay to use non-atomic ops here, because we control |
2805 // the entire span, and each bitmap word has bits for only | 1997 // the entire span, and each bitmap word has bits for only |
2806 // one span, so no other goroutines are changing these | 1998 // one span, so no other goroutines are changing these |
2807 // bitmap words. | 1999 // bitmap words. |
2808 off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // wor d offset | 2000 off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // wor d offset |
2809 b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2001 b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2810 » » shift = off % wordsPerBitmapWord; | 2002 » » shift = (off % wordsPerBitmapWord) * gcBits; |
2811 if(b0 != b) { | 2003 if(b0 != b) { |
2812 if(b0 != nil) | 2004 if(b0 != nil) |
2813 *b0 = x; | 2005 *b0 = x; |
2814 b0 = b; | 2006 b0 = b; |
2815 x = 0; | 2007 x = 0; |
2816 } | 2008 } |
2817 » » x |= bitAllocated<<shift; | 2009 » » x |= bitBoundary<<shift; |
2818 } | 2010 } |
2819 *b0 = x; | 2011 *b0 = x; |
2820 } | 2012 } |
2821 | 2013 |
2822 // unmark the span of memory at v of length n bytes. | 2014 // unmark the span of memory at v of length n bytes. |
2823 void | 2015 void |
2824 runtime·unmarkspan(void *v, uintptr n) | 2016 runtime·unmarkspan(void *v, uintptr n) |
2825 { | 2017 { |
2826 uintptr *p, *b, off; | 2018 uintptr *p, *b, off; |
2827 | 2019 |
2828 if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) | 2020 if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) |
2829 runtime·throw("markspan: bad pointer"); | 2021 runtime·throw("markspan: bad pointer"); |
2830 | 2022 |
2831 p = v; | 2023 p = v; |
2832 off = p - (uintptr*)runtime·mheap.arena_start; // word offset | 2024 off = p - (uintptr*)runtime·mheap.arena_start; // word offset |
2833 » if(off % wordsPerBitmapWord != 0) | 2025 » if((off % wordsPerBitmapWord) != 0) |
2834 runtime·throw("markspan: unaligned pointer"); | 2026 runtime·throw("markspan: unaligned pointer"); |
2835 b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2027 b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2836 n /= PtrSize; | 2028 n /= PtrSize; |
2837 if(n%wordsPerBitmapWord != 0) | 2029 if(n%wordsPerBitmapWord != 0) |
2838 runtime·throw("unmarkspan: unaligned length"); | 2030 runtime·throw("unmarkspan: unaligned length"); |
2839 // Okay to use non-atomic ops here, because we control | 2031 // Okay to use non-atomic ops here, because we control |
2840 // the entire span, and each bitmap word has bits for only | 2032 // the entire span, and each bitmap word has bits for only |
2841 // one span, so no other goroutines are changing these | 2033 // one span, so no other goroutines are changing these |
2842 // bitmap words. | 2034 // bitmap words. |
2843 n /= wordsPerBitmapWord; | 2035 n /= wordsPerBitmapWord; |
(...skipping 14 matching lines...) Expand all Loading... | |
2858 | 2050 |
2859 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; | 2051 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; |
2860 n = ROUND(n, bitmapChunk); | 2052 n = ROUND(n, bitmapChunk); |
2861 n = ROUND(n, PhysPageSize); | 2053 n = ROUND(n, PhysPageSize); |
2862 if(h->bitmap_mapped >= n) | 2054 if(h->bitmap_mapped >= n) |
2863 return; | 2055 return; |
2864 | 2056 |
2865 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped, h->arena_reserv ed, &mstats.gc_sys); | 2057 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped, h->arena_reserv ed, &mstats.gc_sys); |
2866 h->bitmap_mapped = n; | 2058 h->bitmap_mapped = n; |
2867 } | 2059 } |
2060 | |
2061 static bool | |
2062 getgcmaskcb(Stkframe *frame, void *ctxt) | |
2063 { | |
2064 Stkframe *frame0; | |
2065 | |
2066 frame0 = ctxt; | |
2067 if(frame0->sp >= (uintptr)frame->varp - frame->sp && frame0->sp < (uintp tr)frame->varp) { | |
2068 *frame0 = *frame; | |
2069 return false; | |
2070 } | |
2071 return true; | |
2072 } | |
2073 | |
2074 // Returns GC type info for object p for testing. | |
2075 void | |
2076 runtime·getgcmask(byte *p, Type *t, byte **mask, uintptr *len) | |
2077 { | |
2078 Stkframe frame; | |
2079 uintptr i, n, off, bits, shift, *b; | |
2080 byte *base; | |
2081 | |
2082 *mask = nil; | |
2083 *len = 0; | |
2084 | |
2085 // data | |
2086 if(p >= data && p < edata) { | |
2087 n = ((PtrType*)t)->elem->size; | |
2088 *len = n/PtrSize; | |
2089 *mask = runtime·mallocgc(*len, nil, 0); | |
2090 for(i = 0; i < n; i += PtrSize) { | |
2091 off = (p+i-data)/PtrSize; | |
2092 bits = (work.gcdata[off/PointersPerByte] >> ((off%Pointe rsPerByte)*BitsPerPointer))&BitsMask; | |
2093 (*mask)[i/PtrSize] = bits; | |
2094 } | |
2095 return; | |
2096 } | |
2097 // bss | |
2098 if(p >= bss && p < ebss) { | |
2099 n = ((PtrType*)t)->elem->size; | |
2100 *len = n/PtrSize; | |
2101 *mask = runtime·mallocgc(*len, nil, 0); | |
2102 for(i = 0; i < n; i += PtrSize) { | |
2103 off = (p+i-bss)/PtrSize; | |
2104 bits = (work.gcbss[off/PointersPerByte] >> ((off%Pointer sPerByte)*BitsPerPointer))&BitsMask; | |
2105 (*mask)[i/PtrSize] = bits; | |
2106 } | |
2107 return; | |
2108 } | |
2109 // heap | |
2110 if(runtime·mlookup(p, &base, &n, nil)) { | |
2111 *len = n/PtrSize; | |
2112 *mask = runtime·mallocgc(*len, nil, 0); | |
2113 for(i = 0; i < n; i += PtrSize) { | |
2114 off = (uintptr*)(base+i) - (uintptr*)runtime·mheap.arena _start; | |
2115 b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBi tmapWord - 1; | |
2116 shift = (off % wordsPerBitmapWord) * gcBits; | |
2117 bits = (*b >> (shift+2))&BitsMask; | |
2118 (*mask)[i/PtrSize] = bits; | |
2119 } | |
2120 return; | |
2121 } | |
2122 // stack | |
2123 frame.fn = nil; | |
2124 frame.sp = (uintptr)p; | |
2125 runtime·gentraceback((uintptr)runtime·getcallerpc(&p), (uintptr)runtime· getcallersp(&p), 0, g, 0, nil, 1000, getgcmaskcb, &frame, false); | |
2126 if(frame.fn != nil) { | |
2127 Func *f; | |
2128 StackMap *stackmap; | |
2129 BitVector bv; | |
2130 uintptr size; | |
2131 uintptr targetpc; | |
2132 int32 pcdata; | |
2133 | |
2134 f = frame.fn; | |
2135 targetpc = frame.continpc; | |
2136 if(targetpc == 0) | |
2137 return; | |
2138 if(targetpc != f->entry) | |
2139 targetpc--; | |
2140 pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc); | |
2141 if(pcdata == -1) | |
2142 return; | |
2143 stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); | |
2144 if(stackmap == nil || stackmap->n <= 0) | |
2145 return; | |
2146 bv = runtime·stackmapdata(stackmap, pcdata); | |
2147 size = bv.n/BitsPerPointer*PtrSize; | |
2148 n = ((PtrType*)t)->elem->size; | |
2149 *len = n/PtrSize; | |
2150 *mask = runtime·mallocgc(*len, nil, 0); | |
2151 for(i = 0; i < n; i += PtrSize) { | |
2152 off = (p+i-frame.varp+size)/PtrSize; | |
2153 bits = (bv.data[off/PointersPerByte] >> ((off%PointersPe rByte)*BitsPerPointer))&BitsMask; | |
2154 (*mask)[i/PtrSize] = bits; | |
2155 } | |
2156 } | |
2157 } | |
OLD | NEW |