LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Garbage collector. | 5 // Garbage collector. |
6 | 6 |
7 #include "runtime.h" | 7 #include "runtime.h" |
8 #include "arch_GOARCH.h" | 8 #include "arch_GOARCH.h" |
9 #include "malloc.h" | 9 #include "malloc.h" |
10 #include "stack.h" | 10 #include "stack.h" |
11 #include "mgc0.h" | 11 #include "mgc0.h" |
12 #include "race.h" | 12 #include "race.h" |
13 #include "type.h" | 13 #include "type.h" |
14 #include "typekind.h" | 14 #include "typekind.h" |
15 #include "hashmap.h" | 15 #include "hashmap.h" |
16 | 16 |
17 enum { | 17 enum { |
18 Debug = 0, | 18 Debug = 0, |
19 DebugMark = 0, // run second pass to check mark | 19 DebugMark = 0, // run second pass to check mark |
20 CollectStats = 0, | 20 CollectStats = 0, |
21 » ScanStackByFrames = 0, | 21 » ScanStackByFrames = 1, |
22 IgnorePreciseGC = 0, | 22 IgnorePreciseGC = 0, |
23 | 23 |
24 // Four bits per word (see #defines below). | 24 // Four bits per word (see #defines below). |
25 wordsPerBitmapWord = sizeof(void*)*8/4, | 25 wordsPerBitmapWord = sizeof(void*)*8/4, |
26 bitShift = sizeof(void*)*8/4, | 26 bitShift = sizeof(void*)*8/4, |
27 | 27 |
28 handoffThreshold = 4, | 28 handoffThreshold = 4, |
29 IntermediateBufferCapacity = 64, | 29 IntermediateBufferCapacity = 64, |
30 | 30 |
31 // Bits in type information | 31 // Bits in type information |
(...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
197 // This function doesn't append the object to any buffer. | 197 // This function doesn't append the object to any buffer. |
198 static bool | 198 static bool |
199 markonly(void *obj) | 199 markonly(void *obj) |
200 { | 200 { |
201 byte *p; | 201 byte *p; |
202 uintptr *bitp, bits, shift, x, xbits, off; | 202 uintptr *bitp, bits, shift, x, xbits, off; |
203 MSpan *s; | 203 MSpan *s; |
204 PageID k; | 204 PageID k; |
205 | 205 |
206 // Words outside the arena cannot be pointers. | 206 // Words outside the arena cannot be pointers. |
207 » if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) | 207 » if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used) |
208 return false; | 208 return false; |
209 | 209 |
210 // obj may be a pointer to a live object. | 210 // obj may be a pointer to a live object. |
211 // Try to find the beginning of the object. | 211 // Try to find the beginning of the object. |
212 | 212 |
213 // Round down to word boundary. | 213 // Round down to word boundary. |
214 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 214 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
215 | 215 |
216 // Find bits for this word. | 216 // Find bits for this word. |
217 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; | 217 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; |
218 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1
; | 218 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
219 shift = off % wordsPerBitmapWord; | 219 shift = off % wordsPerBitmapWord; |
220 xbits = *bitp; | 220 xbits = *bitp; |
221 bits = xbits >> shift; | 221 bits = xbits >> shift; |
222 | 222 |
223 // Pointing at the beginning of a block? | 223 // Pointing at the beginning of a block? |
224 if((bits & (bitAllocated|bitBlockBoundary)) != 0) | 224 if((bits & (bitAllocated|bitBlockBoundary)) != 0) |
225 goto found; | 225 goto found; |
226 | 226 |
227 // Otherwise consult span table to find beginning. | 227 // Otherwise consult span table to find beginning. |
228 // (Manually inlined copy of MHeap_LookupMaybe.) | 228 // (Manually inlined copy of MHeap_LookupMaybe.) |
229 k = (uintptr)obj>>PageShift; | 229 k = (uintptr)obj>>PageShift; |
230 x = k; | 230 x = k; |
231 if(sizeof(void*) == 8) | 231 if(sizeof(void*) == 8) |
232 » » x -= (uintptr)runtime·mheap->arena_start>>PageShift; | 232 » » x -= (uintptr)runtime·mheap.arena_start>>PageShift; |
233 » s = runtime·mheap->map[x]; | 233 » s = runtime·mheap.map[x]; |
234 if(s == nil || k < s->start || k - s->start >= s->npages || s->state !=
MSpanInUse) | 234 if(s == nil || k < s->start || k - s->start >= s->npages || s->state !=
MSpanInUse) |
235 return false; | 235 return false; |
236 p = (byte*)((uintptr)s->start<<PageShift); | 236 p = (byte*)((uintptr)s->start<<PageShift); |
237 if(s->sizeclass == 0) { | 237 if(s->sizeclass == 0) { |
238 obj = p; | 238 obj = p; |
239 } else { | 239 } else { |
240 if((byte*)obj >= (byte*)s->limit) | 240 if((byte*)obj >= (byte*)s->limit) |
241 return false; | 241 return false; |
242 uintptr size = s->elemsize; | 242 uintptr size = s->elemsize; |
243 int32 i = ((byte*)obj - p)/size; | 243 int32 i = ((byte*)obj - p)/size; |
244 obj = p+i*size; | 244 obj = p+i*size; |
245 } | 245 } |
246 | 246 |
247 // Now that we know the object header, reload bits. | 247 // Now that we know the object header, reload bits. |
248 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; | 248 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; |
249 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1
; | 249 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
250 shift = off % wordsPerBitmapWord; | 250 shift = off % wordsPerBitmapWord; |
251 xbits = *bitp; | 251 xbits = *bitp; |
252 bits = xbits >> shift; | 252 bits = xbits >> shift; |
253 | 253 |
254 found: | 254 found: |
255 // Now we have bits, bitp, and shift correct for | 255 // Now we have bits, bitp, and shift correct for |
256 // obj pointing at the base of the object. | 256 // obj pointing at the base of the object. |
257 // Only care about allocated and not marked. | 257 // Only care about allocated and not marked. |
258 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | 258 if((bits & (bitAllocated|bitMarked)) != bitAllocated) |
259 return false; | 259 return false; |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
321 flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
, uintptr *_nobj) | 321 flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
, uintptr *_nobj) |
322 { | 322 { |
323 byte *p, *arena_start, *obj; | 323 byte *p, *arena_start, *obj; |
324 uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; | 324 uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; |
325 MSpan *s; | 325 MSpan *s; |
326 PageID k; | 326 PageID k; |
327 Obj *wp; | 327 Obj *wp; |
328 Workbuf *wbuf; | 328 Workbuf *wbuf; |
329 PtrTarget *ptrbuf_end; | 329 PtrTarget *ptrbuf_end; |
330 | 330 |
331 » arena_start = runtime·mheap->arena_start; | 331 » arena_start = runtime·mheap.arena_start; |
332 | 332 |
333 wp = *_wp; | 333 wp = *_wp; |
334 wbuf = *_wbuf; | 334 wbuf = *_wbuf; |
335 nobj = *_nobj; | 335 nobj = *_nobj; |
336 | 336 |
337 ptrbuf_end = *ptrbufpos; | 337 ptrbuf_end = *ptrbufpos; |
338 n = ptrbuf_end - ptrbuf; | 338 n = ptrbuf_end - ptrbuf; |
339 *ptrbufpos = ptrbuf; | 339 *ptrbufpos = ptrbuf; |
340 | 340 |
341 if(CollectStats) { | 341 if(CollectStats) { |
(...skipping 18 matching lines...) Expand all Loading... |
360 { | 360 { |
361 // Multi-threaded version. | 361 // Multi-threaded version. |
362 | 362 |
363 while(ptrbuf < ptrbuf_end) { | 363 while(ptrbuf < ptrbuf_end) { |
364 obj = ptrbuf->p; | 364 obj = ptrbuf->p; |
365 ti = ptrbuf->ti; | 365 ti = ptrbuf->ti; |
366 ptrbuf++; | 366 ptrbuf++; |
367 | 367 |
368 // obj belongs to interval [mheap.arena_start, mheap.are
na_used). | 368 // obj belongs to interval [mheap.arena_start, mheap.are
na_used). |
369 if(Debug > 1) { | 369 if(Debug > 1) { |
370 » » » » if(obj < runtime·mheap->arena_start || obj >= ru
ntime·mheap->arena_used) | 370 » » » » if(obj < runtime·mheap.arena_start || obj >= run
time·mheap.arena_used) |
371 runtime·throw("object is outside of mhea
p"); | 371 runtime·throw("object is outside of mhea
p"); |
372 } | 372 } |
373 | 373 |
374 // obj may be a pointer to a live object. | 374 // obj may be a pointer to a live object. |
375 // Try to find the beginning of the object. | 375 // Try to find the beginning of the object. |
376 | 376 |
377 // Round down to word boundary. | 377 // Round down to word boundary. |
378 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { | 378 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { |
379 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-
1)); | 379 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-
1)); |
380 ti = 0; | 380 ti = 0; |
(...skipping 22 matching lines...) Expand all Loading... |
403 goto found; | 403 goto found; |
404 } | 404 } |
405 } | 405 } |
406 | 406 |
407 // Otherwise consult span table to find beginning. | 407 // Otherwise consult span table to find beginning. |
408 // (Manually inlined copy of MHeap_LookupMaybe.) | 408 // (Manually inlined copy of MHeap_LookupMaybe.) |
409 k = (uintptr)obj>>PageShift; | 409 k = (uintptr)obj>>PageShift; |
410 x = k; | 410 x = k; |
411 if(sizeof(void*) == 8) | 411 if(sizeof(void*) == 8) |
412 x -= (uintptr)arena_start>>PageShift; | 412 x -= (uintptr)arena_start>>PageShift; |
413 » » » s = runtime·mheap->map[x]; | 413 » » » s = runtime·mheap.map[x]; |
414 if(s == nil || k < s->start || k - s->start >= s->npages
|| s->state != MSpanInUse) | 414 if(s == nil || k < s->start || k - s->start >= s->npages
|| s->state != MSpanInUse) |
415 continue; | 415 continue; |
416 p = (byte*)((uintptr)s->start<<PageShift); | 416 p = (byte*)((uintptr)s->start<<PageShift); |
417 if(s->sizeclass == 0) { | 417 if(s->sizeclass == 0) { |
418 obj = p; | 418 obj = p; |
419 } else { | 419 } else { |
420 if((byte*)obj >= (byte*)s->limit) | 420 if((byte*)obj >= (byte*)s->limit) |
421 continue; | 421 continue; |
422 size = s->elemsize; | 422 size = s->elemsize; |
423 int32 i = ((byte*)obj - p)/size; | 423 int32 i = ((byte*)obj - p)/size; |
(...skipping 27 matching lines...) Expand all Loading... |
451 | 451 |
452 // If object has no pointers, don't need to scan further
. | 452 // If object has no pointers, don't need to scan further
. |
453 if((bits & bitNoPointers) != 0) | 453 if((bits & bitNoPointers) != 0) |
454 continue; | 454 continue; |
455 | 455 |
456 // Ask span about size class. | 456 // Ask span about size class. |
457 // (Manually inlined copy of MHeap_Lookup.) | 457 // (Manually inlined copy of MHeap_Lookup.) |
458 x = (uintptr)obj >> PageShift; | 458 x = (uintptr)obj >> PageShift; |
459 if(sizeof(void*) == 8) | 459 if(sizeof(void*) == 8) |
460 x -= (uintptr)arena_start>>PageShift; | 460 x -= (uintptr)arena_start>>PageShift; |
461 » » » s = runtime·mheap->map[x]; | 461 » » » s = runtime·mheap.map[x]; |
462 | 462 |
463 PREFETCH(obj); | 463 PREFETCH(obj); |
464 | 464 |
465 *wp = (Obj){obj, s->elemsize, ti}; | 465 *wp = (Obj){obj, s->elemsize, ti}; |
466 wp++; | 466 wp++; |
467 nobj++; | 467 nobj++; |
468 continue_obj:; | 468 continue_obj:; |
469 } | 469 } |
470 | 470 |
471 // If another proc wants a pointer, give it some. | 471 // If another proc wants a pointer, give it some. |
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
559 checkptr(void *obj, uintptr objti) | 559 checkptr(void *obj, uintptr objti) |
560 { | 560 { |
561 uintptr *pc1, *pc2, type, tisize, i, j, x; | 561 uintptr *pc1, *pc2, type, tisize, i, j, x; |
562 byte *objstart; | 562 byte *objstart; |
563 Type *t; | 563 Type *t; |
564 MSpan *s; | 564 MSpan *s; |
565 | 565 |
566 if(!Debug) | 566 if(!Debug) |
567 runtime·throw("checkptr is debug only"); | 567 runtime·throw("checkptr is debug only"); |
568 | 568 |
569 » if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) | 569 » if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used) |
570 return; | 570 return; |
571 type = runtime·gettype(obj); | 571 type = runtime·gettype(obj); |
572 t = (Type*)(type & ~(uintptr)(PtrSize-1)); | 572 t = (Type*)(type & ~(uintptr)(PtrSize-1)); |
573 if(t == nil) | 573 if(t == nil) |
574 return; | 574 return; |
575 x = (uintptr)obj >> PageShift; | 575 x = (uintptr)obj >> PageShift; |
576 if(sizeof(void*) == 8) | 576 if(sizeof(void*) == 8) |
577 » » x -= (uintptr)(runtime·mheap->arena_start)>>PageShift; | 577 » » x -= (uintptr)(runtime·mheap.arena_start)>>PageShift; |
578 » s = runtime·mheap->map[x]; | 578 » s = runtime·mheap.map[x]; |
579 objstart = (byte*)((uintptr)s->start<<PageShift); | 579 objstart = (byte*)((uintptr)s->start<<PageShift); |
580 if(s->sizeclass != 0) { | 580 if(s->sizeclass != 0) { |
581 i = ((byte*)obj - objstart)/s->elemsize; | 581 i = ((byte*)obj - objstart)/s->elemsize; |
582 objstart += i*s->elemsize; | 582 objstart += i*s->elemsize; |
583 } | 583 } |
584 tisize = *(uintptr*)objti; | 584 tisize = *(uintptr*)objti; |
585 // Sanity check for object size: it should fit into the memory block. | 585 // Sanity check for object size: it should fit into the memory block. |
586 if((byte*)obj + tisize > objstart + s->elemsize) | 586 if((byte*)obj + tisize > objstart + s->elemsize) |
587 runtime·throw("invalid gc type info"); | 587 runtime·throw("invalid gc type info"); |
588 if(obj != objstart) | 588 if(obj != objstart) |
(...skipping 27 matching lines...) Expand all Loading... |
616 // | 616 // |
617 // wbuf: current work buffer | 617 // wbuf: current work buffer |
618 // wp: storage for next queued pointer (write pointer) | 618 // wp: storage for next queued pointer (write pointer) |
619 // nobj: number of queued objects | 619 // nobj: number of queued objects |
620 static void | 620 static void |
621 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) | 621 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) |
622 { | 622 { |
623 byte *b, *arena_start, *arena_used; | 623 byte *b, *arena_start, *arena_used; |
624 uintptr n, i, end_b, elemsize, size, ti, objti, count, type; | 624 uintptr n, i, end_b, elemsize, size, ti, objti, count, type; |
625 uintptr *pc, precise_type, nominal_size; | 625 uintptr *pc, precise_type, nominal_size; |
626 » uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_
ret; | 626 » uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_
ret, chancap; |
627 void *obj; | 627 void *obj; |
628 Type *t; | 628 Type *t; |
629 Slice *sliceptr; | 629 Slice *sliceptr; |
630 Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; | 630 Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; |
631 BufferList *scanbuffers; | 631 BufferList *scanbuffers; |
632 PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; | 632 PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; |
633 Obj *objbuf, *objbuf_end, *objbufpos; | 633 Obj *objbuf, *objbuf_end, *objbufpos; |
634 Eface *eface; | 634 Eface *eface; |
635 Iface *iface; | 635 Iface *iface; |
636 Hmap *hmap; | 636 Hmap *hmap; |
637 MapType *maptype; | 637 MapType *maptype; |
638 bool mapkey_kind, mapval_kind; | 638 bool mapkey_kind, mapval_kind; |
639 struct hash_gciter map_iter; | 639 struct hash_gciter map_iter; |
640 struct hash_gciter_data d; | 640 struct hash_gciter_data d; |
641 Hchan *chan; | 641 Hchan *chan; |
642 ChanType *chantype; | 642 ChanType *chantype; |
643 | 643 |
644 if(sizeof(Workbuf) % PageSize != 0) | 644 if(sizeof(Workbuf) % PageSize != 0) |
645 runtime·throw("scanblock: size of Workbuf is suboptimal"); | 645 runtime·throw("scanblock: size of Workbuf is suboptimal"); |
646 | 646 |
647 // Memory arena parameters. | 647 // Memory arena parameters. |
648 » arena_start = runtime·mheap->arena_start; | 648 » arena_start = runtime·mheap.arena_start; |
649 » arena_used = runtime·mheap->arena_used; | 649 » arena_used = runtime·mheap.arena_used; |
650 | 650 |
651 stack_ptr = stack+nelem(stack)-1; | 651 stack_ptr = stack+nelem(stack)-1; |
652 ········ | 652 ········ |
653 precise_type = false; | 653 precise_type = false; |
654 nominal_size = 0; | 654 nominal_size = 0; |
655 | 655 |
656 // Allocate ptrbuf | 656 // Allocate ptrbuf |
657 { | 657 { |
658 scanbuffers = &bufferList[m->helpgc]; | 658 scanbuffers = &bufferList[m->helpgc]; |
659 ptrbuf = &scanbuffers->ptrtarget[0]; | 659 ptrbuf = &scanbuffers->ptrtarget[0]; |
(...skipping 132 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
792 objti = pc[2]; | 792 objti = pc[2]; |
793 pc += 3; | 793 pc += 3; |
794 if(Debug) | 794 if(Debug) |
795 checkptr(obj, objti); | 795 checkptr(obj, objti); |
796 break; | 796 break; |
797 | 797 |
798 case GC_SLICE: | 798 case GC_SLICE: |
799 sliceptr = (Slice*)(stack_top.b + pc[1]); | 799 sliceptr = (Slice*)(stack_top.b + pc[1]); |
800 if(sliceptr->cap != 0) { | 800 if(sliceptr->cap != 0) { |
801 obj = sliceptr->array; | 801 obj = sliceptr->array; |
802 » » » » objti = pc[2] | PRECISE | LOOP; | 802 » » » » // Can't use slice element type for scanning, |
| 803 » » » » // because if it points to an array embedded |
| 804 » » » » // in the beginning of a struct, |
| 805 » » » » // we will scan the whole struct as the slice. |
| 806 » » » » // So just obtain type info from heap. |
803 } | 807 } |
804 pc += 3; | 808 pc += 3; |
805 break; | 809 break; |
806 | 810 |
807 case GC_APTR: | 811 case GC_APTR: |
808 obj = *(void**)(stack_top.b + pc[1]); | 812 obj = *(void**)(stack_top.b + pc[1]); |
809 pc += 2; | 813 pc += 2; |
810 break; | 814 break; |
811 | 815 |
812 case GC_STRING: | 816 case GC_STRING: |
(...skipping 238 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1051 } | 1055 } |
1052 pc += 3; | 1056 pc += 3; |
1053 continue; | 1057 continue; |
1054 | 1058 |
1055 case GC_CHAN: | 1059 case GC_CHAN: |
1056 // There are no heap pointers in struct Hchan, | 1060 // There are no heap pointers in struct Hchan, |
1057 // so we can ignore the leading sizeof(Hchan) bytes. | 1061 // so we can ignore the leading sizeof(Hchan) bytes. |
1058 if(!(chantype->elem->kind & KindNoPointers)) { | 1062 if(!(chantype->elem->kind & KindNoPointers)) { |
1059 // Channel's buffer follows Hchan immediately in
memory. | 1063 // Channel's buffer follows Hchan immediately in
memory. |
1060 // Size of buffer (cap(c)) is second int in the
chan struct. | 1064 // Size of buffer (cap(c)) is second int in the
chan struct. |
1061 » » » » n = ((uintgo*)chan)[1]; | 1065 » » » » chancap = ((uintgo*)chan)[1]; |
1062 » » » » if(n > 0) { | 1066 » » » » if(chancap > 0) { |
1063 // TODO(atom): split into two chunks so
that only the | 1067 // TODO(atom): split into two chunks so
that only the |
1064 // in-use part of the circular buffer is
scanned. | 1068 // in-use part of the circular buffer is
scanned. |
1065 // (Channel routines zero the unused par
t, so the current | 1069 // (Channel routines zero the unused par
t, so the current |
1066 // code does not lead to leaks, it's jus
t a little inefficient.) | 1070 // code does not lead to leaks, it's jus
t a little inefficient.) |
1067 » » » » » *objbufpos++ = (Obj){(byte*)chan+runtime
·Hchansize, n*chantype->elem->size, | 1071 » » » » » *objbufpos++ = (Obj){(byte*)chan+runtime
·Hchansize, chancap*chantype->elem->size, |
1068 (uintptr)chantype->elem->gc | PR
ECISE | LOOP}; | 1072 (uintptr)chantype->elem->gc | PR
ECISE | LOOP}; |
1069 if(objbufpos == objbuf_end) | 1073 if(objbufpos == objbuf_end) |
1070 flushobjbuf(objbuf, &objbufpos,
&wp, &wbuf, &nobj); | 1074 flushobjbuf(objbuf, &objbufpos,
&wp, &wbuf, &nobj); |
1071 } | 1075 } |
1072 } | 1076 } |
1073 if(chan_ret == nil) | 1077 if(chan_ret == nil) |
1074 goto next_block; | 1078 goto next_block; |
1075 pc = chan_ret; | 1079 pc = chan_ret; |
1076 continue; | 1080 continue; |
1077 | 1081 |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1146 b += PtrSize - off; | 1150 b += PtrSize - off; |
1147 n -= PtrSize - off; | 1151 n -= PtrSize - off; |
1148 } | 1152 } |
1149 | 1153 |
1150 vp = (void**)b; | 1154 vp = (void**)b; |
1151 n /= PtrSize; | 1155 n /= PtrSize; |
1152 for(i=0; i<n; i++) { | 1156 for(i=0; i<n; i++) { |
1153 obj = (byte*)vp[i]; | 1157 obj = (byte*)vp[i]; |
1154 | 1158 |
1155 // Words outside the arena cannot be pointers. | 1159 // Words outside the arena cannot be pointers. |
1156 » » if((byte*)obj < runtime·mheap->arena_start || (byte*)obj >= runt
ime·mheap->arena_used) | 1160 » » if((byte*)obj < runtime·mheap.arena_start || (byte*)obj >= runti
me·mheap.arena_used) |
1157 continue; | 1161 continue; |
1158 | 1162 |
1159 // Round down to word boundary. | 1163 // Round down to word boundary. |
1160 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 1164 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
1161 | 1165 |
1162 // Consult span table to find beginning. | 1166 // Consult span table to find beginning. |
1163 » » s = runtime·MHeap_LookupMaybe(runtime·mheap, obj); | 1167 » » s = runtime·MHeap_LookupMaybe(&runtime·mheap, obj); |
1164 if(s == nil) | 1168 if(s == nil) |
1165 continue; | 1169 continue; |
1166 | 1170 |
1167 p = (byte*)((uintptr)s->start<<PageShift); | 1171 p = (byte*)((uintptr)s->start<<PageShift); |
1168 size = s->elemsize; | 1172 size = s->elemsize; |
1169 if(s->sizeclass == 0) { | 1173 if(s->sizeclass == 0) { |
1170 obj = p; | 1174 obj = p; |
1171 } else { | 1175 } else { |
1172 if((byte*)obj >= (byte*)s->limit) | 1176 if((byte*)obj >= (byte*)s->limit) |
1173 continue; | 1177 continue; |
1174 int32 i = ((byte*)obj - p)/size; | 1178 int32 i = ((byte*)obj - p)/size; |
1175 obj = p+i*size; | 1179 obj = p+i*size; |
1176 } | 1180 } |
1177 | 1181 |
1178 // Now that we know the object header, reload bits. | 1182 // Now that we know the object header, reload bits. |
1179 » » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; | 1183 » » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; |
1180 » » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmap
Word - 1; | 1184 » » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapW
ord - 1; |
1181 shift = off % wordsPerBitmapWord; | 1185 shift = off % wordsPerBitmapWord; |
1182 xbits = *bitp; | 1186 xbits = *bitp; |
1183 bits = xbits >> shift; | 1187 bits = xbits >> shift; |
1184 | 1188 |
1185 // Now we have bits, bitp, and shift correct for | 1189 // Now we have bits, bitp, and shift correct for |
1186 // obj pointing at the base of the object. | 1190 // obj pointing at the base of the object. |
1187 // If not allocated or already marked, done. | 1191 // If not allocated or already marked, done. |
1188 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N
OTE: bitSpecial not bitMarked | 1192 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N
OTE: bitSpecial not bitMarked |
1189 continue; | 1193 continue; |
1190 *bitp |= bitSpecial<<shift; | 1194 *bitp |= bitSpecial<<shift; |
(...skipping 189 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1380 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); | 1384 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); |
1381 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); | 1385 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); |
1382 } | 1386 } |
1383 work.roots = new; | 1387 work.roots = new; |
1384 work.rootcap = cap; | 1388 work.rootcap = cap; |
1385 } | 1389 } |
1386 work.roots[work.nroot] = obj; | 1390 work.roots[work.nroot] = obj; |
1387 work.nroot++; | 1391 work.nroot++; |
1388 } | 1392 } |
1389 | 1393 |
1390 // Scan a stack frame. The doframe parameter is a signal that the previously | 1394 // Scan a stack frame. Normally, this scans the locals area, |
1391 // scanned activation has an unknown argument size. When *doframe is true the | 1395 // belonging to the current frame, and the arguments area, belonging |
1392 // current activation must have its entire frame scanned. Otherwise, only the | 1396 // to the calling frame. When the arguments area size is unknown, the |
1393 // locals need to be scanned. | 1397 // arguments area scanning is delayed and the doframe parameter |
| 1398 // signals that the previously scanned activation has an unknown |
| 1399 // argument size. When *doframe is true, the possible arguments area |
| 1400 // for the callee, located between the stack pointer and the bottom of |
| 1401 // the locals area, is additionally scanned. Otherwise, this area is |
| 1402 // ignored, as it must have been scanned when the callee was scanned. |
1394 static void | 1403 static void |
1395 addframeroots(Func *f, byte*, byte *sp, void *doframe) | 1404 addframeroots(Func *f, byte*, byte *sp, void *doframe) |
1396 { | 1405 { |
1397 byte *fp, *ap; | 1406 byte *fp, *ap; |
1398 uintptr outs; | 1407 uintptr outs; |
1399 int32 i, j, rem; | 1408 int32 i, j, rem; |
1400 uint32 w, b; | 1409 uint32 w, b; |
1401 | 1410 |
1402 if(thechar == '5') | 1411 if(thechar == '5') |
1403 sp += sizeof(uintptr); | 1412 sp += sizeof(uintptr); |
1404 fp = sp + f->frame; | 1413 fp = sp + f->frame; |
1405 if(f->locals == 0 || *(bool*)doframe == true) | 1414 if(f->locals == 0 || *(bool*)doframe == true) |
| 1415 // Scan the entire stack frame. |
1406 addroot((Obj){sp, f->frame - sizeof(uintptr), 0}); | 1416 addroot((Obj){sp, f->frame - sizeof(uintptr), 0}); |
1407 else if(f->locals > 0) { | 1417 else if(f->locals > 0) { |
| 1418 // Scan the locals area. |
1408 outs = f->frame - sizeof(uintptr) - f->locals; | 1419 outs = f->frame - sizeof(uintptr) - f->locals; |
1409 addroot((Obj){sp + outs, f->locals, 0}); | 1420 addroot((Obj){sp + outs, f->locals, 0}); |
1410 } | 1421 } |
1411 if(f->args > 0) { | 1422 if(f->args > 0) { |
1412 » » if(f->ptrs.len > 0) { | 1423 » » // Scan the arguments area. |
| 1424 » » if(f->ptrs.array != nil) { |
1413 ap = fp; | 1425 ap = fp; |
1414 » » » rem = f->args; | 1426 » » » rem = f->args / sizeof(uintptr); |
1415 » » » for(i = 0; i < f->ptrs.len; ++i) { | 1427 » » » for(i = 0; i < f->ptrs.len; i++) { |
1416 w = ((uint32*)f->ptrs.array)[i]; | 1428 w = ((uint32*)f->ptrs.array)[i]; |
1417 b = 1; | 1429 b = 1; |
1418 » » » » for((j = (rem < 32) ? rem : 32); j > 0; --j) { | 1430 » » » » for((j = (rem < 32) ? rem : 32); j > 0; j--) { |
1419 if(w & b) | 1431 if(w & b) |
1420 addroot((Obj){ap, sizeof(uintptr
), 0}); | 1432 addroot((Obj){ap, sizeof(uintptr
), 0}); |
1421 b <<= 1; | 1433 b <<= 1; |
1422 ap += sizeof(uintptr); | 1434 ap += sizeof(uintptr); |
1423 } | 1435 } |
1424 rem -= 32; | 1436 rem -= 32; |
1425 } | 1437 } |
1426 } else | 1438 } else |
1427 addroot((Obj){fp, f->args, 0}); | 1439 addroot((Obj){fp, f->args, 0}); |
1428 } | 1440 } |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1463 // Scanning another goroutine's stack. | 1475 // Scanning another goroutine's stack. |
1464 // The goroutine is usually asleep (the world is stopped). | 1476 // The goroutine is usually asleep (the world is stopped). |
1465 sp = (byte*)gp->sched.sp; | 1477 sp = (byte*)gp->sched.sp; |
1466 pc = gp->sched.pc; | 1478 pc = gp->sched.pc; |
1467 if(ScanStackByFrames && pc == (byte*)runtime·goexit && gp->fnsta
rt != nil) { | 1479 if(ScanStackByFrames && pc == (byte*)runtime·goexit && gp->fnsta
rt != nil) { |
1468 // The goroutine has not started. However, its incoming | 1480 // The goroutine has not started. However, its incoming |
1469 // arguments are live at the top of the stack and must | 1481 // arguments are live at the top of the stack and must |
1470 // be scanned. No other live values should be on the | 1482 // be scanned. No other live values should be on the |
1471 // stack. | 1483 // stack. |
1472 f = runtime·findfunc((uintptr)gp->fnstart->fn); | 1484 f = runtime·findfunc((uintptr)gp->fnstart->fn); |
1473 » » » if(f->args > 0) { | 1485 » » » if(f->args != 0) { |
1474 if(thechar == '5') | 1486 if(thechar == '5') |
1475 sp += sizeof(uintptr); | 1487 sp += sizeof(uintptr); |
1476 » » » » addroot((Obj){sp, f->args, 0}); | 1488 » » » » // If the size of the arguments is known |
1477 » » » } | 1489 » » » » // scan just the incoming arguments. |
| 1490 » » » » // Otherwise, scan everything between the |
| 1491 » » » » // top and the bottom of the stack. |
| 1492 » » » » if(f->args > 0) |
| 1493 » » » » » addroot((Obj){sp, f->args, 0}); |
| 1494 » » » » else |
| 1495 » » » » » addroot((Obj){sp, (byte*)stk - sp, 0});· |
| 1496 » » » }· |
1478 return; | 1497 return; |
1479 } | 1498 } |
1480 } | 1499 } |
1481 if(ScanStackByFrames) { | 1500 if(ScanStackByFrames) { |
1482 USED(stk); | 1501 USED(stk); |
1483 USED(guard); | 1502 USED(guard); |
1484 doframe = false; | 1503 doframe = false; |
1485 runtime·gentraceback(pc, sp, nil, gp, 0, nil, 0x7fffffff, addfra
meroots, &doframe); | 1504 runtime·gentraceback(pc, sp, nil, gp, 0, nil, 0x7fffffff, addfra
meroots, &doframe); |
1486 } else { | 1505 } else { |
1487 USED(pc); | 1506 USED(pc); |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1523 uint32 spanidx; | 1542 uint32 spanidx; |
1524 | 1543 |
1525 work.nroot = 0; | 1544 work.nroot = 0; |
1526 | 1545 |
1527 // data & bss | 1546 // data & bss |
1528 // TODO(atom): load balancing | 1547 // TODO(atom): load balancing |
1529 addroot((Obj){data, edata - data, (uintptr)gcdata}); | 1548 addroot((Obj){data, edata - data, (uintptr)gcdata}); |
1530 addroot((Obj){bss, ebss - bss, (uintptr)gcbss}); | 1549 addroot((Obj){bss, ebss - bss, (uintptr)gcbss}); |
1531 | 1550 |
1532 // MSpan.types | 1551 // MSpan.types |
1533 » allspans = runtime·mheap->allspans; | 1552 » allspans = runtime·mheap.allspans; |
1534 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { | 1553 » for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { |
1535 s = allspans[spanidx]; | 1554 s = allspans[spanidx]; |
1536 if(s->state == MSpanInUse) { | 1555 if(s->state == MSpanInUse) { |
1537 // The garbage collector ignores type pointers stored in
MSpan.types: | 1556 // The garbage collector ignores type pointers stored in
MSpan.types: |
1538 // - Compiler-generated types are stored outside of hea
p. | 1557 // - Compiler-generated types are stored outside of hea
p. |
1539 // - The reflect package has runtime-generated types ca
ched in its data structures. | 1558 // - The reflect package has runtime-generated types ca
ched in its data structures. |
1540 // The garbage collector relies on finding the refere
nces via that cache. | 1559 // The garbage collector relies on finding the refere
nces via that cache. |
1541 switch(s->types.compression) { | 1560 switch(s->types.compression) { |
1542 case MTypes_Empty: | 1561 case MTypes_Empty: |
1543 case MTypes_Single: | 1562 case MTypes_Single: |
1544 break; | 1563 break; |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1626 MCache *c; | 1645 MCache *c; |
1627 byte *arena_start; | 1646 byte *arena_start; |
1628 MLink head, *end; | 1647 MLink head, *end; |
1629 int32 nfree; | 1648 int32 nfree; |
1630 byte *type_data; | 1649 byte *type_data; |
1631 byte compression; | 1650 byte compression; |
1632 uintptr type_data_inc; | 1651 uintptr type_data_inc; |
1633 MSpan *s; | 1652 MSpan *s; |
1634 | 1653 |
1635 USED(&desc); | 1654 USED(&desc); |
1636 » s = runtime·mheap->allspans[idx]; | 1655 » s = runtime·mheap.allspans[idx]; |
1637 if(s->state != MSpanInUse) | 1656 if(s->state != MSpanInUse) |
1638 return; | 1657 return; |
1639 » arena_start = runtime·mheap->arena_start; | 1658 » arena_start = runtime·mheap.arena_start; |
1640 p = (byte*)(s->start << PageShift); | 1659 p = (byte*)(s->start << PageShift); |
1641 cl = s->sizeclass; | 1660 cl = s->sizeclass; |
1642 size = s->elemsize; | 1661 size = s->elemsize; |
1643 if(cl == 0) { | 1662 if(cl == 0) { |
1644 n = 1; | 1663 n = 1; |
1645 } else { | 1664 } else { |
1646 // Chunk full of small blocks. | 1665 // Chunk full of small blocks. |
1647 npages = runtime·class_to_allocnpages[cl]; | 1666 npages = runtime·class_to_allocnpages[cl]; |
1648 n = (npages << PageShift) / size; | 1667 n = (npages << PageShift) / size; |
1649 } | 1668 } |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1693 continue; | 1712 continue; |
1694 } | 1713 } |
1695 | 1714 |
1696 // Mark freed; restore block boundary bit. | 1715 // Mark freed; restore block boundary bit. |
1697 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 1716 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
1698 | 1717 |
1699 if(cl == 0) { | 1718 if(cl == 0) { |
1700 // Free large span. | 1719 // Free large span. |
1701 runtime·unmarkspan(p, 1<<PageShift); | 1720 runtime·unmarkspan(p, 1<<PageShift); |
1702 *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll; // needs
zeroing | 1721 *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll; // needs
zeroing |
1703 » » » runtime·MHeap_Free(runtime·mheap, s, 1); | 1722 » » » runtime·MHeap_Free(&runtime·mheap, s, 1); |
1704 c->local_alloc -= size; | 1723 c->local_alloc -= size; |
1705 c->local_nfree++; | 1724 c->local_nfree++; |
1706 } else { | 1725 } else { |
1707 // Free small object. | 1726 // Free small object. |
1708 switch(compression) { | 1727 switch(compression) { |
1709 case MTypes_Words: | 1728 case MTypes_Words: |
1710 *(uintptr*)type_data = 0; | 1729 *(uintptr*)type_data = 0; |
1711 break; | 1730 break; |
1712 case MTypes_Bytes: | 1731 case MTypes_Bytes: |
1713 *(byte*)type_data = 0; | 1732 *(byte*)type_data = 0; |
1714 break; | 1733 break; |
1715 } | 1734 } |
1716 if(size > sizeof(uintptr)) | 1735 if(size > sizeof(uintptr)) |
1717 ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll
; // mark as "needs to be zeroed" | 1736 ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll
; // mark as "needs to be zeroed" |
1718 ························ | 1737 ························ |
1719 end->next = (MLink*)p; | 1738 end->next = (MLink*)p; |
1720 end = (MLink*)p; | 1739 end = (MLink*)p; |
1721 nfree++; | 1740 nfree++; |
1722 } | 1741 } |
1723 } | 1742 } |
1724 | 1743 |
1725 if(nfree) { | 1744 if(nfree) { |
1726 c->local_by_size[cl].nfree += nfree; | 1745 c->local_by_size[cl].nfree += nfree; |
1727 c->local_alloc -= size * nfree; | 1746 c->local_alloc -= size * nfree; |
1728 c->local_nfree += nfree; | 1747 c->local_nfree += nfree; |
1729 c->local_cachealloc -= nfree * size; | 1748 c->local_cachealloc -= nfree * size; |
1730 c->local_objects -= nfree; | 1749 c->local_objects -= nfree; |
1731 » » runtime·MCentral_FreeSpan(&runtime·mheap->central[cl], s, nfree,
head.next, end); | 1750 » » runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, nfree,
head.next, end); |
1732 } | 1751 } |
1733 } | 1752 } |
1734 | 1753 |
1735 static void | 1754 static void |
1736 dumpspan(uint32 idx) | 1755 dumpspan(uint32 idx) |
1737 { | 1756 { |
1738 int32 sizeclass, n, npages, i, column; | 1757 int32 sizeclass, n, npages, i, column; |
1739 uintptr size; | 1758 uintptr size; |
1740 byte *p; | 1759 byte *p; |
1741 byte *arena_start; | 1760 byte *arena_start; |
1742 MSpan *s; | 1761 MSpan *s; |
1743 bool allocated, special; | 1762 bool allocated, special; |
1744 | 1763 |
1745 » s = runtime·mheap->allspans[idx]; | 1764 » s = runtime·mheap.allspans[idx]; |
1746 if(s->state != MSpanInUse) | 1765 if(s->state != MSpanInUse) |
1747 return; | 1766 return; |
1748 » arena_start = runtime·mheap->arena_start; | 1767 » arena_start = runtime·mheap.arena_start; |
1749 p = (byte*)(s->start << PageShift); | 1768 p = (byte*)(s->start << PageShift); |
1750 sizeclass = s->sizeclass; | 1769 sizeclass = s->sizeclass; |
1751 size = s->elemsize; | 1770 size = s->elemsize; |
1752 if(sizeclass == 0) { | 1771 if(sizeclass == 0) { |
1753 n = 1; | 1772 n = 1; |
1754 } else { | 1773 } else { |
1755 npages = runtime·class_to_allocnpages[sizeclass]; | 1774 npages = runtime·class_to_allocnpages[sizeclass]; |
1756 n = (npages << PageShift) / size; | 1775 n = (npages << PageShift) / size; |
1757 } | 1776 } |
1758 ········ | 1777 ········ |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1796 } | 1815 } |
1797 runtime·printf("\n"); | 1816 runtime·printf("\n"); |
1798 } | 1817 } |
1799 | 1818 |
1800 // A debugging function to dump the contents of memory | 1819 // A debugging function to dump the contents of memory |
1801 void | 1820 void |
1802 runtime·memorydump(void) | 1821 runtime·memorydump(void) |
1803 { | 1822 { |
1804 uint32 spanidx; | 1823 uint32 spanidx; |
1805 | 1824 |
1806 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { | 1825 » for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { |
1807 dumpspan(spanidx); | 1826 dumpspan(spanidx); |
1808 } | 1827 } |
1809 } | 1828 } |
1810 | 1829 |
1811 void | 1830 void |
1812 runtime·gchelper(void) | 1831 runtime·gchelper(void) |
1813 { | 1832 { |
1814 gchelperstart(); | 1833 gchelperstart(); |
1815 | 1834 |
1816 // parallel mark for over gc roots | 1835 // parallel mark for over gc roots |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1865 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) | 1884 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) |
1866 dst[i] += src[i]; | 1885 dst[i] += src[i]; |
1867 runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats))
; | 1886 runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats))
; |
1868 } | 1887 } |
1869 } | 1888 } |
1870 for(pp=runtime·allp; p=*pp; pp++) { | 1889 for(pp=runtime·allp; p=*pp; pp++) { |
1871 c = p->mcache; | 1890 c = p->mcache; |
1872 if(c==nil) | 1891 if(c==nil) |
1873 continue; | 1892 continue; |
1874 runtime·purgecachedstats(c); | 1893 runtime·purgecachedstats(c); |
1875 for(i=0; i<nelem(c->local_by_size); i++) { | |
1876 mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc
; | |
1877 c->local_by_size[i].nmalloc = 0; | |
1878 mstats.by_size[i].nfree += c->local_by_size[i].nfree; | |
1879 c->local_by_size[i].nfree = 0; | |
1880 } | |
1881 } | 1894 } |
1882 mstats.stacks_inuse = stacks_inuse; | 1895 mstats.stacks_inuse = stacks_inuse; |
1883 } | 1896 } |
1884 | 1897 |
1885 // Structure of arguments passed to function gc(). | 1898 // Structure of arguments passed to function gc(). |
1886 // This allows the arguments to be passed via reflect·call. | 1899 // This allows the arguments to be passed via reflect·call. |
1887 struct gc_args | 1900 struct gc_args |
1888 { | 1901 { |
1889 int32 force; | 1902 int32 force; |
1890 }; | 1903 }; |
(...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2003 runtime·gc_itab_ptr(&eface); | 2016 runtime·gc_itab_ptr(&eface); |
2004 itabtype = ((PtrType*)eface.type)->elem; | 2017 itabtype = ((PtrType*)eface.type)->elem; |
2005 } | 2018 } |
2006 | 2019 |
2007 work.nwait = 0; | 2020 work.nwait = 0; |
2008 work.ndone = 0; | 2021 work.ndone = 0; |
2009 work.debugmarkdone = 0; | 2022 work.debugmarkdone = 0; |
2010 work.nproc = runtime·gcprocs(); | 2023 work.nproc = runtime·gcprocs(); |
2011 addroots(); | 2024 addroots(); |
2012 runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma
rkroot); | 2025 runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma
rkroot); |
2013 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap->nspan, nil
, true, sweepspan); | 2026 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil,
true, sweepspan); |
2014 if(work.nproc > 1) { | 2027 if(work.nproc > 1) { |
2015 runtime·noteclear(&work.alldone); | 2028 runtime·noteclear(&work.alldone); |
2016 runtime·helpgc(work.nproc); | 2029 runtime·helpgc(work.nproc); |
2017 } | 2030 } |
2018 | 2031 |
2019 t1 = runtime·nanotime(); | 2032 t1 = runtime·nanotime(); |
2020 | 2033 |
2021 gchelperstart(); | 2034 gchelperstart(); |
2022 runtime·parfordo(work.markfor); | 2035 runtime·parfordo(work.markfor); |
2023 scanblock(nil, nil, 0, true); | 2036 scanblock(nil, nil, 0, true); |
(...skipping 13 matching lines...) Expand all Loading... |
2037 runtime·notesleep(&work.alldone); | 2050 runtime·notesleep(&work.alldone); |
2038 | 2051 |
2039 cachestats(&stats); | 2052 cachestats(&stats); |
2040 | 2053 |
2041 stats.nprocyield += work.sweepfor->nprocyield; | 2054 stats.nprocyield += work.sweepfor->nprocyield; |
2042 stats.nosyield += work.sweepfor->nosyield; | 2055 stats.nosyield += work.sweepfor->nosyield; |
2043 stats.nsleep += work.sweepfor->nsleep; | 2056 stats.nsleep += work.sweepfor->nsleep; |
2044 | 2057 |
2045 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; | 2058 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; |
2046 m->gcing = 0; | 2059 m->gcing = 0; |
2047 | |
2048 if(finq != nil) { | |
2049 m->locks++; // disable gc during the mallocs in newproc | |
2050 // kick off or wake up goroutine to run queued finalizers | |
2051 if(fing == nil) | |
2052 fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc
); | |
2053 else if(fingwait) { | |
2054 fingwait = 0; | |
2055 runtime·ready(fing); | |
2056 } | |
2057 m->locks--; | |
2058 } | |
2059 | 2060 |
2060 heap1 = mstats.heap_alloc; | 2061 heap1 = mstats.heap_alloc; |
2061 obj1 = mstats.nmalloc - mstats.nfree; | 2062 obj1 = mstats.nmalloc - mstats.nfree; |
2062 | 2063 |
2063 t4 = runtime·nanotime(); | 2064 t4 = runtime·nanotime(); |
2064 mstats.last_gc = t4; | 2065 mstats.last_gc = t4; |
2065 mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0; | 2066 mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0; |
2066 mstats.pause_total_ns += t4 - t0; | 2067 mstats.pause_total_ns += t4 - t0; |
2067 mstats.numgc++; | 2068 mstats.numgc++; |
2068 if(mstats.debuggc) | 2069 if(mstats.debuggc) |
(...skipping 28 matching lines...) Expand all Loading... |
2097 runtime·printf("\ttotal:\t%D\n", ninstr); | 2098 runtime·printf("\ttotal:\t%D\n", ninstr); |
2098 | 2099 |
2099 runtime·printf("putempty: %D, getfull: %D\n", gcstats.pu
tempty, gcstats.getfull); | 2100 runtime·printf("putempty: %D, getfull: %D\n", gcstats.pu
tempty, gcstats.getfull); |
2100 } | 2101 } |
2101 } | 2102 } |
2102 | 2103 |
2103 runtime·MProf_GC(); | 2104 runtime·MProf_GC(); |
2104 runtime·semrelease(&runtime·worldsema); | 2105 runtime·semrelease(&runtime·worldsema); |
2105 runtime·starttheworld(); | 2106 runtime·starttheworld(); |
2106 | 2107 |
2107 » // give the queued finalizers, if any, a chance to run | 2108 » if(finq != nil) { |
2108 » if(finq != nil) | 2109 » » runtime·lock(&finlock); |
| 2110 » » // kick off or wake up goroutine to run queued finalizers |
| 2111 » » if(fing == nil) |
| 2112 » » » fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc
); |
| 2113 » » else if(fingwait) { |
| 2114 » » » fingwait = 0; |
| 2115 » » » runtime·ready(fing); |
| 2116 » » } |
| 2117 » » runtime·unlock(&finlock); |
| 2118 » » // give the queued finalizers, if any, a chance to run |
2109 runtime·gosched(); | 2119 runtime·gosched(); |
| 2120 } |
2110 } | 2121 } |
2111 | 2122 |
2112 void | 2123 void |
2113 runtime·ReadMemStats(MStats *stats) | 2124 runtime·ReadMemStats(MStats *stats) |
2114 { | 2125 { |
2115 // Have to acquire worldsema to stop the world, | 2126 // Have to acquire worldsema to stop the world, |
2116 // because stoptheworld can only be used by | 2127 // because stoptheworld can only be used by |
2117 // one goroutine at a time, and there might be | 2128 // one goroutine at a time, and there might be |
2118 // a pending garbage collection already calling it. | 2129 // a pending garbage collection already calling it. |
2119 runtime·semacquire(&runtime·worldsema); | 2130 runtime·semacquire(&runtime·worldsema); |
(...skipping 11 matching lines...) Expand all Loading... |
2131 { | 2142 { |
2132 uint64 *p; | 2143 uint64 *p; |
2133 uint32 i, n; | 2144 uint32 i, n; |
2134 | 2145 |
2135 // Calling code in runtime/debug should make the slice large enough. | 2146 // Calling code in runtime/debug should make the slice large enough. |
2136 if(pauses->cap < nelem(mstats.pause_ns)+3) | 2147 if(pauses->cap < nelem(mstats.pause_ns)+3) |
2137 runtime·throw("runtime: short slice passed to readGCStats"); | 2148 runtime·throw("runtime: short slice passed to readGCStats"); |
2138 | 2149 |
2139 // Pass back: pauses, last gc (absolute time), number of gc, total pause
ns. | 2150 // Pass back: pauses, last gc (absolute time), number of gc, total pause
ns. |
2140 p = (uint64*)pauses->array; | 2151 p = (uint64*)pauses->array; |
2141 » runtime·lock(runtime·mheap); | 2152 » runtime·lock(&runtime·mheap); |
2142 n = mstats.numgc; | 2153 n = mstats.numgc; |
2143 if(n > nelem(mstats.pause_ns)) | 2154 if(n > nelem(mstats.pause_ns)) |
2144 n = nelem(mstats.pause_ns); | 2155 n = nelem(mstats.pause_ns); |
2145 ········ | 2156 ········ |
2146 // The pause buffer is circular. The most recent pause is at | 2157 // The pause buffer is circular. The most recent pause is at |
2147 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward | 2158 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward |
2148 // from there to go back farther in time. We deliver the times | 2159 // from there to go back farther in time. We deliver the times |
2149 // most recent first (in p[0]). | 2160 // most recent first (in p[0]). |
2150 for(i=0; i<n; i++) | 2161 for(i=0; i<n; i++) |
2151 p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)
]; | 2162 p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)
]; |
2152 | 2163 |
2153 p[n] = mstats.last_gc; | 2164 p[n] = mstats.last_gc; |
2154 p[n+1] = mstats.numgc; | 2165 p[n+1] = mstats.numgc; |
2155 p[n+2] = mstats.pause_total_ns;· | 2166 p[n+2] = mstats.pause_total_ns;· |
2156 » runtime·unlock(runtime·mheap); | 2167 » runtime·unlock(&runtime·mheap); |
2157 pauses->len = n+3; | 2168 pauses->len = n+3; |
2158 } | 2169 } |
2159 | 2170 |
2160 void | 2171 void |
2161 runtime∕debug·setGCPercent(intgo in, intgo out) | 2172 runtime∕debug·setGCPercent(intgo in, intgo out) |
2162 { | 2173 { |
2163 » runtime·lock(runtime·mheap); | 2174 » runtime·lock(&runtime·mheap); |
2164 if(gcpercent == GcpercentUnknown) | 2175 if(gcpercent == GcpercentUnknown) |
2165 gcpercent = readgogc(); | 2176 gcpercent = readgogc(); |
2166 out = gcpercent; | 2177 out = gcpercent; |
2167 if(in < 0) | 2178 if(in < 0) |
2168 in = -1; | 2179 in = -1; |
2169 gcpercent = in; | 2180 gcpercent = in; |
2170 » runtime·unlock(runtime·mheap); | 2181 » runtime·unlock(&runtime·mheap); |
2171 FLUSH(&out); | 2182 FLUSH(&out); |
2172 } | 2183 } |
2173 | 2184 |
2174 static void | 2185 static void |
2175 gchelperstart(void) | 2186 gchelperstart(void) |
2176 { | 2187 { |
2177 if(m->helpgc < 0 || m->helpgc >= MaxGcproc) | 2188 if(m->helpgc < 0 || m->helpgc >= MaxGcproc) |
2178 runtime·throw("gchelperstart: bad m->helpgc"); | 2189 runtime·throw("gchelperstart: bad m->helpgc"); |
2179 if(runtime·xchg(&bufferList[m->helpgc].busy, 1)) | 2190 if(runtime·xchg(&bufferList[m->helpgc].busy, 1)) |
2180 runtime·throw("gchelperstart: already busy"); | 2191 runtime·throw("gchelperstart: already busy"); |
2181 } | 2192 } |
2182 | 2193 |
2183 static void | 2194 static void |
2184 runfinq(void) | 2195 runfinq(void) |
2185 { | 2196 { |
2186 Finalizer *f; | 2197 Finalizer *f; |
2187 FinBlock *fb, *next; | 2198 FinBlock *fb, *next; |
2188 byte *frame; | 2199 byte *frame; |
2189 uint32 framesz, framecap, i; | 2200 uint32 framesz, framecap, i; |
2190 | 2201 |
2191 frame = nil; | 2202 frame = nil; |
2192 framecap = 0; | 2203 framecap = 0; |
2193 for(;;) { | 2204 for(;;) { |
2194 » » // There's no need for a lock in this section | 2205 » » runtime·lock(&finlock); |
2195 » » // because it only conflicts with the garbage | |
2196 » » // collector, and the garbage collector only | |
2197 » » // runs when everyone else is stopped, and | |
2198 » » // runfinq only stops at the gosched() or | |
2199 » » // during the calls in the for loop. | |
2200 fb = finq; | 2206 fb = finq; |
2201 finq = nil; | 2207 finq = nil; |
2202 if(fb == nil) { | 2208 if(fb == nil) { |
2203 fingwait = 1; | 2209 fingwait = 1; |
2204 » » » runtime·park(nil, nil, "finalizer wait"); | 2210 » » » runtime·park(runtime·unlock, &finlock, "finalizer wait")
; |
2205 continue; | 2211 continue; |
2206 } | 2212 } |
| 2213 runtime·unlock(&finlock); |
2207 if(raceenabled) | 2214 if(raceenabled) |
2208 runtime·racefingo(); | 2215 runtime·racefingo(); |
2209 for(; fb; fb=next) { | 2216 for(; fb; fb=next) { |
2210 next = fb->next; | 2217 next = fb->next; |
2211 for(i=0; i<fb->cnt; i++) { | 2218 for(i=0; i<fb->cnt; i++) { |
2212 f = &fb->fin[i]; | 2219 f = &fb->fin[i]; |
2213 framesz = sizeof(uintptr) + f->nret; | 2220 framesz = sizeof(uintptr) + f->nret; |
2214 if(framecap < framesz) { | 2221 if(framecap < framesz) { |
2215 runtime·free(frame); | 2222 runtime·free(frame); |
2216 frame = runtime·mal(framesz); | 2223 frame = runtime·mal(framesz); |
(...skipping 15 matching lines...) Expand all Loading... |
2232 // mark the block at v of size n as allocated. | 2239 // mark the block at v of size n as allocated. |
2233 // If noptr is true, mark it as having no pointers. | 2240 // If noptr is true, mark it as having no pointers. |
2234 void | 2241 void |
2235 runtime·markallocated(void *v, uintptr n, bool noptr) | 2242 runtime·markallocated(void *v, uintptr n, bool noptr) |
2236 { | 2243 { |
2237 uintptr *b, obits, bits, off, shift; | 2244 uintptr *b, obits, bits, off, shift; |
2238 | 2245 |
2239 if(0) | 2246 if(0) |
2240 runtime·printf("markallocated %p+%p\n", v, n); | 2247 runtime·printf("markallocated %p+%p\n", v, n); |
2241 | 2248 |
2242 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) | 2249 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) |
2243 runtime·throw("markallocated: bad pointer"); | 2250 runtime·throw("markallocated: bad pointer"); |
2244 | 2251 |
2245 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse
t | 2252 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset |
2246 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2253 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2247 shift = off % wordsPerBitmapWord; | 2254 shift = off % wordsPerBitmapWord; |
2248 | 2255 |
2249 for(;;) { | 2256 for(;;) { |
2250 obits = *b; | 2257 obits = *b; |
2251 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); | 2258 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); |
2252 if(noptr) | 2259 if(noptr) |
2253 bits |= bitNoPointers<<shift; | 2260 bits |= bitNoPointers<<shift; |
2254 if(runtime·singleproc) { | 2261 if(runtime·singleproc) { |
2255 *b = bits; | 2262 *b = bits; |
2256 break; | 2263 break; |
2257 } else { | 2264 } else { |
2258 // more than one goroutine is potentially running: use a
tomic op | 2265 // more than one goroutine is potentially running: use a
tomic op |
2259 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2266 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
2260 break; | 2267 break; |
2261 } | 2268 } |
2262 } | 2269 } |
2263 } | 2270 } |
2264 | 2271 |
2265 // mark the block at v of size n as freed. | 2272 // mark the block at v of size n as freed. |
2266 void | 2273 void |
2267 runtime·markfreed(void *v, uintptr n) | 2274 runtime·markfreed(void *v, uintptr n) |
2268 { | 2275 { |
2269 uintptr *b, obits, bits, off, shift; | 2276 uintptr *b, obits, bits, off, shift; |
2270 | 2277 |
2271 if(0) | 2278 if(0) |
2272 runtime·printf("markallocated %p+%p\n", v, n); | 2279 runtime·printf("markallocated %p+%p\n", v, n); |
2273 | 2280 |
2274 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) | 2281 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) |
2275 runtime·throw("markallocated: bad pointer"); | 2282 runtime·throw("markallocated: bad pointer"); |
2276 | 2283 |
2277 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse
t | 2284 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset |
2278 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2285 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2279 shift = off % wordsPerBitmapWord; | 2286 shift = off % wordsPerBitmapWord; |
2280 | 2287 |
2281 for(;;) { | 2288 for(;;) { |
2282 obits = *b; | 2289 obits = *b; |
2283 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2290 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
2284 if(runtime·singleproc) { | 2291 if(runtime·singleproc) { |
2285 *b = bits; | 2292 *b = bits; |
2286 break; | 2293 break; |
2287 } else { | 2294 } else { |
2288 // more than one goroutine is potentially running: use a
tomic op | 2295 // more than one goroutine is potentially running: use a
tomic op |
2289 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2296 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
2290 break; | 2297 break; |
2291 } | 2298 } |
2292 } | 2299 } |
2293 } | 2300 } |
2294 | 2301 |
2295 // check that the block at v of size n is marked freed. | 2302 // check that the block at v of size n is marked freed. |
2296 void | 2303 void |
2297 runtime·checkfreed(void *v, uintptr n) | 2304 runtime·checkfreed(void *v, uintptr n) |
2298 { | 2305 { |
2299 uintptr *b, bits, off, shift; | 2306 uintptr *b, bits, off, shift; |
2300 | 2307 |
2301 if(!runtime·checking) | 2308 if(!runtime·checking) |
2302 return; | 2309 return; |
2303 | 2310 |
2304 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) | 2311 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) |
2305 return; // not allocated, so okay | 2312 return; // not allocated, so okay |
2306 | 2313 |
2307 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse
t | 2314 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset |
2308 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2315 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2309 shift = off % wordsPerBitmapWord; | 2316 shift = off % wordsPerBitmapWord; |
2310 | 2317 |
2311 bits = *b>>shift; | 2318 bits = *b>>shift; |
2312 if((bits & bitAllocated) != 0) { | 2319 if((bits & bitAllocated) != 0) { |
2313 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", | 2320 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", |
2314 v, n, off, bits & bitMask); | 2321 v, n, off, bits & bitMask); |
2315 runtime·throw("checkfreed: not freed"); | 2322 runtime·throw("checkfreed: not freed"); |
2316 } | 2323 } |
2317 } | 2324 } |
2318 | 2325 |
2319 // mark the span of memory at v as having n blocks of the given size. | 2326 // mark the span of memory at v as having n blocks of the given size. |
2320 // if leftover is true, there is left over space at the end of the span. | 2327 // if leftover is true, there is left over space at the end of the span. |
2321 void | 2328 void |
2322 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) | 2329 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) |
2323 { | 2330 { |
2324 uintptr *b, off, shift; | 2331 uintptr *b, off, shift; |
2325 byte *p; | 2332 byte *p; |
2326 | 2333 |
2327 » if((byte*)v+size*n > (byte*)runtime·mheap->arena_used || (byte*)v < runt
ime·mheap->arena_start) | 2334 » if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runti
me·mheap.arena_start) |
2328 runtime·throw("markspan: bad pointer"); | 2335 runtime·throw("markspan: bad pointer"); |
2329 | 2336 |
2330 p = v; | 2337 p = v; |
2331 if(leftover) // mark a boundary just past end of last block too | 2338 if(leftover) // mark a boundary just past end of last block too |
2332 n++; | 2339 n++; |
2333 for(; n-- > 0; p += size) { | 2340 for(; n-- > 0; p += size) { |
2334 // Okay to use non-atomic ops here, because we control | 2341 // Okay to use non-atomic ops here, because we control |
2335 // the entire span, and each bitmap word has bits for only | 2342 // the entire span, and each bitmap word has bits for only |
2336 // one span, so no other goroutines are changing these | 2343 // one span, so no other goroutines are changing these |
2337 // bitmap words. | 2344 // bitmap words. |
2338 » » off = (uintptr*)p - (uintptr*)runtime·mheap->arena_start; // wo
rd offset | 2345 » » off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // wor
d offset |
2339 » » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWor
d - 1; | 2346 » » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord
- 1; |
2340 shift = off % wordsPerBitmapWord; | 2347 shift = off % wordsPerBitmapWord; |
2341 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2348 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
2342 } | 2349 } |
2343 } | 2350 } |
2344 | 2351 |
2345 // unmark the span of memory at v of length n bytes. | 2352 // unmark the span of memory at v of length n bytes. |
2346 void | 2353 void |
2347 runtime·unmarkspan(void *v, uintptr n) | 2354 runtime·unmarkspan(void *v, uintptr n) |
2348 { | 2355 { |
2349 uintptr *p, *b, off; | 2356 uintptr *p, *b, off; |
2350 | 2357 |
2351 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) | 2358 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) |
2352 runtime·throw("markspan: bad pointer"); | 2359 runtime·throw("markspan: bad pointer"); |
2353 | 2360 |
2354 p = v; | 2361 p = v; |
2355 » off = p - (uintptr*)runtime·mheap->arena_start; // word offset | 2362 » off = p - (uintptr*)runtime·mheap.arena_start; // word offset |
2356 if(off % wordsPerBitmapWord != 0) | 2363 if(off % wordsPerBitmapWord != 0) |
2357 runtime·throw("markspan: unaligned pointer"); | 2364 runtime·throw("markspan: unaligned pointer"); |
2358 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2365 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2359 n /= PtrSize; | 2366 n /= PtrSize; |
2360 if(n%wordsPerBitmapWord != 0) | 2367 if(n%wordsPerBitmapWord != 0) |
2361 runtime·throw("unmarkspan: unaligned length"); | 2368 runtime·throw("unmarkspan: unaligned length"); |
2362 // Okay to use non-atomic ops here, because we control | 2369 // Okay to use non-atomic ops here, because we control |
2363 // the entire span, and each bitmap word has bits for only | 2370 // the entire span, and each bitmap word has bits for only |
2364 // one span, so no other goroutines are changing these | 2371 // one span, so no other goroutines are changing these |
2365 // bitmap words. | 2372 // bitmap words. |
2366 n /= wordsPerBitmapWord; | 2373 n /= wordsPerBitmapWord; |
2367 while(n-- > 0) | 2374 while(n-- > 0) |
2368 *b-- = 0; | 2375 *b-- = 0; |
2369 } | 2376 } |
2370 | 2377 |
2371 bool | 2378 bool |
2372 runtime·blockspecial(void *v) | 2379 runtime·blockspecial(void *v) |
2373 { | 2380 { |
2374 uintptr *b, off, shift; | 2381 uintptr *b, off, shift; |
2375 | 2382 |
2376 if(DebugMark) | 2383 if(DebugMark) |
2377 return true; | 2384 return true; |
2378 | 2385 |
2379 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; | 2386 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; |
2380 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2387 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2381 shift = off % wordsPerBitmapWord; | 2388 shift = off % wordsPerBitmapWord; |
2382 | 2389 |
2383 return (*b & (bitSpecial<<shift)) != 0; | 2390 return (*b & (bitSpecial<<shift)) != 0; |
2384 } | 2391 } |
2385 | 2392 |
2386 void | 2393 void |
2387 runtime·setblockspecial(void *v, bool s) | 2394 runtime·setblockspecial(void *v, bool s) |
2388 { | 2395 { |
2389 uintptr *b, off, shift, bits, obits; | 2396 uintptr *b, off, shift, bits, obits; |
2390 | 2397 |
2391 if(DebugMark) | 2398 if(DebugMark) |
2392 return; | 2399 return; |
2393 | 2400 |
2394 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; | 2401 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; |
2395 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2402 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2396 shift = off % wordsPerBitmapWord; | 2403 shift = off % wordsPerBitmapWord; |
2397 | 2404 |
2398 for(;;) { | 2405 for(;;) { |
2399 obits = *b; | 2406 obits = *b; |
2400 if(s) | 2407 if(s) |
2401 bits = obits | (bitSpecial<<shift); | 2408 bits = obits | (bitSpecial<<shift); |
2402 else | 2409 else |
2403 bits = obits & ~(bitSpecial<<shift); | 2410 bits = obits & ~(bitSpecial<<shift); |
2404 if(runtime·singleproc) { | 2411 if(runtime·singleproc) { |
2405 *b = bits; | 2412 *b = bits; |
(...skipping 11 matching lines...) Expand all Loading... |
2417 { | 2424 { |
2418 // Caller has added extra mappings to the arena. | 2425 // Caller has added extra mappings to the arena. |
2419 // Add extra mappings of bitmap words as needed. | 2426 // Add extra mappings of bitmap words as needed. |
2420 // We allocate extra bitmap pieces in chunks of bitmapChunk. | 2427 // We allocate extra bitmap pieces in chunks of bitmapChunk. |
2421 enum { | 2428 enum { |
2422 bitmapChunk = 8192 | 2429 bitmapChunk = 8192 |
2423 }; | 2430 }; |
2424 uintptr n; | 2431 uintptr n; |
2425 | 2432 |
2426 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; | 2433 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; |
2427 » n = (n+bitmapChunk-1) & ~(bitmapChunk-1); | 2434 » n = ROUND(n, bitmapChunk); |
2428 if(h->bitmap_mapped >= n) | 2435 if(h->bitmap_mapped >= n) |
2429 return; | 2436 return; |
2430 | 2437 |
2431 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); | 2438 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); |
2432 h->bitmap_mapped = n; | 2439 h->bitmap_mapped = n; |
2433 } | 2440 } |
LEFT | RIGHT |