Left: | ||
Right: |
LEFT | RIGHT |
---|---|
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Garbage collector. | 5 // Garbage collector. |
6 | 6 |
7 #include "runtime.h" | 7 #include "runtime.h" |
8 #include "arch_GOARCH.h" | 8 #include "arch_GOARCH.h" |
9 #include "malloc.h" | 9 #include "malloc.h" |
10 #include "stack.h" | 10 #include "stack.h" |
(...skipping 186 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
197 // This function doesn't append the object to any buffer. | 197 // This function doesn't append the object to any buffer. |
198 static bool | 198 static bool |
199 markonly(void *obj) | 199 markonly(void *obj) |
200 { | 200 { |
201 byte *p; | 201 byte *p; |
202 uintptr *bitp, bits, shift, x, xbits, off; | 202 uintptr *bitp, bits, shift, x, xbits, off; |
203 MSpan *s; | 203 MSpan *s; |
204 PageID k; | 204 PageID k; |
205 | 205 |
206 // Words outside the arena cannot be pointers. | 206 // Words outside the arena cannot be pointers. |
207 » if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) | 207 » if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used) |
208 return false; | 208 return false; |
209 | 209 |
210 // obj may be a pointer to a live object. | 210 // obj may be a pointer to a live object. |
211 // Try to find the beginning of the object. | 211 // Try to find the beginning of the object. |
212 | 212 |
213 // Round down to word boundary. | 213 // Round down to word boundary. |
214 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 214 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
215 | 215 |
216 // Find bits for this word. | 216 // Find bits for this word. |
217 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; | 217 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; |
218 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1 ; | 218 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
219 shift = off % wordsPerBitmapWord; | 219 shift = off % wordsPerBitmapWord; |
220 xbits = *bitp; | 220 xbits = *bitp; |
221 bits = xbits >> shift; | 221 bits = xbits >> shift; |
222 | 222 |
223 // Pointing at the beginning of a block? | 223 // Pointing at the beginning of a block? |
224 if((bits & (bitAllocated|bitBlockBoundary)) != 0) | 224 if((bits & (bitAllocated|bitBlockBoundary)) != 0) |
225 goto found; | 225 goto found; |
226 | 226 |
227 // Otherwise consult span table to find beginning. | 227 // Otherwise consult span table to find beginning. |
228 // (Manually inlined copy of MHeap_LookupMaybe.) | 228 // (Manually inlined copy of MHeap_LookupMaybe.) |
229 k = (uintptr)obj>>PageShift; | 229 k = (uintptr)obj>>PageShift; |
230 x = k; | 230 x = k; |
231 if(sizeof(void*) == 8) | 231 if(sizeof(void*) == 8) |
232 » » x -= (uintptr)runtime·mheap->arena_start>>PageShift; | 232 » » x -= (uintptr)runtime·mheap.arena_start>>PageShift; |
233 » s = runtime·mheap->map[x]; | 233 » s = runtime·mheap.map[x]; |
234 if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) | 234 if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) |
235 return false; | 235 return false; |
236 p = (byte*)((uintptr)s->start<<PageShift); | 236 p = (byte*)((uintptr)s->start<<PageShift); |
237 if(s->sizeclass == 0) { | 237 if(s->sizeclass == 0) { |
238 obj = p; | 238 obj = p; |
239 } else { | 239 } else { |
240 if((byte*)obj >= (byte*)s->limit) | 240 if((byte*)obj >= (byte*)s->limit) |
241 return false; | 241 return false; |
242 uintptr size = s->elemsize; | 242 uintptr size = s->elemsize; |
243 int32 i = ((byte*)obj - p)/size; | 243 int32 i = ((byte*)obj - p)/size; |
244 obj = p+i*size; | 244 obj = p+i*size; |
245 } | 245 } |
246 | 246 |
247 // Now that we know the object header, reload bits. | 247 // Now that we know the object header, reload bits. |
248 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; | 248 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; |
249 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1 ; | 249 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
250 shift = off % wordsPerBitmapWord; | 250 shift = off % wordsPerBitmapWord; |
251 xbits = *bitp; | 251 xbits = *bitp; |
252 bits = xbits >> shift; | 252 bits = xbits >> shift; |
253 | 253 |
254 found: | 254 found: |
255 // Now we have bits, bitp, and shift correct for | 255 // Now we have bits, bitp, and shift correct for |
256 // obj pointing at the base of the object. | 256 // obj pointing at the base of the object. |
257 // Only care about allocated and not marked. | 257 // Only care about allocated and not marked. |
258 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | 258 if((bits & (bitAllocated|bitMarked)) != bitAllocated) |
259 return false; | 259 return false; |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
321 flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf , uintptr *_nobj) | 321 flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf , uintptr *_nobj) |
322 { | 322 { |
323 byte *p, *arena_start, *obj; | 323 byte *p, *arena_start, *obj; |
324 uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; | 324 uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; |
325 MSpan *s; | 325 MSpan *s; |
326 PageID k; | 326 PageID k; |
327 Obj *wp; | 327 Obj *wp; |
328 Workbuf *wbuf; | 328 Workbuf *wbuf; |
329 PtrTarget *ptrbuf_end; | 329 PtrTarget *ptrbuf_end; |
330 | 330 |
331 » arena_start = runtime·mheap->arena_start; | 331 » arena_start = runtime·mheap.arena_start; |
332 | 332 |
333 wp = *_wp; | 333 wp = *_wp; |
334 wbuf = *_wbuf; | 334 wbuf = *_wbuf; |
335 nobj = *_nobj; | 335 nobj = *_nobj; |
336 | 336 |
337 ptrbuf_end = *ptrbufpos; | 337 ptrbuf_end = *ptrbufpos; |
338 n = ptrbuf_end - ptrbuf; | 338 n = ptrbuf_end - ptrbuf; |
339 *ptrbufpos = ptrbuf; | 339 *ptrbufpos = ptrbuf; |
340 | 340 |
341 if(CollectStats) { | 341 if(CollectStats) { |
(...skipping 18 matching lines...) Expand all Loading... | |
360 { | 360 { |
361 // Multi-threaded version. | 361 // Multi-threaded version. |
362 | 362 |
363 while(ptrbuf < ptrbuf_end) { | 363 while(ptrbuf < ptrbuf_end) { |
364 obj = ptrbuf->p; | 364 obj = ptrbuf->p; |
365 ti = ptrbuf->ti; | 365 ti = ptrbuf->ti; |
366 ptrbuf++; | 366 ptrbuf++; |
367 | 367 |
368 // obj belongs to interval [mheap.arena_start, mheap.are na_used). | 368 // obj belongs to interval [mheap.arena_start, mheap.are na_used). |
369 if(Debug > 1) { | 369 if(Debug > 1) { |
370 » » » » if(obj < runtime·mheap->arena_start || obj >= ru ntime·mheap->arena_used) | 370 » » » » if(obj < runtime·mheap.arena_start || obj >= run time·mheap.arena_used) |
371 runtime·throw("object is outside of mhea p"); | 371 runtime·throw("object is outside of mhea p"); |
372 } | 372 } |
373 | 373 |
374 // obj may be a pointer to a live object. | 374 // obj may be a pointer to a live object. |
375 // Try to find the beginning of the object. | 375 // Try to find the beginning of the object. |
376 | 376 |
377 // Round down to word boundary. | 377 // Round down to word boundary. |
378 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { | 378 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { |
379 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize- 1)); | 379 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize- 1)); |
380 ti = 0; | 380 ti = 0; |
(...skipping 22 matching lines...) Expand all Loading... | |
403 goto found; | 403 goto found; |
404 } | 404 } |
405 } | 405 } |
406 | 406 |
407 // Otherwise consult span table to find beginning. | 407 // Otherwise consult span table to find beginning. |
408 // (Manually inlined copy of MHeap_LookupMaybe.) | 408 // (Manually inlined copy of MHeap_LookupMaybe.) |
409 k = (uintptr)obj>>PageShift; | 409 k = (uintptr)obj>>PageShift; |
410 x = k; | 410 x = k; |
411 if(sizeof(void*) == 8) | 411 if(sizeof(void*) == 8) |
412 x -= (uintptr)arena_start>>PageShift; | 412 x -= (uintptr)arena_start>>PageShift; |
413 » » » s = runtime·mheap->map[x]; | 413 » » » s = runtime·mheap.map[x]; |
414 if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) | 414 if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) |
415 continue; | 415 continue; |
416 p = (byte*)((uintptr)s->start<<PageShift); | 416 p = (byte*)((uintptr)s->start<<PageShift); |
417 if(s->sizeclass == 0) { | 417 if(s->sizeclass == 0) { |
418 obj = p; | 418 obj = p; |
419 } else { | 419 } else { |
420 if((byte*)obj >= (byte*)s->limit) | 420 if((byte*)obj >= (byte*)s->limit) |
421 continue; | 421 continue; |
422 size = s->elemsize; | 422 size = s->elemsize; |
423 int32 i = ((byte*)obj - p)/size; | 423 int32 i = ((byte*)obj - p)/size; |
(...skipping 27 matching lines...) Expand all Loading... | |
451 | 451 |
452 // If object has no pointers, don't need to scan further . | 452 // If object has no pointers, don't need to scan further . |
453 if((bits & bitNoPointers) != 0) | 453 if((bits & bitNoPointers) != 0) |
454 continue; | 454 continue; |
455 | 455 |
456 // Ask span about size class. | 456 // Ask span about size class. |
457 // (Manually inlined copy of MHeap_Lookup.) | 457 // (Manually inlined copy of MHeap_Lookup.) |
458 x = (uintptr)obj >> PageShift; | 458 x = (uintptr)obj >> PageShift; |
459 if(sizeof(void*) == 8) | 459 if(sizeof(void*) == 8) |
460 x -= (uintptr)arena_start>>PageShift; | 460 x -= (uintptr)arena_start>>PageShift; |
461 » » » s = runtime·mheap->map[x]; | 461 » » » s = runtime·mheap.map[x]; |
462 | 462 |
463 PREFETCH(obj); | 463 PREFETCH(obj); |
464 | 464 |
465 *wp = (Obj){obj, s->elemsize, ti}; | 465 *wp = (Obj){obj, s->elemsize, ti}; |
466 wp++; | 466 wp++; |
467 nobj++; | 467 nobj++; |
468 continue_obj:; | 468 continue_obj:; |
469 } | 469 } |
470 | 470 |
471 // If another proc wants a pointer, give it some. | 471 // If another proc wants a pointer, give it some. |
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
559 checkptr(void *obj, uintptr objti) | 559 checkptr(void *obj, uintptr objti) |
560 { | 560 { |
561 uintptr *pc1, *pc2, type, tisize, i, j, x; | 561 uintptr *pc1, *pc2, type, tisize, i, j, x; |
562 byte *objstart; | 562 byte *objstart; |
563 Type *t; | 563 Type *t; |
564 MSpan *s; | 564 MSpan *s; |
565 | 565 |
566 if(!Debug) | 566 if(!Debug) |
567 runtime·throw("checkptr is debug only"); | 567 runtime·throw("checkptr is debug only"); |
568 | 568 |
569 » if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) | 569 » if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used) |
570 return; | 570 return; |
571 type = runtime·gettype(obj); | 571 type = runtime·gettype(obj); |
572 t = (Type*)(type & ~(uintptr)(PtrSize-1)); | 572 t = (Type*)(type & ~(uintptr)(PtrSize-1)); |
573 if(t == nil) | 573 if(t == nil) |
574 return; | 574 return; |
575 x = (uintptr)obj >> PageShift; | 575 x = (uintptr)obj >> PageShift; |
576 if(sizeof(void*) == 8) | 576 if(sizeof(void*) == 8) |
577 » » x -= (uintptr)(runtime·mheap->arena_start)>>PageShift; | 577 » » x -= (uintptr)(runtime·mheap.arena_start)>>PageShift; |
578 » s = runtime·mheap->map[x]; | 578 » s = runtime·mheap.map[x]; |
579 objstart = (byte*)((uintptr)s->start<<PageShift); | 579 objstart = (byte*)((uintptr)s->start<<PageShift); |
580 if(s->sizeclass != 0) { | 580 if(s->sizeclass != 0) { |
581 i = ((byte*)obj - objstart)/s->elemsize; | 581 i = ((byte*)obj - objstart)/s->elemsize; |
582 objstart += i*s->elemsize; | 582 objstart += i*s->elemsize; |
583 } | 583 } |
584 tisize = *(uintptr*)objti; | 584 tisize = *(uintptr*)objti; |
585 // Sanity check for object size: it should fit into the memory block. | 585 // Sanity check for object size: it should fit into the memory block. |
586 if((byte*)obj + tisize > objstart + s->elemsize) | 586 if((byte*)obj + tisize > objstart + s->elemsize) |
587 runtime·throw("invalid gc type info"); | 587 runtime·throw("invalid gc type info"); |
588 if(obj != objstart) | 588 if(obj != objstart) |
(...skipping 27 matching lines...) Expand all Loading... | |
616 // | 616 // |
617 // wbuf: current work buffer | 617 // wbuf: current work buffer |
618 // wp: storage for next queued pointer (write pointer) | 618 // wp: storage for next queued pointer (write pointer) |
619 // nobj: number of queued objects | 619 // nobj: number of queued objects |
620 static void | 620 static void |
621 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) | 621 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) |
622 { | 622 { |
623 byte *b, *arena_start, *arena_used; | 623 byte *b, *arena_start, *arena_used; |
624 uintptr n, i, end_b, elemsize, size, ti, objti, count, type; | 624 uintptr n, i, end_b, elemsize, size, ti, objti, count, type; |
625 uintptr *pc, precise_type, nominal_size; | 625 uintptr *pc, precise_type, nominal_size; |
626 » uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_ ret; | 626 » uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_ ret, chancap; |
627 void *obj; | 627 void *obj; |
628 Type *t; | 628 Type *t; |
629 Slice *sliceptr; | 629 Slice *sliceptr; |
630 Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; | 630 Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; |
631 BufferList *scanbuffers; | 631 BufferList *scanbuffers; |
632 PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; | 632 PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; |
633 Obj *objbuf, *objbuf_end, *objbufpos; | 633 Obj *objbuf, *objbuf_end, *objbufpos; |
634 Eface *eface; | 634 Eface *eface; |
635 Iface *iface; | 635 Iface *iface; |
636 Hmap *hmap; | 636 Hmap *hmap; |
637 MapType *maptype; | 637 MapType *maptype; |
638 bool mapkey_kind, mapval_kind; | 638 bool mapkey_kind, mapval_kind; |
639 struct hash_gciter map_iter; | 639 struct hash_gciter map_iter; |
640 struct hash_gciter_data d; | 640 struct hash_gciter_data d; |
641 Hchan *chan; | 641 Hchan *chan; |
642 ChanType *chantype; | 642 ChanType *chantype; |
643 | 643 |
644 if(sizeof(Workbuf) % PageSize != 0) | 644 if(sizeof(Workbuf) % PageSize != 0) |
645 runtime·throw("scanblock: size of Workbuf is suboptimal"); | 645 runtime·throw("scanblock: size of Workbuf is suboptimal"); |
646 | 646 |
647 // Memory arena parameters. | 647 // Memory arena parameters. |
648 » arena_start = runtime·mheap->arena_start; | 648 » arena_start = runtime·mheap.arena_start; |
649 » arena_used = runtime·mheap->arena_used; | 649 » arena_used = runtime·mheap.arena_used; |
650 | 650 |
651 stack_ptr = stack+nelem(stack)-1; | 651 stack_ptr = stack+nelem(stack)-1; |
652 ········ | 652 ········ |
653 precise_type = false; | 653 precise_type = false; |
654 nominal_size = 0; | 654 nominal_size = 0; |
655 | 655 |
656 // Allocate ptrbuf | 656 // Allocate ptrbuf |
657 { | 657 { |
658 scanbuffers = &bufferList[m->helpgc]; | 658 scanbuffers = &bufferList[m->helpgc]; |
659 ptrbuf = &scanbuffers->ptrtarget[0]; | 659 ptrbuf = &scanbuffers->ptrtarget[0]; |
(...skipping 395 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1055 } | 1055 } |
1056 pc += 3; | 1056 pc += 3; |
1057 continue; | 1057 continue; |
1058 | 1058 |
1059 case GC_CHAN: | 1059 case GC_CHAN: |
1060 // There are no heap pointers in struct Hchan, | 1060 // There are no heap pointers in struct Hchan, |
1061 // so we can ignore the leading sizeof(Hchan) bytes. | 1061 // so we can ignore the leading sizeof(Hchan) bytes. |
1062 if(!(chantype->elem->kind & KindNoPointers)) { | 1062 if(!(chantype->elem->kind & KindNoPointers)) { |
1063 // Channel's buffer follows Hchan immediately in memory. | 1063 // Channel's buffer follows Hchan immediately in memory. |
1064 // Size of buffer (cap(c)) is second int in the chan struct. | 1064 // Size of buffer (cap(c)) is second int in the chan struct. |
1065 » » » » n = ((uintgo*)chan)[1]; | 1065 » » » » chancap = ((uintgo*)chan)[1]; |
1066 » » » » if(n > 0) { | 1066 » » » » if(chancap > 0) { |
1067 // TODO(atom): split into two chunks so that only the | 1067 // TODO(atom): split into two chunks so that only the |
1068 // in-use part of the circular buffer is scanned. | 1068 // in-use part of the circular buffer is scanned. |
1069 // (Channel routines zero the unused par t, so the current | 1069 // (Channel routines zero the unused par t, so the current |
1070 // code does not lead to leaks, it's jus t a little inefficient.) | 1070 // code does not lead to leaks, it's jus t a little inefficient.) |
1071 » » » » » *objbufpos++ = (Obj){(byte*)chan+runtime ·Hchansize, n*chantype->elem->size, | 1071 » » » » » *objbufpos++ = (Obj){(byte*)chan+runtime ·Hchansize, chancap*chantype->elem->size, |
1072 (uintptr)chantype->elem->gc | PR ECISE | LOOP}; | 1072 (uintptr)chantype->elem->gc | PR ECISE | LOOP}; |
1073 if(objbufpos == objbuf_end) | 1073 if(objbufpos == objbuf_end) |
1074 flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); | 1074 flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); |
1075 } | 1075 } |
1076 } | 1076 } |
1077 if(chan_ret == nil) | 1077 if(chan_ret == nil) |
1078 goto next_block; | 1078 goto next_block; |
1079 pc = chan_ret; | 1079 pc = chan_ret; |
1080 continue; | 1080 continue; |
1081 | 1081 |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1150 b += PtrSize - off; | 1150 b += PtrSize - off; |
1151 n -= PtrSize - off; | 1151 n -= PtrSize - off; |
1152 } | 1152 } |
1153 | 1153 |
1154 vp = (void**)b; | 1154 vp = (void**)b; |
1155 n /= PtrSize; | 1155 n /= PtrSize; |
1156 for(i=0; i<n; i++) { | 1156 for(i=0; i<n; i++) { |
1157 obj = (byte*)vp[i]; | 1157 obj = (byte*)vp[i]; |
1158 | 1158 |
1159 // Words outside the arena cannot be pointers. | 1159 // Words outside the arena cannot be pointers. |
1160 » » if((byte*)obj < runtime·mheap->arena_start || (byte*)obj >= runt ime·mheap->arena_used) | 1160 » » if((byte*)obj < runtime·mheap.arena_start || (byte*)obj >= runti me·mheap.arena_used) |
1161 continue; | 1161 continue; |
1162 | 1162 |
1163 // Round down to word boundary. | 1163 // Round down to word boundary. |
1164 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 1164 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
1165 | 1165 |
1166 // Consult span table to find beginning. | 1166 // Consult span table to find beginning. |
1167 » » s = runtime·MHeap_LookupMaybe(runtime·mheap, obj); | 1167 » » s = runtime·MHeap_LookupMaybe(&runtime·mheap, obj); |
1168 if(s == nil) | 1168 if(s == nil) |
1169 continue; | 1169 continue; |
1170 | 1170 |
1171 p = (byte*)((uintptr)s->start<<PageShift); | 1171 p = (byte*)((uintptr)s->start<<PageShift); |
1172 size = s->elemsize; | 1172 size = s->elemsize; |
1173 if(s->sizeclass == 0) { | 1173 if(s->sizeclass == 0) { |
1174 obj = p; | 1174 obj = p; |
1175 } else { | 1175 } else { |
1176 if((byte*)obj >= (byte*)s->limit) | 1176 if((byte*)obj >= (byte*)s->limit) |
1177 continue; | 1177 continue; |
1178 int32 i = ((byte*)obj - p)/size; | 1178 int32 i = ((byte*)obj - p)/size; |
1179 obj = p+i*size; | 1179 obj = p+i*size; |
1180 } | 1180 } |
1181 | 1181 |
1182 // Now that we know the object header, reload bits. | 1182 // Now that we know the object header, reload bits. |
1183 » » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; | 1183 » » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; |
1184 » » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmap Word - 1; | 1184 » » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapW ord - 1; |
1185 shift = off % wordsPerBitmapWord; | 1185 shift = off % wordsPerBitmapWord; |
1186 xbits = *bitp; | 1186 xbits = *bitp; |
1187 bits = xbits >> shift; | 1187 bits = xbits >> shift; |
1188 | 1188 |
1189 // Now we have bits, bitp, and shift correct for | 1189 // Now we have bits, bitp, and shift correct for |
1190 // obj pointing at the base of the object. | 1190 // obj pointing at the base of the object. |
1191 // If not allocated or already marked, done. | 1191 // If not allocated or already marked, done. |
1192 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N OTE: bitSpecial not bitMarked | 1192 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N OTE: bitSpecial not bitMarked |
1193 continue; | 1193 continue; |
1194 *bitp |= bitSpecial<<shift; | 1194 *bitp |= bitSpecial<<shift; |
(...skipping 189 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1384 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj )); | 1384 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj )); |
1385 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); | 1385 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); |
1386 } | 1386 } |
1387 work.roots = new; | 1387 work.roots = new; |
1388 work.rootcap = cap; | 1388 work.rootcap = cap; |
1389 } | 1389 } |
1390 work.roots[work.nroot] = obj; | 1390 work.roots[work.nroot] = obj; |
1391 work.nroot++; | 1391 work.nroot++; |
1392 } | 1392 } |
1393 | 1393 |
1394 // Scan a stack frame. The doframe parameter is a signal that the previously | 1394 // Scan a stack frame. Normally, this scans the locals area, |
1395 // scanned activation has an unknown argument size. When *doframe is true the | 1395 // belonging to the current frame, and the arguments area, belonging |
1396 // current activation must have its entire frame scanned. Otherwise, only the | 1396 // to the calling frame. When the arguments area size is unknown, the |
1397 // locals need to be scanned. | 1397 // arguments area scanning is delayed and the doframe parameter |
1398 // signals that the previously scanned activation has an unknown | |
1399 // argument size. When *doframe is true, the possible arguments area | |
1400 // for the callee, located between the stack pointer and the bottom of | |
1401 // the locals area, is additionally scanned. Otherwise, this area is | |
1402 // ignored, as it must have been scanned when the callee was scanned. | |
1398 static void | 1403 static void |
1399 addframeroots(Func *f, byte*, byte *sp, void *doframe) | 1404 addframeroots(Func *f, byte*, byte *sp, void *doframe) |
1400 { | 1405 { |
1401 byte *fp, *ap; | 1406 byte *fp, *ap; |
1402 uintptr outs; | 1407 uintptr outs; |
1403 int32 i, j, rem; | 1408 int32 i, j, rem; |
1404 uint32 w, b; | 1409 uint32 w, b; |
1405 | 1410 |
1406 if(thechar == '5') | 1411 if(thechar == '5') |
1407 sp += sizeof(uintptr); | 1412 sp += sizeof(uintptr); |
1408 fp = sp + f->frame; | 1413 fp = sp + f->frame; |
1409 if(f->locals == 0 || *(bool*)doframe == true) | 1414 if(f->locals == 0 || *(bool*)doframe == true) |
1410 // Scan the entire stack frame. | 1415 // Scan the entire stack frame. |
1411 addroot((Obj){sp, f->frame - sizeof(uintptr), 0}); | 1416 addroot((Obj){sp, f->frame - sizeof(uintptr), 0}); |
1412 else if(f->locals > 0) { | 1417 else if(f->locals > 0) { |
1413 // Scan the locals area. | 1418 // Scan the locals area. |
1414 outs = f->frame - sizeof(uintptr) - f->locals; | 1419 outs = f->frame - sizeof(uintptr) - f->locals; |
1415 addroot((Obj){sp + outs, f->locals, 0}); | 1420 addroot((Obj){sp + outs, f->locals, 0}); |
1416 } | 1421 } |
1417 if(f->args > 0) { | 1422 if(f->args > 0) { |
1418 // Scan the arguments area. | 1423 // Scan the arguments area. |
iant
2013/05/28 21:23:36
Should we have a check here that f->ptrs.len is la
cshapiro1
2013/05/29 00:29:20
These values are produced by the symbol table cons
| |
1419 if(f->ptrs.array != nil) { | 1424 if(f->ptrs.array != nil) { |
1420 ap = fp; | 1425 ap = fp; |
1421 rem = f->args / sizeof(uintptr); | 1426 rem = f->args / sizeof(uintptr); |
1422 for(i = 0; i < f->ptrs.len; i++) { | 1427 for(i = 0; i < f->ptrs.len; i++) { |
1423 w = ((uint32*)f->ptrs.array)[i]; | 1428 w = ((uint32*)f->ptrs.array)[i]; |
1424 b = 1; | 1429 b = 1; |
1425 for((j = (rem < 32) ? rem : 32); j > 0; j--) { | 1430 for((j = (rem < 32) ? rem : 32); j > 0; j--) { |
1426 if(w & b) | 1431 if(w & b) |
1427 addroot((Obj){ap, sizeof(uintptr ), 0}); | 1432 addroot((Obj){ap, sizeof(uintptr ), 0}); |
1428 b <<= 1; | 1433 b <<= 1; |
(...skipping 108 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1537 uint32 spanidx; | 1542 uint32 spanidx; |
1538 | 1543 |
1539 work.nroot = 0; | 1544 work.nroot = 0; |
1540 | 1545 |
1541 // data & bss | 1546 // data & bss |
1542 // TODO(atom): load balancing | 1547 // TODO(atom): load balancing |
1543 addroot((Obj){data, edata - data, (uintptr)gcdata}); | 1548 addroot((Obj){data, edata - data, (uintptr)gcdata}); |
1544 addroot((Obj){bss, ebss - bss, (uintptr)gcbss}); | 1549 addroot((Obj){bss, ebss - bss, (uintptr)gcbss}); |
1545 | 1550 |
1546 // MSpan.types | 1551 // MSpan.types |
1547 » allspans = runtime·mheap->allspans; | 1552 » allspans = runtime·mheap.allspans; |
1548 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { | 1553 » for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { |
1549 s = allspans[spanidx]; | 1554 s = allspans[spanidx]; |
1550 if(s->state == MSpanInUse) { | 1555 if(s->state == MSpanInUse) { |
1551 // The garbage collector ignores type pointers stored in MSpan.types: | 1556 // The garbage collector ignores type pointers stored in MSpan.types: |
1552 // - Compiler-generated types are stored outside of hea p. | 1557 // - Compiler-generated types are stored outside of hea p. |
1553 // - The reflect package has runtime-generated types ca ched in its data structures. | 1558 // - The reflect package has runtime-generated types ca ched in its data structures. |
1554 // The garbage collector relies on finding the refere nces via that cache. | 1559 // The garbage collector relies on finding the refere nces via that cache. |
1555 switch(s->types.compression) { | 1560 switch(s->types.compression) { |
1556 case MTypes_Empty: | 1561 case MTypes_Empty: |
1557 case MTypes_Single: | 1562 case MTypes_Single: |
1558 break; | 1563 break; |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1640 MCache *c; | 1645 MCache *c; |
1641 byte *arena_start; | 1646 byte *arena_start; |
1642 MLink head, *end; | 1647 MLink head, *end; |
1643 int32 nfree; | 1648 int32 nfree; |
1644 byte *type_data; | 1649 byte *type_data; |
1645 byte compression; | 1650 byte compression; |
1646 uintptr type_data_inc; | 1651 uintptr type_data_inc; |
1647 MSpan *s; | 1652 MSpan *s; |
1648 | 1653 |
1649 USED(&desc); | 1654 USED(&desc); |
1650 » s = runtime·mheap->allspans[idx]; | 1655 » s = runtime·mheap.allspans[idx]; |
1651 if(s->state != MSpanInUse) | 1656 if(s->state != MSpanInUse) |
1652 return; | 1657 return; |
1653 » arena_start = runtime·mheap->arena_start; | 1658 » arena_start = runtime·mheap.arena_start; |
1654 p = (byte*)(s->start << PageShift); | 1659 p = (byte*)(s->start << PageShift); |
1655 cl = s->sizeclass; | 1660 cl = s->sizeclass; |
1656 size = s->elemsize; | 1661 size = s->elemsize; |
1657 if(cl == 0) { | 1662 if(cl == 0) { |
1658 n = 1; | 1663 n = 1; |
1659 } else { | 1664 } else { |
1660 // Chunk full of small blocks. | 1665 // Chunk full of small blocks. |
1661 npages = runtime·class_to_allocnpages[cl]; | 1666 npages = runtime·class_to_allocnpages[cl]; |
1662 n = (npages << PageShift) / size; | 1667 n = (npages << PageShift) / size; |
1663 } | 1668 } |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1707 continue; | 1712 continue; |
1708 } | 1713 } |
1709 | 1714 |
1710 // Mark freed; restore block boundary bit. | 1715 // Mark freed; restore block boundary bit. |
1711 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 1716 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
1712 | 1717 |
1713 if(cl == 0) { | 1718 if(cl == 0) { |
1714 // Free large span. | 1719 // Free large span. |
1715 runtime·unmarkspan(p, 1<<PageShift); | 1720 runtime·unmarkspan(p, 1<<PageShift); |
1716 *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll; // needs zeroing | 1721 *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll; // needs zeroing |
1717 » » » runtime·MHeap_Free(runtime·mheap, s, 1); | 1722 » » » runtime·MHeap_Free(&runtime·mheap, s, 1); |
1718 c->local_alloc -= size; | 1723 c->local_alloc -= size; |
1719 c->local_nfree++; | 1724 c->local_nfree++; |
1720 } else { | 1725 } else { |
1721 // Free small object. | 1726 // Free small object. |
1722 switch(compression) { | 1727 switch(compression) { |
1723 case MTypes_Words: | 1728 case MTypes_Words: |
1724 *(uintptr*)type_data = 0; | 1729 *(uintptr*)type_data = 0; |
1725 break; | 1730 break; |
1726 case MTypes_Bytes: | 1731 case MTypes_Bytes: |
1727 *(byte*)type_data = 0; | 1732 *(byte*)type_data = 0; |
1728 break; | 1733 break; |
1729 } | 1734 } |
1730 if(size > sizeof(uintptr)) | 1735 if(size > sizeof(uintptr)) |
1731 ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll ; // mark as "needs to be zeroed" | 1736 ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll ; // mark as "needs to be zeroed" |
1732 ························ | 1737 ························ |
1733 end->next = (MLink*)p; | 1738 end->next = (MLink*)p; |
1734 end = (MLink*)p; | 1739 end = (MLink*)p; |
1735 nfree++; | 1740 nfree++; |
1736 } | 1741 } |
1737 } | 1742 } |
1738 | 1743 |
1739 if(nfree) { | 1744 if(nfree) { |
1740 c->local_by_size[cl].nfree += nfree; | 1745 c->local_by_size[cl].nfree += nfree; |
1741 c->local_alloc -= size * nfree; | 1746 c->local_alloc -= size * nfree; |
1742 c->local_nfree += nfree; | 1747 c->local_nfree += nfree; |
1743 c->local_cachealloc -= nfree * size; | 1748 c->local_cachealloc -= nfree * size; |
1744 c->local_objects -= nfree; | 1749 c->local_objects -= nfree; |
1745 » » runtime·MCentral_FreeSpan(&runtime·mheap->central[cl], s, nfree, head.next, end); | 1750 » » runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, nfree, head.next, end); |
1746 } | 1751 } |
1747 } | 1752 } |
1748 | 1753 |
1749 static void | 1754 static void |
1750 dumpspan(uint32 idx) | 1755 dumpspan(uint32 idx) |
1751 { | 1756 { |
1752 int32 sizeclass, n, npages, i, column; | 1757 int32 sizeclass, n, npages, i, column; |
1753 uintptr size; | 1758 uintptr size; |
1754 byte *p; | 1759 byte *p; |
1755 byte *arena_start; | 1760 byte *arena_start; |
1756 MSpan *s; | 1761 MSpan *s; |
1757 bool allocated, special; | 1762 bool allocated, special; |
1758 | 1763 |
1759 » s = runtime·mheap->allspans[idx]; | 1764 » s = runtime·mheap.allspans[idx]; |
1760 if(s->state != MSpanInUse) | 1765 if(s->state != MSpanInUse) |
1761 return; | 1766 return; |
1762 » arena_start = runtime·mheap->arena_start; | 1767 » arena_start = runtime·mheap.arena_start; |
1763 p = (byte*)(s->start << PageShift); | 1768 p = (byte*)(s->start << PageShift); |
1764 sizeclass = s->sizeclass; | 1769 sizeclass = s->sizeclass; |
1765 size = s->elemsize; | 1770 size = s->elemsize; |
1766 if(sizeclass == 0) { | 1771 if(sizeclass == 0) { |
1767 n = 1; | 1772 n = 1; |
1768 } else { | 1773 } else { |
1769 npages = runtime·class_to_allocnpages[sizeclass]; | 1774 npages = runtime·class_to_allocnpages[sizeclass]; |
1770 n = (npages << PageShift) / size; | 1775 n = (npages << PageShift) / size; |
1771 } | 1776 } |
1772 ········ | 1777 ········ |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1810 } | 1815 } |
1811 runtime·printf("\n"); | 1816 runtime·printf("\n"); |
1812 } | 1817 } |
1813 | 1818 |
1814 // A debugging function to dump the contents of memory | 1819 // A debugging function to dump the contents of memory |
1815 void | 1820 void |
1816 runtime·memorydump(void) | 1821 runtime·memorydump(void) |
1817 { | 1822 { |
1818 uint32 spanidx; | 1823 uint32 spanidx; |
1819 | 1824 |
1820 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { | 1825 » for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { |
1821 dumpspan(spanidx); | 1826 dumpspan(spanidx); |
1822 } | 1827 } |
1823 } | 1828 } |
1824 | 1829 |
1825 void | 1830 void |
1826 runtime·gchelper(void) | 1831 runtime·gchelper(void) |
1827 { | 1832 { |
1828 gchelperstart(); | 1833 gchelperstart(); |
1829 | 1834 |
1830 // parallel mark for over gc roots | 1835 // parallel mark for over gc roots |
(...skipping 180 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2011 runtime·gc_itab_ptr(&eface); | 2016 runtime·gc_itab_ptr(&eface); |
2012 itabtype = ((PtrType*)eface.type)->elem; | 2017 itabtype = ((PtrType*)eface.type)->elem; |
2013 } | 2018 } |
2014 | 2019 |
2015 work.nwait = 0; | 2020 work.nwait = 0; |
2016 work.ndone = 0; | 2021 work.ndone = 0; |
2017 work.debugmarkdone = 0; | 2022 work.debugmarkdone = 0; |
2018 work.nproc = runtime·gcprocs(); | 2023 work.nproc = runtime·gcprocs(); |
2019 addroots(); | 2024 addroots(); |
2020 runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma rkroot); | 2025 runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma rkroot); |
2021 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap->nspan, nil , true, sweepspan); | 2026 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil, true, sweepspan); |
2022 if(work.nproc > 1) { | 2027 if(work.nproc > 1) { |
2023 runtime·noteclear(&work.alldone); | 2028 runtime·noteclear(&work.alldone); |
2024 runtime·helpgc(work.nproc); | 2029 runtime·helpgc(work.nproc); |
2025 } | 2030 } |
2026 | 2031 |
2027 t1 = runtime·nanotime(); | 2032 t1 = runtime·nanotime(); |
2028 | 2033 |
2029 gchelperstart(); | 2034 gchelperstart(); |
2030 runtime·parfordo(work.markfor); | 2035 runtime·parfordo(work.markfor); |
2031 scanblock(nil, nil, 0, true); | 2036 scanblock(nil, nil, 0, true); |
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2137 { | 2142 { |
2138 uint64 *p; | 2143 uint64 *p; |
2139 uint32 i, n; | 2144 uint32 i, n; |
2140 | 2145 |
2141 // Calling code in runtime/debug should make the slice large enough. | 2146 // Calling code in runtime/debug should make the slice large enough. |
2142 if(pauses->cap < nelem(mstats.pause_ns)+3) | 2147 if(pauses->cap < nelem(mstats.pause_ns)+3) |
2143 runtime·throw("runtime: short slice passed to readGCStats"); | 2148 runtime·throw("runtime: short slice passed to readGCStats"); |
2144 | 2149 |
2145 // Pass back: pauses, last gc (absolute time), number of gc, total pause ns. | 2150 // Pass back: pauses, last gc (absolute time), number of gc, total pause ns. |
2146 p = (uint64*)pauses->array; | 2151 p = (uint64*)pauses->array; |
2147 » runtime·lock(runtime·mheap); | 2152 » runtime·lock(&runtime·mheap); |
2148 n = mstats.numgc; | 2153 n = mstats.numgc; |
2149 if(n > nelem(mstats.pause_ns)) | 2154 if(n > nelem(mstats.pause_ns)) |
2150 n = nelem(mstats.pause_ns); | 2155 n = nelem(mstats.pause_ns); |
2151 ········ | 2156 ········ |
2152 // The pause buffer is circular. The most recent pause is at | 2157 // The pause buffer is circular. The most recent pause is at |
2153 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward | 2158 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward |
2154 // from there to go back farther in time. We deliver the times | 2159 // from there to go back farther in time. We deliver the times |
2155 // most recent first (in p[0]). | 2160 // most recent first (in p[0]). |
2156 for(i=0; i<n; i++) | 2161 for(i=0; i<n; i++) |
2157 p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns) ]; | 2162 p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns) ]; |
2158 | 2163 |
2159 p[n] = mstats.last_gc; | 2164 p[n] = mstats.last_gc; |
2160 p[n+1] = mstats.numgc; | 2165 p[n+1] = mstats.numgc; |
2161 p[n+2] = mstats.pause_total_ns;· | 2166 p[n+2] = mstats.pause_total_ns;· |
2162 » runtime·unlock(runtime·mheap); | 2167 » runtime·unlock(&runtime·mheap); |
2163 pauses->len = n+3; | 2168 pauses->len = n+3; |
2164 } | 2169 } |
2165 | 2170 |
2166 void | 2171 void |
2167 runtime∕debug·setGCPercent(intgo in, intgo out) | 2172 runtime∕debug·setGCPercent(intgo in, intgo out) |
2168 { | 2173 { |
2169 » runtime·lock(runtime·mheap); | 2174 » runtime·lock(&runtime·mheap); |
2170 if(gcpercent == GcpercentUnknown) | 2175 if(gcpercent == GcpercentUnknown) |
2171 gcpercent = readgogc(); | 2176 gcpercent = readgogc(); |
2172 out = gcpercent; | 2177 out = gcpercent; |
2173 if(in < 0) | 2178 if(in < 0) |
2174 in = -1; | 2179 in = -1; |
2175 gcpercent = in; | 2180 gcpercent = in; |
2176 » runtime·unlock(runtime·mheap); | 2181 » runtime·unlock(&runtime·mheap); |
2177 FLUSH(&out); | 2182 FLUSH(&out); |
2178 } | 2183 } |
2179 | 2184 |
2180 static void | 2185 static void |
2181 gchelperstart(void) | 2186 gchelperstart(void) |
2182 { | 2187 { |
2183 if(m->helpgc < 0 || m->helpgc >= MaxGcproc) | 2188 if(m->helpgc < 0 || m->helpgc >= MaxGcproc) |
2184 runtime·throw("gchelperstart: bad m->helpgc"); | 2189 runtime·throw("gchelperstart: bad m->helpgc"); |
2185 if(runtime·xchg(&bufferList[m->helpgc].busy, 1)) | 2190 if(runtime·xchg(&bufferList[m->helpgc].busy, 1)) |
2186 runtime·throw("gchelperstart: already busy"); | 2191 runtime·throw("gchelperstart: already busy"); |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2234 // mark the block at v of size n as allocated. | 2239 // mark the block at v of size n as allocated. |
2235 // If noptr is true, mark it as having no pointers. | 2240 // If noptr is true, mark it as having no pointers. |
2236 void | 2241 void |
2237 runtime·markallocated(void *v, uintptr n, bool noptr) | 2242 runtime·markallocated(void *v, uintptr n, bool noptr) |
2238 { | 2243 { |
2239 uintptr *b, obits, bits, off, shift; | 2244 uintptr *b, obits, bits, off, shift; |
2240 | 2245 |
2241 if(0) | 2246 if(0) |
2242 runtime·printf("markallocated %p+%p\n", v, n); | 2247 runtime·printf("markallocated %p+%p\n", v, n); |
2243 | 2248 |
2244 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m heap->arena_start) | 2249 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) |
2245 runtime·throw("markallocated: bad pointer"); | 2250 runtime·throw("markallocated: bad pointer"); |
2246 | 2251 |
2247 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse t | 2252 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset |
2248 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2253 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2249 shift = off % wordsPerBitmapWord; | 2254 shift = off % wordsPerBitmapWord; |
2250 | 2255 |
2251 for(;;) { | 2256 for(;;) { |
2252 obits = *b; | 2257 obits = *b; |
2253 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); | 2258 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); |
2254 if(noptr) | 2259 if(noptr) |
2255 bits |= bitNoPointers<<shift; | 2260 bits |= bitNoPointers<<shift; |
2256 if(runtime·singleproc) { | 2261 if(runtime·singleproc) { |
2257 *b = bits; | 2262 *b = bits; |
2258 break; | 2263 break; |
2259 } else { | 2264 } else { |
2260 // more than one goroutine is potentially running: use a tomic op | 2265 // more than one goroutine is potentially running: use a tomic op |
2261 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2266 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
2262 break; | 2267 break; |
2263 } | 2268 } |
2264 } | 2269 } |
2265 } | 2270 } |
2266 | 2271 |
2267 // mark the block at v of size n as freed. | 2272 // mark the block at v of size n as freed. |
2268 void | 2273 void |
2269 runtime·markfreed(void *v, uintptr n) | 2274 runtime·markfreed(void *v, uintptr n) |
2270 { | 2275 { |
2271 uintptr *b, obits, bits, off, shift; | 2276 uintptr *b, obits, bits, off, shift; |
2272 | 2277 |
2273 if(0) | 2278 if(0) |
2274 runtime·printf("markallocated %p+%p\n", v, n); | 2279 runtime·printf("markallocated %p+%p\n", v, n); |
2275 | 2280 |
2276 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m heap->arena_start) | 2281 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) |
2277 runtime·throw("markallocated: bad pointer"); | 2282 runtime·throw("markallocated: bad pointer"); |
2278 | 2283 |
2279 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse t | 2284 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset |
2280 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2285 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2281 shift = off % wordsPerBitmapWord; | 2286 shift = off % wordsPerBitmapWord; |
2282 | 2287 |
2283 for(;;) { | 2288 for(;;) { |
2284 obits = *b; | 2289 obits = *b; |
2285 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2290 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
2286 if(runtime·singleproc) { | 2291 if(runtime·singleproc) { |
2287 *b = bits; | 2292 *b = bits; |
2288 break; | 2293 break; |
2289 } else { | 2294 } else { |
2290 // more than one goroutine is potentially running: use a tomic op | 2295 // more than one goroutine is potentially running: use a tomic op |
2291 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2296 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
2292 break; | 2297 break; |
2293 } | 2298 } |
2294 } | 2299 } |
2295 } | 2300 } |
2296 | 2301 |
2297 // check that the block at v of size n is marked freed. | 2302 // check that the block at v of size n is marked freed. |
2298 void | 2303 void |
2299 runtime·checkfreed(void *v, uintptr n) | 2304 runtime·checkfreed(void *v, uintptr n) |
2300 { | 2305 { |
2301 uintptr *b, bits, off, shift; | 2306 uintptr *b, bits, off, shift; |
2302 | 2307 |
2303 if(!runtime·checking) | 2308 if(!runtime·checking) |
2304 return; | 2309 return; |
2305 | 2310 |
2306 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m heap->arena_start) | 2311 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) |
2307 return; // not allocated, so okay | 2312 return; // not allocated, so okay |
2308 | 2313 |
2309 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse t | 2314 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset |
2310 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2315 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2311 shift = off % wordsPerBitmapWord; | 2316 shift = off % wordsPerBitmapWord; |
2312 | 2317 |
2313 bits = *b>>shift; | 2318 bits = *b>>shift; |
2314 if((bits & bitAllocated) != 0) { | 2319 if((bits & bitAllocated) != 0) { |
2315 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", | 2320 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", |
2316 v, n, off, bits & bitMask); | 2321 v, n, off, bits & bitMask); |
2317 runtime·throw("checkfreed: not freed"); | 2322 runtime·throw("checkfreed: not freed"); |
2318 } | 2323 } |
2319 } | 2324 } |
2320 | 2325 |
2321 // mark the span of memory at v as having n blocks of the given size. | 2326 // mark the span of memory at v as having n blocks of the given size. |
2322 // if leftover is true, there is left over space at the end of the span. | 2327 // if leftover is true, there is left over space at the end of the span. |
2323 void | 2328 void |
2324 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) | 2329 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) |
2325 { | 2330 { |
2326 uintptr *b, off, shift; | 2331 uintptr *b, off, shift; |
2327 byte *p; | 2332 byte *p; |
2328 | 2333 |
2329 » if((byte*)v+size*n > (byte*)runtime·mheap->arena_used || (byte*)v < runt ime·mheap->arena_start) | 2334 » if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runti me·mheap.arena_start) |
2330 runtime·throw("markspan: bad pointer"); | 2335 runtime·throw("markspan: bad pointer"); |
2331 | 2336 |
2332 p = v; | 2337 p = v; |
2333 if(leftover) // mark a boundary just past end of last block too | 2338 if(leftover) // mark a boundary just past end of last block too |
2334 n++; | 2339 n++; |
2335 for(; n-- > 0; p += size) { | 2340 for(; n-- > 0; p += size) { |
2336 // Okay to use non-atomic ops here, because we control | 2341 // Okay to use non-atomic ops here, because we control |
2337 // the entire span, and each bitmap word has bits for only | 2342 // the entire span, and each bitmap word has bits for only |
2338 // one span, so no other goroutines are changing these | 2343 // one span, so no other goroutines are changing these |
2339 // bitmap words. | 2344 // bitmap words. |
2340 » » off = (uintptr*)p - (uintptr*)runtime·mheap->arena_start; // wo rd offset | 2345 » » off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // wor d offset |
2341 » » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWor d - 1; | 2346 » » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2342 shift = off % wordsPerBitmapWord; | 2347 shift = off % wordsPerBitmapWord; |
2343 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2348 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
2344 } | 2349 } |
2345 } | 2350 } |
2346 | 2351 |
2347 // unmark the span of memory at v of length n bytes. | 2352 // unmark the span of memory at v of length n bytes. |
2348 void | 2353 void |
2349 runtime·unmarkspan(void *v, uintptr n) | 2354 runtime·unmarkspan(void *v, uintptr n) |
2350 { | 2355 { |
2351 uintptr *p, *b, off; | 2356 uintptr *p, *b, off; |
2352 | 2357 |
2353 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m heap->arena_start) | 2358 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) |
2354 runtime·throw("markspan: bad pointer"); | 2359 runtime·throw("markspan: bad pointer"); |
2355 | 2360 |
2356 p = v; | 2361 p = v; |
2357 » off = p - (uintptr*)runtime·mheap->arena_start; // word offset | 2362 » off = p - (uintptr*)runtime·mheap.arena_start; // word offset |
2358 if(off % wordsPerBitmapWord != 0) | 2363 if(off % wordsPerBitmapWord != 0) |
2359 runtime·throw("markspan: unaligned pointer"); | 2364 runtime·throw("markspan: unaligned pointer"); |
2360 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2365 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2361 n /= PtrSize; | 2366 n /= PtrSize; |
2362 if(n%wordsPerBitmapWord != 0) | 2367 if(n%wordsPerBitmapWord != 0) |
2363 runtime·throw("unmarkspan: unaligned length"); | 2368 runtime·throw("unmarkspan: unaligned length"); |
2364 // Okay to use non-atomic ops here, because we control | 2369 // Okay to use non-atomic ops here, because we control |
2365 // the entire span, and each bitmap word has bits for only | 2370 // the entire span, and each bitmap word has bits for only |
2366 // one span, so no other goroutines are changing these | 2371 // one span, so no other goroutines are changing these |
2367 // bitmap words. | 2372 // bitmap words. |
2368 n /= wordsPerBitmapWord; | 2373 n /= wordsPerBitmapWord; |
2369 while(n-- > 0) | 2374 while(n-- > 0) |
2370 *b-- = 0; | 2375 *b-- = 0; |
2371 } | 2376 } |
2372 | 2377 |
2373 bool | 2378 bool |
2374 runtime·blockspecial(void *v) | 2379 runtime·blockspecial(void *v) |
2375 { | 2380 { |
2376 uintptr *b, off, shift; | 2381 uintptr *b, off, shift; |
2377 | 2382 |
2378 if(DebugMark) | 2383 if(DebugMark) |
2379 return true; | 2384 return true; |
2380 | 2385 |
2381 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; | 2386 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; |
2382 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2387 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2383 shift = off % wordsPerBitmapWord; | 2388 shift = off % wordsPerBitmapWord; |
2384 | 2389 |
2385 return (*b & (bitSpecial<<shift)) != 0; | 2390 return (*b & (bitSpecial<<shift)) != 0; |
2386 } | 2391 } |
2387 | 2392 |
2388 void | 2393 void |
2389 runtime·setblockspecial(void *v, bool s) | 2394 runtime·setblockspecial(void *v, bool s) |
2390 { | 2395 { |
2391 uintptr *b, off, shift, bits, obits; | 2396 uintptr *b, off, shift, bits, obits; |
2392 | 2397 |
2393 if(DebugMark) | 2398 if(DebugMark) |
2394 return; | 2399 return; |
2395 | 2400 |
2396 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; | 2401 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; |
2397 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; | 2402 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; |
2398 shift = off % wordsPerBitmapWord; | 2403 shift = off % wordsPerBitmapWord; |
2399 | 2404 |
2400 for(;;) { | 2405 for(;;) { |
2401 obits = *b; | 2406 obits = *b; |
2402 if(s) | 2407 if(s) |
2403 bits = obits | (bitSpecial<<shift); | 2408 bits = obits | (bitSpecial<<shift); |
2404 else | 2409 else |
2405 bits = obits & ~(bitSpecial<<shift); | 2410 bits = obits & ~(bitSpecial<<shift); |
2406 if(runtime·singleproc) { | 2411 if(runtime·singleproc) { |
2407 *b = bits; | 2412 *b = bits; |
(...skipping 11 matching lines...) Expand all Loading... | |
2419 { | 2424 { |
2420 // Caller has added extra mappings to the arena. | 2425 // Caller has added extra mappings to the arena. |
2421 // Add extra mappings of bitmap words as needed. | 2426 // Add extra mappings of bitmap words as needed. |
2422 // We allocate extra bitmap pieces in chunks of bitmapChunk. | 2427 // We allocate extra bitmap pieces in chunks of bitmapChunk. |
2423 enum { | 2428 enum { |
2424 bitmapChunk = 8192 | 2429 bitmapChunk = 8192 |
2425 }; | 2430 }; |
2426 uintptr n; | 2431 uintptr n; |
2427 | 2432 |
2428 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; | 2433 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; |
2429 » n = (n+bitmapChunk-1) & ~(bitmapChunk-1); | 2434 » n = ROUND(n, bitmapChunk); |
2430 if(h->bitmap_mapped >= n) | 2435 if(h->bitmap_mapped >= n) |
2431 return; | 2436 return; |
2432 | 2437 |
2433 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); | 2438 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); |
2434 h->bitmap_mapped = n; | 2439 h->bitmap_mapped = n; |
2435 } | 2440 } |
LEFT | RIGHT |