LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Garbage collector. | 5 // Garbage collector. |
6 | 6 |
7 #include "runtime.h" | 7 #include "runtime.h" |
8 #include "arch_GOARCH.h" | 8 #include "arch_GOARCH.h" |
9 #include "malloc.h" | 9 #include "malloc.h" |
10 #include "stack.h" | 10 #include "stack.h" |
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
97 LFNode node; // must be first | 97 LFNode node; // must be first |
98 uintptr nobj; | 98 uintptr nobj; |
99 Obj obj[SIZE/sizeof(Obj) - 1]; | 99 Obj obj[SIZE/sizeof(Obj) - 1]; |
100 uint8 _padding[SIZE%sizeof(Obj) + sizeof(Obj)]; | 100 uint8 _padding[SIZE%sizeof(Obj) + sizeof(Obj)]; |
101 #undef SIZE | 101 #undef SIZE |
102 }; | 102 }; |
103 | 103 |
104 typedef struct Finalizer Finalizer; | 104 typedef struct Finalizer Finalizer; |
105 struct Finalizer | 105 struct Finalizer |
106 { | 106 { |
107 » void (*fn)(void*); | 107 » FuncVal *fn; |
108 void *arg; | 108 void *arg; |
109 uintptr nret; | 109 uintptr nret; |
110 }; | 110 }; |
111 | 111 |
112 typedef struct FinBlock FinBlock; | 112 typedef struct FinBlock FinBlock; |
113 struct FinBlock | 113 struct FinBlock |
114 { | 114 { |
115 FinBlock *alllink; | 115 FinBlock *alllink; |
116 FinBlock *next; | 116 FinBlock *next; |
117 int32 cnt; | 117 int32 cnt; |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
157 uintptr nchunk; | 157 uintptr nchunk; |
158 | 158 |
159 Obj *roots; | 159 Obj *roots; |
160 uint32 nroot; | 160 uint32 nroot; |
161 uint32 rootcap; | 161 uint32 rootcap; |
162 } work; | 162 } work; |
163 | 163 |
164 enum { | 164 enum { |
165 GC_DEFAULT_PTR = GC_NUM_INSTR, | 165 GC_DEFAULT_PTR = GC_NUM_INSTR, |
166 GC_MAP_NEXT, | 166 GC_MAP_NEXT, |
| 167 GC_CHAN, |
167 }; | 168 }; |
168 | 169 |
169 // markonly marks an object. It returns true if the object | 170 // markonly marks an object. It returns true if the object |
170 // has been marked by this function, false otherwise. | 171 // has been marked by this function, false otherwise. |
171 // This function isn't thread-safe and doesn't append the object to any buffer. | 172 // This function isn't thread-safe and doesn't append the object to any buffer. |
172 static bool | 173 static bool |
173 markonly(void *obj) | 174 markonly(void *obj) |
174 { | 175 { |
175 byte *p; | 176 byte *p; |
176 uintptr *bitp, bits, shift, x, xbits, off; | 177 uintptr *bitp, bits, shift, x, xbits, off; |
177 MSpan *s; | 178 MSpan *s; |
178 PageID k; | 179 PageID k; |
179 | 180 |
180 // Words outside the arena cannot be pointers. | 181 // Words outside the arena cannot be pointers. |
181 » if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used) | 182 » if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) |
182 return false; | 183 return false; |
183 | 184 |
184 // obj may be a pointer to a live object. | 185 // obj may be a pointer to a live object. |
185 // Try to find the beginning of the object. | 186 // Try to find the beginning of the object. |
186 | 187 |
187 // Round down to word boundary. | 188 // Round down to word boundary. |
188 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 189 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
189 | 190 |
190 // Find bits for this word. | 191 // Find bits for this word. |
191 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | 192 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; |
192 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 193 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1
; |
193 shift = off % wordsPerBitmapWord; | 194 shift = off % wordsPerBitmapWord; |
194 xbits = *bitp; | 195 xbits = *bitp; |
195 bits = xbits >> shift; | 196 bits = xbits >> shift; |
196 | 197 |
197 // Pointing at the beginning of a block? | 198 // Pointing at the beginning of a block? |
198 if((bits & (bitAllocated|bitBlockBoundary)) != 0) | 199 if((bits & (bitAllocated|bitBlockBoundary)) != 0) |
199 goto found; | 200 goto found; |
200 | 201 |
201 // Otherwise consult span table to find beginning. | 202 // Otherwise consult span table to find beginning. |
202 // (Manually inlined copy of MHeap_LookupMaybe.) | 203 // (Manually inlined copy of MHeap_LookupMaybe.) |
203 k = (uintptr)obj>>PageShift; | 204 k = (uintptr)obj>>PageShift; |
204 x = k; | 205 x = k; |
205 if(sizeof(void*) == 8) | 206 if(sizeof(void*) == 8) |
206 » » x -= (uintptr)runtime·mheap.arena_start>>PageShift; | 207 » » x -= (uintptr)runtime·mheap->arena_start>>PageShift; |
207 » s = runtime·mheap.map[x]; | 208 » s = runtime·mheap->map[x]; |
208 if(s == nil || k < s->start || k - s->start >= s->npages || s->state !=
MSpanInUse) | 209 if(s == nil || k < s->start || k - s->start >= s->npages || s->state !=
MSpanInUse) |
209 return false; | 210 return false; |
210 p = (byte*)((uintptr)s->start<<PageShift); | 211 p = (byte*)((uintptr)s->start<<PageShift); |
211 if(s->sizeclass == 0) { | 212 if(s->sizeclass == 0) { |
212 obj = p; | 213 obj = p; |
213 } else { | 214 } else { |
214 if((byte*)obj >= (byte*)s->limit) | 215 if((byte*)obj >= (byte*)s->limit) |
215 return false; | 216 return false; |
216 uintptr size = s->elemsize; | 217 uintptr size = s->elemsize; |
217 int32 i = ((byte*)obj - p)/size; | 218 int32 i = ((byte*)obj - p)/size; |
218 obj = p+i*size; | 219 obj = p+i*size; |
219 } | 220 } |
220 | 221 |
221 // Now that we know the object header, reload bits. | 222 // Now that we know the object header, reload bits. |
222 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | 223 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; |
223 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 224 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1
; |
224 shift = off % wordsPerBitmapWord; | 225 shift = off % wordsPerBitmapWord; |
225 xbits = *bitp; | 226 xbits = *bitp; |
226 bits = xbits >> shift; | 227 bits = xbits >> shift; |
227 | 228 |
228 found: | 229 found: |
229 // Now we have bits, bitp, and shift correct for | 230 // Now we have bits, bitp, and shift correct for |
230 // obj pointing at the base of the object. | 231 // obj pointing at the base of the object. |
231 // Only care about allocated and not marked. | 232 // Only care about allocated and not marked. |
232 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | 233 if((bits & (bitAllocated|bitMarked)) != bitAllocated) |
233 return false; | 234 return false; |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
297 { | 298 { |
298 byte *p, *arena_start, *obj; | 299 byte *p, *arena_start, *obj; |
299 uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; | 300 uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; |
300 MSpan *s; | 301 MSpan *s; |
301 PageID k; | 302 PageID k; |
302 Obj *wp; | 303 Obj *wp; |
303 Workbuf *wbuf; | 304 Workbuf *wbuf; |
304 PtrTarget *ptrbuf_end; | 305 PtrTarget *ptrbuf_end; |
305 BitTarget *bitbufpos, *bt; | 306 BitTarget *bitbufpos, *bt; |
306 | 307 |
307 » arena_start = runtime·mheap.arena_start; | 308 » arena_start = runtime·mheap->arena_start; |
308 | 309 |
309 wp = *_wp; | 310 wp = *_wp; |
310 wbuf = *_wbuf; | 311 wbuf = *_wbuf; |
311 nobj = *_nobj; | 312 nobj = *_nobj; |
312 | 313 |
313 ptrbuf_end = *ptrbufpos; | 314 ptrbuf_end = *ptrbufpos; |
314 n = ptrbuf_end - ptrbuf; | 315 n = ptrbuf_end - ptrbuf; |
315 *ptrbufpos = ptrbuf; | 316 *ptrbufpos = ptrbuf; |
316 | 317 |
317 // If buffer is nearly full, get a new one. | 318 // If buffer is nearly full, get a new one. |
(...skipping 15 matching lines...) Expand all Loading... |
333 | 334 |
334 bitbufpos = bitbuf; | 335 bitbufpos = bitbuf; |
335 | 336 |
336 while(ptrbuf < ptrbuf_end) { | 337 while(ptrbuf < ptrbuf_end) { |
337 obj = ptrbuf->p; | 338 obj = ptrbuf->p; |
338 ti = ptrbuf->ti; | 339 ti = ptrbuf->ti; |
339 ptrbuf++; | 340 ptrbuf++; |
340 | 341 |
341 // obj belongs to interval [mheap.arena_start, mheap.are
na_used). | 342 // obj belongs to interval [mheap.arena_start, mheap.are
na_used). |
342 if(Debug > 1) { | 343 if(Debug > 1) { |
343 » » » » if(obj < runtime·mheap.arena_start || obj >= run
time·mheap.arena_used) | 344 » » » » if(obj < runtime·mheap->arena_start || obj >= ru
ntime·mheap->arena_used) |
344 runtime·throw("object is outside of mhea
p"); | 345 runtime·throw("object is outside of mhea
p"); |
345 } | 346 } |
346 | 347 |
347 // obj may be a pointer to a live object. | 348 // obj may be a pointer to a live object. |
348 // Try to find the beginning of the object. | 349 // Try to find the beginning of the object. |
349 | 350 |
350 // Round down to word boundary. | 351 // Round down to word boundary. |
351 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { | 352 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { |
352 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-
1)); | 353 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-
1)); |
353 ti = 0; | 354 ti = 0; |
(...skipping 22 matching lines...) Expand all Loading... |
376 goto found; | 377 goto found; |
377 } | 378 } |
378 } | 379 } |
379 | 380 |
380 // Otherwise consult span table to find beginning. | 381 // Otherwise consult span table to find beginning. |
381 // (Manually inlined copy of MHeap_LookupMaybe.) | 382 // (Manually inlined copy of MHeap_LookupMaybe.) |
382 k = (uintptr)obj>>PageShift; | 383 k = (uintptr)obj>>PageShift; |
383 x = k; | 384 x = k; |
384 if(sizeof(void*) == 8) | 385 if(sizeof(void*) == 8) |
385 x -= (uintptr)arena_start>>PageShift; | 386 x -= (uintptr)arena_start>>PageShift; |
386 » » » s = runtime·mheap.map[x]; | 387 » » » s = runtime·mheap->map[x]; |
387 if(s == nil || k < s->start || k - s->start >= s->npages
|| s->state != MSpanInUse) | 388 if(s == nil || k < s->start || k - s->start >= s->npages
|| s->state != MSpanInUse) |
388 continue; | 389 continue; |
389 p = (byte*)((uintptr)s->start<<PageShift); | 390 p = (byte*)((uintptr)s->start<<PageShift); |
390 if(s->sizeclass == 0) { | 391 if(s->sizeclass == 0) { |
391 obj = p; | 392 obj = p; |
392 } else { | 393 } else { |
393 if((byte*)obj >= (byte*)s->limit) | 394 if((byte*)obj >= (byte*)s->limit) |
394 continue; | 395 continue; |
395 size = s->elemsize; | 396 size = s->elemsize; |
396 int32 i = ((byte*)obj - p)/size; | 397 int32 i = ((byte*)obj - p)/size; |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
428 if((bits & bitNoPointers) != 0) | 429 if((bits & bitNoPointers) != 0) |
429 continue; | 430 continue; |
430 | 431 |
431 obj = bt->p; | 432 obj = bt->p; |
432 | 433 |
433 // Ask span about size class. | 434 // Ask span about size class. |
434 // (Manually inlined copy of MHeap_Lookup.) | 435 // (Manually inlined copy of MHeap_Lookup.) |
435 x = (uintptr)obj >> PageShift; | 436 x = (uintptr)obj >> PageShift; |
436 if(sizeof(void*) == 8) | 437 if(sizeof(void*) == 8) |
437 x -= (uintptr)arena_start>>PageShift; | 438 x -= (uintptr)arena_start>>PageShift; |
438 » » » s = runtime·mheap.map[x]; | 439 » » » s = runtime·mheap->map[x]; |
439 | 440 |
440 PREFETCH(obj); | 441 PREFETCH(obj); |
441 | 442 |
442 *wp = (Obj){obj, s->elemsize, bt->ti}; | 443 *wp = (Obj){obj, s->elemsize, bt->ti}; |
443 wp++; | 444 wp++; |
444 nobj++; | 445 nobj++; |
445 } | 446 } |
446 runtime·unlock(&lock); | 447 runtime·unlock(&lock); |
447 | 448 |
448 // If another proc wants a pointer, give it some. | 449 // If another proc wants a pointer, give it some. |
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
513 *_wp = wp; | 514 *_wp = wp; |
514 *_wbuf = wbuf; | 515 *_wbuf = wbuf; |
515 *_nobj = nobj; | 516 *_nobj = nobj; |
516 } | 517 } |
517 | 518 |
518 // Program that scans the whole block and treats every block element as a potent
ial pointer | 519 // Program that scans the whole block and treats every block element as a potent
ial pointer |
519 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; | 520 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; |
520 | 521 |
521 // Hashmap iterator program | 522 // Hashmap iterator program |
522 static uintptr mapProg[2] = {0, GC_MAP_NEXT}; | 523 static uintptr mapProg[2] = {0, GC_MAP_NEXT}; |
| 524 |
| 525 // Hchan program |
| 526 static uintptr chanProg[2] = {0, GC_CHAN}; |
523 | 527 |
524 // Local variables of a program fragment or loop | 528 // Local variables of a program fragment or loop |
525 typedef struct Frame Frame; | 529 typedef struct Frame Frame; |
526 struct Frame { | 530 struct Frame { |
527 uintptr count, elemsize, b; | 531 uintptr count, elemsize, b; |
528 uintptr *loop_or_ret; | 532 uintptr *loop_or_ret; |
529 }; | 533 }; |
530 | 534 |
531 // scanblock scans a block of n bytes starting at pointer b for references | 535 // scanblock scans a block of n bytes starting at pointer b for references |
532 // to other objects, scanning any it finds recursively until there are no | 536 // to other objects, scanning any it finds recursively until there are no |
533 // unscanned objects left. Instead of using an explicit recursion, it keeps | 537 // unscanned objects left. Instead of using an explicit recursion, it keeps |
534 // a work list in the Workbuf* structures and loops in the main function | 538 // a work list in the Workbuf* structures and loops in the main function |
535 // body. Keeping an explicit work list is easier on the stack allocator and | 539 // body. Keeping an explicit work list is easier on the stack allocator and |
536 // more efficient. | 540 // more efficient. |
537 // | 541 // |
538 // wbuf: current work buffer | 542 // wbuf: current work buffer |
539 // wp: storage for next queued pointer (write pointer) | 543 // wp: storage for next queued pointer (write pointer) |
540 // nobj: number of queued objects | 544 // nobj: number of queued objects |
541 static void | 545 static void |
542 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) | 546 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) |
543 { | 547 { |
544 byte *b, *arena_start, *arena_used; | 548 byte *b, *arena_start, *arena_used; |
545 » uintptr n, i, end_b, elemsize, ti, objti, count, type; | 549 » uintptr n, i, end_b, elemsize, size, ti, objti, count, type; |
546 uintptr *pc, precise_type, nominal_size; | 550 uintptr *pc, precise_type, nominal_size; |
547 uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti; | 551 uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti; |
548 void *obj; | 552 void *obj; |
549 Type *t; | 553 Type *t; |
550 Slice *sliceptr; | 554 Slice *sliceptr; |
551 Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; | 555 Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; |
552 BufferList *scanbuffers; | 556 BufferList *scanbuffers; |
553 PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; | 557 PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; |
554 BitTarget *bitbuf; | 558 BitTarget *bitbuf; |
555 Obj *objbuf, *objbuf_end, *objbufpos; | 559 Obj *objbuf, *objbuf_end, *objbufpos; |
556 Eface *eface; | 560 Eface *eface; |
557 Iface *iface; | 561 Iface *iface; |
558 Hmap *hmap; | 562 Hmap *hmap; |
559 MapType *maptype; | 563 MapType *maptype; |
560 bool didmark, mapkey_kind, mapval_kind; | 564 bool didmark, mapkey_kind, mapval_kind; |
561 struct hash_gciter map_iter; | 565 struct hash_gciter map_iter; |
562 struct hash_gciter_data d; | 566 struct hash_gciter_data d; |
| 567 Hchan *chan; |
| 568 ChanType *chantype; |
563 | 569 |
564 if(sizeof(Workbuf) % PageSize != 0) | 570 if(sizeof(Workbuf) % PageSize != 0) |
565 runtime·throw("scanblock: size of Workbuf is suboptimal"); | 571 runtime·throw("scanblock: size of Workbuf is suboptimal"); |
566 | 572 |
567 // Memory arena parameters. | 573 // Memory arena parameters. |
568 » arena_start = runtime·mheap.arena_start; | 574 » arena_start = runtime·mheap->arena_start; |
569 » arena_used = runtime·mheap.arena_used; | 575 » arena_used = runtime·mheap->arena_used; |
570 | 576 |
571 stack_ptr = stack+nelem(stack)-1; | 577 stack_ptr = stack+nelem(stack)-1; |
572 ········ | 578 ········ |
573 precise_type = false; | 579 precise_type = false; |
574 nominal_size = 0; | 580 nominal_size = 0; |
575 | 581 |
576 // Allocate ptrbuf, bitbuf | 582 // Allocate ptrbuf, bitbuf |
577 { | 583 { |
578 runtime·lock(&lock); | 584 runtime·lock(&lock); |
579 | 585 |
580 if(bufferList == nil) { | 586 if(bufferList == nil) { |
581 bufferList = runtime·SysAlloc(sizeof(*bufferList)); | 587 bufferList = runtime·SysAlloc(sizeof(*bufferList)); |
| 588 if(bufferList == nil) |
| 589 runtime·throw("runtime: cannot allocate memory")
; |
582 bufferList->next = nil; | 590 bufferList->next = nil; |
583 } | 591 } |
584 scanbuffers = bufferList; | 592 scanbuffers = bufferList; |
585 bufferList = bufferList->next; | 593 bufferList = bufferList->next; |
586 | 594 |
587 ptrbuf = &scanbuffers->ptrtarget[0]; | 595 ptrbuf = &scanbuffers->ptrtarget[0]; |
588 ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptr
target); | 596 ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptr
target); |
589 bitbuf = &scanbuffers->bittarget[0]; | 597 bitbuf = &scanbuffers->bittarget[0]; |
590 objbuf = &scanbuffers->obj[0]; | 598 objbuf = &scanbuffers->obj[0]; |
591 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); | 599 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); |
592 | 600 |
593 runtime·unlock(&lock); | 601 runtime·unlock(&lock); |
594 } | 602 } |
595 | 603 |
596 ptrbufpos = ptrbuf; | 604 ptrbufpos = ptrbuf; |
597 objbufpos = objbuf; | 605 objbufpos = objbuf; |
598 | 606 |
599 // (Silence the compiler) | 607 // (Silence the compiler) |
600 map_ret = nil; | 608 map_ret = nil; |
601 mapkey_size = mapval_size = 0; | 609 mapkey_size = mapval_size = 0; |
602 mapkey_kind = mapval_kind = false; | 610 mapkey_kind = mapval_kind = false; |
603 mapkey_ti = mapval_ti = 0; | 611 mapkey_ti = mapval_ti = 0; |
| 612 chan = nil; |
| 613 chantype = nil; |
604 | 614 |
605 goto next_block; | 615 goto next_block; |
606 | 616 |
607 for(;;) { | 617 for(;;) { |
608 // Each iteration scans the block b of length n, queueing pointe
rs in | 618 // Each iteration scans the block b of length n, queueing pointe
rs in |
609 // the work buffer. | 619 // the work buffer. |
610 if(Debug > 1) { | 620 if(Debug > 1) { |
611 runtime·printf("scanblock %p %D\n", b, (int64)n); | 621 runtime·printf("scanblock %p %D\n", b, (int64)n); |
612 } | 622 } |
613 | 623 |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
653 mapval_size = maptype->elem->siz
e; | 663 mapval_size = maptype->elem->siz
e; |
654 mapval_kind = maptype->elem->kin
d; | 664 mapval_kind = maptype->elem->kin
d; |
655 mapval_ti = (uintptr)maptype->
elem->gc | PRECISE; | 665 mapval_ti = (uintptr)maptype->
elem->gc | PRECISE; |
656 | 666 |
657 map_ret = 0; | 667 map_ret = 0; |
658 pc = mapProg; | 668 pc = mapProg; |
659 } else { | 669 } else { |
660 goto next_block; | 670 goto next_block; |
661 } | 671 } |
662 break; | 672 break; |
| 673 case TypeInfo_Chan: |
| 674 chan = (Hchan*)b; |
| 675 chantype = (ChanType*)t; |
| 676 pc = chanProg; |
| 677 break; |
663 default: | 678 default: |
664 runtime·throw("scanblock: invalid type")
; | 679 runtime·throw("scanblock: invalid type")
; |
665 return; | 680 return; |
666 } | 681 } |
667 } else { | 682 } else { |
668 pc = defaultProg; | 683 pc = defaultProg; |
669 } | 684 } |
670 } else { | 685 } else { |
671 pc = defaultProg; | 686 pc = defaultProg; |
672 } | 687 } |
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
817 // Stack pop. | 832 // Stack pop. |
818 stack_top = *(++stack_ptr); | 833 stack_top = *(++stack_ptr); |
819 pc += 1; | 834 pc += 1; |
820 } | 835 } |
821 continue; | 836 continue; |
822 | 837 |
823 case GC_CALL: | 838 case GC_CALL: |
824 // Stack push. | 839 // Stack push. |
825 *stack_ptr-- = stack_top; | 840 *stack_ptr-- = stack_top; |
826 stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*re
turn address*/}; | 841 stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*re
turn address*/}; |
827 » » » pc = (uintptr*)((byte*)pc + (uintptr)pc[2]); // target
of the CALL instruction | 842 » » » pc = (uintptr*)((byte*)pc + *(int32*)(pc+2)); // target
of the CALL instruction |
828 continue; | 843 continue; |
829 | 844 |
830 case GC_MAP_PTR: | 845 case GC_MAP_PTR: |
831 hmap = *(Hmap**)(stack_top.b + pc[1]); | 846 hmap = *(Hmap**)(stack_top.b + pc[1]); |
832 if(hmap == nil) { | 847 if(hmap == nil) { |
833 pc += 3; | 848 pc += 3; |
834 continue; | 849 continue; |
835 } | 850 } |
836 runtime·lock(&lock); | 851 runtime·lock(&lock); |
837 didmark = markonly(hmap); | 852 didmark = markonly(hmap); |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
885 *ptrbufpos++ = (PtrTarge
t){*(void**)d.val_data, mapval_ti}; | 900 *ptrbufpos++ = (PtrTarge
t){*(void**)d.val_data, mapval_ti}; |
886 } | 901 } |
887 } | 902 } |
888 } | 903 } |
889 if(map_ret == 0) | 904 if(map_ret == 0) |
890 goto next_block; | 905 goto next_block; |
891 pc = map_ret; | 906 pc = map_ret; |
892 continue; | 907 continue; |
893 | 908 |
894 case GC_REGION: | 909 case GC_REGION: |
895 // TODO(atom): to be expanded in a next CL. Same as GC_A
PTR for now. | |
896 obj = (void*)(stack_top.b + pc[1]); | 910 obj = (void*)(stack_top.b + pc[1]); |
| 911 size = pc[2]; |
| 912 objti = pc[3]; |
897 pc += 4; | 913 pc += 4; |
| 914 |
| 915 *objbufpos++ = (Obj){obj, size, objti}; |
| 916 if(objbufpos == objbuf_end) |
| 917 flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nob
j); |
898 break; | 918 break; |
| 919 |
| 920 case GC_CHAN: |
| 921 // There are no heap pointers in struct Hchan, |
| 922 // so we can ignore the leading sizeof(Hchan) bytes. |
| 923 if(!(chantype->elem->kind & KindNoPointers)) { |
| 924 // Channel's buffer follows Hchan immediately in
memory. |
| 925 // Size of buffer (cap(c)) is second int in the
chan struct. |
| 926 n = ((uintgo*)chan)[1]; |
| 927 if(n > 0) { |
| 928 // TODO(atom): split into two chunks so
that only the |
| 929 // in-use part of the circular buffer is
scanned. |
| 930 // (Channel routines zero the unused par
t, so the current |
| 931 // code does not lead to leaks, it's jus
t a little inefficient.) |
| 932 *objbufpos++ = (Obj){(byte*)chan+runtime
·Hchansize, n*chantype->elem->size, |
| 933 (uintptr)chantype->elem->gc | PR
ECISE | LOOP}; |
| 934 if(objbufpos == objbuf_end) |
| 935 flushobjbuf(objbuf, &objbufpos,
&wp, &wbuf, &nobj); |
| 936 } |
| 937 } |
| 938 goto next_block; |
899 | 939 |
900 default: | 940 default: |
901 runtime·throw("scanblock: invalid GC instruction"); | 941 runtime·throw("scanblock: invalid GC instruction"); |
902 return; | 942 return; |
903 } | 943 } |
904 | 944 |
905 if(obj >= arena_start && obj < arena_used) { | 945 if(obj >= arena_start && obj < arena_used) { |
906 *ptrbufpos++ = (PtrTarget){obj, objti}; | 946 *ptrbufpos++ = (PtrTarget){obj, objti}; |
907 if(ptrbufpos == ptrbuf_end) | 947 if(ptrbufpos == ptrbuf_end) |
908 flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nob
j, bitbuf); | 948 flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nob
j, bitbuf); |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
972 b += PtrSize - off; | 1012 b += PtrSize - off; |
973 n -= PtrSize - off; | 1013 n -= PtrSize - off; |
974 } | 1014 } |
975 | 1015 |
976 vp = (void**)b; | 1016 vp = (void**)b; |
977 n /= PtrSize; | 1017 n /= PtrSize; |
978 for(i=0; i<n; i++) { | 1018 for(i=0; i<n; i++) { |
979 obj = (byte*)vp[i]; | 1019 obj = (byte*)vp[i]; |
980 | 1020 |
981 // Words outside the arena cannot be pointers. | 1021 // Words outside the arena cannot be pointers. |
982 » » if((byte*)obj < runtime·mheap.arena_start || (byte*)obj >= runti
me·mheap.arena_used) | 1022 » » if((byte*)obj < runtime·mheap->arena_start || (byte*)obj >= runt
ime·mheap->arena_used) |
983 continue; | 1023 continue; |
984 | 1024 |
985 // Round down to word boundary. | 1025 // Round down to word boundary. |
986 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 1026 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
987 | 1027 |
988 // Consult span table to find beginning. | 1028 // Consult span table to find beginning. |
989 » » s = runtime·MHeap_LookupMaybe(&runtime·mheap, obj); | 1029 » » s = runtime·MHeap_LookupMaybe(runtime·mheap, obj); |
990 if(s == nil) | 1030 if(s == nil) |
991 continue; | 1031 continue; |
992 | 1032 |
993 p = (byte*)((uintptr)s->start<<PageShift); | 1033 p = (byte*)((uintptr)s->start<<PageShift); |
994 size = s->elemsize; | 1034 size = s->elemsize; |
995 if(s->sizeclass == 0) { | 1035 if(s->sizeclass == 0) { |
996 obj = p; | 1036 obj = p; |
997 } else { | 1037 } else { |
998 if((byte*)obj >= (byte*)s->limit) | 1038 if((byte*)obj >= (byte*)s->limit) |
999 continue; | 1039 continue; |
1000 int32 i = ((byte*)obj - p)/size; | 1040 int32 i = ((byte*)obj - p)/size; |
1001 obj = p+i*size; | 1041 obj = p+i*size; |
1002 } | 1042 } |
1003 | 1043 |
1004 // Now that we know the object header, reload bits. | 1044 // Now that we know the object header, reload bits. |
1005 » » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | 1045 » » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; |
1006 » » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapW
ord - 1; | 1046 » » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmap
Word - 1; |
1007 shift = off % wordsPerBitmapWord; | 1047 shift = off % wordsPerBitmapWord; |
1008 xbits = *bitp; | 1048 xbits = *bitp; |
1009 bits = xbits >> shift; | 1049 bits = xbits >> shift; |
1010 | 1050 |
1011 // Now we have bits, bitp, and shift correct for | 1051 // Now we have bits, bitp, and shift correct for |
1012 // obj pointing at the base of the object. | 1052 // obj pointing at the base of the object. |
1013 // If not allocated or already marked, done. | 1053 // If not allocated or already marked, done. |
1014 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N
OTE: bitSpecial not bitMarked | 1054 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N
OTE: bitSpecial not bitMarked |
1015 continue; | 1055 continue; |
1016 *bitp |= bitSpecial<<shift; | 1056 *bitp |= bitSpecial<<shift; |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1102 { | 1142 { |
1103 if(b != nil) | 1143 if(b != nil) |
1104 runtime·lfstackpush(&work.full, &b->node); | 1144 runtime·lfstackpush(&work.full, &b->node); |
1105 b = (Workbuf*)runtime·lfstackpop(&work.empty); | 1145 b = (Workbuf*)runtime·lfstackpop(&work.empty); |
1106 if(b == nil) { | 1146 if(b == nil) { |
1107 // Need to allocate. | 1147 // Need to allocate. |
1108 runtime·lock(&work); | 1148 runtime·lock(&work); |
1109 if(work.nchunk < sizeof *b) { | 1149 if(work.nchunk < sizeof *b) { |
1110 work.nchunk = 1<<20; | 1150 work.nchunk = 1<<20; |
1111 work.chunk = runtime·SysAlloc(work.nchunk); | 1151 work.chunk = runtime·SysAlloc(work.nchunk); |
| 1152 if(work.chunk == nil) |
| 1153 runtime·throw("runtime: cannot allocate memory")
; |
1112 } | 1154 } |
1113 b = (Workbuf*)work.chunk; | 1155 b = (Workbuf*)work.chunk; |
1114 work.chunk += sizeof *b; | 1156 work.chunk += sizeof *b; |
1115 work.nchunk -= sizeof *b; | 1157 work.nchunk -= sizeof *b; |
1116 runtime·unlock(&work); | 1158 runtime·unlock(&work); |
1117 } | 1159 } |
1118 b->nobj = 0; | 1160 b->nobj = 0; |
1119 return b; | 1161 return b; |
1120 } | 1162 } |
1121 | 1163 |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1185 addroot(Obj obj) | 1227 addroot(Obj obj) |
1186 { | 1228 { |
1187 uint32 cap; | 1229 uint32 cap; |
1188 Obj *new; | 1230 Obj *new; |
1189 | 1231 |
1190 if(work.nroot >= work.rootcap) { | 1232 if(work.nroot >= work.rootcap) { |
1191 cap = PageSize/sizeof(Obj); | 1233 cap = PageSize/sizeof(Obj); |
1192 if(cap < 2*work.rootcap) | 1234 if(cap < 2*work.rootcap) |
1193 cap = 2*work.rootcap; | 1235 cap = 2*work.rootcap; |
1194 new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj)); | 1236 new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj)); |
| 1237 if(new == nil) |
| 1238 runtime·throw("runtime: cannot allocate memory"); |
1195 if(work.roots != nil) { | 1239 if(work.roots != nil) { |
1196 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); | 1240 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); |
1197 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); | 1241 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); |
1198 } | 1242 } |
1199 work.roots = new; | 1243 work.roots = new; |
1200 work.rootcap = cap; | 1244 work.rootcap = cap; |
1201 } | 1245 } |
1202 work.roots[work.nroot] = obj; | 1246 work.roots[work.nroot] = obj; |
1203 work.nroot++; | 1247 work.nroot++; |
1204 } | 1248 } |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1274 uint32 spanidx; | 1318 uint32 spanidx; |
1275 | 1319 |
1276 work.nroot = 0; | 1320 work.nroot = 0; |
1277 | 1321 |
1278 // data & bss | 1322 // data & bss |
1279 // TODO(atom): load balancing | 1323 // TODO(atom): load balancing |
1280 addroot((Obj){data, edata - data, (uintptr)gcdata}); | 1324 addroot((Obj){data, edata - data, (uintptr)gcdata}); |
1281 addroot((Obj){bss, ebss - bss, (uintptr)gcbss}); | 1325 addroot((Obj){bss, ebss - bss, (uintptr)gcbss}); |
1282 | 1326 |
1283 // MSpan.types | 1327 // MSpan.types |
1284 » allspans = runtime·mheap.allspans; | 1328 » allspans = runtime·mheap->allspans; |
1285 » for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { | 1329 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { |
1286 s = allspans[spanidx]; | 1330 s = allspans[spanidx]; |
1287 if(s->state == MSpanInUse) { | 1331 if(s->state == MSpanInUse) { |
1288 switch(s->types.compression) { | 1332 switch(s->types.compression) { |
1289 case MTypes_Empty: | 1333 case MTypes_Empty: |
1290 case MTypes_Single: | 1334 case MTypes_Single: |
1291 break; | 1335 break; |
1292 case MTypes_Words: | 1336 case MTypes_Words: |
1293 case MTypes_Bytes: | 1337 case MTypes_Bytes: |
1294 // TODO(atom): consider using defaultProg instea
d of 0 | 1338 // TODO(atom): consider using defaultProg instea
d of 0 |
1295 addroot((Obj){(byte*)&s->types.data, sizeof(void
*), 0}); | 1339 addroot((Obj){(byte*)&s->types.data, sizeof(void
*), 0}); |
(...skipping 25 matching lines...) Expand all Loading... |
1321 | 1365 |
1322 runtime·walkfintab(addfinroots); | 1366 runtime·walkfintab(addfinroots); |
1323 | 1367 |
1324 for(fb=allfin; fb; fb=fb->alllink) | 1368 for(fb=allfin; fb; fb=fb->alllink) |
1325 addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0}); | 1369 addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0}); |
1326 } | 1370 } |
1327 | 1371 |
1328 static bool | 1372 static bool |
1329 handlespecial(byte *p, uintptr size) | 1373 handlespecial(byte *p, uintptr size) |
1330 { | 1374 { |
1331 » void (*fn)(void*); | 1375 » FuncVal *fn; |
1332 uintptr nret; | 1376 uintptr nret; |
1333 FinBlock *block; | 1377 FinBlock *block; |
1334 Finalizer *f; | 1378 Finalizer *f; |
1335 | 1379 |
1336 if(!runtime·getfinalizer(p, true, &fn, &nret)) { | 1380 if(!runtime·getfinalizer(p, true, &fn, &nret)) { |
1337 runtime·setblockspecial(p, false); | 1381 runtime·setblockspecial(p, false); |
1338 runtime·MProf_Free(p, size); | 1382 runtime·MProf_Free(p, size); |
1339 return false; | 1383 return false; |
1340 } | 1384 } |
1341 | 1385 |
1342 runtime·lock(&finlock); | 1386 runtime·lock(&finlock); |
1343 if(finq == nil || finq->cnt == finq->cap) { | 1387 if(finq == nil || finq->cnt == finq->cap) { |
1344 if(finc == nil) { | 1388 if(finc == nil) { |
1345 finc = runtime·SysAlloc(PageSize); | 1389 finc = runtime·SysAlloc(PageSize); |
| 1390 if(finc == nil) |
| 1391 runtime·throw("runtime: cannot allocate memory")
; |
1346 finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Final
izer) + 1; | 1392 finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Final
izer) + 1; |
1347 finc->alllink = allfin; | 1393 finc->alllink = allfin; |
1348 allfin = finc; | 1394 allfin = finc; |
1349 } | 1395 } |
1350 block = finc; | 1396 block = finc; |
1351 finc = block->next; | 1397 finc = block->next; |
1352 block->next = finq; | 1398 block->next = finq; |
1353 finq = block; | 1399 finq = block; |
1354 } | 1400 } |
1355 f = &finq->fin[finq->cnt]; | 1401 f = &finq->fin[finq->cnt]; |
(...skipping 16 matching lines...) Expand all Loading... |
1372 MCache *c; | 1418 MCache *c; |
1373 byte *arena_start; | 1419 byte *arena_start; |
1374 MLink head, *end; | 1420 MLink head, *end; |
1375 int32 nfree; | 1421 int32 nfree; |
1376 byte *type_data; | 1422 byte *type_data; |
1377 byte compression; | 1423 byte compression; |
1378 uintptr type_data_inc; | 1424 uintptr type_data_inc; |
1379 MSpan *s; | 1425 MSpan *s; |
1380 | 1426 |
1381 USED(&desc); | 1427 USED(&desc); |
1382 » s = runtime·mheap.allspans[idx]; | 1428 » s = runtime·mheap->allspans[idx]; |
1383 if(s->state != MSpanInUse) | 1429 if(s->state != MSpanInUse) |
1384 return; | 1430 return; |
1385 » arena_start = runtime·mheap.arena_start; | 1431 » arena_start = runtime·mheap->arena_start; |
1386 p = (byte*)(s->start << PageShift); | 1432 p = (byte*)(s->start << PageShift); |
1387 cl = s->sizeclass; | 1433 cl = s->sizeclass; |
1388 size = s->elemsize; | 1434 size = s->elemsize; |
1389 if(cl == 0) { | 1435 if(cl == 0) { |
1390 n = 1; | 1436 n = 1; |
1391 } else { | 1437 } else { |
1392 // Chunk full of small blocks. | 1438 // Chunk full of small blocks. |
1393 npages = runtime·class_to_allocnpages[cl]; | 1439 npages = runtime·class_to_allocnpages[cl]; |
1394 n = (npages << PageShift) / size; | 1440 n = (npages << PageShift) / size; |
1395 } | 1441 } |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1439 continue; | 1485 continue; |
1440 } | 1486 } |
1441 | 1487 |
1442 // Mark freed; restore block boundary bit. | 1488 // Mark freed; restore block boundary bit. |
1443 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 1489 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
1444 | 1490 |
1445 if(cl == 0) { | 1491 if(cl == 0) { |
1446 // Free large span. | 1492 // Free large span. |
1447 runtime·unmarkspan(p, 1<<PageShift); | 1493 runtime·unmarkspan(p, 1<<PageShift); |
1448 *(uintptr*)p = 1; // needs zeroing | 1494 *(uintptr*)p = 1; // needs zeroing |
1449 » » » runtime·MHeap_Free(&runtime·mheap, s, 1); | 1495 » » » runtime·MHeap_Free(runtime·mheap, s, 1); |
1450 c->local_alloc -= size; | 1496 c->local_alloc -= size; |
1451 c->local_nfree++; | 1497 c->local_nfree++; |
1452 } else { | 1498 } else { |
1453 // Free small object. | 1499 // Free small object. |
1454 switch(compression) { | 1500 switch(compression) { |
1455 case MTypes_Words: | 1501 case MTypes_Words: |
1456 *(uintptr*)type_data = 0; | 1502 *(uintptr*)type_data = 0; |
1457 break; | 1503 break; |
1458 case MTypes_Bytes: | 1504 case MTypes_Bytes: |
1459 *(byte*)type_data = 0; | 1505 *(byte*)type_data = 0; |
1460 break; | 1506 break; |
1461 } | 1507 } |
1462 if(size > sizeof(uintptr)) | 1508 if(size > sizeof(uintptr)) |
1463 ((uintptr*)p)[1] = 1; // mark as "needs to be
zeroed" | 1509 ((uintptr*)p)[1] = 1; // mark as "needs to be
zeroed" |
1464 ························ | 1510 ························ |
1465 end->next = (MLink*)p; | 1511 end->next = (MLink*)p; |
1466 end = (MLink*)p; | 1512 end = (MLink*)p; |
1467 nfree++; | 1513 nfree++; |
1468 } | 1514 } |
1469 } | 1515 } |
1470 | 1516 |
1471 if(nfree) { | 1517 if(nfree) { |
1472 c->local_by_size[cl].nfree += nfree; | 1518 c->local_by_size[cl].nfree += nfree; |
1473 c->local_alloc -= size * nfree; | 1519 c->local_alloc -= size * nfree; |
1474 c->local_nfree += nfree; | 1520 c->local_nfree += nfree; |
1475 c->local_cachealloc -= nfree * size; | 1521 c->local_cachealloc -= nfree * size; |
1476 c->local_objects -= nfree; | 1522 c->local_objects -= nfree; |
1477 » » runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, nfree,
head.next, end); | 1523 » » runtime·MCentral_FreeSpan(&runtime·mheap->central[cl], s, nfree,
head.next, end); |
1478 } | 1524 } |
1479 } | 1525 } |
1480 | 1526 |
1481 static void | 1527 static void |
1482 dumpspan(uint32 idx) | 1528 dumpspan(uint32 idx) |
1483 { | 1529 { |
1484 int32 sizeclass, n, npages, i, column; | 1530 int32 sizeclass, n, npages, i, column; |
1485 uintptr size; | 1531 uintptr size; |
1486 byte *p; | 1532 byte *p; |
1487 byte *arena_start; | 1533 byte *arena_start; |
1488 MSpan *s; | 1534 MSpan *s; |
1489 bool allocated, special; | 1535 bool allocated, special; |
1490 | 1536 |
1491 » s = runtime·mheap.allspans[idx]; | 1537 » s = runtime·mheap->allspans[idx]; |
1492 if(s->state != MSpanInUse) | 1538 if(s->state != MSpanInUse) |
1493 return; | 1539 return; |
1494 » arena_start = runtime·mheap.arena_start; | 1540 » arena_start = runtime·mheap->arena_start; |
1495 p = (byte*)(s->start << PageShift); | 1541 p = (byte*)(s->start << PageShift); |
1496 sizeclass = s->sizeclass; | 1542 sizeclass = s->sizeclass; |
1497 size = s->elemsize; | 1543 size = s->elemsize; |
1498 if(sizeclass == 0) { | 1544 if(sizeclass == 0) { |
1499 n = 1; | 1545 n = 1; |
1500 } else { | 1546 } else { |
1501 npages = runtime·class_to_allocnpages[sizeclass]; | 1547 npages = runtime·class_to_allocnpages[sizeclass]; |
1502 n = (npages << PageShift) / size; | 1548 n = (npages << PageShift) / size; |
1503 } | 1549 } |
1504 ········ | 1550 ········ |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1542 } | 1588 } |
1543 runtime·printf("\n"); | 1589 runtime·printf("\n"); |
1544 } | 1590 } |
1545 | 1591 |
1546 // A debugging function to dump the contents of memory | 1592 // A debugging function to dump the contents of memory |
1547 void | 1593 void |
1548 runtime·memorydump(void) | 1594 runtime·memorydump(void) |
1549 { | 1595 { |
1550 uint32 spanidx; | 1596 uint32 spanidx; |
1551 | 1597 |
1552 » for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { | 1598 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { |
1553 dumpspan(spanidx); | 1599 dumpspan(spanidx); |
1554 } | 1600 } |
1555 } | 1601 } |
1556 | 1602 |
1557 void | 1603 void |
1558 runtime·gchelper(void) | 1604 runtime·gchelper(void) |
1559 { | 1605 { |
1560 // parallel mark for over gc roots | 1606 // parallel mark for over gc roots |
1561 runtime·parfordo(work.markfor); | 1607 runtime·parfordo(work.markfor); |
1562 | 1608 |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1594 MCache *c; | 1640 MCache *c; |
1595 P *p, **pp; | 1641 P *p, **pp; |
1596 int32 i; | 1642 int32 i; |
1597 uint64 stacks_inuse; | 1643 uint64 stacks_inuse; |
1598 uint64 *src, *dst; | 1644 uint64 *src, *dst; |
1599 | 1645 |
1600 if(stats) | 1646 if(stats) |
1601 runtime·memclr((byte*)stats, sizeof(*stats)); | 1647 runtime·memclr((byte*)stats, sizeof(*stats)); |
1602 stacks_inuse = 0; | 1648 stacks_inuse = 0; |
1603 for(mp=runtime·allm; mp; mp=mp->alllink) { | 1649 for(mp=runtime·allm; mp; mp=mp->alllink) { |
1604 » » stacks_inuse += mp->stackinuse; | 1650 » » stacks_inuse += mp->stackinuse*FixedStack; |
1605 if(stats) { | 1651 if(stats) { |
1606 src = (uint64*)&mp->gcstats; | 1652 src = (uint64*)&mp->gcstats; |
1607 dst = (uint64*)stats; | 1653 dst = (uint64*)stats; |
1608 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) | 1654 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) |
1609 dst[i] += src[i]; | 1655 dst[i] += src[i]; |
1610 runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats))
; | 1656 runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats))
; |
1611 } | 1657 } |
1612 } | 1658 } |
1613 for(pp=runtime·allp; p=*pp; pp++) { | 1659 for(pp=runtime·allp; p=*pp; pp++) { |
1614 c = p->mcache; | 1660 c = p->mcache; |
(...skipping 30 matching lines...) Expand all Loading... |
1645 if(runtime·strcmp(p, (byte*)"off") == 0) | 1691 if(runtime·strcmp(p, (byte*)"off") == 0) |
1646 return -1; | 1692 return -1; |
1647 return runtime·atoi(p); | 1693 return runtime·atoi(p); |
1648 } | 1694 } |
1649 | 1695 |
1650 void | 1696 void |
1651 runtime·gc(int32 force) | 1697 runtime·gc(int32 force) |
1652 { | 1698 { |
1653 byte *p; | 1699 byte *p; |
1654 struct gc_args a, *ap; | 1700 struct gc_args a, *ap; |
| 1701 FuncVal gcv; |
1655 | 1702 |
1656 // The atomic operations are not atomic if the uint64s | 1703 // The atomic operations are not atomic if the uint64s |
1657 // are not aligned on uint64 boundaries. This has been | 1704 // are not aligned on uint64 boundaries. This has been |
1658 // a problem in the past. | 1705 // a problem in the past. |
1659 if((((uintptr)&work.empty) & 7) != 0) | 1706 if((((uintptr)&work.empty) & 7) != 0) |
1660 runtime·throw("runtime: gc work buffer is misaligned"); | 1707 runtime·throw("runtime: gc work buffer is misaligned"); |
1661 | 1708 |
1662 // The gc is turned off (via enablegc) until | 1709 // The gc is turned off (via enablegc) until |
1663 // the bootstrap has completed. | 1710 // the bootstrap has completed. |
1664 // Also, malloc gets called in the guts | 1711 // Also, malloc gets called in the guts |
(...skipping 13 matching lines...) Expand all Loading... |
1678 gctrace = runtime·atoi(p); | 1725 gctrace = runtime·atoi(p); |
1679 } | 1726 } |
1680 if(gcpercent < 0) | 1727 if(gcpercent < 0) |
1681 return; | 1728 return; |
1682 | 1729 |
1683 // Run gc on a bigger stack to eliminate | 1730 // Run gc on a bigger stack to eliminate |
1684 // a potentially large number of calls to runtime·morestack. | 1731 // a potentially large number of calls to runtime·morestack. |
1685 a.force = force; | 1732 a.force = force; |
1686 ap = &a; | 1733 ap = &a; |
1687 m->moreframesize_minalloc = StackBig; | 1734 m->moreframesize_minalloc = StackBig; |
1688 » reflect·call((byte*)gc, (byte*)&ap, sizeof(ap)); | 1735 » gcv.fn = (void*)gc; |
| 1736 » reflect·call(&gcv, (byte*)&ap, sizeof(ap)); |
1689 | 1737 |
1690 if(gctrace > 1 && !force) { | 1738 if(gctrace > 1 && !force) { |
1691 a.force = 1; | 1739 a.force = 1; |
1692 gc(&a); | 1740 gc(&a); |
1693 } | 1741 } |
1694 } | 1742 } |
| 1743 |
| 1744 static FuncVal runfinqv = {runfinq}; |
1695 | 1745 |
1696 static void | 1746 static void |
1697 gc(struct gc_args *args) | 1747 gc(struct gc_args *args) |
1698 { | 1748 { |
1699 int64 t0, t1, t2, t3, t4; | 1749 int64 t0, t1, t2, t3, t4; |
1700 uint64 heap0, heap1, obj0, obj1; | 1750 uint64 heap0, heap1, obj0, obj1; |
1701 GCStats stats; | 1751 GCStats stats; |
1702 M *mp; | 1752 M *mp; |
1703 uint32 i; | 1753 uint32 i; |
1704 Eface eface; | 1754 Eface eface; |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1737 runtime·gc_itab_ptr(&eface); | 1787 runtime·gc_itab_ptr(&eface); |
1738 itabtype = ((PtrType*)eface.type)->elem; | 1788 itabtype = ((PtrType*)eface.type)->elem; |
1739 } | 1789 } |
1740 | 1790 |
1741 work.nwait = 0; | 1791 work.nwait = 0; |
1742 work.ndone = 0; | 1792 work.ndone = 0; |
1743 work.debugmarkdone = 0; | 1793 work.debugmarkdone = 0; |
1744 work.nproc = runtime·gcprocs(); | 1794 work.nproc = runtime·gcprocs(); |
1745 addroots(); | 1795 addroots(); |
1746 runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma
rkroot); | 1796 runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma
rkroot); |
1747 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil,
true, sweepspan); | 1797 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap->nspan, nil
, true, sweepspan); |
1748 if(work.nproc > 1) { | 1798 if(work.nproc > 1) { |
1749 runtime·noteclear(&work.alldone); | 1799 runtime·noteclear(&work.alldone); |
1750 runtime·helpgc(work.nproc); | 1800 runtime·helpgc(work.nproc); |
1751 } | 1801 } |
1752 | 1802 |
1753 t1 = runtime·nanotime(); | 1803 t1 = runtime·nanotime(); |
1754 | 1804 |
1755 runtime·parfordo(work.markfor); | 1805 runtime·parfordo(work.markfor); |
1756 scanblock(nil, nil, 0, true); | 1806 scanblock(nil, nil, 0, true); |
1757 | 1807 |
(...skipping 16 matching lines...) Expand all Loading... |
1774 stats.nosyield += work.sweepfor->nosyield; | 1824 stats.nosyield += work.sweepfor->nosyield; |
1775 stats.nsleep += work.sweepfor->nsleep; | 1825 stats.nsleep += work.sweepfor->nsleep; |
1776 | 1826 |
1777 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; | 1827 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; |
1778 m->gcing = 0; | 1828 m->gcing = 0; |
1779 | 1829 |
1780 if(finq != nil) { | 1830 if(finq != nil) { |
1781 m->locks++; // disable gc during the mallocs in newproc | 1831 m->locks++; // disable gc during the mallocs in newproc |
1782 // kick off or wake up goroutine to run queued finalizers | 1832 // kick off or wake up goroutine to run queued finalizers |
1783 if(fing == nil) | 1833 if(fing == nil) |
1784 » » » fing = runtime·newproc1((byte*)runfinq, nil, 0, 0, runti
me·gc); | 1834 » » » fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc
); |
1785 else if(fingwait) { | 1835 else if(fingwait) { |
1786 fingwait = 0; | 1836 fingwait = 0; |
1787 runtime·ready(fing); | 1837 runtime·ready(fing); |
1788 } | 1838 } |
1789 m->locks--; | 1839 m->locks--; |
1790 } | 1840 } |
1791 | 1841 |
1792 heap1 = mstats.heap_alloc; | 1842 heap1 = mstats.heap_alloc; |
1793 obj1 = mstats.nmalloc - mstats.nfree; | 1843 obj1 = mstats.nmalloc - mstats.nfree; |
1794 | 1844 |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1842 { | 1892 { |
1843 uint64 *p; | 1893 uint64 *p; |
1844 uint32 i, n; | 1894 uint32 i, n; |
1845 | 1895 |
1846 // Calling code in runtime/debug should make the slice large enough. | 1896 // Calling code in runtime/debug should make the slice large enough. |
1847 if(pauses->cap < nelem(mstats.pause_ns)+3) | 1897 if(pauses->cap < nelem(mstats.pause_ns)+3) |
1848 runtime·throw("runtime: short slice passed to readGCStats"); | 1898 runtime·throw("runtime: short slice passed to readGCStats"); |
1849 | 1899 |
1850 // Pass back: pauses, last gc (absolute time), number of gc, total pause
ns. | 1900 // Pass back: pauses, last gc (absolute time), number of gc, total pause
ns. |
1851 p = (uint64*)pauses->array; | 1901 p = (uint64*)pauses->array; |
1852 » runtime·lock(&runtime·mheap); | 1902 » runtime·lock(runtime·mheap); |
1853 n = mstats.numgc; | 1903 n = mstats.numgc; |
1854 if(n > nelem(mstats.pause_ns)) | 1904 if(n > nelem(mstats.pause_ns)) |
1855 n = nelem(mstats.pause_ns); | 1905 n = nelem(mstats.pause_ns); |
1856 ········ | 1906 ········ |
1857 // The pause buffer is circular. The most recent pause is at | 1907 // The pause buffer is circular. The most recent pause is at |
1858 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward | 1908 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward |
1859 // from there to go back farther in time. We deliver the times | 1909 // from there to go back farther in time. We deliver the times |
1860 // most recent first (in p[0]). | 1910 // most recent first (in p[0]). |
1861 for(i=0; i<n; i++) | 1911 for(i=0; i<n; i++) |
1862 p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)
]; | 1912 p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)
]; |
1863 | 1913 |
1864 p[n] = mstats.last_gc; | 1914 p[n] = mstats.last_gc; |
1865 p[n+1] = mstats.numgc; | 1915 p[n+1] = mstats.numgc; |
1866 p[n+2] = mstats.pause_total_ns;· | 1916 p[n+2] = mstats.pause_total_ns;· |
1867 » runtime·unlock(&runtime·mheap); | 1917 » runtime·unlock(runtime·mheap); |
1868 pauses->len = n+3; | 1918 pauses->len = n+3; |
1869 } | 1919 } |
1870 | 1920 |
1871 void | 1921 void |
1872 runtime∕debug·setGCPercent(intgo in, intgo out) | 1922 runtime∕debug·setGCPercent(intgo in, intgo out) |
1873 { | 1923 { |
1874 » runtime·lock(&runtime·mheap); | 1924 » runtime·lock(runtime·mheap); |
1875 if(gcpercent == GcpercentUnknown) | 1925 if(gcpercent == GcpercentUnknown) |
1876 gcpercent = readgogc(); | 1926 gcpercent = readgogc(); |
1877 out = gcpercent; | 1927 out = gcpercent; |
1878 if(in < 0) | 1928 if(in < 0) |
1879 in = -1; | 1929 in = -1; |
1880 gcpercent = in; | 1930 gcpercent = in; |
1881 » runtime·unlock(&runtime·mheap); | 1931 » runtime·unlock(runtime·mheap); |
1882 FLUSH(&out); | 1932 FLUSH(&out); |
1883 } | 1933 } |
1884 | 1934 |
1885 static void | 1935 static void |
1886 runfinq(void) | 1936 runfinq(void) |
1887 { | 1937 { |
1888 Finalizer *f; | 1938 Finalizer *f; |
1889 FinBlock *fb, *next; | 1939 FinBlock *fb, *next; |
1890 byte *frame; | 1940 byte *frame; |
1891 uint32 framesz, framecap, i; | 1941 uint32 framesz, framecap, i; |
(...skipping 20 matching lines...) Expand all Loading... |
1912 next = fb->next; | 1962 next = fb->next; |
1913 for(i=0; i<fb->cnt; i++) { | 1963 for(i=0; i<fb->cnt; i++) { |
1914 f = &fb->fin[i]; | 1964 f = &fb->fin[i]; |
1915 framesz = sizeof(uintptr) + f->nret; | 1965 framesz = sizeof(uintptr) + f->nret; |
1916 if(framecap < framesz) { | 1966 if(framecap < framesz) { |
1917 runtime·free(frame); | 1967 runtime·free(frame); |
1918 frame = runtime·mal(framesz); | 1968 frame = runtime·mal(framesz); |
1919 framecap = framesz; | 1969 framecap = framesz; |
1920 } | 1970 } |
1921 *(void**)frame = f->arg; | 1971 *(void**)frame = f->arg; |
1922 » » » » reflect·call((byte*)f->fn, frame, sizeof(uintptr
) + f->nret); | 1972 » » » » reflect·call(f->fn, frame, sizeof(uintptr) + f->
nret); |
1923 f->fn = nil; | 1973 f->fn = nil; |
1924 f->arg = nil; | 1974 f->arg = nil; |
1925 } | 1975 } |
1926 fb->cnt = 0; | 1976 fb->cnt = 0; |
1927 fb->next = finc; | 1977 fb->next = finc; |
1928 finc = fb; | 1978 finc = fb; |
1929 } | 1979 } |
1930 runtime·gc(1); // trigger another gc to clean up the finalized
objects, if possible | 1980 runtime·gc(1); // trigger another gc to clean up the finalized
objects, if possible |
1931 } | 1981 } |
1932 } | 1982 } |
1933 | 1983 |
1934 // mark the block at v of size n as allocated. | 1984 // mark the block at v of size n as allocated. |
1935 // If noptr is true, mark it as having no pointers. | 1985 // If noptr is true, mark it as having no pointers. |
1936 void | 1986 void |
1937 runtime·markallocated(void *v, uintptr n, bool noptr) | 1987 runtime·markallocated(void *v, uintptr n, bool noptr) |
1938 { | 1988 { |
1939 uintptr *b, obits, bits, off, shift; | 1989 uintptr *b, obits, bits, off, shift; |
1940 | 1990 |
1941 if(0) | 1991 if(0) |
1942 runtime·printf("markallocated %p+%p\n", v, n); | 1992 runtime·printf("markallocated %p+%p\n", v, n); |
1943 | 1993 |
1944 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) | 1994 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) |
1945 runtime·throw("markallocated: bad pointer"); | 1995 runtime·throw("markallocated: bad pointer"); |
1946 | 1996 |
1947 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 1997 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse
t |
1948 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 1998 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
1949 shift = off % wordsPerBitmapWord; | 1999 shift = off % wordsPerBitmapWord; |
1950 | 2000 |
1951 for(;;) { | 2001 for(;;) { |
1952 obits = *b; | 2002 obits = *b; |
1953 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); | 2003 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); |
1954 if(noptr) | 2004 if(noptr) |
1955 bits |= bitNoPointers<<shift; | 2005 bits |= bitNoPointers<<shift; |
1956 if(runtime·singleproc) { | 2006 if(runtime·singleproc) { |
1957 *b = bits; | 2007 *b = bits; |
1958 break; | 2008 break; |
1959 } else { | 2009 } else { |
1960 // more than one goroutine is potentially running: use a
tomic op | 2010 // more than one goroutine is potentially running: use a
tomic op |
1961 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2011 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
1962 break; | 2012 break; |
1963 } | 2013 } |
1964 } | 2014 } |
1965 } | 2015 } |
1966 | 2016 |
1967 // mark the block at v of size n as freed. | 2017 // mark the block at v of size n as freed. |
1968 void | 2018 void |
1969 runtime·markfreed(void *v, uintptr n) | 2019 runtime·markfreed(void *v, uintptr n) |
1970 { | 2020 { |
1971 uintptr *b, obits, bits, off, shift; | 2021 uintptr *b, obits, bits, off, shift; |
1972 | 2022 |
1973 if(0) | 2023 if(0) |
1974 runtime·printf("markallocated %p+%p\n", v, n); | 2024 runtime·printf("markallocated %p+%p\n", v, n); |
1975 | 2025 |
1976 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) | 2026 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) |
1977 runtime·throw("markallocated: bad pointer"); | 2027 runtime·throw("markallocated: bad pointer"); |
1978 | 2028 |
1979 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 2029 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse
t |
1980 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2030 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
1981 shift = off % wordsPerBitmapWord; | 2031 shift = off % wordsPerBitmapWord; |
1982 | 2032 |
1983 for(;;) { | 2033 for(;;) { |
1984 obits = *b; | 2034 obits = *b; |
1985 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2035 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
1986 if(runtime·singleproc) { | 2036 if(runtime·singleproc) { |
1987 *b = bits; | 2037 *b = bits; |
1988 break; | 2038 break; |
1989 } else { | 2039 } else { |
1990 // more than one goroutine is potentially running: use a
tomic op | 2040 // more than one goroutine is potentially running: use a
tomic op |
1991 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2041 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
1992 break; | 2042 break; |
1993 } | 2043 } |
1994 } | 2044 } |
1995 } | 2045 } |
1996 | 2046 |
1997 // check that the block at v of size n is marked freed. | 2047 // check that the block at v of size n is marked freed. |
1998 void | 2048 void |
1999 runtime·checkfreed(void *v, uintptr n) | 2049 runtime·checkfreed(void *v, uintptr n) |
2000 { | 2050 { |
2001 uintptr *b, bits, off, shift; | 2051 uintptr *b, bits, off, shift; |
2002 | 2052 |
2003 if(!runtime·checking) | 2053 if(!runtime·checking) |
2004 return; | 2054 return; |
2005 | 2055 |
2006 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) | 2056 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) |
2007 return; // not allocated, so okay | 2057 return; // not allocated, so okay |
2008 | 2058 |
2009 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 2059 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse
t |
2010 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2060 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2011 shift = off % wordsPerBitmapWord; | 2061 shift = off % wordsPerBitmapWord; |
2012 | 2062 |
2013 bits = *b>>shift; | 2063 bits = *b>>shift; |
2014 if((bits & bitAllocated) != 0) { | 2064 if((bits & bitAllocated) != 0) { |
2015 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", | 2065 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", |
2016 v, n, off, bits & bitMask); | 2066 v, n, off, bits & bitMask); |
2017 runtime·throw("checkfreed: not freed"); | 2067 runtime·throw("checkfreed: not freed"); |
2018 } | 2068 } |
2019 } | 2069 } |
2020 | 2070 |
2021 // mark the span of memory at v as having n blocks of the given size. | 2071 // mark the span of memory at v as having n blocks of the given size. |
2022 // if leftover is true, there is left over space at the end of the span. | 2072 // if leftover is true, there is left over space at the end of the span. |
2023 void | 2073 void |
2024 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) | 2074 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) |
2025 { | 2075 { |
2026 uintptr *b, off, shift; | 2076 uintptr *b, off, shift; |
2027 byte *p; | 2077 byte *p; |
2028 | 2078 |
2029 » if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runti
me·mheap.arena_start) | 2079 » if((byte*)v+size*n > (byte*)runtime·mheap->arena_used || (byte*)v < runt
ime·mheap->arena_start) |
2030 runtime·throw("markspan: bad pointer"); | 2080 runtime·throw("markspan: bad pointer"); |
2031 | 2081 |
2032 p = v; | 2082 p = v; |
2033 if(leftover) // mark a boundary just past end of last block too | 2083 if(leftover) // mark a boundary just past end of last block too |
2034 n++; | 2084 n++; |
2035 for(; n-- > 0; p += size) { | 2085 for(; n-- > 0; p += size) { |
2036 // Okay to use non-atomic ops here, because we control | 2086 // Okay to use non-atomic ops here, because we control |
2037 // the entire span, and each bitmap word has bits for only | 2087 // the entire span, and each bitmap word has bits for only |
2038 // one span, so no other goroutines are changing these | 2088 // one span, so no other goroutines are changing these |
2039 // bitmap words. | 2089 // bitmap words. |
2040 » » off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // wor
d offset | 2090 » » off = (uintptr*)p - (uintptr*)runtime·mheap->arena_start; // wo
rd offset |
2041 » » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord
- 1; | 2091 » » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWor
d - 1; |
2042 shift = off % wordsPerBitmapWord; | 2092 shift = off % wordsPerBitmapWord; |
2043 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2093 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
2044 } | 2094 } |
2045 } | 2095 } |
2046 | 2096 |
2047 // unmark the span of memory at v of length n bytes. | 2097 // unmark the span of memory at v of length n bytes. |
2048 void | 2098 void |
2049 runtime·unmarkspan(void *v, uintptr n) | 2099 runtime·unmarkspan(void *v, uintptr n) |
2050 { | 2100 { |
2051 uintptr *p, *b, off; | 2101 uintptr *p, *b, off; |
2052 | 2102 |
2053 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) | 2103 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) |
2054 runtime·throw("markspan: bad pointer"); | 2104 runtime·throw("markspan: bad pointer"); |
2055 | 2105 |
2056 p = v; | 2106 p = v; |
2057 » off = p - (uintptr*)runtime·mheap.arena_start; // word offset | 2107 » off = p - (uintptr*)runtime·mheap->arena_start; // word offset |
2058 if(off % wordsPerBitmapWord != 0) | 2108 if(off % wordsPerBitmapWord != 0) |
2059 runtime·throw("markspan: unaligned pointer"); | 2109 runtime·throw("markspan: unaligned pointer"); |
2060 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2110 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2061 n /= PtrSize; | 2111 n /= PtrSize; |
2062 if(n%wordsPerBitmapWord != 0) | 2112 if(n%wordsPerBitmapWord != 0) |
2063 runtime·throw("unmarkspan: unaligned length"); | 2113 runtime·throw("unmarkspan: unaligned length"); |
2064 // Okay to use non-atomic ops here, because we control | 2114 // Okay to use non-atomic ops here, because we control |
2065 // the entire span, and each bitmap word has bits for only | 2115 // the entire span, and each bitmap word has bits for only |
2066 // one span, so no other goroutines are changing these | 2116 // one span, so no other goroutines are changing these |
2067 // bitmap words. | 2117 // bitmap words. |
2068 n /= wordsPerBitmapWord; | 2118 n /= wordsPerBitmapWord; |
2069 while(n-- > 0) | 2119 while(n-- > 0) |
2070 *b-- = 0; | 2120 *b-- = 0; |
2071 } | 2121 } |
2072 | 2122 |
2073 bool | 2123 bool |
2074 runtime·blockspecial(void *v) | 2124 runtime·blockspecial(void *v) |
2075 { | 2125 { |
2076 uintptr *b, off, shift; | 2126 uintptr *b, off, shift; |
2077 | 2127 |
2078 if(DebugMark) | 2128 if(DebugMark) |
2079 return true; | 2129 return true; |
2080 | 2130 |
2081 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; | 2131 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; |
2082 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2132 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2083 shift = off % wordsPerBitmapWord; | 2133 shift = off % wordsPerBitmapWord; |
2084 | 2134 |
2085 return (*b & (bitSpecial<<shift)) != 0; | 2135 return (*b & (bitSpecial<<shift)) != 0; |
2086 } | 2136 } |
2087 | 2137 |
2088 void | 2138 void |
2089 runtime·setblockspecial(void *v, bool s) | 2139 runtime·setblockspecial(void *v, bool s) |
2090 { | 2140 { |
2091 uintptr *b, off, shift, bits, obits; | 2141 uintptr *b, off, shift, bits, obits; |
2092 | 2142 |
2093 if(DebugMark) | 2143 if(DebugMark) |
2094 return; | 2144 return; |
2095 | 2145 |
2096 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; | 2146 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; |
2097 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2147 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2098 shift = off % wordsPerBitmapWord; | 2148 shift = off % wordsPerBitmapWord; |
2099 | 2149 |
2100 for(;;) { | 2150 for(;;) { |
2101 obits = *b; | 2151 obits = *b; |
2102 if(s) | 2152 if(s) |
2103 bits = obits | (bitSpecial<<shift); | 2153 bits = obits | (bitSpecial<<shift); |
2104 else | 2154 else |
2105 bits = obits & ~(bitSpecial<<shift); | 2155 bits = obits & ~(bitSpecial<<shift); |
2106 if(runtime·singleproc) { | 2156 if(runtime·singleproc) { |
2107 *b = bits; | 2157 *b = bits; |
(...skipping 18 matching lines...) Expand all Loading... |
2126 uintptr n; | 2176 uintptr n; |
2127 | 2177 |
2128 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; | 2178 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; |
2129 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); | 2179 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); |
2130 if(h->bitmap_mapped >= n) | 2180 if(h->bitmap_mapped >= n) |
2131 return; | 2181 return; |
2132 | 2182 |
2133 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); | 2183 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); |
2134 h->bitmap_mapped = n; | 2184 h->bitmap_mapped = n; |
2135 } | 2185 } |
LEFT | RIGHT |