LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Garbage collector. | 5 // Garbage collector. |
6 | 6 |
7 #include "runtime.h" | 7 #include "runtime.h" |
8 #include "arch_GOARCH.h" | 8 #include "arch_GOARCH.h" |
9 #include "malloc.h" | 9 #include "malloc.h" |
10 #include "stack.h" | 10 #include "stack.h" |
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
97 LFNode node; // must be first | 97 LFNode node; // must be first |
98 uintptr nobj; | 98 uintptr nobj; |
99 Obj obj[SIZE/sizeof(Obj) - 1]; | 99 Obj obj[SIZE/sizeof(Obj) - 1]; |
100 uint8 _padding[SIZE%sizeof(Obj) + sizeof(Obj)]; | 100 uint8 _padding[SIZE%sizeof(Obj) + sizeof(Obj)]; |
101 #undef SIZE | 101 #undef SIZE |
102 }; | 102 }; |
103 | 103 |
104 typedef struct Finalizer Finalizer; | 104 typedef struct Finalizer Finalizer; |
105 struct Finalizer | 105 struct Finalizer |
106 { | 106 { |
107 » void (*fn)(void*); | 107 » FuncVal *fn; |
108 void *arg; | 108 void *arg; |
109 uintptr nret; | 109 uintptr nret; |
110 }; | 110 }; |
111 | 111 |
112 typedef struct FinBlock FinBlock; | 112 typedef struct FinBlock FinBlock; |
113 struct FinBlock | 113 struct FinBlock |
114 { | 114 { |
115 FinBlock *alllink; | 115 FinBlock *alllink; |
116 FinBlock *next; | 116 FinBlock *next; |
117 int32 cnt; | 117 int32 cnt; |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
157 uintptr nchunk; | 157 uintptr nchunk; |
158 | 158 |
159 Obj *roots; | 159 Obj *roots; |
160 uint32 nroot; | 160 uint32 nroot; |
161 uint32 rootcap; | 161 uint32 rootcap; |
162 } work; | 162 } work; |
163 | 163 |
164 enum { | 164 enum { |
165 GC_DEFAULT_PTR = GC_NUM_INSTR, | 165 GC_DEFAULT_PTR = GC_NUM_INSTR, |
166 GC_MAP_NEXT, | 166 GC_MAP_NEXT, |
| 167 GC_CHAN, |
167 }; | 168 }; |
168 | 169 |
169 // markonly marks an object. It returns true if the object | 170 // markonly marks an object. It returns true if the object |
170 // has been marked by this function, false otherwise. | 171 // has been marked by this function, false otherwise. |
171 // This function isn't thread-safe and doesn't append the object to any buffer. | 172 // This function isn't thread-safe and doesn't append the object to any buffer. |
172 static bool | 173 static bool |
173 markonly(void *obj) | 174 markonly(void *obj) |
174 { | 175 { |
175 byte *p; | 176 byte *p; |
176 uintptr *bitp, bits, shift, x, xbits, off; | 177 uintptr *bitp, bits, shift, x, xbits, off; |
(...skipping 337 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
514 *_wbuf = wbuf; | 515 *_wbuf = wbuf; |
515 *_nobj = nobj; | 516 *_nobj = nobj; |
516 } | 517 } |
517 | 518 |
518 // Program that scans the whole block and treats every block element as a potent
ial pointer | 519 // Program that scans the whole block and treats every block element as a potent
ial pointer |
519 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; | 520 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; |
520 | 521 |
521 // Hashmap iterator program | 522 // Hashmap iterator program |
522 static uintptr mapProg[2] = {0, GC_MAP_NEXT}; | 523 static uintptr mapProg[2] = {0, GC_MAP_NEXT}; |
523 | 524 |
| 525 // Hchan program |
| 526 static uintptr chanProg[2] = {0, GC_CHAN}; |
| 527 |
524 // Local variables of a program fragment or loop | 528 // Local variables of a program fragment or loop |
525 typedef struct Frame Frame; | 529 typedef struct Frame Frame; |
526 struct Frame { | 530 struct Frame { |
527 uintptr count, elemsize, b; | 531 uintptr count, elemsize, b; |
528 uintptr *loop_or_ret; | 532 uintptr *loop_or_ret; |
529 }; | 533 }; |
530 | 534 |
531 // scanblock scans a block of n bytes starting at pointer b for references | 535 // scanblock scans a block of n bytes starting at pointer b for references |
532 // to other objects, scanning any it finds recursively until there are no | 536 // to other objects, scanning any it finds recursively until there are no |
533 // unscanned objects left. Instead of using an explicit recursion, it keeps | 537 // unscanned objects left. Instead of using an explicit recursion, it keeps |
534 // a work list in the Workbuf* structures and loops in the main function | 538 // a work list in the Workbuf* structures and loops in the main function |
535 // body. Keeping an explicit work list is easier on the stack allocator and | 539 // body. Keeping an explicit work list is easier on the stack allocator and |
536 // more efficient. | 540 // more efficient. |
537 // | 541 // |
538 // wbuf: current work buffer | 542 // wbuf: current work buffer |
539 // wp: storage for next queued pointer (write pointer) | 543 // wp: storage for next queued pointer (write pointer) |
540 // nobj: number of queued objects | 544 // nobj: number of queued objects |
541 static void | 545 static void |
542 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) | 546 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) |
543 { | 547 { |
544 byte *b, *arena_start, *arena_used; | 548 byte *b, *arena_start, *arena_used; |
545 » uintptr n, i, end_b, elemsize, ti, objti, count, type; | 549 » uintptr n, i, end_b, elemsize, size, ti, objti, count, type; |
546 uintptr *pc, precise_type, nominal_size; | 550 uintptr *pc, precise_type, nominal_size; |
547 uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti; | 551 uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti; |
548 void *obj; | 552 void *obj; |
549 Type *t; | 553 Type *t; |
550 Slice *sliceptr; | 554 Slice *sliceptr; |
551 Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; | 555 Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; |
552 BufferList *scanbuffers; | 556 BufferList *scanbuffers; |
553 PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; | 557 PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; |
554 BitTarget *bitbuf; | 558 BitTarget *bitbuf; |
555 Obj *objbuf, *objbuf_end, *objbufpos; | 559 Obj *objbuf, *objbuf_end, *objbufpos; |
556 Eface *eface; | 560 Eface *eface; |
557 Iface *iface; | 561 Iface *iface; |
558 Hmap *hmap; | 562 Hmap *hmap; |
559 MapType *maptype; | 563 MapType *maptype; |
560 bool didmark, mapkey_kind, mapval_kind; | 564 bool didmark, mapkey_kind, mapval_kind; |
561 struct hash_gciter map_iter; | 565 struct hash_gciter map_iter; |
562 struct hash_gciter_data d; | 566 struct hash_gciter_data d; |
| 567 Hchan *chan; |
| 568 ChanType *chantype; |
563 | 569 |
564 if(sizeof(Workbuf) % PageSize != 0) | 570 if(sizeof(Workbuf) % PageSize != 0) |
565 runtime·throw("scanblock: size of Workbuf is suboptimal"); | 571 runtime·throw("scanblock: size of Workbuf is suboptimal"); |
566 | 572 |
567 // Memory arena parameters. | 573 // Memory arena parameters. |
568 arena_start = runtime·mheap->arena_start; | 574 arena_start = runtime·mheap->arena_start; |
569 arena_used = runtime·mheap->arena_used; | 575 arena_used = runtime·mheap->arena_used; |
570 | 576 |
571 stack_ptr = stack+nelem(stack)-1; | 577 stack_ptr = stack+nelem(stack)-1; |
572 ········ | 578 ········ |
573 precise_type = false; | 579 precise_type = false; |
574 nominal_size = 0; | 580 nominal_size = 0; |
575 | 581 |
576 // Allocate ptrbuf, bitbuf | 582 // Allocate ptrbuf, bitbuf |
577 { | 583 { |
578 runtime·lock(&lock); | 584 runtime·lock(&lock); |
579 | 585 |
580 if(bufferList == nil) { | 586 if(bufferList == nil) { |
581 bufferList = runtime·SysAlloc(sizeof(*bufferList)); | 587 bufferList = runtime·SysAlloc(sizeof(*bufferList)); |
| 588 if(bufferList == nil) |
| 589 runtime·throw("runtime: cannot allocate memory")
; |
582 bufferList->next = nil; | 590 bufferList->next = nil; |
583 } | 591 } |
584 scanbuffers = bufferList; | 592 scanbuffers = bufferList; |
585 bufferList = bufferList->next; | 593 bufferList = bufferList->next; |
586 | 594 |
587 ptrbuf = &scanbuffers->ptrtarget[0]; | 595 ptrbuf = &scanbuffers->ptrtarget[0]; |
588 ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptr
target); | 596 ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptr
target); |
589 bitbuf = &scanbuffers->bittarget[0]; | 597 bitbuf = &scanbuffers->bittarget[0]; |
590 objbuf = &scanbuffers->obj[0]; | 598 objbuf = &scanbuffers->obj[0]; |
591 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); | 599 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); |
592 | 600 |
593 runtime·unlock(&lock); | 601 runtime·unlock(&lock); |
594 } | 602 } |
595 | 603 |
596 ptrbufpos = ptrbuf; | 604 ptrbufpos = ptrbuf; |
597 objbufpos = objbuf; | 605 objbufpos = objbuf; |
598 | 606 |
599 // (Silence the compiler) | 607 // (Silence the compiler) |
600 map_ret = nil; | 608 map_ret = nil; |
601 mapkey_size = mapval_size = 0; | 609 mapkey_size = mapval_size = 0; |
602 mapkey_kind = mapval_kind = false; | 610 mapkey_kind = mapval_kind = false; |
603 mapkey_ti = mapval_ti = 0; | 611 mapkey_ti = mapval_ti = 0; |
| 612 chan = nil; |
| 613 chantype = nil; |
604 | 614 |
605 goto next_block; | 615 goto next_block; |
606 | 616 |
607 for(;;) { | 617 for(;;) { |
608 // Each iteration scans the block b of length n, queueing pointe
rs in | 618 // Each iteration scans the block b of length n, queueing pointe
rs in |
609 // the work buffer. | 619 // the work buffer. |
610 if(Debug > 1) { | 620 if(Debug > 1) { |
611 runtime·printf("scanblock %p %D\n", b, (int64)n); | 621 runtime·printf("scanblock %p %D\n", b, (int64)n); |
612 } | 622 } |
613 | 623 |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
653 mapval_size = maptype->elem->siz
e; | 663 mapval_size = maptype->elem->siz
e; |
654 mapval_kind = maptype->elem->kin
d; | 664 mapval_kind = maptype->elem->kin
d; |
655 mapval_ti = (uintptr)maptype->
elem->gc | PRECISE; | 665 mapval_ti = (uintptr)maptype->
elem->gc | PRECISE; |
656 | 666 |
657 map_ret = 0; | 667 map_ret = 0; |
658 pc = mapProg; | 668 pc = mapProg; |
659 } else { | 669 } else { |
660 goto next_block; | 670 goto next_block; |
661 } | 671 } |
662 break; | 672 break; |
| 673 case TypeInfo_Chan: |
| 674 chan = (Hchan*)b; |
| 675 chantype = (ChanType*)t; |
| 676 pc = chanProg; |
| 677 break; |
663 default: | 678 default: |
664 runtime·throw("scanblock: invalid type")
; | 679 runtime·throw("scanblock: invalid type")
; |
665 return; | 680 return; |
666 } | 681 } |
667 } else { | 682 } else { |
668 pc = defaultProg; | 683 pc = defaultProg; |
669 } | 684 } |
670 } else { | 685 } else { |
671 pc = defaultProg; | 686 pc = defaultProg; |
672 } | 687 } |
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
817 // Stack pop. | 832 // Stack pop. |
818 stack_top = *(++stack_ptr); | 833 stack_top = *(++stack_ptr); |
819 pc += 1; | 834 pc += 1; |
820 } | 835 } |
821 continue; | 836 continue; |
822 | 837 |
823 case GC_CALL: | 838 case GC_CALL: |
824 // Stack push. | 839 // Stack push. |
825 *stack_ptr-- = stack_top; | 840 *stack_ptr-- = stack_top; |
826 stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*re
turn address*/}; | 841 stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*re
turn address*/}; |
827 » » » pc = (uintptr*)((byte*)pc + (uintptr)pc[2]); // target
of the CALL instruction | 842 » » » pc = (uintptr*)((byte*)pc + *(int32*)(pc+2)); // target
of the CALL instruction |
828 continue; | 843 continue; |
829 | 844 |
830 case GC_MAP_PTR: | 845 case GC_MAP_PTR: |
831 hmap = *(Hmap**)(stack_top.b + pc[1]); | 846 hmap = *(Hmap**)(stack_top.b + pc[1]); |
832 if(hmap == nil) { | 847 if(hmap == nil) { |
833 pc += 3; | 848 pc += 3; |
834 continue; | 849 continue; |
835 } | 850 } |
836 runtime·lock(&lock); | 851 runtime·lock(&lock); |
837 didmark = markonly(hmap); | 852 didmark = markonly(hmap); |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
885 *ptrbufpos++ = (PtrTarge
t){*(void**)d.val_data, mapval_ti}; | 900 *ptrbufpos++ = (PtrTarge
t){*(void**)d.val_data, mapval_ti}; |
886 } | 901 } |
887 } | 902 } |
888 } | 903 } |
889 if(map_ret == 0) | 904 if(map_ret == 0) |
890 goto next_block; | 905 goto next_block; |
891 pc = map_ret; | 906 pc = map_ret; |
892 continue; | 907 continue; |
893 | 908 |
894 case GC_REGION: | 909 case GC_REGION: |
895 // TODO(atom): to be expanded in a next CL. Same as GC_A
PTR for now. | |
896 obj = (void*)(stack_top.b + pc[1]); | 910 obj = (void*)(stack_top.b + pc[1]); |
| 911 size = pc[2]; |
| 912 objti = pc[3]; |
897 pc += 4; | 913 pc += 4; |
| 914 |
| 915 *objbufpos++ = (Obj){obj, size, objti}; |
| 916 if(objbufpos == objbuf_end) |
| 917 flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nob
j); |
898 break; | 918 break; |
| 919 |
| 920 case GC_CHAN: |
| 921 // There are no heap pointers in struct Hchan, |
| 922 // so we can ignore the leading sizeof(Hchan) bytes. |
| 923 if(!(chantype->elem->kind & KindNoPointers)) { |
| 924 // Channel's buffer follows Hchan immediately in
memory. |
| 925 // Size of buffer (cap(c)) is second int in the
chan struct. |
| 926 n = ((uintgo*)chan)[1]; |
| 927 if(n > 0) { |
| 928 // TODO(atom): split into two chunks so
that only the |
| 929 // in-use part of the circular buffer is
scanned. |
| 930 // (Channel routines zero the unused par
t, so the current |
| 931 // code does not lead to leaks, it's jus
t a little inefficient.) |
| 932 *objbufpos++ = (Obj){(byte*)chan+runtime
·Hchansize, n*chantype->elem->size, |
| 933 (uintptr)chantype->elem->gc | PR
ECISE | LOOP}; |
| 934 if(objbufpos == objbuf_end) |
| 935 flushobjbuf(objbuf, &objbufpos,
&wp, &wbuf, &nobj); |
| 936 } |
| 937 } |
| 938 goto next_block; |
899 | 939 |
900 default: | 940 default: |
901 runtime·throw("scanblock: invalid GC instruction"); | 941 runtime·throw("scanblock: invalid GC instruction"); |
902 return; | 942 return; |
903 } | 943 } |
904 | 944 |
905 if(obj >= arena_start && obj < arena_used) { | 945 if(obj >= arena_start && obj < arena_used) { |
906 *ptrbufpos++ = (PtrTarget){obj, objti}; | 946 *ptrbufpos++ = (PtrTarget){obj, objti}; |
907 if(ptrbufpos == ptrbuf_end) | 947 if(ptrbufpos == ptrbuf_end) |
908 flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nob
j, bitbuf); | 948 flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nob
j, bitbuf); |
(...skipping 193 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1102 { | 1142 { |
1103 if(b != nil) | 1143 if(b != nil) |
1104 runtime·lfstackpush(&work.full, &b->node); | 1144 runtime·lfstackpush(&work.full, &b->node); |
1105 b = (Workbuf*)runtime·lfstackpop(&work.empty); | 1145 b = (Workbuf*)runtime·lfstackpop(&work.empty); |
1106 if(b == nil) { | 1146 if(b == nil) { |
1107 // Need to allocate. | 1147 // Need to allocate. |
1108 runtime·lock(&work); | 1148 runtime·lock(&work); |
1109 if(work.nchunk < sizeof *b) { | 1149 if(work.nchunk < sizeof *b) { |
1110 work.nchunk = 1<<20; | 1150 work.nchunk = 1<<20; |
1111 work.chunk = runtime·SysAlloc(work.nchunk); | 1151 work.chunk = runtime·SysAlloc(work.nchunk); |
| 1152 if(work.chunk == nil) |
| 1153 runtime·throw("runtime: cannot allocate memory")
; |
1112 } | 1154 } |
1113 b = (Workbuf*)work.chunk; | 1155 b = (Workbuf*)work.chunk; |
1114 work.chunk += sizeof *b; | 1156 work.chunk += sizeof *b; |
1115 work.nchunk -= sizeof *b; | 1157 work.nchunk -= sizeof *b; |
1116 runtime·unlock(&work); | 1158 runtime·unlock(&work); |
1117 } | 1159 } |
1118 b->nobj = 0; | 1160 b->nobj = 0; |
1119 return b; | 1161 return b; |
1120 } | 1162 } |
1121 | 1163 |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1185 addroot(Obj obj) | 1227 addroot(Obj obj) |
1186 { | 1228 { |
1187 uint32 cap; | 1229 uint32 cap; |
1188 Obj *new; | 1230 Obj *new; |
1189 | 1231 |
1190 if(work.nroot >= work.rootcap) { | 1232 if(work.nroot >= work.rootcap) { |
1191 cap = PageSize/sizeof(Obj); | 1233 cap = PageSize/sizeof(Obj); |
1192 if(cap < 2*work.rootcap) | 1234 if(cap < 2*work.rootcap) |
1193 cap = 2*work.rootcap; | 1235 cap = 2*work.rootcap; |
1194 new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj)); | 1236 new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj)); |
| 1237 if(new == nil) |
| 1238 runtime·throw("runtime: cannot allocate memory"); |
1195 if(work.roots != nil) { | 1239 if(work.roots != nil) { |
1196 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); | 1240 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); |
1197 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); | 1241 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); |
1198 } | 1242 } |
1199 work.roots = new; | 1243 work.roots = new; |
1200 work.rootcap = cap; | 1244 work.rootcap = cap; |
1201 } | 1245 } |
1202 work.roots[work.nroot] = obj; | 1246 work.roots[work.nroot] = obj; |
1203 work.nroot++; | 1247 work.nroot++; |
1204 } | 1248 } |
(...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1321 | 1365 |
1322 runtime·walkfintab(addfinroots); | 1366 runtime·walkfintab(addfinroots); |
1323 | 1367 |
1324 for(fb=allfin; fb; fb=fb->alllink) | 1368 for(fb=allfin; fb; fb=fb->alllink) |
1325 addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0}); | 1369 addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0}); |
1326 } | 1370 } |
1327 | 1371 |
1328 static bool | 1372 static bool |
1329 handlespecial(byte *p, uintptr size) | 1373 handlespecial(byte *p, uintptr size) |
1330 { | 1374 { |
1331 » void (*fn)(void*); | 1375 » FuncVal *fn; |
1332 uintptr nret; | 1376 uintptr nret; |
1333 FinBlock *block; | 1377 FinBlock *block; |
1334 Finalizer *f; | 1378 Finalizer *f; |
1335 | 1379 |
1336 if(!runtime·getfinalizer(p, true, &fn, &nret)) { | 1380 if(!runtime·getfinalizer(p, true, &fn, &nret)) { |
1337 runtime·setblockspecial(p, false); | 1381 runtime·setblockspecial(p, false); |
1338 runtime·MProf_Free(p, size); | 1382 runtime·MProf_Free(p, size); |
1339 return false; | 1383 return false; |
1340 } | 1384 } |
1341 | 1385 |
1342 runtime·lock(&finlock); | 1386 runtime·lock(&finlock); |
1343 if(finq == nil || finq->cnt == finq->cap) { | 1387 if(finq == nil || finq->cnt == finq->cap) { |
1344 if(finc == nil) { | 1388 if(finc == nil) { |
1345 finc = runtime·SysAlloc(PageSize); | 1389 finc = runtime·SysAlloc(PageSize); |
| 1390 if(finc == nil) |
| 1391 runtime·throw("runtime: cannot allocate memory")
; |
1346 finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Final
izer) + 1; | 1392 finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Final
izer) + 1; |
1347 finc->alllink = allfin; | 1393 finc->alllink = allfin; |
1348 allfin = finc; | 1394 allfin = finc; |
1349 } | 1395 } |
1350 block = finc; | 1396 block = finc; |
1351 finc = block->next; | 1397 finc = block->next; |
1352 block->next = finq; | 1398 block->next = finq; |
1353 finq = block; | 1399 finq = block; |
1354 } | 1400 } |
1355 f = &finq->fin[finq->cnt]; | 1401 f = &finq->fin[finq->cnt]; |
(...skipping 238 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1594 MCache *c; | 1640 MCache *c; |
1595 P *p, **pp; | 1641 P *p, **pp; |
1596 int32 i; | 1642 int32 i; |
1597 uint64 stacks_inuse; | 1643 uint64 stacks_inuse; |
1598 uint64 *src, *dst; | 1644 uint64 *src, *dst; |
1599 | 1645 |
1600 if(stats) | 1646 if(stats) |
1601 runtime·memclr((byte*)stats, sizeof(*stats)); | 1647 runtime·memclr((byte*)stats, sizeof(*stats)); |
1602 stacks_inuse = 0; | 1648 stacks_inuse = 0; |
1603 for(mp=runtime·allm; mp; mp=mp->alllink) { | 1649 for(mp=runtime·allm; mp; mp=mp->alllink) { |
1604 » » stacks_inuse += mp->stackinuse; | 1650 » » stacks_inuse += mp->stackinuse*FixedStack; |
1605 if(stats) { | 1651 if(stats) { |
1606 src = (uint64*)&mp->gcstats; | 1652 src = (uint64*)&mp->gcstats; |
1607 dst = (uint64*)stats; | 1653 dst = (uint64*)stats; |
1608 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) | 1654 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) |
1609 dst[i] += src[i]; | 1655 dst[i] += src[i]; |
1610 runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats))
; | 1656 runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats))
; |
1611 } | 1657 } |
1612 } | 1658 } |
1613 for(pp=runtime·allp; p=*pp; pp++) { | 1659 for(pp=runtime·allp; p=*pp; pp++) { |
1614 c = p->mcache; | 1660 c = p->mcache; |
(...skipping 30 matching lines...) Expand all Loading... |
1645 if(runtime·strcmp(p, (byte*)"off") == 0) | 1691 if(runtime·strcmp(p, (byte*)"off") == 0) |
1646 return -1; | 1692 return -1; |
1647 return runtime·atoi(p); | 1693 return runtime·atoi(p); |
1648 } | 1694 } |
1649 | 1695 |
1650 void | 1696 void |
1651 runtime·gc(int32 force) | 1697 runtime·gc(int32 force) |
1652 { | 1698 { |
1653 byte *p; | 1699 byte *p; |
1654 struct gc_args a, *ap; | 1700 struct gc_args a, *ap; |
| 1701 FuncVal gcv; |
1655 | 1702 |
1656 // The atomic operations are not atomic if the uint64s | 1703 // The atomic operations are not atomic if the uint64s |
1657 // are not aligned on uint64 boundaries. This has been | 1704 // are not aligned on uint64 boundaries. This has been |
1658 // a problem in the past. | 1705 // a problem in the past. |
1659 if((((uintptr)&work.empty) & 7) != 0) | 1706 if((((uintptr)&work.empty) & 7) != 0) |
1660 runtime·throw("runtime: gc work buffer is misaligned"); | 1707 runtime·throw("runtime: gc work buffer is misaligned"); |
1661 | 1708 |
1662 // The gc is turned off (via enablegc) until | 1709 // The gc is turned off (via enablegc) until |
1663 // the bootstrap has completed. | 1710 // the bootstrap has completed. |
1664 // Also, malloc gets called in the guts | 1711 // Also, malloc gets called in the guts |
(...skipping 13 matching lines...) Expand all Loading... |
1678 gctrace = runtime·atoi(p); | 1725 gctrace = runtime·atoi(p); |
1679 } | 1726 } |
1680 if(gcpercent < 0) | 1727 if(gcpercent < 0) |
1681 return; | 1728 return; |
1682 | 1729 |
1683 // Run gc on a bigger stack to eliminate | 1730 // Run gc on a bigger stack to eliminate |
1684 // a potentially large number of calls to runtime·morestack. | 1731 // a potentially large number of calls to runtime·morestack. |
1685 a.force = force; | 1732 a.force = force; |
1686 ap = &a; | 1733 ap = &a; |
1687 m->moreframesize_minalloc = StackBig; | 1734 m->moreframesize_minalloc = StackBig; |
1688 » reflect·call((byte*)gc, (byte*)&ap, sizeof(ap)); | 1735 » gcv.fn = (void*)gc; |
| 1736 » reflect·call(&gcv, (byte*)&ap, sizeof(ap)); |
1689 | 1737 |
1690 if(gctrace > 1 && !force) { | 1738 if(gctrace > 1 && !force) { |
1691 a.force = 1; | 1739 a.force = 1; |
1692 gc(&a); | 1740 gc(&a); |
1693 } | 1741 } |
1694 } | 1742 } |
| 1743 |
| 1744 static FuncVal runfinqv = {runfinq}; |
1695 | 1745 |
1696 static void | 1746 static void |
1697 gc(struct gc_args *args) | 1747 gc(struct gc_args *args) |
1698 { | 1748 { |
1699 int64 t0, t1, t2, t3, t4; | 1749 int64 t0, t1, t2, t3, t4; |
1700 uint64 heap0, heap1, obj0, obj1; | 1750 uint64 heap0, heap1, obj0, obj1; |
1701 GCStats stats; | 1751 GCStats stats; |
1702 M *mp; | 1752 M *mp; |
1703 uint32 i; | 1753 uint32 i; |
1704 Eface eface; | 1754 Eface eface; |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1774 stats.nosyield += work.sweepfor->nosyield; | 1824 stats.nosyield += work.sweepfor->nosyield; |
1775 stats.nsleep += work.sweepfor->nsleep; | 1825 stats.nsleep += work.sweepfor->nsleep; |
1776 | 1826 |
1777 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; | 1827 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; |
1778 m->gcing = 0; | 1828 m->gcing = 0; |
1779 | 1829 |
1780 if(finq != nil) { | 1830 if(finq != nil) { |
1781 m->locks++; // disable gc during the mallocs in newproc | 1831 m->locks++; // disable gc during the mallocs in newproc |
1782 // kick off or wake up goroutine to run queued finalizers | 1832 // kick off or wake up goroutine to run queued finalizers |
1783 if(fing == nil) | 1833 if(fing == nil) |
1784 » » » fing = runtime·newproc1((byte*)runfinq, nil, 0, 0, runti
me·gc); | 1834 » » » fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc
); |
1785 else if(fingwait) { | 1835 else if(fingwait) { |
1786 fingwait = 0; | 1836 fingwait = 0; |
1787 runtime·ready(fing); | 1837 runtime·ready(fing); |
1788 } | 1838 } |
1789 m->locks--; | 1839 m->locks--; |
1790 } | 1840 } |
1791 | 1841 |
1792 heap1 = mstats.heap_alloc; | 1842 heap1 = mstats.heap_alloc; |
1793 obj1 = mstats.nmalloc - mstats.nfree; | 1843 obj1 = mstats.nmalloc - mstats.nfree; |
1794 | 1844 |
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1912 next = fb->next; | 1962 next = fb->next; |
1913 for(i=0; i<fb->cnt; i++) { | 1963 for(i=0; i<fb->cnt; i++) { |
1914 f = &fb->fin[i]; | 1964 f = &fb->fin[i]; |
1915 framesz = sizeof(uintptr) + f->nret; | 1965 framesz = sizeof(uintptr) + f->nret; |
1916 if(framecap < framesz) { | 1966 if(framecap < framesz) { |
1917 runtime·free(frame); | 1967 runtime·free(frame); |
1918 frame = runtime·mal(framesz); | 1968 frame = runtime·mal(framesz); |
1919 framecap = framesz; | 1969 framecap = framesz; |
1920 } | 1970 } |
1921 *(void**)frame = f->arg; | 1971 *(void**)frame = f->arg; |
1922 » » » » reflect·call((byte*)f->fn, frame, sizeof(uintptr
) + f->nret); | 1972 » » » » reflect·call(f->fn, frame, sizeof(uintptr) + f->
nret); |
1923 f->fn = nil; | 1973 f->fn = nil; |
1924 f->arg = nil; | 1974 f->arg = nil; |
1925 } | 1975 } |
1926 fb->cnt = 0; | 1976 fb->cnt = 0; |
1927 fb->next = finc; | 1977 fb->next = finc; |
1928 finc = fb; | 1978 finc = fb; |
1929 } | 1979 } |
1930 runtime·gc(1); // trigger another gc to clean up the finalized
objects, if possible | 1980 runtime·gc(1); // trigger another gc to clean up the finalized
objects, if possible |
1931 } | 1981 } |
1932 } | 1982 } |
(...skipping 193 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2126 uintptr n; | 2176 uintptr n; |
2127 | 2177 |
2128 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; | 2178 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; |
2129 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); | 2179 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); |
2130 if(h->bitmap_mapped >= n) | 2180 if(h->bitmap_mapped >= n) |
2131 return; | 2181 return; |
2132 | 2182 |
2133 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); | 2183 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); |
2134 h->bitmap_mapped = n; | 2184 h->bitmap_mapped = n; |
2135 } | 2185 } |
LEFT | RIGHT |