LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Garbage collector. | 5 // Garbage collector. |
6 | 6 |
7 #include "runtime.h" | 7 #include "runtime.h" |
8 #include "arch_GOARCH.h" | 8 #include "arch_GOARCH.h" |
9 #include "malloc.h" | 9 #include "malloc.h" |
10 #include "stack.h" | 10 #include "stack.h" |
11 #include "mgc0.h" | 11 #include "mgc0.h" |
12 #include "race.h" | 12 #include "race.h" |
13 #include "type.h" | 13 #include "type.h" |
14 #include "typekind.h" | 14 #include "typekind.h" |
15 #include "hashmap.h" | 15 #include "hashmap.h" |
16 | 16 |
17 enum { | 17 enum { |
18 Debug = 0, | 18 Debug = 0, |
19 DebugMark = 0, // run second pass to check mark | 19 DebugMark = 0, // run second pass to check mark |
20 CollectStats = 0, | 20 CollectStats = 0, |
| 21 ScanStackByFrames = 1, |
| 22 IgnorePreciseGC = 0, |
21 DebugType = 0, | 23 DebugType = 0, |
22 MemScramble = 0, // scramble free memory, and don't actually free it | 24 MemScramble = 0, // scramble free memory, and don't actually free it |
23 | 25 |
24 // Four bits per word (see #defines below). | 26 // Four bits per word (see #defines below). |
25 wordsPerBitmapWord = sizeof(void*)*8/4, | 27 wordsPerBitmapWord = sizeof(void*)*8/4, |
26 bitShift = sizeof(void*)*8/4, | 28 bitShift = sizeof(void*)*8/4, |
27 | 29 |
28 handoffThreshold = 4, | 30 handoffThreshold = 4, |
29 IntermediateBufferCapacity = 64, | 31 IntermediateBufferCapacity = 64, |
30 | 32 |
(...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
135 static FinBlock *finc; // cache of free blocks | 137 static FinBlock *finc; // cache of free blocks |
136 static FinBlock *allfin; // list of all blocks | 138 static FinBlock *allfin; // list of all blocks |
137 static Lock finlock; | 139 static Lock finlock; |
138 static int32 fingwait; | 140 static int32 fingwait; |
139 | 141 |
140 static void runfinq(void); | 142 static void runfinq(void); |
141 static Workbuf* getempty(Workbuf*); | 143 static Workbuf* getempty(Workbuf*); |
142 static Workbuf* getfull(Workbuf*); | 144 static Workbuf* getfull(Workbuf*); |
143 static void putempty(Workbuf*); | 145 static void putempty(Workbuf*); |
144 static Workbuf* handoff(Workbuf*); | 146 static Workbuf* handoff(Workbuf*); |
| 147 static void gchelperstart(void); |
145 | 148 |
146 static struct { | 149 static struct { |
147 uint64 full; // lock-free list of full blocks | 150 uint64 full; // lock-free list of full blocks |
148 uint64 empty; // lock-free list of empty blocks | 151 uint64 empty; // lock-free list of empty blocks |
149 byte pad0[CacheLineSize]; // prevents false-sharing between full/empt
y and nproc/nwait | 152 byte pad0[CacheLineSize]; // prevents false-sharing between full/empt
y and nproc/nwait |
150 uint32 nproc; | 153 uint32 nproc; |
151 byte pad1[CacheLineSize]; // prevents false-sharing between nproc and
nwait | 154 byte pad1[CacheLineSize]; // prevents false-sharing between nproc and
nwait |
152 volatile uint32 nwait; | 155 volatile uint32 nwait; |
153 volatile uint32 ndone; | 156 volatile uint32 ndone; |
154 volatile uint32 debugmarkdone; | 157 volatile uint32 debugmarkdone; |
(...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
283 { | 286 { |
284 void *p; | 287 void *p; |
285 uintptr ti; | 288 uintptr ti; |
286 }; | 289 }; |
287 | 290 |
288 typedef struct BufferList BufferList; | 291 typedef struct BufferList BufferList; |
289 struct BufferList | 292 struct BufferList |
290 { | 293 { |
291 PtrTarget ptrtarget[IntermediateBufferCapacity]; | 294 PtrTarget ptrtarget[IntermediateBufferCapacity]; |
292 Obj obj[IntermediateBufferCapacity]; | 295 Obj obj[IntermediateBufferCapacity]; |
293 » BufferList *next; | 296 » uint32 busy; |
| 297 » byte pad[CacheLineSize]; |
294 }; | 298 }; |
295 static BufferList *bufferList; | 299 #pragma dataflag 16 // no pointers |
| 300 static BufferList bufferList[MaxGcproc]; |
296 | 301 |
297 static Lock lock; | 302 static Lock lock; |
298 static Type *itabtype; | 303 static Type *itabtype; |
299 | 304 |
300 static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); | 305 static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); |
301 | 306 |
302 // flushptrbuf moves data from the PtrTarget buffer to the work buffer. | 307 // flushptrbuf moves data from the PtrTarget buffer to the work buffer. |
303 // The PtrTarget buffer contains blocks irrespective of whether the blocks have
been marked or scanned, | 308 // The PtrTarget buffer contains blocks irrespective of whether the blocks have
been marked or scanned, |
304 // while the work buffer contains blocks which have been marked | 309 // while the work buffer contains blocks which have been marked |
305 // and are prepared to be scanned by the garbage collector. | 310 // and are prepared to be scanned by the garbage collector. |
(...skipping 339 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
645 | 650 |
646 // Hchan program | 651 // Hchan program |
647 static uintptr chanProg[2] = {0, GC_CHAN}; | 652 static uintptr chanProg[2] = {0, GC_CHAN}; |
648 | 653 |
649 // Local variables of a program fragment or loop | 654 // Local variables of a program fragment or loop |
650 typedef struct Frame Frame; | 655 typedef struct Frame Frame; |
651 struct Frame { | 656 struct Frame { |
652 uintptr count, elemsize, b; | 657 uintptr count, elemsize, b; |
653 uintptr *loop_or_ret; | 658 uintptr *loop_or_ret; |
654 }; | 659 }; |
| 660 |
| 661 // Sanity check for the derived type info objti. |
| 662 static void |
| 663 checkptr(void *obj, uintptr objti) |
| 664 { |
| 665 uintptr *pc1, *pc2, type, tisize, i, j, x; |
| 666 byte *objstart; |
| 667 Type *t; |
| 668 MSpan *s; |
| 669 |
| 670 if(!Debug) |
| 671 runtime·throw("checkptr is debug only"); |
| 672 |
| 673 if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) |
| 674 return; |
| 675 type = runtime·gettype(obj); |
| 676 t = (Type*)(type & ~(uintptr)(PtrSize-1)); |
| 677 if(t == nil) |
| 678 return; |
| 679 x = (uintptr)obj >> PageShift; |
| 680 if(sizeof(void*) == 8) |
| 681 x -= (uintptr)(runtime·mheap->arena_start)>>PageShift; |
| 682 s = runtime·mheap->map[x]; |
| 683 objstart = (byte*)((uintptr)s->start<<PageShift); |
| 684 if(s->sizeclass != 0) { |
| 685 i = ((byte*)obj - objstart)/s->elemsize; |
| 686 objstart += i*s->elemsize; |
| 687 } |
| 688 tisize = *(uintptr*)objti; |
| 689 // Sanity check for object size: it should fit into the memory block. |
| 690 if((byte*)obj + tisize > objstart + s->elemsize) |
| 691 runtime·throw("invalid gc type info"); |
| 692 if(obj != objstart) |
| 693 return; |
| 694 // If obj points to the beginning of the memory block, |
| 695 // check type info as well. |
| 696 if(t->string == nil || |
| 697 // Gob allocates unsafe pointers for indirection. |
| 698 (runtime·strcmp(t->string->str, (byte*)"unsafe.Pointer") && |
| 699 // Runtime and gc think differently about closures. |
| 700 runtime·strstr(t->string->str, (byte*)"struct { F uintptr") != t
->string->str)) { |
| 701 pc1 = (uintptr*)objti; |
| 702 pc2 = (uintptr*)t->gc; |
| 703 // A simple best-effort check until first GC_END. |
| 704 for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) { |
| 705 if(pc1[j] != pc2[j]) { |
| 706 runtime·printf("invalid gc type info for '%s' at
%p, type info %p, block info %p\n", |
| 707 t->string ? (int8*)t->string->str : (int
8*)"?", j, pc1[j], pc2[j]); |
| 708 runtime·throw("invalid gc type info"); |
| 709 } |
| 710 } |
| 711 } |
| 712 }······································· |
655 | 713 |
656 // scanblock scans a block of n bytes starting at pointer b for references | 714 // scanblock scans a block of n bytes starting at pointer b for references |
657 // to other objects, scanning any it finds recursively until there are no | 715 // to other objects, scanning any it finds recursively until there are no |
658 // unscanned objects left. Instead of using an explicit recursion, it keeps | 716 // unscanned objects left. Instead of using an explicit recursion, it keeps |
659 // a work list in the Workbuf* structures and loops in the main function | 717 // a work list in the Workbuf* structures and loops in the main function |
660 // body. Keeping an explicit work list is easier on the stack allocator and | 718 // body. Keeping an explicit work list is easier on the stack allocator and |
661 // more efficient. | 719 // more efficient. |
662 // | 720 // |
663 // wbuf: current work buffer | 721 // wbuf: current work buffer |
664 // wp: storage for next queued pointer (write pointer) | 722 // wp: storage for next queued pointer (write pointer) |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
696 arena_start = runtime·mheap->arena_start; | 754 arena_start = runtime·mheap->arena_start; |
697 arena_used = runtime·mheap->arena_used; | 755 arena_used = runtime·mheap->arena_used; |
698 | 756 |
699 stack_ptr = stack+nelem(stack)-1; | 757 stack_ptr = stack+nelem(stack)-1; |
700 ········ | 758 ········ |
701 precise_type = false; | 759 precise_type = false; |
702 nominal_size = 0; | 760 nominal_size = 0; |
703 | 761 |
704 // Allocate ptrbuf | 762 // Allocate ptrbuf |
705 { | 763 { |
706 » » runtime·lock(&lock); | 764 » » scanbuffers = &bufferList[m->helpgc]; |
707 | |
708 » » if(bufferList == nil) { | |
709 » » » bufferList = runtime·SysAlloc(sizeof(*bufferList)); | |
710 » » » if(bufferList == nil) | |
711 » » » » runtime·throw("runtime: cannot allocate memory")
; | |
712 » » » bufferList->next = nil; | |
713 » » } | |
714 » » scanbuffers = bufferList; | |
715 » » bufferList = bufferList->next; | |
716 | |
717 ptrbuf = &scanbuffers->ptrtarget[0]; | 765 ptrbuf = &scanbuffers->ptrtarget[0]; |
718 ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptr
target); | 766 ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptr
target); |
719 objbuf = &scanbuffers->obj[0]; | 767 objbuf = &scanbuffers->obj[0]; |
720 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); | 768 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); |
721 | |
722 runtime·unlock(&lock); | |
723 } | 769 } |
724 | 770 |
725 ptrbufpos = ptrbuf; | 771 ptrbufpos = ptrbuf; |
726 objbufpos = objbuf; | 772 objbufpos = objbuf; |
727 | 773 |
728 // (Silence the compiler) | 774 // (Silence the compiler) |
729 map_ret = nil; | 775 map_ret = nil; |
730 mapkey_size = mapval_size = 0; | 776 mapkey_size = mapval_size = 0; |
731 mapkey_kind = mapval_kind = false; | 777 mapkey_kind = mapval_kind = false; |
732 mapkey_ti = mapval_ti = 0; | 778 mapkey_ti = mapval_ti = 0; |
(...skipping 23 matching lines...) Expand all Loading... |
756 pc = (uintptr*)(ti & ~(uintptr)PC_BITS); | 802 pc = (uintptr*)(ti & ~(uintptr)PC_BITS); |
757 precise_type = (ti & PRECISE); | 803 precise_type = (ti & PRECISE); |
758 stack_top.elemsize = pc[0]; | 804 stack_top.elemsize = pc[0]; |
759 if(!precise_type) | 805 if(!precise_type) |
760 nominal_size = pc[0]; | 806 nominal_size = pc[0]; |
761 if(ti & LOOP) { | 807 if(ti & LOOP) { |
762 stack_top.count = 0; // 0 means an infinite n
umber of iterations | 808 stack_top.count = 0; // 0 means an infinite n
umber of iterations |
763 stack_top.loop_or_ret = pc+1; | 809 stack_top.loop_or_ret = pc+1; |
764 } else { | 810 } else { |
765 stack_top.count = 1; | 811 stack_top.count = 1; |
| 812 } |
| 813 if(Debug) { |
| 814 // Simple sanity check for provided type info ti
: |
| 815 // The declared size of the object must be not l
arger than the actual size |
| 816 // (it can be smaller due to inferior pointers). |
| 817 // It's difficult to make a comprehensive check
due to inferior pointers, |
| 818 // reflection, gob, etc. |
| 819 if(pc[0] > n) { |
| 820 runtime·printf("invalid gc type info: ty
pe info size %p, block size %p\n", pc[0], n); |
| 821 runtime·throw("invalid gc type info"); |
| 822 } |
766 } | 823 } |
767 } else if(UseSpanType) { | 824 } else if(UseSpanType) { |
768 if(CollectStats) | 825 if(CollectStats) |
769 runtime·xadd64(&gcstats.obj.notype, 1); | 826 runtime·xadd64(&gcstats.obj.notype, 1); |
770 | 827 |
771 type = runtime·gettype(b); | 828 type = runtime·gettype(b); |
772 if(type != 0) { | 829 if(type != 0) { |
773 if(CollectStats) | 830 if(CollectStats) |
774 runtime·xadd64(&gcstats.obj.typelookup,
1); | 831 runtime·xadd64(&gcstats.obj.typelookup,
1); |
775 | 832 |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
817 runtime·throw("scanblock: invalid type")
; | 874 runtime·throw("scanblock: invalid type")
; |
818 return; | 875 return; |
819 } | 876 } |
820 } else { | 877 } else { |
821 pc = defaultProg; | 878 pc = defaultProg; |
822 } | 879 } |
823 } else { | 880 } else { |
824 pc = defaultProg; | 881 pc = defaultProg; |
825 } | 882 } |
826 | 883 |
| 884 if(IgnorePreciseGC) |
| 885 pc = defaultProg; |
| 886 |
827 if(DebugTypeAtBlockEnd) { | 887 if(DebugTypeAtBlockEnd) { |
828 s = runtime·MHeap_LookupMaybe(runtime·mheap, b); | 888 s = runtime·MHeap_LookupMaybe(runtime·mheap, b); |
829 | 889 |
830 if(s != nil) { | 890 if(s != nil) { |
831 elemsize = s->elemsize; | 891 elemsize = s->elemsize; |
832 p = (byte*)((uintptr)s->start<<PageShift); | 892 p = (byte*)((uintptr)s->start<<PageShift); |
833 if(s->sizeclass == 0) { | 893 if(s->sizeclass == 0) { |
834 b2 = p; | 894 b2 = p; |
835 } | 895 } |
836 else { | 896 else { |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
892 if(CollectStats) | 952 if(CollectStats) |
893 runtime·xadd64(&gcstats.instr[pc[0]], 1); | 953 runtime·xadd64(&gcstats.instr[pc[0]], 1); |
894 | 954 |
895 obj = nil; | 955 obj = nil; |
896 objti = 0; | 956 objti = 0; |
897 switch(pc[0]) { | 957 switch(pc[0]) { |
898 case GC_PTR: | 958 case GC_PTR: |
899 obj = *(void**)(stack_top.b + pc[1]); | 959 obj = *(void**)(stack_top.b + pc[1]); |
900 objti = pc[2]; | 960 objti = pc[2]; |
901 pc += 3; | 961 pc += 3; |
| 962 if(Debug) |
| 963 checkptr(obj, objti); |
902 break; | 964 break; |
903 | 965 |
904 case GC_SLICE: | 966 case GC_SLICE: |
905 sliceptr = (Slice*)(stack_top.b + pc[1]); | 967 sliceptr = (Slice*)(stack_top.b + pc[1]); |
906 if(sliceptr->cap != 0) { | 968 if(sliceptr->cap != 0) { |
907 obj = sliceptr->array; | 969 obj = sliceptr->array; |
908 » » » » objti = pc[2] | PRECISE | LOOP; | 970 » » » » // Can't use slice element type for scanning, |
| 971 » » » » // because if it points to an array embedded |
| 972 » » » » // in the beginning of a struct, |
| 973 » » » » // we will scan the whole struct as the slice. |
| 974 » » » » // So just obtain type info from heap. |
909 } | 975 } |
910 pc += 3; | 976 pc += 3; |
911 break; | 977 break; |
912 | 978 |
913 case GC_APTR: | 979 case GC_APTR: |
914 obj = *(void**)(stack_top.b + pc[1]); | 980 obj = *(void**)(stack_top.b + pc[1]); |
915 pc += 2; | 981 pc += 2; |
916 break; | 982 break; |
917 | 983 |
918 case GC_STRING: | 984 case GC_STRING: |
919 obj = *(void**)(stack_top.b + pc[1]); | 985 obj = *(void**)(stack_top.b + pc[1]); |
| 986 markonly(obj); |
920 pc += 2; | 987 pc += 2; |
921 » » » break; | 988 » » » continue; |
922 | 989 |
923 case GC_EFACE: | 990 case GC_EFACE: |
924 eface = (Eface*)(stack_top.b + pc[1]); | 991 eface = (Eface*)(stack_top.b + pc[1]); |
925 pc += 2; | 992 pc += 2; |
926 if(eface->type == nil) | 993 if(eface->type == nil) |
927 continue; | 994 continue; |
928 | 995 |
929 // eface->type | 996 // eface->type |
930 t = eface->type; | 997 t = eface->type; |
931 if((void*)t >= arena_start && (void*)t < arena_used) { | 998 if((void*)t >= arena_start && (void*)t < arena_used) { |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
974 if((t->kind & ~KindNoPointers) == KindPt
r) | 1041 if((t->kind & ~KindNoPointers) == KindPt
r) |
975 objti = (uintptr)((PtrType*)t)->
elem->gc; | 1042 objti = (uintptr)((PtrType*)t)->
elem->gc; |
976 } else { | 1043 } else { |
977 obj = iface->data; | 1044 obj = iface->data; |
978 objti = (uintptr)t->gc; | 1045 objti = (uintptr)t->gc; |
979 } | 1046 } |
980 } | 1047 } |
981 break; | 1048 break; |
982 | 1049 |
983 case GC_DEFAULT_PTR: | 1050 case GC_DEFAULT_PTR: |
984 » » » while((i = stack_top.b) <= end_b) { | 1051 » » » while(stack_top.b <= end_b) { |
| 1052 » » » » obj = *(byte**)stack_top.b; |
985 stack_top.b += PtrSize; | 1053 stack_top.b += PtrSize; |
986 obj = *(byte**)i; | |
987 if(obj >= arena_start && obj < arena_used) { | 1054 if(obj >= arena_start && obj < arena_used) { |
988 *ptrbufpos++ = (PtrTarget){obj, 0}; | 1055 *ptrbufpos++ = (PtrTarget){obj, 0}; |
989 if(ptrbufpos == ptrbuf_end) | 1056 if(ptrbufpos == ptrbuf_end) |
990 flushptrbuf(ptrbuf, &ptrbufpos,
&wp, &wbuf, &nobj); | 1057 flushptrbuf(ptrbuf, &ptrbufpos,
&wp, &wbuf, &nobj); |
991 } | 1058 } |
992 } | 1059 } |
993 goto next_block; | 1060 goto next_block; |
994 | 1061 |
995 case GC_END: | 1062 case GC_END: |
996 if(--stack_top.count != 0) { | 1063 if(--stack_top.count != 0) { |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1096 if(objbufpos+2 >= objbuf_end) | 1163 if(objbufpos+2 >= objbuf_end) |
1097 flushobjbuf(objbuf, &objbufpos, &wp, &wb
uf, &nobj); | 1164 flushobjbuf(objbuf, &objbufpos, &wp, &wb
uf, &nobj); |
1098 | 1165 |
1099 if(d.st != nil) | 1166 if(d.st != nil) |
1100 markonly(d.st); | 1167 markonly(d.st); |
1101 | 1168 |
1102 if(d.key_data != nil) { | 1169 if(d.key_data != nil) { |
1103 if(!(mapkey_kind & KindNoPointers) || d.
indirectkey) { | 1170 if(!(mapkey_kind & KindNoPointers) || d.
indirectkey) { |
1104 if(!d.indirectkey) | 1171 if(!d.indirectkey) |
1105 *objbufpos++ = (Obj){d.k
ey_data, mapkey_size, mapkey_ti}; | 1172 *objbufpos++ = (Obj){d.k
ey_data, mapkey_size, mapkey_ti}; |
1106 » » » » » » else | 1173 » » » » » » else { |
| 1174 » » » » » » » if(Debug) { |
| 1175 » » » » » » » » obj = *(void**)d
.key_data; |
| 1176 » » » » » » » » if(!(arena_start
<= obj && obj < arena_used)) |
| 1177 » » » » » » » » » runtime·
throw("scanblock: inconsistent hashmap"); |
| 1178 » » » » » » » } |
1107 *ptrbufpos++ = (PtrTarge
t){*(void**)d.key_data, mapkey_ti}; | 1179 *ptrbufpos++ = (PtrTarge
t){*(void**)d.key_data, mapkey_ti}; |
| 1180 } |
1108 } | 1181 } |
1109 if(!(mapval_kind & KindNoPointers) || d.
indirectval) { | 1182 if(!(mapval_kind & KindNoPointers) || d.
indirectval) { |
1110 if(!d.indirectval) | 1183 if(!d.indirectval) |
1111 *objbufpos++ = (Obj){d.v
al_data, mapval_size, mapval_ti}; | 1184 *objbufpos++ = (Obj){d.v
al_data, mapval_size, mapval_ti}; |
1112 » » » » » » else | 1185 » » » » » » else { |
| 1186 » » » » » » » if(Debug) { |
| 1187 » » » » » » » » obj = *(void**)d
.val_data; |
| 1188 » » » » » » » » if(!(arena_start
<= obj && obj < arena_used)) |
| 1189 » » » » » » » » » runtime·
throw("scanblock: inconsistent hashmap"); |
| 1190 » » » » » » » } |
1113 *ptrbufpos++ = (PtrTarge
t){*(void**)d.val_data, mapval_ti}; | 1191 *ptrbufpos++ = (PtrTarge
t){*(void**)d.val_data, mapval_ti}; |
| 1192 } |
1114 } | 1193 } |
1115 } | 1194 } |
1116 } | 1195 } |
1117 if(map_ret == nil) | 1196 if(map_ret == nil) |
1118 goto next_block; | 1197 goto next_block; |
1119 pc = map_ret; | 1198 pc = map_ret; |
1120 continue; | 1199 continue; |
1121 | 1200 |
1122 case GC_REGION: | 1201 case GC_REGION: |
1123 obj = (void*)(stack_top.b + pc[1]); | 1202 obj = (void*)(stack_top.b + pc[1]); |
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1251 } | 1330 } |
1252 } | 1331 } |
1253 | 1332 |
1254 // Fetch b from the work buffer. | 1333 // Fetch b from the work buffer. |
1255 --wp; | 1334 --wp; |
1256 b = wp->p; | 1335 b = wp->p; |
1257 n = wp->n; | 1336 n = wp->n; |
1258 nobj--; | 1337 nobj--; |
1259 } | 1338 } |
1260 | 1339 |
1261 endscan: | 1340 endscan:; |
1262 » runtime·lock(&lock); | |
1263 » scanbuffers->next = bufferList; | |
1264 » bufferList = scanbuffers; | |
1265 » runtime·unlock(&lock); | |
1266 } | 1341 } |
1267 | 1342 |
1268 // debug_scanblock is the debug copy of scanblock. | 1343 // debug_scanblock is the debug copy of scanblock. |
1269 // it is simpler, slower, less precise, single-threaded, recursive, | 1344 // it is simpler, slower, less precise, single-threaded, recursive, |
1270 // and uses bitSpecial as the mark bit. | 1345 // and uses bitSpecial as the mark bit. |
1271 static void | 1346 static void |
1272 debug_scanblock(byte *b, uintptr n) | 1347 debug_scanblock(byte *b, uintptr n) |
1273 { | 1348 { |
1274 byte *obj, *p; | 1349 byte *obj, *p; |
1275 void **vp; | 1350 void **vp; |
(...skipping 255 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1531 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); | 1606 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); |
1532 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); | 1607 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); |
1533 } | 1608 } |
1534 work.roots = new; | 1609 work.roots = new; |
1535 work.rootcap = cap; | 1610 work.rootcap = cap; |
1536 } | 1611 } |
1537 work.roots[work.nroot] = obj; | 1612 work.roots[work.nroot] = obj; |
1538 work.nroot++; | 1613 work.nroot++; |
1539 } | 1614 } |
1540 | 1615 |
| 1616 // Scan a stack frame. The doframe parameter is a signal that the previously |
| 1617 // scanned activation has an unknown argument size. When *doframe is true the |
| 1618 // current activation must have its entire frame scanned. Otherwise, only the |
| 1619 // locals need to be scanned. |
| 1620 static void |
| 1621 addframeroots(Func *f, byte*, byte *sp, void *doframe) |
| 1622 { |
| 1623 uintptr outs; |
| 1624 |
| 1625 if(thechar == '5') |
| 1626 sp += sizeof(uintptr); |
| 1627 if(f->locals == 0 || *(bool*)doframe == true) |
| 1628 addroot((Obj){sp, f->frame - sizeof(uintptr), 0}); |
| 1629 else if(f->locals > 0) { |
| 1630 outs = f->frame - sizeof(uintptr) - f->locals; |
| 1631 addroot((Obj){sp + outs, f->locals, 0}); |
| 1632 } |
| 1633 if(f->args > 0) |
| 1634 addroot((Obj){sp + f->frame, f->args, 0}); |
| 1635 *(bool*)doframe = (f->args == ArgsSizeUnknown); |
| 1636 } |
| 1637 |
1541 static void | 1638 static void |
1542 addstackroots(G *gp) | 1639 addstackroots(G *gp) |
1543 { | 1640 { |
1544 M *mp; | 1641 M *mp; |
1545 int32 n; | 1642 int32 n; |
1546 Stktop *stk; | 1643 Stktop *stk; |
1547 » byte *sp, *guard; | 1644 » byte *sp, *guard, *pc; |
| 1645 » Func *f; |
| 1646 » bool doframe; |
1548 | 1647 |
1549 stk = (Stktop*)gp->stackbase; | 1648 stk = (Stktop*)gp->stackbase; |
1550 guard = (byte*)gp->stackguard; | 1649 guard = (byte*)gp->stackguard; |
1551 | 1650 |
1552 if(gp == g) { | 1651 if(gp == g) { |
1553 // Scanning our own stack: start at &gp. | 1652 // Scanning our own stack: start at &gp. |
1554 » » sp = (byte*)&gp; | 1653 » » sp = runtime·getcallersp(&gp); |
| 1654 » » pc = runtime·getcallerpc(&gp); |
1555 } else if((mp = gp->m) != nil && mp->helpgc) { | 1655 } else if((mp = gp->m) != nil && mp->helpgc) { |
1556 // gchelper's stack is in active use and has no interesting poin
ters. | 1656 // gchelper's stack is in active use and has no interesting poin
ters. |
1557 return; | 1657 return; |
| 1658 } else if(gp->gcstack != (uintptr)nil) { |
| 1659 // Scanning another goroutine that is about to enter or might |
| 1660 // have just exited a system call. It may be executing code such |
| 1661 // as schedlock and may have needed to start a new stack segment
. |
| 1662 // Use the stack segment and stack pointer at the time of |
| 1663 // the system call instead, since that won't change underfoot. |
| 1664 sp = (byte*)gp->gcsp; |
| 1665 pc = gp->gcpc; |
| 1666 stk = (Stktop*)gp->gcstack; |
| 1667 guard = (byte*)gp->gcguard; |
1558 } else { | 1668 } else { |
1559 // Scanning another goroutine's stack. | 1669 // Scanning another goroutine's stack. |
1560 // The goroutine is usually asleep (the world is stopped). | 1670 // The goroutine is usually asleep (the world is stopped). |
1561 sp = (byte*)gp->sched.sp; | 1671 sp = (byte*)gp->sched.sp; |
1562 | 1672 » » pc = gp->sched.pc; |
1563 » » // The exception is that if the goroutine is about to enter or m
ight | 1673 » » if(ScanStackByFrames && pc == (byte*)runtime·goexit && gp->fnsta
rt != nil) { |
1564 » » // have just exited a system call, it may be executing code such | 1674 » » » // The goroutine has not started. However, its incoming |
1565 » » // as schedlock and may have needed to start a new stack segment
. | 1675 » » » // arguments are live at the top of the stack and must |
1566 » » // Use the stack segment and stack pointer at the time of | 1676 » » » // be scanned. No other live values should be on the |
1567 » » // the system call instead, since that won't change underfoot. | 1677 » » » // stack. |
1568 » » if(gp->gcstack != (uintptr)nil) { | 1678 » » » f = runtime·findfunc((uintptr)gp->fnstart->fn); |
1569 » » » stk = (Stktop*)gp->gcstack; | 1679 » » » if(f->args != 0) { |
1570 » » » sp = (byte*)gp->gcsp; | 1680 » » » » if(thechar == '5') |
1571 » » » guard = (byte*)gp->gcguard; | 1681 » » » » » sp += sizeof(uintptr); |
1572 » » } | 1682 » » » » // If the size of the arguments is known |
1573 » } | 1683 » » » » // scan just the incoming arguments. |
1574 | 1684 » » » » // Otherwise, scan everything between the |
1575 » n = 0; | 1685 » » » » // top and the bottom of the stack. |
1576 » while(stk) { | 1686 » » » » if(f->args > 0) |
1577 » » if(sp < guard-StackGuard || (byte*)stk < sp) { | 1687 » » » » » addroot((Obj){sp, f->args, 0}); |
1578 » » » runtime·printf("addstackroots inconsistent: g%D#%d sp=%p
not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); | 1688 » » » » else |
1579 » » » runtime·throw("addstackroots"); | 1689 » » » » » addroot((Obj){sp, (byte*)stk - sp, 0});· |
1580 » » } | 1690 » » » }· |
1581 » » addroot((Obj){sp, (byte*)stk - sp, (uintptr)defaultProg | PRECIS
E | LOOP}); | 1691 » » » return; |
1582 » » sp = (byte*)stk->gobuf.sp; | 1692 » » } |
1583 » » guard = stk->stackguard; | 1693 » } |
1584 » » stk = (Stktop*)stk->stackbase; | 1694 |
1585 » » n++; | 1695 » if (ScanStackByFrames) { |
| 1696 » » USED(stk); |
| 1697 » » USED(guard); |
| 1698 » » doframe = false; |
| 1699 » » runtime·gentraceback(pc, sp, nil, gp, 0, nil, 0x7fffffff, addfra
meroots, &doframe); |
| 1700 » } else { |
| 1701 » » USED(pc); |
| 1702 » » n = 0; |
| 1703 » » while(stk) { |
| 1704 » » » if(sp < guard-StackGuard || (byte*)stk < sp) { |
| 1705 » » » » runtime·printf("addstackroots inconsistent: g%D#
%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); |
| 1706 » » » » runtime·throw("addstackroots"); |
| 1707 » » » » runtime·printf("scanstack inconsistent: g%D#%d s
p=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); |
| 1708 » » » » runtime·throw("scanstack"); |
| 1709 » » » } |
| 1710 » » » addroot((Obj){sp, (byte*)stk - sp, (uintptr)defaultProg
| PRECISE | LOOP}); |
| 1711 » » » sp = (byte*)stk->gobuf.sp; |
| 1712 » » » guard = stk->stackguard; |
| 1713 » » » stk = (Stktop*)stk->stackbase; |
| 1714 » » » n++; |
| 1715 » » } |
1586 } | 1716 } |
1587 } | 1717 } |
1588 | 1718 |
1589 static void | 1719 static void |
1590 addfinroots(void *v) | 1720 addfinroots(void *v) |
1591 { | 1721 { |
1592 uintptr size; | 1722 uintptr size; |
1593 void *base; | 1723 void *base; |
1594 | 1724 |
1595 size = 0; | 1725 size = 0; |
(...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1790 *bitp |= (bitSpecial<<shift); | 1920 *bitp |= (bitSpecial<<shift); |
1791 continue; | 1921 continue; |
1792 } | 1922 } |
1793 | 1923 |
1794 // Mark freed; restore block boundary bit. | 1924 // Mark freed; restore block boundary bit. |
1795 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 1925 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
1796 | 1926 |
1797 if(cl == 0) { | 1927 if(cl == 0) { |
1798 // Free large span. | 1928 // Free large span. |
1799 runtime·unmarkspan(p, 1<<PageShift); | 1929 runtime·unmarkspan(p, 1<<PageShift); |
1800 » » » *(uintptr*)p = 1;» // needs zeroing | 1930 » » » *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll;» // needs
zeroing |
1801 runtime·MHeap_Free(runtime·mheap, s, 1); | 1931 runtime·MHeap_Free(runtime·mheap, s, 1); |
1802 c->local_alloc -= size; | 1932 c->local_alloc -= size; |
1803 c->local_nfree++; | 1933 c->local_nfree++; |
1804 } else { | 1934 } else { |
1805 // Free small object. | 1935 // Free small object. |
1806 switch(compression) { | 1936 switch(compression) { |
1807 case MTypes_Words: | 1937 case MTypes_Words: |
1808 *(uintptr*)type_data = 0; | 1938 *(uintptr*)type_data = 0; |
1809 break; | 1939 break; |
1810 case MTypes_Bytes: | 1940 case MTypes_Bytes: |
1811 *(byte*)type_data = 0; | 1941 *(byte*)type_data = 0; |
1812 break; | 1942 break; |
1813 } | 1943 } |
1814 if(size > sizeof(uintptr)) | 1944 if(size > sizeof(uintptr)) |
1815 » » » » ((uintptr*)p)[1] = 1;» // mark as "needs to be
zeroed" | 1945 » » » » ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll
;» // mark as "needs to be zeroed" |
1816 ························ | 1946 ························ |
1817 end->next = (MLink*)p; | 1947 end->next = (MLink*)p; |
1818 end = (MLink*)p; | 1948 end = (MLink*)p; |
1819 nfree++; | 1949 nfree++; |
1820 } | 1950 } |
1821 } | 1951 } |
1822 | 1952 |
1823 if(nfree) { | 1953 if(nfree) { |
1824 c->local_by_size[cl].nfree += nfree; | 1954 c->local_by_size[cl].nfree += nfree; |
1825 c->local_alloc -= size * nfree; | 1955 c->local_alloc -= size * nfree; |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1902 uint32 spanidx; | 2032 uint32 spanidx; |
1903 | 2033 |
1904 for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { | 2034 for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { |
1905 dumpspan(spanidx); | 2035 dumpspan(spanidx); |
1906 } | 2036 } |
1907 } | 2037 } |
1908 | 2038 |
1909 void | 2039 void |
1910 runtime·gchelper(void) | 2040 runtime·gchelper(void) |
1911 { | 2041 { |
| 2042 gchelperstart(); |
| 2043 |
1912 // parallel mark for over gc roots | 2044 // parallel mark for over gc roots |
1913 runtime·parfordo(work.markfor); | 2045 runtime·parfordo(work.markfor); |
1914 | 2046 |
1915 // help other threads scan secondary blocks | 2047 // help other threads scan secondary blocks |
1916 scanblock(nil, nil, 0, true); | 2048 scanblock(nil, nil, 0, true); |
1917 | 2049 |
1918 if(DebugMark) { | 2050 if(DebugMark) { |
1919 // wait while the main thread executes mark(debug_scanblock) | 2051 // wait while the main thread executes mark(debug_scanblock) |
1920 while(runtime·atomicload(&work.debugmarkdone) == 0) | 2052 while(runtime·atomicload(&work.debugmarkdone) == 0) |
1921 runtime·usleep(10); | 2053 runtime·usleep(10); |
1922 } | 2054 } |
1923 | 2055 |
1924 runtime·parfordo(work.sweepfor); | 2056 runtime·parfordo(work.sweepfor); |
| 2057 bufferList[m->helpgc].busy = 0; |
1925 if(runtime·xadd(&work.ndone, +1) == work.nproc-1) | 2058 if(runtime·xadd(&work.ndone, +1) == work.nproc-1) |
1926 runtime·notewakeup(&work.alldone); | 2059 runtime·notewakeup(&work.alldone); |
1927 } | 2060 } |
1928 | 2061 |
1929 #define GcpercentUnknown (-2) | 2062 #define GcpercentUnknown (-2) |
1930 | 2063 |
1931 // Initialized from $GOGC. GOGC=off means no gc. | 2064 // Initialized from $GOGC. GOGC=off means no gc. |
1932 // | 2065 // |
1933 // Next gc is after we've allocated an extra amount of | 2066 // Next gc is after we've allocated an extra amount of |
1934 // memory proportional to the amount already in use. | 2067 // memory proportional to the amount already in use. |
(...skipping 25 matching lines...) Expand all Loading... |
1960 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) | 2093 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) |
1961 dst[i] += src[i]; | 2094 dst[i] += src[i]; |
1962 runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats))
; | 2095 runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats))
; |
1963 } | 2096 } |
1964 } | 2097 } |
1965 for(pp=runtime·allp; p=*pp; pp++) { | 2098 for(pp=runtime·allp; p=*pp; pp++) { |
1966 c = p->mcache; | 2099 c = p->mcache; |
1967 if(c==nil) | 2100 if(c==nil) |
1968 continue; | 2101 continue; |
1969 runtime·purgecachedstats(c); | 2102 runtime·purgecachedstats(c); |
1970 for(i=0; i<nelem(c->local_by_size); i++) { | |
1971 mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc
; | |
1972 c->local_by_size[i].nmalloc = 0; | |
1973 mstats.by_size[i].nfree += c->local_by_size[i].nfree; | |
1974 c->local_by_size[i].nfree = 0; | |
1975 } | |
1976 } | 2103 } |
1977 mstats.stacks_inuse = stacks_inuse; | 2104 mstats.stacks_inuse = stacks_inuse; |
1978 } | 2105 } |
1979 | 2106 |
1980 // Structure of arguments passed to function gc(). | 2107 // Structure of arguments passed to function gc(). |
1981 // This allows the arguments to be passed via reflect·call. | 2108 // This allows the arguments to be passed via reflect·call. |
1982 struct gc_args | 2109 struct gc_args |
1983 { | 2110 { |
1984 int32 force; | 2111 int32 force; |
1985 }; | 2112 }; |
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2106 addroots(); | 2233 addroots(); |
2107 runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma
rkroot); | 2234 runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma
rkroot); |
2108 runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap->nspan, nil
, true, sweepspan); | 2235 runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap->nspan, nil
, true, sweepspan); |
2109 if(work.nproc > 1) { | 2236 if(work.nproc > 1) { |
2110 runtime·noteclear(&work.alldone); | 2237 runtime·noteclear(&work.alldone); |
2111 runtime·helpgc(work.nproc); | 2238 runtime·helpgc(work.nproc); |
2112 } | 2239 } |
2113 | 2240 |
2114 t1 = runtime·nanotime(); | 2241 t1 = runtime·nanotime(); |
2115 | 2242 |
| 2243 gchelperstart(); |
2116 runtime·parfordo(work.markfor); | 2244 runtime·parfordo(work.markfor); |
2117 scanblock(nil, nil, 0, true); | 2245 scanblock(nil, nil, 0, true); |
2118 | 2246 |
2119 if(DebugMark) { | 2247 if(DebugMark) { |
2120 for(i=0; i<work.nroot; i++) | 2248 for(i=0; i<work.nroot; i++) |
2121 debug_scanblock(work.roots[i].p, work.roots[i].n); | 2249 debug_scanblock(work.roots[i].p, work.roots[i].n); |
2122 runtime·atomicstore(&work.debugmarkdone, 1); | 2250 runtime·atomicstore(&work.debugmarkdone, 1); |
2123 } | 2251 } |
2124 t2 = runtime·nanotime(); | 2252 t2 = runtime·nanotime(); |
2125 | 2253 |
2126 runtime·parfordo(work.sweepfor); | 2254 runtime·parfordo(work.sweepfor); |
| 2255 bufferList[m->helpgc].busy = 0; |
2127 t3 = runtime·nanotime(); | 2256 t3 = runtime·nanotime(); |
2128 | 2257 |
2129 if(work.nproc > 1) | 2258 if(work.nproc > 1) |
2130 runtime·notesleep(&work.alldone); | 2259 runtime·notesleep(&work.alldone); |
2131 | 2260 |
2132 cachestats(&stats); | 2261 cachestats(&stats); |
2133 | 2262 |
2134 stats.nprocyield += work.sweepfor->nprocyield; | 2263 stats.nprocyield += work.sweepfor->nprocyield; |
2135 stats.nosyield += work.sweepfor->nosyield; | 2264 stats.nosyield += work.sweepfor->nosyield; |
2136 stats.nsleep += work.sweepfor->nsleep; | 2265 stats.nsleep += work.sweepfor->nsleep; |
2137 | 2266 |
2138 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; | 2267 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; |
2139 m->gcing = 0; | 2268 m->gcing = 0; |
2140 | |
2141 if(finq != nil) { | |
2142 m->locks++; // disable gc during the mallocs in newproc | |
2143 // kick off or wake up goroutine to run queued finalizers | |
2144 if(fing == nil) | |
2145 fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc
); | |
2146 else if(fingwait) { | |
2147 fingwait = 0; | |
2148 runtime·ready(fing); | |
2149 } | |
2150 m->locks--; | |
2151 } | |
2152 | 2269 |
2153 heap1 = mstats.heap_alloc; | 2270 heap1 = mstats.heap_alloc; |
2154 obj1 = mstats.nmalloc - mstats.nfree; | 2271 obj1 = mstats.nmalloc - mstats.nfree; |
2155 | 2272 |
2156 t4 = runtime·nanotime(); | 2273 t4 = runtime·nanotime(); |
2157 mstats.last_gc = t4; | 2274 mstats.last_gc = t4; |
2158 mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0; | 2275 mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0; |
2159 mstats.pause_total_ns += t4 - t0; | 2276 mstats.pause_total_ns += t4 - t0; |
2160 mstats.numgc++; | 2277 mstats.numgc++; |
2161 if(mstats.debuggc) | 2278 if(mstats.debuggc) |
(...skipping 28 matching lines...) Expand all Loading... |
2190 runtime·printf("\ttotal:\t%D\n", ninstr); | 2307 runtime·printf("\ttotal:\t%D\n", ninstr); |
2191 | 2308 |
2192 runtime·printf("putempty: %D, getfull: %D\n", gcstats.pu
tempty, gcstats.getfull); | 2309 runtime·printf("putempty: %D, getfull: %D\n", gcstats.pu
tempty, gcstats.getfull); |
2193 } | 2310 } |
2194 } | 2311 } |
2195 | 2312 |
2196 runtime·MProf_GC(); | 2313 runtime·MProf_GC(); |
2197 runtime·semrelease(&runtime·worldsema); | 2314 runtime·semrelease(&runtime·worldsema); |
2198 runtime·starttheworld(); | 2315 runtime·starttheworld(); |
2199 | 2316 |
2200 » // give the queued finalizers, if any, a chance to run | 2317 » if(finq != nil) { |
2201 » if(finq != nil) | 2318 » » runtime·lock(&finlock); |
| 2319 » » // kick off or wake up goroutine to run queued finalizers |
| 2320 » » if(fing == nil) |
| 2321 » » » fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc
); |
| 2322 » » else if(fingwait) { |
| 2323 » » » fingwait = 0; |
| 2324 » » » runtime·ready(fing); |
| 2325 » » } |
| 2326 » » runtime·unlock(&finlock); |
| 2327 » » // give the queued finalizers, if any, a chance to run |
2202 runtime·gosched(); | 2328 runtime·gosched(); |
| 2329 } |
2203 } | 2330 } |
2204 | 2331 |
2205 void | 2332 void |
2206 runtime·ReadMemStats(MStats *stats) | 2333 runtime·ReadMemStats(MStats *stats) |
2207 { | 2334 { |
2208 // Have to acquire worldsema to stop the world, | 2335 // Have to acquire worldsema to stop the world, |
2209 // because stoptheworld can only be used by | 2336 // because stoptheworld can only be used by |
2210 // one goroutine at a time, and there might be | 2337 // one goroutine at a time, and there might be |
2211 // a pending garbage collection already calling it. | 2338 // a pending garbage collection already calling it. |
2212 runtime·semacquire(&runtime·worldsema); | 2339 runtime·semacquire(&runtime·worldsema); |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2258 gcpercent = readgogc(); | 2385 gcpercent = readgogc(); |
2259 out = gcpercent; | 2386 out = gcpercent; |
2260 if(in < 0) | 2387 if(in < 0) |
2261 in = -1; | 2388 in = -1; |
2262 gcpercent = in; | 2389 gcpercent = in; |
2263 runtime·unlock(runtime·mheap); | 2390 runtime·unlock(runtime·mheap); |
2264 FLUSH(&out); | 2391 FLUSH(&out); |
2265 } | 2392 } |
2266 | 2393 |
2267 static void | 2394 static void |
| 2395 gchelperstart(void) |
| 2396 { |
| 2397 if(m->helpgc < 0 || m->helpgc >= MaxGcproc) |
| 2398 runtime·throw("gchelperstart: bad m->helpgc"); |
| 2399 if(runtime·xchg(&bufferList[m->helpgc].busy, 1)) |
| 2400 runtime·throw("gchelperstart: already busy"); |
| 2401 } |
| 2402 |
| 2403 static void |
2268 runfinq(void) | 2404 runfinq(void) |
2269 { | 2405 { |
2270 Finalizer *f; | 2406 Finalizer *f; |
2271 FinBlock *fb, *next; | 2407 FinBlock *fb, *next; |
2272 byte *frame; | 2408 byte *frame; |
2273 uint32 framesz, framecap, i; | 2409 uint32 framesz, framecap, i; |
2274 | 2410 |
2275 frame = nil; | 2411 frame = nil; |
2276 framecap = 0; | 2412 framecap = 0; |
2277 for(;;) { | 2413 for(;;) { |
2278 » » // There's no need for a lock in this section | 2414 » » runtime·lock(&finlock); |
2279 » » // because it only conflicts with the garbage | |
2280 » » // collector, and the garbage collector only | |
2281 » » // runs when everyone else is stopped, and | |
2282 » » // runfinq only stops at the gosched() or | |
2283 » » // during the calls in the for loop. | |
2284 fb = finq; | 2415 fb = finq; |
2285 finq = nil; | 2416 finq = nil; |
2286 if(fb == nil) { | 2417 if(fb == nil) { |
2287 fingwait = 1; | 2418 fingwait = 1; |
2288 » » » runtime·park(nil, nil, "finalizer wait"); | 2419 » » » runtime·park(runtime·unlock, &finlock, "finalizer wait")
; |
2289 continue; | 2420 continue; |
2290 } | 2421 } |
| 2422 runtime·unlock(&finlock); |
2291 if(raceenabled) | 2423 if(raceenabled) |
2292 runtime·racefingo(); | 2424 runtime·racefingo(); |
2293 for(; fb; fb=next) { | 2425 for(; fb; fb=next) { |
2294 next = fb->next; | 2426 next = fb->next; |
2295 for(i=0; i<fb->cnt; i++) { | 2427 for(i=0; i<fb->cnt; i++) { |
2296 f = &fb->fin[i]; | 2428 f = &fb->fin[i]; |
2297 framesz = sizeof(uintptr) + f->nret; | 2429 framesz = sizeof(uintptr) + f->nret; |
2298 if(framecap < framesz) { | 2430 if(framecap < framesz) { |
2299 runtime·free(frame); | 2431 runtime·free(frame); |
2300 frame = runtime·mal(framesz); | 2432 frame = runtime·mal(framesz); |
(...skipping 207 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2508 uintptr n; | 2640 uintptr n; |
2509 | 2641 |
2510 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; | 2642 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; |
2511 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); | 2643 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); |
2512 if(h->bitmap_mapped >= n) | 2644 if(h->bitmap_mapped >= n) |
2513 return; | 2645 return; |
2514 | 2646 |
2515 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); | 2647 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); |
2516 h->bitmap_mapped = n; | 2648 h->bitmap_mapped = n; |
2517 } | 2649 } |
LEFT | RIGHT |