LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "zaexperiment.h" | 7 #include "zaexperiment.h" |
8 #include "malloc.h" | 8 #include "malloc.h" |
9 #include "stack.h" | 9 #include "stack.h" |
10 #include "race.h" | 10 #include "race.h" |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
51 uint32 gcwaiting; // gc is waiting to run | 51 uint32 gcwaiting; // gc is waiting to run |
52 int32 stopwait; | 52 int32 stopwait; |
53 Note stopnote; | 53 Note stopnote; |
54 uint32 sysmonwait; | 54 uint32 sysmonwait; |
55 Note sysmonnote; | 55 Note sysmonnote; |
56 uint64 lastpoll; | 56 uint64 lastpoll; |
57 | 57 |
58 int32 profilehz; // cpu profiling rate | 58 int32 profilehz; // cpu profiling rate |
59 }; | 59 }; |
60 | 60 |
61 // The max value of GOMAXPROCS. | 61 enum |
62 // There are no fundamental restrictions on the value. | 62 { |
63 enum { MaxGomaxprocs = 1<<8 }; | 63 » // The max value of GOMAXPROCS. |
| 64 » // There are no fundamental restrictions on the value. |
| 65 » MaxGomaxprocs = 1<<8, |
| 66 |
| 67 » // Number of goroutine ids to grab from runtime·sched.goidgen to local p
er-P cache at once. |
| 68 » // 16 seems to provide enough amortization, but other than that it's mos
tly arbitrary number. |
| 69 » GoidCacheBatch = 16, |
| 70 }; |
64 | 71 |
65 Sched runtime·sched; | 72 Sched runtime·sched; |
66 int32 runtime·gomaxprocs; | 73 int32 runtime·gomaxprocs; |
67 uint32 runtime·needextram; | 74 uint32 runtime·needextram; |
68 bool runtime·iscgo; | 75 bool runtime·iscgo; |
69 M runtime·m0; | 76 M runtime·m0; |
70 G» runtime·g0;» // idle goroutine for m0 | 77 G» runtime·g0;» // idle goroutine for m0 |
71 G*» runtime·allg; | |
72 G* runtime·lastg; | 78 G* runtime·lastg; |
73 M* runtime·allm; | 79 M* runtime·allm; |
74 M* runtime·extram; | 80 M* runtime·extram; |
75 int8* runtime·goos; | 81 int8* runtime·goos; |
76 int32 runtime·ncpu; | 82 int32 runtime·ncpu; |
77 static int32 newprocs; | 83 static int32 newprocs; |
78 | 84 |
| 85 static Lock allglock; // the following vars are protected by this lock or by s
toptheworld |
| 86 G** runtime·allg; |
| 87 uintptr runtime·allglen; |
| 88 static uintptr allgcap; |
| 89 |
79 void runtime·mstart(void); | 90 void runtime·mstart(void); |
80 static void runqput(P*, G*); | 91 static void runqput(P*, G*); |
81 static G* runqget(P*); | 92 static G* runqget(P*); |
82 static void runqgrow(P*); | 93 static bool runqputslow(P*, G*, uint32, uint32); |
83 static G* runqsteal(P*, P*); | 94 static G* runqsteal(P*, P*); |
84 static void mput(M*); | 95 static void mput(M*); |
85 static M* mget(void); | 96 static M* mget(void); |
86 static void mcommoninit(M*); | 97 static void mcommoninit(M*); |
87 static void schedule(void); | 98 static void schedule(void); |
88 static void procresize(int32); | 99 static void procresize(int32); |
89 static void acquirep(P*); | 100 static void acquirep(P*); |
90 static P* releasep(void); | 101 static P* releasep(void); |
91 static void newm(void(*)(void), P*); | 102 static void newm(void(*)(void), P*); |
92 static void stopm(void); | 103 static void stopm(void); |
93 static void startm(P*, bool); | 104 static void startm(P*, bool); |
94 static void handoffp(P*); | 105 static void handoffp(P*); |
95 static void wakep(void); | 106 static void wakep(void); |
96 static void stoplockedm(void); | 107 static void stoplockedm(void); |
97 static void startlockedm(G*); | 108 static void startlockedm(G*); |
98 static void sysmon(void); | 109 static void sysmon(void); |
99 static uint32 retake(int64); | 110 static uint32 retake(int64); |
100 static void incidlelocked(int32); | 111 static void incidlelocked(int32); |
101 static void checkdead(void); | 112 static void checkdead(void); |
102 static void exitsyscall0(G*); | 113 static void exitsyscall0(G*); |
103 static void park0(G*); | 114 static void park0(G*); |
104 static void goexit0(G*); | 115 static void goexit0(G*); |
105 static void gfput(P*, G*); | 116 static void gfput(P*, G*); |
106 static G* gfget(P*); | 117 static G* gfget(P*); |
107 static void gfpurge(P*); | 118 static void gfpurge(P*); |
108 static void globrunqput(G*); | 119 static void globrunqput(G*); |
| 120 static void globrunqputbatch(G*, G*, int32); |
109 static G* globrunqget(P*, int32); | 121 static G* globrunqget(P*, int32); |
110 static P* pidleget(void); | 122 static P* pidleget(void); |
111 static void pidleput(P*); | 123 static void pidleput(P*); |
112 static void injectglist(G*); | 124 static void injectglist(G*); |
113 static bool preemptall(void); | 125 static bool preemptall(void); |
114 static bool preemptone(P*); | 126 static bool preemptone(P*); |
115 static bool exitsyscallfast(void); | 127 static bool exitsyscallfast(void); |
116 static bool haveexperiment(int8*); | 128 static bool haveexperiment(int8*); |
| 129 static void allgadd(G*); |
117 | 130 |
118 // The bootstrap sequence is: | 131 // The bootstrap sequence is: |
119 // | 132 // |
120 // call osinit | 133 // call osinit |
121 // call schedinit | 134 // call schedinit |
122 // make & queue new G | 135 // make & queue new G |
123 // call runtime·mstart | 136 // call runtime·mstart |
124 // | 137 // |
125 // The new G calls runtime·main. | 138 // The new G calls runtime·main. |
126 void | 139 void |
127 runtime·schedinit(void) | 140 runtime·schedinit(void) |
128 { | 141 { |
129 int32 n, procs; | 142 int32 n, procs; |
130 byte *p; | 143 byte *p; |
131 Eface i; | 144 Eface i; |
132 | 145 |
133 runtime·sched.maxmcount = 10000; | 146 runtime·sched.maxmcount = 10000; |
134 runtime·precisestack = haveexperiment("precisestack"); | 147 runtime·precisestack = haveexperiment("precisestack"); |
135 | 148 |
136 runtime·mprofinit(); | |
137 runtime·mallocinit(); | 149 runtime·mallocinit(); |
138 mcommoninit(m); | 150 mcommoninit(m); |
139 ········ | 151 ········ |
140 // Initialize the itable value for newErrorCString, | 152 // Initialize the itable value for newErrorCString, |
141 // so that the next time it gets called, possibly | 153 // so that the next time it gets called, possibly |
142 // in a fault during a garbage collection, it will not | 154 // in a fault during a garbage collection, it will not |
143 // need to allocated memory. | 155 // need to allocated memory. |
144 runtime·newErrorCString(0, &i); | 156 runtime·newErrorCString(0, &i); |
145 | 157 |
146 runtime·goargs(); | 158 runtime·goargs(); |
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
197 // by calling runtime.LockOSThread during initialization | 209 // by calling runtime.LockOSThread during initialization |
198 // to preserve the lock. | 210 // to preserve the lock. |
199 runtime·lockOSThread(); | 211 runtime·lockOSThread(); |
200 ········ | 212 ········ |
201 // Defer unlock so that runtime.Goexit during init does the unlock too. | 213 // Defer unlock so that runtime.Goexit during init does the unlock too. |
202 d.fn = &initDone; | 214 d.fn = &initDone; |
203 d.siz = 0; | 215 d.siz = 0; |
204 d.link = g->defer; | 216 d.link = g->defer; |
205 d.argp = (void*)-1; | 217 d.argp = (void*)-1; |
206 d.special = true; | 218 d.special = true; |
207 d.free = false; | |
208 g->defer = &d; | 219 g->defer = &d; |
209 | 220 |
210 if(m != &runtime·m0) | 221 if(m != &runtime·m0) |
211 runtime·throw("runtime·main not on m0"); | 222 runtime·throw("runtime·main not on m0"); |
212 runtime·newproc1(&scavenger, nil, 0, 0, runtime·main); | 223 runtime·newproc1(&scavenger, nil, 0, 0, runtime·main); |
213 main·init(); | 224 main·init(); |
214 | 225 |
215 if(g->defer != &d || d.fn != &initDone) | 226 if(g->defer != &d || d.fn != &initDone) |
216 runtime·throw("runtime: bad defer entry after init"); | 227 runtime·throw("runtime: bad defer entry after init"); |
217 g->defer = d.link; | 228 g->defer = d.link; |
(...skipping 12 matching lines...) Expand all Loading... |
230 | 241 |
231 runtime·exit(0); | 242 runtime·exit(0); |
232 for(;;) | 243 for(;;) |
233 *(int32*)runtime·main = 0; | 244 *(int32*)runtime·main = 0; |
234 } | 245 } |
235 | 246 |
236 void | 247 void |
237 runtime·goroutineheader(G *gp) | 248 runtime·goroutineheader(G *gp) |
238 { | 249 { |
239 int8 *status; | 250 int8 *status; |
| 251 int64 waitfor; |
240 | 252 |
241 switch(gp->status) { | 253 switch(gp->status) { |
242 case Gidle: | 254 case Gidle: |
243 status = "idle"; | 255 status = "idle"; |
244 break; | 256 break; |
245 case Grunnable: | 257 case Grunnable: |
246 status = "runnable"; | 258 status = "runnable"; |
247 break; | 259 break; |
248 case Grunning: | 260 case Grunning: |
249 status = "running"; | 261 status = "running"; |
250 break; | 262 break; |
251 case Gsyscall: | 263 case Gsyscall: |
252 status = "syscall"; | 264 status = "syscall"; |
253 break; | 265 break; |
254 case Gwaiting: | 266 case Gwaiting: |
255 if(gp->waitreason) | 267 if(gp->waitreason) |
256 status = gp->waitreason; | 268 status = gp->waitreason; |
257 else | 269 else |
258 status = "waiting"; | 270 status = "waiting"; |
259 break; | 271 break; |
260 default: | 272 default: |
261 status = "???"; | 273 status = "???"; |
262 break; | 274 break; |
263 } | 275 } |
264 » runtime·printf("goroutine %D [%s]:\n", gp->goid, status); | 276 |
| 277 » // approx time the G is blocked, in minutes |
| 278 » waitfor = 0; |
| 279 » if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->waitsince !
= 0) |
| 280 » » waitfor = (runtime·nanotime() - gp->waitsince) / (60LL*1000*1000
*1000); |
| 281 |
| 282 » if(waitfor < 1) |
| 283 » » runtime·printf("goroutine %D [%s]:\n", gp->goid, status); |
| 284 » else |
| 285 » » runtime·printf("goroutine %D [%s, %D minutes]:\n", gp->goid, sta
tus, waitfor); |
265 } | 286 } |
266 | 287 |
267 void | 288 void |
268 runtime·tracebackothers(G *me) | 289 runtime·tracebackothers(G *me) |
269 { | 290 { |
270 G *gp; | 291 G *gp; |
271 int32 traceback; | 292 int32 traceback; |
| 293 uintptr i; |
272 | 294 |
273 traceback = runtime·gotraceback(nil); | 295 traceback = runtime·gotraceback(nil); |
274 ········ | 296 ········ |
275 // Show the current goroutine first, if we haven't already. | 297 // Show the current goroutine first, if we haven't already. |
276 if((gp = m->curg) != nil && gp != me) { | 298 if((gp = m->curg) != nil && gp != me) { |
277 runtime·printf("\n"); | 299 runtime·printf("\n"); |
278 runtime·goroutineheader(gp); | 300 runtime·goroutineheader(gp); |
279 runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp); | 301 runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp); |
280 } | 302 } |
281 | 303 |
282 » for(gp = runtime·allg; gp != nil; gp = gp->alllink) { | 304 » runtime·lock(&allglock); |
| 305 » for(i = 0; i < runtime·allglen; i++) { |
| 306 » » gp = runtime·allg[i]; |
283 if(gp == me || gp == m->curg || gp->status == Gdead) | 307 if(gp == me || gp == m->curg || gp->status == Gdead) |
284 continue; | 308 continue; |
285 if(gp->issystem && traceback < 2) | 309 if(gp->issystem && traceback < 2) |
286 continue; | 310 continue; |
287 runtime·printf("\n"); | 311 runtime·printf("\n"); |
288 runtime·goroutineheader(gp); | 312 runtime·goroutineheader(gp); |
289 if(gp->status == Grunning) { | 313 if(gp->status == Grunning) { |
290 runtime·printf("\tgoroutine running on other thread; sta
ck unavailable\n"); | 314 runtime·printf("\tgoroutine running on other thread; sta
ck unavailable\n"); |
291 runtime·printcreatedby(gp); | 315 runtime·printcreatedby(gp); |
292 } else | 316 } else |
293 runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp); | 317 runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp); |
294 } | 318 } |
| 319 runtime·unlock(&allglock); |
295 } | 320 } |
296 | 321 |
297 static void | 322 static void |
298 checkmcount(void) | 323 checkmcount(void) |
299 { | 324 { |
300 // sched lock is held | 325 // sched lock is held |
301 if(runtime·sched.mcount > runtime·sched.maxmcount) { | 326 if(runtime·sched.mcount > runtime·sched.maxmcount) { |
302 runtime·printf("runtime: program exceeds %d-thread limit\n", run
time·sched.maxmcount); | 327 runtime·printf("runtime: program exceeds %d-thread limit\n", run
time·sched.maxmcount); |
303 runtime·throw("thread exhaustion"); | 328 runtime·throw("thread exhaustion"); |
304 } | 329 } |
(...skipping 308 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
613 // When running with cgo, we call _cgo_thread_start | 638 // When running with cgo, we call _cgo_thread_start |
614 // to start threads for us so that we can play nicely with | 639 // to start threads for us so that we can play nicely with |
615 // foreign code. | 640 // foreign code. |
616 void (*_cgo_thread_start)(void*); | 641 void (*_cgo_thread_start)(void*); |
617 | 642 |
618 typedef struct CgoThreadStart CgoThreadStart; | 643 typedef struct CgoThreadStart CgoThreadStart; |
619 struct CgoThreadStart | 644 struct CgoThreadStart |
620 { | 645 { |
621 M *m; | 646 M *m; |
622 G *g; | 647 G *g; |
| 648 uintptr *tls; |
623 void (*fn)(void); | 649 void (*fn)(void); |
624 }; | 650 }; |
625 | 651 |
626 // Allocate a new m unassociated with any thread. | 652 // Allocate a new m unassociated with any thread. |
627 // Can use p for allocation context if needed. | 653 // Can use p for allocation context if needed. |
628 M* | 654 M* |
629 runtime·allocm(P *p) | 655 runtime·allocm(P *p) |
630 { | 656 { |
631 M *mp; | 657 M *mp; |
632 static Type *mtype; // The Go type M | 658 static Type *mtype; // The Go type M |
633 | 659 |
634 m->locks++; // disable GC because it can be called from sysmon | 660 m->locks++; // disable GC because it can be called from sysmon |
635 if(m->p == nil) | 661 if(m->p == nil) |
636 acquirep(p); // temporarily borrow p for mallocs in this functi
on | 662 acquirep(p); // temporarily borrow p for mallocs in this functi
on |
637 if(mtype == nil) { | 663 if(mtype == nil) { |
638 Eface e; | 664 Eface e; |
639 runtime·gc_m_ptr(&e); | 665 runtime·gc_m_ptr(&e); |
640 mtype = ((PtrType*)e.type)->elem; | 666 mtype = ((PtrType*)e.type)->elem; |
641 } | 667 } |
642 | 668 |
643 mp = runtime·cnew(mtype); | 669 mp = runtime·cnew(mtype); |
644 mcommoninit(mp); | 670 mcommoninit(mp); |
645 | 671 |
646 » // In case of cgo, pthread_create will make us a stack. | 672 » // In case of cgo or Solaris, pthread_create will make us a stack. |
647 // Windows will layout sched stack on OS stack. | 673 // Windows will layout sched stack on OS stack. |
648 » if(runtime·iscgo || Windows) | 674 » if(runtime·iscgo || Solaris || Windows) |
649 mp->g0 = runtime·malg(-1); | 675 mp->g0 = runtime·malg(-1); |
650 else | 676 else |
651 mp->g0 = runtime·malg(8192); | 677 mp->g0 = runtime·malg(8192); |
652 | 678 |
653 if(p == m->p) | 679 if(p == m->p) |
654 releasep(); | 680 releasep(); |
655 m->locks--; | 681 m->locks--; |
656 if(m->locks == 0 && g->preempt) // restore the preemption request in ca
se we've cleared it in newstack | 682 if(m->locks == 0 && g->preempt) // restore the preemption request in ca
se we've cleared it in newstack |
657 g->stackguard0 = StackPreempt; | 683 g->stackguard0 = StackPreempt; |
658 | 684 |
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
776 gp->syscallguard = gp->stackguard; | 802 gp->syscallguard = gp->stackguard; |
777 gp->status = Gsyscall; | 803 gp->status = Gsyscall; |
778 mp->curg = gp; | 804 mp->curg = gp; |
779 mp->locked = LockInternal; | 805 mp->locked = LockInternal; |
780 mp->lockedg = gp; | 806 mp->lockedg = gp; |
781 gp->lockedm = mp; | 807 gp->lockedm = mp; |
782 gp->goid = runtime·xadd64(&runtime·sched.goidgen, 1); | 808 gp->goid = runtime·xadd64(&runtime·sched.goidgen, 1); |
783 if(raceenabled) | 809 if(raceenabled) |
784 gp->racectx = runtime·racegostart(runtime·newextram); | 810 gp->racectx = runtime·racegostart(runtime·newextram); |
785 // put on allg for garbage collector | 811 // put on allg for garbage collector |
786 » runtime·lock(&runtime·sched); | 812 » allgadd(gp); |
787 » if(runtime·lastg == nil) | |
788 » » runtime·allg = gp; | |
789 » else | |
790 » » runtime·lastg->alllink = gp; | |
791 » runtime·lastg = gp; | |
792 » runtime·unlock(&runtime·sched); | |
793 | 813 |
794 // Add m to the extra list. | 814 // Add m to the extra list. |
795 mnext = lockextra(true); | 815 mnext = lockextra(true); |
796 mp->schedlink = mnext; | 816 mp->schedlink = mnext; |
797 unlockextra(mp); | 817 unlockextra(mp); |
798 } | 818 } |
799 | 819 |
800 // dropm is called when a cgo callback has called needm but is now | 820 // dropm is called when a cgo callback has called needm but is now |
801 // done with the callback and returning back into the non-Go thread. | 821 // done with the callback and returning back into the non-Go thread. |
802 // It puts the current m back onto the extra list. | 822 // It puts the current m back onto the extra list. |
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
897 mp->nextp = p; | 917 mp->nextp = p; |
898 mp->mstartfn = fn; | 918 mp->mstartfn = fn; |
899 | 919 |
900 if(runtime·iscgo) { | 920 if(runtime·iscgo) { |
901 CgoThreadStart ts; | 921 CgoThreadStart ts; |
902 | 922 |
903 if(_cgo_thread_start == nil) | 923 if(_cgo_thread_start == nil) |
904 runtime·throw("_cgo_thread_start missing"); | 924 runtime·throw("_cgo_thread_start missing"); |
905 ts.m = mp; | 925 ts.m = mp; |
906 ts.g = mp->g0; | 926 ts.g = mp->g0; |
| 927 ts.tls = mp->tls; |
907 ts.fn = runtime·mstart; | 928 ts.fn = runtime·mstart; |
908 runtime·asmcgocall(_cgo_thread_start, &ts); | 929 runtime·asmcgocall(_cgo_thread_start, &ts); |
909 return; | 930 return; |
910 } | 931 } |
911 runtime·newosproc(mp, (byte*)mp->g0->stackbase); | 932 runtime·newosproc(mp, (byte*)mp->g0->stackbase); |
912 } | 933 } |
913 | 934 |
914 // Stops execution of the current m until new work is available. | 935 // Stops execution of the current m until new work is available. |
915 // Returns with acquired P. | 936 // Returns with acquired P. |
916 static void | 937 static void |
(...skipping 24 matching lines...) Expand all Loading... |
941 m->nextp = nil; | 962 m->nextp = nil; |
942 } | 963 } |
943 | 964 |
944 static void | 965 static void |
945 mspinning(void) | 966 mspinning(void) |
946 { | 967 { |
947 m->spinning = true; | 968 m->spinning = true; |
948 } | 969 } |
949 | 970 |
950 // Schedules some M to run the p (creates an M if necessary). | 971 // Schedules some M to run the p (creates an M if necessary). |
951 // If p==nil, tries to get an idle P, if no idle P's returns false. | 972 // If p==nil, tries to get an idle P, if no idle P's does nothing. |
952 static void | 973 static void |
953 startm(P *p, bool spinning) | 974 startm(P *p, bool spinning) |
954 { | 975 { |
955 M *mp; | 976 M *mp; |
956 void (*fn)(void); | 977 void (*fn)(void); |
957 | 978 |
958 runtime·lock(&runtime·sched); | 979 runtime·lock(&runtime·sched); |
959 if(p == nil) { | 980 if(p == nil) { |
960 p = pidleget(); | 981 p = pidleget(); |
961 if(p == nil) { | 982 if(p == nil) { |
(...skipping 143 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1105 static void | 1126 static void |
1106 execute(G *gp) | 1127 execute(G *gp) |
1107 { | 1128 { |
1108 int32 hz; | 1129 int32 hz; |
1109 | 1130 |
1110 if(gp->status != Grunnable) { | 1131 if(gp->status != Grunnable) { |
1111 runtime·printf("execute: bad g status %d\n", gp->status); | 1132 runtime·printf("execute: bad g status %d\n", gp->status); |
1112 runtime·throw("execute: bad g status"); | 1133 runtime·throw("execute: bad g status"); |
1113 } | 1134 } |
1114 gp->status = Grunning; | 1135 gp->status = Grunning; |
| 1136 gp->waitsince = 0; |
1115 gp->preempt = false; | 1137 gp->preempt = false; |
1116 gp->stackguard0 = gp->stackguard; | 1138 gp->stackguard0 = gp->stackguard; |
1117 m->p->schedtick++; | 1139 m->p->schedtick++; |
1118 m->curg = gp; | 1140 m->curg = gp; |
1119 gp->m = m; | 1141 gp->m = m; |
1120 | 1142 |
1121 // Check whether the profiler needs to be turned on or off. | 1143 // Check whether the profiler needs to be turned on or off. |
1122 hz = runtime·sched.profilehz; | 1144 hz = runtime·sched.profilehz; |
1123 if(m->profilehz != hz) | 1145 if(m->profilehz != hz) |
1124 runtime·resetcpuprofiler(hz); | 1146 runtime·resetcpuprofiler(hz); |
(...skipping 199 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1324 if(gp->lockedm) { | 1346 if(gp->lockedm) { |
1325 // Hands off own p to the locked m, | 1347 // Hands off own p to the locked m, |
1326 // then blocks waiting for a new p. | 1348 // then blocks waiting for a new p. |
1327 startlockedm(gp); | 1349 startlockedm(gp); |
1328 goto top; | 1350 goto top; |
1329 } | 1351 } |
1330 | 1352 |
1331 execute(gp); | 1353 execute(gp); |
1332 } | 1354 } |
1333 | 1355 |
1334 // Puts the current goroutine into a waiting state and unlocks the lock. | 1356 // Puts the current goroutine into a waiting state and calls unlockf. |
1335 // The goroutine can be made runnable again by calling runtime·ready(gp). | 1357 // If unlockf returns false, the goroutine is resumed. |
1336 void | 1358 void |
1337 runtime·park(void(*unlockf)(Lock*), Lock *lock, int8 *reason) | 1359 runtime·park(bool(*unlockf)(G*, void*), void *lock, int8 *reason) |
1338 { | 1360 { |
1339 m->waitlock = lock; | 1361 m->waitlock = lock; |
1340 m->waitunlockf = unlockf; | 1362 m->waitunlockf = unlockf; |
1341 g->waitreason = reason; | 1363 g->waitreason = reason; |
1342 runtime·mcall(park0); | 1364 runtime·mcall(park0); |
1343 } | 1365 } |
1344 | 1366 |
| 1367 static bool |
| 1368 parkunlock(G *gp, void *lock) |
| 1369 { |
| 1370 USED(gp); |
| 1371 runtime·unlock(lock); |
| 1372 return true; |
| 1373 } |
| 1374 |
| 1375 // Puts the current goroutine into a waiting state and unlocks the lock. |
| 1376 // The goroutine can be made runnable again by calling runtime·ready(gp). |
| 1377 void |
| 1378 runtime·parkunlock(Lock *lock, int8 *reason) |
| 1379 { |
| 1380 runtime·park(parkunlock, lock, reason); |
| 1381 } |
| 1382 |
1345 // runtime·park continuation on g0. | 1383 // runtime·park continuation on g0. |
1346 static void | 1384 static void |
1347 park0(G *gp) | 1385 park0(G *gp) |
1348 { | 1386 { |
| 1387 bool ok; |
| 1388 |
1349 gp->status = Gwaiting; | 1389 gp->status = Gwaiting; |
1350 gp->m = nil; | 1390 gp->m = nil; |
1351 m->curg = nil; | 1391 m->curg = nil; |
1352 if(m->waitunlockf) { | 1392 if(m->waitunlockf) { |
1353 » » m->waitunlockf(m->waitlock); | 1393 » » ok = m->waitunlockf(gp, m->waitlock); |
1354 m->waitunlockf = nil; | 1394 m->waitunlockf = nil; |
1355 m->waitlock = nil; | 1395 m->waitlock = nil; |
| 1396 if(!ok) { |
| 1397 gp->status = Grunnable; |
| 1398 execute(gp); // Schedule it back, never returns. |
| 1399 } |
1356 } | 1400 } |
1357 if(m->lockedg) { | 1401 if(m->lockedg) { |
1358 stoplockedm(); | 1402 stoplockedm(); |
1359 execute(gp); // Never returns. | 1403 execute(gp); // Never returns. |
1360 } | 1404 } |
1361 schedule(); | 1405 schedule(); |
1362 } | 1406 } |
1363 | 1407 |
1364 // Scheduler yield. | 1408 // Scheduler yield. |
1365 void | 1409 void |
(...skipping 162 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1528 // from the low-level system calls used by the runtime. | 1572 // from the low-level system calls used by the runtime. |
1529 #pragma textflag NOSPLIT | 1573 #pragma textflag NOSPLIT |
1530 void | 1574 void |
1531 runtime·exitsyscall(void) | 1575 runtime·exitsyscall(void) |
1532 { | 1576 { |
1533 m->locks++; // see comment in entersyscall | 1577 m->locks++; // see comment in entersyscall |
1534 | 1578 |
1535 if(g->isbackground) // do not consider blocked scavenger for deadlock d
etection | 1579 if(g->isbackground) // do not consider blocked scavenger for deadlock d
etection |
1536 incidlelocked(-1); | 1580 incidlelocked(-1); |
1537 | 1581 |
| 1582 g->waitsince = 0; |
1538 if(exitsyscallfast()) { | 1583 if(exitsyscallfast()) { |
1539 // There's a cpu for us, so we can run. | 1584 // There's a cpu for us, so we can run. |
1540 m->p->syscalltick++; | 1585 m->p->syscalltick++; |
1541 g->status = Grunning; | 1586 g->status = Grunning; |
1542 // Garbage collector isn't running (since we are), | 1587 // Garbage collector isn't running (since we are), |
1543 // so okay to clear gcstack and gcsp. | 1588 // so okay to clear gcstack and gcsp. |
1544 g->syscallstack = (uintptr)nil; | 1589 g->syscallstack = (uintptr)nil; |
1545 g->syscallsp = (uintptr)nil; | 1590 g->syscallsp = (uintptr)nil; |
1546 m->locks--; | 1591 m->locks--; |
1547 if(g->preempt) { | 1592 if(g->preempt) { |
(...skipping 181 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1729 | 1774 |
1730 // Create a new g running fn with narg bytes of arguments starting | 1775 // Create a new g running fn with narg bytes of arguments starting |
1731 // at argp and returning nret bytes of results. callerpc is the | 1776 // at argp and returning nret bytes of results. callerpc is the |
1732 // address of the go statement that created this. The new g is put | 1777 // address of the go statement that created this. The new g is put |
1733 // on the queue of g's waiting to run. | 1778 // on the queue of g's waiting to run. |
1734 G* | 1779 G* |
1735 runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerpc
) | 1780 runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerpc
) |
1736 { | 1781 { |
1737 byte *sp; | 1782 byte *sp; |
1738 G *newg; | 1783 G *newg; |
| 1784 P *p; |
1739 int32 siz; | 1785 int32 siz; |
1740 | 1786 |
1741 //runtime·printf("newproc1 %p %p narg=%d nret=%d\n", fn->fn, argp, narg, nret); | 1787 //runtime·printf("newproc1 %p %p narg=%d nret=%d\n", fn->fn, argp, narg, nret); |
1742 m->locks++; // disable preemption because it can be holding p in a loca
l var | 1788 m->locks++; // disable preemption because it can be holding p in a loca
l var |
1743 siz = narg + nret; | 1789 siz = narg + nret; |
1744 siz = (siz+7) & ~7; | 1790 siz = (siz+7) & ~7; |
1745 | 1791 |
1746 // We could instead create a secondary stack frame | 1792 // We could instead create a secondary stack frame |
1747 // and make it look like goexit was on the original but | 1793 // and make it look like goexit was on the original but |
1748 // the call to the actual goroutine function was split. | 1794 // the call to the actual goroutine function was split. |
1749 // Not worth it: this is almost always an error. | 1795 // Not worth it: this is almost always an error. |
1750 if(siz > StackMin - 1024) | 1796 if(siz > StackMin - 1024) |
1751 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); | 1797 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); |
1752 | 1798 |
1753 » if((newg = gfget(m->p)) != nil) { | 1799 » p = m->p; |
| 1800 » if((newg = gfget(p)) != nil) { |
1754 if(newg->stackguard - StackGuard != newg->stack0) | 1801 if(newg->stackguard - StackGuard != newg->stack0) |
1755 runtime·throw("invalid stack in newg"); | 1802 runtime·throw("invalid stack in newg"); |
1756 } else { | 1803 } else { |
1757 newg = runtime·malg(StackMin); | 1804 newg = runtime·malg(StackMin); |
1758 » » runtime·lock(&runtime·sched); | 1805 » » allgadd(newg); |
1759 » » if(runtime·lastg == nil) | |
1760 » » » runtime·allg = newg; | |
1761 » » else | |
1762 » » » runtime·lastg->alllink = newg; | |
1763 » » runtime·lastg = newg; | |
1764 » » runtime·unlock(&runtime·sched); | |
1765 } | 1806 } |
1766 | 1807 |
1767 sp = (byte*)newg->stackbase; | 1808 sp = (byte*)newg->stackbase; |
1768 sp -= siz; | 1809 sp -= siz; |
1769 runtime·memmove(sp, argp, narg); | 1810 runtime·memmove(sp, argp, narg); |
1770 if(thechar == '5') { | 1811 if(thechar == '5') { |
1771 // caller's LR | 1812 // caller's LR |
1772 sp -= sizeof(void*); | 1813 sp -= sizeof(void*); |
1773 *(void**)sp = nil; | 1814 *(void**)sp = nil; |
1774 } | 1815 } |
1775 | 1816 |
1776 runtime·memclr((byte*)&newg->sched, sizeof newg->sched); | 1817 runtime·memclr((byte*)&newg->sched, sizeof newg->sched); |
1777 newg->sched.sp = (uintptr)sp; | 1818 newg->sched.sp = (uintptr)sp; |
1778 newg->sched.pc = (uintptr)runtime·goexit; | 1819 newg->sched.pc = (uintptr)runtime·goexit; |
1779 newg->sched.g = newg; | 1820 newg->sched.g = newg; |
1780 runtime·gostartcallfn(&newg->sched, fn); | 1821 runtime·gostartcallfn(&newg->sched, fn); |
1781 newg->gopc = (uintptr)callerpc; | 1822 newg->gopc = (uintptr)callerpc; |
1782 newg->status = Grunnable; | 1823 newg->status = Grunnable; |
1783 » newg->goid = runtime·xadd64(&runtime·sched.goidgen, 1); | 1824 » if(p->goidcache == p->goidcacheend) { |
| 1825 » » p->goidcache = runtime·xadd64(&runtime·sched.goidgen, GoidCacheB
atch); |
| 1826 » » p->goidcacheend = p->goidcache + GoidCacheBatch; |
| 1827 » } |
| 1828 » newg->goid = p->goidcache++; |
1784 newg->panicwrap = 0; | 1829 newg->panicwrap = 0; |
1785 if(raceenabled) | 1830 if(raceenabled) |
1786 newg->racectx = runtime·racegostart((void*)callerpc); | 1831 newg->racectx = runtime·racegostart((void*)callerpc); |
1787 » runqput(m->p, newg); | 1832 » runqput(p, newg); |
1788 | 1833 |
1789 if(runtime·atomicload(&runtime·sched.npidle) != 0 && runtime·atomicload(
&runtime·sched.nmspinning) == 0 && fn->fn != runtime·main) // TODO: fast atomic | 1834 if(runtime·atomicload(&runtime·sched.npidle) != 0 && runtime·atomicload(
&runtime·sched.nmspinning) == 0 && fn->fn != runtime·main) // TODO: fast atomic |
1790 wakep(); | 1835 wakep(); |
1791 m->locks--; | 1836 m->locks--; |
1792 if(m->locks == 0 && g->preempt) // restore the preemption request in ca
se we've cleared it in newstack | 1837 if(m->locks == 0 && g->preempt) // restore the preemption request in ca
se we've cleared it in newstack |
1793 g->stackguard0 = StackPreempt; | 1838 g->stackguard0 = StackPreempt; |
1794 return newg; | 1839 return newg; |
| 1840 } |
| 1841 |
| 1842 static void |
| 1843 allgadd(G *gp) |
| 1844 { |
| 1845 G **new; |
| 1846 uintptr cap; |
| 1847 |
| 1848 runtime·lock(&allglock); |
| 1849 if(runtime·allglen >= allgcap) { |
| 1850 cap = 4096/sizeof(new[0]); |
| 1851 if(cap < 2*allgcap) |
| 1852 cap = 2*allgcap; |
| 1853 new = runtime·malloc(cap*sizeof(new[0])); |
| 1854 if(new == nil) |
| 1855 runtime·throw("runtime: cannot allocate memory"); |
| 1856 if(runtime·allg != nil) { |
| 1857 runtime·memmove(new, runtime·allg, runtime·allglen*sizeo
f(new[0])); |
| 1858 runtime·free(runtime·allg); |
| 1859 } |
| 1860 runtime·allg = new; |
| 1861 allgcap = cap; |
| 1862 } |
| 1863 runtime·allg[runtime·allglen++] = gp; |
| 1864 runtime·unlock(&allglock); |
1795 } | 1865 } |
1796 | 1866 |
1797 // Put on gfree list. | 1867 // Put on gfree list. |
1798 // If local list is too long, transfer a batch to the global list. | 1868 // If local list is too long, transfer a batch to the global list. |
1799 static void | 1869 static void |
1800 gfput(P *p, G *gp) | 1870 gfput(P *p, G *gp) |
1801 { | 1871 { |
1802 if(gp->stackguard - StackGuard != gp->stack0) | 1872 if(gp->stackguard - StackGuard != gp->stack0) |
1803 runtime·throw("invalid stack in gfput"); | 1873 runtime·throw("invalid stack in gfput"); |
1804 gp->schedlink = p->gfree; | 1874 gp->schedlink = p->gfree; |
(...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1976 { | 2046 { |
1977 ret = runtime·gcount(); | 2047 ret = runtime·gcount(); |
1978 FLUSH(&ret); | 2048 FLUSH(&ret); |
1979 } | 2049 } |
1980 | 2050 |
1981 int32 | 2051 int32 |
1982 runtime·gcount(void) | 2052 runtime·gcount(void) |
1983 { | 2053 { |
1984 G *gp; | 2054 G *gp; |
1985 int32 n, s; | 2055 int32 n, s; |
| 2056 uintptr i; |
1986 | 2057 |
1987 n = 0; | 2058 n = 0; |
1988 » runtime·lock(&runtime·sched); | 2059 » runtime·lock(&allglock); |
1989 // TODO(dvyukov): runtime.NumGoroutine() is O(N). | 2060 // TODO(dvyukov): runtime.NumGoroutine() is O(N). |
1990 // We do not want to increment/decrement centralized counter in newproc/
goexit, | 2061 // We do not want to increment/decrement centralized counter in newproc/
goexit, |
1991 // just to make runtime.NumGoroutine() faster. | 2062 // just to make runtime.NumGoroutine() faster. |
1992 // Compromise solution is to introduce per-P counters of active goroutin
es. | 2063 // Compromise solution is to introduce per-P counters of active goroutin
es. |
1993 » for(gp = runtime·allg; gp; gp = gp->alllink) { | 2064 » for(i = 0; i < runtime·allglen; i++) { |
| 2065 » » gp = runtime·allg[i]; |
1994 s = gp->status; | 2066 s = gp->status; |
1995 if(s == Grunnable || s == Grunning || s == Gsyscall || s == Gwai
ting) | 2067 if(s == Grunnable || s == Grunning || s == Gsyscall || s == Gwai
ting) |
1996 n++; | 2068 n++; |
1997 } | 2069 } |
1998 » runtime·unlock(&runtime·sched); | 2070 » runtime·unlock(&allglock); |
1999 return n; | 2071 return n; |
2000 } | 2072 } |
2001 | 2073 |
2002 int32 | 2074 int32 |
2003 runtime·mcount(void) | 2075 runtime·mcount(void) |
2004 { | 2076 { |
2005 return runtime·sched.mcount; | 2077 return runtime·sched.mcount; |
2006 } | 2078 } |
2007 | 2079 |
2008 void | 2080 void |
(...skipping 23 matching lines...) Expand all Loading... |
2032 uintptr pcbuf[100]; | 2104 uintptr pcbuf[100]; |
2033 } prof; | 2105 } prof; |
2034 | 2106 |
2035 static void | 2107 static void |
2036 System(void) | 2108 System(void) |
2037 { | 2109 { |
2038 } | 2110 } |
2039 | 2111 |
2040 // Called if we receive a SIGPROF signal. | 2112 // Called if we receive a SIGPROF signal. |
2041 void | 2113 void |
2042 runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp) | 2114 runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp) |
2043 { | 2115 { |
2044 int32 n; | 2116 int32 n; |
2045 bool traceback; | 2117 bool traceback; |
| 2118 MCache *mcache; |
| 2119 // Do not use global m in this function, use mp instead. |
| 2120 // On windows one m is sending reports about all the g's, so m means a w
rong thing. |
| 2121 byte m; |
| 2122 |
| 2123 m = 0; |
| 2124 USED(m); |
2046 | 2125 |
2047 if(prof.fn == nil || prof.hz == 0) | 2126 if(prof.fn == nil || prof.hz == 0) |
2048 return; | 2127 return; |
2049 » traceback = true; | 2128 |
2050 » // Windows does profiling in a dedicated thread w/o m. | 2129 » // Profiling runs concurrently with GC, so it must not allocate. |
2051 » if(!Windows && (m == nil || m->mcache == nil)) | 2130 » mcache = mp->mcache; |
2052 » » traceback = false; | 2131 » mp->mcache = nil; |
2053 »······· | 2132 |
2054 // Define that a "user g" is a user-created goroutine, and a "system g" | 2133 // Define that a "user g" is a user-created goroutine, and a "system g" |
2055 // is one that is m->g0 or m->gsignal. We've only made sure that we | 2134 // is one that is m->g0 or m->gsignal. We've only made sure that we |
2056 // can unwind user g's, so exclude the system g's. | 2135 // can unwind user g's, so exclude the system g's. |
2057 // | 2136 // |
2058 // It is not quite as easy as testing gp == m->curg (the current user g) | 2137 // It is not quite as easy as testing gp == m->curg (the current user g) |
2059 // because we might be interrupted for profiling halfway through a | 2138 // because we might be interrupted for profiling halfway through a |
2060 // goroutine switch. The switch involves updating three (or four) values
: | 2139 // goroutine switch. The switch involves updating three (or four) values
: |
2061 // g, PC, SP, and (on arm) LR. The PC must be the last to be updated, | 2140 // g, PC, SP, and (on arm) LR. The PC must be the last to be updated, |
2062 // because once it gets updated the new g is running. | 2141 // because once it gets updated the new g is running. |
2063 // | 2142 // |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2116 // The biggest drawback to this solution is that it requires that we can
tell | 2195 // The biggest drawback to this solution is that it requires that we can
tell |
2117 // whether it's safe to read from the memory pointed at by PC. | 2196 // whether it's safe to read from the memory pointed at by PC. |
2118 // In a correct program, we can test PC == nil and otherwise read, | 2197 // In a correct program, we can test PC == nil and otherwise read, |
2119 // but if a profiling signal happens at the instant that a program execu
tes | 2198 // but if a profiling signal happens at the instant that a program execu
tes |
2120 // a bad jump (before the program manages to handle the resulting fault) | 2199 // a bad jump (before the program manages to handle the resulting fault) |
2121 // the profiling handler could fault trying to read nonexistent memory. | 2200 // the profiling handler could fault trying to read nonexistent memory. |
2122 // | 2201 // |
2123 // To recap, there are no constraints on the assembly being used for the | 2202 // To recap, there are no constraints on the assembly being used for the |
2124 // transition. We simply require that g and SP match and that the PC is
not | 2203 // transition. We simply require that g and SP match and that the PC is
not |
2125 // in runtime.gogo. | 2204 // in runtime.gogo. |
2126 » // | 2205 » traceback = true; |
2127 » // On Windows, one m is sending reports about all the g's, so gp == m->c
urg | 2206 » if(gp == nil || gp != mp->curg || |
2128 » // is not a useful comparison. The profilem function in os_windows.c has | |
2129 » // already checked that gp is a user g. | |
2130 » if(gp == nil || | |
2131 » (!Windows && gp != m->curg) || | |
2132 (uintptr)sp < gp->stackguard - StackGuard || gp->stackbase < (uintptr
)sp || | 2207 (uintptr)sp < gp->stackguard - StackGuard || gp->stackbase < (uintptr
)sp || |
2133 ((uint8*)runtime·gogo <= pc && pc < (uint8*)runtime·gogo + RuntimeGog
oBytes)) | 2208 ((uint8*)runtime·gogo <= pc && pc < (uint8*)runtime·gogo + RuntimeGog
oBytes)) |
2134 traceback = false; | 2209 traceback = false; |
2135 | 2210 |
2136 // Race detector calls asmcgocall w/o entersyscall/exitsyscall, | 2211 // Race detector calls asmcgocall w/o entersyscall/exitsyscall, |
2137 // we can not currently unwind through asmcgocall. | 2212 // we can not currently unwind through asmcgocall. |
2138 » if(m != nil && m->racecall) | 2213 » if(mp != nil && mp->racecall) |
2139 traceback = false; | 2214 traceback = false; |
2140 | 2215 |
2141 runtime·lock(&prof); | 2216 runtime·lock(&prof); |
2142 if(prof.fn == nil) { | 2217 if(prof.fn == nil) { |
2143 runtime·unlock(&prof); | 2218 runtime·unlock(&prof); |
| 2219 mp->mcache = mcache; |
2144 return; | 2220 return; |
2145 } | 2221 } |
2146 n = 0; | 2222 n = 0; |
2147 if(traceback) | 2223 if(traceback) |
2148 n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr,
gp, 0, prof.pcbuf, nelem(prof.pcbuf), nil, nil, false); | 2224 n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr,
gp, 0, prof.pcbuf, nelem(prof.pcbuf), nil, nil, false); |
2149 if(!traceback || n <= 0) { | 2225 if(!traceback || n <= 0) { |
2150 n = 2; | 2226 n = 2; |
2151 prof.pcbuf[0] = (uintptr)pc; | 2227 prof.pcbuf[0] = (uintptr)pc; |
2152 prof.pcbuf[1] = (uintptr)System + 1; | 2228 prof.pcbuf[1] = (uintptr)System + 1; |
2153 } | 2229 } |
2154 prof.fn(prof.pcbuf, n); | 2230 prof.fn(prof.pcbuf, n); |
2155 runtime·unlock(&prof); | 2231 runtime·unlock(&prof); |
| 2232 mp->mcache = mcache; |
2156 } | 2233 } |
2157 | 2234 |
2158 // Arrange to call fn with a traceback hz times a second. | 2235 // Arrange to call fn with a traceback hz times a second. |
2159 void | 2236 void |
2160 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) | 2237 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) |
2161 { | 2238 { |
2162 // Force sane arguments. | 2239 // Force sane arguments. |
2163 if(hz < 0) | 2240 if(hz < 0) |
2164 hz = 0; | 2241 hz = 0; |
2165 if(hz == 0) | 2242 if(hz == 0) |
(...skipping 22 matching lines...) Expand all Loading... |
2188 runtime·resetcpuprofiler(hz); | 2265 runtime·resetcpuprofiler(hz); |
2189 | 2266 |
2190 m->locks--; | 2267 m->locks--; |
2191 } | 2268 } |
2192 | 2269 |
2193 // Change number of processors. The world is stopped, sched is locked. | 2270 // Change number of processors. The world is stopped, sched is locked. |
2194 static void | 2271 static void |
2195 procresize(int32 new) | 2272 procresize(int32 new) |
2196 { | 2273 { |
2197 int32 i, old; | 2274 int32 i, old; |
| 2275 bool empty; |
2198 G *gp; | 2276 G *gp; |
2199 P *p; | 2277 P *p; |
2200 | 2278 |
2201 old = runtime·gomaxprocs; | 2279 old = runtime·gomaxprocs; |
2202 if(old < 0 || old > MaxGomaxprocs || new <= 0 || new >MaxGomaxprocs) | 2280 if(old < 0 || old > MaxGomaxprocs || new <= 0 || new >MaxGomaxprocs) |
2203 runtime·throw("procresize: invalid arg"); | 2281 runtime·throw("procresize: invalid arg"); |
2204 // initialize new P's | 2282 // initialize new P's |
2205 for(i = 0; i < new; i++) { | 2283 for(i = 0; i < new; i++) { |
2206 p = runtime·allp[i]; | 2284 p = runtime·allp[i]; |
2207 if(p == nil) { | 2285 if(p == nil) { |
2208 p = (P*)runtime·mallocgc(sizeof(*p), 0, FlagNoInvokeGC); | 2286 p = (P*)runtime·mallocgc(sizeof(*p), 0, FlagNoInvokeGC); |
2209 p->id = i; | 2287 p->id = i; |
2210 p->status = Pgcstop; | 2288 p->status = Pgcstop; |
2211 runtime·atomicstorep(&runtime·allp[i], p); | 2289 runtime·atomicstorep(&runtime·allp[i], p); |
2212 } | 2290 } |
2213 if(p->mcache == nil) { | 2291 if(p->mcache == nil) { |
2214 if(old==0 && i==0) | 2292 if(old==0 && i==0) |
2215 p->mcache = m->mcache; // bootstrap | 2293 p->mcache = m->mcache; // bootstrap |
2216 else | 2294 else |
2217 p->mcache = runtime·allocmcache(); | 2295 p->mcache = runtime·allocmcache(); |
2218 } | 2296 } |
2219 if(p->runq == nil) { | |
2220 p->runqsize = 128; | |
2221 p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*),
0, FlagNoInvokeGC); | |
2222 } | |
2223 } | 2297 } |
2224 | 2298 |
2225 // redistribute runnable G's evenly | 2299 // redistribute runnable G's evenly |
2226 » for(i = 0; i < old; i++) { | 2300 » // collect all runnable goroutines in global queue preserving FIFO order |
2227 » » p = runtime·allp[i]; | 2301 » // FIFO order is required to ensure fairness even during frequent GCs |
2228 » » while(gp = runqget(p)) | 2302 » // see http://golang.org/issue/7126 |
2229 » » » globrunqput(gp); | 2303 » empty = false; |
2230 » } | 2304 » while(!empty) { |
| 2305 » » empty = true; |
| 2306 » » for(i = 0; i < old; i++) { |
| 2307 » » » p = runtime·allp[i]; |
| 2308 » » » if(p->runqhead == p->runqtail) |
| 2309 » » » » continue; |
| 2310 » » » empty = false; |
| 2311 » » » // pop from tail of local queue |
| 2312 » » » p->runqtail--; |
| 2313 » » » gp = p->runq[p->runqtail%nelem(p->runq)]; |
| 2314 » » » // push onto head of global queue |
| 2315 » » » gp->schedlink = runtime·sched.runqhead; |
| 2316 » » » runtime·sched.runqhead = gp; |
| 2317 » » » if(runtime·sched.runqtail == nil) |
| 2318 » » » » runtime·sched.runqtail = gp; |
| 2319 » » » runtime·sched.runqsize++; |
| 2320 » » } |
| 2321 » } |
| 2322 » // fill local queues with at most nelem(p->runq)/2 goroutines |
2231 // start at 1 because current M already executes some G and will acquire
allp[0] below, | 2323 // start at 1 because current M already executes some G and will acquire
allp[0] below, |
2232 // so if we have a spare G we want to put it into allp[1]. | 2324 // so if we have a spare G we want to put it into allp[1]. |
2233 » for(i = 1; runtime·sched.runqhead; i++) { | 2325 » for(i = 1; i < new * nelem(p->runq)/2 && runtime·sched.runqsize > 0; i++
) { |
2234 gp = runtime·sched.runqhead; | 2326 gp = runtime·sched.runqhead; |
2235 runtime·sched.runqhead = gp->schedlink; | 2327 runtime·sched.runqhead = gp->schedlink; |
| 2328 if(runtime·sched.runqhead == nil) |
| 2329 runtime·sched.runqtail = nil; |
| 2330 runtime·sched.runqsize--; |
2236 runqput(runtime·allp[i%new], gp); | 2331 runqput(runtime·allp[i%new], gp); |
2237 } | 2332 } |
2238 runtime·sched.runqtail = nil; | |
2239 runtime·sched.runqsize = 0; | |
2240 | 2333 |
2241 // free unused P's | 2334 // free unused P's |
2242 for(i = new; i < old; i++) { | 2335 for(i = new; i < old; i++) { |
2243 p = runtime·allp[i]; | 2336 p = runtime·allp[i]; |
2244 runtime·freemcache(p->mcache); | 2337 runtime·freemcache(p->mcache); |
2245 p->mcache = nil; | 2338 p->mcache = nil; |
2246 gfpurge(p); | 2339 gfpurge(p); |
2247 p->status = Pdead; | 2340 p->status = Pdead; |
2248 // can't free P itself because it can be referenced by an M in s
yscall | 2341 // can't free P itself because it can be referenced by an M in s
yscall |
2249 } | 2342 } |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2311 runtime·unlock(&runtime·sched); | 2404 runtime·unlock(&runtime·sched); |
2312 } | 2405 } |
2313 | 2406 |
2314 // Check for deadlock situation. | 2407 // Check for deadlock situation. |
2315 // The check is based on number of running M's, if 0 -> deadlock. | 2408 // The check is based on number of running M's, if 0 -> deadlock. |
2316 static void | 2409 static void |
2317 checkdead(void) | 2410 checkdead(void) |
2318 { | 2411 { |
2319 G *gp; | 2412 G *gp; |
2320 int32 run, grunning, s; | 2413 int32 run, grunning, s; |
| 2414 uintptr i; |
2321 | 2415 |
2322 // -1 for sysmon | 2416 // -1 for sysmon |
2323 run = runtime·sched.mcount - runtime·sched.nmidle - runtime·sched.nmidle
locked - 1; | 2417 run = runtime·sched.mcount - runtime·sched.nmidle - runtime·sched.nmidle
locked - 1; |
2324 if(run > 0) | 2418 if(run > 0) |
2325 return; | 2419 return; |
2326 if(run < 0) { | 2420 if(run < 0) { |
2327 runtime·printf("checkdead: nmidle=%d nmidlelocked=%d mcount=%d\n
", | 2421 runtime·printf("checkdead: nmidle=%d nmidlelocked=%d mcount=%d\n
", |
2328 runtime·sched.nmidle, runtime·sched.nmidlelocked, runtim
e·sched.mcount); | 2422 runtime·sched.nmidle, runtime·sched.nmidlelocked, runtim
e·sched.mcount); |
2329 runtime·throw("checkdead: inconsistent counts"); | 2423 runtime·throw("checkdead: inconsistent counts"); |
2330 } | 2424 } |
2331 grunning = 0; | 2425 grunning = 0; |
2332 » for(gp = runtime·allg; gp; gp = gp->alllink) { | 2426 » runtime·lock(&allglock); |
| 2427 » for(i = 0; i < runtime·allglen; i++) { |
| 2428 » » gp = runtime·allg[i]; |
2333 if(gp->isbackground) | 2429 if(gp->isbackground) |
2334 continue; | 2430 continue; |
2335 s = gp->status; | 2431 s = gp->status; |
2336 if(s == Gwaiting) | 2432 if(s == Gwaiting) |
2337 grunning++; | 2433 grunning++; |
2338 else if(s == Grunnable || s == Grunning || s == Gsyscall) { | 2434 else if(s == Grunnable || s == Grunning || s == Gsyscall) { |
| 2435 runtime·unlock(&allglock); |
2339 runtime·printf("checkdead: find g %D in status %d\n", gp
->goid, s); | 2436 runtime·printf("checkdead: find g %D in status %d\n", gp
->goid, s); |
2340 runtime·throw("checkdead: runnable g"); | 2437 runtime·throw("checkdead: runnable g"); |
2341 } | 2438 } |
2342 } | 2439 } |
| 2440 runtime·unlock(&allglock); |
2343 if(grunning == 0) // possible if main goroutine calls runtime·Goexit() | 2441 if(grunning == 0) // possible if main goroutine calls runtime·Goexit() |
2344 runtime·exit(0); | 2442 runtime·exit(0); |
2345 m->throwing = -1; // do not dump full stacks | 2443 m->throwing = -1; // do not dump full stacks |
2346 runtime·throw("all goroutines are asleep - deadlock!"); | 2444 runtime·throw("all goroutines are asleep - deadlock!"); |
2347 } | 2445 } |
2348 | 2446 |
2349 static void | 2447 static void |
2350 sysmon(void) | 2448 sysmon(void) |
2351 { | 2449 { |
2352 uint32 idle, delay; | 2450 uint32 idle, delay; |
(...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2518 gp->stackguard0 = StackPreempt; | 2616 gp->stackguard0 = StackPreempt; |
2519 return true; | 2617 return true; |
2520 } | 2618 } |
2521 | 2619 |
2522 void | 2620 void |
2523 runtime·schedtrace(bool detailed) | 2621 runtime·schedtrace(bool detailed) |
2524 { | 2622 { |
2525 static int64 starttime; | 2623 static int64 starttime; |
2526 int64 now; | 2624 int64 now; |
2527 int64 id1, id2, id3; | 2625 int64 id1, id2, id3; |
2528 » int32 i, q, t, h, s; | 2626 » int32 i, t, h; |
| 2627 » uintptr gi; |
2529 int8 *fmt; | 2628 int8 *fmt; |
2530 M *mp, *lockedm; | 2629 M *mp, *lockedm; |
2531 G *gp, *lockedg; | 2630 G *gp, *lockedg; |
2532 P *p; | 2631 P *p; |
2533 | 2632 |
2534 now = runtime·nanotime(); | 2633 now = runtime·nanotime(); |
2535 if(starttime == 0) | 2634 if(starttime == 0) |
2536 starttime = now; | 2635 starttime = now; |
2537 | 2636 |
2538 runtime·lock(&runtime·sched); | 2637 runtime·lock(&runtime·sched); |
2539 runtime·printf("SCHED %Dms: gomaxprocs=%d idleprocs=%d threads=%d idleth
reads=%d runqueue=%d", | 2638 runtime·printf("SCHED %Dms: gomaxprocs=%d idleprocs=%d threads=%d idleth
reads=%d runqueue=%d", |
2540 (now-starttime)/1000000, runtime·gomaxprocs, runtime·sched.npidl
e, runtime·sched.mcount, | 2639 (now-starttime)/1000000, runtime·gomaxprocs, runtime·sched.npidl
e, runtime·sched.mcount, |
2541 runtime·sched.nmidle, runtime·sched.runqsize); | 2640 runtime·sched.nmidle, runtime·sched.runqsize); |
2542 if(detailed) { | 2641 if(detailed) { |
2543 runtime·printf(" gcwaiting=%d nmidlelocked=%d nmspinning=%d stop
wait=%d sysmonwait=%d\n", | 2642 runtime·printf(" gcwaiting=%d nmidlelocked=%d nmspinning=%d stop
wait=%d sysmonwait=%d\n", |
2544 runtime·sched.gcwaiting, runtime·sched.nmidlelocked, run
time·sched.nmspinning, | 2643 runtime·sched.gcwaiting, runtime·sched.nmidlelocked, run
time·sched.nmspinning, |
2545 runtime·sched.stopwait, runtime·sched.sysmonwait); | 2644 runtime·sched.stopwait, runtime·sched.sysmonwait); |
2546 } | 2645 } |
2547 // We must be careful while reading data from P's, M's and G's. | 2646 // We must be careful while reading data from P's, M's and G's. |
2548 // Even if we hold schedlock, most data can be changed concurrently. | 2647 // Even if we hold schedlock, most data can be changed concurrently. |
2549 // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to
nil. | 2648 // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to
nil. |
2550 for(i = 0; i < runtime·gomaxprocs; i++) { | 2649 for(i = 0; i < runtime·gomaxprocs; i++) { |
2551 p = runtime·allp[i]; | 2650 p = runtime·allp[i]; |
2552 if(p == nil) | 2651 if(p == nil) |
2553 continue; | 2652 continue; |
2554 mp = p->m; | 2653 mp = p->m; |
2555 » » t = p->runqtail; | 2654 » » h = runtime·atomicload(&p->runqhead); |
2556 » » h = p->runqhead; | 2655 » » t = runtime·atomicload(&p->runqtail); |
2557 » » s = p->runqsize; | |
2558 » » q = t - h; | |
2559 » » if(q < 0) | |
2560 » » » q += s; | |
2561 if(detailed) | 2656 if(detailed) |
2562 » » » runtime·printf(" P%d: status=%d schedtick=%d syscalltic
k=%d m=%d runqsize=%d/%d gfreecnt=%d\n", | 2657 » » » runtime·printf(" P%d: status=%d schedtick=%d syscalltic
k=%d m=%d runqsize=%d gfreecnt=%d\n", |
2563 » » » » i, p->status, p->schedtick, p->syscalltick, mp ?
mp->id : -1, q, s, p->gfreecnt); | 2658 » » » » i, p->status, p->schedtick, p->syscalltick, mp ?
mp->id : -1, t-h, p->gfreecnt); |
2564 else { | 2659 else { |
2565 // In non-detailed mode format lengths of per-P run queu
es as: | 2660 // In non-detailed mode format lengths of per-P run queu
es as: |
2566 // [len1 len2 len3 len4] | 2661 // [len1 len2 len3 len4] |
2567 fmt = " %d"; | 2662 fmt = " %d"; |
2568 if(runtime·gomaxprocs == 1) | 2663 if(runtime·gomaxprocs == 1) |
2569 fmt = " [%d]\n"; | 2664 fmt = " [%d]\n"; |
2570 else if(i == 0) | 2665 else if(i == 0) |
2571 fmt = " [%d"; | 2666 fmt = " [%d"; |
2572 else if(i == runtime·gomaxprocs-1) | 2667 else if(i == runtime·gomaxprocs-1) |
2573 fmt = " %d]\n"; | 2668 fmt = " %d]\n"; |
2574 » » » runtime·printf(fmt, q); | 2669 » » » runtime·printf(fmt, t-h); |
2575 } | 2670 } |
2576 } | 2671 } |
2577 if(!detailed) { | 2672 if(!detailed) { |
2578 runtime·unlock(&runtime·sched); | 2673 runtime·unlock(&runtime·sched); |
2579 return; | 2674 return; |
2580 } | 2675 } |
2581 for(mp = runtime·allm; mp; mp = mp->alllink) { | 2676 for(mp = runtime·allm; mp; mp = mp->alllink) { |
2582 p = mp->p; | 2677 p = mp->p; |
2583 gp = mp->curg; | 2678 gp = mp->curg; |
2584 lockedg = mp->lockedg; | 2679 lockedg = mp->lockedg; |
2585 id1 = -1; | 2680 id1 = -1; |
2586 if(p) | 2681 if(p) |
2587 id1 = p->id; | 2682 id1 = p->id; |
2588 id2 = -1; | 2683 id2 = -1; |
2589 if(gp) | 2684 if(gp) |
2590 id2 = gp->goid; | 2685 id2 = gp->goid; |
2591 id3 = -1; | 2686 id3 = -1; |
2592 if(lockedg) | 2687 if(lockedg) |
2593 id3 = lockedg->goid; | 2688 id3 = lockedg->goid; |
2594 runtime·printf(" M%d: p=%D curg=%D mallocing=%d throwing=%d gci
ng=%d" | 2689 runtime·printf(" M%d: p=%D curg=%D mallocing=%d throwing=%d gci
ng=%d" |
2595 " locks=%d dying=%d helpgc=%d spinning=%d lockedg=%D\n", | 2690 " locks=%d dying=%d helpgc=%d spinning=%d lockedg=%D\n", |
2596 mp->id, id1, id2, | 2691 mp->id, id1, id2, |
2597 mp->mallocing, mp->throwing, mp->gcing, mp->locks, mp->d
ying, mp->helpgc, | 2692 mp->mallocing, mp->throwing, mp->gcing, mp->locks, mp->d
ying, mp->helpgc, |
2598 mp->spinning, id3); | 2693 mp->spinning, id3); |
2599 } | 2694 } |
2600 » for(gp = runtime·allg; gp; gp = gp->alllink) { | 2695 » runtime·lock(&allglock); |
| 2696 » for(gi = 0; gi < runtime·allglen; gi++) { |
| 2697 » » gp = runtime·allg[gi]; |
2601 mp = gp->m; | 2698 mp = gp->m; |
2602 lockedm = gp->lockedm; | 2699 lockedm = gp->lockedm; |
2603 runtime·printf(" G%D: status=%d(%s) m=%d lockedm=%d\n", | 2700 runtime·printf(" G%D: status=%d(%s) m=%d lockedm=%d\n", |
2604 gp->goid, gp->status, gp->waitreason, mp ? mp->id : -1, | 2701 gp->goid, gp->status, gp->waitreason, mp ? mp->id : -1, |
2605 lockedm ? lockedm->id : -1); | 2702 lockedm ? lockedm->id : -1); |
2606 } | 2703 } |
| 2704 runtime·unlock(&allglock); |
2607 runtime·unlock(&runtime·sched); | 2705 runtime·unlock(&runtime·sched); |
2608 } | 2706 } |
2609 | 2707 |
2610 // Put mp on midle list. | 2708 // Put mp on midle list. |
2611 // Sched must be locked. | 2709 // Sched must be locked. |
2612 static void | 2710 static void |
2613 mput(M *mp) | 2711 mput(M *mp) |
2614 { | 2712 { |
2615 mp->schedlink = runtime·sched.midle; | 2713 mp->schedlink = runtime·sched.midle; |
2616 runtime·sched.midle = mp; | 2714 runtime·sched.midle = mp; |
(...skipping 22 matching lines...) Expand all Loading... |
2639 { | 2737 { |
2640 gp->schedlink = nil; | 2738 gp->schedlink = nil; |
2641 if(runtime·sched.runqtail) | 2739 if(runtime·sched.runqtail) |
2642 runtime·sched.runqtail->schedlink = gp; | 2740 runtime·sched.runqtail->schedlink = gp; |
2643 else | 2741 else |
2644 runtime·sched.runqhead = gp; | 2742 runtime·sched.runqhead = gp; |
2645 runtime·sched.runqtail = gp; | 2743 runtime·sched.runqtail = gp; |
2646 runtime·sched.runqsize++; | 2744 runtime·sched.runqsize++; |
2647 } | 2745 } |
2648 | 2746 |
| 2747 // Put a batch of runnable goroutines on the global runnable queue. |
| 2748 // Sched must be locked. |
| 2749 static void |
| 2750 globrunqputbatch(G *ghead, G *gtail, int32 n) |
| 2751 { |
| 2752 gtail->schedlink = nil; |
| 2753 if(runtime·sched.runqtail) |
| 2754 runtime·sched.runqtail->schedlink = ghead; |
| 2755 else |
| 2756 runtime·sched.runqhead = ghead; |
| 2757 runtime·sched.runqtail = gtail; |
| 2758 runtime·sched.runqsize += n; |
| 2759 } |
| 2760 |
2649 // Try get a batch of G's from the global runnable queue. | 2761 // Try get a batch of G's from the global runnable queue. |
2650 // Sched must be locked. | 2762 // Sched must be locked. |
2651 static G* | 2763 static G* |
2652 globrunqget(P *p, int32 max) | 2764 globrunqget(P *p, int32 max) |
2653 { | 2765 { |
2654 G *gp, *gp1; | 2766 G *gp, *gp1; |
2655 int32 n; | 2767 int32 n; |
2656 | 2768 |
2657 if(runtime·sched.runqsize == 0) | 2769 if(runtime·sched.runqsize == 0) |
2658 return nil; | 2770 return nil; |
2659 n = runtime·sched.runqsize/runtime·gomaxprocs+1; | 2771 n = runtime·sched.runqsize/runtime·gomaxprocs+1; |
2660 if(n > runtime·sched.runqsize) | 2772 if(n > runtime·sched.runqsize) |
2661 n = runtime·sched.runqsize; | 2773 n = runtime·sched.runqsize; |
2662 if(max > 0 && n > max) | 2774 if(max > 0 && n > max) |
2663 n = max; | 2775 n = max; |
| 2776 if(n > nelem(p->runq)/2) |
| 2777 n = nelem(p->runq)/2; |
2664 runtime·sched.runqsize -= n; | 2778 runtime·sched.runqsize -= n; |
2665 if(runtime·sched.runqsize == 0) | 2779 if(runtime·sched.runqsize == 0) |
2666 runtime·sched.runqtail = nil; | 2780 runtime·sched.runqtail = nil; |
2667 gp = runtime·sched.runqhead; | 2781 gp = runtime·sched.runqhead; |
2668 runtime·sched.runqhead = gp->schedlink; | 2782 runtime·sched.runqhead = gp->schedlink; |
2669 n--; | 2783 n--; |
2670 while(n--) { | 2784 while(n--) { |
2671 gp1 = runtime·sched.runqhead; | 2785 gp1 = runtime·sched.runqhead; |
2672 runtime·sched.runqhead = gp1->schedlink; | 2786 runtime·sched.runqhead = gp1->schedlink; |
2673 runqput(p, gp1); | 2787 runqput(p, gp1); |
(...skipping 19 matching lines...) Expand all Loading... |
2693 P *p; | 2807 P *p; |
2694 | 2808 |
2695 p = runtime·sched.pidle; | 2809 p = runtime·sched.pidle; |
2696 if(p) { | 2810 if(p) { |
2697 runtime·sched.pidle = p->link; | 2811 runtime·sched.pidle = p->link; |
2698 runtime·xadd(&runtime·sched.npidle, -1); // TODO: fast atomic | 2812 runtime·xadd(&runtime·sched.npidle, -1); // TODO: fast atomic |
2699 } | 2813 } |
2700 return p; | 2814 return p; |
2701 } | 2815 } |
2702 | 2816 |
2703 // Put g on local runnable queue. | 2817 // Try to put g on local runnable queue. |
2704 // TODO(dvyukov): consider using lock-free queue. | 2818 // If it's full, put onto global queue. |
| 2819 // Executed only by the owner P. |
2705 static void | 2820 static void |
2706 runqput(P *p, G *gp) | 2821 runqput(P *p, G *gp) |
2707 { | 2822 { |
2708 » int32 h, t, s; | 2823 » uint32 h, t; |
2709 | 2824 |
2710 » runtime·lock(p); | |
2711 retry: | 2825 retry: |
2712 » h = p->runqhead; | 2826 » h = runtime·atomicload(&p->runqhead); // load-acquire, synchronize with
consumers |
2713 t = p->runqtail; | 2827 t = p->runqtail; |
2714 » s = p->runqsize; | 2828 » if(t - h < nelem(p->runq)) { |
2715 » if(t == h-1 || (h == 0 && t == s-1)) { | 2829 » » p->runq[t%nelem(p->runq)] = gp; |
2716 » » runqgrow(p); | 2830 » » runtime·atomicstore(&p->runqtail, t+1); // store-release, makes
the item available for consumption |
2717 » » goto retry; | 2831 » » return; |
2718 » } | 2832 » } |
2719 » p->runq[t++] = gp; | 2833 » if(runqputslow(p, gp, h, t)) |
2720 » if(t == s) | 2834 » » return; |
2721 » » t = 0; | 2835 » // the queue is not full, now the put above must suceed |
2722 » p->runqtail = t; | 2836 » goto retry; |
2723 » runtime·unlock(p); | 2837 } |
| 2838 |
| 2839 // Put g and a batch of work from local runnable queue on global queue. |
| 2840 // Executed only by the owner P. |
| 2841 static bool |
| 2842 runqputslow(P *p, G *gp, uint32 h, uint32 t) |
| 2843 { |
| 2844 » G *batch[nelem(p->runq)/2+1]; |
| 2845 » uint32 n, i; |
| 2846 |
| 2847 » // First, grab a batch from local queue. |
| 2848 » n = t-h; |
| 2849 » n = n/2; |
| 2850 » if(n != nelem(p->runq)/2) |
| 2851 » » runtime·throw("runqputslow: queue is not full"); |
| 2852 » for(i=0; i<n; i++) |
| 2853 » » batch[i] = p->runq[(h+i)%nelem(p->runq)]; |
| 2854 » if(!runtime·cas(&p->runqhead, h, h+n)) // cas-release, commits consume |
| 2855 » » return false; |
| 2856 » batch[n] = gp; |
| 2857 » // Link the goroutines. |
| 2858 » for(i=0; i<n; i++) |
| 2859 » » batch[i]->schedlink = batch[i+1]; |
| 2860 » // Now put the batch on global queue. |
| 2861 » runtime·lock(&runtime·sched); |
| 2862 » globrunqputbatch(batch[0], batch[n], n+1); |
| 2863 » runtime·unlock(&runtime·sched); |
| 2864 » return true; |
2724 } | 2865 } |
2725 | 2866 |
2726 // Get g from local runnable queue. | 2867 // Get g from local runnable queue. |
| 2868 // Executed only by the owner P. |
2727 static G* | 2869 static G* |
2728 runqget(P *p) | 2870 runqget(P *p) |
2729 { | 2871 { |
2730 G *gp; | 2872 G *gp; |
2731 » int32 t, h, s; | 2873 » uint32 t, h; |
2732 | 2874 |
2733 » if(p->runqhead == p->runqtail) | 2875 » for(;;) { |
2734 » » return nil; | 2876 » » h = runtime·atomicload(&p->runqhead); // load-acquire, synchron
ize with other consumers |
2735 » runtime·lock(p); | 2877 » » t = p->runqtail; |
2736 » h = p->runqhead; | 2878 » » if(t == h) |
2737 » t = p->runqtail; | 2879 » » » return nil; |
2738 » s = p->runqsize; | 2880 » » gp = p->runq[h%nelem(p->runq)]; |
2739 » if(t == h) { | 2881 » » if(runtime·cas(&p->runqhead, h, h+1)) // cas-release, commits c
onsume |
2740 » » runtime·unlock(p); | 2882 » » » return gp; |
2741 » » return nil; | 2883 » } |
2742 » } | 2884 } |
2743 » gp = p->runq[h++]; | 2885 |
2744 » if(h == s) | 2886 // Grabs a batch of goroutines from local runnable queue. |
2745 » » h = 0; | 2887 // batch array must be of size nelem(p->runq)/2. Returns number of grabbed gorou
tines. |
2746 » p->runqhead = h; | 2888 // Can be executed by any P. |
2747 » runtime·unlock(p); | 2889 static uint32 |
2748 » return gp; | 2890 runqgrab(P *p, G **batch) |
2749 } | 2891 { |
2750 | 2892 » uint32 t, h, n, i; |
2751 // Grow local runnable queue. | 2893 |
2752 // TODO(dvyukov): consider using fixed-size array | 2894 » for(;;) { |
2753 // and transfer excess to the global list (local queue can grow way too big). | 2895 » » h = runtime·atomicload(&p->runqhead); // load-acquire, synchron
ize with other consumers |
2754 static void | 2896 » » t = runtime·atomicload(&p->runqtail); // load-acquire, synchron
ize with the producer |
2755 runqgrow(P *p) | 2897 » » n = t-h; |
2756 { | 2898 » » n = n - n/2; |
2757 » G **q; | 2899 » » if(n == 0) |
2758 » int32 s, t, h, t2; | 2900 » » » break; |
2759 | 2901 » » if(n > nelem(p->runq)/2) // read inconsistent h and t |
2760 » h = p->runqhead; | 2902 » » » continue; |
2761 » t = p->runqtail; | 2903 » » for(i=0; i<n; i++) |
2762 » s = p->runqsize; | 2904 » » » batch[i] = p->runq[(h+i)%nelem(p->runq)]; |
2763 » t2 = 0; | 2905 » » if(runtime·cas(&p->runqhead, h, h+n)) // cas-release, commits c
onsume |
2764 » q = runtime·malloc(2*s*sizeof(*q)); | 2906 » » » break; |
2765 » while(t != h) { | 2907 » } |
2766 » » q[t2++] = p->runq[h++]; | 2908 » return n; |
2767 » » if(h == s) | |
2768 » » » h = 0; | |
2769 » } | |
2770 » runtime·free(p->runq); | |
2771 » p->runq = q; | |
2772 » p->runqhead = 0; | |
2773 » p->runqtail = t2; | |
2774 » p->runqsize = 2*s; | |
2775 } | 2909 } |
2776 | 2910 |
2777 // Steal half of elements from local runnable queue of p2 | 2911 // Steal half of elements from local runnable queue of p2 |
2778 // and put onto local runnable queue of p. | 2912 // and put onto local runnable queue of p. |
2779 // Returns one of the stolen elements (or nil if failed). | 2913 // Returns one of the stolen elements (or nil if failed). |
2780 static G* | 2914 static G* |
2781 runqsteal(P *p, P *p2) | 2915 runqsteal(P *p, P *p2) |
2782 { | 2916 { |
2783 » G *gp, *gp1; | 2917 » G *gp; |
2784 » int32 t, h, s, t2, h2, s2, c, i; | 2918 » G *batch[nelem(p->runq)/2]; |
2785 | 2919 » uint32 t, h, n, i; |
2786 » if(p2->runqhead == p2->runqtail) | 2920 |
| 2921 » n = runqgrab(p2, batch); |
| 2922 » if(n == 0) |
2787 return nil; | 2923 return nil; |
2788 » // sort locks to prevent deadlocks | 2924 » n--; |
2789 » if(p < p2) | 2925 » gp = batch[n]; |
2790 » » runtime·lock(p); | 2926 » if(n == 0) |
2791 » runtime·lock(p2); | 2927 » » return gp; |
2792 » if(p2->runqhead == p2->runqtail) { | 2928 » h = runtime·atomicload(&p->runqhead); // load-acquire, synchronize with
consumers |
2793 » » runtime·unlock(p2); | |
2794 » » if(p < p2) | |
2795 » » » runtime·unlock(p); | |
2796 » » return nil; | |
2797 » } | |
2798 » if(p >= p2) | |
2799 » » runtime·lock(p); | |
2800 » // now we've locked both queues and know the victim is not empty | |
2801 » h = p->runqhead; | |
2802 t = p->runqtail; | 2929 t = p->runqtail; |
2803 » s = p->runqsize; | 2930 » if(t - h + n >= nelem(p->runq)) |
2804 » h2 = p2->runqhead; | 2931 » » runtime·throw("runqsteal: runq overflow"); |
2805 » t2 = p2->runqtail; | 2932 » for(i=0; i<n; i++, t++) |
2806 » s2 = p2->runqsize; | 2933 » » p->runq[t%nelem(p->runq)] = batch[i]; |
2807 » gp = p2->runq[h2++]; // return value | 2934 » runtime·atomicstore(&p->runqtail, t); // store-release, makes the item
available for consumption |
2808 » if(h2 == s2) | |
2809 » » h2 = 0; | |
2810 » // steal roughly half | |
2811 » if(t2 > h2) | |
2812 » » c = (t2 - h2) / 2; | |
2813 » else | |
2814 » » c = (s2 - h2 + t2) / 2; | |
2815 » // copy | |
2816 » for(i = 0; i != c; i++) { | |
2817 » » // the target queue is full? | |
2818 » » if(t == h-1 || (h == 0 && t == s-1)) | |
2819 » » » break; | |
2820 » » // the victim queue is empty? | |
2821 » » if(t2 == h2) | |
2822 » » » break; | |
2823 » » gp1 = p2->runq[h2++]; | |
2824 » » if(h2 == s2) | |
2825 » » » h2 = 0; | |
2826 » » p->runq[t++] = gp1; | |
2827 » » if(t == s) | |
2828 » » » t = 0; | |
2829 » } | |
2830 » p->runqtail = t; | |
2831 » p2->runqhead = h2; | |
2832 » runtime·unlock(p2); | |
2833 » runtime·unlock(p); | |
2834 return gp; | 2935 return gp; |
2835 } | 2936 } |
2836 | 2937 |
2837 void | 2938 void |
2838 runtime·testSchedLocalQueue(void) | 2939 runtime·testSchedLocalQueue(void) |
2839 { | 2940 { |
2840 P p; | 2941 P p; |
2841 » G gs[1000]; | 2942 » G gs[nelem(p.runq)]; |
2842 int32 i, j; | 2943 int32 i, j; |
2843 | 2944 |
2844 runtime·memclr((byte*)&p, sizeof(p)); | 2945 runtime·memclr((byte*)&p, sizeof(p)); |
2845 p.runqsize = 1; | |
2846 p.runqhead = 0; | |
2847 p.runqtail = 0; | |
2848 p.runq = runtime·malloc(p.runqsize*sizeof(*p.runq)); | |
2849 | 2946 |
2850 for(i = 0; i < nelem(gs); i++) { | 2947 for(i = 0; i < nelem(gs); i++) { |
2851 if(runqget(&p) != nil) | 2948 if(runqget(&p) != nil) |
2852 runtime·throw("runq is not empty initially"); | 2949 runtime·throw("runq is not empty initially"); |
2853 for(j = 0; j < i; j++) | 2950 for(j = 0; j < i; j++) |
2854 runqput(&p, &gs[i]); | 2951 runqput(&p, &gs[i]); |
2855 for(j = 0; j < i; j++) { | 2952 for(j = 0; j < i; j++) { |
2856 if(runqget(&p) != &gs[i]) { | 2953 if(runqget(&p) != &gs[i]) { |
2857 runtime·printf("bad element at iter %d/%d\n", i,
j); | 2954 runtime·printf("bad element at iter %d/%d\n", i,
j); |
2858 runtime·throw("bad element"); | 2955 runtime·throw("bad element"); |
2859 } | 2956 } |
2860 } | 2957 } |
2861 if(runqget(&p) != nil) | 2958 if(runqget(&p) != nil) |
2862 runtime·throw("runq is not empty afterwards"); | 2959 runtime·throw("runq is not empty afterwards"); |
2863 } | 2960 } |
2864 } | 2961 } |
2865 | 2962 |
2866 void | 2963 void |
2867 runtime·testSchedLocalQueueSteal(void) | 2964 runtime·testSchedLocalQueueSteal(void) |
2868 { | 2965 { |
2869 P p1, p2; | 2966 P p1, p2; |
2870 » G gs[1000], *gp; | 2967 » G gs[nelem(p1.runq)], *gp; |
2871 int32 i, j, s; | 2968 int32 i, j, s; |
2872 | 2969 |
2873 runtime·memclr((byte*)&p1, sizeof(p1)); | 2970 runtime·memclr((byte*)&p1, sizeof(p1)); |
2874 p1.runqsize = 1; | |
2875 p1.runqhead = 0; | |
2876 p1.runqtail = 0; | |
2877 p1.runq = runtime·malloc(p1.runqsize*sizeof(*p1.runq)); | |
2878 | |
2879 runtime·memclr((byte*)&p2, sizeof(p2)); | 2971 runtime·memclr((byte*)&p2, sizeof(p2)); |
2880 p2.runqsize = nelem(gs); | |
2881 p2.runqhead = 0; | |
2882 p2.runqtail = 0; | |
2883 p2.runq = runtime·malloc(p2.runqsize*sizeof(*p2.runq)); | |
2884 | 2972 |
2885 for(i = 0; i < nelem(gs); i++) { | 2973 for(i = 0; i < nelem(gs); i++) { |
2886 for(j = 0; j < i; j++) { | 2974 for(j = 0; j < i; j++) { |
2887 gs[j].sig = 0; | 2975 gs[j].sig = 0; |
2888 runqput(&p1, &gs[j]); | 2976 runqput(&p1, &gs[j]); |
2889 } | 2977 } |
2890 gp = runqsteal(&p2, &p1); | 2978 gp = runqsteal(&p2, &p1); |
2891 s = 0; | 2979 s = 0; |
2892 if(gp) { | 2980 if(gp) { |
2893 s++; | 2981 s++; |
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2971 p = mp->p->id; | 3059 p = mp->p->id; |
2972 FLUSH(&p); | 3060 FLUSH(&p); |
2973 } | 3061 } |
2974 | 3062 |
2975 // func runtime_procUnpin() | 3063 // func runtime_procUnpin() |
2976 void | 3064 void |
2977 sync·runtime_procUnpin(void) | 3065 sync·runtime_procUnpin(void) |
2978 { | 3066 { |
2979 m->locks--; | 3067 m->locks--; |
2980 } | 3068 } |
LEFT | RIGHT |