LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "malloc.h" | 7 #include "malloc.h" |
8 #include "stack.h" | 8 #include "stack.h" |
9 #include "race.h" | 9 #include "race.h" |
10 #include "type.h" | 10 #include "type.h" |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
80 static void runqgrow(P*); | 80 static void runqgrow(P*); |
81 static G* runqsteal(P*, P*); | 81 static G* runqsteal(P*, P*); |
82 static void mput(M*); | 82 static void mput(M*); |
83 static M* mget(void); | 83 static M* mget(void); |
84 static void mcommoninit(M*); | 84 static void mcommoninit(M*); |
85 static void schedule(void); | 85 static void schedule(void); |
86 static void procresize(int32); | 86 static void procresize(int32); |
87 static void acquirep(P*); | 87 static void acquirep(P*); |
88 static P* releasep(void); | 88 static P* releasep(void); |
89 static void newm(void(*)(void), P*); | 89 static void newm(void(*)(void), P*); |
90 static void goidle(void); | |
91 static void stopm(void); | 90 static void stopm(void); |
92 static void startm(P*, bool); | 91 static void startm(P*, bool); |
93 static void handoffp(P*); | 92 static void handoffp(P*); |
94 static void wakep(void); | 93 static void wakep(void); |
95 static void stoplockedm(void); | 94 static void stoplockedm(void); |
96 static void startlockedm(G*); | 95 static void startlockedm(G*); |
97 static void sysmon(void); | 96 static void sysmon(void); |
98 static uint32 retake(uint32*); | 97 static uint32 retake(uint32*); |
99 static void inclocked(int32); | 98 static void inclocked(int32); |
100 static void checkdead(void); | 99 static void checkdead(void); |
101 static void exitsyscall0(G*); | 100 static void exitsyscall0(G*); |
102 static void park0(G*); | 101 static void park0(G*); |
103 static void goexit0(G*); | 102 static void goexit0(G*); |
104 static void gfput(P*, G*); | 103 static void gfput(P*, G*); |
105 static G* gfget(P*); | 104 static G* gfget(P*); |
106 static void gfpurge(P*); | 105 static void gfpurge(P*); |
107 static void globrunqput(G*); | 106 static void globrunqput(G*); |
108 static G* globrunqget(P*); | 107 static G* globrunqget(P*, int32); |
109 static P* pidleget(void); | 108 static P* pidleget(void); |
110 static void pidleput(P*); | 109 static void pidleput(P*); |
111 static void injectglist(G*); | 110 static void injectglist(G*); |
112 static void preemptall(void); | 111 static void preemptall(void); |
113 static void preemptone(P*); | 112 static void preemptone(P*); |
114 | 113 |
115 // The bootstrap sequence is: | 114 // The bootstrap sequence is: |
116 // | 115 // |
117 // call osinit | 116 // call osinit |
118 // call schedinit | 117 // call schedinit |
(...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
199 runtime·goroutineheader(G *gp) | 198 runtime·goroutineheader(G *gp) |
200 { | 199 { |
201 int8 *status; | 200 int8 *status; |
202 | 201 |
203 switch(gp->status) { | 202 switch(gp->status) { |
204 case Gidle: | 203 case Gidle: |
205 status = "idle"; | 204 status = "idle"; |
206 break; | 205 break; |
207 case Grunnable: | 206 case Grunnable: |
208 status = "runnable"; | 207 status = "runnable"; |
209 if(gp->preempted) | |
210 status = "runnable (preempted)"; | |
211 break; | 208 break; |
212 case Grunning: | 209 case Grunning: |
213 status = "running"; | 210 status = "running"; |
214 break; | 211 break; |
215 case Gsyscall: | 212 case Gsyscall: |
216 status = "syscall"; | 213 status = "syscall"; |
217 break; | 214 break; |
218 case Gwaiting: | 215 case Gwaiting: |
219 if(gp->waitreason) | 216 if(gp->waitreason) |
220 status = gp->waitreason; | 217 status = gp->waitreason; |
221 else | 218 else |
222 status = "waiting"; | 219 status = "waiting"; |
223 break; | 220 break; |
224 default: | 221 default: |
225 status = "???"; | 222 status = "???"; |
226 break; | 223 break; |
227 } | 224 } |
228 runtime·printf("goroutine %D [%s]:\n", gp->goid, status); | 225 runtime·printf("goroutine %D [%s]:\n", gp->goid, status); |
229 } | 226 } |
230 | 227 |
231 void | 228 void |
232 runtime·tracebackothers(G *me) | 229 runtime·tracebackothers(G *me) |
233 { | 230 { |
234 G *gp; | 231 G *gp; |
235 int32 traceback; | 232 int32 traceback; |
236 | 233 |
237 traceback = runtime·gotraceback(nil); | 234 traceback = runtime·gotraceback(nil); |
| 235 ········ |
| 236 // Show the current goroutine first, if we haven't already. |
| 237 if((gp = m->curg) != nil && gp != me) { |
| 238 runtime·printf("\n"); |
| 239 runtime·goroutineheader(gp); |
| 240 runtime·traceback(gp->sched.pc, gp->sched.sp, gp->sched.lr, gp); |
| 241 } |
| 242 |
238 for(gp = runtime·allg; gp != nil; gp = gp->alllink) { | 243 for(gp = runtime·allg; gp != nil; gp = gp->alllink) { |
239 » » if(gp == me || gp->status == Gdead) | 244 » » if(gp == me || gp == m->curg || gp->status == Gdead) |
240 continue; | 245 continue; |
241 if(gp->issystem && traceback < 2) | 246 if(gp->issystem && traceback < 2) |
242 continue; | 247 continue; |
243 runtime·printf("\n"); | 248 runtime·printf("\n"); |
244 runtime·goroutineheader(gp); | 249 runtime·goroutineheader(gp); |
245 » » runtime·traceback(gp->sched.pc, gp->sched.sp, 0, gp); | 250 » » if(gp->status == Grunning) |
| 251 » » » runtime·printf("\tgoroutine running on other thread; sta
ck unavailable\n"); |
| 252 » » else |
| 253 » » » runtime·traceback(gp->sched.pc, gp->sched.sp, gp->sched.
lr, gp); |
246 } | 254 } |
247 } | 255 } |
248 | 256 |
249 static void | 257 static void |
250 mcommoninit(M *mp) | 258 mcommoninit(M *mp) |
251 { | 259 { |
252 // If there is no mcache runtime·callers() will crash, | 260 // If there is no mcache runtime·callers() will crash, |
253 // and we are most likely in sysmon thread so the stack is senseless any
way. | 261 // and we are most likely in sysmon thread so the stack is senseless any
way. |
254 if(m->mcache) | 262 if(m->mcache) |
255 runtime·callers(1, mp->createstack, nelem(mp->createstack)); | 263 runtime·callers(1, mp->createstack, nelem(mp->createstack)); |
(...skipping 111 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
367 runtime·sched.stopwait--; | 375 runtime·sched.stopwait--; |
368 } | 376 } |
369 // stop idle P's | 377 // stop idle P's |
370 while(p = pidleget()) { | 378 while(p = pidleget()) { |
371 p->status = Pgcstop; | 379 p->status = Pgcstop; |
372 runtime·sched.stopwait--; | 380 runtime·sched.stopwait--; |
373 } | 381 } |
374 wait = runtime·sched.stopwait > 0; | 382 wait = runtime·sched.stopwait > 0; |
375 runtime·unlock(&runtime·sched); | 383 runtime·unlock(&runtime·sched); |
376 | 384 |
377 » // wait for remaining P's to stop voluntary | 385 » // wait for remaining P's to stop voluntarily |
378 if(wait) { | 386 if(wait) { |
379 for(;;) { | 387 for(;;) { |
380 // wait for 100us, then try to re-preempt in case of any
races | 388 // wait for 100us, then try to re-preempt in case of any
races |
381 if(runtime·notetsleep(&runtime·sched.stopnote, 100*1000)
) { | 389 if(runtime·notetsleep(&runtime·sched.stopnote, 100*1000)
) { |
382 runtime·noteclear(&runtime·sched.stopnote); | 390 runtime·noteclear(&runtime·sched.stopnote); |
383 break; | 391 break; |
384 } | 392 } |
385 preemptall(); | 393 preemptall(); |
386 } | 394 } |
387 } | 395 } |
(...skipping 269 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
657 | 665 |
658 // Create extra goroutine locked to extra m. | 666 // Create extra goroutine locked to extra m. |
659 // The goroutine is the context in which the cgo callback will run. | 667 // The goroutine is the context in which the cgo callback will run. |
660 // The sched.pc will never be returned to, but setting it to | 668 // The sched.pc will never be returned to, but setting it to |
661 // runtime.goexit makes clear to the traceback routines where | 669 // runtime.goexit makes clear to the traceback routines where |
662 // the goroutine stack ends. | 670 // the goroutine stack ends. |
663 mp = runtime·allocm(nil); | 671 mp = runtime·allocm(nil); |
664 gp = runtime·malg(4096); | 672 gp = runtime·malg(4096); |
665 gp->sched.pc = (uintptr)runtime·goexit; | 673 gp->sched.pc = (uintptr)runtime·goexit; |
666 gp->sched.sp = gp->stackbase; | 674 gp->sched.sp = gp->stackbase; |
| 675 gp->sched.lr = 0; |
667 gp->sched.g = gp; | 676 gp->sched.g = gp; |
668 gp->status = Gsyscall; | 677 gp->status = Gsyscall; |
669 mp->curg = gp; | 678 mp->curg = gp; |
670 mp->locked = LockInternal; | 679 mp->locked = LockInternal; |
671 mp->lockedg = gp; | 680 mp->lockedg = gp; |
672 gp->lockedm = mp; | 681 gp->lockedm = mp; |
673 // put on allg for garbage collector | 682 // put on allg for garbage collector |
674 runtime·lock(&runtime·sched); | 683 runtime·lock(&runtime·sched); |
675 if(runtime·lastg == nil) | 684 if(runtime·lastg == nil) |
676 runtime·allg = gp; | 685 runtime·allg = gp; |
(...skipping 314 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
991 static void | 1000 static void |
992 execute(G *gp) | 1001 execute(G *gp) |
993 { | 1002 { |
994 int32 hz; | 1003 int32 hz; |
995 | 1004 |
996 if(gp->status != Grunnable) { | 1005 if(gp->status != Grunnable) { |
997 runtime·printf("execute: bad g status %d\n", gp->status); | 1006 runtime·printf("execute: bad g status %d\n", gp->status); |
998 runtime·throw("execute: bad g status"); | 1007 runtime·throw("execute: bad g status"); |
999 } | 1008 } |
1000 gp->status = Grunning; | 1009 gp->status = Grunning; |
1001 gp->preempted = 0; | |
1002 gp->stackguard0 = gp->stackguard; | 1010 gp->stackguard0 = gp->stackguard; |
1003 m->p->tick++; | 1011 m->p->tick++; |
1004 m->curg = gp; | 1012 m->curg = gp; |
1005 gp->m = m; | 1013 gp->m = m; |
1006 | 1014 |
1007 // Check whether the profiler needs to be turned on or off. | 1015 // Check whether the profiler needs to be turned on or off. |
1008 hz = runtime·sched.profilehz; | 1016 hz = runtime·sched.profilehz; |
1009 if(m->profilehz != hz) | 1017 if(m->profilehz != hz) |
1010 runtime·resetcpuprofiler(hz); | 1018 runtime·resetcpuprofiler(hz); |
1011 | 1019 |
(...skipping 14 matching lines...) Expand all Loading... |
1026 gcstopm(); | 1034 gcstopm(); |
1027 goto top; | 1035 goto top; |
1028 } | 1036 } |
1029 // local runq | 1037 // local runq |
1030 gp = runqget(m->p); | 1038 gp = runqget(m->p); |
1031 if(gp) | 1039 if(gp) |
1032 return gp; | 1040 return gp; |
1033 // global runq | 1041 // global runq |
1034 if(runtime·sched.runqsize) { | 1042 if(runtime·sched.runqsize) { |
1035 runtime·lock(&runtime·sched); | 1043 runtime·lock(&runtime·sched); |
1036 » » gp = globrunqget(m->p); | 1044 » » gp = globrunqget(m->p, 0); |
1037 runtime·unlock(&runtime·sched); | 1045 runtime·unlock(&runtime·sched); |
1038 if(gp) | 1046 if(gp) |
1039 return gp; | 1047 return gp; |
1040 } | 1048 } |
1041 // poll network | 1049 // poll network |
1042 gp = runtime·netpoll(false); // non-blocking | 1050 gp = runtime·netpoll(false); // non-blocking |
1043 if(gp) { | 1051 if(gp) { |
1044 injectglist(gp->schedlink); | 1052 injectglist(gp->schedlink); |
1045 gp->status = Grunnable; | 1053 gp->status = Grunnable; |
1046 return gp; | 1054 return gp; |
(...skipping 20 matching lines...) Expand all Loading... |
1067 return gp; | 1075 return gp; |
1068 } | 1076 } |
1069 stop: | 1077 stop: |
1070 // return P and block | 1078 // return P and block |
1071 runtime·lock(&runtime·sched); | 1079 runtime·lock(&runtime·sched); |
1072 if(runtime·gcwaiting) { | 1080 if(runtime·gcwaiting) { |
1073 runtime·unlock(&runtime·sched); | 1081 runtime·unlock(&runtime·sched); |
1074 goto top; | 1082 goto top; |
1075 } | 1083 } |
1076 if(runtime·sched.runqsize) { | 1084 if(runtime·sched.runqsize) { |
1077 » » gp = globrunqget(m->p); | 1085 » » gp = globrunqget(m->p, 0); |
1078 runtime·unlock(&runtime·sched); | 1086 runtime·unlock(&runtime·sched); |
1079 return gp; | 1087 return gp; |
1080 } | 1088 } |
1081 p = releasep(); | 1089 p = releasep(); |
1082 pidleput(p); | 1090 pidleput(p); |
1083 runtime·unlock(&runtime·sched); | 1091 runtime·unlock(&runtime·sched); |
1084 if(m->spinning) { | 1092 if(m->spinning) { |
1085 m->spinning = false; | 1093 m->spinning = false; |
1086 runtime·xadd(&runtime·sched.nmspinning, -1); | 1094 runtime·xadd(&runtime·sched.nmspinning, -1); |
1087 } | 1095 } |
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1146 for(; n && runtime·sched.npidle; n--) | 1154 for(; n && runtime·sched.npidle; n--) |
1147 startm(nil, false); | 1155 startm(nil, false); |
1148 } | 1156 } |
1149 | 1157 |
1150 // One round of scheduler: find a runnable goroutine and execute it. | 1158 // One round of scheduler: find a runnable goroutine and execute it. |
1151 // Never returns. | 1159 // Never returns. |
1152 static void | 1160 static void |
1153 schedule(void) | 1161 schedule(void) |
1154 { | 1162 { |
1155 G *gp; | 1163 G *gp; |
| 1164 uint32 tick; |
1156 | 1165 |
1157 if(m->locks) | 1166 if(m->locks) |
1158 runtime·throw("schedule: holding locks"); | 1167 runtime·throw("schedule: holding locks"); |
1159 | 1168 |
1160 top: | 1169 top: |
1161 if(runtime·gcwaiting) { | 1170 if(runtime·gcwaiting) { |
1162 gcstopm(); | 1171 gcstopm(); |
1163 goto top; | 1172 goto top; |
1164 } | 1173 } |
1165 | 1174 |
1166 » gp = runqget(m->p); | 1175 » gp = nil; |
1167 » if(gp && m->spinning) | 1176 » // Check the global runnable queue once in a while to ensure fairness. |
1168 » » runtime·throw("schedule: spinning with local work"); | 1177 » // Otherwise two goroutines can completely occupy the local runqueue |
| 1178 » // by constantly respawning each other. |
| 1179 » tick = m->p->tick; |
| 1180 » // This is a fancy way to say tick%61==0, |
| 1181 » // it uses 2 MUL instructions instead of a single DIV and so is faster o
n modern processors. |
| 1182 » if(tick - (((uint64)tick*0x4325c53fu)>>36)*61 == 0 && runtime·sched.runq
size > 0) { |
| 1183 » » runtime·lock(&runtime·sched); |
| 1184 » » gp = globrunqget(m->p, 1); |
| 1185 » » runtime·unlock(&runtime·sched); |
| 1186 » } |
| 1187 » if(gp == nil) { |
| 1188 » » gp = runqget(m->p); |
| 1189 » » if(gp && m->spinning) |
| 1190 » » » runtime·throw("schedule: spinning with local work"); |
| 1191 » } |
1169 if(gp == nil) | 1192 if(gp == nil) |
1170 gp = findrunnable(); | 1193 gp = findrunnable(); |
1171 | 1194 |
1172 if(m->spinning) { | 1195 if(m->spinning) { |
1173 m->spinning = false; | 1196 m->spinning = false; |
1174 runtime·xadd(&runtime·sched.nmspinning, -1); | 1197 runtime·xadd(&runtime·sched.nmspinning, -1); |
1175 } | 1198 } |
1176 | 1199 |
1177 // M wakeup policy is deliberately somewhat conservative (see nmspinning
handling), | 1200 // M wakeup policy is deliberately somewhat conservative (see nmspinning
handling), |
1178 // so see if we need to wakeup another M here. | 1201 // so see if we need to wakeup another M here. |
(...skipping 638 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1817 if(!Windows && (m == nil || m->mcache == nil)) | 1840 if(!Windows && (m == nil || m->mcache == nil)) |
1818 return; | 1841 return; |
1819 if(prof.fn == nil || prof.hz == 0) | 1842 if(prof.fn == nil || prof.hz == 0) |
1820 return; | 1843 return; |
1821 | 1844 |
1822 runtime·lock(&prof); | 1845 runtime·lock(&prof); |
1823 if(prof.fn == nil) { | 1846 if(prof.fn == nil) { |
1824 runtime·unlock(&prof); | 1847 runtime·unlock(&prof); |
1825 return; | 1848 return; |
1826 } | 1849 } |
1827 » n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, p
rof.pcbuf, nelem(prof.pcbuf), nil, nil); | 1850 » n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, p
rof.pcbuf, nelem(prof.pcbuf), nil, nil, false); |
1828 if(n > 0) | 1851 if(n > 0) |
1829 prof.fn(prof.pcbuf, n); | 1852 prof.fn(prof.pcbuf, n); |
1830 runtime·unlock(&prof); | 1853 runtime·unlock(&prof); |
1831 } | 1854 } |
1832 | 1855 |
1833 // Arrange to call fn with a traceback hz times a second. | 1856 // Arrange to call fn with a traceback hz times a second. |
1834 void | 1857 void |
1835 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) | 1858 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) |
1836 { | 1859 { |
1837 // Force sane arguments. | 1860 // Force sane arguments. |
(...skipping 331 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2169 runtime·sched.runqtail->schedlink = gp; | 2192 runtime·sched.runqtail->schedlink = gp; |
2170 else | 2193 else |
2171 runtime·sched.runqhead = gp; | 2194 runtime·sched.runqhead = gp; |
2172 runtime·sched.runqtail = gp; | 2195 runtime·sched.runqtail = gp; |
2173 runtime·sched.runqsize++; | 2196 runtime·sched.runqsize++; |
2174 } | 2197 } |
2175 | 2198 |
2176 // Try get a batch of G's from the global runnable queue. | 2199 // Try get a batch of G's from the global runnable queue. |
2177 // Sched must be locked. | 2200 // Sched must be locked. |
2178 static G* | 2201 static G* |
2179 globrunqget(P *p) | 2202 globrunqget(P *p, int32 max) |
2180 { | 2203 { |
2181 G *gp, *gp1; | 2204 G *gp, *gp1; |
2182 int32 n; | 2205 int32 n; |
2183 | 2206 |
2184 if(runtime·sched.runqsize == 0) | 2207 if(runtime·sched.runqsize == 0) |
2185 return nil; | 2208 return nil; |
2186 n = runtime·sched.runqsize/runtime·gomaxprocs+1; | 2209 n = runtime·sched.runqsize/runtime·gomaxprocs+1; |
2187 if(n > runtime·sched.runqsize) | 2210 if(n > runtime·sched.runqsize) |
2188 n = runtime·sched.runqsize; | 2211 n = runtime·sched.runqsize; |
| 2212 if(max > 0 && n > max) |
| 2213 n = max; |
2189 runtime·sched.runqsize -= n; | 2214 runtime·sched.runqsize -= n; |
2190 if(runtime·sched.runqsize == 0) | 2215 if(runtime·sched.runqsize == 0) |
2191 runtime·sched.runqtail = nil; | 2216 runtime·sched.runqtail = nil; |
2192 gp = runtime·sched.runqhead; | 2217 gp = runtime·sched.runqhead; |
2193 runtime·sched.runqhead = gp->schedlink; | 2218 runtime·sched.runqhead = gp->schedlink; |
2194 n--; | 2219 n--; |
2195 while(n--) { | 2220 while(n--) { |
2196 gp1 = runtime·sched.runqhead; | 2221 gp1 = runtime·sched.runqhead; |
2197 runtime·sched.runqhead = gp1->schedlink; | 2222 runtime·sched.runqhead = gp1->schedlink; |
2198 runqput(p, gp1); | 2223 runqput(p, gp1); |
(...skipping 232 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2431 } | 2456 } |
2432 } | 2457 } |
2433 if(s != i/2 && s != i/2+1) { | 2458 if(s != i/2 && s != i/2+1) { |
2434 runtime·printf("bad steal %d, want %d or %d, iter %d\n", | 2459 runtime·printf("bad steal %d, want %d or %d, iter %d\n", |
2435 s, i/2, i/2+1, i); | 2460 s, i/2, i/2+1, i); |
2436 runtime·throw("bad steal"); | 2461 runtime·throw("bad steal"); |
2437 } | 2462 } |
2438 } | 2463 } |
2439 } | 2464 } |
2440 | 2465 |
| 2466 extern void runtime·morestack(void); |
| 2467 |
2441 bool | 2468 bool |
2442 runtime·haszeroargs(uintptr pc) | 2469 runtime·haszeroargs(uintptr pc) |
2443 { | 2470 { |
2444 return pc == (uintptr)runtime·goexit || | 2471 return pc == (uintptr)runtime·goexit || |
2445 pc == (uintptr)runtime·mcall || | 2472 pc == (uintptr)runtime·mcall || |
2446 pc == (uintptr)runtime·mstart || | 2473 pc == (uintptr)runtime·mstart || |
| 2474 pc == (uintptr)runtime·lessstack || |
| 2475 pc == (uintptr)runtime·morestack || |
2447 pc == (uintptr)_rt0_go; | 2476 pc == (uintptr)_rt0_go; |
2448 } | 2477 } |
2449 | 2478 |
LEFT | RIGHT |