LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "malloc.h" | 7 #include "malloc.h" |
8 #include "stack.h" | 8 #include "stack.h" |
9 #include "race.h" | 9 #include "race.h" |
10 #include "type.h" | 10 #include "type.h" |
11 | 11 |
12 G* | 12 // Goroutine scheduler |
13 runtime·netwait(uint32 ms, uint32 cnt); | 13 // The scheduler's job is to distribute ready-to-run goroutines over worker thre
ads. |
14 | |
15 enum { maxgomaxprocs = 1<<10 }; | |
16 #define LOG if(0) runtime·printf | |
17 #define LOG1 runtime·printf | |
18 #define CHECK(cond, fmt) if(cond) {} else { runtime·printf fmt; runtime·throw("C
HECK"); } | |
19 | |
20 G*» runtime·allg; | |
21 G*» runtime·lastg; | |
22 M*» runtime·allm; | |
23 | |
24 int8*» runtime·goos; | |
25 int32» runtime·ncpu; | |
26 | |
27 // Go scheduler | |
28 // | 14 // |
29 // The go scheduler's job is to match ready-to-run goroutines (`g's) | 15 // The main concepts are: |
30 // with waiting-for-work schedulers (`m's). If there are ready g's | 16 // G - goroutine. |
31 // and no waiting m's, ready() will start a new m running in a new | 17 // M - worker thread, or machine. |
32 // OS thread, so that all ready g's can run simultaneously, up to a limit. | 18 // P - processor, a resource that is required to execute Go code. |
33 // For now, m's never go away. | 19 // M must have an associated P to execute Go code, however it can be |
34 // | 20 // blocked or in a syscall w/o an associated P. |
35 // By default, Go keeps only one kernel thread (m) running user code | |
36 // at a single time; other threads may be blocked in the operating system. | |
37 // Setting the environment variable $GOMAXPROCS or calling | |
38 // runtime.GOMAXPROCS() will change the number of user threads | |
39 // allowed to execute simultaneously. $GOMAXPROCS is thus an | |
40 // approximation of the maximum number of cores to use. | |
41 // | |
42 // Even a program that can run without deadlock in a single process | |
43 // might use more m's if given the chance. For example, the prime | |
44 // sieve will use as many m's as there are primes (up to $GOMAXPROCS), | |
45 // allowing different stages of the pipeline to execute in parallel. | |
46 | 21 |
47 typedef struct Sched Sched; | 22 typedef struct Sched Sched; |
48 struct Sched { | 23 struct Sched { |
49 Lock; | 24 Lock; |
50 | 25 |
51 » M*» mhead;» // m's waiting for work | 26 » uint64» goidgen; |
52 » int32» mwait;» // number of m's waiting for work | 27 |
53 » int32» mcount;»// number of m's that have been created | 28 » M*» midle;» // idle m's waiting for work |
| 29 » int32» nmidle;» // number of idle m's waiting for work |
| 30 » int32» mlocked; // number of locked m's waiting for work |
| 31 » int32» mcount;» // number of m's that have been created |
54 | 32 |
55 P* pidle; // idle P's | 33 P* pidle; // idle P's |
56 » int32» npidle; | 34 » uint32» npidle; |
| 35 » uint32» nmspinning; |
57 | 36 |
58 // Global runnable queue. | 37 // Global runnable queue. |
59 G* runqhead; | 38 G* runqhead; |
60 G* runqtail; | 39 G* runqtail; |
61 int32 runqsize; | 40 int32 runqsize; |
62 | 41 |
63 // Global cache of dead G's. | 42 // Global cache of dead G's. |
64 » Lock gflock; | 43 » Lock» gflock; |
65 G* gfree; | 44 G* gfree; |
66 » int32» goidseq; | 45 |
67 | |
68 » volatile uint32»poller; | |
69 int32 stopwait; | 46 int32 stopwait; |
70 Note stopnote; | 47 Note stopnote; |
71 » int32» sysmonwait; | 48 » bool» sysmonwait; |
72 Note sysmonnote; | 49 Note sysmonnote; |
73 | 50 |
74 int32 profilehz; // cpu profiling rate | 51 int32 profilehz; // cpu profiling rate |
75 bool init; // running initialization | |
76 bool lockmain; // init called runtime.LockOSThread | |
77 }; | 52 }; |
| 53 |
| 54 // The max value of GOMAXPROCS. |
| 55 // There are no fundamental restrictions on the value. |
| 56 enum { MaxGomaxprocs = 1<<8 }; |
78 | 57 |
79 Sched runtime·sched; | 58 Sched runtime·sched; |
80 int32 runtime·gomaxprocs; | 59 int32 runtime·gomaxprocs; |
81 bool runtime·singleproc; | 60 bool runtime·singleproc; |
82 bool runtime·iscgo; | 61 bool runtime·iscgo; |
83 int32 runtime·gcwaiting; | 62 int32 runtime·gcwaiting; |
84 M runtime·m0; | 63 M runtime·m0; |
85 G» runtime·g0;» // idle goroutine for m0 | 64 G» runtime·g0;» // idle goroutine for m0 |
| 65 G*» runtime·allg; |
| 66 G*» runtime·lastg; |
| 67 M*» runtime·allm; |
| 68 M*» runtime·extram; |
| 69 int8*» runtime·goos; |
| 70 int32» runtime·ncpu; |
86 static int32 newprocs; | 71 static int32 newprocs; |
87 | |
88 // Keep trace of scavenger's goroutine for deadlock detection. | 72 // Keep trace of scavenger's goroutine for deadlock detection. |
89 static G *scvg; | 73 static G *scvg; |
90 | 74 |
91 // Scheduling helpers. Sched must be locked. | 75 void runtime·mstart(void); |
92 static void runqput(P*, G*);» // put/get on ghead/gtail | 76 static void runqput(P*, G*); |
93 static G* runqget(P*); | 77 static G* runqget(P*); |
94 static void runqgrow(P*); | 78 static void runqgrow(P*); |
95 static G* runqsteal(P*, P*); | 79 static G* runqsteal(P*, P*); |
96 static void globrunqput(G*); | 80 static void mput(M*); |
97 static G* globrunqget(void); | 81 static M* mget(void); |
98 static void mput(M*);» // put/get on mhead | |
99 static M* mtryget(void); | |
100 static void gfput(P*, G*);» // put/get on gfree | |
101 static G* gfget(P*); | |
102 static void mcommoninit(M*); | 82 static void mcommoninit(M*); |
103 static void schedule(void); | 83 static void schedule(void); |
104 static void procresize(int32); | 84 static void procresize(int32); |
105 static void acquirep(M*, P*); | 85 static void acquirep(P*); |
106 static P* releasep(void); | 86 static P* releasep(void); |
107 static void newm(void(*)(void), P*, bool); | 87 static void newm(void(*)(void), P*, bool, bool); |
108 static void goidle(void); | 88 static void goidle(void); |
109 static void stopm(void); | 89 static void stopm(void); |
| 90 static void startm(P*, bool); |
| 91 static void handoffp(P*); |
| 92 static void wakep(void); |
| 93 static void stoplockedm(void); |
| 94 static void startlockedm(G*); |
110 static void sysmon(void); | 95 static void sysmon(void); |
111 void runtime·inject(G*, int32*, int32*); | 96 static uint32 retake(uint32*); |
| 97 static void inclocked(int32); |
| 98 static void checkdead(void); |
| 99 static void exitsyscall0(G*); |
| 100 static void park0(G*); |
| 101 static void gosched0(G*); |
| 102 static void goexit0(G*); |
| 103 static void gfput(P*, G*); |
| 104 static G* gfget(P*); |
| 105 static void gfpurge(P*); |
| 106 static void globrunqput(G*); |
| 107 static G* globrunqget(P*); |
112 static P* pidleget(void); | 108 static P* pidleget(void); |
113 static void pidleput(P*); | 109 static void pidleput(P*); |
114 static bool startm(P*); | |
115 | 110 |
116 // The bootstrap sequence is: | 111 // The bootstrap sequence is: |
117 // | 112 // |
118 // call osinit | 113 // call osinit |
119 // call schedinit | 114 // call schedinit |
120 // make & queue new G | 115 // make & queue new G |
121 // call runtime·mstart | 116 // call runtime·mstart |
122 // | 117 // |
123 // The new G calls runtime·main. | 118 // The new G calls runtime·main. |
124 void | 119 void |
125 runtime·schedinit(void) | 120 runtime·schedinit(void) |
126 { | 121 { |
127 int32 n, procs; | 122 int32 n, procs; |
128 byte *p; | 123 byte *p; |
129 | 124 |
130 LOG("%d: runtime·schedinit\n", m->id); | |
131 m->nomemprof++; | 125 m->nomemprof++; |
132 runtime·mprofinit(); | 126 runtime·mprofinit(); |
133 runtime·mallocinit(); | 127 runtime·mallocinit(); |
134 mcommoninit(m); | 128 mcommoninit(m); |
135 | 129 |
136 runtime·goargs(); | 130 runtime·goargs(); |
137 runtime·goenvs(); | 131 runtime·goenvs(); |
138 | 132 |
139 // For debugging: | 133 // For debugging: |
140 // Allocate internal symbol table representation now, | 134 // Allocate internal symbol table representation now, |
141 // so that we don't need to call malloc when we crash. | 135 // so that we don't need to call malloc when we crash. |
142 // runtime·findfunc(0); | 136 // runtime·findfunc(0); |
143 | 137 |
144 procs = 1; | 138 procs = 1; |
145 p = runtime·getenv("GOMAXPROCS"); | 139 p = runtime·getenv("GOMAXPROCS"); |
146 if(p != nil && (n = runtime·atoi(p)) > 0) { | 140 if(p != nil && (n = runtime·atoi(p)) > 0) { |
147 » » if(n > maxgomaxprocs) | 141 » » if(n > MaxGomaxprocs) |
148 » » » n = maxgomaxprocs; | 142 » » » n = MaxGomaxprocs; |
149 procs = n; | 143 procs = n; |
150 } | 144 } |
151 » runtime·allp = (P**)runtime·malloc((maxgomaxprocs+1)*sizeof(runtime·allp
[0])); | 145 » runtime·allp = runtime·malloc((MaxGomaxprocs+1)*sizeof(runtime·allp[0]))
; |
152 procresize(procs); | 146 procresize(procs); |
153 | 147 |
154 mstats.enablegc = 1; | 148 mstats.enablegc = 1; |
155 m->nomemprof--; | 149 m->nomemprof--; |
156 | 150 |
157 if(raceenabled) | 151 if(raceenabled) |
158 g->racectx = runtime·raceinit(); | 152 g->racectx = runtime·raceinit(); |
159 } | 153 } |
160 | 154 |
161 extern void main·init(void); | 155 extern void main·init(void); |
162 extern void main·main(void); | 156 extern void main·main(void); |
163 | 157 |
| 158 static FuncVal scavenger = {runtime·MHeap_Scavenger}; |
| 159 |
164 // The main goroutine. | 160 // The main goroutine. |
165 void | 161 void |
166 runtime·main(void) | 162 runtime·main(void) |
167 { | 163 { |
168 » LOG("%d: runtime·main\n", m->id); | 164 » newm(sysmon, nil, false, false); |
169 | |
170 » newm(sysmon, nil, false); | |
171 | 165 |
172 // Lock the main goroutine onto this, the main OS thread, | 166 // Lock the main goroutine onto this, the main OS thread, |
173 // during initialization. Most programs won't care, but a few | 167 // during initialization. Most programs won't care, but a few |
174 // do require certain calls to be made by the main thread. | 168 // do require certain calls to be made by the main thread. |
175 // Those can arrange for main.main to run in the main thread | 169 // Those can arrange for main.main to run in the main thread |
176 // by calling runtime.LockOSThread during initialization | 170 // by calling runtime.LockOSThread during initialization |
177 // to preserve the lock. | 171 // to preserve the lock. |
178 runtime·lockOSThread(); | 172 runtime·lockOSThread(); |
179 runtime·sched.init = true; | |
180 if(m != &runtime·m0) | 173 if(m != &runtime·m0) |
181 runtime·throw("runtime·main not on m0"); | 174 runtime·throw("runtime·main not on m0"); |
182 » scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runti
me·main); | 175 » scvg = runtime·newproc1(&scavenger, nil, 0, 0, runtime·main); |
183 scvg->issystem = true; | 176 scvg->issystem = true; |
184 main·init(); | 177 main·init(); |
185 runtime·sched.init = false; | |
186 runtime·unlockOSThread(); | 178 runtime·unlockOSThread(); |
187 | 179 |
188 main·main(); | 180 main·main(); |
189 if(raceenabled) | 181 if(raceenabled) |
190 runtime·racefini(); | 182 runtime·racefini(); |
| 183 |
| 184 // Make racy client program work: if panicking on |
| 185 // another goroutine at the same time as main returns, |
| 186 // let the other goroutine finish printing the panic trace. |
| 187 // Once it does, it will exit. See issue 3934. |
| 188 if(runtime·panicking) |
| 189 runtime·park(nil, nil, "panicwait"); |
| 190 |
191 runtime·exit(0); | 191 runtime·exit(0); |
192 for(;;) | 192 for(;;) |
193 *(int32*)runtime·main = 0; | 193 *(int32*)runtime·main = 0; |
194 } | 194 } |
195 | 195 |
196 void | 196 void |
197 runtime·goroutineheader(G *gp) | 197 runtime·goroutineheader(G *gp) |
198 { | 198 { |
199 int8 *status; | 199 int8 *status; |
200 | 200 |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
244 | 244 |
245 static void | 245 static void |
246 mcommoninit(M *mp) | 246 mcommoninit(M *mp) |
247 { | 247 { |
248 // If there is no mcache runtime·callers() will crash, | 248 // If there is no mcache runtime·callers() will crash, |
249 // and we are most likely in sysmon thread so the stack is senseless any
way. | 249 // and we are most likely in sysmon thread so the stack is senseless any
way. |
250 if(m->mcache) | 250 if(m->mcache) |
251 runtime·callers(1, mp->createstack, nelem(mp->createstack)); | 251 runtime·callers(1, mp->createstack, nelem(mp->createstack)); |
252 | 252 |
253 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); | 253 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); |
254 if(runtime·gsignalstk) | |
255 mp->gsignal = runtime·malg(runtime·gsignalstk);· | |
256 | 254 |
257 runtime·lock(&runtime·sched); | 255 runtime·lock(&runtime·sched); |
258 mp->id = runtime·sched.mcount++; | 256 mp->id = runtime·sched.mcount++; |
| 257 |
| 258 runtime·mpreinit(mp); |
259 | 259 |
260 // Add to runtime·allm so garbage collector doesn't free m | 260 // Add to runtime·allm so garbage collector doesn't free m |
261 // when it is just in a register or thread-local storage. | 261 // when it is just in a register or thread-local storage. |
262 mp->alllink = runtime·allm; | 262 mp->alllink = runtime·allm; |
263 » // runtime·NumCgoCall() iterates over allm w/o locks, | 263 » // runtime·NumCgoCall() iterates over allm w/o schedlock, |
264 // so we need to publish it safely. | 264 // so we need to publish it safely. |
265 runtime·atomicstorep(&runtime·allm, mp); | 265 runtime·atomicstorep(&runtime·allm, mp); |
266 runtime·unlock(&runtime·sched); | 266 runtime·unlock(&runtime·sched); |
267 | |
268 LOG("%d: mcommoninit %d m=%p\n", m->id, mp->id, mp); | |
269 } | 267 } |
270 | 268 |
271 // Mark gp ready to run. | 269 // Mark gp ready to run. |
272 void | 270 void |
273 runtime·ready(G *gp) | 271 runtime·ready(G *gp) |
274 { | 272 { |
275 CHECK(gp->m == nil, ("bad g->m in ready")); | |
276 // Mark runnable. | 273 // Mark runnable. |
277 » if(gp->status == Grunnable || gp->status == Grunning) { | 274 » if(gp->status != Gwaiting) { |
278 runtime·printf("goroutine %D has status %d\n", gp->goid, gp->sta
tus); | 275 runtime·printf("goroutine %D has status %d\n", gp->goid, gp->sta
tus); |
279 runtime·throw("bad g->status in ready"); | 276 runtime·throw("bad g->status in ready"); |
280 } | 277 } |
281 gp->status = Grunnable; | 278 gp->status = Grunnable; |
282 runqput(m->p, gp); | 279 runqput(m->p, gp); |
283 » if(runtime·sched.pidle) | 280 » if(runtime·sched.npidle != 0 && runtime·sched.nmspinning == 0) // TODO:
fast atomic |
284 » » startm(nil); | 281 » » wakep(); |
285 } | |
286 | |
287 // Schedules some M to run the p (creates an M if necessary). | |
288 // If p==nil, tries to get an idle P, if no idle P's returns false. | |
289 static bool | |
290 startm(P *p) | |
291 { | |
292 » M *mp; | |
293 | |
294 » runtime·lock(&runtime·sched); | |
295 » if(p == nil) { | |
296 » » p = pidleget(); | |
297 » » if(p == nil) { | |
298 » » » runtime·unlock(&runtime·sched); | |
299 » » » return false;» » »······· | |
300 » » } | |
301 » } | |
302 » mp = mtryget(); | |
303 » runtime·unlock(&runtime·sched); | |
304 » if(mp) { | |
305 » » acquirep(mp, p); | |
306 » » runtime·notewakeup(&mp->park); | |
307 » » return true; | |
308 » } | |
309 » newm(runtime·mstart, p, false); | |
310 » return true; | |
311 } | 282 } |
312 | 283 |
313 int32 | 284 int32 |
314 runtime·gcprocs(void) | 285 runtime·gcprocs(void) |
315 { | 286 { |
316 int32 n; | 287 int32 n; |
317 | 288 |
318 // Figure out how many CPUs to use during GC. | 289 // Figure out how many CPUs to use during GC. |
319 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. | 290 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. |
320 runtime·lock(&runtime·sched); | 291 runtime·lock(&runtime·sched); |
321 n = runtime·gomaxprocs; | 292 n = runtime·gomaxprocs; |
322 if(n > runtime·ncpu) | 293 if(n > runtime·ncpu) |
323 n = runtime·ncpu; | 294 n = runtime·ncpu; |
324 if(n > MaxGcproc) | 295 if(n > MaxGcproc) |
325 n = MaxGcproc; | 296 n = MaxGcproc; |
326 » if(n > runtime·sched.mwait+1) // one M is currently running | 297 » if(n > runtime·sched.nmidle+1) // one M is currently running |
327 » » n = runtime·sched.mwait+1; | 298 » » n = runtime·sched.nmidle+1; |
328 runtime·unlock(&runtime·sched); | 299 runtime·unlock(&runtime·sched); |
329 return n; | 300 return n; |
330 } | 301 } |
331 | 302 |
332 static bool | 303 static bool |
333 needaddgcproc(void) | 304 needaddgcproc(void) |
334 { | 305 { |
335 int32 n; | 306 int32 n; |
336 | 307 |
337 runtime·lock(&runtime·sched); | 308 runtime·lock(&runtime·sched); |
338 n = runtime·gomaxprocs; | 309 n = runtime·gomaxprocs; |
339 if(n > runtime·ncpu) | 310 if(n > runtime·ncpu) |
340 n = runtime·ncpu; | 311 n = runtime·ncpu; |
341 if(n > MaxGcproc) | 312 if(n > MaxGcproc) |
342 n = MaxGcproc; | 313 n = MaxGcproc; |
343 » n -= runtime·sched.mwait+1; // one M is currently running | 314 » n -= runtime·sched.nmidle+1; // one M is currently running |
344 runtime·unlock(&runtime·sched); | 315 runtime·unlock(&runtime·sched); |
345 return n > 0; | 316 return n > 0; |
346 } | 317 } |
347 | 318 |
348 void | 319 void |
349 runtime·helpgc(int32 nproc) | 320 runtime·helpgc(int32 nproc) |
350 { | 321 { |
351 M *mp; | 322 M *mp; |
352 int32 n, pos; | 323 int32 n, pos; |
353 | 324 |
354 LOG("%d: helpgc(%d)\n", m->id, nproc); | |
355 runtime·lock(&runtime·sched); | 325 runtime·lock(&runtime·sched); |
356 pos = 0; | 326 pos = 0; |
357 for(n = 1; n < nproc; n++) { // one M is currently running | 327 for(n = 1; n < nproc; n++) { // one M is currently running |
358 if(runtime·allp[pos]->mcache == m->mcache) | 328 if(runtime·allp[pos]->mcache == m->mcache) |
359 pos++; | 329 pos++; |
360 » » mp = mtryget(); | 330 » » mp = mget(); |
361 if(mp == nil) | 331 if(mp == nil) |
362 runtime·throw("runtime·gcprocs inconsistency"); | 332 runtime·throw("runtime·gcprocs inconsistency"); |
363 mp->helpgc = 1; | 333 mp->helpgc = 1; |
364 mp->mcache = runtime·allp[pos]->mcache; | 334 mp->mcache = runtime·allp[pos]->mcache; |
365 pos++; | 335 pos++; |
366 LOG("%d: helpgc wake %d\n", m->id, mp->id); | |
367 runtime·notewakeup(&mp->park); | 336 runtime·notewakeup(&mp->park); |
368 } | 337 } |
369 runtime·unlock(&runtime·sched); | 338 runtime·unlock(&runtime·sched); |
370 } | 339 } |
371 | 340 |
372 void | 341 void |
373 runtime·stoptheworld(void) | 342 runtime·stoptheworld(void) |
374 { | 343 { |
375 int32 i; | 344 int32 i; |
376 uint32 s; | 345 uint32 s; |
377 P *p; | 346 P *p; |
378 bool wait; | 347 bool wait; |
379 | 348 |
380 LOG("%d: stoptheworld\n", m->id); | |
381 runtime·lock(&runtime·sched); | 349 runtime·lock(&runtime·sched); |
382 runtime·gcwaiting = 1; | |
383 runtime·sched.stopwait = runtime·gomaxprocs; | 350 runtime·sched.stopwait = runtime·gomaxprocs; |
| 351 runtime·atomicstore((uint32*)&runtime·gcwaiting, 1); |
| 352 // stop current P |
384 m->p->status = Pgcstop; | 353 m->p->status = Pgcstop; |
385 runtime·sched.stopwait--; | 354 runtime·sched.stopwait--; |
386 » for(i=0; i<runtime·gomaxprocs; i++) { | 355 » // try to retake all P's in Psyscall status |
387 » » s = runtime·allp[i]->status; | 356 » for(i = 0; i < runtime·gomaxprocs; i++) { |
388 » » if(s == Psyscall && runtime·cas(&runtime·allp[i]->status, s, Pgc
stop)) { | 357 » » p = runtime·allp[i]; |
389 » » » LOG(" acquired syscall %d\n", i); | 358 » » s = p->status; |
| 359 » » if(s == Psyscall && runtime·cas(&p->status, s, Pgcstop)) |
390 runtime·sched.stopwait--; | 360 runtime·sched.stopwait--; |
391 » » } | 361 » } |
392 » } | 362 » // stop idle P's |
393 » while(runtime·sched.pidle) { | 363 » while(p = pidleget()) { |
394 » » p = pidleget(); | |
395 p->status = Pgcstop; | 364 p->status = Pgcstop; |
396 runtime·sched.stopwait--; | 365 runtime·sched.stopwait--; |
397 } | 366 } |
398 CHECK(runtime·sched.stopwait >= 0, ("")); | |
399 wait = runtime·sched.stopwait > 0; | 367 wait = runtime·sched.stopwait > 0; |
400 runtime·unlock(&runtime·sched); | 368 runtime·unlock(&runtime·sched); |
| 369 |
| 370 // wait for remaining P's to stop voluntary |
401 if(wait) { | 371 if(wait) { |
402 runtime·notesleep(&runtime·sched.stopnote); | 372 runtime·notesleep(&runtime·sched.stopnote); |
403 runtime·noteclear(&runtime·sched.stopnote); | 373 runtime·noteclear(&runtime·sched.stopnote); |
404 } | 374 } |
405 » LOG("%d: stoptheworld stopped\n", m->id); | 375 » if(runtime·sched.stopwait) |
406 » CHECK(runtime·sched.stopwait == 0, ("stoptheworld: stopwait == %d\n", ru
ntime·sched.stopwait)); | 376 » » runtime·throw("stoptheworld: not stopped"); |
407 » for(i=0; i<runtime·gomaxprocs; i++) { | 377 » for(i = 0; i < runtime·gomaxprocs; i++) { |
408 » » CHECK(runtime·allp[i]->status == Pgcstop, ("stoptheworld: not st
opped (%d)\n", runtime·allp[i]->status)); | 378 » » p = runtime·allp[i]; |
| 379 » » if(p->status != Pgcstop) |
| 380 » » » runtime·throw("stoptheworld: not stopped"); |
409 } | 381 } |
410 } | 382 } |
411 | 383 |
412 void | 384 void |
413 runtime·starttheworld(void) | 385 runtime·starttheworld(void) |
414 { | 386 { |
415 /* | 387 » P *p; |
416 » G *gp; | |
417 » *p; | |
418 M *mp; | 388 M *mp; |
419 int32 n, w; | |
420 */ | |
421 bool add; | 389 bool add; |
422 | 390 |
423 LOG("%d: starttheworld\n", m->id); | |
424 add = needaddgcproc(); | 391 add = needaddgcproc(); |
425 » runtime·gcwaiting = 0; | 392 » runtime·lock(&runtime·sched); |
426 if(newprocs) { | 393 if(newprocs) { |
427 procresize(newprocs); | 394 procresize(newprocs); |
428 newprocs = 0; | 395 newprocs = 0; |
429 » } else { | 396 » } else |
430 procresize(runtime·gomaxprocs); | 397 procresize(runtime·gomaxprocs); |
431 » } | 398 » runtime·gcwaiting = 0; |
432 » runtime·lock(&runtime·sched); | 399 |
433 » /* | 400 » while(p = pidleget()) { |
434 » gp = runtime·netwait(0, 4 * runtime·gomaxprocs); | 401 » » // procresize() puts p's with work at the beginning of the list. |
435 » n = w = 0; | 402 » » // Once we reach a p without a run queue, the rest don't have on
e either. |
436 » runtime·inject(gp, &w, &n); | 403 » » if(p->runqhead == p->runqtail) { |
437 » while(runtime·sched.pidle) { | 404 » » » pidleput(p); |
438 » » p = pidleget(); | 405 » » » break; |
439 » » mp = mtryget(); | 406 » » } |
440 » » if(mp) { | 407 » » mp = mget(); |
441 » » » acquirep(mp, p); | 408 » » if(mp == nil) { |
442 » » » runtime·notewakeup(&mp->park); | 409 » » » pidleput(p); |
443 » » } else { | 410 » » » break; |
444 » » » runtime·unlock(&runtime·sched); | 411 » » } |
445 » » » newm(runtime·mstart, p, false); | 412 » » if(mp->nextp) |
446 » » » runtime·lock(&runtime·sched); | 413 » » » runtime·throw("starttheworld: inconsistent mp->nextp"); |
447 » » } | 414 » » mp->nextp = p; |
448 » } | 415 » » runtime·notewakeup(&mp->park); |
449 » */ | 416 » } |
450 if(runtime·sched.sysmonwait) { | 417 if(runtime·sched.sysmonwait) { |
451 » » runtime·sched.sysmonwait = 0; | 418 » » runtime·sched.sysmonwait = false; |
452 runtime·notewakeup(&runtime·sched.sysmonnote); | 419 runtime·notewakeup(&runtime·sched.sysmonnote); |
453 } | 420 } |
454 runtime·unlock(&runtime·sched); | 421 runtime·unlock(&runtime·sched); |
455 | 422 |
456 if(add) { | 423 if(add) { |
457 // If GC could have used another helper proc, start one now, | 424 // If GC could have used another helper proc, start one now, |
458 // in the hope that it will be available next time. | 425 // in the hope that it will be available next time. |
459 // It would have been even better to start it before the collect
ion, | 426 // It would have been even better to start it before the collect
ion, |
460 // but doing so requires allocating memory, so it's tricky to | 427 // but doing so requires allocating memory, so it's tricky to |
461 // coordinate. This lazy approach works out in practice: | 428 // coordinate. This lazy approach works out in practice: |
462 // we don't mind if the first couple gc rounds don't have quite | 429 // we don't mind if the first couple gc rounds don't have quite |
463 // the maximum number of procs. | 430 // the maximum number of procs. |
464 » » newm(runtime·mstart, nil, true); | 431 » » newm(runtime·mstart, nil, true, false); |
465 } | 432 } |
466 } | 433 } |
467 | 434 |
468 // Called to start an M. | 435 // Called to start an M. |
469 void | 436 void |
470 runtime·mstart(void) | 437 runtime·mstart(void) |
471 { | 438 { |
472 // It is used by windows-386 only. Unfortunately, seh needs | 439 // It is used by windows-386 only. Unfortunately, seh needs |
473 // to be located on os stack, and mstart runs on os stack | 440 // to be located on os stack, and mstart runs on os stack |
474 // for both m0 and m. | 441 // for both m0 and m. |
475 SEH seh; | 442 SEH seh; |
476 » P *p; | 443 |
477 | |
478 » LOG("%d: mstart m=%p\n", m->id, m); | |
479 if(g != m->g0) | 444 if(g != m->g0) |
480 runtime·throw("bad runtime·mstart"); | 445 runtime·throw("bad runtime·mstart"); |
481 | 446 |
482 // Record top of stack for use by mcall. | 447 // Record top of stack for use by mcall. |
483 // Once we call schedule we're never coming back, | 448 // Once we call schedule we're never coming back, |
484 // so other calls can reuse this stack space. | 449 // so other calls can reuse this stack space. |
485 runtime·gosave(&m->g0->sched); | 450 runtime·gosave(&m->g0->sched); |
486 m->g0->sched.pc = (void*)-1; // make sure it is never used | 451 m->g0->sched.pc = (void*)-1; // make sure it is never used |
487 m->seh = &seh; | 452 m->seh = &seh; |
488 runtime·asminit(); | 453 runtime·asminit(); |
489 runtime·minit(); | 454 runtime·minit(); |
490 | 455 |
491 // Install signal handlers; after minit so that minit can | 456 // Install signal handlers; after minit so that minit can |
492 // prepare the thread to be able to handle the signals. | 457 // prepare the thread to be able to handle the signals. |
493 » if(m == &runtime·m0) | 458 » if(m == &runtime·m0) { |
494 runtime·initsig(); | 459 runtime·initsig(); |
| 460 if(runtime·iscgo) |
| 461 runtime·newextram(); |
| 462 } |
495 | 463 |
496 if(m->helpgc) { | 464 if(m->helpgc) { |
497 LOG("%d: mstart helpgc\n", m->id); | |
498 m->helpgc = false; | 465 m->helpgc = false; |
499 stopm(); | 466 stopm(); |
500 } else if(m != &runtime·m0) { | 467 } else if(m != &runtime·m0) { |
501 » » p = m->p; | 468 » » acquirep(m->nextp); |
502 » » m->p = nil; | 469 » » m->nextp = nil; |
503 » » acquirep(m, p); | 470 » } |
504 » } | |
505 » LOG("%d: calling schedule\n", m->id); | |
506 schedule(); | 471 schedule(); |
507 | 472 |
508 // TODO(brainman): This point is never reached, because scheduler | 473 // TODO(brainman): This point is never reached, because scheduler |
509 // does not release os threads at the moment. But once this path | 474 // does not release os threads at the moment. But once this path |
510 // is enabled, we must remove our seh here. | 475 // is enabled, we must remove our seh here. |
511 } | 476 } |
512 | 477 |
513 // When running with cgo, we call libcgo_thread_start | 478 // When running with cgo, we call _cgo_thread_start |
514 // to start threads for us so that we can play nicely with | 479 // to start threads for us so that we can play nicely with |
515 // foreign code. | 480 // foreign code. |
516 void (*libcgo_thread_start)(void*); | 481 void (*_cgo_thread_start)(void*); |
517 | 482 |
518 typedef struct CgoThreadStart CgoThreadStart; | 483 typedef struct CgoThreadStart CgoThreadStart; |
519 struct CgoThreadStart | 484 struct CgoThreadStart |
520 { | 485 { |
521 M *m; | 486 M *m; |
522 G *g; | 487 G *g; |
523 void (*fn)(void); | 488 void (*fn)(void); |
524 }; | 489 }; |
525 | 490 |
526 // Create a new m. It will start off with a call to fn. | 491 // Allocate a new m unassociated with any thread. |
527 static void | 492 // Can use p for allocation context if needed. |
528 newm(void(*fn)(void), P *p, bool helpgc) | 493 M* |
| 494 runtime·allocm(P *p) |
529 { | 495 { |
530 M *mp; | 496 M *mp; |
531 static Type *mtype; // The Go type M | 497 static Type *mtype; // The Go type M |
532 | 498 |
533 » LOG("%d: newm\n", m->id); | 499 » m->locks++; // disable GC because it can be called from sysmon |
534 » if(m->p == nil) { | 500 » if(m->p == nil) |
535 » » CHECK(p, ("newm: no p\n")); | 501 » » acquirep(p); // temporarily borrow p for mallocs in this functi
on |
536 » » CHECK(!helpgc, ("newm: helpgc\n")); | |
537 » » acquirep(m, p); // temporarily borrow p for mallocs in this fun
ction | |
538 » } | |
539 » CHECK(m->mcache, ("newm: no mcache\n")); | |
540 » m->locks++; //!!! needed? | |
541 if(mtype == nil) { | 502 if(mtype == nil) { |
542 Eface e; | 503 Eface e; |
543 runtime·gc_m_ptr(&e); | 504 runtime·gc_m_ptr(&e); |
544 mtype = ((PtrType*)e.type)->elem; | 505 mtype = ((PtrType*)e.type)->elem; |
545 } | 506 } |
546 | 507 |
547 mp = runtime·cnew(mtype); | 508 mp = runtime·cnew(mtype); |
548 mcommoninit(mp); | 509 mcommoninit(mp); |
549 mp->p = p; | |
550 mp->helpgc = helpgc; | |
551 | 510 |
552 // In case of cgo, pthread_create will make us a stack. | 511 // In case of cgo, pthread_create will make us a stack. |
553 // Windows will layout sched stack on OS stack. | 512 // Windows will layout sched stack on OS stack. |
554 » mp->g0 = runtime·malg(runtime·iscgo || Windows ? -1 : 8192); | 513 » if(runtime·iscgo || Windows) |
555 | 514 » » mp->g0 = runtime·malg(-1); |
| 515 » else |
| 516 » » mp->g0 = runtime·malg(8192); |
| 517 |
| 518 » if(p == m->p) |
| 519 » » releasep(); |
556 m->locks--; | 520 m->locks--; |
557 » if(p == m->p) { | 521 |
558 » » CHECK(!helpgc, ("newm: releasep helpgc")); | 522 » return mp; |
559 » » releasep(); | 523 } |
560 » } | 524 |
561 » // Can not allocate memory after this point. | 525 static M* lockextra(bool nilokay); |
| 526 static void unlockextra(M*); |
| 527 |
| 528 // needm is called when a cgo callback happens on a |
| 529 // thread without an m (a thread not created by Go). |
| 530 // In this case, needm is expected to find an m to use |
| 531 // and return with m, g initialized correctly. |
| 532 // Since m and g are not set now (likely nil, but see below) |
| 533 // needm is limited in what routines it can call. In particular |
| 534 // it can only call nosplit functions (textflag 7) and cannot |
| 535 // do any scheduling that requires an m. |
| 536 // |
| 537 // In order to avoid needing heavy lifting here, we adopt |
| 538 // the following strategy: there is a stack of available m's |
| 539 // that can be stolen. Using compare-and-swap |
| 540 // to pop from the stack has ABA races, so we simulate |
| 541 // a lock by doing an exchange (via casp) to steal the stack |
| 542 // head and replace the top pointer with MLOCKED (1). |
| 543 // This serves as a simple spin lock that we can use even |
| 544 // without an m. The thread that locks the stack in this way |
| 545 // unlocks the stack by storing a valid stack head pointer. |
| 546 // |
| 547 // In order to make sure that there is always an m structure |
| 548 // available to be stolen, we maintain the invariant that there |
| 549 // is always one more than needed. At the beginning of the |
| 550 // program (if cgo is in use) the list is seeded with a single m. |
| 551 // If needm finds that it has taken the last m off the list, its job |
| 552 // is - once it has installed its own m so that it can do things like |
| 553 // allocate memory - to create a spare m and put it on the list. |
| 554 // |
| 555 // Each of these extra m's also has a g0 and a curg that are |
| 556 // pressed into service as the scheduling stack and current |
| 557 // goroutine for the duration of the cgo callback. |
| 558 // |
| 559 // When the callback is done with the m, it calls dropm to |
| 560 // put the m back on the list. |
| 561 #pragma textflag 7 |
| 562 void |
| 563 runtime·needm(byte x) |
| 564 { |
| 565 » M *mp; |
| 566 |
| 567 » // Lock extra list, take head, unlock popped list. |
| 568 » // nilokay=false is safe here because of the invariant above, |
| 569 » // that the extra list always contains or will soon contain |
| 570 » // at least one m. |
| 571 » mp = lockextra(false); |
| 572 |
| 573 » // Set needextram when we've just emptied the list, |
| 574 » // so that the eventual call into cgocallbackg will |
| 575 » // allocate a new m for the extra list. We delay the |
| 576 » // allocation until then so that it can be done |
| 577 » // after exitsyscall makes sure it is okay to be |
| 578 » // running at all (that is, there's no garbage collection |
| 579 » // running right now). |
| 580 » mp->needextram = mp->schedlink == nil; |
| 581 » unlockextra(mp->schedlink); |
| 582 |
| 583 » // Install m and g (= m->g0) and set the stack bounds |
| 584 » // to match the current stack. We don't actually know |
| 585 » // how big the stack is, like we don't know how big any |
| 586 » // scheduling stack is, but we assume there's at least 32 kB, |
| 587 » // which is more than enough for us. |
| 588 » runtime·setmg(mp, mp->g0); |
| 589 » g->stackbase = (uintptr)(&x + 1024); |
| 590 » g->stackguard = (uintptr)(&x - 32*1024); |
| 591 |
| 592 » // On windows/386, we need to put an SEH frame (two words) |
| 593 » // somewhere on the current stack. We are called |
| 594 » // from needm, and we know there is some available |
| 595 » // space one word into the argument frame. Use that. |
| 596 » m->seh = (SEH*)((uintptr*)&x + 1); |
| 597 |
| 598 » // Initialize this thread to use the m. |
| 599 » runtime·asminit(); |
| 600 » runtime·minit(); |
| 601 } |
| 602 |
| 603 // newextram allocates an m and puts it on the extra list. |
| 604 // It is called with a working local m, so that it can do things |
| 605 // like call schedlock and allocate. |
| 606 void |
| 607 runtime·newextram(void) |
| 608 { |
| 609 » M *mp, *mnext; |
| 610 » G *gp; |
| 611 |
| 612 » // Create extra goroutine locked to extra m. |
| 613 » // The goroutine is the context in which the cgo callback will run. |
| 614 » // The sched.pc will never be returned to, but setting it to |
| 615 » // runtime.goexit makes clear to the traceback routines where |
| 616 » // the goroutine stack ends. |
| 617 » mp = runtime·allocm(nil); |
| 618 » gp = runtime·malg(4096); |
| 619 » gp->sched.pc = (void*)runtime·goexit; |
| 620 » gp->sched.sp = gp->stackbase; |
| 621 » gp->sched.g = gp; |
| 622 » gp->status = Gsyscall; |
| 623 » mp->curg = gp; |
| 624 » mp->locked = LockInternal; |
| 625 » mp->lockedg = gp; |
| 626 » gp->lockedm = mp; |
| 627 » // put on allg for garbage collector |
| 628 » runtime·lock(&runtime·sched); |
| 629 » if(runtime·lastg == nil) |
| 630 » » runtime·allg = gp; |
| 631 » else |
| 632 » » runtime·lastg->alllink = gp; |
| 633 » runtime·lastg = gp; |
| 634 » runtime·unlock(&runtime·sched); |
| 635 » gp->goid = runtime·xadd64(&runtime·sched.goidgen, 1); |
| 636 » if(raceenabled) |
| 637 » » gp->racectx = runtime·racegostart(runtime·newextram); |
| 638 |
| 639 » // Add m to the extra list. |
| 640 » mnext = lockextra(true); |
| 641 » mp->schedlink = mnext; |
| 642 » unlockextra(mp); |
| 643 } |
| 644 |
| 645 // dropm is called when a cgo callback has called needm but is now |
| 646 // done with the callback and returning back into the non-Go thread. |
| 647 // It puts the current m back onto the extra list. |
| 648 // |
| 649 // The main expense here is the call to signalstack to release the |
| 650 // m's signal stack, and then the call to needm on the next callback |
| 651 // from this thread. It is tempting to try to save the m for next time, |
| 652 // which would eliminate both these costs, but there might not be |
| 653 // a next time: the current thread (which Go does not control) might exit. |
| 654 // If we saved the m for that thread, there would be an m leak each time |
| 655 // such a thread exited. Instead, we acquire and release an m on each |
| 656 // call. These should typically not be scheduling operations, just a few |
| 657 // atomics, so the cost should be small. |
| 658 // |
| 659 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread |
| 660 // variable using pthread_key_create. Unlike the pthread keys we already use |
| 661 // on OS X, this dummy key would never be read by Go code. It would exist |
| 662 // only so that we could register at thread-exit-time destructor. |
| 663 // That destructor would put the m back onto the extra list. |
| 664 // This is purely a performance optimization. The current version, |
| 665 // in which dropm happens on each cgo call, is still correct too. |
| 666 // We may have to keep the current version on systems with cgo |
| 667 // but without pthreads, like Windows. |
| 668 void |
| 669 runtime·dropm(void) |
| 670 { |
| 671 » M *mp, *mnext; |
| 672 |
| 673 » // Undo whatever initialization minit did during needm. |
| 674 » runtime·unminit(); |
| 675 |
| 676 » // Clear m and g, and return m to the extra list. |
| 677 » // After the call to setmg we can only call nosplit functions. |
| 678 » mp = m; |
| 679 » runtime·setmg(nil, nil); |
| 680 |
| 681 » mnext = lockextra(true); |
| 682 » mp->schedlink = mnext; |
| 683 » unlockextra(mp); |
| 684 } |
| 685 |
| 686 #define MLOCKED ((M*)1) |
| 687 |
| 688 // lockextra locks the extra list and returns the list head. |
| 689 // The caller must unlock the list by storing a new list head |
| 690 // to runtime.extram. If nilokay is true, then lockextra will |
| 691 // return a nil list head if that's what it finds. If nilokay is false, |
| 692 // lockextra will keep waiting until the list head is no longer nil. |
| 693 #pragma textflag 7 |
| 694 static M* |
| 695 lockextra(bool nilokay) |
| 696 { |
| 697 » M *mp; |
| 698 » void (*yield)(void); |
| 699 |
| 700 » for(;;) { |
| 701 » » mp = runtime·atomicloadp(&runtime·extram); |
| 702 » » if(mp == MLOCKED) { |
| 703 » » » yield = runtime·osyield; |
| 704 » » » yield(); |
| 705 » » » continue; |
| 706 » » } |
| 707 » » if(mp == nil && !nilokay) { |
| 708 » » » runtime·usleep(1); |
| 709 » » » continue; |
| 710 » » } |
| 711 » » if(!runtime·casp(&runtime·extram, mp, MLOCKED)) { |
| 712 » » » yield = runtime·osyield; |
| 713 » » » yield(); |
| 714 » » » continue; |
| 715 » » } |
| 716 » » break; |
| 717 » } |
| 718 » return mp; |
| 719 } |
| 720 |
| 721 #pragma textflag 7 |
| 722 static void |
| 723 unlockextra(M *mp) |
| 724 { |
| 725 » runtime·atomicstorep(&runtime·extram, mp); |
| 726 } |
| 727 |
| 728 |
| 729 // Create a new m. It will start off with a call to fn. |
| 730 static void |
| 731 newm(void(*fn)(void), P *p, bool helpgc, bool spinning) |
| 732 { |
| 733 » M *mp; |
| 734 |
| 735 » mp = runtime·allocm(p); |
| 736 » mp->nextp = p; |
| 737 » mp->helpgc = helpgc; |
| 738 » mp->spinning = spinning; |
562 | 739 |
563 if(runtime·iscgo) { | 740 if(runtime·iscgo) { |
564 CgoThreadStart ts; | 741 CgoThreadStart ts; |
565 | 742 |
566 » » if(libcgo_thread_start == nil) | 743 » » if(_cgo_thread_start == nil) |
567 » » » runtime·throw("libcgo_thread_start missing"); | 744 » » » runtime·throw("_cgo_thread_start missing"); |
568 ts.m = mp; | 745 ts.m = mp; |
569 ts.g = mp->g0; | 746 ts.g = mp->g0; |
570 ts.fn = fn; | 747 ts.fn = fn; |
571 » » runtime·asmcgocall(libcgo_thread_start, &ts); | 748 » » runtime·asmcgocall(_cgo_thread_start, &ts); |
572 return; | 749 return; |
573 } | 750 } |
574 runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, fn); | 751 runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, fn); |
575 } | 752 } |
576 | 753 |
577 // Stops execution of the current m until new work is available. | 754 // Stops execution of the current m until new work is available. |
578 // Returns with acquired P. | 755 // Returns with acquired P. |
579 static void | 756 static void |
580 stopm(void) | 757 stopm(void) |
581 { | 758 { |
582 » LOG("%d: mstop\n", m->id); | 759 » if(m->locks) |
583 » CHECK(m->locks == 0, ("")); | 760 » » runtime·throw("stopm holding locks"); |
584 » CHECK(m->p == nil, ("mstop: p != nil\n")); | 761 » if(m->p) |
| 762 » » runtime·throw("stopm holding p"); |
| 763 » if(m->spinning) { |
| 764 » » m->spinning = false; |
| 765 » » runtime·xadd(&runtime·sched.nmspinning, -1); |
| 766 » } |
585 | 767 |
586 retry: | 768 retry: |
587 runtime·lock(&runtime·sched); | 769 runtime·lock(&runtime·sched); |
588 mput(m); | 770 mput(m); |
589 runtime·unlock(&runtime·sched); | 771 runtime·unlock(&runtime·sched); |
590 runtime·notesleep(&m->park); | 772 runtime·notesleep(&m->park); |
591 runtime·noteclear(&m->park); | 773 runtime·noteclear(&m->park); |
592 if(m->helpgc) { | 774 if(m->helpgc) { |
593 LOG("%d: gchelper\n", m->id); | |
594 m->helpgc = 0; | 775 m->helpgc = 0; |
595 runtime·gchelper(); | 776 runtime·gchelper(); |
596 m->mcache = nil; | 777 m->mcache = nil; |
597 LOG("%d: gchelper end\n", m->id); | |
598 goto retry; | 778 goto retry; |
599 } | 779 } |
600 » LOG("%d: mstop wake\n", m->id); | 780 » acquirep(m->nextp); |
601 » if(m->p == nil) | 781 » m->nextp = nil; |
602 » » runtime·throw("mstop: p == nil"); | 782 } |
| 783 |
| 784 // Schedules some M to run the p (creates an M if necessary). |
| 785 // If p==nil, tries to get an idle P, if no idle P's returns false. |
| 786 static void |
| 787 startm(P *p, bool spinning) |
| 788 { |
| 789 » M *mp; |
| 790 |
| 791 » runtime·lock(&runtime·sched); |
| 792 » if(p == nil) { |
| 793 » » p = pidleget(); |
| 794 » » if(p == nil) { |
| 795 » » » runtime·unlock(&runtime·sched); |
| 796 » » » if(spinning) |
| 797 » » » » runtime·xadd(&runtime·sched.nmspinning, -1); |
| 798 » » » return; |
| 799 » » } |
| 800 » } |
| 801 » mp = mget(); |
| 802 » runtime·unlock(&runtime·sched); |
| 803 » if(mp == nil) { |
| 804 » » newm(runtime·mstart, p, false, spinning); |
| 805 » » return; |
| 806 » } |
| 807 » if(mp->spinning) |
| 808 » » runtime·throw("startm: m is spinning"); |
| 809 » if(mp->nextp) |
| 810 » » runtime·throw("startm: m has p"); |
| 811 » mp->spinning = spinning; |
| 812 » mp->nextp = p; |
| 813 » runtime·notewakeup(&mp->park); |
| 814 } |
| 815 |
| 816 // Hands off P from syscall or locked M. |
| 817 static void |
| 818 handoffp(P *p) |
| 819 { |
| 820 » // if it has local work, start it straight away |
| 821 » if(p->runqhead != p->runqtail || runtime·sched.runqsize) { |
| 822 » » startm(p, false); |
| 823 » » return; |
| 824 » } |
| 825 » // no local work, check that there are no spinning/idle M's, |
| 826 » // otherwise our help is not required |
| 827 » if(runtime·sched.nmspinning + runtime·sched.npidle == 0 && // TODO: fas
t atomic |
| 828 » » runtime·cas(&runtime·sched.nmspinning, 0, 1)) { |
| 829 » » startm(p, true); |
| 830 » » return; |
| 831 » } |
| 832 » runtime·lock(&runtime·sched); |
| 833 » if(runtime·gcwaiting) { |
| 834 » » p->status = Pgcstop; |
| 835 » » if(--runtime·sched.stopwait == 0) |
| 836 » » » runtime·notewakeup(&runtime·sched.stopnote); |
| 837 » » runtime·unlock(&runtime·sched); |
| 838 » » return; |
| 839 » } |
| 840 » if(runtime·sched.runqsize) { |
| 841 » » runtime·unlock(&runtime·sched); |
| 842 » » startm(p, false); |
| 843 » » return; |
| 844 » } |
| 845 » pidleput(p); |
| 846 » runtime·unlock(&runtime·sched); |
| 847 } |
| 848 |
| 849 // Tries to add one more P to execute G's. |
| 850 // Called when a G is made runnable (newproc, ready). |
| 851 static void |
| 852 wakep(void) |
| 853 { |
| 854 » // be conservative about spinning threads |
| 855 » if(!runtime·cas(&runtime·sched.nmspinning, 0, 1)) |
| 856 » » return; |
| 857 » startm(nil, true); |
| 858 } |
| 859 |
| 860 // Stops execution of the current m that is locked to a g until the g is runnabl
e again. |
| 861 // Returns with acquired P. |
| 862 static void |
| 863 stoplockedm(void) |
| 864 { |
| 865 » P *p; |
| 866 |
| 867 » if(m->lockedg == nil || m->lockedg->lockedm != m) |
| 868 » » runtime·throw("stoplockedm: inconsistent locking"); |
| 869 » if(m->p) { |
| 870 » » // Schedule another M to run this p. |
| 871 » » p = releasep(); |
| 872 » » handoffp(p); |
| 873 » } |
| 874 » inclocked(1); |
| 875 » // Wait until another thread schedules lockedg again. |
| 876 » runtime·notesleep(&m->park); |
| 877 » runtime·noteclear(&m->park); |
| 878 » if(m->lockedg->status != Grunnable) |
| 879 » » runtime·throw("stoplockedm: not runnable"); |
| 880 » acquirep(m->nextp); |
| 881 » m->nextp = nil; |
| 882 } |
| 883 |
| 884 // Schedules the locked m to run the locked gp. |
| 885 static void |
| 886 startlockedm(G *gp) |
| 887 { |
| 888 » M *mp; |
| 889 » P *p; |
| 890 |
| 891 » mp = gp->lockedm; |
| 892 » if(mp == m) |
| 893 » » runtime·throw("startlockedm: locked to me"); |
| 894 » if(mp->nextp) |
| 895 » » runtime·throw("startlockedm: m has p"); |
| 896 » // directly handoff current P to the locked m |
| 897 » inclocked(-1); |
| 898 » p = releasep(); |
| 899 » mp->nextp = p; |
| 900 » runtime·notewakeup(&mp->park); |
| 901 » stopm(); |
| 902 } |
| 903 |
| 904 // Stops the current m for stoptheworld. |
| 905 // Returns when the world is restarted. |
| 906 static void |
| 907 gcstopm(void) |
| 908 { |
| 909 » P *p; |
| 910 |
| 911 » if(!runtime·gcwaiting) |
| 912 » » runtime·throw("gcstopm: not waiting for gc"); |
| 913 » if(m->spinning) { |
| 914 » » m->spinning = false; |
| 915 » » runtime·xadd(&runtime·sched.nmspinning, -1); |
| 916 » } |
| 917 » p = releasep(); |
| 918 » runtime·lock(&runtime·sched); |
| 919 » p->status = Pgcstop; |
| 920 » if(--runtime·sched.stopwait == 0) |
| 921 » » runtime·notewakeup(&runtime·sched.stopnote); |
| 922 » runtime·unlock(&runtime·sched); |
| 923 » stopm(); |
603 } | 924 } |
604 | 925 |
605 // Schedules gp to run on the current M. | 926 // Schedules gp to run on the current M. |
606 // Never returns. | 927 // Never returns. |
607 static void | 928 static void |
608 execute(G *gp) | 929 execute(G *gp) |
609 { | 930 { |
610 int32 hz; | 931 int32 hz; |
611 | 932 |
612 » LOG("%d: start running goroutine %p\n", m->id, gp); | 933 » if(gp->status != Grunnable) { |
613 » CHECK(m->locks == 0, ("")); | 934 » » runtime·printf("execute: bad g status %d\n", gp->status); |
614 » CHECK(g == m->g0, ("execute: not on g0\n")); | 935 » » runtime·throw("execute: bad g status"); |
615 » CHECK(m->p != nil, ("execute: no p\n")); | 936 » } |
616 » CHECK(gp->status == Grunnable, ("execute: gp=%D gp->status=%d\n", gp->go
id, gp->status)); | 937 » gp->status = Grunning; |
617 » CHECK(gp->m == nil, ("execute: gp->m=%p\n", gp->m)); | |
618 » CHECK(gp->lockedm == nil && m->lockedg == nil || gp->lockedm == m && m->
lockedg == gp, | |
619 » » ("bad locking: gp->lockedm=%p m->lockedg=%p\n", gp->lockedm, m->
lockedg)); | |
620 m->p->tick++; | 938 m->p->tick++; |
621 gp->status = Grunning; | |
622 m->curg = gp; | 939 m->curg = gp; |
623 gp->m = m; | 940 gp->m = m; |
624 | 941 |
625 // Check whether the profiler needs to be turned on or off. | 942 // Check whether the profiler needs to be turned on or off. |
626 hz = runtime·sched.profilehz; | 943 hz = runtime·sched.profilehz; |
627 if(m->profilehz != hz) | 944 if(m->profilehz != hz) |
628 runtime·resetcpuprofiler(hz); | 945 runtime·resetcpuprofiler(hz); |
629 | 946 |
630 if(gp->sched.pc == (byte*)runtime·goexit) // kickoff | 947 if(gp->sched.pc == (byte*)runtime·goexit) // kickoff |
631 » » runtime·gogocall(&gp->sched, (void(*)(void))gp->entry); | 948 » » runtime·gogocallfn(&gp->sched, gp->fnstart); |
632 runtime·gogo(&gp->sched, 0); | 949 runtime·gogo(&gp->sched, 0); |
633 } | 950 } |
634 | 951 |
635 // Stops the current m for stoptheworld. | |
636 // Returns when the world is restarted. | |
637 static void | |
638 gcstopm(void) | |
639 { | |
640 P *p; | |
641 | |
642 CHECK(runtime·gcwaiting, ("")); | |
643 p = releasep(); | |
644 p->status = Pgcstop; | |
645 runtime·lock(&runtime·sched); | |
646 runtime·sched.stopwait--; | |
647 if(runtime·sched.stopwait == 0) | |
648 runtime·notewakeup(&runtime·sched.stopnote); | |
649 runtime·unlock(&runtime·sched); | |
650 stopm(); | |
651 } | |
652 | |
653 // Finds a runnable goroutine to execute. | 952 // Finds a runnable goroutine to execute. |
654 // Tries to steal from other P's, get g from global queue, polls network connect
ions. | 953 // Tries to steal from other P's and get g from global queue. |
655 static G* | 954 static G* |
656 findrunnable(void) | 955 findrunnable(void) |
657 { | 956 { |
658 » G *gp, *gp1; | 957 » G *gp; |
659 P *p; | 958 P *p; |
660 int32 i; | 959 int32 i; |
661 | 960 |
662 top: | 961 top: |
663 if(runtime·gcwaiting) { | 962 if(runtime·gcwaiting) { |
664 gcstopm(); | 963 gcstopm(); |
665 goto top; | 964 goto top; |
666 } | 965 } |
667 | 966 » // local runq |
668 gp = runqget(m->p); | 967 gp = runqget(m->p); |
669 if(gp) | 968 if(gp) |
670 return gp; | 969 return gp; |
671 » if (runtime·sched.poller == 0 && runtime·xchg(&runtime·sched.poller, 1)
== 0) { | 970 » // global runq |
672 » » G *gp2; | |
673 » » int32 w, n; | |
674 | |
675 » » gp2 = runtime·netwait(0, 10); | |
676 » » runtime·atomicstore(&runtime·sched.poller, 0); | |
677 » » if (gp2) { | |
678 » » » gp = gp2; | |
679 » » » gp->status = Grunnable; | |
680 » » » gp2 = gp->schedlink; | |
681 » » » if (gp2) { | |
682 » » » » n = w = 0; | |
683 » » » » runtime·inject(gp2, &w, &n); | |
684 » » » } | |
685 » » » return gp; | |
686 » » } | |
687 » } | |
688 | |
689 if(runtime·sched.runqsize) { | 971 if(runtime·sched.runqsize) { |
690 runtime·lock(&runtime·sched); | 972 runtime·lock(&runtime·sched); |
691 » » gp = globrunqget(); | 973 » » gp = globrunqget(m->p); |
692 » » if(gp) { | |
693 » » » while(gp->schedlink != nil) { | |
694 » » » » gp1 = gp; | |
695 » » » » gp = gp1->schedlink; | |
696 » » » » runqput(m->p, gp1); | |
697 » » » } | |
698 » » } | |
699 runtime·unlock(&runtime·sched); | 974 runtime·unlock(&runtime·sched); |
700 if(gp) | 975 if(gp) |
701 return gp; | 976 return gp; |
702 } | 977 } |
703 » for(i=0; i<runtime·gomaxprocs; i++) { | 978 » // If number of spinning M's >= number of busy P's, block. |
| 979 » // This is necessary to prevent excessive CPU consumption |
| 980 » // when GOMAXPROCS>>1 but the program parallelism is low. |
| 981 » if(!m->spinning && 2 * runtime·sched.nmspinning >= runtime·gomaxprocs -
runtime·sched.npidle) // TODO: fast atomic |
| 982 » » goto stop; |
| 983 » if(!m->spinning) { |
| 984 » » m->spinning = true; |
| 985 » » runtime·xadd(&runtime·sched.nmspinning, 1); |
| 986 » } |
| 987 » // random steal from other P's |
| 988 » for(i = 0; i < 2*runtime·gomaxprocs; i++) { |
704 if(runtime·gcwaiting) | 989 if(runtime·gcwaiting) |
705 goto top; | 990 goto top; |
706 p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]; | 991 p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]; |
707 if(p == m->p) | 992 if(p == m->p) |
708 gp = runqget(p); | 993 gp = runqget(p); |
709 else | 994 else |
710 gp = runqsteal(m->p, p); | 995 gp = runqsteal(m->p, p); |
711 if(gp) | 996 if(gp) |
712 return gp; | 997 return gp; |
713 } | 998 } |
714 » if(gp) | 999 stop: |
715 » » return gp; | 1000 » // return P and block |
716 runtime·lock(&runtime·sched); | 1001 runtime·lock(&runtime·sched); |
717 if(runtime·gcwaiting) { | 1002 if(runtime·gcwaiting) { |
718 runtime·unlock(&runtime·sched); | 1003 runtime·unlock(&runtime·sched); |
719 goto top; | 1004 goto top; |
720 } | 1005 } |
| 1006 if(runtime·sched.runqsize) { |
| 1007 gp = globrunqget(m->p); |
| 1008 runtime·unlock(&runtime·sched); |
| 1009 return gp; |
| 1010 } |
721 p = releasep(); | 1011 p = releasep(); |
722 pidleput(p); | 1012 pidleput(p); |
723 if(runtime·sched.runqsize) { | |
724 p = pidleget(); | |
725 runtime·unlock(&runtime·sched); | |
726 acquirep(m, p); | |
727 goto top; | |
728 } | |
729 runtime·unlock(&runtime·sched); | 1013 runtime·unlock(&runtime·sched); |
730 » for(i=0; i<runtime·gomaxprocs; i++) { | 1014 » if(m->spinning) { |
| 1015 » » m->spinning = false; |
| 1016 » » runtime·xadd(&runtime·sched.nmspinning, -1); |
| 1017 » } |
| 1018 » // check all runqueues once again |
| 1019 » for(i = 0; i < runtime·gomaxprocs; i++) { |
731 p = runtime·allp[i]; | 1020 p = runtime·allp[i]; |
732 if(p && p->runqhead != p->runqtail) { | 1021 if(p && p->runqhead != p->runqtail) { |
733 //!!! steal directly from this p | |
734 runtime·lock(&runtime·sched); | 1022 runtime·lock(&runtime·sched); |
735 p = pidleget(); | 1023 p = pidleget(); |
736 runtime·unlock(&runtime·sched); | 1024 runtime·unlock(&runtime·sched); |
737 if(p) { | 1025 if(p) { |
738 » » » » acquirep(m, p); | 1026 » » » » acquirep(p); |
739 goto top; | 1027 goto top; |
740 } | 1028 } |
741 break; | 1029 break; |
742 } | 1030 } |
743 } | 1031 } |
744 if (runtime·sched.poller == 0 && runtime·xchg(&runtime·sched.poller, 1)
== 0) { | |
745 G *gp2; | |
746 int32 w, n; | |
747 | |
748 gp2 = runtime·netwait(-1, 100); | |
749 runtime·atomicstore(&runtime·sched.poller, 0); | |
750 if (gp2) { | |
751 n = w = 0; | |
752 runtime·inject(gp2, &w, &n); | |
753 } | |
754 } | |
755 stopm(); | 1032 stopm(); |
756 goto top; | 1033 goto top; |
757 } | 1034 } |
758 | 1035 |
759 // One round of scheduler: find a goroutine and run it. | 1036 // One round of scheduler: find a runnable goroutine and execute it. |
760 // Never returns. | 1037 // Never returns. |
761 static void | 1038 static void |
762 schedule(void) | 1039 schedule(void) |
763 { | 1040 { |
764 G *gp; | 1041 G *gp; |
765 » P *p; | 1042 |
766 » M *mp; | 1043 » if(m->locks) |
767 | 1044 » » runtime·throw("schedule: holding locks"); |
768 » LOG("%d: schedule p=%p\n", m->id, m->p); | |
769 » USED(&gp); | |
770 » CHECK(m->locks == 0, ("schedule: holding locks\n")); | |
771 » CHECK(m->lockedg == nil, ("schedule: locked M\n")); | |
772 | 1045 |
773 top: | 1046 top: |
774 if(runtime·gcwaiting) { | 1047 if(runtime·gcwaiting) { |
775 gcstopm(); | 1048 gcstopm(); |
776 goto top; | 1049 goto top; |
777 } | 1050 } |
778 | 1051 |
779 gp = runqget(m->p); | 1052 gp = runqget(m->p); |
780 if(gp == nil) | 1053 if(gp == nil) |
781 gp = findrunnable(); | 1054 gp = findrunnable(); |
782 | 1055 |
| 1056 if(m->spinning) { |
| 1057 m->spinning = false; |
| 1058 runtime·xadd(&runtime·sched.nmspinning, -1); |
| 1059 } |
| 1060 |
| 1061 // M wakeup policy is deliberately somewhat conservative (see nmspinning
handling), |
| 1062 // so see if we need to wakeup another M here. |
| 1063 if (m->p->runqhead != m->p->runqtail && |
| 1064 runtime·sched.nmspinning == 0 && |
| 1065 runtime·sched.npidle > 0) // TODO: fast atomic |
| 1066 wakep(); |
| 1067 |
783 if(gp->lockedm) { | 1068 if(gp->lockedm) { |
784 » » //!!! rename to startlockedm | 1069 » » startlockedm(gp); |
785 » » CHECK(gp->lockedm != m, ("schedule: locked to myself")); | |
786 » » mp = gp->lockedm; | |
787 » » p = releasep(); | |
788 » » mp->lockedp = p; | |
789 » » // acquirep(mp, p); | |
790 » » runtime·notewakeup(&mp->park); | |
791 » » stopm(); | |
792 goto top; | 1070 goto top; |
793 } | 1071 } |
794 | 1072 |
795 execute(gp); | 1073 execute(gp); |
796 } | |
797 | |
798 // Suspends execution of gp that is locked to the current m. | |
799 // Never returns. | |
800 static void | |
801 waitlockedg(G *gp) | |
802 { | |
803 //!!! rename to stoplockedm | |
804 P *p; | |
805 | |
806 if(m->p) { | |
807 // Schedule another M to run this p. | |
808 p = releasep(); | |
809 startm(p); | |
810 } | |
811 // Wait until another thread schedules gp again. | |
812 runtime·notesleep(&m->park); | |
813 runtime·noteclear(&m->park); | |
814 CHECK(m->p == nil, ("waitlockedg: p != nil\n")); | |
815 CHECK(m->lockedp, ("waitlockedg: no lockedp\n")); | |
816 CHECK(gp->status == Grunnable, ("waitlockedg: bad gp status %d\n", gp->s
tatus)); | |
817 acquirep(m, m->lockedp); | |
818 m->lockedp = nil; | |
819 execute(gp); // Never returns. | |
820 } | |
821 | |
822 // runtime·park continuation on g0. | |
823 static void | |
824 park0(G *gp) | |
825 { | |
826 CHECK(m->lockedp == nil || !m->lockedg, ("waitlockedg: lockedp != nil\n"
)); | |
827 gp->status = Gwaiting; | |
828 gp->m = nil; | |
829 if(m->waitunlockf) { | |
830 m->waitunlockf(m->waitlock); | |
831 m->waitunlockf = nil; | |
832 } | |
833 if(m->lockedg) | |
834 waitlockedg(gp); // Never returns. | |
835 schedule(); | |
836 } | 1074 } |
837 | 1075 |
838 // Puts the current goroutine into a waiting state and unlocks the lock. | 1076 // Puts the current goroutine into a waiting state and unlocks the lock. |
839 // The goroutine can be made runnable again by calling runtime·ready(gp). | 1077 // The goroutine can be made runnable again by calling runtime·ready(gp). |
840 void | 1078 void |
841 runtime·park(void(*unlockf)(Lock*), Lock *lock, int8 *reason) | 1079 runtime·park(void(*unlockf)(Lock*), Lock *lock, int8 *reason) |
842 { | 1080 { |
843 LOG("%d: park l=%p reason=%s\n", m->id, lock, reason); | |
844 CHECK(g != m->g0, ("park of g0\n")); | |
845 m->waitlock = lock; | 1081 m->waitlock = lock; |
846 m->waitunlockf = unlockf; | 1082 m->waitunlockf = unlockf; |
847 g->waitreason = reason; | 1083 g->waitreason = reason; |
848 runtime·mcall(park0); | 1084 runtime·mcall(park0); |
849 } | 1085 } |
850 | 1086 |
| 1087 // runtime·park continuation on g0. |
| 1088 static void |
| 1089 park0(G *gp) |
| 1090 { |
| 1091 gp->status = Gwaiting; |
| 1092 gp->m = nil; |
| 1093 m->curg = nil; |
| 1094 if(m->waitunlockf) { |
| 1095 m->waitunlockf(m->waitlock); |
| 1096 m->waitunlockf = nil; |
| 1097 } |
| 1098 if(m->lockedg) { |
| 1099 stoplockedm(); |
| 1100 execute(gp); // Never returns. |
| 1101 } |
| 1102 schedule(); |
| 1103 } |
| 1104 |
| 1105 // Scheduler yield. |
| 1106 void |
| 1107 runtime·gosched(void) |
| 1108 { |
| 1109 runtime·mcall(gosched0); |
| 1110 } |
| 1111 |
851 // runtime·gosched continuation on g0. | 1112 // runtime·gosched continuation on g0. |
852 static void | 1113 static void |
853 gosched0(G *gp) | 1114 gosched0(G *gp) |
854 { | 1115 { |
855 LOG("%d: gosched0 gp=%p\n", m->id, gp); | |
856 gp->status = Grunnable; | 1116 gp->status = Grunnable; |
857 gp->m = nil; | 1117 gp->m = nil; |
858 » CHECK(m->lockedp == nil || !m->lockedg, ("waitlockedg: lockedp != nil\n"
)); | 1118 » m->curg = nil; |
859 runtime·lock(&runtime·sched); | 1119 runtime·lock(&runtime·sched); |
860 globrunqput(gp); | 1120 globrunqput(gp); |
861 runtime·unlock(&runtime·sched); | 1121 runtime·unlock(&runtime·sched); |
862 » if(m->lockedg) | 1122 » if(m->lockedg) { |
863 » » waitlockedg(gp); // Never returns. | 1123 » » stoplockedm(); |
| 1124 » » execute(gp); // Never returns. |
| 1125 » } |
864 schedule(); | 1126 schedule(); |
865 } | 1127 } |
866 | 1128 |
867 // Scheduler yield. | 1129 // Finishes execution of the current goroutine. |
868 void | 1130 void |
869 runtime·gosched(void) | 1131 runtime·goexit(void) |
870 { | 1132 { |
871 » runtime·mcall(gosched0); | 1133 » if(raceenabled) |
| 1134 » » runtime·racegoend(); |
| 1135 » runtime·mcall(goexit0); |
872 } | 1136 } |
873 | 1137 |
874 // runtime·goexit continuation on g0. | 1138 // runtime·goexit continuation on g0. |
875 static void | 1139 static void |
876 goexit0(G *gp) | 1140 goexit0(G *gp) |
877 { | 1141 { |
878 gp->status = Gdead; | 1142 gp->status = Gdead; |
879 gp->m = nil; | 1143 gp->m = nil; |
880 gp->lockedm = nil; | 1144 gp->lockedm = nil; |
| 1145 m->curg = nil; |
881 m->lockedg = nil; | 1146 m->lockedg = nil; |
882 runtime·unwindstack(gp, nil); | 1147 runtime·unwindstack(gp, nil); |
883 gfput(m->p, gp); | 1148 gfput(m->p, gp); |
884 schedule(); | 1149 schedule(); |
885 } | |
886 | |
887 // Finishes execution of the current goroutine. | |
888 void | |
889 runtime·goexit(void) | |
890 { | |
891 if(raceenabled) | |
892 runtime·racegoend(); | |
893 runtime·mcall(goexit0); | |
894 } | 1150 } |
895 | 1151 |
896 // The goroutine g is about to enter a system call. | 1152 // The goroutine g is about to enter a system call. |
897 // Record that it's not using the cpu anymore. | 1153 // Record that it's not using the cpu anymore. |
898 // This is called only from the go syscall library and cgocall, | 1154 // This is called only from the go syscall library and cgocall, |
899 // not from the low-level system calls used by the runtime. | 1155 // not from the low-level system calls used by the runtime. |
900 // | 1156 // |
901 // Entersyscall cannot split the stack: the runtime·gosave must | 1157 // Entersyscall cannot split the stack: the runtime·gosave must |
902 // make g->sched refer to the caller's stack segment, because | 1158 // make g->sched refer to the caller's stack segment, because |
903 // entersyscall is going to return immediately after. | 1159 // entersyscall is going to return immediately after. |
904 #pragma textflag 7 | 1160 #pragma textflag 7 |
905 void | 1161 void |
906 runtime·entersyscall(void) | 1162 ·entersyscall(int32 dummy) |
907 { | 1163 { |
908 » P *p; | |
909 | |
910 » //LOG("%d: entersyscall g=%p p=%p\n", m->id, g, m->p); | |
911 if(m->profilehz > 0) | 1164 if(m->profilehz > 0) |
912 runtime·setprof(false); | 1165 runtime·setprof(false); |
913 | 1166 |
914 // Leave SP around for gc and traceback. | 1167 // Leave SP around for gc and traceback. |
915 » runtime·gosave(&g->sched); | 1168 » g->sched.sp = (uintptr)runtime·getcallersp(&dummy); |
| 1169 » g->sched.pc = runtime·getcallerpc(&dummy); |
| 1170 » g->sched.g = g; |
916 g->gcsp = g->sched.sp; | 1171 g->gcsp = g->sched.sp; |
| 1172 g->gcpc = g->sched.pc; |
917 g->gcstack = g->stackbase; | 1173 g->gcstack = g->stackbase; |
918 g->gcguard = g->stackguard; | 1174 g->gcguard = g->stackguard; |
919 g->status = Gsyscall; | 1175 g->status = Gsyscall; |
920 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { | 1176 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { |
921 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", | 1177 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", |
922 // g->gcsp, g->gcguard-StackGuard, g->gcstack); | 1178 // g->gcsp, g->gcguard-StackGuard, g->gcstack); |
923 runtime·throw("entersyscall"); | 1179 runtime·throw("entersyscall"); |
924 } | 1180 } |
925 | 1181 |
926 » if(m->blockingsyscall) { | 1182 » if(runtime·sched.sysmonwait) { // TODO: fast atomic |
927 » » m->blockingsyscall = false; | 1183 » » runtime·lock(&runtime·sched); |
928 » » p = releasep(); | 1184 » » if(runtime·sched.sysmonwait) { |
929 » » startm(p); | 1185 » » » runtime·sched.sysmonwait = false; |
930 » » return; | 1186 » » » runtime·notewakeup(&runtime·sched.sysmonnote); |
| 1187 » » } |
| 1188 » » runtime·unlock(&runtime·sched); |
| 1189 » » runtime·gosave(&g->sched); // re-save for traceback |
931 } | 1190 } |
932 | 1191 |
933 m->mcache = nil; | 1192 m->mcache = nil; |
| 1193 m->p->tick++; |
934 m->p->m = nil; | 1194 m->p->m = nil; |
935 runtime·atomicstore(&m->p->status, Psyscall); | 1195 runtime·atomicstore(&m->p->status, Psyscall); |
936 if(runtime·gcwaiting) { | 1196 if(runtime·gcwaiting) { |
937 runtime·lock(&runtime·sched); | 1197 runtime·lock(&runtime·sched); |
938 if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psy
scall, Pgcstop)) { | 1198 if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psy
scall, Pgcstop)) { |
939 » » » runtime·sched.stopwait--; | 1199 » » » if(--runtime·sched.stopwait == 0) |
940 » » » if(runtime·sched.stopwait == 0) | |
941 runtime·notewakeup(&runtime·sched.stopnote); | 1200 runtime·notewakeup(&runtime·sched.stopnote); |
942 } | 1201 } |
943 runtime·unlock(&runtime·sched); | 1202 runtime·unlock(&runtime·sched); |
944 » } | 1203 » » runtime·gosave(&g->sched); // re-save for traceback |
945 } | 1204 » } |
946 | 1205 } |
| 1206 |
| 1207 // The same as runtime·entersyscall(), but with a hint that the syscall is block
ing. |
947 #pragma textflag 7 | 1208 #pragma textflag 7 |
948 void | 1209 void |
949 runtime·entersyscallblock(void) | 1210 ·entersyscallblock(int32 dummy) |
950 { | |
951 » m->blockingsyscall = true; | |
952 » runtime·entersyscall(); | |
953 } | |
954 | |
955 // runtime·exitsyscall slow path on g0. | |
956 // Failed to acquire P, enqueue gp as runnable. | |
957 static void | |
958 exitsyscall0(G *gp) | |
959 { | 1211 { |
960 P *p; | 1212 P *p; |
961 | 1213 |
962 » LOG("%d: exitsyscall0\n", m->id); | 1214 » if(m->profilehz > 0) |
963 » gp->status = Grunnable; | 1215 » » runtime·setprof(false); |
964 » gp->m = nil; | 1216 |
965 » CHECK(m->park.waitm == nil, ("exitsyscall0: park is signalled\n")); | 1217 » // Leave SP around for gc and traceback. |
966 » runtime·lock(&runtime·sched); | 1218 » g->sched.sp = (uintptr)runtime·getcallersp(&dummy); |
967 » p = pidleget(); | 1219 » g->sched.pc = runtime·getcallerpc(&dummy); |
968 » if(p == nil) | 1220 » g->sched.g = g; |
969 » » globrunqput(gp); | 1221 » g->gcsp = g->sched.sp; |
970 » runtime·unlock(&runtime·sched); | 1222 » g->gcpc = g->sched.pc; |
971 » if(p) { | 1223 » g->gcstack = g->stackbase; |
972 » » acquirep(m, p); | 1224 » g->gcguard = g->stackguard; |
973 » » execute(gp); // Never returns. | 1225 » g->status = Gsyscall; |
974 » } | 1226 » if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { |
975 » if(m->lockedg) { | 1227 » » // runtime·printf("entersyscallblock inconsistent %p [%p,%p]\n", |
976 » » CHECK(m->lockedg == gp, ("exitsyscall0: inconsistent locking\n")
); | 1228 » » //» g->gcsp, g->gcguard-StackGuard, g->gcstack); |
977 » » // Wait until another thread schedules gp and so m again. | 1229 » » runtime·throw("entersyscallblock"); |
978 » » waitlockedg(gp); // Never returns. | 1230 » } |
979 » } | 1231 |
980 » stopm(); | 1232 » p = releasep(); |
981 » schedule(); // Never returns. | 1233 » handoffp(p); |
| 1234 » if(g == scvg) // do not consider blocked scavenger for deadlock detecti
on |
| 1235 » » inclocked(1); |
| 1236 » runtime·gosave(&g->sched); // re-save for traceback |
982 } | 1237 } |
983 | 1238 |
984 // The goroutine g exited its system call. | 1239 // The goroutine g exited its system call. |
985 // Arrange for it to run on a cpu again. | 1240 // Arrange for it to run on a cpu again. |
986 // This is called only from the go syscall library, not | 1241 // This is called only from the go syscall library, not |
987 // from the low-level system calls used by the runtime. | 1242 // from the low-level system calls used by the runtime. |
988 void | 1243 void |
989 runtime·exitsyscall(void) | 1244 runtime·exitsyscall(void) |
990 { | 1245 { |
991 uint32 s; | |
992 P *p; | 1246 P *p; |
993 | |
994 LOG("%d: exitsyscall g=%p\n", m->id, g); | |
995 | 1247 |
996 // Check whether the profiler needs to be turned on. | 1248 // Check whether the profiler needs to be turned on. |
997 if(m->profilehz > 0) | 1249 if(m->profilehz > 0) |
998 runtime·setprof(true); | 1250 runtime·setprof(true); |
999 | 1251 |
1000 // Try to re-acquire the last P. | 1252 // Try to re-acquire the last P. |
1001 » s = m->p ? m->p->status : Pidle; | 1253 » if(m->p && m->p->status == Psyscall && runtime·cas(&m->p->status, Psysca
ll, Prunning)) { |
1002 » if(s == Psyscall && runtime·cas(&m->p->status, s, Prunning)) { | |
1003 » » LOG("%d: exitsyscall fast\n", m->id); | |
1004 // There's a cpu for us, so we can run. | 1254 // There's a cpu for us, so we can run. |
1005 m->mcache = m->p->mcache; | 1255 m->mcache = m->p->mcache; |
1006 m->p->m = m; | 1256 m->p->m = m; |
1007 m->p->tick++; | 1257 m->p->tick++; |
1008 g->status = Grunning; | 1258 g->status = Grunning; |
1009 // Garbage collector isn't running (since we are), | 1259 // Garbage collector isn't running (since we are), |
1010 » » // so okay to clear gcstack. | 1260 » » // so okay to clear gcstack and gcsp. |
1011 g->gcstack = (uintptr)nil; | 1261 g->gcstack = (uintptr)nil; |
| 1262 g->gcsp = (uintptr)nil; |
1012 return; | 1263 return; |
1013 } | 1264 } |
1014 | 1265 |
| 1266 if(g == scvg) // do not consider blocked scavenger for deadlock detecti
on |
| 1267 inclocked(-1); |
1015 // Try to get any other idle P. | 1268 // Try to get any other idle P. |
1016 m->p = nil; | 1269 m->p = nil; |
1017 if(runtime·sched.pidle) { | 1270 if(runtime·sched.pidle) { |
1018 runtime·lock(&runtime·sched); | 1271 runtime·lock(&runtime·sched); |
1019 p = pidleget(); | 1272 p = pidleget(); |
1020 runtime·unlock(&runtime·sched); | 1273 runtime·unlock(&runtime·sched); |
1021 if(p) { | 1274 if(p) { |
1022 » » » acquirep(m, p); | 1275 » » » acquirep(p); |
1023 g->gcstack = (uintptr)nil; | 1276 g->gcstack = (uintptr)nil; |
| 1277 g->gcsp = (uintptr)nil; |
1024 return; | 1278 return; |
1025 } | 1279 } |
1026 } | 1280 } |
1027 | 1281 |
1028 // Call the scheduler. | 1282 // Call the scheduler. |
1029 LOG("%d: exitsyscall slow p->status=%d\n", m->id, s); | |
1030 runtime·mcall(exitsyscall0); | 1283 runtime·mcall(exitsyscall0); |
1031 | 1284 |
1032 // Scheduler returned, so we're allowed to run now. | 1285 // Scheduler returned, so we're allowed to run now. |
1033 // Delete the gcstack information that we left for | 1286 // Delete the gcstack information that we left for |
1034 // the garbage collector during the system call. | 1287 // the garbage collector during the system call. |
1035 // Must wait until now because until gosched returns | 1288 // Must wait until now because until gosched returns |
1036 // we don't know for sure that the garbage collector | 1289 // we don't know for sure that the garbage collector |
1037 // is not running. | 1290 // is not running. |
1038 g->gcstack = (uintptr)nil; | 1291 g->gcstack = (uintptr)nil; |
| 1292 g->gcsp = (uintptr)nil; |
| 1293 } |
| 1294 |
| 1295 // runtime·exitsyscall slow path on g0. |
| 1296 // Failed to acquire P, enqueue gp as runnable. |
| 1297 static void |
| 1298 exitsyscall0(G *gp) |
| 1299 { |
| 1300 P *p; |
| 1301 |
| 1302 gp->status = Grunnable; |
| 1303 gp->m = nil; |
| 1304 m->curg = nil; |
| 1305 runtime·lock(&runtime·sched); |
| 1306 p = pidleget(); |
| 1307 if(p == nil) |
| 1308 globrunqput(gp); |
| 1309 runtime·unlock(&runtime·sched); |
| 1310 if(p) { |
| 1311 acquirep(p); |
| 1312 execute(gp); // Never returns. |
| 1313 } |
| 1314 if(m->lockedg) { |
| 1315 // Wait until another thread schedules gp and so m again. |
| 1316 stoplockedm(); |
| 1317 execute(gp); // Never returns. |
| 1318 } |
| 1319 stopm(); |
| 1320 schedule(); // Never returns. |
1039 } | 1321 } |
1040 | 1322 |
1041 // Hook used by runtime·malg to call runtime·stackalloc on the | 1323 // Hook used by runtime·malg to call runtime·stackalloc on the |
1042 // scheduler stack. This exists because runtime·stackalloc insists | 1324 // scheduler stack. This exists because runtime·stackalloc insists |
1043 // on being called on the scheduler stack, to avoid trying to grow | 1325 // on being called on the scheduler stack, to avoid trying to grow |
1044 // the stack while allocating a new stack segment. | 1326 // the stack while allocating a new stack segment. |
1045 static void | 1327 static void |
1046 mstackalloc(G *gp) | 1328 mstackalloc(G *gp) |
1047 { | 1329 { |
1048 gp->param = runtime·stackalloc((uintptr)gp->param); | 1330 gp->param = runtime·stackalloc((uintptr)gp->param); |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1083 | 1365 |
1084 // Create a new g running fn with siz bytes of arguments. | 1366 // Create a new g running fn with siz bytes of arguments. |
1085 // Put it on the queue of g's waiting to run. | 1367 // Put it on the queue of g's waiting to run. |
1086 // The compiler turns a go statement into a call to this. | 1368 // The compiler turns a go statement into a call to this. |
1087 // Cannot split the stack because it assumes that the arguments | 1369 // Cannot split the stack because it assumes that the arguments |
1088 // are available sequentially after &fn; they would not be | 1370 // are available sequentially after &fn; they would not be |
1089 // copied if a stack split occurred. It's OK for this to call | 1371 // copied if a stack split occurred. It's OK for this to call |
1090 // functions that split the stack. | 1372 // functions that split the stack. |
1091 #pragma textflag 7 | 1373 #pragma textflag 7 |
1092 void | 1374 void |
1093 runtime·newproc(int32 siz, byte* fn, ...) | 1375 runtime·newproc(int32 siz, FuncVal* fn, ...) |
1094 { | 1376 { |
1095 byte *argp; | 1377 byte *argp; |
1096 | 1378 |
1097 if(thechar == '5') | 1379 if(thechar == '5') |
1098 argp = (byte*)(&fn+2); // skip caller's saved LR | 1380 argp = (byte*)(&fn+2); // skip caller's saved LR |
1099 else | 1381 else |
1100 argp = (byte*)(&fn+1); | 1382 argp = (byte*)(&fn+1); |
1101 runtime·newproc1(fn, argp, siz, 0, runtime·getcallerpc(&siz)); | 1383 runtime·newproc1(fn, argp, siz, 0, runtime·getcallerpc(&siz)); |
1102 } | 1384 } |
1103 | 1385 |
1104 // Create a new g running fn with narg bytes of arguments starting | 1386 // Create a new g running fn with narg bytes of arguments starting |
1105 // at argp and returning nret bytes of results. callerpc is the | 1387 // at argp and returning nret bytes of results. callerpc is the |
1106 // address of the go statement that created this. The new g is put | 1388 // address of the go statement that created this. The new g is put |
1107 // on the queue of g's waiting to run. | 1389 // on the queue of g's waiting to run. |
1108 G* | 1390 G* |
1109 runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) | 1391 runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerpc
) |
1110 { | 1392 { |
1111 byte *sp; | 1393 byte *sp; |
1112 G *newg; | 1394 G *newg; |
1113 int32 siz; | 1395 int32 siz; |
1114 | 1396 |
1115 //printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret); | 1397 //printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret); |
1116 siz = narg + nret; | 1398 siz = narg + nret; |
1117 siz = (siz+7) & ~7; | 1399 siz = (siz+7) & ~7; |
1118 | 1400 |
1119 // We could instead create a secondary stack frame | 1401 // We could instead create a secondary stack frame |
1120 // and make it look like goexit was on the original but | 1402 // and make it look like goexit was on the original but |
1121 // the call to the actual goroutine function was split. | 1403 // the call to the actual goroutine function was split. |
1122 // Not worth it: this is almost always an error. | 1404 // Not worth it: this is almost always an error. |
1123 if(siz > StackMin - 1024) | 1405 if(siz > StackMin - 1024) |
1124 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); | 1406 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); |
1125 | 1407 |
1126 if((newg = gfget(m->p)) != nil) { | 1408 if((newg = gfget(m->p)) != nil) { |
1127 if(newg->stackguard - StackGuard != newg->stack0) | 1409 if(newg->stackguard - StackGuard != newg->stack0) |
1128 runtime·throw("invalid stack in newg"); | 1410 runtime·throw("invalid stack in newg"); |
1129 } else { | 1411 } else { |
1130 newg = runtime·malg(StackMin); | 1412 newg = runtime·malg(StackMin); |
1131 runtime·lock(&runtime·sched); | 1413 runtime·lock(&runtime·sched); |
1132 newg->goid = ++runtime·sched.goidseq; | |
1133 if(runtime·lastg == nil) | 1414 if(runtime·lastg == nil) |
1134 runtime·allg = newg; | 1415 runtime·allg = newg; |
1135 else | 1416 else |
1136 runtime·lastg->alllink = newg; | 1417 runtime·lastg->alllink = newg; |
1137 runtime·lastg = newg; | 1418 runtime·lastg = newg; |
1138 runtime·unlock(&runtime·sched); | 1419 runtime·unlock(&runtime·sched); |
1139 } | 1420 } |
1140 | 1421 |
1141 sp = (byte*)newg->stackbase; | 1422 sp = (byte*)newg->stackbase; |
1142 sp -= siz; | 1423 sp -= siz; |
1143 runtime·memmove(sp, argp, narg); | 1424 runtime·memmove(sp, argp, narg); |
1144 if(thechar == '5') { | 1425 if(thechar == '5') { |
1145 // caller's LR | 1426 // caller's LR |
1146 sp -= sizeof(void*); | 1427 sp -= sizeof(void*); |
1147 *(void**)sp = nil; | 1428 *(void**)sp = nil; |
1148 } | 1429 } |
1149 | 1430 |
1150 LOG("%d: newproc %p\n", m->id, newg); | |
1151 newg->sched.sp = (uintptr)sp; | 1431 newg->sched.sp = (uintptr)sp; |
1152 newg->sched.pc = (byte*)runtime·goexit; | 1432 newg->sched.pc = (byte*)runtime·goexit; |
1153 newg->sched.g = newg; | 1433 newg->sched.g = newg; |
1154 » newg->entry = fn; | 1434 » newg->fnstart = fn; |
1155 newg->gopc = (uintptr)callerpc; | 1435 newg->gopc = (uintptr)callerpc; |
1156 newg->status = Grunnable; | 1436 newg->status = Grunnable; |
| 1437 newg->goid = runtime·xadd64(&runtime·sched.goidgen, 1); |
1157 if(raceenabled) | 1438 if(raceenabled) |
1158 newg->racectx = runtime·racegostart(callerpc); | 1439 newg->racectx = runtime·racegostart(callerpc); |
1159 runqput(m->p, newg); | 1440 runqput(m->p, newg); |
1160 | 1441 |
1161 » if(runtime·sched.pidle && fn != (byte*)runtime·main) | 1442 » if(runtime·sched.npidle != 0 && runtime·sched.nmspinning == 0 && fn->fn
!= runtime·main) // TODO: fast atomic |
1162 » » startm(nil); | 1443 » » wakep(); |
1163 return newg; | 1444 return newg; |
1164 } | |
1165 | |
1166 void | |
1167 runtime·Breakpoint(void) | |
1168 { | |
1169 runtime·breakpoint(); | |
1170 } | |
1171 | |
1172 void | |
1173 runtime·Gosched(void) | |
1174 { | |
1175 runtime·gosched(); | |
1176 } | |
1177 | |
1178 // Implementation of runtime.GOMAXPROCS. | |
1179 int32 | |
1180 runtime·gomaxprocsfunc(int32 n) | |
1181 { | |
1182 int32 ret; | |
1183 | |
1184 LOG("%d: gomaxprocsfunc %d\n", m->id, n); | |
1185 if(n > maxgomaxprocs) | |
1186 n = maxgomaxprocs; | |
1187 runtime·lock(&runtime·sched); | |
1188 ret = runtime·gomaxprocs; | |
1189 if(n <= 0 || n == ret) { | |
1190 runtime·unlock(&runtime·sched); | |
1191 return ret; | |
1192 } | |
1193 runtime·unlock(&runtime·sched); | |
1194 | |
1195 runtime·semacquire(&runtime·worldsema); | |
1196 m->gcing = 1; | |
1197 runtime·stoptheworld(); | |
1198 newprocs = n; | |
1199 m->gcing = 0; | |
1200 runtime·semrelease(&runtime·worldsema); | |
1201 runtime·starttheworld(); | |
1202 | |
1203 return ret; | |
1204 } | |
1205 | |
1206 static void | |
1207 LockOSThread(void) | |
1208 { | |
1209 m->lockedg = g; | |
1210 g->lockedm = m; | |
1211 } | |
1212 | |
1213 void | |
1214 runtime·LockOSThread(void) | |
1215 { | |
1216 m->locked |= LockExternal; | |
1217 LockOSThread(); | |
1218 } | |
1219 | |
1220 void | |
1221 runtime·lockOSThread(void) | |
1222 { | |
1223 m->locked += LockInternal; | |
1224 LockOSThread(); | |
1225 } | |
1226 | |
1227 static void | |
1228 UnlockOSThread(void) | |
1229 { | |
1230 if(m->locked != 0) | |
1231 return; | |
1232 m->lockedg = nil; | |
1233 g->lockedm = nil; | |
1234 }······· | |
1235 | |
1236 void | |
1237 runtime·UnlockOSThread(void) | |
1238 { | |
1239 m->locked &= ~LockExternal; | |
1240 UnlockOSThread(); | |
1241 } | |
1242 | |
1243 void | |
1244 runtime·unlockOSThread(void) | |
1245 { | |
1246 if(m->locked < LockInternal) | |
1247 runtime·throw("runtime: internal error: misuse of lockOSThread/u
nlockOSThread"); | |
1248 m->locked -= LockInternal; | |
1249 UnlockOSThread(); | |
1250 } | |
1251 | |
1252 bool | |
1253 runtime·lockedOSThread(void) | |
1254 { | |
1255 return g->lockedm != nil && m->lockedg != nil; | |
1256 } | |
1257 | |
1258 // for testing of callbacks | |
1259 void | |
1260 runtime·golockedOSThread(bool ret) | |
1261 { | |
1262 ret = runtime·lockedOSThread(); | |
1263 FLUSH(&ret); | |
1264 } | |
1265 | |
1266 // for testing of wire, unwire | |
1267 void | |
1268 runtime·mid(uint32 ret) | |
1269 { | |
1270 ret = m->id; | |
1271 FLUSH(&ret); | |
1272 } | |
1273 | |
1274 void | |
1275 runtime·NumGoroutine(intgo ret) | |
1276 { | |
1277 // TODO(dvyukov): implement me plz | |
1278 //ret = runtime·sched.gcount; | |
1279 ret = 1; | |
1280 FLUSH(&ret); | |
1281 } | |
1282 | |
1283 int32 | |
1284 runtime·gcount(void) | |
1285 { | |
1286 // TODO(dvyukov): implement me plz | |
1287 //return runtime·sched.gcount; | |
1288 return 1; | |
1289 } | |
1290 | |
1291 int32 | |
1292 runtime·mcount(void) | |
1293 { | |
1294 return runtime·sched.mcount; | |
1295 } | |
1296 | |
1297 void | |
1298 runtime·badmcall(void) // called from assembly | |
1299 { | |
1300 runtime·throw("runtime: mcall called on m->g0 stack"); | |
1301 } | |
1302 | |
1303 void | |
1304 runtime·badmcall2(void) // called from assembly | |
1305 { | |
1306 runtime·throw("runtime: mcall function returned"); | |
1307 } | |
1308 | |
1309 static struct { | |
1310 Lock; | |
1311 void (*fn)(uintptr*, int32); | |
1312 int32 hz; | |
1313 uintptr pcbuf[100]; | |
1314 } prof; | |
1315 | |
1316 // Called if we receive a SIGPROF signal. | |
1317 void | |
1318 runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp) | |
1319 { | |
1320 int32 n; | |
1321 | |
1322 if (m == nil || m->mcache == nil) | |
1323 return; | |
1324 if(prof.fn == nil || prof.hz == 0) | |
1325 return; | |
1326 | |
1327 runtime·lock(&prof); | |
1328 if(prof.fn == nil) { | |
1329 runtime·unlock(&prof); | |
1330 return; | |
1331 } | |
1332 n = runtime·gentraceback(pc, sp, lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf
)); | |
1333 if(n > 0) | |
1334 prof.fn(prof.pcbuf, n); | |
1335 runtime·unlock(&prof); | |
1336 } | |
1337 | |
1338 // Arrange to call fn with a traceback hz times a second. | |
1339 void | |
1340 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) | |
1341 { | |
1342 // Force sane arguments. | |
1343 if(hz < 0) | |
1344 hz = 0; | |
1345 if(hz == 0) | |
1346 fn = nil; | |
1347 if(fn == nil) | |
1348 hz = 0; | |
1349 | |
1350 // Stop profiler on this cpu so that it is safe to lock prof. | |
1351 // if a profiling signal came in while we had prof locked, | |
1352 // it would deadlock. | |
1353 runtime·resetcpuprofiler(0); | |
1354 | |
1355 runtime·lock(&prof); | |
1356 prof.fn = fn; | |
1357 prof.hz = hz; | |
1358 runtime·unlock(&prof); | |
1359 runtime·lock(&runtime·sched); | |
1360 runtime·sched.profilehz = hz; | |
1361 runtime·unlock(&runtime·sched); | |
1362 | |
1363 if(hz != 0) | |
1364 runtime·resetcpuprofiler(hz); | |
1365 } | |
1366 | |
1367 // Change number of processors. The world is stopped. | |
1368 static void | |
1369 procresize(int32 new) | |
1370 { | |
1371 int32 i, old; | |
1372 G *gp; | |
1373 P *p; | |
1374 | |
1375 runtime·lock(&runtime·sched); //!!! | |
1376 old = runtime·gomaxprocs; | |
1377 LOG("%d: procresize %d->%d\n", m->id, old, new); | |
1378 if(old < 0 || old > maxgomaxprocs || new <= 0 || new > maxgomaxprocs) | |
1379 runtime·throw("procresize: invalid arg"); | |
1380 if(old == new) { | |
1381 for(i=0; i<new; i++) { | |
1382 p = runtime·allp[i]; | |
1383 if(p == m->p) | |
1384 p->status = Prunning; | |
1385 else { | |
1386 p->status = Pidle; | |
1387 pidleput(p); | |
1388 } | |
1389 } | |
1390 runtime·unlock(&runtime·sched); | |
1391 return; | |
1392 } | |
1393 | |
1394 runtime·singleproc = new == 1; | |
1395 runtime·gomaxprocs = new; | |
1396 for(i=0; i<new; i++) { | |
1397 p = runtime·allp[i]; | |
1398 if(p == nil) { | |
1399 p = (P*)runtime·mallocgc(sizeof(runtime·allp[i][0]), 0,
0, 1); | |
1400 p->status = Pgcstop; | |
1401 runtime·allp[i] = p; //@@@ store-release | |
1402 } | |
1403 if(p->mcache == nil) { | |
1404 if(old==0 && i==0) | |
1405 p->mcache = m->mcache; | |
1406 else | |
1407 p->mcache = runtime·allocmcache(); | |
1408 } | |
1409 if(p->runq == nil) { | |
1410 p->runqsize = 1024; | |
1411 p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*),
0, 0, 1); | |
1412 } | |
1413 } | |
1414 | |
1415 for(i=1; i<old; i++) { | |
1416 for(;;) { | |
1417 gp = runqget(runtime·allp[i]); | |
1418 if(gp == nil) | |
1419 break; | |
1420 //TODO: spread more evenly. | |
1421 runqput(runtime·allp[0], gp); | |
1422 } | |
1423 } | |
1424 | |
1425 for(i=new; i<old; i++) { | |
1426 runtime·freemcache(runtime·allp[i]->mcache); | |
1427 runtime·allp[i]->mcache = nil; | |
1428 runtime·allp[i]->status = Pdead; | |
1429 //TODO: free freeg | |
1430 } | |
1431 | |
1432 if(m->p) | |
1433 m->p->m = nil; | |
1434 m->p = nil; | |
1435 m->mcache = nil; | |
1436 runtime·allp[0]->m = nil; | |
1437 runtime·allp[0]->status = Pidle; | |
1438 acquirep(m, runtime·allp[0]); | |
1439 for(i=1; i<new; i++) { | |
1440 p = runtime·allp[i]; | |
1441 p->status = Pidle; | |
1442 pidleput(p); | |
1443 } | |
1444 runtime·unlock(&runtime·sched); | |
1445 } | |
1446 | |
1447 // Associate p and mp. | |
1448 static void | |
1449 acquirep(M *mp, P *p) | |
1450 { | |
1451 LOG("%d: acquirep m=%d p=%p p->m=%p, p->status=%d, p->mcache=%p\n", m->i
d, mp->id, p, p->m, p->status, p->mcache); | |
1452 if(mp->p || mp->mcache) | |
1453 runtime·throw("acquirep: already in go"); | |
1454 if(p->m || p->status != Pidle) { | |
1455 runtime·printf("acquirep: p->m=%p(%d) p->status=%d\n", p->m, p->
m ? p->m->id : 0, p->status); | |
1456 runtime·throw("acquirep: invalid p state"); | |
1457 } | |
1458 mp->mcache = p->mcache; | |
1459 mp->p = p; | |
1460 p->m = mp; | |
1461 p->status = Prunning; | |
1462 } | |
1463 | |
1464 // Disassociate p and the current m. | |
1465 static P* | |
1466 releasep(void) | |
1467 { | |
1468 M *mp; | |
1469 P *p; | |
1470 | |
1471 mp = m; | |
1472 LOG("%d: releasep\n", mp->id); | |
1473 // sched is locked | |
1474 if(mp->p == nil || mp->mcache == nil) | |
1475 runtime·throw("releasep: invalid arg"); | |
1476 p = mp->p; | |
1477 if(p->m != mp || p->mcache != mp->mcache || p->status != Prunning) { | |
1478 runtime·printf("releasep: m=%p m->p=%p p->m=%p m->mcache=%p p->m
cache=%p p->status=%d\n", | |
1479 mp, mp->p, p->m, m->mcache, p->mcache, p->status); | |
1480 runtime·throw("releasep: invalid p state"); | |
1481 } | |
1482 mp->p = nil; | |
1483 mp->mcache = nil; | |
1484 p->m = nil; | |
1485 p->status = Pidle; | |
1486 return p; | |
1487 } | |
1488 | |
1489 typedef struct Pdesc Pdesc; | |
1490 struct Pdesc | |
1491 { | |
1492 uint32 tick; | |
1493 int64 when; | |
1494 }; | |
1495 | |
1496 static void | |
1497 retake(int64 now, Pdesc *ps) | |
1498 { | |
1499 uint32 i, s; | |
1500 int64 t; | |
1501 P *p; | |
1502 | |
1503 for(i=0; i<runtime·gomaxprocs; i++) { | |
1504 p = runtime·allp[i]; | |
1505 //!!! procresize may be in progress | |
1506 // do something if GC is in progress (help). | |
1507 if(p==nil) | |
1508 continue; | |
1509 t = p->tick; | |
1510 if(ps[i].tick != t) { | |
1511 ps[i].tick = t; | |
1512 ps[i].when = now; | |
1513 } | |
1514 if(ps[i].when + 20*1000 > now) | |
1515 continue; | |
1516 s = p->status; | |
1517 if(s == Psyscall && runtime·cas(&p->status, s, Pidle)) { | |
1518 LOG("retake %p(%d)\n", p, i); | |
1519 startm(p); | |
1520 } | |
1521 } | |
1522 } | |
1523 | |
1524 static Pdesc ps[maxgomaxprocs]; | |
1525 | |
1526 static void | |
1527 sysmon(void) | |
1528 { | |
1529 int64 now; | |
1530 | |
1531 // This is a special dedicated thread. | |
1532 // It works w/o mcache nor stackalloc, it may work concurrently with GC. | |
1533 runtime·asminit(); | |
1534 runtime·minit(); | |
1535 LOG("sysmon\n"); | |
1536 for(;;) { | |
1537 //!!! become poller if it's not polled for too long | |
1538 //!!! if no running P's -> block | |
1539 //!!! if do not wake anybody for some time -> sleep longer | |
1540 runtime·usleep(20); | |
1541 now = runtime·nanotime(); | |
1542 if(runtime·gcwaiting) { | |
1543 runtime·lock(&runtime·sched); | |
1544 if(runtime·gcwaiting) { | |
1545 runtime·sched.sysmonwait = 1; | |
1546 runtime·unlock(&runtime·sched); | |
1547 runtime·notesleep(&runtime·sched.sysmonnote); | |
1548 runtime·noteclear(&runtime·sched.sysmonnote); | |
1549 } else | |
1550 runtime·unlock(&runtime·sched); | |
1551 } | |
1552 retake(now, ps); | |
1553 } | |
1554 } | |
1555 | |
1556 void | |
1557 runtime·inject(G *gp0, int32 *w, int32 *n) | |
1558 { | |
1559 int32 nw; | |
1560 G *gp; | |
1561 | |
1562 runtime·lock(&runtime·sched); | |
1563 while(gp0) { | |
1564 gp = gp0; | |
1565 gp0 = gp->schedlink; | |
1566 gp->status = Grunnable; | |
1567 globrunqput(gp); | |
1568 (*n)++; | |
1569 } | |
1570 LOG("%d: inject %d\n", m->id, *n); | |
1571 runtime·unlock(&runtime·sched); | |
1572 | |
1573 nw = *n; | |
1574 while(runtime·sched.pidle && nw) { | |
1575 if(!startm(nil)) | |
1576 break; | |
1577 (*w)++; | |
1578 nw--; | |
1579 } | |
1580 } | |
1581 | |
1582 // Put gp on the global runnable queue. | |
1583 // Sched must be locked. | |
1584 static void | |
1585 globrunqput(G *gp) | |
1586 { | |
1587 gp->schedlink = nil; | |
1588 if(runtime·sched.runqtail) | |
1589 runtime·sched.runqtail->schedlink = gp; | |
1590 else | |
1591 runtime·sched.runqhead = gp; | |
1592 runtime·sched.runqtail = gp; | |
1593 runtime·sched.runqsize++; | |
1594 } | |
1595 | |
1596 // Try get a batch of G's from the global runnable queue. | |
1597 // Sched must be locked. | |
1598 static G* | |
1599 globrunqget(void) | |
1600 { | |
1601 G *gp, *gp1; | |
1602 int32 n; | |
1603 | |
1604 if(runtime·sched.runqsize == 0) | |
1605 return nil; | |
1606 n = runtime·sched.runqsize/runtime·gomaxprocs+1; | |
1607 if(n > runtime·sched.runqsize) | |
1608 n = runtime·sched.runqsize; | |
1609 runtime·sched.runqsize -= n; | |
1610 if(runtime·sched.runqsize == 0) | |
1611 runtime·sched.runqtail = nil; | |
1612 gp1 = nil; | |
1613 while(n--) { | |
1614 gp = runtime·sched.runqhead; | |
1615 runtime·sched.runqhead = gp->schedlink; | |
1616 gp->schedlink = gp1; | |
1617 gp1 = gp; | |
1618 } | |
1619 return gp1; | |
1620 } | |
1621 | |
1622 // Put g on local runnable queue. | |
1623 static void | |
1624 runqput(P *p, G *gp) | |
1625 { | |
1626 int32 h, t, s; | |
1627 | |
1628 CHECK(m->mcache, ("runqput: no mcache\n")); | |
1629 runtime·lock(p); | |
1630 retry: | |
1631 h = p->runqhead; | |
1632 t = p->runqtail; | |
1633 s = p->runqsize; | |
1634 if(t==h-1 || (h==0 && t==s-1)) { | |
1635 runqgrow(p); | |
1636 goto retry; | |
1637 } | |
1638 p->runq[t] = gp; | |
1639 t++; | |
1640 if(t==s) | |
1641 t = 0; | |
1642 p->runqtail = t; | |
1643 runtime·unlock(p); | |
1644 } | |
1645 | |
1646 // Get g from local runnable queue. | |
1647 static G* | |
1648 runqget(P *p) | |
1649 { | |
1650 G *gp; | |
1651 int32 t, h, s; | |
1652 | |
1653 if(p->runqhead==p->runqtail) | |
1654 return nil; | |
1655 runtime·lock(p); | |
1656 h = p->runqhead; | |
1657 t = p->runqtail; | |
1658 s = p->runqsize; | |
1659 if(t==h) { | |
1660 runtime·unlock(p); | |
1661 return nil; | |
1662 } | |
1663 gp = p->runq[h]; | |
1664 h++; | |
1665 if(h==s) | |
1666 h = 0; | |
1667 p->runqhead = h; | |
1668 runtime·unlock(p); | |
1669 return gp; | |
1670 } | |
1671 | |
1672 // Grow local runnable queue. | |
1673 // TODO(dvyukov): consider using fixed-size array | |
1674 // and transfer excess to the global list (local queue can grow way too big). | |
1675 static void | |
1676 runqgrow(P *p) | |
1677 { | |
1678 G **q; | |
1679 int32 s, t, h, t2; | |
1680 | |
1681 h = p->runqhead; | |
1682 t = p->runqtail; | |
1683 s = p->runqsize; | |
1684 t2 = 0; | |
1685 q = (G**)runtime·malloc(2*s*sizeof(*q)); | |
1686 while(t!=h) { | |
1687 q[t2] = p->runq[h]; | |
1688 t2++; | |
1689 h++; | |
1690 if(h==s) | |
1691 h = 0; | |
1692 } | |
1693 runtime·free(p->runq); | |
1694 p->runq = q; | |
1695 p->runqhead = 0; | |
1696 p->runqtail = t2; | |
1697 p->runqsize = 2*s; | |
1698 } | |
1699 | |
1700 // Steal half of elements from local runnable queue of p2 | |
1701 // and put onto local runnable queue of p. | |
1702 // Returns one of the stolen elements (or nil if failed). | |
1703 static G* | |
1704 runqsteal(P *p, P *p2) | |
1705 { | |
1706 G *gp, *gp1; | |
1707 int32 t, h, s, t2, h2, s2, c, c1; | |
1708 | |
1709 if(p2->runqhead==p2->runqtail) | |
1710 return nil; | |
1711 if(p < p2) | |
1712 runtime·lock(p); | |
1713 runtime·lock(p2); | |
1714 if(p2->runqhead==p2->runqtail) { | |
1715 runtime·unlock(p2); | |
1716 if(p < p2) | |
1717 runtime·unlock(p); | |
1718 return nil; | |
1719 } | |
1720 if(p >= p2) | |
1721 runtime·lock(p); | |
1722 h = p->runqhead; | |
1723 t = p->runqtail; | |
1724 s = p->runqsize; | |
1725 h2 = p2->runqhead; | |
1726 t2 = p2->runqtail; | |
1727 s2 = p2->runqsize; | |
1728 gp = p2->runq[h2]; | |
1729 h2++; | |
1730 if(h2 == s2) | |
1731 h2 = 0; | |
1732 if(t2 > h2) | |
1733 c = (t2 - h2) / 2; | |
1734 else | |
1735 c = (s2 - h2 + t2) / 2; | |
1736 c1 = 0; | |
1737 for(;;) { | |
1738 if(c1 == c) | |
1739 break; | |
1740 if(t==h-1 || (h==0 && t==s-1)) | |
1741 break; | |
1742 if(t2==h2) | |
1743 break; | |
1744 gp1 = p2->runq[h2]; | |
1745 h2++; | |
1746 if(h2 == s2) | |
1747 h2 = 0; | |
1748 p->runq[t] = gp1; | |
1749 t++; | |
1750 if(t==s) | |
1751 t = 0; | |
1752 c1++; | |
1753 } | |
1754 p->runqtail = t; | |
1755 p2->runqhead = h2; | |
1756 runtime·unlock(p2); | |
1757 runtime·unlock(p); | |
1758 return gp; | |
1759 } | 1445 } |
1760 | 1446 |
1761 // Put on gfree list. | 1447 // Put on gfree list. |
1762 // If local list is too long, transfer a batch to the global list. | 1448 // If local list is too long, transfer a batch to the global list. |
1763 static void | 1449 static void |
1764 gfput(P *p, G *gp) | 1450 gfput(P *p, G *gp) |
1765 { | 1451 { |
1766 if(gp->stackguard - StackGuard != gp->stack0) | 1452 if(gp->stackguard - StackGuard != gp->stack0) |
1767 runtime·throw("invalid stack in gfput"); | 1453 runtime·throw("invalid stack in gfput"); |
1768 gp->schedlink = p->gfree; | 1454 gp->schedlink = p->gfree; |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1802 runtime·unlock(&runtime·sched.gflock); | 1488 runtime·unlock(&runtime·sched.gflock); |
1803 goto retry; | 1489 goto retry; |
1804 } | 1490 } |
1805 if(gp) { | 1491 if(gp) { |
1806 p->gfree = gp->schedlink; | 1492 p->gfree = gp->schedlink; |
1807 p->gfreecnt--; | 1493 p->gfreecnt--; |
1808 } | 1494 } |
1809 return gp; | 1495 return gp; |
1810 } | 1496 } |
1811 | 1497 |
1812 // Put mp on mwait list. | 1498 // Purge all cached G's from gfree list to the global list. |
| 1499 static void |
| 1500 gfpurge(P *p) |
| 1501 { |
| 1502 » G *gp; |
| 1503 |
| 1504 » runtime·lock(&runtime·sched.gflock); |
| 1505 » while(p->gfreecnt) { |
| 1506 » » p->gfreecnt--; |
| 1507 » » gp = p->gfree; |
| 1508 » » p->gfree = gp->schedlink; |
| 1509 » » gp->schedlink = runtime·sched.gfree; |
| 1510 » » runtime·sched.gfree = gp; |
| 1511 » } |
| 1512 » runtime·unlock(&runtime·sched.gflock); |
| 1513 } |
| 1514 |
| 1515 void |
| 1516 runtime·Breakpoint(void) |
| 1517 { |
| 1518 » runtime·breakpoint(); |
| 1519 } |
| 1520 |
| 1521 void |
| 1522 runtime·Gosched(void) |
| 1523 { |
| 1524 » runtime·gosched(); |
| 1525 } |
| 1526 |
| 1527 // Implementation of runtime.GOMAXPROCS. |
| 1528 // delete when scheduler is even stronger |
| 1529 int32 |
| 1530 runtime·gomaxprocsfunc(int32 n) |
| 1531 { |
| 1532 » int32 ret; |
| 1533 |
| 1534 » if(n > MaxGomaxprocs) |
| 1535 » » n = MaxGomaxprocs; |
| 1536 » runtime·lock(&runtime·sched); |
| 1537 » ret = runtime·gomaxprocs; |
| 1538 » if(n <= 0 || n == ret) { |
| 1539 » » runtime·unlock(&runtime·sched); |
| 1540 » » return ret; |
| 1541 » } |
| 1542 » runtime·unlock(&runtime·sched); |
| 1543 |
| 1544 » runtime·semacquire(&runtime·worldsema); |
| 1545 » m->gcing = 1; |
| 1546 » runtime·stoptheworld(); |
| 1547 » newprocs = n; |
| 1548 » m->gcing = 0; |
| 1549 » runtime·semrelease(&runtime·worldsema); |
| 1550 » runtime·starttheworld(); |
| 1551 |
| 1552 » return ret; |
| 1553 } |
| 1554 |
| 1555 static void |
| 1556 LockOSThread(void) |
| 1557 { |
| 1558 » m->lockedg = g; |
| 1559 » g->lockedm = m; |
| 1560 } |
| 1561 |
| 1562 void |
| 1563 runtime·LockOSThread(void) |
| 1564 { |
| 1565 » m->locked |= LockExternal; |
| 1566 » LockOSThread(); |
| 1567 } |
| 1568 |
| 1569 void |
| 1570 runtime·lockOSThread(void) |
| 1571 { |
| 1572 » m->locked += LockInternal; |
| 1573 » LockOSThread(); |
| 1574 } |
| 1575 |
| 1576 static void |
| 1577 UnlockOSThread(void) |
| 1578 { |
| 1579 » if(m->locked != 0) |
| 1580 » » return; |
| 1581 » m->lockedg = nil; |
| 1582 » g->lockedm = nil; |
| 1583 } |
| 1584 |
| 1585 void |
| 1586 runtime·UnlockOSThread(void) |
| 1587 { |
| 1588 » m->locked &= ~LockExternal; |
| 1589 » UnlockOSThread(); |
| 1590 } |
| 1591 |
| 1592 void |
| 1593 runtime·unlockOSThread(void) |
| 1594 { |
| 1595 » if(m->locked < LockInternal) |
| 1596 » » runtime·throw("runtime: internal error: misuse of lockOSThread/u
nlockOSThread"); |
| 1597 » m->locked -= LockInternal; |
| 1598 » UnlockOSThread(); |
| 1599 } |
| 1600 |
| 1601 bool |
| 1602 runtime·lockedOSThread(void) |
| 1603 { |
| 1604 » return g->lockedm != nil && m->lockedg != nil; |
| 1605 } |
| 1606 |
| 1607 // for testing of callbacks |
| 1608 void |
| 1609 runtime·golockedOSThread(bool ret) |
| 1610 { |
| 1611 » ret = runtime·lockedOSThread(); |
| 1612 » FLUSH(&ret); |
| 1613 } |
| 1614 |
| 1615 // for testing of wire, unwire |
| 1616 void |
| 1617 runtime·mid(uint32 ret) |
| 1618 { |
| 1619 » ret = m->id; |
| 1620 » FLUSH(&ret); |
| 1621 } |
| 1622 |
| 1623 void |
| 1624 runtime·NumGoroutine(intgo ret) |
| 1625 { |
| 1626 » ret = runtime·gcount(); |
| 1627 » FLUSH(&ret); |
| 1628 } |
| 1629 |
| 1630 int32 |
| 1631 runtime·gcount(void) |
| 1632 { |
| 1633 » G *gp; |
| 1634 » int32 n, s; |
| 1635 |
| 1636 » n = 0; |
| 1637 » runtime·lock(&runtime·sched); |
| 1638 » // TODO(dvyukov): runtime.NumGoroutine() is O(N). |
| 1639 » // We do not want to increment/decrement centralized counter in newproc/
goexit, |
| 1640 » // just to make runtime.NumGoroutine() faster. |
| 1641 » // Compromise solution is to introduce per-P counters of active goroutin
es. |
| 1642 » for(gp = runtime·allg; gp; gp = gp->alllink) { |
| 1643 » » s = gp->status; |
| 1644 » » if(s == Grunnable || s == Grunning || s == Gsyscall || s == Gwai
ting) |
| 1645 » » » n++; |
| 1646 » } |
| 1647 » runtime·unlock(&runtime·sched); |
| 1648 » return n; |
| 1649 } |
| 1650 |
| 1651 int32 |
| 1652 runtime·mcount(void) |
| 1653 { |
| 1654 » return runtime·sched.mcount; |
| 1655 } |
| 1656 |
| 1657 void |
| 1658 runtime·badmcall(void) // called from assembly |
| 1659 { |
| 1660 » runtime·throw("runtime: mcall called on m->g0 stack"); |
| 1661 } |
| 1662 |
| 1663 void |
| 1664 runtime·badmcall2(void) // called from assembly |
| 1665 { |
| 1666 » runtime·throw("runtime: mcall function returned"); |
| 1667 } |
| 1668 |
| 1669 static struct { |
| 1670 » Lock; |
| 1671 » void (*fn)(uintptr*, int32); |
| 1672 » int32 hz; |
| 1673 » uintptr pcbuf[100]; |
| 1674 } prof; |
| 1675 |
| 1676 // Called if we receive a SIGPROF signal. |
| 1677 void |
| 1678 runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp) |
| 1679 { |
| 1680 » int32 n; |
| 1681 |
| 1682 » // Windows does profiling in a dedicated thread w/o m. |
| 1683 » if(!Windows && (m == nil || m->mcache == nil)) |
| 1684 » » return; |
| 1685 » if(prof.fn == nil || prof.hz == 0) |
| 1686 » » return; |
| 1687 |
| 1688 » runtime·lock(&prof); |
| 1689 » if(prof.fn == nil) { |
| 1690 » » runtime·unlock(&prof); |
| 1691 » » return; |
| 1692 » } |
| 1693 » n = runtime·gentraceback(pc, sp, lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf
)); |
| 1694 » if(n > 0) |
| 1695 » » prof.fn(prof.pcbuf, n); |
| 1696 » runtime·unlock(&prof); |
| 1697 } |
| 1698 |
| 1699 // Arrange to call fn with a traceback hz times a second. |
| 1700 void |
| 1701 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) |
| 1702 { |
| 1703 » // Force sane arguments. |
| 1704 » if(hz < 0) |
| 1705 » » hz = 0; |
| 1706 » if(hz == 0) |
| 1707 » » fn = nil; |
| 1708 » if(fn == nil) |
| 1709 » » hz = 0; |
| 1710 |
| 1711 » // Stop profiler on this cpu so that it is safe to lock prof. |
| 1712 » // if a profiling signal came in while we had prof locked, |
| 1713 » // it would deadlock. |
| 1714 » runtime·resetcpuprofiler(0); |
| 1715 |
| 1716 » runtime·lock(&prof); |
| 1717 » prof.fn = fn; |
| 1718 » prof.hz = hz; |
| 1719 » runtime·unlock(&prof); |
| 1720 » runtime·lock(&runtime·sched); |
| 1721 » runtime·sched.profilehz = hz; |
| 1722 » runtime·unlock(&runtime·sched); |
| 1723 |
| 1724 » if(hz != 0) |
| 1725 » » runtime·resetcpuprofiler(hz); |
| 1726 } |
| 1727 |
| 1728 // Change number of processors. The world is stopped, sched is locked. |
| 1729 static void |
| 1730 procresize(int32 new) |
| 1731 { |
| 1732 » int32 i, old; |
| 1733 » G *gp; |
| 1734 » P *p; |
| 1735 |
| 1736 » old = runtime·gomaxprocs; |
| 1737 » if(old < 0 || old > MaxGomaxprocs || new <= 0 || new >MaxGomaxprocs) |
| 1738 » » runtime·throw("procresize: invalid arg"); |
| 1739 » // initialize new P's |
| 1740 » for(i = 0; i < new; i++) { |
| 1741 » » p = runtime·allp[i]; |
| 1742 » » if(p == nil) { |
| 1743 » » » p = (P*)runtime·mallocgc(sizeof(*p), 0, 0, 1); |
| 1744 » » » p->status = Pgcstop; |
| 1745 » » » runtime·atomicstorep(&runtime·allp[i], p); |
| 1746 » » } |
| 1747 » » if(p->mcache == nil) { |
| 1748 » » » if(old==0 && i==0) |
| 1749 » » » » p->mcache = m->mcache; // bootstrap |
| 1750 » » » else |
| 1751 » » » » p->mcache = runtime·allocmcache(); |
| 1752 » » } |
| 1753 » » if(p->runq == nil) { |
| 1754 » » » p->runqsize = 128; |
| 1755 » » » p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*),
0, 0, 1); |
| 1756 » » } |
| 1757 » } |
| 1758 |
| 1759 » // redistribute runnable G's evenly |
| 1760 » for(i = 0; i < old; i++) { |
| 1761 » » p = runtime·allp[i]; |
| 1762 » » while(gp = runqget(p)) |
| 1763 » » » globrunqput(gp); |
| 1764 » } |
| 1765 » // start at 1 because current M already executes some G and will acquire
allp[0] below, |
| 1766 » // so if we have a spare G we want to put it into allp[1]. |
| 1767 » for(i = 1; runtime·sched.runqhead; i++) { |
| 1768 » » gp = runtime·sched.runqhead; |
| 1769 » » runtime·sched.runqhead = gp->schedlink; |
| 1770 » » runqput(runtime·allp[i%new], gp); |
| 1771 » } |
| 1772 » runtime·sched.runqtail = nil; |
| 1773 » runtime·sched.runqsize = 0; |
| 1774 |
| 1775 » // free unused P's |
| 1776 » for(i = new; i < old; i++) { |
| 1777 » » p = runtime·allp[i]; |
| 1778 » » runtime·freemcache(p->mcache); |
| 1779 » » p->mcache = nil; |
| 1780 » » gfpurge(p); |
| 1781 » » p->status = Pdead; |
| 1782 » » // can't free P itself because it can be referenced by an M in s
yscall |
| 1783 » } |
| 1784 |
| 1785 » if(m->p) |
| 1786 » » m->p->m = nil; |
| 1787 » m->p = nil; |
| 1788 » m->mcache = nil; |
| 1789 » p = runtime·allp[0]; |
| 1790 » p->m = nil; |
| 1791 » p->status = Pidle; |
| 1792 » acquirep(p); |
| 1793 » for(i = new-1; i > 0; i--) { |
| 1794 » » p = runtime·allp[i]; |
| 1795 » » p->status = Pidle; |
| 1796 » » pidleput(p); |
| 1797 » } |
| 1798 » runtime·singleproc = new == 1; |
| 1799 » runtime·atomicstore((uint32*)&runtime·gomaxprocs, new); |
| 1800 } |
| 1801 |
| 1802 // Associate p and the current m. |
| 1803 static void |
| 1804 acquirep(P *p) |
| 1805 { |
| 1806 » if(m->p || m->mcache) |
| 1807 » » runtime·throw("acquirep: already in go"); |
| 1808 » if(p->m || p->status != Pidle) { |
| 1809 » » runtime·printf("acquirep: p->m=%p(%d) p->status=%d\n", p->m, p->
m ? p->m->id : 0, p->status); |
| 1810 » » runtime·throw("acquirep: invalid p state"); |
| 1811 » } |
| 1812 » m->mcache = p->mcache; |
| 1813 » m->p = p; |
| 1814 » p->m = m; |
| 1815 » p->status = Prunning; |
| 1816 } |
| 1817 |
| 1818 // Disassociate p and the current m. |
| 1819 static P* |
| 1820 releasep(void) |
| 1821 { |
| 1822 » P *p; |
| 1823 |
| 1824 » if(m->p == nil || m->mcache == nil) |
| 1825 » » runtime·throw("releasep: invalid arg"); |
| 1826 » p = m->p; |
| 1827 » if(p->m != m || p->mcache != m->mcache || p->status != Prunning) { |
| 1828 » » runtime·printf("releasep: m=%p m->p=%p p->m=%p m->mcache=%p p->m
cache=%p p->status=%d\n", |
| 1829 » » » m, m->p, p->m, m->mcache, p->mcache, p->status); |
| 1830 » » runtime·throw("releasep: invalid p state"); |
| 1831 » } |
| 1832 » m->p = nil; |
| 1833 » m->mcache = nil; |
| 1834 » p->m = nil; |
| 1835 » p->status = Pidle; |
| 1836 » return p; |
| 1837 } |
| 1838 |
| 1839 static void |
| 1840 inclocked(int32 v) |
| 1841 { |
| 1842 » runtime·lock(&runtime·sched); |
| 1843 » runtime·sched.mlocked += v; |
| 1844 » if(v > 0) |
| 1845 » » checkdead(); |
| 1846 » runtime·unlock(&runtime·sched); |
| 1847 } |
| 1848 |
| 1849 // Check for deadlock situation. |
| 1850 // The check is based on number of running M's, if 0 -> deadlock. |
| 1851 static void |
| 1852 checkdead(void) |
| 1853 { |
| 1854 » G *gp; |
| 1855 » int32 run, grunning, s; |
| 1856 |
| 1857 » // -1 for sysmon |
| 1858 » run = runtime·sched.mcount - runtime·sched.nmidle - runtime·sched.mlocke
d - 1; |
| 1859 » if(run > 0) |
| 1860 » » return; |
| 1861 » if(run < 0) { |
| 1862 » » runtime·printf("checkdead: nmidle=%d mlocked=%d mcount=%d\n", |
| 1863 » » » runtime·sched.nmidle, runtime·sched.mlocked, runtime·sch
ed.mcount); |
| 1864 » » runtime·throw("checkdead: inconsistent counts"); |
| 1865 » } |
| 1866 » grunning = 0; |
| 1867 » for(gp = runtime·allg; gp; gp = gp->alllink) { |
| 1868 » » if(gp == scvg) |
| 1869 » » » continue; |
| 1870 » » s = gp->status; |
| 1871 » » if(s == Gwaiting) |
| 1872 » » » grunning++; |
| 1873 » » else if(s == Grunnable || s == Grunning || s == Gsyscall) { |
| 1874 » » » runtime·printf("checkdead: find g %D in status %d\n", gp
->goid, s); |
| 1875 » » » runtime·throw("checkdead: runnable g"); |
| 1876 » » } |
| 1877 » } |
| 1878 » if(grunning == 0) // possible if main goroutine calls runtime·Goexit() |
| 1879 » » runtime·exit(0); |
| 1880 » m->throwing = -1; // do not dump full stacks |
| 1881 » runtime·throw("all goroutines are asleep - deadlock!"); |
| 1882 } |
| 1883 |
| 1884 static void |
| 1885 sysmon(void) |
| 1886 { |
| 1887 » uint32 idle, delay; |
| 1888 » uint32 ticks[MaxGomaxprocs]; |
| 1889 |
| 1890 » // This is a special dedicated thread that retakes P's from blocking sys
calls. |
| 1891 » // It works w/o mcache nor stackalloc, it may work concurrently with GC. |
| 1892 » runtime·asminit(); |
| 1893 » runtime·minit(); |
| 1894 |
| 1895 » idle = 0; // how many cycles in succession we had not wokeup somebody |
| 1896 » delay = 0; |
| 1897 » for(;;) { |
| 1898 » » if(idle == 0) // start with 20us sleep... |
| 1899 » » » delay = 20; |
| 1900 » » else if(idle > 50) // start doubling the sleep after 1ms... |
| 1901 » » » delay *= 2; |
| 1902 » » if(delay > 10*1000) // up to 10ms |
| 1903 » » » delay = 10*1000; |
| 1904 » » runtime·usleep(delay); |
| 1905 » » if(runtime·gcwaiting || runtime·sched.npidle == runtime·gomaxpro
cs) { // TODO: fast atomic |
| 1906 » » » runtime·lock(&runtime·sched); |
| 1907 » » » if(runtime·gcwaiting || runtime·sched.npidle == runtime·
gomaxprocs) { |
| 1908 » » » » runtime·sched.sysmonwait = true; |
| 1909 » » » » runtime·unlock(&runtime·sched); |
| 1910 » » » » runtime·notesleep(&runtime·sched.sysmonnote); |
| 1911 » » » » runtime·noteclear(&runtime·sched.sysmonnote); |
| 1912 » » » » idle = 0; |
| 1913 » » » » delay = 20; |
| 1914 » » » } else |
| 1915 » » » » runtime·unlock(&runtime·sched); |
| 1916 » » } |
| 1917 » » if(retake(ticks)) |
| 1918 » » » idle = 0; |
| 1919 » » else |
| 1920 » » » idle++; |
| 1921 » } |
| 1922 } |
| 1923 |
| 1924 static uint32 |
| 1925 retake(uint32 *ticks) |
| 1926 { |
| 1927 » uint32 i, s, n; |
| 1928 » int64 t; |
| 1929 » P *p; |
| 1930 |
| 1931 » n = 0; |
| 1932 » for(i = 0; i < runtime·gomaxprocs; i++) { |
| 1933 » » p = runtime·allp[i]; |
| 1934 » » if(p==nil) |
| 1935 » » » continue; |
| 1936 » » t = p->tick; |
| 1937 » » if(ticks[i] != t) { |
| 1938 » » » ticks[i] = t; |
| 1939 » » » continue; |
| 1940 » » } |
| 1941 » » s = p->status; |
| 1942 » » if(s != Psyscall) |
| 1943 » » » continue; |
| 1944 » » if(p->runqhead == p->runqtail && runtime·sched.nmspinning + runt
ime·sched.npidle > 0) // TODO: fast atomic |
| 1945 » » » continue; |
| 1946 » » // Need to increment number of locked M's before the CAS. |
| 1947 » » // Otherwise the M from which we retake can exit the syscall, |
| 1948 » » // increment nmidle and report deadlock. |
| 1949 » » inclocked(-1); |
| 1950 » » if(runtime·cas(&p->status, s, Pidle)) { |
| 1951 » » » n++; |
| 1952 » » » handoffp(p); |
| 1953 » » } |
| 1954 » » inclocked(1); |
| 1955 » } |
| 1956 » return n; |
| 1957 } |
| 1958 |
| 1959 // Put mp on midle list. |
1813 // Sched must be locked. | 1960 // Sched must be locked. |
1814 static void | 1961 static void |
1815 mput(M *mp) | 1962 mput(M *mp) |
1816 { | 1963 { |
1817 » mp->schedlink = runtime·sched.mhead; | 1964 » mp->schedlink = runtime·sched.midle; |
1818 » runtime·sched.mhead = mp; | 1965 » runtime·sched.midle = mp; |
1819 » runtime·sched.mwait++; | 1966 » runtime·sched.nmidle++; |
1820 } | 1967 » checkdead(); |
1821 | 1968 } |
1822 // Try to get an m from mwait list. | 1969 |
| 1970 // Try to get an m from midle list. |
1823 // Sched must be locked. | 1971 // Sched must be locked. |
1824 static M* | 1972 static M* |
1825 mtryget(void) | 1973 mget(void) |
1826 { | 1974 { |
1827 M *mp; | 1975 M *mp; |
1828 | 1976 |
1829 » if((mp = runtime·sched.mhead) != nil){ | 1977 » if((mp = runtime·sched.midle) != nil){ |
1830 » » runtime·sched.mhead = mp->schedlink; | 1978 » » runtime·sched.midle = mp->schedlink; |
1831 » » runtime·sched.mwait--; | 1979 » » runtime·sched.nmidle--; |
1832 } | 1980 } |
1833 return mp; | 1981 return mp; |
| 1982 } |
| 1983 |
| 1984 // Put gp on the global runnable queue. |
| 1985 // Sched must be locked. |
| 1986 static void |
| 1987 globrunqput(G *gp) |
| 1988 { |
| 1989 gp->schedlink = nil; |
| 1990 if(runtime·sched.runqtail) |
| 1991 runtime·sched.runqtail->schedlink = gp; |
| 1992 else |
| 1993 runtime·sched.runqhead = gp; |
| 1994 runtime·sched.runqtail = gp; |
| 1995 runtime·sched.runqsize++; |
| 1996 } |
| 1997 |
| 1998 // Try get a batch of G's from the global runnable queue. |
| 1999 // Sched must be locked. |
| 2000 static G* |
| 2001 globrunqget(P *p) |
| 2002 { |
| 2003 G *gp, *gp1; |
| 2004 int32 n; |
| 2005 |
| 2006 if(runtime·sched.runqsize == 0) |
| 2007 return nil; |
| 2008 n = runtime·sched.runqsize/runtime·gomaxprocs+1; |
| 2009 if(n > runtime·sched.runqsize) |
| 2010 n = runtime·sched.runqsize; |
| 2011 runtime·sched.runqsize -= n; |
| 2012 if(runtime·sched.runqsize == 0) |
| 2013 runtime·sched.runqtail = nil; |
| 2014 gp = runtime·sched.runqhead; |
| 2015 runtime·sched.runqhead = gp->schedlink; |
| 2016 n--; |
| 2017 while(n--) { |
| 2018 gp1 = runtime·sched.runqhead; |
| 2019 runtime·sched.runqhead = gp1->schedlink; |
| 2020 runqput(p, gp1); |
| 2021 } |
| 2022 return gp; |
1834 } | 2023 } |
1835 | 2024 |
1836 // Put p to on pidle list. | 2025 // Put p to on pidle list. |
1837 // Sched must be locked. | 2026 // Sched must be locked. |
1838 static void | 2027 static void |
1839 pidleput(P *p) | 2028 pidleput(P *p) |
1840 { | 2029 { |
1841 p->link = runtime·sched.pidle; | 2030 p->link = runtime·sched.pidle; |
1842 runtime·sched.pidle = p; | 2031 runtime·sched.pidle = p; |
1843 » runtime·sched.npidle++; | 2032 » runtime·sched.npidle++; // TODO: fast atomic |
1844 } | 2033 } |
1845 | 2034 |
1846 // Try get a p from pidle list. | 2035 // Try get a p from pidle list. |
1847 // Sched must be locked. | 2036 // Sched must be locked. |
1848 static P* | 2037 static P* |
1849 pidleget(void) | 2038 pidleget(void) |
1850 { | 2039 { |
1851 P *p; | 2040 P *p; |
1852 » | 2041 |
1853 p = runtime·sched.pidle; | 2042 p = runtime·sched.pidle; |
1854 if(p) { | 2043 if(p) { |
1855 runtime·sched.pidle = p->link; | 2044 runtime·sched.pidle = p->link; |
1856 » » runtime·sched.npidle--; | 2045 » » runtime·sched.npidle--; // TODO: fast atomic |
1857 } | 2046 } |
1858 return p; | 2047 return p; |
1859 } | 2048 } |
| 2049 |
| 2050 // Put g on local runnable queue. |
| 2051 // TODO(dvyukov): consider using lock-free queue. |
| 2052 static void |
| 2053 runqput(P *p, G *gp) |
| 2054 { |
| 2055 int32 h, t, s; |
| 2056 |
| 2057 runtime·lock(p); |
| 2058 retry: |
| 2059 h = p->runqhead; |
| 2060 t = p->runqtail; |
| 2061 s = p->runqsize; |
| 2062 if(t == h-1 || (h == 0 && t == s-1)) { |
| 2063 runqgrow(p); |
| 2064 goto retry; |
| 2065 } |
| 2066 p->runq[t++] = gp; |
| 2067 if(t == s) |
| 2068 t = 0; |
| 2069 p->runqtail = t; |
| 2070 runtime·unlock(p); |
| 2071 } |
| 2072 |
| 2073 // Get g from local runnable queue. |
| 2074 static G* |
| 2075 runqget(P *p) |
| 2076 { |
| 2077 G *gp; |
| 2078 int32 t, h, s; |
| 2079 |
| 2080 if(p->runqhead == p->runqtail) |
| 2081 return nil; |
| 2082 runtime·lock(p); |
| 2083 h = p->runqhead; |
| 2084 t = p->runqtail; |
| 2085 s = p->runqsize; |
| 2086 if(t == h) { |
| 2087 runtime·unlock(p); |
| 2088 return nil; |
| 2089 } |
| 2090 gp = p->runq[h++]; |
| 2091 if(h == s) |
| 2092 h = 0; |
| 2093 p->runqhead = h; |
| 2094 runtime·unlock(p); |
| 2095 return gp; |
| 2096 } |
| 2097 |
| 2098 // Grow local runnable queue. |
| 2099 // TODO(dvyukov): consider using fixed-size array |
| 2100 // and transfer excess to the global list (local queue can grow way too big). |
| 2101 static void |
| 2102 runqgrow(P *p) |
| 2103 { |
| 2104 G **q; |
| 2105 int32 s, t, h, t2; |
| 2106 |
| 2107 h = p->runqhead; |
| 2108 t = p->runqtail; |
| 2109 s = p->runqsize; |
| 2110 t2 = 0; |
| 2111 q = runtime·malloc(2*s*sizeof(*q)); |
| 2112 while(t != h) { |
| 2113 q[t2++] = p->runq[h++]; |
| 2114 if(h == s) |
| 2115 h = 0; |
| 2116 } |
| 2117 runtime·free(p->runq); |
| 2118 p->runq = q; |
| 2119 p->runqhead = 0; |
| 2120 p->runqtail = t2; |
| 2121 p->runqsize = 2*s; |
| 2122 } |
| 2123 |
| 2124 // Steal half of elements from local runnable queue of p2 |
| 2125 // and put onto local runnable queue of p. |
| 2126 // Returns one of the stolen elements (or nil if failed). |
| 2127 static G* |
| 2128 runqsteal(P *p, P *p2) |
| 2129 { |
| 2130 G *gp, *gp1; |
| 2131 int32 t, h, s, t2, h2, s2, c, i; |
| 2132 |
| 2133 if(p2->runqhead == p2->runqtail) |
| 2134 return nil; |
| 2135 // sort locks to prevent deadlocks |
| 2136 if(p < p2) |
| 2137 runtime·lock(p); |
| 2138 runtime·lock(p2); |
| 2139 if(p2->runqhead == p2->runqtail) { |
| 2140 runtime·unlock(p2); |
| 2141 if(p < p2) |
| 2142 runtime·unlock(p); |
| 2143 return nil; |
| 2144 } |
| 2145 if(p >= p2) |
| 2146 runtime·lock(p); |
| 2147 // now we've locked both queues and know the victim is not empty |
| 2148 h = p->runqhead; |
| 2149 t = p->runqtail; |
| 2150 s = p->runqsize; |
| 2151 h2 = p2->runqhead; |
| 2152 t2 = p2->runqtail; |
| 2153 s2 = p2->runqsize; |
| 2154 gp = p2->runq[h2++]; // return value |
| 2155 if(h2 == s2) |
| 2156 h2 = 0; |
| 2157 // steal roughly half |
| 2158 if(t2 > h2) |
| 2159 c = (t2 - h2) / 2; |
| 2160 else |
| 2161 c = (s2 - h2 + t2) / 2; |
| 2162 // copy |
| 2163 for(i = 0; i != c; i++) { |
| 2164 // the target queue is full? |
| 2165 if(t == h-1 || (h == 0 && t == s-1)) |
| 2166 break; |
| 2167 // the victim queue is empty? |
| 2168 if(t2 == h2) |
| 2169 break; |
| 2170 gp1 = p2->runq[h2++]; |
| 2171 if(h2 == s2) |
| 2172 h2 = 0; |
| 2173 p->runq[t++] = gp1; |
| 2174 if(t == s) |
| 2175 t = 0; |
| 2176 } |
| 2177 p->runqtail = t; |
| 2178 p2->runqhead = h2; |
| 2179 runtime·unlock(p2); |
| 2180 runtime·unlock(p); |
| 2181 return gp; |
| 2182 } |
| 2183 |
| 2184 void |
| 2185 runtime·testSchedLocalQueue(void) |
| 2186 { |
| 2187 P p; |
| 2188 G gs[1000]; |
| 2189 int32 i, j; |
| 2190 |
| 2191 runtime·memclr((byte*)&p, sizeof(p)); |
| 2192 p.runqsize = 1; |
| 2193 p.runqhead = 0; |
| 2194 p.runqtail = 0; |
| 2195 p.runq = runtime·malloc(p.runqsize*sizeof(*p.runq)); |
| 2196 |
| 2197 for(i = 0; i < nelem(gs); i++) { |
| 2198 if(runqget(&p) != nil) |
| 2199 runtime·throw("runq is not empty initially"); |
| 2200 for(j = 0; j < i; j++) |
| 2201 runqput(&p, &gs[i]); |
| 2202 for(j = 0; j < i; j++) { |
| 2203 if(runqget(&p) != &gs[i]) { |
| 2204 runtime·printf("bad element at iter %d/%d\n", i,
j); |
| 2205 runtime·throw("bad element"); |
| 2206 } |
| 2207 } |
| 2208 if(runqget(&p) != nil) |
| 2209 runtime·throw("runq is not empty afterwards"); |
| 2210 } |
| 2211 } |
| 2212 |
| 2213 void |
| 2214 runtime·testSchedLocalQueueSteal(void) |
| 2215 { |
| 2216 P p1, p2; |
| 2217 G gs[1000], *gp; |
| 2218 int32 i, j, s; |
| 2219 |
| 2220 runtime·memclr((byte*)&p1, sizeof(p1)); |
| 2221 p1.runqsize = 1; |
| 2222 p1.runqhead = 0; |
| 2223 p1.runqtail = 0; |
| 2224 p1.runq = runtime·malloc(p1.runqsize*sizeof(*p1.runq)); |
| 2225 |
| 2226 runtime·memclr((byte*)&p2, sizeof(p2)); |
| 2227 p2.runqsize = nelem(gs); |
| 2228 p2.runqhead = 0; |
| 2229 p2.runqtail = 0; |
| 2230 p2.runq = runtime·malloc(p2.runqsize*sizeof(*p2.runq)); |
| 2231 |
| 2232 for(i = 0; i < nelem(gs); i++) { |
| 2233 for(j = 0; j < i; j++) { |
| 2234 gs[j].sig = 0; |
| 2235 runqput(&p1, &gs[j]); |
| 2236 } |
| 2237 gp = runqsteal(&p2, &p1); |
| 2238 s = 0; |
| 2239 if(gp) { |
| 2240 s++; |
| 2241 gp->sig++; |
| 2242 } |
| 2243 while(gp = runqget(&p2)) { |
| 2244 s++; |
| 2245 gp->sig++; |
| 2246 } |
| 2247 while(gp = runqget(&p1)) |
| 2248 gp->sig++; |
| 2249 for(j = 0; j < i; j++) { |
| 2250 if(gs[j].sig != 1) { |
| 2251 runtime·printf("bad element %d(%d) at iter %d\n"
, j, gs[j].sig, i); |
| 2252 runtime·throw("bad element"); |
| 2253 } |
| 2254 } |
| 2255 if(s != i/2 && s != i/2+1) { |
| 2256 runtime·printf("bad steal %d, want %d or %d, iter %d\n", |
| 2257 s, i/2, i/2+1, i); |
| 2258 runtime·throw("bad steal"); |
| 2259 } |
| 2260 } |
| 2261 } |
| 2262 |
LEFT | RIGHT |