LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "malloc.h" | 7 #include "malloc.h" |
8 #include "stack.h" | 8 #include "stack.h" |
9 #include "race.h" | 9 #include "race.h" |
10 #include "type.h" | 10 #include "type.h" |
11 | |
12 #define CHECK(cond, fmt) /*if(cond) {} else { runtime·printf fmt; runtime·throw(
"CHECK"); }*/ | |
13 | 11 |
14 // Goroutine scheduler | 12 // Goroutine scheduler |
15 // The scheduler's job is to distribute ready-to-run goroutines over worker thre
ads. | 13 // The scheduler's job is to distribute ready-to-run goroutines over worker thre
ads. |
16 // | 14 // |
17 // The main concepts are: | 15 // The main concepts are: |
18 // G - goroutine. | 16 // G - goroutine. |
19 // M - worker thread, or machine. | 17 // M - worker thread, or machine. |
20 // P - processor, a resource that is required to execute Go code. | 18 // P - processor, a resource that is required to execute Go code. |
21 // M must have an associated P to execute Go code, however it can be | 19 // M must have an associated P to execute Go code, however it can be |
22 // blocked or in a syscall w/o an associated P. | 20 // blocked or in a syscall w/o an associated P. |
23 | 21 |
24 typedef struct Sched Sched; | 22 typedef struct Sched Sched; |
25 struct Sched { | 23 struct Sched { |
26 Lock; | 24 Lock; |
27 | 25 |
28 » M*» mhead;» // m's waiting for work | 26 » uint64» goidgen; |
29 » int32» mwait;» // number of m's waiting for work | 27 |
| 28 » M*» midle;» // idle m's waiting for work |
| 29 » int32» nmidle;» // number of idle m's waiting for work |
30 int32 mlocked; // number of locked m's waiting for work | 30 int32 mlocked; // number of locked m's waiting for work |
31 int32 mcount; // number of m's that have been created | 31 int32 mcount; // number of m's that have been created |
32 | 32 |
33 P* pidle; // idle P's | 33 P* pidle; // idle P's |
34 uint32 npidle; | 34 uint32 npidle; |
35 uint32 nmspinning; | 35 uint32 nmspinning; |
36 | 36 |
37 // Global runnable queue. | 37 // Global runnable queue. |
38 G* runqhead; | 38 G* runqhead; |
39 G* runqtail; | 39 G* runqtail; |
40 int32 runqsize; | 40 int32 runqsize; |
41 | 41 |
42 // Global cache of dead G's. | 42 // Global cache of dead G's. |
43 » Lock gflock; | 43 » Lock» gflock; |
44 G* gfree; | 44 G* gfree; |
45 uint64 goidgen; | |
46 | 45 |
47 int32 stopwait; | 46 int32 stopwait; |
48 Note stopnote; | 47 Note stopnote; |
49 bool sysmonwait; | 48 bool sysmonwait; |
50 Note sysmonnote; | 49 Note sysmonnote; |
51 | 50 |
52 int32 profilehz; // cpu profiling rate | 51 int32 profilehz; // cpu profiling rate |
53 }; | 52 }; |
54 | 53 |
55 enum { maxgomaxprocs = 1<<8 }; | 54 // The max value of GOMAXPROCS. |
| 55 // There are no fundamental restrictions on the value. |
| 56 enum { MaxGomaxprocs = 1<<8 }; |
56 | 57 |
57 Sched runtime·sched; | 58 Sched runtime·sched; |
58 int32 runtime·gomaxprocs; | 59 int32 runtime·gomaxprocs; |
59 bool runtime·singleproc; | 60 bool runtime·singleproc; |
60 bool runtime·iscgo; | 61 bool runtime·iscgo; |
61 int32 runtime·gcwaiting; | 62 int32 runtime·gcwaiting; |
62 M runtime·m0; | 63 M runtime·m0; |
63 G runtime·g0; // idle goroutine for m0 | 64 G runtime·g0; // idle goroutine for m0 |
64 G* runtime·allg; | 65 G* runtime·allg; |
65 G* runtime·lastg; | 66 G* runtime·lastg; |
66 M* runtime·allm; | 67 M* runtime·allm; |
67 M* runtime·extram; | 68 M* runtime·extram; |
68 int8* runtime·goos; | 69 int8* runtime·goos; |
69 int32 runtime·ncpu; | 70 int32 runtime·ncpu; |
70 static int32 newprocs; | 71 static int32 newprocs; |
71 // Keep trace of scavenger's goroutine for deadlock detection. | 72 // Keep trace of scavenger's goroutine for deadlock detection. |
72 static G *scvg; | 73 static G *scvg; |
73 | 74 |
| 75 void runtime·mstart(void); |
74 static void runqput(P*, G*); | 76 static void runqput(P*, G*); |
75 static G* runqget(P*); | 77 static G* runqget(P*); |
76 static void runqgrow(P*); | 78 static void runqgrow(P*); |
77 static G* runqsteal(P*, P*); | 79 static G* runqsteal(P*, P*); |
78 static void globrunqput(G*); | |
79 static G* globrunqget(P*); | |
80 static void mput(M*); | 80 static void mput(M*); |
81 static M* mget(void); | 81 static M* mget(void); |
82 static void gfput(P*, G*); | |
83 static G* gfget(P*); | |
84 static void gfpurge(P*); | |
85 static void mcommoninit(M*); | 82 static void mcommoninit(M*); |
86 static void schedule(void); | 83 static void schedule(void); |
87 static void procresize(int32); | 84 static void procresize(int32); |
88 static void acquirep(P*); | 85 static void acquirep(P*); |
89 static P* releasep(void); | 86 static P* releasep(void); |
90 static void newm(void(*)(void), P*, bool, bool); | 87 static void newm(void(*)(void), P*, bool, bool); |
91 static void goidle(void); | 88 static void goidle(void); |
92 static void stopm(void); | 89 static void stopm(void); |
93 static void startm(P*, bool); | 90 static void startm(P*, bool); |
94 static void handoffp(P*); | 91 static void handoffp(P*); |
95 static void wakep(void); | 92 static void wakep(void); |
96 static void stoplockedm(void); | 93 static void stoplockedm(void); |
97 static void startlockedm(G*); | 94 static void startlockedm(G*); |
98 static void sysmon(void); | 95 static void sysmon(void); |
99 static uint32 retake(uint32*); | 96 static uint32 retake(uint32*); |
100 static P* pidleget(void); | |
101 static void pidleput(P*); | |
102 static void inclocked(int32); | 97 static void inclocked(int32); |
103 static void checkdead(void); | 98 static void checkdead(void); |
104 static void exitsyscall0(G*); | 99 static void exitsyscall0(G*); |
105 static void park0(G*); | 100 static void park0(G*); |
106 static void gosched0(G*); | 101 static void gosched0(G*); |
| 102 static void goexit0(G*); |
| 103 static void gfput(P*, G*); |
| 104 static G* gfget(P*); |
| 105 static void gfpurge(P*); |
| 106 static void globrunqput(G*); |
| 107 static G* globrunqget(P*); |
| 108 static P* pidleget(void); |
| 109 static void pidleput(P*); |
107 | 110 |
108 // The bootstrap sequence is: | 111 // The bootstrap sequence is: |
109 // | 112 // |
110 // call osinit | 113 // call osinit |
111 // call schedinit | 114 // call schedinit |
112 // make & queue new G | 115 // make & queue new G |
113 // call runtime·mstart | 116 // call runtime·mstart |
114 // | 117 // |
115 // The new G calls runtime·main. | 118 // The new G calls runtime·main. |
116 void | 119 void |
(...skipping 11 matching lines...) Expand all Loading... |
128 runtime·goenvs(); | 131 runtime·goenvs(); |
129 | 132 |
130 // For debugging: | 133 // For debugging: |
131 // Allocate internal symbol table representation now, | 134 // Allocate internal symbol table representation now, |
132 // so that we don't need to call malloc when we crash. | 135 // so that we don't need to call malloc when we crash. |
133 // runtime·findfunc(0); | 136 // runtime·findfunc(0); |
134 | 137 |
135 procs = 1; | 138 procs = 1; |
136 p = runtime·getenv("GOMAXPROCS"); | 139 p = runtime·getenv("GOMAXPROCS"); |
137 if(p != nil && (n = runtime·atoi(p)) > 0) { | 140 if(p != nil && (n = runtime·atoi(p)) > 0) { |
138 » » if(n > maxgomaxprocs) | 141 » » if(n > MaxGomaxprocs) |
139 » » » n = maxgomaxprocs; | 142 » » » n = MaxGomaxprocs; |
140 procs = n; | 143 procs = n; |
141 } | 144 } |
142 » runtime·allp = (P**)runtime·malloc((maxgomaxprocs+1)*sizeof(runtime·allp
[0])); | 145 » runtime·allp = runtime·malloc((MaxGomaxprocs+1)*sizeof(runtime·allp[0]))
; |
143 procresize(procs); | 146 procresize(procs); |
144 | 147 |
145 mstats.enablegc = 1; | 148 mstats.enablegc = 1; |
146 m->nomemprof--; | 149 m->nomemprof--; |
147 | 150 |
148 if(raceenabled) | 151 if(raceenabled) |
149 g->racectx = runtime·raceinit(); | 152 g->racectx = runtime·raceinit(); |
150 } | 153 } |
151 | 154 |
152 extern void main·init(void); | 155 extern void main·init(void); |
(...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
241 | 244 |
242 static void | 245 static void |
243 mcommoninit(M *mp) | 246 mcommoninit(M *mp) |
244 { | 247 { |
245 // If there is no mcache runtime·callers() will crash, | 248 // If there is no mcache runtime·callers() will crash, |
246 // and we are most likely in sysmon thread so the stack is senseless any
way. | 249 // and we are most likely in sysmon thread so the stack is senseless any
way. |
247 if(m->mcache) | 250 if(m->mcache) |
248 runtime·callers(1, mp->createstack, nelem(mp->createstack)); | 251 runtime·callers(1, mp->createstack, nelem(mp->createstack)); |
249 | 252 |
250 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); | 253 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); |
251 if(runtime·gsignalstk) | |
252 mp->gsignal = runtime·malg(runtime·gsignalstk); | |
253 | 254 |
254 runtime·lock(&runtime·sched); | 255 runtime·lock(&runtime·sched); |
255 mp->id = runtime·sched.mcount++; | 256 mp->id = runtime·sched.mcount++; |
256 | 257 |
257 runtime·mpreinit(mp); | 258 runtime·mpreinit(mp); |
258 | 259 |
259 // Add to runtime·allm so garbage collector doesn't free m | 260 // Add to runtime·allm so garbage collector doesn't free m |
260 // when it is just in a register or thread-local storage. | 261 // when it is just in a register or thread-local storage. |
261 mp->alllink = runtime·allm; | 262 mp->alllink = runtime·allm; |
262 // runtime·NumCgoCall() iterates over allm w/o schedlock, | 263 // runtime·NumCgoCall() iterates over allm w/o schedlock, |
263 // so we need to publish it safely. | 264 // so we need to publish it safely. |
264 runtime·atomicstorep(&runtime·allm, mp); | 265 runtime·atomicstorep(&runtime·allm, mp); |
265 runtime·unlock(&runtime·sched); | 266 runtime·unlock(&runtime·sched); |
266 } | 267 } |
267 | 268 |
268 // Mark gp ready to run. | 269 // Mark gp ready to run. |
269 void | 270 void |
270 runtime·ready(G *gp) | 271 runtime·ready(G *gp) |
271 { | 272 { |
272 // Mark runnable. | 273 // Mark runnable. |
273 if(gp->status != Gwaiting) { | 274 if(gp->status != Gwaiting) { |
274 runtime·printf("goroutine %D has status %d\n", gp->goid, gp->sta
tus); | 275 runtime·printf("goroutine %D has status %d\n", gp->goid, gp->sta
tus); |
275 runtime·throw("bad g->status in ready"); | 276 runtime·throw("bad g->status in ready"); |
276 } | 277 } |
277 gp->status = Grunnable; | 278 gp->status = Grunnable; |
278 runqput(m->p, gp); | 279 runqput(m->p, gp); |
279 » if(runtime·sched.npidle != 0 && runtime·sched.nmspinning == 0) | 280 » if(runtime·sched.npidle != 0 && runtime·sched.nmspinning == 0) // TODO:
fast atomic |
280 wakep(); | 281 wakep(); |
281 } | 282 } |
282 | 283 |
283 int32 | 284 int32 |
284 runtime·gcprocs(void) | 285 runtime·gcprocs(void) |
285 { | 286 { |
286 int32 n; | 287 int32 n; |
287 | 288 |
288 // Figure out how many CPUs to use during GC. | 289 // Figure out how many CPUs to use during GC. |
289 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. | 290 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. |
290 runtime·lock(&runtime·sched); | 291 runtime·lock(&runtime·sched); |
291 n = runtime·gomaxprocs; | 292 n = runtime·gomaxprocs; |
292 if(n > runtime·ncpu) | 293 if(n > runtime·ncpu) |
293 n = runtime·ncpu; | 294 n = runtime·ncpu; |
294 if(n > MaxGcproc) | 295 if(n > MaxGcproc) |
295 n = MaxGcproc; | 296 n = MaxGcproc; |
296 » if(n > runtime·sched.mwait+1) // one M is currently running | 297 » if(n > runtime·sched.nmidle+1) // one M is currently running |
297 » » n = runtime·sched.mwait+1; | 298 » » n = runtime·sched.nmidle+1; |
298 runtime·unlock(&runtime·sched); | 299 runtime·unlock(&runtime·sched); |
299 return n; | 300 return n; |
300 } | 301 } |
301 | 302 |
302 static bool | 303 static bool |
303 needaddgcproc(void) | 304 needaddgcproc(void) |
304 { | 305 { |
305 int32 n; | 306 int32 n; |
306 | 307 |
307 runtime·lock(&runtime·sched); | 308 runtime·lock(&runtime·sched); |
308 n = runtime·gomaxprocs; | 309 n = runtime·gomaxprocs; |
309 if(n > runtime·ncpu) | 310 if(n > runtime·ncpu) |
310 n = runtime·ncpu; | 311 n = runtime·ncpu; |
311 if(n > MaxGcproc) | 312 if(n > MaxGcproc) |
312 n = MaxGcproc; | 313 n = MaxGcproc; |
313 » n -= runtime·sched.mwait+1; // one M is currently running | 314 » n -= runtime·sched.nmidle+1; // one M is currently running |
314 runtime·unlock(&runtime·sched); | 315 runtime·unlock(&runtime·sched); |
315 return n > 0; | 316 return n > 0; |
316 } | 317 } |
317 | 318 |
318 void | 319 void |
319 runtime·helpgc(int32 nproc) | 320 runtime·helpgc(int32 nproc) |
320 { | 321 { |
321 M *mp; | 322 M *mp; |
322 int32 n, pos; | 323 int32 n, pos; |
323 | 324 |
(...skipping 15 matching lines...) Expand all Loading... |
339 | 340 |
340 void | 341 void |
341 runtime·stoptheworld(void) | 342 runtime·stoptheworld(void) |
342 { | 343 { |
343 int32 i; | 344 int32 i; |
344 uint32 s; | 345 uint32 s; |
345 P *p; | 346 P *p; |
346 bool wait; | 347 bool wait; |
347 | 348 |
348 runtime·lock(&runtime·sched); | 349 runtime·lock(&runtime·sched); |
349 runtime·gcwaiting = 1; | |
350 runtime·sched.stopwait = runtime·gomaxprocs; | 350 runtime·sched.stopwait = runtime·gomaxprocs; |
| 351 runtime·atomicstore((uint32*)&runtime·gcwaiting, 1); |
351 // stop current P | 352 // stop current P |
352 m->p->status = Pgcstop; | 353 m->p->status = Pgcstop; |
353 runtime·sched.stopwait--; | 354 runtime·sched.stopwait--; |
354 // try to retake all P's in Psyscall status | 355 // try to retake all P's in Psyscall status |
355 for(i = 0; i < runtime·gomaxprocs; i++) { | 356 for(i = 0; i < runtime·gomaxprocs; i++) { |
356 p = runtime·allp[i]; | 357 p = runtime·allp[i]; |
357 s = p->status; | 358 s = p->status; |
358 if(s == Psyscall && runtime·cas(&p->status, s, Pgcstop)) | 359 if(s == Psyscall && runtime·cas(&p->status, s, Pgcstop)) |
359 runtime·sched.stopwait--; | 360 runtime·sched.stopwait--; |
360 } | 361 } |
(...skipping 20 matching lines...) Expand all Loading... |
381 } | 382 } |
382 | 383 |
383 void | 384 void |
384 runtime·starttheworld(void) | 385 runtime·starttheworld(void) |
385 { | 386 { |
386 P *p; | 387 P *p; |
387 M *mp; | 388 M *mp; |
388 bool add; | 389 bool add; |
389 | 390 |
390 add = needaddgcproc(); | 391 add = needaddgcproc(); |
| 392 runtime·lock(&runtime·sched); |
391 if(newprocs) { | 393 if(newprocs) { |
392 procresize(newprocs); | 394 procresize(newprocs); |
393 newprocs = 0; | 395 newprocs = 0; |
394 } else | 396 } else |
395 procresize(runtime·gomaxprocs); | 397 procresize(runtime·gomaxprocs); |
396 runtime·gcwaiting = 0; | 398 runtime·gcwaiting = 0; |
397 | 399 |
398 runtime·lock(&runtime·sched); | |
399 while(p = pidleget()) { | 400 while(p = pidleget()) { |
| 401 // procresize() puts p's with work at the beginning of the list. |
| 402 // Once we reach a p without a run queue, the rest don't have on
e either. |
400 if(p->runqhead == p->runqtail) { | 403 if(p->runqhead == p->runqtail) { |
401 pidleput(p); | 404 pidleput(p); |
402 break; | 405 break; |
403 } | 406 } |
404 mp = mget(); | 407 mp = mget(); |
405 if(mp == nil) { | 408 if(mp == nil) { |
406 pidleput(p); | 409 pidleput(p); |
407 break; | 410 break; |
408 } | 411 } |
409 if(mp->nextp) | 412 if(mp->nextp) |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
465 acquirep(m->nextp); | 468 acquirep(m->nextp); |
466 m->nextp = nil; | 469 m->nextp = nil; |
467 } | 470 } |
468 schedule(); | 471 schedule(); |
469 | 472 |
470 // TODO(brainman): This point is never reached, because scheduler | 473 // TODO(brainman): This point is never reached, because scheduler |
471 // does not release os threads at the moment. But once this path | 474 // does not release os threads at the moment. But once this path |
472 // is enabled, we must remove our seh here. | 475 // is enabled, we must remove our seh here. |
473 } | 476 } |
474 | 477 |
475 // When running with cgo, we call libcgo_thread_start | 478 // When running with cgo, we call _cgo_thread_start |
476 // to start threads for us so that we can play nicely with | 479 // to start threads for us so that we can play nicely with |
477 // foreign code. | 480 // foreign code. |
478 void (*libcgo_thread_start)(void*); | 481 void (*_cgo_thread_start)(void*); |
479 | 482 |
480 typedef struct CgoThreadStart CgoThreadStart; | 483 typedef struct CgoThreadStart CgoThreadStart; |
481 struct CgoThreadStart | 484 struct CgoThreadStart |
482 { | 485 { |
483 M *m; | 486 M *m; |
484 G *g; | 487 G *g; |
485 void (*fn)(void); | 488 void (*fn)(void); |
486 }; | 489 }; |
487 | 490 |
488 // Allocate a new m unassociated with any thread. | 491 // Allocate a new m unassociated with any thread. |
| 492 // Can use p for allocation context if needed. |
489 M* | 493 M* |
490 runtime·allocm(P *p) | 494 runtime·allocm(P *p) |
491 { | 495 { |
492 M *mp; | 496 M *mp; |
493 static Type *mtype; // The Go type M | 497 static Type *mtype; // The Go type M |
494 | 498 |
495 m->locks++; // disable GC because it can be called from sysmon | 499 m->locks++; // disable GC because it can be called from sysmon |
496 if(m->p == nil) | 500 if(m->p == nil) |
497 acquirep(p); // temporarily borrow p for mallocs in this functi
on | 501 acquirep(p); // temporarily borrow p for mallocs in this functi
on |
498 if(mtype == nil) { | 502 if(mtype == nil) { |
499 Eface e; | 503 Eface e; |
500 runtime·gc_m_ptr(&e); | 504 runtime·gc_m_ptr(&e); |
501 mtype = ((PtrType*)e.type)->elem; | 505 mtype = ((PtrType*)e.type)->elem; |
502 } | 506 } |
| 507 |
503 mp = runtime·cnew(mtype); | 508 mp = runtime·cnew(mtype); |
504 mcommoninit(mp); | 509 mcommoninit(mp); |
505 | 510 |
506 // In case of cgo, pthread_create will make us a stack. | 511 // In case of cgo, pthread_create will make us a stack. |
507 // Windows will layout sched stack on OS stack. | 512 // Windows will layout sched stack on OS stack. |
508 » mp->g0 = runtime·malg(runtime·iscgo || Windows ? -1 : 8192); | 513 » if(runtime·iscgo || Windows) |
| 514 » » mp->g0 = runtime·malg(-1); |
| 515 » else |
| 516 » » mp->g0 = runtime·malg(8192); |
509 | 517 |
510 if(p == m->p) | 518 if(p == m->p) |
511 releasep(); | 519 releasep(); |
512 m->locks--; | 520 m->locks--; |
513 | 521 |
514 return mp; | 522 return mp; |
515 } | 523 } |
516 | 524 |
517 static M* lockextra(bool nilokay); | 525 static M* lockextra(bool nilokay); |
518 static void unlockextra(M*); | 526 static void unlockextra(M*); |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
558 | 566 |
559 // Lock extra list, take head, unlock popped list. | 567 // Lock extra list, take head, unlock popped list. |
560 // nilokay=false is safe here because of the invariant above, | 568 // nilokay=false is safe here because of the invariant above, |
561 // that the extra list always contains or will soon contain | 569 // that the extra list always contains or will soon contain |
562 // at least one m. | 570 // at least one m. |
563 mp = lockextra(false); | 571 mp = lockextra(false); |
564 | 572 |
565 // Set needextram when we've just emptied the list, | 573 // Set needextram when we've just emptied the list, |
566 // so that the eventual call into cgocallbackg will | 574 // so that the eventual call into cgocallbackg will |
567 // allocate a new m for the extra list. We delay the | 575 // allocate a new m for the extra list. We delay the |
568 » // allocation until then so that it can be done | 576 » // allocation until then so that it can be done |
569 // after exitsyscall makes sure it is okay to be | 577 // after exitsyscall makes sure it is okay to be |
570 // running at all (that is, there's no garbage collection | 578 // running at all (that is, there's no garbage collection |
571 » // running right now).» | 579 » // running right now). |
572 mp->needextram = mp->schedlink == nil; | 580 mp->needextram = mp->schedlink == nil; |
573 unlockextra(mp->schedlink); | 581 unlockextra(mp->schedlink); |
574 » | 582 |
575 // Install m and g (= m->g0) and set the stack bounds | 583 // Install m and g (= m->g0) and set the stack bounds |
576 // to match the current stack. We don't actually know | 584 // to match the current stack. We don't actually know |
577 // how big the stack is, like we don't know how big any | 585 // how big the stack is, like we don't know how big any |
578 // scheduling stack is, but we assume there's at least 32 kB, | 586 // scheduling stack is, but we assume there's at least 32 kB, |
579 // which is more than enough for us. | 587 // which is more than enough for us. |
580 runtime·setmg(mp, mp->g0); | 588 runtime·setmg(mp, mp->g0); |
581 g->stackbase = (uintptr)(&x + 1024); | 589 g->stackbase = (uintptr)(&x + 1024); |
582 g->stackguard = (uintptr)(&x - 32*1024); | 590 g->stackguard = (uintptr)(&x - 32*1024); |
583 | 591 |
584 // On windows/386, we need to put an SEH frame (two words) | 592 // On windows/386, we need to put an SEH frame (two words) |
(...skipping 25 matching lines...) Expand all Loading... |
610 gp = runtime·malg(4096); | 618 gp = runtime·malg(4096); |
611 gp->sched.pc = (void*)runtime·goexit; | 619 gp->sched.pc = (void*)runtime·goexit; |
612 gp->sched.sp = gp->stackbase; | 620 gp->sched.sp = gp->stackbase; |
613 gp->sched.g = gp; | 621 gp->sched.g = gp; |
614 gp->status = Gsyscall; | 622 gp->status = Gsyscall; |
615 mp->curg = gp; | 623 mp->curg = gp; |
616 mp->locked = LockInternal; | 624 mp->locked = LockInternal; |
617 mp->lockedg = gp; | 625 mp->lockedg = gp; |
618 gp->lockedm = mp; | 626 gp->lockedm = mp; |
619 // put on allg for garbage collector | 627 // put on allg for garbage collector |
| 628 runtime·lock(&runtime·sched); |
620 if(runtime·lastg == nil) | 629 if(runtime·lastg == nil) |
621 runtime·allg = gp; | 630 runtime·allg = gp; |
622 else | 631 else |
623 runtime·lastg->alllink = gp; | 632 runtime·lastg->alllink = gp; |
624 runtime·lastg = gp; | 633 runtime·lastg = gp; |
| 634 runtime·unlock(&runtime·sched); |
| 635 gp->goid = runtime·xadd64(&runtime·sched.goidgen, 1); |
| 636 if(raceenabled) |
| 637 gp->racectx = runtime·racegostart(runtime·newextram); |
625 | 638 |
626 // Add m to the extra list. | 639 // Add m to the extra list. |
627 mnext = lockextra(true); | 640 mnext = lockextra(true); |
628 mp->schedlink = mnext; | 641 mp->schedlink = mnext; |
629 unlockextra(mp); | 642 unlockextra(mp); |
630 } | 643 } |
631 | 644 |
632 // dropm is called when a cgo callback has called needm but is now | 645 // dropm is called when a cgo callback has called needm but is now |
633 // done with the callback and returning back into the non-Go thread. | 646 // done with the callback and returning back into the non-Go thread. |
634 // It puts the current m back onto the extra list. | 647 // It puts the current m back onto the extra list. |
635 // | 648 // |
636 // The main expense here is the call to signalstack to release the | 649 // The main expense here is the call to signalstack to release the |
637 // m's signal stack, and then the call to needm on the next callback | 650 // m's signal stack, and then the call to needm on the next callback |
638 // from this thread. It is tempting to try to save the m for next time, | 651 // from this thread. It is tempting to try to save the m for next time, |
639 // which would eliminate both these costs, but there might not be | 652 // which would eliminate both these costs, but there might not be |
640 // a next time: the current thread (which Go does not control) might exit. | 653 // a next time: the current thread (which Go does not control) might exit. |
641 // If we saved the m for that thread, there would be an m leak each time | 654 // If we saved the m for that thread, there would be an m leak each time |
642 // such a thread exited. Instead, we acquire and release an m on each | 655 // such a thread exited. Instead, we acquire and release an m on each |
643 // call. These should typically not be scheduling operations, just a few | 656 // call. These should typically not be scheduling operations, just a few |
644 // atomics, so the cost should be small. | 657 // atomics, so the cost should be small. |
645 // | 658 // |
646 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread | 659 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread |
647 // variable using pthread_key_create. Unlike the pthread keys we already use | 660 // variable using pthread_key_create. Unlike the pthread keys we already use |
648 // on OS X, this dummy key would never be read by Go code. It would exist | 661 // on OS X, this dummy key would never be read by Go code. It would exist |
649 // only so that we could register at thread-exit-time destructor. | 662 // only so that we could register at thread-exit-time destructor. |
(...skipping 26 matching lines...) Expand all Loading... |
676 // The caller must unlock the list by storing a new list head | 689 // The caller must unlock the list by storing a new list head |
677 // to runtime.extram. If nilokay is true, then lockextra will | 690 // to runtime.extram. If nilokay is true, then lockextra will |
678 // return a nil list head if that's what it finds. If nilokay is false, | 691 // return a nil list head if that's what it finds. If nilokay is false, |
679 // lockextra will keep waiting until the list head is no longer nil. | 692 // lockextra will keep waiting until the list head is no longer nil. |
680 #pragma textflag 7 | 693 #pragma textflag 7 |
681 static M* | 694 static M* |
682 lockextra(bool nilokay) | 695 lockextra(bool nilokay) |
683 { | 696 { |
684 M *mp; | 697 M *mp; |
685 void (*yield)(void); | 698 void (*yield)(void); |
686 » | 699 |
687 for(;;) { | 700 for(;;) { |
688 mp = runtime·atomicloadp(&runtime·extram); | 701 mp = runtime·atomicloadp(&runtime·extram); |
689 if(mp == MLOCKED) { | 702 if(mp == MLOCKED) { |
690 yield = runtime·osyield; | 703 yield = runtime·osyield; |
691 yield(); | 704 yield(); |
692 continue; | 705 continue; |
693 } | 706 } |
694 if(mp == nil && !nilokay) { | 707 if(mp == nil && !nilokay) { |
695 runtime·usleep(1); | 708 runtime·usleep(1); |
696 continue; | 709 continue; |
(...skipping 14 matching lines...) Expand all Loading... |
711 { | 724 { |
712 runtime·atomicstorep(&runtime·extram, mp); | 725 runtime·atomicstorep(&runtime·extram, mp); |
713 } | 726 } |
714 | 727 |
715 | 728 |
716 // Create a new m. It will start off with a call to fn. | 729 // Create a new m. It will start off with a call to fn. |
717 static void | 730 static void |
718 newm(void(*fn)(void), P *p, bool helpgc, bool spinning) | 731 newm(void(*fn)(void), P *p, bool helpgc, bool spinning) |
719 { | 732 { |
720 M *mp; | 733 M *mp; |
721 » | 734 |
722 mp = runtime·allocm(p); | 735 mp = runtime·allocm(p); |
723 mp->nextp = p; | 736 mp->nextp = p; |
724 mp->helpgc = helpgc; | 737 mp->helpgc = helpgc; |
725 mp->spinning = spinning; | 738 mp->spinning = spinning; |
726 | 739 |
727 if(runtime·iscgo) { | 740 if(runtime·iscgo) { |
728 CgoThreadStart ts; | 741 CgoThreadStart ts; |
729 | 742 |
730 » » if(libcgo_thread_start == nil) | 743 » » if(_cgo_thread_start == nil) |
731 » » » runtime·throw("libcgo_thread_start missing"); | 744 » » » runtime·throw("_cgo_thread_start missing"); |
732 ts.m = mp; | 745 ts.m = mp; |
733 ts.g = mp->g0; | 746 ts.g = mp->g0; |
734 ts.fn = fn; | 747 ts.fn = fn; |
735 » » runtime·asmcgocall(libcgo_thread_start, &ts); | 748 » » runtime·asmcgocall(_cgo_thread_start, &ts); |
736 return; | 749 return; |
737 } | 750 } |
738 runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, fn); | 751 runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, fn); |
739 } | 752 } |
740 | 753 |
741 // Stops execution of the current m until new work is available. | 754 // Stops execution of the current m until new work is available. |
742 // Returns with acquired P. | 755 // Returns with acquired P. |
743 static void | 756 static void |
744 stopm(void) | 757 stopm(void) |
745 { | 758 { |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
798 mp->spinning = spinning; | 811 mp->spinning = spinning; |
799 mp->nextp = p; | 812 mp->nextp = p; |
800 runtime·notewakeup(&mp->park); | 813 runtime·notewakeup(&mp->park); |
801 } | 814 } |
802 | 815 |
803 // Hands off P from syscall or locked M. | 816 // Hands off P from syscall or locked M. |
804 static void | 817 static void |
805 handoffp(P *p) | 818 handoffp(P *p) |
806 { | 819 { |
807 // if it has local work, start it straight away | 820 // if it has local work, start it straight away |
808 » if(p->runqhead != p->runqtail) { | 821 » if(p->runqhead != p->runqtail || runtime·sched.runqsize) { |
809 startm(p, false); | 822 startm(p, false); |
810 return; | 823 return; |
811 } | 824 } |
812 » // no local work and there are already spinning/idle M's, | 825 » // no local work, check that there are no spinning/idle M's, |
813 » // our help is not required | 826 » // otherwise our help is not required |
814 » if(runtime·sched.nmspinning + runtime·sched.npidle > 0) | 827 » if(runtime·sched.nmspinning + runtime·sched.npidle == 0 && // TODO: fas
t atomic |
815 » » goto dontstart; | 828 » » runtime·cas(&runtime·sched.nmspinning, 0, 1)) { |
816 » // our help can be required, but be conservative about spinning threads | 829 » » startm(p, true); |
817 » if(runtime·xadd(&runtime·sched.nmspinning, 1) > 1) { | 830 » » return; |
818 » » runtime·xadd(&runtime·sched.nmspinning, -1); | 831 » } |
819 » » goto dontstart; | |
820 » } | |
821 » startm(p, true); | |
822 » return; | |
823 | |
824 dontstart: | |
825 runtime·lock(&runtime·sched); | 832 runtime·lock(&runtime·sched); |
826 » // need to recheck runtime·sched.runqsize, otherwise can deadlock | 833 » if(runtime·gcwaiting) { |
| 834 » » p->status = Pgcstop; |
| 835 » » if(--runtime·sched.stopwait == 0) |
| 836 » » » runtime·notewakeup(&runtime·sched.stopnote); |
| 837 » » runtime·unlock(&runtime·sched); |
| 838 » » return; |
| 839 » } |
827 if(runtime·sched.runqsize) { | 840 if(runtime·sched.runqsize) { |
828 runtime·unlock(&runtime·sched); | 841 runtime·unlock(&runtime·sched); |
829 startm(p, false); | 842 startm(p, false); |
830 return; | 843 return; |
831 } | 844 } |
832 pidleput(p); | 845 pidleput(p); |
833 runtime·unlock(&runtime·sched); | 846 runtime·unlock(&runtime·sched); |
834 } | 847 } |
835 | 848 |
836 // Tries to add one more P to execute G's. | 849 // Tries to add one more P to execute G's. |
837 // Called when a G is made runnable (newproc, ready). | 850 // Called when a G is made runnable (newproc, ready). |
838 static void | 851 static void |
839 wakep(void) | 852 wakep(void) |
840 { | 853 { |
841 // be conservative about spinning threads | 854 // be conservative about spinning threads |
842 » if(runtime·xadd(&runtime·sched.nmspinning, 1) > 1) { | 855 » if(!runtime·cas(&runtime·sched.nmspinning, 0, 1)) |
843 » » runtime·xadd(&runtime·sched.nmspinning, -1); | |
844 return; | 856 return; |
845 } | |
846 startm(nil, true); | 857 startm(nil, true); |
847 } | 858 } |
848 | 859 |
849 // Stops execution of the current m that is locked to a g until the g is runnabl
e again. | 860 // Stops execution of the current m that is locked to a g until the g is runnabl
e again. |
850 // Returns with acquired P. | 861 // Returns with acquired P. |
851 static void | 862 static void |
852 stoplockedm(void) | 863 stoplockedm(void) |
853 { | 864 { |
854 P *p; | 865 P *p; |
855 | 866 |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
892 | 903 |
893 // Stops the current m for stoptheworld. | 904 // Stops the current m for stoptheworld. |
894 // Returns when the world is restarted. | 905 // Returns when the world is restarted. |
895 static void | 906 static void |
896 gcstopm(void) | 907 gcstopm(void) |
897 { | 908 { |
898 P *p; | 909 P *p; |
899 | 910 |
900 if(!runtime·gcwaiting) | 911 if(!runtime·gcwaiting) |
901 runtime·throw("gcstopm: not waiting for gc"); | 912 runtime·throw("gcstopm: not waiting for gc"); |
| 913 if(m->spinning) { |
| 914 m->spinning = false; |
| 915 runtime·xadd(&runtime·sched.nmspinning, -1); |
| 916 } |
902 p = releasep(); | 917 p = releasep(); |
| 918 runtime·lock(&runtime·sched); |
903 p->status = Pgcstop; | 919 p->status = Pgcstop; |
904 runtime·lock(&runtime·sched); | |
905 if(--runtime·sched.stopwait == 0) | 920 if(--runtime·sched.stopwait == 0) |
906 runtime·notewakeup(&runtime·sched.stopnote); | 921 runtime·notewakeup(&runtime·sched.stopnote); |
907 runtime·unlock(&runtime·sched); | 922 runtime·unlock(&runtime·sched); |
908 stopm(); | 923 stopm(); |
909 } | 924 } |
910 | 925 |
911 // Schedules gp to run on the current M. | 926 // Schedules gp to run on the current M. |
912 // Never returns. | 927 // Never returns. |
913 static void | 928 static void |
914 execute(G *gp) | 929 execute(G *gp) |
915 { | 930 { |
916 int32 hz; | 931 int32 hz; |
917 | 932 |
918 » CHECK(m->locks == 0, ("")); | 933 » if(gp->status != Grunnable) { |
919 » CHECK(g == m->g0, ("execute: not on g0\n")); | 934 » » runtime·printf("execute: bad g status %d\n", gp->status); |
920 » CHECK(gp->status == Grunnable, ("execute: gp=%D gp->status=%d\n", gp->go
id, gp->status)); | 935 » » runtime·throw("execute: bad g status"); |
921 | 936 » } |
| 937 » gp->status = Grunning; |
922 m->p->tick++; | 938 m->p->tick++; |
923 gp->status = Grunning; | |
924 m->curg = gp; | 939 m->curg = gp; |
925 gp->m = m; | 940 gp->m = m; |
926 | 941 |
927 // Check whether the profiler needs to be turned on or off. | 942 // Check whether the profiler needs to be turned on or off. |
928 hz = runtime·sched.profilehz; | 943 hz = runtime·sched.profilehz; |
929 if(m->profilehz != hz) | 944 if(m->profilehz != hz) |
930 runtime·resetcpuprofiler(hz); | 945 runtime·resetcpuprofiler(hz); |
931 | 946 |
932 if(gp->sched.pc == (byte*)runtime·goexit) // kickoff | 947 if(gp->sched.pc == (byte*)runtime·goexit) // kickoff |
933 runtime·gogocallfn(&gp->sched, gp->fnstart); | 948 runtime·gogocallfn(&gp->sched, gp->fnstart); |
934 runtime·gogo(&gp->sched, 0); | 949 runtime·gogo(&gp->sched, 0); |
935 } | 950 } |
936 | 951 |
937 // Finds a runnable goroutine to execute. | 952 // Finds a runnable goroutine to execute. |
938 // Tries to steal from other P's, get g from global queue, polls network connect
ions. | 953 // Tries to steal from other P's and get g from global queue. |
939 static G* | 954 static G* |
940 findrunnable(void) | 955 findrunnable(void) |
941 { | 956 { |
942 G *gp; | 957 G *gp; |
943 P *p; | 958 P *p; |
944 int32 i; | 959 int32 i; |
945 | 960 |
946 top: | 961 top: |
947 if(runtime·gcwaiting) { | 962 if(runtime·gcwaiting) { |
948 gcstopm(); | 963 gcstopm(); |
949 goto top; | 964 goto top; |
950 } | 965 } |
951 // local runq | 966 // local runq |
952 gp = runqget(m->p); | 967 gp = runqget(m->p); |
953 if(gp) | 968 if(gp) |
954 return gp; | 969 return gp; |
955 // global runq | 970 // global runq |
956 if(runtime·sched.runqsize) { | 971 if(runtime·sched.runqsize) { |
957 runtime·lock(&runtime·sched); | 972 runtime·lock(&runtime·sched); |
958 gp = globrunqget(m->p); | 973 gp = globrunqget(m->p); |
959 runtime·unlock(&runtime·sched); | 974 runtime·unlock(&runtime·sched); |
960 if(gp) | 975 if(gp) |
961 return gp; | 976 return gp; |
962 } | 977 } |
963 // If number of spinning M's >= number of busy P's, block. | 978 // If number of spinning M's >= number of busy P's, block. |
964 // This is necessary to prevent excessive CPU consumption | 979 // This is necessary to prevent excessive CPU consumption |
965 // when GOMAXPROCS>>1 but the program parallelism is low. | 980 // when GOMAXPROCS>>1 but the program parallelism is low. |
966 » if(!m->spinning && 2 * runtime·sched.nmspinning >= runtime·gomaxprocs -
runtime·sched.npidle) | 981 » if(!m->spinning && 2 * runtime·sched.nmspinning >= runtime·gomaxprocs -
runtime·sched.npidle) // TODO: fast atomic |
967 goto stop; | 982 goto stop; |
968 if(!m->spinning) { | 983 if(!m->spinning) { |
969 m->spinning = true; | 984 m->spinning = true; |
970 runtime·xadd(&runtime·sched.nmspinning, 1); | 985 runtime·xadd(&runtime·sched.nmspinning, 1); |
971 } | 986 } |
972 // random steal from other P's | 987 // random steal from other P's |
973 for(i = 0; i < 2*runtime·gomaxprocs; i++) { | 988 for(i = 0; i < 2*runtime·gomaxprocs; i++) { |
974 if(runtime·gcwaiting) | 989 if(runtime·gcwaiting) |
975 goto top; | 990 goto top; |
976 p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]; | 991 p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]; |
977 if(p == m->p) | 992 if(p == m->p) |
978 gp = runqget(p); | 993 gp = runqget(p); |
979 else | 994 else |
980 gp = runqsteal(m->p, p); | 995 gp = runqsteal(m->p, p); |
981 if(gp) | 996 if(gp) |
982 return gp; | 997 return gp; |
983 } | 998 } |
984 if(gp) | |
985 return gp; | |
986 stop: | 999 stop: |
987 // return P and block | 1000 // return P and block |
988 runtime·lock(&runtime·sched); | 1001 runtime·lock(&runtime·sched); |
989 if(runtime·gcwaiting) { | 1002 if(runtime·gcwaiting) { |
990 runtime·unlock(&runtime·sched); | 1003 runtime·unlock(&runtime·sched); |
991 goto top; | 1004 goto top; |
992 } | 1005 } |
993 if(runtime·sched.runqsize) { | 1006 if(runtime·sched.runqsize) { |
994 gp = globrunqget(m->p); | 1007 gp = globrunqget(m->p); |
995 runtime·unlock(&runtime·sched); | 1008 runtime·unlock(&runtime·sched); |
996 return gp; | 1009 return gp; |
997 } | 1010 } |
998 p = releasep(); | 1011 p = releasep(); |
999 pidleput(p); | 1012 pidleput(p); |
1000 runtime·unlock(&runtime·sched); | 1013 runtime·unlock(&runtime·sched); |
| 1014 if(m->spinning) { |
| 1015 m->spinning = false; |
| 1016 runtime·xadd(&runtime·sched.nmspinning, -1); |
| 1017 } |
1001 // check all runqueues once again | 1018 // check all runqueues once again |
1002 » if(m->spinning) { //!!! need to check in either case, otherwise deadloc
k!!! | 1019 » for(i = 0; i < runtime·gomaxprocs; i++) { |
1003 » » for(i = 0; i < runtime·gomaxprocs; i++) { | 1020 » » p = runtime·allp[i]; |
1004 » » » p = runtime·allp[i]; | 1021 » » if(p && p->runqhead != p->runqtail) { |
1005 » » » if(p && p->runqhead != p->runqtail) { | 1022 » » » runtime·lock(&runtime·sched); |
1006 » » » » runtime·lock(&runtime·sched); | 1023 » » » p = pidleget(); |
1007 » » » » p = pidleget(); | 1024 » » » runtime·unlock(&runtime·sched); |
1008 » » » » runtime·unlock(&runtime·sched); | 1025 » » » if(p) { |
1009 » » » » if(p) { | 1026 » » » » acquirep(p); |
1010 » » » » » acquirep(p); | 1027 » » » » goto top; |
1011 » » » » » goto top; | |
1012 » » » » } | |
1013 » » » » break; | |
1014 } | 1028 } |
| 1029 break; |
1015 } | 1030 } |
1016 } | 1031 } |
1017 stopm(); | 1032 stopm(); |
1018 goto top; | 1033 goto top; |
1019 } | 1034 } |
1020 | 1035 |
1021 // One round of scheduler: find a runnable goroutine and execute it. | 1036 // One round of scheduler: find a runnable goroutine and execute it. |
1022 // Never returns. | 1037 // Never returns. |
1023 static void | 1038 static void |
1024 schedule(void) | 1039 schedule(void) |
1025 { | 1040 { |
1026 G *gp; | 1041 G *gp; |
1027 | 1042 |
1028 » USED(&gp); | 1043 » if(m->locks) |
1029 » CHECK(m->locks == 0, ("schedule: holding locks\n")); | 1044 » » runtime·throw("schedule: holding locks"); |
1030 » CHECK(m->lockedg == nil, ("schedule: locked M\n")); | |
1031 » CHECK((int32)runtime·sched.nmspinning >= 0, ("schedule: nmspinning < 0\n
")); | |
1032 | 1045 |
1033 top: | 1046 top: |
1034 if(runtime·gcwaiting) { | 1047 if(runtime·gcwaiting) { |
1035 gcstopm(); | 1048 gcstopm(); |
1036 goto top; | 1049 goto top; |
1037 } | 1050 } |
1038 | 1051 |
1039 gp = runqget(m->p); | 1052 gp = runqget(m->p); |
1040 if(gp == nil) | 1053 if(gp == nil) |
1041 gp = findrunnable(); | 1054 gp = findrunnable(); |
1042 | 1055 |
1043 if(m->spinning) { | 1056 if(m->spinning) { |
1044 m->spinning = false; | 1057 m->spinning = false; |
1045 runtime·xadd(&runtime·sched.nmspinning, -1); | 1058 runtime·xadd(&runtime·sched.nmspinning, -1); |
1046 } | 1059 } |
1047 | 1060 |
1048 // M wakeup policy is deliberately somewhat conservative (see nmspinning
handling), | 1061 // M wakeup policy is deliberately somewhat conservative (see nmspinning
handling), |
1049 // so see if we need to wakeup another M here. | 1062 // so see if we need to wakeup another M here. |
1050 » if (m->p->runqhead != m->p->runqtail && runtime·sched.nmspinning == 0 &&
runtime·sched.npidle > 0) | 1063 » if (m->p->runqhead != m->p->runqtail && |
| 1064 » » runtime·sched.nmspinning == 0 && |
| 1065 » » runtime·sched.npidle > 0) // TODO: fast atomic |
1051 wakep(); | 1066 wakep(); |
1052 | 1067 |
1053 if(gp->lockedm) { | 1068 if(gp->lockedm) { |
1054 startlockedm(gp); | 1069 startlockedm(gp); |
1055 goto top; | 1070 goto top; |
1056 } | 1071 } |
1057 | 1072 |
1058 execute(gp); | 1073 execute(gp); |
1059 } | 1074 } |
1060 | 1075 |
1061 // Puts the current goroutine into a waiting state and unlocks the lock. | 1076 // Puts the current goroutine into a waiting state and unlocks the lock. |
1062 // The goroutine can be made runnable again by calling runtime·ready(gp). | 1077 // The goroutine can be made runnable again by calling runtime·ready(gp). |
1063 void | 1078 void |
1064 runtime·park(void(*unlockf)(Lock*), Lock *lock, int8 *reason) | 1079 runtime·park(void(*unlockf)(Lock*), Lock *lock, int8 *reason) |
1065 { | 1080 { |
1066 CHECK(g != m->g0, ("park of g0\n")); | |
1067 CHECK(!m->spinning, ("runtime·park: m is spinning")); | |
1068 m->waitlock = lock; | 1081 m->waitlock = lock; |
1069 m->waitunlockf = unlockf; | 1082 m->waitunlockf = unlockf; |
1070 g->waitreason = reason; | 1083 g->waitreason = reason; |
1071 runtime·mcall(park0); | 1084 runtime·mcall(park0); |
1072 } | 1085 } |
1073 | 1086 |
1074 // runtime·park continuation on g0. | 1087 // runtime·park continuation on g0. |
1075 static void | 1088 static void |
1076 park0(G *gp) | 1089 park0(G *gp) |
1077 { | 1090 { |
1078 CHECK(m->nextp == nil || !m->lockedg, ("park0: nextp != nil\n")); | |
1079 gp->status = Gwaiting; | 1091 gp->status = Gwaiting; |
1080 gp->m = nil; | 1092 gp->m = nil; |
1081 m->curg = nil; | 1093 m->curg = nil; |
1082 if(m->waitunlockf) { | 1094 if(m->waitunlockf) { |
1083 m->waitunlockf(m->waitlock); | 1095 m->waitunlockf(m->waitlock); |
1084 m->waitunlockf = nil; | 1096 m->waitunlockf = nil; |
1085 } | 1097 } |
1086 if(m->lockedg) { | 1098 if(m->lockedg) { |
1087 stoplockedm(); | 1099 stoplockedm(); |
1088 execute(gp); // Never returns. | 1100 execute(gp); // Never returns. |
1089 } | 1101 } |
1090 schedule(); | 1102 schedule(); |
1091 } | 1103 } |
1092 | 1104 |
1093 // Scheduler yield. | 1105 // Scheduler yield. |
1094 void | 1106 void |
1095 runtime·gosched(void) | 1107 runtime·gosched(void) |
1096 { | 1108 { |
1097 runtime·mcall(gosched0); | 1109 runtime·mcall(gosched0); |
1098 } | 1110 } |
1099 | 1111 |
1100 // runtime·gosched continuation on g0. | 1112 // runtime·gosched continuation on g0. |
1101 static void | 1113 static void |
1102 gosched0(G *gp) | 1114 gosched0(G *gp) |
1103 { | 1115 { |
1104 gp->status = Grunnable; | 1116 gp->status = Grunnable; |
1105 gp->m = nil; | 1117 gp->m = nil; |
1106 m->curg = nil; | 1118 m->curg = nil; |
1107 CHECK(m->nextp == nil || !m->lockedg, ("gosched0: nextp != nil\n")); | |
1108 runtime·lock(&runtime·sched); | 1119 runtime·lock(&runtime·sched); |
1109 globrunqput(gp); | 1120 globrunqput(gp); |
1110 runtime·unlock(&runtime·sched); | 1121 runtime·unlock(&runtime·sched); |
1111 if(m->lockedg) { | 1122 if(m->lockedg) { |
1112 stoplockedm(); | 1123 stoplockedm(); |
1113 execute(gp); // Never returns. | 1124 execute(gp); // Never returns. |
1114 } | 1125 } |
1115 schedule(); | 1126 schedule(); |
1116 } | 1127 } |
1117 | 1128 |
| 1129 // Finishes execution of the current goroutine. |
| 1130 void |
| 1131 runtime·goexit(void) |
| 1132 { |
| 1133 if(raceenabled) |
| 1134 runtime·racegoend(); |
| 1135 runtime·mcall(goexit0); |
| 1136 } |
| 1137 |
1118 // runtime·goexit continuation on g0. | 1138 // runtime·goexit continuation on g0. |
1119 static void | 1139 static void |
1120 goexit0(G *gp) | 1140 goexit0(G *gp) |
1121 { | 1141 { |
1122 CHECK(!m->spinning, ("goexit0: m is spinning")); | |
1123 gp->status = Gdead; | 1142 gp->status = Gdead; |
1124 gp->m = nil; | 1143 gp->m = nil; |
1125 gp->lockedm = nil; | 1144 gp->lockedm = nil; |
1126 m->curg = nil; | 1145 m->curg = nil; |
1127 m->lockedg = nil; | 1146 m->lockedg = nil; |
1128 runtime·unwindstack(gp, nil); | 1147 runtime·unwindstack(gp, nil); |
1129 gfput(m->p, gp); | 1148 gfput(m->p, gp); |
1130 schedule(); | 1149 schedule(); |
1131 } | |
1132 | |
1133 // Finishes execution of the current goroutine. | |
1134 void | |
1135 runtime·goexit(void) | |
1136 { | |
1137 if(raceenabled) | |
1138 runtime·racegoend(); | |
1139 runtime·mcall(goexit0); | |
1140 } | 1150 } |
1141 | 1151 |
1142 // The goroutine g is about to enter a system call. | 1152 // The goroutine g is about to enter a system call. |
1143 // Record that it's not using the cpu anymore. | 1153 // Record that it's not using the cpu anymore. |
1144 // This is called only from the go syscall library and cgocall, | 1154 // This is called only from the go syscall library and cgocall, |
1145 // not from the low-level system calls used by the runtime. | 1155 // not from the low-level system calls used by the runtime. |
1146 // | 1156 // |
1147 // Entersyscall cannot split the stack: the runtime·gosave must | 1157 // Entersyscall cannot split the stack: the runtime·gosave must |
1148 // make g->sched refer to the caller's stack segment, because | 1158 // make g->sched refer to the caller's stack segment, because |
1149 // entersyscall is going to return immediately after. | 1159 // entersyscall is going to return immediately after. |
1150 #pragma textflag 7 | 1160 #pragma textflag 7 |
1151 void | 1161 void |
1152 runtime·entersyscall(void) | 1162 ·entersyscall(int32 dummy) |
1153 { | 1163 { |
1154 if(m->profilehz > 0) | 1164 if(m->profilehz > 0) |
1155 runtime·setprof(false); | 1165 runtime·setprof(false); |
1156 | 1166 |
1157 // Leave SP around for gc and traceback. | 1167 // Leave SP around for gc and traceback. |
1158 » runtime·gosave(&g->sched); | 1168 » g->sched.sp = (uintptr)runtime·getcallersp(&dummy); |
| 1169 » g->sched.pc = runtime·getcallerpc(&dummy); |
| 1170 » g->sched.g = g; |
1159 g->gcsp = g->sched.sp; | 1171 g->gcsp = g->sched.sp; |
| 1172 g->gcpc = g->sched.pc; |
1160 g->gcstack = g->stackbase; | 1173 g->gcstack = g->stackbase; |
1161 g->gcguard = g->stackguard; | 1174 g->gcguard = g->stackguard; |
1162 g->status = Gsyscall; | 1175 g->status = Gsyscall; |
1163 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { | 1176 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { |
1164 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", | 1177 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", |
1165 // g->gcsp, g->gcguard-StackGuard, g->gcstack); | 1178 // g->gcsp, g->gcguard-StackGuard, g->gcstack); |
1166 runtime·throw("entersyscall"); | 1179 runtime·throw("entersyscall"); |
1167 } | 1180 } |
1168 | 1181 |
1169 » if(runtime·sched.sysmonwait) { | 1182 » if(runtime·sched.sysmonwait) { // TODO: fast atomic |
1170 runtime·lock(&runtime·sched); | 1183 runtime·lock(&runtime·sched); |
1171 if(runtime·sched.sysmonwait) { | 1184 if(runtime·sched.sysmonwait) { |
1172 runtime·sched.sysmonwait = false; | 1185 runtime·sched.sysmonwait = false; |
1173 runtime·notewakeup(&runtime·sched.sysmonnote); | 1186 runtime·notewakeup(&runtime·sched.sysmonnote); |
1174 } | 1187 } |
1175 runtime·unlock(&runtime·sched); | 1188 runtime·unlock(&runtime·sched); |
1176 runtime·gosave(&g->sched); // re-save for traceback | 1189 runtime·gosave(&g->sched); // re-save for traceback |
1177 } | 1190 } |
1178 | 1191 |
1179 m->mcache = nil; | 1192 m->mcache = nil; |
1180 m->p->tick++; | 1193 m->p->tick++; |
1181 m->p->m = nil; | 1194 m->p->m = nil; |
1182 runtime·atomicstore(&m->p->status, Psyscall); | 1195 runtime·atomicstore(&m->p->status, Psyscall); |
1183 if(runtime·gcwaiting) { | 1196 if(runtime·gcwaiting) { |
1184 runtime·lock(&runtime·sched); | 1197 runtime·lock(&runtime·sched); |
1185 if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psy
scall, Pgcstop)) { | 1198 if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psy
scall, Pgcstop)) { |
1186 » » » runtime·sched.stopwait--; | 1199 » » » if(--runtime·sched.stopwait == 0) |
1187 » » » if(runtime·sched.stopwait == 0) | |
1188 runtime·notewakeup(&runtime·sched.stopnote); | 1200 runtime·notewakeup(&runtime·sched.stopnote); |
1189 } | 1201 } |
1190 runtime·unlock(&runtime·sched); | 1202 runtime·unlock(&runtime·sched); |
1191 runtime·gosave(&g->sched); // re-save for traceback | 1203 runtime·gosave(&g->sched); // re-save for traceback |
1192 } | 1204 } |
1193 } | 1205 } |
1194 | 1206 |
1195 // The same as runtime·entersyscall(), but with a hint that the syscall is block
ing. | 1207 // The same as runtime·entersyscall(), but with a hint that the syscall is block
ing. |
1196 #pragma textflag 7 | 1208 #pragma textflag 7 |
1197 void | 1209 void |
1198 runtime·entersyscallblock(void) | 1210 ·entersyscallblock(int32 dummy) |
1199 { | 1211 { |
1200 P *p; | 1212 P *p; |
1201 | 1213 |
1202 if(m->profilehz > 0) | 1214 if(m->profilehz > 0) |
1203 runtime·setprof(false); | 1215 runtime·setprof(false); |
1204 | 1216 |
1205 // Leave SP around for gc and traceback. | 1217 // Leave SP around for gc and traceback. |
1206 » runtime·gosave(&g->sched); | 1218 » g->sched.sp = (uintptr)runtime·getcallersp(&dummy); |
| 1219 » g->sched.pc = runtime·getcallerpc(&dummy); |
| 1220 » g->sched.g = g; |
1207 g->gcsp = g->sched.sp; | 1221 g->gcsp = g->sched.sp; |
| 1222 g->gcpc = g->sched.pc; |
1208 g->gcstack = g->stackbase; | 1223 g->gcstack = g->stackbase; |
1209 g->gcguard = g->stackguard; | 1224 g->gcguard = g->stackguard; |
1210 g->status = Gsyscall; | 1225 g->status = Gsyscall; |
1211 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { | 1226 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { |
1212 » » // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", | 1227 » » // runtime·printf("entersyscallblock inconsistent %p [%p,%p]\n", |
1213 // g->gcsp, g->gcguard-StackGuard, g->gcstack); | 1228 // g->gcsp, g->gcguard-StackGuard, g->gcstack); |
1214 » » runtime·throw("entersyscall"); | 1229 » » runtime·throw("entersyscallblock"); |
1215 } | 1230 } |
1216 | 1231 |
1217 p = releasep(); | 1232 p = releasep(); |
1218 handoffp(p); | 1233 handoffp(p); |
1219 if(g == scvg) // do not consider blocked scavenger for deadlock detecti
on | 1234 if(g == scvg) // do not consider blocked scavenger for deadlock detecti
on |
1220 inclocked(1); | 1235 inclocked(1); |
1221 runtime·gosave(&g->sched); // re-save for traceback | 1236 runtime·gosave(&g->sched); // re-save for traceback |
1222 } | 1237 } |
1223 | 1238 |
1224 // The goroutine g exited its system call. | 1239 // The goroutine g exited its system call. |
1225 // Arrange for it to run on a cpu again. | 1240 // Arrange for it to run on a cpu again. |
1226 // This is called only from the go syscall library, not | 1241 // This is called only from the go syscall library, not |
1227 // from the low-level system calls used by the runtime. | 1242 // from the low-level system calls used by the runtime. |
1228 void | 1243 void |
1229 runtime·exitsyscall(void) | 1244 runtime·exitsyscall(void) |
1230 { | 1245 { |
1231 uint32 s; | |
1232 P *p; | 1246 P *p; |
1233 | 1247 |
1234 // Check whether the profiler needs to be turned on. | 1248 // Check whether the profiler needs to be turned on. |
1235 if(m->profilehz > 0) | 1249 if(m->profilehz > 0) |
1236 runtime·setprof(true); | 1250 runtime·setprof(true); |
1237 | 1251 |
1238 // Try to re-acquire the last P. | 1252 // Try to re-acquire the last P. |
1239 » s = m->p ? m->p->status : Pidle; | 1253 » if(m->p && m->p->status == Psyscall && runtime·cas(&m->p->status, Psysca
ll, Prunning)) { |
1240 » if(s == Psyscall && runtime·cas(&m->p->status, s, Prunning)) { | |
1241 // There's a cpu for us, so we can run. | 1254 // There's a cpu for us, so we can run. |
1242 m->mcache = m->p->mcache; | 1255 m->mcache = m->p->mcache; |
1243 m->p->m = m; | 1256 m->p->m = m; |
1244 m->p->tick++; | 1257 m->p->tick++; |
1245 g->status = Grunning; | 1258 g->status = Grunning; |
1246 // Garbage collector isn't running (since we are), | 1259 // Garbage collector isn't running (since we are), |
1247 » » // so okay to clear gcstack. | 1260 » » // so okay to clear gcstack and gcsp. |
1248 g->gcstack = (uintptr)nil; | 1261 g->gcstack = (uintptr)nil; |
| 1262 g->gcsp = (uintptr)nil; |
1249 return; | 1263 return; |
1250 } | 1264 } |
1251 | 1265 |
1252 if(g == scvg) // do not consider blocked scavenger for deadlock detecti
on | 1266 if(g == scvg) // do not consider blocked scavenger for deadlock detecti
on |
1253 inclocked(-1); | 1267 inclocked(-1); |
1254 // Try to get any other idle P. | 1268 // Try to get any other idle P. |
1255 m->p = nil; | 1269 m->p = nil; |
1256 if(runtime·sched.pidle) { | 1270 if(runtime·sched.pidle) { |
1257 runtime·lock(&runtime·sched); | 1271 runtime·lock(&runtime·sched); |
1258 p = pidleget(); | 1272 p = pidleget(); |
1259 runtime·unlock(&runtime·sched); | 1273 runtime·unlock(&runtime·sched); |
1260 if(p) { | 1274 if(p) { |
1261 acquirep(p); | 1275 acquirep(p); |
1262 g->gcstack = (uintptr)nil; | 1276 g->gcstack = (uintptr)nil; |
| 1277 g->gcsp = (uintptr)nil; |
1263 return; | 1278 return; |
1264 } | 1279 } |
1265 } | 1280 } |
1266 | 1281 |
1267 // Call the scheduler. | 1282 // Call the scheduler. |
1268 runtime·mcall(exitsyscall0); | 1283 runtime·mcall(exitsyscall0); |
1269 | 1284 |
1270 // Scheduler returned, so we're allowed to run now. | 1285 // Scheduler returned, so we're allowed to run now. |
1271 // Delete the gcstack information that we left for | 1286 // Delete the gcstack information that we left for |
1272 // the garbage collector during the system call. | 1287 // the garbage collector during the system call. |
1273 // Must wait until now because until gosched returns | 1288 // Must wait until now because until gosched returns |
1274 // we don't know for sure that the garbage collector | 1289 // we don't know for sure that the garbage collector |
1275 // is not running. | 1290 // is not running. |
1276 g->gcstack = (uintptr)nil; | 1291 g->gcstack = (uintptr)nil; |
| 1292 g->gcsp = (uintptr)nil; |
1277 } | 1293 } |
1278 | 1294 |
1279 // runtime·exitsyscall slow path on g0. | 1295 // runtime·exitsyscall slow path on g0. |
1280 // Failed to acquire P, enqueue gp as runnable. | 1296 // Failed to acquire P, enqueue gp as runnable. |
1281 static void | 1297 static void |
1282 exitsyscall0(G *gp) | 1298 exitsyscall0(G *gp) |
1283 { | 1299 { |
1284 P *p; | 1300 P *p; |
1285 | 1301 |
1286 gp->status = Grunnable; | 1302 gp->status = Grunnable; |
1287 gp->m = nil; | 1303 gp->m = nil; |
1288 m->curg = nil; | 1304 m->curg = nil; |
1289 CHECK(m->park.waitm == nil, ("exitsyscall0: park is signalled\n")); | |
1290 runtime·lock(&runtime·sched); | 1305 runtime·lock(&runtime·sched); |
1291 p = pidleget(); | 1306 p = pidleget(); |
1292 if(p == nil) | 1307 if(p == nil) |
1293 globrunqput(gp); | 1308 globrunqput(gp); |
1294 runtime·unlock(&runtime·sched); | 1309 runtime·unlock(&runtime·sched); |
1295 if(p) { | 1310 if(p) { |
1296 acquirep(p); | 1311 acquirep(p); |
1297 execute(gp); // Never returns. | 1312 execute(gp); // Never returns. |
1298 } | 1313 } |
1299 if(m->lockedg) { | 1314 if(m->lockedg) { |
1300 CHECK(m->lockedg == gp, ("exitsyscall0: inconsistent locking\n")
); | |
1301 // Wait until another thread schedules gp and so m again. | 1315 // Wait until another thread schedules gp and so m again. |
1302 stoplockedm(); | 1316 stoplockedm(); |
1303 execute(gp); // Never returns. | 1317 execute(gp); // Never returns. |
1304 } | 1318 } |
1305 stopm(); | 1319 stopm(); |
1306 schedule(); // Never returns. | 1320 schedule(); // Never returns. |
1307 } | 1321 } |
1308 | 1322 |
1309 // Hook used by runtime·malg to call runtime·stackalloc on the | 1323 // Hook used by runtime·malg to call runtime·stackalloc on the |
1310 // scheduler stack. This exists because runtime·stackalloc insists | 1324 // scheduler stack. This exists because runtime·stackalloc insists |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1373 // at argp and returning nret bytes of results. callerpc is the | 1387 // at argp and returning nret bytes of results. callerpc is the |
1374 // address of the go statement that created this. The new g is put | 1388 // address of the go statement that created this. The new g is put |
1375 // on the queue of g's waiting to run. | 1389 // on the queue of g's waiting to run. |
1376 G* | 1390 G* |
1377 runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerpc
) | 1391 runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerpc
) |
1378 { | 1392 { |
1379 byte *sp; | 1393 byte *sp; |
1380 G *newg; | 1394 G *newg; |
1381 int32 siz; | 1395 int32 siz; |
1382 | 1396 |
| 1397 //printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret); |
1383 siz = narg + nret; | 1398 siz = narg + nret; |
1384 siz = (siz+7) & ~7; | 1399 siz = (siz+7) & ~7; |
1385 | 1400 |
1386 // We could instead create a secondary stack frame | 1401 // We could instead create a secondary stack frame |
1387 // and make it look like goexit was on the original but | 1402 // and make it look like goexit was on the original but |
1388 // the call to the actual goroutine function was split. | 1403 // the call to the actual goroutine function was split. |
1389 // Not worth it: this is almost always an error. | 1404 // Not worth it: this is almost always an error. |
1390 if(siz > StackMin - 1024) | 1405 if(siz > StackMin - 1024) |
1391 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); | 1406 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); |
1392 | 1407 |
(...skipping 24 matching lines...) Expand all Loading... |
1417 newg->sched.pc = (byte*)runtime·goexit; | 1432 newg->sched.pc = (byte*)runtime·goexit; |
1418 newg->sched.g = newg; | 1433 newg->sched.g = newg; |
1419 newg->fnstart = fn; | 1434 newg->fnstart = fn; |
1420 newg->gopc = (uintptr)callerpc; | 1435 newg->gopc = (uintptr)callerpc; |
1421 newg->status = Grunnable; | 1436 newg->status = Grunnable; |
1422 newg->goid = runtime·xadd64(&runtime·sched.goidgen, 1); | 1437 newg->goid = runtime·xadd64(&runtime·sched.goidgen, 1); |
1423 if(raceenabled) | 1438 if(raceenabled) |
1424 newg->racectx = runtime·racegostart(callerpc); | 1439 newg->racectx = runtime·racegostart(callerpc); |
1425 runqput(m->p, newg); | 1440 runqput(m->p, newg); |
1426 | 1441 |
1427 » if(runtime·sched.npidle != 0 && runtime·sched.nmspinning == 0 && fn->fn
!= runtime·main) | 1442 » if(runtime·sched.npidle != 0 && runtime·sched.nmspinning == 0 && fn->fn
!= runtime·main) // TODO: fast atomic |
1428 wakep(); | 1443 wakep(); |
1429 return newg; | 1444 return newg; |
1430 } | 1445 } |
1431 | 1446 |
| 1447 // Put on gfree list. |
| 1448 // If local list is too long, transfer a batch to the global list. |
| 1449 static void |
| 1450 gfput(P *p, G *gp) |
| 1451 { |
| 1452 if(gp->stackguard - StackGuard != gp->stack0) |
| 1453 runtime·throw("invalid stack in gfput"); |
| 1454 gp->schedlink = p->gfree; |
| 1455 p->gfree = gp; |
| 1456 p->gfreecnt++; |
| 1457 if(p->gfreecnt >= 64) { |
| 1458 runtime·lock(&runtime·sched.gflock); |
| 1459 while(p->gfreecnt >= 32) { |
| 1460 p->gfreecnt--; |
| 1461 gp = p->gfree; |
| 1462 p->gfree = gp->schedlink; |
| 1463 gp->schedlink = runtime·sched.gfree; |
| 1464 runtime·sched.gfree = gp; |
| 1465 } |
| 1466 runtime·unlock(&runtime·sched.gflock); |
| 1467 } |
| 1468 } |
| 1469 |
| 1470 // Get from gfree list. |
| 1471 // If local list is empty, grab a batch from global list. |
| 1472 static G* |
| 1473 gfget(P *p) |
| 1474 { |
| 1475 G *gp; |
| 1476 |
| 1477 retry: |
| 1478 gp = p->gfree; |
| 1479 if(gp == nil && runtime·sched.gfree) { |
| 1480 runtime·lock(&runtime·sched.gflock); |
| 1481 while(p->gfreecnt < 32 && runtime·sched.gfree) { |
| 1482 p->gfreecnt++; |
| 1483 gp = runtime·sched.gfree; |
| 1484 runtime·sched.gfree = gp->schedlink; |
| 1485 gp->schedlink = p->gfree; |
| 1486 p->gfree = gp; |
| 1487 } |
| 1488 runtime·unlock(&runtime·sched.gflock); |
| 1489 goto retry; |
| 1490 } |
| 1491 if(gp) { |
| 1492 p->gfree = gp->schedlink; |
| 1493 p->gfreecnt--; |
| 1494 } |
| 1495 return gp; |
| 1496 } |
| 1497 |
| 1498 // Purge all cached G's from gfree list to the global list. |
| 1499 static void |
| 1500 gfpurge(P *p) |
| 1501 { |
| 1502 G *gp; |
| 1503 |
| 1504 runtime·lock(&runtime·sched.gflock); |
| 1505 while(p->gfreecnt) { |
| 1506 p->gfreecnt--; |
| 1507 gp = p->gfree; |
| 1508 p->gfree = gp->schedlink; |
| 1509 gp->schedlink = runtime·sched.gfree; |
| 1510 runtime·sched.gfree = gp; |
| 1511 } |
| 1512 runtime·unlock(&runtime·sched.gflock); |
| 1513 } |
| 1514 |
1432 void | 1515 void |
1433 runtime·Breakpoint(void) | 1516 runtime·Breakpoint(void) |
1434 { | 1517 { |
1435 runtime·breakpoint(); | 1518 runtime·breakpoint(); |
1436 } | 1519 } |
1437 | 1520 |
1438 void | 1521 void |
1439 runtime·Gosched(void) | 1522 runtime·Gosched(void) |
1440 { | 1523 { |
1441 runtime·gosched(); | 1524 runtime·gosched(); |
1442 } | 1525 } |
1443 | 1526 |
1444 // Implementation of runtime.GOMAXPROCS. | 1527 // Implementation of runtime.GOMAXPROCS. |
| 1528 // delete when scheduler is even stronger |
1445 int32 | 1529 int32 |
1446 runtime·gomaxprocsfunc(int32 n) | 1530 runtime·gomaxprocsfunc(int32 n) |
1447 { | 1531 { |
1448 int32 ret; | 1532 int32 ret; |
1449 | 1533 |
1450 » if(n > maxgomaxprocs) | 1534 » if(n > MaxGomaxprocs) |
1451 » » n = maxgomaxprocs; | 1535 » » n = MaxGomaxprocs; |
1452 runtime·lock(&runtime·sched); | 1536 runtime·lock(&runtime·sched); |
1453 ret = runtime·gomaxprocs; | 1537 ret = runtime·gomaxprocs; |
1454 if(n <= 0 || n == ret) { | 1538 if(n <= 0 || n == ret) { |
1455 runtime·unlock(&runtime·sched); | 1539 runtime·unlock(&runtime·sched); |
1456 return ret; | 1540 return ret; |
1457 } | 1541 } |
1458 runtime·unlock(&runtime·sched); | 1542 runtime·unlock(&runtime·sched); |
1459 | 1543 |
1460 runtime·semacquire(&runtime·worldsema); | 1544 runtime·semacquire(&runtime·worldsema); |
1461 m->gcing = 1; | 1545 m->gcing = 1; |
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1544 } | 1628 } |
1545 | 1629 |
1546 int32 | 1630 int32 |
1547 runtime·gcount(void) | 1631 runtime·gcount(void) |
1548 { | 1632 { |
1549 G *gp; | 1633 G *gp; |
1550 int32 n, s; | 1634 int32 n, s; |
1551 | 1635 |
1552 n = 0; | 1636 n = 0; |
1553 runtime·lock(&runtime·sched); | 1637 runtime·lock(&runtime·sched); |
| 1638 // TODO(dvyukov): runtime.NumGoroutine() is O(N). |
| 1639 // We do not want to increment/decrement centralized counter in newproc/
goexit, |
| 1640 // just to make runtime.NumGoroutine() faster. |
| 1641 // Compromise solution is to introduce per-P counters of active goroutin
es. |
1554 for(gp = runtime·allg; gp; gp = gp->alllink) { | 1642 for(gp = runtime·allg; gp; gp = gp->alllink) { |
1555 s = gp->status; | 1643 s = gp->status; |
1556 if(s == Grunnable || s == Grunning || s == Gsyscall || s == Gwai
ting) | 1644 if(s == Grunnable || s == Grunning || s == Gsyscall || s == Gwai
ting) |
1557 n++; | 1645 n++; |
1558 } | 1646 } |
1559 runtime·unlock(&runtime·sched); | 1647 runtime·unlock(&runtime·sched); |
1560 return n; | 1648 return n; |
1561 } | 1649 } |
1562 | 1650 |
1563 int32 | 1651 int32 |
(...skipping 20 matching lines...) Expand all Loading... |
1584 int32 hz; | 1672 int32 hz; |
1585 uintptr pcbuf[100]; | 1673 uintptr pcbuf[100]; |
1586 } prof; | 1674 } prof; |
1587 | 1675 |
1588 // Called if we receive a SIGPROF signal. | 1676 // Called if we receive a SIGPROF signal. |
1589 void | 1677 void |
1590 runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp) | 1678 runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp) |
1591 { | 1679 { |
1592 int32 n; | 1680 int32 n; |
1593 | 1681 |
1594 » if(m == nil || m->mcache == nil) | 1682 » // Windows does profiling in a dedicated thread w/o m. |
| 1683 » if(!Windows && (m == nil || m->mcache == nil)) |
1595 return; | 1684 return; |
1596 if(prof.fn == nil || prof.hz == 0) | 1685 if(prof.fn == nil || prof.hz == 0) |
1597 return; | 1686 return; |
1598 | 1687 |
1599 runtime·lock(&prof); | 1688 runtime·lock(&prof); |
1600 if(prof.fn == nil) { | 1689 if(prof.fn == nil) { |
1601 runtime·unlock(&prof); | 1690 runtime·unlock(&prof); |
1602 return; | 1691 return; |
1603 } | 1692 } |
1604 n = runtime·gentraceback(pc, sp, lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf
)); | 1693 n = runtime·gentraceback(pc, sp, lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf
)); |
(...skipping 24 matching lines...) Expand all Loading... |
1629 prof.hz = hz; | 1718 prof.hz = hz; |
1630 runtime·unlock(&prof); | 1719 runtime·unlock(&prof); |
1631 runtime·lock(&runtime·sched); | 1720 runtime·lock(&runtime·sched); |
1632 runtime·sched.profilehz = hz; | 1721 runtime·sched.profilehz = hz; |
1633 runtime·unlock(&runtime·sched); | 1722 runtime·unlock(&runtime·sched); |
1634 | 1723 |
1635 if(hz != 0) | 1724 if(hz != 0) |
1636 runtime·resetcpuprofiler(hz); | 1725 runtime·resetcpuprofiler(hz); |
1637 } | 1726 } |
1638 | 1727 |
1639 // Change number of processors. The world is stopped. | 1728 // Change number of processors. The world is stopped, sched is locked. |
1640 static void | 1729 static void |
1641 procresize(int32 new) | 1730 procresize(int32 new) |
1642 { | 1731 { |
1643 int32 i, old; | 1732 int32 i, old; |
1644 G *gp; | 1733 G *gp; |
1645 P *p; | 1734 P *p; |
1646 | 1735 |
1647 runtime·lock(&runtime·sched); | |
1648 old = runtime·gomaxprocs; | 1736 old = runtime·gomaxprocs; |
1649 » if(old < 0 || old > maxgomaxprocs || new <= 0 || new > maxgomaxprocs) | 1737 » if(old < 0 || old > MaxGomaxprocs || new <= 0 || new >MaxGomaxprocs) |
1650 runtime·throw("procresize: invalid arg"); | 1738 runtime·throw("procresize: invalid arg"); |
1651 // initialize new P's | 1739 // initialize new P's |
1652 for(i = 0; i < new; i++) { | 1740 for(i = 0; i < new; i++) { |
1653 p = runtime·allp[i]; | 1741 p = runtime·allp[i]; |
1654 if(p == nil) { | 1742 if(p == nil) { |
1655 p = (P*)runtime·mallocgc(sizeof(*p), 0, 0, 1); | 1743 p = (P*)runtime·mallocgc(sizeof(*p), 0, 0, 1); |
1656 p->status = Pgcstop; | 1744 p->status = Pgcstop; |
1657 » » » runtime·allp[i] = p; //@@@ store-release | 1745 » » » runtime·atomicstorep(&runtime·allp[i], p); |
1658 } | 1746 } |
1659 if(p->mcache == nil) { | 1747 if(p->mcache == nil) { |
1660 if(old==0 && i==0) | 1748 if(old==0 && i==0) |
1661 p->mcache = m->mcache; // bootstrap | 1749 p->mcache = m->mcache; // bootstrap |
1662 else | 1750 else |
1663 p->mcache = runtime·allocmcache(); | 1751 p->mcache = runtime·allocmcache(); |
1664 } | 1752 } |
1665 if(p->runq == nil) { | 1753 if(p->runq == nil) { |
1666 p->runqsize = 128; | 1754 p->runqsize = 128; |
1667 p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*),
0, 0, 1); | 1755 p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*),
0, 0, 1); |
1668 } | 1756 } |
1669 } | 1757 } |
1670 | 1758 |
1671 // redistribute runnable G's evenly | 1759 // redistribute runnable G's evenly |
1672 for(i = 0; i < old; i++) { | 1760 for(i = 0; i < old; i++) { |
1673 p = runtime·allp[i]; | 1761 p = runtime·allp[i]; |
1674 while(gp = runqget(p)) | 1762 while(gp = runqget(p)) |
1675 globrunqput(gp); | 1763 globrunqput(gp); |
1676 } | 1764 } |
1677 » for(i = 0; runtime·sched.runqhead; i++) { | 1765 » // start at 1 because current M already executes some G and will acquire
allp[0] below, |
| 1766 » // so if we have a spare G we want to put it into allp[1]. |
| 1767 » for(i = 1; runtime·sched.runqhead; i++) { |
1678 gp = runtime·sched.runqhead; | 1768 gp = runtime·sched.runqhead; |
1679 runtime·sched.runqhead = gp->schedlink; | 1769 runtime·sched.runqhead = gp->schedlink; |
1680 runqput(runtime·allp[i%new], gp); | 1770 runqput(runtime·allp[i%new], gp); |
1681 } | 1771 } |
1682 runtime·sched.runqtail = nil; | 1772 runtime·sched.runqtail = nil; |
1683 runtime·sched.runqsize = 0; | 1773 runtime·sched.runqsize = 0; |
1684 | 1774 |
1685 // free unused P's | 1775 // free unused P's |
1686 for(i = new; i < old; i++) { | 1776 for(i = new; i < old; i++) { |
1687 p = runtime·allp[i]; | 1777 p = runtime·allp[i]; |
(...skipping 12 matching lines...) Expand all Loading... |
1700 p->m = nil; | 1790 p->m = nil; |
1701 p->status = Pidle; | 1791 p->status = Pidle; |
1702 acquirep(p); | 1792 acquirep(p); |
1703 for(i = new-1; i > 0; i--) { | 1793 for(i = new-1; i > 0; i--) { |
1704 p = runtime·allp[i]; | 1794 p = runtime·allp[i]; |
1705 p->status = Pidle; | 1795 p->status = Pidle; |
1706 pidleput(p); | 1796 pidleput(p); |
1707 } | 1797 } |
1708 runtime·singleproc = new == 1; | 1798 runtime·singleproc = new == 1; |
1709 runtime·atomicstore((uint32*)&runtime·gomaxprocs, new); | 1799 runtime·atomicstore((uint32*)&runtime·gomaxprocs, new); |
1710 runtime·unlock(&runtime·sched); | |
1711 } | 1800 } |
1712 | 1801 |
1713 // Associate p and the current m. | 1802 // Associate p and the current m. |
1714 static void | 1803 static void |
1715 acquirep(P *p) | 1804 acquirep(P *p) |
1716 { | 1805 { |
1717 if(m->p || m->mcache) | 1806 if(m->p || m->mcache) |
1718 runtime·throw("acquirep: already in go"); | 1807 runtime·throw("acquirep: already in go"); |
1719 if(p->m || p->status != Pidle) { | 1808 if(p->m || p->status != Pidle) { |
1720 runtime·printf("acquirep: p->m=%p(%d) p->status=%d\n", p->m, p->
m ? p->m->id : 0, p->status); | 1809 runtime·printf("acquirep: p->m=%p(%d) p->status=%d\n", p->m, p->
m ? p->m->id : 0, p->status); |
(...skipping 20 matching lines...) Expand all Loading... |
1741 runtime·throw("releasep: invalid p state"); | 1830 runtime·throw("releasep: invalid p state"); |
1742 } | 1831 } |
1743 m->p = nil; | 1832 m->p = nil; |
1744 m->mcache = nil; | 1833 m->mcache = nil; |
1745 p->m = nil; | 1834 p->m = nil; |
1746 p->status = Pidle; | 1835 p->status = Pidle; |
1747 return p; | 1836 return p; |
1748 } | 1837 } |
1749 | 1838 |
1750 static void | 1839 static void |
1751 inclocked(int32 v) { | 1840 inclocked(int32 v) |
| 1841 { |
1752 runtime·lock(&runtime·sched); | 1842 runtime·lock(&runtime·sched); |
1753 runtime·sched.mlocked += v; | 1843 runtime·sched.mlocked += v; |
1754 if(v > 0) | 1844 if(v > 0) |
1755 checkdead(); | 1845 checkdead(); |
1756 runtime·unlock(&runtime·sched); | 1846 runtime·unlock(&runtime·sched); |
1757 } | 1847 } |
1758 | 1848 |
1759 // Check for deadlock situation. | 1849 // Check for deadlock situation. |
1760 // The check is based on number of running M's, if 0 -> deadlock. | 1850 // The check is based on number of running M's, if 0 -> deadlock. |
1761 static void | 1851 static void |
1762 checkdead(void) | 1852 checkdead(void) |
1763 { | 1853 { |
1764 » int32 run; | 1854 » G *gp; |
| 1855 » int32 run, grunning, s; |
1765 | 1856 |
1766 // -1 for sysmon | 1857 // -1 for sysmon |
1767 » run = runtime·sched.mcount - runtime·sched.mwait - runtime·sched.mlocked
- 1; | 1858 » run = runtime·sched.mcount - runtime·sched.nmidle - runtime·sched.mlocke
d - 1; |
1768 if(run > 0) | 1859 if(run > 0) |
1769 return; | 1860 return; |
1770 if(run < 0) { | 1861 if(run < 0) { |
1771 » » runtime·printf("checkdead: mwait=%d mlocked=%d mcount=%d\n", | 1862 » » runtime·printf("checkdead: nmidle=%d mlocked=%d mcount=%d\n", |
1772 » » » runtime·sched.mwait, runtime·sched.mlocked, runtime·sche
d.mcount); | 1863 » » » runtime·sched.nmidle, runtime·sched.mlocked, runtime·sch
ed.mcount); |
1773 runtime·throw("checkdead: inconsistent counts"); | 1864 runtime·throw("checkdead: inconsistent counts"); |
1774 } | 1865 } |
| 1866 grunning = 0; |
| 1867 for(gp = runtime·allg; gp; gp = gp->alllink) { |
| 1868 if(gp == scvg) |
| 1869 continue; |
| 1870 s = gp->status; |
| 1871 if(s == Gwaiting) |
| 1872 grunning++; |
| 1873 else if(s == Grunnable || s == Grunning || s == Gsyscall) { |
| 1874 runtime·printf("checkdead: find g %D in status %d\n", gp
->goid, s); |
| 1875 runtime·throw("checkdead: runnable g"); |
| 1876 } |
| 1877 } |
| 1878 if(grunning == 0) // possible if main goroutine calls runtime·Goexit() |
| 1879 runtime·exit(0); |
1775 m->throwing = -1; // do not dump full stacks | 1880 m->throwing = -1; // do not dump full stacks |
1776 runtime·throw("all goroutines are asleep - deadlock!"); | 1881 runtime·throw("all goroutines are asleep - deadlock!"); |
1777 } | 1882 } |
1778 | 1883 |
1779 static void | 1884 static void |
1780 sysmon(void) | 1885 sysmon(void) |
1781 { | 1886 { |
1782 uint32 idle, delay; | 1887 uint32 idle, delay; |
1783 » uint32 ticks[maxgomaxprocs]; | 1888 » uint32 ticks[MaxGomaxprocs]; |
1784 | 1889 |
1785 // This is a special dedicated thread that retakes P's from blocking sys
calls. | 1890 // This is a special dedicated thread that retakes P's from blocking sys
calls. |
1786 // It works w/o mcache nor stackalloc, it may work concurrently with GC. | 1891 // It works w/o mcache nor stackalloc, it may work concurrently with GC. |
1787 runtime·asminit(); | 1892 runtime·asminit(); |
1788 runtime·minit(); | 1893 runtime·minit(); |
1789 | 1894 |
1790 idle = 0; // how many cycles in succession we had not wokeup somebody | 1895 idle = 0; // how many cycles in succession we had not wokeup somebody |
1791 delay = 0; | 1896 delay = 0; |
1792 for(;;) { | 1897 for(;;) { |
1793 if(idle == 0) // start with 20us sleep... | 1898 if(idle == 0) // start with 20us sleep... |
1794 delay = 20; | 1899 delay = 20; |
1795 else if(idle > 50) // start doubling the sleep after 1ms... | 1900 else if(idle > 50) // start doubling the sleep after 1ms... |
1796 delay *= 2; | 1901 delay *= 2; |
1797 if(delay > 10*1000) // up to 10ms | 1902 if(delay > 10*1000) // up to 10ms |
1798 delay = 10*1000; | 1903 delay = 10*1000; |
1799 runtime·usleep(delay); | 1904 runtime·usleep(delay); |
1800 » » if(runtime·gcwaiting || runtime·sched.npidle == runtime·gomaxpro
cs) { | 1905 » » if(runtime·gcwaiting || runtime·sched.npidle == runtime·gomaxpro
cs) { // TODO: fast atomic |
1801 runtime·lock(&runtime·sched); | 1906 runtime·lock(&runtime·sched); |
1802 if(runtime·gcwaiting || runtime·sched.npidle == runtime·
gomaxprocs) { | 1907 if(runtime·gcwaiting || runtime·sched.npidle == runtime·
gomaxprocs) { |
1803 runtime·sched.sysmonwait = true; | 1908 runtime·sched.sysmonwait = true; |
1804 runtime·unlock(&runtime·sched); | 1909 runtime·unlock(&runtime·sched); |
1805 runtime·notesleep(&runtime·sched.sysmonnote); | 1910 runtime·notesleep(&runtime·sched.sysmonnote); |
1806 runtime·noteclear(&runtime·sched.sysmonnote); | 1911 runtime·noteclear(&runtime·sched.sysmonnote); |
1807 CHECK(!runtime·sched.sysmonwait, ("sysmon: sysmo
nwait is set after wait")); | |
1808 idle = 0; | 1912 idle = 0; |
1809 delay = 20; | 1913 delay = 20; |
1810 } else | 1914 } else |
1811 runtime·unlock(&runtime·sched); | 1915 runtime·unlock(&runtime·sched); |
1812 } | 1916 } |
1813 if(retake(ticks)) | 1917 if(retake(ticks)) |
1814 idle = 0; | 1918 idle = 0; |
1815 else | 1919 else |
1816 idle++; | 1920 idle++; |
1817 } | 1921 } |
(...skipping 12 matching lines...) Expand all Loading... |
1830 if(p==nil) | 1934 if(p==nil) |
1831 continue; | 1935 continue; |
1832 t = p->tick; | 1936 t = p->tick; |
1833 if(ticks[i] != t) { | 1937 if(ticks[i] != t) { |
1834 ticks[i] = t; | 1938 ticks[i] = t; |
1835 continue; | 1939 continue; |
1836 } | 1940 } |
1837 s = p->status; | 1941 s = p->status; |
1838 if(s != Psyscall) | 1942 if(s != Psyscall) |
1839 continue; | 1943 continue; |
1840 » » if(p->runqhead == p->runqtail && runtime·sched.nmspinning + runt
ime·sched.npidle > 0) | 1944 » » if(p->runqhead == p->runqtail && runtime·sched.nmspinning + runt
ime·sched.npidle > 0) // TODO: fast atomic |
1841 continue; | 1945 continue; |
1842 // Need to increment number of locked M's before the CAS. | 1946 // Need to increment number of locked M's before the CAS. |
1843 // Otherwise the M from which we retake can exit the syscall, | 1947 // Otherwise the M from which we retake can exit the syscall, |
1844 » » // increment mwait and report deadlock. | 1948 » » // increment nmidle and report deadlock. |
1845 inclocked(-1); | 1949 inclocked(-1); |
1846 if(runtime·cas(&p->status, s, Pidle)) { | 1950 if(runtime·cas(&p->status, s, Pidle)) { |
1847 n++; | 1951 n++; |
1848 handoffp(p); | 1952 handoffp(p); |
1849 } | 1953 } |
1850 inclocked(1); | 1954 inclocked(1); |
1851 } | 1955 } |
1852 return n; | 1956 return n; |
| 1957 } |
| 1958 |
| 1959 // Put mp on midle list. |
| 1960 // Sched must be locked. |
| 1961 static void |
| 1962 mput(M *mp) |
| 1963 { |
| 1964 mp->schedlink = runtime·sched.midle; |
| 1965 runtime·sched.midle = mp; |
| 1966 runtime·sched.nmidle++; |
| 1967 checkdead(); |
| 1968 } |
| 1969 |
| 1970 // Try to get an m from midle list. |
| 1971 // Sched must be locked. |
| 1972 static M* |
| 1973 mget(void) |
| 1974 { |
| 1975 M *mp; |
| 1976 |
| 1977 if((mp = runtime·sched.midle) != nil){ |
| 1978 runtime·sched.midle = mp->schedlink; |
| 1979 runtime·sched.nmidle--; |
| 1980 } |
| 1981 return mp; |
1853 } | 1982 } |
1854 | 1983 |
1855 // Put gp on the global runnable queue. | 1984 // Put gp on the global runnable queue. |
1856 // Sched must be locked. | 1985 // Sched must be locked. |
1857 static void | 1986 static void |
1858 globrunqput(G *gp) | 1987 globrunqput(G *gp) |
1859 { | 1988 { |
1860 gp->schedlink = nil; | 1989 gp->schedlink = nil; |
1861 if(runtime·sched.runqtail) | 1990 if(runtime·sched.runqtail) |
1862 runtime·sched.runqtail->schedlink = gp; | 1991 runtime·sched.runqtail->schedlink = gp; |
(...skipping 23 matching lines...) Expand all Loading... |
1886 runtime·sched.runqhead = gp->schedlink; | 2015 runtime·sched.runqhead = gp->schedlink; |
1887 n--; | 2016 n--; |
1888 while(n--) { | 2017 while(n--) { |
1889 gp1 = runtime·sched.runqhead; | 2018 gp1 = runtime·sched.runqhead; |
1890 runtime·sched.runqhead = gp1->schedlink; | 2019 runtime·sched.runqhead = gp1->schedlink; |
1891 runqput(p, gp1); | 2020 runqput(p, gp1); |
1892 } | 2021 } |
1893 return gp; | 2022 return gp; |
1894 } | 2023 } |
1895 | 2024 |
1896 // Put on gfree list. | |
1897 // If local list is too long, transfer a batch to the global list. | |
1898 static void | |
1899 gfput(P *p, G *gp) | |
1900 { | |
1901 if(gp->stackguard - StackGuard != gp->stack0) | |
1902 runtime·throw("invalid stack in gfput"); | |
1903 gp->schedlink = p->gfree; | |
1904 p->gfree = gp; | |
1905 p->gfreecnt++; | |
1906 if(p->gfreecnt >= 64) { | |
1907 runtime·lock(&runtime·sched.gflock); | |
1908 while(p->gfreecnt >= 32) { | |
1909 p->gfreecnt--; | |
1910 gp = p->gfree; | |
1911 p->gfree = gp->schedlink; | |
1912 gp->schedlink = runtime·sched.gfree; | |
1913 runtime·sched.gfree = gp; | |
1914 } | |
1915 runtime·unlock(&runtime·sched.gflock); | |
1916 } | |
1917 } | |
1918 | |
1919 // Get from gfree list. | |
1920 // If local list is empty, grab a batch from global list. | |
1921 static G* | |
1922 gfget(P *p) | |
1923 { | |
1924 G *gp; | |
1925 | |
1926 retry: | |
1927 gp = p->gfree; | |
1928 if(gp == nil && runtime·sched.gfree) { | |
1929 runtime·lock(&runtime·sched.gflock); | |
1930 while(p->gfreecnt < 32 && runtime·sched.gfree) { | |
1931 p->gfreecnt++; | |
1932 gp = runtime·sched.gfree; | |
1933 runtime·sched.gfree = gp->schedlink; | |
1934 gp->schedlink = p->gfree; | |
1935 p->gfree = gp; | |
1936 } | |
1937 runtime·unlock(&runtime·sched.gflock); | |
1938 goto retry; | |
1939 } | |
1940 if(gp) { | |
1941 p->gfree = gp->schedlink; | |
1942 p->gfreecnt--; | |
1943 } | |
1944 return gp; | |
1945 } | |
1946 | |
1947 // Purge all cached G's from gfree list to the global list. | |
1948 static void | |
1949 gfpurge(P *p) | |
1950 { | |
1951 G *gp; | |
1952 | |
1953 runtime·lock(&runtime·sched.gflock); | |
1954 while(p->gfreecnt) { | |
1955 p->gfreecnt--; | |
1956 gp = p->gfree; | |
1957 p->gfree = gp->schedlink; | |
1958 gp->schedlink = runtime·sched.gfree; | |
1959 runtime·sched.gfree = gp; | |
1960 } | |
1961 runtime·unlock(&runtime·sched.gflock); | |
1962 } | |
1963 | |
1964 // Put mp on mwait list. | |
1965 // Sched must be locked. | |
1966 static void | |
1967 mput(M *mp) | |
1968 { | |
1969 mp->schedlink = runtime·sched.mhead; | |
1970 runtime·sched.mhead = mp; | |
1971 runtime·sched.mwait++; | |
1972 checkdead(); | |
1973 } | |
1974 | |
1975 // Try to get an m from mwait list. | |
1976 // Sched must be locked. | |
1977 static M* | |
1978 mget(void) | |
1979 { | |
1980 M *mp; | |
1981 | |
1982 if((mp = runtime·sched.mhead) != nil){ | |
1983 runtime·sched.mhead = mp->schedlink; | |
1984 runtime·sched.mwait--; | |
1985 } | |
1986 return mp; | |
1987 } | |
1988 | |
1989 // Put p to on pidle list. | 2025 // Put p to on pidle list. |
1990 // Sched must be locked. | 2026 // Sched must be locked. |
1991 static void | 2027 static void |
1992 pidleput(P *p) | 2028 pidleput(P *p) |
1993 { | 2029 { |
1994 CHECK(p->status == Pidle, ("pidleput: not idle (%d)\n", p->status)); | |
1995 p->link = runtime·sched.pidle; | 2030 p->link = runtime·sched.pidle; |
1996 runtime·sched.pidle = p; | 2031 runtime·sched.pidle = p; |
1997 » runtime·sched.npidle++; | 2032 » runtime·sched.npidle++; // TODO: fast atomic |
1998 } | 2033 } |
1999 | 2034 |
2000 // Try get a p from pidle list. | 2035 // Try get a p from pidle list. |
2001 // Sched must be locked. | 2036 // Sched must be locked. |
2002 static P* | 2037 static P* |
2003 pidleget(void) | 2038 pidleget(void) |
2004 { | 2039 { |
2005 P *p; | 2040 P *p; |
2006 | 2041 |
2007 p = runtime·sched.pidle; | 2042 p = runtime·sched.pidle; |
2008 if(p) { | 2043 if(p) { |
2009 runtime·sched.pidle = p->link; | 2044 runtime·sched.pidle = p->link; |
2010 » » runtime·sched.npidle--; | 2045 » » runtime·sched.npidle--; // TODO: fast atomic |
2011 } | 2046 } |
2012 return p; | 2047 return p; |
2013 } | 2048 } |
2014 | 2049 |
2015 // Put g on local runnable queue. | 2050 // Put g on local runnable queue. |
2016 // TODO(dvyukov): consider using lock-free queue. | 2051 // TODO(dvyukov): consider using lock-free queue. |
2017 static void | 2052 static void |
2018 runqput(P *p, G *gp) | 2053 runqput(P *p, G *gp) |
2019 { | 2054 { |
2020 int32 h, t, s; | 2055 int32 h, t, s; |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2066 static void | 2101 static void |
2067 runqgrow(P *p) | 2102 runqgrow(P *p) |
2068 { | 2103 { |
2069 G **q; | 2104 G **q; |
2070 int32 s, t, h, t2; | 2105 int32 s, t, h, t2; |
2071 | 2106 |
2072 h = p->runqhead; | 2107 h = p->runqhead; |
2073 t = p->runqtail; | 2108 t = p->runqtail; |
2074 s = p->runqsize; | 2109 s = p->runqsize; |
2075 t2 = 0; | 2110 t2 = 0; |
2076 » q = (G**)runtime·malloc(2*s*sizeof(*q)); | 2111 » q = runtime·malloc(2*s*sizeof(*q)); |
2077 while(t != h) { | 2112 while(t != h) { |
2078 q[t2++] = p->runq[h++]; | 2113 q[t2++] = p->runq[h++]; |
2079 if(h == s) | 2114 if(h == s) |
2080 h = 0; | 2115 h = 0; |
2081 } | 2116 } |
2082 runtime·free(p->runq); | 2117 runtime·free(p->runq); |
2083 p->runq = q; | 2118 p->runq = q; |
2084 p->runqhead = 0; | 2119 p->runqhead = 0; |
2085 p->runqtail = t2; | 2120 p->runqtail = t2; |
2086 p->runqsize = 2*s; | 2121 p->runqsize = 2*s; |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2150 runtime·testSchedLocalQueue(void) | 2185 runtime·testSchedLocalQueue(void) |
2151 { | 2186 { |
2152 P p; | 2187 P p; |
2153 G gs[1000]; | 2188 G gs[1000]; |
2154 int32 i, j; | 2189 int32 i, j; |
2155 | 2190 |
2156 runtime·memclr((byte*)&p, sizeof(p)); | 2191 runtime·memclr((byte*)&p, sizeof(p)); |
2157 p.runqsize = 1; | 2192 p.runqsize = 1; |
2158 p.runqhead = 0; | 2193 p.runqhead = 0; |
2159 p.runqtail = 0; | 2194 p.runqtail = 0; |
2160 » p.runq = (G**)runtime·malloc(p.runqsize*sizeof(*p.runq)); | 2195 » p.runq = runtime·malloc(p.runqsize*sizeof(*p.runq)); |
2161 | 2196 |
2162 for(i = 0; i < nelem(gs); i++) { | 2197 for(i = 0; i < nelem(gs); i++) { |
2163 if(runqget(&p) != nil) | 2198 if(runqget(&p) != nil) |
2164 runtime·throw("runq is not empty initially"); | 2199 runtime·throw("runq is not empty initially"); |
2165 for(j = 0; j < i; j++) | 2200 for(j = 0; j < i; j++) |
2166 runqput(&p, &gs[i]); | 2201 runqput(&p, &gs[i]); |
2167 for(j = 0; j < i; j++) { | 2202 for(j = 0; j < i; j++) { |
2168 if(runqget(&p) != &gs[i]) { | 2203 if(runqget(&p) != &gs[i]) { |
2169 runtime·printf("bad element at iter %d/%d\n", i,
j); | 2204 runtime·printf("bad element at iter %d/%d\n", i,
j); |
2170 runtime·throw("bad element"); | 2205 runtime·throw("bad element"); |
2171 } | 2206 } |
2172 } | 2207 } |
2173 if(runqget(&p) != nil) | 2208 if(runqget(&p) != nil) |
2174 runtime·throw("runq is not empty afterwards"); | 2209 runtime·throw("runq is not empty afterwards"); |
2175 } | 2210 } |
2176 } | 2211 } |
2177 | 2212 |
2178 void | 2213 void |
2179 runtime·testSchedLocalQueueSteal(void) | 2214 runtime·testSchedLocalQueueSteal(void) |
2180 { | 2215 { |
2181 P p1, p2; | 2216 P p1, p2; |
2182 G gs[1000], *gp; | 2217 G gs[1000], *gp; |
2183 int32 i, j, s; | 2218 int32 i, j, s; |
2184 | 2219 |
2185 runtime·memclr((byte*)&p1, sizeof(p1)); | 2220 runtime·memclr((byte*)&p1, sizeof(p1)); |
2186 p1.runqsize = 1; | 2221 p1.runqsize = 1; |
2187 p1.runqhead = 0; | 2222 p1.runqhead = 0; |
2188 p1.runqtail = 0; | 2223 p1.runqtail = 0; |
2189 » p1.runq = (G**)runtime·malloc(p1.runqsize*sizeof(*p1.runq)); | 2224 » p1.runq = runtime·malloc(p1.runqsize*sizeof(*p1.runq)); |
2190 | 2225 |
2191 runtime·memclr((byte*)&p2, sizeof(p2)); | 2226 runtime·memclr((byte*)&p2, sizeof(p2)); |
2192 p2.runqsize = nelem(gs); | 2227 p2.runqsize = nelem(gs); |
2193 p2.runqhead = 0; | 2228 p2.runqhead = 0; |
2194 p2.runqtail = 0; | 2229 p2.runqtail = 0; |
2195 » p2.runq = (G**)runtime·malloc(p2.runqsize*sizeof(*p2.runq)); | 2230 » p2.runq = runtime·malloc(p2.runqsize*sizeof(*p2.runq)); |
2196 | 2231 |
2197 for(i = 0; i < nelem(gs); i++) { | 2232 for(i = 0; i < nelem(gs); i++) { |
2198 for(j = 0; j < i; j++) { | 2233 for(j = 0; j < i; j++) { |
2199 gs[j].sig = 0; | 2234 gs[j].sig = 0; |
2200 runqput(&p1, &gs[j]); | 2235 runqput(&p1, &gs[j]); |
2201 } | 2236 } |
2202 gp = runqsteal(&p2, &p1); | 2237 gp = runqsteal(&p2, &p1); |
2203 s = 0; | 2238 s = 0; |
2204 if(gp) { | 2239 if(gp) { |
2205 s++; | 2240 s++; |
2206 gp->sig++; | 2241 gp->sig++; |
2207 } | 2242 } |
2208 while(gp = runqget(&p2)) { | 2243 while(gp = runqget(&p2)) { |
2209 s++; | 2244 s++; |
2210 gp->sig++; | 2245 gp->sig++; |
2211 } | 2246 } |
2212 while(gp = runqget(&p1)) | 2247 while(gp = runqget(&p1)) |
2213 gp->sig++; | 2248 gp->sig++; |
2214 for(j = 0; j < i; j++) { | 2249 for(j = 0; j < i; j++) { |
2215 if(gs[j].sig != 1) { | 2250 if(gs[j].sig != 1) { |
2216 runtime·printf("bad element %d(%d) at iter %d\n"
, j, gs[j].sig, i); | 2251 runtime·printf("bad element %d(%d) at iter %d\n"
, j, gs[j].sig, i); |
2217 runtime·throw("bad element"); | 2252 runtime·throw("bad element"); |
2218 } | 2253 } |
2219 } | 2254 } |
2220 if(s != i/2 && s != i/2+1) { | 2255 if(s != i/2 && s != i/2+1) { |
2221 runtime·printf("bad steal %d, want %d or %d, iter %d\n", | 2256 runtime·printf("bad steal %d, want %d or %d, iter %d\n", |
2222 » » » » » » s, i/2, i/2+1, i); | 2257 » » » » s, i/2, i/2+1, i); |
2223 runtime·throw("bad steal"); | 2258 runtime·throw("bad steal"); |
2224 } | 2259 } |
2225 } | 2260 } |
2226 } | 2261 } |
| 2262 |
LEFT | RIGHT |