LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "malloc.h" | 7 #include "malloc.h" |
8 #include "stack.h" | 8 #include "stack.h" |
9 #include "race.h" | 9 #include "race.h" |
10 #include "type.h" | 10 #include "type.h" |
11 | 11 |
12 //!!! deadlock detection | 12 // Goroutine scheduler |
13 | 13 // The scheduler's job is to distribute ready-to-run goroutines over worker thre
ads. |
14 G* | |
15 runtime·netwait(uint32 ms, uint32 cnt); | |
16 | |
17 enum { maxgomaxprocs = 1<<10 }; | |
18 #define LOG if(0) runtime·printf | |
19 #define LOG1 runtime·printf | |
20 #define CHECK(cond, fmt) if(cond) {} else { runtime·printf fmt; runtime·throw("C
HECK"); } | |
21 | |
22 G*» runtime·allg; | |
23 G*» runtime·lastg; | |
24 M*» runtime·allm; | |
25 | |
26 int8*» runtime·goos; | |
27 int32» runtime·ncpu; | |
28 | |
29 // Go scheduler | |
30 // | 14 // |
31 // The go scheduler's job is to match ready-to-run goroutines (`g's) | 15 // The main concepts are: |
32 // with waiting-for-work schedulers (`m's). If there are ready g's | 16 // G - goroutine. |
33 // and no waiting m's, ready() will start a new m running in a new | 17 // M - worker thread, or machine. |
34 // OS thread, so that all ready g's can run simultaneously, up to a limit. | 18 // P - processor, a resource that is required to execute Go code. |
35 // For now, m's never go away. | 19 // M must have an associated P to execute Go code, however it can be |
36 // | 20 // blocked or in a syscall w/o an associated P. |
37 // By default, Go keeps only one kernel thread (m) running user code | |
38 // at a single time; other threads may be blocked in the operating system. | |
39 // Setting the environment variable $GOMAXPROCS or calling | |
40 // runtime.GOMAXPROCS() will change the number of user threads | |
41 // allowed to execute simultaneously. $GOMAXPROCS is thus an | |
42 // approximation of the maximum number of cores to use. | |
43 // | |
44 // Even a program that can run without deadlock in a single process | |
45 // might use more m's if given the chance. For example, the prime | |
46 // sieve will use as many m's as there are primes (up to $GOMAXPROCS), | |
47 // allowing different stages of the pipeline to execute in parallel. | |
48 | 21 |
49 typedef struct Sched Sched; | 22 typedef struct Sched Sched; |
50 struct Sched { | 23 struct Sched { |
51 Lock; | 24 Lock; |
52 | 25 |
53 » M*» mhead;» // m's waiting for work | 26 » uint64» goidgen; |
54 » int32» mwait;» // number of m's waiting for work | 27 |
| 28 » M*» midle;» // idle m's waiting for work |
| 29 » int32» nmidle;» // number of idle m's waiting for work |
55 int32 mlocked; // number of locked m's waiting for work | 30 int32 mlocked; // number of locked m's waiting for work |
56 int32 mcount; // number of m's that have been created | 31 int32 mcount; // number of m's that have been created |
57 | 32 |
58 P* pidle; // idle P's | 33 P* pidle; // idle P's |
59 » int32» npidle; | 34 » uint32» npidle; |
| 35 » uint32» nmspinning; |
60 | 36 |
61 // Global runnable queue. | 37 // Global runnable queue. |
62 G* runqhead; | 38 G* runqhead; |
63 G* runqtail; | 39 G* runqtail; |
64 int32 runqsize; | 40 int32 runqsize; |
65 | 41 |
66 // Global cache of dead G's. | 42 // Global cache of dead G's. |
67 » Lock gflock; | 43 » Lock» gflock; |
68 G* gfree; | 44 G* gfree; |
69 » int32» goidseq; | 45 |
70 | |
71 » volatile uint32»poller; | |
72 int32 stopwait; | 46 int32 stopwait; |
73 Note stopnote; | 47 Note stopnote; |
74 » int32» sysmonwait; | 48 » bool» sysmonwait; |
75 Note sysmonnote; | 49 Note sysmonnote; |
76 | 50 |
77 int32 profilehz; // cpu profiling rate | 51 int32 profilehz; // cpu profiling rate |
78 bool init; // running initialization | |
79 bool lockmain; // init called runtime.LockOSThread | |
80 }; | 52 }; |
| 53 |
| 54 // The max value of GOMAXPROCS. |
| 55 // There are no fundamental restrictions on the value. |
| 56 enum { MaxGomaxprocs = 1<<8 }; |
81 | 57 |
82 Sched runtime·sched; | 58 Sched runtime·sched; |
83 int32 runtime·gomaxprocs; | 59 int32 runtime·gomaxprocs; |
84 bool runtime·singleproc; | 60 bool runtime·singleproc; |
85 bool runtime·iscgo; | 61 bool runtime·iscgo; |
86 int32 runtime·gcwaiting; | 62 int32 runtime·gcwaiting; |
87 M runtime·m0; | 63 M runtime·m0; |
88 G» runtime·g0;» // idle goroutine for m0 | 64 G» runtime·g0;» // idle goroutine for m0 |
| 65 G*» runtime·allg; |
| 66 G*» runtime·lastg; |
| 67 M*» runtime·allm; |
| 68 M*» runtime·extram; |
| 69 int8*» runtime·goos; |
| 70 int32» runtime·ncpu; |
89 static int32 newprocs; | 71 static int32 newprocs; |
90 | |
91 // Keep trace of scavenger's goroutine for deadlock detection. | 72 // Keep trace of scavenger's goroutine for deadlock detection. |
92 static G *scvg; | 73 static G *scvg; |
93 | 74 |
94 // Scheduling helpers. Sched must be locked. | 75 void runtime·mstart(void); |
95 static void runqput(P*, G*);» // put/get on ghead/gtail | 76 static void runqput(P*, G*); |
96 static G* runqget(P*); | 77 static G* runqget(P*); |
97 static void runqgrow(P*); | 78 static void runqgrow(P*); |
98 static G* runqsteal(P*, P*); | 79 static G* runqsteal(P*, P*); |
99 static void globrunqput(G*); | 80 static void mput(M*); |
100 static G* globrunqget(void); | 81 static M* mget(void); |
101 static void mput(M*);» // put/get on mhead | |
102 static M* mtryget(void); | |
103 static void gfput(P*, G*);» // put/get on gfree | |
104 static G* gfget(P*); | |
105 static void mcommoninit(M*); | 82 static void mcommoninit(M*); |
106 static void schedule(void); | 83 static void schedule(void); |
107 static void procresize(int32); | 84 static void procresize(int32); |
108 static void acquirep(M*, P*); | 85 static void acquirep(P*); |
109 static P* releasep(void); | 86 static P* releasep(void); |
110 static void newm(void(*)(void), P*, bool); | 87 static void newm(void(*)(void), P*, bool, bool); |
111 static void goidle(void); | 88 static void goidle(void); |
112 static void stopm(void); | 89 static void stopm(void); |
113 static bool startm(P*); | 90 static void startm(P*, bool); |
| 91 static void handoffp(P*); |
| 92 static void wakep(void); |
114 static void stoplockedm(void); | 93 static void stoplockedm(void); |
115 static void startlockedm(G*); | 94 static void startlockedm(G*); |
116 static void sysmon(void); | 95 static void sysmon(void); |
117 void runtime·inject(G*, int32*, int32*); | 96 static uint32 retake(uint32*); |
| 97 static void inclocked(int32); |
| 98 static void checkdead(void); |
| 99 static void exitsyscall0(G*); |
| 100 static void park0(G*); |
| 101 static void gosched0(G*); |
| 102 static void goexit0(G*); |
| 103 static void gfput(P*, G*); |
| 104 static G* gfget(P*); |
| 105 static void gfpurge(P*); |
| 106 static void globrunqput(G*); |
| 107 static G* globrunqget(P*); |
118 static P* pidleget(void); | 108 static P* pidleget(void); |
119 static void pidleput(P*); | 109 static void pidleput(P*); |
120 static void checkdead(void); | |
121 | 110 |
122 // The bootstrap sequence is: | 111 // The bootstrap sequence is: |
123 // | 112 // |
124 // call osinit | 113 // call osinit |
125 // call schedinit | 114 // call schedinit |
126 // make & queue new G | 115 // make & queue new G |
127 // call runtime·mstart | 116 // call runtime·mstart |
128 // | 117 // |
129 // The new G calls runtime·main. | 118 // The new G calls runtime·main. |
130 void | 119 void |
131 runtime·schedinit(void) | 120 runtime·schedinit(void) |
132 { | 121 { |
133 int32 n, procs; | 122 int32 n, procs; |
134 byte *p; | 123 byte *p; |
135 | 124 |
136 LOG("%d: runtime·schedinit\n", m->id); | |
137 m->nomemprof++; | 125 m->nomemprof++; |
138 runtime·mprofinit(); | 126 runtime·mprofinit(); |
139 runtime·mallocinit(); | 127 runtime·mallocinit(); |
140 mcommoninit(m); | 128 mcommoninit(m); |
141 | 129 |
142 runtime·goargs(); | 130 runtime·goargs(); |
143 runtime·goenvs(); | 131 runtime·goenvs(); |
144 | 132 |
145 // For debugging: | 133 // For debugging: |
146 // Allocate internal symbol table representation now, | 134 // Allocate internal symbol table representation now, |
147 // so that we don't need to call malloc when we crash. | 135 // so that we don't need to call malloc when we crash. |
148 // runtime·findfunc(0); | 136 // runtime·findfunc(0); |
149 | 137 |
150 procs = 1; | 138 procs = 1; |
151 p = runtime·getenv("GOMAXPROCS"); | 139 p = runtime·getenv("GOMAXPROCS"); |
152 if(p != nil && (n = runtime·atoi(p)) > 0) { | 140 if(p != nil && (n = runtime·atoi(p)) > 0) { |
153 » » if(n > maxgomaxprocs) | 141 » » if(n > MaxGomaxprocs) |
154 » » » n = maxgomaxprocs; | 142 » » » n = MaxGomaxprocs; |
155 procs = n; | 143 procs = n; |
156 } | 144 } |
157 » runtime·allp = (P**)runtime·malloc((maxgomaxprocs+1)*sizeof(runtime·allp
[0])); | 145 » runtime·allp = runtime·malloc((MaxGomaxprocs+1)*sizeof(runtime·allp[0]))
; |
158 procresize(procs); | 146 procresize(procs); |
159 | 147 |
160 mstats.enablegc = 1; | 148 mstats.enablegc = 1; |
161 m->nomemprof--; | 149 m->nomemprof--; |
162 | 150 |
163 if(raceenabled) | 151 if(raceenabled) |
164 g->racectx = runtime·raceinit(); | 152 g->racectx = runtime·raceinit(); |
165 } | 153 } |
166 | 154 |
167 extern void main·init(void); | 155 extern void main·init(void); |
168 extern void main·main(void); | 156 extern void main·main(void); |
169 | 157 |
| 158 static FuncVal scavenger = {runtime·MHeap_Scavenger}; |
| 159 |
170 // The main goroutine. | 160 // The main goroutine. |
171 void | 161 void |
172 runtime·main(void) | 162 runtime·main(void) |
173 { | 163 { |
174 » LOG("%d: runtime·main\n", m->id); | 164 » newm(sysmon, nil, false, false); |
175 | |
176 » newm(sysmon, nil, false); | |
177 | 165 |
178 // Lock the main goroutine onto this, the main OS thread, | 166 // Lock the main goroutine onto this, the main OS thread, |
179 // during initialization. Most programs won't care, but a few | 167 // during initialization. Most programs won't care, but a few |
180 // do require certain calls to be made by the main thread. | 168 // do require certain calls to be made by the main thread. |
181 // Those can arrange for main.main to run in the main thread | 169 // Those can arrange for main.main to run in the main thread |
182 // by calling runtime.LockOSThread during initialization | 170 // by calling runtime.LockOSThread during initialization |
183 // to preserve the lock. | 171 // to preserve the lock. |
184 runtime·lockOSThread(); | 172 runtime·lockOSThread(); |
185 runtime·sched.init = true; | |
186 if(m != &runtime·m0) | 173 if(m != &runtime·m0) |
187 runtime·throw("runtime·main not on m0"); | 174 runtime·throw("runtime·main not on m0"); |
188 » scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runti
me·main); | 175 » scvg = runtime·newproc1(&scavenger, nil, 0, 0, runtime·main); |
189 scvg->issystem = true; | 176 scvg->issystem = true; |
190 runtime·gosched(); // kickoff scavenger for deadlock detector | |
191 main·init(); | 177 main·init(); |
192 runtime·sched.init = false; | |
193 runtime·unlockOSThread(); | 178 runtime·unlockOSThread(); |
194 | 179 |
195 main·main(); | 180 main·main(); |
196 if(raceenabled) | 181 if(raceenabled) |
197 runtime·racefini(); | 182 runtime·racefini(); |
| 183 |
| 184 // Make racy client program work: if panicking on |
| 185 // another goroutine at the same time as main returns, |
| 186 // let the other goroutine finish printing the panic trace. |
| 187 // Once it does, it will exit. See issue 3934. |
| 188 if(runtime·panicking) |
| 189 runtime·park(nil, nil, "panicwait"); |
| 190 |
198 runtime·exit(0); | 191 runtime·exit(0); |
199 for(;;) | 192 for(;;) |
200 *(int32*)runtime·main = 0; | 193 *(int32*)runtime·main = 0; |
201 } | 194 } |
202 | 195 |
203 void | 196 void |
204 runtime·goroutineheader(G *gp) | 197 runtime·goroutineheader(G *gp) |
205 { | 198 { |
206 int8 *status; | 199 int8 *status; |
207 | 200 |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
251 | 244 |
252 static void | 245 static void |
253 mcommoninit(M *mp) | 246 mcommoninit(M *mp) |
254 { | 247 { |
255 // If there is no mcache runtime·callers() will crash, | 248 // If there is no mcache runtime·callers() will crash, |
256 // and we are most likely in sysmon thread so the stack is senseless any
way. | 249 // and we are most likely in sysmon thread so the stack is senseless any
way. |
257 if(m->mcache) | 250 if(m->mcache) |
258 runtime·callers(1, mp->createstack, nelem(mp->createstack)); | 251 runtime·callers(1, mp->createstack, nelem(mp->createstack)); |
259 | 252 |
260 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); | 253 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); |
261 if(runtime·gsignalstk) | |
262 mp->gsignal = runtime·malg(runtime·gsignalstk);· | |
263 | 254 |
264 runtime·lock(&runtime·sched); | 255 runtime·lock(&runtime·sched); |
265 mp->id = runtime·sched.mcount++; | 256 mp->id = runtime·sched.mcount++; |
| 257 |
| 258 runtime·mpreinit(mp); |
266 | 259 |
267 // Add to runtime·allm so garbage collector doesn't free m | 260 // Add to runtime·allm so garbage collector doesn't free m |
268 // when it is just in a register or thread-local storage. | 261 // when it is just in a register or thread-local storage. |
269 mp->alllink = runtime·allm; | 262 mp->alllink = runtime·allm; |
270 » // runtime·NumCgoCall() iterates over allm w/o locks, | 263 » // runtime·NumCgoCall() iterates over allm w/o schedlock, |
271 // so we need to publish it safely. | 264 // so we need to publish it safely. |
272 runtime·atomicstorep(&runtime·allm, mp); | 265 runtime·atomicstorep(&runtime·allm, mp); |
273 runtime·unlock(&runtime·sched); | 266 runtime·unlock(&runtime·sched); |
274 | |
275 LOG("%d: mcommoninit %d m=%p\n", m->id, mp->id, mp); | |
276 } | 267 } |
277 | 268 |
278 // Mark gp ready to run. | 269 // Mark gp ready to run. |
279 void | 270 void |
280 runtime·ready(G *gp) | 271 runtime·ready(G *gp) |
281 { | 272 { |
282 CHECK(gp->m == nil, ("bad g->m in ready")); | |
283 // Mark runnable. | 273 // Mark runnable. |
284 » if(gp->status == Grunnable || gp->status == Grunning) { | 274 » if(gp->status != Gwaiting) { |
285 runtime·printf("goroutine %D has status %d\n", gp->goid, gp->sta
tus); | 275 runtime·printf("goroutine %D has status %d\n", gp->goid, gp->sta
tus); |
286 runtime·throw("bad g->status in ready"); | 276 runtime·throw("bad g->status in ready"); |
287 } | 277 } |
288 gp->status = Grunnable; | 278 gp->status = Grunnable; |
289 runqput(m->p, gp); | 279 runqput(m->p, gp); |
290 » if(runtime·sched.pidle) | 280 » if(runtime·sched.npidle != 0 && runtime·sched.nmspinning == 0) // TODO:
fast atomic |
291 » » startm(nil); | 281 » » wakep(); |
292 } | 282 } |
293 | 283 |
294 int32 | 284 int32 |
295 runtime·gcprocs(void) | 285 runtime·gcprocs(void) |
296 { | 286 { |
297 int32 n; | 287 int32 n; |
298 | 288 |
299 // Figure out how many CPUs to use during GC. | 289 // Figure out how many CPUs to use during GC. |
300 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. | 290 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. |
301 runtime·lock(&runtime·sched); | 291 runtime·lock(&runtime·sched); |
302 n = runtime·gomaxprocs; | 292 n = runtime·gomaxprocs; |
303 if(n > runtime·ncpu) | 293 if(n > runtime·ncpu) |
304 n = runtime·ncpu; | 294 n = runtime·ncpu; |
305 if(n > MaxGcproc) | 295 if(n > MaxGcproc) |
306 n = MaxGcproc; | 296 n = MaxGcproc; |
307 » if(n > runtime·sched.mwait+1) // one M is currently running | 297 » if(n > runtime·sched.nmidle+1) // one M is currently running |
308 » » n = runtime·sched.mwait+1; | 298 » » n = runtime·sched.nmidle+1; |
309 runtime·unlock(&runtime·sched); | 299 runtime·unlock(&runtime·sched); |
310 return n; | 300 return n; |
311 } | 301 } |
312 | 302 |
313 static bool | 303 static bool |
314 needaddgcproc(void) | 304 needaddgcproc(void) |
315 { | 305 { |
316 int32 n; | 306 int32 n; |
317 | 307 |
318 runtime·lock(&runtime·sched); | 308 runtime·lock(&runtime·sched); |
319 n = runtime·gomaxprocs; | 309 n = runtime·gomaxprocs; |
320 if(n > runtime·ncpu) | 310 if(n > runtime·ncpu) |
321 n = runtime·ncpu; | 311 n = runtime·ncpu; |
322 if(n > MaxGcproc) | 312 if(n > MaxGcproc) |
323 n = MaxGcproc; | 313 n = MaxGcproc; |
324 » n -= runtime·sched.mwait+1; // one M is currently running | 314 » n -= runtime·sched.nmidle+1; // one M is currently running |
325 runtime·unlock(&runtime·sched); | 315 runtime·unlock(&runtime·sched); |
326 return n > 0; | 316 return n > 0; |
327 } | 317 } |
328 | 318 |
329 void | 319 void |
330 runtime·helpgc(int32 nproc) | 320 runtime·helpgc(int32 nproc) |
331 { | 321 { |
332 M *mp; | 322 M *mp; |
333 int32 n, pos; | 323 int32 n, pos; |
334 | 324 |
335 LOG("%d: helpgc(%d)\n", m->id, nproc); | |
336 runtime·lock(&runtime·sched); | 325 runtime·lock(&runtime·sched); |
337 pos = 0; | 326 pos = 0; |
338 for(n = 1; n < nproc; n++) { // one M is currently running | 327 for(n = 1; n < nproc; n++) { // one M is currently running |
339 if(runtime·allp[pos]->mcache == m->mcache) | 328 if(runtime·allp[pos]->mcache == m->mcache) |
340 pos++; | 329 pos++; |
341 » » mp = mtryget(); | 330 » » mp = mget(); |
342 if(mp == nil) | 331 if(mp == nil) |
343 runtime·throw("runtime·gcprocs inconsistency"); | 332 runtime·throw("runtime·gcprocs inconsistency"); |
344 mp->helpgc = 1; | 333 mp->helpgc = 1; |
345 mp->mcache = runtime·allp[pos]->mcache; | 334 mp->mcache = runtime·allp[pos]->mcache; |
346 pos++; | 335 pos++; |
347 LOG("%d: helpgc wake %d\n", m->id, mp->id); | |
348 runtime·notewakeup(&mp->park); | 336 runtime·notewakeup(&mp->park); |
349 } | 337 } |
350 runtime·unlock(&runtime·sched); | 338 runtime·unlock(&runtime·sched); |
351 } | 339 } |
352 | 340 |
353 void | 341 void |
354 runtime·stoptheworld(void) | 342 runtime·stoptheworld(void) |
355 { | 343 { |
356 int32 i; | 344 int32 i; |
357 uint32 s; | 345 uint32 s; |
358 P *p; | 346 P *p; |
359 bool wait; | 347 bool wait; |
360 | 348 |
361 LOG("%d: stoptheworld\n", m->id); | |
362 runtime·lock(&runtime·sched); | 349 runtime·lock(&runtime·sched); |
363 runtime·gcwaiting = 1; | |
364 runtime·sched.stopwait = runtime·gomaxprocs; | 350 runtime·sched.stopwait = runtime·gomaxprocs; |
| 351 runtime·atomicstore((uint32*)&runtime·gcwaiting, 1); |
| 352 // stop current P |
365 m->p->status = Pgcstop; | 353 m->p->status = Pgcstop; |
366 runtime·sched.stopwait--; | 354 runtime·sched.stopwait--; |
367 » for(i=0; i<runtime·gomaxprocs; i++) { | 355 » // try to retake all P's in Psyscall status |
368 » » s = runtime·allp[i]->status; | 356 » for(i = 0; i < runtime·gomaxprocs; i++) { |
369 » » if(s == Psyscall && runtime·cas(&runtime·allp[i]->status, s, Pgc
stop)) { | 357 » » p = runtime·allp[i]; |
370 » » » LOG(" acquired syscall %d\n", i); | 358 » » s = p->status; |
| 359 » » if(s == Psyscall && runtime·cas(&p->status, s, Pgcstop)) |
371 runtime·sched.stopwait--; | 360 runtime·sched.stopwait--; |
372 » » } | 361 » } |
373 » } | 362 » // stop idle P's |
374 » while(runtime·sched.pidle) { | 363 » while(p = pidleget()) { |
375 » » p = pidleget(); | |
376 p->status = Pgcstop; | 364 p->status = Pgcstop; |
377 runtime·sched.stopwait--; | 365 runtime·sched.stopwait--; |
378 } | 366 } |
379 CHECK(runtime·sched.stopwait >= 0, ("")); | |
380 wait = runtime·sched.stopwait > 0; | 367 wait = runtime·sched.stopwait > 0; |
381 runtime·unlock(&runtime·sched); | 368 runtime·unlock(&runtime·sched); |
| 369 |
| 370 // wait for remaining P's to stop voluntary |
382 if(wait) { | 371 if(wait) { |
383 runtime·notesleep(&runtime·sched.stopnote); | 372 runtime·notesleep(&runtime·sched.stopnote); |
384 runtime·noteclear(&runtime·sched.stopnote); | 373 runtime·noteclear(&runtime·sched.stopnote); |
385 } | 374 } |
386 » LOG("%d: stoptheworld stopped\n", m->id); | 375 » if(runtime·sched.stopwait) |
387 » CHECK(runtime·sched.stopwait == 0, ("stoptheworld: stopwait == %d\n", ru
ntime·sched.stopwait)); | 376 » » runtime·throw("stoptheworld: not stopped"); |
388 » for(i=0; i<runtime·gomaxprocs; i++) { | 377 » for(i = 0; i < runtime·gomaxprocs; i++) { |
389 » » CHECK(runtime·allp[i]->status == Pgcstop, ("stoptheworld: not st
opped (%d)\n", runtime·allp[i]->status)); | 378 » » p = runtime·allp[i]; |
| 379 » » if(p->status != Pgcstop) |
| 380 » » » runtime·throw("stoptheworld: not stopped"); |
390 } | 381 } |
391 } | 382 } |
392 | 383 |
393 void | 384 void |
394 runtime·starttheworld(void) | 385 runtime·starttheworld(void) |
395 { | 386 { |
396 /* | 387 » P *p; |
397 » G *gp; | |
398 » *p; | |
399 M *mp; | 388 M *mp; |
400 int32 n, w; | |
401 */ | |
402 bool add; | 389 bool add; |
403 | 390 |
404 LOG("%d: starttheworld\n", m->id); | |
405 add = needaddgcproc(); | 391 add = needaddgcproc(); |
406 » runtime·gcwaiting = 0; | 392 » runtime·lock(&runtime·sched); |
407 if(newprocs) { | 393 if(newprocs) { |
408 procresize(newprocs); | 394 procresize(newprocs); |
409 newprocs = 0; | 395 newprocs = 0; |
410 » } else { | 396 » } else |
411 procresize(runtime·gomaxprocs); | 397 procresize(runtime·gomaxprocs); |
412 » } | 398 » runtime·gcwaiting = 0; |
413 » runtime·lock(&runtime·sched); | 399 |
414 » /* | 400 » while(p = pidleget()) { |
415 » gp = runtime·netwait(0, 4 * runtime·gomaxprocs); | 401 » » // procresize() puts p's with work at the beginning of the list. |
416 » n = w = 0; | 402 » » // Once we reach a p without a run queue, the rest don't have on
e either. |
417 » runtime·inject(gp, &w, &n); | 403 » » if(p->runqhead == p->runqtail) { |
418 » while(runtime·sched.pidle) { | 404 » » » pidleput(p); |
419 » » p = pidleget(); | 405 » » » break; |
420 » » mp = mtryget(); | 406 » » } |
421 » » if(mp) { | 407 » » mp = mget(); |
422 » » » acquirep(mp, p); | 408 » » if(mp == nil) { |
423 » » » runtime·notewakeup(&mp->park); | 409 » » » pidleput(p); |
424 » » } else { | 410 » » » break; |
425 » » » runtime·unlock(&runtime·sched); | 411 » » } |
426 » » » newm(runtime·mstart, p, false); | 412 » » if(mp->nextp) |
427 » » » runtime·lock(&runtime·sched); | 413 » » » runtime·throw("starttheworld: inconsistent mp->nextp"); |
428 » » } | 414 » » mp->nextp = p; |
429 » } | 415 » » runtime·notewakeup(&mp->park); |
430 » */ | 416 » } |
431 if(runtime·sched.sysmonwait) { | 417 if(runtime·sched.sysmonwait) { |
432 » » runtime·sched.sysmonwait = 0; | 418 » » runtime·sched.sysmonwait = false; |
433 runtime·notewakeup(&runtime·sched.sysmonnote); | 419 runtime·notewakeup(&runtime·sched.sysmonnote); |
434 } | 420 } |
435 runtime·unlock(&runtime·sched); | 421 runtime·unlock(&runtime·sched); |
436 | 422 |
437 if(add) { | 423 if(add) { |
438 // If GC could have used another helper proc, start one now, | 424 // If GC could have used another helper proc, start one now, |
439 // in the hope that it will be available next time. | 425 // in the hope that it will be available next time. |
440 // It would have been even better to start it before the collect
ion, | 426 // It would have been even better to start it before the collect
ion, |
441 // but doing so requires allocating memory, so it's tricky to | 427 // but doing so requires allocating memory, so it's tricky to |
442 // coordinate. This lazy approach works out in practice: | 428 // coordinate. This lazy approach works out in practice: |
443 // we don't mind if the first couple gc rounds don't have quite | 429 // we don't mind if the first couple gc rounds don't have quite |
444 // the maximum number of procs. | 430 // the maximum number of procs. |
445 » » newm(runtime·mstart, nil, true); | 431 » » newm(runtime·mstart, nil, true, false); |
446 } | 432 } |
447 } | 433 } |
448 | 434 |
449 // Called to start an M. | 435 // Called to start an M. |
450 void | 436 void |
451 runtime·mstart(void) | 437 runtime·mstart(void) |
452 { | 438 { |
453 // It is used by windows-386 only. Unfortunately, seh needs | 439 // It is used by windows-386 only. Unfortunately, seh needs |
454 // to be located on os stack, and mstart runs on os stack | 440 // to be located on os stack, and mstart runs on os stack |
455 // for both m0 and m. | 441 // for both m0 and m. |
456 SEH seh; | 442 SEH seh; |
457 » P *p; | 443 |
458 | |
459 » LOG("%d: mstart m=%p\n", m->id, m); | |
460 if(g != m->g0) | 444 if(g != m->g0) |
461 runtime·throw("bad runtime·mstart"); | 445 runtime·throw("bad runtime·mstart"); |
462 | 446 |
463 // Record top of stack for use by mcall. | 447 // Record top of stack for use by mcall. |
464 // Once we call schedule we're never coming back, | 448 // Once we call schedule we're never coming back, |
465 // so other calls can reuse this stack space. | 449 // so other calls can reuse this stack space. |
466 runtime·gosave(&m->g0->sched); | 450 runtime·gosave(&m->g0->sched); |
467 m->g0->sched.pc = (void*)-1; // make sure it is never used | 451 m->g0->sched.pc = (void*)-1; // make sure it is never used |
468 m->seh = &seh; | 452 m->seh = &seh; |
469 runtime·asminit(); | 453 runtime·asminit(); |
470 runtime·minit(); | 454 runtime·minit(); |
471 | 455 |
472 // Install signal handlers; after minit so that minit can | 456 // Install signal handlers; after minit so that minit can |
473 // prepare the thread to be able to handle the signals. | 457 // prepare the thread to be able to handle the signals. |
474 » if(m == &runtime·m0) | 458 » if(m == &runtime·m0) { |
475 runtime·initsig(); | 459 runtime·initsig(); |
| 460 if(runtime·iscgo) |
| 461 runtime·newextram(); |
| 462 } |
476 | 463 |
477 if(m->helpgc) { | 464 if(m->helpgc) { |
478 LOG("%d: mstart helpgc\n", m->id); | |
479 m->helpgc = false; | 465 m->helpgc = false; |
480 stopm(); | 466 stopm(); |
481 } else if(m != &runtime·m0) { | 467 } else if(m != &runtime·m0) { |
482 » » p = m->p; | 468 » » acquirep(m->nextp); |
483 » » m->p = nil; | 469 » » m->nextp = nil; |
484 » » acquirep(m, p); | 470 » } |
485 » } | |
486 » LOG("%d: calling schedule\n", m->id); | |
487 schedule(); | 471 schedule(); |
488 | 472 |
489 // TODO(brainman): This point is never reached, because scheduler | 473 // TODO(brainman): This point is never reached, because scheduler |
490 // does not release os threads at the moment. But once this path | 474 // does not release os threads at the moment. But once this path |
491 // is enabled, we must remove our seh here. | 475 // is enabled, we must remove our seh here. |
492 } | 476 } |
493 | 477 |
494 // When running with cgo, we call libcgo_thread_start | 478 // When running with cgo, we call _cgo_thread_start |
495 // to start threads for us so that we can play nicely with | 479 // to start threads for us so that we can play nicely with |
496 // foreign code. | 480 // foreign code. |
497 void (*libcgo_thread_start)(void*); | 481 void (*_cgo_thread_start)(void*); |
498 | 482 |
499 typedef struct CgoThreadStart CgoThreadStart; | 483 typedef struct CgoThreadStart CgoThreadStart; |
500 struct CgoThreadStart | 484 struct CgoThreadStart |
501 { | 485 { |
502 M *m; | 486 M *m; |
503 G *g; | 487 G *g; |
504 void (*fn)(void); | 488 void (*fn)(void); |
505 }; | 489 }; |
506 | 490 |
507 // Create a new m. It will start off with a call to fn. | 491 // Allocate a new m unassociated with any thread. |
508 static void | 492 // Can use p for allocation context if needed. |
509 newm(void(*fn)(void), P *p, bool helpgc) | 493 M* |
| 494 runtime·allocm(P *p) |
510 { | 495 { |
511 M *mp; | 496 M *mp; |
512 static Type *mtype; // The Go type M | 497 static Type *mtype; // The Go type M |
513 | 498 |
514 » LOG("%d: newm\n", m->id); | 499 » m->locks++; // disable GC because it can be called from sysmon |
515 » if(m->p == nil) { | 500 » if(m->p == nil) |
516 » » CHECK(p, ("newm: no p\n")); | 501 » » acquirep(p); // temporarily borrow p for mallocs in this functi
on |
517 » » CHECK(!helpgc, ("newm: helpgc\n")); | |
518 » » acquirep(m, p); // temporarily borrow p for mallocs in this fun
ction | |
519 » } | |
520 » CHECK(m->mcache, ("newm: no mcache\n")); | |
521 » m->locks++; //!!! needed? | |
522 if(mtype == nil) { | 502 if(mtype == nil) { |
523 Eface e; | 503 Eface e; |
524 runtime·gc_m_ptr(&e); | 504 runtime·gc_m_ptr(&e); |
525 mtype = ((PtrType*)e.type)->elem; | 505 mtype = ((PtrType*)e.type)->elem; |
526 } | 506 } |
527 | 507 |
528 mp = runtime·cnew(mtype); | 508 mp = runtime·cnew(mtype); |
529 mcommoninit(mp); | 509 mcommoninit(mp); |
530 mp->p = p; | |
531 mp->helpgc = helpgc; | |
532 | 510 |
533 // In case of cgo, pthread_create will make us a stack. | 511 // In case of cgo, pthread_create will make us a stack. |
534 // Windows will layout sched stack on OS stack. | 512 // Windows will layout sched stack on OS stack. |
535 » mp->g0 = runtime·malg(runtime·iscgo || Windows ? -1 : 8192); | 513 » if(runtime·iscgo || Windows) |
536 | 514 » » mp->g0 = runtime·malg(-1); |
| 515 » else |
| 516 » » mp->g0 = runtime·malg(8192); |
| 517 |
| 518 » if(p == m->p) |
| 519 » » releasep(); |
537 m->locks--; | 520 m->locks--; |
538 » if(p == m->p) { | 521 |
539 » » CHECK(!helpgc, ("newm: releasep helpgc")); | 522 » return mp; |
540 » » releasep(); | 523 } |
541 » } | 524 |
542 » // Can not allocate memory after this point. | 525 static M* lockextra(bool nilokay); |
| 526 static void unlockextra(M*); |
| 527 |
| 528 // needm is called when a cgo callback happens on a |
| 529 // thread without an m (a thread not created by Go). |
| 530 // In this case, needm is expected to find an m to use |
| 531 // and return with m, g initialized correctly. |
| 532 // Since m and g are not set now (likely nil, but see below) |
| 533 // needm is limited in what routines it can call. In particular |
| 534 // it can only call nosplit functions (textflag 7) and cannot |
| 535 // do any scheduling that requires an m. |
| 536 // |
| 537 // In order to avoid needing heavy lifting here, we adopt |
| 538 // the following strategy: there is a stack of available m's |
| 539 // that can be stolen. Using compare-and-swap |
| 540 // to pop from the stack has ABA races, so we simulate |
| 541 // a lock by doing an exchange (via casp) to steal the stack |
| 542 // head and replace the top pointer with MLOCKED (1). |
| 543 // This serves as a simple spin lock that we can use even |
| 544 // without an m. The thread that locks the stack in this way |
| 545 // unlocks the stack by storing a valid stack head pointer. |
| 546 // |
| 547 // In order to make sure that there is always an m structure |
| 548 // available to be stolen, we maintain the invariant that there |
| 549 // is always one more than needed. At the beginning of the |
| 550 // program (if cgo is in use) the list is seeded with a single m. |
| 551 // If needm finds that it has taken the last m off the list, its job |
| 552 // is - once it has installed its own m so that it can do things like |
| 553 // allocate memory - to create a spare m and put it on the list. |
| 554 // |
| 555 // Each of these extra m's also has a g0 and a curg that are |
| 556 // pressed into service as the scheduling stack and current |
| 557 // goroutine for the duration of the cgo callback. |
| 558 // |
| 559 // When the callback is done with the m, it calls dropm to |
| 560 // put the m back on the list. |
| 561 #pragma textflag 7 |
| 562 void |
| 563 runtime·needm(byte x) |
| 564 { |
| 565 » M *mp; |
| 566 |
| 567 » // Lock extra list, take head, unlock popped list. |
| 568 » // nilokay=false is safe here because of the invariant above, |
| 569 » // that the extra list always contains or will soon contain |
| 570 » // at least one m. |
| 571 » mp = lockextra(false); |
| 572 |
| 573 » // Set needextram when we've just emptied the list, |
| 574 » // so that the eventual call into cgocallbackg will |
| 575 » // allocate a new m for the extra list. We delay the |
| 576 » // allocation until then so that it can be done |
| 577 » // after exitsyscall makes sure it is okay to be |
| 578 » // running at all (that is, there's no garbage collection |
| 579 » // running right now). |
| 580 » mp->needextram = mp->schedlink == nil; |
| 581 » unlockextra(mp->schedlink); |
| 582 |
| 583 » // Install m and g (= m->g0) and set the stack bounds |
| 584 » // to match the current stack. We don't actually know |
| 585 » // how big the stack is, like we don't know how big any |
| 586 » // scheduling stack is, but we assume there's at least 32 kB, |
| 587 » // which is more than enough for us. |
| 588 » runtime·setmg(mp, mp->g0); |
| 589 » g->stackbase = (uintptr)(&x + 1024); |
| 590 » g->stackguard = (uintptr)(&x - 32*1024); |
| 591 |
| 592 » // On windows/386, we need to put an SEH frame (two words) |
| 593 » // somewhere on the current stack. We are called |
| 594 » // from needm, and we know there is some available |
| 595 » // space one word into the argument frame. Use that. |
| 596 » m->seh = (SEH*)((uintptr*)&x + 1); |
| 597 |
| 598 » // Initialize this thread to use the m. |
| 599 » runtime·asminit(); |
| 600 » runtime·minit(); |
| 601 } |
| 602 |
| 603 // newextram allocates an m and puts it on the extra list. |
| 604 // It is called with a working local m, so that it can do things |
| 605 // like call schedlock and allocate. |
| 606 void |
| 607 runtime·newextram(void) |
| 608 { |
| 609 » M *mp, *mnext; |
| 610 » G *gp; |
| 611 |
| 612 » // Create extra goroutine locked to extra m. |
| 613 » // The goroutine is the context in which the cgo callback will run. |
| 614 » // The sched.pc will never be returned to, but setting it to |
| 615 » // runtime.goexit makes clear to the traceback routines where |
| 616 » // the goroutine stack ends. |
| 617 » mp = runtime·allocm(nil); |
| 618 » gp = runtime·malg(4096); |
| 619 » gp->sched.pc = (void*)runtime·goexit; |
| 620 » gp->sched.sp = gp->stackbase; |
| 621 » gp->sched.g = gp; |
| 622 » gp->status = Gsyscall; |
| 623 » mp->curg = gp; |
| 624 » mp->locked = LockInternal; |
| 625 » mp->lockedg = gp; |
| 626 » gp->lockedm = mp; |
| 627 » // put on allg for garbage collector |
| 628 » runtime·lock(&runtime·sched); |
| 629 » if(runtime·lastg == nil) |
| 630 » » runtime·allg = gp; |
| 631 » else |
| 632 » » runtime·lastg->alllink = gp; |
| 633 » runtime·lastg = gp; |
| 634 » runtime·unlock(&runtime·sched); |
| 635 » gp->goid = runtime·xadd64(&runtime·sched.goidgen, 1); |
| 636 » if(raceenabled) |
| 637 » » gp->racectx = runtime·racegostart(runtime·newextram); |
| 638 |
| 639 » // Add m to the extra list. |
| 640 » mnext = lockextra(true); |
| 641 » mp->schedlink = mnext; |
| 642 » unlockextra(mp); |
| 643 } |
| 644 |
| 645 // dropm is called when a cgo callback has called needm but is now |
| 646 // done with the callback and returning back into the non-Go thread. |
| 647 // It puts the current m back onto the extra list. |
| 648 // |
| 649 // The main expense here is the call to signalstack to release the |
| 650 // m's signal stack, and then the call to needm on the next callback |
| 651 // from this thread. It is tempting to try to save the m for next time, |
| 652 // which would eliminate both these costs, but there might not be |
| 653 // a next time: the current thread (which Go does not control) might exit. |
| 654 // If we saved the m for that thread, there would be an m leak each time |
| 655 // such a thread exited. Instead, we acquire and release an m on each |
| 656 // call. These should typically not be scheduling operations, just a few |
| 657 // atomics, so the cost should be small. |
| 658 // |
| 659 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread |
| 660 // variable using pthread_key_create. Unlike the pthread keys we already use |
| 661 // on OS X, this dummy key would never be read by Go code. It would exist |
| 662 // only so that we could register at thread-exit-time destructor. |
| 663 // That destructor would put the m back onto the extra list. |
| 664 // This is purely a performance optimization. The current version, |
| 665 // in which dropm happens on each cgo call, is still correct too. |
| 666 // We may have to keep the current version on systems with cgo |
| 667 // but without pthreads, like Windows. |
| 668 void |
| 669 runtime·dropm(void) |
| 670 { |
| 671 » M *mp, *mnext; |
| 672 |
| 673 » // Undo whatever initialization minit did during needm. |
| 674 » runtime·unminit(); |
| 675 |
| 676 » // Clear m and g, and return m to the extra list. |
| 677 » // After the call to setmg we can only call nosplit functions. |
| 678 » mp = m; |
| 679 » runtime·setmg(nil, nil); |
| 680 |
| 681 » mnext = lockextra(true); |
| 682 » mp->schedlink = mnext; |
| 683 » unlockextra(mp); |
| 684 } |
| 685 |
| 686 #define MLOCKED ((M*)1) |
| 687 |
| 688 // lockextra locks the extra list and returns the list head. |
| 689 // The caller must unlock the list by storing a new list head |
| 690 // to runtime.extram. If nilokay is true, then lockextra will |
| 691 // return a nil list head if that's what it finds. If nilokay is false, |
| 692 // lockextra will keep waiting until the list head is no longer nil. |
| 693 #pragma textflag 7 |
| 694 static M* |
| 695 lockextra(bool nilokay) |
| 696 { |
| 697 » M *mp; |
| 698 » void (*yield)(void); |
| 699 |
| 700 » for(;;) { |
| 701 » » mp = runtime·atomicloadp(&runtime·extram); |
| 702 » » if(mp == MLOCKED) { |
| 703 » » » yield = runtime·osyield; |
| 704 » » » yield(); |
| 705 » » » continue; |
| 706 » » } |
| 707 » » if(mp == nil && !nilokay) { |
| 708 » » » runtime·usleep(1); |
| 709 » » » continue; |
| 710 » » } |
| 711 » » if(!runtime·casp(&runtime·extram, mp, MLOCKED)) { |
| 712 » » » yield = runtime·osyield; |
| 713 » » » yield(); |
| 714 » » » continue; |
| 715 » » } |
| 716 » » break; |
| 717 » } |
| 718 » return mp; |
| 719 } |
| 720 |
| 721 #pragma textflag 7 |
| 722 static void |
| 723 unlockextra(M *mp) |
| 724 { |
| 725 » runtime·atomicstorep(&runtime·extram, mp); |
| 726 } |
| 727 |
| 728 |
| 729 // Create a new m. It will start off with a call to fn. |
| 730 static void |
| 731 newm(void(*fn)(void), P *p, bool helpgc, bool spinning) |
| 732 { |
| 733 » M *mp; |
| 734 |
| 735 » mp = runtime·allocm(p); |
| 736 » mp->nextp = p; |
| 737 » mp->helpgc = helpgc; |
| 738 » mp->spinning = spinning; |
543 | 739 |
544 if(runtime·iscgo) { | 740 if(runtime·iscgo) { |
545 CgoThreadStart ts; | 741 CgoThreadStart ts; |
546 | 742 |
547 » » if(libcgo_thread_start == nil) | 743 » » if(_cgo_thread_start == nil) |
548 » » » runtime·throw("libcgo_thread_start missing"); | 744 » » » runtime·throw("_cgo_thread_start missing"); |
549 ts.m = mp; | 745 ts.m = mp; |
550 ts.g = mp->g0; | 746 ts.g = mp->g0; |
551 ts.fn = fn; | 747 ts.fn = fn; |
552 » » runtime·asmcgocall(libcgo_thread_start, &ts); | 748 » » runtime·asmcgocall(_cgo_thread_start, &ts); |
553 return; | 749 return; |
554 } | 750 } |
555 runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, fn); | 751 runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, fn); |
556 } | 752 } |
557 | 753 |
558 // Stops execution of the current m until new work is available. | 754 // Stops execution of the current m until new work is available. |
559 // Returns with acquired P. | 755 // Returns with acquired P. |
560 static void | 756 static void |
561 stopm(void) | 757 stopm(void) |
562 { | 758 { |
563 » LOG("%d: mstop\n", m->id); | 759 » if(m->locks) |
564 » CHECK(m->locks == 0, ("")); | 760 » » runtime·throw("stopm holding locks"); |
565 » CHECK(m->p == nil, ("mstop: p != nil\n")); | 761 » if(m->p) |
| 762 » » runtime·throw("stopm holding p"); |
| 763 » if(m->spinning) { |
| 764 » » m->spinning = false; |
| 765 » » runtime·xadd(&runtime·sched.nmspinning, -1); |
| 766 » } |
566 | 767 |
567 retry: | 768 retry: |
568 runtime·lock(&runtime·sched); | 769 runtime·lock(&runtime·sched); |
569 mput(m); | 770 mput(m); |
570 runtime·unlock(&runtime·sched); | 771 runtime·unlock(&runtime·sched); |
571 runtime·notesleep(&m->park); | 772 runtime·notesleep(&m->park); |
572 runtime·noteclear(&m->park); | 773 runtime·noteclear(&m->park); |
573 if(m->helpgc) { | 774 if(m->helpgc) { |
574 LOG("%d: gchelper\n", m->id); | |
575 m->helpgc = 0; | 775 m->helpgc = 0; |
576 runtime·gchelper(); | 776 runtime·gchelper(); |
577 m->mcache = nil; | 777 m->mcache = nil; |
578 LOG("%d: gchelper end\n", m->id); | |
579 goto retry; | 778 goto retry; |
580 } | 779 } |
581 » LOG("%d: mstop wake\n", m->id); | 780 » acquirep(m->nextp); |
582 » if(m->p == nil) | 781 » m->nextp = nil; |
583 » » runtime·throw("mstop: p == nil"); | |
584 } | 782 } |
585 | 783 |
586 // Schedules some M to run the p (creates an M if necessary). | 784 // Schedules some M to run the p (creates an M if necessary). |
587 // If p==nil, tries to get an idle P, if no idle P's returns false. | 785 // If p==nil, tries to get an idle P, if no idle P's returns false. |
588 static bool | 786 static void |
589 startm(P *p) | 787 startm(P *p, bool spinning) |
590 { | 788 { |
591 M *mp; | 789 M *mp; |
592 | 790 |
593 runtime·lock(&runtime·sched); | 791 runtime·lock(&runtime·sched); |
594 if(p == nil) { | 792 if(p == nil) { |
595 p = pidleget(); | 793 p = pidleget(); |
596 if(p == nil) { | 794 if(p == nil) { |
597 runtime·unlock(&runtime·sched); | 795 runtime·unlock(&runtime·sched); |
598 » » » return false;» » »······· | 796 » » » if(spinning) |
599 » » } | 797 » » » » runtime·xadd(&runtime·sched.nmspinning, -1); |
600 » } | 798 » » » return; |
601 » mp = mtryget(); | 799 » » } |
| 800 » } |
| 801 » mp = mget(); |
602 runtime·unlock(&runtime·sched); | 802 runtime·unlock(&runtime·sched); |
603 » if(mp) { | 803 » if(mp == nil) { |
604 » » acquirep(mp, p); | 804 » » newm(runtime·mstart, p, false, spinning); |
605 » » runtime·notewakeup(&mp->park); | 805 » » return; |
606 » » return true; | 806 » } |
607 » } | 807 » if(mp->spinning) |
608 » newm(runtime·mstart, p, false); | 808 » » runtime·throw("startm: m is spinning"); |
609 » return true; | 809 » if(mp->nextp) |
| 810 » » runtime·throw("startm: m has p"); |
| 811 » mp->spinning = spinning; |
| 812 » mp->nextp = p; |
| 813 » runtime·notewakeup(&mp->park); |
| 814 } |
| 815 |
| 816 // Hands off P from syscall or locked M. |
| 817 static void |
| 818 handoffp(P *p) |
| 819 { |
| 820 » // if it has local work, start it straight away |
| 821 » if(p->runqhead != p->runqtail || runtime·sched.runqsize) { |
| 822 » » startm(p, false); |
| 823 » » return; |
| 824 » } |
| 825 » // no local work, check that there are no spinning/idle M's, |
| 826 » // otherwise our help is not required |
| 827 » if(runtime·sched.nmspinning + runtime·sched.npidle == 0 && // TODO: fas
t atomic |
| 828 » » runtime·cas(&runtime·sched.nmspinning, 0, 1)) { |
| 829 » » startm(p, true); |
| 830 » » return; |
| 831 » } |
| 832 » runtime·lock(&runtime·sched); |
| 833 » if(runtime·gcwaiting) { |
| 834 » » p->status = Pgcstop; |
| 835 » » if(--runtime·sched.stopwait == 0) |
| 836 » » » runtime·notewakeup(&runtime·sched.stopnote); |
| 837 » » runtime·unlock(&runtime·sched); |
| 838 » » return; |
| 839 » } |
| 840 » if(runtime·sched.runqsize) { |
| 841 » » runtime·unlock(&runtime·sched); |
| 842 » » startm(p, false); |
| 843 » » return; |
| 844 » } |
| 845 » pidleput(p); |
| 846 » runtime·unlock(&runtime·sched); |
| 847 } |
| 848 |
| 849 // Tries to add one more P to execute G's. |
| 850 // Called when a G is made runnable (newproc, ready). |
| 851 static void |
| 852 wakep(void) |
| 853 { |
| 854 » // be conservative about spinning threads |
| 855 » if(!runtime·cas(&runtime·sched.nmspinning, 0, 1)) |
| 856 » » return; |
| 857 » startm(nil, true); |
610 } | 858 } |
611 | 859 |
612 // Stops execution of the current m that is locked to a g until the g is runnabl
e again. | 860 // Stops execution of the current m that is locked to a g until the g is runnabl
e again. |
613 // Returns with acquired P. | 861 // Returns with acquired P. |
614 static void | 862 static void |
615 stoplockedm(void) | 863 stoplockedm(void) |
616 { | 864 { |
617 P *p; | 865 P *p; |
618 | 866 |
619 » CHECK(m->lockedg, ("stoplockedm: no g\n")); | 867 » if(m->lockedg == nil || m->lockedg->lockedm != m) |
620 » CHECK(m->lockedg->lockedm == m, ("stoplockedm: bad lockedm\n")); | 868 » » runtime·throw("stoplockedm: inconsistent locking"); |
621 if(m->p) { | 869 if(m->p) { |
622 // Schedule another M to run this p. | 870 // Schedule another M to run this p. |
623 p = releasep(); | 871 p = releasep(); |
624 » » startm(p); | 872 » » handoffp(p); |
625 » } | 873 » } |
626 » runtime·lock(&runtime·sched); | 874 » inclocked(1); |
627 » runtime·sched.mlocked++; | |
628 » checkdead(); | |
629 » runtime·unlock(&runtime·sched); | |
630 // Wait until another thread schedules lockedg again. | 875 // Wait until another thread schedules lockedg again. |
631 runtime·notesleep(&m->park); | 876 runtime·notesleep(&m->park); |
632 runtime·noteclear(&m->park); | 877 runtime·noteclear(&m->park); |
633 » CHECK(m->p == nil, ("stoplockedm: p != nil\n")); | 878 » if(m->lockedg->status != Grunnable) |
634 » CHECK(m->lockedp, ("stoplockedm: no lockedp\n")); | 879 » » runtime·throw("stoplockedm: not runnable"); |
635 » CHECK(m->lockedg->status == Grunnable, ("stoplockedm: bad g status %d\n"
, m->lockedg->status)); | 880 » acquirep(m->nextp); |
636 » acquirep(m, m->lockedp); | 881 » m->nextp = nil; |
637 » m->lockedp = nil; | |
638 } | 882 } |
639 | 883 |
640 // Schedules the locked m to run the locked gp. | 884 // Schedules the locked m to run the locked gp. |
641 static void | 885 static void |
642 startlockedm(G *gp) | 886 startlockedm(G *gp) |
643 { | 887 { |
644 M *mp; | 888 M *mp; |
645 P *p; | 889 P *p; |
646 | 890 |
647 // Directly handoff current P to the locked m. | |
648 CHECK(gp->lockedm != m, ("schedule: locked to myself\n")); | |
649 runtime·lock(&runtime·sched); | |
650 runtime·sched.mlocked--; // can be negative due to race with stoplocked
m | |
651 runtime·unlock(&runtime·sched); | |
652 mp = gp->lockedm; | 891 mp = gp->lockedm; |
| 892 if(mp == m) |
| 893 runtime·throw("startlockedm: locked to me"); |
| 894 if(mp->nextp) |
| 895 runtime·throw("startlockedm: m has p"); |
| 896 // directly handoff current P to the locked m |
| 897 inclocked(-1); |
653 p = releasep(); | 898 p = releasep(); |
654 » mp->lockedp = p; | 899 » mp->nextp = p; |
655 runtime·notewakeup(&mp->park); | 900 runtime·notewakeup(&mp->park); |
656 stopm(); | 901 stopm(); |
657 } | 902 } |
658 | 903 |
659 // Stops the current m for stoptheworld. | 904 // Stops the current m for stoptheworld. |
660 // Returns when the world is restarted. | 905 // Returns when the world is restarted. |
661 static void | 906 static void |
662 gcstopm(void) | 907 gcstopm(void) |
663 { | 908 { |
664 P *p; | 909 P *p; |
665 | 910 |
666 » CHECK(runtime·gcwaiting, ("")); | 911 » if(!runtime·gcwaiting) |
| 912 » » runtime·throw("gcstopm: not waiting for gc"); |
| 913 » if(m->spinning) { |
| 914 » » m->spinning = false; |
| 915 » » runtime·xadd(&runtime·sched.nmspinning, -1); |
| 916 » } |
667 p = releasep(); | 917 p = releasep(); |
| 918 runtime·lock(&runtime·sched); |
668 p->status = Pgcstop; | 919 p->status = Pgcstop; |
669 » runtime·lock(&runtime·sched); | 920 » if(--runtime·sched.stopwait == 0) |
670 » runtime·sched.stopwait--; | |
671 » if(runtime·sched.stopwait == 0) | |
672 runtime·notewakeup(&runtime·sched.stopnote); | 921 runtime·notewakeup(&runtime·sched.stopnote); |
673 runtime·unlock(&runtime·sched); | 922 runtime·unlock(&runtime·sched); |
674 stopm(); | 923 stopm(); |
675 } | 924 } |
676 | 925 |
677 // Schedules gp to run on the current M. | 926 // Schedules gp to run on the current M. |
678 // Never returns. | 927 // Never returns. |
679 static void | 928 static void |
680 execute(G *gp) | 929 execute(G *gp) |
681 { | 930 { |
682 int32 hz; | 931 int32 hz; |
683 | 932 |
684 » LOG("%d: start running goroutine %p\n", m->id, gp); | 933 » if(gp->status != Grunnable) { |
685 » CHECK(m->locks == 0, ("")); | 934 » » runtime·printf("execute: bad g status %d\n", gp->status); |
686 » CHECK(g == m->g0, ("execute: not on g0\n")); | 935 » » runtime·throw("execute: bad g status"); |
687 » CHECK(m->p != nil, ("execute: no p\n")); | 936 » } |
688 » CHECK(gp->status == Grunnable, ("execute: gp=%D gp->status=%d\n", gp->go
id, gp->status)); | 937 » gp->status = Grunning; |
689 » CHECK(gp->m == nil, ("execute: gp->m=%p\n", gp->m)); | |
690 » CHECK(gp->lockedm == nil && m->lockedg == nil || gp->lockedm == m && m->
lockedg == gp, | |
691 » » ("bad locking: gp->lockedm=%p m->lockedg=%p\n", gp->lockedm, m->
lockedg)); | |
692 m->p->tick++; | 938 m->p->tick++; |
693 gp->status = Grunning; | |
694 m->curg = gp; | 939 m->curg = gp; |
695 gp->m = m; | 940 gp->m = m; |
696 | 941 |
697 // Check whether the profiler needs to be turned on or off. | 942 // Check whether the profiler needs to be turned on or off. |
698 hz = runtime·sched.profilehz; | 943 hz = runtime·sched.profilehz; |
699 if(m->profilehz != hz) | 944 if(m->profilehz != hz) |
700 runtime·resetcpuprofiler(hz); | 945 runtime·resetcpuprofiler(hz); |
701 | 946 |
702 if(gp->sched.pc == (byte*)runtime·goexit) // kickoff | 947 if(gp->sched.pc == (byte*)runtime·goexit) // kickoff |
703 » » runtime·gogocall(&gp->sched, (void(*)(void))gp->entry); | 948 » » runtime·gogocallfn(&gp->sched, gp->fnstart); |
704 runtime·gogo(&gp->sched, 0); | 949 runtime·gogo(&gp->sched, 0); |
705 } | 950 } |
706 | 951 |
707 // Finds a runnable goroutine to execute. | 952 // Finds a runnable goroutine to execute. |
708 // Tries to steal from other P's, get g from global queue, polls network connect
ions. | 953 // Tries to steal from other P's and get g from global queue. |
709 static G* | 954 static G* |
710 findrunnable(void) | 955 findrunnable(void) |
711 { | 956 { |
712 » G *gp, *gp1; | 957 » G *gp; |
713 P *p; | 958 P *p; |
714 int32 i; | 959 int32 i; |
715 | 960 |
716 top: | 961 top: |
717 if(runtime·gcwaiting) { | 962 if(runtime·gcwaiting) { |
718 gcstopm(); | 963 gcstopm(); |
719 goto top; | 964 goto top; |
720 } | 965 } |
721 | 966 » // local runq |
722 gp = runqget(m->p); | 967 gp = runqget(m->p); |
723 if(gp) | 968 if(gp) |
724 return gp; | 969 return gp; |
725 » if (runtime·sched.poller == 0 && runtime·xchg(&runtime·sched.poller, 1)
== 0) { | 970 » // global runq |
726 » » G *gp2; | |
727 » » int32 w, n; | |
728 | |
729 » » gp2 = runtime·netwait(0, 10); | |
730 » » runtime·atomicstore(&runtime·sched.poller, 0); | |
731 » » if (gp2) { | |
732 » » » gp = gp2; | |
733 » » » gp->status = Grunnable; | |
734 » » » gp2 = gp->schedlink; | |
735 » » » if (gp2) { | |
736 » » » » n = w = 0; | |
737 » » » » runtime·inject(gp2, &w, &n); | |
738 » » » } | |
739 » » » return gp; | |
740 » » } | |
741 » } | |
742 | |
743 if(runtime·sched.runqsize) { | 971 if(runtime·sched.runqsize) { |
744 runtime·lock(&runtime·sched); | 972 runtime·lock(&runtime·sched); |
745 » » gp = globrunqget(); | 973 » » gp = globrunqget(m->p); |
746 » » if(gp) { | |
747 » » » while(gp->schedlink != nil) { | |
748 » » » » gp1 = gp; | |
749 » » » » gp = gp1->schedlink; | |
750 » » » » runqput(m->p, gp1); | |
751 » » » } | |
752 » » } | |
753 runtime·unlock(&runtime·sched); | 974 runtime·unlock(&runtime·sched); |
754 if(gp) | 975 if(gp) |
755 return gp; | 976 return gp; |
756 } | 977 } |
757 » for(i=0; i<runtime·gomaxprocs; i++) { | 978 » // If number of spinning M's >= number of busy P's, block. |
| 979 » // This is necessary to prevent excessive CPU consumption |
| 980 » // when GOMAXPROCS>>1 but the program parallelism is low. |
| 981 » if(!m->spinning && 2 * runtime·sched.nmspinning >= runtime·gomaxprocs -
runtime·sched.npidle) // TODO: fast atomic |
| 982 » » goto stop; |
| 983 » if(!m->spinning) { |
| 984 » » m->spinning = true; |
| 985 » » runtime·xadd(&runtime·sched.nmspinning, 1); |
| 986 » } |
| 987 » // random steal from other P's |
| 988 » for(i = 0; i < 2*runtime·gomaxprocs; i++) { |
758 if(runtime·gcwaiting) | 989 if(runtime·gcwaiting) |
759 goto top; | 990 goto top; |
760 p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]; | 991 p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]; |
761 if(p == m->p) | 992 if(p == m->p) |
762 gp = runqget(p); | 993 gp = runqget(p); |
763 else | 994 else |
764 gp = runqsteal(m->p, p); | 995 gp = runqsteal(m->p, p); |
765 if(gp) | 996 if(gp) |
766 return gp; | 997 return gp; |
767 } | 998 } |
768 » if(gp) | 999 stop: |
769 » » return gp; | 1000 » // return P and block |
770 runtime·lock(&runtime·sched); | 1001 runtime·lock(&runtime·sched); |
771 if(runtime·gcwaiting) { | 1002 if(runtime·gcwaiting) { |
772 runtime·unlock(&runtime·sched); | 1003 runtime·unlock(&runtime·sched); |
773 goto top; | 1004 goto top; |
774 } | 1005 } |
| 1006 if(runtime·sched.runqsize) { |
| 1007 gp = globrunqget(m->p); |
| 1008 runtime·unlock(&runtime·sched); |
| 1009 return gp; |
| 1010 } |
775 p = releasep(); | 1011 p = releasep(); |
776 pidleput(p); | 1012 pidleput(p); |
777 if(runtime·sched.runqsize) { | |
778 p = pidleget(); | |
779 runtime·unlock(&runtime·sched); | |
780 acquirep(m, p); | |
781 goto top; | |
782 } | |
783 runtime·unlock(&runtime·sched); | 1013 runtime·unlock(&runtime·sched); |
784 » for(i=0; i<runtime·gomaxprocs; i++) { | 1014 » if(m->spinning) { |
| 1015 » » m->spinning = false; |
| 1016 » » runtime·xadd(&runtime·sched.nmspinning, -1); |
| 1017 » } |
| 1018 » // check all runqueues once again |
| 1019 » for(i = 0; i < runtime·gomaxprocs; i++) { |
785 p = runtime·allp[i]; | 1020 p = runtime·allp[i]; |
786 if(p && p->runqhead != p->runqtail) { | 1021 if(p && p->runqhead != p->runqtail) { |
787 //!!! steal directly from this p | |
788 runtime·lock(&runtime·sched); | 1022 runtime·lock(&runtime·sched); |
789 p = pidleget(); | 1023 p = pidleget(); |
790 runtime·unlock(&runtime·sched); | 1024 runtime·unlock(&runtime·sched); |
791 if(p) { | 1025 if(p) { |
792 » » » » acquirep(m, p); | 1026 » » » » acquirep(p); |
793 goto top; | 1027 goto top; |
794 } | 1028 } |
795 break; | 1029 break; |
796 } | 1030 } |
797 } | 1031 } |
798 if (runtime·sched.poller == 0 && runtime·xchg(&runtime·sched.poller, 1)
== 0) { | |
799 G *gp2; | |
800 int32 w, n; | |
801 | |
802 gp2 = runtime·netwait(-1, 100); | |
803 runtime·atomicstore(&runtime·sched.poller, 0); | |
804 if (gp2) { | |
805 n = w = 0; | |
806 runtime·inject(gp2, &w, &n); | |
807 } | |
808 } | |
809 stopm(); | 1032 stopm(); |
810 goto top; | 1033 goto top; |
811 } | 1034 } |
812 | 1035 |
813 // One round of scheduler: find a goroutine and run it. | 1036 // One round of scheduler: find a runnable goroutine and execute it. |
814 // Never returns. | 1037 // Never returns. |
815 static void | 1038 static void |
816 schedule(void) | 1039 schedule(void) |
817 { | 1040 { |
818 G *gp; | 1041 G *gp; |
819 | 1042 |
820 » LOG("%d: schedule p=%p\n", m->id, m->p); | 1043 » if(m->locks) |
821 » USED(&gp); | 1044 » » runtime·throw("schedule: holding locks"); |
822 » CHECK(m->locks == 0, ("schedule: holding locks\n")); | |
823 » CHECK(m->lockedg == nil, ("schedule: locked M\n")); | |
824 | 1045 |
825 top: | 1046 top: |
826 if(runtime·gcwaiting) { | 1047 if(runtime·gcwaiting) { |
827 gcstopm(); | 1048 gcstopm(); |
828 goto top; | 1049 goto top; |
829 } | 1050 } |
830 | 1051 |
831 gp = runqget(m->p); | 1052 gp = runqget(m->p); |
832 if(gp == nil) | 1053 if(gp == nil) |
833 gp = findrunnable(); | 1054 gp = findrunnable(); |
834 | 1055 |
| 1056 if(m->spinning) { |
| 1057 m->spinning = false; |
| 1058 runtime·xadd(&runtime·sched.nmspinning, -1); |
| 1059 } |
| 1060 |
| 1061 // M wakeup policy is deliberately somewhat conservative (see nmspinning
handling), |
| 1062 // so see if we need to wakeup another M here. |
| 1063 if (m->p->runqhead != m->p->runqtail && |
| 1064 runtime·sched.nmspinning == 0 && |
| 1065 runtime·sched.npidle > 0) // TODO: fast atomic |
| 1066 wakep(); |
| 1067 |
835 if(gp->lockedm) { | 1068 if(gp->lockedm) { |
836 startlockedm(gp); | 1069 startlockedm(gp); |
837 goto top; | 1070 goto top; |
838 } | 1071 } |
839 | 1072 |
840 execute(gp); | 1073 execute(gp); |
841 } | 1074 } |
842 | 1075 |
843 // runtime·park continuation on g0. | |
844 static void | |
845 park0(G *gp) | |
846 { | |
847 CHECK(m->lockedp == nil || !m->lockedg, ("park0: lockedp != nil\n")); | |
848 gp->status = Gwaiting; | |
849 gp->m = nil; | |
850 if(m->waitunlockf) { | |
851 m->waitunlockf(m->waitlock); | |
852 m->waitunlockf = nil; | |
853 } | |
854 if(m->lockedg) { | |
855 stoplockedm(); | |
856 execute(gp); // Never returns. | |
857 } | |
858 schedule(); | |
859 } | |
860 | |
861 // Puts the current goroutine into a waiting state and unlocks the lock. | 1076 // Puts the current goroutine into a waiting state and unlocks the lock. |
862 // The goroutine can be made runnable again by calling runtime·ready(gp). | 1077 // The goroutine can be made runnable again by calling runtime·ready(gp). |
863 void | 1078 void |
864 runtime·park(void(*unlockf)(Lock*), Lock *lock, int8 *reason) | 1079 runtime·park(void(*unlockf)(Lock*), Lock *lock, int8 *reason) |
865 { | 1080 { |
866 LOG("%d: park l=%p reason=%s\n", m->id, lock, reason); | |
867 CHECK(g != m->g0, ("park of g0\n")); | |
868 m->waitlock = lock; | 1081 m->waitlock = lock; |
869 m->waitunlockf = unlockf; | 1082 m->waitunlockf = unlockf; |
870 g->waitreason = reason; | 1083 g->waitreason = reason; |
871 runtime·mcall(park0); | 1084 runtime·mcall(park0); |
872 } | 1085 } |
873 | 1086 |
| 1087 // runtime·park continuation on g0. |
| 1088 static void |
| 1089 park0(G *gp) |
| 1090 { |
| 1091 gp->status = Gwaiting; |
| 1092 gp->m = nil; |
| 1093 m->curg = nil; |
| 1094 if(m->waitunlockf) { |
| 1095 m->waitunlockf(m->waitlock); |
| 1096 m->waitunlockf = nil; |
| 1097 } |
| 1098 if(m->lockedg) { |
| 1099 stoplockedm(); |
| 1100 execute(gp); // Never returns. |
| 1101 } |
| 1102 schedule(); |
| 1103 } |
| 1104 |
| 1105 // Scheduler yield. |
| 1106 void |
| 1107 runtime·gosched(void) |
| 1108 { |
| 1109 runtime·mcall(gosched0); |
| 1110 } |
| 1111 |
874 // runtime·gosched continuation on g0. | 1112 // runtime·gosched continuation on g0. |
875 static void | 1113 static void |
876 gosched0(G *gp) | 1114 gosched0(G *gp) |
877 { | 1115 { |
878 LOG("%d: gosched0 gp=%p\n", m->id, gp); | |
879 gp->status = Grunnable; | 1116 gp->status = Grunnable; |
880 gp->m = nil; | 1117 gp->m = nil; |
881 » CHECK(m->lockedp == nil || !m->lockedg, ("gosched0: lockedp != nil\n")); | 1118 » m->curg = nil; |
882 runtime·lock(&runtime·sched); | 1119 runtime·lock(&runtime·sched); |
883 globrunqput(gp); | 1120 globrunqput(gp); |
884 runtime·unlock(&runtime·sched); | 1121 runtime·unlock(&runtime·sched); |
885 if(m->lockedg) { | 1122 if(m->lockedg) { |
886 stoplockedm(); | 1123 stoplockedm(); |
887 execute(gp); // Never returns. | 1124 execute(gp); // Never returns. |
888 } | 1125 } |
889 schedule(); | 1126 schedule(); |
890 } | 1127 } |
891 | 1128 |
892 // Scheduler yield. | 1129 // Finishes execution of the current goroutine. |
893 void | 1130 void |
894 runtime·gosched(void) | 1131 runtime·goexit(void) |
895 { | 1132 { |
896 » runtime·mcall(gosched0); | 1133 » if(raceenabled) |
| 1134 » » runtime·racegoend(); |
| 1135 » runtime·mcall(goexit0); |
897 } | 1136 } |
898 | 1137 |
899 // runtime·goexit continuation on g0. | 1138 // runtime·goexit continuation on g0. |
900 static void | 1139 static void |
901 goexit0(G *gp) | 1140 goexit0(G *gp) |
902 { | 1141 { |
903 gp->status = Gdead; | 1142 gp->status = Gdead; |
904 gp->m = nil; | 1143 gp->m = nil; |
905 gp->lockedm = nil; | 1144 gp->lockedm = nil; |
| 1145 m->curg = nil; |
906 m->lockedg = nil; | 1146 m->lockedg = nil; |
907 runtime·unwindstack(gp, nil); | 1147 runtime·unwindstack(gp, nil); |
908 gfput(m->p, gp); | 1148 gfput(m->p, gp); |
909 schedule(); | 1149 schedule(); |
910 } | |
911 | |
912 // Finishes execution of the current goroutine. | |
913 void | |
914 runtime·goexit(void) | |
915 { | |
916 if(raceenabled) | |
917 runtime·racegoend(); | |
918 runtime·mcall(goexit0); | |
919 } | 1150 } |
920 | 1151 |
921 // The goroutine g is about to enter a system call. | 1152 // The goroutine g is about to enter a system call. |
922 // Record that it's not using the cpu anymore. | 1153 // Record that it's not using the cpu anymore. |
923 // This is called only from the go syscall library and cgocall, | 1154 // This is called only from the go syscall library and cgocall, |
924 // not from the low-level system calls used by the runtime. | 1155 // not from the low-level system calls used by the runtime. |
925 // | 1156 // |
926 // Entersyscall cannot split the stack: the runtime·gosave must | 1157 // Entersyscall cannot split the stack: the runtime·gosave must |
927 // make g->sched refer to the caller's stack segment, because | 1158 // make g->sched refer to the caller's stack segment, because |
928 // entersyscall is going to return immediately after. | 1159 // entersyscall is going to return immediately after. |
929 #pragma textflag 7 | 1160 #pragma textflag 7 |
930 void | 1161 void |
931 runtime·entersyscall(void) | 1162 ·entersyscall(int32 dummy) |
932 { | 1163 { |
933 » P *p; | |
934 | |
935 » //LOG("%d: entersyscall g=%p p=%p scvg=%d\n", m->id, g, m->p, g==scvg); | |
936 if(m->profilehz > 0) | 1164 if(m->profilehz > 0) |
937 runtime·setprof(false); | 1165 runtime·setprof(false); |
938 | 1166 |
939 // Leave SP around for gc and traceback. | 1167 // Leave SP around for gc and traceback. |
940 » runtime·gosave(&g->sched); | 1168 » g->sched.sp = (uintptr)runtime·getcallersp(&dummy); |
| 1169 » g->sched.pc = runtime·getcallerpc(&dummy); |
| 1170 » g->sched.g = g; |
941 g->gcsp = g->sched.sp; | 1171 g->gcsp = g->sched.sp; |
| 1172 g->gcpc = g->sched.pc; |
942 g->gcstack = g->stackbase; | 1173 g->gcstack = g->stackbase; |
943 g->gcguard = g->stackguard; | 1174 g->gcguard = g->stackguard; |
944 g->status = Gsyscall; | 1175 g->status = Gsyscall; |
945 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { | 1176 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { |
946 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", | 1177 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", |
947 // g->gcsp, g->gcguard-StackGuard, g->gcstack); | 1178 // g->gcsp, g->gcguard-StackGuard, g->gcstack); |
948 runtime·throw("entersyscall"); | 1179 runtime·throw("entersyscall"); |
949 } | 1180 } |
950 | 1181 |
951 » if(m->blockingsyscall) { | 1182 » if(runtime·sched.sysmonwait) { // TODO: fast atomic |
952 » » m->blockingsyscall = false; | 1183 » » runtime·lock(&runtime·sched); |
953 » » p = releasep(); | 1184 » » if(runtime·sched.sysmonwait) { |
954 » » //!!! can switch to g0 and change g->sched. is it important? | 1185 » » » runtime·sched.sysmonwait = false; |
955 » » startm(p); | 1186 » » » runtime·notewakeup(&runtime·sched.sysmonnote); |
956 » » if(g == scvg) { | 1187 » » } |
957 » » » LOG("%d: scvg enters syscall\n", m->id); | 1188 » » runtime·unlock(&runtime·sched); |
958 » » » runtime·lock(&runtime·sched); | 1189 » » runtime·gosave(&g->sched); // re-save for traceback |
959 » » » runtime·sched.mlocked++; | |
960 » » » checkdead(); | |
961 » » » runtime·unlock(&runtime·sched); | |
962 » » } | |
963 » » return; | |
964 } | 1190 } |
965 | 1191 |
966 m->mcache = nil; | 1192 m->mcache = nil; |
| 1193 m->p->tick++; |
967 m->p->m = nil; | 1194 m->p->m = nil; |
968 runtime·atomicstore(&m->p->status, Psyscall); | 1195 runtime·atomicstore(&m->p->status, Psyscall); |
969 if(runtime·gcwaiting) { | 1196 if(runtime·gcwaiting) { |
970 runtime·lock(&runtime·sched); | 1197 runtime·lock(&runtime·sched); |
971 if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psy
scall, Pgcstop)) { | 1198 if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psy
scall, Pgcstop)) { |
972 » » » runtime·sched.stopwait--; | 1199 » » » if(--runtime·sched.stopwait == 0) |
973 » » » if(runtime·sched.stopwait == 0) | |
974 runtime·notewakeup(&runtime·sched.stopnote); | 1200 runtime·notewakeup(&runtime·sched.stopnote); |
975 } | 1201 } |
976 runtime·unlock(&runtime·sched); | 1202 runtime·unlock(&runtime·sched); |
977 » } | 1203 » » runtime·gosave(&g->sched); // re-save for traceback |
978 } | 1204 » } |
979 | 1205 } |
| 1206 |
| 1207 // The same as runtime·entersyscall(), but with a hint that the syscall is block
ing. |
980 #pragma textflag 7 | 1208 #pragma textflag 7 |
981 void | 1209 void |
982 runtime·entersyscallblock(void) | 1210 ·entersyscallblock(int32 dummy) |
983 { | |
984 » m->blockingsyscall = true; | |
985 » runtime·entersyscall(); | |
986 } | |
987 | |
988 // runtime·exitsyscall slow path on g0. | |
989 // Failed to acquire P, enqueue gp as runnable. | |
990 static void | |
991 exitsyscall0(G *gp) | |
992 { | 1211 { |
993 P *p; | 1212 P *p; |
994 | 1213 |
995 » LOG("%d: exitsyscall0\n", m->id); | 1214 » if(m->profilehz > 0) |
996 » gp->status = Grunnable; | 1215 » » runtime·setprof(false); |
997 » gp->m = nil; | 1216 |
998 » CHECK(m->park.waitm == nil, ("exitsyscall0: park is signalled\n")); | 1217 » // Leave SP around for gc and traceback. |
999 » runtime·lock(&runtime·sched); | 1218 » g->sched.sp = (uintptr)runtime·getcallersp(&dummy); |
1000 » p = pidleget(); | 1219 » g->sched.pc = runtime·getcallerpc(&dummy); |
1001 » if(p == nil) | 1220 » g->sched.g = g; |
1002 » » globrunqput(gp); | 1221 » g->gcsp = g->sched.sp; |
1003 » runtime·unlock(&runtime·sched); | 1222 » g->gcpc = g->sched.pc; |
1004 » if(p) { | 1223 » g->gcstack = g->stackbase; |
1005 » » acquirep(m, p); | 1224 » g->gcguard = g->stackguard; |
1006 » » execute(gp); // Never returns. | 1225 » g->status = Gsyscall; |
1007 » } | 1226 » if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { |
1008 » if(m->lockedg) { | 1227 » » // runtime·printf("entersyscallblock inconsistent %p [%p,%p]\n", |
1009 » » CHECK(m->lockedg == gp, ("exitsyscall0: inconsistent locking\n")
); | 1228 » » //» g->gcsp, g->gcguard-StackGuard, g->gcstack); |
1010 » » // Wait until another thread schedules gp and so m again. | 1229 » » runtime·throw("entersyscallblock"); |
1011 » » stoplockedm(); | 1230 » } |
1012 » » execute(gp); // Never returns. | 1231 |
1013 » } | 1232 » p = releasep(); |
1014 » stopm(); | 1233 » handoffp(p); |
1015 » schedule(); // Never returns. | 1234 » if(g == scvg) // do not consider blocked scavenger for deadlock detecti
on |
| 1235 » » inclocked(1); |
| 1236 » runtime·gosave(&g->sched); // re-save for traceback |
1016 } | 1237 } |
1017 | 1238 |
1018 // The goroutine g exited its system call. | 1239 // The goroutine g exited its system call. |
1019 // Arrange for it to run on a cpu again. | 1240 // Arrange for it to run on a cpu again. |
1020 // This is called only from the go syscall library, not | 1241 // This is called only from the go syscall library, not |
1021 // from the low-level system calls used by the runtime. | 1242 // from the low-level system calls used by the runtime. |
1022 void | 1243 void |
1023 runtime·exitsyscall(void) | 1244 runtime·exitsyscall(void) |
1024 { | 1245 { |
1025 uint32 s; | |
1026 P *p; | 1246 P *p; |
1027 | |
1028 LOG("%d: exitsyscall g=%p scvg=%d\n", m->id, g, g==scvg); | |
1029 | 1247 |
1030 // Check whether the profiler needs to be turned on. | 1248 // Check whether the profiler needs to be turned on. |
1031 if(m->profilehz > 0) | 1249 if(m->profilehz > 0) |
1032 runtime·setprof(true); | 1250 runtime·setprof(true); |
1033 | 1251 |
1034 // Try to re-acquire the last P. | 1252 // Try to re-acquire the last P. |
1035 » s = m->p ? m->p->status : Pidle; | 1253 » if(m->p && m->p->status == Psyscall && runtime·cas(&m->p->status, Psysca
ll, Prunning)) { |
1036 » if(s == Psyscall && runtime·cas(&m->p->status, s, Prunning)) { | |
1037 » » LOG("%d: exitsyscall fast\n", m->id); | |
1038 // There's a cpu for us, so we can run. | 1254 // There's a cpu for us, so we can run. |
1039 m->mcache = m->p->mcache; | 1255 m->mcache = m->p->mcache; |
1040 m->p->m = m; | 1256 m->p->m = m; |
1041 m->p->tick++; | 1257 m->p->tick++; |
1042 g->status = Grunning; | 1258 g->status = Grunning; |
1043 // Garbage collector isn't running (since we are), | 1259 // Garbage collector isn't running (since we are), |
1044 » » // so okay to clear gcstack. | 1260 » » // so okay to clear gcstack and gcsp. |
1045 g->gcstack = (uintptr)nil; | 1261 g->gcstack = (uintptr)nil; |
| 1262 g->gcsp = (uintptr)nil; |
1046 return; | 1263 return; |
1047 } | 1264 } |
1048 | 1265 |
1049 » if(g == scvg) { | 1266 » if(g == scvg) // do not consider blocked scavenger for deadlock detecti
on |
1050 » » runtime·lock(&runtime·sched); | 1267 » » inclocked(-1); |
1051 » » runtime·sched.mlocked--; | |
1052 » » runtime·unlock(&runtime·sched); | |
1053 » } | |
1054 | |
1055 // Try to get any other idle P. | 1268 // Try to get any other idle P. |
1056 m->p = nil; | 1269 m->p = nil; |
1057 if(runtime·sched.pidle) { | 1270 if(runtime·sched.pidle) { |
1058 runtime·lock(&runtime·sched); | 1271 runtime·lock(&runtime·sched); |
1059 p = pidleget(); | 1272 p = pidleget(); |
1060 runtime·unlock(&runtime·sched); | 1273 runtime·unlock(&runtime·sched); |
1061 if(p) { | 1274 if(p) { |
1062 » » » acquirep(m, p); | 1275 » » » acquirep(p); |
1063 g->gcstack = (uintptr)nil; | 1276 g->gcstack = (uintptr)nil; |
| 1277 g->gcsp = (uintptr)nil; |
1064 return; | 1278 return; |
1065 } | 1279 } |
1066 } | 1280 } |
1067 | 1281 |
1068 // Call the scheduler. | 1282 // Call the scheduler. |
1069 LOG("%d: exitsyscall slow p->status=%d\n", m->id, s); | |
1070 runtime·mcall(exitsyscall0); | 1283 runtime·mcall(exitsyscall0); |
1071 | 1284 |
1072 // Scheduler returned, so we're allowed to run now. | 1285 // Scheduler returned, so we're allowed to run now. |
1073 // Delete the gcstack information that we left for | 1286 // Delete the gcstack information that we left for |
1074 // the garbage collector during the system call. | 1287 // the garbage collector during the system call. |
1075 // Must wait until now because until gosched returns | 1288 // Must wait until now because until gosched returns |
1076 // we don't know for sure that the garbage collector | 1289 // we don't know for sure that the garbage collector |
1077 // is not running. | 1290 // is not running. |
1078 g->gcstack = (uintptr)nil; | 1291 g->gcstack = (uintptr)nil; |
| 1292 g->gcsp = (uintptr)nil; |
| 1293 } |
| 1294 |
| 1295 // runtime·exitsyscall slow path on g0. |
| 1296 // Failed to acquire P, enqueue gp as runnable. |
| 1297 static void |
| 1298 exitsyscall0(G *gp) |
| 1299 { |
| 1300 P *p; |
| 1301 |
| 1302 gp->status = Grunnable; |
| 1303 gp->m = nil; |
| 1304 m->curg = nil; |
| 1305 runtime·lock(&runtime·sched); |
| 1306 p = pidleget(); |
| 1307 if(p == nil) |
| 1308 globrunqput(gp); |
| 1309 runtime·unlock(&runtime·sched); |
| 1310 if(p) { |
| 1311 acquirep(p); |
| 1312 execute(gp); // Never returns. |
| 1313 } |
| 1314 if(m->lockedg) { |
| 1315 // Wait until another thread schedules gp and so m again. |
| 1316 stoplockedm(); |
| 1317 execute(gp); // Never returns. |
| 1318 } |
| 1319 stopm(); |
| 1320 schedule(); // Never returns. |
1079 } | 1321 } |
1080 | 1322 |
1081 // Hook used by runtime·malg to call runtime·stackalloc on the | 1323 // Hook used by runtime·malg to call runtime·stackalloc on the |
1082 // scheduler stack. This exists because runtime·stackalloc insists | 1324 // scheduler stack. This exists because runtime·stackalloc insists |
1083 // on being called on the scheduler stack, to avoid trying to grow | 1325 // on being called on the scheduler stack, to avoid trying to grow |
1084 // the stack while allocating a new stack segment. | 1326 // the stack while allocating a new stack segment. |
1085 static void | 1327 static void |
1086 mstackalloc(G *gp) | 1328 mstackalloc(G *gp) |
1087 { | 1329 { |
1088 gp->param = runtime·stackalloc((uintptr)gp->param); | 1330 gp->param = runtime·stackalloc((uintptr)gp->param); |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1123 | 1365 |
1124 // Create a new g running fn with siz bytes of arguments. | 1366 // Create a new g running fn with siz bytes of arguments. |
1125 // Put it on the queue of g's waiting to run. | 1367 // Put it on the queue of g's waiting to run. |
1126 // The compiler turns a go statement into a call to this. | 1368 // The compiler turns a go statement into a call to this. |
1127 // Cannot split the stack because it assumes that the arguments | 1369 // Cannot split the stack because it assumes that the arguments |
1128 // are available sequentially after &fn; they would not be | 1370 // are available sequentially after &fn; they would not be |
1129 // copied if a stack split occurred. It's OK for this to call | 1371 // copied if a stack split occurred. It's OK for this to call |
1130 // functions that split the stack. | 1372 // functions that split the stack. |
1131 #pragma textflag 7 | 1373 #pragma textflag 7 |
1132 void | 1374 void |
1133 runtime·newproc(int32 siz, byte* fn, ...) | 1375 runtime·newproc(int32 siz, FuncVal* fn, ...) |
1134 { | 1376 { |
1135 byte *argp; | 1377 byte *argp; |
1136 | 1378 |
1137 if(thechar == '5') | 1379 if(thechar == '5') |
1138 argp = (byte*)(&fn+2); // skip caller's saved LR | 1380 argp = (byte*)(&fn+2); // skip caller's saved LR |
1139 else | 1381 else |
1140 argp = (byte*)(&fn+1); | 1382 argp = (byte*)(&fn+1); |
1141 runtime·newproc1(fn, argp, siz, 0, runtime·getcallerpc(&siz)); | 1383 runtime·newproc1(fn, argp, siz, 0, runtime·getcallerpc(&siz)); |
1142 } | 1384 } |
1143 | 1385 |
1144 // Create a new g running fn with narg bytes of arguments starting | 1386 // Create a new g running fn with narg bytes of arguments starting |
1145 // at argp and returning nret bytes of results. callerpc is the | 1387 // at argp and returning nret bytes of results. callerpc is the |
1146 // address of the go statement that created this. The new g is put | 1388 // address of the go statement that created this. The new g is put |
1147 // on the queue of g's waiting to run. | 1389 // on the queue of g's waiting to run. |
1148 G* | 1390 G* |
1149 runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) | 1391 runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerpc
) |
1150 { | 1392 { |
1151 byte *sp; | 1393 byte *sp; |
1152 G *newg; | 1394 G *newg; |
1153 int32 siz; | 1395 int32 siz; |
1154 | 1396 |
| 1397 //printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret); |
1155 siz = narg + nret; | 1398 siz = narg + nret; |
1156 siz = (siz+7) & ~7; | 1399 siz = (siz+7) & ~7; |
1157 | 1400 |
1158 // We could instead create a secondary stack frame | 1401 // We could instead create a secondary stack frame |
1159 // and make it look like goexit was on the original but | 1402 // and make it look like goexit was on the original but |
1160 // the call to the actual goroutine function was split. | 1403 // the call to the actual goroutine function was split. |
1161 // Not worth it: this is almost always an error. | 1404 // Not worth it: this is almost always an error. |
1162 if(siz > StackMin - 1024) | 1405 if(siz > StackMin - 1024) |
1163 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); | 1406 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); |
1164 | 1407 |
1165 if((newg = gfget(m->p)) != nil) { | 1408 if((newg = gfget(m->p)) != nil) { |
1166 if(newg->stackguard - StackGuard != newg->stack0) | 1409 if(newg->stackguard - StackGuard != newg->stack0) |
1167 runtime·throw("invalid stack in newg"); | 1410 runtime·throw("invalid stack in newg"); |
1168 } else { | 1411 } else { |
1169 newg = runtime·malg(StackMin); | 1412 newg = runtime·malg(StackMin); |
1170 runtime·lock(&runtime·sched); | 1413 runtime·lock(&runtime·sched); |
1171 newg->goid = ++runtime·sched.goidseq; | |
1172 if(runtime·lastg == nil) | 1414 if(runtime·lastg == nil) |
1173 runtime·allg = newg; | 1415 runtime·allg = newg; |
1174 else | 1416 else |
1175 runtime·lastg->alllink = newg; | 1417 runtime·lastg->alllink = newg; |
1176 runtime·lastg = newg; | 1418 runtime·lastg = newg; |
1177 runtime·unlock(&runtime·sched); | 1419 runtime·unlock(&runtime·sched); |
1178 } | 1420 } |
1179 | 1421 |
1180 sp = (byte*)newg->stackbase; | 1422 sp = (byte*)newg->stackbase; |
1181 sp -= siz; | 1423 sp -= siz; |
1182 runtime·memmove(sp, argp, narg); | 1424 runtime·memmove(sp, argp, narg); |
1183 if(thechar == '5') { | 1425 if(thechar == '5') { |
1184 // caller's LR | 1426 // caller's LR |
1185 sp -= sizeof(void*); | 1427 sp -= sizeof(void*); |
1186 *(void**)sp = nil; | 1428 *(void**)sp = nil; |
1187 } | 1429 } |
1188 | 1430 |
1189 LOG("%d: newproc %p\n", m->id, newg); | |
1190 newg->sched.sp = (uintptr)sp; | 1431 newg->sched.sp = (uintptr)sp; |
1191 newg->sched.pc = (byte*)runtime·goexit; | 1432 newg->sched.pc = (byte*)runtime·goexit; |
1192 newg->sched.g = newg; | 1433 newg->sched.g = newg; |
1193 » newg->entry = fn; | 1434 » newg->fnstart = fn; |
1194 newg->gopc = (uintptr)callerpc; | 1435 newg->gopc = (uintptr)callerpc; |
1195 newg->status = Grunnable; | 1436 newg->status = Grunnable; |
| 1437 newg->goid = runtime·xadd64(&runtime·sched.goidgen, 1); |
1196 if(raceenabled) | 1438 if(raceenabled) |
1197 newg->racectx = runtime·racegostart(callerpc); | 1439 newg->racectx = runtime·racegostart(callerpc); |
1198 runqput(m->p, newg); | 1440 runqput(m->p, newg); |
1199 | 1441 |
1200 » if(runtime·sched.pidle && fn != (byte*)runtime·main) | 1442 » if(runtime·sched.npidle != 0 && runtime·sched.nmspinning == 0 && fn->fn
!= runtime·main) // TODO: fast atomic |
1201 » » startm(nil); | 1443 » » wakep(); |
1202 return newg; | 1444 return newg; |
1203 } | |
1204 | |
1205 void | |
1206 runtime·Breakpoint(void) | |
1207 { | |
1208 runtime·breakpoint(); | |
1209 } | |
1210 | |
1211 void | |
1212 runtime·Gosched(void) | |
1213 { | |
1214 runtime·gosched(); | |
1215 } | |
1216 | |
1217 // Implementation of runtime.GOMAXPROCS. | |
1218 int32 | |
1219 runtime·gomaxprocsfunc(int32 n) | |
1220 { | |
1221 int32 ret; | |
1222 | |
1223 LOG("%d: gomaxprocsfunc %d\n", m->id, n); | |
1224 if(n > maxgomaxprocs) | |
1225 n = maxgomaxprocs; | |
1226 runtime·lock(&runtime·sched); | |
1227 ret = runtime·gomaxprocs; | |
1228 if(n <= 0 || n == ret) { | |
1229 runtime·unlock(&runtime·sched); | |
1230 return ret; | |
1231 } | |
1232 runtime·unlock(&runtime·sched); | |
1233 | |
1234 runtime·semacquire(&runtime·worldsema); | |
1235 m->gcing = 1; | |
1236 runtime·stoptheworld(); | |
1237 newprocs = n; | |
1238 m->gcing = 0; | |
1239 runtime·semrelease(&runtime·worldsema); | |
1240 runtime·starttheworld(); | |
1241 | |
1242 return ret; | |
1243 } | |
1244 | |
1245 static void | |
1246 LockOSThread(void) | |
1247 { | |
1248 m->lockedg = g; | |
1249 g->lockedm = m; | |
1250 } | |
1251 | |
1252 void | |
1253 runtime·LockOSThread(void) | |
1254 { | |
1255 m->locked |= LockExternal; | |
1256 LockOSThread(); | |
1257 } | |
1258 | |
1259 void | |
1260 runtime·lockOSThread(void) | |
1261 { | |
1262 m->locked += LockInternal; | |
1263 LockOSThread(); | |
1264 } | |
1265 | |
1266 static void | |
1267 UnlockOSThread(void) | |
1268 { | |
1269 if(m->locked != 0) | |
1270 return; | |
1271 m->lockedg = nil; | |
1272 g->lockedm = nil; | |
1273 }······· | |
1274 | |
1275 void | |
1276 runtime·UnlockOSThread(void) | |
1277 { | |
1278 m->locked &= ~LockExternal; | |
1279 UnlockOSThread(); | |
1280 } | |
1281 | |
1282 void | |
1283 runtime·unlockOSThread(void) | |
1284 { | |
1285 if(m->locked < LockInternal) | |
1286 runtime·throw("runtime: internal error: misuse of lockOSThread/u
nlockOSThread"); | |
1287 m->locked -= LockInternal; | |
1288 UnlockOSThread(); | |
1289 } | |
1290 | |
1291 bool | |
1292 runtime·lockedOSThread(void) | |
1293 { | |
1294 return g->lockedm != nil && m->lockedg != nil; | |
1295 } | |
1296 | |
1297 // for testing of callbacks | |
1298 void | |
1299 runtime·golockedOSThread(bool ret) | |
1300 { | |
1301 ret = runtime·lockedOSThread(); | |
1302 FLUSH(&ret); | |
1303 } | |
1304 | |
1305 // for testing of wire, unwire | |
1306 void | |
1307 runtime·mid(uint32 ret) | |
1308 { | |
1309 ret = m->id; | |
1310 FLUSH(&ret); | |
1311 } | |
1312 | |
1313 void | |
1314 runtime·NumGoroutine(intgo ret) | |
1315 { | |
1316 // TODO(dvyukov): implement me plz | |
1317 //ret = runtime·sched.gcount; | |
1318 ret = 1; | |
1319 FLUSH(&ret); | |
1320 } | |
1321 | |
1322 int32 | |
1323 runtime·gcount(void) | |
1324 { | |
1325 // TODO(dvyukov): implement me plz | |
1326 //return runtime·sched.gcount; | |
1327 return 1; | |
1328 } | |
1329 | |
1330 int32 | |
1331 runtime·mcount(void) | |
1332 { | |
1333 return runtime·sched.mcount; | |
1334 } | |
1335 | |
1336 void | |
1337 runtime·badmcall(void) // called from assembly | |
1338 { | |
1339 runtime·throw("runtime: mcall called on m->g0 stack"); | |
1340 } | |
1341 | |
1342 void | |
1343 runtime·badmcall2(void) // called from assembly | |
1344 { | |
1345 runtime·throw("runtime: mcall function returned"); | |
1346 } | |
1347 | |
1348 static struct { | |
1349 Lock; | |
1350 void (*fn)(uintptr*, int32); | |
1351 int32 hz; | |
1352 uintptr pcbuf[100]; | |
1353 } prof; | |
1354 | |
1355 // Called if we receive a SIGPROF signal. | |
1356 void | |
1357 runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp) | |
1358 { | |
1359 int32 n; | |
1360 | |
1361 if (m == nil || m->mcache == nil) | |
1362 return; | |
1363 if(prof.fn == nil || prof.hz == 0) | |
1364 return; | |
1365 | |
1366 runtime·lock(&prof); | |
1367 if(prof.fn == nil) { | |
1368 runtime·unlock(&prof); | |
1369 return; | |
1370 } | |
1371 n = runtime·gentraceback(pc, sp, lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf
)); | |
1372 if(n > 0) | |
1373 prof.fn(prof.pcbuf, n); | |
1374 runtime·unlock(&prof); | |
1375 } | |
1376 | |
1377 // Arrange to call fn with a traceback hz times a second. | |
1378 void | |
1379 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) | |
1380 { | |
1381 // Force sane arguments. | |
1382 if(hz < 0) | |
1383 hz = 0; | |
1384 if(hz == 0) | |
1385 fn = nil; | |
1386 if(fn == nil) | |
1387 hz = 0; | |
1388 | |
1389 // Stop profiler on this cpu so that it is safe to lock prof. | |
1390 // if a profiling signal came in while we had prof locked, | |
1391 // it would deadlock. | |
1392 runtime·resetcpuprofiler(0); | |
1393 | |
1394 runtime·lock(&prof); | |
1395 prof.fn = fn; | |
1396 prof.hz = hz; | |
1397 runtime·unlock(&prof); | |
1398 runtime·lock(&runtime·sched); | |
1399 runtime·sched.profilehz = hz; | |
1400 runtime·unlock(&runtime·sched); | |
1401 | |
1402 if(hz != 0) | |
1403 runtime·resetcpuprofiler(hz); | |
1404 } | |
1405 | |
1406 // Change number of processors. The world is stopped. | |
1407 static void | |
1408 procresize(int32 new) | |
1409 { | |
1410 int32 i, old; | |
1411 G *gp; | |
1412 P *p; | |
1413 | |
1414 runtime·lock(&runtime·sched); //!!! | |
1415 old = runtime·gomaxprocs; | |
1416 LOG("%d: procresize %d->%d\n", m->id, old, new); | |
1417 if(old < 0 || old > maxgomaxprocs || new <= 0 || new > maxgomaxprocs) | |
1418 runtime·throw("procresize: invalid arg"); | |
1419 if(old == new) { | |
1420 for(i=0; i<new; i++) { | |
1421 p = runtime·allp[i]; | |
1422 if(p == m->p) | |
1423 p->status = Prunning; | |
1424 else { | |
1425 p->status = Pidle; | |
1426 pidleput(p); | |
1427 } | |
1428 } | |
1429 runtime·unlock(&runtime·sched); | |
1430 return; | |
1431 } | |
1432 | |
1433 runtime·singleproc = new == 1; | |
1434 runtime·gomaxprocs = new; | |
1435 for(i=0; i<new; i++) { | |
1436 p = runtime·allp[i]; | |
1437 if(p == nil) { | |
1438 p = (P*)runtime·mallocgc(sizeof(runtime·allp[i][0]), 0,
0, 1); | |
1439 p->status = Pgcstop; | |
1440 runtime·allp[i] = p; //@@@ store-release | |
1441 } | |
1442 if(p->mcache == nil) { | |
1443 if(old==0 && i==0) | |
1444 p->mcache = m->mcache; | |
1445 else | |
1446 p->mcache = runtime·allocmcache(); | |
1447 } | |
1448 if(p->runq == nil) { | |
1449 p->runqsize = 1024; | |
1450 p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*),
0, 0, 1); | |
1451 } | |
1452 } | |
1453 | |
1454 for(i=1; i<old; i++) { | |
1455 for(;;) { | |
1456 gp = runqget(runtime·allp[i]); | |
1457 if(gp == nil) | |
1458 break; | |
1459 //TODO: spread more evenly. | |
1460 runqput(runtime·allp[0], gp); | |
1461 } | |
1462 } | |
1463 | |
1464 for(i=new; i<old; i++) { | |
1465 runtime·freemcache(runtime·allp[i]->mcache); | |
1466 runtime·allp[i]->mcache = nil; | |
1467 runtime·allp[i]->status = Pdead; | |
1468 //TODO: free freeg | |
1469 } | |
1470 | |
1471 if(m->p) | |
1472 m->p->m = nil; | |
1473 m->p = nil; | |
1474 m->mcache = nil; | |
1475 runtime·allp[0]->m = nil; | |
1476 runtime·allp[0]->status = Pidle; | |
1477 acquirep(m, runtime·allp[0]); | |
1478 for(i=1; i<new; i++) { | |
1479 p = runtime·allp[i]; | |
1480 p->status = Pidle; | |
1481 pidleput(p); | |
1482 } | |
1483 runtime·unlock(&runtime·sched); | |
1484 } | |
1485 | |
1486 // Associate p and mp. | |
1487 static void | |
1488 acquirep(M *mp, P *p) | |
1489 { | |
1490 LOG("%d: acquirep m=%d p=%p p->m=%p, p->status=%d, p->mcache=%p\n", m->i
d, mp->id, p, p->m, p->status, p->mcache); | |
1491 if(mp->p || mp->mcache) | |
1492 runtime·throw("acquirep: already in go"); | |
1493 if(p->m || p->status != Pidle) { | |
1494 runtime·printf("acquirep: p->m=%p(%d) p->status=%d\n", p->m, p->
m ? p->m->id : 0, p->status); | |
1495 runtime·throw("acquirep: invalid p state"); | |
1496 } | |
1497 mp->mcache = p->mcache; | |
1498 mp->p = p; | |
1499 p->m = mp; | |
1500 p->status = Prunning; | |
1501 } | |
1502 | |
1503 // Disassociate p and the current m. | |
1504 static P* | |
1505 releasep(void) | |
1506 { | |
1507 M *mp; | |
1508 P *p; | |
1509 | |
1510 mp = m; | |
1511 LOG("%d: releasep\n", mp->id); | |
1512 // sched is locked | |
1513 if(mp->p == nil || mp->mcache == nil) | |
1514 runtime·throw("releasep: invalid arg"); | |
1515 p = mp->p; | |
1516 if(p->m != mp || p->mcache != mp->mcache || p->status != Prunning) { | |
1517 runtime·printf("releasep: m=%p m->p=%p p->m=%p m->mcache=%p p->m
cache=%p p->status=%d\n", | |
1518 mp, mp->p, p->m, m->mcache, p->mcache, p->status); | |
1519 runtime·throw("releasep: invalid p state"); | |
1520 } | |
1521 mp->p = nil; | |
1522 mp->mcache = nil; | |
1523 p->m = nil; | |
1524 p->status = Pidle; | |
1525 return p; | |
1526 } | |
1527 | |
1528 static void | |
1529 checkdead(void) | |
1530 { | |
1531 int32 running; | |
1532 | |
1533 LOG("%d: checkdead: mwait=%d mlocked=%d mcount=%d\n", m->id, runtime·sch
ed.mwait, runtime·sched.mlocked, runtime·sched.mcount); | |
1534 // -2 for sysmon and scvg | |
1535 running = runtime·sched.mcount - runtime·sched.mwait - runtime·sched.mlo
cked - 1; | |
1536 if(running > 0) | |
1537 return; | |
1538 runtime·printf("checkdead: mwait=%d mlocked=%d mcount=%d\n", | |
1539 runtime·sched.mwait, runtime·sched.mlocked, runtime·sche
d.mcount); | |
1540 if(running < 0) { | |
1541 runtime·printf("checkdead: mwait=%d mlocked=%d mcount=%d\n", | |
1542 runtime·sched.mwait, runtime·sched.mlocked, runtime·sche
d.mcount); | |
1543 runtime·throw("checkdead: inconsistent counts"); | |
1544 } | |
1545 m->throwing = -1; // do not dump full stacks | |
1546 runtime·throw("all goroutines are asleep - deadlock!"); | |
1547 } | |
1548 | |
1549 typedef struct Pdesc Pdesc; | |
1550 struct Pdesc | |
1551 { | |
1552 uint32 tick; | |
1553 int64 when; | |
1554 }; | |
1555 | |
1556 static void | |
1557 retake(int64 now, Pdesc *ps) | |
1558 { | |
1559 uint32 i, s; | |
1560 int64 t; | |
1561 P *p; | |
1562 | |
1563 for(i=0; i<runtime·gomaxprocs; i++) { | |
1564 p = runtime·allp[i]; | |
1565 //!!! procresize may be in progress | |
1566 // do something if GC is in progress (help). | |
1567 if(p==nil) | |
1568 continue; | |
1569 t = p->tick; | |
1570 if(ps[i].tick != t) { | |
1571 ps[i].tick = t; | |
1572 ps[i].when = now; | |
1573 } | |
1574 if(ps[i].when + 20*1000 > now) | |
1575 continue; | |
1576 s = p->status; | |
1577 if(s != Psyscall) | |
1578 continue; | |
1579 ················ | |
1580 runtime·lock(&runtime·sched); | |
1581 runtime·sched.mlocked--; | |
1582 runtime·unlock(&runtime·sched); | |
1583 ················ | |
1584 if(runtime·cas(&p->status, s, Pidle)) { | |
1585 LOG("retake %p(%d)\n", p, i); | |
1586 startm(p); | |
1587 } | |
1588 runtime·lock(&runtime·sched); | |
1589 runtime·sched.mlocked++; | |
1590 checkdead(); | |
1591 runtime·unlock(&runtime·sched); | |
1592 } | |
1593 } | |
1594 | |
1595 static Pdesc ps[maxgomaxprocs]; | |
1596 | |
1597 static void | |
1598 sysmon(void) | |
1599 { | |
1600 int64 now; | |
1601 | |
1602 // This is a special dedicated thread. | |
1603 // It works w/o mcache nor stackalloc, it may work concurrently with GC. | |
1604 runtime·asminit(); | |
1605 runtime·minit(); | |
1606 LOG("sysmon\n"); | |
1607 for(;;) { | |
1608 //!!! become poller if it's not polled for too long | |
1609 //!!! if no running P's -> block | |
1610 //!!! if do not wake anybody for some time -> sleep longer | |
1611 runtime·usleep(20); | |
1612 now = runtime·nanotime(); | |
1613 if(runtime·gcwaiting) { | |
1614 runtime·lock(&runtime·sched); | |
1615 if(runtime·gcwaiting) { | |
1616 runtime·sched.sysmonwait = 1; | |
1617 runtime·unlock(&runtime·sched); | |
1618 runtime·notesleep(&runtime·sched.sysmonnote); | |
1619 runtime·noteclear(&runtime·sched.sysmonnote); | |
1620 } else | |
1621 runtime·unlock(&runtime·sched); | |
1622 } | |
1623 retake(now, ps); | |
1624 } | |
1625 } | |
1626 | |
1627 void | |
1628 runtime·inject(G *gp0, int32 *w, int32 *n) | |
1629 { | |
1630 int32 nw; | |
1631 G *gp; | |
1632 | |
1633 runtime·lock(&runtime·sched); | |
1634 while(gp0) { | |
1635 gp = gp0; | |
1636 gp0 = gp->schedlink; | |
1637 gp->status = Grunnable; | |
1638 globrunqput(gp); | |
1639 (*n)++; | |
1640 } | |
1641 LOG("%d: inject %d\n", m->id, *n); | |
1642 runtime·unlock(&runtime·sched); | |
1643 | |
1644 nw = *n; | |
1645 while(runtime·sched.pidle && nw) { | |
1646 if(!startm(nil)) | |
1647 break; | |
1648 (*w)++; | |
1649 nw--; | |
1650 } | |
1651 } | |
1652 | |
1653 // Put gp on the global runnable queue. | |
1654 // Sched must be locked. | |
1655 static void | |
1656 globrunqput(G *gp) | |
1657 { | |
1658 gp->schedlink = nil; | |
1659 if(runtime·sched.runqtail) | |
1660 runtime·sched.runqtail->schedlink = gp; | |
1661 else | |
1662 runtime·sched.runqhead = gp; | |
1663 runtime·sched.runqtail = gp; | |
1664 runtime·sched.runqsize++; | |
1665 } | |
1666 | |
1667 // Try get a batch of G's from the global runnable queue. | |
1668 // Sched must be locked. | |
1669 static G* | |
1670 globrunqget(void) | |
1671 { | |
1672 G *gp, *gp1; | |
1673 int32 n; | |
1674 | |
1675 if(runtime·sched.runqsize == 0) | |
1676 return nil; | |
1677 n = runtime·sched.runqsize/runtime·gomaxprocs+1; | |
1678 if(n > runtime·sched.runqsize) | |
1679 n = runtime·sched.runqsize; | |
1680 runtime·sched.runqsize -= n; | |
1681 if(runtime·sched.runqsize == 0) | |
1682 runtime·sched.runqtail = nil; | |
1683 gp1 = nil; | |
1684 while(n--) { | |
1685 gp = runtime·sched.runqhead; | |
1686 runtime·sched.runqhead = gp->schedlink; | |
1687 gp->schedlink = gp1; | |
1688 gp1 = gp; | |
1689 } | |
1690 return gp1; | |
1691 } | |
1692 | |
1693 // Put g on local runnable queue. | |
1694 static void | |
1695 runqput(P *p, G *gp) | |
1696 { | |
1697 int32 h, t, s; | |
1698 | |
1699 CHECK(m->mcache, ("runqput: no mcache\n")); | |
1700 runtime·lock(p); | |
1701 retry: | |
1702 h = p->runqhead; | |
1703 t = p->runqtail; | |
1704 s = p->runqsize; | |
1705 if(t==h-1 || (h==0 && t==s-1)) { | |
1706 runqgrow(p); | |
1707 goto retry; | |
1708 } | |
1709 p->runq[t] = gp; | |
1710 t++; | |
1711 if(t==s) | |
1712 t = 0; | |
1713 p->runqtail = t; | |
1714 runtime·unlock(p); | |
1715 } | |
1716 | |
1717 // Get g from local runnable queue. | |
1718 static G* | |
1719 runqget(P *p) | |
1720 { | |
1721 G *gp; | |
1722 int32 t, h, s; | |
1723 | |
1724 if(p->runqhead==p->runqtail) | |
1725 return nil; | |
1726 runtime·lock(p); | |
1727 h = p->runqhead; | |
1728 t = p->runqtail; | |
1729 s = p->runqsize; | |
1730 if(t==h) { | |
1731 runtime·unlock(p); | |
1732 return nil; | |
1733 } | |
1734 gp = p->runq[h]; | |
1735 h++; | |
1736 if(h==s) | |
1737 h = 0; | |
1738 p->runqhead = h; | |
1739 runtime·unlock(p); | |
1740 return gp; | |
1741 } | |
1742 | |
1743 // Grow local runnable queue. | |
1744 // TODO(dvyukov): consider using fixed-size array | |
1745 // and transfer excess to the global list (local queue can grow way too big). | |
1746 static void | |
1747 runqgrow(P *p) | |
1748 { | |
1749 G **q; | |
1750 int32 s, t, h, t2; | |
1751 | |
1752 h = p->runqhead; | |
1753 t = p->runqtail; | |
1754 s = p->runqsize; | |
1755 t2 = 0; | |
1756 q = (G**)runtime·malloc(2*s*sizeof(*q)); | |
1757 while(t!=h) { | |
1758 q[t2] = p->runq[h]; | |
1759 t2++; | |
1760 h++; | |
1761 if(h==s) | |
1762 h = 0; | |
1763 } | |
1764 runtime·free(p->runq); | |
1765 p->runq = q; | |
1766 p->runqhead = 0; | |
1767 p->runqtail = t2; | |
1768 p->runqsize = 2*s; | |
1769 } | |
1770 | |
1771 // Steal half of elements from local runnable queue of p2 | |
1772 // and put onto local runnable queue of p. | |
1773 // Returns one of the stolen elements (or nil if failed). | |
1774 static G* | |
1775 runqsteal(P *p, P *p2) | |
1776 { | |
1777 G *gp, *gp1; | |
1778 int32 t, h, s, t2, h2, s2, c, c1; | |
1779 | |
1780 if(p2->runqhead==p2->runqtail) | |
1781 return nil; | |
1782 if(p < p2) | |
1783 runtime·lock(p); | |
1784 runtime·lock(p2); | |
1785 if(p2->runqhead==p2->runqtail) { | |
1786 runtime·unlock(p2); | |
1787 if(p < p2) | |
1788 runtime·unlock(p); | |
1789 return nil; | |
1790 } | |
1791 if(p >= p2) | |
1792 runtime·lock(p); | |
1793 h = p->runqhead; | |
1794 t = p->runqtail; | |
1795 s = p->runqsize; | |
1796 h2 = p2->runqhead; | |
1797 t2 = p2->runqtail; | |
1798 s2 = p2->runqsize; | |
1799 gp = p2->runq[h2]; | |
1800 h2++; | |
1801 if(h2 == s2) | |
1802 h2 = 0; | |
1803 if(t2 > h2) | |
1804 c = (t2 - h2) / 2; | |
1805 else | |
1806 c = (s2 - h2 + t2) / 2; | |
1807 c1 = 0; | |
1808 for(;;) { | |
1809 if(c1 == c) | |
1810 break; | |
1811 if(t==h-1 || (h==0 && t==s-1)) | |
1812 break; | |
1813 if(t2==h2) | |
1814 break; | |
1815 gp1 = p2->runq[h2]; | |
1816 h2++; | |
1817 if(h2 == s2) | |
1818 h2 = 0; | |
1819 p->runq[t] = gp1; | |
1820 t++; | |
1821 if(t==s) | |
1822 t = 0; | |
1823 c1++; | |
1824 } | |
1825 p->runqtail = t; | |
1826 p2->runqhead = h2; | |
1827 runtime·unlock(p2); | |
1828 runtime·unlock(p); | |
1829 return gp; | |
1830 } | 1445 } |
1831 | 1446 |
1832 // Put on gfree list. | 1447 // Put on gfree list. |
1833 // If local list is too long, transfer a batch to the global list. | 1448 // If local list is too long, transfer a batch to the global list. |
1834 static void | 1449 static void |
1835 gfput(P *p, G *gp) | 1450 gfput(P *p, G *gp) |
1836 { | 1451 { |
1837 if(gp->stackguard - StackGuard != gp->stack0) | 1452 if(gp->stackguard - StackGuard != gp->stack0) |
1838 runtime·throw("invalid stack in gfput"); | 1453 runtime·throw("invalid stack in gfput"); |
1839 gp->schedlink = p->gfree; | 1454 gp->schedlink = p->gfree; |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1873 runtime·unlock(&runtime·sched.gflock); | 1488 runtime·unlock(&runtime·sched.gflock); |
1874 goto retry; | 1489 goto retry; |
1875 } | 1490 } |
1876 if(gp) { | 1491 if(gp) { |
1877 p->gfree = gp->schedlink; | 1492 p->gfree = gp->schedlink; |
1878 p->gfreecnt--; | 1493 p->gfreecnt--; |
1879 } | 1494 } |
1880 return gp; | 1495 return gp; |
1881 } | 1496 } |
1882 | 1497 |
1883 // Put mp on mwait list. | 1498 // Purge all cached G's from gfree list to the global list. |
| 1499 static void |
| 1500 gfpurge(P *p) |
| 1501 { |
| 1502 » G *gp; |
| 1503 |
| 1504 » runtime·lock(&runtime·sched.gflock); |
| 1505 » while(p->gfreecnt) { |
| 1506 » » p->gfreecnt--; |
| 1507 » » gp = p->gfree; |
| 1508 » » p->gfree = gp->schedlink; |
| 1509 » » gp->schedlink = runtime·sched.gfree; |
| 1510 » » runtime·sched.gfree = gp; |
| 1511 » } |
| 1512 » runtime·unlock(&runtime·sched.gflock); |
| 1513 } |
| 1514 |
| 1515 void |
| 1516 runtime·Breakpoint(void) |
| 1517 { |
| 1518 » runtime·breakpoint(); |
| 1519 } |
| 1520 |
| 1521 void |
| 1522 runtime·Gosched(void) |
| 1523 { |
| 1524 » runtime·gosched(); |
| 1525 } |
| 1526 |
| 1527 // Implementation of runtime.GOMAXPROCS. |
| 1528 // delete when scheduler is even stronger |
| 1529 int32 |
| 1530 runtime·gomaxprocsfunc(int32 n) |
| 1531 { |
| 1532 » int32 ret; |
| 1533 |
| 1534 » if(n > MaxGomaxprocs) |
| 1535 » » n = MaxGomaxprocs; |
| 1536 » runtime·lock(&runtime·sched); |
| 1537 » ret = runtime·gomaxprocs; |
| 1538 » if(n <= 0 || n == ret) { |
| 1539 » » runtime·unlock(&runtime·sched); |
| 1540 » » return ret; |
| 1541 » } |
| 1542 » runtime·unlock(&runtime·sched); |
| 1543 |
| 1544 » runtime·semacquire(&runtime·worldsema); |
| 1545 » m->gcing = 1; |
| 1546 » runtime·stoptheworld(); |
| 1547 » newprocs = n; |
| 1548 » m->gcing = 0; |
| 1549 » runtime·semrelease(&runtime·worldsema); |
| 1550 » runtime·starttheworld(); |
| 1551 |
| 1552 » return ret; |
| 1553 } |
| 1554 |
| 1555 static void |
| 1556 LockOSThread(void) |
| 1557 { |
| 1558 » m->lockedg = g; |
| 1559 » g->lockedm = m; |
| 1560 } |
| 1561 |
| 1562 void |
| 1563 runtime·LockOSThread(void) |
| 1564 { |
| 1565 » m->locked |= LockExternal; |
| 1566 » LockOSThread(); |
| 1567 } |
| 1568 |
| 1569 void |
| 1570 runtime·lockOSThread(void) |
| 1571 { |
| 1572 » m->locked += LockInternal; |
| 1573 » LockOSThread(); |
| 1574 } |
| 1575 |
| 1576 static void |
| 1577 UnlockOSThread(void) |
| 1578 { |
| 1579 » if(m->locked != 0) |
| 1580 » » return; |
| 1581 » m->lockedg = nil; |
| 1582 » g->lockedm = nil; |
| 1583 } |
| 1584 |
| 1585 void |
| 1586 runtime·UnlockOSThread(void) |
| 1587 { |
| 1588 » m->locked &= ~LockExternal; |
| 1589 » UnlockOSThread(); |
| 1590 } |
| 1591 |
| 1592 void |
| 1593 runtime·unlockOSThread(void) |
| 1594 { |
| 1595 » if(m->locked < LockInternal) |
| 1596 » » runtime·throw("runtime: internal error: misuse of lockOSThread/u
nlockOSThread"); |
| 1597 » m->locked -= LockInternal; |
| 1598 » UnlockOSThread(); |
| 1599 } |
| 1600 |
| 1601 bool |
| 1602 runtime·lockedOSThread(void) |
| 1603 { |
| 1604 » return g->lockedm != nil && m->lockedg != nil; |
| 1605 } |
| 1606 |
| 1607 // for testing of callbacks |
| 1608 void |
| 1609 runtime·golockedOSThread(bool ret) |
| 1610 { |
| 1611 » ret = runtime·lockedOSThread(); |
| 1612 » FLUSH(&ret); |
| 1613 } |
| 1614 |
| 1615 // for testing of wire, unwire |
| 1616 void |
| 1617 runtime·mid(uint32 ret) |
| 1618 { |
| 1619 » ret = m->id; |
| 1620 » FLUSH(&ret); |
| 1621 } |
| 1622 |
| 1623 void |
| 1624 runtime·NumGoroutine(intgo ret) |
| 1625 { |
| 1626 » ret = runtime·gcount(); |
| 1627 » FLUSH(&ret); |
| 1628 } |
| 1629 |
| 1630 int32 |
| 1631 runtime·gcount(void) |
| 1632 { |
| 1633 » G *gp; |
| 1634 » int32 n, s; |
| 1635 |
| 1636 » n = 0; |
| 1637 » runtime·lock(&runtime·sched); |
| 1638 » // TODO(dvyukov): runtime.NumGoroutine() is O(N). |
| 1639 » // We do not want to increment/decrement centralized counter in newproc/
goexit, |
| 1640 » // just to make runtime.NumGoroutine() faster. |
| 1641 » // Compromise solution is to introduce per-P counters of active goroutin
es. |
| 1642 » for(gp = runtime·allg; gp; gp = gp->alllink) { |
| 1643 » » s = gp->status; |
| 1644 » » if(s == Grunnable || s == Grunning || s == Gsyscall || s == Gwai
ting) |
| 1645 » » » n++; |
| 1646 » } |
| 1647 » runtime·unlock(&runtime·sched); |
| 1648 » return n; |
| 1649 } |
| 1650 |
| 1651 int32 |
| 1652 runtime·mcount(void) |
| 1653 { |
| 1654 » return runtime·sched.mcount; |
| 1655 } |
| 1656 |
| 1657 void |
| 1658 runtime·badmcall(void) // called from assembly |
| 1659 { |
| 1660 » runtime·throw("runtime: mcall called on m->g0 stack"); |
| 1661 } |
| 1662 |
| 1663 void |
| 1664 runtime·badmcall2(void) // called from assembly |
| 1665 { |
| 1666 » runtime·throw("runtime: mcall function returned"); |
| 1667 } |
| 1668 |
| 1669 static struct { |
| 1670 » Lock; |
| 1671 » void (*fn)(uintptr*, int32); |
| 1672 » int32 hz; |
| 1673 » uintptr pcbuf[100]; |
| 1674 } prof; |
| 1675 |
| 1676 // Called if we receive a SIGPROF signal. |
| 1677 void |
| 1678 runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp) |
| 1679 { |
| 1680 » int32 n; |
| 1681 |
| 1682 » // Windows does profiling in a dedicated thread w/o m. |
| 1683 » if(!Windows && (m == nil || m->mcache == nil)) |
| 1684 » » return; |
| 1685 » if(prof.fn == nil || prof.hz == 0) |
| 1686 » » return; |
| 1687 |
| 1688 » runtime·lock(&prof); |
| 1689 » if(prof.fn == nil) { |
| 1690 » » runtime·unlock(&prof); |
| 1691 » » return; |
| 1692 » } |
| 1693 » n = runtime·gentraceback(pc, sp, lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf
)); |
| 1694 » if(n > 0) |
| 1695 » » prof.fn(prof.pcbuf, n); |
| 1696 » runtime·unlock(&prof); |
| 1697 } |
| 1698 |
| 1699 // Arrange to call fn with a traceback hz times a second. |
| 1700 void |
| 1701 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) |
| 1702 { |
| 1703 » // Force sane arguments. |
| 1704 » if(hz < 0) |
| 1705 » » hz = 0; |
| 1706 » if(hz == 0) |
| 1707 » » fn = nil; |
| 1708 » if(fn == nil) |
| 1709 » » hz = 0; |
| 1710 |
| 1711 » // Stop profiler on this cpu so that it is safe to lock prof. |
| 1712 » // if a profiling signal came in while we had prof locked, |
| 1713 » // it would deadlock. |
| 1714 » runtime·resetcpuprofiler(0); |
| 1715 |
| 1716 » runtime·lock(&prof); |
| 1717 » prof.fn = fn; |
| 1718 » prof.hz = hz; |
| 1719 » runtime·unlock(&prof); |
| 1720 » runtime·lock(&runtime·sched); |
| 1721 » runtime·sched.profilehz = hz; |
| 1722 » runtime·unlock(&runtime·sched); |
| 1723 |
| 1724 » if(hz != 0) |
| 1725 » » runtime·resetcpuprofiler(hz); |
| 1726 } |
| 1727 |
| 1728 // Change number of processors. The world is stopped, sched is locked. |
| 1729 static void |
| 1730 procresize(int32 new) |
| 1731 { |
| 1732 » int32 i, old; |
| 1733 » G *gp; |
| 1734 » P *p; |
| 1735 |
| 1736 » old = runtime·gomaxprocs; |
| 1737 » if(old < 0 || old > MaxGomaxprocs || new <= 0 || new >MaxGomaxprocs) |
| 1738 » » runtime·throw("procresize: invalid arg"); |
| 1739 » // initialize new P's |
| 1740 » for(i = 0; i < new; i++) { |
| 1741 » » p = runtime·allp[i]; |
| 1742 » » if(p == nil) { |
| 1743 » » » p = (P*)runtime·mallocgc(sizeof(*p), 0, 0, 1); |
| 1744 » » » p->status = Pgcstop; |
| 1745 » » » runtime·atomicstorep(&runtime·allp[i], p); |
| 1746 » » } |
| 1747 » » if(p->mcache == nil) { |
| 1748 » » » if(old==0 && i==0) |
| 1749 » » » » p->mcache = m->mcache; // bootstrap |
| 1750 » » » else |
| 1751 » » » » p->mcache = runtime·allocmcache(); |
| 1752 » » } |
| 1753 » » if(p->runq == nil) { |
| 1754 » » » p->runqsize = 128; |
| 1755 » » » p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*),
0, 0, 1); |
| 1756 » » } |
| 1757 » } |
| 1758 |
| 1759 » // redistribute runnable G's evenly |
| 1760 » for(i = 0; i < old; i++) { |
| 1761 » » p = runtime·allp[i]; |
| 1762 » » while(gp = runqget(p)) |
| 1763 » » » globrunqput(gp); |
| 1764 » } |
| 1765 » // start at 1 because current M already executes some G and will acquire
allp[0] below, |
| 1766 » // so if we have a spare G we want to put it into allp[1]. |
| 1767 » for(i = 1; runtime·sched.runqhead; i++) { |
| 1768 » » gp = runtime·sched.runqhead; |
| 1769 » » runtime·sched.runqhead = gp->schedlink; |
| 1770 » » runqput(runtime·allp[i%new], gp); |
| 1771 » } |
| 1772 » runtime·sched.runqtail = nil; |
| 1773 » runtime·sched.runqsize = 0; |
| 1774 |
| 1775 » // free unused P's |
| 1776 » for(i = new; i < old; i++) { |
| 1777 » » p = runtime·allp[i]; |
| 1778 » » runtime·freemcache(p->mcache); |
| 1779 » » p->mcache = nil; |
| 1780 » » gfpurge(p); |
| 1781 » » p->status = Pdead; |
| 1782 » » // can't free P itself because it can be referenced by an M in s
yscall |
| 1783 » } |
| 1784 |
| 1785 » if(m->p) |
| 1786 » » m->p->m = nil; |
| 1787 » m->p = nil; |
| 1788 » m->mcache = nil; |
| 1789 » p = runtime·allp[0]; |
| 1790 » p->m = nil; |
| 1791 » p->status = Pidle; |
| 1792 » acquirep(p); |
| 1793 » for(i = new-1; i > 0; i--) { |
| 1794 » » p = runtime·allp[i]; |
| 1795 » » p->status = Pidle; |
| 1796 » » pidleput(p); |
| 1797 » } |
| 1798 » runtime·singleproc = new == 1; |
| 1799 » runtime·atomicstore((uint32*)&runtime·gomaxprocs, new); |
| 1800 } |
| 1801 |
| 1802 // Associate p and the current m. |
| 1803 static void |
| 1804 acquirep(P *p) |
| 1805 { |
| 1806 » if(m->p || m->mcache) |
| 1807 » » runtime·throw("acquirep: already in go"); |
| 1808 » if(p->m || p->status != Pidle) { |
| 1809 » » runtime·printf("acquirep: p->m=%p(%d) p->status=%d\n", p->m, p->
m ? p->m->id : 0, p->status); |
| 1810 » » runtime·throw("acquirep: invalid p state"); |
| 1811 » } |
| 1812 » m->mcache = p->mcache; |
| 1813 » m->p = p; |
| 1814 » p->m = m; |
| 1815 » p->status = Prunning; |
| 1816 } |
| 1817 |
| 1818 // Disassociate p and the current m. |
| 1819 static P* |
| 1820 releasep(void) |
| 1821 { |
| 1822 » P *p; |
| 1823 |
| 1824 » if(m->p == nil || m->mcache == nil) |
| 1825 » » runtime·throw("releasep: invalid arg"); |
| 1826 » p = m->p; |
| 1827 » if(p->m != m || p->mcache != m->mcache || p->status != Prunning) { |
| 1828 » » runtime·printf("releasep: m=%p m->p=%p p->m=%p m->mcache=%p p->m
cache=%p p->status=%d\n", |
| 1829 » » » m, m->p, p->m, m->mcache, p->mcache, p->status); |
| 1830 » » runtime·throw("releasep: invalid p state"); |
| 1831 » } |
| 1832 » m->p = nil; |
| 1833 » m->mcache = nil; |
| 1834 » p->m = nil; |
| 1835 » p->status = Pidle; |
| 1836 » return p; |
| 1837 } |
| 1838 |
| 1839 static void |
| 1840 inclocked(int32 v) |
| 1841 { |
| 1842 » runtime·lock(&runtime·sched); |
| 1843 » runtime·sched.mlocked += v; |
| 1844 » if(v > 0) |
| 1845 » » checkdead(); |
| 1846 » runtime·unlock(&runtime·sched); |
| 1847 } |
| 1848 |
| 1849 // Check for deadlock situation. |
| 1850 // The check is based on number of running M's, if 0 -> deadlock. |
| 1851 static void |
| 1852 checkdead(void) |
| 1853 { |
| 1854 » G *gp; |
| 1855 » int32 run, grunning, s; |
| 1856 |
| 1857 » // -1 for sysmon |
| 1858 » run = runtime·sched.mcount - runtime·sched.nmidle - runtime·sched.mlocke
d - 1; |
| 1859 » if(run > 0) |
| 1860 » » return; |
| 1861 » if(run < 0) { |
| 1862 » » runtime·printf("checkdead: nmidle=%d mlocked=%d mcount=%d\n", |
| 1863 » » » runtime·sched.nmidle, runtime·sched.mlocked, runtime·sch
ed.mcount); |
| 1864 » » runtime·throw("checkdead: inconsistent counts"); |
| 1865 » } |
| 1866 » grunning = 0; |
| 1867 » for(gp = runtime·allg; gp; gp = gp->alllink) { |
| 1868 » » if(gp == scvg) |
| 1869 » » » continue; |
| 1870 » » s = gp->status; |
| 1871 » » if(s == Gwaiting) |
| 1872 » » » grunning++; |
| 1873 » » else if(s == Grunnable || s == Grunning || s == Gsyscall) { |
| 1874 » » » runtime·printf("checkdead: find g %D in status %d\n", gp
->goid, s); |
| 1875 » » » runtime·throw("checkdead: runnable g"); |
| 1876 » » } |
| 1877 » } |
| 1878 » if(grunning == 0) // possible if main goroutine calls runtime·Goexit() |
| 1879 » » runtime·exit(0); |
| 1880 » m->throwing = -1; // do not dump full stacks |
| 1881 » runtime·throw("all goroutines are asleep - deadlock!"); |
| 1882 } |
| 1883 |
| 1884 static void |
| 1885 sysmon(void) |
| 1886 { |
| 1887 » uint32 idle, delay; |
| 1888 » uint32 ticks[MaxGomaxprocs]; |
| 1889 |
| 1890 » // This is a special dedicated thread that retakes P's from blocking sys
calls. |
| 1891 » // It works w/o mcache nor stackalloc, it may work concurrently with GC. |
| 1892 » runtime·asminit(); |
| 1893 » runtime·minit(); |
| 1894 |
| 1895 » idle = 0; // how many cycles in succession we had not wokeup somebody |
| 1896 » delay = 0; |
| 1897 » for(;;) { |
| 1898 » » if(idle == 0) // start with 20us sleep... |
| 1899 » » » delay = 20; |
| 1900 » » else if(idle > 50) // start doubling the sleep after 1ms... |
| 1901 » » » delay *= 2; |
| 1902 » » if(delay > 10*1000) // up to 10ms |
| 1903 » » » delay = 10*1000; |
| 1904 » » runtime·usleep(delay); |
| 1905 » » if(runtime·gcwaiting || runtime·sched.npidle == runtime·gomaxpro
cs) { // TODO: fast atomic |
| 1906 » » » runtime·lock(&runtime·sched); |
| 1907 » » » if(runtime·gcwaiting || runtime·sched.npidle == runtime·
gomaxprocs) { |
| 1908 » » » » runtime·sched.sysmonwait = true; |
| 1909 » » » » runtime·unlock(&runtime·sched); |
| 1910 » » » » runtime·notesleep(&runtime·sched.sysmonnote); |
| 1911 » » » » runtime·noteclear(&runtime·sched.sysmonnote); |
| 1912 » » » » idle = 0; |
| 1913 » » » » delay = 20; |
| 1914 » » » } else |
| 1915 » » » » runtime·unlock(&runtime·sched); |
| 1916 » » } |
| 1917 » » if(retake(ticks)) |
| 1918 » » » idle = 0; |
| 1919 » » else |
| 1920 » » » idle++; |
| 1921 » } |
| 1922 } |
| 1923 |
| 1924 static uint32 |
| 1925 retake(uint32 *ticks) |
| 1926 { |
| 1927 » uint32 i, s, n; |
| 1928 » int64 t; |
| 1929 » P *p; |
| 1930 |
| 1931 » n = 0; |
| 1932 » for(i = 0; i < runtime·gomaxprocs; i++) { |
| 1933 » » p = runtime·allp[i]; |
| 1934 » » if(p==nil) |
| 1935 » » » continue; |
| 1936 » » t = p->tick; |
| 1937 » » if(ticks[i] != t) { |
| 1938 » » » ticks[i] = t; |
| 1939 » » » continue; |
| 1940 » » } |
| 1941 » » s = p->status; |
| 1942 » » if(s != Psyscall) |
| 1943 » » » continue; |
| 1944 » » if(p->runqhead == p->runqtail && runtime·sched.nmspinning + runt
ime·sched.npidle > 0) // TODO: fast atomic |
| 1945 » » » continue; |
| 1946 » » // Need to increment number of locked M's before the CAS. |
| 1947 » » // Otherwise the M from which we retake can exit the syscall, |
| 1948 » » // increment nmidle and report deadlock. |
| 1949 » » inclocked(-1); |
| 1950 » » if(runtime·cas(&p->status, s, Pidle)) { |
| 1951 » » » n++; |
| 1952 » » » handoffp(p); |
| 1953 » » } |
| 1954 » » inclocked(1); |
| 1955 » } |
| 1956 » return n; |
| 1957 } |
| 1958 |
| 1959 // Put mp on midle list. |
1884 // Sched must be locked. | 1960 // Sched must be locked. |
1885 static void | 1961 static void |
1886 mput(M *mp) | 1962 mput(M *mp) |
1887 { | 1963 { |
1888 » mp->schedlink = runtime·sched.mhead; | 1964 » mp->schedlink = runtime·sched.midle; |
1889 » runtime·sched.mhead = mp; | 1965 » runtime·sched.midle = mp; |
1890 » runtime·sched.mwait++; | 1966 » runtime·sched.nmidle++; |
1891 checkdead(); | 1967 checkdead(); |
1892 } | 1968 } |
1893 | 1969 |
1894 // Try to get an m from mwait list. | 1970 // Try to get an m from midle list. |
1895 // Sched must be locked. | 1971 // Sched must be locked. |
1896 static M* | 1972 static M* |
1897 mtryget(void) | 1973 mget(void) |
1898 { | 1974 { |
1899 M *mp; | 1975 M *mp; |
1900 | 1976 |
1901 » if((mp = runtime·sched.mhead) != nil){ | 1977 » if((mp = runtime·sched.midle) != nil){ |
1902 » » runtime·sched.mhead = mp->schedlink; | 1978 » » runtime·sched.midle = mp->schedlink; |
1903 » » runtime·sched.mwait--; | 1979 » » runtime·sched.nmidle--; |
1904 } | 1980 } |
1905 return mp; | 1981 return mp; |
| 1982 } |
| 1983 |
| 1984 // Put gp on the global runnable queue. |
| 1985 // Sched must be locked. |
| 1986 static void |
| 1987 globrunqput(G *gp) |
| 1988 { |
| 1989 gp->schedlink = nil; |
| 1990 if(runtime·sched.runqtail) |
| 1991 runtime·sched.runqtail->schedlink = gp; |
| 1992 else |
| 1993 runtime·sched.runqhead = gp; |
| 1994 runtime·sched.runqtail = gp; |
| 1995 runtime·sched.runqsize++; |
| 1996 } |
| 1997 |
| 1998 // Try get a batch of G's from the global runnable queue. |
| 1999 // Sched must be locked. |
| 2000 static G* |
| 2001 globrunqget(P *p) |
| 2002 { |
| 2003 G *gp, *gp1; |
| 2004 int32 n; |
| 2005 |
| 2006 if(runtime·sched.runqsize == 0) |
| 2007 return nil; |
| 2008 n = runtime·sched.runqsize/runtime·gomaxprocs+1; |
| 2009 if(n > runtime·sched.runqsize) |
| 2010 n = runtime·sched.runqsize; |
| 2011 runtime·sched.runqsize -= n; |
| 2012 if(runtime·sched.runqsize == 0) |
| 2013 runtime·sched.runqtail = nil; |
| 2014 gp = runtime·sched.runqhead; |
| 2015 runtime·sched.runqhead = gp->schedlink; |
| 2016 n--; |
| 2017 while(n--) { |
| 2018 gp1 = runtime·sched.runqhead; |
| 2019 runtime·sched.runqhead = gp1->schedlink; |
| 2020 runqput(p, gp1); |
| 2021 } |
| 2022 return gp; |
1906 } | 2023 } |
1907 | 2024 |
1908 // Put p to on pidle list. | 2025 // Put p to on pidle list. |
1909 // Sched must be locked. | 2026 // Sched must be locked. |
1910 static void | 2027 static void |
1911 pidleput(P *p) | 2028 pidleput(P *p) |
1912 { | 2029 { |
1913 p->link = runtime·sched.pidle; | 2030 p->link = runtime·sched.pidle; |
1914 runtime·sched.pidle = p; | 2031 runtime·sched.pidle = p; |
1915 » runtime·sched.npidle++; | 2032 » runtime·sched.npidle++; // TODO: fast atomic |
1916 } | 2033 } |
1917 | 2034 |
1918 // Try get a p from pidle list. | 2035 // Try get a p from pidle list. |
1919 // Sched must be locked. | 2036 // Sched must be locked. |
1920 static P* | 2037 static P* |
1921 pidleget(void) | 2038 pidleget(void) |
1922 { | 2039 { |
1923 P *p; | 2040 P *p; |
1924 » | 2041 |
1925 p = runtime·sched.pidle; | 2042 p = runtime·sched.pidle; |
1926 if(p) { | 2043 if(p) { |
1927 runtime·sched.pidle = p->link; | 2044 runtime·sched.pidle = p->link; |
1928 » » runtime·sched.npidle--; | 2045 » » runtime·sched.npidle--; // TODO: fast atomic |
1929 } | 2046 } |
1930 return p; | 2047 return p; |
1931 } | 2048 } |
| 2049 |
| 2050 // Put g on local runnable queue. |
| 2051 // TODO(dvyukov): consider using lock-free queue. |
| 2052 static void |
| 2053 runqput(P *p, G *gp) |
| 2054 { |
| 2055 int32 h, t, s; |
| 2056 |
| 2057 runtime·lock(p); |
| 2058 retry: |
| 2059 h = p->runqhead; |
| 2060 t = p->runqtail; |
| 2061 s = p->runqsize; |
| 2062 if(t == h-1 || (h == 0 && t == s-1)) { |
| 2063 runqgrow(p); |
| 2064 goto retry; |
| 2065 } |
| 2066 p->runq[t++] = gp; |
| 2067 if(t == s) |
| 2068 t = 0; |
| 2069 p->runqtail = t; |
| 2070 runtime·unlock(p); |
| 2071 } |
| 2072 |
| 2073 // Get g from local runnable queue. |
| 2074 static G* |
| 2075 runqget(P *p) |
| 2076 { |
| 2077 G *gp; |
| 2078 int32 t, h, s; |
| 2079 |
| 2080 if(p->runqhead == p->runqtail) |
| 2081 return nil; |
| 2082 runtime·lock(p); |
| 2083 h = p->runqhead; |
| 2084 t = p->runqtail; |
| 2085 s = p->runqsize; |
| 2086 if(t == h) { |
| 2087 runtime·unlock(p); |
| 2088 return nil; |
| 2089 } |
| 2090 gp = p->runq[h++]; |
| 2091 if(h == s) |
| 2092 h = 0; |
| 2093 p->runqhead = h; |
| 2094 runtime·unlock(p); |
| 2095 return gp; |
| 2096 } |
| 2097 |
| 2098 // Grow local runnable queue. |
| 2099 // TODO(dvyukov): consider using fixed-size array |
| 2100 // and transfer excess to the global list (local queue can grow way too big). |
| 2101 static void |
| 2102 runqgrow(P *p) |
| 2103 { |
| 2104 G **q; |
| 2105 int32 s, t, h, t2; |
| 2106 |
| 2107 h = p->runqhead; |
| 2108 t = p->runqtail; |
| 2109 s = p->runqsize; |
| 2110 t2 = 0; |
| 2111 q = runtime·malloc(2*s*sizeof(*q)); |
| 2112 while(t != h) { |
| 2113 q[t2++] = p->runq[h++]; |
| 2114 if(h == s) |
| 2115 h = 0; |
| 2116 } |
| 2117 runtime·free(p->runq); |
| 2118 p->runq = q; |
| 2119 p->runqhead = 0; |
| 2120 p->runqtail = t2; |
| 2121 p->runqsize = 2*s; |
| 2122 } |
| 2123 |
| 2124 // Steal half of elements from local runnable queue of p2 |
| 2125 // and put onto local runnable queue of p. |
| 2126 // Returns one of the stolen elements (or nil if failed). |
| 2127 static G* |
| 2128 runqsteal(P *p, P *p2) |
| 2129 { |
| 2130 G *gp, *gp1; |
| 2131 int32 t, h, s, t2, h2, s2, c, i; |
| 2132 |
| 2133 if(p2->runqhead == p2->runqtail) |
| 2134 return nil; |
| 2135 // sort locks to prevent deadlocks |
| 2136 if(p < p2) |
| 2137 runtime·lock(p); |
| 2138 runtime·lock(p2); |
| 2139 if(p2->runqhead == p2->runqtail) { |
| 2140 runtime·unlock(p2); |
| 2141 if(p < p2) |
| 2142 runtime·unlock(p); |
| 2143 return nil; |
| 2144 } |
| 2145 if(p >= p2) |
| 2146 runtime·lock(p); |
| 2147 // now we've locked both queues and know the victim is not empty |
| 2148 h = p->runqhead; |
| 2149 t = p->runqtail; |
| 2150 s = p->runqsize; |
| 2151 h2 = p2->runqhead; |
| 2152 t2 = p2->runqtail; |
| 2153 s2 = p2->runqsize; |
| 2154 gp = p2->runq[h2++]; // return value |
| 2155 if(h2 == s2) |
| 2156 h2 = 0; |
| 2157 // steal roughly half |
| 2158 if(t2 > h2) |
| 2159 c = (t2 - h2) / 2; |
| 2160 else |
| 2161 c = (s2 - h2 + t2) / 2; |
| 2162 // copy |
| 2163 for(i = 0; i != c; i++) { |
| 2164 // the target queue is full? |
| 2165 if(t == h-1 || (h == 0 && t == s-1)) |
| 2166 break; |
| 2167 // the victim queue is empty? |
| 2168 if(t2 == h2) |
| 2169 break; |
| 2170 gp1 = p2->runq[h2++]; |
| 2171 if(h2 == s2) |
| 2172 h2 = 0; |
| 2173 p->runq[t++] = gp1; |
| 2174 if(t == s) |
| 2175 t = 0; |
| 2176 } |
| 2177 p->runqtail = t; |
| 2178 p2->runqhead = h2; |
| 2179 runtime·unlock(p2); |
| 2180 runtime·unlock(p); |
| 2181 return gp; |
| 2182 } |
| 2183 |
| 2184 void |
| 2185 runtime·testSchedLocalQueue(void) |
| 2186 { |
| 2187 P p; |
| 2188 G gs[1000]; |
| 2189 int32 i, j; |
| 2190 |
| 2191 runtime·memclr((byte*)&p, sizeof(p)); |
| 2192 p.runqsize = 1; |
| 2193 p.runqhead = 0; |
| 2194 p.runqtail = 0; |
| 2195 p.runq = runtime·malloc(p.runqsize*sizeof(*p.runq)); |
| 2196 |
| 2197 for(i = 0; i < nelem(gs); i++) { |
| 2198 if(runqget(&p) != nil) |
| 2199 runtime·throw("runq is not empty initially"); |
| 2200 for(j = 0; j < i; j++) |
| 2201 runqput(&p, &gs[i]); |
| 2202 for(j = 0; j < i; j++) { |
| 2203 if(runqget(&p) != &gs[i]) { |
| 2204 runtime·printf("bad element at iter %d/%d\n", i,
j); |
| 2205 runtime·throw("bad element"); |
| 2206 } |
| 2207 } |
| 2208 if(runqget(&p) != nil) |
| 2209 runtime·throw("runq is not empty afterwards"); |
| 2210 } |
| 2211 } |
| 2212 |
| 2213 void |
| 2214 runtime·testSchedLocalQueueSteal(void) |
| 2215 { |
| 2216 P p1, p2; |
| 2217 G gs[1000], *gp; |
| 2218 int32 i, j, s; |
| 2219 |
| 2220 runtime·memclr((byte*)&p1, sizeof(p1)); |
| 2221 p1.runqsize = 1; |
| 2222 p1.runqhead = 0; |
| 2223 p1.runqtail = 0; |
| 2224 p1.runq = runtime·malloc(p1.runqsize*sizeof(*p1.runq)); |
| 2225 |
| 2226 runtime·memclr((byte*)&p2, sizeof(p2)); |
| 2227 p2.runqsize = nelem(gs); |
| 2228 p2.runqhead = 0; |
| 2229 p2.runqtail = 0; |
| 2230 p2.runq = runtime·malloc(p2.runqsize*sizeof(*p2.runq)); |
| 2231 |
| 2232 for(i = 0; i < nelem(gs); i++) { |
| 2233 for(j = 0; j < i; j++) { |
| 2234 gs[j].sig = 0; |
| 2235 runqput(&p1, &gs[j]); |
| 2236 } |
| 2237 gp = runqsteal(&p2, &p1); |
| 2238 s = 0; |
| 2239 if(gp) { |
| 2240 s++; |
| 2241 gp->sig++; |
| 2242 } |
| 2243 while(gp = runqget(&p2)) { |
| 2244 s++; |
| 2245 gp->sig++; |
| 2246 } |
| 2247 while(gp = runqget(&p1)) |
| 2248 gp->sig++; |
| 2249 for(j = 0; j < i; j++) { |
| 2250 if(gs[j].sig != 1) { |
| 2251 runtime·printf("bad element %d(%d) at iter %d\n"
, j, gs[j].sig, i); |
| 2252 runtime·throw("bad element"); |
| 2253 } |
| 2254 } |
| 2255 if(s != i/2 && s != i/2+1) { |
| 2256 runtime·printf("bad steal %d, want %d or %d, iter %d\n", |
| 2257 s, i/2, i/2+1, i); |
| 2258 runtime·throw("bad steal"); |
| 2259 } |
| 2260 } |
| 2261 } |
| 2262 |
LEFT | RIGHT |