LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "defs_GOOS_GOARCH.h" | 7 #include "defs_GOOS_GOARCH.h" |
8 #include "malloc.h" | 8 #include "malloc.h" |
9 #include "os_GOOS.h" | 9 #include "os_GOOS.h" |
10 #include "stack.h" | 10 #include "stack.h" |
| 11 #include "race.h" |
| 12 #include "type.h" |
| 13 |
| 14 // TODO(dvyukov): if a thread w/o mcache catches a signal (in particular SIGABOR
T), |
| 15 // then it can't print dump. |
11 | 16 |
12 enum { maxgomaxprocs = 1<<10 }; | 17 enum { maxgomaxprocs = 1<<10 }; |
13 | 18 #define LOG if(0) runtime·printf |
14 enum { debug = 0 }; | 19 #define LOG1 runtime·printf |
15 #define LOG if(debug) runtime·printf | 20 #define CHECK(cond, fmt) /*if(cond) {} else { runtime·printf fmt; runtime·throw(
"CHECK"); }*/ |
16 | |
17 //FIXME: fix the comment. | |
18 | 21 |
19 // Go scheduler | 22 // Go scheduler |
20 // | 23 // |
21 // The go scheduler's job is to match ready-to-run goroutines (`g's) | 24 // The go scheduler's job is to match ready-to-run goroutines (`g's) |
22 // with waiting-for-work schedulers (`m's). If there are ready g's | 25 // with waiting-for-work schedulers (`m's). If there are ready g's |
23 // and no waiting m's, ready() will start a new m running in a new | 26 // and no waiting m's, ready() will start a new m running in a new |
24 // OS thread, so that all ready g's can run simultaneously, up to a limit. | 27 // OS thread, so that all ready g's can run simultaneously, up to a limit. |
25 // For now, m's never go away. | 28 // For now, m's never go away. |
26 // | 29 // |
27 // By default, Go keeps only one kernel thread (m) running user code | 30 // By default, Go keeps only one kernel thread (m) running user code |
28 // at a single time; other threads may be blocked in the operating system. | 31 // at a single time; other threads may be blocked in the operating system. |
29 // Setting the environment variable $GOMAXPROCS or calling | 32 // Setting the environment variable $GOMAXPROCS or calling |
30 // runtime.GOMAXPROCS() will change the number of user threads | 33 // runtime.GOMAXPROCS() will change the number of user threads |
31 // allowed to execute simultaneously. $GOMAXPROCS is thus an | 34 // allowed to execute simultaneously. $GOMAXPROCS is thus an |
32 // approximation of the maximum number of cores to use. | 35 // approximation of the maximum number of cores to use. |
33 // | 36 // |
34 // Even a program that can run without deadlock in a single process | 37 // Even a program that can run without deadlock in a single process |
35 // might use more m's if given the chance. For example, the prime | 38 // might use more m's if given the chance. For example, the prime |
36 // sieve will use as many m's as there are primes (up to runtime·sched.mmax), | 39 // sieve will use as many m's as there are primes (up to $GOMAXPROCS), |
37 // allowing different stages of the pipeline to execute in parallel. | 40 // allowing different stages of the pipeline to execute in parallel. |
38 // We could revisit this choice, only kicking off new m's for blocking | |
39 // system calls, but that would limit the amount of parallel computation | |
40 // that go would try to do. | |
41 // | |
42 // In general, one could imagine all sorts of refinements to the | |
43 // scheduler, but the goal now is just to get something working on | |
44 // Linux and OS X. | |
45 | 41 |
46 typedef struct Sched Sched; | 42 typedef struct Sched Sched; |
47 struct Sched { | 43 struct Sched { |
48 Lock; | 44 Lock; |
49 | 45 |
50 M* mhead; // m's waiting for work | 46 M* mhead; // m's waiting for work |
51 int32 mwait; // number of m's waiting for work | 47 int32 mwait; // number of m's waiting for work |
52 int32 mcount; // number of m's that have been created | 48 int32 mcount; // number of m's that have been created |
53 | 49 |
| 50 P* pidle; // idle P's |
| 51 int32 npidle; |
| 52 |
| 53 G* runqhead; |
| 54 G* runqtail; |
| 55 int32 runqsize; |
| 56 |
| 57 Lock gflock; |
| 58 G* gfree; |
| 59 int32 goidseq; |
| 60 |
| 61 int32 stopwait; |
| 62 Note stopnote; |
| 63 int32 sysmonwait; |
| 64 Note sysmonnote; |
| 65 |
54 int32 profilehz; // cpu profiling rate | 66 int32 profilehz; // cpu profiling rate |
55 | 67 |
56 bool init; // running initialization | 68 bool init; // running initialization |
57 bool lockmain; // init called runtime.LockOSThread | 69 bool lockmain; // init called runtime.LockOSThread |
58 bool bootstrap; | |
59 }; | 70 }; |
60 | 71 |
61 Sched runtime·sched; | 72 Sched runtime·sched; |
62 int32 runtime·gomaxprocs; | 73 int32 runtime·gomaxprocs; |
63 bool runtime·singleproc; | 74 bool runtime·singleproc; |
64 bool runtime·iscgo; | 75 bool runtime·iscgo; |
65 int32 runtime·gcwaiting; | 76 int32 runtime·gcwaiting; |
66 M» » runtime·m0; | 77 M» runtime·m0; |
67 G» » runtime·g0;» // idle goroutine for m0 | 78 G» runtime·g0;» // idle goroutine for m0 |
| 79 static int32» newprocs; |
68 | 80 |
69 // Keep trace of scavenger's goroutine for deadlock detection. | 81 // Keep trace of scavenger's goroutine for deadlock detection. |
70 static G *scvg; | 82 static G *scvg; |
71 | 83 |
72 // Scheduling helpers. Sched must be locked. | 84 // Scheduling helpers. Sched must be locked. |
73 static void gput(P*, G*);» // put/get on ghead/gtail | 85 static void runqput(P*, G*);» // put/get on ghead/gtail |
74 static G* gget(P*); | 86 static G* runqget(P*); |
| 87 static void runqgrow(P*); |
| 88 static G* runqsteal(P*, P*); |
| 89 static void globrunqput(G*); |
| 90 static G* globrunqget(void); |
75 static void mput(M*); // put/get on mhead | 91 static void mput(M*); // put/get on mhead |
76 static M* mget(void); | 92 static M* mget(void); |
77 static void gfput(P*, G*); // put/get on gfree | 93 static void gfput(P*, G*); // put/get on gfree |
78 static G* gfget(P*); | 94 static G* gfget(P*); |
79 static void mcommoninit(M*); | 95 static void mcommoninit(M*); |
80 static void schedule(G*); | 96 static void schedule(void); |
81 static void procresize(void); | 97 static void procresize(int32); |
82 static void entergo(M*, P*); | 98 static void entergo(M*, P*); |
83 static void leavego(M*, uint32); | 99 static P* releasep(void); |
84 static void newm(void); | 100 static M* newm(void(*)(void), P*, bool); |
85 static void goidle(void); | 101 static void goidle(void); |
| 102 static void mstop(void); |
| 103 static void initgstack(G*, byte*, int32); |
| 104 static void sysmon(void); |
| 105 static void inject(G*, int32*, int32*); |
| 106 static P* pidleget(void); |
| 107 static void pidleput(P*); |
86 | 108 |
87 // The bootstrap sequence is: | 109 // The bootstrap sequence is: |
88 // | 110 // |
89 // call osinit | 111 // call osinit |
90 // call schedinit | 112 // call schedinit |
91 // make & queue new G | 113 // make & queue new G |
92 // call runtime·mstart | 114 // call runtime·mstart |
93 // | 115 // |
94 // The new G calls runtime·main. | 116 // The new G calls runtime·main. |
95 void | 117 void |
96 runtime·schedinit(void) | 118 runtime·schedinit(void) |
97 { | 119 { |
98 » int32 n; | 120 » int32 n, procs; |
99 byte *p; | 121 byte *p; |
100 | 122 |
101 LOG("%d: runtime·schedinit\n", m->id); | 123 LOG("%d: runtime·schedinit\n", m->id); |
102 m->nomemprof++; | 124 m->nomemprof++; |
103 runtime·mallocinit(); | 125 runtime·mallocinit(); |
| 126 m->stackalloc = runtime·malloc(sizeof(*m->stackalloc)); //!!! it may be
GCed |
104 mcommoninit(m); | 127 mcommoninit(m); |
| 128 if(runtime·gsignalstk) |
| 129 m->gsignal = runtime·malg(runtime·gsignalstk); |
105 | 130 |
106 runtime·goargs(); | 131 runtime·goargs(); |
107 runtime·goenvs(); | 132 runtime·goenvs(); |
108 | 133 |
109 // For debugging: | 134 // For debugging: |
110 // Allocate internal symbol table representation now, | 135 // Allocate internal symbol table representation now, |
111 // so that we don't need to call malloc when we crash. | 136 // so that we don't need to call malloc when we crash. |
112 // runtime·findfunc(0); | 137 // runtime·findfunc(0); |
113 | 138 |
114 » runtime·sched.bootstrap = true; | 139 » procs = 1; |
115 » runtime·gomaxprocs = 1; | |
116 p = runtime·getenv("GOMAXPROCS"); | 140 p = runtime·getenv("GOMAXPROCS"); |
117 if(p != nil && (n = runtime·atoi(p)) > 0) { | 141 if(p != nil && (n = runtime·atoi(p)) > 0) { |
118 if(n > maxgomaxprocs) | 142 if(n > maxgomaxprocs) |
119 n = maxgomaxprocs; | 143 n = maxgomaxprocs; |
120 » » runtime·gomaxprocs = n; | 144 » » procs = n; |
121 } | 145 } |
122 runtime·allp = (P**)runtime·malloc((maxgomaxprocs+1)*sizeof(runtime·allp
[0])); | 146 runtime·allp = (P**)runtime·malloc((maxgomaxprocs+1)*sizeof(runtime·allp
[0])); |
123 » procresize(); | 147 » procresize(procs); |
124 | 148 |
125 mstats.enablegc = 1; | 149 mstats.enablegc = 1; |
126 m->nomemprof--; | 150 m->nomemprof--; |
| 151 |
| 152 if(raceenabled) |
| 153 runtime·raceinit(); |
127 } | 154 } |
128 | 155 |
129 extern void main·init(void); | 156 extern void main·init(void); |
130 extern void main·main(void); | 157 extern void main·main(void); |
131 | 158 |
132 // The main goroutine. | 159 // The main goroutine. |
133 void | 160 void |
134 runtime·main(void) | 161 runtime·main(void) |
135 { | 162 { |
136 LOG("%d: runtime·main\n", m->id); | 163 LOG("%d: runtime·main\n", m->id); |
137 » runtime·sched.bootstrap = false; | 164 |
| 165 » //TODO(dvyukov): block signals because that thread can't handle them |
| 166 » newm(sysmon, nil, false); |
| 167 |
138 // Lock the main goroutine onto this, the main OS thread, | 168 // Lock the main goroutine onto this, the main OS thread, |
139 // during initialization. Most programs won't care, but a few | 169 // during initialization. Most programs won't care, but a few |
140 // do require certain calls to be made by the main thread. | 170 // do require certain calls to be made by the main thread. |
141 // Those can arrange for main.main to run in the main thread | 171 // Those can arrange for main.main to run in the main thread |
142 // by calling runtime.LockOSThread during initialization | 172 // by calling runtime.LockOSThread during initialization |
143 // to preserve the lock. | 173 // to preserve the lock. |
144 runtime·LockOSThread(); | 174 runtime·LockOSThread(); |
145 runtime·sched.init = true; | 175 runtime·sched.init = true; |
146 if(m != &runtime·m0) | 176 if(m != &runtime·m0) |
147 runtime·throw("runtime·main not on m0"); | 177 runtime·throw("runtime·main not on m0"); |
148 scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runti
me·main); | 178 scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runti
me·main); |
149 main·init(); | 179 main·init(); |
150 runtime·sched.init = false; | 180 runtime·sched.init = false; |
151 if(!runtime·sched.lockmain) | 181 if(!runtime·sched.lockmain) |
152 runtime·UnlockOSThread(); | 182 runtime·UnlockOSThread(); |
153 | 183 |
154 main·main(); | 184 main·main(); |
| 185 if(raceenabled) |
| 186 runtime·racefini(); |
155 runtime·exit(0); | 187 runtime·exit(0); |
156 for(;;) | 188 for(;;) |
157 *(int32*)runtime·main = 0; | 189 *(int32*)runtime·main = 0; |
158 } | 190 } |
159 | 191 |
160 void | 192 void |
161 runtime·goroutineheader(G *gp) | 193 runtime·goroutineheader(G *gp) |
162 { | 194 { |
163 int8 *status; | 195 int8 *status; |
164 | 196 |
(...skipping 13 matching lines...) Expand all Loading... |
178 case Gwaiting: | 210 case Gwaiting: |
179 if(gp->waitreason) | 211 if(gp->waitreason) |
180 status = gp->waitreason; | 212 status = gp->waitreason; |
181 else | 213 else |
182 status = "waiting"; | 214 status = "waiting"; |
183 break; | 215 break; |
184 default: | 216 default: |
185 status = "???"; | 217 status = "???"; |
186 break; | 218 break; |
187 } | 219 } |
188 » runtime·printf("goroutine %p [%s]:\n", gp, status); | 220 » runtime·printf("goroutine %D [%s]:\n", gp->goid, status); |
189 } | 221 } |
190 | 222 |
191 void | 223 void |
192 runtime·tracebackothers(G *me) | 224 runtime·tracebackothers(G *me) |
193 { | 225 { |
194 G *gp; | 226 G *gp; |
195 » P *p, **pp; | 227 |
196 | 228 » for(gp = runtime·allg; gp != nil; gp = gp->alllink) { |
197 » for(pp=runtime·allp; p=*pp; pp++) { | 229 » » if(gp == me || gp->status == Gdead) |
198 » » for(gp = p->allg; gp != nil; gp = gp->alllink) { | 230 » » » continue; |
199 » » » if(gp == me || gp->status == Gdead) | 231 » » runtime·printf("\n"); |
200 » » » » continue; | 232 » » runtime·goroutineheader(gp); |
201 » » » runtime·printf("\n"); | 233 » » runtime·traceback(gp->sched.pc, (byte*)gp->sched.sp, 0, gp); |
202 » » » runtime·goroutineheader(gp); | |
203 » » » runtime·traceback(gp->sched.pc, (byte*)gp->sched.sp, 0,
gp); | |
204 » » } | |
205 } | 234 } |
206 } | 235 } |
207 | 236 |
208 static void | 237 static void |
209 mcommoninit(M *mp) | 238 mcommoninit(M *mp) |
210 { | 239 { |
| 240 runtime·lock(&runtime·sched); |
211 mp->id = runtime·sched.mcount++; | 241 mp->id = runtime·sched.mcount++; |
212 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); | 242 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); |
213 mp->stackalloc = runtime·malloc(sizeof(*mp->stackalloc)); | |
214 runtime·FixAlloc_Init(mp->stackalloc, FixedStack, runtime·SysAlloc, nil,
nil); | 243 runtime·FixAlloc_Init(mp->stackalloc, FixedStack, runtime·SysAlloc, nil,
nil); |
215 | 244 |
216 » runtime·callers(1, mp->createstack, nelem(mp->createstack)); | 245 » // does it allocate? |
| 246 » //runtime·callers(1, mp->createstack, nelem(mp->createstack)); |
217 | 247 |
218 // Add to runtime·allm so garbage collector doesn't free m | 248 // Add to runtime·allm so garbage collector doesn't free m |
219 // when it is just in a register or thread-local storage. | 249 // when it is just in a register or thread-local storage. |
220 mp->alllink = runtime·allm; | 250 mp->alllink = runtime·allm; |
221 // runtime·NumCgoCall() iterates over allm w/o locks, | 251 // runtime·NumCgoCall() iterates over allm w/o locks, |
222 // so we need to publish it safely. | 252 // so we need to publish it safely. |
223 runtime·atomicstorep(&runtime·allm, mp); | 253 runtime·atomicstorep(&runtime·allm, mp); |
224 } | 254 » LOG("%d: mcommoninit %d m=%p stackalloc=%p\n", m->id, mp->id, mp, mp->st
ackalloc); |
225 | 255 » runtime·unlock(&runtime·sched); |
226 // Put g on runnable queue. P is locked. | 256 } |
227 static void | 257 |
228 gput(P *p, G *gp) | 258 // Mark g ready to run. |
229 { | 259 void |
230 » gp->schedlink = nil; | 260 runtime·ready(G *gp) |
231 » if(p->ghead) | 261 { |
232 » » p->gtail->schedlink = gp; | 262 » P *p; |
233 » else | |
234 » » p->ghead = gp; | |
235 » p->gtail = gp; | |
236 » p->gwait++; | |
237 } | |
238 | |
239 // Get g from runnable queue. P is locked. | |
240 static G* | |
241 gget(P *p) | |
242 { | |
243 » G *gp; | |
244 | |
245 » gp = p->ghead; | |
246 » if(gp) { | |
247 » » p->ghead = gp->schedlink; | |
248 » » if(p->ghead == nil) | |
249 » » » p->gtail = nil; | |
250 » » p->gwait--; | |
251 » } | |
252 » return gp; | |
253 } | |
254 | |
255 // Put on `m' list. Sched must be locked. | |
256 static void | |
257 mput(M *mp) | |
258 { | |
259 » mp->schedlink = runtime·sched.mhead; | |
260 » runtime·sched.mhead = mp; | |
261 » runtime·sched.mwait++; | |
262 } | |
263 | |
264 // Sched must be locked. | |
265 static M* | |
266 mget(void) | |
267 { | |
268 M *mp; | 263 M *mp; |
269 | 264 |
270 if((mp = runtime·sched.mhead) != nil){ | |
271 runtime·sched.mhead = mp->schedlink; | |
272 runtime·sched.mwait--; | |
273 } | |
274 return mp; | |
275 } | |
276 | |
277 // Mark g ready to run. | |
278 void | |
279 runtime·ready(G *gp) | |
280 { | |
281 if(gp->m) | 265 if(gp->m) |
282 runtime·throw("bad g->m in ready"); | 266 runtime·throw("bad g->m in ready"); |
283 | 267 |
284 // Mark runnable. | 268 // Mark runnable. |
285 if(gp->status == Grunnable || gp->status == Grunning) { | 269 if(gp->status == Grunnable || gp->status == Grunning) { |
286 » » runtime·printf("goroutine %p has status %d\n", gp, gp->status); | 270 » » runtime·printf("goroutine %D has status %d\n", gp->goid, gp->sta
tus); |
287 runtime·throw("bad g->status in ready"); | 271 runtime·throw("bad g->status in ready"); |
288 } | 272 } |
289 gp->status = Grunnable; | 273 gp->status = Grunnable; |
290 » runtime·lock(m->p); | 274 » runqput(m->p, gp); |
291 » gput(m->p, gp); | 275 » if(runtime·sched.pidle) { |
292 » runtime·unlock(m->p); | 276 » » runtime·lock(&runtime·sched); |
293 } | 277 » » p = pidleget(); |
294 | 278 » » if(p) { |
| 279 » » » mp = mget(); |
| 280 » » » runtime·unlock(&runtime·sched); |
| 281 » » » if(mp) { |
| 282 » » » » entergo(mp, p); |
| 283 » » » » runtime·notewakeup(&mp->park); |
| 284 » » » } else { |
| 285 » » » » newm(runtime·mstart, p, false); |
| 286 » » » } |
| 287 » » } else |
| 288 » » » runtime·unlock(&runtime·sched); |
| 289 » } |
| 290 } |
| 291 |
| 292 static void |
| 293 munpark(M *mp, P *p) |
| 294 { |
| 295 » if(mp) { |
| 296 » » entergo(mp, p); |
| 297 » » runtime·notewakeup(&mp->park); |
| 298 » } else |
| 299 » » newm(runtime·mstart, p, false); |
| 300 } |
| 301 » » » »······· |
295 int32 | 302 int32 |
296 runtime·gcprocs(void) | 303 runtime·gcprocs(void) |
297 { | 304 { |
298 int32 n; | 305 int32 n; |
299 | 306 |
300 /* | |
301 runtime·lock(&runtime·sched); | 307 runtime·lock(&runtime·sched); |
302 // Figure out how many CPUs to use during GC. | |
303 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. | |
304 n = runtime·gomaxprocs; | 308 n = runtime·gomaxprocs; |
305 if(n > runtime·ncpu) | 309 if(n > runtime·ncpu) |
306 n = runtime·ncpu; | 310 n = runtime·ncpu; |
307 if(n > MaxGcproc) | 311 if(n > MaxGcproc) |
308 n = MaxGcproc; | 312 n = MaxGcproc; |
309 if(n > runtime·sched.mwait+1) // one M is currently running | |
310 n = runtime·sched.mwait+1; | |
311 runtime·unlock(&runtime·sched); | 313 runtime·unlock(&runtime·sched); |
| 314 return n; |
| 315 } |
| 316 |
| 317 void |
| 318 runtime·helpgc(int32 nproc) |
| 319 { |
| 320 M *mp; |
| 321 int32 n, pos; |
| 322 |
| 323 LOG("%d: helpgc(%d)\n", m->id, nproc); |
| 324 runtime·lock(&runtime·sched); |
| 325 pos = 0; |
| 326 for(n = 1; n < nproc; n++) { // one M is currently running |
| 327 if(runtime·allp[pos]->mcache == m->mcache) |
| 328 pos++; |
| 329 mp = mget(); |
| 330 if(mp == nil) { |
| 331 runtime·unlock(&runtime·sched); |
| 332 newm(runtime·mstart, runtime·allp[pos], true); |
| 333 runtime·lock(&runtime·sched); |
| 334 pos++; |
| 335 continue; |
| 336 } |
| 337 mp->helpgc = 1; |
| 338 mp->mcache = runtime·allp[pos]->mcache; |
| 339 pos++; |
| 340 LOG("%d: helpgc wake %d\n", m->id, mp->id); |
| 341 runtime·notewakeup(&mp->park); |
| 342 } |
| 343 runtime·unlock(&runtime·sched); |
| 344 } |
| 345 |
| 346 void |
| 347 runtime·stoptheworld(void) |
| 348 { |
| 349 int32 i; |
| 350 uint32 s; |
| 351 P *p; |
| 352 bool wait; |
| 353 |
| 354 LOG("%d: stoptheworld\n", m->id); |
| 355 runtime·lock(&runtime·sched); |
| 356 runtime·gcwaiting = 1; |
| 357 runtime·sched.stopwait = runtime·gomaxprocs; |
| 358 m->p->status = Plocked; |
| 359 runtime·sched.stopwait--; |
| 360 for(i=0; i<runtime·gomaxprocs; i++) { |
| 361 s = runtime·allp[i]->status; |
| 362 if(s == Psyscall && runtime·cas(&runtime·allp[i]->status, s, Plo
cked)) { |
| 363 LOG(" acquired syscall %d\n", i); |
| 364 runtime·sched.stopwait--; |
| 365 } |
| 366 } |
| 367 while(runtime·sched.pidle) { |
| 368 p = pidleget(); |
| 369 p->status = Plocked; |
| 370 runtime·sched.stopwait--; |
| 371 } |
| 372 CHECK(runtime·sched.stopwait >= 0, ("")); |
| 373 wait = runtime·sched.stopwait > 0; |
| 374 runtime·unlock(&runtime·sched); |
| 375 if(wait) { |
| 376 runtime·notesleep(&runtime·sched.stopnote); |
| 377 runtime·noteclear(&runtime·sched.stopnote); |
| 378 } |
| 379 LOG("%d: stoptheworld stopped\n", m->id); |
| 380 CHECK(runtime·sched.stopwait == 0, ("stoptheworld: stopwait == %d\n", ru
ntime·sched.stopwait)); |
| 381 for(i=0; i<runtime·gomaxprocs; i++) { |
| 382 CHECK(runtime·allp[i]->status == Plocked, ("stoptheworld: not st
opped (%d)\n", runtime·allp[i]->status)); |
| 383 } |
| 384 } |
| 385 |
| 386 void |
| 387 runtime·starttheworld(void) |
| 388 { |
| 389 //G *gp; |
| 390 //P *p; |
| 391 //M *mp; |
| 392 //int32 n, w; |
| 393 |
| 394 LOG("%d: starttheworld\n", m->id); |
| 395 runtime·gcwaiting = 0; |
| 396 if(newprocs) { |
| 397 procresize(newprocs); |
| 398 newprocs = 0; |
| 399 } else { |
| 400 procresize(runtime·gomaxprocs); |
| 401 } |
| 402 runtime·lock(&runtime·sched); |
| 403 /* |
| 404 gp = runtime·netwait(0, runtime·gomaxprocs); |
| 405 n = w = 0; |
| 406 inject(gp, &w, &n); |
| 407 while(runtime·sched.pidle) { |
| 408 p = pidleget(); |
| 409 mp = mget(); |
| 410 if(mp) { |
| 411 entergo(mp, p); |
| 412 runtime·notewakeup(&mp->park); |
| 413 } else { |
| 414 runtime·unlock(&runtime·sched); |
| 415 newm(runtime·mstart, p, false); |
| 416 runtime·lock(&runtime·sched); |
| 417 } |
| 418 } |
312 */ | 419 */ |
313 » n = 1; | 420 » if(runtime·sched.sysmonwait) { |
314 » return n; | 421 » » runtime·sched.sysmonwait = 0; |
315 } | 422 » » runtime·notewakeup(&runtime·sched.sysmonnote); |
316 | |
317 void | |
318 runtime·helpgc(int32 nproc) | |
319 { | |
320 » USED(&nproc); | |
321 » /* | |
322 » M *mp; | |
323 » int32 n; | |
324 | |
325 » runtime·lock(&runtime·sched); | |
326 » for(n = 1; n < nproc; n++) { // one M is currently running | |
327 » » mp = mget(); | |
328 » » if(mp == nil) | |
329 » » » runtime·throw("runtime·gcprocs inconsistency"); | |
330 » » mp->helpgc = 1; | |
331 » » runtime·notewakeup(&mp->havenextg); | |
332 } | 423 } |
333 runtime·unlock(&runtime·sched); | 424 runtime·unlock(&runtime·sched); |
334 */ | |
335 } | |
336 | |
337 void | |
338 runtime·stoptheworld(void) | |
339 { | |
340 int32 acquired, i; | |
341 uint32 s; | |
342 | |
343 LOG("%d: stoptheworld\n", m->id); | |
344 runtime·gcwaiting = 1; //!!! atomic | |
345 acquired = 1; | |
346 m->p->status = Plocked; | |
347 while(acquired != runtime·np) { | |
348 for(i=0; i<runtime·np; i++) { | |
349 s = runtime·allp[i]->status; | |
350 if(s == Pidle || s == Psyscall) { | |
351 if(runtime·cas(&runtime·allp[i]->status, s, Ploc
ked)) | |
352 acquired++; | |
353 } | |
354 } | |
355 //!!! replace with blocking | |
356 if(acquired != runtime·np) | |
357 runtime·usleep(1); | |
358 } | |
359 } | |
360 | |
361 void | |
362 runtime·starttheworld(void) | |
363 { | |
364 LOG("%d: starttheworld\n", m->id); | |
365 //FIXME: start additional M's | |
366 /* | |
367 int32 max, cur; | |
368 | |
369 cur = runtime·gcprocs(); | |
370 // Figure out how many CPUs GC could possibly use. | |
371 max = runtime·gomaxprocs; | |
372 if(max > runtime·ncpu) | |
373 max = runtime·ncpu; | |
374 if(max > MaxGcproc) | |
375 max = MaxGcproc; | |
376 */ | |
377 | |
378 runtime·gcwaiting = 0; | |
379 procresize(); | |
380 runtime·gosched(); | |
381 } | 425 } |
382 | 426 |
383 // Called to start an M. | 427 // Called to start an M. |
384 void | 428 void |
385 runtime·mstart(void) | 429 runtime·mstart(void) |
386 { | 430 { |
387 // It is used by windows-386 only. Unfortunately, seh needs | 431 // It is used by windows-386 only. Unfortunately, seh needs |
388 // to be located on os stack, and mstart runs on os stack | 432 // to be located on os stack, and mstart runs on os stack |
389 // for both m0 and m. | 433 // for both m0 and m. |
390 SEH seh; | 434 SEH seh; |
391 | 435 » P *p; |
392 » LOG("%d: mstart\n", m->id); | 436 |
| 437 » LOG("%d: mstart m=%p\n", m->id, m); |
393 if(g != m->g0) | 438 if(g != m->g0) |
394 runtime·throw("bad runtime·mstart"); | 439 runtime·throw("bad runtime·mstart"); |
395 | 440 |
396 // Record top of stack for use by mcall. | 441 // Record top of stack for use by mcall. |
397 // Once we call schedule we're never coming back, | 442 // Once we call schedule we're never coming back, |
398 // so other calls can reuse this stack space. | 443 // so other calls can reuse this stack space. |
399 runtime·gosave(&m->g0->sched); | 444 runtime·gosave(&m->g0->sched); |
400 m->g0->sched.pc = (void*)-1; // make sure it is never used | 445 m->g0->sched.pc = (void*)-1; // make sure it is never used |
401 m->seh = &seh; | 446 m->seh = &seh; |
402 runtime·asminit(); | 447 runtime·asminit(); |
403 if(m == &runtime·m0) | |
404 runtime·minitparent(m); | |
405 runtime·minit(); | 448 runtime·minit(); |
406 | 449 |
407 // Install signal handlers; after minit so that minit can | 450 // Install signal handlers; after minit so that minit can |
408 // prepare the thread to be able to handle the signals. | 451 // prepare the thread to be able to handle the signals. |
409 if(m == &runtime·m0) | 452 if(m == &runtime·m0) |
410 runtime·initsig(); | 453 runtime·initsig(); |
411 | 454 |
412 » if(m == &runtime·m0) | 455 » if(m->helpgc) { |
413 » » leavego(m, Pidle); | 456 » » LOG("%d: mstart helpgc\n", m->id); |
414 » else { | 457 » » m->helpgc = 0; |
415 » » //!!! this is to ensure that runtime.main is executed on m0 | 458 » » m->mcache = m->p->mcache; |
416 » » // think of something better | 459 » » runtime·gchelper(); |
417 » » while(runtime·sched.bootstrap) | 460 » » m->mcache = nil; |
418 » » » runtime·usleep(10); | 461 » » m->p = nil; |
419 » } | 462 » » LOG("%d: gchelper end\n", m->id); |
420 » goidle(); | 463 » » mstop(); |
421 » schedule(nil); | 464 » } else if(m != &runtime·m0) { |
| 465 » » p = m->p; |
| 466 » » m->p = nil; |
| 467 » » entergo(m, p); |
| 468 » } |
| 469 » LOG("%d: calling schedule\n", m->id); |
| 470 » schedule(); |
422 | 471 |
423 // TODO(brainman): This point is never reached, because scheduler | 472 // TODO(brainman): This point is never reached, because scheduler |
424 // does not release os threads at the moment. But once this path | 473 // does not release os threads at the moment. But once this path |
425 // is enabled, we must remove our seh here. | 474 // is enabled, we must remove our seh here. |
426 } | 475 } |
427 | 476 |
428 // When running with cgo, we call libcgo_thread_start | 477 // When running with cgo, we call libcgo_thread_start |
429 // to start threads for us so that we can play nicely with | 478 // to start threads for us so that we can play nicely with |
430 // foreign code. | 479 // foreign code. |
431 void (*libcgo_thread_start)(void*); | 480 void (*libcgo_thread_start)(void*); |
432 | 481 |
433 typedef struct CgoThreadStart CgoThreadStart; | 482 typedef struct CgoThreadStart CgoThreadStart; |
434 struct CgoThreadStart | 483 struct CgoThreadStart |
435 { | 484 { |
436 M *m; | 485 M *m; |
437 G *g; | 486 G *g; |
438 void (*fn)(void); | 487 void (*fn)(void); |
439 }; | 488 }; |
440 | 489 |
| 490 static void |
| 491 initgstack(G *newg, byte *stk, int32 stacksize) |
| 492 { |
| 493 newg->stack0 = (uintptr)stk; |
| 494 newg->stackguard = (uintptr)stk + StackGuard; |
| 495 newg->stackbase = (uintptr)stk + stacksize - sizeof(Stktop); |
| 496 runtime·memclr((byte*)newg->stackbase, sizeof(Stktop)); |
| 497 } |
| 498 |
441 // Create a new m. It will start off with a call to runtime·mstart. | 499 // Create a new m. It will start off with a call to runtime·mstart. |
442 static void | 500 static M* |
443 newm(void) | 501 newm(void(*fn)(void), P *p, bool helpgc) |
444 { | 502 { |
445 M *mp; | 503 M *mp; |
| 504 int32 addmem,stksiz, stkoff; |
| 505 //!!!static Type *mtype; // The Go type M |
446 | 506 |
447 LOG("%d: newm\n", m->id); | 507 LOG("%d: newm\n", m->id); |
448 » mp = runtime·malloc(sizeof(M)); | 508 » addmem = sizeof(*mp->stackalloc); |
| 509 » if(runtime·gsignalstk) |
| 510 » » addmem += sizeof(G) + runtime·gsignalstk; |
| 511 » stkoff = sizeof(M) + addmem; |
| 512 » stksiz = StackSystem + (fn == runtime·mstart ? 8192 : 64*1024); |
| 513 » if(!runtime·iscgo && !Windows) |
| 514 » » addmem += stksiz; |
| 515 » //!!! all that is now non-GC, can it break something? |
| 516 » mp = runtime·SysAlloc(sizeof(M) + addmem); |
| 517 » mp->stackalloc = (FixAlloc*)(mp+1); |
| 518 » //!!!if(mtype == nil) { |
| 519 » //!!!» Eface e; |
| 520 » //!!!» runtime·gc_m_ptr(&e); |
| 521 » //!!!» mtype = ((PtrType*)e.type)->elem; |
| 522 » //!!!} |
| 523 » //!!! mp = runtime·cnew(mtype); |
449 mcommoninit(mp); | 524 mcommoninit(mp); |
450 » runtime·minitparent(mp); | 525 » mp->g0 = &mp->g0buf; |
| 526 » mp->p = p; |
| 527 » mp->helpgc = helpgc; |
| 528 » if(runtime·gsignalstk) { |
| 529 » » mp->gsignal = (G*)((byte*)mp+sizeof(*mp)+sizeof(*mp->stackalloc)
); |
| 530 » » initgstack(mp->gsignal, (byte*)(mp->gsignal+1), runtime·gsignals
tk); |
| 531 » } |
451 | 532 |
452 if(runtime·iscgo) { | 533 if(runtime·iscgo) { |
453 CgoThreadStart ts; | 534 CgoThreadStart ts; |
454 | 535 |
455 if(libcgo_thread_start == nil) | 536 if(libcgo_thread_start == nil) |
456 runtime·throw("libcgo_thread_start missing"); | 537 runtime·throw("libcgo_thread_start missing"); |
457 // pthread_create will make us a stack. | 538 // pthread_create will make us a stack. |
458 mp->g0 = runtime·malg(-1); | |
459 ts.m = mp; | 539 ts.m = mp; |
460 ts.g = mp->g0; | 540 ts.g = mp->g0; |
461 » » ts.fn = runtime·mstart; | 541 » » ts.fn = fn; |
462 runtime·asmcgocall(libcgo_thread_start, &ts); | 542 runtime·asmcgocall(libcgo_thread_start, &ts); |
463 } else { | 543 } else { |
464 » » if(Windows) | 544 » » // windows will layout sched stack on os stack |
465 » » » // windows will layout sched stack on os stack | 545 » » if(!Windows) |
466 » » » mp->g0 = runtime·malg(-1); | 546 » » » initgstack(mp->g0, (byte*)mp+stkoff, stksiz); |
467 » » else | 547 » » runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, fn); |
468 » » » mp->g0 = runtime·malg(8192); | 548 » } |
469 » » runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, runtime·
mstart); | 549 » return mp; |
470 » } | |
471 } | 550 } |
472 | 551 |
473 static void | 552 static void |
474 mstop(void) | 553 mstop(void) |
475 { | 554 { |
476 LOG("%d: mstop\n", m->id); | 555 LOG("%d: mstop\n", m->id); |
477 » runtime·noteclear(&m->park); | 556 » CHECK(m->locks == 0, ("")); |
| 557 » CHECK(m->p == nil, ("mstop: p != nil\n")); |
| 558 retry: |
478 runtime·lock(&runtime·sched); | 559 runtime·lock(&runtime·sched); |
479 mput(m); | 560 mput(m); |
480 runtime·unlock(&runtime·sched); | 561 runtime·unlock(&runtime·sched); |
481 runtime·notesleep(&m->park); | 562 runtime·notesleep(&m->park); |
482 » goidle(); | 563 » runtime·noteclear(&m->park); |
| 564 » if(m->helpgc) { |
| 565 » » LOG("%d: gchelper\n", m->id); |
| 566 » » m->helpgc = 0; |
| 567 » » runtime·gchelper(); |
| 568 » » m->mcache = nil; |
| 569 » » LOG("%d: gchelper end\n", m->id); |
| 570 » » goto retry; |
| 571 » } |
| 572 » LOG("%d: mstop wake\n", m->id); |
| 573 » if(m->p == nil) |
| 574 » » runtime·throw("mstop: p == nil"); |
| 575 } |
| 576 |
| 577 // Schedules gp to run on M. Never returns. |
| 578 static void |
| 579 execute(G *gp) |
| 580 { |
| 581 » int32 hz; |
| 582 |
| 583 » LOG("%d: start running goroutine %p\n", m->id, gp); |
| 584 » CHECK(m->locks == 0, ("")); |
| 585 » CHECK(g == m->g0, ("execute: not on g0\n")); |
| 586 » CHECK(m->p != nil, ("execute: no p\n")); |
| 587 » CHECK(gp->status == Grunnable, ("execute: gp=%d gp->status=%d\n", gp->go
id, gp->status)); |
| 588 » CHECK(gp->m == nil, ("execute: gp->m=%p\n", gp->m)); |
| 589 » CHECK(gp->lockedm == nil && m->lockedg == nil || gp->lockedm == m && m->
lockedg == gp, |
| 590 » » ("bad locking: gp->lockedm=%p m->lockedg=%p\n", gp->lockedm, m->
lockedg)); |
| 591 » m->p->tick++; |
| 592 » gp->status = Grunning; |
| 593 » m->curg = gp; |
| 594 » gp->m = m; |
| 595 |
| 596 » // Check whether the profiler needs to be turned on or off. |
| 597 » hz = runtime·sched.profilehz; |
| 598 » if(m->profilehz != hz) |
| 599 » » runtime·resetcpuprofiler(hz); |
| 600 |
| 601 » if(gp->sched.pc == (byte*)runtime·goexit) // kickoff |
| 602 » » runtime·gogocall(&gp->sched, (void(*)(void))gp->entry); |
| 603 » runtime·gogo(&gp->sched, 0); |
483 } | 604 } |
484 | 605 |
485 // One round of scheduler: find a goroutine and run it. | 606 // One round of scheduler: find a goroutine and run it. |
486 // The argument is the goroutine that was running before | 607 // The argument is the goroutine that was running before |
487 // schedule was called, or nil if this is the first call. | 608 // schedule was called, or nil if this is the first call. |
488 // Never returns. | 609 // Never returns. |
489 static void | 610 static void |
490 schedule(G *gp) | 611 schedule(void) |
491 { | 612 { |
492 » int32 hz, i, try; | 613 » int32 i, try; |
| 614 » G *gp, *gp1; |
493 P *p; | 615 P *p; |
| 616 M *mp; |
494 | 617 |
495 LOG("%d: schedule p=%p\n", m->id, m->p); | 618 LOG("%d: schedule p=%p\n", m->id, m->p); |
496 USED(&gp); | 619 USED(&gp); |
497 » if(m->locks != 0) | 620 » CHECK(m->locks == 0, ("schedule: holding locks\n")); |
498 » » runtime·throw("schedule holding locks"); | 621 » CHECK(m->lockedg == nil, ("schedule: locked M\n")); |
499 » if(gp == m->g0) | |
500 » » runtime·throw("schedule of g0"); | |
501 | 622 |
502 top: | 623 top: |
503 if(runtime·gcwaiting) { | 624 if(runtime·gcwaiting) { |
504 » » leavego(m, Pidle); | 625 » » p = releasep(); |
| 626 » » p->status = Plocked; |
| 627 » » runtime·lock(&runtime·sched); |
| 628 » » runtime·sched.stopwait--; |
| 629 » » if(runtime·sched.stopwait == 0) |
| 630 » » » runtime·notewakeup(&runtime·sched.stopnote); |
| 631 » » runtime·unlock(&runtime·sched); |
505 mstop(); | 632 mstop(); |
506 goto top; | 633 goto top; |
507 } | 634 } |
508 | 635 |
509 » runtime·lock(m->p); | 636 » gp = runqget(m->p); |
510 » gp = gget(m->p); | |
511 » runtime·unlock(m->p); | |
512 if(gp == nil) { | 637 if(gp == nil) { |
513 » » for(try=0; try<10; try++) { | 638 » » for(try=0; try<2; try++) { |
514 » » for(i=0; i<runtime·np; i++) { | 639 » » if(runtime·sched.runqsize) { |
515 » » » p = runtime·allp[i]; | 640 » » » runtime·lock(&runtime·sched); |
516 » » » if(p->ghead) { | 641 » » » gp = globrunqget(); |
517 » » » » runtime·lock(p); | 642 » » » if(gp) { |
518 » » » » gp = gget(p); | 643 » » » » while(gp->schedlink != nil) { |
519 » » » » runtime·unlock(p); | 644 » » » » » gp1 = gp; |
| 645 » » » » » gp = gp1->schedlink; |
| 646 » » » » » runqput(m->p, gp1); |
| 647 » » » » } |
520 } | 648 } |
| 649 runtime·unlock(&runtime·sched); |
| 650 if(gp) |
| 651 goto haveg; |
| 652 } |
| 653 for(i=0; i<runtime·gomaxprocs; i++) { |
| 654 if(runtime·gcwaiting) |
| 655 goto top; |
| 656 p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]
; |
| 657 if(p == m->p) |
| 658 gp = runqget(p); |
| 659 else |
| 660 gp = runqsteal(m->p, p); |
521 if(gp) | 661 if(gp) |
522 break; | 662 break; |
523 } | 663 } |
524 » » » if(gp) | 664 » » if(gp) |
525 » » » » break; | 665 » » » break; |
526 » » » runtime·usleep(20); | 666 » » if(try==0 && runtime·gcwaiting == 0) |
| 667 » » » runtime·osyield(); |
527 } | 668 } |
528 if(gp == nil) { | 669 if(gp == nil) { |
529 » » » leavego(m, Pidle); | 670 » » » p = releasep(); |
| 671 » » » runtime·lock(&runtime·sched); |
| 672 » » » if(runtime·gcwaiting) { |
| 673 » » » » p->status = Plocked; |
| 674 » » » » runtime·sched.stopwait--; |
| 675 » » » » if(runtime·sched.stopwait == 0) |
| 676 » » » » » runtime·notewakeup(&runtime·sched.stopno
te); |
| 677 » » » » runtime·unlock(&runtime·sched); |
| 678 » » » » mstop(); |
| 679 » » » » goto top; |
| 680 » » » } |
| 681 » » » pidleput(p); |
| 682 » » » if(runtime·sched.runqsize) { |
| 683 » » » » p = pidleget(); |
| 684 » » » » runtime·unlock(&runtime·sched); |
| 685 » » » » entergo(m, p); |
| 686 » » » » goto top; |
| 687 » » » } |
| 688 » » » runtime·unlock(&runtime·sched); |
| 689 » » » for(i=0; i<runtime·gomaxprocs; i++) { |
| 690 » » » » p = runtime·allp[i]; |
| 691 » » » » if(p && p->runqhead != p->runqtail) { |
| 692 » » » » » runtime·lock(&runtime·sched); |
| 693 » » » » » p = pidleget(); |
| 694 » » » » » runtime·unlock(&runtime·sched); |
| 695 » » » » » if(p) { |
| 696 » » » » » » entergo(m, p); |
| 697 » » » » » » goto top; |
| 698 » » » » » } |
| 699 » » » » » break; |
| 700 » » » » } |
| 701 » » » } |
530 mstop(); | 702 mstop(); |
531 goto top; | 703 goto top; |
532 } | 704 } |
533 } | 705 } |
534 | 706 |
535 » LOG("%d: start running goroutine %p\n", m->id, gp); | 707 haveg: |
536 » m->p->tick++; | 708 » if(gp->lockedm) { |
537 » gp->status = Grunning; | 709 » » mp = gp->lockedm; |
538 » m->curg = gp; | 710 » » p = releasep(); |
539 » gp->m = m; | 711 » » entergo(mp, p); |
540 | 712 » » runtime·notewakeup(&mp->park); |
541 » // Check whether the profiler needs to be turned on or off. | 713 » » mstop(); |
542 » hz = runtime·sched.profilehz; | 714 » » goto top; |
543 » if(m->profilehz != hz) | 715 » } |
544 » » runtime·resetcpuprofiler(hz); | 716 |
545 | 717 » execute(gp); |
546 » if(gp->sched.pc == (byte*)runtime·goexit) {» // kickoff | |
547 » » runtime·gogocall(&gp->sched, (void(*)(void))gp->entry); | |
548 » } | |
549 » runtime·gogo(&gp->sched, 0); | |
550 } | 718 } |
551 | 719 |
552 static void | 720 static void |
553 park0(G *gp) | 721 park0(G *gp) |
554 { | 722 { |
| 723 P *p; |
| 724 M *mp; |
| 725 |
555 USED(&gp); | 726 USED(&gp); |
| 727 if(m->lockedg) { |
| 728 p = releasep(); |
| 729 if(m->waitunlockf) { |
| 730 m->waitunlockf(m->waitlock); |
| 731 m->waitunlockf = nil; |
| 732 } |
| 733 // After this point another thread may schedule gp on m again. |
| 734 // Schedule another M to run P. |
| 735 runtime·lock(&runtime·sched); |
| 736 mp = mget(); |
| 737 runtime·unlock(&runtime·sched); |
| 738 munpark(mp, p); |
| 739 // Wait until another thread schedules gp and so m again. |
| 740 runtime·notesleep(&m->park); |
| 741 runtime·noteclear(&m->park); |
| 742 execute(gp); // Never returns. |
| 743 } |
556 if(m->waitunlockf) { | 744 if(m->waitunlockf) { |
557 m->waitunlockf(m->waitlock); | 745 m->waitunlockf(m->waitlock); |
558 m->waitunlockf = nil; | 746 m->waitunlockf = nil; |
559 } | 747 } |
560 » schedule(nil); | 748 » schedule(); |
561 } | 749 } |
562 | 750 |
563 // Atomically parks g and unlocks the lock. | 751 // Puts the current goroutine into a waiting state and unlocks the lock. |
564 void | 752 // The goroutine can be made runnable again by calling runtime·ready(gp). |
565 runtime·park(void *l, void(*unlockf)(void*), int8 *reason) | 753 void |
| 754 runtime·park(void(*unlockf)(Lock*), Lock *l, int8 *reason) |
566 { | 755 { |
567 LOG("%d: park l=%p reason=%s\n", m->id, l, reason); | 756 LOG("%d: park l=%p reason=%s\n", m->id, l, reason); |
568 » if(g == m->g0) | 757 » CHECK(g != m->g0, ("park of g0\n")); |
569 » » runtime·throw("park of g0"); | |
570 m->waitlock = l; | 758 m->waitlock = l; |
571 m->waitunlockf = unlockf; | 759 m->waitunlockf = unlockf; |
572 g->status = Gwaiting; | 760 g->status = Gwaiting; |
573 g->waitreason = reason; | 761 g->waitreason = reason; |
574 g->m = nil; | 762 g->m = nil; |
575 runtime·mcall(park0); | 763 runtime·mcall(park0); |
576 } | 764 } |
577 | 765 |
578 static void | 766 static void |
579 gosched0(G *gp) | 767 gosched0(G *gp) |
580 { | 768 { |
| 769 P *p; |
| 770 M *mp; |
| 771 |
581 LOG("%d: gosched0 gp=%p\n", m->id, gp); | 772 LOG("%d: gosched0 gp=%p\n", m->id, gp); |
582 gp->status = Grunnable; | 773 gp->status = Grunnable; |
583 gp->m = nil; | 774 gp->m = nil; |
584 » runtime·lock(m->p); | 775 » if(m->lockedg) { |
585 » gput(m->p, gp); | 776 » » p = releasep(); |
586 » runtime·unlock(m->p); | 777 » » runtime·lock(&runtime·sched); |
587 » schedule(nil); | 778 » » globrunqput(gp); |
| 779 » » // After this point another thread may schedule gp on m again. |
| 780 » » // Schedule another M to run P. |
| 781 » » mp = mget(); |
| 782 » » runtime·unlock(&runtime·sched); |
| 783 » » munpark(mp, p); |
| 784 » » // Wait until another thread schedules gp and so m again. |
| 785 » » runtime·notesleep(&m->park); |
| 786 » » runtime·noteclear(&m->park); |
| 787 » » execute(gp); // Never returns. |
| 788 » } |
| 789 » runtime·lock(&runtime·sched); |
| 790 » globrunqput(gp); |
| 791 » runtime·unlock(&runtime·sched); |
| 792 » schedule(); |
588 } | 793 } |
589 | 794 |
590 void | 795 void |
591 runtime·gosched(void) | 796 runtime·gosched(void) |
592 { | 797 { |
593 runtime·mcall(gosched0); | 798 runtime·mcall(gosched0); |
594 } | 799 } |
595 | 800 |
596 // On g0. | 801 // On g0. |
597 static void | 802 static void |
598 goexit0(G *gp) | 803 goexit0(G *gp) |
599 { | 804 { |
600 gp->status = Gdead; | 805 gp->status = Gdead; |
601 gp->m = nil; | 806 gp->m = nil; |
602 » if(gp->lockedm) { | 807 » gp->lockedm = nil; |
603 » » gp->lockedm = nil; | 808 » m->lockedg = nil; |
604 » » m->lockedg = nil; | |
605 » } | |
606 runtime·unwindstack(gp, nil); | 809 runtime·unwindstack(gp, nil); |
607 gfput(m->p, gp); | 810 gfput(m->p, gp); |
608 » schedule(nil); | 811 » schedule(); |
609 } | 812 } |
610 | 813 |
611 void | 814 void |
612 runtime·goexit(void) | 815 runtime·goexit(void) |
613 { | 816 { |
614 runtime·mcall(goexit0); | 817 runtime·mcall(goexit0); |
615 } | 818 } |
616 | 819 |
617 // The goroutine g is about to enter a system call. | 820 // The goroutine g is about to enter a system call. |
618 // Record that it's not using the cpu anymore. | 821 // Record that it's not using the cpu anymore. |
619 // This is called only from the go syscall library and cgocall, | 822 // This is called only from the go syscall library and cgocall, |
620 // not from the low-level system calls used by the runtime. | 823 // not from the low-level system calls used by the runtime. |
621 // | 824 // |
622 // Entersyscall cannot split the stack: the runtime·gosave must | 825 // Entersyscall cannot split the stack: the runtime·gosave must |
623 // make g->sched refer to the caller's stack segment, because | 826 // make g->sched refer to the caller's stack segment, because |
624 // entersyscall is going to return immediately after. | 827 // entersyscall is going to return immediately after. |
625 #pragma textflag 7 | 828 #pragma textflag 7 |
626 void | 829 void |
627 runtime·entersyscall(void) | 830 runtime·entersyscall(void) |
628 { | 831 { |
629 » LOG("%d: entersyscall g=%p\n", m->id, g); | 832 » P *p; |
630 » //m->sysenterticks = runtime·cputicks(); | 833 » M *mp; |
| 834 |
| 835 » LOG("%d: entersyscall g=%p p=%p\n", m->id, g, m->p); |
631 if(m->profilehz > 0) | 836 if(m->profilehz > 0) |
632 runtime·setprof(false); | 837 runtime·setprof(false); |
633 | 838 |
634 // Leave SP around for gc and traceback. | 839 // Leave SP around for gc and traceback. |
635 runtime·gosave(&g->sched); | 840 runtime·gosave(&g->sched); |
636 g->gcsp = g->sched.sp; | 841 g->gcsp = g->sched.sp; |
637 g->gcstack = g->stackbase; | 842 g->gcstack = g->stackbase; |
638 g->gcguard = g->stackguard; | 843 g->gcguard = g->stackguard; |
639 g->status = Gsyscall; | 844 g->status = Gsyscall; |
640 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { | 845 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { |
641 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", | 846 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", |
642 // g->gcsp, g->gcguard-StackGuard, g->gcstack); | 847 // g->gcsp, g->gcguard-StackGuard, g->gcstack); |
643 runtime·throw("entersyscall"); | 848 runtime·throw("entersyscall"); |
644 } | 849 } |
645 | 850 |
| 851 if(m->blockingsyscall) { |
| 852 m->blockingsyscall = false; |
| 853 p = releasep(); |
| 854 runtime·lock(&runtime·sched); |
| 855 mp = mget(); |
| 856 runtime·unlock(&runtime·sched); |
| 857 munpark(mp, p); |
| 858 return; |
| 859 } |
| 860 |
646 m->mcache = nil; | 861 m->mcache = nil; |
647 m->p->tick++; | |
648 m->p->m = nil; | 862 m->p->m = nil; |
649 » m->p->status = Psyscall; //!!! store-release | 863 » runtime·atomicstore(&m->p->status, Psyscall); |
| 864 » if(runtime·gcwaiting) { |
| 865 » » runtime·lock(&runtime·sched); |
| 866 » » if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psy
scall, Plocked)) { |
| 867 » » » runtime·sched.stopwait--; |
| 868 » » » if(runtime·sched.stopwait == 0) |
| 869 » » » » runtime·notewakeup(&runtime·sched.stopnote); |
| 870 » » } |
| 871 » » runtime·unlock(&runtime·sched); |
| 872 » } |
| 873 } |
| 874 |
| 875 #pragma textflag 7 |
| 876 void |
| 877 runtime·entersyscallblock(void) |
| 878 { |
| 879 » m->blockingsyscall = true; |
| 880 » runtime·entersyscall(); |
650 } | 881 } |
651 | 882 |
652 static void | 883 static void |
653 exitsyscall0(G *gp) | 884 exitsyscall0(G *gp) |
654 { | 885 { |
| 886 P *p; |
| 887 |
655 LOG("%d: exitsyscall0\n", m->id); | 888 LOG("%d: exitsyscall0\n", m->id); |
656 gp->status = Grunnable; | 889 gp->status = Grunnable; |
657 gp->m = nil; | 890 gp->m = nil; |
658 » runtime·lock(runtime·allp[0]); | 891 » CHECK(m->park.waitm == nil, ("exitsyscall0: park is signalled\n")); |
659 » //!!! anything better than put to allp[0]? | 892 » runtime·lock(&runtime·sched); |
660 » gput(runtime·allp[0], gp); | 893 » p = pidleget(); |
661 » runtime·unlock(runtime·allp[0]); | 894 » if(p == nil) |
| 895 » » globrunqput(gp); |
| 896 » runtime·unlock(&runtime·sched); |
| 897 » if(p) { |
| 898 » » entergo(m, p); |
| 899 » » execute(gp); // Never returns. |
| 900 » } |
| 901 » if(m->lockedg) { |
| 902 » » CHECK(m->lockedg == gp, ("exitsyscall0: inconsistent locking\n")
); |
| 903 » » // Wait until another thread schedules gp and so m again. |
| 904 » » runtime·notesleep(&m->park); |
| 905 » » runtime·noteclear(&m->park); |
| 906 » » execute(gp); // Never returns. |
| 907 » } |
662 mstop(); | 908 mstop(); |
663 » schedule(nil); | 909 » schedule(); |
664 } | 910 } |
665 | 911 |
666 // The goroutine g exited its system call. | 912 // The goroutine g exited its system call. |
667 // Arrange for it to run on a cpu again. | 913 // Arrange for it to run on a cpu again. |
668 // This is called only from the go syscall library, not | 914 // This is called only from the go syscall library, not |
669 // from the low-level system calls used by the runtime. | 915 // from the low-level system calls used by the runtime. |
670 void | 916 void |
671 runtime·exitsyscall(void) | 917 runtime·exitsyscall(void) |
672 { | 918 { |
| 919 uint32 s; |
| 920 P *p; |
| 921 |
673 LOG("%d: exitsyscall g=%p\n", m->id, g); | 922 LOG("%d: exitsyscall g=%p\n", m->id, g); |
674 | 923 |
675 //runtime·printf("syscall=%d\n", (int32)(runtime·cputicks() - m->sysenterticks))
; | 924 » // Check whether the profiler needs to be turned on. |
676 | 925 » if(m->profilehz > 0) |
677 » if(runtime·cas(&m->p->status, Psyscall, Pbusy)) { | 926 » » runtime·setprof(true); |
678 LOG("%d: exitsyscall 1\n", m->id); | 927 |
| 928 » // Try to re-acquire the P. |
| 929 » s = m->p ? m->p->status : Pidle; |
| 930 » if(s == Psyscall && runtime·cas(&m->p->status, s, Pbusy)) { |
| 931 » » LOG("%d: exitsyscall fast\n", m->id); |
679 // There's a cpu for us, so we can run. | 932 // There's a cpu for us, so we can run. |
680 m->mcache = m->p->mcache; | 933 m->mcache = m->p->mcache; |
681 m->p->m = m; | 934 m->p->m = m; |
682 g->status = Grunning; | 935 g->status = Grunning; |
683 // Garbage collector isn't running (since we are), | 936 // Garbage collector isn't running (since we are), |
684 // so okay to clear gcstack. | 937 // so okay to clear gcstack. |
685 g->gcstack = (uintptr)nil; | 938 g->gcstack = (uintptr)nil; |
686 | |
687 // Check whether the profiler needs to be turned on or off. | |
688 if(m->profilehz > 0) | |
689 runtime·setprof(true); | |
690 return; | 939 return; |
691 } | 940 } |
692 | 941 |
693 LOG("%d: exitsyscall 2\n", m->id); | 942 » // Try to get idle P. |
694 //runtime·printf("%d: exitsyscall slow\n", m->id); | |
695 » if(m->profilehz > 0) | |
696 » » runtime·setprof(true); | |
697 | |
698 m->p = nil; | 943 m->p = nil; |
| 944 if(runtime·sched.pidle) { |
| 945 runtime·lock(&runtime·sched); |
| 946 p = pidleget(); |
| 947 runtime·unlock(&runtime·sched); |
| 948 if(p) { |
| 949 entergo(m, p); |
| 950 g->gcstack = (uintptr)nil; |
| 951 return; |
| 952 } |
| 953 } |
| 954 |
| 955 LOG("%d: exitsyscall slow p->status=%d\n", m->id, s); |
| 956 |
699 runtime·mcall(exitsyscall0); | 957 runtime·mcall(exitsyscall0); |
700 | 958 |
701 // Gosched returned, so we're allowed to run now. | 959 // Gosched returned, so we're allowed to run now. |
702 // Delete the gcstack information that we left for | 960 // Delete the gcstack information that we left for |
703 // the garbage collector during the system call. | 961 // the garbage collector during the system call. |
704 // Must wait until now because until gosched returns | 962 // Must wait until now because until gosched returns |
705 // we don't know for sure that the garbage collector | 963 // we don't know for sure that the garbage collector |
706 // is not running. | 964 // is not running. |
707 g->gcstack = (uintptr)nil; | 965 g->gcstack = (uintptr)nil; |
708 } | 966 } |
(...skipping 192 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
901 | 1159 |
902 // Create a new g running fn with narg bytes of arguments starting | 1160 // Create a new g running fn with narg bytes of arguments starting |
903 // at argp and returning nret bytes of results. callerpc is the | 1161 // at argp and returning nret bytes of results. callerpc is the |
904 // address of the go statement that created this. The new g is put | 1162 // address of the go statement that created this. The new g is put |
905 // on the queue of g's waiting to run. | 1163 // on the queue of g's waiting to run. |
906 G* | 1164 G* |
907 runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) | 1165 runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) |
908 { | 1166 { |
909 byte *sp; | 1167 byte *sp; |
910 G *newg; | 1168 G *newg; |
| 1169 M *mp; |
| 1170 P *p; |
911 int32 siz; | 1171 int32 siz; |
| 1172 //int64 goid; |
912 | 1173 |
913 //printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret); | 1174 //printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret); |
914 siz = narg + nret; | 1175 siz = narg + nret; |
915 siz = (siz+7) & ~7; | 1176 siz = (siz+7) & ~7; |
916 | 1177 |
917 // We could instead create a secondary stack frame | 1178 // We could instead create a secondary stack frame |
918 // and make it look like goexit was on the original but | 1179 // and make it look like goexit was on the original but |
919 // the call to the actual goroutine function was split. | 1180 // the call to the actual goroutine function was split. |
920 // Not worth it: this is almost always an error. | 1181 // Not worth it: this is almost always an error. |
921 if(siz > StackMin - 1024) | 1182 if(siz > StackMin - 1024) |
922 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); | 1183 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); |
923 | 1184 |
924 if((newg = gfget(m->p)) != nil) { | 1185 if((newg = gfget(m->p)) != nil) { |
| 1186 //!!!if(raceenabled) |
| 1187 //!!! runtime·racegostart(goid, callerpc); |
925 if(newg->stackguard - StackGuard != newg->stack0) | 1188 if(newg->stackguard - StackGuard != newg->stack0) |
926 runtime·throw("invalid stack in newg"); | 1189 runtime·throw("invalid stack in newg"); |
927 } else { | 1190 } else { |
928 newg = runtime·malg(StackMin); | 1191 newg = runtime·malg(StackMin); |
929 » » newg->alllink = m->p->allg; | 1192 » » runtime·lock(&runtime·sched); |
930 » » m->p->allg = newg; | 1193 » » newg->goid = ++runtime·sched.goidseq; |
| 1194 » » if(runtime·lastg == nil) |
| 1195 » » » runtime·allg = newg; |
| 1196 » » else |
| 1197 » » » runtime·lastg->alllink = newg; |
| 1198 » » runtime·lastg = newg;» »······· |
| 1199 » » runtime·unlock(&runtime·sched); |
931 } | 1200 } |
932 | 1201 |
933 sp = (byte*)newg->stackbase; | 1202 sp = (byte*)newg->stackbase; |
934 sp -= siz; | 1203 sp -= siz; |
935 runtime·memmove(sp, argp, narg); | 1204 runtime·memmove(sp, argp, narg); |
936 if(thechar == '5') { | 1205 if(thechar == '5') { |
937 // caller's LR | 1206 // caller's LR |
938 sp -= sizeof(void*); | 1207 sp -= sizeof(void*); |
939 *(void**)sp = nil; | 1208 *(void**)sp = nil; |
940 } | 1209 } |
941 | 1210 |
942 LOG("%d: newproc %p\n", m->id, newg); | 1211 LOG("%d: newproc %p\n", m->id, newg); |
943 newg->sched.sp = (uintptr)sp; | 1212 newg->sched.sp = (uintptr)sp; |
944 newg->sched.pc = (byte*)runtime·goexit; | 1213 newg->sched.pc = (byte*)runtime·goexit; |
945 newg->sched.g = newg; | 1214 newg->sched.g = newg; |
946 newg->entry = fn; | 1215 newg->entry = fn; |
947 newg->gopc = (uintptr)callerpc; | 1216 newg->gopc = (uintptr)callerpc; |
948 newg->status = Grunnable; | 1217 newg->status = Grunnable; |
949 » runtime·lock(m->p); | 1218 » runqput(m->p, newg); |
950 » gput(m->p, newg); | 1219 |
951 » runtime·unlock(m->p); | 1220 » if(runtime·sched.pidle && fn != (byte*)runtime·main) { |
| 1221 » » runtime·lock(&runtime·sched); |
| 1222 » » p = pidleget(); |
| 1223 » » if(p) { |
| 1224 » » » mp = mget(); |
| 1225 » » » runtime·unlock(&runtime·sched); |
| 1226 » » » if(mp) { |
| 1227 » » » » entergo(mp, p); |
| 1228 » » » » runtime·notewakeup(&mp->park); |
| 1229 » » » } else |
| 1230 » » » » newm(runtime·mstart, p, false); |
| 1231 » » } else |
| 1232 » » » runtime·unlock(&runtime·sched); |
| 1233 » } |
952 return newg; | 1234 return newg; |
953 } | |
954 | |
955 // Put on gfree list. Sched must be locked. | |
956 static void | |
957 gfput(P *p, G *gp) | |
958 { | |
959 if(gp->stackguard - StackGuard != gp->stack0) | |
960 runtime·throw("invalid stack in gfput"); | |
961 gp->schedlink = p->gfree; | |
962 p->gfree = gp; | |
963 } | |
964 | |
965 // Get from gfree list. Sched must be locked. | |
966 static G* | |
967 gfget(P *p) | |
968 { | |
969 G *gp; | |
970 | |
971 gp = p->gfree; | |
972 if(gp) | |
973 p->gfree = gp->schedlink; | |
974 return gp; | |
975 } | 1235 } |
976 | 1236 |
977 void | 1237 void |
978 runtime·Breakpoint(void) | 1238 runtime·Breakpoint(void) |
979 { | 1239 { |
980 runtime·breakpoint(); | 1240 runtime·breakpoint(); |
981 } | 1241 } |
982 | 1242 |
983 void | 1243 void |
984 runtime·Gosched(void) | 1244 runtime·Gosched(void) |
985 { | 1245 { |
986 runtime·gosched(); | 1246 runtime·gosched(); |
987 } | 1247 } |
988 | 1248 |
989 // Implementation of runtime.GOMAXPROCS. | 1249 // Implementation of runtime.GOMAXPROCS. |
990 // delete when scheduler is stronger | 1250 // delete when scheduler is stronger |
991 int32 | 1251 int32 |
992 runtime·gomaxprocsfunc(int32 n) | 1252 runtime·gomaxprocsfunc(int32 n) |
993 { | 1253 { |
994 int32 ret; | 1254 int32 ret; |
995 | 1255 |
| 1256 LOG("%d: gomaxprocsfunc %d\n", m->id, n); |
996 if(n > maxgomaxprocs) | 1257 if(n > maxgomaxprocs) |
997 n = maxgomaxprocs; | 1258 n = maxgomaxprocs; |
998 runtime·lock(&runtime·sched); | 1259 runtime·lock(&runtime·sched); |
999 ret = runtime·gomaxprocs; | 1260 ret = runtime·gomaxprocs; |
1000 if(n <= 0 || n == ret) { | 1261 if(n <= 0 || n == ret) { |
1001 runtime·unlock(&runtime·sched); | 1262 runtime·unlock(&runtime·sched); |
1002 return ret; | 1263 return ret; |
1003 } | 1264 } |
1004 runtime·unlock(&runtime·sched); | 1265 runtime·unlock(&runtime·sched); |
1005 | 1266 |
1006 runtime·semacquire(&runtime·worldsema); | 1267 runtime·semacquire(&runtime·worldsema); |
1007 m->gcing = 1; | 1268 m->gcing = 1; |
1008 runtime·stoptheworld(); | 1269 runtime·stoptheworld(); |
1009 | 1270 » newprocs = n; |
1010 » runtime·gomaxprocs = n; | |
1011 | |
1012 m->gcing = 0; | 1271 m->gcing = 0; |
1013 runtime·semrelease(&runtime·worldsema); | 1272 runtime·semrelease(&runtime·worldsema); |
1014 runtime·starttheworld(); | 1273 runtime·starttheworld(); |
1015 | 1274 |
1016 return ret; | 1275 return ret; |
1017 } | 1276 } |
1018 | 1277 |
1019 void | 1278 void |
1020 runtime·LockOSThread(void) | 1279 runtime·LockOSThread(void) |
1021 { | 1280 { |
1022 //!!! implement me. | |
1023 /* | |
1024 if(m == &runtime·m0 && runtime·sched.init) { | 1281 if(m == &runtime·m0 && runtime·sched.init) { |
1025 runtime·sched.lockmain = true; | 1282 runtime·sched.lockmain = true; |
1026 return; | 1283 return; |
1027 } | 1284 } |
1028 m->lockedg = g; | 1285 m->lockedg = g; |
1029 g->lockedm = m; | 1286 g->lockedm = m; |
1030 */ | |
1031 } | 1287 } |
1032 | 1288 |
1033 void | 1289 void |
1034 runtime·UnlockOSThread(void) | 1290 runtime·UnlockOSThread(void) |
1035 { | 1291 { |
1036 /* | |
1037 if(m == &runtime·m0 && runtime·sched.init) { | 1292 if(m == &runtime·m0 && runtime·sched.init) { |
1038 runtime·sched.lockmain = false; | 1293 runtime·sched.lockmain = false; |
1039 return; | 1294 return; |
1040 } | 1295 } |
1041 m->lockedg = nil; | 1296 m->lockedg = nil; |
1042 g->lockedm = nil; | 1297 g->lockedm = nil; |
1043 */ | |
1044 } | 1298 } |
1045 | 1299 |
1046 bool | 1300 bool |
1047 runtime·lockedOSThread(void) | 1301 runtime·lockedOSThread(void) |
1048 { | 1302 { |
1049 return g->lockedm != nil && m->lockedg != nil; | 1303 return g->lockedm != nil && m->lockedg != nil; |
1050 } | 1304 } |
1051 | 1305 |
1052 // for testing of callbacks | 1306 // for testing of callbacks |
1053 void | 1307 void |
1054 runtime·golockedOSThread(bool ret) | 1308 runtime·golockedOSThread(bool ret) |
1055 { | 1309 { |
1056 ret = runtime·lockedOSThread(); | 1310 ret = runtime·lockedOSThread(); |
1057 FLUSH(&ret); | 1311 FLUSH(&ret); |
1058 } | 1312 } |
1059 | 1313 |
1060 // for testing of wire, unwire | 1314 // for testing of wire, unwire |
1061 void | 1315 void |
1062 runtime·mid(uint32 ret) | 1316 runtime·mid(uint32 ret) |
1063 { | 1317 { |
1064 ret = m->id; | 1318 ret = m->id; |
1065 FLUSH(&ret); | 1319 FLUSH(&ret); |
1066 } | 1320 } |
1067 | 1321 |
1068 void | 1322 void |
1069 runtime·NumGoroutine(int32 ret) | 1323 runtime·NumGoroutine(intgo ret) |
1070 { | 1324 { |
1071 //ret = runtime·sched.gcount; | 1325 //ret = runtime·sched.gcount; |
1072 ret = 1; | 1326 ret = 1; |
1073 FLUSH(&ret); | 1327 FLUSH(&ret); |
1074 } | 1328 } |
1075 | 1329 |
1076 int32 | 1330 int32 |
1077 runtime·gcount(void) | 1331 runtime·gcount(void) |
1078 { | 1332 { |
1079 //return runtime·sched.gcount; | 1333 //return runtime·sched.gcount; |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1149 runtime·lock(&runtime·sched); | 1403 runtime·lock(&runtime·sched); |
1150 runtime·sched.profilehz = hz; | 1404 runtime·sched.profilehz = hz; |
1151 runtime·unlock(&runtime·sched); | 1405 runtime·unlock(&runtime·sched); |
1152 | 1406 |
1153 if(hz != 0) | 1407 if(hz != 0) |
1154 runtime·resetcpuprofiler(hz); | 1408 runtime·resetcpuprofiler(hz); |
1155 } | 1409 } |
1156 | 1410 |
1157 // Change number of processors. The world is stopped. | 1411 // Change number of processors. The world is stopped. |
1158 static void | 1412 static void |
1159 procresize(void) | 1413 procresize(int32 new) |
1160 { | 1414 { |
1161 » int32 i, old, new; | 1415 » int32 i, old; |
1162 G *gp; | 1416 G *gp; |
1163 | 1417 » P *p; |
1164 » old = runtime·np; | 1418 |
1165 » new = runtime·gomaxprocs; | 1419 » runtime·lock(&runtime·sched); //!!! |
| 1420 » old = runtime·gomaxprocs; |
1166 LOG("%d: procresize %d->%d\n", m->id, old, new); | 1421 LOG("%d: procresize %d->%d\n", m->id, old, new); |
1167 if(old < 0 || old > maxgomaxprocs || new <= 0 || new > maxgomaxprocs) | 1422 if(old < 0 || old > maxgomaxprocs || new <= 0 || new > maxgomaxprocs) |
1168 runtime·throw("procresize: invalid arg"); | 1423 runtime·throw("procresize: invalid arg"); |
1169 if(old == new) { | 1424 if(old == new) { |
1170 for(i=0; i<new; i++) { | 1425 for(i=0; i<new; i++) { |
1171 » » » if(runtime·allp[i] == m->p) | 1426 » » » p = runtime·allp[i]; |
1172 » » » » runtime·allp[i]->status = Pbusy; | 1427 » » » if(p == m->p) |
| 1428 » » » » p->status = Pbusy; |
| 1429 » » » else { |
| 1430 » » » » p->status = Pidle; |
| 1431 » » » » pidleput(p); |
| 1432 » » » } |
| 1433 » » } |
| 1434 » » runtime·unlock(&runtime·sched); |
| 1435 » » return; |
| 1436 » } |
| 1437 |
| 1438 » runtime·singleproc = new == 1; |
| 1439 » runtime·gomaxprocs = new; |
| 1440 » for(i=0; i<new; i++) { |
| 1441 » » p = runtime·allp[i]; |
| 1442 » » if(p == nil) { |
| 1443 » » » p = (P*)runtime·mallocgc(sizeof(runtime·allp[i][0]), 0,
0, 1); |
| 1444 » » » p->status = Plocked; |
| 1445 » » » runtime·allp[i] = p; //@@@ store-release |
| 1446 » » } |
| 1447 » » if(p->mcache == nil) { |
| 1448 » » » if(old==0 && i==0) |
| 1449 » » » » p->mcache = m->mcache; |
1173 else | 1450 else |
1174 » » » » runtime·allp[i]->status = Pidle; //!!! store-rel
ease | 1451 » » » » p->mcache = runtime·allocmcache(); |
1175 » » } | 1452 » » } |
1176 » » return; | 1453 » » if(p->runq == nil) { |
1177 » } | 1454 » » » p->runqsize = 1024; |
1178 | 1455 » » » p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*),
0, 0, 1); |
1179 » runtime·singleproc = new == 1; | |
1180 » runtime·np = new; | |
1181 » for(i=0; i<new; i++) { | |
1182 » » if(runtime·allp[i] == nil) | |
1183 » » » runtime·allp[i] = (P*)runtime·malloc(sizeof(runtime·allp
[i][0])); | |
1184 » » if(runtime·allp[i]->mcache == nil) { | |
1185 » » » if(old==0 && i==0) | |
1186 » » » » runtime·allp[i]->mcache = m->mcache; | |
1187 » » » else | |
1188 » » » » runtime·allp[i]->mcache = runtime·allocmcache(); | |
1189 } | 1456 } |
1190 } | 1457 } |
1191 | 1458 |
1192 for(i=1; i<old; i++) { | 1459 for(i=1; i<old; i++) { |
1193 for(;;) { | 1460 for(;;) { |
1194 » » » gp = gget(runtime·allp[i]); | 1461 » » » gp = runqget(runtime·allp[i]); |
1195 if(gp == nil) | 1462 if(gp == nil) |
1196 break; | 1463 break; |
1197 //TODO: spread more evenly. | 1464 //TODO: spread more evenly. |
1198 » » » gput(runtime·allp[0], gp); | 1465 » » » runqput(runtime·allp[0], gp); |
1199 } | 1466 } |
1200 } | 1467 } |
1201 | 1468 |
1202 for(i=new; i<old; i++) { | 1469 for(i=new; i<old; i++) { |
1203 runtime·freemcache(runtime·allp[i]->mcache); | 1470 runtime·freemcache(runtime·allp[i]->mcache); |
1204 runtime·allp[i]->mcache = nil; | 1471 runtime·allp[i]->mcache = nil; |
1205 runtime·allp[i]->status = Pdead; | 1472 runtime·allp[i]->status = Pdead; |
1206 //TODO: free freeg | 1473 //TODO: free freeg |
1207 } | 1474 } |
1208 | 1475 |
1209 if(m->p) | 1476 if(m->p) |
1210 m->p->m = nil; | 1477 m->p->m = nil; |
1211 m->p = nil; | 1478 m->p = nil; |
1212 m->mcache = nil; | 1479 m->mcache = nil; |
1213 runtime·allp[0]->m = nil; | 1480 runtime·allp[0]->m = nil; |
1214 » runtime·allp[0]->status = Pbusy; | 1481 » runtime·allp[0]->status = Pidle; |
1215 entergo(m, runtime·allp[0]); | 1482 entergo(m, runtime·allp[0]); |
1216 » for(i=1; i<new; i++) | 1483 » for(i=1; i<new; i++) { |
1217 » » runtime·allp[i]->status = Pidle; //!!! store-release | 1484 » » p = runtime·allp[i]; |
| 1485 » » p->status = Pidle; |
| 1486 » » pidleput(p); |
| 1487 » } |
| 1488 » runtime·unlock(&runtime·sched); |
1218 } | 1489 } |
1219 | 1490 |
1220 static void | 1491 static void |
1221 entergo(M *mp, P *p) | 1492 entergo(M *mp, P *p) |
1222 { | 1493 { |
1223 » LOG("%d: entergo p=%p\n", mp->id, p); | 1494 » LOG("%d: entergo m=%d p=%p p->m=%p, p->status=%d, p->mcache=%p\n", m->id
, mp->id, p, p->m, p->status, p->mcache); |
1224 if(mp->p || mp->mcache) | 1495 if(mp->p || mp->mcache) |
1225 runtime·throw("entergo: already in go"); | 1496 runtime·throw("entergo: already in go"); |
1226 » if(p->m || p->status != Pbusy) { | 1497 » if(p->m || p->status != Pidle) { |
1227 » » runtime·printf("entergo: p->m=%d p->status=%d\n", p->m->id, p->s
tatus); | 1498 » » runtime·printf("entergo: p->m=%p(%d) p->status=%d\n", p->m, p->m
? p->m->id : 0, p->status); |
1228 runtime·throw("entergo: invalid p state"); | 1499 runtime·throw("entergo: invalid p state"); |
1229 } | 1500 } |
1230 mp->mcache = p->mcache; | 1501 mp->mcache = p->mcache; |
1231 mp->p = p; | 1502 mp->p = p; |
1232 p->m = mp; | 1503 p->m = mp; |
1233 } | 1504 » p->status = Pbusy; |
1234 | 1505 } |
1235 static void | 1506 |
1236 leavego(M *mp, uint32 status) | 1507 static P* |
1237 { | 1508 releasep(void) |
| 1509 { |
| 1510 » M *mp; |
1238 P *p; | 1511 P *p; |
1239 | 1512 |
1240 » LOG("%d: leavego %d\n", mp->id, status); | 1513 » mp = m; |
| 1514 » LOG("%d: releasep\n", mp->id); |
1241 // sched is locked | 1515 // sched is locked |
1242 if(mp->p == nil || mp->mcache == nil) | 1516 if(mp->p == nil || mp->mcache == nil) |
1243 » » runtime·throw("leavego: invalid arg"); | 1517 » » runtime·throw("releasep: invalid arg"); |
1244 p = mp->p; | 1518 p = mp->p; |
1245 » if(p->m != mp || p->mcache != mp->mcache || p->status != Pbusy) | 1519 » if(p->m != mp || p->mcache != mp->mcache || p->status != Pbusy) { |
1246 » » runtime·throw("leavego: invalid p state"); | 1520 » » runtime·printf("releasep: m=%p m->p=%p p->m=%p m->mcache=%p p->m
cache=%p p->status=%d\n", |
| 1521 » » » mp, mp->p, p->m, m->mcache, p->mcache, p->status); |
| 1522 » » runtime·throw("releasep: invalid p state"); |
| 1523 » } |
1247 mp->p = nil; | 1524 mp->p = nil; |
1248 mp->mcache = nil; | 1525 mp->mcache = nil; |
1249 p->m = nil; | 1526 p->m = nil; |
1250 » p->status = status; //!!! store-release | 1527 » p->status = Pidle; |
1251 } | 1528 » return p; |
1252 | 1529 } |
1253 static void | 1530 |
1254 goidle(void) | 1531 typedef struct Pdesc Pdesc; |
1255 { | 1532 struct Pdesc |
1256 » uint32 i, s, try; | 1533 { |
1257 » P *p, *idlep; | 1534 » uint32» tick; |
1258 » bool haveg; | 1535 » int64» when; |
| 1536 }; |
| 1537 |
| 1538 static void |
| 1539 retake(int64 now, Pdesc *ps) |
| 1540 { |
| 1541 » uint32 i, s; |
| 1542 » int64 t; |
| 1543 » P *p; |
1259 M *mp; | 1544 M *mp; |
1260 | 1545 |
1261 » LOG("%d: goidle\n", m->id); | 1546 » for(i=0; i<runtime·gomaxprocs; i++) { |
1262 » if(g != m->g0) | 1547 » » p = runtime·allp[i]; |
1263 » » runtime·throw("goidle not on g0"); | 1548 » » //!!! procresize may be in progress |
1264 » if(m->p) | 1549 » » // do something if GC is in progress (help). |
1265 » » runtime·throw("goidle not idle"); | 1550 » » if(p==nil) |
1266 | |
1267 » for(try=0;; try++) { | |
1268 » » idlep = nil; | |
1269 » » haveg = false; | |
1270 » » for(i=0; i<runtime·np; i++) { | |
1271 » » » p = runtime·allp[i]; | |
1272 » » » //!!! procresize may be in progress | |
1273 » » » // do something if GC is in progress. | |
1274 » » » if(p==nil) | |
1275 » » » » continue; | |
1276 » » » //runtime·printf("%d: goidle p=%d status=%d ghead=%p\n",
m->id, i, p->status, p->ghead); | |
1277 » » » if(p->status == Pidle) | |
1278 » » » » idlep = p; | |
1279 » » » else if(p->status == Psyscall) { | |
1280 » » » » if(p->tick == p->lasttick) | |
1281 » » » » » idlep = p; | |
1282 » » » » else | |
1283 » » » » » p->lasttick = p->tick; | |
1284 » » » } | |
1285 » » » if(p->ghead != nil) | |
1286 » » » » haveg = true; | |
1287 » » } | |
1288 » » if(idlep == nil || !haveg) { | |
1289 » » » runtime·usleep(20); | |
1290 continue; | 1551 continue; |
1291 » » } | 1552 » » t = p->tick; |
1292 » » s = idlep->status; | 1553 » » if(ps[i].tick != t) { |
1293 » » if(s == Pidle || s == Psyscall) { | 1554 » » » ps[i].tick = t; |
1294 » » » //runtime·printf("%d: trying to steal p in status %d\n",
m->id, s); | 1555 » » » ps[i].when = now; |
1295 » » » if(runtime·cas(&idlep->status, s, Pbusy)) { | 1556 » » } |
1296 » » » » entergo(m, idlep); | 1557 » » if(ps[i].when + 20*1000 > now) |
1297 » » » » runtime·lock(&runtime·sched); | 1558 » » » continue; |
1298 » » » » mp = mget(); | 1559 » » s = p->status; |
| 1560 » » if(s == Psyscall && runtime·cas(&p->status, s, Pidle)) { |
| 1561 » » » LOG("retake %p(%d)\n", p, i); |
| 1562 » » » runtime·lock(&runtime·sched); |
| 1563 » » » mp = mget(); |
| 1564 » » » runtime·unlock(&runtime·sched); |
| 1565 » » » munpark(mp, p); |
| 1566 » » } |
| 1567 » } |
| 1568 } |
| 1569 |
| 1570 static Pdesc ps[maxgomaxprocs]; |
| 1571 |
| 1572 static void |
| 1573 sysmon(void) |
| 1574 { |
| 1575 » int64 t0, now; |
| 1576 |
| 1577 » // This is a special dedicated thread. |
| 1578 » // It works w/o mcache nor stackalloc, it may work concurrently with GC. |
| 1579 » runtime·asminit(); |
| 1580 » runtime·minit(); |
| 1581 » LOG("sysmon\n"); |
| 1582 » t0 = runtime·nanotime(); |
| 1583 » for(;;) { |
| 1584 » » //!!! sleep more if possible |
| 1585 » » runtime·usleep(20); |
| 1586 » » if(runtime·gcwaiting) { |
| 1587 » » » runtime·lock(&runtime·sched); |
| 1588 » » » if(runtime·gcwaiting) { |
| 1589 » » » » runtime·sched.sysmonwait = 1; |
1299 runtime·unlock(&runtime·sched); | 1590 runtime·unlock(&runtime·sched); |
1300 » » » » if(mp) | 1591 » » » » runtime·notesleep(&runtime·sched.sysmonnote); |
1301 » » » » » runtime·notewakeup(&mp->park); | 1592 » » » » runtime·noteclear(&runtime·sched.sysmonnote); |
1302 » » » » else | 1593 » » » } else |
1303 » » » » » newm(); | 1594 » » » » runtime·unlock(&runtime·sched); |
1304 » » » » return; | 1595 » » } |
1305 » » » } | 1596 » » now = runtime·nanotime() - t0; |
1306 » » } | 1597 » » retake(now, ps); |
1307 » } | 1598 » } |
1308 } | 1599 } |
| 1600 |
| 1601 static void |
| 1602 inject(G *gp0, int32 *w, int32 *n) |
| 1603 { |
| 1604 » int32 nw; |
| 1605 » G *gp; |
| 1606 » M *mp; |
| 1607 » P *p; |
| 1608 |
| 1609 » runtime·lock(&runtime·sched); |
| 1610 » while(gp0) { |
| 1611 » » gp = gp0; |
| 1612 » » gp0 = gp->schedlink; |
| 1613 » » gp->status = Grunnable; |
| 1614 » » globrunqput(gp); |
| 1615 » » (*n)++; |
| 1616 » } |
| 1617 » runtime·unlock(&runtime·sched); |
| 1618 |
| 1619 » nw = *n; |
| 1620 » while(runtime·sched.pidle && nw) { |
| 1621 » » runtime·lock(&runtime·sched); |
| 1622 » » if(runtime·sched.pidle == nil) { |
| 1623 » » » runtime·unlock(&runtime·sched); |
| 1624 » » » break; |
| 1625 » » } |
| 1626 » » (*w)++; |
| 1627 » » nw--; |
| 1628 » » p = pidleget(); |
| 1629 » » mp = mget(); |
| 1630 » » runtime·unlock(&runtime·sched); |
| 1631 » » if(mp) { |
| 1632 » » » entergo(mp, p); |
| 1633 » » » runtime·notewakeup(&mp->park); |
| 1634 » » } else |
| 1635 » » » newm(runtime·mstart, p, false); |
| 1636 » } |
| 1637 } |
| 1638 |
| 1639 static void |
| 1640 globrunqput(G *gp) |
| 1641 { |
| 1642 » gp->schedlink = nil; |
| 1643 » if(runtime·sched.runqtail) |
| 1644 » » runtime·sched.runqtail->schedlink = gp; |
| 1645 » else |
| 1646 » » runtime·sched.runqhead = gp; |
| 1647 » runtime·sched.runqtail = gp; |
| 1648 » runtime·sched.runqsize++; |
| 1649 } |
| 1650 |
| 1651 static G* |
| 1652 globrunqget(void) |
| 1653 { |
| 1654 » G *gp, *gp1; |
| 1655 » int32 n; |
| 1656 |
| 1657 » if(runtime·sched.runqsize == 0) |
| 1658 » » return nil; |
| 1659 » n = runtime·sched.runqsize/runtime·gomaxprocs+1; |
| 1660 » if(n > runtime·sched.runqsize) |
| 1661 » » n = runtime·sched.runqsize; |
| 1662 » runtime·sched.runqsize -= n; |
| 1663 » if(runtime·sched.runqsize == 0) |
| 1664 » » runtime·sched.runqtail = nil; |
| 1665 » gp1 = nil; |
| 1666 » while(n--) { |
| 1667 » » gp = runtime·sched.runqhead; |
| 1668 » » runtime·sched.runqhead = gp->schedlink; |
| 1669 » » gp->schedlink = gp1; |
| 1670 » » gp1 = gp; |
| 1671 » } |
| 1672 » return gp1; |
| 1673 } |
| 1674 |
| 1675 // sched is locked |
| 1676 static P* |
| 1677 pidleget(void) |
| 1678 { |
| 1679 » P *p; |
| 1680 »······· |
| 1681 » p = runtime·sched.pidle; |
| 1682 » if(p) { |
| 1683 » » runtime·sched.pidle = p->link; |
| 1684 » » runtime·sched.npidle--; |
| 1685 » } |
| 1686 » return p; |
| 1687 } |
| 1688 |
| 1689 // sched is locked |
| 1690 static void |
| 1691 pidleput(P *p) |
| 1692 { |
| 1693 » p->link = runtime·sched.pidle; |
| 1694 » runtime·sched.pidle = p; |
| 1695 » runtime·sched.npidle++; |
| 1696 } |
| 1697 |
| 1698 static void |
| 1699 runqgrow(P *p) |
| 1700 { |
| 1701 » G **q; |
| 1702 » int32 s, t, h, t2; |
| 1703 |
| 1704 » h = p->runqhead; |
| 1705 » t = p->runqtail; |
| 1706 » s = p->runqsize; |
| 1707 » t2 = 0; |
| 1708 » q = (G**)runtime·malloc(2*s*sizeof(*q)); |
| 1709 » while(t!=h) { |
| 1710 » » q[t2] = p->runq[h]; |
| 1711 » » t2++; |
| 1712 » » h++; |
| 1713 » » if(h==s) |
| 1714 » » » h = 0; |
| 1715 » } |
| 1716 » runtime·free(p->runq); |
| 1717 » p->runq = q; |
| 1718 » p->runqhead = 0; |
| 1719 » p->runqtail = t2; |
| 1720 » p->runqsize = 2*s; |
| 1721 } |
| 1722 |
| 1723 static G* |
| 1724 runqsteal(P *p, P *p2) |
| 1725 { |
| 1726 » G *gp, *gp1; |
| 1727 » int32 t, h, s, t2, h2, s2, c, c1; |
| 1728 |
| 1729 » if(p2->runqhead==p2->runqtail) |
| 1730 » » return nil; |
| 1731 » if(p < p2) |
| 1732 » » runtime·lock(p); |
| 1733 » runtime·lock(p2); |
| 1734 » if(p2->runqhead==p2->runqtail) { |
| 1735 » » runtime·unlock(p2); |
| 1736 » » if(p < p2) |
| 1737 » » » runtime·unlock(p); |
| 1738 » » return nil; |
| 1739 » } |
| 1740 » if(p >= p2) |
| 1741 » » runtime·lock(p); |
| 1742 » h = p->runqhead; |
| 1743 » t = p->runqtail; |
| 1744 » s = p->runqsize; |
| 1745 » h2 = p2->runqhead; |
| 1746 » t2 = p2->runqtail; |
| 1747 » s2 = p2->runqsize; |
| 1748 » gp = p2->runq[h2]; |
| 1749 » h2++; |
| 1750 » if(h2 == s2) |
| 1751 » » h2 = 0; |
| 1752 » if(t2 > h2) |
| 1753 » » c = (t2 - h2) / 2; |
| 1754 » else |
| 1755 » » c = (s2 - h2 + t2) / 2; |
| 1756 » c1 = 0; |
| 1757 » for(;;) { |
| 1758 » » if(c1 == c) |
| 1759 » » » break; |
| 1760 » » if(t==h-1 || (h==0 && t==s-1)) |
| 1761 » » » break; |
| 1762 » » if(t2==h2) |
| 1763 » » » break; |
| 1764 » » gp1 = p2->runq[h2]; |
| 1765 » » h2++; |
| 1766 » » if(h2 == s2) |
| 1767 » » » h2 = 0; |
| 1768 » » p->runq[t] = gp1; |
| 1769 » » t++; |
| 1770 » » if(t==s) |
| 1771 » » » t = 0; |
| 1772 » » c1++; |
| 1773 » } |
| 1774 » p->runqtail = t; |
| 1775 » p2->runqhead = h2; |
| 1776 » runtime·unlock(p2); |
| 1777 » runtime·unlock(p); |
| 1778 » return gp; |
| 1779 } |
| 1780 |
| 1781 // Put g on runnable queue. |
| 1782 static void |
| 1783 runqput(P *p, G *gp) |
| 1784 { |
| 1785 » int32 h, t, s; |
| 1786 |
| 1787 » runtime·lock(p); |
| 1788 retry: |
| 1789 » h = p->runqhead; |
| 1790 » t = p->runqtail; |
| 1791 » s = p->runqsize; |
| 1792 » if(t==h-1 || (h==0 && t==s-1)) { |
| 1793 » » runqgrow(p); |
| 1794 » » goto retry; |
| 1795 » } |
| 1796 » p->runq[t] = gp; |
| 1797 » t++; |
| 1798 » if(t==s) |
| 1799 » » t = 0; |
| 1800 » p->runqtail = t; |
| 1801 » runtime·unlock(p); |
| 1802 } |
| 1803 |
| 1804 // Get g from runnable queue. |
| 1805 static G* |
| 1806 runqget(P *p) |
| 1807 { |
| 1808 » G *gp; |
| 1809 » int32 t, h, s; |
| 1810 |
| 1811 » if(p->runqhead==p->runqtail) |
| 1812 » » return nil; |
| 1813 » runtime·lock(p); |
| 1814 » h = p->runqhead; |
| 1815 » t = p->runqtail; |
| 1816 » s = p->runqsize; |
| 1817 » if(t==h) { |
| 1818 » » runtime·unlock(p); |
| 1819 » » return nil; |
| 1820 » } |
| 1821 » gp = p->runq[h]; |
| 1822 » h++; |
| 1823 » if(h==s) |
| 1824 » » h = 0; |
| 1825 » p->runqhead = h; |
| 1826 » runtime·unlock(p); |
| 1827 » return gp; |
| 1828 } |
| 1829 |
| 1830 // Put on `m' list. Sched must be locked. |
| 1831 static void |
| 1832 mput(M *mp) |
| 1833 { |
| 1834 » mp->schedlink = runtime·sched.mhead; |
| 1835 » runtime·sched.mhead = mp; |
| 1836 » runtime·sched.mwait++; |
| 1837 } |
| 1838 |
| 1839 // Sched must be locked. |
| 1840 static M* |
| 1841 mget(void) |
| 1842 { |
| 1843 » M *mp; |
| 1844 |
| 1845 » if((mp = runtime·sched.mhead) != nil){ |
| 1846 » » runtime·sched.mhead = mp->schedlink; |
| 1847 » » runtime·sched.mwait--; |
| 1848 » } |
| 1849 » return mp; |
| 1850 } |
| 1851 |
| 1852 // Put on gfree list. |
| 1853 static void |
| 1854 gfput(P *p, G *gp) |
| 1855 { |
| 1856 » if(gp->stackguard - StackGuard != gp->stack0) |
| 1857 » » runtime·throw("invalid stack in gfput"); |
| 1858 » gp->schedlink = p->gfree; |
| 1859 » p->gfree = gp; |
| 1860 » p->gfreecnt++; |
| 1861 » if(p->gfreecnt >= 64) { |
| 1862 » » runtime·lock(&runtime·sched.gflock); |
| 1863 » » while(p->gfreecnt >= 32) { |
| 1864 » » » p->gfreecnt--; |
| 1865 » » » gp = p->gfree; |
| 1866 » » » p->gfree = gp->schedlink; |
| 1867 » » » gp->schedlink = runtime·sched.gfree; |
| 1868 » » » runtime·sched.gfree = gp; |
| 1869 » » } |
| 1870 » » runtime·unlock(&runtime·sched.gflock); |
| 1871 » } |
| 1872 } |
| 1873 |
| 1874 // Get from gfree list. |
| 1875 static G* |
| 1876 gfget(P *p) |
| 1877 { |
| 1878 » G *gp; |
| 1879 |
| 1880 retry: |
| 1881 » gp = p->gfree; |
| 1882 » if(gp == nil && runtime·sched.gfree) { |
| 1883 » » runtime·lock(&runtime·sched.gflock); |
| 1884 » » while(p->gfreecnt < 32 && runtime·sched.gfree) { |
| 1885 » » » p->gfreecnt++; |
| 1886 » » » gp = runtime·sched.gfree; |
| 1887 » » » runtime·sched.gfree = gp->schedlink; |
| 1888 » » » gp->schedlink = p->gfree; |
| 1889 » » » p->gfree = gp; |
| 1890 » » } |
| 1891 » » runtime·unlock(&runtime·sched.gflock); |
| 1892 » » goto retry; |
| 1893 » } |
| 1894 » if(gp) { |
| 1895 » » p->gfree = gp->schedlink; |
| 1896 » » p->gfreecnt--; |
| 1897 » } |
| 1898 » return gp; |
| 1899 } |
LEFT | RIGHT |