LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "defs_GOOS_GOARCH.h" | 7 #include "defs_GOOS_GOARCH.h" |
8 #include "malloc.h" | 8 #include "malloc.h" |
9 #include "os_GOOS.h" | 9 #include "os_GOOS.h" |
10 #include "stack.h" | 10 #include "stack.h" |
| 11 #include "race.h" |
| 12 #include "type.h" |
| 13 |
| 14 // TODO(dvyukov): if a thread w/o mcache catches a signal (in particular SIGABOR
T), |
| 15 // then it can't print dump. |
11 | 16 |
12 enum { maxgomaxprocs = 1<<10 }; | 17 enum { maxgomaxprocs = 1<<10 }; |
13 | 18 #define LOG if(0) runtime·printf |
14 enum { debug = 0 }; | |
15 #define LOG if(debug) runtime·printf | |
16 #define LOG1 runtime·printf | 19 #define LOG1 runtime·printf |
17 | 20 #define CHECK(cond, fmt) /*if(cond) {} else { runtime·printf fmt; runtime·throw(
"CHECK"); }*/ |
18 //FIXME: fix the comment. | |
19 | 21 |
20 // Go scheduler | 22 // Go scheduler |
21 // | 23 // |
22 // The go scheduler's job is to match ready-to-run goroutines (`g's) | 24 // The go scheduler's job is to match ready-to-run goroutines (`g's) |
23 // with waiting-for-work schedulers (`m's). If there are ready g's | 25 // with waiting-for-work schedulers (`m's). If there are ready g's |
24 // and no waiting m's, ready() will start a new m running in a new | 26 // and no waiting m's, ready() will start a new m running in a new |
25 // OS thread, so that all ready g's can run simultaneously, up to a limit. | 27 // OS thread, so that all ready g's can run simultaneously, up to a limit. |
26 // For now, m's never go away. | 28 // For now, m's never go away. |
27 // | 29 // |
28 // By default, Go keeps only one kernel thread (m) running user code | 30 // By default, Go keeps only one kernel thread (m) running user code |
29 // at a single time; other threads may be blocked in the operating system. | 31 // at a single time; other threads may be blocked in the operating system. |
30 // Setting the environment variable $GOMAXPROCS or calling | 32 // Setting the environment variable $GOMAXPROCS or calling |
31 // runtime.GOMAXPROCS() will change the number of user threads | 33 // runtime.GOMAXPROCS() will change the number of user threads |
32 // allowed to execute simultaneously. $GOMAXPROCS is thus an | 34 // allowed to execute simultaneously. $GOMAXPROCS is thus an |
33 // approximation of the maximum number of cores to use. | 35 // approximation of the maximum number of cores to use. |
34 // | 36 // |
35 // Even a program that can run without deadlock in a single process | 37 // Even a program that can run without deadlock in a single process |
36 // might use more m's if given the chance. For example, the prime | 38 // might use more m's if given the chance. For example, the prime |
37 // sieve will use as many m's as there are primes (up to runtime·sched.mmax), | 39 // sieve will use as many m's as there are primes (up to $GOMAXPROCS), |
38 // allowing different stages of the pipeline to execute in parallel. | 40 // allowing different stages of the pipeline to execute in parallel. |
39 // We could revisit this choice, only kicking off new m's for blocking | |
40 // system calls, but that would limit the amount of parallel computation | |
41 // that go would try to do. | |
42 // | |
43 // In general, one could imagine all sorts of refinements to the | |
44 // scheduler, but the goal now is just to get something working on | |
45 // Linux and OS X. | |
46 | 41 |
47 typedef struct Sched Sched; | 42 typedef struct Sched Sched; |
48 struct Sched { | 43 struct Sched { |
49 Lock; | 44 Lock; |
50 | 45 |
51 M* mhead; // m's waiting for work | 46 M* mhead; // m's waiting for work |
52 int32 mwait; // number of m's waiting for work | 47 int32 mwait; // number of m's waiting for work |
53 int32 mcount; // number of m's that have been created | 48 int32 mcount; // number of m's that have been created |
54 | 49 |
| 50 P* pidle; // idle P's |
| 51 int32 npidle; |
| 52 |
| 53 G* runqhead; |
| 54 G* runqtail; |
| 55 int32 runqsize; |
| 56 |
| 57 Lock gflock; |
| 58 G* gfree; |
| 59 int32 goidseq; |
| 60 |
| 61 int32 stopwait; |
| 62 Note stopnote; |
| 63 int32 sysmonwait; |
| 64 Note sysmonnote; |
| 65 |
55 int32 profilehz; // cpu profiling rate | 66 int32 profilehz; // cpu profiling rate |
56 | 67 |
57 bool init; // running initialization | 68 bool init; // running initialization |
58 bool lockmain; // init called runtime.LockOSThread | 69 bool lockmain; // init called runtime.LockOSThread |
59 bool bootstrap; | |
60 }; | 70 }; |
61 | 71 |
62 Sched runtime·sched; | 72 Sched runtime·sched; |
63 int32 runtime·gomaxprocs; | 73 int32 runtime·gomaxprocs; |
64 bool runtime·singleproc; | 74 bool runtime·singleproc; |
65 bool runtime·iscgo; | 75 bool runtime·iscgo; |
66 int32 runtime·gcwaiting; | 76 int32 runtime·gcwaiting; |
67 M» » runtime·m0; | 77 M» runtime·m0; |
68 G» » runtime·g0;» // idle goroutine for m0 | 78 G» runtime·g0;» // idle goroutine for m0 |
| 79 static int32» newprocs; |
69 | 80 |
70 // Keep trace of scavenger's goroutine for deadlock detection. | 81 // Keep trace of scavenger's goroutine for deadlock detection. |
71 static G *scvg; | 82 static G *scvg; |
72 | 83 |
73 // Scheduling helpers. Sched must be locked. | 84 // Scheduling helpers. Sched must be locked. |
74 static void gput(P*, G*);» // put/get on ghead/gtail | 85 static void runqput(P*, G*);» // put/get on ghead/gtail |
75 static G* gget(P*); | 86 static G* runqget(P*); |
| 87 static void runqgrow(P*); |
| 88 static G* runqsteal(P*, P*); |
| 89 static void globrunqput(G*); |
| 90 static G* globrunqget(void); |
76 static void mput(M*); // put/get on mhead | 91 static void mput(M*); // put/get on mhead |
77 static M* mget(void); | 92 static M* mget(void); |
78 static void gfput(P*, G*); // put/get on gfree | 93 static void gfput(P*, G*); // put/get on gfree |
79 static G* gfget(P*); | 94 static G* gfget(P*); |
80 static void mcommoninit(M*); | 95 static void mcommoninit(M*); |
81 static void schedule(G*); | 96 static void schedule(void); |
82 static void procresize(void); | 97 static void procresize(int32); |
83 static void entergo(M*, P*); | 98 static void entergo(M*, P*); |
84 static void leavego(M*, uint32); | 99 static P* releasep(void); |
85 static void newm(void); | 100 static M* newm(void(*)(void), P*, bool); |
86 static void goidle(void); | 101 static void goidle(void); |
87 | 102 static void mstop(void); |
88 static void | 103 static void initgstack(G*, byte*, int32); |
89 outputstats(void) | 104 static void sysmon(void); |
90 { | 105 static void inject(G*, int32*, int32*); |
91 » M *mp; | 106 static P* pidleget(void); |
92 » SchedStats s; | 107 static void pidleput(P*); |
93 » int32 i; | |
94 » uint64 *src, *dst; | |
95 | |
96 » runtime·memclr((byte*)&s, sizeof(s)); | |
97 » for(mp=runtime·allm; mp; mp=mp->alllink) { | |
98 » » src = (uint64*)&mp->schedstats; | |
99 » » dst = (uint64*)&s; | |
100 » » for(i=0; i<sizeof(s)/sizeof(uint64); i++) | |
101 » » » dst[i] += src[i]; | |
102 » } | |
103 » runtime·printf("SchedStats:\n"); | |
104 » runtime·printf("sysexitfast %D\n", s.sysexitfast); | |
105 » runtime·printf("sysexitslow %D\n", s.sysexitslow); | |
106 } | |
107 | 108 |
108 // The bootstrap sequence is: | 109 // The bootstrap sequence is: |
109 // | 110 // |
110 // call osinit | 111 // call osinit |
111 // call schedinit | 112 // call schedinit |
112 // make & queue new G | 113 // make & queue new G |
113 // call runtime·mstart | 114 // call runtime·mstart |
114 // | 115 // |
115 // The new G calls runtime·main. | 116 // The new G calls runtime·main. |
116 void | 117 void |
117 runtime·schedinit(void) | 118 runtime·schedinit(void) |
118 { | 119 { |
119 » int32 n; | 120 » int32 n, procs; |
120 byte *p; | 121 byte *p; |
121 | 122 |
122 LOG("%d: runtime·schedinit\n", m->id); | 123 LOG("%d: runtime·schedinit\n", m->id); |
123 m->nomemprof++; | 124 m->nomemprof++; |
124 runtime·mallocinit(); | 125 runtime·mallocinit(); |
| 126 m->stackalloc = runtime·malloc(sizeof(*m->stackalloc)); //!!! it may be
GCed |
125 mcommoninit(m); | 127 mcommoninit(m); |
| 128 if(runtime·gsignalstk) |
| 129 m->gsignal = runtime·malg(runtime·gsignalstk); |
126 | 130 |
127 runtime·goargs(); | 131 runtime·goargs(); |
128 runtime·goenvs(); | 132 runtime·goenvs(); |
129 | 133 |
130 // For debugging: | 134 // For debugging: |
131 // Allocate internal symbol table representation now, | 135 // Allocate internal symbol table representation now, |
132 // so that we don't need to call malloc when we crash. | 136 // so that we don't need to call malloc when we crash. |
133 // runtime·findfunc(0); | 137 // runtime·findfunc(0); |
134 | 138 |
135 » runtime·sched.bootstrap = true; | 139 » procs = 1; |
136 » runtime·gomaxprocs = 1; | |
137 p = runtime·getenv("GOMAXPROCS"); | 140 p = runtime·getenv("GOMAXPROCS"); |
138 if(p != nil && (n = runtime·atoi(p)) > 0) { | 141 if(p != nil && (n = runtime·atoi(p)) > 0) { |
139 if(n > maxgomaxprocs) | 142 if(n > maxgomaxprocs) |
140 n = maxgomaxprocs; | 143 n = maxgomaxprocs; |
141 » » runtime·gomaxprocs = n; | 144 » » procs = n; |
142 } | 145 } |
143 runtime·allp = (P**)runtime·malloc((maxgomaxprocs+1)*sizeof(runtime·allp
[0])); | 146 runtime·allp = (P**)runtime·malloc((maxgomaxprocs+1)*sizeof(runtime·allp
[0])); |
144 » procresize(); | 147 » procresize(procs); |
145 | 148 |
146 mstats.enablegc = 1; | 149 mstats.enablegc = 1; |
147 m->nomemprof--; | 150 m->nomemprof--; |
| 151 |
| 152 if(raceenabled) |
| 153 runtime·raceinit(); |
148 } | 154 } |
149 | 155 |
150 extern void main·init(void); | 156 extern void main·init(void); |
151 extern void main·main(void); | 157 extern void main·main(void); |
152 | 158 |
153 // The main goroutine. | 159 // The main goroutine. |
154 void | 160 void |
155 runtime·main(void) | 161 runtime·main(void) |
156 { | 162 { |
157 LOG("%d: runtime·main\n", m->id); | 163 LOG("%d: runtime·main\n", m->id); |
158 » runtime·sched.bootstrap = false; | 164 |
| 165 » //TODO(dvyukov): block signals because that thread can't handle them |
| 166 » newm(sysmon, nil, false); |
| 167 |
159 // Lock the main goroutine onto this, the main OS thread, | 168 // Lock the main goroutine onto this, the main OS thread, |
160 // during initialization. Most programs won't care, but a few | 169 // during initialization. Most programs won't care, but a few |
161 // do require certain calls to be made by the main thread. | 170 // do require certain calls to be made by the main thread. |
162 // Those can arrange for main.main to run in the main thread | 171 // Those can arrange for main.main to run in the main thread |
163 // by calling runtime.LockOSThread during initialization | 172 // by calling runtime.LockOSThread during initialization |
164 // to preserve the lock. | 173 // to preserve the lock. |
165 runtime·LockOSThread(); | 174 runtime·LockOSThread(); |
166 runtime·sched.init = true; | 175 runtime·sched.init = true; |
167 if(m != &runtime·m0) | 176 if(m != &runtime·m0) |
168 runtime·throw("runtime·main not on m0"); | 177 runtime·throw("runtime·main not on m0"); |
169 scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runti
me·main); | 178 scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runti
me·main); |
170 main·init(); | 179 main·init(); |
171 runtime·sched.init = false; | 180 runtime·sched.init = false; |
172 if(!runtime·sched.lockmain) | 181 if(!runtime·sched.lockmain) |
173 runtime·UnlockOSThread(); | 182 runtime·UnlockOSThread(); |
174 | 183 |
175 main·main(); | 184 main·main(); |
176 » outputstats(); | 185 » if(raceenabled) |
| 186 » » runtime·racefini(); |
177 runtime·exit(0); | 187 runtime·exit(0); |
178 for(;;) | 188 for(;;) |
179 *(int32*)runtime·main = 0; | 189 *(int32*)runtime·main = 0; |
180 } | 190 } |
181 | 191 |
182 void | 192 void |
183 runtime·goroutineheader(G *gp) | 193 runtime·goroutineheader(G *gp) |
184 { | 194 { |
185 int8 *status; | 195 int8 *status; |
186 | 196 |
(...skipping 13 matching lines...) Expand all Loading... |
200 case Gwaiting: | 210 case Gwaiting: |
201 if(gp->waitreason) | 211 if(gp->waitreason) |
202 status = gp->waitreason; | 212 status = gp->waitreason; |
203 else | 213 else |
204 status = "waiting"; | 214 status = "waiting"; |
205 break; | 215 break; |
206 default: | 216 default: |
207 status = "???"; | 217 status = "???"; |
208 break; | 218 break; |
209 } | 219 } |
210 » runtime·printf("goroutine %p [%s]:\n", gp, status); | 220 » runtime·printf("goroutine %D [%s]:\n", gp->goid, status); |
211 } | 221 } |
212 | 222 |
213 void | 223 void |
214 runtime·tracebackothers(G *me) | 224 runtime·tracebackothers(G *me) |
215 { | 225 { |
216 G *gp; | 226 G *gp; |
217 » P *p, **pp; | 227 |
218 | 228 » for(gp = runtime·allg; gp != nil; gp = gp->alllink) { |
219 » for(pp=runtime·allp; p=*pp; pp++) { | 229 » » if(gp == me || gp->status == Gdead) |
220 » » for(gp = p->allg; gp != nil; gp = gp->alllink) { | 230 » » » continue; |
221 » » » if(gp == me || gp->status == Gdead) | 231 » » runtime·printf("\n"); |
222 » » » » continue; | 232 » » runtime·goroutineheader(gp); |
223 » » » runtime·printf("\n"); | 233 » » runtime·traceback(gp->sched.pc, (byte*)gp->sched.sp, 0, gp); |
224 » » » runtime·goroutineheader(gp); | |
225 » » » runtime·traceback(gp->sched.pc, (byte*)gp->sched.sp, 0,
gp); | |
226 » » } | |
227 } | 234 } |
228 } | 235 } |
229 | 236 |
230 static void | 237 static void |
231 mcommoninit(M *mp) | 238 mcommoninit(M *mp) |
232 { | 239 { |
| 240 runtime·lock(&runtime·sched); |
233 mp->id = runtime·sched.mcount++; | 241 mp->id = runtime·sched.mcount++; |
234 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); | 242 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); |
235 mp->stackalloc = runtime·malloc(sizeof(*mp->stackalloc)); | |
236 runtime·FixAlloc_Init(mp->stackalloc, FixedStack, runtime·SysAlloc, nil,
nil); | 243 runtime·FixAlloc_Init(mp->stackalloc, FixedStack, runtime·SysAlloc, nil,
nil); |
237 | 244 |
238 » runtime·callers(1, mp->createstack, nelem(mp->createstack)); | 245 » // does it allocate? |
| 246 » //runtime·callers(1, mp->createstack, nelem(mp->createstack)); |
239 | 247 |
240 // Add to runtime·allm so garbage collector doesn't free m | 248 // Add to runtime·allm so garbage collector doesn't free m |
241 // when it is just in a register or thread-local storage. | 249 // when it is just in a register or thread-local storage. |
242 mp->alllink = runtime·allm; | 250 mp->alllink = runtime·allm; |
243 // runtime·NumCgoCall() iterates over allm w/o locks, | 251 // runtime·NumCgoCall() iterates over allm w/o locks, |
244 // so we need to publish it safely. | 252 // so we need to publish it safely. |
245 runtime·atomicstorep(&runtime·allm, mp); | 253 runtime·atomicstorep(&runtime·allm, mp); |
246 } | 254 » LOG("%d: mcommoninit %d m=%p stackalloc=%p\n", m->id, mp->id, mp, mp->st
ackalloc); |
247 | 255 » runtime·unlock(&runtime·sched); |
248 // Put g on runnable queue. P is locked. | 256 } |
249 static void | 257 |
250 gput(P *p, G *gp) | 258 // Mark g ready to run. |
251 { | 259 void |
252 » gp->schedlink = nil; | 260 runtime·ready(G *gp) |
253 » if(p->ghead) | 261 { |
254 » » p->gtail->schedlink = gp; | 262 » P *p; |
255 » else | |
256 » » p->ghead = gp; | |
257 » p->gtail = gp; | |
258 » p->gwait++; | |
259 } | |
260 | |
261 // Get g from runnable queue. P is locked. | |
262 static G* | |
263 gget(P *p) | |
264 { | |
265 » G *gp; | |
266 | |
267 » gp = p->ghead; | |
268 » if(gp) { | |
269 » » p->ghead = gp->schedlink; | |
270 » » if(p->ghead == nil) | |
271 » » » p->gtail = nil; | |
272 » » p->gwait--; | |
273 » } | |
274 » return gp; | |
275 } | |
276 | |
277 // Put on `m' list. Sched must be locked. | |
278 static void | |
279 mput(M *mp) | |
280 { | |
281 » mp->schedlink = runtime·sched.mhead; | |
282 » runtime·sched.mhead = mp; | |
283 » runtime·sched.mwait++; | |
284 } | |
285 | |
286 // Sched must be locked. | |
287 static M* | |
288 mget(void) | |
289 { | |
290 M *mp; | 263 M *mp; |
291 | 264 |
292 if((mp = runtime·sched.mhead) != nil){ | |
293 runtime·sched.mhead = mp->schedlink; | |
294 runtime·sched.mwait--; | |
295 } | |
296 return mp; | |
297 } | |
298 | |
299 // Mark g ready to run. | |
300 void | |
301 runtime·ready(G *gp) | |
302 { | |
303 if(gp->m) | 265 if(gp->m) |
304 runtime·throw("bad g->m in ready"); | 266 runtime·throw("bad g->m in ready"); |
305 | 267 |
306 // Mark runnable. | 268 // Mark runnable. |
307 if(gp->status == Grunnable || gp->status == Grunning) { | 269 if(gp->status == Grunnable || gp->status == Grunning) { |
308 » » runtime·printf("goroutine %p has status %d\n", gp, gp->status); | 270 » » runtime·printf("goroutine %D has status %d\n", gp->goid, gp->sta
tus); |
309 runtime·throw("bad g->status in ready"); | 271 runtime·throw("bad g->status in ready"); |
310 } | 272 } |
311 gp->status = Grunnable; | 273 gp->status = Grunnable; |
312 » runtime·lock(m->p); | 274 » runqput(m->p, gp); |
313 » gput(m->p, gp); | 275 » if(runtime·sched.pidle) { |
314 » runtime·unlock(m->p); | 276 » » runtime·lock(&runtime·sched); |
315 } | 277 » » p = pidleget(); |
316 | 278 » » if(p) { |
| 279 » » » mp = mget(); |
| 280 » » » runtime·unlock(&runtime·sched); |
| 281 » » » if(mp) { |
| 282 » » » » entergo(mp, p); |
| 283 » » » » runtime·notewakeup(&mp->park); |
| 284 » » » } else { |
| 285 » » » » newm(runtime·mstart, p, false); |
| 286 » » » } |
| 287 » » } else |
| 288 » » » runtime·unlock(&runtime·sched); |
| 289 » } |
| 290 } |
| 291 |
| 292 static void |
| 293 munpark(M *mp, P *p) |
| 294 { |
| 295 » if(mp) { |
| 296 » » entergo(mp, p); |
| 297 » » runtime·notewakeup(&mp->park); |
| 298 » } else |
| 299 » » newm(runtime·mstart, p, false); |
| 300 } |
| 301 » » » »······· |
317 int32 | 302 int32 |
318 runtime·gcprocs(void) | 303 runtime·gcprocs(void) |
319 { | 304 { |
320 int32 n; | 305 int32 n; |
321 | 306 |
322 /* | |
323 runtime·lock(&runtime·sched); | 307 runtime·lock(&runtime·sched); |
324 // Figure out how many CPUs to use during GC. | |
325 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. | |
326 n = runtime·gomaxprocs; | 308 n = runtime·gomaxprocs; |
327 if(n > runtime·ncpu) | 309 if(n > runtime·ncpu) |
328 n = runtime·ncpu; | 310 n = runtime·ncpu; |
329 if(n > MaxGcproc) | 311 if(n > MaxGcproc) |
330 n = MaxGcproc; | 312 n = MaxGcproc; |
331 if(n > runtime·sched.mwait+1) // one M is currently running | |
332 n = runtime·sched.mwait+1; | |
333 runtime·unlock(&runtime·sched); | 313 runtime·unlock(&runtime·sched); |
| 314 return n; |
| 315 } |
| 316 |
| 317 void |
| 318 runtime·helpgc(int32 nproc) |
| 319 { |
| 320 M *mp; |
| 321 int32 n, pos; |
| 322 |
| 323 LOG("%d: helpgc(%d)\n", m->id, nproc); |
| 324 runtime·lock(&runtime·sched); |
| 325 pos = 0; |
| 326 for(n = 1; n < nproc; n++) { // one M is currently running |
| 327 if(runtime·allp[pos]->mcache == m->mcache) |
| 328 pos++; |
| 329 mp = mget(); |
| 330 if(mp == nil) { |
| 331 runtime·unlock(&runtime·sched); |
| 332 newm(runtime·mstart, runtime·allp[pos], true); |
| 333 runtime·lock(&runtime·sched); |
| 334 pos++; |
| 335 continue; |
| 336 } |
| 337 mp->helpgc = 1; |
| 338 mp->mcache = runtime·allp[pos]->mcache; |
| 339 pos++; |
| 340 LOG("%d: helpgc wake %d\n", m->id, mp->id); |
| 341 runtime·notewakeup(&mp->park); |
| 342 } |
| 343 runtime·unlock(&runtime·sched); |
| 344 } |
| 345 |
| 346 void |
| 347 runtime·stoptheworld(void) |
| 348 { |
| 349 int32 i; |
| 350 uint32 s; |
| 351 P *p; |
| 352 bool wait; |
| 353 |
| 354 LOG("%d: stoptheworld\n", m->id); |
| 355 runtime·lock(&runtime·sched); |
| 356 runtime·gcwaiting = 1; |
| 357 runtime·sched.stopwait = runtime·gomaxprocs; |
| 358 m->p->status = Plocked; |
| 359 runtime·sched.stopwait--; |
| 360 for(i=0; i<runtime·gomaxprocs; i++) { |
| 361 s = runtime·allp[i]->status; |
| 362 if(s == Psyscall && runtime·cas(&runtime·allp[i]->status, s, Plo
cked)) { |
| 363 LOG(" acquired syscall %d\n", i); |
| 364 runtime·sched.stopwait--; |
| 365 } |
| 366 } |
| 367 while(runtime·sched.pidle) { |
| 368 p = pidleget(); |
| 369 p->status = Plocked; |
| 370 runtime·sched.stopwait--; |
| 371 } |
| 372 CHECK(runtime·sched.stopwait >= 0, ("")); |
| 373 wait = runtime·sched.stopwait > 0; |
| 374 runtime·unlock(&runtime·sched); |
| 375 if(wait) { |
| 376 runtime·notesleep(&runtime·sched.stopnote); |
| 377 runtime·noteclear(&runtime·sched.stopnote); |
| 378 } |
| 379 LOG("%d: stoptheworld stopped\n", m->id); |
| 380 CHECK(runtime·sched.stopwait == 0, ("stoptheworld: stopwait == %d\n", ru
ntime·sched.stopwait)); |
| 381 for(i=0; i<runtime·gomaxprocs; i++) { |
| 382 CHECK(runtime·allp[i]->status == Plocked, ("stoptheworld: not st
opped (%d)\n", runtime·allp[i]->status)); |
| 383 } |
| 384 } |
| 385 |
| 386 void |
| 387 runtime·starttheworld(void) |
| 388 { |
| 389 //G *gp; |
| 390 //P *p; |
| 391 //M *mp; |
| 392 //int32 n, w; |
| 393 |
| 394 LOG("%d: starttheworld\n", m->id); |
| 395 runtime·gcwaiting = 0; |
| 396 if(newprocs) { |
| 397 procresize(newprocs); |
| 398 newprocs = 0; |
| 399 } else { |
| 400 procresize(runtime·gomaxprocs); |
| 401 } |
| 402 runtime·lock(&runtime·sched); |
| 403 /* |
| 404 gp = runtime·netwait(0, runtime·gomaxprocs); |
| 405 n = w = 0; |
| 406 inject(gp, &w, &n); |
| 407 while(runtime·sched.pidle) { |
| 408 p = pidleget(); |
| 409 mp = mget(); |
| 410 if(mp) { |
| 411 entergo(mp, p); |
| 412 runtime·notewakeup(&mp->park); |
| 413 } else { |
| 414 runtime·unlock(&runtime·sched); |
| 415 newm(runtime·mstart, p, false); |
| 416 runtime·lock(&runtime·sched); |
| 417 } |
| 418 } |
334 */ | 419 */ |
335 » n = 1; | 420 » if(runtime·sched.sysmonwait) { |
336 » return n; | 421 » » runtime·sched.sysmonwait = 0; |
337 } | 422 » » runtime·notewakeup(&runtime·sched.sysmonnote); |
338 | |
339 void | |
340 runtime·helpgc(int32 nproc) | |
341 { | |
342 » USED(&nproc); | |
343 » /* | |
344 » M *mp; | |
345 » int32 n; | |
346 | |
347 » runtime·lock(&runtime·sched); | |
348 » for(n = 1; n < nproc; n++) { // one M is currently running | |
349 » » mp = mget(); | |
350 » » if(mp == nil) | |
351 » » » runtime·throw("runtime·gcprocs inconsistency"); | |
352 » » mp->helpgc = 1; | |
353 » » runtime·notewakeup(&mp->havenextg); | |
354 } | 423 } |
355 runtime·unlock(&runtime·sched); | 424 runtime·unlock(&runtime·sched); |
356 */ | |
357 } | |
358 | |
359 void | |
360 runtime·stoptheworld(void) | |
361 { | |
362 int32 acquired, i; | |
363 uint32 s; | |
364 | |
365 LOG("%d: stoptheworld\n", m->id); | |
366 runtime·gcwaiting = 1; //!!! atomic | |
367 acquired = 1; | |
368 m->p->status = Plocked; | |
369 while(acquired != runtime·np) { | |
370 for(i=0; i<runtime·np; i++) { | |
371 s = runtime·allp[i]->status; | |
372 if(s == Pidle || s == Psyscall) { | |
373 if(runtime·cas(&runtime·allp[i]->status, s, Ploc
ked)) | |
374 acquired++; | |
375 } | |
376 } | |
377 //!!! replace with blocking | |
378 if(acquired != runtime·np) | |
379 runtime·usleep(1); | |
380 } | |
381 } | |
382 | |
383 void | |
384 runtime·starttheworld(void) | |
385 { | |
386 LOG("%d: starttheworld\n", m->id); | |
387 //FIXME: start additional M's | |
388 /* | |
389 int32 max, cur; | |
390 | |
391 cur = runtime·gcprocs(); | |
392 // Figure out how many CPUs GC could possibly use. | |
393 max = runtime·gomaxprocs; | |
394 if(max > runtime·ncpu) | |
395 max = runtime·ncpu; | |
396 if(max > MaxGcproc) | |
397 max = MaxGcproc; | |
398 */ | |
399 | |
400 runtime·gcwaiting = 0; | |
401 procresize(); | |
402 runtime·gosched(); | |
403 } | 425 } |
404 | 426 |
405 // Called to start an M. | 427 // Called to start an M. |
406 void | 428 void |
407 runtime·mstart(void) | 429 runtime·mstart(void) |
408 { | 430 { |
409 // It is used by windows-386 only. Unfortunately, seh needs | 431 // It is used by windows-386 only. Unfortunately, seh needs |
410 // to be located on os stack, and mstart runs on os stack | 432 // to be located on os stack, and mstart runs on os stack |
411 // for both m0 and m. | 433 // for both m0 and m. |
412 SEH seh; | 434 SEH seh; |
413 | 435 » P *p; |
414 » LOG("%d: mstart\n", m->id); | 436 |
| 437 » LOG("%d: mstart m=%p\n", m->id, m); |
415 if(g != m->g0) | 438 if(g != m->g0) |
416 runtime·throw("bad runtime·mstart"); | 439 runtime·throw("bad runtime·mstart"); |
417 | 440 |
418 // Record top of stack for use by mcall. | 441 // Record top of stack for use by mcall. |
419 // Once we call schedule we're never coming back, | 442 // Once we call schedule we're never coming back, |
420 // so other calls can reuse this stack space. | 443 // so other calls can reuse this stack space. |
421 runtime·gosave(&m->g0->sched); | 444 runtime·gosave(&m->g0->sched); |
422 m->g0->sched.pc = (void*)-1; // make sure it is never used | 445 m->g0->sched.pc = (void*)-1; // make sure it is never used |
423 m->seh = &seh; | 446 m->seh = &seh; |
424 runtime·asminit(); | 447 runtime·asminit(); |
425 if(m == &runtime·m0) | |
426 runtime·minitparent(m); | |
427 runtime·minit(); | 448 runtime·minit(); |
428 | 449 |
429 // Install signal handlers; after minit so that minit can | 450 // Install signal handlers; after minit so that minit can |
430 // prepare the thread to be able to handle the signals. | 451 // prepare the thread to be able to handle the signals. |
431 if(m == &runtime·m0) | 452 if(m == &runtime·m0) |
432 runtime·initsig(); | 453 runtime·initsig(); |
433 | 454 |
434 » if(m == &runtime·m0) | 455 » if(m->helpgc) { |
435 » » leavego(m, Pidle); | 456 » » LOG("%d: mstart helpgc\n", m->id); |
436 » else { | 457 » » m->helpgc = 0; |
437 » » //!!! this is to ensure that runtime.main is executed on m0 | 458 » » m->mcache = m->p->mcache; |
438 » » // think of something better | 459 » » runtime·gchelper(); |
439 » » while(runtime·sched.bootstrap) | 460 » » m->mcache = nil; |
440 » » » runtime·usleep(10); | 461 » » m->p = nil; |
441 » } | 462 » » LOG("%d: gchelper end\n", m->id); |
442 » goidle(); | 463 » » mstop(); |
443 » schedule(nil); | 464 » } else if(m != &runtime·m0) { |
| 465 » » p = m->p; |
| 466 » » m->p = nil; |
| 467 » » entergo(m, p); |
| 468 » } |
| 469 » LOG("%d: calling schedule\n", m->id); |
| 470 » schedule(); |
444 | 471 |
445 // TODO(brainman): This point is never reached, because scheduler | 472 // TODO(brainman): This point is never reached, because scheduler |
446 // does not release os threads at the moment. But once this path | 473 // does not release os threads at the moment. But once this path |
447 // is enabled, we must remove our seh here. | 474 // is enabled, we must remove our seh here. |
448 } | 475 } |
449 | 476 |
450 // When running with cgo, we call libcgo_thread_start | 477 // When running with cgo, we call libcgo_thread_start |
451 // to start threads for us so that we can play nicely with | 478 // to start threads for us so that we can play nicely with |
452 // foreign code. | 479 // foreign code. |
453 void (*libcgo_thread_start)(void*); | 480 void (*libcgo_thread_start)(void*); |
454 | 481 |
455 typedef struct CgoThreadStart CgoThreadStart; | 482 typedef struct CgoThreadStart CgoThreadStart; |
456 struct CgoThreadStart | 483 struct CgoThreadStart |
457 { | 484 { |
458 M *m; | 485 M *m; |
459 G *g; | 486 G *g; |
460 void (*fn)(void); | 487 void (*fn)(void); |
461 }; | 488 }; |
462 | 489 |
| 490 static void |
| 491 initgstack(G *newg, byte *stk, int32 stacksize) |
| 492 { |
| 493 newg->stack0 = (uintptr)stk; |
| 494 newg->stackguard = (uintptr)stk + StackGuard; |
| 495 newg->stackbase = (uintptr)stk + stacksize - sizeof(Stktop); |
| 496 runtime·memclr((byte*)newg->stackbase, sizeof(Stktop)); |
| 497 } |
| 498 |
463 // Create a new m. It will start off with a call to runtime·mstart. | 499 // Create a new m. It will start off with a call to runtime·mstart. |
464 static void | 500 static M* |
465 newm(void) | 501 newm(void(*fn)(void), P *p, bool helpgc) |
466 { | 502 { |
467 M *mp; | 503 M *mp; |
| 504 int32 addmem,stksiz, stkoff; |
| 505 //!!!static Type *mtype; // The Go type M |
468 | 506 |
469 LOG("%d: newm\n", m->id); | 507 LOG("%d: newm\n", m->id); |
470 » mp = runtime·malloc(sizeof(M)); | 508 » addmem = sizeof(*mp->stackalloc); |
| 509 » if(runtime·gsignalstk) |
| 510 » » addmem += sizeof(G) + runtime·gsignalstk; |
| 511 » stkoff = sizeof(M) + addmem; |
| 512 » stksiz = StackSystem + (fn == runtime·mstart ? 8192 : 64*1024); |
| 513 » if(!runtime·iscgo && !Windows) |
| 514 » » addmem += stksiz; |
| 515 » //!!! all that is now non-GC, can it break something? |
| 516 » mp = runtime·SysAlloc(sizeof(M) + addmem); |
| 517 » mp->stackalloc = (FixAlloc*)(mp+1); |
| 518 » //!!!if(mtype == nil) { |
| 519 » //!!!» Eface e; |
| 520 » //!!!» runtime·gc_m_ptr(&e); |
| 521 » //!!!» mtype = ((PtrType*)e.type)->elem; |
| 522 » //!!!} |
| 523 » //!!! mp = runtime·cnew(mtype); |
471 mcommoninit(mp); | 524 mcommoninit(mp); |
472 » runtime·minitparent(mp); | 525 » mp->g0 = &mp->g0buf; |
| 526 » mp->p = p; |
| 527 » mp->helpgc = helpgc; |
| 528 » if(runtime·gsignalstk) { |
| 529 » » mp->gsignal = (G*)((byte*)mp+sizeof(*mp)+sizeof(*mp->stackalloc)
); |
| 530 » » initgstack(mp->gsignal, (byte*)(mp->gsignal+1), runtime·gsignals
tk); |
| 531 » } |
473 | 532 |
474 if(runtime·iscgo) { | 533 if(runtime·iscgo) { |
475 CgoThreadStart ts; | 534 CgoThreadStart ts; |
476 | 535 |
477 if(libcgo_thread_start == nil) | 536 if(libcgo_thread_start == nil) |
478 runtime·throw("libcgo_thread_start missing"); | 537 runtime·throw("libcgo_thread_start missing"); |
479 // pthread_create will make us a stack. | 538 // pthread_create will make us a stack. |
480 mp->g0 = runtime·malg(-1); | |
481 ts.m = mp; | 539 ts.m = mp; |
482 ts.g = mp->g0; | 540 ts.g = mp->g0; |
483 » » ts.fn = runtime·mstart; | 541 » » ts.fn = fn; |
484 runtime·asmcgocall(libcgo_thread_start, &ts); | 542 runtime·asmcgocall(libcgo_thread_start, &ts); |
485 } else { | 543 } else { |
486 » » if(Windows) | 544 » » // windows will layout sched stack on os stack |
487 » » » // windows will layout sched stack on os stack | 545 » » if(!Windows) |
488 » » » mp->g0 = runtime·malg(-1); | 546 » » » initgstack(mp->g0, (byte*)mp+stkoff, stksiz); |
489 » » else | 547 » » runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, fn); |
490 » » » mp->g0 = runtime·malg(8192); | 548 » } |
491 » » runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, runtime·
mstart); | 549 » return mp; |
492 » } | |
493 } | 550 } |
494 | 551 |
495 static void | 552 static void |
496 mstop(void) | 553 mstop(void) |
497 { | 554 { |
498 LOG("%d: mstop\n", m->id); | 555 LOG("%d: mstop\n", m->id); |
499 » runtime·noteclear(&m->park); | 556 » CHECK(m->locks == 0, ("")); |
| 557 » CHECK(m->p == nil, ("mstop: p != nil\n")); |
| 558 retry: |
500 runtime·lock(&runtime·sched); | 559 runtime·lock(&runtime·sched); |
501 mput(m); | 560 mput(m); |
502 runtime·unlock(&runtime·sched); | 561 runtime·unlock(&runtime·sched); |
503 runtime·notesleep(&m->park); | 562 runtime·notesleep(&m->park); |
504 » goidle(); | 563 » runtime·noteclear(&m->park); |
| 564 » if(m->helpgc) { |
| 565 » » LOG("%d: gchelper\n", m->id); |
| 566 » » m->helpgc = 0; |
| 567 » » runtime·gchelper(); |
| 568 » » m->mcache = nil; |
| 569 » » LOG("%d: gchelper end\n", m->id); |
| 570 » » goto retry; |
| 571 » } |
| 572 » LOG("%d: mstop wake\n", m->id); |
| 573 » if(m->p == nil) |
| 574 » » runtime·throw("mstop: p == nil"); |
| 575 } |
| 576 |
| 577 // Schedules gp to run on M. Never returns. |
| 578 static void |
| 579 execute(G *gp) |
| 580 { |
| 581 » int32 hz; |
| 582 |
| 583 » LOG("%d: start running goroutine %p\n", m->id, gp); |
| 584 » CHECK(m->locks == 0, ("")); |
| 585 » CHECK(g == m->g0, ("execute: not on g0\n")); |
| 586 » CHECK(m->p != nil, ("execute: no p\n")); |
| 587 » CHECK(gp->status == Grunnable, ("execute: gp=%d gp->status=%d\n", gp->go
id, gp->status)); |
| 588 » CHECK(gp->m == nil, ("execute: gp->m=%p\n", gp->m)); |
| 589 » CHECK(gp->lockedm == nil && m->lockedg == nil || gp->lockedm == m && m->
lockedg == gp, |
| 590 » » ("bad locking: gp->lockedm=%p m->lockedg=%p\n", gp->lockedm, m->
lockedg)); |
| 591 » m->p->tick++; |
| 592 » gp->status = Grunning; |
| 593 » m->curg = gp; |
| 594 » gp->m = m; |
| 595 |
| 596 » // Check whether the profiler needs to be turned on or off. |
| 597 » hz = runtime·sched.profilehz; |
| 598 » if(m->profilehz != hz) |
| 599 » » runtime·resetcpuprofiler(hz); |
| 600 |
| 601 » if(gp->sched.pc == (byte*)runtime·goexit) // kickoff |
| 602 » » runtime·gogocall(&gp->sched, (void(*)(void))gp->entry); |
| 603 » runtime·gogo(&gp->sched, 0); |
505 } | 604 } |
506 | 605 |
507 // One round of scheduler: find a goroutine and run it. | 606 // One round of scheduler: find a goroutine and run it. |
508 // The argument is the goroutine that was running before | 607 // The argument is the goroutine that was running before |
509 // schedule was called, or nil if this is the first call. | 608 // schedule was called, or nil if this is the first call. |
510 // Never returns. | 609 // Never returns. |
511 static void | 610 static void |
512 schedule(G *gp) | 611 schedule(void) |
513 { | 612 { |
514 » int32 hz, i, try; | 613 » int32 i, try; |
| 614 » G *gp, *gp1; |
515 P *p; | 615 P *p; |
| 616 M *mp; |
516 | 617 |
517 LOG("%d: schedule p=%p\n", m->id, m->p); | 618 LOG("%d: schedule p=%p\n", m->id, m->p); |
518 USED(&gp); | 619 USED(&gp); |
519 » if(m->locks != 0) | 620 » CHECK(m->locks == 0, ("schedule: holding locks\n")); |
520 » » runtime·throw("schedule holding locks"); | 621 » CHECK(m->lockedg == nil, ("schedule: locked M\n")); |
521 » if(gp == m->g0) | |
522 » » runtime·throw("schedule of g0"); | |
523 | 622 |
524 top: | 623 top: |
525 if(runtime·gcwaiting) { | 624 if(runtime·gcwaiting) { |
526 » » leavego(m, Pidle); | 625 » » p = releasep(); |
| 626 » » p->status = Plocked; |
| 627 » » runtime·lock(&runtime·sched); |
| 628 » » runtime·sched.stopwait--; |
| 629 » » if(runtime·sched.stopwait == 0) |
| 630 » » » runtime·notewakeup(&runtime·sched.stopnote); |
| 631 » » runtime·unlock(&runtime·sched); |
527 mstop(); | 632 mstop(); |
528 goto top; | 633 goto top; |
529 } | 634 } |
530 | 635 |
531 » runtime·lock(m->p); | 636 » gp = runqget(m->p); |
532 » gp = gget(m->p); | |
533 » runtime·unlock(m->p); | |
534 if(gp == nil) { | 637 if(gp == nil) { |
535 » » for(try=0; try<1; try++) { | 638 » » for(try=0; try<2; try++) { |
536 » » for(i=0; i<runtime·np; i++) { | 639 » » if(runtime·sched.runqsize) { |
537 » » » p = runtime·allp[i]; | 640 » » » runtime·lock(&runtime·sched); |
538 » » » if(p->ghead) { | 641 » » » gp = globrunqget(); |
539 » » » » runtime·lock(p); | 642 » » » if(gp) { |
540 » » » » gp = gget(p); | 643 » » » » while(gp->schedlink != nil) { |
541 » » » » runtime·unlock(p); | 644 » » » » » gp1 = gp; |
| 645 » » » » » gp = gp1->schedlink; |
| 646 » » » » » runqput(m->p, gp1); |
| 647 » » » » } |
542 } | 648 } |
| 649 runtime·unlock(&runtime·sched); |
| 650 if(gp) |
| 651 goto haveg; |
| 652 } |
| 653 for(i=0; i<runtime·gomaxprocs; i++) { |
| 654 if(runtime·gcwaiting) |
| 655 goto top; |
| 656 p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]
; |
| 657 if(p == m->p) |
| 658 gp = runqget(p); |
| 659 else |
| 660 gp = runqsteal(m->p, p); |
543 if(gp) | 661 if(gp) |
544 break; | 662 break; |
545 } | 663 } |
546 » » » if(gp) | 664 » » if(gp) |
547 » » » » break; | 665 » » » break; |
548 » » » //runtime·usleep(20); | 666 » » if(try==0 && runtime·gcwaiting == 0) |
| 667 » » » runtime·osyield(); |
549 } | 668 } |
550 if(gp == nil) { | 669 if(gp == nil) { |
551 » » » leavego(m, Pidle); | 670 » » » p = releasep(); |
| 671 » » » runtime·lock(&runtime·sched); |
| 672 » » » if(runtime·gcwaiting) { |
| 673 » » » » p->status = Plocked; |
| 674 » » » » runtime·sched.stopwait--; |
| 675 » » » » if(runtime·sched.stopwait == 0) |
| 676 » » » » » runtime·notewakeup(&runtime·sched.stopno
te); |
| 677 » » » » runtime·unlock(&runtime·sched); |
| 678 » » » » mstop(); |
| 679 » » » » goto top; |
| 680 » » » } |
| 681 » » » pidleput(p); |
| 682 » » » if(runtime·sched.runqsize) { |
| 683 » » » » p = pidleget(); |
| 684 » » » » runtime·unlock(&runtime·sched); |
| 685 » » » » entergo(m, p); |
| 686 » » » » goto top; |
| 687 » » » } |
| 688 » » » runtime·unlock(&runtime·sched); |
| 689 » » » for(i=0; i<runtime·gomaxprocs; i++) { |
| 690 » » » » p = runtime·allp[i]; |
| 691 » » » » if(p && p->runqhead != p->runqtail) { |
| 692 » » » » » runtime·lock(&runtime·sched); |
| 693 » » » » » p = pidleget(); |
| 694 » » » » » runtime·unlock(&runtime·sched); |
| 695 » » » » » if(p) { |
| 696 » » » » » » entergo(m, p); |
| 697 » » » » » » goto top; |
| 698 » » » » » } |
| 699 » » » » » break; |
| 700 » » » » } |
| 701 » » » } |
552 mstop(); | 702 mstop(); |
553 goto top; | 703 goto top; |
554 } | 704 } |
555 } | 705 } |
556 | 706 |
557 » LOG("%d: start running goroutine %p\n", m->id, gp); | 707 haveg: |
558 » m->p->tick++; | 708 » if(gp->lockedm) { |
559 » gp->status = Grunning; | 709 » » mp = gp->lockedm; |
560 » m->curg = gp; | 710 » » p = releasep(); |
561 » gp->m = m; | 711 » » entergo(mp, p); |
562 | 712 » » runtime·notewakeup(&mp->park); |
563 » // Check whether the profiler needs to be turned on or off. | 713 » » mstop(); |
564 » hz = runtime·sched.profilehz; | 714 » » goto top; |
565 » if(m->profilehz != hz) | 715 » } |
566 » » runtime·resetcpuprofiler(hz); | 716 |
567 | 717 » execute(gp); |
568 » if(gp->sched.pc == (byte*)runtime·goexit) {» // kickoff | |
569 » » runtime·gogocall(&gp->sched, (void(*)(void))gp->entry); | |
570 » } | |
571 » runtime·gogo(&gp->sched, 0); | |
572 } | 718 } |
573 | 719 |
574 static void | 720 static void |
575 park0(G *gp) | 721 park0(G *gp) |
576 { | 722 { |
| 723 P *p; |
| 724 M *mp; |
| 725 |
577 USED(&gp); | 726 USED(&gp); |
| 727 if(m->lockedg) { |
| 728 p = releasep(); |
| 729 if(m->waitunlockf) { |
| 730 m->waitunlockf(m->waitlock); |
| 731 m->waitunlockf = nil; |
| 732 } |
| 733 // After this point another thread may schedule gp on m again. |
| 734 // Schedule another M to run P. |
| 735 runtime·lock(&runtime·sched); |
| 736 mp = mget(); |
| 737 runtime·unlock(&runtime·sched); |
| 738 munpark(mp, p); |
| 739 // Wait until another thread schedules gp and so m again. |
| 740 runtime·notesleep(&m->park); |
| 741 runtime·noteclear(&m->park); |
| 742 execute(gp); // Never returns. |
| 743 } |
578 if(m->waitunlockf) { | 744 if(m->waitunlockf) { |
579 m->waitunlockf(m->waitlock); | 745 m->waitunlockf(m->waitlock); |
580 m->waitunlockf = nil; | 746 m->waitunlockf = nil; |
581 } | 747 } |
582 » schedule(nil); | 748 » schedule(); |
583 } | 749 } |
584 | 750 |
585 // Atomically parks g and unlocks the lock. | 751 // Puts the current goroutine into a waiting state and unlocks the lock. |
586 void | 752 // The goroutine can be made runnable again by calling runtime·ready(gp). |
587 runtime·park(void *l, void(*unlockf)(void*), int8 *reason) | 753 void |
| 754 runtime·park(void(*unlockf)(Lock*), Lock *l, int8 *reason) |
588 { | 755 { |
589 LOG("%d: park l=%p reason=%s\n", m->id, l, reason); | 756 LOG("%d: park l=%p reason=%s\n", m->id, l, reason); |
590 » if(g == m->g0) | 757 » CHECK(g != m->g0, ("park of g0\n")); |
591 » » runtime·throw("park of g0"); | |
592 m->waitlock = l; | 758 m->waitlock = l; |
593 m->waitunlockf = unlockf; | 759 m->waitunlockf = unlockf; |
594 g->status = Gwaiting; | 760 g->status = Gwaiting; |
595 g->waitreason = reason; | 761 g->waitreason = reason; |
596 g->m = nil; | 762 g->m = nil; |
597 runtime·mcall(park0); | 763 runtime·mcall(park0); |
598 } | 764 } |
599 | 765 |
600 static void | 766 static void |
601 gosched0(G *gp) | 767 gosched0(G *gp) |
602 { | 768 { |
| 769 P *p; |
| 770 M *mp; |
| 771 |
603 LOG("%d: gosched0 gp=%p\n", m->id, gp); | 772 LOG("%d: gosched0 gp=%p\n", m->id, gp); |
604 gp->status = Grunnable; | 773 gp->status = Grunnable; |
605 gp->m = nil; | 774 gp->m = nil; |
606 » runtime·lock(m->p); | 775 » if(m->lockedg) { |
607 » gput(m->p, gp); | 776 » » p = releasep(); |
608 » runtime·unlock(m->p); | 777 » » runtime·lock(&runtime·sched); |
609 » schedule(nil); | 778 » » globrunqput(gp); |
| 779 » » // After this point another thread may schedule gp on m again. |
| 780 » » // Schedule another M to run P. |
| 781 » » mp = mget(); |
| 782 » » runtime·unlock(&runtime·sched); |
| 783 » » munpark(mp, p); |
| 784 » » // Wait until another thread schedules gp and so m again. |
| 785 » » runtime·notesleep(&m->park); |
| 786 » » runtime·noteclear(&m->park); |
| 787 » » execute(gp); // Never returns. |
| 788 » } |
| 789 » runtime·lock(&runtime·sched); |
| 790 » globrunqput(gp); |
| 791 » runtime·unlock(&runtime·sched); |
| 792 » schedule(); |
610 } | 793 } |
611 | 794 |
612 void | 795 void |
613 runtime·gosched(void) | 796 runtime·gosched(void) |
614 { | 797 { |
615 runtime·mcall(gosched0); | 798 runtime·mcall(gosched0); |
616 } | 799 } |
617 | 800 |
618 // On g0. | 801 // On g0. |
619 static void | 802 static void |
620 goexit0(G *gp) | 803 goexit0(G *gp) |
621 { | 804 { |
622 gp->status = Gdead; | 805 gp->status = Gdead; |
623 gp->m = nil; | 806 gp->m = nil; |
624 » if(gp->lockedm) { | 807 » gp->lockedm = nil; |
625 » » gp->lockedm = nil; | 808 » m->lockedg = nil; |
626 » » m->lockedg = nil; | |
627 » } | |
628 runtime·unwindstack(gp, nil); | 809 runtime·unwindstack(gp, nil); |
629 gfput(m->p, gp); | 810 gfput(m->p, gp); |
630 » schedule(nil); | 811 » schedule(); |
631 } | 812 } |
632 | 813 |
633 void | 814 void |
634 runtime·goexit(void) | 815 runtime·goexit(void) |
635 { | 816 { |
636 runtime·mcall(goexit0); | 817 runtime·mcall(goexit0); |
637 } | 818 } |
638 | 819 |
639 // The goroutine g is about to enter a system call. | 820 // The goroutine g is about to enter a system call. |
640 // Record that it's not using the cpu anymore. | 821 // Record that it's not using the cpu anymore. |
641 // This is called only from the go syscall library and cgocall, | 822 // This is called only from the go syscall library and cgocall, |
642 // not from the low-level system calls used by the runtime. | 823 // not from the low-level system calls used by the runtime. |
643 // | 824 // |
644 // Entersyscall cannot split the stack: the runtime·gosave must | 825 // Entersyscall cannot split the stack: the runtime·gosave must |
645 // make g->sched refer to the caller's stack segment, because | 826 // make g->sched refer to the caller's stack segment, because |
646 // entersyscall is going to return immediately after. | 827 // entersyscall is going to return immediately after. |
647 #pragma textflag 7 | 828 #pragma textflag 7 |
648 void | 829 void |
649 runtime·entersyscall(void) | 830 runtime·entersyscall(void) |
650 { | 831 { |
651 » LOG("%d: entersyscall g=%p\n", m->id, g); | 832 » P *p; |
652 » //m->sysenterticks = runtime·cputicks(); | 833 » M *mp; |
| 834 |
| 835 » LOG("%d: entersyscall g=%p p=%p\n", m->id, g, m->p); |
653 if(m->profilehz > 0) | 836 if(m->profilehz > 0) |
654 runtime·setprof(false); | 837 runtime·setprof(false); |
655 | 838 |
656 // Leave SP around for gc and traceback. | 839 // Leave SP around for gc and traceback. |
657 runtime·gosave(&g->sched); | 840 runtime·gosave(&g->sched); |
658 g->gcsp = g->sched.sp; | 841 g->gcsp = g->sched.sp; |
659 g->gcstack = g->stackbase; | 842 g->gcstack = g->stackbase; |
660 g->gcguard = g->stackguard; | 843 g->gcguard = g->stackguard; |
661 g->status = Gsyscall; | 844 g->status = Gsyscall; |
662 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { | 845 if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) { |
663 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", | 846 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", |
664 // g->gcsp, g->gcguard-StackGuard, g->gcstack); | 847 // g->gcsp, g->gcguard-StackGuard, g->gcstack); |
665 runtime·throw("entersyscall"); | 848 runtime·throw("entersyscall"); |
666 } | 849 } |
667 | 850 |
| 851 if(m->blockingsyscall) { |
| 852 m->blockingsyscall = false; |
| 853 p = releasep(); |
| 854 runtime·lock(&runtime·sched); |
| 855 mp = mget(); |
| 856 runtime·unlock(&runtime·sched); |
| 857 munpark(mp, p); |
| 858 return; |
| 859 } |
| 860 |
668 m->mcache = nil; | 861 m->mcache = nil; |
669 m->p->tick++; | |
670 m->p->m = nil; | 862 m->p->m = nil; |
671 » m->p->status = Psyscall; //!!! store-release | 863 » runtime·atomicstore(&m->p->status, Psyscall); |
| 864 » if(runtime·gcwaiting) { |
| 865 » » runtime·lock(&runtime·sched); |
| 866 » » if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psy
scall, Plocked)) { |
| 867 » » » runtime·sched.stopwait--; |
| 868 » » » if(runtime·sched.stopwait == 0) |
| 869 » » » » runtime·notewakeup(&runtime·sched.stopnote); |
| 870 » » } |
| 871 » » runtime·unlock(&runtime·sched); |
| 872 » } |
| 873 } |
| 874 |
| 875 #pragma textflag 7 |
| 876 void |
| 877 runtime·entersyscallblock(void) |
| 878 { |
| 879 » m->blockingsyscall = true; |
| 880 » runtime·entersyscall(); |
672 } | 881 } |
673 | 882 |
674 static void | 883 static void |
675 exitsyscall0(G *gp) | 884 exitsyscall0(G *gp) |
676 { | 885 { |
| 886 P *p; |
| 887 |
677 LOG("%d: exitsyscall0\n", m->id); | 888 LOG("%d: exitsyscall0\n", m->id); |
678 gp->status = Grunnable; | 889 gp->status = Grunnable; |
679 gp->m = nil; | 890 gp->m = nil; |
680 » runtime·lock(runtime·allp[0]); | 891 » CHECK(m->park.waitm == nil, ("exitsyscall0: park is signalled\n")); |
681 » //!!! anything better than put to allp[0]? | 892 » runtime·lock(&runtime·sched); |
682 » gput(runtime·allp[0], gp); | 893 » p = pidleget(); |
683 » runtime·unlock(runtime·allp[0]); | 894 » if(p == nil) |
| 895 » » globrunqput(gp); |
| 896 » runtime·unlock(&runtime·sched); |
| 897 » if(p) { |
| 898 » » entergo(m, p); |
| 899 » » execute(gp); // Never returns. |
| 900 » } |
| 901 » if(m->lockedg) { |
| 902 » » CHECK(m->lockedg == gp, ("exitsyscall0: inconsistent locking\n")
); |
| 903 » » // Wait until another thread schedules gp and so m again. |
| 904 » » runtime·notesleep(&m->park); |
| 905 » » runtime·noteclear(&m->park); |
| 906 » » execute(gp); // Never returns. |
| 907 » } |
684 mstop(); | 908 mstop(); |
685 » schedule(nil); | 909 » schedule(); |
686 } | 910 } |
687 | 911 |
688 // The goroutine g exited its system call. | 912 // The goroutine g exited its system call. |
689 // Arrange for it to run on a cpu again. | 913 // Arrange for it to run on a cpu again. |
690 // This is called only from the go syscall library, not | 914 // This is called only from the go syscall library, not |
691 // from the low-level system calls used by the runtime. | 915 // from the low-level system calls used by the runtime. |
692 void | 916 void |
693 runtime·exitsyscall(void) | 917 runtime·exitsyscall(void) |
694 { | 918 { |
695 uint32 s; | 919 uint32 s; |
| 920 P *p; |
696 | 921 |
697 LOG("%d: exitsyscall g=%p\n", m->id, g); | 922 LOG("%d: exitsyscall g=%p\n", m->id, g); |
698 | 923 |
699 //runtime·printf("syscall=%d\n", (int32)(runtime·cputicks() - m->sysenterticks))
; | 924 » // Check whether the profiler needs to be turned on. |
700 | 925 » if(m->profilehz > 0) |
701 » s = m->p->status; | 926 » » runtime·setprof(true); |
702 » if((s==Psyscall || s==Pidle) && runtime·cas(&m->p->status, s, Pbusy)) { | 927 |
703 » » m->schedstats.sysexitfast++; | 928 » // Try to re-acquire the P. |
| 929 » s = m->p ? m->p->status : Pidle; |
| 930 » if(s == Psyscall && runtime·cas(&m->p->status, s, Pbusy)) { |
704 LOG("%d: exitsyscall fast\n", m->id); | 931 LOG("%d: exitsyscall fast\n", m->id); |
705 // There's a cpu for us, so we can run. | 932 // There's a cpu for us, so we can run. |
706 m->mcache = m->p->mcache; | 933 m->mcache = m->p->mcache; |
707 m->p->m = m; | 934 m->p->m = m; |
708 g->status = Grunning; | 935 g->status = Grunning; |
709 // Garbage collector isn't running (since we are), | 936 // Garbage collector isn't running (since we are), |
710 // so okay to clear gcstack. | 937 // so okay to clear gcstack. |
711 g->gcstack = (uintptr)nil; | 938 g->gcstack = (uintptr)nil; |
712 | |
713 // Check whether the profiler needs to be turned on or off. | |
714 if(m->profilehz > 0) | |
715 runtime·setprof(true); | |
716 return; | 939 return; |
717 } | 940 } |
718 | 941 |
719 » m->schedstats.sysexitslow++; | 942 » // Try to get idle P. |
| 943 » m->p = nil; |
| 944 » if(runtime·sched.pidle) { |
| 945 » » runtime·lock(&runtime·sched); |
| 946 » » p = pidleget(); |
| 947 » » runtime·unlock(&runtime·sched); |
| 948 » » if(p) { |
| 949 » » » entergo(m, p); |
| 950 » » » g->gcstack = (uintptr)nil; |
| 951 » » » return; |
| 952 » » } |
| 953 » } |
| 954 |
720 LOG("%d: exitsyscall slow p->status=%d\n", m->id, s); | 955 LOG("%d: exitsyscall slow p->status=%d\n", m->id, s); |
721 » if(m->profilehz > 0) | 956 |
722 » » runtime·setprof(true); | |
723 | |
724 » m->p = nil; | |
725 runtime·mcall(exitsyscall0); | 957 runtime·mcall(exitsyscall0); |
726 | 958 |
727 // Gosched returned, so we're allowed to run now. | 959 // Gosched returned, so we're allowed to run now. |
728 // Delete the gcstack information that we left for | 960 // Delete the gcstack information that we left for |
729 // the garbage collector during the system call. | 961 // the garbage collector during the system call. |
730 // Must wait until now because until gosched returns | 962 // Must wait until now because until gosched returns |
731 // we don't know for sure that the garbage collector | 963 // we don't know for sure that the garbage collector |
732 // is not running. | 964 // is not running. |
733 g->gcstack = (uintptr)nil; | 965 g->gcstack = (uintptr)nil; |
734 } | 966 } |
(...skipping 192 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
927 | 1159 |
928 // Create a new g running fn with narg bytes of arguments starting | 1160 // Create a new g running fn with narg bytes of arguments starting |
929 // at argp and returning nret bytes of results. callerpc is the | 1161 // at argp and returning nret bytes of results. callerpc is the |
930 // address of the go statement that created this. The new g is put | 1162 // address of the go statement that created this. The new g is put |
931 // on the queue of g's waiting to run. | 1163 // on the queue of g's waiting to run. |
932 G* | 1164 G* |
933 runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) | 1165 runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) |
934 { | 1166 { |
935 byte *sp; | 1167 byte *sp; |
936 G *newg; | 1168 G *newg; |
| 1169 M *mp; |
| 1170 P *p; |
937 int32 siz; | 1171 int32 siz; |
| 1172 //int64 goid; |
938 | 1173 |
939 //printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret); | 1174 //printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret); |
940 siz = narg + nret; | 1175 siz = narg + nret; |
941 siz = (siz+7) & ~7; | 1176 siz = (siz+7) & ~7; |
942 | 1177 |
943 // We could instead create a secondary stack frame | 1178 // We could instead create a secondary stack frame |
944 // and make it look like goexit was on the original but | 1179 // and make it look like goexit was on the original but |
945 // the call to the actual goroutine function was split. | 1180 // the call to the actual goroutine function was split. |
946 // Not worth it: this is almost always an error. | 1181 // Not worth it: this is almost always an error. |
947 if(siz > StackMin - 1024) | 1182 if(siz > StackMin - 1024) |
948 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); | 1183 runtime·throw("runtime.newproc: function arguments too large for
new goroutine"); |
949 | 1184 |
950 if((newg = gfget(m->p)) != nil) { | 1185 if((newg = gfget(m->p)) != nil) { |
| 1186 //!!!if(raceenabled) |
| 1187 //!!! runtime·racegostart(goid, callerpc); |
951 if(newg->stackguard - StackGuard != newg->stack0) | 1188 if(newg->stackguard - StackGuard != newg->stack0) |
952 runtime·throw("invalid stack in newg"); | 1189 runtime·throw("invalid stack in newg"); |
953 } else { | 1190 } else { |
954 newg = runtime·malg(StackMin); | 1191 newg = runtime·malg(StackMin); |
955 » » newg->alllink = m->p->allg; | 1192 » » runtime·lock(&runtime·sched); |
956 » » m->p->allg = newg; | 1193 » » newg->goid = ++runtime·sched.goidseq; |
| 1194 » » if(runtime·lastg == nil) |
| 1195 » » » runtime·allg = newg; |
| 1196 » » else |
| 1197 » » » runtime·lastg->alllink = newg; |
| 1198 » » runtime·lastg = newg;» »······· |
| 1199 » » runtime·unlock(&runtime·sched); |
957 } | 1200 } |
958 | 1201 |
959 sp = (byte*)newg->stackbase; | 1202 sp = (byte*)newg->stackbase; |
960 sp -= siz; | 1203 sp -= siz; |
961 runtime·memmove(sp, argp, narg); | 1204 runtime·memmove(sp, argp, narg); |
962 if(thechar == '5') { | 1205 if(thechar == '5') { |
963 // caller's LR | 1206 // caller's LR |
964 sp -= sizeof(void*); | 1207 sp -= sizeof(void*); |
965 *(void**)sp = nil; | 1208 *(void**)sp = nil; |
966 } | 1209 } |
967 | 1210 |
968 LOG("%d: newproc %p\n", m->id, newg); | 1211 LOG("%d: newproc %p\n", m->id, newg); |
969 newg->sched.sp = (uintptr)sp; | 1212 newg->sched.sp = (uintptr)sp; |
970 newg->sched.pc = (byte*)runtime·goexit; | 1213 newg->sched.pc = (byte*)runtime·goexit; |
971 newg->sched.g = newg; | 1214 newg->sched.g = newg; |
972 newg->entry = fn; | 1215 newg->entry = fn; |
973 newg->gopc = (uintptr)callerpc; | 1216 newg->gopc = (uintptr)callerpc; |
974 newg->status = Grunnable; | 1217 newg->status = Grunnable; |
975 » runtime·lock(m->p); | 1218 » runqput(m->p, newg); |
976 » gput(m->p, newg); | 1219 |
977 » runtime·unlock(m->p); | 1220 » if(runtime·sched.pidle && fn != (byte*)runtime·main) { |
| 1221 » » runtime·lock(&runtime·sched); |
| 1222 » » p = pidleget(); |
| 1223 » » if(p) { |
| 1224 » » » mp = mget(); |
| 1225 » » » runtime·unlock(&runtime·sched); |
| 1226 » » » if(mp) { |
| 1227 » » » » entergo(mp, p); |
| 1228 » » » » runtime·notewakeup(&mp->park); |
| 1229 » » » } else |
| 1230 » » » » newm(runtime·mstart, p, false); |
| 1231 » » } else |
| 1232 » » » runtime·unlock(&runtime·sched); |
| 1233 » } |
978 return newg; | 1234 return newg; |
979 } | |
980 | |
981 // Put on gfree list. Sched must be locked. | |
982 static void | |
983 gfput(P *p, G *gp) | |
984 { | |
985 if(gp->stackguard - StackGuard != gp->stack0) | |
986 runtime·throw("invalid stack in gfput"); | |
987 gp->schedlink = p->gfree; | |
988 p->gfree = gp; | |
989 } | |
990 | |
991 // Get from gfree list. Sched must be locked. | |
992 static G* | |
993 gfget(P *p) | |
994 { | |
995 G *gp; | |
996 | |
997 gp = p->gfree; | |
998 if(gp) | |
999 p->gfree = gp->schedlink; | |
1000 return gp; | |
1001 } | 1235 } |
1002 | 1236 |
1003 void | 1237 void |
1004 runtime·Breakpoint(void) | 1238 runtime·Breakpoint(void) |
1005 { | 1239 { |
1006 runtime·breakpoint(); | 1240 runtime·breakpoint(); |
1007 } | 1241 } |
1008 | 1242 |
1009 void | 1243 void |
1010 runtime·Gosched(void) | 1244 runtime·Gosched(void) |
1011 { | 1245 { |
1012 runtime·gosched(); | 1246 runtime·gosched(); |
1013 } | 1247 } |
1014 | 1248 |
1015 // Implementation of runtime.GOMAXPROCS. | 1249 // Implementation of runtime.GOMAXPROCS. |
1016 // delete when scheduler is stronger | 1250 // delete when scheduler is stronger |
1017 int32 | 1251 int32 |
1018 runtime·gomaxprocsfunc(int32 n) | 1252 runtime·gomaxprocsfunc(int32 n) |
1019 { | 1253 { |
1020 int32 ret; | 1254 int32 ret; |
1021 | 1255 |
| 1256 LOG("%d: gomaxprocsfunc %d\n", m->id, n); |
1022 if(n > maxgomaxprocs) | 1257 if(n > maxgomaxprocs) |
1023 n = maxgomaxprocs; | 1258 n = maxgomaxprocs; |
1024 runtime·lock(&runtime·sched); | 1259 runtime·lock(&runtime·sched); |
1025 ret = runtime·gomaxprocs; | 1260 ret = runtime·gomaxprocs; |
1026 if(n <= 0 || n == ret) { | 1261 if(n <= 0 || n == ret) { |
1027 runtime·unlock(&runtime·sched); | 1262 runtime·unlock(&runtime·sched); |
1028 return ret; | 1263 return ret; |
1029 } | 1264 } |
1030 runtime·unlock(&runtime·sched); | 1265 runtime·unlock(&runtime·sched); |
1031 | 1266 |
1032 runtime·semacquire(&runtime·worldsema); | 1267 runtime·semacquire(&runtime·worldsema); |
1033 m->gcing = 1; | 1268 m->gcing = 1; |
1034 runtime·stoptheworld(); | 1269 runtime·stoptheworld(); |
1035 | 1270 » newprocs = n; |
1036 » runtime·gomaxprocs = n; | |
1037 | |
1038 m->gcing = 0; | 1271 m->gcing = 0; |
1039 runtime·semrelease(&runtime·worldsema); | 1272 runtime·semrelease(&runtime·worldsema); |
1040 runtime·starttheworld(); | 1273 runtime·starttheworld(); |
1041 | 1274 |
1042 return ret; | 1275 return ret; |
1043 } | 1276 } |
1044 | 1277 |
1045 void | 1278 void |
1046 runtime·LockOSThread(void) | 1279 runtime·LockOSThread(void) |
1047 { | 1280 { |
1048 //!!! implement me. | |
1049 /* | |
1050 if(m == &runtime·m0 && runtime·sched.init) { | 1281 if(m == &runtime·m0 && runtime·sched.init) { |
1051 runtime·sched.lockmain = true; | 1282 runtime·sched.lockmain = true; |
1052 return; | 1283 return; |
1053 } | 1284 } |
1054 m->lockedg = g; | 1285 m->lockedg = g; |
1055 g->lockedm = m; | 1286 g->lockedm = m; |
1056 */ | |
1057 } | 1287 } |
1058 | 1288 |
1059 void | 1289 void |
1060 runtime·UnlockOSThread(void) | 1290 runtime·UnlockOSThread(void) |
1061 { | 1291 { |
1062 /* | |
1063 if(m == &runtime·m0 && runtime·sched.init) { | 1292 if(m == &runtime·m0 && runtime·sched.init) { |
1064 runtime·sched.lockmain = false; | 1293 runtime·sched.lockmain = false; |
1065 return; | 1294 return; |
1066 } | 1295 } |
1067 m->lockedg = nil; | 1296 m->lockedg = nil; |
1068 g->lockedm = nil; | 1297 g->lockedm = nil; |
1069 */ | |
1070 } | 1298 } |
1071 | 1299 |
1072 bool | 1300 bool |
1073 runtime·lockedOSThread(void) | 1301 runtime·lockedOSThread(void) |
1074 { | 1302 { |
1075 return g->lockedm != nil && m->lockedg != nil; | 1303 return g->lockedm != nil && m->lockedg != nil; |
1076 } | 1304 } |
1077 | 1305 |
1078 // for testing of callbacks | 1306 // for testing of callbacks |
1079 void | 1307 void |
1080 runtime·golockedOSThread(bool ret) | 1308 runtime·golockedOSThread(bool ret) |
1081 { | 1309 { |
1082 ret = runtime·lockedOSThread(); | 1310 ret = runtime·lockedOSThread(); |
1083 FLUSH(&ret); | 1311 FLUSH(&ret); |
1084 } | 1312 } |
1085 | 1313 |
1086 // for testing of wire, unwire | 1314 // for testing of wire, unwire |
1087 void | 1315 void |
1088 runtime·mid(uint32 ret) | 1316 runtime·mid(uint32 ret) |
1089 { | 1317 { |
1090 ret = m->id; | 1318 ret = m->id; |
1091 FLUSH(&ret); | 1319 FLUSH(&ret); |
1092 } | 1320 } |
1093 | 1321 |
1094 void | 1322 void |
1095 runtime·NumGoroutine(int32 ret) | 1323 runtime·NumGoroutine(intgo ret) |
1096 { | 1324 { |
1097 //ret = runtime·sched.gcount; | 1325 //ret = runtime·sched.gcount; |
1098 ret = 1; | 1326 ret = 1; |
1099 FLUSH(&ret); | 1327 FLUSH(&ret); |
1100 } | 1328 } |
1101 | 1329 |
1102 int32 | 1330 int32 |
1103 runtime·gcount(void) | 1331 runtime·gcount(void) |
1104 { | 1332 { |
1105 //return runtime·sched.gcount; | 1333 //return runtime·sched.gcount; |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1175 runtime·lock(&runtime·sched); | 1403 runtime·lock(&runtime·sched); |
1176 runtime·sched.profilehz = hz; | 1404 runtime·sched.profilehz = hz; |
1177 runtime·unlock(&runtime·sched); | 1405 runtime·unlock(&runtime·sched); |
1178 | 1406 |
1179 if(hz != 0) | 1407 if(hz != 0) |
1180 runtime·resetcpuprofiler(hz); | 1408 runtime·resetcpuprofiler(hz); |
1181 } | 1409 } |
1182 | 1410 |
1183 // Change number of processors. The world is stopped. | 1411 // Change number of processors. The world is stopped. |
1184 static void | 1412 static void |
1185 procresize(void) | 1413 procresize(int32 new) |
1186 { | 1414 { |
1187 » int32 i, old, new; | 1415 » int32 i, old; |
1188 G *gp; | 1416 G *gp; |
1189 | 1417 » P *p; |
1190 » old = runtime·np; | 1418 |
1191 » new = runtime·gomaxprocs; | 1419 » runtime·lock(&runtime·sched); //!!! |
| 1420 » old = runtime·gomaxprocs; |
1192 LOG("%d: procresize %d->%d\n", m->id, old, new); | 1421 LOG("%d: procresize %d->%d\n", m->id, old, new); |
1193 if(old < 0 || old > maxgomaxprocs || new <= 0 || new > maxgomaxprocs) | 1422 if(old < 0 || old > maxgomaxprocs || new <= 0 || new > maxgomaxprocs) |
1194 runtime·throw("procresize: invalid arg"); | 1423 runtime·throw("procresize: invalid arg"); |
1195 if(old == new) { | 1424 if(old == new) { |
1196 for(i=0; i<new; i++) { | 1425 for(i=0; i<new; i++) { |
1197 » » » if(runtime·allp[i] == m->p) | 1426 » » » p = runtime·allp[i]; |
1198 » » » » runtime·allp[i]->status = Pbusy; | 1427 » » » if(p == m->p) |
| 1428 » » » » p->status = Pbusy; |
| 1429 » » » else { |
| 1430 » » » » p->status = Pidle; |
| 1431 » » » » pidleput(p); |
| 1432 » » » } |
| 1433 » » } |
| 1434 » » runtime·unlock(&runtime·sched); |
| 1435 » » return; |
| 1436 » } |
| 1437 |
| 1438 » runtime·singleproc = new == 1; |
| 1439 » runtime·gomaxprocs = new; |
| 1440 » for(i=0; i<new; i++) { |
| 1441 » » p = runtime·allp[i]; |
| 1442 » » if(p == nil) { |
| 1443 » » » p = (P*)runtime·mallocgc(sizeof(runtime·allp[i][0]), 0,
0, 1); |
| 1444 » » » p->status = Plocked; |
| 1445 » » » runtime·allp[i] = p; //@@@ store-release |
| 1446 » » } |
| 1447 » » if(p->mcache == nil) { |
| 1448 » » » if(old==0 && i==0) |
| 1449 » » » » p->mcache = m->mcache; |
1199 else | 1450 else |
1200 » » » » runtime·allp[i]->status = Pidle; //!!! store-rel
ease | 1451 » » » » p->mcache = runtime·allocmcache(); |
1201 » » } | 1452 » » } |
1202 » » return; | 1453 » » if(p->runq == nil) { |
1203 » } | 1454 » » » p->runqsize = 1024; |
1204 | 1455 » » » p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*),
0, 0, 1); |
1205 » runtime·singleproc = new == 1; | |
1206 » runtime·np = new; | |
1207 » for(i=0; i<new; i++) { | |
1208 » » if(runtime·allp[i] == nil) | |
1209 » » » runtime·allp[i] = (P*)runtime·malloc(sizeof(runtime·allp
[i][0])); | |
1210 » » if(runtime·allp[i]->mcache == nil) { | |
1211 » » » if(old==0 && i==0) | |
1212 » » » » runtime·allp[i]->mcache = m->mcache; | |
1213 » » » else | |
1214 » » » » runtime·allp[i]->mcache = runtime·allocmcache(); | |
1215 } | 1456 } |
1216 } | 1457 } |
1217 | 1458 |
1218 for(i=1; i<old; i++) { | 1459 for(i=1; i<old; i++) { |
1219 for(;;) { | 1460 for(;;) { |
1220 » » » gp = gget(runtime·allp[i]); | 1461 » » » gp = runqget(runtime·allp[i]); |
1221 if(gp == nil) | 1462 if(gp == nil) |
1222 break; | 1463 break; |
1223 //TODO: spread more evenly. | 1464 //TODO: spread more evenly. |
1224 » » » gput(runtime·allp[0], gp); | 1465 » » » runqput(runtime·allp[0], gp); |
1225 } | 1466 } |
1226 } | 1467 } |
1227 | 1468 |
1228 for(i=new; i<old; i++) { | 1469 for(i=new; i<old; i++) { |
1229 runtime·freemcache(runtime·allp[i]->mcache); | 1470 runtime·freemcache(runtime·allp[i]->mcache); |
1230 runtime·allp[i]->mcache = nil; | 1471 runtime·allp[i]->mcache = nil; |
1231 runtime·allp[i]->status = Pdead; | 1472 runtime·allp[i]->status = Pdead; |
1232 //TODO: free freeg | 1473 //TODO: free freeg |
1233 } | 1474 } |
1234 | 1475 |
1235 if(m->p) | 1476 if(m->p) |
1236 m->p->m = nil; | 1477 m->p->m = nil; |
1237 m->p = nil; | 1478 m->p = nil; |
1238 m->mcache = nil; | 1479 m->mcache = nil; |
1239 runtime·allp[0]->m = nil; | 1480 runtime·allp[0]->m = nil; |
1240 » runtime·allp[0]->status = Pbusy; | 1481 » runtime·allp[0]->status = Pidle; |
1241 entergo(m, runtime·allp[0]); | 1482 entergo(m, runtime·allp[0]); |
1242 » for(i=1; i<new; i++) | 1483 » for(i=1; i<new; i++) { |
1243 » » runtime·allp[i]->status = Pidle; //!!! store-release | 1484 » » p = runtime·allp[i]; |
| 1485 » » p->status = Pidle; |
| 1486 » » pidleput(p); |
| 1487 » } |
| 1488 » runtime·unlock(&runtime·sched); |
1244 } | 1489 } |
1245 | 1490 |
1246 static void | 1491 static void |
1247 entergo(M *mp, P *p) | 1492 entergo(M *mp, P *p) |
1248 { | 1493 { |
1249 » LOG("%d: entergo p=%p\n", mp->id, p); | 1494 » LOG("%d: entergo m=%d p=%p p->m=%p, p->status=%d, p->mcache=%p\n", m->id
, mp->id, p, p->m, p->status, p->mcache); |
1250 if(mp->p || mp->mcache) | 1495 if(mp->p || mp->mcache) |
1251 runtime·throw("entergo: already in go"); | 1496 runtime·throw("entergo: already in go"); |
1252 » if(p->m || p->status != Pbusy) { | 1497 » if(p->m || p->status != Pidle) { |
1253 » » runtime·printf("entergo: p->m=%d p->status=%d\n", p->m->id, p->s
tatus); | 1498 » » runtime·printf("entergo: p->m=%p(%d) p->status=%d\n", p->m, p->m
? p->m->id : 0, p->status); |
1254 runtime·throw("entergo: invalid p state"); | 1499 runtime·throw("entergo: invalid p state"); |
1255 } | 1500 } |
1256 mp->mcache = p->mcache; | 1501 mp->mcache = p->mcache; |
1257 mp->p = p; | 1502 mp->p = p; |
1258 p->m = mp; | 1503 p->m = mp; |
1259 } | 1504 » p->status = Pbusy; |
1260 | 1505 } |
1261 static void | 1506 |
1262 leavego(M *mp, uint32 status) | 1507 static P* |
1263 { | 1508 releasep(void) |
| 1509 { |
| 1510 » M *mp; |
1264 P *p; | 1511 P *p; |
1265 | 1512 |
1266 » LOG("%d: leavego %d\n", mp->id, status); | 1513 » mp = m; |
| 1514 » LOG("%d: releasep\n", mp->id); |
1267 // sched is locked | 1515 // sched is locked |
1268 if(mp->p == nil || mp->mcache == nil) | 1516 if(mp->p == nil || mp->mcache == nil) |
1269 » » runtime·throw("leavego: invalid arg"); | 1517 » » runtime·throw("releasep: invalid arg"); |
1270 p = mp->p; | 1518 p = mp->p; |
1271 » if(p->m != mp || p->mcache != mp->mcache || p->status != Pbusy) | 1519 » if(p->m != mp || p->mcache != mp->mcache || p->status != Pbusy) { |
1272 » » runtime·throw("leavego: invalid p state"); | 1520 » » runtime·printf("releasep: m=%p m->p=%p p->m=%p m->mcache=%p p->m
cache=%p p->status=%d\n", |
| 1521 » » » mp, mp->p, p->m, m->mcache, p->mcache, p->status); |
| 1522 » » runtime·throw("releasep: invalid p state"); |
| 1523 » } |
1273 mp->p = nil; | 1524 mp->p = nil; |
1274 mp->mcache = nil; | 1525 mp->mcache = nil; |
1275 p->m = nil; | 1526 p->m = nil; |
1276 » p->status = status; //!!! store-release | 1527 » p->status = Pidle; |
1277 } | 1528 » return p; |
1278 | 1529 } |
1279 static void | 1530 |
1280 goidle(void) | 1531 typedef struct Pdesc Pdesc; |
1281 { | 1532 struct Pdesc |
1282 » uint32 i, s, try; | 1533 { |
1283 » P *p, *idlep; | 1534 » uint32» tick; |
1284 » bool haveg; | 1535 » int64» when; |
| 1536 }; |
| 1537 |
| 1538 static void |
| 1539 retake(int64 now, Pdesc *ps) |
| 1540 { |
| 1541 » uint32 i, s; |
| 1542 » int64 t; |
| 1543 » P *p; |
1285 M *mp; | 1544 M *mp; |
1286 | 1545 |
1287 » LOG("%d: goidle\n", m->id); | 1546 » for(i=0; i<runtime·gomaxprocs; i++) { |
1288 » if(g != m->g0) | 1547 » » p = runtime·allp[i]; |
1289 » » runtime·throw("goidle not on g0"); | 1548 » » //!!! procresize may be in progress |
1290 » if(m->p) | 1549 » » // do something if GC is in progress (help). |
1291 » » runtime·throw("goidle not idle"); | 1550 » » if(p==nil) |
1292 | |
1293 » for(try=0;; try++) { | |
1294 » » idlep = nil; | |
1295 » » haveg = false; | |
1296 » » for(i=0; i<runtime·np; i++) { | |
1297 » » » p = runtime·allp[i]; | |
1298 » » » //!!! procresize may be in progress | |
1299 » » » // do something if GC is in progress. | |
1300 » » » if(p==nil) | |
1301 » » » » continue; | |
1302 » » » s = p->status; | |
1303 » » » LOG("%d: goidle p=%d status=%d ghead=%p tick=%d lasttick
=%d\n", | |
1304 » » » » m->id, i, s, p->ghead, p->tick, p->lasttick); | |
1305 » » » if(s == Pidle) | |
1306 » » » » idlep = p; | |
1307 » » » else if(s == Psyscall) { | |
1308 » » » » if(p->tick == p->lasttick) | |
1309 » » » » » idlep = p; | |
1310 » » » » else | |
1311 » » » » » p->lasttick = p->tick; | |
1312 » » » } | |
1313 » » » if(p->ghead != nil) | |
1314 » » » » haveg = true; | |
1315 » » } | |
1316 » » if(idlep == nil || !haveg) { | |
1317 » » » runtime·usleep(20); | |
1318 continue; | 1551 continue; |
1319 » » } | 1552 » » t = p->tick; |
1320 » » s = idlep->status; | 1553 » » if(ps[i].tick != t) { |
1321 » » if(s == Pidle || s == Psyscall) { | 1554 » » » ps[i].tick = t; |
1322 » » » if(runtime·cas(&idlep->status, s, Pbusy)) { | 1555 » » » ps[i].when = now; |
1323 » » » » LOG("%d: goidle steal p in status %d\n", m->id,
s); | 1556 » » } |
1324 » » » » entergo(m, idlep); | 1557 » » if(ps[i].when + 20*1000 > now) |
1325 » » » » runtime·lock(&runtime·sched); | 1558 » » » continue; |
1326 » » » » mp = mget(); | 1559 » » s = p->status; |
| 1560 » » if(s == Psyscall && runtime·cas(&p->status, s, Pidle)) { |
| 1561 » » » LOG("retake %p(%d)\n", p, i); |
| 1562 » » » runtime·lock(&runtime·sched); |
| 1563 » » » mp = mget(); |
| 1564 » » » runtime·unlock(&runtime·sched); |
| 1565 » » » munpark(mp, p); |
| 1566 » » } |
| 1567 » } |
| 1568 } |
| 1569 |
| 1570 static Pdesc ps[maxgomaxprocs]; |
| 1571 |
| 1572 static void |
| 1573 sysmon(void) |
| 1574 { |
| 1575 » int64 t0, now; |
| 1576 |
| 1577 » // This is a special dedicated thread. |
| 1578 » // It works w/o mcache nor stackalloc, it may work concurrently with GC. |
| 1579 » runtime·asminit(); |
| 1580 » runtime·minit(); |
| 1581 » LOG("sysmon\n"); |
| 1582 » t0 = runtime·nanotime(); |
| 1583 » for(;;) { |
| 1584 » » //!!! sleep more if possible |
| 1585 » » runtime·usleep(20); |
| 1586 » » if(runtime·gcwaiting) { |
| 1587 » » » runtime·lock(&runtime·sched); |
| 1588 » » » if(runtime·gcwaiting) { |
| 1589 » » » » runtime·sched.sysmonwait = 1; |
1327 runtime·unlock(&runtime·sched); | 1590 runtime·unlock(&runtime·sched); |
1328 » » » » if(mp) | 1591 » » » » runtime·notesleep(&runtime·sched.sysmonnote); |
1329 » » » » » runtime·notewakeup(&mp->park); | 1592 » » » » runtime·noteclear(&runtime·sched.sysmonnote); |
1330 » » » » else | 1593 » » » } else |
1331 » » » » » newm(); | 1594 » » » » runtime·unlock(&runtime·sched); |
1332 » » » » return; | 1595 » » } |
1333 » » » } | 1596 » » now = runtime·nanotime() - t0; |
1334 » » } | 1597 » » retake(now, ps); |
1335 » » runtime·usleep(20); | 1598 » } |
1336 » } | 1599 } |
1337 } | 1600 |
| 1601 static void |
| 1602 inject(G *gp0, int32 *w, int32 *n) |
| 1603 { |
| 1604 » int32 nw; |
| 1605 » G *gp; |
| 1606 » M *mp; |
| 1607 » P *p; |
| 1608 |
| 1609 » runtime·lock(&runtime·sched); |
| 1610 » while(gp0) { |
| 1611 » » gp = gp0; |
| 1612 » » gp0 = gp->schedlink; |
| 1613 » » gp->status = Grunnable; |
| 1614 » » globrunqput(gp); |
| 1615 » » (*n)++; |
| 1616 » } |
| 1617 » runtime·unlock(&runtime·sched); |
| 1618 |
| 1619 » nw = *n; |
| 1620 » while(runtime·sched.pidle && nw) { |
| 1621 » » runtime·lock(&runtime·sched); |
| 1622 » » if(runtime·sched.pidle == nil) { |
| 1623 » » » runtime·unlock(&runtime·sched); |
| 1624 » » » break; |
| 1625 » » } |
| 1626 » » (*w)++; |
| 1627 » » nw--; |
| 1628 » » p = pidleget(); |
| 1629 » » mp = mget(); |
| 1630 » » runtime·unlock(&runtime·sched); |
| 1631 » » if(mp) { |
| 1632 » » » entergo(mp, p); |
| 1633 » » » runtime·notewakeup(&mp->park); |
| 1634 » » } else |
| 1635 » » » newm(runtime·mstart, p, false); |
| 1636 » } |
| 1637 } |
| 1638 |
| 1639 static void |
| 1640 globrunqput(G *gp) |
| 1641 { |
| 1642 » gp->schedlink = nil; |
| 1643 » if(runtime·sched.runqtail) |
| 1644 » » runtime·sched.runqtail->schedlink = gp; |
| 1645 » else |
| 1646 » » runtime·sched.runqhead = gp; |
| 1647 » runtime·sched.runqtail = gp; |
| 1648 » runtime·sched.runqsize++; |
| 1649 } |
| 1650 |
| 1651 static G* |
| 1652 globrunqget(void) |
| 1653 { |
| 1654 » G *gp, *gp1; |
| 1655 » int32 n; |
| 1656 |
| 1657 » if(runtime·sched.runqsize == 0) |
| 1658 » » return nil; |
| 1659 » n = runtime·sched.runqsize/runtime·gomaxprocs+1; |
| 1660 » if(n > runtime·sched.runqsize) |
| 1661 » » n = runtime·sched.runqsize; |
| 1662 » runtime·sched.runqsize -= n; |
| 1663 » if(runtime·sched.runqsize == 0) |
| 1664 » » runtime·sched.runqtail = nil; |
| 1665 » gp1 = nil; |
| 1666 » while(n--) { |
| 1667 » » gp = runtime·sched.runqhead; |
| 1668 » » runtime·sched.runqhead = gp->schedlink; |
| 1669 » » gp->schedlink = gp1; |
| 1670 » » gp1 = gp; |
| 1671 » } |
| 1672 » return gp1; |
| 1673 } |
| 1674 |
| 1675 // sched is locked |
| 1676 static P* |
| 1677 pidleget(void) |
| 1678 { |
| 1679 » P *p; |
| 1680 »······· |
| 1681 » p = runtime·sched.pidle; |
| 1682 » if(p) { |
| 1683 » » runtime·sched.pidle = p->link; |
| 1684 » » runtime·sched.npidle--; |
| 1685 » } |
| 1686 » return p; |
| 1687 } |
| 1688 |
| 1689 // sched is locked |
| 1690 static void |
| 1691 pidleput(P *p) |
| 1692 { |
| 1693 » p->link = runtime·sched.pidle; |
| 1694 » runtime·sched.pidle = p; |
| 1695 » runtime·sched.npidle++; |
| 1696 } |
| 1697 |
| 1698 static void |
| 1699 runqgrow(P *p) |
| 1700 { |
| 1701 » G **q; |
| 1702 » int32 s, t, h, t2; |
| 1703 |
| 1704 » h = p->runqhead; |
| 1705 » t = p->runqtail; |
| 1706 » s = p->runqsize; |
| 1707 » t2 = 0; |
| 1708 » q = (G**)runtime·malloc(2*s*sizeof(*q)); |
| 1709 » while(t!=h) { |
| 1710 » » q[t2] = p->runq[h]; |
| 1711 » » t2++; |
| 1712 » » h++; |
| 1713 » » if(h==s) |
| 1714 » » » h = 0; |
| 1715 » } |
| 1716 » runtime·free(p->runq); |
| 1717 » p->runq = q; |
| 1718 » p->runqhead = 0; |
| 1719 » p->runqtail = t2; |
| 1720 » p->runqsize = 2*s; |
| 1721 } |
| 1722 |
| 1723 static G* |
| 1724 runqsteal(P *p, P *p2) |
| 1725 { |
| 1726 » G *gp, *gp1; |
| 1727 » int32 t, h, s, t2, h2, s2, c, c1; |
| 1728 |
| 1729 » if(p2->runqhead==p2->runqtail) |
| 1730 » » return nil; |
| 1731 » if(p < p2) |
| 1732 » » runtime·lock(p); |
| 1733 » runtime·lock(p2); |
| 1734 » if(p2->runqhead==p2->runqtail) { |
| 1735 » » runtime·unlock(p2); |
| 1736 » » if(p < p2) |
| 1737 » » » runtime·unlock(p); |
| 1738 » » return nil; |
| 1739 » } |
| 1740 » if(p >= p2) |
| 1741 » » runtime·lock(p); |
| 1742 » h = p->runqhead; |
| 1743 » t = p->runqtail; |
| 1744 » s = p->runqsize; |
| 1745 » h2 = p2->runqhead; |
| 1746 » t2 = p2->runqtail; |
| 1747 » s2 = p2->runqsize; |
| 1748 » gp = p2->runq[h2]; |
| 1749 » h2++; |
| 1750 » if(h2 == s2) |
| 1751 » » h2 = 0; |
| 1752 » if(t2 > h2) |
| 1753 » » c = (t2 - h2) / 2; |
| 1754 » else |
| 1755 » » c = (s2 - h2 + t2) / 2; |
| 1756 » c1 = 0; |
| 1757 » for(;;) { |
| 1758 » » if(c1 == c) |
| 1759 » » » break; |
| 1760 » » if(t==h-1 || (h==0 && t==s-1)) |
| 1761 » » » break; |
| 1762 » » if(t2==h2) |
| 1763 » » » break; |
| 1764 » » gp1 = p2->runq[h2]; |
| 1765 » » h2++; |
| 1766 » » if(h2 == s2) |
| 1767 » » » h2 = 0; |
| 1768 » » p->runq[t] = gp1; |
| 1769 » » t++; |
| 1770 » » if(t==s) |
| 1771 » » » t = 0; |
| 1772 » » c1++; |
| 1773 » } |
| 1774 » p->runqtail = t; |
| 1775 » p2->runqhead = h2; |
| 1776 » runtime·unlock(p2); |
| 1777 » runtime·unlock(p); |
| 1778 » return gp; |
| 1779 } |
| 1780 |
| 1781 // Put g on runnable queue. |
| 1782 static void |
| 1783 runqput(P *p, G *gp) |
| 1784 { |
| 1785 » int32 h, t, s; |
| 1786 |
| 1787 » runtime·lock(p); |
| 1788 retry: |
| 1789 » h = p->runqhead; |
| 1790 » t = p->runqtail; |
| 1791 » s = p->runqsize; |
| 1792 » if(t==h-1 || (h==0 && t==s-1)) { |
| 1793 » » runqgrow(p); |
| 1794 » » goto retry; |
| 1795 » } |
| 1796 » p->runq[t] = gp; |
| 1797 » t++; |
| 1798 » if(t==s) |
| 1799 » » t = 0; |
| 1800 » p->runqtail = t; |
| 1801 » runtime·unlock(p); |
| 1802 } |
| 1803 |
| 1804 // Get g from runnable queue. |
| 1805 static G* |
| 1806 runqget(P *p) |
| 1807 { |
| 1808 » G *gp; |
| 1809 » int32 t, h, s; |
| 1810 |
| 1811 » if(p->runqhead==p->runqtail) |
| 1812 » » return nil; |
| 1813 » runtime·lock(p); |
| 1814 » h = p->runqhead; |
| 1815 » t = p->runqtail; |
| 1816 » s = p->runqsize; |
| 1817 » if(t==h) { |
| 1818 » » runtime·unlock(p); |
| 1819 » » return nil; |
| 1820 » } |
| 1821 » gp = p->runq[h]; |
| 1822 » h++; |
| 1823 » if(h==s) |
| 1824 » » h = 0; |
| 1825 » p->runqhead = h; |
| 1826 » runtime·unlock(p); |
| 1827 » return gp; |
| 1828 } |
| 1829 |
| 1830 // Put on `m' list. Sched must be locked. |
| 1831 static void |
| 1832 mput(M *mp) |
| 1833 { |
| 1834 » mp->schedlink = runtime·sched.mhead; |
| 1835 » runtime·sched.mhead = mp; |
| 1836 » runtime·sched.mwait++; |
| 1837 } |
| 1838 |
| 1839 // Sched must be locked. |
| 1840 static M* |
| 1841 mget(void) |
| 1842 { |
| 1843 » M *mp; |
| 1844 |
| 1845 » if((mp = runtime·sched.mhead) != nil){ |
| 1846 » » runtime·sched.mhead = mp->schedlink; |
| 1847 » » runtime·sched.mwait--; |
| 1848 » } |
| 1849 » return mp; |
| 1850 } |
| 1851 |
| 1852 // Put on gfree list. |
| 1853 static void |
| 1854 gfput(P *p, G *gp) |
| 1855 { |
| 1856 » if(gp->stackguard - StackGuard != gp->stack0) |
| 1857 » » runtime·throw("invalid stack in gfput"); |
| 1858 » gp->schedlink = p->gfree; |
| 1859 » p->gfree = gp; |
| 1860 » p->gfreecnt++; |
| 1861 » if(p->gfreecnt >= 64) { |
| 1862 » » runtime·lock(&runtime·sched.gflock); |
| 1863 » » while(p->gfreecnt >= 32) { |
| 1864 » » » p->gfreecnt--; |
| 1865 » » » gp = p->gfree; |
| 1866 » » » p->gfree = gp->schedlink; |
| 1867 » » » gp->schedlink = runtime·sched.gfree; |
| 1868 » » » runtime·sched.gfree = gp; |
| 1869 » » } |
| 1870 » » runtime·unlock(&runtime·sched.gflock); |
| 1871 » } |
| 1872 } |
| 1873 |
| 1874 // Get from gfree list. |
| 1875 static G* |
| 1876 gfget(P *p) |
| 1877 { |
| 1878 » G *gp; |
| 1879 |
| 1880 retry: |
| 1881 » gp = p->gfree; |
| 1882 » if(gp == nil && runtime·sched.gfree) { |
| 1883 » » runtime·lock(&runtime·sched.gflock); |
| 1884 » » while(p->gfreecnt < 32 && runtime·sched.gfree) { |
| 1885 » » » p->gfreecnt++; |
| 1886 » » » gp = runtime·sched.gfree; |
| 1887 » » » runtime·sched.gfree = gp->schedlink; |
| 1888 » » » gp->schedlink = p->gfree; |
| 1889 » » » p->gfree = gp; |
| 1890 » » } |
| 1891 » » runtime·unlock(&runtime·sched.gflock); |
| 1892 » » goto retry; |
| 1893 » } |
| 1894 » if(gp) { |
| 1895 » » p->gfree = gp->schedlink; |
| 1896 » » p->gfreecnt--; |
| 1897 » } |
| 1898 » return gp; |
| 1899 } |
LEFT | RIGHT |