LEFT | RIGHT |
1 // Copyright 2013 The Go Authors. All rights reserved. | 1 // Copyright 2013 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "malloc.h" | 7 #include "malloc.h" |
8 #include "stack.h" | 8 #include "stack.h" |
| 9 |
| 10 enum |
| 11 { |
| 12 StackDebug = 0, |
| 13 }; |
9 | 14 |
10 typedef struct StackCacheNode StackCacheNode; | 15 typedef struct StackCacheNode StackCacheNode; |
11 struct StackCacheNode | 16 struct StackCacheNode |
12 { | 17 { |
13 StackCacheNode *next; | 18 StackCacheNode *next; |
14 void* batch[StackCacheBatch-1]; | 19 void* batch[StackCacheBatch-1]; |
15 }; | 20 }; |
16 | 21 |
17 static StackCacheNode *stackcache; | 22 static StackCacheNode *stackcache; |
18 static Lock stackcachemu; | 23 static Lock stackcachemu; |
(...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
121 runtime·free(v); | 126 runtime·free(v); |
122 } | 127 } |
123 | 128 |
124 // Called from runtime·lessstack when returning from a function which | 129 // Called from runtime·lessstack when returning from a function which |
125 // allocated a new stack segment. The function's return value is in | 130 // allocated a new stack segment. The function's return value is in |
126 // m->cret. | 131 // m->cret. |
127 void | 132 void |
128 runtime·oldstack(void) | 133 runtime·oldstack(void) |
129 { | 134 { |
130 Stktop *top; | 135 Stktop *top; |
131 Gobuf label; | |
132 uint32 argsize; | 136 uint32 argsize; |
133 byte *sp, *old; | 137 byte *sp, *old; |
134 uintptr *src, *dst, *dstend; | 138 uintptr *src, *dst, *dstend; |
135 G *gp; | 139 G *gp; |
136 int64 goid; | 140 int64 goid; |
137 | 141 » int32 oldstatus; |
138 //printf("oldstack m->cret=%p\n", m->cret); | |
139 | 142 |
140 gp = m->curg; | 143 gp = m->curg; |
141 top = (Stktop*)gp->stackbase; | 144 top = (Stktop*)gp->stackbase; |
142 old = (byte*)gp->stackguard - StackGuard; | 145 old = (byte*)gp->stackguard - StackGuard; |
143 sp = (byte*)top; | 146 sp = (byte*)top; |
144 argsize = top->argsize; | 147 argsize = top->argsize; |
| 148 |
| 149 if(StackDebug) { |
| 150 runtime·printf("runtime: oldstack gobuf={pc:%p sp:%p lr:%p} cret
=%p argsize=%p\n", |
| 151 top->gobuf.pc, top->gobuf.sp, top->gobuf.lr, m->cret, (u
intptr)argsize); |
| 152 } |
| 153 |
| 154 // gp->status is usually Grunning, but it could be Gsyscall if a stack s
plit |
| 155 // happens during a function call inside entersyscall. |
| 156 oldstatus = gp->status; |
| 157 ········ |
| 158 gp->sched = top->gobuf; |
| 159 gp->sched.ret = m->cret; |
| 160 m->cret = 0; // drop reference |
| 161 gp->status = Gwaiting; |
| 162 gp->waitreason = "stack unsplit"; |
| 163 |
145 if(argsize > 0) { | 164 if(argsize > 0) { |
146 sp -= argsize; | 165 sp -= argsize; |
147 dst = (uintptr*)top->argp; | 166 dst = (uintptr*)top->argp; |
148 dstend = dst + argsize/sizeof(*dst); | 167 dstend = dst + argsize/sizeof(*dst); |
149 src = (uintptr*)sp; | 168 src = (uintptr*)sp; |
150 while(dst < dstend) | 169 while(dst < dstend) |
151 *dst++ = *src++; | 170 *dst++ = *src++; |
152 } | 171 } |
153 goid = top->gobuf.g->goid; // fault if g is bad, before gogo | 172 goid = top->gobuf.g->goid; // fault if g is bad, before gogo |
154 USED(goid); | 173 USED(goid); |
155 | 174 |
156 label = top->gobuf; | |
157 gp->stackbase = top->stackbase; | 175 gp->stackbase = top->stackbase; |
158 gp->stackguard = top->stackguard; | 176 gp->stackguard = top->stackguard; |
159 gp->stackguard0 = gp->stackguard; | 177 gp->stackguard0 = gp->stackguard; |
| 178 |
160 if(top->free != 0) | 179 if(top->free != 0) |
161 runtime·stackfree(old, top->free); | 180 runtime·stackfree(old, top->free); |
162 | 181 |
163 » label.ret = m->cret; | 182 » gp->status = oldstatus; |
164 » m->cret = 0; // drop reference | 183 » runtime·gogo(&gp->sched); |
165 » runtime·gogo(&label); | |
166 } | 184 } |
167 | 185 |
168 // Called from reflect·call or from runtime·morestack when a new | 186 // Called from reflect·call or from runtime·morestack when a new |
169 // stack segment is needed. Allocate a new stack big enough for | 187 // stack segment is needed. Allocate a new stack big enough for |
170 // m->moreframesize bytes, copy m->moreargsize bytes to the new frame, | 188 // m->moreframesize bytes, copy m->moreargsize bytes to the new frame, |
171 // and then act as though runtime·lessstack called the function at | 189 // and then act as though runtime·lessstack called the function at |
172 // m->morepc. | 190 // m->morepc. |
173 void | 191 void |
174 runtime·newstack(void) | 192 runtime·newstack(void) |
175 { | 193 { |
176 » int32 framesize, minalloc, argsize; | 194 » int32 framesize, argsize, oldstatus; |
177 Stktop *top; | 195 Stktop *top; |
178 byte *stk; | 196 byte *stk; |
179 uintptr sp; | 197 uintptr sp; |
180 uintptr *src, *dst, *dstend; | 198 uintptr *src, *dst, *dstend; |
181 G *gp; | 199 G *gp; |
182 Gobuf label; | 200 Gobuf label; |
183 » bool reflectcall, preempt; | 201 » bool reflectcall; |
184 uintptr free; | 202 uintptr free; |
| 203 |
| 204 // gp->status is usually Grunning, but it could be Gsyscall if a stack s
plit |
| 205 // happens during a function call inside entersyscall. |
| 206 gp = m->curg; |
| 207 oldstatus = gp->status; |
185 | 208 |
186 framesize = m->moreframesize; | 209 framesize = m->moreframesize; |
187 argsize = m->moreargsize; | 210 argsize = m->moreargsize; |
188 » gp = m->curg; | 211 » gp->status = Gwaiting; |
189 | 212 » gp->waitreason = "stack split"; |
190 » if(m->morebuf.sp < gp->stackguard - StackGuard) { | 213 » reflectcall = framesize==1; |
191 » » runtime·printf("runtime: split stack overflow: %p < %p\n", m->mo
rebuf.sp, gp->stackguard - StackGuard); | 214 » if(reflectcall) |
| 215 » » framesize = 0; |
| 216 |
| 217 » // For reflectcall the context already points to beginning of reflect·ca
ll. |
| 218 » if(!reflectcall) |
| 219 » » runtime·rewindmorestack(&gp->sched); |
| 220 |
| 221 » sp = gp->sched.sp; |
| 222 » if(thechar == '6' || thechar == '8') { |
| 223 » » // The call to morestack cost a word. |
| 224 » » sp -= sizeof(uintptr); |
| 225 » } |
| 226 » if(StackDebug || sp < gp->stackguard - StackGuard) { |
| 227 » » runtime·printf("runtime: newstack framesize=%p argsize=%p sp=%p
stack=[%p, %p]\n" |
| 228 » » » "\tmorebuf={pc:%p sp:%p lr:%p}\n" |
| 229 » » » "\tsched={pc:%p sp:%p lr:%p ctxt:%p}\n", |
| 230 » » » (uintptr)framesize, (uintptr)argsize, sp, gp->stackguard
- StackGuard, gp->stackbase, |
| 231 » » » m->morebuf.pc, m->morebuf.sp, m->morebuf.lr, |
| 232 » » » gp->sched.pc, gp->sched.sp, gp->sched.lr, gp->sched.ctxt
); |
| 233 » } |
| 234 » if(sp < gp->stackguard - StackGuard) { |
| 235 » » runtime·printf("runtime: split stack overflow: %p < %p\n", sp, g
p->stackguard - StackGuard); |
192 runtime·throw("runtime: split stack overflow"); | 236 runtime·throw("runtime: split stack overflow"); |
193 } | 237 } |
| 238 |
194 if(argsize % sizeof(uintptr) != 0) { | 239 if(argsize % sizeof(uintptr) != 0) { |
195 runtime·printf("runtime: stack split with misaligned argsize %d\
n", argsize); | 240 runtime·printf("runtime: stack split with misaligned argsize %d\
n", argsize); |
196 runtime·throw("runtime: stack split argsize"); | 241 runtime·throw("runtime: stack split argsize"); |
197 } | 242 } |
198 | 243 |
199 minalloc = 0; | |
200 reflectcall = framesize==1; | |
201 if(reflectcall) { | |
202 framesize = 0; | |
203 // moreframesize_minalloc is only set in runtime·gc(), | |
204 // that calls newstack via reflect·call(). | |
205 minalloc = m->moreframesize_minalloc; | |
206 m->moreframesize_minalloc = 0; | |
207 if(framesize < minalloc) | |
208 framesize = minalloc; | |
209 } | |
210 | |
211 preempt = false; | |
212 if(gp->stackguard0 == StackPreempt) { | 244 if(gp->stackguard0 == StackPreempt) { |
213 if(gp == m->g0) | 245 if(gp == m->g0) |
214 runtime·throw("runtime: preempt g0"); | 246 runtime·throw("runtime: preempt g0"); |
| 247 if(oldstatus == Grunning && (m->p == nil || m->p->status != Prun
ning)) |
| 248 runtime·throw("runtime: g is running but p is not"); |
215 // Be conservative about where we preempt. | 249 // Be conservative about where we preempt. |
216 // We are interested in preempting user Go code, not runtime cod
e. | 250 // We are interested in preempting user Go code, not runtime cod
e. |
217 » » preempt = gp->status == Grunning && m->locks == 0 && m->mallocin
g == 0 && | 251 » » if(oldstatus != Grunning || m->locks || m->mallocing || m->gcing
) { |
218 » » » m->gcing == 0 && m->p && m->p->status == Prunning && !re
flectcall; | 252 » » » // Let the goroutine keep running for now. |
219 » } | 253 » » » // TODO(dvyukov): remember but delay the preemption. |
220 | 254 » » » gp->stackguard0 = gp->stackguard; |
221 » // TODO(dvyukov): try to reuse the current stack segment | 255 » » » gp->status = oldstatus; |
222 » // instead of creating new segment on every preemption. | 256 » » » runtime·gogo(&gp->sched);» // never return |
223 » if(reflectcall && minalloc == 0 && m->morebuf.sp - sizeof(Stktop) - args
ize - 32 > gp->stackguard) { | 257 » » } |
| 258 » » // Act like goroutine called runtime.Gosched. |
| 259 » » gp->status = oldstatus; |
| 260 » » runtime·gosched0(gp);» // never return |
| 261 » } |
| 262 |
| 263 » if(reflectcall && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > gp->st
ackguard) { |
224 // special case: called from reflect.call (framesize==1) | 264 // special case: called from reflect.call (framesize==1) |
225 // to call code with an arbitrary argument size, | 265 // to call code with an arbitrary argument size, |
226 // and we have enough space on the current stack. | 266 // and we have enough space on the current stack. |
227 // the new Stktop* is necessary to unwind, but | 267 // the new Stktop* is necessary to unwind, but |
228 // we don't need to create a new segment. | 268 // we don't need to create a new segment. |
229 top = (Stktop*)(m->morebuf.sp - sizeof(*top)); | 269 top = (Stktop*)(m->morebuf.sp - sizeof(*top)); |
230 stk = (byte*)gp->stackguard - StackGuard; | 270 stk = (byte*)gp->stackguard - StackGuard; |
231 free = 0; | 271 free = 0; |
232 } else { | 272 } else { |
233 // allocate new segment. | 273 // allocate new segment. |
234 framesize += argsize; | 274 framesize += argsize; |
235 framesize += StackExtra; // room for more functions, Stkt
op. | 275 framesize += StackExtra; // room for more functions, Stkt
op. |
236 if(framesize < StackMin) | 276 if(framesize < StackMin) |
237 framesize = StackMin; | 277 framesize = StackMin; |
238 framesize += StackSystem; | 278 framesize += StackSystem; |
239 stk = runtime·stackalloc(framesize); | 279 stk = runtime·stackalloc(framesize); |
240 top = (Stktop*)(stk+framesize-sizeof(*top)); | 280 top = (Stktop*)(stk+framesize-sizeof(*top)); |
241 free = framesize; | 281 free = framesize; |
242 } | 282 } |
243 | 283 |
244 » if(0) { | 284 » if(StackDebug) { |
245 » » runtime·printf("newstack framesize=%d argsize=%d morepc=%p morea
rgp=%p gobuf=%p, %p top=%p old=%p\n", | 285 » » runtime·printf("\t-> new stack [%p, %p]\n", stk, top); |
246 » » » framesize, argsize, m->morepc, m->moreargp, m->morebuf.p
c, m->morebuf.sp, top, gp->stackbase); | |
247 } | 286 } |
248 | 287 |
249 top->stackbase = gp->stackbase; | 288 top->stackbase = gp->stackbase; |
250 top->stackguard = gp->stackguard; | 289 top->stackguard = gp->stackguard; |
251 top->gobuf = m->morebuf; | 290 top->gobuf = m->morebuf; |
252 top->argp = m->moreargp; | 291 top->argp = m->moreargp; |
253 top->argsize = argsize; | 292 top->argsize = argsize; |
254 top->free = free; | 293 top->free = free; |
255 m->moreargp = nil; | 294 m->moreargp = nil; |
256 m->morebuf.pc = (uintptr)nil; | 295 m->morebuf.pc = (uintptr)nil; |
| 296 m->morebuf.lr = (uintptr)nil; |
257 m->morebuf.sp = (uintptr)nil; | 297 m->morebuf.sp = (uintptr)nil; |
258 | 298 |
259 // copy flag from panic | 299 // copy flag from panic |
260 top->panic = gp->ispanic; | 300 top->panic = gp->ispanic; |
261 gp->ispanic = false; | 301 gp->ispanic = false; |
262 | 302 |
263 gp->stackbase = (uintptr)top; | 303 gp->stackbase = (uintptr)top; |
264 gp->stackguard = (uintptr)stk + StackGuard; | 304 gp->stackguard = (uintptr)stk + StackGuard; |
265 » // If preemption was requested but can't preempt right now, | 305 » gp->stackguard0 = gp->stackguard; |
266 » // leave StackPreempt there to try to preempt next time. | |
267 » if(gp->stackguard0 != StackPreempt || preempt) | |
268 » » gp->stackguard0 = gp->stackguard; | |
269 | 306 |
270 sp = (uintptr)top; | 307 sp = (uintptr)top; |
271 if(argsize > 0) { | 308 if(argsize > 0) { |
272 sp -= argsize; | 309 sp -= argsize; |
273 dst = (uintptr*)sp; | 310 dst = (uintptr*)sp; |
274 dstend = dst + argsize/sizeof(*dst); | 311 dstend = dst + argsize/sizeof(*dst); |
275 src = (uintptr*)top->argp; | 312 src = (uintptr*)top->argp; |
276 while(dst < dstend) | 313 while(dst < dstend) |
277 *dst++ = *src++; | 314 *dst++ = *src++; |
278 } | 315 } |
279 if(thechar == '5') { | 316 if(thechar == '5') { |
280 // caller would have saved its LR below args. | 317 // caller would have saved its LR below args. |
281 sp -= sizeof(void*); | 318 sp -= sizeof(void*); |
282 *(void**)sp = nil; | 319 *(void**)sp = nil; |
283 } | 320 } |
284 | 321 |
285 // Continue as if lessstack had just called m->morepc | 322 // Continue as if lessstack had just called m->morepc |
286 // (the PC that decided to grow the stack). | 323 // (the PC that decided to grow the stack). |
287 runtime·memclr((byte*)&label, sizeof label); | 324 runtime·memclr((byte*)&label, sizeof label); |
288 label.sp = sp; | 325 label.sp = sp; |
289 label.pc = (uintptr)runtime·lessstack; | 326 label.pc = (uintptr)runtime·lessstack; |
290 label.g = m->curg; | 327 label.g = m->curg; |
291 if(reflectcall) | 328 if(reflectcall) |
292 » » runtime·gostartcallfn(&label, (FuncVal*)m->morepc); | 329 » » runtime·gostartcallfn(&label, (FuncVal*)m->cret); |
293 else { | 330 else { |
294 » » // The stack growth code saves ctxt (not ret) in m->cret. | 331 » » runtime·gostartcall(&label, (void(*)(void))gp->sched.pc, gp->sch
ed.ctxt); |
295 » » runtime·gostartcall(&label, m->morepc, (void*)m->cret); | 332 » » gp->sched.ctxt = nil; |
296 » » m->cret = 0; | 333 » } |
297 » } | 334 » gp->status = oldstatus; |
298 » if(preempt) { | |
299 » » gp->preempted = 1; | |
300 » » gp->sched = label; | |
301 » » runtime·gosched0(gp); | |
302 » » *(int32*)678 = 123;» // never return | |
303 » } | |
304 runtime·gogo(&label); | 335 runtime·gogo(&label); |
305 | 336 |
306 *(int32*)345 = 123; // never return | 337 *(int32*)345 = 123; // never return |
307 } | 338 } |
308 | 339 |
309 // adjust Gobuf as if it executed a call to fn | 340 // adjust Gobuf as if it executed a call to fn |
310 // and then did an immediate gosave. | 341 // and then did an immediate gosave. |
311 void | 342 void |
312 runtime·gostartcallfn(Gobuf *gobuf, FuncVal *fv) | 343 runtime·gostartcallfn(Gobuf *gobuf, FuncVal *fv) |
313 { | 344 { |
314 runtime·gostartcall(gobuf, fv->fn, fv); | 345 runtime·gostartcall(gobuf, fv->fn, fv); |
315 } | 346 } |
LEFT | RIGHT |