code review 10136043: runtime: refactor mallocgc

src/pkg/runtime/proc.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 #include "runtime.h"
 #include "arch_GOARCH.h"
 #include "malloc.h"
 #include "stack.h"
 #include "race.h"
 #include "type.h"
@@ skipping 69 matching lines @@
 static void runqgrow(P*);
 static G* runqsteal(P*, P*);
 static void mput(M*);
 static M* mget(void);
 static void mcommoninit(M*);
 static void schedule(void);
 static void procresize(int32);
 static void acquirep(P*);
 static P* releasep(void);
 static void newm(void(*)(void), P*);
-static void goidle(void);
 static void stopm(void);
 static void startm(P*, bool);
 static void handoffp(P*);
 static void wakep(void);
 static void stoplockedm(void);
 static void startlockedm(G*);
 static void sysmon(void);
-static uint32 retake(uint32*);
+static uint32 retake(int64);
 static void inclocked(int32);
 static void checkdead(void);
 static void exitsyscall0(G*);
 static void park0(G*);
-static void gosched0(G*);
 static void goexit0(G*);
 static void gfput(P*, G*);
 static G* gfget(P*);
 static void gfpurge(P*);
 static void globrunqput(G*);
-static G* globrunqget(P*);
+static G* globrunqget(P*, int32);
 static P* pidleget(void);
 static void pidleput(P*);
 static void injectglist(G*);
 static void preemptall(void);
 static void preemptone(P*);
 
 // The bootstrap sequence is:
 //
 //      call osinit
 //      call schedinit
 //      make & queue new G
 //      call runtime·mstart
 //
 // The new G calls runtime·main.
 void
 runtime·schedinit(void)
 {
         int32 n, procs;
         byte *p;
 
         m->nomemprof++;
         runtime·mprofinit();
         runtime·mallocinit();
         mcommoninit(m);
 
         runtime·goargs();
         runtime·goenvs();
+        runtime·parsedebugvars();
 
         // Allocate internal symbol table representation now, we need it for GC anyway.
         runtime·symtabinit();
 
         runtime·sched.lastpoll = runtime·nanotime();
         procs = 1;
         p = runtime·getenv("GOMAXPROCS");
         if(p != nil && (n = runtime·atoi(p)) > 0) {
                 if(n > MaxGomaxprocs)
                         n = MaxGomaxprocs;
@@ skipping 80 matching lines @@
         runtime·printf("goroutine %D [%s]:\n", gp->goid, status);
 }
 
 void
 runtime·tracebackothers(G *me)
 {
         G *gp;
         int32 traceback;
 
         traceback = runtime·gotraceback(nil);
+········
+        // Show the current goroutine first, if we haven't already.
+        if((gp = m->curg) != nil && gp != me) {
+                runtime·printf("\n");
+                runtime·goroutineheader(gp);
+                runtime·traceback(gp->sched.pc, gp->sched.sp, gp->sched.lr, gp);
+        }
+
         for(gp = runtime·allg; gp != nil; gp = gp->alllink) {
-                if(gp == me || gp->status == Gdead)
+                if(gp == me || gp == m->curg || gp->status == Gdead)
                         continue;
                 if(gp->issystem && traceback < 2)
                         continue;
                 runtime·printf("\n");
                 runtime·goroutineheader(gp);
-                runtime·traceback(gp->sched.pc, gp->sched.sp, 0, gp);
+                if(gp->status == Grunning)
+                        runtime·printf("\tgoroutine running on other thread; stack unavailable\n");
+                else
+                        runtime·traceback(gp->sched.pc, gp->sched.sp, gp->sched.lr, gp);
         }
 }
 
 static void
 mcommoninit(M *mp)
 {
         // If there is no mcache runtime·callers() will crash,
         // and we are most likely in sysmon thread so the stack is senseless anyway.
         if(m->mcache)
                 runtime·callers(1, mp->createstack, nelem(mp->createstack));
@@ skipping 22 matching lines @@
         m->locks++;  // disable preemption because it can be holding p in a local var
         if(gp->status != Gwaiting) {
                 runtime·printf("goroutine %D has status %d\n", gp->goid, gp->status);
                 runtime·throw("bad g->status in ready");
         }
         gp->status = Grunnable;
         runqput(m->p, gp);
         if(runtime·atomicload(&runtime·sched.npidle) != 0 && runtime·atomicload(&runtime·sched.nmspinning) == 0)  // TODO: fast atomic
                 wakep();
         m->locks--;
+        if(m->locks == 0 && g->preempt)  // restore the preemption request in case we've cleared it in newstack
+                g->stackguard0 = StackPreempt;
 }
 
 int32
 runtime·gcprocs(void)
 {
         int32 n;
 
         // Figure out how many CPUs to use during GC.
         // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
         runtime·lock(&runtime·sched);
@@ skipping 50 matching lines @@
 runtime·stoptheworld(void)
 {
         int32 i;
         uint32 s;
         P *p;
         bool wait;
 
         runtime·lock(&runtime·sched);
         runtime·sched.stopwait = runtime·gomaxprocs;
         runtime·atomicstore((uint32*)&runtime·gcwaiting, 1);
+        preemptall();
         // stop current P
         m->p->status = Pgcstop;
         runtime·sched.stopwait--;
         // try to retake all P's in Psyscall status
         for(i = 0; i < runtime·gomaxprocs; i++) {
                 p = runtime·allp[i];
                 s = p->status;
                 if(s == Psyscall && runtime·cas(&p->status, s, Pgcstop))
                         runtime·sched.stopwait--;
         }
         // stop idle P's
         while(p = pidleget()) {
                 p->status = Pgcstop;
                 runtime·sched.stopwait--;
         }
         wait = runtime·sched.stopwait > 0;
         runtime·unlock(&runtime·sched);
 
-        // wait for remaining P's to stop voluntary
+        // wait for remaining P's to stop voluntarily
         if(wait) {
-                runtime·notesleep(&runtime·sched.stopnote);
-                runtime·noteclear(&runtime·sched.stopnote);
+                for(;;) {
+                        // wait for 100us, then try to re-preempt in case of any races
+                        if(runtime·notetsleep(&runtime·sched.stopnote, 100*1000)) {
+                                runtime·noteclear(&runtime·sched.stopnote);
+                                break;
+                        }
+                        preemptall();
+                }
         }
         if(runtime·sched.stopwait)
                 runtime·throw("stoptheworld: not stopped");
         for(i = 0; i < runtime·gomaxprocs; i++) {
                 p = runtime·allp[i];
                 if(p->status != Pgcstop)
                         runtime·throw("stoptheworld: not stopped");
         }
 }
 
@@ skipping 62 matching lines @@
                 // If GC could have used another helper proc, start one now,
                 // in the hope that it will be available next time.
                 // It would have been even better to start it before the collection,
                 // but doing so requires allocating memory, so it's tricky to
                 // coordinate.  This lazy approach works out in practice:
                 // we don't mind if the first couple gc rounds don't have quite
                 // the maximum number of procs.
                 newm(mhelpgc, nil);
         }
         m->locks--;
+        if(m->locks == 0 && g->preempt)  // restore the preemption request in case we've cleared it in newstack
+                g->stackguard0 = StackPreempt;
 }
 
 // Called to start an M.
 void
 runtime·mstart(void)
 {
+#ifdef GOOS_windows
+#ifdef GOARCH_386
         // It is used by windows-386 only. Unfortunately, seh needs
         // to be located on os stack, and mstart runs on os stack
         // for both m0 and m.
         SEH seh;
+#endif
+#endif
 
         if(g != m->g0)
                 runtime·throw("bad runtime·mstart");
 
         // Record top of stack for use by mcall.
         // Once we call schedule we're never coming back,
         // so other calls can reuse this stack space.
         runtime·gosave(&m->g0->sched);
         m->g0->sched.pc = (uintptr)-1;  // make sure it is never used
         m->g0->stackguard = m->g0->stackguard0;  // cgo sets only stackguard0, copy it to stackguard
+#ifdef GOOS_windows
+#ifdef GOARCH_386
         m->seh = &seh;
+#endif
+#endif
         runtime·asminit();
         runtime·minit();
 
         // Install signal handlers; after minit so that minit can
         // prepare the thread to be able to handle the signals.
         if(m == &runtime·m0)
                 runtime·initsig();
 ········
         if(m->mstartfn)
                 m->mstartfn();
@@ skipping 48 matching lines @@
         // In case of cgo, pthread_create will make us a stack.
         // Windows will layout sched stack on OS stack.
         if(runtime·iscgo || Windows)
                 mp->g0 = runtime·malg(-1);
         else
                 mp->g0 = runtime·malg(8192);
 
         if(p == m->p)
                 releasep();
         m->locks--;
+        if(m->locks == 0 && g->preempt)  // restore the preemption request in case we've cleared it in newstack
+                g->stackguard0 = StackPreempt;
 
         return mp;
 }
 
 static M* lockextra(bool nilokay);
 static void unlockextra(M*);
 
 // needm is called when a cgo callback happens on a
 // thread without an m (a thread not created by Go).
 // In this case, needm is expected to find an m to use
@@ skipping 60 matching lines @@
         // Install m and g (= m->g0) and set the stack bounds
         // to match the current stack. We don't actually know
         // how big the stack is, like we don't know how big any
         // scheduling stack is, but we assume there's at least 32 kB,
         // which is more than enough for us.
         runtime·setmg(mp, mp->g0);
         g->stackbase = (uintptr)(&x + 1024);
         g->stackguard = (uintptr)(&x - 32*1024);
         g->stackguard0 = g->stackguard;
 
+#ifdef GOOS_windows
+#ifdef GOARCH_386
         // On windows/386, we need to put an SEH frame (two words)
-        // somewhere on the current stack. We are called
-        // from needm, and we know there is some available
-        // space one word into the argument frame. Use that.
+        // somewhere on the current stack. We are called from cgocallback_gofunc
+        // and we know that it will leave two unused words below m->curg->sched.sp.
+        // Use those.
         m->seh = (SEH*)((uintptr*)&x + 1);
+#endif
+#endif
 
         // Initialize this thread to use the m.
         runtime·asminit();
         runtime·minit();
 }
 
 // newextram allocates an m and puts it on the extra list.
 // It is called with a working local m, so that it can do things
 // like call schedlock and allocate.
 void
 runtime·newextram(void)
 {
         M *mp, *mnext;
         G *gp;
 
         // Create extra goroutine locked to extra m.
         // The goroutine is the context in which the cgo callback will run.
         // The sched.pc will never be returned to, but setting it to
         // runtime.goexit makes clear to the traceback routines where
         // the goroutine stack ends.
         mp = runtime·allocm(nil);
         gp = runtime·malg(4096);
         gp->sched.pc = (uintptr)runtime·goexit;
         gp->sched.sp = gp->stackbase;
+        gp->sched.lr = 0;
         gp->sched.g = gp;
         gp->status = Gsyscall;
         mp->curg = gp;
         mp->locked = LockInternal;
         mp->lockedg = gp;
         gp->lockedm = mp;
         // put on allg for garbage collector
         runtime·lock(&runtime·sched);
         if(runtime·lastg == nil)
                 runtime·allg = gp;
@@ skipping 34 matching lines @@
 // in which dropm happens on each cgo call, is still correct too.
 // We may have to keep the current version on systems with cgo
 // but without pthreads, like Windows.
 void
 runtime·dropm(void)
 {
         M *mp, *mnext;
 
         // Undo whatever initialization minit did during needm.
         runtime·unminit();
+
+#ifdef GOOS_windows
+#ifdef GOARCH_386
         m->seh = nil;  // reset dangling typed pointer
+#endif
+#endif
 
         // Clear m and g, and return m to the extra list.
         // After the call to setmg we can only call nosplit functions.
         mp = m;
         runtime·setmg(nil, nil);
 
         mnext = lockextra(true);
         mp->schedlink = mnext;
         unlockextra(mp);
 }
@@ skipping 259 matching lines @@
 static void
 execute(G *gp)
 {
         int32 hz;
 
         if(gp->status != Grunnable) {
                 runtime·printf("execute: bad g status %d\n", gp->status);
                 runtime·throw("execute: bad g status");
         }
         gp->status = Grunning;
+        gp->preempt = false;
         gp->stackguard0 = gp->stackguard;
         m->p->tick++;
         m->curg = gp;
         gp->m = m;
 
         // Check whether the profiler needs to be turned on or off.
         hz = runtime·sched.profilehz;
         if(m->profilehz != hz)
                 runtime·resetcpuprofiler(hz);
 
@@ skipping 14 matching lines @@
                 gcstopm();
                 goto top;
         }
         // local runq
         gp = runqget(m->p);
         if(gp)
                 return gp;
         // global runq
         if(runtime·sched.runqsize) {
                 runtime·lock(&runtime·sched);
-                gp = globrunqget(m->p);
+                gp = globrunqget(m->p, 0);
                 runtime·unlock(&runtime·sched);
                 if(gp)
                         return gp;
         }
         // poll network
         gp = runtime·netpoll(false);  // non-blocking
         if(gp) {
                 injectglist(gp->schedlink);
                 gp->status = Grunnable;
                 return gp;
@@ skipping 20 matching lines @@
                         return gp;
         }
 stop:
         // return P and block
         runtime·lock(&runtime·sched);
         if(runtime·gcwaiting) {
                 runtime·unlock(&runtime·sched);
                 goto top;
         }
         if(runtime·sched.runqsize) {
-                gp = globrunqget(m->p);
+                gp = globrunqget(m->p, 0);
                 runtime·unlock(&runtime·sched);
                 return gp;
         }
         p = releasep();
         pidleput(p);
         runtime·unlock(&runtime·sched);
         if(m->spinning) {
                 m->spinning = false;
                 runtime·xadd(&runtime·sched.nmspinning, -1);
         }
@@ skipping 29 matching lines @@
                                 gp->status = Grunnable;
                                 return gp;
                         }
                         injectglist(gp);
                 }
         }
         stopm();
         goto top;
 }
 
+static void
+resetspinning(void)
+{
+        int32 nmspinning;
+
+        if(m->spinning) {
+                m->spinning = false;
+                nmspinning = runtime·xadd(&runtime·sched.nmspinning, -1);
+                if(nmspinning < 0)
+                        runtime·throw("findrunnable: negative nmspinning");
+        } else
+                nmspinning = runtime·atomicload(&runtime·sched.nmspinning);
+
+        // M wakeup policy is deliberately somewhat conservative (see nmspinning handling),
+        // so see if we need to wakeup another P here.
+        if (nmspinning == 0 && runtime·atomicload(&runtime·sched.npidle) > 0)
+                wakep();
+}
+
 // Injects the list of runnable G's into the scheduler.
 // Can run concurrently with GC.
 static void
 injectglist(G *glist)
 {
         int32 n;
         G *gp;
 
         if(glist == nil)
                 return;
         runtime·lock(&runtime·sched);
         for(n = 0; glist; n++) {
                 gp = glist;
                 glist = gp->schedlink;
                 gp->status = Grunnable;
                 globrunqput(gp);
         }
         runtime·unlock(&runtime·sched);
 
         for(; n && runtime·sched.npidle; n--)
                 startm(nil, false);
 }
 
 // One round of scheduler: find a runnable goroutine and execute it.
 // Never returns.
 static void
 schedule(void)
 {
         G *gp;
+        uint32 tick;
 
         if(m->locks)
                 runtime·throw("schedule: holding locks");
 
 top:
         if(runtime·gcwaiting) {
                 gcstopm();
                 goto top;
         }
 
-        gp = runqget(m->p);
-        if(gp && m->spinning)
-                runtime·throw("schedule: spinning with local work");
-        if(gp == nil)
-                gp = findrunnable();
-
-        if(m->spinning) {
-                m->spinning = false;
-                runtime·xadd(&runtime·sched.nmspinning, -1);
-        }
-
-        // M wakeup policy is deliberately somewhat conservative (see nmspinning handling),
-        // so see if we need to wakeup another M here.
-        if (m->p->runqhead != m->p->runqtail &&
-                runtime·atomicload(&runtime·sched.nmspinning) == 0 &&
-                runtime·atomicload(&runtime·sched.npidle) > 0)  // TODO: fast atomic
-                wakep();
+        gp = nil;
+        // Check the global runnable queue once in a while to ensure fairness.
+        // Otherwise two goroutines can completely occupy the local runqueue
+        // by constantly respawning each other.
+        tick = m->p->tick;
+        // This is a fancy way to say tick%61==0,
+        // it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors.
+        if(tick - (((uint64)tick*0x4325c53fu)>>36)*61 == 0 && runtime·sched.runqsize > 0) {
+                runtime·lock(&runtime·sched);
+                gp = globrunqget(m->p, 1);
+                runtime·unlock(&runtime·sched);
+                if(gp)
+                        resetspinning();
+        }
+        if(gp == nil) {
+                gp = runqget(m->p);
+                if(gp && m->spinning)
+                        runtime·throw("schedule: spinning with local work");
+        }
+        if(gp == nil) {
+                gp = findrunnable();  // blocks until work is available
+                resetspinning();
+        }
 
         if(gp->lockedm) {
+                // Hands off own p to the locked m,
+                // then blocks waiting for a new p.
                 startlockedm(gp);
                 goto top;
         }
 
         execute(gp);
 }
 
 // Puts the current goroutine into a waiting state and unlocks the lock.
 // The goroutine can be made runnable again by calling runtime·ready(gp).
 void
@@ skipping 21 matching lines @@
                 stoplockedm();
                 execute(gp);  // Never returns.
         }
         schedule();
 }
 
 // Scheduler yield.
 void
 runtime·gosched(void)
 {
-        runtime·mcall(gosched0);
+        runtime·mcall(runtime·gosched0);
 }
 
 // runtime·gosched continuation on g0.
-static void
-gosched0(G *gp)
+void
+runtime·gosched0(G *gp)
 {
         gp->status = Grunnable;
         gp->m = nil;
         m->curg = nil;
         runtime·lock(&runtime·sched);
         globrunqput(gp);
         runtime·unlock(&runtime·sched);
         if(m->lockedg) {
                 stoplockedm();
                 execute(gp);  // Never returns.
@@ skipping 17 matching lines @@
 // runtime·goexit continuation on g0.
 static void
 goexit0(G *gp)
 {
         gp->status = Gdead;
         gp->m = nil;
         gp->lockedm = nil;
         m->curg = nil;
         m->lockedg = nil;
         if(m->locked & ~LockExternal) {
-                runtime·printf("invalid m->locked = %d", m->locked);
+                runtime·printf("invalid m->locked = %d\n", m->locked);
                 runtime·throw("internal lockOSThread error");
         }·······
         m->locked = 0;
         runtime·unwindstack(gp, nil);
         gfput(m->p, gp);
         schedule();
 }
 
 static void
 save(void *pc, uintptr sp)
@@ skipping 109 matching lines @@
         if(m->p && m->p->status == Psyscall && runtime·cas(&m->p->status, Psyscall, Prunning)) {
                 // There's a cpu for us, so we can run.
                 m->mcache = m->p->mcache;
                 m->p->m = m;
                 m->p->tick++;
                 g->status = Grunning;
                 // Garbage collector isn't running (since we are),
                 // so okay to clear gcstack and gcsp.
                 g->gcstack = (uintptr)nil;
                 g->gcsp = (uintptr)nil;
+                if(g->preempt)  // restore the preemption request in case we've cleared it in newstack
+                        g->stackguard0 = StackPreempt;
                 return;
         }
 
         if(g->isbackground)  // do not consider blocked scavenger for deadlock detection
                 inclocked(-1);
         // Try to get any other idle P.
         m->p = nil;
         if(runtime·sched.pidle) {
                 runtime·lock(&runtime·sched);
                 p = pidleget();
                 runtime·unlock(&runtime·sched);
                 if(p) {
                         acquirep(p);
                         m->p->tick++;
                         g->status = Grunning;
                         g->gcstack = (uintptr)nil;
                         g->gcsp = (uintptr)nil;
+                        if(g->preempt)  // restore the preemption request in case we've cleared it in newstack
+                                g->stackguard0 = StackPreempt;
                         return;
                 }
         }
 
         // Call the scheduler.
         runtime·mcall(exitsyscall0);
 
         // Scheduler returned, so we're allowed to run now.
         // Delete the gcstack information that we left for
         // the garbage collector during the system call.
@@ skipping 150 matching lines @@
         newg->gopc = (uintptr)callerpc;
         newg->status = Grunnable;
         newg->goid = runtime·xadd64(&runtime·sched.goidgen, 1);
         if(raceenabled)
                 newg->racectx = runtime·racegostart((void*)callerpc);
         runqput(m->p, newg);
 
         if(runtime·atomicload(&runtime·sched.npidle) != 0 && runtime·atomicload(&runtime·sched.nmspinning) == 0 && fn->fn != runtime·main)  // TODO: fast atomic
                 wakep();
         m->locks--;
+        if(m->locks == 0 && g->preempt)  // restore the preemption request in case we've cleared it in newstack
+                g->stackguard0 = StackPreempt;
         return newg;
 }
 
 // Put on gfree list.
 // If local list is too long, transfer a batch to the global list.
 static void
 gfput(P *p, G *gp)
 {
         if(gp->stackguard - StackGuard != gp->stack0)
                 runtime·throw("invalid stack in gfput");
@@ skipping 221 matching lines @@
         if(!Windows && (m == nil || m->mcache == nil))
                 return;
         if(prof.fn == nil || prof.hz == 0)
                 return;
 
         runtime·lock(&prof);
         if(prof.fn == nil) {
                 runtime·unlock(&prof);
                 return;
         }
-        n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf), nil, nil);
+        n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf), nil, nil, false);
         if(n > 0)
                 prof.fn(prof.pcbuf, n);
         runtime·unlock(&prof);
 }
 
 // Arrange to call fn with a traceback hz times a second.
 void
 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz)
 {
         // Force sane arguments.
@@ skipping 176 matching lines @@
         m->throwing = -1;  // do not dump full stacks
         runtime·throw("all goroutines are asleep - deadlock!");
 }
 
 static void
 sysmon(void)
 {
         uint32 idle, delay;
         int64 now, lastpoll;
         G *gp;
-        uint32 ticks[MaxGomaxprocs];
 
         idle = 0;  // how many cycles in succession we had not wokeup somebody
         delay = 0;
         for(;;) {
                 if(idle == 0)  // start with 20us sleep...
                         delay = 20;
                 else if(idle > 50)  // start doubling the sleep after 1ms...
                         delay *= 2;
                 if(delay > 10*1000)  // up to 10ms
                         delay = 10*1000;
                 runtime·usleep(delay);
                 if(runtime·gcwaiting || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs) {  // TODO: fast atomic
                         runtime·lock(&runtime·sched);
                         if(runtime·atomicload(&runtime·gcwaiting) || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs) {
                                 runtime·atomicstore(&runtime·sched.sysmonwait, 1);
                                 runtime·unlock(&runtime·sched);
                                 runtime·notesleep(&runtime·sched.sysmonnote);
                                 runtime·noteclear(&runtime·sched.sysmonnote);
                                 idle = 0;
                                 delay = 20;
                         } else
                                 runtime·unlock(&runtime·sched);
                 }
                 // poll network if not polled for more than 10ms
                 lastpoll = runtime·atomicload64(&runtime·sched.lastpoll);
                 now = runtime·nanotime();
                 if(lastpoll != 0 && lastpoll + 10*1000*1000 > now) {
+                        runtime·cas64(&runtime·sched.lastpoll, lastpoll, now);
                         gp = runtime·netpoll(false);  // non-blocking
                         injectglist(gp);
                 }
                 // retake P's blocked in syscalls
-                if(retake(ticks))
+                // and preempt long running G's
+                if(retake(now))
                         idle = 0;
                 else
                         idle++;
         }
 }
 
+typedef struct Pdesc Pdesc;
+struct Pdesc
+{
+        uint32  tick;
+        int64   when;
+};
+static Pdesc pdesc[MaxGomaxprocs];
+
 static uint32
-retake(uint32 *ticks)
+retake(int64 now)
 {
         uint32 i, s, n;
         int64 t;
         P *p;
+        Pdesc *pd;
 
         n = 0;
         for(i = 0; i < runtime·gomaxprocs; i++) {
                 p = runtime·allp[i];
                 if(p==nil)
                         continue;
                 t = p->tick;
-                if(ticks[i] != t) {
-                        ticks[i] = t;
+                pd = &pdesc[i];
+                if(pd->tick != t) {
+                        pd->tick = t;
+                        pd->when = now;
                         continue;
                 }
                 s = p->status;
-                if(s != Psyscall)
-                        continue;
-                if(p->runqhead == p->runqtail && runtime·atomicload(&runtime·sched.nmspinning) + runtime·atomicload(&runtime·sched.npidle) > 0)  // TODO: fast atomic
-                        continue;
-                // Need to increment number of locked M's before the CAS.
-                // Otherwise the M from which we retake can exit the syscall,
-                // increment nmidle and report deadlock.
-                inclocked(-1);
-                if(runtime·cas(&p->status, s, Pidle)) {
-                        n++;
-                        handoffp(p);
-                }
-                inclocked(1);
+                if(s == Psyscall) {
+                        // Retake P from syscall if it's there for more than 1 sysmon tick (20us).
+                        // But only if there is other work to do.
+                        if(p->runqhead == p->runqtail &&
+                                runtime·atomicload(&runtime·sched.nmspinning) + runtime·atomicload(&runtime·sched.npidle) > 0)
+                                continue;
+                        // Need to increment number of locked M's before the CAS.
+                        // Otherwise the M from which we retake can exit the syscall,
+                        // increment nmidle and report deadlock.
+                        inclocked(-1);
+                        if(runtime·cas(&p->status, s, Pidle)) {
+                                n++;
+                                handoffp(p);
+                        }
+                        inclocked(1);
+                } else if(s == Prunning) {
+                        // Preempt G if it's running for more than 10ms.
+                        if(pd->when + 10*1000*1000 > now)
+                                continue;
+                        preemptone(p);
+                }
         }
         return n;
 }
 
 // Tell all goroutines that they have been preempted and they should stop.
 // This function is purely best-effort.  It can fail to inform a goroutine if a
 // processor just started running it.
 // No locks need to be held.
 static void
 preemptall(void)
@@ skipping 14 matching lines @@
 // goroutine.  It can send inform the wrong goroutine.  Even if it informs the
 // correct goroutine, that goroutine might ignore the request if it is
 // simultaneously executing runtime·newstack.
 // No lock needs to be held.
 static void
 preemptone(P *p)
 {
         M *mp;
         G *gp;
 
+// Preemption requires more robust traceback routines.
+// For now, disable.
+// The if(1) silences a compiler warning about the rest of the
+// function being unreachable.
+if(1) return;
+
         mp = p->m;
         if(mp == nil || mp == m)
                 return;
         gp = mp->curg;
         if(gp == nil || gp == mp->g0)
                 return;
+        gp->preempt = true;
         gp->stackguard0 = StackPreempt;
 }
 
 // Put mp on midle list.
 // Sched must be locked.
 static void
 mput(M *mp)
 {
         mp->schedlink = runtime·sched.midle;
         runtime·sched.midle = mp;
@@ skipping 25 matching lines @@
                 runtime·sched.runqtail->schedlink = gp;
         else
                 runtime·sched.runqhead = gp;
         runtime·sched.runqtail = gp;
         runtime·sched.runqsize++;
 }
 
 // Try get a batch of G's from the global runnable queue.
 // Sched must be locked.
 static G*
-globrunqget(P *p)
+globrunqget(P *p, int32 max)
 {
         G *gp, *gp1;
         int32 n;
 
         if(runtime·sched.runqsize == 0)
                 return nil;
         n = runtime·sched.runqsize/runtime·gomaxprocs+1;
         if(n > runtime·sched.runqsize)
                 n = runtime·sched.runqsize;
+        if(max > 0 && n > max)
+                n = max;
         runtime·sched.runqsize -= n;
         if(runtime·sched.runqsize == 0)
                 runtime·sched.runqtail = nil;
         gp = runtime·sched.runqhead;
         runtime·sched.runqhead = gp->schedlink;
         n--;
         while(n--) {
                 gp1 = runtime·sched.runqhead;
                 runtime·sched.runqhead = gp1->schedlink;
                 runqput(p, gp1);
@@ skipping 232 matching lines @@
                         }
                 }
                 if(s != i/2 && s != i/2+1) {
                         runtime·printf("bad steal %d, want %d or %d, iter %d\n",
                                 s, i/2, i/2+1, i);
                         runtime·throw("bad steal");
                 }
         }
 }
 
+extern void runtime·morestack(void);
+
+// Does f mark the top of a goroutine stack?
 bool
-runtime·haszeroargs(uintptr pc)
-{
-        return pc == (uintptr)runtime·goexit ||
-                pc == (uintptr)runtime·mcall ||
-                pc == (uintptr)runtime·mstart ||
-                pc == (uintptr)_rt0_go;
-}
-
+runtime·topofstack(Func *f)
+{
+        return f->entry == (uintptr)runtime·goexit ||
+                f->entry == (uintptr)runtime·mstart ||
+                f->entry == (uintptr)runtime·mcall ||
+                f->entry == (uintptr)runtime·morestack ||
+                f->entry == (uintptr)runtime·lessstack ||
+                f->entry == (uintptr)_rt0_go;
+}