code review 137070043: runtime: adapt race detector for runtime written in Goadapt

src/pkg/runtime/proc.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 #include "runtime.h"
 #include "arch_GOARCH.h"
 #include "zaexperiment.h"
 #include "malloc.h"
 #include "stack.h"
 #include "race.h"
@@ skipping 110 matching lines @@
 static void globrunqputbatch(G*, G*, int32);
 static G* globrunqget(P*, int32);
 static P* pidleget(void);
 static void pidleput(P*);
 static void injectglist(G*);
 static bool preemptall(void);
 static bool preemptone(P*);
 static bool exitsyscallfast(void);
 static bool haveexperiment(int8*);
 static void allgadd(G*);
+static void dropg(void);
 
 extern String runtime·buildVersion;
 
 // The bootstrap sequence is:
 //
 //      call osinit
 //      call schedinit
 //      make & queue new G
 //      call runtime·mstart
 //
@@ skipping 37 matching lines @@
                         n = MaxGomaxprocs;
                 procs = n;
         }
         procresize(procs);
 
         runtime·copystack = runtime·precisestack;
         p = runtime·getenv("GOCOPYSTACK");
         if(p != nil && !runtime·strcmp(p, (byte*)"0"))
                 runtime·copystack = false;
 
-        mstats.enablegc = 1;
-
         if(runtime·buildVersion.str == nil) {
                 // Condition should never trigger.  This code just serves
                 // to ensure runtime·buildVersion is kept in the resulting binary.
                 runtime·buildVersion.str = (uint8*)"unknown";
                 runtime·buildVersion.len = 7;
         }
 }
 
 extern void main·init(void);
 extern void runtime·init(void);
@@ skipping 41 matching lines @@
         // Defer unlock so that runtime.Goexit during init does the unlock too.
         d.fn = &initDone;
         d.siz = 0;
         d.link = g->defer;
         d.argp = NoArgs;
         d.special = true;
         g->defer = &d;
 
         if(g->m != &runtime·m0)
                 runtime·throw("runtime·main not on m0");
+
         runtime·init();
+        mstats.enablegc = 1; // now that runtime is initialized, GC is okay
+
         main·init();
 
         if(g->defer != &d || d.fn != &initDone)
                 runtime·throw("runtime: bad defer entry after init");
         g->defer = d.link;
         runtime·unlockOSThread();
 
         main·main();
         if(raceenabled)
                 runtime·racefini();
 
         // Make racy client program work: if panicking on
         // another goroutine at the same time as main returns,
         // let the other goroutine finish printing the panic trace.
         // Once it does, it will exit. See issue 3934.
         if(runtime·panicking)
                 runtime·park(nil, nil, runtime·gostringnocopy((byte*)"panicwait"));
 
         runtime·exit(0);
         for(;;)
                 *(int32*)runtime·main = 0;
 }
 
-void
-runtime·goroutineheader(G *gp)
-{
-        String status;
-        int64 waitfor;
-        uint32 gpstatus;
-
-        gpstatus = runtime·readgstatus(gp);
-        switch(gpstatus) {
-        case Gidle:
-                status = runtime·gostringnocopy((byte*)"idle");
-                break;
-        case Grunnable:
-                status = runtime·gostringnocopy((byte*)"runnable");
-                break;
-        case Grunning:
-                status = runtime·gostringnocopy((byte*)"running");
-                break;
-        case Gsyscall:
-                status = runtime·gostringnocopy((byte*)"syscall");
-                break;
-        case Gwaiting:
-                if(gp->waitreason.str != nil)
-                        status = gp->waitreason;
-                else
-                        status = runtime·gostringnocopy((byte*)"waiting");
-                break;
-        case Gscan:
-                status = runtime·gostringnocopy((byte*)"scan");
-                break;
-        case Gscanrunnable:
-                status =  runtime·gostringnocopy((byte*)"scanrunnable");
-                break;
-        case Gscanrunning:
-                status = runtime·gostringnocopy((byte*)"scanrunning");
-                break;
-        case Gscansyscall:
-                status = runtime·gostringnocopy((byte*)"scansyscall");
-                break;
-        case Gscanenqueue:
-                status = runtime·gostringnocopy((byte*)"scanenqueue");
-                break;
-        case Gscanwaiting:
-                if(gp->waitreason.str != nil)
-                        status = gp->waitreason;
-                else
-                        status = runtime·gostringnocopy((byte*)"scanwaiting");
-                break;
-        case Gcopystack:
-                status = runtime·gostringnocopy((byte*)"copystack");
-                break;
-        default:
-                status = runtime·gostringnocopy((byte*)"???");
-                break;
-        }
-
-        // approx time the G is blocked, in minutes
-        waitfor = 0;
-        gpstatus = gpstatus&~Gscan; // drop the scan bit
-        if((gpstatus == Gwaiting || gpstatus == Gsyscall) && gp->waitsince != 0)
-                waitfor = (runtime·nanotime() - gp->waitsince) / (60LL*1000*1000*1000);
-
-        runtime·printf("goroutine %D [%S", gp->goid, status);
-        if(waitfor >= 1)
-                runtime·printf(", %D minutes", waitfor);
-        if(gp->lockedm != nil)
-                runtime·printf(", locked to thread");
-        runtime·printf("]:\n");
-}
-
 static void
 dumpgstatus(G* gp)
 {
-        runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp));
-}
-
-void
-runtime·tracebackothers(G *me)
-{
-        G *gp;
-        int32 traceback;
-        uintptr i;
-        uint32 status;
-
-        traceback = runtime·gotraceback(nil);
- ·······
-        // Show the current goroutine first, if we haven't already.
-        if((gp = g->m->curg) != nil && gp != me) {
-                runtime·printf("\n");
-                runtime·goroutineheader(gp);
-                runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp);
-        }
-
-        runtime·lock(&allglock);
-        for(i = 0; i < runtime·allglen; i++) {
-                gp = runtime·allg[i];
-                if(gp == me || gp == g->m->curg || runtime·readgstatus(gp) == Gdead)
-                        continue;
-                if(gp->issystem && traceback < 2)
-                        continue;
-                runtime·printf("\n");
-                runtime·goroutineheader(gp);
-                status = runtime·readgstatus(gp);
-                if((status&~Gscan) == Grunning){
-                        runtime·printf("\tgoroutine running on other thread; stack unavailable\n");
-                        runtime·printcreatedby(gp);
-                } else
-                        runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp);
-        }
-        runtime·unlock(&allglock);
+        runtime·printf("runtime: gp: gp=%p, goid=%D, gp->atomicstatus=%x\n", gp, gp->goid, runtime·readgstatus(gp));
+        runtime·printf("runtime:  g:  g=%p, goid=%D,  g->atomicstatus=%x\n", g, g->goid, runtime·readgstatus(g));
 }
 
 static void
 checkmcount(void)
 {
         // sched lock is held
         if(runtime·sched.mcount > runtime·sched.maxmcount){
                 runtime·printf("runtime: program exceeds %d-thread limit\n", runtime·sched.maxmcount);
                 runtime·throw("thread exhaustion");
         }
@@ skipping 215 matching lines @@
 // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that·
 // put it in the Gscan state is finished.
 void
 runtime·casgstatus(G *gp, uint32 oldval, uint32 newval)
 {
         if(isscanstatus(oldval) || isscanstatus(newval) || oldval == newval) {
                 runtime·printf("casgstatus: oldval=%d, newval=%d\n", oldval, newval);
                 runtime·throw("casgstatus: bad incoming values");
         }
 
+        // loop if gp->atomicstatus is in a scan state giving
+        // GC time to finish and change the state to oldval.
         while(!runtime·cas(&gp->atomicstatus, oldval, newval)) {
-                // loop if gp->atomicstatus is in a  scan state giving
-                // GC time to finish and change the state to oldval.
-        }
+                // Help GC if needed.·
+                if(gp->preemptscan && !gp->gcworkdone && (oldval == Grunning || oldval == Gsyscall)) {
+                        gp->preemptscan = false;
+                        runtime·gcphasework(gp);
+                }
+        } ······
+}
+
+// stopg ensures that gp is stopped at a GC safe point where its stack can be scanned
+// or in the context of a moving collector the pointers can be flipped from pointing·
+// to old object to pointing to new objects.·
+// If stopg returns true, the caller knows gp is at a GC safe point and will remain there until
+// the caller calls restartg.
+// If stopg returns false, the caller is not responsible for calling restartg. This can happen
+// if another thread, either the gp itself or another GC thread is taking the responsibility·
+// to do the GC work related to this thread.
+bool
+runtime·stopg(G *gp)
+{
+        uint32 s;
+
+        for(;;) {
+                if(gp->gcworkdone)
+                        return false;
+
+                s = runtime·readgstatus(gp);
+                switch(s) {
+                default:
+                        dumpgstatus(gp);
+                        runtime·throw("stopg: gp->atomicstatus is not valid");
+
+                case Gdead:
+                        return false;
+
+                case Gcopystack:
+                        // Loop until a new stack is in place.
+                        break;
+
+                case Grunnable:
+                case Gsyscall:
+                case Gwaiting:
+                        // Claim goroutine by setting scan bit.
+                        if(!runtime·castogscanstatus(gp, s, s|Gscan))
+                                break;
+                        // In scan state, do work.
+                        runtime·gcphasework(gp);
+                        return true;
+
+                case Gscanrunnable:
+                case Gscanwaiting:
+                case Gscansyscall:
+                        // Goroutine already claimed by another GC helper.
+                        return false;
+
+                case Grunning:
+                        // Claim goroutine, so we aren't racing with a status
+                        // transition away from Grunning.
+                        if(!runtime·castogscanstatus(gp, Grunning, Gscanrunning))
+                                break;
+
+                        // Mark gp for preemption.
+                        if(!gp->gcworkdone) {
+                                gp->preemptscan = true;
+                                gp->preempt = true;
+                                gp->stackguard0 = StackPreempt;
+                        }
+
+                        // Unclaim.
+                        runtime·casfromgscanstatus(gp, Gscanrunning, Grunning);
+                        return false;
+                }
+        }
+        // Should not be here....
+}
+
+// The GC requests that this routine be moved from a scanmumble state to a mumble state.
+void·
+runtime·restartg (G *gp)
+{
+        uint32 s;
+
+        s = runtime·readgstatus(gp);
+        switch(s) {
+        default:
+                dumpgstatus(gp);·
+                runtime·throw("restartg: unexpected status");
+
+        case Gdead:
+                break;
+
+        case Gscanrunnable:
+        case Gscanwaiting:
+        case Gscansyscall:
+                runtime·casfromgscanstatus(gp, s, s&~Gscan);
+                break;
+
+        case Gscanenqueue:
+                // Scan is now completed.
+                // Goroutine now needs to be made runnable.
+                // We put it on the global run queue; ready blocks on the global scheduler lock.
+                runtime·casfromgscanstatus(gp, Gscanenqueue, Gwaiting);
+                if(gp != g->m->curg)
+                        runtime·throw("processing Gscanenqueue on wrong m");
+                dropg();
+                runtime·ready(gp);
+                break;
+        }
+}
+
+static void
+stopscanstart(G* gp)
+{
+        if(g == gp)
+                runtime·throw("GC not moved to G0");
+        if(runtime·stopg(gp)) {
+                if(!isscanstatus(runtime·readgstatus(gp))) {
+                        dumpgstatus(gp);
+                        runtime·throw("GC not in scan state");
+                }
+                runtime·restartg(gp);
+        }
+}
+
+// Runs on g0 and does the actual work after putting the g back on the run queue.
+static void
+mquiesce(G *gpmaster)
+{
+        G* gp;
+        uint32 i;
+        uint32 status;
+        uint32 activeglen;
+
+        activeglen = runtime·allglen;
+        // enqueue the calling goroutine.
+        runtime·restartg(gpmaster);
+        for(i = 0; i < activeglen; i++) {
+                gp = runtime·allg[i];
+                if(runtime·readgstatus(gp) == Gdead)·
+                        gp->gcworkdone = true; // noop scan.
+                else·
+                        gp->gcworkdone = false;·
+                stopscanstart(gp);·
+        }
+
+        // Check that the G's gcwork (such as scanning) has been done. If not do it now.·
+        // You can end up doing work here if the page trap on a Grunning Goroutine has
+        // not been sprung or in some race situations. For example a runnable goes dead
+        // and is started up again with a gp->gcworkdone set to false.
+        for(i = 0; i < activeglen; i++) {
+                gp = runtime·allg[i];
+                while (!gp->gcworkdone) {
+                        status = runtime·readgstatus(gp);
+                        if(status == Gdead) {
+                                gp->gcworkdone = true; // scan is a noop
+                                break;
+                                //do nothing, scan not needed.·
+                        }
+                        if(status == Grunning && gp->stackguard0 == (uintptr)StackPreempt && runtime·notetsleep(&runtime·sched.stopnote, 100*1000)) // nanosecond arg·
+                                runtime·noteclear(&runtime·sched.stopnote);
+                        else·
+                                stopscanstart(gp);
+                }
+        }
+
+        for(i = 0; i < activeglen; i++) {
+                gp = runtime·allg[i];
+                status = runtime·readgstatus(gp);
+                if(isscanstatus(status)) {
+                        runtime·printf("mstopandscang:bottom: post scan bad status gp=%p has status %x\n", gp, status);
+                        dumpgstatus(gp);
+                }
+                if(!gp->gcworkdone && status != Gdead) {
+                        runtime·printf("mstopandscang:bottom: post scan gp=%p->gcworkdone still false\n", gp);
+                        dumpgstatus(gp);
+                }
+        }
+
+        schedule(); // Never returns.
+}
+
+// quiesce moves all the goroutines to a GC safepoint which for now is a at preemption point.
+// If the global runtime·gcphase is GCmark quiesce will ensure that all of the goroutine's stacks
+// have been scanned before it returns.
+void
+runtime·quiesce(G* mastergp)
+{
+        void (*fn)(G*);
+
+        runtime·castogscanstatus(mastergp, Grunning, Gscanenqueue);
+        // Now move this to the g0 (aka m) stack.
+        // g0 will potentially scan this thread and put mastergp on the runqueue·
+        fn = mquiesce;
+        runtime·mcall(&fn);
 }
 
 // This is used by the GC as well as the routines that do stack dumps. In the case
 // of GC all the routines can be reliably stopped. This is not always the case
 // when the system is in panic or being exited.
 void
 runtime·stoptheworld(void)
 {
         int32 i;
         uint32 s;
@@ skipping 905 matching lines @@
                 g->m->curg->m = nil;
                 g->m->curg = nil;
         }
 }
 
 // Puts the current goroutine into a waiting state and calls unlockf.
 // If unlockf returns false, the goroutine is resumed.
 void
 runtime·park(bool(*unlockf)(G*, void*), void *lock, String reason)
 {
+        void (*fn)(G*);
+
         g->m->waitlock = lock;
         g->m->waitunlockf = unlockf;
         g->waitreason = reason;
-        runtime·mcall(runtime·park_m);
+        fn = runtime·park_m;
+        runtime·mcall(&fn);
 }
 
 bool
 runtime·parkunlock_c(G *gp, void *lock)
 {
         USED(gp);
         runtime·unlock(lock);
         return true;
 }
 
@@ skipping 24 matching lines @@
                 }
         }
 
         schedule();
 }
 
 // Scheduler yield.
 void
 runtime·gosched(void)
 {
-        runtime·mcall(runtime·gosched_m);
+        void (*fn)(G*);
+ ·······
+        fn = runtime·gosched_m;
+        runtime·mcall(&fn);
 }
 
 // runtime·gosched continuation on g0.
 void
 runtime·gosched_m(G *gp)
 {
         uint32 status;
 
         status = runtime·readgstatus(gp);
-        if ((status&~Gscan) != Grunning){
+        if((status&~Gscan) != Grunning){
                 dumpgstatus(gp);
                 runtime·throw("bad g status");
         }
         runtime·casgstatus(gp, Grunning, Grunnable);
         dropg();
         runtime·lock(&runtime·sched.lock);
         globrunqput(gp);
         runtime·unlock(&runtime·sched.lock);
 
         schedule();
 }
 
 // Finishes execution of the current goroutine.
 // Need to mark it as nosplit, because it runs with sp > stackbase (as runtime·lessstack).
 // Since it does not return it does not matter.  But if it is preempted
 // at the split stack check, GC will complain about inconsistent sp.
 #pragma textflag NOSPLIT
 void
 runtime·goexit(void)
 {
+        void (*fn)(G*);
+
         if(raceenabled)
                 runtime·racegoend();
-        runtime·mcall(goexit0);
+        fn = goexit0;
+        runtime·mcall(&fn);
 }
 
 // runtime·goexit continuation on g0.
 static void
 goexit0(G *gp)
 {
         runtime·casgstatus(gp, Grunning, Gdead);
         gp->m = nil;
         gp->lockedm = nil;
         g->m->lockedg = nil;
@@ skipping 148 matching lines @@
 }
 
 // The goroutine g exited its system call.
 // Arrange for it to run on a cpu again.
 // This is called only from the go syscall library, not
 // from the low-level system calls used by the runtime.
 #pragma textflag NOSPLIT
 void
 runtime·exitsyscall(void)
 {
+        void (*fn)(G*);
+
         g->m->locks++;  // see comment in entersyscall
 
         g->waitsince = 0;
         if(exitsyscallfast()) {
                 // There's a cpu for us, so we can run.
                 g->m->p->syscalltick++;
                 // We need to cas the status and scan before resuming...
                 runtime·casgstatus(g, Gsyscall, Grunning);
 
                 // Garbage collector isn't running (since we are),
                 // so okay to clear gcstack and gcsp.
                 g->syscallstack = (uintptr)nil;
                 g->syscallsp = (uintptr)nil;
                 g->m->locks--;
                 if(g->preempt) {
                         // restore the preemption request in case we've cleared it in newstack
                         g->stackguard0 = StackPreempt;
                 } else {
                         // otherwise restore the real stackguard, we've spoiled it in entersyscall/entersyscallblock
                         g->stackguard0 = g->stackguard;
                 }
                 return;
         }
 
         g->m->locks--;
 
         // Call the scheduler.
-        runtime·mcall(exitsyscall0);
+        fn = exitsyscall0;
+        runtime·mcall(&fn);
 
         // Scheduler returned, so we're allowed to run now.
         // Delete the gcstack information that we left for
         // the garbage collector during the system call.
         // Must wait until now because until gosched returns
         // we don't know for sure that the garbage collector
         // is not running.
         g->syscallstack = (uintptr)nil;
         g->syscallsp = (uintptr)nil;
         g->m->p->syscalltick++;
@@ skipping 121 matching lines @@
         gp->param = runtime·stackalloc(newg, size);
         runtime·gogo(&gp->sched);
 }
 
 // Allocate a new g, with a stack big enough for stacksize bytes.
 G*
 runtime·malg(int32 stacksize)
 {
         G *newg;
         byte *stk;
+        void (*fn)(G*);
 
         if(StackTop < sizeof(Stktop)) {
                 runtime·printf("runtime: SizeofStktop=%d, should be >=%d\n", (int32)StackTop, (int32)sizeof(Stktop));
                 runtime·throw("runtime: bad stack.h");
         }
 
         newg = allocg();
         if(stacksize >= 0) {
                 stacksize = runtime·round2(StackSystem + stacksize);
                 if(g == g->m->g0) {
                         // running on scheduler stack already.
                         stk = runtime·stackalloc(newg, stacksize);
                 } else {
                         // have to call stackalloc on scheduler stack.
                         newg->stacksize = stacksize;
                         g->param = newg;
-                        runtime·mcall(mstackalloc);
+                        fn = mstackalloc;
+                        runtime·mcall(&fn);
                         stk = g->param;
                         g->param = nil;
                 }
                 newg->stack0 = (uintptr)stk;
                 newg->stackguard = (uintptr)stk + StackGuard;
                 newg->stackguard0 = newg->stackguard;
                 newg->stackbase = (uintptr)stk + stacksize - sizeof(Stktop);
         }
         return newg;
 }
@@ skipping 20 matching lines @@
 // Put it on the queue of g's waiting to run.
 // The compiler turns a go statement into a call to this.
 // Cannot split the stack because it assumes that the arguments
 // are available sequentially after &fn; they would not be
 // copied if a stack split occurred.
 #pragma textflag NOSPLIT
 void
 runtime·newproc(int32 siz, FuncVal* fn, ...)
 {
         byte *argp;
+        void (*mfn)(void);
 
         if(thechar == '5')
                 argp = (byte*)(&fn+2);  // skip caller's saved LR
         else
                 argp = (byte*)(&fn+1);
 
         g->m->locks++;
         g->m->scalararg[0] = siz;
         g->m->scalararg[1] = (uintptr)runtime·getcallerpc(&siz);
         g->m->ptrarg[0] = argp;
         g->m->ptrarg[1] = fn;
-        runtime·onM(newproc_m);
+        mfn = newproc_m;
+        runtime·onM(&mfn);
         g->m->locks--;
 }
 
 // Create a new g running fn with narg bytes of arguments starting
 // at argp and returning nret bytes of results.  callerpc is the
 // address of the go statement that created this.  The new g is put
 // on the queue of g's waiting to run.
 G*
 runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerpc)
 {
@@ skipping 68 matching lines @@
                 g->stackguard0 = StackPreempt;
         return newg;
 }
 
 static void
 allgadd(G *gp)
 {
         G **new;
         uintptr cap;
 
-        if (runtime·readgstatus(gp) == Gidle)·
+        if(runtime·readgstatus(gp) == Gidle)·
                 runtime·throw("allgadd: bad status Gidle");
 
         runtime·lock(&allglock);
         if(runtime·allglen >= allgcap) {
                 cap = 4096/sizeof(new[0]);
                 if(cap < 2*allgcap)
                         cap = 2*allgcap;
                 new = runtime·mallocgc(cap*sizeof(new[0]), nil, 0);
                 if(new == nil)
                         runtime·throw("runtime: cannot allocate memory");
@@ skipping 10 matching lines @@
 }
 
 // Put on gfree list.
 // If local list is too long, transfer a batch to the global list.
 static void
 gfput(P *p, G *gp)
 {
         uintptr stksize;
         Stktop *top;
 
-        if (runtime·readgstatus(gp) != Gdead)·
+        if(runtime·readgstatus(gp) != Gdead)·
                 runtime·throw("gfput: bad status (not Gdead)");
 
         if(gp->stackguard - StackGuard != gp->stack0)
                 runtime·throw("invalid stack in gfput");
         stksize = gp->stackbase + sizeof(Stktop) - gp->stack0;
         if(stksize != gp->stacksize) {
                 runtime·printf("runtime: bad stacksize, goroutine %D, remain=%d, last=%d\n",
                         gp->goid, (int32)gp->stacksize, (int32)stksize);
                 runtime·throw("gfput: bad stacksize");
         }
@@ skipping 23 matching lines @@
         }
 }
 
 // Get from gfree list.
 // If local list is empty, grab a batch from global list.
 static G*
 gfget(P *p)
 {
         G *gp;
         byte *stk;
+        void (*fn)(G*);
 
 retry:
         gp = p->gfree;
         if(gp == nil && runtime·sched.gfree) {
                 runtime·lock(&runtime·sched.gflock);
                 while(p->gfreecnt < 32 && runtime·sched.gfree != nil) {
                         p->gfreecnt++;
                         gp = runtime·sched.gfree;
                         runtime·sched.gfree = gp->schedlink;
                         runtime·sched.ngfree--;
                         gp->schedlink = p->gfree;
                         p->gfree = gp;
                 }
                 runtime·unlock(&runtime·sched.gflock);
                 goto retry;
         }
         if(gp) {
                 p->gfree = gp->schedlink;
                 p->gfreecnt--;
 
                 if(gp->stack0 == 0) {
                         // Stack was deallocated in gfput.  Allocate a new one.
                         if(g == g->m->g0) {
                                 stk = runtime·stackalloc(gp, FixedStack);
                         } else {
                                 gp->stacksize = FixedStack;
                                 g->param = gp;
-                                runtime·mcall(mstackalloc);
+                                fn = mstackalloc;
+                                runtime·mcall(&fn);
                                 stk = g->param;
                                 g->param = nil;
                         }
                         gp->stack0 = (uintptr)stk;
                         gp->stackbase = (uintptr)stk + FixedStack - sizeof(Stktop);
                         gp->stackguard = (uintptr)stk + StackGuard;
                         gp->stackguard0 = gp->stackguard;
                 } else {
                         if(raceenabled)
                                 runtime·racemalloc((void*)gp->stack0, gp->stackbase + sizeof(Stktop) - gp->stack0);
@@ skipping 603 matching lines @@
                 // and preempt long running G's
                 if(retake(now))
                         idle = 0;
                 else
                         idle++;
 
                 // check if we need to force a GC
                 lastgc = runtime·atomicload64(&mstats.last_gc);
                 if(lastgc != 0 && unixnow - lastgc > forcegcperiod && runtime·atomicload(&runtime·forcegc.idle)) {
                         runtime·lock(&runtime·forcegc.lock);
-                        runtime·forcegc.idle = 0;
-                        runtime·forcegc.g->schedlink = nil;
-                        injectglist(runtime·forcegc.g);
+                        if(runtime·forcegc.g != nil) {
+                                // Goroutine may be started but has not initialized g yet.
+                                runtime·forcegc.idle = 0;
+                                runtime·forcegc.g->schedlink = nil;
+                                injectglist(runtime·forcegc.g);
+                        }
                         runtime·unlock(&runtime·forcegc.lock);
                 }
 
                 // scavenge heap once in a while
                 if(lastscavenge + scavengelimit/2 < now) {
                         runtime·MHeap_Scavenge(nscavenge, now, scavengelimit);
                         lastscavenge = now;
                         nscavenge++;
                 }
 
@@ skipping 501 matching lines @@
                         }
                 }
                 if(s != i/2 && s != i/2+1) {
                         runtime·printf("bad steal %d, want %d or %d, iter %d\n",
                                 s, i/2, i/2+1, i);
                         runtime·throw("bad steal");
                 }
         }
 }
 
-extern void runtime·morestack(void);
-uintptr runtime·externalthreadhandlerp;
-
-// Does f mark the top of a goroutine stack?
-bool
-runtime·topofstack(Func *f)
-{
-        return f->entry == (uintptr)runtime·goexit ||
-                f->entry == (uintptr)runtime·mstart ||
-                f->entry == (uintptr)runtime·mcall ||
-                f->entry == (uintptr)runtime·onM ||
-                f->entry == (uintptr)runtime·morestack ||
-                f->entry == (uintptr)runtime·lessstack ||
-                f->entry == (uintptr)_rt0_go ||
-                (runtime·externalthreadhandlerp != 0 && f->entry == runtime·externalthreadhandlerp);
-}
-
 void
 runtime·setmaxthreads_m(void)
 {
         int32 in;
         int32 out;
 
         in = g->m->scalararg[0];
 
         runtime·lock(&runtime·sched.lock);
         out = runtime·sched.maxmcount;
@@ skipping 37 matching lines @@
         p = mp->p->id;
         FLUSH(&p);
 }
 
 #pragma textflag NOSPLIT
 void
 sync·runtime_procUnpin()
 {
         g->m->locks--;
 }