code review 5279048: runtime: faster and more scalable GC

src/pkg/runtime/proc.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 #include "runtime.h"
-#include "arch.h"
-#include "defs.h"
+#include "arch_GOARCH.h"
+#include "defs_GOOS_GOARCH.h"
 #include "malloc.h"
-#include "os.h"
+#include "os_GOOS.h"
 #include "stack.h"
 
 bool    runtime·iscgo;
 
 static void unwindstack(G*, byte*);
 static void schedule(G*);
 
 typedef struct Sched Sched;
 
 M       runtime·m0;
@@ skipping 137 matching lines @@
         for(;;) {
                 v = runtime·sched.atomic;
                 w = v;
                 w &= ~(mcpuMask<<mcpumaxShift);
                 w |= n<<mcpumaxShift;
                 if(runtime·cas(&runtime·sched.atomic, v, w))
                         break;
         }
 }
 
+// Keep trace of scavenger's goroutine for deadlock detection.
+static G *scvg;
+
 // The bootstrap sequence is:
 //
 //      call osinit
 //      call schedinit
 //      make & queue new G
 //      call runtime·mstart
 //
 // The new G calls runtime·main.
 void
 runtime·schedinit(void)
@@ skipping 13 matching lines @@
         // so that we don't need to call malloc when we crash.
         // runtime·findfunc(0);
 
         runtime·gomaxprocs = 1;
         p = runtime·getenv("GOMAXPROCS");
         if(p != nil && (n = runtime·atoi(p)) != 0) {
                 if(n > maxgomaxprocs)
                         n = maxgomaxprocs;
                 runtime·gomaxprocs = n;
         }
-        setmcpumax(runtime·gomaxprocs);
+        // wait for the main goroutine to start before taking
+        // GOMAXPROCS into account.
+        setmcpumax(1);
         runtime·singleproc = runtime·gomaxprocs == 1;
 
         canaddmcpu();   // mcpu++ to account for bootstrap m
         m->helpgc = 1;  // flag to tell schedule() to mcpu--
         runtime·sched.grunning++;
 
         mstats.enablegc = 1;
         m->nomemprof--;
 }
 
 extern void main·init(void);
 extern void main·main(void);
 
 // The main goroutine.
 void
 runtime·main(void)
 {
         // Lock the main goroutine onto this, the main OS thread,
         // during initialization.  Most programs won't care, but a few
         // do require certain calls to be made by the main thread.
         // Those can arrange for main.main to run in the main thread
         // by calling runtime.LockOSThread during initialization
         // to preserve the lock.
         runtime·LockOSThread();
+        // From now on, newgoroutines may use non-main threads.
+        setmcpumax(runtime·gomaxprocs);
         runtime·sched.init = true;
+        scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runtime·main);
         main·init();
         runtime·sched.init = false;
         if(!runtime·sched.lockmain)
                 runtime·UnlockOSThread();
 
         main·main();
         runtime·exit(0);
         for(;;)
                 *(int32*)runtime·main = 0;
 }
@@ skipping 81 matching lines @@
 runtime·idlegoroutine(void)
 {
         if(g->idlem != nil)
                 runtime·throw("g is already an idle goroutine");
         g->idlem = m;
 }
 
 static void
 mcommoninit(M *m)
 {
+        m->id = runtime·sched.mcount++;
+        m->fastrand = 0x49f6428aUL + m->id + runtime·cputicks();
+        m->stackalloc = runtime·malloc(sizeof(*m->stackalloc));
+        runtime·FixAlloc_Init(m->stackalloc, FixedStack, runtime·SysAlloc, nil, nil);
+
+        if(m->mcache == nil)
+                m->mcache = runtime·allocmcache();
+
+        runtime·callers(1, m->createstack, nelem(m->createstack));
+
         // Add to runtime·allm so garbage collector doesn't free m
         // when it is just in a register or thread-local storage.
         m->alllink = runtime·allm;
-        // runtime·Cgocalls() iterates over allm w/o schedlock,
+        // runtime·NumCgoCall() iterates over allm w/o schedlock,
         // so we need to publish it safely.
         runtime·atomicstorep(&runtime·allm, m);
-
-        m->id = runtime·sched.mcount++;
-        m->fastrand = 0x49f6428aUL + m->id;
-        m->stackalloc = runtime·malloc(sizeof(*m->stackalloc));
-        runtime·FixAlloc_Init(m->stackalloc, FixedStack, runtime·SysAlloc, nil, nil);
-
-        if(m->mcache == nil)
-                m->mcache = runtime·allocmcache();
 }
 
 // Try to increment mcpu.  Report whether succeeded.
 static bool
 canaddmcpu(void)
 {
         uint32 v;
 
         for(;;) {
                 v = runtime·sched.atomic;
@@ skipping 222 matching lines @@
                 // We hold the sched lock, so no one else is manipulating the
                 // g queue or changing mcpumax.  Entersyscall can decrement
                 // mcpu, but if does so when there is something on the g queue,
                 // the gwait bit will be set, so entersyscall will take the slow path
                 // and use the sched lock.  So it cannot invalidate our decision.
                 //
                 // Wait on global m queue.
                 mput(m);
         }
 
-        v = runtime·atomicload(&runtime·sched.atomic);
-        if(runtime·sched.grunning == 0)
+        // Look for deadlock situation.
+        if((scvg == nil && runtime·sched.grunning == 0) ||
+           (scvg != nil && runtime·sched.grunning == 1 && runtime·sched.gwait == 0 &&
+            (scvg->status == Grunning || scvg->status == Gsyscall))) {
                 runtime·throw("all goroutines are asleep - deadlock!");
+        }
+
         m->nextg = nil;
         m->waitnextg = 1;
         runtime·noteclear(&m->havenextg);
 
         // Stoptheworld is waiting for all but its cpu to go to stop.
         // Entersyscall might have decremented mcpu too, but if so
         // it will see the waitstop and take the slow path.
         // Exitsyscall never increments mcpu beyond mcpumax.
+        v = runtime·atomicload(&runtime·sched.atomic);
         if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
                 // set waitstop = 0 (known to be 1)
                 runtime·xadd(&runtime·sched.atomic, -1<<waitstopShift);
                 runtime·notewakeup(&runtime·sched.stopped);
         }
         schedunlock();
 
         runtime·notesleep(&m->havenextg);
         if(m->helpgc) {
                 runtime·gchelper();
@@ skipping 18 matching lines @@
         if(n > runtime·ncpu)
                 n = runtime·ncpu;
         if(n > MaxGcproc)
                 n = MaxGcproc;
         if(n > runtime·sched.mwait+1) // one M is currently running
                 n = runtime·sched.mwait+1;
         return n;
 }
 
 void
-runtime·helpgc(void)
+runtime·helpgc(int32 nproc)
 {
         M *mp;
         int32 n;
 
         runtime·lock(&runtime·sched);
-        for(n = runtime·gcprocs()-1; n; n--) {
+        for(n = nproc-1; n; n--) {
                 mp = mget(nil);
                 if(mp == nil)
                         runtime·throw("runtime·gcprocs inconsistency");
                 mp->helpgc = 1;
                 mp->waitnextg = 0;
                 runtime·notewakeup(&mp->havenextg);
         }
         runtime·unlock(&runtime·sched);
 }
 
@@ skipping 64 matching lines @@
                 // but m is not running a specific goroutine,
                 // so set the helpgc flag as a signal to m's
                 // first schedule(nil) to mcpu-- and grunning--.
                 m = runtime·newm();
                 m->helpgc = 1;
                 runtime·sched.grunning++;
         }
         schedunlock();
 }
 
+// Tests that GC helper procs are properly started.
+// That is, on one hand number of GC procs eventually reaches the maximum,
+// and on the other hand no unbounded proc creation takes place.
+void
+runtime·CTestGcprocs(bool isShort)
+{
+        int32 ncpu, i, procs, mwait0, gcprocs;
+
+        USED(isShort);
+
+        // temprary bump ncpu, restore before return
+        ncpu = runtime·ncpu;
+        runtime·ncpu = MaxGcproc;
+        procs = runtime·gomaxprocsfunc(-1);
+        gcprocs = 0;
+
+        // set GOMAXPROCS=1, so new procs should not be created
+        runtime·gomaxprocsfunc(1);
+        mwait0 = runtime·sched.mwait;
+
+        for(i=0; i<MaxGcproc*2; i++) {
+                // emulate GC cycle
+                runtime·stoptheworld();
+                gcprocs = runtime·gcprocs();
+                runtime·starttheworld();
+
+                if(gcprocs != 1) {
+                        runtime·printf("gcprocs=%d\n", gcprocs);
+                        runtime·panicstring("gcprocs != 1");
+                }
+
+                // give a new proc a change to start
+                g->status = Gwaiting;
+                g->waitreason = "test sleep";
+                runtime·tsleep(10000);
+        }
+
+        if(runtime·sched.mwait != mwait0) {
+                runtime·printf("mwait=%d, mwait0=%d\n", runtime·sched.mwait, mwait0);
+                runtime·panicstring("new procs started when GOMAXPROCS=1");
+        }
+
+        // set GOMAXPROCS=MaxGcproc, so new procs can be created
+        runtime·gomaxprocsfunc(MaxGcproc);
+        mwait0 = runtime·sched.mwait;
+
+        for(i=0; i<MaxGcproc*2; i++) {
+                // emulate GC cycle
+                runtime·stoptheworld();
+                gcprocs = runtime·gcprocs();
+                runtime·starttheworld();
+
+                // give a new proc a change to start
+                g->status = Gwaiting;
+                g->waitreason = "test sleep";
+                runtime·tsleep(10000);
+        }
+
+        if(gcprocs != MaxGcproc) {
+                runtime·printf("gcprocs=%d, MaxGcproc=%d\n", gcprocs, MaxGcproc);
+                runtime·panicstring("new procs are not started");
+        }
+
+        if(runtime·sched.mwait > mwait0 + MaxGcproc - 1) {
+                runtime·printf("MaxGcproc=%d, mwait=%d, mwait0=%d\n",
+                        MaxGcproc, runtime·sched.mwait, mwait0);
+                runtime·panicstring("a way too many procs started");
+        }
+
+        runtime·gomaxprocsfunc(procs);
+        runtime·ncpu = ncpu;
+}
+
 // Called to start an M.
 void
 runtime·mstart(void)
 {
         if(g != m->g0)
                 runtime·throw("bad runtime·mstart");
 
         // Record top of stack for use by mcall.
         // Once we call schedule we're never coming back,
         // so other calls can reuse this stack space.
         runtime·gosave(&m->g0->sched);
         m->g0->sched.pc = (void*)-1;  // make sure it is never used
-
+        runtime·asminit();
         runtime·minit();
+
+        // Install signal handlers; after minit so that minit can
+        // prepare the thread to be able to handle the signals.
+        if(m == &runtime·m0)
+                runtime·initsig();
+
         schedule(nil);
 }
 
 // When running with cgo, we call libcgo_thread_start
 // to start threads for us so that we can play nicely with
 // foreign code.
 void (*libcgo_thread_start)(void*);
 
 typedef struct CgoThreadStart CgoThreadStart;
 struct CgoThreadStart
@@ skipping 165 matching lines @@
 // entersyscall is going to return immediately after.
 // It's okay to call matchmg and notewakeup even after
 // decrementing mcpu, because we haven't released the
 // sched lock yet, so the garbage collector cannot be running.
 #pragma textflag 7
 void
 runtime·entersyscall(void)
 {
         uint32 v;
 
+        if(m->profilehz > 0)
+                runtime·setprof(false);
+
         // Leave SP around for gc and traceback.
         runtime·gosave(&g->sched);
         g->gcsp = g->sched.sp;
         g->gcstack = g->stackbase;
         g->gcguard = g->stackguard;
         g->status = Gsyscall;
         if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) {
                 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n",
                 //      g->gcsp, g->gcguard-StackGuard, g->gcstack);
                 runtime·throw("entersyscall");
@@ skipping 43 matching lines @@
         // find that we still have mcpu <= mcpumax, then we can
         // start executing Go code immediately, without having to
         // schedlock/schedunlock.
         v = runtime·xadd(&runtime·sched.atomic, (1<<mcpuShift));
         if(m->profilehz == runtime·sched.profilehz && atomic_mcpu(v) <= atomic_mcpumax(v)) {
                 // There's a cpu for us, so we can run.
                 g->status = Grunning;
                 // Garbage collector isn't running (since we are),
                 // so okay to clear gcstack.
                 g->gcstack = nil;
+
+                if(m->profilehz > 0)
+                        runtime·setprof(true);
                 return;
         }
 
         // Tell scheduler to put g back on the run queue:
         // mostly equivalent to g->status = Grunning,
         // but keeps the garbage collector from thinking
         // that g is running right now, which it's not.
         g->readyonstop = 1;
 
         // All the cpus are taken.
@@ skipping 12 matching lines @@
 }
 
 // Called from runtime·lessstack when returning from a function which
 // allocated a new stack segment.  The function's return value is in
 // m->cret.
 void
 runtime·oldstack(void)
 {
         Stktop *top, old;
         uint32 argsize;
+        uintptr cret;
         byte *sp;
         G *g1;
         int32 goid;
 
 //printf("oldstack m->cret=%p\n", m->cret);
 
         g1 = m->curg;
         top = (Stktop*)g1->stackbase;
         sp = (byte*)top;
         old = *top;
         argsize = old.argsize;
         if(argsize > 0) {
                 sp -= argsize;
                 runtime·memmove(top->argp, sp, argsize);
         }
         goid = old.gobuf.g->goid;       // fault if g is bad, before gogo
         USED(goid);
 
         if(old.free != 0)
                 runtime·stackfree(g1->stackguard - StackGuard, old.free);
         g1->stackbase = old.stackbase;
         g1->stackguard = old.stackguard;
 
-        runtime·gogo(&old.gobuf, m->cret);
+        cret = m->cret;
+        m->cret = 0;  // drop reference
+        runtime·gogo(&old.gobuf, cret);
 }
 
 // Called from reflect·call or from runtime·morestack when a new
 // stack segment is needed.  Allocate a new stack big enough for
 // m->moreframesize bytes, copy m->moreargsize bytes to the new frame,
 // and then act as though runtime·lessstack called the function at
 // m->morepc.
 void
 runtime·newstack(void)
 {
@@ skipping 45 matching lines @@
 
 //runtime·printf("newstack framesize=%d argsize=%d morepc=%p moreargp=%p gobuf=%p, %p top=%p old=%p\n",
 //framesize, argsize, m->morepc, m->moreargp, m->morebuf.pc, m->morebuf.sp, top, g1->stackbase);
 
         top->stackbase = g1->stackbase;
         top->stackguard = g1->stackguard;
         top->gobuf = m->morebuf;
         top->argp = m->moreargp;
         top->argsize = argsize;
         top->free = free;
+        m->moreargp = nil;
+        m->morebuf.pc = nil;
+        m->morebuf.sp = nil;
 
         // copy flag from panic
         top->panic = g1->ispanic;
         g1->ispanic = false;
 
         g1->stackbase = (byte*)top;
         g1->stackguard = stk + StackGuard;
 
         sp = (byte*)top;
         if(argsize > 0) {
                 sp -= argsize;
-                runtime·memmove(sp, m->moreargp, argsize);
+                runtime·memmove(sp, top->argp, argsize);
         }
         if(thechar == '5') {
                 // caller would have saved its LR below args.
                 sp -= sizeof(void*);
                 *(void**)sp = nil;
         }
 
         // Continue as if lessstack had just called m->morepc
         // (the PC that decided to grow the stack).
         label.sp = sp;
@@ skipping 14 matching lines @@
         gp->param = runtime·stackalloc((uintptr)gp->param);
         runtime·gogo(&gp->sched, 0);
 }
 
 // Allocate a new g, with a stack big enough for stacksize bytes.
 G*
 runtime·malg(int32 stacksize)
 {
         G *newg;
         byte *stk;
+
+        if(StackTop < sizeof(Stktop)) {
+                runtime·printf("runtime: SizeofStktop=%d, should be >=%d\n", (int32)StackTop, (int32)sizeof(Stktop));
+                runtime·throw("runtime: bad stack.h");
+        }
 
         newg = runtime·malloc(sizeof(G));
         if(stacksize >= 0) {
                 if(g == m->g0) {
                         // running on scheduler stack already.
                         stk = runtime·stackalloc(StackSystem + stacksize);
                 } else {
                         // have to call stackalloc on scheduler stack.
                         g->param = (void*)(StackSystem + stacksize);
                         runtime·mcall(mstackalloc);
@@ skipping 486 matching lines @@
 
 // for testing of wire, unwire
 void
 runtime·mid(uint32 ret)
 {
         ret = m->id;
         FLUSH(&ret);
 }
 
 void
-runtime·Goroutines(int32 ret)
+runtime·NumGoroutine(int32 ret)
 {
         ret = runtime·sched.gcount;
         FLUSH(&ret);
+}
+
+int32
+runtime·gcount(void)
+{
+        return runtime·sched.gcount;
 }
 
 int32
 runtime·mcount(void)
 {
         return runtime·sched.mcount;
 }
 
 void
 runtime·badmcall(void)  // called from assembly
@@ skipping 80 matching lines @@
         arg[0][k.len] = 0;
 
         arg[1] = runtime·malloc(v.len + 1);
         runtime·memmove(arg[1], v.str, v.len);
         arg[1][v.len] = 0;
 
         runtime·asmcgocall((void*)libcgo_setenv, arg);
         runtime·free(arg[0]);
         runtime·free(arg[1]);
 }