code review 123700044: runtime: convert timers to Go

src/pkg/runtime/mgc0.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Garbage collector (GC).
 //
 // GC is:
 // - mark&sweep
 // - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc)
 // - parallel (up to MaxGcproc threads)
@@ skipping 184 matching lines @@
 static struct {
         uint64  full;  // lock-free list of full blocks
         uint64  empty; // lock-free list of empty blocks
         byte    pad0[CacheLineSize]; // prevents false-sharing between full/empty and nproc/nwait
         uint32  nproc;
         int64   tstart;
         volatile uint32 nwait;
         volatile uint32 ndone;
         Note    alldone;
         ParFor* markfor;
+
+        // Copy of mheap.allspans for marker or sweeper.
+        MSpan** spans;
+        uint32  nspan;
 } work;
 
 // scanblock scans a block of n bytes starting at pointer b for references
 // to other objects, scanning any it finds recursively until there are no
 // unscanned objects left.  Instead of using an explicit recursion, it keeps
 // a work list in the Workbuf* structures and loops in the main function
 // body.  Keeping an explicit work list is easier on the stack allocator and
 // more efficient.
 static void
 scanblock(byte *b, uintptr n, byte *ptrmask)
@@ skipping 278 matching lines @@
                                 runtime·throw("scanblock: scanned invalid object");
                         }
                 }
         }
 }
 
 static void
 markroot(ParFor *desc, uint32 i)
 {
         FinBlock *fb;
-        MHeap *h;
-        MSpan **allspans, *s;
+        MSpan *s;
         uint32 spanidx, sg;
         G *gp;
         void *p;
 
         USED(&desc);
         // Note: if you add a case here, please also update heapdump.c:dumproots.
         switch(i) {
         case RootData:
                 scanblock(data, edata - data, runtime·gcdatamask);
                 break;
 
         case RootBss:
                 scanblock(bss, ebss - bss, runtime·gcbssmask);
                 break;
 
         case RootFinalizers:
                 for(fb=allfin; fb; fb=fb->alllink)
                         scanblock((byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), ScanConservatively);
                 break;
 
         case RootSpans:
                 // mark MSpan.specials
-                h = &runtime·mheap;
-                sg = h->sweepgen;
-                allspans = h->allspans;
-                for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
+                sg = runtime·mheap.sweepgen;
+                for(spanidx=0; spanidx<work.nspan; spanidx++) {
                         Special *sp;
                         SpecialFinalizer *spf;
 
-                        s = allspans[spanidx];
+                        s = work.spans[spanidx];
                         if(s->state != MSpanInUse)
                                 continue;
                         if(s->sweepgen != sg) {
                                 runtime·printf("sweep %d %d\n", s->sweepgen, sg);
                                 runtime·throw("gc: unswept span");
                         }
                         for(sp = s->specials; sp != nil; sp = sp->next) {
                                 if(sp->kind != KindSpecialFinalizer)
                                         continue;
                                 // don't mark finalized object, but scan it so we
@@ skipping 532 matching lines @@
         return res;
 }
 
 // State of background sweep.
 // Pretected by gclock.
 static struct
 {
         G*      g;
         bool    parked;
 
-        MSpan** spans;
-        uint32  nspan;
-        uint32  spanidx;
+        uint32  spanidx;        // background sweeper position
 
         uint32  nbgsweep;
         uint32  npausesweep;
 } sweep;
 
 // background sweeping goroutine
 static void
 bgsweep(void)
 {
         g->issystem = 1;
@@ skipping 24 matching lines @@
         MSpan *s;
         uint32 idx, sg;
         uintptr npages;
 
         // increment locks to ensure that the goroutine is not preempted
         // in the middle of sweep thus leaving the span in an inconsistent state for next GC
         g->m->locks++;
         sg = runtime·mheap.sweepgen;
         for(;;) {
                 idx = runtime·xadd(&sweep.spanidx, 1) - 1;
-                if(idx >= sweep.nspan) {
+                if(idx >= work.nspan) {
                         runtime·mheap.sweepdone = true;
                         g->m->locks--;
                         return -1;
                 }
-                s = sweep.spans[idx];
+                s = work.spans[idx];
                 if(s->state != MSpanInUse) {
                         s->sweepgen = sg;
                         continue;
                 }
                 if(s->sweepgen != sg-2 || !runtime·cas(&s->sweepgen, sg-2, sg-1))
                         continue;
                 npages = s->npages;
                 if(!runtime·MSpan_Sweep(s, false))
                         npages = 0;
                 g->m->locks--;
@@ skipping 102 matching lines @@
         // Flush MCache's to MCentral.
         if(g == g->m->g0)
                 flushallmcaches();
         else
                 runtime·mcall(flushallmcaches_m);
 
         // Aggregate local stats.
         cachestats();
 
         // Scan all spans and count number of alive objects.
+        runtime·lock(&runtime·mheap.lock);
         for(i = 0; i < runtime·mheap.nspan; i++) {
                 s = runtime·mheap.allspans[i];
                 if(s->state != MSpanInUse)
                         continue;
                 if(s->sizeclass == 0) {
                         mstats.nmalloc++;
                         mstats.alloc += s->elemsize;
                 } else {
                         mstats.nmalloc += s->ref;
                         mstats.by_size[s->sizeclass].nmalloc += s->ref;
                         mstats.alloc += s->ref*s->elemsize;
                 }
         }
+        runtime·unlock(&runtime·mheap.lock);
 
         // Aggregate by size class.
         smallfree = 0;
         mstats.nfree = runtime·mheap.nlargefree;
         for(i = 0; i < nelem(mstats.by_size); i++) {
                 mstats.nfree += runtime·mheap.nsmallfree[i];
                 mstats.by_size[i].nfree = runtime·mheap.nsmallfree[i];
                 mstats.by_size[i].nmalloc += runtime·mheap.nsmallfree[i];
                 smallfree += runtime·mheap.nsmallfree[i] * runtime·class_to_size[i];
         }
@@ skipping 149 matching lines @@
         work.tstart = args->start_time;·
 
         t1 = 0;
         if(runtime·debug.gctrace)
                 t1 = runtime·nanotime();
 
         // Sweep what is not sweeped by bgsweep.
         while(runtime·sweepone() != -1)
                 sweep.npausesweep++;
 
+        // Cache runtime.mheap.allspans in work.spans to avoid conflicts with
+        // resizing/freeing allspans.
+        // New spans can be created while GC progresses, but they are not garbage for
+        // this round:
+        //  - new stack spans can be created even while the world is stopped.
+        //  - new malloc spans can be created during the concurrent sweep
+
+        // Even if this is stop-the-world, a concurrent exitsyscall can allocate a stack from heap.
+        runtime·lock(&runtime·mheap.lock);
+        // Free the old cached sweep array if necessary.
+        if(work.spans != nil && work.spans != runtime·mheap.allspans)
+                runtime·SysFree(work.spans, work.nspan*sizeof(work.spans[0]), &mstats.other_sys);
+        // Cache the current array for marking.
+        runtime·mheap.gcspans = runtime·mheap.allspans;
+        work.spans = runtime·mheap.allspans;
+        work.nspan = runtime·mheap.nspan;
+        runtime·unlock(&runtime·mheap.lock);
+
         work.nwait = 0;
         work.ndone = 0;
         work.nproc = runtime·gcprocs();
         runtime·parforsetup(work.markfor, work.nproc, RootCount + runtime·allglen, nil, false, markroot);
         if(work.nproc > 1) {
                 runtime·noteclear(&work.alldone);
                 runtime·helpgc(work.nproc);
         }
 
         t2 = 0;
@@ skipping 39 matching lines @@
                 stats.nprocyield += work.markfor->nprocyield;
                 stats.nosyield += work.markfor->nosyield;
                 stats.nsleep += work.markfor->nsleep;
 
                 runtime·printf("gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D) objects,"
                                 " %d/%d/%d sweeps,"
                                 " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\n",
                         mstats.numgc, work.nproc, (t1-t0)/1000, (t2-t1)/1000, (t3-t2)/1000, (t4-t3)/1000,
                         heap0>>20, heap1>>20, obj,
                         mstats.nmalloc, mstats.nfree,
-                        sweep.nspan, sweep.nbgsweep, sweep.npausesweep,
+                        work.nspan, sweep.nbgsweep, sweep.npausesweep,
                         stats.nhandoff, stats.nhandoffcnt,
                         work.markfor->nsteal, work.markfor->nstealcnt,
                         stats.nprocyield, stats.nosyield, stats.nsleep);
                 sweep.nbgsweep = sweep.npausesweep = 0;
         }
 
-        // We cache current runtime·mheap.allspans array in sweep.spans,
-        // because the former can be resized and freed.
-        // Otherwise we would need to take heap lock every time
-        // we want to convert span index to span pointer.
-
-        // Free the old cached array if necessary.
-        if(sweep.spans && sweep.spans != runtime·mheap.allspans)
-                runtime·SysFree(sweep.spans, sweep.nspan*sizeof(sweep.spans[0]), &mstats.other_sys);
-        // Cache the current array.
-        runtime·mheap.sweepspans = runtime·mheap.allspans;
+        // See the comment in the beginning of this function as to why we need the following.
+        // Even if this is still stop-the-world, a concurrent exitsyscall can allocate a stack from heap.
+        runtime·lock(&runtime·mheap.lock);
+        // Free the old cached mark array if necessary.
+        if(work.spans != nil && work.spans != runtime·mheap.allspans)
+                runtime·SysFree(work.spans, work.nspan*sizeof(work.spans[0]), &mstats.other_sys);
+        // Cache the current array for sweeping.
+        runtime·mheap.gcspans = runtime·mheap.allspans;
         runtime·mheap.sweepgen += 2;
         runtime·mheap.sweepdone = false;
-        sweep.spans = runtime·mheap.allspans;
-        sweep.nspan = runtime·mheap.nspan;
+        work.spans = runtime·mheap.allspans;
+        work.nspan = runtime·mheap.nspan;
         sweep.spanidx = 0;
+        runtime·unlock(&runtime·mheap.lock);
 
         // Temporary disable concurrent sweep, because we see failures on builders.
         if(ConcurrentSweep && !args->eagersweep) {
                 runtime·lock(&gclock);
                 if(sweep.g == nil)
                         sweep.g = runtime·newproc1(&bgsweepv, nil, 0, 0, runtime·gc);
                 else if(sweep.parked) {
                         sweep.parked = false;
                         runtime·ready(sweep.g);
                 }
@@ skipping 61 matching lines @@
         for(i=0; i<n; i++)
                 p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)];
 
         p[n] = mstats.last_gc;
         p[n+1] = mstats.numgc;
         p[n+2] = mstats.pause_total_ns;·
         runtime·unlock(&runtime·mheap.lock);
         pauses->len = n+3;
 }
 
-int32
-runtime·setgcpercent(int32 in) {
+void
+runtime·setgcpercent_m(void) {
+        int32 in;
         int32 out;
+
+        in = (int32)(intptr)g->m->scalararg[0];
 
         runtime·lock(&runtime·mheap.lock);
         out = runtime·gcpercent;
         if(in < 0)
                 in = -1;
         runtime·gcpercent = in;
         runtime·unlock(&runtime·mheap.lock);
-        return out;
+
+        g->m->scalararg[0] = (uintptr)(intptr)out;
 }
 
 static void
 gchelperstart(void)
 {
         if(g->m->helpgc < 0 || g->m->helpgc >= MaxGcproc)
                 runtime·throw("gchelperstart: bad m->helpgc");
         if(g != g->m->g0)
                 runtime·throw("gchelper not running on g0 stack");
 }
@@ skipping 484 matching lines @@
 
 void runtime·gc_unixnanotime(int64 *now);
 
 int64 runtime·unixnanotime(void)
 {
         int64 now;
 
         runtime·gc_unixnanotime(&now);
         return now;
 }