code review 37540043: runtime: heap dump experiment

src/pkg/runtime/mgc0.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Garbage collector (GC).
 //
 // GC is:
 // - mark&sweep
 // - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc)
 // - parallel (up to MaxGcproc threads)
@@ skipping 49 matching lines @@
 #include "typekind.h"
 #include "funcdata.h"
 #include "../../cmd/ld/textflag.h"
 
 enum {
         Debug = 0,
         CollectStats = 0,
         ConcurrentSweep = 1,
 
         WorkbufSize     = 16*1024,
-        RootBlockSize   = 4*1024,
         FinBlockSize    = 4*1024,
 
         handoffThreshold = 4,
         IntermediateBufferCapacity = 64,
 
         // Bits in type information
         PRECISE = 1,
         LOOP = 2,
         PC_BITS = PRECISE | LOOP,
 
@@ skipping 123 matching lines @@
 extern byte gcbss[];
 
 static G        *fing;
 static FinBlock *finq; // list of finalizers that are to be executed
 static FinBlock *finc; // cache of free blocks
 static FinBlock *allfin; // list of all blocks
 static int32    fingwait;
 static Lock     gclock;
 
 static void     runfinq(void);
+static void     wakefing(void);
 static void     bgsweep(void);
 static Workbuf* getempty(Workbuf*);
 static Workbuf* getfull(Workbuf*);
 static void     putempty(Workbuf*);
 static Workbuf* handoff(Workbuf*);
 static void     gchelperstart(void);
-static void     addfinroots(void *wbufp, void *v);
 static void     flushallmcaches(void);
 static bool     scanframe(Stkframe *frame, void *wbufp);
 static void     addstackroots(G *gp, Workbuf **wbufp);
 
 static FuncVal runfinqv = {runfinq};
 static FuncVal bgsweepv = {bgsweep};
 
 static struct {
         uint64  full;  // lock-free list of full blocks
         uint64  empty; // lock-free list of empty blocks
@@ skipping 504 matching lines @@
 static void
 scanblock(Workbuf *wbuf, bool keepworking)
 {
         byte *b, *arena_start, *arena_used;
         uintptr n, i, end_b, elemsize, size, ti, objti, count, type, nobj;
         uintptr *pc, precise_type, nominal_size;
         uintptr *chan_ret, chancap;
         void *obj;
         Type *t;
         Slice *sliceptr;
+        String *stringptr;
         Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4];
         BufferList *scanbuffers;
         Scanbuf sbuf;
         Eface *eface;
         Iface *iface;
         Hchan *chan;
         ChanType *chantype;
         Obj *wp;
 
         if(sizeof(Workbuf) % WorkbufSize != 0)
@@ skipping 150 matching lines @@
                         }
                         pc += 3;
                         break;
 
                 case GC_APTR:
                         obj = *(void**)(stack_top.b + pc[1]);
                         pc += 2;
                         break;
 
                 case GC_STRING:
-                        obj = *(void**)(stack_top.b + pc[1]);
-                        markonly(obj);
+                        stringptr = (String*)(stack_top.b + pc[1]);
+                        if(stringptr->len != 0) {
+                                obj = stringptr->str;
+                                markonly(obj);
+                        }
                         pc += 2;
                         continue;
 
                 case GC_EFACE:
                         eface = (Eface*)(stack_top.b + pc[1]);
                         pc += 2;
                         if(eface->type == nil)
                                 continue;
 
                         // eface->type
@@ skipping 295 matching lines @@
         Workbuf *wbuf;
         FinBlock *fb;
         MHeap *h;
         MSpan **allspans, *s;
         uint32 spanidx, sg;
         G *gp;
         void *p;
 
         USED(&desc);
         wbuf = getempty(nil);
+        // Note: if you add a case here, please also update heapdump.c:dumproots.
         switch(i) {
         case RootData:
                 enqueue1(&wbuf, (Obj){data, edata - data, (uintptr)gcdata});
                 break;
 
         case RootBss:
                 enqueue1(&wbuf, (Obj){bss, ebss - bss, (uintptr)gcbss});
                 break;
 
         case RootFinalizers:
@@ skipping 199 matching lines @@
         enqueue1(wbufp, (Obj){scanp+PtrSize, PtrSize, 0});
 }
 
 // Starting from scanp, scans words corresponding to set bits.
 static void
 scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, void *wbufp)
 {
         uintptr word, bits;
         uint32 *wordp;
         int32 i, remptrs;
+        byte *p;
 
         wordp = bv->data;
         for(remptrs = bv->n; remptrs > 0; remptrs -= 32) {
                 word = *wordp++;
                 if(remptrs < 32)
                         i = remptrs;
                 else
                         i = 32;
                 i /= BitsPerPointer;
                 for(; i > 0; i--) {
                         bits = word & 3;
-                        if(bits != BitsNoPointer && *(void**)scanp != nil)
-                                if(bits == BitsPointer)
+                        switch(bits) {
+                        case BitsDead:
+                                if(runtime·debug.gcdead)
+                                        *(uintptr*)scanp = (uintptr)0x6969696969696969LL;
+                                break;
+                        case BitsScalar:
+                                break;
+                        case BitsPointer:
+                                p = *(byte**)scanp;
+                                if(p != nil)
                                         enqueue1(wbufp, (Obj){scanp, PtrSize, 0});
-                                else
-                                        scaninterfacedata(bits, scanp, afterprologue, wbufp);
+                                break;
+                        case BitsMultiWord:
+                                p = *(byte**)scanp;
+                                if(p != nil) {
+                                        word >>= BitsPerPointer;
+                                        scanp += PtrSize;
+                                        i--;
+                                        if(i == 0) {
+                                                // Get next chunk of bits
+                                                remptrs -= 32;
+                                                word = *wordp++;
+                                                if(remptrs < 32)
+                                                        i = remptrs;
+                                                else
+                                                        i = 32;
+                                                i /= BitsPerPointer;
+                                        }
+                                        switch(word & 3) {
+                                        case BitsString:
+                                                if(((String*)(scanp - PtrSize))->len != 0)
+                                                        markonly(p);
+                                                break;
+                                        case BitsSlice:
+                                                if(((Slice*)(scanp - PtrSize))->cap < ((Slice*)(scanp - PtrSize))->len)
+                                                        runtime·throw("slice capacity smaller than length");
+                                                if(((Slice*)(scanp - PtrSize))->cap != 0)
+                                                        enqueue1(wbufp, (Obj){scanp - PtrSize, PtrSize, 0});
+                                                break;
+                                        case BitsIface:
+                                        case BitsEface:
+                                                scaninterfacedata(word & 3, scanp - PtrSize, afterprologue, wbufp);
+                                                break;
+                                        }
+                                }
+                        }
                         word >>= BitsPerPointer;
                         scanp += PtrSize;
                 }
         }
 }
 
 // Scan a stack frame: local variables and function arguments/results.
 static bool
 scanframe(Stkframe *frame, void *wbufp)
 {
@@ skipping 148 matching lines @@
         finq->cnt++;
         f->fn = fn;
         f->nret = nret;
         f->fint = fint;
         f->ot = ot;
         f->arg = p;
         runtime·unlock(&gclock);
 }
 
 void
+runtime·iterate_finq(void (*callback)(FuncVal*, byte*, uintptr, Type*, PtrType*))
+{
+        FinBlock *fb;
+        Finalizer *f;
+        uintptr i;
+
+        for(fb = allfin; fb; fb = fb->alllink) {
+                for(i = 0; i < fb->cnt; i++) {
+                        f = &fb->fin[i];
+                        callback(f->fn, f->arg, f->nret, f->fint, f->ot);
+                }
+        }
+}
+
+void
 runtime·MSpan_EnsureSwept(MSpan *s)
 {
         uint32 sg;
 
         // Caller must disable preemption.
         // Otherwise when this function returns the span can become unswept again
         // (if GC is triggered on another goroutine).
-        if(m->locks == 0 && m->mallocing == 0)
+        if(m->locks == 0 && m->mallocing == 0 && g != m->g0)
                 runtime·throw("MSpan_EnsureSwept: m is not locked");
 
         sg = runtime·mheap.sweepgen;
         if(runtime·atomicload(&s->sweepgen) == sg)
                 return;
         if(runtime·cas(&s->sweepgen, sg-2, sg-1)) {
                 runtime·MSpan_Sweep(s);
                 return;
         }
         // unfortunate condition, and we don't have efficient means to wait
@@ skipping 108 matching lines @@
                 bits = *bitp>>shift;
 
                 if((bits & bitAllocated) == 0)
                         continue;
 
                 if((bits & bitMarked) != 0) {
                         *bitp &= ~(bitMarked<<shift);
                         continue;
                 }
 
-                // Clear mark, scan, and special bits.
-                *bitp &= ~((bitScan|bitMarked|bitSpecial)<<shift);
+                // Clear mark and scan bits.
+                *bitp &= ~((bitScan|bitMarked)<<shift);
 
                 if(cl == 0) {
                         // Free large span.
                         runtime·unmarkspan(p, 1<<PageShift);
                         s->needzero = 1;
                         // important to set sweepgen before returning it to heap
                         runtime·atomicstore(&s->sweepgen, sweepgen);
                         sweepgenset = true;
                         // See note about SysFault vs SysFree in malloc.goc.
                         if(runtime·debug.efence)
@@ skipping 50 matching lines @@
         }
         return res;
 }
 
 // State of background sweep.
 // Pretected by gclock.
 static struct
 {
         G*      g;
         bool    parked;
+        uint32  lastsweepgen;
 
         MSpan** spans;
         uint32  nspan;
         uint32  spanidx;
 } sweep;
 
 // background sweeping goroutine
 static void
 bgsweep(void)
 {
         g->issystem = 1;
         for(;;) {
                 while(runtime·sweepone() != -1) {
                         gcstats.nbgsweep++;
+                        if(sweep.lastsweepgen != runtime·mheap.sweepgen) {
+                                // If bgsweep does not catch up for any reason
+                                // (does not finish before next GC),
+                                // we still need to kick off runfinq at least once per GC.
+                                sweep.lastsweepgen = runtime·mheap.sweepgen;
+                                wakefing();
+                        }
                         runtime·gosched();
                 }
+                // kick off goroutine to run queued finalizers
+                wakefing();
                 runtime·lock(&gclock);
-                if(finq != nil) {
-                        // kick off or wake up goroutine to run queued finalizers
-                        if(fing == nil)
-                                fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc);
-                        else if(fingwait) {
-                                fingwait = 0;
-                                runtime·ready(fing);
-                        }
+                if(!runtime·mheap.sweepdone) {
+                        // It's possible if GC has happened between sweepone has
+                        // returned -1 and gclock lock.
+                        runtime·unlock(&gclock);
+                        continue;
                 }
                 sweep.parked = true;
                 runtime·parkunlock(&gclock, "GC sweep wait");
         }
 }
 
 // sweeps one span
 // returns number of pages returned to heap, or -1 if there is nothing to sweep
 uintptr
 runtime·sweepone(void)
@@ skipping 31 matching lines @@
 }
 
 static void
 dumpspan(uint32 idx)
 {
         int32 sizeclass, n, npages, i, column;
         uintptr size;
         byte *p;
         byte *arena_start;
         MSpan *s;
-        bool allocated, special;
+        bool allocated;
 
         s = runtime·mheap.allspans[idx];
         if(s->state != MSpanInUse)
                 return;
         arena_start = runtime·mheap.arena_start;
         p = (byte*)(s->start << PageShift);
         sizeclass = s->sizeclass;
         size = s->elemsize;
         if(sizeclass == 0) {
                 n = 1;
         } else {
                 npages = runtime·class_to_allocnpages[sizeclass];
                 n = (npages << PageShift) / size;
         }
 ········
         runtime·printf("%p .. %p:\n", p, p+n*size);
         column = 0;
         for(; n>0; n--, p+=size) {
                 uintptr off, *bitp, shift, bits;
 
                 off = (uintptr*)p - (uintptr*)arena_start;
                 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
                 shift = off % wordsPerBitmapWord;
                 bits = *bitp>>shift;
 
                 allocated = ((bits & bitAllocated) != 0);
-                special = ((bits & bitSpecial) != 0);
 
                 for(i=0; i<size; i+=sizeof(void*)) {
                         if(column == 0) {
                                 runtime·printf("\t");
                         }
                         if(i == 0) {
                                 runtime·printf(allocated ? "(" : "[");
-                                runtime·printf(special ? "@" : "");
                                 runtime·printf("%p: ", p+i);
                         } else {
                                 runtime·printf(" ");
                         }
 
                         runtime·printf("%p", *(void**)(p+i));
 
                         if(i+sizeof(void*) >= size) {
                                 runtime·printf(allocated ? ") " : "] ");
                         }
@@ skipping 15 matching lines @@
         uint32 spanidx;
 
         for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
                 dumpspan(spanidx);
         }
 }
 
 void
 runtime·gchelper(void)
 {
-        int32 nproc;
+        uint32 nproc;
 
         gchelperstart();
 
         // parallel mark for over gc roots
         runtime·parfordo(work.markfor);
 
         // help other threads scan secondary blocks
         scanblock(nil, true);
 
         bufferList[m->helpgc].busy = 0;
@@ skipping 24 matching lines @@
 
         // Flush MCache's to MCentral.
         for(pp=runtime·allp; p=*pp; pp++) {
                 c = p->mcache;
                 if(c==nil)
                         continue;
                 runtime·MCache_ReleaseAll(c);
         }
 }
 
-static void
-updatememstats(GCStats *stats)
+void
+runtime·updatememstats(GCStats *stats)
 {
         M *mp;
         MSpan *s;
         int32 i;
         uint64 stacks_inuse, smallfree;
         uint64 *src, *dst;
 
         if(stats)
                 runtime·memclr((byte*)stats, sizeof(*stats));
         stacks_inuse = 0;
@@ skipping 159 matching lines @@
 
         // all done
         m->gcing = 0;
         m->locks++;
         runtime·semrelease(&runtime·worldsema);
         runtime·starttheworld();
         m->locks--;
 
         // now that gc is done, kick off finalizer thread if needed
         if(!ConcurrentSweep) {
-                if(finq != nil) {
-                        runtime·lock(&gclock);
-                        // kick off or wake up goroutine to run queued finalizers
-                        if(fing == nil)
-                                fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc);
-                        else if(fingwait) {
-                                fingwait = 0;
-                                runtime·ready(fing);
-                        }
-                        runtime·unlock(&gclock);
-                }
+                // kick off goroutine to run queued finalizers
+                wakefing();
                 // give the queued finalizers, if any, a chance to run
                 runtime·gosched();
         }
 }
 
 static void
 mgc(G *gp)
 {
         gc(gp->param);
         gp->param = nil;
@@ skipping 65 matching lines @@
         t4 = runtime·nanotime();
         mstats.last_gc = t4;
         mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0;
         mstats.pause_total_ns += t4 - t0;
         mstats.numgc++;
         if(mstats.debuggc)
                 runtime·printf("pause %D\n", t4-t0);
 
         if(runtime·debug.gctrace) {
                 heap1 = mstats.heap_alloc;
-                updatememstats(&stats);
+                runtime·updatememstats(&stats);
                 if(heap1 != mstats.heap_alloc) {
                         runtime·printf("runtime: mstats skew: heap=%D/%D\n", heap1, mstats.heap_alloc);
                         runtime·throw("mstats skew");
                 }
                 obj = mstats.nmalloc - mstats.nfree;
 
                 stats.nprocyield += work.markfor->nprocyield;
                 stats.nosyield += work.markfor->nosyield;
                 stats.nsleep += work.markfor->nsleep;
 
@@ skipping 79 matching lines @@
 void
 runtime·ReadMemStats(MStats *stats)
 {
         // Have to acquire worldsema to stop the world,
         // because stoptheworld can only be used by
         // one goroutine at a time, and there might be
         // a pending garbage collection already calling it.
         runtime·semacquire(&runtime·worldsema, false);
         m->gcing = 1;
         runtime·stoptheworld();
-        updatememstats(nil);
+        runtime·updatememstats(nil);
         // Size of the trailing by_size array differs between Go and C,
         // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
         runtime·memcopy(runtime·sizeof_C_MStats, stats, &mstats);
         m->gcing = 0;
         m->locks++;
         runtime·semrelease(&runtime·worldsema);
         runtime·starttheworld();
         m->locks--;
 }
 
@@ skipping 147 matching lines @@
                 fb = nil;
                 next = nil;
                 i = 0;
                 ef = nil;
                 ef1.type = nil;
                 ef1.data = nil;
                 runtime·gc(1);  // trigger another gc to clean up the finalized objects, if possible
         }
 }
 
+static void
+wakefing(void)
+{
+        if(finq == nil)
+                return;
+        runtime·lock(&gclock);
+        // kick off or wake up goroutine to run queued finalizers
+        if(fing == nil)
+                fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc);
+        else if(fingwait) {
+                fingwait = 0;
+                runtime·ready(fing);
+        }
+        runtime·unlock(&gclock);
+}
+
 void
 runtime·marknogc(void *v)
 {
-        uintptr *b, obits, bits, off, shift;
+        uintptr *b, off, shift;
 
         off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;  // word offset
         b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
         shift = off % wordsPerBitmapWord;
-
-        for(;;) {
-                obits = *b;
-                if((obits>>shift & bitMask) != bitAllocated)
-                        runtime·throw("bad initial state for marknogc");
-                bits = (obits & ~(bitAllocated<<shift)) | bitBlockBoundary<<shift;
-                if(runtime·gomaxprocs == 1) {
-                        *b = bits;
-                        break;
-                } else {
-                        // more than one goroutine is potentially running: use atomic op
-                        if(runtime·casp((void**)b, (void*)obits, (void*)bits))
-                                break;
-                }
-        }
+        *b = (*b & ~(bitAllocated<<shift)) | bitBlockBoundary<<shift;
 }
 
 void
 runtime·markscan(void *v)
 {
         uintptr *b, off, shift;
 
         off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;  // word offset
         b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
         shift = off % wordsPerBitmapWord;
@@ skipping 114 matching lines @@
                 bitmapChunk = 8192
         };
         uintptr n;
 
         n = (h->arena_used - h->arena_start) / wordsPerBitmapWord;
         n = ROUND(n, bitmapChunk);
         n = ROUND(n, PhysPageSize);
         if(h->bitmap_mapped >= n)
                 return;
 
-        runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped, &mstats.gc_sys);
+        runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped, h->arena_reserved, &mstats.gc_sys);
         h->bitmap_mapped = n;
 }