code review 45770044: runtime: increase page size to 8K

src/pkg/runtime/mgc0.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Garbage collector.
 
 #include "runtime.h"
 #include "arch_GOARCH.h"
 #include "malloc.h"
 #include "stack.h"
 #include "mgc0.h"
 #include "race.h"
 #include "type.h"
 #include "typekind.h"
 #include "funcdata.h"
 #include "../../cmd/ld/textflag.h"
 
 enum {
         Debug = 0,
-        DebugMark = 0,  // run second pass to check mark
         CollectStats = 0,
         ScanStackByFrames = 1,
         IgnorePreciseGC = 0,
 
         // Four bits per word (see #defines below).
         wordsPerBitmapWord = sizeof(void*)*8/4,
         bitShift = sizeof(void*)*8/4,
 
         WorkbufSize     = 16*1024,
         RootBlockSize   = 4*1024,
         FinBlockSize    = 4*1024,
 
         handoffThreshold = 4,
         IntermediateBufferCapacity = 64,
 
         // Bits in type information
         PRECISE = 1,
         LOOP = 2,
         PC_BITS = PRECISE | LOOP,
 
         // Pointer map
         BitsPerPointer = 2,
         BitsNoPointer = 0,
         BitsPointer = 1,
         BitsIface = 2,
         BitsEface = 3,
+
+        RootData        = 0,
+        RootBss         = 1,
+        RootFinalizers  = 2,
+        RootSpanTypes   = 3,
+        RootFlushCaches = 4,
+        RootCount       = 5,
 };
 
 static struct
 {
         Lock;··
         void* head;
 } pools;
 
 void
 sync·runtime_registerPool(void **p)
 {
         runtime·lock(&pools);
         p[0] = pools.head;
         pools.head = p;
         runtime·unlock(&pools);
 }
 
 static void
 clearpools(void)
 {
-        void **p, **next;
-
-        for(p = pools.head; p != nil; p = next) {
-                next = p[0];
-                p[0] = nil; // next
-                p[1] = nil; // slice
-                p[2] = nil;
-                p[3] = nil;
+        void **pool, **next;
+        P *p, **pp;
+        int32 i;
+
+        // clear sync.Pool's
+        for(pool = pools.head; pool != nil; pool = next) {
+                next = pool[0];
+                pool[0] = nil; // next
+                pool[1] = nil; // slice
+                pool[2] = nil;
+                pool[3] = nil;
         }
         pools.head = nil;
+
+        // clear defer pools
+        for(pp=runtime·allp; p=*pp; pp++) {
+                for(i=0; i<nelem(p->deferpool); i++)
+                        p->deferpool[i] = nil;
+        }
 }
 
 // Bits in per-word bitmap.
 // #defines because enum might not be able to hold the values.
 //
 // Each word in the bitmap describes wordsPerBitmapWord words
 // of heap memory.  There are 4 bitmap bits dedicated to each heap word,
 // so on a 64-bit system there is one bitmap word per 16 heap words.
 // The bits in the word are packed together by type first, then by
 // heap location, so each 64-bit bitmap word consists of, from top to bottom,
@@ skipping 90 matching lines @@
 static FinBlock *allfin; // list of all blocks
 static Lock finlock;
 static int32 fingwait;
 
 static void runfinq(void);
 static Workbuf* getempty(Workbuf*);
 static Workbuf* getfull(Workbuf*);
 static void     putempty(Workbuf*);
 static Workbuf* handoff(Workbuf*);
 static void     gchelperstart(void);
-static void     scanstack(G* gp, void *scanbuf);
+static void     addfinroots(void *wbufp, void *v);
+static void     flushallmcaches(void);
+static void     scanframe(Stkframe *frame, void *wbufp);
+static void     addstackroots(G *gp, Workbuf **wbufp);
 
 static struct {
         uint64  full;  // lock-free list of full blocks
         uint64  empty; // lock-free list of empty blocks
         byte    pad0[CacheLineSize]; // prevents false-sharing between full/empty and nproc/nwait
         uint32  nproc;
+        int64   tstart;
         volatile uint32 nwait;
         volatile uint32 ndone;
-        volatile uint32 debugmarkdone;
         Note    alldone;
         ParFor  *markfor;
         ParFor  *sweepfor;
 
         Lock;
         byte    *chunk;
         uintptr nchunk;
-
-        Obj     *roots;
-        uint32  nroot;
-        uint32  rootcap;
 } work;
 
 enum {
         GC_DEFAULT_PTR = GC_NUM_INSTR,
         GC_CHAN,
-        GC_G_PTR,
 
         GC_NUM_INSTR2
 };
 
 static struct {
         struct {
                 uint64 sum;
                 uint64 cnt;
         } ptr;
         uint64 nbytes;
@@ skipping 64 matching lines @@
                         if(CollectStats)
                                 runtime·xadd64(&gcstats.markonly.foundword, 1);
                         goto found;
                 }
         }
 
         // Otherwise consult span table to find beginning.
         // (Manually inlined copy of MHeap_LookupMaybe.)
         k = (uintptr)obj>>PageShift;
         x = k;
-        if(sizeof(void*) == 8)
-                x -= (uintptr)runtime·mheap.arena_start>>PageShift;
+        x -= (uintptr)runtime·mheap.arena_start>>PageShift;
         s = runtime·mheap.spans[x];
         if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
                 return false;
         p = (byte*)((uintptr)s->start<<PageShift);
         if(s->sizeclass == 0) {
                 obj = p;
         } else {
                 uintptr size = s->elemsize;
                 int32 i = ((byte*)obj - p)/size;
                 obj = p+i*size;
@@ skipping 178 matching lines @@
                                 if(CollectStats)
                                         runtime·xadd64(&gcstats.flushptrbuf.foundword, 1);
                                 goto found;
                         }
                 }
 
                 // Otherwise consult span table to find beginning.
                 // (Manually inlined copy of MHeap_LookupMaybe.)
                 k = (uintptr)obj>>PageShift;
                 x = k;
-                if(sizeof(void*) == 8)
-                        x -= (uintptr)arena_start>>PageShift;
+                x -= (uintptr)arena_start>>PageShift;
                 s = runtime·mheap.spans[x];
                 if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
                         continue;
                 p = (byte*)((uintptr)s->start<<PageShift);
                 if(s->sizeclass == 0) {
                         obj = p;
                 } else {
                         size = s->elemsize;
                         int32 i = ((byte*)obj - p)/size;
                         obj = p+i*size;
@@ skipping 26 matching lines @@
                         }
                 }
 
                 // If object has no pointers, don't need to scan further.
                 if((bits & bitScan) == 0)
                         continue;
 
                 // Ask span about size class.
                 // (Manually inlined copy of MHeap_Lookup.)
                 x = (uintptr)obj >> PageShift;
-                if(sizeof(void*) == 8)
-                        x -= (uintptr)arena_start>>PageShift;
+                x -= (uintptr)arena_start>>PageShift;
                 s = runtime·mheap.spans[x];
 
                 PREFETCH(obj);
 
                 *wp = (Obj){obj, s->elemsize, ti};
                 wp++;
                 nobj++;
         continue_obj:;
         }
 
@@ skipping 67 matching lines @@
         sbuf->wbuf = wbuf;
         sbuf->nobj = nobj;
 }
 
 // Program that scans the whole block and treats every block element as a potential pointer
 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR};
 
 // Hchan program
 static uintptr chanProg[2] = {0, GC_CHAN};
 
-// G* program
-static uintptr gptrProg[2] = {0, GC_G_PTR};
-
 // Local variables of a program fragment or loop
 typedef struct Frame Frame;
 struct Frame {
         uintptr count, elemsize, b;
         uintptr *loop_or_ret;
 };
 
 // Sanity check for the derived type info objti.
 static void
 checkptr(void *obj, uintptr objti)
 {
         uintptr *pc1, *pc2, type, tisize, i, j, x;
         byte *objstart;
         Type *t;
         MSpan *s;
 
         if(!Debug)
                 runtime·throw("checkptr is debug only");
 
         if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
                 return;
         type = runtime·gettype(obj);
         t = (Type*)(type & ~(uintptr)(PtrSize-1));
         if(t == nil)
                 return;
         x = (uintptr)obj >> PageShift;
-        if(sizeof(void*) == 8)
-                x -= (uintptr)(runtime·mheap.arena_start)>>PageShift;
+        x -= (uintptr)(runtime·mheap.arena_start)>>PageShift;
         s = runtime·mheap.spans[x];
         objstart = (byte*)((uintptr)s->start<<PageShift);
         if(s->sizeclass != 0) {
                 i = ((byte*)obj - objstart)/s->elemsize;
                 objstart += i*s->elemsize;
         }
         tisize = *(uintptr*)objti;
         // Sanity check for object size: it should fit into the memory block.
         if((byte*)obj + tisize > objstart + s->elemsize) {
                 runtime·printf("object of type '%S' at %p/%p does not fit in block %p/%p\n",
@@ skipping 21 matching lines @@
                 }
         }
 }·······································
 
 // scanblock scans a block of n bytes starting at pointer b for references
 // to other objects, scanning any it finds recursively until there are no
 // unscanned objects left.  Instead of using an explicit recursion, it keeps
 // a work list in the Workbuf* structures and loops in the main function
 // body.  Keeping an explicit work list is easier on the stack allocator and
 // more efficient.
-//
-// wbuf: current work buffer
-// wp:   storage for next queued pointer (write pointer)
-// nobj: number of queued objects
 static void
-scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
+scanblock(Workbuf *wbuf, bool keepworking)
 {
         byte *b, *arena_start, *arena_used;
-        uintptr n, i, end_b, elemsize, size, ti, objti, count, type;
+        uintptr n, i, end_b, elemsize, size, ti, objti, count, type, nobj;
         uintptr *pc, precise_type, nominal_size;
         uintptr *chan_ret, chancap;
         void *obj;
         Type *t;
         Slice *sliceptr;
         Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4];
         BufferList *scanbuffers;
         Scanbuf sbuf;
         Eface *eface;
         Iface *iface;
         Hchan *chan;
         ChanType *chantype;
+        Obj *wp;
 
         if(sizeof(Workbuf) % WorkbufSize != 0)
                 runtime·throw("scanblock: size of Workbuf is suboptimal");
 
         // Memory arena parameters.
         arena_start = runtime·mheap.arena_start;
         arena_used = runtime·mheap.arena_used;
 
         stack_ptr = stack+nelem(stack)-1;
 
         precise_type = false;
         nominal_size = 0;
+
+        if(wbuf) {
+                nobj = wbuf->nobj;
+                wp = &wbuf->obj[nobj];
+        } else {
+                nobj = 0;
+                wp = nil;
+        }
 
         // Initialize sbuf
         scanbuffers = &bufferList[m->helpgc];
 
         sbuf.ptr.begin = sbuf.ptr.pos = &scanbuffers->ptrtarget[0];
         sbuf.ptr.end = sbuf.ptr.begin + nelem(scanbuffers->ptrtarget);
 
         sbuf.obj.begin = sbuf.obj.pos = &scanbuffers->obj[0];
         sbuf.obj.end = sbuf.obj.begin + nelem(scanbuffers->obj);
 
@@ skipping 315 matching lines @@
                                                 (uintptr)chantype->elem->gc | PRECISE | LOOP};
                                         if(sbuf.obj.pos == sbuf.obj.end)
                                                 flushobjbuf(&sbuf);
                                 }
                         }
                         if(chan_ret == nil)
                                 goto next_block;
                         pc = chan_ret;
                         continue;
 
-                case GC_G_PTR:
-                        obj = (void*)stack_top.b;
-                        scanstack(obj, &sbuf);
-                        goto next_block;
-
                 default:
                         runtime·throw("scanblock: invalid GC instruction");
                         return;
                 }
 
                 if(obj >= arena_start && obj < arena_used) {
                         *sbuf.ptr.pos++ = (PtrTarget){obj, objti};
                         if(sbuf.ptr.pos == sbuf.ptr.end)
                                 flushptrbuf(&sbuf);
                 }
@@ skipping 24 matching lines @@
 
                 // Fetch b from the work buffer.
                 --sbuf.wp;
                 b = sbuf.wp->p;
                 n = sbuf.wp->n;
                 ti = sbuf.wp->ti;
                 sbuf.nobj--;
         }
 }
 
-// debug_scanblock is the debug copy of scanblock.
-// it is simpler, slower, single-threaded, recursive,
-// and uses bitSpecial as the mark bit.
-static void
-debug_scanblock(byte *b, uintptr n)
-{
-        byte *obj, *p;
-        void **vp;
-        uintptr size, *bitp, bits, shift, i, xbits, off;
-        MSpan *s;
-
-        if(!DebugMark)
-                runtime·throw("debug_scanblock without DebugMark");
-
-        if((intptr)n < 0) {
-                runtime·printf("debug_scanblock %p %D\n", b, (int64)n);
-                runtime·throw("debug_scanblock");
-        }
-
-        // Align b to a word boundary.
-        off = (uintptr)b & (PtrSize-1);
-        if(off != 0) {
-                b += PtrSize - off;
-                n -= PtrSize - off;
-        }
-
-        vp = (void**)b;
-        n /= PtrSize;
-        for(i=0; i<n; i++) {
-                obj = (byte*)vp[i];
-
-                // Words outside the arena cannot be pointers.
-                if((byte*)obj < runtime·mheap.arena_start || (byte*)obj >= runtime·mheap.arena_used)
-                        continue;
-
-                // Round down to word boundary.
-                obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
-
-                // Consult span table to find beginning.
-                s = runtime·MHeap_LookupMaybe(&runtime·mheap, obj);
-                if(s == nil)
-                        continue;
-
-                p =  (byte*)((uintptr)s->start<<PageShift);
-                size = s->elemsize;
-                if(s->sizeclass == 0) {
-                        obj = p;
-                } else {
-                        int32 i = ((byte*)obj - p)/size;
-                        obj = p+i*size;
-                }
-
-                // Now that we know the object header, reload bits.
-                off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
-                bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-                shift = off % wordsPerBitmapWord;
-                xbits = *bitp;
-                bits = xbits >> shift;
-
-                // Now we have bits, bitp, and shift correct for
-                // obj pointing at the base of the object.
-                // If not allocated or already marked, done.
-                if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0)  // NOTE: bitSpecial not bitMarked
-                        continue;
-                *bitp |= bitSpecial<<shift;
-                if(!(bits & bitMarked))
-                        runtime·printf("found unmarked block %p in %p\n", obj, vp+i);
-
-                // If object has no pointers, don't need to scan further.
-                if((bits & bitScan) == 0)
-                        continue;
-
-                debug_scanblock(obj, size);
-        }
-}
-
 // Append obj to the work buffer.
 // _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
 static void
 enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj)
 {
         uintptr nobj, off;
         Obj *wp;
         Workbuf *wbuf;
 
         if(Debug > 1)
@@ skipping 36 matching lines @@
         wp++;
         nobj++;
 
         // Save work buffer state
         *_wp = wp;
         *_wbuf = wbuf;
         *_nobj = nobj;
 }
 
 static void
+enqueue1(Workbuf **wbufp, Obj obj)
+{
+        Workbuf *wbuf;
+
+        wbuf = *wbufp;
+        if(wbuf->nobj >= nelem(wbuf->obj))
+                *wbufp = wbuf = getempty(wbuf);
+        wbuf->obj[wbuf->nobj++] = obj;
+}
+
+static void
 markroot(ParFor *desc, uint32 i)
 {
-        Obj *wp;
         Workbuf *wbuf;
-        uintptr nobj;
+        FinBlock *fb;
+        MSpan **allspans, *s;
+        uint32 spanidx;
+        G *gp;
 
         USED(&desc);
-        wp = nil;
-        wbuf = nil;
-        nobj = 0;
-        enqueue(work.roots[i], &wbuf, &wp, &nobj);
-        scanblock(wbuf, wp, nobj, false);
+        wbuf = getempty(nil);
+        switch(i) {
+        case RootData:
+                enqueue1(&wbuf, (Obj){data, edata - data, (uintptr)gcdata});
+                break;
+
+        case RootBss:
+                enqueue1(&wbuf, (Obj){bss, ebss - bss, (uintptr)gcbss});
+                break;
+
+        case RootFinalizers:
+                for(fb=allfin; fb; fb=fb->alllink)
+                        enqueue1(&wbuf, (Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0});
+                break;
+
+        case RootSpanTypes:
+                // mark span types and MSpan.specials (to walk spans only once)
+                allspans = runtime·mheap.allspans;
+                for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
+                        Special *sp;
+                        SpecialFinalizer *spf;
+
+                        s = allspans[spanidx];
+                        if(s->state != MSpanInUse)
+                                continue;
+                        // The garbage collector ignores type pointers stored in MSpan.types:
+                        //  - Compiler-generated types are stored outside of heap.
+                        //  - The reflect package has runtime-generated types cached in its data structures.
+                        //    The garbage collector relies on finding the references via that cache.
+                        if(s->types.compression == MTypes_Words || s->types.compression == MTypes_Bytes)
+                                markonly((byte*)s->types.data);
+                        for(sp = s->specials; sp != nil; sp = sp->next) {
+                                if(sp->kind != KindSpecialFinalizer)
+                                        continue;
+                                // don't mark finalized object, but scan it so we
+                                // retain everything it points to.
+                                spf = (SpecialFinalizer*)sp;
+                                enqueue1(&wbuf, (Obj){(void*)((s->start << PageShift) + spf->offset), s->elemsize, 0});
+                                enqueue1(&wbuf, (Obj){(void*)&spf->fn, PtrSize, 0});
+                                enqueue1(&wbuf, (Obj){(void*)&spf->fint, PtrSize, 0});
+                                enqueue1(&wbuf, (Obj){(void*)&spf->ot, PtrSize, 0});
+                        }
+                }
+                break;
+
+        case RootFlushCaches:
+                flushallmcaches();
+                break;
+
+        default:
+                // the rest is scanning goroutine stacks
+                if(i - RootCount >= runtime·allglen)
+                        runtime·throw("markroot: bad index");
+                gp = runtime·allg[i - RootCount];
+                // remember when we've first observed the G blocked
+                // needed only to output in traceback
+                if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->waitsince == 0)
+                        gp->waitsince = work.tstart;
+                addstackroots(gp, &wbuf);
+                break;
+         ·······
+        }
+
+        if(wbuf)
+                scanblock(wbuf, false);
 }
 
 // Get an empty work buffer off the work.empty list,
 // allocating new buffers as needed.
 static Workbuf*
 getempty(Workbuf *b)
 {
         if(b != nil)
                 runtime·lfstackpush(&work.full, &b->node);
         b = (Workbuf*)runtime·lfstackpop(&work.empty);
@@ skipping 76 matching lines @@
         b1->nobj = n;
         runtime·memmove(b1->obj, b->obj+b->nobj, n*sizeof b1->obj[0]);
         m->gcstats.nhandoff++;
         m->gcstats.nhandoffcnt += n;
 
         // Put b on full list - let first half of b get stolen.
         runtime·lfstackpush(&work.full, &b->node);
         return b1;
 }
 
-static void
-addroot(Obj obj)
-{
-        uint32 cap;
-        Obj *new;
-
-        if(work.nroot >= work.rootcap) {
-                cap = RootBlockSize/sizeof(Obj);
-                if(cap < 2*work.rootcap)
-                        cap = 2*work.rootcap;
-                new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj), &mstats.gc_sys);
-                if(new == nil)
-                        runtime·throw("runtime: cannot allocate memory");
-                if(work.roots != nil) {
-                        runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj));
-                        runtime·SysFree(work.roots, work.rootcap*sizeof(Obj), &mstats.gc_sys);
-                }
-                work.roots = new;
-                work.rootcap = cap;
-        }
-        work.roots[work.nroot] = obj;
-        work.nroot++;
-}
-
 extern byte pclntab[]; // base for f->ptrsoff
 
 typedef struct BitVector BitVector;
 struct BitVector
 {
         int32 n;
         uint32 data[];
 };
 
 typedef struct StackMap StackMap;
@@ skipping 19 matching lines @@
                 bv = (BitVector*)ptr;
                 words = ((bv->n + 31) / 32) + 1;
                 ptr += words;
         }
         return (BitVector*)ptr;
 }
 
 // Scans an interface data value when the interface type indicates
 // that it is a pointer.
 static void
-scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue, Scanbuf *sbuf)
+scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue, void *wbufp)
 {
         Itab *tab;
         Type *type;
 
         if(runtime·precisestack && afterprologue) {
                 if(bits == BitsIface) {
                         tab = *(Itab**)scanp;
                         if(tab->type->size <= sizeof(void*) && (tab->type->kind & KindNoPointers))
                                 return;
                 } else { // bits == BitsEface
                         type = *(Type**)scanp;
                         if(type->size <= sizeof(void*) && (type->kind & KindNoPointers))
                                 return;
                 }
         }
-        *sbuf->obj.pos++ = (Obj){scanp+PtrSize, PtrSize, 0};
-        if(sbuf->obj.pos == sbuf->obj.end)
-                flushobjbuf(sbuf);
+        enqueue1(wbufp, (Obj){scanp+PtrSize, PtrSize, 0});
 }
 
 // Starting from scanp, scans words corresponding to set bits.
 static void
-scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, Scanbuf *sbuf)
+scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, void *wbufp)
 {
         uintptr word, bits;
         uint32 *wordp;
         int32 i, remptrs;
 
         wordp = bv->data;
         for(remptrs = bv->n; remptrs > 0; remptrs -= 32) {
                 word = *wordp++;
                 if(remptrs < 32)
                         i = remptrs;
                 else
                         i = 32;
                 i /= BitsPerPointer;
                 for(; i > 0; i--) {
                         bits = word & 3;
                         if(bits != BitsNoPointer && *(void**)scanp != nil)
-                                if(bits == BitsPointer) {
-                                        *sbuf->obj.pos++ = (Obj){scanp, PtrSize, 0};
-                                        if(sbuf->obj.pos == sbuf->obj.end)
-                                                flushobjbuf(sbuf);
-                                } else
-                                        scaninterfacedata(bits, scanp, afterprologue, sbuf);
+                                if(bits == BitsPointer)
+                                        enqueue1(wbufp, (Obj){scanp, PtrSize, 0});
+                                else
+                                        scaninterfacedata(bits, scanp, afterprologue, wbufp);
                         word >>= BitsPerPointer;
                         scanp += PtrSize;
                 }
         }
 }
 
 // Scan a stack frame: local variables and function arguments/results.
 static void
-scanframe(Stkframe *frame, void *arg)
+scanframe(Stkframe *frame, void *wbufp)
 {
         Func *f;
-        Scanbuf *sbuf;
         StackMap *stackmap;
         BitVector *bv;
         uintptr size;
         uintptr targetpc;
         int32 pcdata;
         bool afterprologue;
 
         f = frame->fn;
         targetpc = frame->pc;
         if(targetpc != f->entry)
                 targetpc--;
         pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc);
         if(pcdata == -1) {
                 // We do not have a valid pcdata value but there might be a
                 // stackmap for this function.  It is likely that we are looking
                 // at the function prologue, assume so and hope for the best.
                 pcdata = 0;
         }
 
-        sbuf = arg;
         // Scan local variables if stack frame has been allocated.
         // Use pointer information if known.
         afterprologue = (frame->varp > (byte*)frame->sp);
         if(afterprologue) {
                 stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps);
                 if(stackmap == nil) {
                         // No locals information, scan everything.
                         size = frame->varp - (byte*)frame->sp;
-                        *sbuf->obj.pos++ = (Obj){frame->varp - size, size, 0};
-                        if(sbuf->obj.pos == sbuf->obj.end)
-                                flushobjbuf(sbuf);
+                        enqueue1(wbufp, (Obj){frame->varp - size, size, 0});
                 } else if(stackmap->n < 0) {
                         // Locals size information, scan just the locals.
                         size = -stackmap->n;
-                        *sbuf->obj.pos++ = (Obj){frame->varp - size, size, 0};
-                        if(sbuf->obj.pos == sbuf->obj.end)
-                                flushobjbuf(sbuf);              } else if(stackmap->n > 0) {
+                        enqueue1(wbufp, (Obj){frame->varp - size, size, 0});
+                } else if(stackmap->n > 0) {
                         // Locals bitmap information, scan just the pointers in
                         // locals.
                         if(pcdata < 0 || pcdata >= stackmap->n) {
                                 // don't know where we are
-                                runtime·printf("pcdata is %d and %d stack map entries\n", pcdata, stackmap->n);
-                                runtime·throw("addframeroots: bad symbol table");
+                                runtime·printf("pcdata is %d and %d stack map entries for %s (targetpc=%p)\n",
+                                        pcdata, stackmap->n, runtime·funcname(f), targetpc);
+                                runtime·throw("scanframe: bad symbol table");
                         }
                         bv = stackmapdata(stackmap, pcdata);
                         size = (bv->n * PtrSize) / BitsPerPointer;
-                        scanbitvector(frame->varp - size, bv, afterprologue, sbuf);
+                        scanbitvector(frame->varp - size, bv, afterprologue, wbufp);
                 }
         }
 
         // Scan arguments.
         // Use pointer information if known.
         stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
         if(stackmap != nil) {
                 bv = stackmapdata(stackmap, pcdata);
-                scanbitvector(frame->argp, bv, false, sbuf);
-        } else {
-                *sbuf->obj.pos++ = (Obj){frame->argp, frame->arglen, 0};
-                if(sbuf->obj.pos == sbuf->obj.end)
-                        flushobjbuf(sbuf);
-        }
+                scanbitvector(frame->argp, bv, false, wbufp);
+        } else
+                enqueue1(wbufp, (Obj){frame->argp, frame->arglen, 0});
 }
 
 static void
-scanstack(G* gp, void *scanbuf)
-{
-        runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, scanframe, scanbuf, false);
-}
-
-static void
-addstackroots(G *gp)
+addstackroots(G *gp, Workbuf **wbufp)
 {
         M *mp;
         int32 n;
         Stktop *stk;
         uintptr sp, guard;
         void *base;
         uintptr size;
+
+        switch(gp->status){
+        default:
+                runtime·printf("unexpected G.status %d (goroutine %p %D)\n", gp->status, gp, gp->goid);
+                runtime·throw("mark - bad status");
+        case Gdead:
+                return;
+        case Grunning:
+                runtime·throw("mark - world not stopped");
+        case Grunnable:
+        case Gsyscall:
+        case Gwaiting:
+                break;
+        }
 
         if(gp == g)
                 runtime·throw("can't scan our own stack");
         if((mp = gp->m) != nil && mp->helpgc)
                 runtime·throw("can't scan gchelper stack");
         if(gp->syscallstack != (uintptr)nil) {
                 // Scanning another goroutine that is about to enter or might
                 // have just exited a system call. It may be executing code such
                 // as schedlock and may have needed to start a new stack segment.
                 // Use the stack segment and stack pointer at the time of
                 // the system call instead, since that won't change underfoot.
                 sp = gp->syscallsp;
                 stk = (Stktop*)gp->syscallstack;
                 guard = gp->syscallguard;
         } else {
                 // Scanning another goroutine's stack.
                 // The goroutine is usually asleep (the world is stopped).
                 sp = gp->sched.sp;
                 stk = (Stktop*)gp->stackbase;
                 guard = gp->stackguard;
                 // For function about to start, context argument is a root too.
                 if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil))
-                        addroot((Obj){base, size, 0});
+                        enqueue1(wbufp, (Obj){base, size, 0});
         }
         if(ScanStackByFrames) {
                 USED(sp);
                 USED(stk);
                 USED(guard);
-                addroot((Obj){(byte*)gp, PtrSize, (uintptr)gptrProg});
+                runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, scanframe, wbufp, false);
         } else {
                 n = 0;
                 while(stk) {
                         if(sp < guard-StackGuard || (uintptr)stk < sp) {
                                 runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
                                 runtime·throw("scanstack");
                         }
-                        addroot((Obj){(byte*)sp, (uintptr)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
+                        enqueue1(wbufp, (Obj){(byte*)sp, (uintptr)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
                         sp = stk->gobuf.sp;
                         guard = stk->stackguard;
                         stk = (Stktop*)stk->stackbase;
                         n++;
                 }
         }
 }
 
-static void
-addfinroots(void *v)
-{
-        uintptr size;
-        void *base;
-
-        size = 0;
-        if(!runtime·mlookup(v, &base, &size, nil) || !runtime·blockspecial(base))
-                runtime·throw("mark - finalizer inconsistency");
-
-        // do not mark the finalizer block itself.  just mark the things it points at.
-        addroot((Obj){base, size, 0});
-}
-
-static void
-addroots(void)
-{
-        G *gp;
-        FinBlock *fb;
-        MSpan *s, **allspans;
-        uint32 spanidx;
-
-        work.nroot = 0;
-
-        // data & bss
-        // TODO(atom): load balancing
-        addroot((Obj){data, edata - data, (uintptr)gcdata});
-        addroot((Obj){bss, ebss - bss, (uintptr)gcbss});
-
-        // MSpan.types
-        allspans = runtime·mheap.allspans;
-        for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
-                s = allspans[spanidx];
-                if(s->state == MSpanInUse) {
-                        // The garbage collector ignores type pointers stored in MSpan.types:
-                        //  - Compiler-generated types are stored outside of heap.
-                        //  - The reflect package has runtime-generated types cached in its data structures.
-                        //    The garbage collector relies on finding the references via that cache.
-                        switch(s->types.compression) {
-                        case MTypes_Empty:
-                        case MTypes_Single:
-                                break;
-                        case MTypes_Words:
-                        case MTypes_Bytes:
-                                markonly((byte*)s->types.data);
-                                break;
-                        }
-                }
-        }
-
-        // stacks
-        for(gp=runtime·allg; gp!=nil; gp=gp->alllink) {
-                switch(gp->status){
-                default:
-                        runtime·printf("unexpected G.status %d\n", gp->status);
-                        runtime·throw("mark - bad status");
-                case Gdead:
-                        break;
-                case Grunning:
-                        runtime·throw("mark - world not stopped");
-                case Grunnable:
-                case Gsyscall:
-                case Gwaiting:
-                        addstackroots(gp);
-                        break;
-                }
-        }
-
-        runtime·walkfintab(addfinroots);
-
-        for(fb=allfin; fb; fb=fb->alllink)
-                addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0});
-}
-
-static void
-addfreelists(void)
-{
-        int32 i;
-        P *p, **pp;
-        MCache *c;
-        MLink *m;
-
-        // Mark objects in the MCache of each P so we don't collect them.
-        for(pp=runtime·allp; p=*pp; pp++) {
-                c = p->mcache;
-                if(c==nil)
-                        continue;
-                for(i = 0; i < NumSizeClasses; i++) {
-                        for(m = c->list[i].list; m != nil; m = m->next) {
-                                markonly(m);
-                        }
-                }
-        }
-        // Note: the sweeper will mark objects in each span's freelist.
-}
-
-static bool
-handlespecial(byte *p, uintptr size)
-{
-        FuncVal *fn;
-        uintptr nret;
-        PtrType *ot;
-        Type *fint;
+void
+runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType *ot)
+{
         FinBlock *block;
         Finalizer *f;
-
-        if(!runtime·getfinalizer(p, true, &fn, &nret, &fint, &ot)) {
-                runtime·setblockspecial(p, false);
-                runtime·MProf_Free(p, size);
-                return false;
-        }
 
         runtime·lock(&finlock);
         if(finq == nil || finq->cnt == finq->cap) {
                 if(finc == nil) {
                         finc = runtime·persistentalloc(FinBlockSize, 0, &mstats.gc_sys);
                         finc->cap = (FinBlockSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1;
                         finc->alllink = allfin;
                         allfin = finc;
                 }
                 block = finc;
                 finc = block->next;
                 block->next = finq;
                 finq = block;
         }
         f = &finq->fin[finq->cnt];
         finq->cnt++;
         f->fn = fn;
         f->nret = nret;
         f->fint = fint;
         f->ot = ot;
         f->arg = p;
         runtime·unlock(&finlock);
-        return true;
 }
 
 // Sweep frees or collects finalizers for blocks not marked in the mark phase.
 // It clears the mark bits in preparation for the next GC round.
 static void
 sweepspan(ParFor *desc, uint32 idx)
 {
         int32 cl, n, npages;
-        uintptr size, off, *bitp, shift;
+        uintptr size, off, *bitp, shift, bits;
         byte *p;
         MCache *c;
         byte *arena_start;
         MLink head, *end;
         int32 nfree;
         byte *type_data;
         byte compression;
         uintptr type_data_inc;
         MSpan *s;
         MLink *x;
+        Special *special, **specialp, *y;
 
         USED(&desc);
         s = runtime·mheap.allspans[idx];
         if(s->state != MSpanInUse)
                 return;
         arena_start = runtime·mheap.arena_start;
-        p = (byte*)(s->start << PageShift);
         cl = s->sizeclass;
         size = s->elemsize;
         if(cl == 0) {
                 n = 1;
         } else {
                 // Chunk full of small blocks.
                 npages = runtime·class_to_allocnpages[cl];
                 n = (npages << PageShift) / size;
         }
         nfree = 0;
         end = &head;
         c = m->mcache;
 
         // mark any free objects in this span so we don't collect them
         for(x = s->freelist; x != nil; x = x->next) {
                 // This is markonly(x) but faster because we don't need
                 // atomic access and we're guaranteed to be pointing at
                 // the head of a valid object.
                 off = (uintptr*)x - (uintptr*)runtime·mheap.arena_start;
                 bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
                 shift = off % wordsPerBitmapWord;
                 *bitp |= bitMarked<<shift;
         }
 ········
+        // Unlink & free special records for any objects we're about to free.
+        specialp = &s->specials;
+        special = *specialp;
+        while(special != nil) {
+                p = (byte*)(s->start << PageShift) + special->offset;
+                off = (uintptr*)p - (uintptr*)arena_start;
+                bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
+                shift = off % wordsPerBitmapWord;
+                bits = *bitp>>shift;
+                if((bits & (bitAllocated|bitMarked)) == bitAllocated) {
+                        // about to free object: splice out special record
+                        y = special;
+                        special = special->next;
+                        *specialp = special;
+                        if(!runtime·freespecial(y, p, size)) {
+                                // stop freeing of object if it has a finalizer
+                                *bitp |= bitMarked << shift;
+                        }
+                } else {
+                        // object is still live: keep special record
+                        specialp = &special->next;
+                        special = *specialp;
+                }
+        }
+
         type_data = (byte*)s->types.data;
         type_data_inc = sizeof(uintptr);
         compression = s->types.compression;
         switch(compression) {
         case MTypes_Bytes:
                 type_data += 8*sizeof(uintptr);
                 type_data_inc = 1;
                 break;
         }
 
         // Sweep through n objects of given size starting at p.
         // This thread owns the span now, so it can manipulate
         // the block bitmap without atomic operations.
+        p = (byte*)(s->start << PageShift);
         for(; n > 0; n--, p += size, type_data+=type_data_inc) {
-                uintptr off, *bitp, shift, bits;
-
                 off = (uintptr*)p - (uintptr*)arena_start;
                 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
                 shift = off % wordsPerBitmapWord;
                 bits = *bitp>>shift;
 
                 if((bits & bitAllocated) == 0)
                         continue;
 
                 if((bits & bitMarked) != 0) {
-                        if(DebugMark) {
-                                if(!(bits & bitSpecial))
-                                        runtime·printf("found spurious mark on %p\n", p);
-                                *bitp &= ~(bitSpecial<<shift);
-                        }
                         *bitp &= ~(bitMarked<<shift);
                         continue;
-                }
-
-                // Special means it has a finalizer or is being profiled.
-                // In DebugMark mode, the bit has been coopted so
-                // we have to assume all blocks are special.
-                if(DebugMark || (bits & bitSpecial) != 0) {
-                        if(handlespecial(p, size))
-                                continue;
                 }
 
                 // Clear mark, scan, and special bits.
                 *bitp &= ~((bitScan|bitMarked|bitSpecial)<<shift);
 
                 if(cl == 0) {
                         // Free large span.
                         runtime·unmarkspan(p, 1<<PageShift);
                         *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll;   // needs zeroing
                         if(runtime·debug.efence)
                                 runtime·SysFree(p, size, &mstats.gc_sys);
                         else
                                 runtime·MHeap_Free(&runtime·mheap, s, 1);
                         c->local_nlargefree++;
                         c->local_largefree += size;
                 } else {
                         // Free small object.
                         switch(compression) {
                         case MTypes_Words:
                                 *(uintptr*)type_data = 0;
                                 break;
                         case MTypes_Bytes:
                                 *(byte*)type_data = 0;
                                 break;
                         }
-                        if(size > sizeof(uintptr))
+                        if(size > 2*sizeof(uintptr))
                                 ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll;       // mark as "needs to be zeroed"
-                 ·······
+                        else if(size > sizeof(uintptr))
+                                ((uintptr*)p)[1] = 0;
+
                         end->next = (MLink*)p;
                         end = (MLink*)p;
                         nfree++;
                 }
         }
 
         if(nfree) {
                 c->local_nsmallfree[cl] += nfree;
                 c->local_cachealloc -= nfree * size;
                 runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, nfree, head.next, end);
@@ skipping 72 matching lines @@
         uint32 spanidx;
 
         for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
                 dumpspan(spanidx);
         }
 }
 
 void
 runtime·gchelper(void)
 {
+        int32 nproc;
+
         gchelperstart();
 
         // parallel mark for over gc roots
         runtime·parfordo(work.markfor);
 
         // help other threads scan secondary blocks
-        scanblock(nil, nil, 0, true);
-
-        if(DebugMark) {
-                // wait while the main thread executes mark(debug_scanblock)
-                while(runtime·atomicload(&work.debugmarkdone) == 0)
-                        runtime·usleep(10);
-        }
+        scanblock(nil, true);
 
         runtime·parfordo(work.sweepfor);
         bufferList[m->helpgc].busy = 0;
-        if(runtime·xadd(&work.ndone, +1) == work.nproc-1)
+        nproc = work.nproc;  // work.nproc can change right after we increment work.ndone
+        if(runtime·xadd(&work.ndone, +1) == nproc-1)
                 runtime·notewakeup(&work.alldone);
 }
 
 #define GcpercentUnknown (-2)
 
 // Initialized from $GOGC.  GOGC=off means no gc.
 //
 // Next gc is after we've allocated an extra amount of
 // memory proportional to the amount already in use.
 // If gcpercent=100 and we're using 4M, we'll gc again
@@ skipping 11 matching lines @@
 
         for(pp=runtime·allp; p=*pp; pp++) {
                 c = p->mcache;
                 if(c==nil)
                         continue;
                 runtime·purgecachedstats(c);
         }
 }
 
 static void
+flushallmcaches(void)
+{
+        P *p, **pp;
+        MCache *c;
+
+        // Flush MCache's to MCentral.
+        for(pp=runtime·allp; p=*pp; pp++) {
+                c = p->mcache;
+                if(c==nil)
+                        continue;
+                runtime·MCache_ReleaseAll(c);
+        }
+}
+
+static void
 updatememstats(GCStats *stats)
 {
         M *mp;
         MSpan *s;
-        MCache *c;
-        P *p, **pp;
         int32 i;
         uint64 stacks_inuse, smallfree;
         uint64 *src, *dst;
 
         if(stats)
                 runtime·memclr((byte*)stats, sizeof(*stats));
         stacks_inuse = 0;
         for(mp=runtime·allm; mp; mp=mp->alllink) {
                 stacks_inuse += mp->stackinuse*FixedStack;
                 if(stats) {
@@ skipping 20 matching lines @@
         mstats.alloc = 0;
         mstats.total_alloc = 0;
         mstats.nmalloc = 0;
         mstats.nfree = 0;
         for(i = 0; i < nelem(mstats.by_size); i++) {
                 mstats.by_size[i].nmalloc = 0;
                 mstats.by_size[i].nfree = 0;
         }
 
         // Flush MCache's to MCentral.
-        for(pp=runtime·allp; p=*pp; pp++) {
-                c = p->mcache;
-                if(c==nil)
-                        continue;
-                runtime·MCache_ReleaseAll(c);
-        }
+        flushallmcaches();
 
         // Aggregate local stats.
         cachestats();
 
         // Scan all spans and count number of alive objects.
         for(i = 0; i < runtime·mheap.nspan; i++) {
                 s = runtime·mheap.allspans[i];
                 if(s->state != MSpanInUse)
                         continue;
                 if(s->sizeclass == 0) {
@@ skipping 87 matching lines @@
                 // typically threads which lost the race to grab
                 // worldsema exit here when gc is done.
                 runtime·semrelease(&runtime·worldsema);
                 return;
         }
 
         // Ok, we're doing it!  Stop everybody else
         a.start_time = runtime·nanotime();
         m->gcing = 1;
         runtime·stoptheworld();
+········
+        if(runtime·debug.allocfreetrace)
+                runtime·MProf_TraceGC();
 
         clearpools();
 
         // Run gc on the g0 stack.  We do this so that the g stack
         // we're currently running on will no longer change.  Cuts
         // the root set down a bit (g0 stacks are not scanned, and
         // we don't need to scan gc's internal state).  Also an
         // enabler for copyable stacks.
         for(i = 0; i < (runtime·debug.gctrace > 1 ? 2 : 1); i++) {
                 // switch to g0, call gc(&a), then switch back
@@ skipping 41 matching lines @@
 gc(struct gc_args *args)
 {
         int64 t0, t1, t2, t3, t4;
         uint64 heap0, heap1, obj0, obj1, ninstr;
         GCStats stats;
         M *mp;
         uint32 i;
         Eface eface;
 
         t0 = args->start_time;
+        work.tstart = args->start_time;·
 
         if(CollectStats)
                 runtime·memclr((byte*)&gcstats, sizeof(gcstats));
 
         for(mp=runtime·allm; mp; mp=mp->alllink)
                 runtime·settype_flush(mp);
 
         heap0 = 0;
         obj0 = 0;
         if(runtime·debug.gctrace) {
@@ skipping 10 matching lines @@
         m->locks--;
 
         if(itabtype == nil) {
                 // get C pointer to the Go type "itab"
                 runtime·gc_itab_ptr(&eface);
                 itabtype = ((PtrType*)eface.type)->elem;
         }
 
         work.nwait = 0;
         work.ndone = 0;
-        work.debugmarkdone = 0;
         work.nproc = runtime·gcprocs();
-        addroots();
-        addfreelists();
-        runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, markroot);
+        runtime·parforsetup(work.markfor, work.nproc, RootCount + runtime·allglen, nil, false, markroot);
         runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil, true, sweepspan);
         if(work.nproc > 1) {
                 runtime·noteclear(&work.alldone);
                 runtime·helpgc(work.nproc);
         }
 
         t1 = runtime·nanotime();
 
         gchelperstart();
         runtime·parfordo(work.markfor);
-        scanblock(nil, nil, 0, true);
-
-        if(DebugMark) {
-                for(i=0; i<work.nroot; i++)
-                        debug_scanblock(work.roots[i].p, work.roots[i].n);
-                runtime·atomicstore(&work.debugmarkdone, 1);
-        }
+        scanblock(nil, true);
+
         t2 = runtime·nanotime();
 
         runtime·parfordo(work.sweepfor);
         bufferList[m->helpgc].busy = 0;
         t3 = runtime·nanotime();
 
         if(work.nproc > 1)
                 runtime·notesleep(&work.alldone);
 
         cachestats();
@@ skipping 142 matching lines @@
         Eface *ef, ef1;
 
         frame = nil;
         framecap = 0;
         for(;;) {
                 runtime·lock(&finlock);
                 fb = finq;
                 finq = nil;
                 if(fb == nil) {
                         fingwait = 1;
-                        runtime·park(runtime·unlock, &finlock, "finalizer wait");
+                        runtime·parkunlock(&finlock, "finalizer wait");
                         continue;
                 }
                 runtime·unlock(&finlock);
                 if(raceenabled)
                         runtime·racefingo();
                 for(; fb; fb=next) {
                         next = fb->next;
                         for(i=0; i<fb->cnt; i++) {
                                 f = &fb->fin[i];
                                 framesz = sizeof(Eface) + f->nret;
@@ skipping 198 matching lines @@
                 runtime·throw("unmarkspan: unaligned length");
         // Okay to use non-atomic ops here, because we control
         // the entire span, and each bitmap word has bits for only
         // one span, so no other goroutines are changing these
         // bitmap words.
         n /= wordsPerBitmapWord;
         while(n-- > 0)
                 *b-- = 0;
 }
 
-bool
-runtime·blockspecial(void *v)
-{
-        uintptr *b, off, shift;
-
-        if(DebugMark)
-                return true;
-
-        off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;
-        b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-        shift = off % wordsPerBitmapWord;
-
-        return (*b & (bitSpecial<<shift)) != 0;
-}
-
-void
-runtime·setblockspecial(void *v, bool s)
-{
-        uintptr *b, off, shift, bits, obits;
-
-        if(DebugMark)
-                return;
-
-        off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;
-        b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-        shift = off % wordsPerBitmapWord;
-
-        for(;;) {
-                obits = *b;
-                if(s)
-                        bits = obits | (bitSpecial<<shift);
-                else
-                        bits = obits & ~(bitSpecial<<shift);
-                if(runtime·gomaxprocs == 1) {
-                        *b = bits;
-                        break;
-                } else {
-                        // more than one goroutine is potentially running: use atomic op
-                        if(runtime·casp((void**)b, (void*)obits, (void*)bits))
-                                break;
-                }
-        }
-}
-
 void
 runtime·MHeap_MapBits(MHeap *h)
 {
         // Caller has added extra mappings to the arena.
         // Add extra mappings of bitmap words as needed.
         // We allocate extra bitmap pieces in chunks of bitmapChunk.
         enum {
                 bitmapChunk = 8192
         };
         uintptr n;
 
         n = (h->arena_used - h->arena_start) / wordsPerBitmapWord;
         n = ROUND(n, bitmapChunk);
         if(h->bitmap_mapped >= n)
                 return;
 
         runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped, &mstats.gc_sys);
         h->bitmap_mapped = n;
 }