code review 12328044: cmd/cc, cmd/gc, runtime: emit bitmaps for scanning locals.

src/pkg/runtime/mgc0.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Garbage collector.
 
 #include "runtime.h"
 #include "arch_GOARCH.h"
 #include "malloc.h"
 #include "stack.h"
@@ skipping 1369 matching lines @@
                 }
                 work.roots = new;
                 work.rootcap = cap;
         }
         work.roots[work.nroot] = obj;
         work.nroot++;
 }
 
 extern byte pclntab[]; // base for f->ptrsoff
 
+typedef struct BitVector BitVector;
 struct BitVector
 {
         int32 n;
         uint32 data[];
 };
-typedef struct BitVector BitVector;
 
 // Starting from scanp, scans words corresponding to set bits.
 static void
-scanbitvector(byte* scanp, BitVector *bv)
-{
-        int32 i, j, remptrs;
-        uint32 w, b;
-
-        for(i = 0, remptrs = bv->n; i < bv->n; i++) {
-                w = bv->data[i];
-                b = 1;
-                j = remptrs;
-                if(j > 32)
-                        j = 32;
-                for(; j > 0; j--) {
-                        if(w & b) {
+scanbitvector(byte *scanp, BitVector *bv)
+{
+        uint32 *wp;
+        uint32 w;
+        int32 i, remptrs;
+
+        wp = bv->data;
+        for(remptrs = bv->n; remptrs > 0; remptrs -= 32) {
+                w = *wp++;
+                if(remptrs < 32)
+                        i = remptrs;
+                else
+                        i = 32;
+                for(; i > 0; i--) {
+                        if(w & 1)
                                 addroot((Obj){scanp, PtrSize, 0});
-                        }
-                        b <<= 1;
+                        w >>= 1;
                         scanp += PtrSize;
                 }
-                remptrs -= 32;
         }
 }
 
 // Scan a stack frame: local variables and function arguments/results.
 static void
 addframeroots(Stkframe *frame, void*)
 {
         Func *f;
-        byte *scanp;
         BitVector *args, *locals;
-        intptr sz;
+        uintptr size;
 
         f = frame->fn;
 
         // Scan local variables if stack frame has been allocated.
         // Use pointer information if known.
-        sz = frame->varp - (byte*)frame->sp;
-        if(sz > 0) {
-                locals = runtime·funcdata(f, FUNCDATA_GCLOCALS);
+        if(frame->varp > (byte*)frame->sp) {
+                locals = runtime·funcdata(f, FUNCDATA_GCLocals);
                 if(locals == nil) {
-                        addroot((Obj){frame->varp - sz, sz, 0});
-                } else if(locals->n >= 0) {
-                        if(locals->n > 0) {
-                                scanp = frame->varp - locals->n*PtrSize;
-                                scanbitvector(scanp, locals);
-                        }
-                } else {
-                        addroot((Obj){frame->varp - -locals->n, -locals->n, 0});
+                        // No locals information, scan everything.
+                        size = frame->varp - (byte*)frame->sp;
+                        addroot((Obj){frame->varp - size, size, 0});
+                } else if(locals->n < 0) {
+                        // Locals size information, scan just the
+                        // locals.
+                        size = -locals->n;
+                        addroot((Obj){frame->varp - size, size, 0});
+                } else if(locals->n > 0) {
+                        // Locals bitmap information, scan just the
+                        // pointers in locals.
+                        size = locals->n*PtrSize;
+                        scanbitvector(frame->varp - size, locals);
                 }
         }
 
         // Scan arguments.
         // Use pointer information if known.
-        args = runtime·funcdata(f, FUNCDATA_GCARGS);
-        if(f->args > 0 && args != nil && args->n > 0) {
-                scanp = frame->argp;
-                scanbitvector(scanp, args);
-        } else
+        args = runtime·funcdata(f, FUNCDATA_GCArgs);
+        if(args != nil && args->n > 0)
+                scanbitvector(frame->argp, args);
+        else
                 addroot((Obj){frame->argp, frame->arglen, 0});
 }
 
 static void
 addstackroots(G *gp)
 {
         M *mp;
         int32 n;
         Stktop *stk;
         uintptr sp, guard, pc, lr;
         void *base;
         uintptr size;
 
         stk = (Stktop*)gp->stackbase;
         guard = gp->stackguard;
 
         if(gp == g)
                 runtime·throw("can't scan our own stack");
         if((mp = gp->m) != nil && mp->helpgc)
                 runtime·throw("can't scan gchelper stack");
-        if(gp->gcstack != (uintptr)nil) {
+        if(gp->syscallstack != (uintptr)nil) {
                 // Scanning another goroutine that is about to enter or might
                 // have just exited a system call. It may be executing code such
                 // as schedlock and may have needed to start a new stack segment.
                 // Use the stack segment and stack pointer at the time of
                 // the system call instead, since that won't change underfoot.
-                sp = gp->gcsp;
-                pc = gp->gcpc;
+                sp = gp->syscallsp;
+                pc = gp->syscallpc;
                 lr = 0;
-                stk = (Stktop*)gp->gcstack;
-                guard = gp->gcguard;
+                stk = (Stktop*)gp->syscallstack;
+                guard = gp->syscallguard;
         } else {
                 // Scanning another goroutine's stack.
                 // The goroutine is usually asleep (the world is stopped).
                 sp = gp->sched.sp;
                 pc = gp->sched.pc;
                 lr = gp->sched.lr;
 
                 // For function about to start, context argument is a root too.
                 if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil))
                         addroot((Obj){base, size, 0});
@@ skipping 596 matching lines @@
         M *mp;
         uint32 i;
         Eface eface;
 
         t0 = args->start_time;
 
         if(CollectStats)
                 runtime·memclr((byte*)&gcstats, sizeof(gcstats));
 
         for(mp=runtime·allm; mp; mp=mp->alllink)
-                runtime·settype_flush(mp, false);
+                runtime·settype_flush(mp);
 
         heap0 = 0;
         obj0 = 0;
         if(runtime·debug.gctrace) {
                 updatememstats(nil);
                 heap0 = mstats.heap_alloc;
                 obj0 = mstats.nmalloc - mstats.nfree;
         }
 
         m->locks++;     // disable gc during mallocs in parforalloc
@@ skipping 242 matching lines @@
 
         off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;  // word offset
         b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
         shift = off % wordsPerBitmapWord;
 
         for(;;) {
                 obits = *b;
                 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift);
                 if(noptr)
                         bits |= bitNoPointers<<shift;
-                if(runtime·singleproc) {
+                if(runtime·gomaxprocs == 1) {
                         *b = bits;
                         break;
                 } else {
                         // more than one goroutine is potentially running: use atomic op
                         if(runtime·casp((void**)b, (void*)obits, (void*)bits))
                                 break;
                 }
         }
 }
 
 // mark the block at v of size n as freed.
 void
 runtime·markfreed(void *v, uintptr n)
 {
         uintptr *b, obits, bits, off, shift;
 
         if(0)
                 runtime·printf("markallocated %p+%p\n", v, n);
 
         if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
                 runtime·throw("markallocated: bad pointer");
 
         off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;  // word offset
         b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
         shift = off % wordsPerBitmapWord;
 
         for(;;) {
                 obits = *b;
                 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
-                if(runtime·singleproc) {
+                if(runtime·gomaxprocs == 1) {
                         *b = bits;
                         break;
                 } else {
                         // more than one goroutine is potentially running: use atomic op
                         if(runtime·casp((void**)b, (void*)obits, (void*)bits))
                                 break;
                 }
         }
 }
 
@@ skipping 99 matching lines @@
         off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;
         b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
         shift = off % wordsPerBitmapWord;
 
         for(;;) {
                 obits = *b;
                 if(s)
                         bits = obits | (bitSpecial<<shift);
                 else
                         bits = obits & ~(bitSpecial<<shift);
-                if(runtime·singleproc) {
+                if(runtime·gomaxprocs == 1) {
                         *b = bits;
                         break;
                 } else {
                         // more than one goroutine is potentially running: use atomic op
                         if(runtime·casp((void**)b, (void*)obits, (void*)bits))
                                 break;
                 }
         }
 }
 
 void
 runtime·MHeap_MapBits(MHeap *h)
 {
         // Caller has added extra mappings to the arena.
         // Add extra mappings of bitmap words as needed.
         // We allocate extra bitmap pieces in chunks of bitmapChunk.
         enum {
                 bitmapChunk = 8192
         };
         uintptr n;
 
         n = (h->arena_used - h->arena_start) / wordsPerBitmapWord;
         n = ROUND(n, bitmapChunk);
         if(h->bitmap_mapped >= n)
                 return;
 
         runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped);
         h->bitmap_mapped = n;
 }