code review 5279048: runtime: faster and more scalable GC

src/pkg/runtime/mgc0.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Garbage collector.
 
 #include "runtime.h"
-#include "arch.h"
+#include "arch_GOARCH.h"
 #include "malloc.h"
 #include "stack.h"
 
 enum {
         Debug = 0,
         PtrSize = sizeof(void*),
         DebugMark = 0,  // run second pass to check mark
         DataBlock = 8*1024,
 
         // Four bits per word (see #defines below).
@@ skipping 27 matching lines @@
 //      /* then test bits & bitAllocated, bits & bitMarked, etc. */
 //
 #define bitAllocated            ((uintptr)1<<(bitShift*0))
 #define bitNoPointers           ((uintptr)1<<(bitShift*1))      /* when bitAllocated is set */
 #define bitMarked               ((uintptr)1<<(bitShift*2))      /* when bitAllocated is set */
 #define bitSpecial              ((uintptr)1<<(bitShift*3))      /* when bitAllocated is set - has finalizer or being profiled */
 #define bitBlockBoundary        ((uintptr)1<<(bitShift*1))      /* when bitAllocated is NOT set */
 
 #define bitMask (bitBlockBoundary | bitAllocated | bitMarked | bitSpecial)
 
+// Holding worldsema grants an M the right to try to stop the world.
+// The procedure is:
+//
+//      runtime·semacquire(&runtime·worldsema);
+//      m->gcing = 1;
+//      runtime·stoptheworld();
+//
+//      ... do stuff ...
+//
+//      m->gcing = 0;
+//      runtime·semrelease(&runtime·worldsema);
+//      runtime·starttheworld();
+//
+uint32 runtime·worldsema = 1;
 static int32 gctrace;
 
 typedef struct Workbuf Workbuf;
 struct Workbuf
 {
         Workbuf *next;
         uintptr pushcnt;
         uintptr nobj;
         byte *obj[512-3];
 };
@@ skipping 10 matching lines @@
 struct FinBlock
 {
         FinBlock *alllink;
         FinBlock *next;
         int32 cnt;
         int32 cap;
         Finalizer fin[1];
 };
 
 extern byte data[];
-extern byte end[];
+extern byte etext[];
+extern byte ebss[];
 
 static G *fing;
 static FinBlock *finq; // list of finalizers that are to be executed
 static FinBlock *finc; // cache of free blocks
 static FinBlock *allfin; // list of all blocks
 static Lock finlock;
 static int32 fingwait;
 
 static void runfinq(void);
 static Workbuf* getempty(Workbuf*);
@@ skipping 158 matching lines @@
                                 if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
                                         obj = (byte*)obj - (shift-j)*PtrSize;
                                         shift = j;
                                         bits = xbits>>shift;
                                         goto found;
                                 }
                         }
 
                         // Otherwise consult span table to find beginning.
                         // (Manually inlined copy of MHeap_LookupMaybe.)
-                        m->gcstats.naddrlookup++;
                         k = (uintptr)obj>>PageShift;
                         x = k;
                         if(sizeof(void*) == 8)
                                 x -= (uintptr)arena_start>>PageShift;
                         s = runtime·mheap.map[x];
                         if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse)
                                 continue;
                         p =  (byte*)((uintptr)s->start<<PageShift);
                         if(s->sizeclass == 0) {
                                 obj = p;
@@ skipping 70 matching lines @@
                         // Emptied our buffer: refill.
                         wbuf = getfull(wbuf);
                         if(wbuf == nil)
                                 return;
                         nobj = wbuf->nobj;
                         wp = wbuf->obj + wbuf->nobj;
                 }
                 b = *--wp;
                 nobj--;
 
-                // Figure out n = size of b.  Start by loading bits for b.
-                off = (uintptr*)b - (uintptr*)arena_start;
-                bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
-                shift = off % wordsPerBitmapWord;
-                xbits = *bitp;
-                bits = xbits >> shift;
-
-                // Might be small; look for nearby block boundary.
-                // A block boundary is marked by either bitBlockBoundary
-                // or bitAllocated being set (see notes near their definition).
-                enum {
-                        boundary = bitBlockBoundary|bitAllocated
-                };
-                // Look for a block boundary both after and before b
-                // in the same bitmap word.
-                //
-                // A block boundary j words after b is indicated by
-                //      bits>>j & boundary
-                // assuming shift+j < bitShift.  (If shift+j >= bitShift then
-                // we'll be bleeding other bit types like bitMarked into our test.)
-                // Instead of inserting the conditional shift+j < bitShift into the loop,
-                // we can let j range from 1 to bitShift as long as we first
-                // apply a mask to keep only the bits corresponding
-                // to shift+j < bitShift aka j < bitShift-shift.
-                bits &= (boundary<<(bitShift-shift)) - boundary;
-
-                // A block boundary j words before b is indicated by
-                //      xbits>>(shift-j) & boundary
-                // (assuming shift >= j).  There is no cleverness here
-                // avoid the test, because when j gets too large the shift
-                // turns negative, which is undefined in C.
-
-                for(j=1; j<bitShift; j++) {
-                        if(((bits>>j)&boundary) != 0 || shift>=j && ((xbits>>(shift-j))&boundary) != 0) {
-                                n = j*PtrSize;
-                                goto scan;
-                        }
-                }
-
-                // Fall back to asking span about size class.
+                // Ask span about size class.
                 // (Manually inlined copy of MHeap_Lookup.)
-                m->gcstats.nsizelookup++;
                 x = (uintptr)b>>PageShift;
                 if(sizeof(void*) == 8)
                         x -= (uintptr)arena_start>>PageShift;
                 s = runtime·mheap.map[x];
                 if(s->sizeclass == 0)
                         n = s->npages<<PageShift;
                 else
                         n = runtime·class_to_size[s->sizeclass];
-        scan:;
         }
 }
 
 // debug_scanblock is the debug copy of scanblock.
 // it is simpler, slower, single-threaded, recursive,
 // and uses bitSpecial as the mark bit.
 static void
 debug_scanblock(byte *b, int64 n)
 {
         byte *obj, *p;
@@ skipping 179 matching lines @@
                 }
                 work.roots = new;
                 work.rootcap = cap;
         }
         work.roots[work.nroot].p = p;
         work.roots[work.nroot].n = n;
         work.nroot++;
 }
 
 static void
-collectstackroots(G *gp)
+addstackroots(G *gp)
 {
         M *mp;
         int32 n;
         Stktop *stk;
         byte *sp, *guard;
 
         stk = (Stktop*)gp->stackbase;
         guard = gp->stackguard;
 
         if(gp == g) {
                 // Scanning our own stack: start at &gp.
                 sp = (byte*)&gp;
         } else if((mp = gp->m) != nil && mp->helpgc) {
                 // gchelper's stack is in active use and has no interesting pointers.
                 return;
         } else {
                 // Scanning another goroutine's stack.
                 // The goroutine is usually asleep (the world is stopped).
                 sp = gp->sched.sp;
-                
+
                 // The exception is that if the goroutine is about to enter or might
                 // have just exited a system call, it may be executing code such
                 // as schedlock and may have needed to start a new stack segment.
                 // Use the stack segment and stack pointer at the time of
                 // the system call instead, since that won't change underfoot.
                 if(gp->gcstack != nil) {
                         stk = (Stktop*)gp->gcstack;
                         sp = gp->gcsp;
                         guard = gp->gcguard;
                 }
         }
 
         n = 0;
         while(stk) {
                 if(sp < guard-StackGuard || (byte*)stk < sp) {
                         runtime·printf("scanstack inconsistent: g%d#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
                         runtime·throw("scanstack");
                 }
                 addroot(sp, (byte*)stk - sp);
                 sp = stk->gobuf.sp;
                 guard = stk->stackguard;
                 stk = (Stktop*)stk->stackbase;
                 n++;
         }
 }
 
 static void
-collectfinroots(void *v)
+addfinroots(void *v)
 {
         uintptr size;
 
         size = 0;
         if(!runtime·mlookup(v, &v, &size, nil) || !runtime·blockspecial(v))
                 runtime·throw("mark - finalizer inconsistency");
 
         // do not mark the finalizer block itself.  just mark the things it points at.
         addroot(v, size);
 }
 
 static void
-collectroots(void)
+addroots(void)
 {
         G *gp;
         FinBlock *fb;
-        byte *p, *e;
+        byte *p;
 
         work.nroot = 0;
 
-        e = (byte*)&runtime·mheap;
-        for(p=data; p<e; p+=DataBlock)
-                addroot(p, p+DataBlock<e?DataBlock:e-p);
-
-        e = end;
-        for(p=(byte*)(&runtime·mheap+1); p<e; p+=DataBlock)
-                addroot(p, p+DataBlock<e?DataBlock:e-p);
-
-        runtime·walkfintab(collectfinroots);
+        // mark data+bss.
+        for(p=data; p<ebss; p+=DataBlock)
+                addroot(p, p+DataBlock<ebss?DataBlock:ebss-p);
+
+        runtime·walkfintab(addfinroots);
 
         for(fb=allfin; fb; fb=fb->alllink)
                 addroot((byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]));
 
         for(gp=runtime·allg; gp!=nil; gp=gp->alllink) {
                 switch(gp->status){
                         default:
                                 runtime·printf("unexpected G.status %d\n", gp->status);
                                 runtime·throw("mark - bad status");
                         case Gdead:
                                 break;
                         case Grunning:
                                 if(gp != g)
                                         runtime·throw("mark - world not stopped");
-                                collectstackroots(gp);
+                                addstackroots(gp);
                                 break;
                         case Grunnable:
                         case Gsyscall:
                         case Gwaiting:
-                                collectstackroots(gp);
+                                addstackroots(gp);
                                 break;
                 }
         }
 }
 
 static bool
 handlespecial(byte *p, uintptr size)
 {
         void (*fn)(void*);
         int32 nret;
@@ skipping 38 matching lines @@
         uintptr size;
         byte *p;
         MCache *c;
         byte *arena_start;
         MLink *start, *end;
         int32 nfree;
 
         USED(desc);
 
         s = runtime·mheap.allspans[spanidx];
+        // Stamp newly unused spans. The scavenger will use that
+        // info to potentially give back some pages to the OS.
+        if(s->state == MSpanFree && s->unusedsince == 0)
+                s->unusedsince = runtime·nanotime();
         if(s->state != MSpanInUse)
                 return;
 
         arena_start = runtime·mheap.arena_start;
 
         p = (byte*)(s->start << PageShift);
         cl = s->sizeclass;
         if(cl == 0) {
                 size = s->npages<<PageShift;
                 n = 1;
         } else {
                 // Chunk full of small blocks.
                 size = runtime·class_to_size[cl];
                 npages = runtime·class_to_allocnpages[cl];
                 n = (npages << PageShift) / size;
         }
         c = m->mcache;
         nfree = 0;
         start = end = nil;
 
         // Sweep through n objects of given size starting at p.
         // This thread owns the span now, so it can manipulate
         // the block bitmap without atomic operations.
         for(; n > 0; n--, p += size) {
                 uintptr off, *bitp, shift, bits;
 
                 off = (uintptr*)p - (uintptr*)arena_start;
                 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
-PREFETCH((byte*)bitp - size);
+                PREFETCH((byte*)bitp - size);
                 shift = off % wordsPerBitmapWord;
                 bits = *bitp>>shift;
 
                 if((bits & bitAllocated) == 0)
                         continue;
 
                 if((bits & bitMarked) != 0) {
                         if(DebugMark) {
                                 if(!(bits & bitSpecial))
                                         runtime·printf("found spurious mark on %p\n", p);
@@ skipping 16 matching lines @@
 
                 if(cl == 0) {
                         // Free large span.
                         runtime·unmarkspan(p, 1<<PageShift);
                         *(uintptr*)p = 1;       // needs zeroing
                         runtime·MHeap_Free(&runtime·mheap, s, 1);
                         c->local_alloc -= size;
                         c->local_nfree++;
                 } else {
                         // Free small object.
-//                      if(size > sizeof(uintptr))
-//                              ((uintptr*)p)[1] = 1;   // mark as "needs to be zeroed"
                         if(nfree) {
-PREFETCH(p);
+                                PREFETCH(p);
                                 if(size > sizeof(uintptr))
                                         ((uintptr*)end)[1] = 1;   // mark as "needs to be zeroed"
                                 end->next = (MLink*)p;
                                 end = (MLink*)p;
                         } else {
                                 start = (MLink*)p;
                                 end = (MLink*)p;
                         }
                         nfree++;
                 }
@@ skipping 25 matching lines @@
                 while(runtime·atomicload(&work.debugmarkdone) == 0)
                         runtime·usleep(10);
         }
 
         // parallel sweep for over all spans
         parfor(&work.sweepfor);
 
         if(runtime·xadd(&work.ndone, +1) == work.nproc-1)
                 runtime·notewakeup(&work.alldone);
 }
-
-// Semaphore, not Lock, so that the goroutine
-// reschedules when there is contention rather
-// than spinning.
-static uint32 gcsema = 1;
 
 // Initialized from $GOGC.  GOGC=off means no gc.
 //
 // Next gc is after we've allocated an extra amount of
 // memory proportional to the amount already in use.
 // If gcpercent=100 and we're using 4M, we'll gc again
 // when we get to 8M.  This keeps the gc cost in linear
 // proportion to the allocation cost.  Adjusting gcpercent
 // just changes the linear constant (and also the amount of
 // extra memory used).
@@ skipping 74 matching lines @@
                 else
                         gcpercent = runtime·atoi(p);
 
                 p = runtime·getenv("GOGCTRACE");
                 if(p != nil)
                         gctrace = runtime·atoi(p);
         }
         if(gcpercent < 0)
                 return;
 
-        runtime·semacquire(&gcsema);
+        runtime·semacquire(&runtime·worldsema);
         if(!force && mstats.heap_alloc < mstats.next_gc) {
-                runtime·semrelease(&gcsema);
+                runtime·semrelease(&runtime·worldsema);
                 return;
         }
 
         t0 = runtime·nanotime();
 
         m->gcing = 1;
         runtime·stoptheworld();
 
         heap0 = 0;
         obj0 = 0;
         if(gctrace) {
                 cachestats(0);
                 heap0 = mstats.heap_alloc;
                 obj0 = mstats.nmalloc - mstats.nfree;
         }
 
         work.nproc = runtime·gcprocs();
         work.nwait = 0;
         work.ndone = 0;
         work.debugmarkdone = 0;
-        collectroots();
+        addroots();
         parforsetup(&work.markfor, markroot, work.nproc, work.nroot, nil, false);
         parforsetup(&work.sweepfor, sweepspan, work.nproc, runtime·mheap.nspan, nil, true);
         if(work.nproc > 1) {
                 runtime·noteclear(&work.alldone);
-                runtime·helpgc();
+                runtime·helpgc(work.nproc);
         }
 
         parfor(&work.markfor);
         scanblock(nil, 0);
 
         if(DebugMark) {
                 for(i=0; i<work.nroot; i++)
                         debug_scanblock(work.roots[i].p, work.roots[i].n);
                 runtime·xchg(&work.debugmarkdone, 1);
         }
@@ skipping 20 matching lines @@
                 m->locks--;
         }
 ········
         if(work.nproc > 1)
                 runtime·notesleep(&work.alldone);
 
         heap1 = mstats.heap_alloc;
         obj1 = mstats.nmalloc - mstats.nfree;
 
         t3 = runtime·nanotime();
+        mstats.last_gc = t3;
         mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t3 - t0;
         mstats.pause_total_ns += t3 - t0;
         mstats.numgc++;
         if(mstats.debuggc)
                 runtime·printf("pause %D\n", t3-t0);
 
         if(gctrace) {
-                runtime·printf("gc%d(%d): %D+%D+%D ms %D -> %D MB %D -> %D (%D-%D) objects %D pointer lookups (%D size, %D addr), %D(%D) steals, %D(%D) handoffs, %D/%D/%D yields\n",
+                runtime·printf("gc%d(%d): %D+%D+%D ms %D -> %D MB %D -> %D (%D-%D) objects, %D(%D) steals, %D(%D) handoffs, %D/%D/%D yields\n",
                         mstats.numgc, work.nproc, (t1-t0)/1000000, (t2-t1)/1000000, (t3-t2)/1000000,
                         heap0>>20, heap1>>20, obj0, obj1,
                         mstats.nmalloc, mstats.nfree,
-                        stats.nsizelookup+stats.naddrlookup, stats.nsizelookup, stats.naddrlookup,
                         stats.nsteal, stats.nstealcnt,
                         stats.nhandoff, stats.nhandoffcnt,
                         stats.nprocyield, stats.nosyield, stats.nsleep);
         }
-
-        runtime·semrelease(&gcsema);
+ ·······
+        runtime·MProf_GC();
+        runtime·semrelease(&runtime·worldsema);
         runtime·starttheworld();
 
         // give the queued finalizers, if any, a chance to run
         if(finq != nil)
                 runtime·gosched();
 
         if(gctrace > 1 && !force)
                 runtime·gc(1);
 }
 
 void
-runtime·UpdateMemStats(void)
-{
-        // Have to acquire gcsema to stop the world,
+runtime·ReadMemStats(MStats *stats)
+{
+        // Have to acquire worldsema to stop the world,
         // because stoptheworld can only be used by
         // one goroutine at a time, and there might be
         // a pending garbage collection already calling it.
-        runtime·semacquire(&gcsema);
+        runtime·semacquire(&runtime·worldsema);
         m->gcing = 1;
         runtime·stoptheworld();
         cachestats(0);
+        *stats = mstats;
         m->gcing = 0;
-        runtime·semrelease(&gcsema);
+        runtime·semrelease(&runtime·worldsema);
         runtime·starttheworld();
 }
 
 static void
 runfinq(void)
 {
         Finalizer *f;
         FinBlock *fb, *next;
         byte *frame;
         uint32 framesz, framecap, i;
@@ skipping 236 matching lines @@
 
         n = (h->arena_used - h->arena_start) / wordsPerBitmapWord;
         n = (n+bitmapChunk-1) & ~(bitmapChunk-1);
         if(h->bitmap_mapped >= n)
                 return;
 
         runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped);
         h->bitmap_mapped = n;
 }
 
+#ifdef _64BIT
 // Amd64 uses 48-bit virtual addresses, 47-th bit is used as kernel/user flag.
-// So we use 17msb of pointers as ABA counter. 
-#define PTR_BITS 47
+// So we use 17msb of pointers as ABA counter.
+# define PTR_BITS 47
+#else
+# define PTR_BITS 32
+#endif
 #define PTR_MASK ((1ull<<PTR_BITS)-1)
 
 static void
 lifopush(uint64 *a, Workbuf *b)
 {
         uint64 old, new;
-        
+
         if((uint64)b != ((uint64)b&PTR_MASK)) {
                 runtime·printf("p=%p\n", b);
                 runtime·throw("lifopush: invalid pointer");
         }
-        
+
         if(work.nproc == 1) {
                 b->next = (Workbuf*)(*a&PTR_MASK);
                 *a = (uint64)b;
                 return;
         }
-        
+
         b->pushcnt++;
         new = (uint64)b|(((uint64)b->pushcnt)<<PTR_BITS);
         old = runtime·atomicload64(a);
         for(;;) {
                 b->next = (Workbuf*)(old&PTR_MASK);
                 if(runtime·cas64(a, &old, new))
                         break;
         }
 }
 
@@ skipping 19 matching lines @@
                 b2 = runtime·atomicloadp(&b->next);
                 new = 0;
                 if(b2 != nil)
                         new = (uint64)b2|(((uint64)b2->pushcnt)<<PTR_BITS);
                 if(runtime·cas64(a, &old, new))
                         return b;
         }
 }
 
 void
-runtime·CTestLockFreeStack(void)
+runtime·CTestLockFreeStack(bool isShort)
 {
         uint64 stack;
         Workbuf *b;
+
+        USED(isShort);
 
         for(work.nproc=1; work.nproc<=2; work.nproc++) {
                 // check the stack is initially empty
                 stack = 0;
                 if(lifopop(&stack) != nil)
                         runtime·panicstring("stack is not empty");
 
                 // push one element
                 b = (Workbuf*)runtime·mallocgc(sizeof(Workbuf), FlagNoGC, 0, 0);
                 b->nobj = 42;
                 lifopush(&stack, b);
-        
+
                 // push another
                 b = (Workbuf*)runtime·mallocgc(sizeof(Workbuf), FlagNoGC, 0, 0);
                 b->nobj = 43;
                 lifopush(&stack, b);
 
                 // pop one element
                 b = lifopop(&stack);
                 if(b == nil)
                         runtime·panicstring("stack is empty");
                 if(b->nobj != 43)
@@ skipping 12 matching lines @@
                 if(stack != 0)
                         runtime·panicstring("stack is not empty");
                 if(lifopop(&stack) != nil)
                         runtime·panicstring("stack is not empty");
         }
 }
 
 typedef struct StackTestCtx StackTestCtx;
 struct StackTestCtx
 {
+        uint32 niter;
         uint32 waitsema;
         uint64 stack[2];
 };
 
 static void
 stackTestProc(StackTestCtx *ctx)
 {
         int32 i, n;
         Workbuf *b;
 
-        for(i=0; i<100000; i++) {
+        for(i=0; i<ctx->niter; i++) {
                 n = runtime·fastrand1()%2;
                 b = lifopop(&ctx->stack[n]);
                 if(b==nil)
                         continue;
                 n = runtime·fastrand1()%2;
                 lifopush(&ctx->stack[n], b);
         }
         runtime·semrelease(&ctx->waitsema);
 }
 
 void
-runtime·CTestLockFreeStackStress(void)
+runtime·CTestLockFreeStackStress(bool isShort)
 {
         StackTestCtx ctx, *arg;
         Workbuf *b;
         int32 i, sum, sum2, cnt, procs;
         const int32 N = 100;
         const int32 P = 8;
         const int32 G = 16;
 
         procs = runtime·gomaxprocsfunc(P);
         work.nproc = P;
+        ctx.niter = isShort ? 10000 : 100000;
         ctx.waitsema = 0;
         ctx.stack[0] = 0;
         ctx.stack[1] = 0;
         arg = &ctx;
         sum = 0;
         for(i=0; i<N; i++) {
                 b = (Workbuf*)runtime·mallocgc(sizeof(Workbuf), FlagNoGC, 0, 0);
                 b->nobj = i;
                 sum += i;
                 lifopush(&ctx.stack[i%2], b);
@@ skipping 164 matching lines @@
 }
 
 static void
 testParforProc(Parfor *desc)
 {
         parfor(desc);
 }
 
 // Simple serial sanity test for parallelfor.
 void
-runtime·CTestParfor(void)
+runtime·CTestParfor(bool isShort)
 {
         Parfor desc;
         uint64 *data;
         uint32 i, N;
+
+        USED(isShort);
 
         N = 1000;
         data = (uint64*)runtime·mal(N*sizeof(uint64));
         for(i=0; i<N; i++)
                 data[i] = i;
         parforsetup(&desc, testParforBody, 1, N, data, true);
         parfor(&desc);
         for(i=0; i<N; i++) {
                 if(data[i] != i*i)
                         runtime·panicstring("incorrect result");
         }
         runtime·free(data);
 }
 
 // Test that iterations are properly distributed.
 void
-runtime·CTestParforSetup(void)
+runtime·CTestParforSetup(bool isShort)
 {
         Parfor desc;
         uint32 n, t, i, begin, end, size, end0, size0, sum;
+
+        USED(isShort);
 
         for(n=0; n<100; n++) {
                 for(t=1; t<=MaxGcproc; t++) {
                         parforsetup(&desc, testParforBody, t, n, 0, true);
                         sum = 0;
                         size0 = 0;
                         end0 = 0;
                         for(i=0; i<t; i++) {
                                 begin = (uint32)desc.thr[i].pos;
                                 end = (uint32)(desc.thr[i].pos>>32);
@@ skipping 12 matching lines @@
                                 end0 = end;
                         }
                         if(sum != n)
                                 runtime·panicstring("incorrect sum");
                 }
         }
 }
 
 // Test that nonblocking parallelfor does not block.
 void
-runtime·CTestParforNonblock(void)
+runtime·CTestParforNonblock(bool isShort)
 {
         Parfor desc;
         uint64 *data;
         uint32 i, N;
-        
+
+        USED(isShort);
+
         N = 1000;
         data = (uint64*)runtime·mal(N*sizeof(uint64));
         for(i=0; i<N; i++)
                 data[i] = i;
         parforsetup(&desc, testParforBody, MaxGcproc, N, data, false);
         for(i=0; i<MaxGcproc; i++)
                 parfor(&desc);
         for(i=0; i<N; i++) {
                 if(data[i] != i*i)
                         runtime·panicstring("incorrect result");
         }
         runtime·free(data);
 }
 
 // Test parallel parallelfor.
 void
-runtime·CTestParforParallel(void)
+runtime·CTestParforParallel(bool isShort)
 {
         Parfor desc, *arg;
         uint64 *data;
         uint32 i, N;
         int32 procs;
 
         procs = runtime·gomaxprocsfunc(MaxGcproc+1);
         N = 10000;
+        if(isShort)
+                N /= 10;
         data = (uint64*)runtime·mal(N*sizeof(uint64));
         for(i=0; i<N; i++)
                 data[i] = i;
         parforsetup(&desc, testParforBody, MaxGcproc, N, data, true);
         arg = &desc;
         m->locks++; // disable gc during the mallocs in newproc
         for(i=1; i<MaxGcproc; i++)
                 runtime·newproc1((byte*)testParforProc, (byte*)&arg, sizeof(arg), 0, runtime·CTestParforParallel);
         m->locks--;
         parfor(&desc);
         for(i=0; i<N; i++) {
                 if(data[i] != i*i)
                         runtime·panicstring("incorrect result");
         }
         runtime·free(data);
         runtime·gomaxprocsfunc(procs);
 }