code review 5279048: runtime: faster and more scalable GC

src/pkg/runtime/mcentral.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Central free lists.
 //
 // See malloc.h for an overview.
 //
 // The MCentral doesn't actually contain the list of free objects; the MSpan does.
 // Each MCentral is two lists of MSpans: those with free objects (c->nonempty)
@@ skipping 70 matching lines @@
         v = s->freelist;
         s->freelist = v->next;
         if(s->freelist == nil) {
                 runtime·MSpanList_Remove(s);
                 runtime·MSpanList_Insert(&c->empty, s);
         }
         return v;
 }
 
 // Free n objects back into the central free list.
-// Return the number of objects allocated.
-// The objects are linked together by their first words.
-// On return, *pstart points at the first object and *pend at the last.
 void
 runtime·MCentral_FreeList(MCentral *c, int32 n, MLink *start)
 {
         MLink *v, *next;
 
         // Assume next == nil marks end of list.
         // n and end would be useful if we implemented
         // the transfer cache optimization in the TODO above.
         USED(n);
 
@@ skipping 37 matching lines @@
                 runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
                 *(uintptr*)(s->start<<PageShift) = 1;  // needs zeroing
                 s->freelist = nil;
                 c->nfree -= (s->npages << PageShift) / size;
                 runtime·unlock(c);
                 runtime·MHeap_Free(&runtime·mheap, s, 0);
                 runtime·lock(c);
         }
 }
 
+// Free n objects from a span s back into the central free list c.
+// Called from GC.
+void
+runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end)
+{
+        int32 size;
+
+        runtime·lock(c);
+
+        // Move to nonempty if necessary.
+        if(s->freelist == nil) {
+                runtime·MSpanList_Remove(s);
+                runtime·MSpanList_Insert(&c->nonempty, s);
+        }
+
+        // Add the objects back to s's free list.
+        end->next = s->freelist;
+        s->freelist = start;
+        s->ref -= n;
+        c->nfree += n;
+
+        // If s is completely freed, return it to the heap.
+        if(s->ref == 0) {
+                size = runtime·class_to_size[c->sizeclass];
+                runtime·MSpanList_Remove(s);
+                *(uintptr*)(s->start<<PageShift) = 1;  // needs zeroing
+                s->freelist = nil;
+                c->nfree -= (s->npages << PageShift) / size;
+                runtime·unlock(c);
+                runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
+                runtime·MHeap_Free(&runtime·mheap, s, 0);
+        } else {
+                runtime·unlock(c);
+        }
+}
+
 void
 runtime·MGetSizeClassInfo(int32 sizeclass, uintptr *sizep, int32 *npagesp, int32 *nobj)
 {
         int32 size;
         int32 npages;
 
         npages = runtime·class_to_allocnpages[sizeclass];
         size = runtime·class_to_size[sizeclass];
         *npagesp = npages;
         *sizep = size;
@@ skipping 31 matching lines @@
                 p += size;
         }
         *tailp = nil;
         runtime·markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift));
 
         runtime·lock(c);
         c->nfree += n;
         runtime·MSpanList_Insert(&c->nonempty, s);
         return true;
 }