code review 5279048: runtime: faster and more scalable GC

src/pkg/runtime/malloc.h

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Memory allocator, based on tcmalloc.
 // http://goog-perftools.sourceforge.net/doc/tcmalloc.html
 
 // The main allocator works in runs of pages.
 // Small allocation sizes (up to and including 32 kB) are
 // rounded to one of about 100 size classes, each of which
@@ skipping 105 matching lines @@
         // On 64-bit, we limit the arena to 16G, so 22 bits suffices.
         // On 32-bit, we don't bother limiting anything: 20 bits for 4G.
 #ifdef _64BIT
         MHeapMap_Bits = 22,
 #else
         MHeapMap_Bits = 20,
 #endif
 
         // Max number of threads to run garbage collection.
         // 2, 3, and 4 are all plausible maximums depending
-        // on the hardware details of the machine.  The second
-        // proc is the one that helps the most (after the first),
-        // so start with just 2 for now.
+        // on the hardware details of the machine.  The garbage
+        // collector scales well to 4 cpus.
         MaxGcproc = 4,
 };
 
 // A generic linked list of blocks.  (Typically the block is bigger than sizeof(MLink).)
 struct MLink
 {
         MLink *next;
 };
 
 // SysAlloc obtains a large chunk of zeroed memory from the
@@ skipping 60 matching lines @@
         uint64  nlookup;        // number of pointer lookups
         uint64  nmalloc;        // number of mallocs
         uint64  nfree;  // number of frees
 
         // Statistics about malloc heap.
         // protected by mheap.Lock
         uint64  heap_alloc;     // bytes allocated and still in use
         uint64  heap_sys;       // bytes obtained from system
         uint64  heap_idle;      // bytes in idle spans
         uint64  heap_inuse;     // bytes in non-idle spans
+        uint64  heap_released;  // bytes released to the OS
         uint64  heap_objects;   // total number of allocated objects
 
         // Statistics about allocation of low-level fixed-size structures.
         // Protected by FixAlloc locks.
         uint64  stacks_inuse;   // bootstrap stacks
         uint64  stacks_sys;
         uint64  mspan_inuse;    // MSpan structures
         uint64  mspan_sys;
         uint64  mcache_inuse;   // MCache structures
         uint64  mcache_sys;
         uint64  buckhash_sys;   // profiling bucket hash table
 
         // Statistics about garbage collector.
         // Protected by stopping the world during GC.
         uint64  next_gc;        // next GC (in heap_alloc time)
+        uint64  last_gc;        // last GC (in absolute time)
         uint64  pause_total_ns;
         uint64  pause_ns[256];
         uint32  numgc;
         bool    enablegc;
         bool    debuggc;
 
         // Statistics about allocation size classes.
         struct {
                 uint32 size;
                 uint64 nmalloc;
                 uint64 nfree;
         } by_size[NumSizeClasses];
 };
 
-#define mstats runtime·MemStats /* name shared with Go */
+#define mstats runtime·memStats /* name shared with Go */
 extern MStats mstats;
 
 
 // Size classes.  Computed and initialized by InitSizes.
 //
 // SizeToClass(0 <= n <= MaxSmallSize) returns the size class,
 //      1 <= sizeclass < NumSizeClasses, for n.
 //      Size class 0 is reserved to mean "not small".
 //
 // class_to_size[i] = largest size in class i
@@ skipping 55 matching lines @@
 struct MSpan
 {
         MSpan   *next;          // in a span linked list
         MSpan   *prev;          // in a span linked list
         PageID  start;          // starting page number
         uintptr npages;         // number of pages in span
         MLink   *freelist;      // list of free objects
         uint32  ref;            // number of allocated objects in this span
         uint32  sizeclass;      // size class
         uint32  state;          // MSpanInUse etc
-        byte    *limit; // end of data in span
+        int64   unusedsince;    // First time spotted by GC in MSpanFree state
+        uintptr npreleased;     // number of pages released to the OS
+        byte    *limit;         // end of data in span
 };
 
 void    runtime·MSpan_Init(MSpan *span, PageID start, uintptr npages);
 
 // Every MSpan is in one doubly-linked list,
 // either one of the MHeap's free lists or one of the
 // MCentral's span lists.  We use empty MSpan structures as list heads.
 void    runtime·MSpanList_Init(MSpan *list);
 bool    runtime·MSpanList_IsEmpty(MSpan *list);
 void    runtime·MSpanList_Insert(MSpan *list, MSpan *span);
@@ skipping 52 matching lines @@
 extern MHeap runtime·mheap;
 
 void    runtime·MHeap_Init(MHeap *h, void *(*allocator)(uintptr));
 MSpan*  runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, int32 acct);
 void    runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct);
 MSpan*  runtime·MHeap_Lookup(MHeap *h, void *v);
 MSpan*  runtime·MHeap_LookupMaybe(MHeap *h, void *v);
 void    runtime·MGetSizeClassInfo(int32 sizeclass, uintptr *size, int32 *npages, int32 *nobj);
 void*   runtime·MHeap_SysAlloc(MHeap *h, uintptr n);
 void    runtime·MHeap_MapBits(MHeap *h);
+void    runtime·MHeap_Scavenger(void);
 
 void*   runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed);
 int32   runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **s);
 void    runtime·gc(int32 force);
 void    runtime·markallocated(void *v, uintptr n, bool noptr);
 void    runtime·checkallocated(void *v, uintptr n);
 void    runtime·markfreed(void *v, uintptr n);
 void    runtime·checkfreed(void *v, uintptr n);
 int32   runtime·checking;
 void    runtime·markspan(void *v, uintptr size, uintptr n, bool leftover);
 void    runtime·unmarkspan(void *v, uintptr size);
 bool    runtime·blockspecial(void*);
 void    runtime·setblockspecial(void*, bool);
 void    runtime·purgecachedstats(M*);
 
 enum
 {
         // flags to malloc
         FlagNoPointers = 1<<0,  // no pointers here
         FlagNoProfiling = 1<<1, // must not profile
         FlagNoGC = 1<<2,        // must not free or scan for pointers
 };
 
 void    runtime·MProf_Malloc(void*, uintptr);
 void    runtime·MProf_Free(void*, uintptr);
+void    runtime·MProf_GC(void);
 
 void    runtime·stoptheworld(void);
 void    runtime·starttheworld(void);
 int32   runtime·gcprocs(void);
-void    runtime·helpgc(void);
+void    runtime·helpgc(int32 nproc);
 void    runtime·gchelper(void);
-
-// Malloc profiling settings.
-// Must match definition in extern.go.
-enum {
-        MProf_None = 0,
-        MProf_Sample = 1,
-        MProf_All = 2,
-};
-extern int32 runtime·malloc_profile;
 
 bool    runtime·getfinalizer(void *p, bool del, void (**fn)(void*), int32 *nret);
 void    runtime·walkfintab(void (*fn)(void*));