runtime: concurrent GC sweep

src/pkg/runtime/malloc.h

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Memory allocator, based on tcmalloc.
 // http://goog-perftools.sourceforge.net/doc/tcmalloc.html
 
 // The main allocator works in runs of pages.
 // Small allocation sizes (up to and including 32 kB) are
 // rounded to one of about 100 size classes, each of which
@@ skipping 105 matching lines @@
         // On 64-bit, we limit the arena to 16G, so 22 bits suffices.
         // On 32-bit, we don't bother limiting anything: 20 bits for 4G.
 #ifdef _64BIT
         MHeapMap_Bits = 22,
 #else
         MHeapMap_Bits = 20,
 #endif
 
         // Max number of threads to run garbage collection.
         // 2, 3, and 4 are all plausible maximums depending
-        // on the hardware details of the machine.  The second
-        // proc is the one that helps the most (after the first),
-        // so start with just 2 for now.
-        MaxGcproc = 2,
+        // on the hardware details of the machine.  The garbage
+        // collector scales well to 4 cpus.
+        MaxGcproc = 4,
 };
 
 // A generic linked list of blocks.  (Typically the block is bigger than sizeof(MLink).)
 struct MLink
 {
         MLink *next;
 };
 
 // SysAlloc obtains a large chunk of zeroed memory from the
 // operating system, typically on the order of a hundred kilobytes
@@ skipping 59 matching lines @@
         uint64  nlookup;        // number of pointer lookups
         uint64  nmalloc;        // number of mallocs
         uint64  nfree;  // number of frees
 
         // Statistics about malloc heap.
         // protected by mheap.Lock
         uint64  heap_alloc;     // bytes allocated and still in use
         uint64  heap_sys;       // bytes obtained from system
         uint64  heap_idle;      // bytes in idle spans
         uint64  heap_inuse;     // bytes in non-idle spans
+        uint64  heap_released;  // bytes released to the OS
         uint64  heap_objects;   // total number of allocated objects
 
         // Statistics about allocation of low-level fixed-size structures.
         // Protected by FixAlloc locks.
         uint64  stacks_inuse;   // bootstrap stacks
         uint64  stacks_sys;
         uint64  mspan_inuse;    // MSpan structures
         uint64  mspan_sys;
         uint64  mcache_inuse;   // MCache structures
         uint64  mcache_sys;
         uint64  buckhash_sys;   // profiling bucket hash table
 
         // Statistics about garbage collector.
         // Protected by stopping the world during GC.
         uint64  next_gc;        // next GC (in heap_alloc time)
+        uint64  last_gc;        // last GC (in absolute time)
         uint64  pause_total_ns;
         uint64  pause_ns[256];
         uint32  numgc;
         bool    enablegc;
         bool    debuggc;
 
         // Statistics about allocation size classes.
         struct {
                 uint32 size;
                 uint64 nmalloc;
                 uint64 nfree;
         } by_size[NumSizeClasses];
 };
 
-#define mstats runtime·MemStats /* name shared with Go */
+#define mstats runtime·memStats /* name shared with Go */
 extern MStats mstats;
 
 
 // Size classes.  Computed and initialized by InitSizes.
 //
 // SizeToClass(0 <= n <= MaxSmallSize) returns the size class,
 //      1 <= sizeclass < NumSizeClasses, for n.
 //      Size class 0 is reserved to mean "not small".
 //
 // class_to_size[i] = largest size in class i
@@ skipping 24 matching lines @@
 {
         MCacheList list[NumSizeClasses];
         uint64 size;
         int64 local_cachealloc; // bytes allocated (or freed) from cache since last lock of heap
         int64 local_objects;    // objects allocated (or freed) from cache since last lock of heap
         int64 local_alloc;      // bytes allocated (or freed) since last lock of heap
         int64 local_total_alloc;        // bytes allocated (even if freed) since last lock of heap
         int64 local_nmalloc;    // number of mallocs since last lock of heap
         int64 local_nfree;      // number of frees since last lock of heap
         int64 local_nlookup;    // number of pointer lookups since last lock of heap
+        int64 local_marked;     // number of bytes marked by GC
+        int64 local_nmarked;    // number of objects marked by GC
         int32 next_sample;      // trigger heap sample after allocating this many bytes
         // Statistics about allocation size classes since last lock of heap
         struct {
                 int64 nmalloc;
                 int64 nfree;
         } local_by_size[NumSizeClasses];
-
 };
 
 void*   runtime·MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed);
 void    runtime·MCache_Free(MCache *c, void *p, int32 sizeclass, uintptr size);
 void    runtime·MCache_ReleaseAll(MCache *c);
 // An MSpan is a run of pages.
 enum
 {
         MSpanInUse = 0,
         MSpanFree,
         MSpanListHead,
         MSpanDead,
 };
 struct MSpan
 {
         MSpan   *next;          // in a span linked list
         MSpan   *prev;          // in a span linked list
-        MSpan   *allnext;               // in the list of all spans
+        MSpan   *allnext;       // in the list of all spans
         PageID  start;          // starting page number
         uintptr npages;         // number of pages in span
         MLink   *freelist;      // list of free objects
         uint32  ref;            // number of allocated objects in this span
         uint32  needssweep;     // the span is marked by GC but not yet swept
         uint32  sizeclass;      // size class
         uint32  state;          // MSpanInUse etc
-        byte    *limit; // end of data in span
+        uintptr size;           // size in bytes (actual size for large objects)
+        int64   unusedsince;    // First time spotted by GC in MSpanFree state
+        uintptr npreleased;     // number of pages released to the OS
+        byte    *limit;         // end of data in span
 };
 
 void    runtime·MSpan_Init(MSpan *span, PageID start, uintptr npages);
 void    runtime·MSpan_Sweep(MSpan *span, bool dolock);
 
 // Every MSpan is in one doubly-linked list,
 // either one of the MHeap's free lists or one of the
 // MCentral's span lists.  We use empty MSpan structures as list heads.
 void    runtime·MSpanList_Init(MSpan *list);
 bool    runtime·MSpanList_IsEmpty(MSpan *list);
 void    runtime·MSpanList_Insert(MSpan *list, MSpan *span);
+void    runtime·MSpanList_InsertBack(MSpan *list, MSpan *span);
 void    runtime·MSpanList_Remove(MSpan *span);  // from whatever list it is in
 
 
 // Central list of free objects of a given size.
 struct MCentral
 {
         Lock;
         int32 sizeclass;
         MSpan nonempty;
         MSpan empty;
@@ skipping 40 matching lines @@
 extern MHeap runtime·mheap;
 
 void    runtime·MHeap_Init(MHeap *h, void *(*allocator)(uintptr));
 MSpan*  runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, int32 acct);
 void    runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct);
 MSpan*  runtime·MHeap_Lookup(MHeap *h, void *v);
 MSpan*  runtime·MHeap_LookupMaybe(MHeap *h, void *v);
 void    runtime·MGetSizeClassInfo(int32 sizeclass, uintptr *size, int32 *npages, int32 *nobj);
 void*   runtime·MHeap_SysAlloc(MHeap *h, uintptr n);
 void    runtime·MHeap_MapBits(MHeap *h);
+void    runtime·MHeap_Scavenger(void);
 
 void*   runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed);
 int32   runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **s);
 void    runtime·gc(int32 force);
 void    runtime·markallocated(void *v, uintptr n, bool noptr);
 void    runtime·checkallocated(void *v, uintptr n);
 void    runtime·markfreed(void *v, uintptr n);
 void    runtime·checkfreed(void *v, uintptr n);
 int32   runtime·checking;
 void    runtime·markspan(void *v, uintptr size, uintptr n, bool leftover);
 void    runtime·unmarkspan(void *v, uintptr size);
 bool    runtime·blockspecial(void*);
 void    runtime·setblockspecial(void*, bool);
 void    runtime·purgecachedstats(M*);
 
 enum
 {
         // flags to malloc
         FlagNoPointers = 1<<0,  // no pointers here
         FlagNoProfiling = 1<<1, // must not profile
         FlagNoGC = 1<<2,        // must not free or scan for pointers
 };
 
 void    runtime·MProf_Malloc(void*, uintptr);
 void    runtime·MProf_Free(void*, uintptr);
+void    runtime·MProf_GC(void);
 int32   runtime·helpgc(bool*);
 void    runtime·gchelper(void);
 
-// Malloc profiling settings.
-// Must match definition in extern.go.
-enum {
-        MProf_None = 0,
-        MProf_Sample = 1,
-        MProf_All = 2,
-};
-extern int32 runtime·malloc_profile;
-
 bool    runtime·getfinalizer(void *p, bool del, void (**fn)(void*), int32 *nret);
 void    runtime·walkfintab(void (*fn)(void*));