LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Page heap. | 5 // Page heap. |
6 // | 6 // |
7 // See malloc.h for overview. | 7 // See malloc.h for overview. |
8 // | 8 // |
9 // When a MSpan is in the heap free list, state == MSpanFree | 9 // When a MSpan is in the heap free list, state == MSpanFree |
10 // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. | 10 // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. |
11 // | 11 // |
12 // When a MSpan is allocated, state == MSpanInUse | 12 // When a MSpan is allocated, state == MSpanInUse |
13 // and heapmap(i) == span for all s->start <= i < s->start+s->npages. | 13 // and heapmap(i) == span for all s->start <= i < s->start+s->npages. |
14 | 14 |
15 #include "runtime.h" | 15 #include "runtime.h" |
16 #include "arch.h" | 16 #include "arch_GOARCH.h" |
17 #include "malloc.h" | 17 #include "malloc.h" |
18 | 18 |
19 static MSpan *MHeap_AllocLocked(MHeap*, uintptr, int32); | 19 static MSpan *MHeap_AllocLocked(MHeap*, uintptr, int32); |
20 static bool MHeap_Grow(MHeap*, uintptr); | 20 static bool MHeap_Grow(MHeap*, uintptr); |
21 static void MHeap_FreeLocked(MHeap*, MSpan*); | 21 static void MHeap_FreeLocked(MHeap*, MSpan*); |
22 static MSpan *MHeap_AllocLarge(MHeap*, uintptr); | 22 static MSpan *MHeap_AllocLarge(MHeap*, uintptr); |
23 static MSpan *BestFit(MSpan*, uintptr, MSpan*); | 23 static MSpan *BestFit(MSpan*, uintptr, MSpan*); |
24 | 24 |
25 static void | 25 static void |
26 RecordSpan(void *vh, byte *p) | 26 RecordSpan(void *vh, byte *p) |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
96 | 96 |
97 HaveSpan: | 97 HaveSpan: |
98 // Mark span in use. | 98 // Mark span in use. |
99 if(s->state != MSpanFree) | 99 if(s->state != MSpanFree) |
100 runtime·throw("MHeap_AllocLocked - MSpan not free"); | 100 runtime·throw("MHeap_AllocLocked - MSpan not free"); |
101 if(s->npages < npage) | 101 if(s->npages < npage) |
102 runtime·throw("MHeap_AllocLocked - bad npages"); | 102 runtime·throw("MHeap_AllocLocked - bad npages"); |
103 runtime·MSpanList_Remove(s); | 103 runtime·MSpanList_Remove(s); |
104 s->state = MSpanInUse; | 104 s->state = MSpanInUse; |
105 mstats.heap_idle -= s->npages<<PageShift; | 105 mstats.heap_idle -= s->npages<<PageShift; |
| 106 mstats.heap_released -= s->npreleased<<PageShift; |
| 107 s->npreleased = 0; |
106 | 108 |
107 if(s->npages > npage) { | 109 if(s->npages > npage) { |
108 // Trim extra and put it back in the heap. | 110 // Trim extra and put it back in the heap. |
109 t = runtime·FixAlloc_Alloc(&h->spanalloc); | 111 t = runtime·FixAlloc_Alloc(&h->spanalloc); |
110 mstats.mspan_inuse = h->spanalloc.inuse; | 112 mstats.mspan_inuse = h->spanalloc.inuse; |
111 mstats.mspan_sys = h->spanalloc.sys; | 113 mstats.mspan_sys = h->spanalloc.sys; |
112 runtime·MSpan_Init(t, s->start + npage, s->npages - npage); | 114 runtime·MSpan_Init(t, s->start + npage, s->npages - npage); |
113 s->npages = npage; | 115 s->npages = npage; |
114 p = t->start; | 116 p = t->start; |
115 if(sizeof(void*) == 8) | 117 if(sizeof(void*) == 8) |
(...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
273 uintptr *sp, *tp; | 275 uintptr *sp, *tp; |
274 MSpan *t; | 276 MSpan *t; |
275 PageID p; | 277 PageID p; |
276 | 278 |
277 if(s->state != MSpanInUse || s->ref != 0) { | 279 if(s->state != MSpanInUse || s->ref != 0) { |
278 runtime·printf("MHeap_FreeLocked - span %p ptr %p state %d ref %
d\n", s, s->start<<PageShift, s->state, s->ref); | 280 runtime·printf("MHeap_FreeLocked - span %p ptr %p state %d ref %
d\n", s, s->start<<PageShift, s->state, s->ref); |
279 runtime·throw("MHeap_FreeLocked - invalid free"); | 281 runtime·throw("MHeap_FreeLocked - invalid free"); |
280 } | 282 } |
281 mstats.heap_idle += s->npages<<PageShift; | 283 mstats.heap_idle += s->npages<<PageShift; |
282 s->state = MSpanFree; | 284 s->state = MSpanFree; |
| 285 s->unusedsince = 0; |
| 286 s->npreleased = 0; |
283 runtime·MSpanList_Remove(s); | 287 runtime·MSpanList_Remove(s); |
284 sp = (uintptr*)(s->start<<PageShift); | 288 sp = (uintptr*)(s->start<<PageShift); |
285 | 289 |
286 // Coalesce with earlier, later spans. | 290 // Coalesce with earlier, later spans. |
287 p = s->start; | 291 p = s->start; |
288 if(sizeof(void*) == 8) | 292 if(sizeof(void*) == 8) |
289 p -= (uintptr)h->arena_start >> PageShift; | 293 p -= (uintptr)h->arena_start >> PageShift; |
290 if(p > 0 && (t = h->map[p-1]) != nil && t->state != MSpanInUse) { | 294 if(p > 0 && (t = h->map[p-1]) != nil && t->state != MSpanInUse) { |
291 tp = (uintptr*)(t->start<<PageShift); | 295 tp = (uintptr*)(t->start<<PageShift); |
292 *tp |= *sp; // propagate "needs zeroing" mark | 296 *tp |= *sp; // propagate "needs zeroing" mark |
293 s->start = t->start; | 297 s->start = t->start; |
294 s->npages += t->npages; | 298 s->npages += t->npages; |
| 299 s->npreleased = t->npreleased; // absorb released pages |
295 p -= t->npages; | 300 p -= t->npages; |
296 h->map[p] = s; | 301 h->map[p] = s; |
297 runtime·MSpanList_Remove(t); | 302 runtime·MSpanList_Remove(t); |
298 t->state = MSpanDead; | 303 t->state = MSpanDead; |
299 runtime·FixAlloc_Free(&h->spanalloc, t); | 304 runtime·FixAlloc_Free(&h->spanalloc, t); |
300 mstats.mspan_inuse = h->spanalloc.inuse; | 305 mstats.mspan_inuse = h->spanalloc.inuse; |
301 mstats.mspan_sys = h->spanalloc.sys; | 306 mstats.mspan_sys = h->spanalloc.sys; |
302 } | 307 } |
303 if(p+s->npages < nelem(h->map) && (t = h->map[p+s->npages]) != nil && t-
>state != MSpanInUse) { | 308 if(p+s->npages < nelem(h->map) && (t = h->map[p+s->npages]) != nil && t-
>state != MSpanInUse) { |
304 tp = (uintptr*)(t->start<<PageShift); | 309 tp = (uintptr*)(t->start<<PageShift); |
305 *sp |= *tp; // propagate "needs zeroing" mark | 310 *sp |= *tp; // propagate "needs zeroing" mark |
306 s->npages += t->npages; | 311 s->npages += t->npages; |
| 312 s->npreleased += t->npreleased; |
307 h->map[p + s->npages - 1] = s; | 313 h->map[p + s->npages - 1] = s; |
308 runtime·MSpanList_Remove(t); | 314 runtime·MSpanList_Remove(t); |
309 t->state = MSpanDead; | 315 t->state = MSpanDead; |
310 runtime·FixAlloc_Free(&h->spanalloc, t); | 316 runtime·FixAlloc_Free(&h->spanalloc, t); |
311 mstats.mspan_inuse = h->spanalloc.inuse; | 317 mstats.mspan_inuse = h->spanalloc.inuse; |
312 mstats.mspan_sys = h->spanalloc.sys; | 318 mstats.mspan_sys = h->spanalloc.sys; |
313 } | 319 } |
314 | 320 |
315 // Insert s into appropriate list. | 321 // Insert s into appropriate list. |
316 if(s->npages < nelem(h->free)) | 322 if(s->npages < nelem(h->free)) |
317 runtime·MSpanList_Insert(&h->free[s->npages], s); | 323 runtime·MSpanList_Insert(&h->free[s->npages], s); |
318 else | 324 else |
319 runtime·MSpanList_Insert(&h->large, s); | 325 runtime·MSpanList_Insert(&h->large, s); |
320 | |
321 // TODO(rsc): IncrementalScavenge() to return memory to OS. | |
322 } | 326 } |
323 | 327 |
324 // Release (part of) unused memory to OS. | 328 // Release (part of) unused memory to OS. |
325 // Goroutine created in runtime·schedinit. | 329 // Goroutine created in runtime·schedinit. |
326 // Loop forever. | 330 // Loop forever. |
327 void | 331 void |
328 runtime·MHeap_Scavenger() | 332 runtime·MHeap_Scavenger(void) |
329 { | 333 { |
330 » int64 tickPeriod = 10e9; | |
331 » float64 cutRatio = 0.1; | |
332 | |
333 MHeap *h; | 334 MHeap *h; |
334 » MSpan *s; | 335 » MSpan *s, *list; |
335 » PageID p; | 336 » uint64 tick, now, forcegc, limit; |
336 » uintptr target, cut, n; | 337 » uint32 k, i; |
337 » uint64 prev_idle, prev_sys, sz; | 338 » uintptr released, sumreleased; |
338 byte *env; | 339 byte *env; |
339 bool trace; | 340 bool trace; |
340 » uint32 k; | 341 » Note note; |
341 | 342 |
342 » h = &runtime·mheap; | 343 » // If we go two minutes without a garbage collection, force one to run. |
343 » prev_sys = prev_idle = 0; | 344 » forcegc = 2*60*1e9; |
344 | 345 » // If a span goes unused for 5 minutes after a garbage collection, |
345 » k = 0; | 346 » // we hand it back to the operating system. |
| 347 » limit = 5*60*1e9; |
| 348 » // Make wake-up period small enough for the sampling to be correct. |
| 349 » tick = forcegc < limit ? forcegc/2 : limit/2; |
| 350 |
346 trace = false; | 351 trace = false; |
347 env = runtime·getenv("GOGCTRACE"); | 352 env = runtime·getenv("GOGCTRACE"); |
348 if(env != nil) | 353 if(env != nil) |
349 trace = runtime·atoi(env) > 0; | 354 trace = runtime·atoi(env) > 0; |
350 | 355 |
351 » for (;;) { | 356 » h = &runtime·mheap; |
352 » » g->status = Gwaiting; | 357 » for(k=0;; k++) { |
353 » » g->waitreason = "scavenger asleep"; | 358 » » runtime·noteclear(¬e); |
354 » » runtime·tsleep(tickPeriod); | 359 » » runtime·entersyscall(); |
| 360 » » runtime·notetsleep(¬e, tick); |
| 361 » » runtime·exitsyscall(); |
355 | 362 |
356 runtime·lock(h); | 363 runtime·lock(h); |
357 | 364 » » now = runtime·nanotime(); |
358 » » if (trace) | 365 » » if(now - mstats.last_gc > forcegc) { |
359 » » » runtime·printf("scvg%d: unused: %D of %D MB\n", | 366 » » » runtime·unlock(h); |
360 » » » » k, mstats.heap_idle>>20, mstats.heap_sys>>20); | 367 » » » runtime·gc(1); |
361 | 368 » » » runtime·lock(h); |
362 » » // Try to not hurt transient allocations. | 369 » » » now = runtime·nanotime(); |
363 » » if (mstats.heap_sys > prev_sys) { | |
364 » » » // Some more hysteresis control is needed. | |
365 if (trace) | 370 if (trace) |
366 » » » » runtime·printf("scvg%d: heap has grown, not goin
g against\n", k); | 371 » » » » runtime·printf("scvg%d: GC forced\n", k); |
367 » » » goto NextRound; | 372 » » } |
368 » » } | 373 » » sumreleased = 0; |
369 » » target = cutRatio * (mstats.heap_idle >> PageShift); | 374 » » for(i=0; i < nelem(h->free)+1; i++) { |
370 » » cut = 0; | 375 » » » if(i < nelem(h->free)) |
371 | 376 » » » » list = &h->free[i]; |
372 » » // Small boost toward heap_idle trend. | 377 » » » else |
373 » » if (mstats.heap_idle > prev_idle) | 378 » » » » list = &h->large; |
374 » » » target *= 2; | 379 » » » if(runtime·MSpanList_IsEmpty(list)) |
375 » » else if (mstats.heap_idle < prev_idle) | |
376 » » » target /= 2; | |
377 | |
378 » » if (trace) | |
379 » » » runtime·printf("scvg%d: target: %D MB+\n", k, (target<<P
ageShift)>>20); | |
380 | |
381 » » // Cut until target is overtaken, oldest spans first. | |
382 » » for (s = h->large.prev; s != &h->large && cut < target; s = h->l
arge.prev) { | |
383 | |
384 » » » // Prevent fragmentation. | |
385 » » » if (s->npages < (HeapAllocChunk >> PageShift)) | |
386 » » » » // Won't happen as long as (HeapAllocChunk >> Pa
geShift) == MaxMHeapList | |
387 continue; | 380 continue; |
388 » » » cut += s->npages; | 381 » » » for(s=list->next; s != list; s=s->next) { |
389 | 382 » » » » if(s->unusedsince != 0 && (now - s->unusedsince)
> limit) { |
390 » » » sz = s->npages << PageShift; | 383 » » » » » released = (s->npages - s->npreleased) <
< PageShift; |
391 » » » runtime·SysFree((void*)(s->start << PageShift), sz); | 384 » » » » » mstats.heap_released += released; |
392 » » » mstats.heap_sys -= sz; | 385 » » » » » sumreleased += released; |
393 » » » mstats.heap_idle -= sz; | 386 » » » » » s->npreleased = s->npages; |
394 » » » if (trace) | 387 » » » » » runtime·SysUnused((void*)(s->start << Pa
geShift), s->npages << PageShift); |
395 » » » » runtime·printf("scvg%d: releasing: %D MB\n", k,
sz>>20); | 388 » » » » } |
396 | 389 » » » } |
397 » » » // Make sure MHeap_FreeLocked won't coalesce with unmape
d chunks. | 390 » » } |
398 » » » p = s->start; | |
399 » » » if(sizeof(void*) == 8) | |
400 » » » » p -= ((uintptr)h->arena_start >> PageShift); | |
401 » » » for(n = 0; n < s->npages; n++) | |
402 » » » » h->map[p+n] = nil; | |
403 | |
404 » » » runtime·MSpanList_Remove(s); | |
405 » » » runtime·FixAlloc_Free(&h->spanalloc, s); | |
406 » » » mstats.mspan_inuse = h->spanalloc.inuse; | |
407 » » » mstats.mspan_sys = h->spanalloc.sys; | |
408 » » } | |
409 » » if (trace) { | |
410 » » » if (cut == 0) | |
411 » » » » runtime·printf("scvg%d: no candidate span\n", k)
; | |
412 » » » else | |
413 » » » » runtime·printf("scvg%d: now unused: %D of %D MB\
n", | |
414 » » » » » k, mstats.heap_idle>>20, mstats.heap_sys
>>20); | |
415 » » } | |
416 | |
417 NextRound: | |
418 » » prev_idle = mstats.heap_idle; | |
419 » » prev_sys = mstats.heap_sys; | |
420 » » k += 1; | |
421 runtime·unlock(h); | 391 runtime·unlock(h); |
| 392 |
| 393 if(trace) { |
| 394 if(sumreleased > 0) |
| 395 runtime·printf("scvg%d: %p MB released\n", k, su
mreleased>>20); |
| 396 runtime·printf("scvg%d: inuse: %D, idle: %D, sys: %D, re
leased: %D, consumed: %D (MB)\n", |
| 397 k, mstats.heap_inuse>>20, mstats.heap_idle>>20,
mstats.heap_sys>>20, |
| 398 mstats.heap_released>>20, (mstats.heap_sys - mst
ats.heap_released)>>20); |
| 399 } |
422 } | 400 } |
423 } | 401 } |
424 | 402 |
425 // Initialize a new span with the given start and npages. | 403 // Initialize a new span with the given start and npages. |
426 void | 404 void |
427 runtime·MSpan_Init(MSpan *span, PageID start, uintptr npages) | 405 runtime·MSpan_Init(MSpan *span, PageID start, uintptr npages) |
428 { | 406 { |
429 span->next = nil; | 407 span->next = nil; |
430 span->prev = nil; | 408 span->prev = nil; |
431 span->start = start; | 409 span->start = start; |
432 span->npages = npages; | 410 span->npages = npages; |
433 span->freelist = nil; | 411 span->freelist = nil; |
434 span->ref = 0; | 412 span->ref = 0; |
435 span->sizeclass = 0; | 413 span->sizeclass = 0; |
436 span->state = 0; | 414 span->state = 0; |
| 415 span->unusedsince = 0; |
| 416 span->npreleased = 0; |
437 } | 417 } |
438 | 418 |
439 // Initialize an empty doubly-linked list. | 419 // Initialize an empty doubly-linked list. |
440 void | 420 void |
441 runtime·MSpanList_Init(MSpan *list) | 421 runtime·MSpanList_Init(MSpan *list) |
442 { | 422 { |
443 list->state = MSpanListHead; | 423 list->state = MSpanListHead; |
444 list->next = list; | 424 list->next = list; |
445 list->prev = list; | 425 list->prev = list; |
446 } | 426 } |
(...skipping 22 matching lines...) Expand all Loading... |
469 runtime·printf("failed MSpanList_Insert %p %p %p\n", span, span-
>next, span->prev); | 449 runtime·printf("failed MSpanList_Insert %p %p %p\n", span, span-
>next, span->prev); |
470 runtime·throw("MSpanList_Insert"); | 450 runtime·throw("MSpanList_Insert"); |
471 } | 451 } |
472 span->next = list->next; | 452 span->next = list->next; |
473 span->prev = list; | 453 span->prev = list; |
474 span->next->prev = span; | 454 span->next->prev = span; |
475 span->prev->next = span; | 455 span->prev->next = span; |
476 } | 456 } |
477 | 457 |
478 | 458 |
LEFT | RIGHT |