LEFT | RIGHT |
(no file at all) | |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Page heap. | 5 // Page heap. |
6 // | 6 // |
7 // See malloc.h for overview. | 7 // See malloc.h for overview. |
8 // | 8 // |
9 // When a MSpan is in the heap free list, state == MSpanFree | 9 // When a MSpan is in the heap free list, state == MSpanFree |
10 // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. | 10 // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
96 | 96 |
97 HaveSpan: | 97 HaveSpan: |
98 // Mark span in use. | 98 // Mark span in use. |
99 if(s->state != MSpanFree) | 99 if(s->state != MSpanFree) |
100 runtime·throw("MHeap_AllocLocked - MSpan not free"); | 100 runtime·throw("MHeap_AllocLocked - MSpan not free"); |
101 if(s->npages < npage) | 101 if(s->npages < npage) |
102 runtime·throw("MHeap_AllocLocked - bad npages"); | 102 runtime·throw("MHeap_AllocLocked - bad npages"); |
103 runtime·MSpanList_Remove(s); | 103 runtime·MSpanList_Remove(s); |
104 s->state = MSpanInUse; | 104 s->state = MSpanInUse; |
105 mstats.heap_idle -= s->npages<<PageShift; | 105 mstats.heap_idle -= s->npages<<PageShift; |
| 106 mstats.heap_released -= s->npreleased<<PageShift; |
| 107 s->npreleased = 0; |
106 | 108 |
107 if(s->npages > npage) { | 109 if(s->npages > npage) { |
108 // Trim extra and put it back in the heap. | 110 // Trim extra and put it back in the heap. |
109 t = runtime·FixAlloc_Alloc(&h->spanalloc); | 111 t = runtime·FixAlloc_Alloc(&h->spanalloc); |
110 mstats.mspan_inuse = h->spanalloc.inuse; | 112 mstats.mspan_inuse = h->spanalloc.inuse; |
111 mstats.mspan_sys = h->spanalloc.sys; | 113 mstats.mspan_sys = h->spanalloc.sys; |
112 runtime·MSpan_Init(t, s->start + npage, s->npages - npage); | 114 runtime·MSpan_Init(t, s->start + npage, s->npages - npage); |
113 s->npages = npage; | 115 s->npages = npage; |
114 p = t->start; | 116 p = t->start; |
115 if(sizeof(void*) == 8) | 117 if(sizeof(void*) == 8) |
(...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
273 uintptr *sp, *tp; | 275 uintptr *sp, *tp; |
274 MSpan *t; | 276 MSpan *t; |
275 PageID p; | 277 PageID p; |
276 | 278 |
277 if(s->state != MSpanInUse || s->ref != 0) { | 279 if(s->state != MSpanInUse || s->ref != 0) { |
278 runtime·printf("MHeap_FreeLocked - span %p ptr %p state %d ref %
d\n", s, s->start<<PageShift, s->state, s->ref); | 280 runtime·printf("MHeap_FreeLocked - span %p ptr %p state %d ref %
d\n", s, s->start<<PageShift, s->state, s->ref); |
279 runtime·throw("MHeap_FreeLocked - invalid free"); | 281 runtime·throw("MHeap_FreeLocked - invalid free"); |
280 } | 282 } |
281 mstats.heap_idle += s->npages<<PageShift; | 283 mstats.heap_idle += s->npages<<PageShift; |
282 s->state = MSpanFree; | 284 s->state = MSpanFree; |
| 285 s->unusedsince = 0; |
| 286 s->npreleased = 0; |
283 runtime·MSpanList_Remove(s); | 287 runtime·MSpanList_Remove(s); |
284 sp = (uintptr*)(s->start<<PageShift); | 288 sp = (uintptr*)(s->start<<PageShift); |
285 | 289 |
286 // Coalesce with earlier, later spans. | 290 // Coalesce with earlier, later spans. |
287 p = s->start; | 291 p = s->start; |
288 if(sizeof(void*) == 8) | 292 if(sizeof(void*) == 8) |
289 p -= (uintptr)h->arena_start >> PageShift; | 293 p -= (uintptr)h->arena_start >> PageShift; |
290 if(p > 0 && (t = h->map[p-1]) != nil && t->state != MSpanInUse) { | 294 if(p > 0 && (t = h->map[p-1]) != nil && t->state != MSpanInUse) { |
291 tp = (uintptr*)(t->start<<PageShift); | 295 tp = (uintptr*)(t->start<<PageShift); |
292 *tp |= *sp; // propagate "needs zeroing" mark | 296 *tp |= *sp; // propagate "needs zeroing" mark |
293 s->start = t->start; | 297 s->start = t->start; |
294 s->npages += t->npages; | 298 s->npages += t->npages; |
| 299 s->npreleased = t->npreleased; // absorb released pages |
295 p -= t->npages; | 300 p -= t->npages; |
296 h->map[p] = s; | 301 h->map[p] = s; |
297 runtime·MSpanList_Remove(t); | 302 runtime·MSpanList_Remove(t); |
298 t->state = MSpanDead; | 303 t->state = MSpanDead; |
299 runtime·FixAlloc_Free(&h->spanalloc, t); | 304 runtime·FixAlloc_Free(&h->spanalloc, t); |
300 mstats.mspan_inuse = h->spanalloc.inuse; | 305 mstats.mspan_inuse = h->spanalloc.inuse; |
301 mstats.mspan_sys = h->spanalloc.sys; | 306 mstats.mspan_sys = h->spanalloc.sys; |
302 } | 307 } |
303 if(p+s->npages < nelem(h->map) && (t = h->map[p+s->npages]) != nil && t-
>state != MSpanInUse) { | 308 if(p+s->npages < nelem(h->map) && (t = h->map[p+s->npages]) != nil && t-
>state != MSpanInUse) { |
304 tp = (uintptr*)(t->start<<PageShift); | 309 tp = (uintptr*)(t->start<<PageShift); |
305 *sp |= *tp; // propagate "needs zeroing" mark | 310 *sp |= *tp; // propagate "needs zeroing" mark |
306 s->npages += t->npages; | 311 s->npages += t->npages; |
| 312 s->npreleased += t->npreleased; |
307 h->map[p + s->npages - 1] = s; | 313 h->map[p + s->npages - 1] = s; |
308 runtime·MSpanList_Remove(t); | 314 runtime·MSpanList_Remove(t); |
309 t->state = MSpanDead; | 315 t->state = MSpanDead; |
310 runtime·FixAlloc_Free(&h->spanalloc, t); | 316 runtime·FixAlloc_Free(&h->spanalloc, t); |
311 mstats.mspan_inuse = h->spanalloc.inuse; | 317 mstats.mspan_inuse = h->spanalloc.inuse; |
312 mstats.mspan_sys = h->spanalloc.sys; | 318 mstats.mspan_sys = h->spanalloc.sys; |
313 } | 319 } |
314 | 320 |
315 // Insert s into appropriate list. | 321 // Insert s into appropriate list. |
316 if(s->npages < nelem(h->free)) | 322 if(s->npages < nelem(h->free)) |
317 runtime·MSpanList_Insert(&h->free[s->npages], s); | 323 runtime·MSpanList_Insert(&h->free[s->npages], s); |
318 else | 324 else |
319 runtime·MSpanList_Insert(&h->large, s); | 325 runtime·MSpanList_Insert(&h->large, s); |
320 | 326 } |
321 » // TODO(rsc): IncrementalScavenge() to return memory to OS. | 327 |
| 328 // Release (part of) unused memory to OS. |
| 329 // Goroutine created in runtime·schedinit. |
| 330 // Loop forever. |
| 331 void |
| 332 runtime·MHeap_Scavenger(void) |
| 333 { |
| 334 » MHeap *h; |
| 335 » MSpan *s, *list; |
| 336 » uint64 tick, now, forcegc, limit; |
| 337 » uint32 k, i; |
| 338 » uintptr released, sumreleased; |
| 339 » byte *env; |
| 340 » bool trace; |
| 341 » Note note; |
| 342 |
| 343 » // If we go two minutes without a garbage collection, force one to run. |
| 344 » forcegc = 2*60*1e9; |
| 345 » // If a span goes unused for 5 minutes after a garbage collection, |
| 346 » // we hand it back to the operating system. |
| 347 » limit = 5*60*1e9; |
| 348 » // Make wake-up period small enough for the sampling to be correct. |
| 349 » tick = forcegc < limit ? forcegc/2 : limit/2; |
| 350 |
| 351 » trace = false; |
| 352 » env = runtime·getenv("GOGCTRACE"); |
| 353 » if(env != nil) |
| 354 » » trace = runtime·atoi(env) > 0; |
| 355 |
| 356 » h = &runtime·mheap; |
| 357 » for(k=0;; k++) { |
| 358 » » runtime·noteclear(¬e); |
| 359 » » runtime·entersyscall(); |
| 360 » » runtime·notetsleep(¬e, tick); |
| 361 » » runtime·exitsyscall(); |
| 362 |
| 363 » » runtime·lock(h); |
| 364 » » now = runtime·nanotime(); |
| 365 » » if(now - mstats.last_gc > forcegc) { |
| 366 » » » runtime·unlock(h); |
| 367 » » » runtime·gc(1); |
| 368 » » » runtime·lock(h); |
| 369 » » » now = runtime·nanotime(); |
| 370 » » » if (trace) |
| 371 » » » » runtime·printf("scvg%d: GC forced\n", k); |
| 372 » » } |
| 373 » » sumreleased = 0; |
| 374 » » for(i=0; i < nelem(h->free)+1; i++) { |
| 375 » » » if(i < nelem(h->free)) |
| 376 » » » » list = &h->free[i]; |
| 377 » » » else |
| 378 » » » » list = &h->large; |
| 379 » » » if(runtime·MSpanList_IsEmpty(list)) |
| 380 » » » » continue; |
| 381 » » » for(s=list->next; s != list; s=s->next) { |
| 382 » » » » if(s->unusedsince != 0 && (now - s->unusedsince)
> limit) { |
| 383 » » » » » released = (s->npages - s->npreleased) <
< PageShift; |
| 384 » » » » » mstats.heap_released += released; |
| 385 » » » » » sumreleased += released; |
| 386 » » » » » s->npreleased = s->npages; |
| 387 » » » » » runtime·SysUnused((void*)(s->start << Pa
geShift), s->npages << PageShift); |
| 388 » » » » } |
| 389 » » » } |
| 390 » » } |
| 391 » » runtime·unlock(h); |
| 392 |
| 393 » » if(trace) { |
| 394 » » » if(sumreleased > 0) |
| 395 » » » » runtime·printf("scvg%d: %p MB released\n", k, su
mreleased>>20); |
| 396 » » » runtime·printf("scvg%d: inuse: %D, idle: %D, sys: %D, re
leased: %D, consumed: %D (MB)\n", |
| 397 » » » » k, mstats.heap_inuse>>20, mstats.heap_idle>>20,
mstats.heap_sys>>20, |
| 398 » » » » mstats.heap_released>>20, (mstats.heap_sys - mst
ats.heap_released)>>20); |
| 399 » » } |
| 400 » } |
322 } | 401 } |
323 | 402 |
324 // Initialize a new span with the given start and npages. | 403 // Initialize a new span with the given start and npages. |
325 void | 404 void |
326 runtime·MSpan_Init(MSpan *span, PageID start, uintptr npages) | 405 runtime·MSpan_Init(MSpan *span, PageID start, uintptr npages) |
327 { | 406 { |
328 span->next = nil; | 407 span->next = nil; |
329 span->prev = nil; | 408 span->prev = nil; |
330 span->start = start; | 409 span->start = start; |
331 span->npages = npages; | 410 span->npages = npages; |
332 span->freelist = nil; | 411 span->freelist = nil; |
333 span->ref = 0; | 412 span->ref = 0; |
334 span->sizeclass = 0; | 413 span->sizeclass = 0; |
335 span->state = 0; | 414 span->state = 0; |
| 415 span->unusedsince = 0; |
| 416 span->npreleased = 0; |
336 } | 417 } |
337 | 418 |
338 // Initialize an empty doubly-linked list. | 419 // Initialize an empty doubly-linked list. |
339 void | 420 void |
340 runtime·MSpanList_Init(MSpan *list) | 421 runtime·MSpanList_Init(MSpan *list) |
341 { | 422 { |
342 list->state = MSpanListHead; | 423 list->state = MSpanListHead; |
343 list->next = list; | 424 list->next = list; |
344 list->prev = list; | 425 list->prev = list; |
345 } | 426 } |
(...skipping 22 matching lines...) Expand all Loading... |
368 runtime·printf("failed MSpanList_Insert %p %p %p\n", span, span-
>next, span->prev); | 449 runtime·printf("failed MSpanList_Insert %p %p %p\n", span, span-
>next, span->prev); |
369 runtime·throw("MSpanList_Insert"); | 450 runtime·throw("MSpanList_Insert"); |
370 } | 451 } |
371 span->next = list->next; | 452 span->next = list->next; |
372 span->prev = list; | 453 span->prev = list; |
373 span->next->prev = span; | 454 span->next->prev = span; |
374 span->prev->next = span; | 455 span->prev->next = span; |
375 } | 456 } |
376 | 457 |
377 | 458 |
LEFT | RIGHT |