LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Malloc profiling. | 5 // Malloc profiling. |
6 // Patterned after tcmalloc's algorithms; shorter code. | 6 // Patterned after tcmalloc's algorithms; shorter code. |
7 | 7 |
8 package runtime | 8 package runtime |
9 #include "runtime.h" | 9 #include "runtime.h" |
10 #include "arch_GOARCH.h" | 10 #include "arch_GOARCH.h" |
11 #include "malloc.h" | 11 #include "malloc.h" |
12 #include "defs_GOOS_GOARCH.h" | 12 #include "defs_GOOS_GOARCH.h" |
13 #include "type.h" | 13 #include "type.h" |
14 | 14 |
15 // NOTE(rsc): Everything here could use cas if contention became an issue. | 15 // NOTE(rsc): Everything here could use cas if contention became an issue. |
16 static Lock proflock; | 16 static Lock proflock; |
17 | 17 |
18 // Per-call-stack allocation information. | 18 enum { MProf, BProf }; // profile types |
| 19 |
| 20 // Per-call-stack profiling information. |
19 // Lookup by hashing call stack into a linked-list hash table. | 21 // Lookup by hashing call stack into a linked-list hash table. |
20 typedef struct Bucket Bucket; | 22 typedef struct Bucket Bucket; |
21 struct Bucket | 23 struct Bucket |
22 { | 24 { |
23 Bucket *next; // next in hash list | 25 Bucket *next; // next in hash list |
24 » Bucket» *allnext;» // next in list of all buckets | 26 » Bucket» *allnext;» // next in list of all mbuckets/bbuckets |
25 » uintptr»allocs; | 27 » int32» typ; |
26 » uintptr»frees; | 28 » union |
27 » uintptr»alloc_bytes; | 29 » { |
28 » uintptr»free_bytes; | 30 » » struct // typ == MProf |
29 » uintptr»recent_allocs; // since last gc | 31 » » { |
30 » uintptr»recent_frees; | 32 » » » uintptr»allocs; |
31 » uintptr»recent_alloc_bytes; | 33 » » » uintptr»frees; |
32 » uintptr»recent_free_bytes; | 34 » » » uintptr»alloc_bytes; |
33 » int64» time; | 35 » » » uintptr»free_bytes; |
| 36 » » » uintptr»recent_allocs; // since last gc |
| 37 » » » uintptr»recent_frees; |
| 38 » » » uintptr»recent_alloc_bytes; |
| 39 » » » uintptr»recent_free_bytes; |
| 40 » » }; |
| 41 » » struct // typ == BProf |
| 42 » » { |
| 43 » » » int64» count; |
| 44 » » » int64» cycles; |
| 45 » » }; |
| 46 » }; |
34 uintptr hash; | 47 uintptr hash; |
35 uintptr nstk; | 48 uintptr nstk; |
36 uintptr stk[1]; | 49 uintptr stk[1]; |
37 }; | 50 }; |
38 enum { | 51 enum { |
39 BuckHashSize = 179999, | 52 BuckHashSize = 179999, |
40 }; | 53 }; |
41 static Bucket **buckhash; | 54 static Bucket **buckhash; |
42 static Bucket *buckets; | 55 static Bucket *mbuckets; // memory profile buckets |
| 56 static Bucket *bbuckets; // blocking profile buckets |
43 static uintptr bucketmem; | 57 static uintptr bucketmem; |
44 | 58 |
45 // Return the bucket for stk[0:nstk], allocating new bucket if needed. | 59 // Return the bucket for stk[0:nstk], allocating new bucket if needed. |
46 static Bucket* | 60 static Bucket* |
47 stkbucket(uintptr *stk, int32 nstk, bool alloc) | 61 stkbucket(int32 typ, uintptr *stk, int32 nstk, bool alloc) |
48 { | 62 { |
49 int32 i; | 63 int32 i; |
50 uintptr h; | 64 uintptr h; |
51 Bucket *b; | 65 Bucket *b; |
52 | 66 |
53 if(buckhash == nil) { | 67 if(buckhash == nil) { |
54 buckhash = runtime·SysAlloc(BuckHashSize*sizeof buckhash[0]); | 68 buckhash = runtime·SysAlloc(BuckHashSize*sizeof buckhash[0]); |
55 mstats.buckhash_sys += BuckHashSize*sizeof buckhash[0]; | 69 mstats.buckhash_sys += BuckHashSize*sizeof buckhash[0]; |
56 } | 70 } |
57 | 71 |
58 // Hash stack. | 72 // Hash stack. |
59 h = 0; | 73 h = 0; |
60 for(i=0; i<nstk; i++) { | 74 for(i=0; i<nstk; i++) { |
61 h += stk[i]; | 75 h += stk[i]; |
62 h += h<<10; | 76 h += h<<10; |
63 h ^= h>>6; | 77 h ^= h>>6; |
64 } | 78 } |
65 h += h<<3; | 79 h += h<<3; |
66 h ^= h>>11; | 80 h ^= h>>11; |
67 | 81 |
68 i = h%BuckHashSize; | 82 i = h%BuckHashSize; |
69 for(b = buckhash[i]; b; b=b->next) | 83 for(b = buckhash[i]; b; b=b->next) |
70 » » if(b->hash == h && b->nstk == nstk && | 84 » » if(b->typ == typ && b->hash == h && b->nstk == nstk && |
71 runtime·mcmp((byte*)b->stk, (byte*)stk, nstk*sizeof stk[0]) =
= 0) | 85 runtime·mcmp((byte*)b->stk, (byte*)stk, nstk*sizeof stk[0]) =
= 0) |
72 return b; | 86 return b; |
73 | 87 |
74 if(!alloc) | 88 if(!alloc) |
75 return nil; | 89 return nil; |
76 | 90 |
77 b = runtime·mallocgc(sizeof *b + nstk*sizeof stk[0], FlagNoProfiling, 0,
1); | 91 b = runtime·mallocgc(sizeof *b + nstk*sizeof stk[0], FlagNoProfiling, 0,
1); |
78 bucketmem += sizeof *b + nstk*sizeof stk[0]; | 92 bucketmem += sizeof *b + nstk*sizeof stk[0]; |
79 runtime·memmove(b->stk, stk, nstk*sizeof stk[0]); | 93 runtime·memmove(b->stk, stk, nstk*sizeof stk[0]); |
| 94 b->typ = typ; |
80 b->hash = h; | 95 b->hash = h; |
81 b->nstk = nstk; | 96 b->nstk = nstk; |
82 b->next = buckhash[i]; | 97 b->next = buckhash[i]; |
83 buckhash[i] = b; | 98 buckhash[i] = b; |
84 » b->allnext = buckets; | 99 » if(typ == MProf) { |
85 » buckets = b; | 100 » » b->allnext = mbuckets; |
| 101 » » mbuckets = b; |
| 102 » } else { |
| 103 » » b->allnext = bbuckets; |
| 104 » » bbuckets = b; |
| 105 » } |
86 return b; | 106 return b; |
87 } | 107 } |
88 | 108 |
89 // Record that a gc just happened: all the 'recent' statistics are now real. | 109 // Record that a gc just happened: all the 'recent' statistics are now real. |
90 void | 110 void |
91 runtime·MProf_GC(void) | 111 runtime·MProf_GC(void) |
92 { | 112 { |
93 Bucket *b; | 113 Bucket *b; |
94 ········ | 114 ········ |
95 runtime·lock(&proflock); | 115 runtime·lock(&proflock); |
96 » for(b=buckets; b; b=b->allnext) { | 116 » for(b=mbuckets; b; b=b->allnext) { |
97 b->allocs += b->recent_allocs; | 117 b->allocs += b->recent_allocs; |
98 b->frees += b->recent_frees; | 118 b->frees += b->recent_frees; |
99 b->alloc_bytes += b->recent_alloc_bytes; | 119 b->alloc_bytes += b->recent_alloc_bytes; |
100 b->free_bytes += b->recent_free_bytes; | 120 b->free_bytes += b->recent_free_bytes; |
101 b->recent_allocs = 0; | 121 b->recent_allocs = 0; |
102 b->recent_frees = 0; | 122 b->recent_frees = 0; |
103 b->recent_alloc_bytes = 0; | 123 b->recent_alloc_bytes = 0; |
104 b->recent_free_bytes = 0; | 124 b->recent_free_bytes = 0; |
105 } | 125 } |
106 runtime·unlock(&proflock); | 126 runtime·unlock(&proflock); |
(...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
222 int32 nstk; | 242 int32 nstk; |
223 uintptr stk[32]; | 243 uintptr stk[32]; |
224 Bucket *b; | 244 Bucket *b; |
225 | 245 |
226 if(m->nomemprof > 0) | 246 if(m->nomemprof > 0) |
227 return; | 247 return; |
228 | 248 |
229 m->nomemprof++; | 249 m->nomemprof++; |
230 nstk = runtime·callers(1, stk, 32); | 250 nstk = runtime·callers(1, stk, 32); |
231 runtime·lock(&proflock); | 251 runtime·lock(&proflock); |
232 » b = stkbucket(stk, nstk, true); | 252 » b = stkbucket(MProf, stk, nstk, true); |
233 b->recent_allocs++; | 253 b->recent_allocs++; |
234 b->recent_alloc_bytes += size; | 254 b->recent_alloc_bytes += size; |
235 setaddrbucket((uintptr)p, b); | 255 setaddrbucket((uintptr)p, b); |
236 runtime·unlock(&proflock); | 256 runtime·unlock(&proflock); |
237 m->nomemprof--; | 257 m->nomemprof--; |
238 } | 258 } |
239 | 259 |
240 // Called when freeing a profiled block. | 260 // Called when freeing a profiled block. |
241 void | 261 void |
242 runtime·MProf_Free(void *p, uintptr size) | 262 runtime·MProf_Free(void *p, uintptr size) |
243 { | 263 { |
244 Bucket *b; | 264 Bucket *b; |
245 | 265 |
246 if(m->nomemprof > 0) | 266 if(m->nomemprof > 0) |
247 return; | 267 return; |
248 | 268 |
249 m->nomemprof++; | 269 m->nomemprof++; |
250 runtime·lock(&proflock); | 270 runtime·lock(&proflock); |
251 b = getaddrbucket((uintptr)p); | 271 b = getaddrbucket((uintptr)p); |
252 if(b != nil) { | 272 if(b != nil) { |
253 b->recent_frees++; | 273 b->recent_frees++; |
254 b->recent_free_bytes += size; | 274 b->recent_free_bytes += size; |
255 } | 275 } |
256 runtime·unlock(&proflock); | 276 runtime·unlock(&proflock); |
257 m->nomemprof--; | 277 m->nomemprof--; |
258 } | 278 } |
259 | 279 |
| 280 int64 runtime·blockprofilerate; // in CPU ticks |
| 281 |
260 void | 282 void |
261 runtime·MtxProf(int64 t) | 283 runtime·SetBlockProfileRate(intgo rate) |
| 284 { |
| 285 » runtime·atomicstore64((uint64*)&runtime·blockprofilerate, rate * runtime
·tickspersecond() / (1000*1000*1000)); |
| 286 } |
| 287 |
| 288 void |
| 289 runtime·blockevent(int64 cycles, int32 skip) |
262 { | 290 { |
263 int32 nstk; | 291 int32 nstk; |
| 292 int64 rate; |
264 uintptr stk[32]; | 293 uintptr stk[32]; |
265 Bucket *b; | 294 Bucket *b; |
266 | 295 |
267 » nstk = runtime·callers(3, stk, 32); | 296 » if(cycles <= 0) |
268 » runtime·lock(&proflock); | 297 » » return; |
269 » b = stkbucket(stk, nstk, true); | 298 » rate = runtime·atomicload64((uint64*)&runtime·blockprofilerate); |
270 » b->time += t; | 299 » if(rate <= 0 || (rate > cycles && runtime·fastrand1()%rate > cycles)) |
| 300 » » return; |
| 301 |
| 302 » nstk = runtime·callers(skip, stk, 32); |
| 303 » runtime·lock(&proflock); |
| 304 » b = stkbucket(BProf, stk, nstk, true); |
| 305 » b->count++; |
| 306 » b->cycles += cycles; |
271 runtime·unlock(&proflock); | 307 runtime·unlock(&proflock); |
272 } | 308 } |
273 | 309 |
274 // Go interface to profile data. (Declared in extern.go) | 310 // Go interface to profile data. (Declared in extern.go) |
275 // Assumes Go sizeof(int) == sizeof(int32) | 311 // Assumes Go sizeof(int) == sizeof(int32) |
276 | 312 |
277 // Must match MemProfileRecord in debug.go. | 313 // Must match MemProfileRecord in debug.go. |
278 typedef struct Record Record; | 314 typedef struct Record Record; |
279 struct Record { | 315 struct Record { |
280 int64 alloc_bytes, free_bytes; | 316 int64 alloc_bytes, free_bytes; |
(...skipping 10 matching lines...) Expand all Loading... |
291 r->alloc_bytes = b->alloc_bytes; | 327 r->alloc_bytes = b->alloc_bytes; |
292 r->free_bytes = b->free_bytes; | 328 r->free_bytes = b->free_bytes; |
293 r->alloc_objects = b->allocs; | 329 r->alloc_objects = b->allocs; |
294 r->free_objects = b->frees; | 330 r->free_objects = b->frees; |
295 for(i=0; i<b->nstk && i<nelem(r->stk); i++) | 331 for(i=0; i<b->nstk && i<nelem(r->stk); i++) |
296 r->stk[i] = b->stk[i]; | 332 r->stk[i] = b->stk[i]; |
297 for(; i<nelem(r->stk); i++) | 333 for(; i<nelem(r->stk); i++) |
298 r->stk[i] = 0; | 334 r->stk[i] = 0; |
299 } | 335 } |
300 | 336 |
301 func MemProfile(p Slice, include_inuse_zero bool) (n int32, ok bool) { | 337 func MemProfile(p Slice, include_inuse_zero bool) (n int, ok bool) { |
302 Bucket *b; | 338 Bucket *b; |
303 Record *r; | 339 Record *r; |
304 | 340 |
305 runtime·lock(&proflock); | 341 runtime·lock(&proflock); |
306 n = 0; | 342 n = 0; |
307 » for(b=buckets; b; b=b->allnext) | 343 » for(b=mbuckets; b; b=b->allnext) |
308 if(include_inuse_zero || b->alloc_bytes != b->free_bytes) | 344 if(include_inuse_zero || b->alloc_bytes != b->free_bytes) |
309 n++; | 345 n++; |
310 ok = false; | 346 ok = false; |
311 if(n <= p.len) { | 347 if(n <= p.len) { |
312 ok = true; | 348 ok = true; |
313 r = (Record*)p.array; | 349 r = (Record*)p.array; |
314 » » for(b=buckets; b; b=b->allnext) | 350 » » for(b=mbuckets; b; b=b->allnext) |
315 if(include_inuse_zero || b->alloc_bytes != b->free_bytes
) | 351 if(include_inuse_zero || b->alloc_bytes != b->free_bytes
) |
316 record(r++, b); | 352 record(r++, b); |
317 } | 353 } |
318 runtime·unlock(&proflock); | 354 runtime·unlock(&proflock); |
319 } | 355 } |
320 | 356 |
321 // Must match ContentionRecord in debug.go. | 357 // Must match BlockProfileRecord in debug.go. |
322 typedef struct CRecord CRecord; | 358 typedef struct BRecord BRecord; |
323 struct CRecord { | 359 struct BRecord { |
324 » int64 time; | 360 » int64 count; |
| 361 » int64 cycles; |
325 uintptr stk[32]; | 362 uintptr stk[32]; |
326 }; | 363 }; |
327 | 364 |
328 func ContentionProfile(p Slice) (n int32, ok bool) { | 365 func BlockProfile(p Slice) (n int, ok bool) { |
329 » Bucket *b; | 366 » Bucket *b; |
330 » CRecord *r; | 367 » BRecord *r; |
331 int32 i; | 368 int32 i; |
332 | 369 |
333 runtime·lock(&proflock); | 370 runtime·lock(&proflock); |
334 n = 0; | 371 n = 0; |
335 » for(b=buckets; b; b=b->allnext) | 372 » for(b=bbuckets; b; b=b->allnext) |
336 n++; | 373 n++; |
337 ok = false; | 374 ok = false; |
338 if(n <= p.len) { | 375 if(n <= p.len) { |
339 ok = true; | 376 ok = true; |
340 » » r = (CRecord*)p.array; | 377 » » r = (BRecord*)p.array; |
341 » » for(b=buckets; b; b=b->allnext, r++) { | 378 » » for(b=bbuckets; b; b=b->allnext, r++) { |
342 » » » r->time = b->time; | 379 » » » r->count = b->count; |
| 380 » » » r->cycles = b->cycles; |
343 for(i=0; i<b->nstk && i<nelem(r->stk); i++) | 381 for(i=0; i<b->nstk && i<nelem(r->stk); i++) |
344 r->stk[i] = b->stk[i]; | 382 r->stk[i] = b->stk[i]; |
345 for(; i<nelem(r->stk); i++) | 383 for(; i<nelem(r->stk); i++) |
346 r->stk[i] = 0;·················· | 384 r->stk[i] = 0;·················· |
347 } | 385 } |
348 } | 386 } |
349 runtime·unlock(&proflock); | 387 runtime·unlock(&proflock); |
350 } | 388 } |
351 | 389 |
352 // Must match StackRecord in debug.go. | 390 // Must match StackRecord in debug.go. |
353 typedef struct TRecord TRecord; | 391 typedef struct TRecord TRecord; |
354 struct TRecord { | 392 struct TRecord { |
355 uintptr stk[32]; | 393 uintptr stk[32]; |
356 }; | 394 }; |
357 | 395 |
358 func ThreadCreateProfile(p Slice) (n int32, ok bool) { | 396 func ThreadCreateProfile(p Slice) (n int, ok bool) { |
359 TRecord *r; | 397 TRecord *r; |
360 M *first, *m; | 398 M *first, *m; |
361 ········ | 399 ········ |
362 first = runtime·atomicloadp(&runtime·allm); | 400 first = runtime·atomicloadp(&runtime·allm); |
363 n = 0; | 401 n = 0; |
364 for(m=first; m; m=m->alllink) | 402 for(m=first; m; m=m->alllink) |
365 n++; | 403 n++; |
366 ok = false; | 404 ok = false; |
367 if(n <= p.len) { | 405 if(n <= p.len) { |
368 ok = true; | 406 ok = true; |
369 r = (TRecord*)p.array; | 407 r = (TRecord*)p.array; |
370 for(m=first; m; m=m->alllink) { | 408 for(m=first; m; m=m->alllink) { |
371 runtime·memmove(r->stk, m->createstack, sizeof r->stk); | 409 runtime·memmove(r->stk, m->createstack, sizeof r->stk); |
372 r++; | 410 r++; |
373 } | 411 } |
374 } | 412 } |
375 } | 413 } |
376 | 414 |
377 func Stack(b Slice, all bool) (n int32) { | 415 func Stack(b Slice, all bool) (n int) { |
378 byte *pc, *sp; | 416 byte *pc, *sp; |
379 ········ | 417 ········ |
380 sp = runtime·getcallersp(&b); | 418 sp = runtime·getcallersp(&b); |
381 pc = runtime·getcallerpc(&b); | 419 pc = runtime·getcallerpc(&b); |
382 | 420 |
383 if(all) { | 421 if(all) { |
384 runtime·semacquire(&runtime·worldsema); | 422 runtime·semacquire(&runtime·worldsema); |
385 m->gcing = 1; | 423 m->gcing = 1; |
386 runtime·stoptheworld(); | 424 runtime·stoptheworld(); |
387 } | 425 } |
(...skipping 22 matching lines...) Expand all Loading... |
410 static void | 448 static void |
411 saveg(byte *pc, byte *sp, G *g, TRecord *r) | 449 saveg(byte *pc, byte *sp, G *g, TRecord *r) |
412 { | 450 { |
413 int32 n; | 451 int32 n; |
414 ········ | 452 ········ |
415 n = runtime·gentraceback(pc, sp, 0, g, 0, r->stk, nelem(r->stk)); | 453 n = runtime·gentraceback(pc, sp, 0, g, 0, r->stk, nelem(r->stk)); |
416 if(n < nelem(r->stk)) | 454 if(n < nelem(r->stk)) |
417 r->stk[n] = 0; | 455 r->stk[n] = 0; |
418 } | 456 } |
419 | 457 |
420 func GoroutineProfile(b Slice) (n int32, ok bool) { | 458 func GoroutineProfile(b Slice) (n int, ok bool) { |
421 byte *pc, *sp; | 459 byte *pc, *sp; |
422 TRecord *r; | 460 TRecord *r; |
423 G *gp; | 461 G *gp; |
424 | 462 » |
425 sp = runtime·getcallersp(&b); | 463 sp = runtime·getcallersp(&b); |
426 pc = runtime·getcallerpc(&b); | 464 pc = runtime·getcallerpc(&b); |
427 ········ | 465 ········ |
428 ok = false; | 466 ok = false; |
429 n = runtime·gcount(); | 467 n = runtime·gcount(); |
430 if(n <= b.len) { | 468 if(n <= b.len) { |
431 runtime·semacquire(&runtime·worldsema); | 469 runtime·semacquire(&runtime·worldsema); |
432 m->gcing = 1; | 470 m->gcing = 1; |
433 runtime·stoptheworld(); | 471 runtime·stoptheworld(); |
434 | 472 |
435 n = runtime·gcount(); | 473 n = runtime·gcount(); |
436 if(n <= b.len) { | 474 if(n <= b.len) { |
437 ok = true; | 475 ok = true; |
438 r = (TRecord*)b.array; | 476 r = (TRecord*)b.array; |
439 saveg(pc, sp, g, r++); | 477 saveg(pc, sp, g, r++); |
440 for(gp = runtime·allg; gp != nil; gp = gp->alllink) { | 478 for(gp = runtime·allg; gp != nil; gp = gp->alllink) { |
441 if(gp == g || gp->status == Gdead) | 479 if(gp == g || gp->status == Gdead) |
442 continue; | 480 continue; |
443 saveg(gp->sched.pc, (byte*)gp->sched.sp, gp, r++
); | 481 saveg(gp->sched.pc, (byte*)gp->sched.sp, gp, r++
); |
444 } | 482 } |
445 } | 483 } |
446 ········ | 484 ········ |
447 m->gcing = 0; | 485 m->gcing = 0; |
448 runtime·semrelease(&runtime·worldsema); | 486 runtime·semrelease(&runtime·worldsema); |
449 runtime·starttheworld(); | 487 runtime·starttheworld(); |
450 } | 488 } |
451 } | 489 } |
452 | 490 |
LEFT | RIGHT |