Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(4113)

Delta Between Two Patch Sets: src/cmd/pprof/internal/profile/legacy_profile.go

Issue 153750043: code review 153750043: cmd/pprof: add Go implementation (Closed)
Left Patch Set: Created 9 years, 6 months ago
Right Patch Set: diff -r 2e467bc60e64def06419194f48fe0d6c8b56765d https://code.google.com/p/go/ Created 9 years, 6 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « src/cmd/pprof/internal/profile/filter.go ('k') | src/cmd/pprof/internal/profile/profile.go » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 // This file implements parsers to convert legacy profiles into the
6 // profile.proto format.
7
8 package profile
9
10 import (
11 "bufio"
12 "bytes"
13 "fmt"
14 "io"
15 "math"
16 "regexp"
17 "strconv"
18 "strings"
19 )
20
21 var (
22 countStartRE = regexp.MustCompile(`\A(\w+) profile: total \d+\n\z`)
23 countRE = regexp.MustCompile(`\A(\d+) @(( 0x[0-9a-f]+)+)\n\z`)
24
25 heapHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d +): *(\d+) *\] *@ *(heap[_a-z0-9]*)/?(\d*)`)
26 heapSampleRE = regexp.MustCompile(`(-?\d+): *(-?\d+) *\[ *(\d+): *(\d+) *] @([ x0-9a-f]*)`)
27
28 contentionSampleRE = regexp.MustCompile(`(\d+) *(\d+) @([ x0-9a-f]*)`)
29
30 hexNumberRE = regexp.MustCompile(`0x[0-9a-f]+`)
31
32 growthHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *( \d+): *(\d+) *\] @ growthz`)
33
34 fragmentationHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ fragmentationz`)
35
36 threadzStartRE = regexp.MustCompile(`--- threadz \d+ ---`)
37 threadStartRE = regexp.MustCompile(`--- Thread ([[:xdigit:]]+) \(name: (.*)/(\d+)\) stack: ---`)
38
39 procMapsRE = regexp.MustCompile(`([[:xdigit:]]+)-([[:xdigit:]]+)\s+([-rw xp]+)\s+([[:xdigit:]]+)\s+([[:xdigit:]]+):([[:xdigit:]]+)\s+([[:digit:]]+)\s*(\S +)?`)
40
41 briefMapsRE = regexp.MustCompile(`\s*([[:xdigit:]]+)-([[:xdigit:]]+):\s* (\S+)(\s.*@)?([[:xdigit:]]+)?`)
42
43 // LegacyHeapAllocated instructs the heapz parsers to use the
44 // allocated memory stats instead of the default in-use memory. Note
45 // that tcmalloc doesn't provide all allocated memory, only in-use
46 // stats.
47 LegacyHeapAllocated bool
48 )
49
50 func isSpaceOrComment(line string) bool {
51 trimmed := strings.TrimSpace(line)
52 return len(trimmed) == 0 || trimmed[0] == '#'
53 }
54
55 // parseGoCount parses a Go count profile (e.g., threadcreate or
56 // goroutine) and returns a new Profile.
57 func parseGoCount(b []byte) (*Profile, error) {
58 r := bytes.NewBuffer(b)
59
60 var line string
61 var err error
62 for {
63 // Skip past comments and empty lines seeking a real header.
64 line, err = r.ReadString('\n')
65 if err != nil {
66 return nil, err
67 }
68 if !isSpaceOrComment(line) {
69 break
70 }
71 }
72
73 m := countStartRE.FindStringSubmatch(line)
74 if m == nil {
75 return nil, errUnrecognized
76 }
77 profileType := string(m[1])
78 p := &Profile{
79 PeriodType: &ValueType{Type: profileType, Unit: "count"},
80 Period: 1,
81 SampleType: []*ValueType{{Type: profileType, Unit: "count"}},
82 }
83 locations := make(map[uint64]*Location)
84 for {
85 line, err = r.ReadString('\n')
86 if err != nil {
87 if err == io.EOF {
88 break
89 }
90 return nil, err
91 }
92 if isSpaceOrComment(line) {
93 continue
94 }
95 if strings.HasPrefix(line, "---") {
96 break
97 }
98 m := countRE.FindStringSubmatch(line)
99 if m == nil {
100 return nil, errMalformed
101 }
102 n, err := strconv.ParseInt(string(m[1]), 0, 64)
103 if err != nil {
104 return nil, errMalformed
105 }
106 fields := strings.Fields(string(m[2]))
107 locs := make([]*Location, 0, len(fields))
108 for _, stk := range fields {
109 addr, err := strconv.ParseUint(stk, 0, 64)
110 if err != nil {
111 return nil, errMalformed
112 }
113 // Adjust all frames by -1 (except the leaf) to land on top of
114 // the call instruction.
115 if len(locs) > 0 {
116 addr--
117 }
118 loc := locations[addr]
119 if loc == nil {
120 loc = &Location{
121 Address: addr,
122 }
123 locations[addr] = loc
124 p.Location = append(p.Location, loc)
125 }
126 locs = append(locs, loc)
127 }
128 p.Sample = append(p.Sample, &Sample{
129 Location: locs,
130 Value: []int64{n},
131 })
132 }
133
134 if err = parseAdditionalSections(strings.TrimSpace(line), r, p); err != nil {
135 return nil, err
136 }
137 return p, nil
138 }
139
140 // remapLocationIDs ensures there is a location for each address
141 // referenced by a sample, and remaps the samples to point to the new
142 // location ids.
143 func (p *Profile) remapLocationIDs() {
144 seen := make(map[*Location]bool, len(p.Location))
145 var locs []*Location
146
147 for _, s := range p.Sample {
148 for _, l := range s.Location {
149 if seen[l] {
150 continue
151 }
152 l.ID = uint64(len(locs) + 1)
153 locs = append(locs, l)
154 seen[l] = true
155 }
156 }
157 p.Location = locs
158 }
159
160 func (p *Profile) remapFunctionIDs() {
161 seen := make(map[*Function]bool, len(p.Function))
162 var fns []*Function
163
164 for _, l := range p.Location {
165 for _, ln := range l.Line {
166 fn := ln.Function
167 if fn == nil || seen[fn] {
168 continue
169 }
170 fn.ID = uint64(len(fns) + 1)
171 fns = append(fns, fn)
172 seen[fn] = true
173 }
174 }
175 p.Function = fns
176 }
177
178 // remapMappingIDs matches location addresses with existing mappings
179 // and updates them appropriately. This is O(N*M), if this ever shows
180 // up as a bottleneck, evaluate sorting the mappings and doing a
181 // binary search, which would make it O(N*log(M)).
182 func (p *Profile) remapMappingIDs() {
183 if len(p.Mapping) == 0 {
184 return
185 }
186
187 // Some profile handlers will incorrectly set regions for the main
188 // executable if its section is remapped. Fix them through heuristics.
189
190 // Remove the initial mapping if named '/anon_hugepage' and has a
191 // consecutive adjacent mapping.
192 if m := p.Mapping[0]; strings.HasPrefix(m.File, "/anon_hugepage") {
193 if len(p.Mapping) > 1 && m.Limit == p.Mapping[1].Start {
194 p.Mapping = p.Mapping[1:]
195 }
196 }
197
198 // Subtract the offset from the start of the main mapping if it
199 // ends up at a recognizable start address.
200 const expectedStart = 0x400000
201 if m := p.Mapping[0]; m.Start-m.Offset == expectedStart {
202 m.Start = expectedStart
203 m.Offset = 0
204 }
205
206 for _, l := range p.Location {
207 if a := l.Address; a != 0 {
208 for _, m := range p.Mapping {
209 if m.Start <= a && a < m.Limit {
210 l.Mapping = m
211 break
212 }
213 }
214 }
215 }
216
217 // Reset all mapping IDs.
218 for i, m := range p.Mapping {
219 m.ID = uint64(i + 1)
220 }
221 }
222
223 var cpuInts = []func([]byte) (uint64, []byte){
224 get32l,
225 get32b,
226 get64l,
227 get64b,
228 }
229
230 func get32l(b []byte) (uint64, []byte) {
231 if len(b) < 4 {
232 return 0, nil
233 }
234 return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])< <24, b[4:]
235 }
236
237 func get32b(b []byte) (uint64, []byte) {
238 if len(b) < 4 {
239 return 0, nil
240 }
241 return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])< <24, b[4:]
242 }
243
244 func get64l(b []byte) (uint64, []byte) {
245 if len(b) < 8 {
246 return 0, nil
247 }
248 return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])< <24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56, b[8:]
249 }
250
251 func get64b(b []byte) (uint64, []byte) {
252 if len(b) < 8 {
253 return 0, nil
254 }
255 return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])< <24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56, b[8:]
256 }
257
258 // ParseTracebacks parses a set of tracebacks and returns a newly
259 // populated profile. It will accept any text file and generate a
260 // Profile out of it with any hex addresses it can identify, including
261 // a process map if it can recognize one. Each sample will include a
262 // tag "source" with the addresses recognized in string format.
263 func ParseTracebacks(b []byte) (*Profile, error) {
264 r := bytes.NewBuffer(b)
265
266 p := &Profile{
267 PeriodType: &ValueType{Type: "trace", Unit: "count"},
268 Period: 1,
269 SampleType: []*ValueType{
270 {Type: "trace", Unit: "count"},
271 },
272 }
273
274 var sources []string
275 var sloc []*Location
276
277 locs := make(map[uint64]*Location)
278 for {
279 l, err := r.ReadString('\n')
280 if err != nil {
281 if err != io.EOF {
282 return nil, err
283 }
284 if l == "" {
285 break
286 }
287 }
288 if sectionTrigger(l) == memoryMapSection {
289 break
290 }
291 if s, addrs := extractHexAddresses(l); len(s) > 0 {
292 for _, addr := range addrs {
293 // Addresses from stack traces point to the next instruction after
294 // each call. Adjust by -1 to land somewhere on the actual call
295 // (except for the leaf, which is not a call).
296 if len(sloc) > 0 {
297 addr--
298 }
299 loc := locs[addr]
300 if locs[addr] == nil {
301 loc = &Location{
302 Address: addr,
303 }
304 p.Location = append(p.Location, loc)
305 locs[addr] = loc
306 }
307 sloc = append(sloc, loc)
308 }
309
310 sources = append(sources, s...)
311 } else {
312 if len(sources) > 0 || len(sloc) > 0 {
313 addTracebackSample(sloc, sources, p)
314 sloc, sources = nil, nil
315 }
316 }
317 }
318
319 // Add final sample to save any leftover data.
320 if len(sources) > 0 || len(sloc) > 0 {
321 addTracebackSample(sloc, sources, p)
322 }
323
324 if err := p.ParseMemoryMap(r); err != nil {
325 return nil, err
326 }
327 return p, nil
328 }
329
330 func addTracebackSample(l []*Location, s []string, p *Profile) {
331 p.Sample = append(p.Sample,
332 &Sample{
333 Value: []int64{1},
334 Location: l,
335 Label: map[string][]string{"source": s},
336 })
337 }
338
339 // parseCPU parses a profilez legacy profile and returns a newly
340 // populated Profile.
341 //
342 // The general format for profilez samples is a sequence of words in
343 // binary format. The first words are a header with the following data:
344 // 1st word -- 0
345 // 2nd word -- 3
346 // 3rd word -- 0 if a c++ application, 1 if a java application.
347 // 4th word -- Sampling period (in microseconds).
348 // 5th word -- Padding.
349 func parseCPU(b []byte) (*Profile, error) {
350 var parse func([]byte) (uint64, []byte)
351 var n1, n2, n3, n4, n5 uint64
352 for _, parse = range cpuInts {
353 var tmp []byte
354 n1, tmp = parse(b)
355 n2, tmp = parse(tmp)
356 n3, tmp = parse(tmp)
357 n4, tmp = parse(tmp)
358 n5, tmp = parse(tmp)
359
360 if tmp != nil && n1 == 0 && n2 == 3 && n3 == 0 && n4 > 0 && n5 = = 0 {
361 b = tmp
362 return cpuProfile(b, int64(n4), parse)
363 }
364 }
365 return nil, errUnrecognized
366 }
367
368 // cpuProfile returns a new Profile from C++ profilez data.
369 // b is the profile bytes after the header, period is the profiling
370 // period, and parse is a function to parse 8-byte chunks from the
371 // profile in its native endianness.
372 func cpuProfile(b []byte, period int64, parse func(b []byte) (uint64, []byte)) ( *Profile, error) {
373 p := &Profile{
374 Period: period * 1000,
375 PeriodType: &ValueType{Type: "cpu", Unit: "nanoseconds"},
376 SampleType: []*ValueType{
377 {Type: "samples", Unit: "count"},
378 {Type: "cpu", Unit: "nanoseconds"},
379 },
380 }
381 var err error
382 if b, _, err = parseCPUSamples(b, parse, true, p); err != nil {
383 return nil, err
384 }
385
386 // If all samples have the same second-to-the-bottom frame, it
387 // strongly suggests that it is an uninteresting artifact of
388 // measurement -- a stack frame pushed by the signal handler. The
389 // bottom frame is always correct as it is picked up from the signal
390 // structure, not the stack. Check if this is the case and if so,
391 // remove.
392 if len(p.Sample) > 1 && len(p.Sample[0].Location) > 1 {
393 allSame := true
394 id1 := p.Sample[0].Location[1].Address
395 for _, s := range p.Sample {
396 if len(s.Location) < 2 || id1 != s.Location[1].Address {
397 allSame = false
398 break
399 }
400 }
401 if allSame {
402 for _, s := range p.Sample {
403 s.Location = append(s.Location[:1], s.Location[2 :]...)
404 }
405 }
406 }
407
408 if err := p.ParseMemoryMap(bytes.NewBuffer(b)); err != nil {
409 return nil, err
410 }
411 return p, nil
412 }
413
414 // parseCPUSamples parses a collection of profilez samples from a
415 // profile.
416 //
417 // profilez samples are a repeated sequence of stack frames of the
418 // form:
419 // 1st word -- The number of times this stack was encountered.
420 // 2nd word -- The size of the stack (StackSize).
421 // 3rd word -- The first address on the stack.
422 // ...
423 // StackSize + 2 -- The last address on the stack
424 // The last stack trace is of the form:
425 // 1st word -- 0
426 // 2nd word -- 1
427 // 3rd word -- 0
428 //
429 // Addresses from stack traces may point to the next instruction after
430 // each call. Optionally adjust by -1 to land somewhere on the actual
431 // call (except for the leaf, which is not a call).
432 func parseCPUSamples(b []byte, parse func(b []byte) (uint64, []byte), adjust boo l, p *Profile) ([]byte, map[uint64]*Location, error) {
433 locs := make(map[uint64]*Location)
434 for len(b) > 0 {
435 var count, nstk uint64
436 count, b = parse(b)
437 nstk, b = parse(b)
438 if b == nil || nstk > uint64(len(b)/4) {
439 return nil, nil, errUnrecognized
440 }
441 var sloc []*Location
442 addrs := make([]uint64, nstk)
443 for i := 0; i < int(nstk); i++ {
444 addrs[i], b = parse(b)
445 }
446
447 if count == 0 && nstk == 1 && addrs[0] == 0 {
448 // End of data marker
449 break
450 }
451 for i, addr := range addrs {
452 if adjust && i > 0 {
453 addr--
454 }
455 loc := locs[addr]
456 if loc == nil {
457 loc = &Location{
458 Address: addr,
459 }
460 locs[addr] = loc
461 p.Location = append(p.Location, loc)
462 }
463 sloc = append(sloc, loc)
464 }
465 p.Sample = append(p.Sample,
466 &Sample{
467 Value: []int64{int64(count), int64(count) * i nt64(p.Period)},
468 Location: sloc,
469 })
470 }
471 // Reached the end without finding the EOD marker.
472 return b, locs, nil
473 }
474
475 // parseHeap parses a heapz legacy or a growthz profile and
476 // returns a newly populated Profile.
477 func parseHeap(b []byte) (p *Profile, err error) {
478 r := bytes.NewBuffer(b)
479 l, err := r.ReadString('\n')
480 if err != nil {
481 return nil, errUnrecognized
482 }
483
484 sampling := ""
485
486 if header := heapHeaderRE.FindStringSubmatch(l); header != nil {
487 p = &Profile{
488 SampleType: []*ValueType{
489 {Type: "objects", Unit: "count"},
490 {Type: "space", Unit: "bytes"},
491 },
492 PeriodType: &ValueType{Type: "objects", Unit: "bytes"},
493 }
494
495 var period int64
496 if len(header[6]) > 0 {
497 if period, err = strconv.ParseInt(string(header[6]), 10, 64); err != nil {
498 return nil, errUnrecognized
499 }
500 }
501
502 switch header[5] {
503 case "heapz_v2", "heap_v2":
504 sampling, p.Period = "v2", period
505 case "heapprofile":
506 sampling, p.Period = "", 1
507 case "heap":
508 sampling, p.Period = "v2", period/2
509 default:
510 return nil, errUnrecognized
511 }
512 } else if header = growthHeaderRE.FindStringSubmatch(l); header != nil {
513 p = &Profile{
514 SampleType: []*ValueType{
515 {Type: "objects", Unit: "count"},
516 {Type: "space", Unit: "bytes"},
517 },
518 PeriodType: &ValueType{Type: "heapgrowth", Unit: "count" },
519 Period: 1,
520 }
521 } else if header = fragmentationHeaderRE.FindStringSubmatch(l); header ! = nil {
522 p = &Profile{
523 SampleType: []*ValueType{
524 {Type: "objects", Unit: "count"},
525 {Type: "space", Unit: "bytes"},
526 },
527 PeriodType: &ValueType{Type: "allocations", Unit: "count "},
528 Period: 1,
529 }
530 } else {
531 return nil, errUnrecognized
532 }
533
534 if LegacyHeapAllocated {
535 for _, st := range p.SampleType {
536 st.Type = "alloc_" + st.Type
537 }
538 } else {
539 for _, st := range p.SampleType {
540 st.Type = "inuse_" + st.Type
541 }
542 }
543
544 locs := make(map[uint64]*Location)
545 for {
546 l, err = r.ReadString('\n')
547 if err != nil {
548 if err != io.EOF {
549 return nil, err
550 }
551
552 if l == "" {
553 break
554 }
555 }
556
557 if l = strings.TrimSpace(l); l == "" {
558 continue
559 }
560
561 if sectionTrigger(l) != unrecognizedSection {
562 break
563 }
564
565 value, blocksize, addrs, err := parseHeapSample(l, p.Period, sam pling)
566 if err != nil {
567 return nil, err
568 }
569 var sloc []*Location
570 for i, addr := range addrs {
571 // Addresses from stack traces point to the next instruc tion after
572 // each call. Adjust by -1 to land somewhere on the act ual call
573 // (except for the leaf, which is not a call).
574 if i > 0 {
575 addr--
576 }
577 loc := locs[addr]
578 if locs[addr] == nil {
579 loc = &Location{
580 Address: addr,
581 }
582 p.Location = append(p.Location, loc)
583 locs[addr] = loc
584 }
585 sloc = append(sloc, loc)
586 }
587
588 p.Sample = append(p.Sample, &Sample{
589 Value: value,
590 Location: sloc,
591 NumLabel: map[string][]int64{"bytes": []int64{blocksize} },
592 })
593 }
594
595 if err = parseAdditionalSections(l, r, p); err != nil {
596 return nil, err
597 }
598 return p, nil
599 }
600
601 // parseHeapSample parses a single row from a heap profile into a new Sample.
602 func parseHeapSample(line string, rate int64, sampling string) (value []int64, b locksize int64, addrs []uint64, err error) {
603 sampleData := heapSampleRE.FindStringSubmatch(line)
604 if len(sampleData) != 6 {
605 return value, blocksize, addrs, fmt.Errorf("unexpected number of sample values: got %d, want 6", len(sampleData))
606 }
607
608 // Use first two values by default; tcmalloc sampling generates the
609 // same value for both, only the older heap-profile collect separate
610 // stats for in-use and allocated objects.
611 valueIndex := 1
612 if LegacyHeapAllocated {
613 valueIndex = 3
614 }
615
616 var v1, v2 int64
617 if v1, err = strconv.ParseInt(sampleData[valueIndex], 10, 64); err != ni l {
618 return value, blocksize, addrs, fmt.Errorf("malformed sample: %s : %v", line, err)
619 }
620 if v2, err = strconv.ParseInt(sampleData[valueIndex+1], 10, 64); err != nil {
621 return value, blocksize, addrs, fmt.Errorf("malformed sample: %s : %v", line, err)
622 }
623
624 if v1 == 0 {
625 if v2 != 0 {
626 return value, blocksize, addrs, fmt.Errorf("allocation c ount was 0 but allocation bytes was %d", v2)
627 }
628 } else {
629 blocksize = v2 / v1
630 if sampling == "v2" {
631 v1, v2 = scaleHeapSample(v1, v2, rate)
632 }
633 }
634
635 value = []int64{v1, v2}
636 addrs = parseHexAddresses(sampleData[5])
637
638 return value, blocksize, addrs, nil
639 }
640
641 // extractHexAddresses extracts hex numbers from a string and returns
642 // them, together with their numeric value, in a slice.
643 func extractHexAddresses(s string) ([]string, []uint64) {
644 hexStrings := hexNumberRE.FindAllString(s, -1)
645 var ids []uint64
646 for _, s := range hexStrings {
647 if id, err := strconv.ParseUint(s, 0, 64); err == nil {
648 ids = append(ids, id)
649 } else {
650 // Do not expect any parsing failures due to the regexp matching.
651 panic("failed to parse hex value:" + s)
652 }
653 }
654 return hexStrings, ids
655 }
656
657 // parseHexAddresses parses hex numbers from a string and returns them
658 // in a slice.
659 func parseHexAddresses(s string) []uint64 {
660 _, ids := extractHexAddresses(s)
661 return ids
662 }
663
664 // scaleHeapSample adjusts the data from a heapz Sample to
665 // account for its probability of appearing in the collected
666 // data. heapz profiles are a sampling of the memory allocations
667 // requests in a program. We estimate the unsampled value by dividing
668 // each collected sample by its probability of appearing in the
669 // profile. heapz v2 profiles rely on a poisson process to determine
670 // which samples to collect, based on the desired average collection
671 // rate R. The probability of a sample of size S to appear in that
672 // profile is 1-exp(-S/R).
673 func scaleHeapSample(count, size, rate int64) (int64, int64) {
674 if count == 0 || size == 0 {
675 return 0, 0
676 }
677
678 if rate <= 1 {
679 // if rate==1 all samples were collected so no adjustment is nee ded.
680 // if rate<1 treat as unknown and skip scaling.
681 return count, size
682 }
683
684 avgSize := float64(size) / float64(count)
685 scale := 1 / (1 - math.Exp(-avgSize/float64(rate)))
686
687 return int64(float64(count) * scale), int64(float64(size) * scale)
688 }
689
690 // parseContention parses a contentionz profile and returns a newly
691 // populated Profile.
692 func parseContention(b []byte) (p *Profile, err error) {
693 r := bytes.NewBuffer(b)
694 l, err := r.ReadString('\n')
695 if err != nil {
696 return nil, errUnrecognized
697 }
698
699 if !strings.HasPrefix(l, "--- contention") {
700 return nil, errUnrecognized
701 }
702
703 p = &Profile{
704 PeriodType: &ValueType{Type: "contentions", Unit: "count"},
705 Period: 1,
706 SampleType: []*ValueType{
707 {Type: "contentions", Unit: "count"},
708 {Type: "delay", Unit: "nanoseconds"},
709 },
710 }
711
712 var cpuHz int64
713 // Parse text of the form "attribute = value" before the samples.
714 const delimiter = "="
715 for {
716 l, err = r.ReadString('\n')
717 if err != nil {
718 if err != io.EOF {
719 return nil, err
720 }
721
722 if l == "" {
723 break
724 }
725 }
726
727 if l = strings.TrimSpace(l); l == "" {
728 continue
729 }
730
731 if strings.HasPrefix(l, "---") {
732 break
733 }
734
735 attr := strings.SplitN(l, delimiter, 2)
736 if len(attr) != 2 {
737 break
738 }
739 key, val := strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1 ])
740 var err error
741 switch key {
742 case "cycles/second":
743 if cpuHz, err = strconv.ParseInt(val, 0, 64); err != nil {
744 return nil, errUnrecognized
745 }
746 case "sampling period":
747 if p.Period, err = strconv.ParseInt(val, 0, 64); err != nil {
748 return nil, errUnrecognized
749 }
750 case "ms since reset":
751 ms, err := strconv.ParseInt(val, 0, 64)
752 if err != nil {
753 return nil, errUnrecognized
754 }
755 p.DurationNanos = ms * 1000 * 1000
756 case "format":
757 // CPP contentionz profiles don't have format.
758 return nil, errUnrecognized
759 case "resolution":
760 // CPP contentionz profiles don't have resolution.
761 return nil, errUnrecognized
762 case "discarded samples":
763 default:
764 return nil, errUnrecognized
765 }
766 }
767
768 locs := make(map[uint64]*Location)
769 for {
770 if l = strings.TrimSpace(l); strings.HasPrefix(l, "---") {
771 break
772 }
773 value, addrs, err := parseContentionSample(l, p.Period, cpuHz)
774 if err != nil {
775 return nil, err
776 }
777 var sloc []*Location
778 for i, addr := range addrs {
779 // Addresses from stack traces point to the next instruc tion after
780 // each call. Adjust by -1 to land somewhere on the act ual call
781 // (except for the leaf, which is not a call).
782 if i > 0 {
783 addr--
784 }
785 loc := locs[addr]
786 if locs[addr] == nil {
787 loc = &Location{
788 Address: addr,
789 }
790 p.Location = append(p.Location, loc)
791 locs[addr] = loc
792 }
793 sloc = append(sloc, loc)
794 }
795 p.Sample = append(p.Sample, &Sample{
796 Value: value,
797 Location: sloc,
798 })
799
800 if l, err = r.ReadString('\n'); err != nil {
801 if err != io.EOF {
802 return nil, err
803 }
804 if l == "" {
805 break
806 }
807 }
808 }
809
810 if err = parseAdditionalSections(l, r, p); err != nil {
811 return nil, err
812 }
813
814 return p, nil
815 }
816
817 // parseContentionSample parses a single row from a contention profile
818 // into a new Sample.
819 func parseContentionSample(line string, period, cpuHz int64) (value []int64, add rs []uint64, err error) {
820 sampleData := contentionSampleRE.FindStringSubmatch(line)
821 if sampleData == nil {
822 return value, addrs, errUnrecognized
823 }
824
825 v1, err := strconv.ParseInt(sampleData[1], 10, 64)
826 if err != nil {
827 return value, addrs, fmt.Errorf("malformed sample: %s: %v", line , err)
828 }
829 v2, err := strconv.ParseInt(sampleData[2], 10, 64)
830 if err != nil {
831 return value, addrs, fmt.Errorf("malformed sample: %s: %v", line , err)
832 }
833
834 // Unsample values if period and cpuHz are available.
835 // - Delays are scaled to cycles and then to nanoseconds.
836 // - Contentions are scaled to cycles.
837 if period > 0 {
838 if cpuHz > 0 {
839 cpuGHz := float64(cpuHz) / 1e9
840 v1 = int64(float64(v1) * float64(period) / cpuGHz)
841 }
842 v2 = v2 * period
843 }
844
845 value = []int64{v2, v1}
846 addrs = parseHexAddresses(sampleData[3])
847
848 return value, addrs, nil
849 }
850
851 // parseThread parses a Threadz profile and returns a new Profile.
852 func parseThread(b []byte) (*Profile, error) {
853 r := bytes.NewBuffer(b)
854
855 var line string
856 var err error
857 for {
858 // Skip past comments and empty lines seeking a real header.
859 line, err = r.ReadString('\n')
860 if err != nil {
861 return nil, err
862 }
863 if !isSpaceOrComment(line) {
864 break
865 }
866 }
867
868 if m := threadzStartRE.FindStringSubmatch(line); m != nil {
869 // Advance over initial comments until first stack trace.
870 for {
871 line, err = r.ReadString('\n')
872 if err != nil {
873 if err != io.EOF {
874 return nil, err
875 }
876
877 if line == "" {
878 break
879 }
880 }
881 if sectionTrigger(line) != unrecognizedSection || line[0 ] == '-' {
882 break
883 }
884 }
885 } else if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 {
886 return nil, errUnrecognized
887 }
888
889 p := &Profile{
890 SampleType: []*ValueType{{Type: "thread", Unit: "count"}},
891 PeriodType: &ValueType{Type: "thread", Unit: "count"},
892 Period: 1,
893 }
894
895 locs := make(map[uint64]*Location)
896 // Recognize each thread and populate profile samples.
897 for sectionTrigger(line) == unrecognizedSection {
898 if strings.HasPrefix(line, "---- no stack trace for") {
899 line = ""
900 break
901 }
902 if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 {
903 return nil, errUnrecognized
904 }
905
906 var addrs []uint64
907 line, addrs, err = parseThreadSample(r)
908 if err != nil {
909 return nil, errUnrecognized
910 }
911 if len(addrs) == 0 {
912 // We got a --same as previous threads--. Bump counters.
913 if len(p.Sample) > 0 {
914 s := p.Sample[len(p.Sample)-1]
915 s.Value[0]++
916 }
917 continue
918 }
919
920 var sloc []*Location
921 for i, addr := range addrs {
922 // Addresses from stack traces point to the next instruc tion after
923 // each call. Adjust by -1 to land somewhere on the act ual call
924 // (except for the leaf, which is not a call).
925 if i > 0 {
926 addr--
927 }
928 loc := locs[addr]
929 if locs[addr] == nil {
930 loc = &Location{
931 Address: addr,
932 }
933 p.Location = append(p.Location, loc)
934 locs[addr] = loc
935 }
936 sloc = append(sloc, loc)
937 }
938
939 p.Sample = append(p.Sample, &Sample{
940 Value: []int64{1},
941 Location: sloc,
942 })
943 }
944
945 if err = parseAdditionalSections(line, r, p); err != nil {
946 return nil, err
947 }
948
949 return p, nil
950 }
951
952 // parseThreadSample parses a symbolized or unsymbolized stack trace.
953 // Returns the first line after the traceback, the sample (or nil if
954 // it hits a 'same-as-previous' marker) and an error.
955 func parseThreadSample(b *bytes.Buffer) (nextl string, addrs []uint64, err error ) {
956 var l string
957 sameAsPrevious := false
958 for {
959 if l, err = b.ReadString('\n'); err != nil {
960 if err != io.EOF {
961 return "", nil, err
962 }
963 if l == "" {
964 break
965 }
966 }
967 if l = strings.TrimSpace(l); l == "" {
968 continue
969 }
970
971 if strings.HasPrefix(l, "---") {
972 break
973 }
974 if strings.Contains(l, "same as previous thread") {
975 sameAsPrevious = true
976 continue
977 }
978
979 addrs = append(addrs, parseHexAddresses(l)...)
980 }
981
982 if sameAsPrevious {
983 return l, nil, nil
984 }
985 return l, addrs, nil
986 }
987
988 // parseAdditionalSections parses any additional sections in the
989 // profile, ignoring any unrecognized sections.
990 func parseAdditionalSections(l string, b *bytes.Buffer, p *Profile) (err error) {
991 for {
992 if sectionTrigger(l) == memoryMapSection {
993 break
994 }
995 // Ignore any unrecognized sections.
996 if l, err := b.ReadString('\n'); err != nil {
997 if err != io.EOF {
998 return err
999 }
1000 if l == "" {
1001 break
1002 }
1003 }
1004 }
1005 return p.ParseMemoryMap(b)
1006 }
1007
1008 // ParseMemoryMap parses a memory map in the format of
1009 // /proc/self/maps, and overrides the mappings in the current profile.
1010 // It renumbers the samples and locations in the profile correspondingly.
1011 func (p *Profile) ParseMemoryMap(rd io.Reader) error {
1012 b := bufio.NewReader(rd)
1013
1014 var attrs []string
1015 var r *strings.Replacer
1016 const delimiter = "="
1017 for {
1018 l, err := b.ReadString('\n')
1019 if err != nil {
1020 if err != io.EOF {
1021 return err
1022 }
1023 if l == "" {
1024 break
1025 }
1026 }
1027 if l = strings.TrimSpace(l); l == "" {
1028 continue
1029 }
1030
1031 if r != nil {
1032 l = r.Replace(l)
1033 }
1034 m, err := parseMappingEntry(l)
1035 if err != nil {
1036 if err == errUnrecognized {
1037 // Recognize assignments of the form: attr=value , and replace
1038 // $attr with value on subsequent mappings.
1039 if attr := strings.SplitN(l, delimiter, 2); len( attr) == 2 {
1040 attrs = append(attrs, "$"+strings.TrimSp ace(attr[0]), strings.TrimSpace(attr[1]))
1041 r = strings.NewReplacer(attrs...)
1042 }
1043 // Ignore any unrecognized entries
1044 continue
1045 }
1046 return err
1047 }
1048 if m == nil || (m.File == "" && len(p.Mapping) != 0) {
1049 // In some cases the first entry may include the address range
1050 // but not the name of the file. It should be followed b y
1051 // another entry with the name.
1052 continue
1053 }
1054 if len(p.Mapping) == 1 && p.Mapping[0].File == "" {
1055 // Update the name if this is the entry following that e mpty one.
1056 p.Mapping[0].File = m.File
1057 continue
1058 }
1059 p.Mapping = append(p.Mapping, m)
1060 }
1061 p.remapLocationIDs()
1062 p.remapFunctionIDs()
1063 p.remapMappingIDs()
1064 return nil
1065 }
1066
1067 func parseMappingEntry(l string) (*Mapping, error) {
1068 mapping := &Mapping{}
1069 var err error
1070 if me := procMapsRE.FindStringSubmatch(l); len(me) == 9 {
1071 if !strings.Contains(me[3], "x") {
1072 // Skip non-executable entries.
1073 return nil, nil
1074 }
1075 if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil {
1076 return nil, errUnrecognized
1077 }
1078 if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil {
1079 return nil, errUnrecognized
1080 }
1081 if me[4] != "" {
1082 if mapping.Offset, err = strconv.ParseUint(me[4], 16, 64 ); err != nil {
1083 return nil, errUnrecognized
1084 }
1085 }
1086 mapping.File = me[8]
1087 return mapping, nil
1088 }
1089
1090 if me := briefMapsRE.FindStringSubmatch(l); len(me) == 6 {
1091 if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil {
1092 return nil, errUnrecognized
1093 }
1094 if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil {
1095 return nil, errUnrecognized
1096 }
1097 mapping.File = me[3]
1098 if me[5] != "" {
1099 if mapping.Offset, err = strconv.ParseUint(me[5], 16, 64 ); err != nil {
1100 return nil, errUnrecognized
1101 }
1102 }
1103 return mapping, nil
1104 }
1105
1106 return nil, errUnrecognized
1107 }
1108
1109 type sectionType int
1110
1111 const (
1112 unrecognizedSection sectionType = iota
1113 memoryMapSection
1114 )
1115
1116 var memoryMapTriggers = []string{
1117 "--- Memory map: ---",
1118 "MAPPED_LIBRARIES:",
1119 }
1120
1121 func sectionTrigger(line string) sectionType {
1122 for _, trigger := range memoryMapTriggers {
1123 if strings.Contains(line, trigger) {
1124 return memoryMapSection
1125 }
1126 }
1127 return unrecognizedSection
1128 }
1129
1130 func (p *Profile) addLegacyFrameInfo() {
1131 switch {
1132 case isProfileType(p, heapzSampleTypes) ||
1133 isProfileType(p, heapzInUseSampleTypes) ||
1134 isProfileType(p, heapzAllocSampleTypes):
1135 p.DropFrames, p.KeepFrames = allocRxStr, allocSkipRxStr
1136 case isProfileType(p, contentionzSampleTypes):
1137 p.DropFrames, p.KeepFrames = lockRxStr, ""
1138 default:
1139 p.DropFrames, p.KeepFrames = cpuProfilerRxStr, ""
1140 }
1141 }
1142
1143 var heapzSampleTypes = []string{"allocations", "size"} // early Go pprof profile s
1144 var heapzInUseSampleTypes = []string{"inuse_objects", "inuse_space"}
1145 var heapzAllocSampleTypes = []string{"alloc_objects", "alloc_space"}
1146 var contentionzSampleTypes = []string{"contentions", "delay"}
1147
1148 func isProfileType(p *Profile, t []string) bool {
1149 st := p.SampleType
1150 if len(st) != len(t) {
1151 return false
1152 }
1153
1154 for i := range st {
1155 if st[i].Type != t[i] {
1156 return false
1157 }
1158 }
1159 return true
1160 }
1161
1162 var allocRxStr = strings.Join([]string{
1163 // POSIX entry points.
1164 `calloc`,
1165 `cfree`,
1166 `malloc`,
1167 `free`,
1168 `memalign`,
1169 `do_memalign`,
1170 `(__)?posix_memalign`,
1171 `pvalloc`,
1172 `valloc`,
1173 `realloc`,
1174
1175 // TC malloc.
1176 `tcmalloc::.*`,
1177 `tc_calloc`,
1178 `tc_cfree`,
1179 `tc_malloc`,
1180 `tc_free`,
1181 `tc_memalign`,
1182 `tc_posix_memalign`,
1183 `tc_pvalloc`,
1184 `tc_valloc`,
1185 `tc_realloc`,
1186 `tc_new`,
1187 `tc_delete`,
1188 `tc_newarray`,
1189 `tc_deletearray`,
1190 `tc_new_nothrow`,
1191 `tc_newarray_nothrow`,
1192
1193 // Memory-allocation routines on OS X.
1194 `malloc_zone_malloc`,
1195 `malloc_zone_calloc`,
1196 `malloc_zone_valloc`,
1197 `malloc_zone_realloc`,
1198 `malloc_zone_memalign`,
1199 `malloc_zone_free`,
1200
1201 // Go runtime
1202 `runtime\..*`,
1203
1204 // Other misc. memory allocation routines
1205 `BaseArena::.*`,
1206 `(::)?do_malloc_no_errno`,
1207 `(::)?do_malloc_pages`,
1208 `(::)?do_malloc`,
1209 `DoSampledAllocation`,
1210 `MallocedMemBlock::MallocedMemBlock`,
1211 `_M_allocate`,
1212 `__builtin_(vec_)?delete`,
1213 `__builtin_(vec_)?new`,
1214 `__gnu_cxx::new_allocator::allocate`,
1215 `__libc_malloc`,
1216 `__malloc_alloc_template::allocate`,
1217 `allocate`,
1218 `cpp_alloc`,
1219 `operator new(\[\])?`,
1220 `simple_alloc::allocate`,
1221 }, `|`)
1222
1223 var allocSkipRxStr = strings.Join([]string{
1224 // Preserve Go runtime frames that appear in the middle/bottom of
1225 // the stack.
1226 `runtime\.panic`,
1227 }, `|`)
1228
1229 var cpuProfilerRxStr = strings.Join([]string{
1230 `ProfileData::Add`,
1231 `ProfileData::prof_handler`,
1232 `CpuProfiler::prof_handler`,
1233 `__pthread_sighandler`,
1234 `__restore`,
1235 }, `|`)
1236
1237 var lockRxStr = strings.Join([]string{
1238 `RecordLockProfileData`,
1239 `(base::)?RecordLockProfileData.*`,
1240 `(base::)?SubmitMutexProfileData.*`,
1241 `(base::)?SubmitSpinLockProfileData.*`,
1242 `(Mutex::)?AwaitCommon.*`,
1243 `(Mutex::)?Unlock.*`,
1244 `(Mutex::)?UnlockSlow.*`,
1245 `(Mutex::)?ReaderUnlock.*`,
1246 `(MutexLock::)?~MutexLock.*`,
1247 `(SpinLock::)?Unlock.*`,
1248 `(SpinLock::)?SlowUnlock.*`,
1249 `(SpinLockHolder::)?~SpinLockHolder.*`,
1250 }, `|`)
LEFTRIGHT

Powered by Google App Engine
RSS Feeds Recent Issues | This issue
This is Rietveld f62528b