Left: | ||
Right: |
OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 | 2 |
3 """Tool for comparing the performance of two Python implementations. | 3 """Tool for comparing the performance of two Python implementations. |
4 | 4 |
5 Typical usage looks like | 5 Typical usage looks like |
6 | 6 |
7 ./perf.py -b 2to3,django control/python experiment/python | 7 ./perf.py -b 2to3,django control/python experiment/python |
8 | 8 |
9 This will run the 2to3 and Django template benchmarks, using `control/python` | 9 This will run the 2to3 and Django template benchmarks, using `control/python` |
10 as the baseline and `experiment/python` as the experiment. The --fast and | 10 as the baseline and `experiment/python` as the experiment. The --fast and |
(...skipping 427 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
438 | 438 |
439 Returns: | 439 Returns: |
440 A MemoryUsageResult object. | 440 A MemoryUsageResult object. |
441 """ | 441 """ |
442 max_base, max_changed = max(base_usage), max(changed_usage) | 442 max_base, max_changed = max(base_usage), max(changed_usage) |
443 delta_max = QuantityDelta(max_base, max_changed) | 443 delta_max = QuantityDelta(max_base, max_changed) |
444 | 444 |
445 if options.disable_timelines: | 445 if options.disable_timelines: |
446 chart_link = None | 446 chart_link = None |
447 else: | 447 else: |
448 chart_link = GetChart(SummarizeData(base_usage), | 448 points = 100 |
449 SummarizeData(changed_usage), | 449 if options.verbose: |
450 info("Downsampling %d memory samples to %d" | |
451 % (len(base_usage), points)) | |
452 chart_link = GetChart(SummarizeData(base_usage, points), | |
453 SummarizeData(changed_usage, points), | |
450 options) | 454 options) |
451 | 455 |
452 return MemoryUsageResult(max_base, max_changed, delta_max, chart_link) | 456 return MemoryUsageResult(max_base, max_changed, delta_max, chart_link) |
453 | 457 |
454 | 458 |
455 ### Utility functions | 459 ### Utility functions |
456 | 460 |
457 def SimpleBenchmark(benchmark_function, base_python, changed_python, options, | 461 def SimpleBenchmark(benchmark_function, base_python, changed_python, options, |
458 *args, **kwargs): | 462 *args, **kwargs): |
459 """Abstract out the body for most simple benchmarks. | 463 """Abstract out the body for most simple benchmarks. |
(...skipping 24 matching lines...) Expand all Loading... | |
484 *args, **kwargs) | 488 *args, **kwargs) |
485 except subprocess.CalledProcessError, e: | 489 except subprocess.CalledProcessError, e: |
486 return BenchmarkError(e) | 490 return BenchmarkError(e) |
487 | 491 |
488 return CompareBenchmarkData(base_data, changed_data, options) | 492 return CompareBenchmarkData(base_data, changed_data, options) |
489 | 493 |
490 | 494 |
491 def _FormatData(num): | 495 def _FormatData(num): |
492 return str(round(num, 2)) | 496 return str(round(num, 2)) |
493 | 497 |
494 def GetChart(base_data, changed_data, options, chart_margin=100): | 498 def GetChart(base_data, changed_data, options): |
495 """Build a Google Chart API URL for the given data. | 499 """Build a Google Chart API URL for the given data. |
496 | 500 |
497 Args: | 501 Args: |
498 base_data: data points for the base binary. | 502 base_data: data points for the base binary. |
499 changed_data: data points for the changed binary. | 503 changed_data: data points for the changed binary. |
500 options: optparse.Values instance. | 504 options: optparse.Values instance. |
501 chart_margin: optional integer margin to add/sub from the max/min. | |
502 | 505 |
503 Returns: | 506 Returns: |
504 Google Chart API URL as a string; or None, if options.disable_timelines | 507 Google Chart API URL as a string; or None, if options.disable_timelines |
505 is true. | 508 is true. |
506 """ | 509 """ |
507 if options.disable_timelines: | 510 if options.disable_timelines: |
508 return None | 511 return None |
509 # We use these to scale the graph. | 512 # We use these to scale the graph. |
510 max_data = max(max(base_data), max(changed_data)) + chart_margin | 513 max_data = max(max(base_data), max(changed_data)) |
511 min_data = min(min(base_data), min(changed_data)) - chart_margin | 514 min_data = min(min(base_data), min(changed_data)) |
512 if min_data < 0: | 515 # Add a 10% margin, top and bottom. |
Jeffrey Yasskin
2010/01/27 23:17:45
10% should be of the min->max range, not their abs
| |
516 max_data += max_data * 0.10 | |
517 min_data -= min_data * 0.10 | |
518 if min_data < 10: # Give a nicer Y axis. | |
Jeffrey Yasskin
2010/01/27 23:17:45
This seems like it should be based on a comparison
| |
513 min_data = 0 | 519 min_data = 0 |
514 # Google-bound data, formatted as desired by the Chart API. | 520 # Google-bound data, formatted as desired by the Chart API. |
515 data_for_google = (",".join(map(_FormatData, base_data)) + "|" + | 521 data_for_google = (",".join(map(_FormatData, base_data)) + "|" + |
516 ",".join(map(_FormatData, changed_data))) | 522 ",".join(map(_FormatData, changed_data))) |
517 | 523 |
518 # Come up with labels for the X axis; not too many, though, or they'll be | 524 # Come up with labels for the X axis; not too many, though, or they'll be |
519 # unreadable. | 525 # unreadable. |
520 max_len = max(len(base_data), len(changed_data)) | 526 max_len = max(len(base_data), len(changed_data)) |
521 if max_len <= 20: | 527 if max_len <= 20: |
522 points = range(1, max_len + 1) | 528 points = range(1, max_len + 1) |
(...skipping 195 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
718 # With only one data point, we can't do any of the interesting stats | 724 # With only one data point, we can't do any of the interesting stats |
719 # below. | 725 # below. |
720 base_time, changed_time = base_times[0], changed_times[0] | 726 base_time, changed_time = base_times[0], changed_times[0] |
721 time_delta = TimeDelta(base_time, changed_time) | 727 time_delta = TimeDelta(base_time, changed_time) |
722 return SimpleBenchmarkResult(base_time, changed_time, time_delta) | 728 return SimpleBenchmarkResult(base_time, changed_time, time_delta) |
723 | 729 |
724 # Create a chart showing iteration times over time. We round the times so | 730 # Create a chart showing iteration times over time. We round the times so |
725 # as not to exceed the GET limit for Google's chart server. | 731 # as not to exceed the GET limit for Google's chart server. |
726 timeline_link = GetChart(SummarizeData(base_times), | 732 timeline_link = GetChart(SummarizeData(base_times), |
727 SummarizeData(changed_times), | 733 SummarizeData(changed_times), |
728 options, chart_margin=1) | 734 options) |
729 | 735 |
730 base_times = sorted(base_times) | 736 base_times = sorted(base_times) |
731 changed_times = sorted(changed_times) | 737 changed_times = sorted(changed_times) |
732 | 738 |
733 min_base, min_changed = base_times[0], changed_times[0] | 739 min_base, min_changed = base_times[0], changed_times[0] |
734 avg_base, avg_changed = avg(base_times), avg(changed_times) | 740 avg_base, avg_changed = avg(base_times), avg(changed_times) |
735 std_base = SampleStdDev(base_times) | 741 std_base = SampleStdDev(base_times) |
736 std_changed = SampleStdDev(changed_times) | 742 std_changed = SampleStdDev(changed_times) |
737 delta_min = TimeDelta(min_base, min_changed) | 743 delta_min = TimeDelta(min_base, min_changed) |
738 delta_avg = TimeDelta(avg_base, avg_changed) | 744 delta_avg = TimeDelta(avg_base, avg_changed) |
(...skipping 1032 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1771 if multiprocessing: | 1777 if multiprocessing: |
1772 print "Total CPU cores:", multiprocessing.cpu_count() | 1778 print "Total CPU cores:", multiprocessing.cpu_count() |
1773 for name, result in results: | 1779 for name, result in results: |
1774 print | 1780 print |
1775 print "###", name, "###" | 1781 print "###", name, "###" |
1776 print result.string_representation() | 1782 print result.string_representation() |
1777 return results | 1783 return results |
1778 | 1784 |
1779 if __name__ == "__main__": | 1785 if __name__ == "__main__": |
1780 main(sys.argv[1:]) | 1786 main(sys.argv[1:]) |
OLD | NEW |