mirror of
https://github.com/google/benchmark.git
synced 2025-04-17 23:12:30 +08:00
[Tooling] 'display aggregates only' support. (#674)
Works as you'd expect, both for the benchmark binaries, and
the jsons (naturally, since the tests need to work):
```
$ ~/src/googlebenchmark/tools/compare.py benchmarks ~/rawspeed/build-{0,1}/src/utilities/rsbench/rsbench --benchmark_repetitions=3 *
RUNNING: /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench --benchmark_repetitions=3 2K4A9927.CR2 2K4A9928.CR2 2K4A9929.CR2 --benchmark_out=/tmp/tmpYoui5H
2018-09-12 20:23:47
Running /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench
Run on (8 X 4000 MHz CPU s)
CPU Caches:
L1 Data 16K (x8)
L1 Instruction 64K (x4)
L2 Unified 2048K (x4)
L3 Unified 8192K (x1)
-----------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations UserCounters...
-----------------------------------------------------------------------------------------------
2K4A9927.CR2/threads:8/real_time 447 ms 447 ms 2 CPUTime,s=0.447302 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=117.738M Pixels/WallTime=117.738M Raws/CPUTime=2.23562 Raws/WallTime=2.23564 WallTime,s=0.4473
2K4A9927.CR2/threads:8/real_time 444 ms 444 ms 2 CPUTime,s=0.444228 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=118.552M Pixels/WallTime=118.553M Raws/CPUTime=2.2511 Raws/WallTime=2.25111 WallTime,s=0.444226
2K4A9927.CR2/threads:8/real_time 447 ms 447 ms 2 CPUTime,s=0.446983 CPUTime/WallTime=0.999999 Pixels=52.6643M Pixels/CPUTime=117.822M Pixels/WallTime=117.822M Raws/CPUTime=2.23722 Raws/WallTime=2.23722 WallTime,s=0.446984
2K4A9927.CR2/threads:8/real_time_mean 446 ms 446 ms 2 CPUTime,s=0.446171 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.037M Pixels/WallTime=118.038M Raws/CPUTime=2.24131 Raws/WallTime=2.24132 WallTime,s=0.44617
2K4A9927.CR2/threads:8/real_time_median 447 ms 447 ms 2 CPUTime,s=0.446983 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=117.822M Pixels/WallTime=117.822M Raws/CPUTime=2.23722 Raws/WallTime=2.23722 WallTime,s=0.446984
2K4A9927.CR2/threads:8/real_time_stddev 2 ms 2 ms 2 CPUTime,s=1.69052m CPUTime/WallTime=3.53737u Pixels=0 Pixels/CPUTime=448.178k Pixels/WallTime=448.336k Raws/CPUTime=8.51008m Raws/WallTime=8.51308m WallTime,s=1.6911m
2K4A9928.CR2/threads:8/real_time 563 ms 563 ms 1 CPUTime,s=0.562511 CPUTime/WallTime=0.999824 Pixels=27.9936M Pixels/CPUTime=49.7654M Pixels/WallTime=49.7567M Raws/CPUTime=1.77774 Raws/WallTime=1.77743 WallTime,s=0.56261
2K4A9928.CR2/threads:8/real_time 561 ms 561 ms 1 CPUTime,s=0.561328 CPUTime/WallTime=0.999917 Pixels=27.9936M Pixels/CPUTime=49.8703M Pixels/WallTime=49.8662M Raws/CPUTime=1.78149 Raws/WallTime=1.78134 WallTime,s=0.561375
2K4A9928.CR2/threads:8/real_time 570 ms 570 ms 1 CPUTime,s=0.570423 CPUTime/WallTime=0.999876 Pixels=27.9936M Pixels/CPUTime=49.0752M Pixels/WallTime=49.0691M Raws/CPUTime=1.75308 Raws/WallTime=1.75287 WallTime,s=0.570493
2K4A9928.CR2/threads:8/real_time_mean 565 ms 565 ms 1 CPUTime,s=0.564754 CPUTime/WallTime=0.999872 Pixels=27.9936M Pixels/CPUTime=49.5703M Pixels/WallTime=49.564M Raws/CPUTime=1.77077 Raws/WallTime=1.77055 WallTime,s=0.564826
2K4A9928.CR2/threads:8/real_time_median 563 ms 563 ms 1 CPUTime,s=0.562511 CPUTime/WallTime=0.999876 Pixels=27.9936M Pixels/CPUTime=49.7654M Pixels/WallTime=49.7567M Raws/CPUTime=1.77774 Raws/WallTime=1.77743 WallTime,s=0.56261
2K4A9928.CR2/threads:8/real_time_stddev 5 ms 5 ms 1 CPUTime,s=4.945m CPUTime/WallTime=46.3459u Pixels=0 Pixels/CPUTime=431.997k Pixels/WallTime=432.061k Raws/CPUTime=0.015432 Raws/WallTime=0.0154343 WallTime,s=4.94686m
2K4A9929.CR2/threads:8/real_time 306 ms 306 ms 2 CPUTime,s=0.306476 CPUTime/WallTime=0.999961 Pixels=12.4416M Pixels/CPUTime=40.5957M Pixels/WallTime=40.5941M Raws/CPUTime=3.2629 Raws/WallTime=3.26277 WallTime,s=0.306488
2K4A9929.CR2/threads:8/real_time 310 ms 310 ms 2 CPUTime,s=0.309978 CPUTime/WallTime=0.999939 Pixels=12.4416M Pixels/CPUTime=40.1371M Pixels/WallTime=40.1347M Raws/CPUTime=3.22604 Raws/WallTime=3.22584 WallTime,s=0.309996
2K4A9929.CR2/threads:8/real_time 309 ms 309 ms 2 CPUTime,s=0.30943 CPUTime/WallTime=0.999987 Pixels=12.4416M Pixels/CPUTime=40.2081M Pixels/WallTime=40.2076M Raws/CPUTime=3.23175 Raws/WallTime=3.23171 WallTime,s=0.309434
2K4A9929.CR2/threads:8/real_time_mean 309 ms 309 ms 2 CPUTime,s=0.308628 CPUTime/WallTime=0.999962 Pixels=12.4416M Pixels/CPUTime=40.3136M Pixels/WallTime=40.3121M Raws/CPUTime=3.24023 Raws/WallTime=3.24011 WallTime,s=0.308639
2K4A9929.CR2/threads:8/real_time_median 309 ms 309 ms 2 CPUTime,s=0.30943 CPUTime/WallTime=0.999961 Pixels=12.4416M Pixels/CPUTime=40.2081M Pixels/WallTime=40.2076M Raws/CPUTime=3.23175 Raws/WallTime=3.23171 WallTime,s=0.309434
2K4A9929.CR2/threads:8/real_time_stddev 2 ms 2 ms 2 CPUTime,s=1.88354m CPUTime/WallTime=23.7788u Pixels=0 Pixels/CPUTime=246.821k Pixels/WallTime=246.914k Raws/CPUTime=0.0198384 Raws/WallTime=0.0198458 WallTime,s=1.88442m
RUNNING: /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench --benchmark_repetitions=3 2K4A9927.CR2 2K4A9928.CR2 2K4A9929.CR2 --benchmark_out=/tmp/tmpRShmmf
2018-09-12 20:23:55
Running /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench
Run on (8 X 4000 MHz CPU s)
CPU Caches:
L1 Data 16K (x8)
L1 Instruction 64K (x4)
L2 Unified 2048K (x4)
L3 Unified 8192K (x1)
-----------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations UserCounters...
-----------------------------------------------------------------------------------------------
2K4A9927.CR2/threads:8/real_time 446 ms 446 ms 2 CPUTime,s=0.445589 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.19M Pixels/WallTime=118.19M Raws/CPUTime=2.24422 Raws/WallTime=2.24422 WallTime,s=0.445589
2K4A9927.CR2/threads:8/real_time 446 ms 446 ms 2 CPUTime,s=0.446008 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=118.079M Pixels/WallTime=118.08M Raws/CPUTime=2.24211 Raws/WallTime=2.24213 WallTime,s=0.446005
2K4A9927.CR2/threads:8/real_time 448 ms 448 ms 2 CPUTime,s=0.447763 CPUTime/WallTime=0.999994 Pixels=52.6643M Pixels/CPUTime=117.616M Pixels/WallTime=117.616M Raws/CPUTime=2.23332 Raws/WallTime=2.23331 WallTime,s=0.447766
2K4A9927.CR2/threads:8/real_time_mean 446 ms 446 ms 2 CPUTime,s=0.446453 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=117.962M Pixels/WallTime=117.962M Raws/CPUTime=2.23988 Raws/WallTime=2.23989 WallTime,s=0.446453
2K4A9927.CR2/threads:8/real_time_median 446 ms 446 ms 2 CPUTime,s=0.446008 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.079M Pixels/WallTime=118.08M Raws/CPUTime=2.24211 Raws/WallTime=2.24213 WallTime,s=0.446005
2K4A9927.CR2/threads:8/real_time_stddev 1 ms 1 ms 2 CPUTime,s=1.15367m CPUTime/WallTime=6.48501u Pixels=0 Pixels/CPUTime=304.437k Pixels/WallTime=305.025k Raws/CPUTime=5.7807m Raws/WallTime=5.79188m WallTime,s=1.15591m
2K4A9928.CR2/threads:8/real_time 561 ms 561 ms 1 CPUTime,s=0.560856 CPUTime/WallTime=0.999996 Pixels=27.9936M Pixels/CPUTime=49.9123M Pixels/WallTime=49.9121M Raws/CPUTime=1.78299 Raws/WallTime=1.78298 WallTime,s=0.560858
2K4A9928.CR2/threads:8/real_time 560 ms 560 ms 1 CPUTime,s=0.560023 CPUTime/WallTime=1.00001 Pixels=27.9936M Pixels/CPUTime=49.9865M Pixels/WallTime=49.9872M Raws/CPUTime=1.78564 Raws/WallTime=1.78567 WallTime,s=0.560015
2K4A9928.CR2/threads:8/real_time 562 ms 562 ms 1 CPUTime,s=0.562337 CPUTime/WallTime=0.999994 Pixels=27.9936M Pixels/CPUTime=49.7808M Pixels/WallTime=49.7805M Raws/CPUTime=1.77829 Raws/WallTime=1.77828 WallTime,s=0.56234
2K4A9928.CR2/threads:8/real_time_mean 561 ms 561 ms 1 CPUTime,s=0.561072 CPUTime/WallTime=1 Pixels=27.9936M Pixels/CPUTime=49.8932M Pixels/WallTime=49.8933M Raws/CPUTime=1.78231 Raws/WallTime=1.78231 WallTime,s=0.561071
2K4A9928.CR2/threads:8/real_time_median 561 ms 561 ms 1 CPUTime,s=0.560856 CPUTime/WallTime=0.999996 Pixels=27.9936M Pixels/CPUTime=49.9123M Pixels/WallTime=49.9121M Raws/CPUTime=1.78299 Raws/WallTime=1.78298 WallTime,s=0.560858
2K4A9928.CR2/threads:8/real_time_stddev 1 ms 1 ms 1 CPUTime,s=1.17202m CPUTime/WallTime=10.7929u Pixels=0 Pixels/CPUTime=104.164k Pixels/WallTime=104.612k Raws/CPUTime=3.721m Raws/WallTime=3.73701m WallTime,s=1.17706m
2K4A9929.CR2/threads:8/real_time 305 ms 305 ms 2 CPUTime,s=0.305436 CPUTime/WallTime=0.999926 Pixels=12.4416M Pixels/CPUTime=40.7339M Pixels/WallTime=40.7309M Raws/CPUTime=3.27401 Raws/WallTime=3.27376 WallTime,s=0.305459
2K4A9929.CR2/threads:8/real_time 306 ms 306 ms 2 CPUTime,s=0.30576 CPUTime/WallTime=0.999999 Pixels=12.4416M Pixels/CPUTime=40.6908M Pixels/WallTime=40.6908M Raws/CPUTime=3.27054 Raws/WallTime=3.27054 WallTime,s=0.30576
2K4A9929.CR2/threads:8/real_time 307 ms 307 ms 2 CPUTime,s=0.30724 CPUTime/WallTime=0.999991 Pixels=12.4416M Pixels/CPUTime=40.4947M Pixels/WallTime=40.4944M Raws/CPUTime=3.25478 Raws/WallTime=3.25475 WallTime,s=0.307243
2K4A9929.CR2/threads:8/real_time_mean 306 ms 306 ms 2 CPUTime,s=0.306145 CPUTime/WallTime=0.999972 Pixels=12.4416M Pixels/CPUTime=40.6398M Pixels/WallTime=40.6387M Raws/CPUTime=3.26645 Raws/WallTime=3.26635 WallTime,s=0.306154
2K4A9929.CR2/threads:8/real_time_median 306 ms 306 ms 2 CPUTime,s=0.30576 CPUTime/WallTime=0.999991 Pixels=12.4416M Pixels/CPUTime=40.6908M Pixels/WallTime=40.6908M Raws/CPUTime=3.27054 Raws/WallTime=3.27054 WallTime,s=0.30576
2K4A9929.CR2/threads:8/real_time_stddev 1 ms 1 ms 2 CPUTime,s=961.851u CPUTime/WallTime=40.2708u Pixels=0 Pixels/CPUTime=127.481k Pixels/WallTime=126.577k Raws/CPUTime=0.0102463 Raws/WallTime=0.0101737 WallTime,s=955.105u
Comparing /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench to /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench
Benchmark Time CPU Time Old Time New CPU Old CPU New
--------------------------------------------------------------------------------------------------------------------------------------
2K4A9927.CR2/threads:8/real_time -0.0038 -0.0038 447 446 447 446
2K4A9927.CR2/threads:8/real_time +0.0031 +0.0031 444 446 444 446
2K4A9927.CR2/threads:8/real_time -0.0031 -0.0031 447 446 447 446
2K4A9927.CR2/threads:8/real_time_pvalue 0.6428 0.6428 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended.
2K4A9927.CR2/threads:8/real_time_mean +0.0006 +0.0006 446 446 446 446
2K4A9927.CR2/threads:8/real_time_median -0.0022 -0.0022 447 446 447 446
2K4A9927.CR2/threads:8/real_time_stddev -0.3161 -0.3175 2 1 2 1
2K4A9928.CR2/threads:8/real_time -0.0031 -0.0029 563 561 563 561
2K4A9928.CR2/threads:8/real_time -0.0009 -0.0008 561 561 561 561
2K4A9928.CR2/threads:8/real_time -0.0169 -0.0168 570 561 570 561
2K4A9928.CR2/threads:8/real_time_pvalue 0.0636 0.0636 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended.
2K4A9928.CR2/threads:8/real_time_mean -0.0066 -0.0065 565 561 565 561
2K4A9928.CR2/threads:8/real_time_median -0.0031 -0.0029 563 561 563 561
2K4A9928.CR2/threads:8/real_time_stddev -0.7620 -0.7630 5 1 5 1
2K4A9929.CR2/threads:8/real_time -0.0034 -0.0034 306 305 306 305
2K4A9929.CR2/threads:8/real_time -0.0146 -0.0146 310 305 310 305
2K4A9929.CR2/threads:8/real_time -0.0128 -0.0129 309 305 309 305
2K4A9929.CR2/threads:8/real_time_pvalue 0.0636 0.0636 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended.
2K4A9929.CR2/threads:8/real_time_mean -0.0081 -0.0080 309 306 309 306
2K4A9929.CR2/threads:8/real_time_median -0.0119 -0.0119 309 306 309 306
2K4A9929.CR2/threads:8/real_time_stddev -0.4931 -0.4894 2 1 2 1
$ ~/src/googlebenchmark/tools/compare.py -a benchmarks ~/rawspeed/build-{0,1}/src/utilities/rsbench/rsbench --benchmark_repetitions=3 *
RUNNING: /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench --benchmark_repetitions=3 2K4A9927.CR2 2K4A9928.CR2 2K4A9929.CR2 --benchmark_display_aggregates_only=true --benchmark_out=/tmp/tmpjrD2I0
2018-09-12 20:24:11
Running /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench
Run on (8 X 4000 MHz CPU s)
CPU Caches:
L1 Data 16K (x8)
L1 Instruction 64K (x4)
L2 Unified 2048K (x4)
L3 Unified 8192K (x1)
-----------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations UserCounters...
-----------------------------------------------------------------------------------------------
2K4A9927.CR2/threads:8/real_time_mean 446 ms 446 ms 2 CPUTime,s=0.446077 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.061M Pixels/WallTime=118.062M Raws/CPUTime=2.24177 Raws/WallTime=2.24178 WallTime,s=0.446076
2K4A9927.CR2/threads:8/real_time_median 446 ms 446 ms 2 CPUTime,s=0.445676 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.167M Pixels/WallTime=118.168M Raws/CPUTime=2.24378 Raws/WallTime=2.2438 WallTime,s=0.445673
2K4A9927.CR2/threads:8/real_time_stddev 1 ms 1 ms 2 CPUTime,s=820.275u CPUTime/WallTime=3.51513u Pixels=0 Pixels/CPUTime=216.876k Pixels/WallTime=217.229k Raws/CPUTime=4.11809m Raws/WallTime=4.12478m WallTime,s=821.607u
2K4A9928.CR2/threads:8/real_time_mean 562 ms 562 ms 1 CPUTime,s=0.561843 CPUTime/WallTime=0.999983 Pixels=27.9936M Pixels/CPUTime=49.8247M Pixels/WallTime=49.8238M Raws/CPUTime=1.77986 Raws/WallTime=1.77983 WallTime,s=0.561852
2K4A9928.CR2/threads:8/real_time_median 562 ms 562 ms 1 CPUTime,s=0.561745 CPUTime/WallTime=0.999995 Pixels=27.9936M Pixels/CPUTime=49.8333M Pixels/WallTime=49.8307M Raws/CPUTime=1.78017 Raws/WallTime=1.78008 WallTime,s=0.561774
2K4A9928.CR2/threads:8/real_time_stddev 1 ms 1 ms 1 CPUTime,s=788.58u CPUTime/WallTime=30.2578u Pixels=0 Pixels/CPUTime=69.914k Pixels/WallTime=69.4978k Raws/CPUTime=2.4975m Raws/WallTime=2.48263m WallTime,s=783.873u
2K4A9929.CR2/threads:8/real_time_mean 306 ms 306 ms 2 CPUTime,s=0.305718 CPUTime/WallTime=1.00001 Pixels=12.4416M Pixels/CPUTime=40.6964M Pixels/WallTime=40.6967M Raws/CPUTime=3.271 Raws/WallTime=3.27102 WallTime,s=0.305716
2K4A9929.CR2/threads:8/real_time_median 306 ms 306 ms 2 CPUTime,s=0.305737 CPUTime/WallTime=1 Pixels=12.4416M Pixels/CPUTime=40.6939M Pixels/WallTime=40.6945M Raws/CPUTime=3.27079 Raws/WallTime=3.27084 WallTime,s=0.305732
2K4A9929.CR2/threads:8/real_time_stddev 1 ms 1 ms 2 CPUTime,s=584.969u CPUTime/WallTime=7.87539u Pixels=0 Pixels/CPUTime=77.8769k Pixels/WallTime=77.9118k Raws/CPUTime=6.2594m Raws/WallTime=6.2622m WallTime,s=585.232u
RUNNING: /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench --benchmark_repetitions=3 2K4A9927.CR2 2K4A9928.CR2 2K4A9929.CR2 --benchmark_display_aggregates_only=true --benchmark_out=/tmp/tmpN9Xhk2
2018-09-12 20:24:18
Running /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench
Run on (8 X 4000 MHz CPU s)
CPU Caches:
L1 Data 16K (x8)
L1 Instruction 64K (x4)
L2 Unified 2048K (x4)
L3 Unified 8192K (x1)
-----------------------------------------------------------------------------------------------
Benchmark Time CPU Iterations UserCounters...
-----------------------------------------------------------------------------------------------
2K4A9927.CR2/threads:8/real_time_mean 446 ms 446 ms 2 CPUTime,s=0.445771 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=118.142M Pixels/WallTime=118.143M Raws/CPUTime=2.2433 Raws/WallTime=2.24332 WallTime,s=0.445769
2K4A9927.CR2/threads:8/real_time_median 446 ms 446 ms 2 CPUTime,s=0.445719 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=118.156M Pixels/WallTime=118.156M Raws/CPUTime=2.24356 Raws/WallTime=2.24358 WallTime,s=0.445717
2K4A9927.CR2/threads:8/real_time_stddev 0 ms 0 ms 2 CPUTime,s=184.23u CPUTime/WallTime=1.69441u Pixels=0 Pixels/CPUTime=48.8185k Pixels/WallTime=49.0086k Raws/CPUTime=926.975u Raws/WallTime=930.585u WallTime,s=184.946u
2K4A9928.CR2/threads:8/real_time_mean 562 ms 562 ms 1 CPUTime,s=0.562071 CPUTime/WallTime=0.999969 Pixels=27.9936M Pixels/CPUTime=49.8045M Pixels/WallTime=49.803M Raws/CPUTime=1.77914 Raws/WallTime=1.77908 WallTime,s=0.562088
2K4A9928.CR2/threads:8/real_time_median 562 ms 562 ms 1 CPUTime,s=0.56237 CPUTime/WallTime=0.999986 Pixels=27.9936M Pixels/CPUTime=49.7779M Pixels/WallTime=49.7772M Raws/CPUTime=1.77819 Raws/WallTime=1.77816 WallTime,s=0.562378
2K4A9928.CR2/threads:8/real_time_stddev 1 ms 1 ms 1 CPUTime,s=967.38u CPUTime/WallTime=35.0024u Pixels=0 Pixels/CPUTime=85.7806k Pixels/WallTime=84.0545k Raws/CPUTime=3.06429m Raws/WallTime=3.00263m WallTime,s=947.993u
2K4A9929.CR2/threads:8/real_time_mean 307 ms 307 ms 2 CPUTime,s=0.306511 CPUTime/WallTime=0.999995 Pixels=12.4416M Pixels/CPUTime=40.5924M Pixels/WallTime=40.5922M Raws/CPUTime=3.26264 Raws/WallTime=3.26262 WallTime,s=0.306513
2K4A9929.CR2/threads:8/real_time_median 306 ms 306 ms 2 CPUTime,s=0.306169 CPUTime/WallTime=0.999999 Pixels=12.4416M Pixels/CPUTime=40.6364M Pixels/WallTime=40.637M Raws/CPUTime=3.26618 Raws/WallTime=3.26622 WallTime,s=0.306164
2K4A9929.CR2/threads:8/real_time_stddev 2 ms 2 ms 2 CPUTime,s=2.25763m CPUTime/WallTime=21.4306u Pixels=0 Pixels/CPUTime=298.503k Pixels/WallTime=299.126k Raws/CPUTime=0.0239924 Raws/WallTime=0.0240424 WallTime,s=2.26242m
Comparing /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench to /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench
Benchmark Time CPU Time Old Time New CPU Old CPU New
--------------------------------------------------------------------------------------------------------------------------------------
2K4A9927.CR2/threads:8/real_time_pvalue 0.6428 0.6428 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended.
2K4A9927.CR2/threads:8/real_time_mean -0.0007 -0.0007 446 446 446 446
2K4A9927.CR2/threads:8/real_time_median +0.0001 +0.0001 446 446 446 446
2K4A9927.CR2/threads:8/real_time_stddev -0.7746 -0.7749 1 0 1 0
2K4A9928.CR2/threads:8/real_time_pvalue 0.0636 0.0636 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended.
2K4A9928.CR2/threads:8/real_time_mean +0.0004 +0.0004 562 562 562 562
2K4A9928.CR2/threads:8/real_time_median +0.0011 +0.0011 562 562 562 562
2K4A9928.CR2/threads:8/real_time_stddev +0.2091 +0.2270 1 1 1 1
2K4A9929.CR2/threads:8/real_time_pvalue 0.0636 0.0636 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended.
2K4A9929.CR2/threads:8/real_time_mean +0.0026 +0.0026 306 307 306 307
2K4A9929.CR2/threads:8/real_time_median +0.0014 +0.0014 306 306 306 306
2K4A9929.CR2/threads:8/real_time_stddev +2.8652 +2.8585 1 2 1 2
```
This commit is contained in:
Roman Lebedev
GitHub
parent
1b44120cd1
commit
a5e9c061d9
@ -6,6 +6,15 @@ The `compare.py` can be used to compare the result of benchmarks.
|
||||
|
||||
**NOTE**: the utility relies on the scipy package which can be installed using [these instructions](https://www.scipy.org/install.html).
|
||||
|
||||
### Displaying aggregates only
|
||||
|
||||
The switch `-a` / `--display_aggregates_only` can be used to control the
|
||||
displayment of the normal iterations vs the aggregates. When passed, it will
|
||||
be passthrough to the benchmark binaries to be run, and will be accounted for
|
||||
in the tool itself; only the aggregates will be displayed, but not normal runs.
|
||||
It only affects the display, the separate runs will still be used to calculate
|
||||
the U test.
|
||||
|
||||
### Modes of operation
|
||||
|
||||
There are three modes of operation:
|
||||
|
||||
@ -36,6 +36,17 @@ def create_parser():
|
||||
parser = ArgumentParser(
|
||||
description='versatile benchmark output compare tool')
|
||||
|
||||
parser.add_argument(
|
||||
'-a',
|
||||
'--display_aggregates_only',
|
||||
dest='display_aggregates_only',
|
||||
action="store_true",
|
||||
help="If there are repetitions, by default, we display everything - the"
|
||||
" actual runs, and the aggregates computed. Sometimes, it is "
|
||||
"desirable to only view the aggregates. E.g. when there are a lot "
|
||||
"of repetitions. Do note that only the display is affected. "
|
||||
"Internally, all the actual runs are still used, e.g. for U test.")
|
||||
|
||||
utest = parser.add_argument_group()
|
||||
utest.add_argument(
|
||||
'--no-utest',
|
||||
@ -200,6 +211,9 @@ def main():
|
||||
|
||||
check_inputs(test_baseline, test_contender, benchmark_options)
|
||||
|
||||
if args.display_aggregates_only:
|
||||
benchmark_options += ['--benchmark_display_aggregates_only=true']
|
||||
|
||||
options_baseline = []
|
||||
options_contender = []
|
||||
|
||||
@ -223,7 +237,8 @@ def main():
|
||||
|
||||
# Diff and output
|
||||
output_lines = gbench.report.generate_difference_report(
|
||||
json1, json2, args.utest, args.utest_alpha)
|
||||
json1, json2, args.display_aggregates_only,
|
||||
args.utest, args.utest_alpha)
|
||||
print(description)
|
||||
for ln in output_lines:
|
||||
print(ln)
|
||||
@ -246,6 +261,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_benchmarks_basic(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarks', self.testInput0, self.testInput1])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
@ -255,6 +271,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_benchmarks_basic_without_utest(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['--no-utest', 'benchmarks', self.testInput0, self.testInput1])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertFalse(parsed.utest)
|
||||
self.assertEqual(parsed.utest_alpha, 0.05)
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
@ -262,9 +279,20 @@ class TestParser(unittest.TestCase):
|
||||
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
||||
self.assertFalse(parsed.benchmark_options)
|
||||
|
||||
def test_benchmarks_basic_display_aggregates_only(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['-a', 'benchmarks', self.testInput0, self.testInput1])
|
||||
self.assertTrue(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
||||
self.assertFalse(parsed.benchmark_options)
|
||||
|
||||
def test_benchmarks_basic_with_utest_alpha(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.utest_alpha, 0.314)
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
@ -275,6 +303,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_benchmarks_basic_without_utest_with_utest_alpha(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['--no-utest', '--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertFalse(parsed.utest)
|
||||
self.assertEqual(parsed.utest_alpha, 0.314)
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
@ -285,6 +314,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_benchmarks_with_remainder(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarks', self.testInput0, self.testInput1, 'd'])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
@ -294,6 +324,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_benchmarks_with_remainder_after_doubleminus(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarks', self.testInput0, self.testInput1, '--', 'e'])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'benchmarks')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
@ -303,6 +334,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_filters_basic(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['filters', self.testInput0, 'c', 'd'])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'filters')
|
||||
self.assertEqual(parsed.test[0].name, self.testInput0)
|
||||
@ -313,6 +345,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_filters_with_remainder(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['filters', self.testInput0, 'c', 'd', 'e'])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'filters')
|
||||
self.assertEqual(parsed.test[0].name, self.testInput0)
|
||||
@ -323,6 +356,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_filters_with_remainder_after_doubleminus(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['filters', self.testInput0, 'c', 'd', '--', 'f'])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'filters')
|
||||
self.assertEqual(parsed.test[0].name, self.testInput0)
|
||||
@ -333,6 +367,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_benchmarksfiltered_basic(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e'])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'benchmarksfiltered')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
@ -344,6 +379,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_benchmarksfiltered_with_remainder(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f'])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'benchmarksfiltered')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
@ -355,6 +391,7 @@ class TestParser(unittest.TestCase):
|
||||
def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
|
||||
parsed = self.parser.parse_args(
|
||||
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g'])
|
||||
self.assertFalse(parsed.display_aggregates_only)
|
||||
self.assertTrue(parsed.utest)
|
||||
self.assertEqual(parsed.mode, 'benchmarksfiltered')
|
||||
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
||||
@ -365,7 +402,7 @@ class TestParser(unittest.TestCase):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# unittest.main()
|
||||
#unittest.main()
|
||||
main()
|
||||
|
||||
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
"benchmarks": [
|
||||
{
|
||||
"name": "BM_One",
|
||||
"run_type": "aggregate",
|
||||
"iterations": 1000,
|
||||
"real_time": 10,
|
||||
"cpu_time": 100,
|
||||
@ -30,6 +31,15 @@
|
||||
},
|
||||
{
|
||||
"name": "short",
|
||||
"run_type": "aggregate",
|
||||
"iterations": 1000,
|
||||
"real_time": 8,
|
||||
"cpu_time": 80,
|
||||
"time_unit": "ns"
|
||||
},
|
||||
{
|
||||
"name": "medium",
|
||||
"run_type": "iteration",
|
||||
"iterations": 1000,
|
||||
"real_time": 8,
|
||||
"cpu_time": 80,
|
||||
|
||||
@ -16,6 +16,7 @@
|
||||
},
|
||||
{
|
||||
"name": "BM_Two",
|
||||
"run_type": "aggregate",
|
||||
"iterations": 1000,
|
||||
"real_time": 10,
|
||||
"cpu_time": 89,
|
||||
@ -30,10 +31,19 @@
|
||||
},
|
||||
{
|
||||
"name": "short",
|
||||
"run_type": "aggregate",
|
||||
"iterations": 1000,
|
||||
"real_time": 8,
|
||||
"cpu_time": 80,
|
||||
"time_unit": "ns"
|
||||
},
|
||||
{
|
||||
"name": "medium",
|
||||
"run_type": "iteration",
|
||||
"iterations": 1200,
|
||||
"real_time": 5,
|
||||
"cpu_time": 53,
|
||||
"time_unit": "ns"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -96,6 +96,7 @@ def filter_benchmark(json_orig, family, replacement=""):
|
||||
def generate_difference_report(
|
||||
json1,
|
||||
json2,
|
||||
display_aggregates_only=False,
|
||||
utest=False,
|
||||
utest_alpha=0.05,
|
||||
use_color=True):
|
||||
@ -191,6 +192,13 @@ def generate_difference_report(
|
||||
timings_cpu[0].append(bn['cpu_time'])
|
||||
timings_cpu[1].append(other_bench['cpu_time'])
|
||||
|
||||
# *After* recording this run for u test, *if* we were asked to only
|
||||
# display aggregates, and if it is non-aggregate, then skip it.
|
||||
if display_aggregates_only and 'run_type' in bn and 'run_type' in other_bench:
|
||||
assert bn['run_type'] == other_bench['run_type']
|
||||
if bn['run_type'] != 'aggregate':
|
||||
continue;
|
||||
|
||||
tres = calculate_change(timings_time[0][-1], timings_time[1][-1])
|
||||
cpures = calculate_change(timings_cpu[0][-1], timings_cpu[1][-1])
|
||||
output_strs += [color_format(use_color,
|
||||
@ -331,10 +339,60 @@ class TestReportDifferenceWithUTest(unittest.TestCase):
|
||||
'repetitions',
|
||||
'recommended.'],
|
||||
['short', '+0.0000', '+0.0000', '8', '8', '80', '80'],
|
||||
['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'],
|
||||
]
|
||||
json1, json2 = self.load_results()
|
||||
output_lines_with_header = generate_difference_report(
|
||||
json1, json2, True, 0.05, use_color=False)
|
||||
json1, json2, utest=True, utest_alpha=0.05, use_color=False)
|
||||
output_lines = output_lines_with_header[2:]
|
||||
print("\n")
|
||||
print("\n".join(output_lines_with_header))
|
||||
self.assertEqual(len(output_lines), len(expect_lines))
|
||||
for i in range(0, len(output_lines)):
|
||||
parts = [x for x in output_lines[i].split(' ') if x]
|
||||
self.assertEqual(parts, expect_lines[i])
|
||||
|
||||
|
||||
class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(unittest.TestCase):
|
||||
def load_results(self):
|
||||
import json
|
||||
testInputs = os.path.join(
|
||||
os.path.dirname(
|
||||
os.path.realpath(__file__)),
|
||||
'Inputs')
|
||||
testOutput1 = os.path.join(testInputs, 'test3_run0.json')
|
||||
testOutput2 = os.path.join(testInputs, 'test3_run1.json')
|
||||
with open(testOutput1, 'r') as f:
|
||||
json1 = json.load(f)
|
||||
with open(testOutput2, 'r') as f:
|
||||
json2 = json.load(f)
|
||||
return json1, json2
|
||||
|
||||
def test_utest(self):
|
||||
expect_lines = []
|
||||
expect_lines = [
|
||||
['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
|
||||
['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'],
|
||||
['BM_Two', '+0.2500', '+0.1125', '8', '10', '80', '89'],
|
||||
['BM_Two_pvalue',
|
||||
'0.2207',
|
||||
'0.6831',
|
||||
'U',
|
||||
'Test,',
|
||||
'Repetitions:',
|
||||
'2.',
|
||||
'WARNING:',
|
||||
'Results',
|
||||
'unreliable!',
|
||||
'9+',
|
||||
'repetitions',
|
||||
'recommended.'],
|
||||
['short', '+0.0000', '+0.0000', '8', '8', '80', '80'],
|
||||
]
|
||||
json1, json2 = self.load_results()
|
||||
output_lines_with_header = generate_difference_report(
|
||||
json1, json2, display_aggregates_only=True,
|
||||
utest=True, utest_alpha=0.05, use_color=False)
|
||||
output_lines = output_lines_with_header[2:]
|
||||
print("\n")
|
||||
print("\n".join(output_lines_with_header))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user