mirror of
https://github.com/sharkdp/hyperfine.git
synced 2024-09-11 14:07:02 +03:00
5a80a23bdc
Before this patch, executing directly one of these scripts, for example `./plot_hystogram.py` in a unix-like environment meant that the default system-level python would be used, regardless of an eventual activated virtualenv. This was due to the "#!/usr/bin/python" shebang. Changing it to "/usr/bin/env python" is a fairly standard practice, keeps intact the compatiblity with the system level python, and allows a user to run in a virtualenv if he wants.
39 lines
1018 B
Python
Executable File
39 lines
1018 B
Python
Executable File
#!/usr/bin/env python
|
|
|
|
"""This script performs Welch's t-test on a JSON export file with two
|
|
benchmark results to test whether or not the two distributions are
|
|
the same."""
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from scipy import stats
|
|
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("file", help="JSON file with two benchmark results")
|
|
args = parser.parse_args()
|
|
|
|
with open(args.file) as f:
|
|
results = json.load(f)["results"]
|
|
|
|
if len(results) != 2:
|
|
print("The input file has to contain exactly two benchmarks")
|
|
sys.exit(1)
|
|
|
|
a, b = [x["command"] for x in results[:2]]
|
|
X, Y = [x["times"] for x in results[:2]]
|
|
|
|
print("Command 1: {}".format(a))
|
|
print("Command 2: {}\n".format(b))
|
|
|
|
t, p = stats.ttest_ind(X, Y, equal_var=False)
|
|
th = 0.05
|
|
dispose = p < th
|
|
print("t = {:.3}, p = {:.3}".format(t, p))
|
|
print()
|
|
|
|
if dispose:
|
|
print("There is a difference between the two benchmarks (p < {}).".format(th))
|
|
else:
|
|
print("The two benchmarks are almost the same (p >= {}).".format(th))
|