Add a common entry point for all benchmarks (#3875)

Implements https://www.pivotaltracker.com/story/show/183915527

Adds a common entry point to the `Benchmarks` test suite, so that all benchmarks can be run at once.

Usually we are running only one set of benchmarks that we want to measure, but for the purpose of checking if they are still compiling after a language change, it can be useful to run all of them (currently this takes a pretty long time though, but may work when going for lunch break :)).

Related to https://www.pivotaltracker.com/story/show/183369386/comments/234258671
This commit is contained in:
Radosław Waśko 2022-11-30 19:58:29 +01:00 committed by GitHub
parent 20c22f2422
commit 06070645ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 244 additions and 133 deletions

View File

@ -22,7 +22,7 @@ build_map size =
rand = Random.new
0.up_to size . fold Map.empty (m -> i -> m.insert (rand.nextInt 10000) i)
main =
bench =
mil = 1000000
list = gen_list mil
vec = Vector.new mil (ix -> ix + 1)
@ -32,3 +32,5 @@ main =
Bench.measure (vec.fold 0 (+)) "vector fold" 1000 10
Bench.measure (vec_decimal.fold 0 (+)) "vector decimal fold" 1000 10
Bench.measure (build_map 10000) "build a map" 100 10
main = bench

View File

@ -39,7 +39,7 @@ count_entries_polyglot vector element expected_count=1 =
IO.println msg
Panic.throw (Illegal_State_Error msg)
main =
bench =
n = 100000
iter_size = 100
num_iterations = 10
@ -50,11 +50,11 @@ main =
text_vec = integer_vec.map .to_text
uniform_atom_vec = integer_vec.map x->
Only_Ctor (x + 10) (x * 2)
Uniform_Type.Only_Ctor (x + 10) (x * 2)
simple_variant_atom_vec = integer_vec.map x->
if x % 5 == 0 then Variant_A else Variant_B
if x % 5 == 0 then Simple_Sum_Type.Variant_A else Simple_Sum_Type.Variant_B
complex_variant_atom_vec = integer_vec.map x->
if x % 5 == 0 then Ctor_A (x + 10) (x * 2) else Ctor_B x
if x % 5 == 0 then Sum_Type_With_Values.Ctor_A (x + 10) (x * 2) else Sum_Type_With_Values.Ctor_B x
polyglot_vec = integer_vec.map x->
Graal_Pair.create (x + 10) (x * 2)
@ -72,11 +72,13 @@ main =
Bench.measure (count_entries date_vec (Date.new 1980 12 30)) "Date Equality" iter_size num_iterations
Bench.measure (count_entries datetime_vec (Date_Time.new 1971 1 2 12 30)) "Date_Time Equality" iter_size num_iterations
Bench.measure (count_entries uniform_atom_vec (Only_Ctor 110 200)) "Custom Atom Equality - single constructor" iter_size num_iterations
Bench.measure (count_entries simple_variant_atom_vec Variant_A expected_count=(n . div 5)) "Custom Atom Equality - 2 constructors without data" iter_size num_iterations
Bench.measure (count_entries complex_variant_atom_vec (Ctor_B 123)) "Custom Atom Equality - 2 constructors with data" iter_size num_iterations
Bench.measure (count_entries uniform_atom_vec (Uniform_Type.Only_Ctor 110 200)) "Custom Atom Equality - single constructor" iter_size num_iterations
Bench.measure (count_entries simple_variant_atom_vec Simple_Sum_Type.Variant_A expected_count=(n . div 5)) "Custom Atom Equality - 2 constructors without data" iter_size num_iterations
Bench.measure (count_entries complex_variant_atom_vec (Sum_Type_With_Values.Ctor_B 123)) "Custom Atom Equality - 2 constructors with data" iter_size num_iterations
Bench.measure (count_entries polyglot_vec (Graal_Pair.create 110 200)) "Java Polyglot ==" iter_size num_iterations
Bench.measure (count_entries_polyglot polyglot_vec (Graal_Pair.create 110 200)) "Java Polyglot .equals" iter_size num_iterations
Nothing
main = bench

View File

@ -1,4 +1,5 @@
from Standard.Base import all
import Standard.Base
from Standard.Test import Bench
@ -8,6 +9,8 @@ prep_json size =
str = "[" + (many.join ",") + "]"
str
main =
bench =
large_json = prep_json 1000000
Bench.measure (Json.parse large_json) "parse json" 10 10
main = bench

View File

@ -1,88 +1,28 @@
from Standard.Base import all
import Standard.Base.Runtime.Debug
import Standard.Base.Runtime.State
import project.Collections
import project.Equality
import project.Json_Bench
import project.Natural_Order_Sort
import project.Number_Parse
import project.Sum
from Standard.Test import Bench
polyglot java import java.lang.Long
type Counter
type Sum
sum_tco = sum_to ->
summator = acc -> current ->
if current == 0 then acc else
@Tail_Call summator acc+current current-1
res = summator 0 sum_to
res
sum_tco_decimal = sum_to ->
s = sum_to.to_decimal
summator = acc -> current ->
if current >= s then acc else
@Tail_Call summator acc+current current+1.0
res = summator 0.0 0.0
res
sum_tco_eval = sumTo ->
summator = acc -> current ->
if current == 0 then acc else
Debug.eval "@Tail_Call summator (acc + current) (current - 1)"
res = summator 0 sumTo
res
sum_tco_java = sum_to ->
summator = acc -> current ->
if current == 0 then acc else
@Tail_Call summator (Long.sum acc current) (current - 1)
res = summator 0 sum_to
res
sum_co_state_body =
n = State.get Counter
acc = State.get Sum
State.put Counter n-1
State.put Sum acc+n
if n == 0 then acc else
@Tail_Call sum_co_state_body
sum_co_state n =
res = State.run Counter n (State.run Sum 0 sum_co_state_body)
res
sum_state_body n =
acc = State.get Number
State.put Number (acc + n)
if n == 0 then State.get Number else
@Tail_Call sum_state_body (n - 1)
sum_state = sum_to ->
res = State.run Number 0 (sum_state_body sum_to)
res
sum_co_1 n acc = if n == 0 then acc else @Tail_Call sum_co_2 n-1 acc+n
sum_co_2 n acc = if n == 0 then acc else sum_co_1 n-1 acc+n
sum_co n =
res = sum_co_2 n 0
res
import project.Statistics
import project.Table
import project.Text
import project.Time
import project.Vector
## A common entry point for all benchmarks. Usually just one suite is run, but
sometimes we want to run all benchmarks to ensure our language changes are
not breaking them.
main =
hundred_mil = 100000000
IO.println "Measuring Sum TCO Corecursive"
Bench.measure (sum_co hundred_mil) "sum_tco_corecursive" 100 10
IO.println "Measuring Sum TCO Decimal"
Bench.measure (sum_tco_decimal hundred_mil) "sum_tco_float" 100 10
IO.println "Measuring SumTCO"
Bench.measure (sum_tco hundred_mil) "sum_tco" 100 10
IO.println "Measuring SumTCO Java"
Bench.measure (sum_tco_java hundred_mil) "sum_tco_java" 100 10
IO.println "Measuring SumTCO Eval"
Bench.measure (sum_tco_eval hundred_mil) "sum_tco_eval" 100 10
IO.println "Measuring State"
Bench.measure (sum_state hundred_mil) "sum_state" 100 10
IO.println "Measuring Co-State"
Bench.measure (sum_co_state hundred_mil) "sum_co_state" 100 10
IO.println "Bye."
Collections.bench
Equality.bench
Json_Bench.bench
Natural_Order_Sort.bench
Number_Parse.bench
Sum.bench
Statistics.Main.bench
Table.Main.bench
Text.Main.bench
Time.Main.bench
Vector.Main.bench

View File

@ -11,7 +11,7 @@ num_iterations = 10
# The Benchmarks ==============================================================
main =
bench =
l = Faker.upper_case_letters
n = Faker.numbers
template = [l, l, l, n, n, n, n, n, l]
@ -25,3 +25,5 @@ main =
IO.println <| "Benchmarking..."
Bench.measure (unsorted.sort by=Natural_Order.compare) "Natural Order" iter_size num_iterations
main = bench

View File

@ -10,7 +10,7 @@ num_iterations = 10
# The Benchmarks ==============================================================
main =
bench =
## No specific significance to this constant, just fixed to make generated set deterministic
fixed_random_seed = 1644575867
faker = Faker.new fixed_random_seed
@ -24,3 +24,5 @@ main =
int_strings = Vector.new vector_size _->(faker.integer -1000000000 1000000000).to_text
IO.println <| "Benchmarking Integer.parse"
Bench.measure (int_strings.map Integer.parse) "Integer.parse" iter_size num_iterations
main = bench

View File

@ -15,9 +15,11 @@ create_vector rows (seed=1646322139) =
0.up_to rows . map _-> faker.make_some_nothing (faker.integer 0 1000000)
# The Benchmarks ==============================================================
main =
bench =
IO.println <| "Making data..."
vector = create_vector vector_size
IO.println <| "Testing..."
Bench.measure (vector.compute_bulk [Statistic.Count, Statistic.Minimum, Statistic.Maximum]) "Count Min Max" iter_size num_iterations
main = bench

View File

@ -0,0 +1,6 @@
import project.Statistics.Count_Min_Max
bench =
Count_Min_Max.bench
main = bench

View File

@ -0,0 +1,94 @@
from Standard.Base import all
import Standard.Base.Runtime.Debug
import Standard.Base.Runtime.State
from Standard.Test import Bench
polyglot java import java.lang.Long
type Counter
type Sum
sum_tco = sum_to ->
summator = acc -> current ->
if current == 0 then acc else
@Tail_Call summator acc+current current-1
res = summator 0 sum_to
res
sum_tco_decimal = sum_to ->
s = sum_to.to_decimal
summator = acc -> current ->
if current >= s then acc else
@Tail_Call summator acc+current current+1.0
res = summator 0.0 0.0
res
sum_tco_eval = sumTo ->
summator = acc -> current ->
if current == 0 then acc else
Debug.eval "@Tail_Call summator (acc + current) (current - 1)"
res = summator 0 sumTo
res
sum_tco_java = sum_to ->
summator = acc -> current ->
if current == 0 then acc else
@Tail_Call summator (Long.sum acc current) (current - 1)
res = summator 0 sum_to
res
sum_co_state_body =
n = State.get Counter
acc = State.get Sum
State.put Counter n-1
State.put Sum acc+n
if n == 0 then acc else
@Tail_Call sum_co_state_body
sum_co_state n =
res = State.run Counter n (State.run Sum 0 sum_co_state_body)
res
sum_state_body n =
acc = State.get Number
State.put Number (acc + n)
if n == 0 then State.get Number else
@Tail_Call sum_state_body (n - 1)
sum_state = sum_to ->
res = State.run Number 0 (sum_state_body sum_to)
res
sum_co_1 n acc = if n == 0 then acc else @Tail_Call sum_co_2 n-1 acc+n
sum_co_2 n acc = if n == 0 then acc else sum_co_1 n-1 acc+n
sum_co n =
res = sum_co_2 n 0
res
bench =
hundred_mil = 100000000
IO.println "Measuring Sum TCO Corecursive"
Bench.measure (sum_co hundred_mil) "sum_tco_corecursive" 100 10
IO.println "Measuring Sum TCO Decimal"
Bench.measure (sum_tco_decimal hundred_mil) "sum_tco_float" 100 10
IO.println "Measuring SumTCO"
Bench.measure (sum_tco hundred_mil) "sum_tco" 100 10
IO.println "Measuring SumTCO Java"
Bench.measure (sum_tco_java hundred_mil) "sum_tco_java" 100 10
## This test is disabled because it fails when running from a common entry point.
See the bug: https://www.pivotaltracker.com/story/show/183914580
# IO.println "Measuring SumTCO Eval"
# Bench.measure (sum_tco_eval hundred_mil) "sum_tco_eval" 100 10
IO.println "Measuring State"
Bench.measure (sum_state hundred_mil) "sum_state" 100 10
IO.println "Measuring Co-State"
Bench.measure (sum_co_state hundred_mil) "sum_co_state" 100 10
IO.println "Bye."
main = bench

View File

@ -1,7 +1,7 @@
from Standard.Base import all
from Standard.Table import Table, Column_Selector
from Standard.Table.Data.Aggregate_Column import all
from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all
from Standard.Test import Bench, Faker
@ -24,7 +24,7 @@ create_table rows (seed=1646322139) =
Table.new [key1, key2, key3, value1, value2, text1, text2]
# The Benchmarks ==============================================================
main =
bench =
IO.println <| "Making table data..."
table = create_table vector_size
@ -51,3 +51,5 @@ main =
Bench.measure (table.aggregate [Group_By "Index", Group_By "Flag", Standard_Deviation "Value"]) "StDev 2 level groups" iter_size num_iterations
# Bench.measure (table.aggregate [Group_By "Index", Group_By "Flag", Median "Value"]) "Median 2 level groups" iter_size num_iterations
# Bench.measure (table.aggregate [Group_By "Index", Group_By "Flag", Mode "Index"]) "Mode 2 level groups" iter_size num_iterations
main = bench

View File

@ -0,0 +1,8 @@
import project.Table.Aggregate
import project.Table.Sorting
bench =
Aggregate.bench
Sorting.bench
main = bench

View File

@ -16,7 +16,7 @@ iter_size = 20
num_iterations = 10
# The Benchmarks ==============================================================
main =
bench =
ints = (0.up_to vector_size).to_vector.take (Index_Sub_Range.Sample vector_size 42)
start = Date_Time.new 1990 1 1
dates = ints.map x->
@ -35,3 +35,5 @@ main =
Bench.measure (objects_table.order_by (Sort_Column_Selector.By_Index [0])) "Table.order_by objects" iter_size num_iterations
Bench.measure (objects.sort) "Vector.sort objects" iter_size num_iterations
main = bench

View File

@ -16,7 +16,9 @@ build_long_bldr n =
res = bldr.toString
res
main =
bench =
n = 1000000
Bench.measure (build_long_bldr n) "string append bldr" 1 1
Bench.measure (build_long n) "string append" 1 1
main = bench

View File

@ -8,7 +8,7 @@ compare_all_adjacent text_vector =
acc && compare_result
res
main =
bench =
## The `Text.compare_to` benchmarks check both scenarios where the Texts are
short and very long - both checking the case where the difference appears
early or late in the long string. This is to see well any overheads
@ -48,3 +48,5 @@ main =
bench_strcmp "strcmp Unicode " (Faker.upper_case_letters + 'ę\u{301}\u{302}'.char_vector) 'ę\u{301}'
bench_strcmp "strcmp ASCII " Faker.upper_case_letters 'A'
main = bench

View File

@ -8,7 +8,7 @@ check_all text_vector pattern_vector mode =
result = text.contains pattern mode
acc + if result then 1 else 0
main =
bench =
bench_contains suite_prefix character_template =
faker = Faker.new
## Warning: this relies on the fact that Faker will treat the accent
@ -20,11 +20,13 @@ main =
big_template = make_alpha_template 10000
big_random = Vector.new 200 _-> faker.string_value big_template
Bench.measure (check_all big_random ["AAAAAA"] Text_Matcher) suite_prefix+" exact" 10 10
Bench.measure (check_all big_random ["AAAAAA"] (Text_Matcher Sort_Direction)) suite_prefix+" case-insensitive" 10 10
Bench.measure (check_all big_random ["AAAAAA"] Regex_Matcher) suite_prefix+" exact regex" 10 10
Bench.measure (check_all big_random ["AAAAAA"] Text_Matcher.Case_Sensitive) suite_prefix+" exact" 10 10
Bench.measure (check_all big_random ["AAAAAA"] Text_Matcher.Case_Insensitive) suite_prefix+" case-insensitive" 10 10
Bench.measure (check_all big_random ["AAAAAA"] Regex_Matcher.Regex_Matcher_Data) suite_prefix+" exact regex" 10 10
Bench.measure (check_all big_random ["AAAAAA"] (Regex_Matcher.Regex_Matcher_Data case_sensitivity=Case_Sensitivity.Insensitive)) suite_prefix+" case-insensitive regex" 10 10
Bench.measure (check_all big_random ["A.{5000}A"] Regex_Matcher) suite_prefix+" const-width regex" 10 10
Bench.measure (check_all big_random ["AAAAA.*AAAAA"] Regex_Matcher) suite_prefix+" wildcard regex" 10 10
Bench.measure (check_all big_random ["A.{5000}A"] Regex_Matcher.Regex_Matcher_Data) suite_prefix+" const-width regex" 10 10
Bench.measure (check_all big_random ["AAAAA.*AAAAA"] Regex_Matcher.Regex_Matcher_Data) suite_prefix+" wildcard regex" 10 10
bench_contains "Text.contains " (Faker.upper_case_letters + Faker.lower_case_letters + 'ąę\u{301}\u{302}\u{303}\u{321}'.char_vector)
main = bench

View File

@ -0,0 +1,12 @@
import project.Text.Build
import project.Text.Compare
import project.Text.Contains
import project.Text.Reverse
bench =
Build.bench
Compare.bench
Contains.bench
Reverse.bench
main = bench

View File

@ -4,7 +4,7 @@ from Standard.Test import Bench, Faker
# Benchmarks ##################################################################
main =
bench =
## The `Text.reverse` benchmarks check both scenarios where the Texts are
short and very long. This is to show any overheads related to string
building.
@ -25,3 +25,5 @@ main =
Bench.measure (big_random.reverse) suite_prefix+" 100000" 10 10
bench_reverse "Text.reverse" (Faker.upper_case_letters + Faker.lower_case_letters + 'ąę\u{301}\u{302}\u{303}\u{321}'.char_vector)
main = bench

View File

@ -0,0 +1,6 @@
import project.Time.Work_Days
bench =
Work_Days.bench
main = bench

View File

@ -4,7 +4,7 @@ from Standard.Test import Bench
polyglot java import java.util.Random
main =
bench =
iter_size = 50
num_iterations = 10
@ -14,7 +14,7 @@ main =
shifts = [1, 5, 20, 100]
shifts.each shift->
shifted_dates = dates.map (d -> d + shift.days)
shifted_dates = dates.map (d -> d + (Period.new days=shift))
Bench.measure (dates.zip shifted_dates d1-> d2-> d1.work_days_until d2) "(Shift="+shift.to_text+") work_days_until" iter_size num_iterations
Bench.measure (dates.zip shifted_dates d1-> d2-> d1.work_days_until d2 holidays=holidays) "(Shift="+shift.to_text+") work_days_until with holidays" iter_size num_iterations
@ -23,3 +23,5 @@ main =
Bench.measure (dates.map date-> date.add_work_days shift holidays=holidays) "(Shift="+shift.to_text+") add work days with holidays" iter_size num_iterations
Nothing
main = bench

View File

@ -3,6 +3,8 @@ import Standard.Base
from Standard.Test import Bench
import project.Vector.Utils
polyglot java import java.util.Random
polyglot java import org.enso.base.Time_Utils
@ -12,15 +14,10 @@ polyglot java import org.enso.base.Time_Utils
iter_size = 100
num_iterations = 10
make_random_vec : Integer -> Base.Vector
make_random_vec n =
random_gen = Random.new n
Base.Vector.new n _->random_gen.nextLong
# The Benchmarks ==============================================================
main =
random_vec = make_random_vec 10000
bench =
random_vec = Utils.make_random_vec 10000
uniform_vec = Base.Vector.fill 10000 1
random_text_vec = random_vec.map .to_text
@ -31,3 +28,5 @@ main =
Bench.measure (random_text_vec.distinct) "Random Text Vector Distinct" iter_size num_iterations
Bench.measure (uniform_text_vec.distinct) "Uniform Text Vector Distinct" iter_size num_iterations
main = bench

View File

@ -0,0 +1,10 @@
import project.Vector.Distinct
import project.Vector.Sort
import project.Vector.Operations
bench =
Distinct.bench
Sort.bench
Operations.bench
main = bench

View File

@ -4,9 +4,9 @@ import Standard.Base
from Standard.Test import Bench
polyglot java import java.util.Random
polyglot java import org.enso.base.Time_Utils
import project.Vector.Utils
polyglot java import java.util.Random
## Bench Utilities ============================================================
@ -14,21 +14,16 @@ vector_size = 1000000
iter_size = 100
num_iterations = 10
make_random_vec : Integer -> Base.Vector
make_random_vec n =
random_gen = Random.new n
Base.Vector.fill n random_gen.nextLong
# The Benchmarks ==============================================================
main =
random_vec = make_random_vec vector_size
random_vec_2 = make_random_vec 100000
bench =
random_vec = Utils.make_random_vec vector_size
random_vec_2 = Utils.make_random_vec 100000
random_gen = Random.new 123456
Bench.measure (Base.Vector.new vector_size i->i) "New Vector" iter_size num_iterations
Bench.measure (Base.Vector.new vector_size _->42) "New Constant" iter_size num_iterations
Bench.measure (Base.Vector.new vector_size _->random_gen.nextLong) "New Random" iter_size num_iterations
Bench.measure (Base.Vector.new vector_size _->42) "New Constant" iter_size num_iterations
Bench.measure (Base.Vector.new vector_size _->random_gen.nextLong) "New Random" iter_size num_iterations
Bench.measure (Base.Vector.fill vector_size 42) "Fill Constant" iter_size num_iterations
Bench.measure (Base.Vector.fill vector_size random_gen.nextLong) "Fill Random (constant)" iter_size num_iterations
Bench.measure (random_vec + [1]) "Append Single" iter_size num_iterations
@ -38,6 +33,8 @@ main =
Bench.measure ((random_vec.drop (Last 20)).sum) "Drop Last 20 and Sum" iter_size num_iterations
Bench.measure (random_vec.filter (x -> x % 3 == 1)) "Filter" iter_size num_iterations
Bench.measure (random_vec.filter_with_index (i-> x-> (i+x) % 3 == 1)) "Filter With Index" iter_size num_iterations
random_gen = Random.new 0
Bench.measure (random_vec . map (x -> x + random_gen.nextLong) . filter (x -> x % 3 == 1)) "Map & Filter" iter_size num_iterations
Bench.measure (random_vec.partition (x -> x % 3 == 1)) "Partition" iter_size num_iterations
Bench.measure (random_vec.partition_with_index (i-> x-> (i+x) % 3 == 1)) "Partition With Index" iter_size num_iterations
@ -46,3 +43,5 @@ main =
s = State.get Number
State.put s+x
Bench.measure (State.run Number 0 <| random_vec.each stateful_fun) "Each" iter_size num_iterations
main = bench

View File

@ -3,7 +3,7 @@ import Standard.Base.Runtime.Ref.Ref
from Standard.Test import Bench
import project.Vector.Vector as Vector_Utils
import project.Vector.Utils
polyglot java import java.util.Random
polyglot java import org.enso.base.Time_Utils
@ -52,10 +52,10 @@ make_partially_sorted_vec n =
# The Benchmarks ==============================================================
main =
bench =
sorted_vec = make_sorted_ascending_vec vector_size
partially_sorted_vec = make_partially_sorted_vec vector_size
random_vec = Vector_Utils.make_random_vec vector_size
random_vec = Utils.make_random_vec vector_size
projection = x -> x % 10
comparator = l -> r -> r.compare_to l
@ -67,3 +67,5 @@ main =
Bench.measure (random_vec.sort order=Sort_Direction.Descending) "Random Elements Descending" iter_size num_iterations
Bench.measure (random_vec.sort on=projection) "Sorting with a Custom Projection" iter_size num_iterations
Bench.measure (random_vec.sort by=comparator) "Sorting with a Custom Comparison" iter_size num_iterations
main = bench

View File

@ -0,0 +1,8 @@
from Standard.Base import all
polyglot java import java.util.Random
make_random_vec : Integer -> Vector.Vector
make_random_vec n =
random_gen = Random.new n
Vector.fill n random_gen.nextLong