graphql-engine/server/bench-wrk/results_schema.yaml

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

168 lines
4.2 KiB
YAML
Raw Normal View History

Regression benchmarks setup (#3310) * Benchmark GraphQL queries using wrk * fix console assets dir * Store wrk parameters as well * Add details about storing results in Readme * Remove files in bench-wrk while computing server shasum * Instead of just getting maximum throughput per query per version, create plots using wrk2 for a given set of requests per second. The maximum throughput is used to see what values of requests per second are feasible. * Add id for version dropdown * Allow specifiying env and args for GraphQL Engine 1) Arguments defined after -- will be applied as arguments to Hasura GraphQL Engine 2) Script will also pass the environmental variables to Hasura GraphQL Engine instances Hasura GraphQL engine can be run with the given environmental variables and arguments as follows $ export HASURA_GRAPHQL_...=.... $ python3 hge_wrk_bench.py -- --hge_arg1 val1 --hge_arg2 val2 ... * Use matplotlib instead of plotly for figures * Show throughput graph also. It maybe useful in checking performance regression across versions * Support storing results in s3 Use --upload-root-uri 's3://bucket/path' to upload results inside the given path.When specified, the results will be uploaded to the bucket, including latencies, latency histogram, and the test setup info. The s3 credentials should be provided as given in AWS boto3 documentation. * Allow specifying a name for the test scenario * Fix open latency uri bug * Update wrk docker image * Keep ylim a little higher than maximum so that the throughput plot is clearly visible * Show throughput plots for multiple queries at the same time * 1) Adjust size of dropdowns 2) Make label for requests/sec invisible when plot type is throughput * 1) Adding boto3 to requirements.txt 2) Removing CPU Key print line 3) Adding info about the tests that will be run with wrk2 * Docker builder fo wrk-websocket-server * Make it optional to setup remote graphql-engine * Listen on all interfaces and enable ping thread * Add bench_scripts to wrk-websocket-server docker * Use 127.0.0.1 instead of 'localhost' to address local hge For some reason it seems wrk was hanging trying to resolve 'localhost'. ping was able to fine from the same container, so I'm not sure what the deal was. Probably some local misconfiguration on my machine, but maybe this change will also help others. * Store latency samples in subdirectory, server_shasum just once at start, additional docs * Add a note on running the benchmarks in the simplest way * Add a new section on how to run benchmarks on a new linux hosted instance Co-authored-by: Nizar Malangadan <nizar-m@users.noreply.github.com> Co-authored-by: Brandon Simmons <brandon.m.simmons@gmail.com> Co-authored-by: Karthikeyan Chinnakonda <karthikeyan@hasura.io> Co-authored-by: Brandon Simmons <brandon@hasura.io> Co-authored-by: Vamshi Surabhi <0x777@users.noreply.github.com>
2020-06-19 20:10:17 +03:00
- type: run_sql
args:
sql: |
create schema if not exists hge_bench;
create table if not exists hge_bench.gql_query (
name text primary key not null,
query text not null unique
);
create table if not exists hge_bench.cpu_info (
key text primary key not null,
info jsonb not null unique
);
create table if not exists hge_bench.query_max_rps(
id serial primary key,
cpu_key text references hge_bench.cpu_info (key),
query_name text references hge_bench.gql_query (name) not null,
docker_image text,
version text,
scenario_name text,
postgres_version text,
server_shasum text,
time timestamptz not null default now(),
max_rps integer not null,
wrk_parameters jsonb,
hge_conf jsonb
constraint should_have_tag CHECK (docker_image is not null or version is not null)
);
create table if not exists hge_bench.results(
id serial primary key,
cpu_key text references hge_bench.cpu_info (key),
query_name text references hge_bench.gql_query (name) not null,
docker_image text,
version text,
scenario_name text,
postgres_version text,
server_shasum text,
time timestamptz not null default now(),
requests_per_sec integer not null,
summary jsonb,
latencies_uri text,
wrk2_parameters jsonb,
hge_conf jsonb
constraint should_have_tag CHECK (docker_image is not null or version is not null)
);
create or replace view hge_bench.latest_results as
select
distinct on (cpu_key, docker_image, version, query_name, requests_per_sec)
id, cpu_key, query_name, docker_image, version,
postgres_version, server_shasum, time, requests_per_sec, summary,
latencies_uri, wrk2_parameters
from hge_bench.results
order by cpu_key, docker_image, version, query_name, requests_per_sec, time desc;
create table if not exists hge_bench.latency_histogram (
id integer references hge_bench.results(id),
percentile double precision not null,
latency double precision not null,
total_count integer not null
);
create or replace view hge_bench.avg_query_max_rps as
select cpu_key, query_name, docker_image, version, avg(max_rps) as max_rps
from hge_bench.query_max_rps
group by cpu_key, query_name, docker_image, version;
- type: track_table
args:
schema: hge_bench
name: gql_query
- type: track_table
args:
schema: hge_bench
name: results
- type: track_table
args:
schema: hge_bench
name: query_max_rps
- type: track_table
args:
schema: hge_bench
name: cpu_info
- type: track_table
args:
schema: hge_bench
name: latency_histogram
- type: track_table
args:
schema: hge_bench
name: latest_results
- type: track_table
args:
schema: hge_bench
name: avg_query_max_rps
- type: create_object_relationship
args:
table:
schema: hge_bench
name: results
name: query
using:
foreign_key_constraint_on: query_name
- type: create_object_relationship
args:
table:
schema: hge_bench
name: results
name: cpu
using:
foreign_key_constraint_on: cpu_key
- type: create_object_relationship
args:
table:
schema: hge_bench
name: query_max_rps
name: query
using:
foreign_key_constraint_on: query_name
- type: create_object_relationship
args:
table:
schema: hge_bench
name: query_max_rps
name: cpu
using:
foreign_key_constraint_on: cpu_key
- type: create_array_relationship
args:
table:
schema: hge_bench
name: results
name: latency_histogram
using:
foreign_key_constraint_on:
table:
schema: hge_bench
name: latency_histogram
column: id
- type: create_array_relationship
args:
table:
schema: hge_bench
name: latest_results
name: latency_histogram
using:
manual_configuration:
remote_table:
schema: hge_bench
name: latency_histogram
column_mapping:
id: id