diff --git a/.gitignore b/.gitignore index e8acf96118f..2a57e557da2 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,7 @@ node_modules/ *.iml .enso-sources* .metals +tools/performance/engine-benchmarks/generated_site ############################ ## Rendered Documentation ## diff --git a/tools/performance/engine-benchmarks/README.md b/tools/performance/engine-benchmarks/README.md index d6cb4c4d915..c37cfbe060a 100644 --- a/tools/performance/engine-benchmarks/README.md +++ b/tools/performance/engine-benchmarks/README.md @@ -1,8 +1,8 @@ # Engine benchmarks This directory contains a python script `bench_download.py` for downloading -Engine benchmark results from GitHub and `Engine_Benchs` Enso project for -analysing the downloaded data. +Engine and stdlib benchmark results from GitHub, and `Engine_Benchs` Enso +project for analysing the downloaded data. Dependencies for `bench_download.py`: @@ -14,7 +14,9 @@ Dependencies for `bench_download.py`: `sudo apt-get install gh` Check `bench_download -h` for documentation and usage. Ensure that your -`/usr/bin/env python` links to Python version at least 3.7. +`/usr/bin/env python` links to Python version at least 3.7. `bench_download.py` +creates `generated_site` directory with HTML files for visualizing the benchmark +results. One can also analyze the benchmarks in Enso IDE by running `bench_download.py --create-csv` and then running `Engine_Benchs` project. The diff --git a/tools/performance/engine-benchmarks/bench_download.py b/tools/performance/engine-benchmarks/bench_download.py index 5805d59dd85..3b796de8f21 100755 --- a/tools/performance/engine-benchmarks/bench_download.py +++ b/tools/performance/engine-benchmarks/bench_download.py @@ -3,16 +3,28 @@ """ Script for downloading Engine benchmark results into a single static web page that visualizes all the benchmarks. Without any options, downloads and -visualizes benchmark data for the last 14 days. +visualizes benchmark data for the last 14 days. By default, no data is written +to the disk except for the generated web page, and the data are downloaded +asynchronously. -It downloads the data synchronously and uses a cache directory by default. -It is advised to use `-v|--verbose` option all the time. +Set the `--source` parameter to either `engine` or `stdlib`. + +The generated website is placed under "generated_site" directory + +The default GH artifact retention period is 3 months, which means that all +the artifacts older than 3 months are dropped. If you wish to gather the data +for benchmarks older than 3 months, make sure that the `use_cache` parameter +is set to true, and that the cache directory is populated with older data. +If the script encounters an expired artifact, it prints a warning. + +This script is under continuous development, so it is advised to use +`-v|--verbose` option all the time. It queries only successful benchmark runs. If there are no successful benchmarks in a given period, no results will be written. The process of the script is roughly as follows: -- Gather all the benchmark results from GH API into job reports (JobReport dataclass) +- Asynchronously gather all the benchmark results from GH API into job reports (JobReport dataclass) - Use cache if possible to avoid unnecessary GH API queries - Transform the gathered results into data for a particular benchmark sorted by an appropriate commit timestamp. @@ -33,9 +45,11 @@ Dependencies for the script: - Used as a template engine for the HTML. """ +import asyncio import json import logging import logging.config +import math import os import re import shutil @@ -45,10 +59,13 @@ import tempfile import zipfile from argparse import ArgumentParser, RawDescriptionHelpFormatter from csv import DictWriter -from datetime import datetime, timedelta, date +from datetime import datetime, timedelta +from enum import Enum from os import path from typing import List, Dict, Optional, Any, Union, Set from dataclasses import dataclass +import xml.etree.ElementTree as ET + if not (sys.version_info.major >= 3 and sys.version_info.minor >= 7): print("ERROR: python version lower than 3.7") @@ -58,18 +75,48 @@ try: import numpy as np import jinja2 except ModuleNotFoundError as err: - print("ERROR: One of pandas, numpy, or jinja2 packages not installed") + print("ERROR: One of pandas, numpy, or jinja2 packages not installed", file=sys.stderr) exit(1) -BENCH_RUN_NAME = "Benchmark Engine" DATE_FORMAT = "%Y-%m-%d" -# Workflod ID of engine benchmarks, got via `gh api '/repos/enso-org/enso/actions/workflows'` -BENCH_WORKFLOW_ID = 29450898 +ENGINE_BENCH_WORKFLOW_ID = 29450898 +""" +Workflow ID of engine benchmarks, got via `gh api +'/repos/enso-org/enso/actions/workflows'`. +The name of the workflow is 'Benchmark Engine' +""" +NEW_ENGINE_BENCH_WORKFLOW_ID = 67075764 +""" +Workflow ID for 'Benchmark Engine' workflow, which is the new workflow +since 2023-08-22. +""" +STDLIBS_BENCH_WORKFLOW_ID = 66661001 +""" +Workflow ID of stdlibs benchmarks, got via `gh api +'/repos/enso-org/enso/actions/workflows'`. +The name is 'Benchmark Standard Libraries' +""" GH_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" """ Date format as returned from responses in GH API""" ENSO_COMMIT_BASE_URL = "https://github.com/enso-org/enso/commit/" -JINJA_TEMPLATE = "template_jinja.html" +JINJA_TEMPLATE = "templates/template_jinja.html" """ Path to the Jinja HTML template """ +TEMPLATES_DIR = "templates" +GENERATED_SITE_DIR = "generated_site" +GH_ARTIFACT_RETENTION_PERIOD = timedelta(days=90) + + +class Source(Enum): + ENGINE = "engine" + STDLIB = "stdlib" + + def workflow_ids(self) -> List[int]: + if self == Source.ENGINE: + return [ENGINE_BENCH_WORKFLOW_ID, NEW_ENGINE_BENCH_WORKFLOW_ID] + elif self == Source.STDLIB: + return [STDLIBS_BENCH_WORKFLOW_ID] + else: + raise ValueError(f"Unknown source {self}") @dataclass @@ -155,16 +202,21 @@ class TemplateBenchData: """ Data for one benchmark label (with a unique name and ID) """ id: str """ ID of the benchmark, must not contain dots """ + name: str + """ Human readable name of the benchmark """ branches_datapoints: Dict[str, List[BenchDatapoint]] """ Mapping of branches to datapoints for that branch """ @dataclass class JinjaData: + bench_source: Source bench_datas: List[TemplateBenchData] branches: List[str] since: datetime until: datetime + display_since: datetime + """ The date from which all the datapoints are first displayed """ def _parse_bench_run_from_json(obj: Dict[Any, Any]) -> JobRun: @@ -213,24 +265,22 @@ def _bench_report_to_json(bench_report: JobReport) -> Dict[Any, Any]: } -def _parse_bench_report_from_xml(bench_report_xml: str, bench_run: JobRun) -> "JobReport": - logging.debug(f"Parsing BenchReport from {bench_report_xml}") - with open(bench_report_xml, "r") as f: - lines = f.readlines() - label_pattern = re.compile("