move package.json to root level so it can be shared by tests and benchmarks

This commit is contained in:
Matthew Griffith 2019-07-05 21:53:08 -04:00
parent 67926b4706
commit acfadbd04c
9 changed files with 2398 additions and 374 deletions

30
package.json Normal file
View File

@ -0,0 +1,30 @@
{
"name": "elm-ui-testing",
"version": "1.0.0",
"description": "Run the Elm UI benchmarks and render tests",
"main": "index.js",
"scripts": {
"test": "elm-test",
"test-render": "node tests/automation/run.js --chrome",
"test-render-sauce": "source source.env; node tests/automation/run.js",
"bench": "node benchmarks/runtime/index.js"
},
"repository": {
"type": "git",
"url": "git+https://github.com/mdgriffith/elm-ui.git"
},
"author": "Matthew Griffith <mdg.griffith@gmail.com> (http://mechanical-elephant.com/)",
"license": "ISC",
"bugs": {
"url": "https://github.com/mdgriffith/elm-ui/issues"
},
"homepage": "https://github.com/mdgriffith/elm-ui#readme",
"dependencies": {
"chalk": "^2.4.2",
"commander": "^2.20.0",
"elm-test": "^0.19.0-rev6",
"node-elm-compiler": "^5.0.3",
"puppeteer": "^1.18.0",
"selenium-webdriver": "^4.0.0-alpha.4"
}
}

View File

@ -1,13 +0,0 @@
compile:
elm make Tests/Run.elm --output=elm.js
local_test:
source env/bin/activate
source sauce.env
python3 automation/selenium_test.py --local
remote_test:
source env/bin/activate
source sauce.env
python3 automation/run.py

View File

@ -1,55 +0,0 @@
# Elm UI Test Suite
There are two types of tests for Elm UI. The normal kind, which are located in `tests/suite/`
These can be run using [`elm-test`](https://github.com/elm-explorations/test) and can be run via
```bash
elm-test
```
at the `elm-ui` root.
# Layout Testing
`elm-ui` also needs to ensure that all layouts work as expected on all browsers.
In order to do this, we need a different testing environment.
So, the tests in `elm-ui/tests/Tests` will render output, then harvest bounding boxes form the browser, and run the test on the resulting data.
In order to run the test locally, compile
`elm make Tests/Run.elm --output elm.js`
and then open `gather-styles.html`
Alternatively, you can run the tests locally using selenium by running:
`python3 automation/run.py --local`
from the `tests` directory.
# Running on Sauce Labs
In order to automate some of this stuff, this test can be run on Sauce Labs.
If you have an account, you'll need to create a `elm-ui/sauce.env` file, which has the following fields:
```
export SAUCE_ACCESS_KEY={your key}
export SAUCE_USERNAME={your username}
```
You can then run.
`python3 automation/run.py`
**Note**: The compiled `elm-ui` test needs to be made public somewhere in order for this to work. At the moment it's at my github.io account, though something more permanenet might be set up.
# Sauce Labs References
https://wiki.saucelabs.com/display/DOCS/Platform+Configurator#/
https://wiki.saucelabs.com/display/DOCS/Sauce+Labs+Basics
https://github.com/saucelabs-sample-test-frameworks/Python-Pytest-Selenium

View File

@ -1,91 +0,0 @@
import appium
import selenium
import selenium.webdriver.firefox.options
import os
browsers = {'Windows 10':
{'internet explorer': ['11.285'], 'MicrosoftEdge': ['18.17763'], 'chrome': ['71.0'], 'firefox': ['64.0']
},
"macOS 10.14":
{'chrome': ['71.0'], 'firefox': ['64.0'], 'safari': ['12.0']}
}
def desktop():
caps = []
for platform, brows in browsers.items():
for browser, versions in brows.items():
for vers in versions:
cap = {'platform': platform,
'browserName': browser, 'version': vers}
caps.append(
{'driver_type': 'desktop',
'capabilities': cap})
return caps
def local():
options = selenium.webdriver.firefox.options.Options()
options.set_headless(True)
driver = selenium.webdriver.Firefox(options=options)
caps = {}
caps['browserName'] = "firefox"
caps['platform'] = "macOS 10.14"
return [{'driver_type': 'local', 'capabilities': caps}]
def mobile():
caps = {}
caps['browserName'] = "Safari"
caps['appiumVersion'] = "1.9.1"
caps['deviceName'] = "iPhone XS Simulator"
caps['deviceOrientation'] = "portrait"
caps['platformVersion'] = "12.0"
caps['platformName'] = "iOS"
return [{'capabilities': caps, 'driver_type': 'app'}]
def get_credentials():
return {'username': os.environ['SAUCE_USERNAME'], 'access_key': os.environ['SAUCE_ACCESS_KEY']}
def start_driver(env, capabilities):
if env == 'app':
return app(capabilities)
elif env == 'desktop':
return remote(capabilities)
else:
options = selenium.webdriver.firefox.options.Options()
options.set_headless(True)
return selenium.webdriver.Firefox(options=options)
def remote(desired_cap):
creds = get_credentials()
driver = selenium.webdriver.Remote(
command_executor='http://{username}:{key}@ondemand.saucelabs.com:80/wd/hub'.format(
username=creds['username'],
key=creds['access_key']),
desired_capabilities=desired_cap)
return driver
def app(desired_cap):
creds = get_credentials()
driver = appium.webdriver.Remote(
command_executor='http://{username}:{key}@ondemand.saucelabs.com:80/wd/hub'.format(
username=creds['username'],
key=creds['access_key']),
desired_capabilities=desired_cap)
return driver

195
tests/automation/run.js Normal file
View File

@ -0,0 +1,195 @@
const compileToString = require("node-elm-compiler").compileToString;
const { Builder, By, Key, until } = require('selenium-webdriver');
const chrome = require('selenium-webdriver/chrome');
const firefox = require('selenium-webdriver/firefox');
const fs = require('fs');
const program = require('commander');
const path = require("path");
const chalk = require('chalk');
program.option('-s, --sauce', 'run tests on sauce labs')
.option('--chrome', 'run only chrome tests')
.option('--firefox', 'run only firefox tests')
.option('--build', 'Set build number for sauce labs')
.option('--name', 'Set run name for sauce labs')
.parse(process.argv)
// 'Windows 10'
const windows = [
{
browser: "chrome"
, browserVersion: "latest"
},
{
browser: "firefox"
, browserVersion: "latest"
},
{
browser: "safari"
, browserVersion: "latest"
},
{
browser: "MicrosoftEdge"
, browserVersion: "latest"
},
{
browser: "internet explorer"
, browserVersion: "latest"
}
]
// "macOS 10.14"
const osx = {
chrome: {
platform: "macOS 10.14"
, browser: "chrome"
, browserVersion: "latest"
},
firefox: {
platform: "macOS 10.14"
, browser: "firefox"
, browserVersion: "latest"
},
safari: {
platform: "macOS 10.14"
, browser: "safari"
, browserVersion: "latest"
}
}
function prepare_envs() {
return []
}
async function compile_and_embed(config) {
var template = fs.readFileSync(config.template)
// we embed the compiled js to avoid having to start a server to read the app.
await compileToString(config.elm, config.elmOptions).then(function (compiled_elm_code) {
const compiled = eval(`\`${template}\``)
fs.writeFileSync(config.target, compiled)
});
}
function prepare_sauce_driver(env) {
const username = process.env.SAUCE_USERNAME;
const accessKey = process.env.SAUCE_ACCESS_KEY;
driver = new webdriver.Builder().withCapabilities({
'browserName': env.browser,
'platformName': env.platform,
'browserVersion': env.browserVersion,
/* Pass Sauce User Name and Access Key */
'sauce:options': {
'username': username,
'accessKey': accessKey,
'build': env.build,
'name': env.name
}
}).usingServer("https://@ondemand.saucelabs.com:443/wd/hub").build();
return driver;
}
async function prepare_local_driver(env) {
const firefoxOptions = new firefox.Options().headless()
const chromeOptions = new chrome.Options().headless()
let driver = await new Builder()
.forBrowser(env.browser)
.setChromeOptions(chromeOptions)
.setFirefoxOptions(firefoxOptions)
.build();
return driver;
}
async function run_test(driver, url) {
var results = null
try {
await driver.get(url);
await driver.wait(until.titleIs('tests finished'), 60000);
results = await driver.executeScript("return test_results")
} finally {
await driver.quit();
}
return results
}
(async () => {
var dir = "tmp"
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
await compile_and_embed({
template: "./tests/automation/templates/gather-styles.html",
target: "./tmp/test.html",
elm: ["Tests/Run.elm"],
elmOptions: { cwd: "./tests" }
})
console.log("Tests done compiling")
if (program.sauce) {
const build = program.build
const name = program.name
var results = []
console.log("TODO: PUBLISH FILE BEFORE TESTING")
var url = "fail"
// Publish to
const envs = prepare_envs({ build: build, name: name })
for (i = 0; i < envs.length; i++) {
var driver = prepare_sauce_driver(envs[i])
results.push(await run_test(driver, url))
}
} else {
if (program.chrome) {
console.log("Running locally on Chrome...")
const driver = await prepare_local_driver(osx.chrome)
var results = await run_test(driver, "file://" + path.resolve('./tmp/test.html'))
print_results(results);
}
if (program.firefox) {
console.log("Running locally on Firefox...")
const driver = await prepare_local_driver(osx.firefox)
var results = await run_test(driver, "file://" + path.resolve('./tmp/test.html'))
print_results(results);
}
}
})();
function print_results(results) {
var passed = 0
var failed = 0
var i;
for (i = 0; i < results.length; i++) {
passed = 0
failed = 0
console.log(results[i].label)
for (j = 0; j < results[i].results.length; j++) {
if (results[i].results[j][1] == null) {
passed = passed + 1
} else {
failed = failed + 1
console.log(" " + chalk.red("fail") + " -> " + results[i].results[j][0])
}
}
if (failed == 0) {
console.log(chalk.green(` All ${passed} tests passed`))
} else {
console.log(chalk.green(` ${passed} tests passed`))
console.log(chalk.red(` ${failed} tests failed`))
}
console.log()
}
}

View File

@ -1,105 +0,0 @@
import os
import time
import pprint
import json
import sys
import environments
def run(env, url):
driver = environments.start_driver(env['driver_type'], env['capabilities'])
test_run = run_test(driver, url)
return {'capabilities': env['capabilities'], 'failures': test_run, 'success': test_run == []}
def log_results(all_results):
for group in all_results:
print(group["label"])
all_pass = True
for result in group["results"]:
name = result[0]
val = result[1]
if val is not None:
print(" Failed - " + name)
all_pass = False
# { given : Maybe String
# , description : String
# , reason : Failure.Reason
# }
pprint.pprint(val)
if all_pass:
print(" All passed!")
def only_failures(all_results):
failures = []
for group in all_results:
for result in group["results"]:
name = result[0]
val = result[1]
if val is not None:
failures.append(
{'group': group['label'], 'description': name, 'result': val})
return failures
def run_test(driver, url):
print("Running test...")
results = None
try:
# print("Opening Browser")
driver.get(url)
# print("Checking Results")
time.sleep(20)
for x in range(60):
feedback = driver.execute_script('return test_results')
if feedback != "waiting..":
results = only_failures(feedback)
# log_results(feedback)
break
time.sleep(1)
except Exception as inst:
print(type(inst))
print(inst.args)
print(inst)
driver.quit()
else:
print("Finished!")
driver.quit()
return results
if __name__ == "__main__":
url = "http://mdgriffith.github.io/elm-ui/tests/base/"
results = []
if len(sys.argv) > 1:
if sys.argv[1] == "--local":
envs = environments.local()
else:
raise "Unknown argument, run --local to run locally."
else:
envs = environments.desktop()
envs.extend(environments.mobile())
for env in envs:
pprint.pprint(env['capabilities'])
result = run(env, url)
results.append(result)
print("--")
pprint.pprint(results)
with open("automation/results/test-results.json", 'w') as RESULTS:
RESULTS.write(json.dumps(results, indent=4))

View File

@ -0,0 +1,105 @@
<!DOCTYPE HTML>
<html>
<head>
<meta charset="UTF-8">
<title>Rendering Benchmark Viewer</title>
<script>
${compiled_elm_code}
</script>
</head>
<style>
</style>
<body id="root"></body>
<script type="text/javascript">
var app = Elm.Tests.Run.init();
var test_results = "waiting.."
app.ports.report.subscribe(function (results) {
test_results = results;
})
app.ports.analyze.subscribe(function (ids) {
// ids : List String
var idsLength = ids.length;
var results = [];
for (var i = 0; i < idsLength; i++) {
var id = ids[i];
var element = document.getElementById(id);
if (element == null) {
console.log("id " + id + " not found");
}
var style = getStyle(element);
var bbox = getBoundingBox(element);
var visible = isVisible(id, bbox);
var result = { "bbox": bbox, "style": style, "id": id, "isVisible": visible };
results.push(result);
}
app.ports.styles.send(results);
});
function isVisible(id, bbox) {
var current = document.getElementById(id);
var result = 0;
if (current == document.elementFromPoint(bbox['left'], bbox['top'])) {
result++;
}
if (current == document.elementFromPoint(bbox['left'], bbox['bottom'] - 1)) {
result++;
}
if (current == document.elementFromPoint(bbox['right'] - 1, bbox['top'])) {
result++;
}
if (current == document.elementFromPoint(bbox['right'] - 1, bbox['bottom'] - 1)) {
result++;
}
if (result == 4) {
return true;
} else {
return false
}
}
function getStyle(element) {
var props = []
var style = window.getComputedStyle(element);
for (var i = style.length; i--;) {
var name = style.item(i);
var value = style.getPropertyValue(name);
props.push([name, value]);
}
return props;
}
function getBoundingBox(element) {
var bbox = element.getBoundingClientRect();
var style = window.getComputedStyle(element, null);
var padding = {
'top': parseFloat(style.getPropertyValue('padding-top'))
, 'bottom': parseFloat(style.getPropertyValue('padding-bottom'))
, 'left': parseFloat(style.getPropertyValue('padding-left'))
, 'right': parseFloat(style.getPropertyValue('padding-right'))
};
return {
'top': bbox.top
, 'bottom': bbox.bottom
, 'left': bbox.left
, 'right': bbox.right
, 'width': bbox.width
, 'height': bbox.height
, 'padding': padding
}
}
</script>
</html>

View File

@ -1,110 +0,0 @@
<!DOCTYPE HTML>
<html>
<head>
<meta charset="UTF-8">
<title>Rendering Benchmark Viewer</title>
<script src="elm.js"></script>
</head>
<style>
</style>
<body id="root"></body>
<script type="text/javascript">
var node = document.getElementById('root');
var app = Elm.Tests.Run.init({ node: node });
var test_results = "waiting.."
app.ports.report.subscribe(function (results) {
test_results = results;
})
app.ports.analyze.subscribe(function (ids) {
// ids : List String
var idsLength = ids.length;
var results = [];
for (var i = 0; i < idsLength; i++) {
var id = ids[i];
var element = document.getElementById(id);
if (element == null) {
console.log("id " + id + " not found");
}
var style = getStyle(element);
var bbox = getBoundingBox(element);
var visible = isVisible(id, bbox);
var result = { "bbox": bbox, "style": style, "id": id, "isVisible": visible };
results.push(result);
}
app.ports.styles.send(results);
});
function isVisible(id, bbox) {
var current = document.getElementById(id);
var result = 0;
if (current == document.elementFromPoint(bbox['left'], bbox['top'])) {
result++;
}
if (current == document.elementFromPoint(bbox['left'], bbox['bottom'] - 1)) {
result++;
}
if (current == document.elementFromPoint(bbox['right'] - 1, bbox['top'])) {
result++;
}
if (current == document.elementFromPoint(bbox['right'] - 1, bbox['bottom'] - 1)) {
result++;
}
if (result == 4) {
return true;
} else {
return false
}
}
function getStyle(element) {
var props = []
var style = window.getComputedStyle(element);
for (var i = style.length; i--;) {
var name = style.item(i);
var value = style.getPropertyValue(name);
props.push([name, value]);
}
return props;
}
function getBoundingBox(element) {
var bbox = element.getBoundingClientRect();
var style = window.getComputedStyle(element, null);
var padding = {
'top': parseFloat(style.getPropertyValue('padding-top'))
, 'bottom': parseFloat(style.getPropertyValue('padding-bottom'))
, 'left': parseFloat(style.getPropertyValue('padding-left'))
, 'right': parseFloat(style.getPropertyValue('padding-right'))
};
return {
'top': bbox.top
, 'bottom': bbox.bottom
, 'left': bbox.left
, 'right': bbox.right
, 'width': bbox.width
, 'height': bbox.height
, 'padding': padding
}
}
</script>
</html>

2068
yarn.lock Normal file

File diff suppressed because it is too large Load Diff