Automated Browser and App Testing

.gitignore

@@ -14,4 +14,9 @@ elm.js
 
 examples/temporary/
 
-.vscode
+.vscode
+
+
+sauce.env
+
+__pycache__

tests/Makefile

@@ -0,0 +1,13 @@
+compile:
+	elm make Tests/Run.elm --output=elm.js
+
+local_test:
+	source env/bin/activate
+	source sauce.env
+	python3 automation/selenium_test.py --local
+
+remote_test:
+	source env/bin/activate
+	source sauce.env
+	python3 automation/run.py
+	

tests/README.md

@@ -22,4 +22,34 @@ In order to run the test locally, compile
 
 `elm make Tests/Run.elm --output elm.js`
 
-and then open `gather-styles.html`
+and then open `gather-styles.html`
+
+Alternatively, you can run the tests locally using selenium by running:
+
+`python3 automation/run.py --local`
+
+from the `tests` directory.
+
+
+# Running on Sauce Labs
+
+In order to automate some of this stuff, this test can be run on Sauce Labs.
+
+If you have an account, you'll need to create a `elm-ui/sauce.env` file, which has the following fields:
+
+```
+export SAUCE_ACCESS_KEY={your key}
+export SAUCE_USERNAME={your username}
+```
+You can then run.
+
+`python3 automation/run.py`
+
+**Note**: The compiled `elm-ui` test needs to be made public somewhere in order for this to work.  At the moment it's at my github.io account, though something more permanenet might be set up.
+
+
+# Sauce Labs References
+
+https://wiki.saucelabs.com/display/DOCS/Platform+Configurator#/
+https://wiki.saucelabs.com/display/DOCS/Sauce+Labs+Basics
+https://github.com/saucelabs-sample-test-frameworks/Python-Pytest-Selenium

tests/automation/environments.py

@@ -0,0 +1,91 @@
+
+
+import appium
+import selenium
+import selenium.webdriver.firefox.options
+import os
+
+browsers = {'Windows 10':
+            {'internet explorer': ['11.285'], 'MicrosoftEdge': ['18.17763'], 'chrome': ['71.0'], 'firefox': ['64.0']
+             },
+            "macOS 10.14":
+            {'chrome': ['71.0'], 'firefox': ['64.0'], 'safari': ['12.0']}
+
+            }
+
+
+def desktop():
+
+    caps = []
+    for platform, brows in browsers.items():
+        for browser, versions in brows.items():
+            for vers in versions:
+                cap = {'platform': platform,
+                       'browserName': browser, 'version': vers}
+
+                caps.append(
+                    {'driver_type': 'desktop',
+                     'capabilities': cap})
+
+    return caps
+
+
+def local():
+    options = selenium.webdriver.firefox.options.Options()
+    options.set_headless(True)
+
+    driver = selenium.webdriver.Firefox(options=options)
+    caps = {}
+    caps['browserName'] = "firefox"
+    caps['platform'] = "macOS 10.14"
+    return [{'driver_type': 'local', 'capabilities': caps}]
+
+
+def mobile():
+
+    caps = {}
+    caps['browserName'] = "Safari"
+    caps['appiumVersion'] = "1.9.1"
+    caps['deviceName'] = "iPhone XS Simulator"
+    caps['deviceOrientation'] = "portrait"
+    caps['platformVersion'] = "12.0"
+    caps['platformName'] = "iOS"
+    return [{'capabilities': caps, 'driver_type': 'app'}]
+
+
+def get_credentials():
+    return {'username': os.environ['SAUCE_USERNAME'], 'access_key': os.environ['SAUCE_ACCESS_KEY']}
+
+
+def start_driver(env, capabilities):
+    if env == 'app':
+        return app(capabilities)
+    elif env == 'desktop':
+        return remote(capabilities)
+    else:
+        options = selenium.webdriver.firefox.options.Options()
+        options.set_headless(True)
+
+        return selenium.webdriver.Firefox(options=options)
+
+
+def remote(desired_cap):
+    creds = get_credentials()
+    driver = selenium.webdriver.Remote(
+        command_executor='http://{username}:{key}@ondemand.saucelabs.com:80/wd/hub'.format(
+            username=creds['username'],
+            key=creds['access_key']),
+        desired_capabilities=desired_cap)
+
+    return driver
+
+
+def app(desired_cap):
+    creds = get_credentials()
+    driver = appium.webdriver.Remote(
+        command_executor='http://{username}:{key}@ondemand.saucelabs.com:80/wd/hub'.format(
+            username=creds['username'],
+            key=creds['access_key']),
+        desired_capabilities=desired_cap)
+
+    return driver

tests/automation/run.py

@@ -0,0 +1,105 @@
+import os
+import time
+import pprint
+import json
+import sys
+
+import environments
+
+
+def run(env, url):
+    driver = environments.start_driver(env['driver_type'], env['capabilities'])
+
+    test_run = run_test(driver, url)
+    return {'capabilities': env['capabilities'], 'failures': test_run, 'success': test_run == []}
+
+
+def log_results(all_results):
+
+    for group in all_results:
+        print(group["label"])
+        all_pass = True
+        for result in group["results"]:
+            name = result[0]
+            val = result[1]
+
+            if val is not None:
+                print("    Failed - " + name)
+                all_pass = False
+                # { given : Maybe String
+                # , description : String
+                # , reason : Failure.Reason
+                # }
+                pprint.pprint(val)
+
+        if all_pass:
+            print("   All passed!")
+
+
+def only_failures(all_results):
+    failures = []
+    for group in all_results:
+
+        for result in group["results"]:
+            name = result[0]
+            val = result[1]
+
+            if val is not None:
+                failures.append(
+                    {'group': group['label'], 'description': name, 'result': val})
+
+    return failures
+
+
+def run_test(driver, url):
+    print("Running test...")
+    results = None
+    try:
+        # print("Opening Browser")
+        driver.get(url)
+
+        # print("Checking Results")
+        time.sleep(20)
+        for x in range(60):
+            feedback = driver.execute_script('return test_results')
+            if feedback != "waiting..":
+                results = only_failures(feedback)
+                # log_results(feedback)
+                break
+            time.sleep(1)
+    except Exception as inst:
+        print(type(inst))
+        print(inst.args)
+        print(inst)
+        driver.quit()
+    else:
+        print("Finished!")
+        driver.quit()
+
+    return results
+
+
+if __name__ == "__main__":
+
+    url = "http://mdgriffith.github.io/elm-ui/tests/base/"
+
+    results = []
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "--local":
+            envs = environments.local()
+        else:
+            raise "Unknown argument, run --local to run locally."
+    else:
+        envs = environments.desktop()
+        envs.extend(environments.mobile())
+
+    for env in envs:
+        pprint.pprint(env['capabilities'])
+        result = run(env, url)
+        results.append(result)
+        print("--")
+
+    pprint.pprint(results)
+
+    with open("results/test-results.json", 'w') as RESULTS:
+        RESULTS.write(json.dumps(results, indent=4))

tests/automation/selenium_test.py

@@ -1,59 +0,0 @@
-from selenium import webdriver
-from selenium.webdriver.common.keys import Keys
-from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-import os
-import time
-
-
-def browsers():
-
-    capabilities = [
-        {'platform': "Mac OS X 10.9",
-         'browserName': "chrome",
-         'version': "31",
-         }
-    ]
-    return capabilies
-
-
-def run_local():
-    driver = webdriver.Firefox()
-    local_file = "file://" + os.getcwd() + "/gather-styles.html"
-    run_test(driver, local_file)
-
-
-def get_credentials():
-    return {'username': os.environ['SAUCE_USERNAME'], 'access_key': os.environ['SAUCE_ACCESS_KEY']}
-
-
-def run_remote(desired_cap):
-    creds = get_credentials()
-    driver = webdriver.Remote(
-        command_executor='http://{username}:{key}@ondemand.saucelabs.com:80/wd/hub'.format(
-            username=creds['username'],
-            key=creds['access_key']),
-        desired_capabilities=desired_cap)
-    run_test(driver)
-
-
-def run_test(driver, url):
-    try:
-        driver.get(local_file)
-        for x in range(5):
-            results = driver.execute_script('return test_results')
-            if results != "waiting..":
-                print(results)
-                break
-            time.sleep(1)
-    except Exception as inst:
-        print(type(inst))
-        print(inst.args)
-        print(inst)
-        driver.quit()
-    else:
-        print("quitting browser")
-        driver.quit()
-
-
-if __name__ == "__main__":
-    run_local()