Merge pull request #907 from NoRedInk/bat/a11y-axe-checks

Fold axe and percy checks together

Shakefile.hs

@@ -51,8 +51,7 @@ main = do
             "log/elm-test-styleguide.txt",
             "log/elm-review.txt",
             "log/elm-review-styleguide.txt",
-            "log/axe-report.txt",
-            "log/percy-tests.txt",
+            "log/puppeteer-tests.txt",
             "log/forbidden-imports-report.txt",
             "log/check-exposed.txt",
             "log/format.txt",
@@ -112,22 +111,16 @@ main = do
         need elmFiles
         cmd (WithStdout True) (FileStdout out) "elm-format" "--validate" placesToLook
 
-      "log/percy-tests.txt" %> \out -> do
+      "log/puppeteer-tests.txt" %> \out -> do
         percyToken <- getEnv "PERCY_TOKEN"
         case percyToken of
           Nothing -> do
-            writeFileChanged out "Skipped running Percy tests, PERCY_TOKEN not set."
+            writeFileChanged out "PERCY_TOKEN not set, so skipping visual diff testing."
+            need ["log/npm-install.txt", "log/public.txt"]
+            cmd (WithStdout True) (FileStdout out) "script/puppeteer-tests-no-percy.sh"
           Just _ -> do
             need ["log/npm-install.txt", "log/public.txt"]
-            cmd (WithStdout True) (FileStdout out) "script/percy-tests.sh"
-
-      "log/axe-report.json" %> \out -> do
-        need ["log/npm-install.txt", "script/run-axe.sh", "script/axe-puppeteer.js", "log/public.txt"]
-        cmd (WithStdout True) (FileStdout out) "script/run-axe.sh"
-
-      "log/axe-report.txt" %> \out -> do
-        need ["log/axe-report.json", "script/format-axe-report.sh", "script/axe-report.jq"]
-        cmd (WithStdout True) (FileStdout out) "script/format-axe-report.sh" "log/axe-report.json"
+            cmd (WithStdout True) (FileStdout out) "script/puppeteer-tests-percy.sh"
 
       "log/forbidden-imports-report.txt" %> \out -> do
         need ["forbidden-imports.toml"]

package-lock.json

@@ -4,6 +4,21 @@
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
+    "@axe-core/puppeteer": {
+      "version": "4.4.2",
+      "resolved": "https://registry.npmjs.org/@axe-core/puppeteer/-/puppeteer-4.4.2.tgz",
+      "integrity": "sha512-HsiXUALjQ5fcWZZgGUvYGr/b7qvWbQXeDuW2z+2YYOJsavlPV9z0IGdm4rmX0lmsdJ9usA5vq5LNLiz23ZyXmw==",
+      "requires": {
+        "axe-core": "^4.4.1"
+      },
+      "dependencies": {
+        "axe-core": {
+          "version": "4.4.1",
+          "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.1.tgz",
+          "integrity": "sha512-gd1kmb21kwNuWr6BQz8fv6GNECPBnUasepcoLbekws23NVBLODdsClRZ+bQ8+9Uomf3Sm3+Vwn0oYG9NvwnJCw=="
+        }
+      }
+    },
     "@babel/code-frame": {
       "version": "7.16.7",
       "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz",

package.json

@@ -33,6 +33,7 @@
     "request": "^2.88.0"
   },
   "dependencies": {
+    "@axe-core/puppeteer": "^4.4.2",
     "axe-core": "3.5.6",
     "expect": "^27.4.6",
     "http-server": "^14.1.0",

script/axe-puppeteer.js

@@ -1,63 +0,0 @@
-// this script is from the example axe docs at https://github.com/dequelabs/axe-core/blob/develop/doc/examples/puppeteer/axe-puppeteer.js
-// it is licensed MPL 2.0: https://github.com/dequelabs/axe-core/blob/develop/LICENSE
-
-const puppeteer = require('puppeteer');
-const axeCore = require('axe-core');
-const { parse: parseURL } = require('url');
-const assert = require('assert');
-
-// Cheap URL validation
-const isValidURL = input => {
-    const u = parseURL(input);
-    return u.protocol && u.host;
-};
-
-// node axe-puppeteer.js <url>
-const url = process.argv[2];
-assert(isValidURL(url), 'Invalid URL');
-
-const main = async url => {
-    let browser;
-    let results;
-    try {
-        // Setup Puppeteer
-        browser = await puppeteer.launch();
-
-        // Get new page
-        const page = await browser.newPage();
-        await page.goto(url);
-
-        // Inject and run axe-core
-        const handle = await page.evaluateHandle(`
-            // Inject axe source code
-            ${axeCore.source}
-            // Run axe
-            axe.run()
-        `);
-
-        // Get the results from `axe.run()`.
-        results = await handle.jsonValue();
-        // Destroy the handle & return axe results.
-        await handle.dispose();
-    } catch (err) {
-        // Ensure we close the puppeteer connection when possible
-        if (browser) {
-            await browser.close();
-        }
-
-        // Re-throw
-        throw err;
-    }
-
-    await browser.close();
-    return results;
-};
-
-main(url)
-    .then(results => {
-        console.log(JSON.stringify(results));
-    })
-    .catch(err => {
-        console.error('Error running axe-core:', err.message);
-        process.exit(1);
-    });

script/axe-report.jq

@@ -1,17 +0,0 @@
-def node: "  at \(.target | join(" ")):\n\n    \(.failureSummary | gsub("\n"; "\n    "))";
-def violation: "\(.id): \(.impact) violation with \(.nodes | length) instances.\n\n\(.help) (\(.helpUrl))\n\n\(.nodes | map(node) | join("\n\n"))";
-
-"Tested \(.url) with \(.testEngine.name) \(.testEngine.version) at \(.timestamp)
-
-Agent information:
-
-  \(.testEnvironment | to_entries | map("\(.key): \(.value)") | join("\n  "))
-
-Summary: \(.passes | length) passes | \(.violations | map(.nodes | length) | add) violations | \(.incomplete | map(.nodes | length) | add) incomplete | \(.inapplicable | length) inapplicable
-
-Violations:
-
-----------
-
-\(.violations | map(violation) | join("\n\n----------\n\n"))
-"

script/format-axe-report.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-JSON_FILE="${1:-}"
-if test -z "$JSON_FILE"; then
-    echo "Please specify a report JSON file as the first argument."
-    exit 1
-fi
-
-jq -r -f script/axe-report.jq "$JSON_FILE"
-
-NUM_ERRORS="$(jq '.violations | map(.nodes | length) | add' "$JSON_FILE")"
-if test "$NUM_ERRORS" -lt 4;
-then
-    echo "$NUM_ERRORS accessibility errors"
-    exit 1
-fi

script/percy-tests.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-npx percy exec -- mocha script/percy-tests.js --exit

script/puppeteer-tests-no-percy.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -euo pipefail
+npx mocha script/puppeteer-tests.js --timeout 15000 --exit

script/puppeteer-tests-percy.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -euo pipefail
+npx percy exec -- mocha script/puppeteer-tests.js --timeout 50000 --exit

script/puppeteer-tests.js

@@ -8,8 +8,10 @@ const platform = require('os').platform();
 const puppeteerArgs = /^win/.test(platform) ? [] : ['--single-process'];
 const PORT = process.env.PORT_NUMBER || 8000;
 
-describe('Visual tests', function () {
-  this.timeout(30000);
+const { AxePuppeteer } = require('@axe-core/puppeteer');
+const assert = require('assert');
+
+describe('UI tests', function () {
   let page;
   let server;
   let browser;
@@ -29,11 +31,28 @@ describe('Visual tests', function () {
     server.close();
   });
 
+  const handleAxeResults = function(name, results) {
+    const violations = results["violations"];
+    if (violations.length > 0) {
+      violations.map(function(violation) {
+        console.log("\n\n", violation["id"], ":", violation["description"])
+        console.log(violation["help"])
+        console.log(violation["helpUrl"])
+
+        console.table(violation["nodes"], ["html"])
+      });
+      assert.fail(`Expected no axe violations in ${name} but got ${violations.length} violations`)
+    }
+  }
+
   const defaultProcessing = async (name, location) => {
     await page.goto(location)
     await page.waitFor(`#${name.replace(".", "-")}`)
     await percySnapshot(page, name)
     console.log(`Snapshot complete for ${name}`)
+
+    const results = await new AxePuppeteer(page).disableRules(skippedRules[name] || []).analyze();
+    handleAxeResults(name, results);
   }
 
   const iconProcessing = async(name, location) => {
@@ -47,6 +66,39 @@ describe('Visual tests', function () {
     await percySnapshot(page, `${name} - display icon names`)
 
     console.log(`Snapshots complete for ${name}`)
+
+    const results = await new AxePuppeteer(page).disableRules(skippedRules[name] || []).analyze();
+    handleAxeResults(name, results);
+  }
+
+  const skippedRules = {
+    'Accordion': ['heading-order', 'region'],
+    'Balloon': ['color-contrast', 'heading-order', 'label'],
+    'Button': ['heading-order'],
+    'Checkbox': ['heading-order', 'region'],
+    'ClickableSvg': ['heading-order'],
+    'ClickableText': ['heading-order'],
+    'Colors': ['heading-order'],
+    'Confetti': ['heading-order'],
+    'Container': ['heading-order'],
+    'DisclosureIndicator': ['heading-order'],
+    'Fonts': ['heading-order'],
+    'Heading': ['heading-order'],
+    'Loading': ['color-contrast'],
+    'Menu': ['heading-order', 'region'],
+    'Modal': ['region'],
+    'Message': ['heading-order', 'region'],
+    'Page': ['color-contrast', 'heading-order', 'select-name'],
+    'RadioButton': ['duplicate-id', 'region'],
+    'SegmentedControl': ['heading-order', 'region'],
+    'Select': ['label'],
+    'SideNav': ['heading-order'],
+    'SortableTable': ['heading-order'],
+    'Switch': ['aria-allowed-attr', 'heading-order'],
+    'Table': ['heading-order'],
+    'Tabs': ['region'],
+    'Text': ['heading-order'],
+    'Tooltip': ['heading-order'],
   }
 
   const specialProcessing = {
@@ -56,6 +108,10 @@ describe('Visual tests', function () {
       await page.click('#launch-modal')
       await page.waitFor('[role="dialog"]')
       await percySnapshot(page, 'Full Info Modal')
+
+      const results = await new AxePuppeteer(page).disableRules(skippedRules[name] || []).analyze();
+      handleAxeResults(name, results);
+
       await page.click('[aria-label="Close modal"]')
       await page.select('select', 'warning')
       await page.click('#launch-modal')
@@ -74,7 +130,18 @@ describe('Visual tests', function () {
     await page.goto(`http://localhost:${PORT}`);
     await page.$('#maincontent');
     await percySnapshot(page, this.test.fullTitle());
+
+    const results = await new AxePuppeteer(page).
+      disableRules([
+        "aria-hidden-focus",
+        "color-contrast",
+        "duplicate-id-aria",
+        "duplicate-id",
+      ]).analyze();
+
     page.close();
+
+    handleAxeResults("index view", results);
   });
 
   it('Doodads', async function () {
@@ -94,5 +161,7 @@ describe('Visual tests', function () {
         }
       )
     }, Promise.resolve())
+
+    page.close();
   })
 });