Merge pull request #479 from primer/scoreboard

Add "scoreboard" test suite

.travis.yml

@@ -12,6 +12,7 @@ env:
     - secure: "KcGydAqL7ryDh2rTJJB4wU8NE5BRtnrRXDEcPBScSscO3zFsHXHBDvvO04B/9hFVatXzGYXmkn+FZ0P9QikhvebzdwwyqUG2SKFiHhMvbX0m0WtAhn5NqDuKU1r5qy5YQ18r/tiLfC9GSAlEpfLAH58pwpcn8srV3Mn/yKvlrfs="
 
 script:
+  - cd meta/scoreboard; npm install; npm test
   - npm test
 
 after_success:

meta/scoreboard/index.js

@@ -0,0 +1,96 @@
+const {basename, join, resolve} = require('path')
+const PromiseQueue = require('p-queue')
+const execa = require('execa')
+const globby = require('globby')
+const rootDir = resolve(__dirname, '../..')
+const lernaConfig = require(join(rootDir, 'lerna.json'))
+const modulesDir = join(rootDir, 'modules')
+require('console.table')
+
+const unique = list => Array.from(new Set(list)).sort()
+
+const matchAll = (pattern, text) => {
+  const matches = []
+  let match
+  while (match = pattern.exec(text)) {
+    matches.push(match)
+  }
+  return matches
+}
+
+const checks = {
+  'has stories': (module, key) => {
+    return globby(join(module.path, '**/stories.js'))
+      .then(files => ({
+        [key]: files.length > 0 ? 'yes' : 'no'
+      }))
+  },
+  'docs test': (module, key) => {
+    return execa(join(rootDir, 'script/test-docs'), {
+      cwd: module.path
+    })
+    .then(result => ({[key]: 'pass'}))
+    .catch(({stderr}) => {
+      const pattern = /("\.[-\w]+") is not documented/g
+      const matches = matchAll(pattern, stderr)
+        .map(match => match[1])
+      let missing = matches ? Array.from(matches) : []
+      const max = 5
+      if (missing.length > max) {
+        const more = missing.length - max
+        missing = missing.slice(0, max).concat(`and ${more} more...`)
+      }
+      return {
+        [key]: 'FAIL',
+        'missing docs': unique(missing).join(', ')
+      }
+    })
+  }
+}
+
+const args = process.argv.slice(2)
+
+const modules = args.length
+  ? Promise.resolve(args)
+  : globby(join(modulesDir, 'primer-*'))
+
+modules
+  .then(moduleDirs => {
+    console.log('Found %d module directories', moduleDirs.length)
+    return moduleDirs
+      .map(path => ({
+        path,
+        name: basename(path),
+        pkg: require(join(path, 'package.json'))
+      }))
+      .filter(({pkg}) => pkg.primer.module_type !== 'meta')
+  })
+  .then(modules => {
+    console.log('Filtered to %d modules (excluding meta-packages)', modules.length)
+
+    const queue = new PromiseQueue({concurrency: 3})
+
+    for (const module of modules) {
+      module.checks = {}
+      for (const [name, check] of Object.entries(checks)) {
+        queue.add(() => {
+          // console.warn(`? check: ${module.name} ${name}`)
+          return check(module, name)
+            .then(result => {
+              Object.assign(module.checks, result)
+            })
+        })
+      }
+    }
+
+    console.warn(`Running ${queue.size} checks...`)
+    return queue.onIdle().then(() => modules)
+  })
+  .then(modules => {
+    console.warn('ran tests on %d modules', modules.length)
+    const rows = modules.map(({name, checks}) => {
+      return Object.assign({'package': name}, checks)
+    })
+    console.table(rows)
+  })
+

meta/scoreboard/package.json

@@ -0,0 +1,12 @@
+{
+  "private": true,
+  "scripts": {
+    "test": "node index.js"
+  },
+  "devDependencies": {
+    "console.table": "^0.10.0",
+    "execa": "^0.10.0",
+    "globby": "^6.1.0",
+    "p-queue": "^2.4.2"
+  }
+}

modules/primer-alerts/lib/flash.scss

@@ -72,6 +72,7 @@
   border-radius: 0;
 }
 
+// FIXME deprecate this
 .warning {
   padding: $em-spacer-5;
   margin-bottom: 0.8em;

modules/primer-alerts/package.json

@@ -10,7 +10,10 @@
   "main": "build/index.js",
   "primer": {
     "category": "product",
-    "module_type": "components"
+    "module_type": "components",
+    "class_whitelist": [
+      "warning"
+    ]
   },
   "files": [
     "index.scss",

modules/primer-base/package.json

@@ -10,7 +10,11 @@
   "main": "build/index.js",
   "primer": {
     "category": "core",
-    "module_type": "support"
+    "module_type": "support",
+    "class_whitelist": [
+      "octicon",
+      "rule"
+    ]
   },
   "files": [
     "index.scss",

modules/primer-marketing-utilities/package.json

@@ -10,7 +10,11 @@
   "main": "build/index.js",
   "primer": {
     "category": "marketing",
-    "module_type": "utilities"
+    "module_type": "utilities",
+    "class_whitelist": [
+      "border-??-*",
+      "position-??-*"
+    ]
   },
   "files": [
     "index.scss",

package.json

@@ -24,6 +24,7 @@
     "@storybook/addon-options": "^3.2.6",
     "@storybook/react": "^3.2.12",
     "ava": "^0.23.0",
+    "babel-core": "^6.26.3",
     "babel-preset-env": "^1.6.0",
     "babel-preset-minify": "^0.2.0",
     "babel-preset-react": "^6.24.1",
@@ -38,6 +39,7 @@
     "isomorphic-fetch": "^2.2.1",
     "lerna": "2.4.0",
     "lerna-changelog": "^0.7.0",
+    "minimatch": "^3.0.4",
     "node-sass": "^4.5.3",
     "npm-run-all": "^4.0.2",
     "octicons": "^6.0.1",

script/test-docs

@@ -1,3 +1,3 @@
 #!/bin/bash
 set -e
-$(dirname $0)/npm-run ava --verbose $(dirname $0)/../tests/modules/test-document-styles.js
+$(dirname $0)/npm-run ava --verbose $(dirname $0)/../tests/modules/test-document-styles.js "$@"

tests/modules/test-document-styles.js

@@ -1,73 +1,95 @@
-const test = require("ava")
-const css = require(process.cwd())
-const fs = require("fs")
-const glob = require("glob")
+const {join} = require('path')
+const fse = require('fs-extra')
+const globby = require('globby')
+const test = require('ava')
+const minimatch = require('minimatch')
 
-let selectors
-let classnames
+const cwd = process.cwd()
+const css = require(cwd)
+const pkg = require(join(cwd, 'package.json'))
 
+const unique = list => Array.from(new Set(list)).sort()
+
+/*
+ * These are the regular expressions that match what we
+ * expect to be class name instances in the docs.
+ * Patterns should group the matched class name(s) such that:
+ *
+ * ```js
+ * const [, klass, ] = pattern.exec(content)
+ * ```
+ */
 const classPatterns = [
   // HTML class attributes
   /class="([^"]+)"/ig,
+  /:class ?=> "([^"]+)"/g,
+  /class: "([^"]+)"/g,
   // assume that ERB helpers generate an element with the same class
   /<%= (\w+)\b/g,
 ]
 
+const whitelistClasses = (pkg.primer ? pkg.primer.class_whitelist || [] : [])
+  .concat('js*')
+
+const isWhitelisted = klass => {
+  return whitelistClasses.some(glob => minimatch(klass, glob))
+}
+
 // Find unique selectors from the cssstats selector list
-function uniqueSelectors(s) {
-  s = s.map(s => {
+function uniqueSelectors(selectors) {
+  return unique(selectors.map(s => {
     // split multi-selectors into last class used .foo .bar .baz
-    return s.split(" ").pop()
+    return s.split(' ').pop()
   })
   .filter(s => {
-    // remove any selector that aren't just regular classnames eg. ::hover [type]
+    // only match exact class selectors
     return s.match(/^\.[a-z\-_]+$/ig)
-  })
-
-  // return only the unique selectors
-  return [...new Set(s)]
+  }))
 }
 
 // From the given glob sources array, read the files and return found classnames
-function documentedClassnames(sources) {
-  const classes = []
-  const files = sources.reduce((acc, pattern) => {
-    return acc.concat(glob.sync(pattern))
-  }, [])
-
-  files.forEach(file => {
-    let match = null
-    const content = fs.readFileSync(file, "utf8")
-
-    classPatterns.forEach(pattern => {
-      // match each pattern against the source
-      while (match = pattern.exec(content)) {
-        // get the matched classnames and split by whitespace into classes
-        const klasses = match[1].trim().split(/\s+/)
-        classes.push(...klasses)
-      }
+function getDocumentedClassnames(sources) {
+  return globby(sources)
+    .then(paths => {
+      return Promise.all(paths.map(path => {
+        return fse.readFile(path, 'utf8')
+          .then(content => ({path, content}))
+      }))
     })
-  })
-
-  // return only the unique classnames
-  return Array.from(new Set(classes))
+    .then(files => {
+      return files.reduce((classes, {path, content}) => {
+        classPatterns.forEach(pattern => {
+          let match
+          while (match = pattern.exec(content)) {
+            // get the matched classnames and split by whitespace into classes
+            let klasses = match[1].trim().split(/\s+/)
+            classes.push(...klasses)
+          }
+        })
+        return classes
+      }, [])
+    })
+    .then(classes => unique(classes))
 }
 
-// Before all the tests get the selectors and classnames
+const selectors = uniqueSelectors(css.cssstats.selectors.values)
+let classnames
 test.before(t => {
-  selectors = uniqueSelectors(css.cssstats.selectors.values)
-  classnames = documentedClassnames([
-    "docs/*.md",
-    "README.md"
-  ])
+  return getDocumentedClassnames([
+      'docs/*.md',
+      'README.md'
+    ])
+    .then(_ => (classnames = _))
 })
 
-test("Every selector class is documented", t => {
-  const undocumented = []
-  selectors.forEach(selector => {
-    if (!classnames.includes(selector.replace(/^\./, ""))) {
-      undocumented.push(selector)
+selectors.forEach(selector => {
+  const klass = selector.replace(/^\./, '')
+  test(`Selector "${selector}" is documented/whitelisted`, t => {
+    t.plan(1)
+    if (isWhitelisted(klass)) {
+      t.pass(`Selector "${selector}" is whitelisted`)
+    } else {
+      t.is(classnames.includes(klass), true, `Selector "${selector}" is not documented`)
     }
   })
-  t.is(undocumented.length, 0, `I did not find documentation for the "${undocumented.join(", ")}" selector(s) in the ${process.env.npm_package_name} module.`);
 })