refactor syncing with Metalsmith ✨

docs/lib/add-package-meta.js

@@ -0,0 +1,44 @@
+const {existsSync} = require('fs')
+const {dirname, join, resolve} = require('path')
+
+const cache = {}
+
+module.exports = function addPackageMeta(options = {}) {
+  const {fields, namespace = 'data', log = noop} = options
+  return (files, metal, done) => {
+    const root = metal.source()
+    for (const [path, file] of Object.entries(files)) {
+      const fullPath = join(root, path)
+      const pkg = getPackageRelativeTo(dirname(fullPath))
+      if (pkg) {
+        file[namespace].package = fields ? pluck(pkg, fields) : pkg
+      } else {
+        log('no package.json found relative to', fullPath)
+      }
+    }
+    done()
+  }
+}
+
+function getPackageRelativeTo(dir) {
+  if (dir in cache) {
+    return cache[dir]
+  }
+  while (dir !== root) {
+    const pkgPath = join(dir, 'package.json')
+    if (existsSync(pkgPath)) {
+      return (cache[dir] = require(pkgPath))
+    }
+    dir = resolve(dir, '..')
+  }
+  return false
+}
+
+function pluck(data, fields) {
+  return fields.reduce((out, field) => {
+    out[field] = data[field]
+    return out
+  }, {})
+}
+
+function noop() {}

docs/lib/add-source.js

@@ -0,0 +1,14 @@
+const each = require('./each')
+
+module.exports = function addSource(options = {}) {
+  const {namespace = 'data'} = options
+  for (const key of ['branch', 'repo']) {
+    if (!options[key]) {
+      throw new Error(`addSource() plugin requires options.${key} (got ${JSON.stringify(options[key])})`)
+    }
+  }
+  const {branch, repo} = options
+  return each((file, source) => {
+    file[namespace].source = `https://github.com/${repo}/tree/${branch}/modules/${source}`
+  })
+}

docs/lib/config.js

@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 module.exports = (pluginOptions = {}) => (nextConfig = {}) => {
   const test = pluginOptions.extension || /\.mdx?$/
   const primerSCSS = 'primer/index.scss$'
@@ -18,16 +20,13 @@ module.exports = (pluginOptions = {}) => (nextConfig = {}) => {
 
       config.module.rules.push({
         test,
-        use: [
-          options.defaultLoaders.babel,
-          'mdx-loader'
-        ]
+        use: [options.defaultLoaders.babel, 'mdx-loader']
       })
 
       /**
-      * in production we don't have access to ../modules, so we need to
-      * rewrite the 'primer/index.scss' import to the static CSS build
-      */
+       * in production we don't have access to ../modules, so we need to
+       * rewrite the 'primer/index.scss' import to the static CSS build
+       */
       if (!options.dev && !config.resolve[primerSCSS]) {
         console.warn('*** rewriting primer/index.scss to:', primerCSS)
         config.resolve.alias[primerSCSS] = primerCSS

docs/lib/each.js

@@ -0,0 +1,8 @@
+module.exports = function each(fn) {
+  return (files, metal, done) => {
+    for (const path of Object.keys(files)) {
+      fn(files[path], path, metal)
+    }
+    done()
+  }
+}

docs/lib/filter-by.js

@@ -0,0 +1,10 @@
+module.exports = function filterBy(fn) {
+  return (files, metal, done) => {
+    for (const [key, file] of Object.entries(files)) {
+      if (!fn(file, key, metal)) {
+        delete files[key]
+      }
+    }
+    done()
+  }
+}

docs/lib/ignore.js

@@ -1,16 +1,31 @@
-const {green, red, yellow} = require('colorette')
-const {readFileSync, writeFileSync} = require('fs-extra')
+const {join} = require('path')
+const {existsSync, readFileSync, removeSync, writeFileSync} = require('fs')
 
-const HEADER = '# DO NOT EDIT: automatically generated by ignore.js'
+module.exports = function gitIgnore(options = {}) {
+  const {header, log = noop} = options
+  if (!header) {
+    throw new Error(`getIgnore(): the "header" is required (got: ${JSON.stringify(header)})`)
+  }
+  return (files, metal, done) => {
+    const ignoreFile = join(metal.destination(), '.gitignore')
+    // first, get the list of previously ignored files and remove them (sync)
+    const ignored = getIgnored(ignoreFile, header, log)
+    for (const file of ignored) {
+      if (existsSync(file)) {
+        removeSync(file)
+      }
+    }
+    setIgnored(ignoreFile, Object.keys(files), header, log)
+    done()
+  }
+}
 
-module.exports = {getIgnored, setIgnored}
-
-function readLines(file) {
+function readLines(file, log = noop) {
   let content
   try {
     content = readFileSync(file, 'utf8')
   } catch (error) {
-    console.warn(`ignore file ${file} does not exist!`)
+    log(`ignore file ${file} does not exist!`)
     return []
   }
 
@@ -20,25 +35,27 @@ function readLines(file) {
     .map(line => line.trim())
 }
 
-function getIgnored(file) {
+function getIgnored(file, header, log = noop) {
   const lines = readLines(file)
-  const headerIndex = lines.indexOf(HEADER)
+  const headerIndex = lines.indexOf(header)
   if (headerIndex === -1) {
-    console.warn(`ignore file ${file} does not contain the automatically generated header`)
+    log(`ignore file ${file} does not contain the automatically generated header`)
     return []
   }
 
   return lines.slice(headerIndex + 1).filter(line => line)
 }
 
-function setIgnored(file, files) {
-  const lines = readLines(file)
-  const headerIndex = lines.indexOf(HEADER)
+function setIgnored(file, files, header, log) {
+  const lines = readLines(file, log)
+  const headerIndex = lines.indexOf(header)
   if (headerIndex === -1) {
-    lines.push(HEADER)
+    lines.push(header)
   } else {
     lines.splice(headerIndex + 1)
   }
   lines.push(...files)
-  writeFileSync(file, lines.sort().join('\n'), 'utf8')
+  writeFileSync(file, `${lines.sort().join('\n')}\n`, 'utf8')
 }
+
+function noop() {}

docs/lib/parse-doc-comments.js

@@ -0,0 +1,31 @@
+const each = require('./each')
+const START = /<!-- *%docs *\n/
+const SPLIT = /\n *-->/
+const END = /<!-- *%enddocs *-->/
+
+module.exports = function parseDocComments({log = noop}) {
+  return each((file, path) => {
+    const str = String(file.contents)
+
+    let parts = str.split(START)
+    if (parts.length > 1) {
+      // metadata should either be in frontmatter _or_ %docs comment;
+      // it's too tricky to reconcile them here
+      if (str.indexOf('---') === 0) {
+        log('unable to parse doc comments from', path, '(existing frontmatter)')
+        return
+      }
+
+      // take the part between the start and end
+      const middle = parts[1].split(END)[0]
+      // split *that* on the split "marker"
+      parts = middle.split(SPLIT)
+      // the part before that is the "frontmatter"
+      // and everything after that is the actual docs
+      const [meta, docs] = parts
+      file.contents = Buffer.from(`---\n${meta}\n---\n${docs}`)
+    }
+  })
+}
+
+function noop() {}

docs/lib/rename.js

@@ -0,0 +1,20 @@
+module.exports = function rename(fn, options = {}) {
+  const {log = noop} = options
+  return (files, metal, done) => {
+    for (const [key, file] of Object.entries(files)) {
+      const dest = fn(file, key, files, metal)
+      if (dest && dest !== key) {
+        log(`[rename] ${key} -> ${dest}`)
+        file.path = dest
+        files[dest] = file
+        delete files[key]
+      } else if (!dest) {
+        log(`[rename] delete ${key}`)
+        delete files[key]
+      }
+    }
+    done()
+  }
+}
+
+function noop() {}

docs/lib/sync.js

@@ -1,174 +1,80 @@
-const chokidar = require('chokidar')
-const klaw = require('klaw-sync')
-const minimatch = require('minimatch')
-const matter = require('gray-matter')
-const {green, red, yellow} = require('colorette')
-const {basename, dirname, join} = require('path')
-const {copySync, ensureDirSync, existsSync, readFileSync, removeSync} = require('fs-extra')
-const {getIgnored, setIgnored} = require('./ignore')
+const Metalsmith = require('metalsmith')
+const filter = require('metalsmith-filter')
+const frontmatter = require('metalsmith-matters')
+const watch = require('metalsmith-watch')
 
-const sourceDir = join(__dirname, '../../modules')
-const destDir = join(__dirname, '../pages/css')
-const ignoreFile = join(destDir, '.gitignore')
+const addPackageMeta = require('./add-package-meta')
+const addSource = require('./add-source')
+const filterBy = require('./filter-by')
+const parseDocComments = require('./parse-doc-comments')
+const rename = require('./rename')
+const writeMeta = require('./write-meta')
+const gitIgnore = require('./ignore')
 
-const map = {
-  '../CHANGELOG.md': 'whats-new/changelog.md',
-  'primer/README.md': false, // 'packages/primer.md',
-  'primer-base/README.md': false, // 'support/base.md',
-  'primer-core/README.md': false, // 'packages/primer-core.md',
-  'primer-layout/README.md': 'objects/layout.md',
-  'primer-layout/docs/*.md': path => `objects/${basename(path)}`,
-  'primer-marketing-support/README.md': 'support/marketing-variables.md',
-  'primer-marketing-type/docs/index.md': 'utilities/marketing-type.md',
-  'primer-marketing-utilities/README.md': false, // 'utilities/marketing.md',
-  'primer-marketing-utilities/docs/*.md': path => `utilities/marketing-${basename(path)}`,
-  'primer-marketing/README.md': false, // 'packages/primer-marketing.md',
-  'primer-product/README.md': false, // 'packages/primer-product.md',
-  'primer-support/README.md': false, // 'support/index.md',
-  'primer-support/docs/*.md': path => `support/${basename(path)}`,
-  'primer-table-object/README.md': 'objects/table-object.md',
-  'primer-utilities/README.md': false, // 'utilities/index.md',
-  'primer-utilities/docs/*.md': path => `utilities/${basename(path)}`,
-  // this is a catch-all rule that needs to go last so that it doesn't override others
-  'primer-*/README.md': path => `components/${shortName(path)}.md`,
-}
+module.exports = function sync(options = {}) {
+  // eslint-disable-next-line no-console
+  const {log = console.warn} = options
 
-module.exports = {sync, watch}
+  const metaOptions = options.meta || {namespace: 'data', log}
+  const ns = metaOptions.namespace
 
-function sync({debug = false}) {
-  const log = debug ? console.warn : noop
-  const ignored = getIgnored(ignoreFile)
-  for (const file of ignored) {
-    try {
-      removeSync(file)
-      log(`${yellow('x')} removed: ${file}`)
-    } catch (error) {
-      log(`${red('x')} missing: ${file}`)
-    }
-  }
-  console.time('get links')
-  const links = getLinks(sourceDir, destDir, map)
-  console.timeEnd('get links')
-  if (links.length) {
-    log(yellow(`linking ${links.length} files...`))
-    syncLinks(links)
-    const toBeIgnored = links.map(link => link.dest.substr(destDir.length + 1))
-    log(yellow(`adding ${toBeIgnored.length} files to ${ignoreFile}...`))
-    setIgnored(ignoreFile, toBeIgnored)
-    log(green('done!'))
-  } else {
-    log(yellow('(no links to copy)'))
-  }
-}
+  // this is what we'll resolve our Promise with later
+  let files
 
-function watch(options) {
-  const {debug = false} = options
-  const keys = Object.keys(map)
-  const globs = keys.map(path => join(sourceDir, path))
-  const log = debug ? console.warn : noop
-  let timeout
-  const update = path => {
-    if (timeout) return
-    timeout = setTimeout(() => {
-      log(`${yellow('changed')} ${path}`)
-      sync(options)
-      clearTimeout(timeout)
-      timeout = null
-    }, 50)
-  }
-  sync(options)
-  log(`watching in ${yellow(sourceDir)}: ${keys.join(', ')}`)
-  return chokidar.watch(globs)
-    // .on('add', update)
-    .on('change', update)
-    .on('unlink', update)
-}
+  const metal = Metalsmith(process.cwd())
+    .source('../modules')
+    .destination('pages/css')
+    .clean(false)
+    .frontmatter(false)
+    // ignore anything containing "node_modules" in its path
+    .ignore(path => path.includes('node_modules'))
+    // only match files that look like docs
+    .use(filter(['**/README.md', '**/docs/*.md']))
+    // convert <!-- %docs -->...<!-- %enddocs --> blocks into frontmatter
+    .use(parseDocComments({log}))
+    // parse frontmatter into "data" key of each file
+    .use(frontmatter(metaOptions))
+    // only match files that have a "path" key in their frontmatter
+    .use(filterBy(file => file[ns].path))
+    // write the source frontmatter key to the relative source path
+    .use(
+      addSource({
+        branch: 'master',
+        repo: 'primer/primer',
+        log
+      })
+    )
+    // copy a subset of fields from the nearest package.json
+    .use(
+      addPackageMeta({
+        fields: ['name', 'description', 'version'],
+        namespace: ns
+      })
+    )
+    // rename files with their "path" frontmatter key
+    .use(rename(file => `${file[ns].path}.md`), {log})
+    // write frontmatter back out to the file
+    .use(writeMeta(metaOptions))
+    // read the changelog manually
+    .use((_files, metal, done) => {
+      _files['whats-new/changelog.md'] = metal.readFile('../CHANGELOG.md')
+      files = _files
+      done()
+    })
+    // keep .gitignore up-to-date with the list of generated files
+    .use(
+      gitIgnore({
+        header: '# DO NOT EDIT: automatically generated by ignore.js'
+      })
+    )
 
-function syncLinks(links) {
-  const message = `sync ${links.length} links`
-  console.time(message)
-  for (const {source, dest} of links) {
-    console.warn(`${source.substr(sourceDir.length + 1)} ${yellow('->')} ${dest.substr(destDir.length + 1)}`)
-    const destPath = dirname(dest)
-    removeSync(dest)
-    ensureDirSync(destPath)
-    copySync(source, dest)
-  }
-  console.timeEnd(message)
-}
-
-function getLinks(sourceDir, destDir, map) {
-  const links = []
-
-  const mapEntries = Object.entries(map)
-  for (const [source, dest] of mapEntries) {
-    if (source.indexOf('..') === 0 && typeof dest === 'string') {
-      links.push({source, dest})
-    }
+  if (options.watch) {
+    metal.use(watch(typeof options.watch === 'object' ? options.watch : {}))
   }
 
-  console.warn(yellow(`walking: ${sourceDir}...`))
-  const items = klaw(sourceDir, {
-    nodir: true,
-    filter: item => item.path.indexOf('node_modules') === -1
-  })
-
-  let skipped = []
-  for (const item of items) {
-    // item.path is fully-qualified, so we need to remove the sourceDir
-    // from the beginning of it to get the relative path
-    const source = item.path.substr(sourceDir.length + 1)
-    let linked = false
-    for (const [pattern, name] of mapEntries) {
-      if (source === pattern || minimatch(source, pattern)) {
-        const dest = typeof name === 'function' ? name(source) : name
-        if (dest) {
-          links.push({source, dest})
-          linked = true
-        }
-        break
-      }
-    }
-    if (!linked && source.endsWith('.md')) {
-      skipped.push(source)
-    }
-  }
-
-  skipped = skipped.filter(source => source !== 'README.md')
-  if (skipped.length) {
-    console.warn(`ignored ${yellow(skipped.length)} files`)
-    for (const source of skipped) {
-      console.warn(`${yellow('-')} ${source}`)
-    }
-  }
-
-  return links.map(({source, dest}) => ({
-    source: join(sourceDir, source),
-    dest: join(destDir, dest)
-  }))
-  .filter(({source, dest}) => {
-    if (!existsSync(source)) {
-      console.warn(`${red('x')} missing: ${source.substr(sourceDir.length + 1)}`)
-      return false
-    }
-    const sourceContent = readFileSync(source, 'utf8')
-    const {data} = matter(sourceContent)
-    if (data.docs === false) {
-      console.warn(`${yellow('x')} ${source.substr(sourceDir.length + 1)} (docs: false)`)
-      return false
-    }
-    try {
-      const destContent = readFileSync(dest, 'utf8')
-      return sourceContent !== destContent
-    } catch (error) {
-      return true
-    }
+  return new Promise((resolve, reject) => {
+    metal.build(error => {
+      error ? reject(error) : resolve(files)
+    })
   })
 }
-
-function shortName(path) {
-  return path.match(/primer-([-\w]+)/)[1]
-}
-
-function noop() {
-}

docs/lib/write-meta.js

@@ -0,0 +1,11 @@
+const matter = require('gray-matter')
+const each = require('./each')
+
+module.exports = function writeMeta({namespace = 'data'}) {
+  return each(file => {
+    const data = file[namespace]
+    if (data) {
+      file.contents = matter.stringify(String(file.contents), data)
+    }
+  })
+}

docs/pages/css/.gitignore

@@ -21,6 +21,7 @@ objects/grid.md
 objects/layout.md
 objects/table-object.md
 support/breakpoints.md
+support/index.md
 support/marketing-variables.md
 support/spacing.md
 support/typography.md

docs/pages/css/support/index.md

@@ -1,14 +0,0 @@
----
-title: Support
----
-
-Primer is built on systems that form the foundation of our styles, and inform the way we write and organize our CSS. Building upon systems helps us make styles consistent and interoperable with each other, and assists us with visual hierarchy and vertical rhythm.
-
-We use Sass variables to keep color, typography, spacing, and other foundations of our system consistent. Occasionally we use Sass mixins to apply multiple CSS properties, they are a convenient solution for frequently-used verbose patterns.
-
-We've documented variables, mixins, and the systems they are built on for the following:
-
-- [Breakpoints](breakpoints/)
-- [Colors](color-system/)
-- [Spacing](spacing/)
-- [Typography](typography/)

docs/script/copy

@@ -1,15 +0,0 @@
-#!/usr/bin/env node
-const {basename, join} = require('path')
-const {sync, watch} = require('../lib/sync')
-const args = process.argv.slice(2)
-
-const options = {
-  debug: !args.includes('--quiet')
-}
-
-if (args.includes('--watch')) {
-  const watcher = watch(options)
-  process.on('exit', () => watcher.close())
-} else {
-  sync(options)
-}

docs/script/sync

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+const sync = require('../lib/sync')
+const args = process.argv.slice(2)
+
+const options = {
+  debug: !args.includes('--quiet'),
+  watch: args.includes('--watch')
+}
+
+sync(options)
+  .then(files => {
+    const built = Object.keys(files)
+    console.warn(`built ${built.length} files:`)
+    for (const file of built) {
+      console.warn(`- ${file}`)
+    }
+  })
+  .catch(error => {
+    console.error(error)
+    process.exitCode = 1
+  })