Enable automated nightly Atom releases

This change adds automation for producing nightly Atom releases using
VSTS CI.  Most of the changes are just slight modifications to Atom's
existing build scripts to produce another build channel and publish
those artifacts in a way that can be installed and updated when new
releases are available.

script/build

@@ -104,8 +104,10 @@ if (!argv.generateApiDocs) {
           }
           if (argv.createWindowsInstaller) {
             return createWindowsInstaller(packagedAppPath)
-              .then(() => argv.codeSign && codeSignOnWindows([ path.join(CONFIG.buildOutputPath, 'AtomSetup.exe') ]))
-              .then(() => packagedAppPath)
+              .then((installerPath) => {
+                argv.codeSign && codeSignOnWindows([installerPath])
+                return packagedAppPath
+              })
           } else {
             console.log('Skipping creating installer. Specify the --create-windows-installer option to create a Squirrel-based Windows installer.'.gray)
           }

script/config.js

@@ -5,6 +5,7 @@
 
 const fs = require('fs')
 const path = require('path')
+const spawnSync = require('./lib/spawn-sync')
 
 const repositoryRootPath = path.resolve(__dirname, '..')
 const apmRootPath = path.join(repositoryRootPath, 'apm')
@@ -20,11 +21,13 @@ const atomHomeDirPath = process.env.ATOM_HOME || path.join(homeDirPath, '.atom')
 const appMetadata = require(path.join(repositoryRootPath, 'package.json'))
 const apmMetadata = require(path.join(apmRootPath, 'package.json'))
 const channel = getChannel()
+const computedAppVersion = computeAppVersion(appMetadata.version, channel)
 
 module.exports = {
   appMetadata,
   apmMetadata,
   channel,
+  computedAppVersion,
   repositoryRootPath,
   apmRootPath,
   scriptRootPath,
@@ -41,7 +44,9 @@ module.exports = {
 }
 
 function getChannel () {
-  if (appMetadata.version.match(/dev/)) {
+  if (process.env.BUILD_DEFINITIONNAME === 'Atom Nightly') {
+    return 'nightly'
+  } else if (appMetadata.version.match(/dev/)) {
     return 'dev'
   } else if (appMetadata.version.match(/beta/)) {
     return 'beta'
@@ -50,6 +55,17 @@ function getChannel () {
   }
 }
 
+function computeAppVersion (version, channel) {
+  if (channel === 'dev') {
+    const result = spawnSync('git', ['rev-parse', '--short', 'HEAD'], {cwd: repositoryRootPath})
+    const commitHash = result.stdout.toString().trim()
+    version += '-' + commitHash
+  } else if (channel === 'nightly') {
+    version = process.env.BUILD_BUILDNUMBER
+  }
+  return version
+}
+
 function getApmBinPath () {
   const apmBinName = process.platform === 'win32' ? 'apm.cmd' : 'apm'
   return path.join(apmRootPath, 'node_modules', 'atom-package-manager', 'bin', apmBinName)

script/lib/code-sign-on-mac.js

@@ -16,6 +16,36 @@ module.exports = function (packagedAppPath) {
     downloadFileFromGithub(process.env.ATOM_MAC_CODE_SIGNING_CERT_DOWNLOAD_URL, certPath)
   }
   try {
+    console.log(`Ensuring keychain ${process.env.ATOM_MAC_CODE_SIGNING_KEYCHAIN} exists`)
+    try {
+      spawnSync('security', [
+        'show-keychain-info',
+        process.env.ATOM_MAC_CODE_SIGNING_KEYCHAIN
+      ], {stdio: 'inherit'})
+    } catch (err) {
+      console.log(`Creating keychain ${process.env.ATOM_MAC_CODE_SIGNING_KEYCHAIN}`)
+      // The keychain doesn't exist, try to create it
+      spawnSync('security', [
+        'create-keychain',
+        '-p', process.env.ATOM_MAC_CODE_SIGNING_KEYCHAIN_PASSWORD,
+        process.env.ATOM_MAC_CODE_SIGNING_KEYCHAIN
+      ], {stdio: 'inherit'})
+
+      // List the keychain to "activate" it.  Somehow this seems
+      // to be needed otherwise the signing operation fails
+      spawnSync('security', [
+        'list-keychains',
+        '-s', process.env.ATOM_MAC_CODE_SIGNING_KEYCHAIN
+      ], {stdio: 'inherit'})
+
+      // Make sure it doesn't time out before we use it
+      spawnSync('security', [
+        'set-keychain-settings',
+        '-t', '3600',
+        '-u', process.env.ATOM_MAC_CODE_SIGNING_KEYCHAIN
+      ], {stdio: 'inherit'})
+    }
+
     console.log(`Unlocking keychain ${process.env.ATOM_MAC_CODE_SIGNING_KEYCHAIN}`)
     const unlockArgs = ['unlock-keychain']
     // For signing on local workstations, password could be entered interactively

script/lib/compress-artifacts.js

@@ -3,6 +3,7 @@
 const fs = require('fs-extra')
 const path = require('path')
 const spawnSync = require('./spawn-sync')
+const { path7za } = require('7zip-bin')
 
 const CONFIG = require('../config')
 
@@ -19,7 +20,7 @@ module.exports = function (packagedAppPath) {
 function getArchiveName () {
   switch (process.platform) {
     case 'darwin': return 'atom-mac.zip'
-    case 'win32': return 'atom-windows.zip'
+    case 'win32': return `atom-${process.arch === 'x64' ? 'x64-' : ''}windows.zip`
     default: return `atom-${getLinuxArchiveArch()}.tar.gz`
   }
 }
@@ -44,7 +45,7 @@ function compress (inputDirPath, outputArchivePath) {
     compressCommand = 'zip'
     compressArguments = ['-r', '--symlinks']
   } else if (process.platform === 'win32') {
-    compressCommand = '7z.exe'
+    compressCommand = path7za
     compressArguments = ['a', '-r']
   } else {
     compressCommand = 'tar'

script/lib/create-github-release.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const publishRelease = require('publish-release')
+const CONFIG = require('../config')
+
+module.exports = function (assets) {
+  return new Promise(function (resolve, reject) {
+    console.log(`Uploading assets to GitHub release ${CONFIG.computedAppVersion}`)
+    publishRelease({
+      token: process.env.GITHUB_TOKEN,
+      owner: 'atom',
+      repo: CONFIG.channel !== 'nightly' ? 'atom' : 'atom-nightly-releases',
+      name: CONFIG.computedAppVersion,
+      tag: CONFIG.computedAppVersion,
+      draft: true,
+      prerelease: CONFIG.channel !== 'stable',
+      reuseRelease: true,
+      reuseDraftOnly: true,
+      skipIfPublished: true,
+      assets
+    }, function (err, release) {
+      if (err) {
+        reject(err)
+      } else {
+        console.log('Release created successfully: ', release.html_url)
+        resolve(release)
+      }
+    })
+  })
+}

script/lib/create-windows-installer.js

@@ -1,14 +1,14 @@
 'use strict'
 
 const electronInstaller = require('electron-winstaller')
-const fs = require('fs-extra')
+const fs = require('fs')
 const glob = require('glob')
 const path = require('path')
 
 const CONFIG = require('../config')
 
 module.exports = (packagedAppPath) => {
-  const archSuffix = process.arch === 'ia32' ? '' : '-' + process.arch
+  // const archSuffix = process.arch === 'ia32' ? '' : '-' + process.arch
   const options = {
     appDirectory: packagedAppPath,
     authors: 'GitHub Inc.',
@@ -16,19 +16,32 @@ module.exports = (packagedAppPath) => {
     loadingGif: path.join(CONFIG.repositoryRootPath, 'resources', 'win', 'loading.gif'),
     outputDirectory: CONFIG.buildOutputPath,
     noMsi: true,
-    remoteReleases: `https://atom.io/api/updates${archSuffix}?version=${CONFIG.appMetadata.version}`,
+    // remoteReleases: `https://atom.io/api/updates${archSuffix}?version=${CONFIG.appMetadata.version}`,
+    setupExe: `AtomSetup${process.arch === 'x64' ? '-x64' : ''}.exe`,
     setupIcon: path.join(CONFIG.repositoryRootPath, 'resources', 'app-icons', CONFIG.channel, 'atom.ico')
   }
 
   const cleanUp = () => {
-    for (let nupkgPath of glob.sync(`${CONFIG.buildOutputPath}/*.nupkg`)) {
+    const releasesPath = `${CONFIG.buildOutputPath}/RELEASES`
+    if (process.arch === 'x64' && fs.existsSync(releasesPath)) {
+      fs.renameSync(releasesPath, `${releasesPath}-x64`)
+    }
+
+    for (let nupkgPath of glob.sync(`${CONFIG.buildOutputPath}/atom-*.nupkg`)) {
       if (!nupkgPath.includes(CONFIG.appMetadata.version)) {
         console.log(`Deleting downloaded nupkg for previous version at ${nupkgPath} to prevent it from being stored as an artifact`)
-        fs.removeSync(nupkgPath)
+        fs.unlinkSync(nupkgPath)
+      } else {
+        if (process.arch === 'x64') {
+          const newNupkgPath = `${CONFIG.buildOutputPath}/atom-x64${path.basename(nupkgPath).slice(4)}`
+          fs.renameSync(nupkgPath, newNupkgPath)
         }
       }
     }
 
+    return `${CONFIG.buildOutputPath}/${options.setupExe}`
+  }
+
   console.log(`Creating Windows Installer for ${packagedAppPath}`)
   return electronInstaller.createWindowsInstaller(options)
     .then(cleanUp, error => {

script/lib/generate-metadata.js

@@ -6,7 +6,6 @@ const fs = require('fs-plus')
 const normalizePackageData = require('normalize-package-data')
 const path = require('path')
 const semver = require('semver')
-const spawnSync = require('./spawn-sync')
 
 const CONFIG = require('../config')
 
@@ -16,7 +15,7 @@ module.exports = function () {
   CONFIG.appMetadata._atomMenu = buildPlatformMenuMetadata()
   CONFIG.appMetadata._atomKeymaps = buildPlatformKeymapsMetadata()
   CONFIG.appMetadata._deprecatedPackages = deprecatedPackagesMetadata
-  CONFIG.appMetadata.version = computeAppVersion()
+  CONFIG.appMetadata.version = CONFIG.computedAppVersion
   checkDeprecatedPackagesMetadata()
   fs.writeFileSync(path.join(CONFIG.intermediateAppPath, 'package.json'), JSON.stringify(CONFIG.appMetadata))
 }
@@ -162,13 +161,3 @@ function checkDeprecatedPackagesMetadata () {
     }
   }
 }
-
-function computeAppVersion () {
-  let version = CONFIG.appMetadata.version
-  if (CONFIG.channel === 'dev') {
-    const result = spawnSync('git', ['rev-parse', '--short', 'HEAD'], {cwd: CONFIG.repositoryRootPath})
-    const commitHash = result.stdout.toString().trim()
-    version += '-' + commitHash
-  }
-  return version
-}

script/package.json

@@ -2,6 +2,7 @@
   "name": "atom-build-scripts",
   "description": "Atom build scripts",
   "dependencies": {
+    "7zip-bin": "^4.0.2",
     "async": "2.0.1",
     "babel-core": "5.8.38",
     "coffeelint": "1.15.7",
@@ -26,6 +27,7 @@
     "npm": "5.3.0",
     "passwd-user": "2.1.0",
     "pegjs": "0.9.0",
+    "publish-release": "^1.6.0",
     "random-seed": "^0.3.0",
     "season": "5.3.0",
     "semver": "5.3.0",

script/publish-release

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+
+'use strict'
+
+const path = require('path')
+const glob = require('glob')
+const publishRelease = require('publish-release')
+const CONFIG = require('./config')
+
+console.log(`Publishing GitHub release ${CONFIG.computedAppVersion}`)
+
+const yargs = require('yargs')
+const argv = yargs
+  .usage('Usage: $0 [options]')
+  .help('help')
+  .describe('assets-path', 'Path to the folder where all release assets are stored')
+  .wrap(yargs.terminalWidth())
+  .argv
+
+let assetsPath = argv.assetsPath || path.join(CONFIG.repositoryRootPath, 'out')
+let assets = glob.sync(path.join(assetsPath, '*(*.exe|*.zip|*.nupkg|*.tar.gz|*.rpm|*.deb|RELEASES*)'))
+
+publishRelease({
+  token: process.env.GITHUB_TOKEN,
+  owner: 'atom',
+  repo: CONFIG.channel !== 'nightly' ? 'atom' : 'atom-nightly-releases',
+  name: CONFIG.computedAppVersion,
+  tag: CONFIG.computedAppVersion,
+  draft: false,
+  prerelease: CONFIG.channel !== 'stable',
+  reuseRelease: true,
+  skipIfPublished: true,
+  assets
+}, function (err, release) {
+  if (err) {
+    console.log("An error occurred while publishing the release:\n\n", err)
+  } else {
+    console.log("Release published successfully: ", release.html_url)
+  }
+})

script/publish-release.cmd

@@ -0,0 +1,5 @@
+@IF EXIST "%~dp0\node.exe" (
+  "%~dp0\node.exe"  "%~dp0\publish-release" %*
+) ELSE (
+  node  "%~dp0\publish-release" %*
+)

script/vsts/linux.yml

@@ -0,0 +1,52 @@
+phases:
+- phase: Linux
+  queue:
+    name: Hosted Linux Preview
+    timeoutInMinutes: 180
+
+  steps:
+  - task: NodeTool@0
+    inputs:
+      versionSpec: 8.11.3
+    displayName: Install Node.js 8.11.3
+
+  - script: |
+      apt-get update
+      apt-get install -y --no-install-recommends build-essential xvfb clang-3.5 fakeroot git libsecret-1-dev rpm libx11-dev libxkbfile-dev xz-utils xorriso zsync libxss1 libgconf2-4 libgtk-3-0
+    displayName: Install apt dependencies
+
+  - script: |
+      script/build --create-debian-package --create-rpm-package --compress-artifacts
+    displayName: Build Atom
+
+  - script: script/lint
+    displayName: Run linter
+
+  - script: |
+      /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16
+      export DISPLAY=':99.0'
+      Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
+      script/test
+    env:
+      CI: true
+    displayName: Run tests
+
+  # This step is necessary in the short term due to a bug in the *NIX
+  # implementation of the CopyFiles task which scans the entire file
+  # system structure just to resolve the glob pattern.
+  - script: rm -rf $(Build.SourcesDirectory)/out/*/
+    displayName: Delete Intermediate Output
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.SourcesDirectory)/out
+      contents: '?(*.deb|*.rpm|*.tar.gz)'
+      targetFolder: $(Build.ArtifactStagingDirectory)
+    displayName: Stage Artifacts
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      PathtoPublish: $(Build.ArtifactStagingDirectory)
+      ArtifactName: Binaries
+      ArtifactType: Container
+    displayName: Upload Artifacts

script/vsts/macos.yml

@@ -0,0 +1,51 @@
+phases:
+- phase: macOS
+  queue:
+    name: Hosted macOS Preview
+    timeoutInMinutes: 180
+
+  steps:
+  - task: NodeTool@0
+    inputs:
+      versionSpec: 8.11.3
+    displayName: Install Node.js 8.11.3
+
+  - script: |
+      script/build --code-sign --compress-artifacts
+    displayName: Build Atom
+    env:
+      ATOM_MAC_CODE_SIGNING_CERT_DOWNLOAD_URL: $(ATOM_MAC_CODE_SIGNING_CERT_DOWNLOAD_URL)
+      ATOM_MAC_CODE_SIGNING_CERT_PASSWORD: $(ATOM_MAC_CODE_SIGNING_CERT_PASSWORD)
+      ATOM_MAC_CODE_SIGNING_KEYCHAIN: $(ATOM_MAC_CODE_SIGNING_KEYCHAIN)
+      ATOM_MAC_CODE_SIGNING_KEYCHAIN_PASSWORD: $(ATOM_MAC_CODE_SIGNING_KEYCHAIN_PASSWORD)
+
+  - script: script/lint
+    displayName: Run linter
+
+  - script: |
+      osascript -e 'tell application "System Events" to keystroke "x"' # clear screen saver
+      caffeinate -s script/test # Run with caffeinate to prevent screen saver
+    env:
+      CI: true
+      ATOM_GITHUB_DISABLE_KEYTAR: 1
+    displayName: Run tests
+
+  # This step is necessary in the short term due to a bug in the *NIX
+  # implementation of the CopyFiles task which scans the entire file
+  # system structure just to resolve the glob pattern.
+  - script: rm -rf $(Build.SourcesDirectory)/out/*/
+    displayName: Delete Intermediate Output
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.SourcesDirectory)/out
+      contents: '*.zip'
+      targetFolder: $(Build.ArtifactStagingDirectory)
+    displayName: Stage Artifacts
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      PathtoPublish: $(Build.ArtifactStagingDirectory)
+      ArtifactName: Binaries
+      ArtifactType: Container
+    displayName: Upload Artifacts

script/vsts/nightly-release.yml

@@ -0,0 +1,39 @@
+name: 1.29.0-nightly$(Rev:r)
+
+phases:
+
+- template: windows.yml
+- template: macos.yml
+- template: linux.yml
+
+- phase: Release
+  queue: Hosted  # Need this for Python 2.7
+
+  dependsOn:
+  - Windows
+  - Linux
+  - macOS
+
+  steps:
+  - task: NodeTool@0
+    inputs:
+      versionSpec: 8.11.3
+    displayName: Install Node.js 8.11.3
+
+  # This has to be done separately because VSTS inexplicably
+  # exits the script block after `npm install` completes.
+  - script: |
+      cd script
+      npm install
+    displayName: npm install
+
+  - task: DownloadBuildArtifacts@0
+    displayName: Download Release Artifacts
+    inputs:
+      artifactName: Binaries
+
+  - script: |
+      $(Build.SourcesDirectory)\script\publish-release.cmd --assets-path "$(System.ArtifactsDirectory)/Binaries"
+    env:
+      GITHUB_TOKEN: $(GITHUB_TOKEN)
+    displayName: Create Nightly Release

script/vsts/windows.yml

@@ -0,0 +1,49 @@
+phases:
+- phase: Windows
+  queue:
+    name: Hosted
+    timeoutInMinutes: 180
+    parallel: 2
+    matrix:
+      x64:
+        buildArch: x64
+      # TODO: x86 is currently not supported on VSTS
+      # x86:
+      #   buildArch: x86
+
+  steps:
+  - task: NodeTool@0
+    inputs:
+      versionSpec: 8.11.3
+    displayName: Install Node.js 8.11.3
+
+  - script: |
+      IF NOT EXIST C:\tmp MKDIR C:\tmp
+      SET SQUIRREL_TEMP=C:\tmp
+      script\build.cmd --create-windows-installer --code-sign --compress-artifacts
+    env:
+      ATOM_WIN_CODE_SIGNING_CERT_DOWNLOAD_URL: $(ATOM_WIN_CODE_SIGNING_CERT_DOWNLOAD_URL)
+      ATOM_WIN_CODE_SIGNING_CERT_PASSWORD: $(ATOM_WIN_CODE_SIGNING_CERT_PASSWORD)
+    displayName: Build Atom
+
+  - script: script\lint.cmd
+    displayName: Run linter
+
+  - script: script\test.cmd
+    env:
+      CI: true
+    displayName: Run tests
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.SourcesDirectory)/out
+      contents: '?(*.exe|*.zip|*.nupkg|RELEASES*)'
+      targetFolder: $(Build.ArtifactStagingDirectory)
+    displayName: Stage Artifacts
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      PathtoPublish: $(Build.ArtifactStagingDirectory)
+      ArtifactName: Binaries
+      ArtifactType: Container
+    displayName: Upload Artifacts