Merge remote-tracking branch 'origin/master' into dw-repo-local-core-packages

.travis.yml

@@ -27,7 +27,7 @@ install:
   - source /tmp/.nvm/nvm.sh
   - nvm install $NODE_VERSION
   - nvm use --delete-prefix $NODE_VERSION
-  - npm install -g npm@6.1.0
+  - npm install --global npm@6.2.0
   - script/build --create-debian-package --create-rpm-package --compress-artifacts
 
 script:

appveyor.yml

@@ -36,6 +36,7 @@ install:
   - IF NOT EXIST %TEST_JUNIT_XML_ROOT% MKDIR %TEST_JUNIT_XML_ROOT%
   - SET PATH=C:\Program Files\Atom\resources\cli;%PATH%
   - ps: Install-Product node $env:NODE_VERSION $env:PLATFORM
+  - npm install --global npm@6.2.0
 
 build_script:
   - CD %APPVEYOR_BUILD_FOLDER%

script/bootstrap

@@ -18,6 +18,14 @@ process.on('unhandledRejection', function (e) {
   process.exit(1)
 })
 
+// We can't use yargs until installScriptDependencies() is executed, so...
+let ci = process.argv.indexOf('--ci') !== -1
+
+if (!ci && process.env.CI === 'true' && process.argv.indexOf('--no-ci') === -1) {
+  console.log('Automatically enabling --ci because CI is set in the environment')
+  ci = true
+}
+
 verifyMachineRequirements()
 
 if (dependenciesFingerprint.isOutdated()) {
@@ -26,14 +34,14 @@ if (dependenciesFingerprint.isOutdated()) {
 
 if (process.platform === 'win32') deleteMsbuildFromPath()
 
-installScriptDependencies()
-installApm()
+installScriptDependencies(ci)
+installApm(ci)
 childProcess.execFileSync(
   CONFIG.getApmBinPath(),
   ['--version'],
   {stdio: 'inherit'}
 )
-runApmInstall(CONFIG.repositoryRootPath)
+runApmInstall(CONFIG.repositoryRootPath, ci)
 
 if (!process.env.CI) {
   const glob = require('glob')

script/build

@@ -24,6 +24,7 @@ const argv = yargs
   .describe('generate-api-docs', 'Only build the API documentation')
   .describe('install', 'Install Atom')
   .string('install')
+  .describe('ci', 'Install dependencies quickly (package-lock.json files must be up to date)')
   .wrap(yargs.terminalWidth())
   .argv
 

script/config.js

@@ -76,8 +76,8 @@ function getApmBinPath () {
   return path.join(apmRootPath, 'node_modules', 'atom-package-manager', 'bin', apmBinName)
 }
 
-function getNpmBinPath () {
+function getNpmBinPath (external = false) {
   const npmBinName = process.platform === 'win32' ? 'npm.cmd' : 'npm'
   const localNpmBinPath = path.resolve(repositoryRootPath, 'script', 'node_modules', '.bin', npmBinName)
-  return fs.existsSync(localNpmBinPath) ? localNpmBinPath : npmBinName
+  return !external && fs.existsSync(localNpmBinPath) ? localNpmBinPath : npmBinName
 }

script/lib/install-apm.js

@@ -4,8 +4,9 @@ const childProcess = require('child_process')
 
 const CONFIG = require('../config')
 
-module.exports = function () {
+module.exports = function (ci) {
   console.log('Installing apm')
+  // npm ci leaves apm with a bunch of unmet dependencies
   childProcess.execFileSync(
     CONFIG.getNpmBinPath(),
     ['--global-style', '--loglevel=error', 'install'],

script/lib/install-script-dependencies.js

@@ -4,11 +4,11 @@ const childProcess = require('child_process')
 
 const CONFIG = require('../config')
 
-module.exports = function () {
+module.exports = function (ci) {
   console.log('Installing script dependencies')
   childProcess.execFileSync(
-    CONFIG.getNpmBinPath(),
-    ['--loglevel=error', 'install'],
+    CONFIG.getNpmBinPath(ci),
+    ['--loglevel=error', ci ? 'ci' : 'install'],
     {env: process.env, cwd: CONFIG.scriptRootPath}
   )
 }

script/lib/run-apm-install.js

@@ -4,7 +4,7 @@ const childProcess = require('child_process')
 
 const CONFIG = require('../config')
 
-module.exports = function (packagePath, stderrOnly) {
+module.exports = function (packagePath, ci, stderrOnly) {
   const installEnv = Object.assign({}, process.env)
   // Set resource path so that apm can load metadata related to Atom.
   installEnv.ATOM_RESOURCE_PATH = CONFIG.repositoryRootPath
@@ -13,7 +13,7 @@ module.exports = function (packagePath, stderrOnly) {
   installEnv.npm_config_target = CONFIG.appMetadata.electronVersion
   childProcess.execFileSync(
     CONFIG.getApmBinPath(),
-    ['--loglevel=error', 'install'],
+    ['--loglevel=error', ci ? 'ci' : 'install'],
     {env: installEnv, cwd: packagePath, stdio: ['inherit', stderrOnly ? 'pipe' : 'inherit', 'inherit']}
   )
 }

script/vsts/platforms/linux.yml

@@ -13,6 +13,9 @@ phases:
       versionSpec: 8.9.3
     displayName: Install Node.js 8.9.3
 
+  - script: npm install --global npm@6.2.0
+    displayName: Update npm
+
   - script: |
       apt-get update
       apt-get install -y --no-install-recommends build-essential xvfb clang-3.5 fakeroot git libsecret-1-dev rpm libx11-dev libxkbfile-dev xz-utils xorriso zsync libxss1 libgconf2-4 libgtk-3-0

script/vsts/platforms/macos.yml

@@ -15,6 +15,9 @@ phases:
       versionSpec: 8.9.3
     displayName: Install Node.js 8.9.3
 
+  - script: npm install --global npm@6.2.0
+    displayName: Update npm
+
   - script: |
       if [ $IS_RELEASE_BRANCH == "true" ] || [ $IS_SIGNED_ZIP_BRANCH == "true" ]; then
         script/build --code-sign --compress-artifacts

script/vsts/platforms/windows.yml

@@ -22,6 +22,12 @@ phases:
       versionSpec: 8.9.3
     displayName: Install Node.js 8.9.3
 
+  - script: |
+      ECHO Installing npm-windows-upgrade
+      npm install --global --production npm-windows-upgrade
+      ECHO Upgrading npm
+      npm-windows-upgrade --no-spinner --no-prompt --npm-version 6.2.0
+
   - script: |
       IF NOT EXIST C:\tmp MKDIR C:\tmp
       SET SQUIRREL_TEMP=C:\tmp

spec/tree-sitter-language-mode-spec.js

@@ -210,9 +210,6 @@ describe('TreeSitterLanguageMode', () => {
       buffer.setLanguageMode(languageMode)
       await nextHighlightingUpdate(languageMode)
 
-      console.log(languageMode.tree.rootNode.toString());
-      console.log(languageMode.grammar.scopeMap);
-
       expectTokensToEqual(editor, [
         [
           {text: 'a', scopes: ['function']},
@@ -728,6 +725,39 @@ describe('TreeSitterLanguageMode', () => {
           ],
         ])
       })
+
+      it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => {
+        const promise = new Promise(resolve => {
+          editor.onDidTokenize(event => {
+            expectTokensToEqual(editor, [
+              [
+                {text: '<', scopes: ['html']},
+                {text: 'script', scopes: ['html', 'tag']},
+                {text: '>', scopes: ['html']},
+              ],
+              [
+                {text: 'hello', scopes: ['html', 'function']},
+                {text: '();', scopes: ['html']},
+              ],
+              [
+                {text: '</', scopes: ['html']},
+                {text: 'script', scopes: ['html', 'tag']},
+                {text: '>', scopes: ['html']},
+              ]
+            ])
+            resolve()
+          })
+        })
+
+        atom.grammars.addGrammar(jsGrammar)
+        atom.grammars.addGrammar(htmlGrammar)
+        buffer.setText('<script>\nhello();\n</script>')
+
+        const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
+        buffer.setLanguageMode(languageMode)
+
+        await promise
+      })
     })
   })
 

src/tree-sitter-language-mode.js

@@ -1,7 +1,7 @@
 const Parser = require('tree-sitter')
 const {Point, Range, spliceArray} = require('text-buffer')
 const {Patch} = require('superstring')
-const {Emitter, Disposable} = require('event-kit')
+const {Emitter} = require('event-kit')
 const ScopeDescriptor = require('./scope-descriptor')
 const TokenizedLine = require('./tokenized-line')
 const TextMateLanguageMode = require('./text-mate-language-mode')
@@ -63,7 +63,9 @@ class TreeSitterLanguageMode {
       this.rootLanguageLayer.update(null)
     })
 
-    this.rootLanguageLayer.update(null)
+    this.rootLanguageLayer.update(null).then(() =>
+      this.emitter.emit('did-tokenize')
+    )
 
     // TODO: Remove this once TreeSitterLanguageMode implements its own auto-indentation system. This
     // is temporarily needed in order to delegate to the TextMateLanguageMode's auto-indent system.
@@ -119,6 +121,10 @@ class TreeSitterLanguageMode {
     return new HighlightIterator(this, layerIterators)
   }
 
+  onDidTokenize (callback) {
+    return this.emitter.on('did-tokenize', callback)
+  }
+
   onDidChangeHighlighting (callback) {
     return this.emitter.on('did-change-highlighting', callback)
   }
@@ -386,8 +392,6 @@ class TreeSitterLanguageMode {
   Section - Backward compatibility shims
   */
 
-  onDidTokenize (callback) { return new Disposable(() => {}) }
-
   tokenizedLineForRow (row) {
     return new TokenizedLine({
       openScopes: [],