Ensure that ordering of scopes is correct…

…no matter where `HighlightIterator` starts.

If `HighlightIterator` tried to start a highlighting job at a point where more
than one layer reported an already-open scope, those scopes were combined such
that those from shallower layers went first. But that's not necessarily right,
and it's important to get the ordering correct because it affects
`scopeDescriptorForPosition` in particular.

Luckily, all the information we had to compile to find out if already-open
scopes should be covered by deeper injection layers… is exactly what we need to
solve this problem. When we assemble the list of open scopes, we can sort them
based on buffer position, only using layer depth to break ties.

spec/wasm-tree-sitter-language-mode-spec.js

@@ -2399,7 +2399,7 @@ describe('WASMTreeSitterLanguageMode', () => {
       buffer.setLanguageMode(languageMode);
       await languageMode.ready;
       // Wait for injections.
-      await wait(1000);
+      await wait(100);
 
       let injectionLayers = languageMode.getAllInjectionLayers();
       expect(injectionLayers.length).toBe(1);
@@ -2410,6 +2410,73 @@ describe('WASMTreeSitterLanguageMode', () => {
       expect(scopes.includes('regex-outer')).toBe(true);
       expect(scopes.includes('regex-inner')).toBe(false);
     });
+
+    it('arranges scopes in the proper order when scopes from several layers were already open at a given point', async () => {
+      jasmine.useRealClock();
+      const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
+
+      let tempJsRegexConfig = {
+        ...jsRegexConfig,
+        injectionRegex: '^(js-regex-for-test)$'
+      };
+
+      const regexGrammar = new WASMTreeSitterGrammar(atom.grammars, jsRegexGrammarPath, tempJsRegexConfig);
+
+      await regexGrammar.setQueryForTest('syntaxQuery', `
+        (pattern) @string.regexp
+      `);
+
+      jsGrammar.addInjectionPoint({
+        type: 'regex_pattern',
+        language(regex) {
+          return 'js-regex-for-test';
+        },
+        content(regex) {
+          return regex;
+        },
+        includeChildren: true,
+        languageScope: null
+      });
+
+      await jsGrammar.setQueryForTest('syntaxQuery', `
+        ((regex_pattern) @gadfly
+          (#set! startAndEndAroundFirstMatchOf "lor\\\\?em"))
+        (regex) @regex-outer
+        (regex_pattern) @regex-inner
+      `);
+
+      atom.grammars.addGrammar(regexGrammar);
+      atom.grammars.addGrammar(jsGrammar);
+
+      buffer.setText(dedent`
+        let foo = /patt.lor?em.ern/;
+      `);
+
+      const languageMode = new WASMTreeSitterLanguageMode({
+        grammar: jsGrammar,
+        buffer,
+        config: atom.config,
+        grammars: atom.grammars
+      });
+      buffer.setLanguageMode(languageMode);
+      await languageMode.ready;
+      // Wait for injections.
+      await wait(100);
+
+      let injectionLayers = languageMode.getAllInjectionLayers();
+      expect(injectionLayers.length).toBe(1);
+
+      let descriptor = languageMode.scopeDescriptorForPosition(new Point(0, 19));
+      let scopes = descriptor.getScopesArray();
+      expect(scopes).toEqual([
+        "source.js",
+        "regex-outer",
+        "regex-inner",
+        "string.regexp",
+        "gadfly"
+      ]);
+    });
+
   });
 
   describe('.syntaxTreeScopeDescriptorForPosition', () => {

src/wasm-tree-sitter-language-mode.js

@@ -2026,17 +2026,34 @@ class HighlightIterator {
 
     this.detectCoveredScope();
 
-    // Our nightmare is over, and we can flatten this data structure into a
-    // simple list of open scopes.
-    openScopes = [];
+    // Our nightmare is almost over, but one chore remains. The ordering of
+    // already open scopes should be consistent; scopes added earlier in the
+    // buffer should appear in the list before scopes added later. This ensures
+    // that, e.g., `scopeDescriptorForPosition` returns scopes in the proper
+    // hierarchy.
+    let sortedOpenScopes = [];
 
-    for (let layerOpenScopeMap of openScopesByLayer.values()) {
-      for (let layerOpenScopes of layerOpenScopeMap.values()) {
-        openScopes.push(...layerOpenScopes);
+    // First we'll gather all the point/scope-list pairs into a flat list…
+    let unsortedScopeBundles = [];
+    for (let [iterator, layerOpenScopeMap] of openScopesByLayer) {
+      for (let [point, scopes] of layerOpenScopeMap) {
+        unsortedScopeBundles.push({ point, scopes, iterator });
       }
     }
 
-    return openScopes;
+    // …then sort them by buffer position, with shallower layers first in case
+    // of ties.
+    unsortedScopeBundles.sort((a, b) => {
+      return a.point.compare(b.point) ||
+        a.iterator.depth - b.iterator.depth;
+    });
+
+    // Now we can flatten all the scopes themselves, preserving order.
+    for (let { scopes } of unsortedScopeBundles) {
+      sortedOpenScopes.push(...scopes);
+    }
+
+    return sortedOpenScopes;
   }
 
   moveToSuccessor () {