Support importing profiles whose contents exceed V8s maximum string size (#385)

Browsers have a limit on how big you can make strings. In Chrome on a 64 bit machine, this is around 512MB, which explains why in #340, a 600MB file fails to load.

To work around this issue, we avoid making strings this large.

To do so, we need two core changes:
1. Instead of sending large strings as the import mechanism to different file format importers, we introduce a new `TextFileContent` interface which exposes methods to get the lines in the file or the JSON representation. In the case of line splitting, we assume that no single line exceeds the 512MB limit.
2. We introduce a dependency on https://github.com/evanw/uint8array-json-parser to allow us to parse JSON files contained in `Uint8Array` objects

To ensure that this code doesn't code rot without introducing 600MB test files or test file generation into the repository, we also re-run a small set of tests with a mocked maximum string size of 100 bytes. You can see that the chunked string representation code is getting executed via test coverage.

Fixes #340

bin/cli.js

@@ -66,10 +66,20 @@ async function main() {
     const relPath = process.argv[2]
     const sourceBuffer = await getProfileBuffer(relPath)
     const filename = path.basename(relPath)
-    const sourceBase64 = sourceBuffer.toString('base64')
+
-    const jsSource = `speedscope.loadFileFromBase64(${JSON.stringify(filename)}, ${JSON.stringify(
+    let jsSource
-      sourceBase64,
+    try {
-    )})`
+      const sourceBase64 = sourceBuffer.toString('base64')
+      jsSource = `speedscope.loadFileFromBase64(${JSON.stringify(filename)}, ${JSON.stringify(
+        sourceBase64,
+      )})`
+    } catch(e) {
+      if (e && e.message && /Cannot create a string longer than/.exec(e.message)) {
+        jsSource = `alert("Sorry, ${filename} is too large to be loaded via command-line argument! Try dragging it into speedscope instead.")`
+      } else {
+        throw e
+      }
+    }
 
     const filePrefix = `speedscope-${+new Date()}-${process.pid}`
     const jsPath = path.join(os.tmpdir(), `${filePrefix}.js`)

package.json

@@ -53,7 +53,8 @@
     "ts-jest": "24.3.0",
     "typescript": "4.2.3",
     "typescript-json-schema": "0.42.0",
-    "uglify-es": "3.2.2"
+    "uglify-es": "3.2.2",
+    "uint8array-json-parser": "jlfwong/uint8array-json-parser#edce51ce"
   },
   "jest": {
     "transform": {

src/import/__snapshots__/chrome.test.ts.snap

@@ -1756,6 +1756,209 @@ exports[`importFromChromeTimeline Workers Chrome 70: indexToView 1`] = `0`;
 
 exports[`importFromChromeTimeline Workers Chrome 70: profileGroup.name 1`] = `"worker.json"`;
 
+exports[`importFromChromeTimeline chunked 1`] = `
+Object {
+  "frames": Array [
+    Frame {
+      "col": 0,
+      "file": "",
+      "key": "(program)::0:0",
+      "line": 0,
+      "name": "(program)",
+      "selfWeight": 2299,
+      "totalWeight": 2299,
+    },
+    Frame {
+      "col": 11,
+      "file": "file:///Users/jlfwong/code/speedscope/sample/programs/javascript/simple.js",
+      "key": "a:file:///Users/jlfwong/code/speedscope/sample/programs/javascript/simple.js:1:11",
+      "line": 1,
+      "name": "a",
+      "selfWeight": 0,
+      "totalWeight": 33186,
+    },
+    Frame {
+      "col": 11,
+      "file": "file:///Users/jlfwong/code/speedscope/sample/programs/javascript/simple.js",
+      "key": "c:file:///Users/jlfwong/code/speedscope/sample/programs/javascript/simple.js:14:11",
+      "line": 14,
+      "name": "c",
+      "selfWeight": 16275,
+      "totalWeight": 16275,
+    },
+    Frame {
+      "col": 11,
+      "file": "file:///Users/jlfwong/code/speedscope/sample/programs/javascript/simple.js",
+      "key": "b:file:///Users/jlfwong/code/speedscope/sample/programs/javascript/simple.js:8:11",
+      "line": 8,
+      "name": "b",
+      "selfWeight": 16911,
+      "totalWeight": 16911,
+    },
+  ],
+  "name": "simple-timeline.json",
+  "stacks": Array [
+    " 10.06ms",
+    "(program) 202.00µs",
+    " 1.97ms",
+    "(program) 1.04ms",
+    " 85.89ms",
+    "(program) 132.00µs",
+    " 8.40ms",
+    "(program) 134.00µs",
+    "a;c 273.00µs",
+    "a;b 513.00µs",
+    "a;c 512.00µs",
+    "a;b 513.00µs",
+    "a;c 262.00µs",
+    "a;b 514.00µs",
+    "a;c 517.00µs",
+    "a;b 261.00µs",
+    "a;c 515.00µs",
+    "a;b 273.00µs",
+    "a;c 134.00µs",
+    "a;b 146.00µs",
+    "a;c 291.00µs",
+    "a;b 256.00µs",
+    "a;c 145.00µs",
+    "a;b 134.00µs",
+    "a;c 273.00µs",
+    "a;b 141.00µs",
+    "a;c 139.00µs",
+    "a;b 130.00µs",
+    "a;c 393.00µs",
+    "a;b 131.00µs",
+    "a;c 126.00µs",
+    "a;b 129.00µs",
+    "a;c 520.00µs",
+    "a;b 129.00µs",
+    "a;c 259.00µs",
+    "a;b 130.00µs",
+    "a;c 128.00µs",
+    "a;b 136.00µs",
+    "a;c 137.00µs",
+    "a;b 281.00µs",
+    "a;c 150.00µs",
+    "a;b 624.00µs",
+    "a;c 135.00µs",
+    "a;b 129.00µs",
+    "a;c 604.00µs",
+    "a;b 292.00µs",
+    "a;c 257.00µs",
+    "a;b 127.00µs",
+    "a;c 128.00µs",
+    "a;b 118.00µs",
+    "a;c 256.00µs",
+    "a;b 524.00µs",
+    "a;c 129.00µs",
+    "a;b 131.00µs",
+    "a;c 125.00µs",
+    "a;b 263.00µs",
+    "a;c 123.00µs",
+    "a;b 128.00µs",
+    "a;c 129.00µs",
+    "a;b 265.00µs",
+    "a;c 128.00µs",
+    "a;b 280.00µs",
+    "a;c 366.00µs",
+    "a;b 931.00µs",
+    "a;c 148.00µs",
+    "a;b 147.00µs",
+    "a;c 302.00µs",
+    "a;b 151.00µs",
+    "a;c 158.00µs",
+    "a;b 456.00µs",
+    "a;c 170.00µs",
+    "a;b 139.00µs",
+    "a;c 145.00µs",
+    "a;b 260.00µs",
+    "a;c 141.00µs",
+    "a;b 280.00µs",
+    "a;c 268.00µs",
+    "a;b 289.00µs",
+    "a;c 146.00µs",
+    "a;b 120.00µs",
+    "a;c 436.00µs",
+    "a;b 147.00µs",
+    "a;c 164.00µs",
+    "a;b 294.00µs",
+    "a;c 147.00µs",
+    "a;b 856.00µs",
+    "a;c 273.00µs",
+    "a;b 139.00µs",
+    "a;c 269.00µs",
+    "a;b 256.00µs",
+    "a;c 258.00µs",
+    "a;b 128.00µs",
+    "a;c 255.00µs",
+    "a;b 140.00µs",
+    "a;c 514.00µs",
+    "a;b 129.00µs",
+    "a;c 395.00µs",
+    "a;b 389.00µs",
+    "a;c 257.00µs",
+    "a;b 686.00µs",
+    "a;c 147.00µs",
+    "a;b 156.00µs",
+    "a;c 137.00µs",
+    "a;b 141.00µs",
+    "a;c 140.00µs",
+    "a;b 141.00µs",
+    "a;c 141.00µs",
+    "a;b 140.00µs",
+    "a;c 140.00µs",
+    "a;b 139.00µs",
+    "a;c 129.00µs",
+    "a;b 149.00µs",
+    "a;c 267.00µs",
+    "a;b 294.00µs",
+    "a;c 145.00µs",
+    "a;b 294.00µs",
+    "a;c 294.00µs",
+    "a;b 295.00µs",
+    "a;c 140.00µs",
+    "a;b 139.00µs",
+    "a;c 139.00µs",
+    "a;b 127.00µs",
+    "a;c 513.00µs",
+    "a;b 516.00µs",
+    "a;c 385.00µs",
+    "a;b 520.00µs",
+    "a;c 252.00µs",
+    "a;b 255.00µs",
+    "a;c 134.00µs",
+    "a;b 112.00µs",
+    "a;c 140.00µs",
+    "a;b 296.00µs",
+    "a;c 441.00µs",
+    "a;b 145.00µs",
+    "a;c 292.00µs",
+    "a;b 130.00µs",
+    "a;c 277.00µs",
+    "a;b 147.00µs",
+    "a;c 286.00µs",
+    "a;b 140.00µs",
+    "a;c 136.00µs",
+    " 61.59ms",
+    "(program) 134.00µs",
+    " 128.00µs",
+    "(program) 141.00µs",
+    " 16.39ms",
+    "(program) 145.00µs",
+    " 16.63ms",
+    "(program) 131.00µs",
+    " 16.71ms",
+    "(program) 131.00µs",
+    " 294.15ms",
+    "(program) 109.00µs",
+  ],
+}
+`;
+
+exports[`importFromChromeTimeline chunked: indexToView 1`] = `0`;
+
+exports[`importFromChromeTimeline chunked: profileGroup.name 1`] = `"simple-timeline.json"`;
+
 exports[`importFromChromeTimeline: indexToView 1`] = `0`;
 
 exports[`importFromChromeTimeline: profileGroup.name 1`] = `"simple-timeline.json"`;

src/import/__snapshots__/linux-tools-perf.test.ts.snap

@@ -1376,6 +1376,268 @@ Object {
 }
 `;
 
+exports[`importFromLinuxPerf system-wide.linux-perf.txt chunked 1`] = `
+Object {
+  "frames": Array [
+    Frame {
+      "col": undefined,
+      "file": "[unknown]",
+      "key": "[unknown] ([unknown])",
+      "line": undefined,
+      "name": "??? ([unknown])",
+      "selfWeight": 0,
+      "totalWeight": 0.13823499999125488,
+    },
+    Frame {
+      "col": undefined,
+      "file": "/lib/x86_64-linux-gnu/libc-2.27.so",
+      "key": "__libc_start_main (/lib/x86_64-linux-gnu/libc-2.27.so)",
+      "line": undefined,
+      "name": "__libc_start_main",
+      "selfWeight": 0,
+      "totalWeight": 0.13823499999125488,
+    },
+    Frame {
+      "col": undefined,
+      "file": "/workdir/simple-terminates",
+      "key": "main (/workdir/simple-terminates)",
+      "line": undefined,
+      "name": "main",
+      "selfWeight": 0,
+      "totalWeight": 0.13823499999125488,
+    },
+    Frame {
+      "col": undefined,
+      "file": "/workdir/simple-terminates",
+      "key": "_Z5alphav (/workdir/simple-terminates)",
+      "line": undefined,
+      "name": "_Z5alphav",
+      "selfWeight": 0.04059999997843988,
+      "totalWeight": 0.04059999997843988,
+    },
+    Frame {
+      "col": undefined,
+      "file": "/workdir/simple-terminates",
+      "key": "_Z5deltav (/workdir/simple-terminates)",
+      "line": undefined,
+      "name": "_Z5deltav",
+      "selfWeight": 0.023244000098202378,
+      "totalWeight": 0.07264250004664063,
+    },
+    Frame {
+      "col": undefined,
+      "file": "/workdir/simple-terminates",
+      "key": "_Z4betav (/workdir/simple-terminates)",
+      "line": undefined,
+      "name": "_Z4betav",
+      "selfWeight": 0.05354449988226406,
+      "totalWeight": 0.05447799988905899,
+    },
+    Frame {
+      "col": undefined,
+      "file": "/workdir/simple-terminates",
+      "key": "_Z5gammav (/workdir/simple-terminates)",
+      "line": undefined,
+      "name": "_Z5gammav",
+      "selfWeight": 0.01991300002555363,
+      "totalWeight": 0.01991300002555363,
+    },
+    Frame {
+      "col": undefined,
+      "file": "[kernel.kallsyms]",
+      "key": "[unknown] ([kernel.kallsyms])",
+      "line": undefined,
+      "name": "??? ([kernel.kallsyms])",
+      "selfWeight": 0.0009335000067949295,
+      "totalWeight": 0.0009335000067949295,
+    },
+  ],
+  "name": "simple-terminat tid: 9",
+  "stacks": Array [
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 501.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 996.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.03ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 989.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1.07ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.04ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav;??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 933.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.46ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.47ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 997.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.04ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.06ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 965.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 2.12ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 925.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.02ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.07ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 987.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 997.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1000.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 997.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 997.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 999.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 999.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 999.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 921.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.08ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 989.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 974.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 963.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.04ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 1.04ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 991.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 922.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.02ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.07ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 983.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 999.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 999.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 998.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 998.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 974.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.03ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 998.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1000.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 999.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 997.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 999.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 999.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 999.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 997.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 932.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 2.09ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 986.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 998.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 954.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 991.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.05ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 999.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 989.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 998.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1000.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 981.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 921.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 2.09ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.01ms",
+    "??? ([unknown]);__libc_start_main;main;_Z4betav 999.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5gammav 995.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z5alphav 997.50µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav 1.00ms",
+    "??? ([unknown]);__libc_start_main;main;_Z5alphav 980.00µs",
+    "??? ([unknown]);__libc_start_main;main;_Z5deltav;_Z4betav 476.00µs",
+  ],
+}
+`;
+
+exports[`importFromLinuxPerf system-wide.linux-perf.txt chunked 2`] = `
+Object {
+  "frames": Array [
+    Frame {
+      "col": undefined,
+      "file": "[kernel.kallsyms]",
+      "key": "[unknown] ([kernel.kallsyms])",
+      "line": undefined,
+      "name": "??? ([kernel.kallsyms])",
+      "selfWeight": 0.1372890000056941,
+      "totalWeight": 0.1372890000056941,
+    },
+    Frame {
+      "col": undefined,
+      "file": "[unknown]",
+      "key": "[unknown] ([unknown])",
+      "line": undefined,
+      "name": "??? ([unknown])",
+      "selfWeight": 0.0009340000106021762,
+      "totalWeight": 0.0014475000207312405,
+    },
+  ],
+  "name": "swapper",
+  "stacks": Array [
+    "??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 17.00µs",
+    "??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 9.55ms",
+    "??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]) 487.50µs",
+    "??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 486.00µs",
+    "??? ([unknown]);??? ([unknown]) 446.50µs",
+    "??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 34.50µs",
+    "??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 38.00µs",
+    "??? ([kernel.kallsyms]);??? ([unknown]);??? ([unknown]);??? ([unknown]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 479.00µs",
+    "??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 6.52ms",
+    "??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 517.00µs",
+    "??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 111.12ms",
+    "??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 399.50µs",
+    "??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]);??? ([kernel.kallsyms]) 8.13ms",
+  ],
+}
+`;
+
+exports[`importFromLinuxPerf system-wide.linux-perf.txt chunked: indexToView 1`] = `0`;
+
+exports[`importFromLinuxPerf system-wide.linux-perf.txt chunked: profileGroup.name 1`] = `"system-wide.linux-perf.txt"`;
+
 exports[`importFromLinuxPerf system-wide.linux-perf.txt: indexToView 1`] = `0`;
 
 exports[`importFromLinuxPerf system-wide.linux-perf.txt: profileGroup.name 1`] = `"system-wide.linux-perf.txt"`;

src/import/__snapshots__/trace-event.test.ts.snap

@@ -842,6 +842,62 @@ Object {
 }
 `;
 
+exports[`importTraceEvents partial json import chunked 1`] = `
+Object {
+  "frames": Array [
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "alpha",
+      "line": undefined,
+      "name": "alpha",
+      "selfWeight": 2,
+      "totalWeight": 14,
+    },
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "beta",
+      "line": undefined,
+      "name": "beta",
+      "selfWeight": 3,
+      "totalWeight": 12,
+    },
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "gamma {\\"detail\\":\\"foobar\\"}",
+      "line": undefined,
+      "name": "gamma {\\"detail\\":\\"foobar\\"}",
+      "selfWeight": 5,
+      "totalWeight": 5,
+    },
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "epsilon",
+      "line": undefined,
+      "name": "epsilon",
+      "selfWeight": 4,
+      "totalWeight": 4,
+    },
+  ],
+  "name": "pid 0, tid 0",
+  "stacks": Array [
+    "alpha 1.00µs",
+    "alpha;beta 1.00µs",
+    "alpha;beta;gamma {\\"detail\\":\\"foobar\\"} 5.00µs",
+    "alpha;beta;epsilon 4.00µs",
+    "alpha;beta 2.00µs",
+    "alpha 1.00µs",
+  ],
+}
+`;
+
+exports[`importTraceEvents partial json import chunked: indexToView 1`] = `0`;
+
+exports[`importTraceEvents partial json import chunked: profileGroup.name 1`] = `"simple-partial.json"`;
+
 exports[`importTraceEvents partial json import trailing comma 1`] = `
 Object {
   "frames": Array [
@@ -894,6 +950,62 @@ Object {
 }
 `;
 
+exports[`importTraceEvents partial json import trailing comma chunked 1`] = `
+Object {
+  "frames": Array [
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "alpha",
+      "line": undefined,
+      "name": "alpha",
+      "selfWeight": 2,
+      "totalWeight": 14,
+    },
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "beta",
+      "line": undefined,
+      "name": "beta",
+      "selfWeight": 3,
+      "totalWeight": 12,
+    },
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "gamma {\\"detail\\":\\"foobar\\"}",
+      "line": undefined,
+      "name": "gamma {\\"detail\\":\\"foobar\\"}",
+      "selfWeight": 5,
+      "totalWeight": 5,
+    },
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "epsilon",
+      "line": undefined,
+      "name": "epsilon",
+      "selfWeight": 4,
+      "totalWeight": 4,
+    },
+  ],
+  "name": "pid 0, tid 0",
+  "stacks": Array [
+    "alpha 1.00µs",
+    "alpha;beta 1.00µs",
+    "alpha;beta;gamma {\\"detail\\":\\"foobar\\"} 5.00µs",
+    "alpha;beta;epsilon 4.00µs",
+    "alpha;beta 2.00µs",
+    "alpha 1.00µs",
+  ],
+}
+`;
+
+exports[`importTraceEvents partial json import trailing comma chunked: indexToView 1`] = `0`;
+
+exports[`importTraceEvents partial json import trailing comma chunked: profileGroup.name 1`] = `"simple-partial-trailing-comma.json"`;
+
 exports[`importTraceEvents partial json import trailing comma: indexToView 1`] = `0`;
 
 exports[`importTraceEvents partial json import trailing comma: profileGroup.name 1`] = `"simple-partial-trailing-comma.json"`;
@@ -950,6 +1062,62 @@ Object {
 }
 `;
 
+exports[`importTraceEvents partial json import whitespace padding chunked 1`] = `
+Object {
+  "frames": Array [
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "alpha",
+      "line": undefined,
+      "name": "alpha",
+      "selfWeight": 2,
+      "totalWeight": 14,
+    },
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "beta",
+      "line": undefined,
+      "name": "beta",
+      "selfWeight": 3,
+      "totalWeight": 12,
+    },
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "gamma {\\"detail\\":\\"foobar\\"}",
+      "line": undefined,
+      "name": "gamma {\\"detail\\":\\"foobar\\"}",
+      "selfWeight": 5,
+      "totalWeight": 5,
+    },
+    Frame {
+      "col": undefined,
+      "file": undefined,
+      "key": "epsilon",
+      "line": undefined,
+      "name": "epsilon",
+      "selfWeight": 4,
+      "totalWeight": 4,
+    },
+  ],
+  "name": "pid 0, tid 0",
+  "stacks": Array [
+    "alpha 1.00µs",
+    "alpha;beta 1.00µs",
+    "alpha;beta;gamma {\\"detail\\":\\"foobar\\"} 5.00µs",
+    "alpha;beta;epsilon 4.00µs",
+    "alpha;beta 2.00µs",
+    "alpha 1.00µs",
+  ],
+}
+`;
+
+exports[`importTraceEvents partial json import whitespace padding chunked: indexToView 1`] = `0`;
+
+exports[`importTraceEvents partial json import whitespace padding chunked: profileGroup.name 1`] = `"simple-partial-whitespace.json"`;
+
 exports[`importTraceEvents partial json import whitespace padding: indexToView 1`] = `0`;
 
 exports[`importTraceEvents partial json import whitespace padding: profileGroup.name 1`] = `"simple-partial-whitespace.json"`;

src/import/bg-flamegraph.ts

@@ -1,25 +1,31 @@
 // https://github.com/brendangregg/FlameGraph#2-fold-stacks
 
 import {Profile, FrameInfo, StackListProfileBuilder} from '../lib/profile'
+import {TextFileContent} from './utils'
 
 interface BGSample {
   stack: FrameInfo[]
   duration: number
 }
 
-function parseBGFoldedStacks(contents: string): BGSample[] {
+function parseBGFoldedStacks(contents: TextFileContent): BGSample[] {
   const samples: BGSample[] = []
-  contents.replace(/^(.*) (\d+)$/gm, (match: string, stack: string, n: string) => {
+  for (const line of contents.splitLines()) {
+    const match = /^(.*) (\d+)$/gm.exec(line)
+    if (!match) continue
+    const stack = match[1]
+    const n = match[2]
+
     samples.push({
       stack: stack.split(';').map(name => ({key: name, name: name})),
       duration: parseInt(n, 10),
     })
-    return match
+  }
-  })
+
   return samples
 }
 
-export function importFromBGFlameGraph(contents: string): Profile | null {
+export function importFromBGFlameGraph(contents: TextFileContent): Profile | null {
   const parsed = parseBGFoldedStacks(contents)
   const duration = parsed.reduce((prev: number, cur: BGSample) => prev + cur.duration, 0)
   const profile = new StackListProfileBuilder(duration)

src/import/callgrind.ts

@@ -88,6 +88,7 @@
 import {CallTreeProfileBuilder, Frame, FrameInfo, Profile, ProfileGroup} from '../lib/profile'
 import {getOrElse, getOrInsert, KeyedSet} from '../lib/utils'
 import {ByteFormatter, TimeFormatter} from '../lib/value-formatters'
+import {TextFileContent} from './utils'
 
 class CallGraph {
   private frameSet = new KeyedSet<Frame>()
@@ -291,8 +292,8 @@ class CallgrindParser {
   private savedFileNames: {[id: string]: string} = {}
   private savedFunctionNames: {[id: string]: string} = {}
 
-  constructor(contents: string, private importedFileName: string) {
+  constructor(contents: TextFileContent, private importedFileName: string) {
-    this.lines = contents.split('\n')
+    this.lines = contents.splitLines()
     this.lineNum = 0
   }
 
@@ -510,7 +511,7 @@ class CallgrindParser {
 }
 
 export function importFromCallgrind(
-  contents: string,
+  contents: TextFileContent,
   importedFileName: string,
 ): ProfileGroup | null {
   return new CallgrindParser(contents, importedFileName).parse()

src/import/chrome.test.ts

@@ -1,4 +1,5 @@
 import {checkProfileSnapshot} from '../lib/test-utils'
+import {withMockedFileChunkSizeForTests} from './utils'
 
 test('importFromChromeCPUProfile', async () => {
   await checkProfileSnapshot('./sample/profiles/Chrome/65/simple.cpuprofile')
@@ -8,6 +9,12 @@ test('importFromChromeTimeline', async () => {
   await checkProfileSnapshot('./sample/profiles/Chrome/65/simple-timeline.json')
 })
 
+test('importFromChromeTimeline chunked', async () => {
+  await withMockedFileChunkSizeForTests(100, async () => {
+    await checkProfileSnapshot('./sample/profiles/Chrome/65/simple-timeline.json')
+  })
+})
+
 test('importFromChromeTimeline Chrome 69', async () => {
   await checkProfileSnapshot('./sample/profiles/Chrome/69/simple.json')
 })

src/import/index.ts

@@ -73,26 +73,6 @@ function toGroup(profile: Profile | null): ProfileGroup | null {
   return {name: profile.getName(), indexToView: 0, profiles: [profile]}
 }
 
-function fixUpJSON(content: string): string {
-  // This code is similar to the code from here:
-  // https://github.com/catapult-project/catapult/blob/27e047e0494df162022be6aa8a8862742a270232/tracing/tracing/extras/importer/trace_event_importer.html#L197-L208
-  //
-  //   If the event data begins with a [, then we know it should end with a ]. The
-  //   reason we check for this is because some tracing implementations cannot
-  //   guarantee that a ']' gets written to the trace file. So, we are forgiving
-  //   and if this is obviously the case, we fix it up before throwing the string
-  //   at JSON.parse.
-  //
-  content = content.trim()
-  if (content[0] === '[') {
-    content = content.replace(/,\s*$/, '')
-    if (content[content.length - 1] !== ']') {
-      content += ']'
-    }
-  }
-  return content
-}
-
 async function _importProfileGroup(dataSource: ProfileDataSource): Promise<ProfileGroup | null> {
   const fileName = await dataSource.name()
 
@@ -111,13 +91,13 @@ async function _importProfileGroup(dataSource: ProfileDataSource): Promise<Profi
   // First pass: Check known file format names to infer the file type
   if (fileName.endsWith('.speedscope.json')) {
     console.log('Importing as speedscope json file')
-    return importSpeedscopeProfiles(JSON.parse(contents))
+    return importSpeedscopeProfiles(contents.parseAsJSON())
   } else if (fileName.endsWith('.chrome.json') || /Profile-\d{8}T\d{6}/.exec(fileName)) {
     console.log('Importing as Chrome Timeline')
-    return importFromChromeTimeline(JSON.parse(contents), fileName)
+    return importFromChromeTimeline(contents.parseAsJSON(), fileName)
   } else if (fileName.endsWith('.stackprof.json')) {
     console.log('Importing as stackprof profile')
-    return toGroup(importFromStackprof(JSON.parse(contents)))
+    return toGroup(importFromStackprof(contents.parseAsJSON()))
   } else if (fileName.endsWith('.instruments.txt')) {
     console.log('Importing as Instruments.app deep copy')
     return toGroup(importFromInstrumentsDeepCopy(contents))
@@ -129,13 +109,13 @@ async function _importProfileGroup(dataSource: ProfileDataSource): Promise<Profi
     return toGroup(importFromBGFlameGraph(contents))
   } else if (fileName.endsWith('.v8log.json')) {
     console.log('Importing as --prof-process v8 log')
-    return toGroup(importFromV8ProfLog(JSON.parse(contents)))
+    return toGroup(importFromV8ProfLog(contents.parseAsJSON()))
   } else if (fileName.endsWith('.heapprofile')) {
     console.log('Importing as Chrome Heap Profile')
-    return toGroup(importFromChromeHeapProfile(JSON.parse(contents)))
+    return toGroup(importFromChromeHeapProfile(contents.parseAsJSON()))
   } else if (fileName.endsWith('-recording.json')) {
     console.log('Importing as Safari profile')
-    return toGroup(importFromSafari(JSON.parse(contents)))
+    return toGroup(importFromSafari(contents.parseAsJSON()))
   } else if (fileName.startsWith('callgrind.')) {
     console.log('Importing as Callgrind profile')
     return importFromCallgrind(contents, fileName)
@@ -144,12 +124,12 @@ async function _importProfileGroup(dataSource: ProfileDataSource): Promise<Profi
   // Second pass: Try to guess what file format it is based on structure
   let parsed: any
   try {
-    parsed = JSON.parse(fixUpJSON(contents))
+    parsed = contents.parseAsJSON()
   } catch (e) {}
   if (parsed) {
     if (parsed['$schema'] === 'https://www.speedscope.app/file-format-schema.json') {
       console.log('Importing as speedscope json file')
-      return importSpeedscopeProfiles(JSON.parse(contents))
+      return importSpeedscopeProfiles(parsed)
     } else if (parsed['systemHost'] && parsed['systemHost']['name'] == 'Firefox') {
       console.log('Importing as Firefox profile')
       return toGroup(importFromFirefox(parsed))
@@ -173,13 +153,13 @@ async function _importProfileGroup(dataSource: ProfileDataSource): Promise<Profi
       return toGroup(importFromV8ProfLog(parsed))
     } else if ('head' in parsed && 'selfSize' in parsed['head']) {
       console.log('Importing as Chrome Heap Profile')
-      return toGroup(importFromChromeHeapProfile(JSON.parse(contents)))
+      return toGroup(importFromChromeHeapProfile(parsed))
     } else if ('rts_arguments' in parsed && 'initial_capabilities' in parsed) {
       console.log('Importing as Haskell GHC JSON Profile')
       return importFromHaskell(parsed)
     } else if ('recording' in parsed && 'sampleStackTraces' in parsed.recording) {
       console.log('Importing as Safari profile')
-      return toGroup(importFromSafari(JSON.parse(contents)))
+      return toGroup(importFromSafari(parsed))
     }
   } else {
     // Format is not JSON
@@ -187,8 +167,8 @@ async function _importProfileGroup(dataSource: ProfileDataSource): Promise<Profi
     // If the first line is "# callgrind format", it's probably in Callgrind
     // Profile Format.
     if (
-      /^# callgrind format/.exec(contents) ||
+      /^# callgrind format/.exec(contents.firstChunk()) ||
-      (/^events:/m.exec(contents) && /^fn=/m.exec(contents))
+      (/^events:/m.exec(contents.firstChunk()) && /^fn=/m.exec(contents.firstChunk()))
     ) {
       console.log('Importing as Callgrind profile')
       return importFromCallgrind(contents, fileName)
@@ -196,7 +176,7 @@ async function _importProfileGroup(dataSource: ProfileDataSource): Promise<Profi
 
     // If the first line contains "Symbol Name", preceded by a tab, it's probably
     // a deep copy from OS X Instruments.app
-    if (/^[\w \t\(\)]*\tSymbol Name/.exec(contents)) {
+    if (/^[\w \t\(\)]*\tSymbol Name/.exec(contents.firstChunk())) {
       console.log('Importing as Instruments.app deep copy')
       return toGroup(importFromInstrumentsDeepCopy(contents))
     }

src/import/instruments.ts

@@ -11,10 +11,10 @@ import {
 import {sortBy, getOrThrow, getOrInsert, lastOf, getOrElse, zeroPad} from '../lib/utils'
 import {ByteFormatter, TimeFormatter} from '../lib/value-formatters'
 import {FileSystemDirectoryEntry, FileSystemEntry, FileSystemFileEntry} from './file-system-entry'
-import {MaybeCompressedDataReader} from './utils'
+import {MaybeCompressedDataReader, TextFileContent} from './utils'
 
-function parseTSV<T>(contents: string): T[] {
+function parseTSV<T>(contents: TextFileContent): T[] {
-  const lines = contents.split('\n').map(l => l.split('\t'))
+  const lines = contents.splitLines().map(l => l.split('\t'))
 
   const headerLine = lines.shift()
   if (!headerLine) return []
@@ -94,7 +94,7 @@ function getWeight(deepCopyRow: any): number {
 }
 
 // Import from a deep copy made of a profile
-export function importFromInstrumentsDeepCopy(contents: string): Profile {
+export function importFromInstrumentsDeepCopy(contents: TextFileContent): Profile {
   const profile = new CallTreeProfileBuilder()
   const rows = parseTSV<PastedTimeProfileRow | PastedAllocationsProfileRow>(contents)
 
@@ -187,7 +187,7 @@ function readAsArrayBuffer(file: File): Promise<ArrayBuffer> {
   return MaybeCompressedDataReader.fromFile(file).readAsArrayBuffer()
 }
 
-function readAsText(file: File): Promise<string> {
+function readAsText(file: File): Promise<TextFileContent> {
   return MaybeCompressedDataReader.fromFile(file).readAsText()
 }
 
@@ -259,7 +259,7 @@ async function getRawSampleList(core: TraceDirectoryTree): Promise<Sample[]> {
     const schemaFile = storedir.files.get('schema.xml')
     if (!schemaFile) continue
     const schema = await readAsText(schemaFile)
-    if (!/name="time-profile"/.exec(schema)) {
+    if (!/name="time-profile"/.exec(schema.firstChunk())) {
       continue
     }
     const bulkstore = new BinReader(

src/import/linux-tools-perf.test.ts

@@ -1,4 +1,5 @@
 import {checkProfileSnapshot} from '../lib/test-utils'
+import {withMockedFileChunkSizeForTests} from './utils'
 
 describe('importFromLinuxPerf', () => {
   test('simple.linux-perf.txt', async () => {
@@ -19,4 +20,9 @@ describe('importFromLinuxPerf', () => {
   test('system-wide.linux-perf.txt', async () => {
     await checkProfileSnapshot('./sample/profiles/linux-perf/system-wide.linux-perf.txt')
   })
+  test('system-wide.linux-perf.txt chunked', async () => {
+    await withMockedFileChunkSizeForTests(100, async () => {
+      await checkProfileSnapshot('./sample/profiles/linux-perf/system-wide.linux-perf.txt')
+    })
+  })
 })

src/import/linux-tools-perf.ts

@@ -1,6 +1,7 @@
 import {StackListProfileBuilder, ProfileGroup} from '../lib/profile'
 import {itMap, getOrInsert} from '../lib/utils'
 import {TimeFormatter} from '../lib/value-formatters'
+import {TextFileContent} from './utils'
 
 // This imports the output of the "perf script" command on linux.
 //
@@ -80,11 +81,39 @@ function parseEvent(rawEvent: string): PerfEvent | null {
   return event
 }
 
-export function importFromLinuxPerf(contents: string): ProfileGroup | null {
+function splitBlocks(contents: TextFileContent): string[] {
+  // In perf files, blocks are separated by '\n\n'.  If our input was a simple
+  // string, we could use str.split('\n\n'), but since we have a TextFileContent
+  // object which may be backed by several strings, we can't easily split this
+  // way.
+  //
+  // Instead, we'll split into lines, and then re-group runs of non-empty strings.
+  const blocks: string[] = []
+  let pending: string = ''
+  for (let line of contents.splitLines()) {
+    if (line === '') {
+      if (pending.length > 0) {
+        blocks.push(pending)
+      }
+      pending = line
+    } else {
+      if (pending.length > 0) {
+        pending += '\n'
+      }
+      pending += line
+    }
+  }
+  if (pending.length > 0) {
+    blocks.push(pending)
+  }
+  return blocks
+}
+
+export function importFromLinuxPerf(contents: TextFileContent): ProfileGroup | null {
   const profiles = new Map<string, StackListProfileBuilder>()
 
   let eventType: string | null = null
-  const events = contents.split('\n\n').map(parseEvent)
+  const events = splitBlocks(contents).map(parseEvent)
 
   for (let event of events) {
     if (event == null) continue

src/import/trace-event.test.ts

@@ -1,4 +1,5 @@
 import {checkProfileSnapshot} from '../lib/test-utils'
+import {withMockedFileChunkSizeForTests} from './utils'
 
 test('importTraceEvents simple', async () => {
   await checkProfileSnapshot('./sample/profiles/trace-event/simple.json')
@@ -16,14 +17,32 @@ test('importTraceEvents partial json import', async () => {
   await checkProfileSnapshot('./sample/profiles/trace-event/simple-partial.json')
 })
 
+test('importTraceEvents partial json import chunked', async () => {
+  await withMockedFileChunkSizeForTests(100, async () => {
+    await checkProfileSnapshot('./sample/profiles/trace-event/simple-partial.json')
+  })
+})
+
 test('importTraceEvents partial json import trailing comma', async () => {
   await checkProfileSnapshot('./sample/profiles/trace-event/simple-partial-trailing-comma.json')
 })
 
+test('importTraceEvents partial json import trailing comma chunked', async () => {
+  await withMockedFileChunkSizeForTests(100, async () => {
+    await checkProfileSnapshot('./sample/profiles/trace-event/simple-partial-trailing-comma.json')
+  })
+})
+
 test('importTraceEvents partial json import whitespace padding', async () => {
   await checkProfileSnapshot('./sample/profiles/trace-event/simple-partial-whitespace.json')
 })
 
+test('importTraceEvents partial json import whitespace padding chunked', async () => {
+  await withMockedFileChunkSizeForTests(100, async () => {
+    await checkProfileSnapshot('./sample/profiles/trace-event/simple-partial-whitespace.json')
+  })
+})
+
 test('importTraceEvents bad E events', async () => {
   await checkProfileSnapshot('./sample/profiles/trace-event/too-many-end-events.json')
 })

src/import/utils.ts

@@ -1,9 +1,197 @@
 import * as pako from 'pako'
+import {JSON_parse} from 'uint8array-json-parser'
 
 export interface ProfileDataSource {
   name(): Promise<string>
   readAsArrayBuffer(): Promise<ArrayBuffer>
-  readAsText(): Promise<string>
+  readAsText(): Promise<TextFileContent>
+}
+
+export interface TextFileContent {
+  splitLines(): string[]
+  firstChunk(): string
+  parseAsJSON(): any
+}
+
+// V8 has a maximum string size. To support files whose contents exceeds that
+// size, we provide an alternate string interface for text backed by a
+// Uint8Array instead.
+//
+// We provide a simple splitLines() which returns simple strings under the
+// assumption that most extremely large text profiles will be broken into many
+// lines.  This isn't true in the general case, but will be true for most common
+// large files.
+//
+// See: https://github.com/v8/v8/blob/8b663818fc311217c2cdaaab935f020578bfb7a8/src/objects/string.h#L479-L483
+//
+// At time of writing (2021/03/27), the maximum string length in V8 is
+//  32 bit systems: 2^28 - 16 = ~268M chars
+//  64 bit systems: 2^29 - 24 = ~537M chars
+//
+// https://source.chromium.org/chromium/chromium/src/+/main:v8/include/v8-primitive.h;drc=cb88fe94d9044d860cc75c89e1bc270ab4062702;l=125
+//
+// We'll be conservative and feed in 2^27 bytes at a time (~134M chars
+// assuming utf-8 encoding)
+let TEXT_FILE_CHUNK_SIZE = 1 << 27
+
+export async function withMockedFileChunkSizeForTests(chunkSize: number, cb: () => any) {
+  const original = TEXT_FILE_CHUNK_SIZE
+  TEXT_FILE_CHUNK_SIZE = chunkSize
+  try {
+    await cb()
+  } finally {
+    TEXT_FILE_CHUNK_SIZE = original
+  }
+}
+
+function permissivelyParseJSONString(content: string) {
+  // This code is similar to the code from here:
+  // https://github.com/catapult-project/catapult/blob/27e047e0494df162022be6aa8a8862742a270232/tracing/tracing/extras/importer/trace_event_importer.html#L197-L208
+  //
+  //   If the event data begins with a [, then we know it should end with a ]. The
+  //   reason we check for this is because some tracing implementations cannot
+  //   guarantee that a ']' gets written to the trace file. So, we are forgiving
+  //   and if this is obviously the case, we fix it up before throwing the string
+  //   at JSON.parse.
+  content = content.trim()
+  if (content[0] === '[') {
+    content = content.replace(/,\s*$/, '')
+    if (content[content.length - 1] !== ']') {
+      content += ']'
+    }
+  }
+  return JSON.parse(content)
+}
+
+function permissivelyParseJSONUint8Array(byteArray: Uint8Array) {
+  let indexOfFirstNonWhitespaceChar = 0
+  for (let i = 0; i < byteArray.length; i++) {
+    if (!/\s/.exec(String.fromCharCode(byteArray[i]))) {
+      indexOfFirstNonWhitespaceChar = i
+      break
+    }
+  }
+  if (
+    byteArray[indexOfFirstNonWhitespaceChar] === '['.charCodeAt(0) &&
+    byteArray[byteArray.length - 1] !== ']'.charCodeAt(0)
+  ) {
+    // Strip trailing whitespace from the end of the array
+    let trimmedLength = byteArray.length
+    while (trimmedLength > 0 && /\s/.exec(String.fromCharCode(byteArray[trimmedLength - 1]))) {
+      trimmedLength--
+    }
+
+    // Ignore trailing comma
+    if (String.fromCharCode(byteArray[trimmedLength - 1]) === ',') {
+      trimmedLength--
+    }
+
+    if (String.fromCharCode(byteArray[trimmedLength - 1]) !== ']') {
+      // Clone the array, ignoring any whitespace & trailing comma, then append a ']'
+      //
+      // Note: We could save a tiny bit of space here by avoiding copying the
+      // leading whitespace, but it's a trivial perf boost and it complicates
+      // the code.
+      const newByteArray = new Uint8Array(trimmedLength + 1)
+      newByteArray.set(byteArray.subarray(0, trimmedLength))
+      newByteArray[trimmedLength] = ']'.charCodeAt(0)
+      byteArray = newByteArray
+    }
+  }
+  return JSON_parse(byteArray)
+}
+
+export class BufferBackedTextFileContent implements TextFileContent {
+  private chunks: string[] = []
+  private byteArray: Uint8Array
+
+  constructor(buffer: ArrayBuffer) {
+    const byteArray = (this.byteArray = new Uint8Array(buffer))
+
+    let encoding: string = 'utf-8'
+    if (byteArray.length > 2) {
+      if (byteArray[0] === 0xff && byteArray[1] === 0xfe) {
+        // UTF-16, Little Endian encoding
+        encoding = 'utf-16le'
+      } else if (byteArray[0] === 0xfe && byteArray[1] === 0xff) {
+        // UTF-16, Big Endian encoding
+        encoding = 'utf-16be'
+      }
+    }
+
+    if (typeof TextDecoder !== 'undefined') {
+      // If TextDecoder is available, we'll try to use it to decode the string.
+      const decoder = new TextDecoder(encoding)
+
+      for (let chunkNum = 0; chunkNum < buffer.byteLength / TEXT_FILE_CHUNK_SIZE; chunkNum++) {
+        const offset = chunkNum * TEXT_FILE_CHUNK_SIZE
+        const view = new Uint8Array(
+          buffer,
+          offset,
+          Math.min(buffer.byteLength - offset, TEXT_FILE_CHUNK_SIZE),
+        )
+        const chunk = decoder.decode(view, {stream: true})
+        this.chunks.push(chunk)
+      }
+    } else {
+      // JavaScript strings are UTF-16 encoded, but we're reading data from disk
+      // that we're going to blindly assume it's ASCII encoded. This codepath
+      // only exists for older browser support.
+
+      console.warn('This browser does not support TextDecoder. Decoding text as ASCII.')
+      this.chunks.push('')
+      for (let i = 0; i < byteArray.length; i++) {
+        this.chunks[this.chunks.length - 1] += String.fromCharCode(byteArray[i])
+        ;(this.chunks[this.chunks.length - 1] as any) | 0 // This forces the string to be flattened
+
+        if (this.chunks[this.chunks.length - 1].length >= TEXT_FILE_CHUNK_SIZE) {
+          this.chunks.push('')
+        }
+      }
+    }
+  }
+
+  splitLines(): string[] {
+    let parts: string[] = this.chunks[0].split('\n')
+    for (let i = 1; i < this.chunks.length; i++) {
+      const chunkParts = this.chunks[i].split('\n')
+      if (chunkParts.length === 0) continue
+      if (parts.length > 0) {
+        parts[parts.length - 1] += chunkParts.shift()
+      }
+      parts = parts.concat(chunkParts)
+    }
+    return parts
+  }
+
+  firstChunk(): string {
+    return this.chunks[0] || ''
+  }
+
+  parseAsJSON(): any {
+    // We only use the Uint8Array version of JSON.parse when necessary, because
+    // it's around 4x slower than native.
+    if (this.chunks.length === 1) {
+      return permissivelyParseJSONString(this.chunks[0])
+    }
+    return permissivelyParseJSONUint8Array(this.byteArray)
+  }
+}
+
+export class StringBackedTextFileContent implements TextFileContent {
+  constructor(private s: string) {}
+
+  splitLines(): string[] {
+    return this.s.split('\n')
+  }
+
+  firstChunk(): string {
+    return this.s
+  }
+
+  parseAsJSON(): any {
+    return permissivelyParseJSONString(this.s)
+  }
 }
 
 export class TextProfileDataSource implements ProfileDataSource {
@@ -11,16 +199,13 @@ export class TextProfileDataSource implements ProfileDataSource {
   async name() {
     return this.fileName
   }
-  async readAsArrayBuffer() {
-    // JavaScript strings are UTF-16 encoded, but if this string is
-    // constructed based on
 
-    // TODO(jlfwong): Might want to make this construct an array
+  async readAsArrayBuffer() {
-    // buffer based on the text
     return new ArrayBuffer(0)
   }
+
   async readAsText() {
-    return this.contents
+    return new StringBackedTextFileContent(this.contents)
   }
 }
 
@@ -49,37 +234,9 @@ export class MaybeCompressedDataReader implements ProfileDataSource {
     return await this.uncompressedData
   }
 
-  async readAsText(): Promise<string> {
+  async readAsText(): Promise<TextFileContent> {
     const buffer = await this.readAsArrayBuffer()
-
+    return new BufferBackedTextFileContent(buffer)
-    // By default, we assume the file is utf-8 encoded.
-    let encoding = 'utf-8'
-
-    const array = new Uint8Array(buffer)
-    if (array.length > 2) {
-      if (array[0] === 0xff && array[1] === 0xfe) {
-        // UTF-16, Little Endian encoding
-        encoding = 'utf-16le'
-      } else if (array[0] === 0xfe && array[1] === 0xff) {
-        // UTF-16, Big Endian encoding
-        encoding = 'utf-16be'
-      }
-    }
-
-    if (typeof TextDecoder !== 'undefined') {
-      const decoder = new TextDecoder(encoding)
-      return decoder.decode(buffer)
-    } else {
-      // JavaScript strings are UTF-16 encoded, but we're reading data from disk
-      // that we're going to blindly assume it's ASCII encoded. This codepath
-      // only exists for older browser support.
-      console.warn('This browser does not support TextDecoder. Decoding text as ASCII.')
-      let ret: string = ''
-      for (let i = 0; i < array.length; i++) {
-        ret += String.fromCharCode(array[i])
-      }
-      return ret
-    }
   }
 
   static fromFile(file: File): MaybeCompressedDataReader {

src/lib/utils.test.ts

@@ -17,7 +17,14 @@ import {
   findIndexBisect,
 } from './utils'
 
+import {TextEncoder} from 'util'
+
 import * as jsc from 'jsverify'
+import {
+  BufferBackedTextFileContent,
+  StringBackedTextFileContent,
+  withMockedFileChunkSizeForTests,
+} from '../import/utils'
 
 test('sortBy', () => {
   const ls = ['a3', 'b2', 'c1', 'd4']
@@ -229,3 +236,53 @@ test('decodeBase64', () => {
     return true
   })
 })
+
+test('BufferBackedTextFileContent.firstChunk', async () => {
+  await withMockedFileChunkSizeForTests(2, () => {
+    const str = 'may\nyour\nrope\nbe\nlong'
+    const buffer = new TextEncoder().encode(str).buffer
+    const content = new BufferBackedTextFileContent(buffer)
+    expect(content.firstChunk()).toEqual('ma')
+  })
+})
+
+test('BufferBackedTextFileContent.splitLines', async () => {
+  await withMockedFileChunkSizeForTests(2, () => {
+    const str = 'may\nyour\nrope\nbe\nlong'
+    const buffer = new TextEncoder().encode(str).buffer
+    const content = new BufferBackedTextFileContent(buffer)
+    expect(content.splitLines()).toEqual(['may', 'your', 'rope', 'be', 'long'])
+  })
+})
+
+test('BufferBackedTextFileContent.parseAsJSON', async () => {
+  await withMockedFileChunkSizeForTests(2, () => {
+    // parseAsJSON is special cased to permissively allow trailing commas
+    // and a mission closing bracket
+    const str = '[200,300,400,'
+    const buffer = new TextEncoder().encode(str).buffer
+    const content = new BufferBackedTextFileContent(buffer)
+
+    expect(content.parseAsJSON()).toEqual([200, 300, 400])
+  })
+})
+
+test('StringBackedTextFileContent.firstChunk', async () => {
+  const str = 'may\nyour\nrope\nbe\nlong'
+  const content = new StringBackedTextFileContent(str)
+  expect(content.firstChunk()).toEqual(str)
+})
+
+test('StringBackedTextFileContent.splitLines', async () => {
+  const str = 'may\nyour\nrope\nbe\nlong'
+  const content = new StringBackedTextFileContent(str)
+  expect(content.splitLines()).toEqual(['may', 'your', 'rope', 'be', 'long'])
+})
+
+test('StringBackedTextFileContent.parseAsJSON', async () => {
+  // parseAsJSON is special cased to permissively allow trailing commas
+  // and a mission closing bracket
+  const str = '[200,300,400,'
+  const content = new StringBackedTextFileContent(str)
+  expect(content.parseAsJSON()).toEqual([200, 300, 400])
+})