chore: read trace off zip file (#9377)

package-lock.json

@@ -47,6 +47,7 @@
         "source-map-support": "^0.4.18",
         "stack-utils": "^2.0.3",
         "ws": "^7.4.6",
+        "yauzl": "^2.10.0",
         "yazl": "^2.5.1"
       },
       "bin": {

package.json

@@ -79,6 +79,7 @@
     "source-map-support": "^0.4.18",
     "stack-utils": "^2.0.3",
     "ws": "^7.4.6",
+    "yauzl": "^2.10.0",
     "yazl": "^2.5.1"
   },
   "devDependencies": {

src/server/snapshot/inMemorySnapshotter.ts

@@ -96,7 +96,7 @@ export class InMemorySnapshotter extends BaseSnapshotStorage implements Snapshot
     this.addFrameSnapshot(snapshot);
   }
 
-  resourceContent(sha1: string): Buffer | undefined {
+  async resourceContent(sha1: string): Promise<Buffer | undefined> {
     return this._blobs.get(sha1);
   }
 }

src/server/snapshot/snapshotServer.ts

@@ -17,7 +17,7 @@
 import * as http from 'http';
 import querystring from 'querystring';
 import { HttpServer } from '../../utils/httpServer';
-import type { RenderedFrameSnapshot } from './snapshotTypes';
+import type { RenderedFrameSnapshot, ResourceSnapshot } from './snapshotTypes';
 import { SnapshotStorage } from './snapshotStorage';
 import type { Point } from '../../common/types';
 
@@ -176,11 +176,19 @@ export class SnapshotServer {
     const sha1 = resource.response.content._sha1;
     if (!sha1)
       return false;
+    (async () => {
+      this._innerServeResource(sha1, resource, response);
+    })().catch(() => {});
+    return true;
+  }
 
-    try {
-      const content = this._snapshotStorage.resourceContent(sha1);
-      if (!content)
-        return false;
+  private async _innerServeResource(sha1: string, resource: ResourceSnapshot, response: http.ServerResponse) {
+    const content = await this._snapshotStorage.resourceContent(sha1);
+    if (!content) {
+      response.statusCode = 404;
+      response.end();
+      return;
+    }
     response.statusCode = 200;
     let contentType = resource.response.content.mimeType;
     const isTextEncoding = /^text\/|^application\/(javascript|json)/.test(contentType);
@@ -203,10 +211,6 @@ export class SnapshotServer {
     response.setHeader('Content-Length', content.byteLength);
     response.setHeader('Cache-Control', 'public, max-age=31536000');
     response.end(content);
-      return true;
-    } catch (e) {
-      return false;
-    }
   }
 }
 

src/server/snapshot/snapshotStorage.ts

@@ -20,7 +20,7 @@ import { SnapshotRenderer } from './snapshotRenderer';
 
 export interface SnapshotStorage {
   resources(): ResourceSnapshot[];
-  resourceContent(sha1: string): Buffer | undefined;
+  resourceContent(sha1: string): Promise<Buffer | undefined>;
   snapshotByName(pageOrFrameId: string, snapshotName: string): SnapshotRenderer | undefined;
   snapshotByIndex(frameId: string, index: number): SnapshotRenderer | undefined;
 }
@@ -58,7 +58,7 @@ export abstract class BaseSnapshotStorage extends EventEmitter implements Snapsh
     this.emit('snapshot', renderer);
   }
 
-  abstract resourceContent(sha1: string): Buffer | undefined;
+  abstract resourceContent(sha1: string): Promise<Buffer | undefined>;
 
   resources(): ResourceSnapshot[] {
     return this._resources.slice();

src/server/trace/viewer/traceModel.ts

@@ -14,13 +14,12 @@
  * limitations under the License.
  */
 
-import fs from 'fs';
-import path from 'path';
 import * as trace from '../common/traceEvents';
 import { ResourceSnapshot } from '../../snapshot/snapshotTypes';
 import { BaseSnapshotStorage } from '../../snapshot/snapshotStorage';
 import { BrowserContextOptions } from '../../types';
 import { shouldCaptureSnapshot, VERSION } from '../recorder/tracing';
+import { VirtualFileSystem } from '../../../utils/vfs';
 export * as trace from '../common/traceEvents';
 
 export class TraceModel {
@@ -180,14 +179,13 @@ export type PageEntry = {
 };
 
 export class PersistentSnapshotStorage extends BaseSnapshotStorage {
-  private _resourcesDir: string;
-
-  constructor(resourcesDir: string) {
+  private _loader: VirtualFileSystem;
+  constructor(loader: VirtualFileSystem) {
     super();
-    this._resourcesDir = resourcesDir;
+    this._loader = loader;
   }
 
-  resourceContent(sha1: string): Buffer | undefined {
-    return fs.readFileSync(path.join(this._resourcesDir, sha1));
+  async resourceContent(sha1: string): Promise<Buffer | undefined> {
+    return this._loader.read('resources/' + sha1);
   }
 }

src/server/trace/viewer/traceViewer.ts

@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-import extract from 'extract-zip';
 import fs from 'fs';
 import readline from 'readline';
 import os from 'os';
 import path from 'path';
 import rimraf from 'rimraf';
+import stream from 'stream';
 import { createPlaywright } from '../../playwright';
 import { PersistentSnapshotStorage, TraceModel } from './traceModel';
 import { ServerRouteHandler, HttpServer } from '../../../utils/httpServer';
@@ -32,15 +32,20 @@ import { BrowserContext } from '../../browserContext';
 import { findChromiumChannel } from '../../../utils/registry';
 import { installAppIcon } from '../../chromium/crApp';
 import { debugLogger } from '../../../utils/debugLogger';
+import { VirtualFileSystem, RealFileSystem, ZipFileSystem } from '../../../utils/vfs';
 
 export class TraceViewer {
+  private _vfs: VirtualFileSystem;
   private _server: HttpServer;
   private _browserName: string;
 
-  constructor(tracesDir: string, browserName: string) {
+  constructor(vfs: VirtualFileSystem, browserName: string) {
+    this._vfs = vfs;
     this._browserName = browserName;
-    const resourcesDir = path.join(tracesDir, 'resources');
+    this._server = new HttpServer();
+  }
 
+  async init() {
     // Served by TraceServer
     // - "/tracemodel" - json with trace model.
     //
@@ -55,14 +60,11 @@ export class TraceViewer {
     // - "/snapshot/pageId/..." - actual snapshot html.
     // - "/snapshot/service-worker.js" - service worker that intercepts snapshot resources
     //   and translates them into network requests.
-    const actionTraces = fs.readdirSync(tracesDir).filter(name => name.endsWith('.trace'));
-    const debugNames = actionTraces.map(name => {
-      const tracePrefix = path.join(tracesDir, name.substring(0, name.indexOf('.trace')));
-      return path.basename(tracePrefix);
+    const entries = await this._vfs.entries();
+    const debugNames = entries.filter(name => name.endsWith('.trace')).map(name => {
+      return name.substring(0, name.indexOf('.trace'));
     });
 
-    this._server = new HttpServer();
-
     const traceListHandler: ServerRouteHandler = (request, response) => {
       response.statusCode = 200;
       response.setHeader('Content-Type', 'application/json');
@@ -70,7 +72,7 @@ export class TraceViewer {
       return true;
     };
     this._server.routePath('/contexts', traceListHandler);
-    const snapshotStorage = new PersistentSnapshotStorage(resourcesDir);
+    const snapshotStorage = new PersistentSnapshotStorage(this._vfs);
     new SnapshotServer(this._server, snapshotStorage);
 
     const traceModelHandler: ServerRouteHandler = (request, response) => {
@@ -79,12 +81,12 @@ export class TraceViewer {
       response.statusCode = 200;
       response.setHeader('Content-Type', 'application/json');
       (async () => {
-        const traceFile = path.join(tracesDir, debugName + '.trace');
+        const traceFile = await this._vfs.readStream(debugName + '.trace');
         const match = debugName.match(/^(.*)-\d+$/);
-        const networkFile = path.join(tracesDir, (match ? match[1] : debugName) + '.network');
+        const networkFile = await this._vfs.readStream((match ? match[1] : debugName) + '.network').catch(() => undefined);
         const model = new TraceModel(snapshotStorage);
         await appendTraceEvents(model, traceFile);
-        if (fs.existsSync(networkFile))
+        if (networkFile)
           await appendTraceEvents(model, networkFile);
         model.build();
         response.end(JSON.stringify(model.contextEntry));
@@ -117,7 +119,8 @@ export class TraceViewer {
       const sha1 = request.url!.substring('/sha1/'.length);
       if (sha1.includes('/'))
         return false;
-      return this._server.serveFile(response, path.join(resourcesDir!, sha1));
+      this._server.serveVirtualFile(response, this._vfs, 'resources/' + sha1).catch(() => {});
+      return true;
     };
     this._server.routePrefix('/sha1/', sha1Handler);
   }
@@ -163,10 +166,9 @@ export class TraceViewer {
   }
 }
 
-async function appendTraceEvents(model: TraceModel, file: string) {
-  const fileStream = fs.createReadStream(file, 'utf8');
+async function appendTraceEvents(model: TraceModel, input: stream.Readable) {
   const rl = readline.createInterface({
-    input: fileStream,
+    input,
     crlfDelay: Infinity
   });
   for await (const line of rl as any)
@@ -200,17 +202,12 @@ export async function showTraceViewer(tracePath: string, browserName: string, he
   }
 
   if (stat.isDirectory()) {
-    const traceViewer = new TraceViewer(tracePath, browserName);
+    const traceViewer = new TraceViewer(new RealFileSystem(tracePath), browserName);
+    await traceViewer.init();
     return await traceViewer.show(headless);
   }
 
-  const zipFile = tracePath;
-  try {
-    await extract(zipFile, { dir });
-  } catch (e) {
-    console.log(`Invalid trace file: ${zipFile}`);  // eslint-disable-line no-console
-    return;
-  }
-  const traceViewer = new TraceViewer(dir, browserName);
+  const traceViewer = new TraceViewer(new ZipFileSystem(tracePath), browserName);
+  await traceViewer.init();
   return await traceViewer.show(headless);
 }

src/utils/httpServer.ts

@@ -20,6 +20,7 @@ import path from 'path';
 import { Server as WebSocketServer } from 'ws';
 import * as mime from 'mime';
 import { assert } from './utils';
+import { VirtualFileSystem } from './vfs';
 
 export type ServerRouteHandler = (request: http.IncomingMessage, response: http.ServerResponse) => boolean;
 
@@ -95,6 +96,22 @@ export class HttpServer {
     }
   }
 
+  async serveVirtualFile(response: http.ServerResponse, vfs: VirtualFileSystem, entry: string, headers?: { [name: string]: string }) {
+    try {
+      const content = await vfs.read(entry);
+      response.statusCode = 200;
+      const contentType = mime.getType(path.extname(entry)) || 'application/octet-stream';
+      response.setHeader('Content-Type', contentType);
+      response.setHeader('Content-Length', content.byteLength);
+      for (const [name, value] of Object.entries(headers || {}))
+        response.setHeader(name, value);
+      response.end(content);
+      return true;
+    } catch (e) {
+      return false;
+    }
+  }
+
   private _onRequest(request: http.IncomingMessage, response: http.ServerResponse) {
     request.on('error', () => response.end());
     try {

src/utils/vfs.ts

@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import fs from 'fs';
+import stream from 'stream';
+import yauzl from 'yauzl';
+
+export interface VirtualFileSystem {
+  entries(): Promise<string[]>;
+  read(entry: string): Promise<Buffer>;
+  readStream(entryPath: string): Promise<stream.Readable>;
+  close(): void;
+}
+
+abstract class BaseFileSystem {
+
+  abstract readStream(entryPath: string): Promise<stream.Readable>;
+
+  async read(entryPath: string): Promise<Buffer> {
+    const readStream = await this.readStream(entryPath);
+    const buffers: Buffer[] = [];
+    return new Promise(f => {
+      readStream.on('data', d => buffers.push(d));
+      readStream.on('end', () => f(Buffer.concat(buffers)));
+    });
+  }
+
+  close() {
+  }
+}
+
+export class RealFileSystem extends BaseFileSystem implements VirtualFileSystem {
+  private _folder: string;
+
+  constructor(folder: string) {
+    super();
+    this._folder = folder;
+  }
+
+  async entries(): Promise<string[]> {
+    const result: string[] = [];
+    const visit = (dir: string) => {
+      for (const name of fs.readdirSync(dir)) {
+        const fqn = path.join(dir, name);
+        if (fs.statSync(fqn).isDirectory())
+          visit(fqn);
+        if (fs.statSync(fqn).isFile())
+          result.push(fqn);
+      }
+    };
+    visit(this._folder);
+    return result;
+  }
+
+  async readStream(entry: string): Promise<stream.Readable> {
+    return fs.createReadStream(path.join(this._folder, ...entry.split('/')));
+  }
+}
+
+export class ZipFileSystem extends BaseFileSystem implements VirtualFileSystem {
+  private _fileName: string;
+  private _zipFile: yauzl.ZipFile | undefined;
+  private _entries = new Map<string, yauzl.Entry>();
+  private _openedPromise: Promise<void>;
+
+  constructor(fileName: string) {
+    super();
+    this._fileName = fileName;
+    this._openedPromise = this.open();
+  }
+
+  async open() {
+    await new Promise<yauzl.ZipFile>((fulfill, reject) => {
+      yauzl.open(this._fileName, { autoClose: false }, (e, z) => {
+        if (e) {
+          reject(e);
+          return;
+        }
+        this._zipFile = z;
+        this._zipFile!.on('entry', (entry: yauzl.Entry) => {
+          this._entries.set(entry.fileName, entry);
+        });
+        this._zipFile!.on('end', fulfill);
+      });
+    });
+  }
+
+  async entries(): Promise<string[]> {
+    await this._openedPromise;
+    return [...this._entries.keys()];
+  }
+
+  async readStream(entryPath: string): Promise<stream.Readable> {
+    await this._openedPromise;
+    const entry = this._entries.get(entryPath)!;
+    return new Promise((f, r) => {
+      this._zipFile!.openReadStream(entry, (error, readStream) => {
+        if (error || !readStream) {
+          r(error || 'Entry not found');
+          return;
+        }
+        f(readStream);
+      });
+    });
+  }
+
+  override close() {
+    this._zipFile?.close();
+  }
+}

tests/snapshotter.spec.ts

@@ -146,7 +146,7 @@ it.describe('snapshots', () => {
     await page.evaluate(() => { (document.styleSheets[0].cssRules[0] as any).style.color = 'blue'; });
     const snapshot2 = await snapshotter.captureSnapshot(toImpl(page), 'snapshot1');
     const resource = snapshot2.resourceByUrl(`http://localhost:${server.PORT}/style.css`);
-    expect(snapshotter.resourceContent(resource.response.content._sha1).toString()).toBe('button { color: blue; }');
+    expect((await snapshotter.resourceContent(resource.response.content._sha1)).toString()).toBe('button { color: blue; }');
   });
 
   it('should capture iframe', async ({ page, server, toImpl, browserName, snapshotter, showSnapshot }) => {

tests/tracing.spec.ts

@@ -15,7 +15,7 @@
  */
 
 import { expect, contextTest as test, browserTest } from './config/browserTest';
-import yauzl from 'yauzl';
+import { ZipFileSystem } from '../lib/utils/vfs';
 import jpeg from 'jpeg-js';
 
 test.skip(({ trace }) => !!trace);
@@ -284,29 +284,12 @@ test('should not hang for clicks that open dialogs', async ({ context, page }) =
 });
 
 async function parseTrace(file: string): Promise<{ events: any[], resources: Map<string, Buffer> }> {
-  const entries = await new Promise<any[]>(f => {
-    const entries: Promise<any>[] = [];
-    yauzl.open(file, (err, zipFile) => {
-      zipFile.on('entry', entry => {
-        const entryPromise = new Promise(ff => {
-          zipFile.openReadStream(entry, (err, readStream) => {
-            const buffers = [];
-            if (readStream) {
-              readStream.on('data', d => buffers.push(d));
-              readStream.on('end', () => ff({ name: entry.fileName, buffer: Buffer.concat(buffers) }));
-            } else {
-              ff({ name: entry.fileName });
-            }
-          });
-        });
-        entries.push(entryPromise);
-      });
-      zipFile.on('end', () => f(entries));
-    });
-  });
+  const zipFS = new ZipFileSystem(file);
   const resources = new Map<string, Buffer>();
-  for (const { name, buffer } of await Promise.all(entries))
-    resources.set(name, buffer);
+  for (const entry of await zipFS.entries())
+    resources.set(entry, await zipFS.read(entry));
+  zipFS.close();
+
   const events = [];
   for (const line of resources.get('trace.trace').toString().split('\n')) {
     if (line)