feat(screencast): use ffmpeg to produce webm in chromium (#3668)

package-lock.json

@@ -1081,6 +1081,62 @@
         "sumchecker": "^3.0.1"
       }
     },
+    "@ffmpeg-installer/darwin-x64": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz",
+      "integrity": "sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==",
+      "optional": true
+    },
+    "@ffmpeg-installer/ffmpeg": {
+      "version": "1.0.20",
+      "resolved": "https://registry.npmjs.org/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.0.20.tgz",
+      "integrity": "sha512-wbgd//6OdwbFXYgV68ZyKrIcozEQpUKlvV66XHaqO2h3sFbX0jYLzx62Q0v8UcFWN21LoxT98NU2P+K0OWsKNA==",
+      "requires": {
+        "@ffmpeg-installer/darwin-x64": "4.1.0",
+        "@ffmpeg-installer/linux-arm": "4.1.3",
+        "@ffmpeg-installer/linux-arm64": "4.1.4",
+        "@ffmpeg-installer/linux-ia32": "4.1.0",
+        "@ffmpeg-installer/linux-x64": "4.1.0",
+        "@ffmpeg-installer/win32-ia32": "4.1.0",
+        "@ffmpeg-installer/win32-x64": "4.1.0"
+      }
+    },
+    "@ffmpeg-installer/linux-arm": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz",
+      "integrity": "sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==",
+      "optional": true
+    },
+    "@ffmpeg-installer/linux-arm64": {
+      "version": "4.1.4",
+      "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz",
+      "integrity": "sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==",
+      "optional": true
+    },
+    "@ffmpeg-installer/linux-ia32": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz",
+      "integrity": "sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==",
+      "optional": true
+    },
+    "@ffmpeg-installer/linux-x64": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz",
+      "integrity": "sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==",
+      "optional": true
+    },
+    "@ffmpeg-installer/win32-ia32": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz",
+      "integrity": "sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==",
+      "optional": true
+    },
+    "@ffmpeg-installer/win32-x64": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz",
+      "integrity": "sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==",
+      "optional": true
+    },
     "@jest/types": {
       "version": "26.3.0",
       "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz",

package.json

@@ -37,6 +37,7 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
+    "@ffmpeg-installer/ffmpeg": "^1.0.20",
     "debug": "^4.1.1",
     "extract-zip": "^2.0.1",
     "https-proxy-agent": "^5.0.0",

src/server/chromium/crPage.ts

@@ -37,6 +37,7 @@ import { ConsoleMessage } from '../console';
 import * as sourceMap from '../../utils/sourceMap';
 import { rewriteErrorMessage } from '../../utils/stackTrace';
 import { assert, headersArrayToObject } from '../../utils/utils';
+import { VideoRecorder } from './videoRecorder';
 
 
 const UTILITY_WORLD_NAME = '__playwright_utility_world__';
@@ -209,11 +210,11 @@ export class CRPage implements PageDelegate {
   }
 
   async startScreencast(options: types.PageScreencastOptions): Promise<void> {
-    throw new Error('Not implemented');
+    await this._mainFrameSession._startScreencast(options);
   }
 
   async stopScreencast(): Promise<void> {
-    throw new Error('Not implemented');
+    await this._mainFrameSession._stopScreencast();
   }
 
   async takeScreenshot(format: 'png' | 'jpeg', documentRect: types.Rect | undefined, viewportRect: types.Rect | undefined, quality: number | undefined): Promise<Buffer> {
@@ -324,6 +325,7 @@ class FrameSession {
   // Marks the oopif session that remote -> local transition has happened in the parent.
   // See Target.detachedFromTarget handler for details.
   private _swappedIn = false;
+  private _videoRecorder: VideoRecorder | null = null;
 
   constructor(crPage: CRPage, client: CRSession, targetId: string, parentSession: FrameSession | null) {
     this._client = client;
@@ -358,6 +360,7 @@ class FrameSession {
       helper.addEventListener(this._client, 'Page.navigatedWithinDocument', event => this._onFrameNavigatedWithinDocument(event.frameId, event.url)),
       helper.addEventListener(this._client, 'Page.downloadWillBegin', event => this._onDownloadWillBegin(event)),
       helper.addEventListener(this._client, 'Page.downloadProgress', event => this._onDownloadProgress(event)),
+      helper.addEventListener(this._client, 'Page.screencastFrame', event => this._onScreencastFrame(event)),
       helper.addEventListener(this._client, 'Runtime.bindingCalled', event => this._onBindingCalled(event)),
       helper.addEventListener(this._client, 'Runtime.consoleAPICalled', event => this._onConsoleAPI(event)),
       helper.addEventListener(this._client, 'Runtime.exceptionThrown', exception => this._handleException(exception.exceptionDetails)),
@@ -724,6 +727,34 @@ class FrameSession {
       this._crPage._browserContext._browser._downloadFinished(payload.guid, 'canceled');
   }
 
+  _onScreencastFrame(payload: Protocol.Page.screencastFramePayload) {
+    if (!this._videoRecorder)
+      return;
+    const buffer = Buffer.from(payload.data, 'base64');
+    this._videoRecorder.writeFrame(buffer, payload.metadata.timestamp!);
+    this._client.send('Page.screencastFrameAck', {sessionId: payload.sessionId});
+  }
+
+  async _startScreencast(options: types.PageScreencastOptions): Promise<void> {
+    assert(!this._videoRecorder, 'Already started');
+    this._videoRecorder = await VideoRecorder.launch(options);
+    await this._client.send('Page.startScreencast', {
+      format: 'jpeg',
+      quality: 90,
+      maxWidth: options.width,
+      maxHeight: options.height,
+    });
+  }
+
+  async _stopScreencast(): Promise<void> {
+    if (!this._videoRecorder)
+      return;
+    const recorder = this._videoRecorder;
+    this._videoRecorder = null;
+    await this._client.send('Page.stopScreencast');
+    await recorder.stop();
+  }
+
   async _updateExtraHTTPHeaders(): Promise<void> {
     const headers = network.mergeHeaders([
       this._crPage._browserContext._options.extraHTTPHeaders,

src/server/chromium/videoRecorder.ts

@@ -0,0 +1,128 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const ffmpegPath = require('@ffmpeg-installer/ffmpeg').path;
+import { launchProcess } from '../processLauncher';
+import { ChildProcess } from 'child_process';
+import { Progress, runAbortableTask } from '../progress';
+import * as types from '../types';
+import { assert } from '../../utils/utils';
+
+const fps = 25;
+
+export class VideoRecorder {
+  private _process: ChildProcess | null = null;
+  private _gracefullyClose: (() => Promise<void>) | null = null;
+  private _lastWritePromise: Promise<void>;
+  private _lastFrameTimestamp: number = 0;
+  private _lastFrameBuffer: Buffer | null = null;
+  private _lastWriteTimestamp: number = 0;
+  private readonly _progress: Progress;
+
+  static async launch(options: types.PageScreencastOptions): Promise<VideoRecorder> {
+    if (!options.outputFile.endsWith('.webm'))
+      throw new Error('File must have .webm extension');
+
+    return await runAbortableTask(async progress => {
+      const recorder = new VideoRecorder(progress);
+      await recorder._launch(options);
+      return recorder;
+    }, 0, 'browser');
+  }
+
+  private constructor(progress: Progress) {
+    this._progress = progress;
+    this._lastWritePromise = Promise.resolve();
+  }
+
+  private async _launch(options: types.PageScreencastOptions) {
+    assert(!this._isRunning());
+    const w = options.width;
+    const h = options.height;
+    const args = `-f image2pipe -c:v mjpeg -i - -y -an -r ${fps} -c:v vp8 -vf pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`.split(' ');
+    args.push(options.outputFile);
+    const progress = this._progress;
+    const { launchedProcess, gracefullyClose } = await launchProcess({
+      executablePath: ffmpegPath,
+      args,
+      pipeStdin: true,
+      progress,
+      tempDirectories: [],
+      attemptToGracefullyClose: async () => {
+        progress.log('Closing stdin...');
+        launchedProcess.stdin.end();
+      },
+      onExit: (exitCode, signal) => {
+        progress.log(`ffmpeg onkill exitCode=${exitCode} signal=${signal}`);
+      },
+    });
+    launchedProcess.stdin.on('finish', () => {
+      progress.log('ffmpeg finished input.');
+    });
+    launchedProcess.stdin.on('error', () => {
+      progress.log('ffmpeg error.');
+    });
+    this._process = launchedProcess;
+    this._gracefullyClose = gracefullyClose;
+  }
+
+  async writeFrame(frame: Buffer, timestamp: number) {
+    assert(this._process);
+    if (!this._isRunning())
+      return;
+    const duration = this._lastFrameTimestamp ? Math.max(1, Math.round(25 * (timestamp - this._lastFrameTimestamp))) : 1;
+    this._progress.log(`writing ${duration} frame(s)`);
+    this._lastFrameBuffer = frame;
+    this._lastFrameTimestamp = timestamp;
+    this._lastWriteTimestamp = Date.now();
+
+    const previousWrites = this._lastWritePromise;
+    let finishedWriting: () => void;
+    this._lastWritePromise = new Promise(fulfill => finishedWriting = fulfill);
+    const writePromise = this._lastWritePromise;
+    await previousWrites;
+    for (let i = 0; i < duration; i++) {
+      const callFinish = i === (duration - 1);
+      this._process.stdin.write(frame, (error: Error | null | undefined) => {
+        if (error)
+          this._progress.log(`ffmpeg failed to write: ${error}`);
+        if (callFinish)
+          finishedWriting();
+      });
+    }
+    return writePromise;
+  }
+
+  async stop() {
+    if (!this._gracefullyClose)
+      return;
+
+    if (this._lastWriteTimestamp) {
+      const durationSec = (Date.now() - this._lastWriteTimestamp) / 1000;
+      if (durationSec > 1 / fps)
+        this.writeFrame(this._lastFrameBuffer!, this._lastFrameTimestamp + durationSec);
+    }
+
+    const close = this._gracefullyClose;
+    this._gracefullyClose = null;
+    await this._lastWritePromise;
+    await close();
+  }
+
+  private _isRunning(): boolean {
+    return !!this._gracefullyClose;
+  }
+}

src/server/processLauncher.ts

@@ -35,6 +35,7 @@ export type LaunchProcessOptions = {
   handleSIGTERM?: boolean,
   handleSIGHUP?: boolean,
   pipe?: boolean,
+  pipeStdin?: boolean,
   tempDirectories: string[],
 
   cwd?: string,
@@ -62,6 +63,8 @@ export async function launchProcess(options: LaunchProcessOptions): Promise<Laun
 
   const progress = options.progress;
   const stdio: ('ignore' | 'pipe')[] = options.pipe ? ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'] : ['ignore', 'pipe', 'pipe'];
+  if (options.pipeStdin)
+    stdio[0] = 'pipe';
   progress.log(`<launching> ${options.executablePath} ${options.args.join(' ')}`);
   const spawnedProcess = childProcess.spawn(
       options.executablePath,

test/page-goto.spec.ts

@@ -450,7 +450,7 @@ it('should reject referer option when setExtraHTTPHeaders provides referer', asy
 it('should override referrer-policy', async ({page, server}) => {
   server.setRoute('/grid.html', (req, res) => {
     res.setHeader('Referrer-Policy', 'no-referrer');
-    server.serveFile(req, res, '/grid.html');
+    server.serveFile(req, res);
   });
   const [request1, request2] = await Promise.all([
     server.waitForRequest('/grid.html'),
@@ -482,7 +482,7 @@ it('extraHttpHeaders should be pushed to provisional page', test => {
   const pagePath = '/one-style.html';
   server.setRoute(pagePath, async (req, res) => {
     page.setExtraHTTPHeaders({ foo: 'bar' });
-    server.serveFile(req, res, pagePath);
+    server.serveFile(req, res);
   });
   const [htmlReq, cssReq] = await Promise.all([
     server.waitForRequest(pagePath),

test/screencast.spec.ts

@@ -20,7 +20,7 @@ import type { Page } from '..';
 
 import fs from 'fs';
 import path from 'path';
-import url from 'url';
+import { TestServer } from '../utils/testserver';
 
 
 declare global {
@@ -29,7 +29,7 @@ declare global {
   }
 }
 
-registerFixture('videoPlayer', async ({playwright, context}, test) => {
+registerFixture('videoPlayer', async ({playwright, context, server}, test) => {
   let firefox;
   if (options.WEBKIT && !LINUX) {
     // WebKit on Mac & Windows cannot replay webm/vp8 video, so we launch Firefox.
@@ -38,7 +38,7 @@ registerFixture('videoPlayer', async ({playwright, context}, test) => {
   }
 
   const page = await context.newPage();
-  const player = new VideoPlayer(page);
+  const player = new VideoPlayer(page, server);
   await test(player);
   if (firefox)
     await firefox.close();
@@ -90,12 +90,19 @@ function expectAll(pixels, rgbaPredicate) {
 
 class VideoPlayer {
   private readonly _page: Page;
-  constructor(page: Page) {
+  private readonly _server: TestServer;
+  constructor(page: Page, server: TestServer) {
     this._page = page;
+    this._server = server;
   }
 
-  async load(videoFile) {
-    await this._page.goto(url.pathToFileURL(videoFile).href);
+  async load(videoFile: string) {
+    const servertPath = '/v.webm';
+    this._server.setRoute(servertPath, (req, response) => {
+      this._server.serveFile(req, response, videoFile);
+    });
+
+    await this._page.goto(this._server.PREFIX + servertPath);
     await this._page.$eval('video', (v: HTMLVideoElement) => {
       return new Promise(fulfil => {
         // In case video playback autostarts.
@@ -172,36 +179,35 @@ class VideoPlayer {
   }
 }
 
+it('should capture static page', test => {
+  test.skip(options.WIRE);
+}, async ({page, tmpDir, videoPlayer, toImpl}) => {
+  const videoFile = path.join(tmpDir, 'v.webm');
+  await page.evaluate(() => document.body.style.backgroundColor = 'red');
+  await toImpl(page)._delegate.startScreencast({outputFile: videoFile, width: 640, height: 480});
+  // TODO: in WebKit figure out why video size is not reported correctly for
+  // static pictures.
+  if (options.HEADLESS && options.WEBKIT)
+    await page.setViewportSize({width: 1270, height: 950});
+  await new Promise(r => setTimeout(r, 300));
+  await toImpl(page)._delegate.stopScreencast();
+  expect(fs.existsSync(videoFile)).toBe(true);
+
+  await videoPlayer.load(videoFile);
+  const duration = await videoPlayer.duration();
+  expect(duration).toBeGreaterThan(0);
+
+  expect(await videoPlayer.videoWidth()).toBe(640);
+  expect(await videoPlayer.videoHeight()).toBe(480);
+
+  await videoPlayer.seekLastNonEmptyFrame();
+  const pixels = await videoPlayer.pixels();
+  expectAll(pixels, almostRed);
+});
+
 describe('screencast', suite => {
-  suite.skip(options.WIRE);
-  suite.fixme(options.CHROMIUM);
+  suite.skip(options.WIRE || options.CHROMIUM);
 }, () => {
-  it('should capture static page', test => {
-    test.fixme();
-  }, async ({page, tmpDir, videoPlayer, toImpl}) => {
-    const videoFile = path.join(tmpDir, 'v.webm');
-    await page.evaluate(() => document.body.style.backgroundColor = 'red');
-    await toImpl(page)._delegate.startScreencast({outputFile: videoFile, width: 640, height: 480});
-    // TODO: in WebKit figure out why video size is not reported correctly for
-    // static pictures.
-    if (options.HEADLESS && options.WEBKIT)
-      await page.setViewportSize({width: 1270, height: 950});
-    await new Promise(r => setTimeout(r, 300));
-    await toImpl(page)._delegate.stopScreencast();
-    expect(fs.existsSync(videoFile)).toBe(true);
-
-    await videoPlayer.load(videoFile);
-    const duration = await videoPlayer.duration();
-    expect(duration).toBeGreaterThan(0);
-
-    expect(await videoPlayer.videoWidth()).toBe(640);
-    expect(await videoPlayer.videoHeight()).toBe(480);
-
-    await videoPlayer.seekLastNonEmptyFrame();
-    const pixels = await videoPlayer.pixels();
-    expectAll(pixels, almostRed);
-  });
-
   it('should capture navigation', test => {
     test.flaky(options.WEBKIT);
     test.flaky(options.FIREFOX);

utils/testserver/index.d.ts

@@ -29,7 +29,8 @@ export class TestServer {
   setRedirect(from: string, to: string);
   waitForRequest(path: string): Promise<IncomingMessage & {postBody: Buffer}>;
   reset();
-  serveFile(request: IncomingMessage, response: ServerResponse, pathName: string);
+  serveFile(request: IncomingMessage, response: ServerResponse);
+  serveFile(request: IncomingMessage, response: ServerResponse, filePath: string);
 
   PORT: number;
   PREFIX: string;

utils/testserver/index.js

@@ -228,20 +228,22 @@ class TestServer {
     if (handler) {
       handler.call(null, request, response);
     } else {
-      const pathName = url.parse(request.url).path;
-      this.serveFile(request, response, pathName);
+      this.serveFile(request, response);
     }
   }
 
   /**
    * @param {!http.IncomingMessage} request
    * @param {!http.ServerResponse} response
-   * @param {string} pathName
+   * @param {string|undefined} filePath
    */
-  serveFile(request, response, pathName) {
-    if (pathName === '/')
-      pathName = '/index.html';
-    const filePath = path.join(this._dirPath, pathName.substring(1));
+  serveFile(request, response, filePath) {
+    let pathName = url.parse(request.url).path;
+    if (!filePath) {
+      if (pathName === '/')
+        pathName = '/index.html';
+      filePath = path.join(this._dirPath, pathName.substring(1));
+    }
 
     if (this._cachedPathPrefix !== null && filePath.startsWith(this._cachedPathPrefix)) {
       if (request.headers['if-modified-since']) {