From 8d21b12454632430433d46e0a0a3a4fbc468a884 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 10 May 2021 22:01:41 -0700 Subject: [PATCH] browser(firefox): fit screencast images into given frame (#6495) --- browser_patches/firefox-stable/BUILD_NUMBER | 4 +- .../firefox-stable/juggler/TargetRegistry.js | 6 +- .../screencast/nsIScreencastService.idl | 2 +- .../screencast/nsScreencastService.cpp | 72 ++++++++++++++++--- browser_patches/firefox/BUILD_NUMBER | 4 +- .../firefox/juggler/TargetRegistry.js | 6 +- .../screencast/nsIScreencastService.idl | 2 +- .../screencast/nsScreencastService.cpp | 72 ++++++++++++++++--- 8 files changed, 138 insertions(+), 30 deletions(-) diff --git a/browser_patches/firefox-stable/BUILD_NUMBER b/browser_patches/firefox-stable/BUILD_NUMBER index b8e1107465..7e42c4ffc8 100644 --- a/browser_patches/firefox-stable/BUILD_NUMBER +++ b/browser_patches/firefox-stable/BUILD_NUMBER @@ -1,2 +1,2 @@ -1247 -Changed: dgozman@gmail.com Sat May 8 17:46:11 PDT 2021 +1248 +Changed: pavel.feldman@gmail.com Mon 10 May 2021 09:58:08 PM PDT diff --git a/browser_patches/firefox-stable/juggler/TargetRegistry.js b/browser_patches/firefox-stable/juggler/TargetRegistry.js index 9613fc83c6..e308a6c13c 100644 --- a/browser_patches/firefox-stable/juggler/TargetRegistry.js +++ b/browser_patches/firefox-stable/juggler/TargetRegistry.js @@ -514,7 +514,8 @@ class PageTarget { registry.emit(TargetRegistry.Events.ScreencastStopped, sessionId); }, }; - sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, devicePixelRatio * rect.top); + const viewport = this._viewportSize || this._browserContext.defaultViewportSize || { width: 0, height: 0 }; + sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, viewport.width, viewport.height, devicePixelRatio * rect.top); this._videoRecordingInfo = { sessionId, file }; this.emit(PageTarget.Events.ScreencastStarted); } @@ -554,7 +555,8 @@ class PageTarget { screencastStopped() { }, }; - const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, devicePixelRatio * rect.top); + const viewport = this._viewportSize || this._browserContext.defaultViewportSize || { width: 0, height: 0 }; + const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, viewport.width, viewport.height, devicePixelRatio * rect.top); this._screencastRecordingInfo = { screencastId }; return { screencastId }; } diff --git a/browser_patches/firefox-stable/juggler/screencast/nsIScreencastService.idl b/browser_patches/firefox-stable/juggler/screencast/nsIScreencastService.idl index 302a47ce7c..16c94371ba 100644 --- a/browser_patches/firefox-stable/juggler/screencast/nsIScreencastService.idl +++ b/browser_patches/firefox-stable/juggler/screencast/nsIScreencastService.idl @@ -20,7 +20,7 @@ interface nsIScreencastServiceClient : nsISupports [scriptable, uuid(d8c4d9e0-9462-445e-9e43-68d3872ad1de)] interface nsIScreencastService : nsISupports { - AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in int32_t offset_top); + AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in uint32_t viewportWidth, in uint32_t viewportHeight, in uint32_t offset_top); /** * Will emit 'juggler-screencast-stopped' when the video file is saved. diff --git a/browser_patches/firefox-stable/juggler/screencast/nsScreencastService.cpp b/browser_patches/firefox-stable/juggler/screencast/nsScreencastService.cpp index 90699235e9..c7c76ee3c7 100644 --- a/browser_patches/firefox-stable/juggler/screencast/nsScreencastService.cpp +++ b/browser_patches/firefox-stable/juggler/screencast/nsScreencastService.cpp @@ -28,6 +28,7 @@ extern "C" { #include "jpeglib.h" } +#include using namespace mozilla::widget; @@ -79,11 +80,22 @@ nsresult generateUid(nsString& uid) { class nsScreencastService::Session : public rtc::VideoSinkInterface, public webrtc::RawFrameCallback { public: - Session(nsIScreencastServiceClient* client, rtc::scoped_refptr&& capturer, RefPtr&& encoder, gfx::IntMargin margin, uint32_t jpegQuality) + Session( + nsIScreencastServiceClient* client, + rtc::scoped_refptr&& capturer, + RefPtr&& encoder, + int width, int height, + int viewportWidth, int viewportHeight, + gfx::IntMargin margin, + uint32_t jpegQuality) : mClient(client) , mCaptureModule(std::move(capturer)) , mEncoder(std::move(encoder)) , mJpegQuality(jpegQuality) + , mWidth(width) + , mHeight(height) + , mViewportWidth(viewportWidth) + , mViewportHeight(viewportHeight) , mMargin(margin) { } @@ -145,6 +157,14 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface mViewportWidth) + pageWidth = mViewportWidth; + if (mViewportHeight && pageHeight > mViewportHeight) + pageHeight = mViewportHeight; + { rtc::CritScope lock(&mCaptureCallbackCs); if (mFramesInFlight >= kMaxFramesInFlight) { @@ -155,6 +175,36 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface canvas; + uint8_t* canvasPtr = videoFrame; + int canvasStride = videoFrameStride; + + if (mWidth < pageWidth || mHeight < pageHeight) { + double scale = std::min(1., std::min((double)mWidth / pageWidth, (double)mHeight / pageHeight)); + int canvasWidth = frameInfo.width * scale; + int canvasHeight = frameInfo.height * scale; + canvasStride = canvasWidth * 4; + + screenshotWidth *= scale; + screenshotHeight *= scale; + screenshotTopMargin *= scale; + + canvas.reset(new uint8_t[canvasWidth * canvasHeight * 4]); + canvasPtr = canvas.get(); + libyuv::ARGBScale(videoFrame, + videoFrameStride, + frameInfo.width, + frameInfo.height, + canvasPtr, + canvasStride, + canvasWidth, + canvasHeight, + libyuv::kFilterBilinear); + } + jpeg_compress_struct info; jpeg_error_mgr error; info.err = jpeg_std_error(&error); @@ -164,8 +214,8 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface void { + "NotifyScreencastFrame", [client, base64, pageWidth, pageHeight]() -> void { NS_ConvertUTF8toUTF16 utf16(base64); - client->ScreencastFrame(utf16, deviceWidth, deviceHeight); + client->ScreencastFrame(utf16, pageWidth, pageHeight); })); } @@ -221,6 +269,10 @@ class nsScreencastService::Session : public rtc::VideoSinkInterfaceGetPresShell(); @@ -282,7 +334,7 @@ nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aC NS_ENSURE_SUCCESS(rv, rv); sessionId = uid; - auto session = std::make_unique(aClient, std::move(capturer), std::move(encoder), margin, isVideo ? 0 : quality); + auto session = std::make_unique(aClient, std::move(capturer), std::move(encoder), width, height, viewportWidth, viewportHeight, margin, isVideo ? 0 : quality); if (!session->Start()) return NS_ERROR_FAILURE; mIdToSession.emplace(sessionId, std::move(session)); diff --git a/browser_patches/firefox/BUILD_NUMBER b/browser_patches/firefox/BUILD_NUMBER index 9f8a635f23..06d739129d 100644 --- a/browser_patches/firefox/BUILD_NUMBER +++ b/browser_patches/firefox/BUILD_NUMBER @@ -1,2 +1,2 @@ -1257 -Changed: dgozman@gmail.com Sat May 8 17:46:11 PDT 2021 +1258 +Changed: pavel.feldman@gmail.com Mon 10 May 2021 09:56:40 PM PDT diff --git a/browser_patches/firefox/juggler/TargetRegistry.js b/browser_patches/firefox/juggler/TargetRegistry.js index 9613fc83c6..e308a6c13c 100644 --- a/browser_patches/firefox/juggler/TargetRegistry.js +++ b/browser_patches/firefox/juggler/TargetRegistry.js @@ -514,7 +514,8 @@ class PageTarget { registry.emit(TargetRegistry.Events.ScreencastStopped, sessionId); }, }; - sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, devicePixelRatio * rect.top); + const viewport = this._viewportSize || this._browserContext.defaultViewportSize || { width: 0, height: 0 }; + sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, viewport.width, viewport.height, devicePixelRatio * rect.top); this._videoRecordingInfo = { sessionId, file }; this.emit(PageTarget.Events.ScreencastStarted); } @@ -554,7 +555,8 @@ class PageTarget { screencastStopped() { }, }; - const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, devicePixelRatio * rect.top); + const viewport = this._viewportSize || this._browserContext.defaultViewportSize || { width: 0, height: 0 }; + const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, viewport.width, viewport.height, devicePixelRatio * rect.top); this._screencastRecordingInfo = { screencastId }; return { screencastId }; } diff --git a/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl b/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl index 302a47ce7c..16c94371ba 100644 --- a/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl +++ b/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl @@ -20,7 +20,7 @@ interface nsIScreencastServiceClient : nsISupports [scriptable, uuid(d8c4d9e0-9462-445e-9e43-68d3872ad1de)] interface nsIScreencastService : nsISupports { - AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in int32_t offset_top); + AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in uint32_t viewportWidth, in uint32_t viewportHeight, in uint32_t offset_top); /** * Will emit 'juggler-screencast-stopped' when the video file is saved. diff --git a/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp b/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp index 90699235e9..c7c76ee3c7 100644 --- a/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp +++ b/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp @@ -28,6 +28,7 @@ extern "C" { #include "jpeglib.h" } +#include using namespace mozilla::widget; @@ -79,11 +80,22 @@ nsresult generateUid(nsString& uid) { class nsScreencastService::Session : public rtc::VideoSinkInterface, public webrtc::RawFrameCallback { public: - Session(nsIScreencastServiceClient* client, rtc::scoped_refptr&& capturer, RefPtr&& encoder, gfx::IntMargin margin, uint32_t jpegQuality) + Session( + nsIScreencastServiceClient* client, + rtc::scoped_refptr&& capturer, + RefPtr&& encoder, + int width, int height, + int viewportWidth, int viewportHeight, + gfx::IntMargin margin, + uint32_t jpegQuality) : mClient(client) , mCaptureModule(std::move(capturer)) , mEncoder(std::move(encoder)) , mJpegQuality(jpegQuality) + , mWidth(width) + , mHeight(height) + , mViewportWidth(viewportWidth) + , mViewportHeight(viewportHeight) , mMargin(margin) { } @@ -145,6 +157,14 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface mViewportWidth) + pageWidth = mViewportWidth; + if (mViewportHeight && pageHeight > mViewportHeight) + pageHeight = mViewportHeight; + { rtc::CritScope lock(&mCaptureCallbackCs); if (mFramesInFlight >= kMaxFramesInFlight) { @@ -155,6 +175,36 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface canvas; + uint8_t* canvasPtr = videoFrame; + int canvasStride = videoFrameStride; + + if (mWidth < pageWidth || mHeight < pageHeight) { + double scale = std::min(1., std::min((double)mWidth / pageWidth, (double)mHeight / pageHeight)); + int canvasWidth = frameInfo.width * scale; + int canvasHeight = frameInfo.height * scale; + canvasStride = canvasWidth * 4; + + screenshotWidth *= scale; + screenshotHeight *= scale; + screenshotTopMargin *= scale; + + canvas.reset(new uint8_t[canvasWidth * canvasHeight * 4]); + canvasPtr = canvas.get(); + libyuv::ARGBScale(videoFrame, + videoFrameStride, + frameInfo.width, + frameInfo.height, + canvasPtr, + canvasStride, + canvasWidth, + canvasHeight, + libyuv::kFilterBilinear); + } + jpeg_compress_struct info; jpeg_error_mgr error; info.err = jpeg_std_error(&error); @@ -164,8 +214,8 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface void { + "NotifyScreencastFrame", [client, base64, pageWidth, pageHeight]() -> void { NS_ConvertUTF8toUTF16 utf16(base64); - client->ScreencastFrame(utf16, deviceWidth, deviceHeight); + client->ScreencastFrame(utf16, pageWidth, pageHeight); })); } @@ -221,6 +269,10 @@ class nsScreencastService::Session : public rtc::VideoSinkInterfaceGetPresShell(); @@ -282,7 +334,7 @@ nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aC NS_ENSURE_SUCCESS(rv, rv); sessionId = uid; - auto session = std::make_unique(aClient, std::move(capturer), std::move(encoder), margin, isVideo ? 0 : quality); + auto session = std::make_unique(aClient, std::move(capturer), std::move(encoder), width, height, viewportWidth, viewportHeight, margin, isVideo ? 0 : quality); if (!session->Start()) return NS_ERROR_FAILURE; mIdToSession.emplace(sessionId, std::move(session));