From 4f2d09d2ba5749d0d74b6c583c66fab57b87b7c6 Mon Sep 17 00:00:00 2001 From: Axel Eriksson Date: Thu, 23 Jan 2025 13:25:21 +0100 Subject: [PATCH] Adding support for Windows to use navigator.mediaDevices.getDisplayMedia --- spec/screenPicker.spec.ts | 118 ++++++++++++++-------- src/app/display-media-request-handler.ts | 102 +++++++++++++++++-- src/app/window-handler.ts | 60 +++++------ src/renderer/components/screen-picker.tsx | 4 +- src/renderer/desktop-capturer.ts | 13 +-- 5 files changed, 204 insertions(+), 93 deletions(-) diff --git a/spec/screenPicker.spec.ts b/spec/screenPicker.spec.ts index d0f5d07a6..de488b42b 100644 --- a/spec/screenPicker.spec.ts +++ b/spec/screenPicker.spec.ts @@ -11,6 +11,10 @@ jest.mock('../src/common/env', () => { }; }); +const mockThumbnail = { + toDataURL: () => {}, +}; + describe('screen picker', () => { const keyCode = { pageDown: { keyCode: 34 }, @@ -40,26 +44,26 @@ describe('screen picker', () => { display_id: '0', id: '0', name: 'Application screen 0', - thumbnail: undefined, + thumbnail: mockThumbnail, }, { display_id: '1', id: '1', name: 'Application screen 1', - thumbnail: undefined, + thumbnail: mockThumbnail, }, { display_id: '2', id: '2', name: 'Application screen 2', - thumbnail: undefined, + thumbnail: mockThumbnail, }, ], selectedSource: { display_id: '1', id: '1', name: 'Application screen 1', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; @@ -85,7 +89,7 @@ describe('screen picker', () => { display_id: '1', id: '1', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }; const customSelector = 'button.ScreenPicker-share-button'; wrapper.setState({ selectedSource }); @@ -101,7 +105,7 @@ describe('screen picker', () => { display_id: '0', id: '0', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }, ], selectedSource: undefined, @@ -119,14 +123,14 @@ describe('screen picker', () => { display_id: '0', id: '0', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }, ], selectedSource: { display_id: '0', id: '0', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const applicationScreenStateMock = { @@ -135,14 +139,14 @@ describe('screen picker', () => { display_id: '', id: '1', name: 'Application 1', - thumbnail: undefined, + thumbnail: mockThumbnail, }, ], selectedSource: { display_id: '', id: '1', name: 'Application 1', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; @@ -175,7 +179,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '0', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const customSelector = '.ScreenPicker-item-container'; @@ -185,19 +189,19 @@ describe('screen picker', () => { display_id: '0', id: '0', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }, { display_id: '1', id: '1', name: 'Application screen 1', - thumbnail: undefined, + thumbnail: mockThumbnail, }, { display_id: '2', id: '2', name: 'Application screen 2', - thumbnail: undefined, + thumbnail: mockThumbnail, }, ], selectedSource: { @@ -205,7 +209,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '1', name: 'Application screen 1', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; wrapper.setState(applicationScreenStateMock); @@ -222,7 +226,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '2', name: 'Application screen 2', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const customSelector = '.ScreenPicker-item-container'; @@ -232,19 +236,19 @@ describe('screen picker', () => { display_id: '0', id: '0', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }, { display_id: '1', id: '1', name: 'Application screen 1', - thumbnail: undefined, + thumbnail: mockThumbnail, }, { display_id: '2', id: '2', name: 'Application screen 2', - thumbnail: undefined, + thumbnail: mockThumbnail, }, ], selectedSource: { @@ -252,7 +256,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '1', name: 'Application screen 1', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; wrapper.setState(applicationScreenStateMock); @@ -289,7 +293,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '2', name: 'Application screen 2', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const wrapper = shallow(React.createElement(ScreenPicker)); @@ -306,7 +310,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '2', name: 'Application screen 2', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const wrapper = shallow(React.createElement(ScreenPicker)); @@ -323,7 +327,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '0', name: 'Application screen 0', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const wrapper = shallow(React.createElement(ScreenPicker)); @@ -340,7 +344,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '0', name: 'Application screen 0', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const wrapper = shallow(React.createElement(ScreenPicker)); @@ -357,7 +361,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '0', name: 'Application screen 0', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const wrapper = shallow(React.createElement(ScreenPicker)); @@ -374,7 +378,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '2', name: 'Application screen 2', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const wrapper = shallow(React.createElement(ScreenPicker)); @@ -389,7 +393,7 @@ describe('screen picker', () => { display_id: '1', id: '1', name: 'Application screen 1', - thumbnail: undefined, + thumbnail: mockThumbnail, }; const wrapper = shallow(React.createElement(ScreenPicker)); wrapper.setState(stateMock); @@ -418,7 +422,7 @@ describe('screen picker', () => { fileName: 'fullscreen', id: '0', name: 'Application screen 0', - thumbnail: undefined, + thumbnail: mockThumbnail, }, }; const wrapper = shallow(React.createElement(ScreenPicker)); @@ -437,19 +441,19 @@ describe('screen picker', () => { display_id: '', id: '1', name: 'Application Screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }, { display_id: '', id: '2', name: 'Application Screen 2', - thumbnail: undefined, + thumbnail: mockThumbnail, }, { display_id: '', id: '3', name: 'Application Screen 3', - thumbnail: undefined, + thumbnail: mockThumbnail, }, ], }; @@ -466,10 +470,20 @@ describe('screen picker', () => { display_id: '1', id: '1', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, + }, + { + display_id: '2', + id: '2', + name: 'Screen 2', + thumbnail: mockThumbnail, + }, + { + display_id: '3', + id: '3', + name: 'screen 3', + thumbnail: mockThumbnail, }, - { display_id: '2', id: '2', name: 'Screen 2', thumbnail: undefined }, - { display_id: '3', id: '3', name: 'screen 3', thumbnail: undefined }, ], }; wrapper.setState(entireScreenStateMock); @@ -486,10 +500,20 @@ describe('screen picker', () => { display_id: '', id: '1', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, + }, + { + display_id: '', + id: '2', + name: 'Screen 2', + thumbnail: mockThumbnail, + }, + { + display_id: '', + id: '3', + name: 'screen 3', + thumbnail: mockThumbnail, }, - { display_id: '', id: '2', name: 'Screen 2', thumbnail: undefined }, - { display_id: '', id: '3', name: 'screen 3', thumbnail: undefined }, ], }; env.isWindowsOS = true; @@ -509,10 +533,20 @@ describe('screen picker', () => { display_id: '', id: '1', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, + }, + { + display_id: '', + id: '2', + name: 'Screen 2', + thumbnail: mockThumbnail, + }, + { + display_id: '', + id: '3', + name: 'screen 3', + thumbnail: mockThumbnail, }, - { display_id: '', id: '2', name: 'Screen 2', thumbnail: undefined }, - { display_id: '', id: '3', name: 'screen 3', thumbnail: undefined }, ], }; env.isWindowsOS = false; @@ -531,13 +565,13 @@ describe('screen picker', () => { display_id: '1', id: '1', name: 'Entire screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }, { display_id: '', id: '1', name: 'Application screen', - thumbnail: undefined, + thumbnail: mockThumbnail, }, ], }; diff --git a/src/app/display-media-request-handler.ts b/src/app/display-media-request-handler.ts index 70231ea92..5c1b6717f 100644 --- a/src/app/display-media-request-handler.ts +++ b/src/app/display-media-request-handler.ts @@ -1,19 +1,103 @@ -import { session } from 'electron'; +import { desktopCapturer, ipcMain, session } from 'electron'; +import { NOTIFICATION_WINDOW_TITLE } from '../common/api-interface'; +import { isDevEnv, isMac } from '../common/env'; +import { logger } from '../common/logger'; +import { + ICustomBrowserWindowConstructorOpts, + windowHandler, +} from './window-handler'; +import { createComponentWindow, windowExists } from './window-utils'; /** - * This is currently supported only on macOS 15+. - * setDisplayMediaRequestHandler injects into navigator.mediaDevices.getDisplayMedia(). - * With the macOS-only option { useSystemPicker: true }, - * everyting is handled natively by the OS. + * For MacOS 15+ the { useSystemPicker: true } overrides the code set in the handler, + * and uses the native implementation. * - * For all other OSes and versions, the regular screen share flow will be used. + * But for other versions and OSes, the code is executed. */ export const setDisplayMediaRequestHandler = () => { const { defaultSession } = session; - defaultSession.setDisplayMediaRequestHandler( - async (_request, _callback) => { - // TODO - Add support for Windows. + async (_request, callback) => { + logger.info('display-media-request-handler: getting sources'); + const sources = await desktopCapturer.getSources({ + types: ['screen', 'window'], + thumbnailSize: { + height: 150, + width: 150, + }, + }); + + const updatedSources = sources.filter( + (source) => source.name !== NOTIFICATION_WINDOW_TITLE, + ); + + const browserWindowOptions: ICustomBrowserWindowConstructorOpts = + windowHandler.getWindowOpts( + { + alwaysOnTop: true, + autoHideMenuBar: true, + frame: false, + modal: false, + height: isMac ? 519 : 523, + width: 580, + show: false, + fullscreenable: false, + }, + { + devTools: isDevEnv, + }, + ); + const screenPickerWindow = createComponentWindow( + 'screen-picker', + browserWindowOptions, + ); + + screenPickerWindow.webContents.once('did-finish-load', () => { + if (!screenPickerWindow || !windowExists(screenPickerWindow)) { + return; + } + screenPickerWindow.webContents.send('screen-picker-data', { + sources: updatedSources, + }); + }); + + const mainWebContents = windowHandler.getMainWebContents(); + if (!mainWebContents) { + return; + } + mainWebContents.send('screen-picker-data', updatedSources); + + ipcMain.on( + 'screen-source-select', + (_event, source: Electron.DesktopCapturerSource) => { + if (source) { + windowHandler.drawScreenShareIndicatorFrame(source); + } + logger.info('display-media-request-handler: source selected', source); + }, + ); + + ipcMain.once( + 'screen-source-selected', + (_event, source: Electron.DesktopCapturerSource) => { + screenPickerWindow.close(); + logger.info( + 'display-media-request-handler: source to be shared', + source, + ); + if (!source) { + windowHandler.closeScreenSharingIndicator(); + /** + * Passing the empty stream crashes the main process, + * but passing an empty callback throws an AbortError. + */ + // @ts-ignore + callback(); + } else { + callback({ video: source }); + } + }, + ); }, { useSystemPicker: true }, ); diff --git a/src/app/window-handler.ts b/src/app/window-handler.ts index f00280d5f..6e6cf6a63 100644 --- a/src/app/window-handler.ts +++ b/src/app/window-handler.ts @@ -2419,6 +2419,36 @@ export class WindowHandler { app.exit(); }; + /** + * Returns constructor opts for the browser window + * + * @param windowOpts {Electron.BrowserWindowConstructorOptions} + * @param webPreferences {Electron.WebPreferences} + */ + public getWindowOpts( + windowOpts: Electron.BrowserWindowConstructorOptions, + webPreferences: Electron.WebPreferences, + ): ICustomBrowserWindowConstructorOpts { + const defaultPreferencesOpts = { + ...{ + sandbox: IS_SAND_BOXED, + nodeIntegration: IS_NODE_INTEGRATION_ENABLED, + contextIsolation: this.contextIsolation, + backgroundThrottling: this.backgroundThrottling, + enableRemoteModule: true, + disableBlinkFeatures: AUX_CLICK, + }, + ...webPreferences, + }; + const defaultWindowOpts = { + alwaysOnTop: false, + webPreferences: defaultPreferencesOpts, + winKey: getGuid(), + }; + + return { ...defaultWindowOpts, ...windowOpts }; + } + /** * Listens for app load timeouts and reloads if required */ @@ -2507,36 +2537,6 @@ export class WindowHandler { } } - /** - * Returns constructor opts for the browser window - * - * @param windowOpts {Electron.BrowserWindowConstructorOptions} - * @param webPreferences {Electron.WebPreferences} - */ - private getWindowOpts( - windowOpts: Electron.BrowserWindowConstructorOptions, - webPreferences: Electron.WebPreferences, - ): ICustomBrowserWindowConstructorOpts { - const defaultPreferencesOpts = { - ...{ - sandbox: IS_SAND_BOXED, - nodeIntegration: IS_NODE_INTEGRATION_ENABLED, - contextIsolation: this.contextIsolation, - backgroundThrottling: this.backgroundThrottling, - enableRemoteModule: true, - disableBlinkFeatures: AUX_CLICK, - }, - ...webPreferences, - }; - const defaultWindowOpts = { - alwaysOnTop: false, - webPreferences: defaultPreferencesOpts, - winKey: getGuid(), - }; - - return { ...defaultWindowOpts, ...windowOpts }; - } - /** * getUserAgent retrieves current window user-agent and updates it * depending on global config setup diff --git a/src/renderer/components/screen-picker.tsx b/src/renderer/components/screen-picker.tsx index 2865e5dcf..1b320fc1c 100644 --- a/src/renderer/components/screen-picker.tsx +++ b/src/renderer/components/screen-picker.tsx @@ -172,7 +172,7 @@ export default class ScreenPicker extends React.Component<{}, IState> {
thumbnail image
@@ -190,7 +190,7 @@ export default class ScreenPicker extends React.Component<{}, IState> {
thumbnail image
diff --git a/src/renderer/desktop-capturer.ts b/src/renderer/desktop-capturer.ts index 41e3deacb..6a3734973 100644 --- a/src/renderer/desktop-capturer.ts +++ b/src/renderer/desktop-capturer.ts @@ -139,16 +139,9 @@ export const getSource = async ( } } - const updatedSources = sources - .filter((source) => source.name !== NOTIFICATION_WINDOW_TITLE) - .map((source) => { - return { - ...source, - ...{ - thumbnail: source.thumbnail.toDataURL(), - }, - }; - }); + const updatedSources = sources.filter( + (source) => source.name !== NOTIFICATION_WINDOW_TITLE, + ); ipcRenderer.send(apiName.symphonyApi, { cmd: apiCmds.openScreenPickerWindow,