From f9f5f7692ba2b5d1c0299e08ad9e4dc16de0065b Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Thu, 29 Jun 2017 14:06:38 -0500 Subject: [PATCH] feat(remotecontrol): multi monitor support --- README.md | 4 + config.js | 9 +- modules/remotecontrol/constants.js | 17 +- modules/remotecontrol/index.js | 222 ++++++++++-------- node_addons/sourceId2Coordinates/binding.gyp | 27 +++ node_addons/sourceId2Coordinates/index.js | 22 ++ node_addons/sourceId2Coordinates/package.json | 17 ++ node_addons/sourceId2Coordinates/src/index.cc | 37 +++ .../src/sourceId2Coordinates.cc | 45 ++++ .../src/sourceId2Coordinates.h | 7 + package.json | 3 +- windows/jitsi-meet/render.js | 59 +---- 12 files changed, 297 insertions(+), 172 deletions(-) create mode 100644 node_addons/sourceId2Coordinates/binding.gyp create mode 100644 node_addons/sourceId2Coordinates/index.js create mode 100644 node_addons/sourceId2Coordinates/package.json create mode 100644 node_addons/sourceId2Coordinates/src/index.cc create mode 100644 node_addons/sourceId2Coordinates/src/sourceId2Coordinates.cc create mode 100644 node_addons/sourceId2Coordinates/src/sourceId2Coordinates.h diff --git a/README.md b/README.md index b20f0f1..430f630 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ from config.js ```bash npm install ``` +Since node_addons/sourceId2Coordinates add-on is local dependency, every code change requires increasing the version in its package.json. To rebuild the add-on if it is already installed execute: +```bash +npm update +``` ## Statring the application ```bash diff --git a/config.js b/config.js index 7aeb711..f48850b 100644 --- a/config.js +++ b/config.js @@ -2,12 +2,5 @@ module.exports = { /** * The URL of the Jitsi Meet deployment that will be used. */ - jitsiMeetURL: "https://meet.jit.si/", - - /** - * If true, the authorization of remote control will be handled by jitsi - * meet electron app. Otherwise the authorization will be handled in Jitsi - * Meet. - */ - handleRemoteControlAuthorization: false + jitsiMeetURL: "https://meet.jit.si/" }; diff --git a/modules/remotecontrol/constants.js b/modules/remotecontrol/constants.js index e8d53a6..edd2e6b 100644 --- a/modules/remotecontrol/constants.js +++ b/modules/remotecontrol/constants.js @@ -1,8 +1,8 @@ module.exports = { /** - * Types of remote-control-event events. + * Types of remote-control events. */ - EVENT_TYPES: { + EVENTS: { mousemove: "mousemove", mousedown: "mousedown", mouseup: "mouseup", @@ -10,22 +10,19 @@ module.exports = { mousescroll: "mousescroll", keydown: "keydown", keyup: "keyup", - permissions: "permissions", stop: "stop", supported: "supported" }, /** - * Actions for the remote control permission events. + * Types of remote-control requests. */ - PERMISSIONS_ACTIONS: { - request: "request", - grant: "grant", - deny: "deny" + REQUESTS: { + start: "start" }, /** - * The name of remote control events sent trough the API module. + * The name of remote control messages. */ - REMOTE_CONTROL_EVENT_NAME: "remote-control-event" + REMOTE_CONTROL_MESSAGE_NAME: "remote-control" }; diff --git a/modules/remotecontrol/index.js b/modules/remotecontrol/index.js index 1d67e8b..4efcdad 100644 --- a/modules/remotecontrol/index.js +++ b/modules/remotecontrol/index.js @@ -1,6 +1,8 @@ let robot = require("robotjs"); +const sourceId2Coordinates = require("sourceId2Coordinates"); +const electron = require("electron"); const constants = require("../../modules/remotecontrol/constants"); -const {EVENT_TYPES, PERMISSIONS_ACTIONS, REMOTE_CONTROL_EVENT_NAME} = constants; +const { EVENTS, REQUESTS, REMOTE_CONTROL_MESSAGE_NAME } = constants; /** * Attaching to the window for debug purposes. @@ -9,13 +11,8 @@ const {EVENT_TYPES, PERMISSIONS_ACTIONS, REMOTE_CONTROL_EVENT_NAME} = constants; window.robot = robot; /** - * Width/Heught for the screen. - */ -const {width, height} = robot.getScreenSize(); - -/** - * Mouse button mapping between the values in remote-control-event and robotjs - * methods. + * Mouse button mapping between the values in remote control mouse event and + * robotjs methods. */ const MOUSE_BUTTONS = { 1: "left", @@ -24,8 +21,8 @@ const MOUSE_BUTTONS = { }; /** - * Mouse actions mapping between the values in remote-control-event and robotjs - * methods. + * Mouse actions mapping between the values in remote control mouse event and + * robotjs methods. */ const MOUSE_ACTIONS_FROM_EVENT_TYPE = { "mousedown": "down", @@ -33,8 +30,8 @@ const MOUSE_ACTIONS_FROM_EVENT_TYPE = { }; /** - * Key actions mapping between the values in remote-control-event and robotjs - * methods. + * Key actions mapping between the values in remote control key event and + * robotjs methods. */ const KEY_ACTIONS_FROM_EVENT_TYPE = { "keydown": "down", @@ -50,35 +47,25 @@ const KEY_ACTIONS_FROM_EVENT_TYPE = { let mouseButtonStatus = "up"; /** - * Parses the remote-control-events and executes them via robotjs. + * Parses the remote control events and executes them via robotjs. */ class RemoteControl { /** - * Construcs new instance. - * @constructor + * Constructs new instance and initializes the remote control functionality. + * + * @param {Postis} channel the postis channel. */ - constructor() { - this.started = false; - } - - /** - * Initializes the remote control functionality. - */ - init(channel, windowManager, handleAuthorization) { - this.handleAuthorization = handleAuthorization; - this.windowManager = windowManager; + constructor(channel) { + // TODO: if no channel is passed, create one. this.channel = channel; this.channel.ready(() => { this.channel.listen('message', message => { - const event = message.data; - if(event.name === REMOTE_CONTROL_EVENT_NAME) { - this.onRemoteControlEvent(event); + const { name } = message.data; + if(name === REMOTE_CONTROL_MESSAGE_NAME) { + this.onRemoteControlMessage(message); } }); - this.sendEvent({type: EVENT_TYPES.supported}); - if (!handleAuthorization) { - this.start(); - } + this.sendEvent({ type: EVENTS.supported }); }); } @@ -86,114 +73,137 @@ class RemoteControl { * Disposes the remote control functionality. */ dispose() { - this.windowManager = null; this.channel = null; this.stop(); } /** - * Handles permission requests from Jitsi Meet. - * @param {object} userInfo - information about the user that has requested - * permissions: - * @param {string} userInfo.displayName - display name - * @param {string} userInfo.userJID - the JID of the user. - * @param {string} userInfo.userId - the user id (the resource of the JID) - * @param {boolean} userInfo.screenSharing - true if the screen sharing - * is started. + * Handles remote control start messages. + * + * @param {number} id - the id of the request that will be used for the + * response. + * @param {string} sourceId - The source id of the desktop sharing stream. */ - handlePermissionRequest(userInfo) { - this.windowManager.requestRemoteControlPermissions(userInfo) - .then(result => { - this.sendEvent({ - type: EVENT_TYPES.permissions, - action: result ? PERMISSIONS_ACTIONS.grant - : PERMISSIONS_ACTIONS.deny, - userId: userInfo.userId - }); - if(result) { - this.start(); - } - }).catch(e => { - console.error(e); - }); - } + start(id, sourceId) { + const displays = electron.screen.getAllDisplays(); - /** - * Starts processing the events. - */ - start() { - this.started = true; + switch(displays.length) { + case 0: + this.display = undefined; + break; + case 1: + // On Linux probably we'll end up here even if there are + // multiple monitors. + this.display = displays[0]; + break; + // eslint-disable-next-line no-case-declarations + default: // > 1 display + const coordinates = sourceId2Coordinates(sourceId); + if(coordinates) { + // Currently sourceId2Coordinates will return undefined for + // any OS except Windows. This code will be executed only on + // Windows. + const { x, y } = coordinates; + this.display = electron.screen.getDisplayNearestPoint({ + x: x + 1, + y: y + 1 + }); + } else { + // On Mac OS the sourceId = 'screen' + displayId. + // Try to match displayId with sourceId. + const displayId = Number(sourceId.replace('screen:', '')); + this.display + = displays.find(display => display.id === displayId); + } + } + + const response = { + id, + type: 'response' + }; + + if(this.display) { + response.result = true; + } else { + response.error + = 'Error: Can\'t detect the display that is currently shared'; + } + + this.sendMessage(response); } /** * Stops processing the events. */ stop() { - this.started = false; + this.display = undefined; } /** - * Executes the passed event. - * @param {Object} event the remote-control-event. + * Executes the passed message. + * @param {Object} message the remote control message. */ - onRemoteControlEvent(event) { - if(!this.started && event.type !== EVENT_TYPES.permissions) { + onRemoteControlMessage(message) { + const { id, data } = message; + + // If we haven't set the display prop. We haven't received the remote + // control start message or there was an error associating a display. + if(!this.display + && data.type != REQUESTS.start) { return; } - switch(event.type) { - case EVENT_TYPES.mousemove: { - const x = event.x * width, y = event.y * height; + switch(data.type) { + case EVENTS.mousemove: { + const { width, height, x, y } = this.display.workArea; + const destX = data.x * width + x; + const destY = data.y * height + y; if(mouseButtonStatus === "down") { - robot.dragMouse(x, y); + robot.dragMouse(destX, destY); } else { - robot.moveMouse(x, y); + robot.moveMouse(destX, destY); } break; } - case EVENT_TYPES.mousedown: - case EVENT_TYPES.mouseup: { - mouseButtonStatus = MOUSE_ACTIONS_FROM_EVENT_TYPE[event.type]; + case EVENTS.mousedown: + case EVENTS.mouseup: { + mouseButtonStatus + = MOUSE_ACTIONS_FROM_EVENT_TYPE[data.type]; robot.mouseToggle( mouseButtonStatus, - (event.button ? MOUSE_BUTTONS[event.button] : undefined)); + (data.button + ? MOUSE_BUTTONS[data.button] : undefined)); break; } - case EVENT_TYPES.mousedblclick: { + case EVENTS.mousedblclick: { robot.mouseClick( - (event.button ? MOUSE_BUTTONS[event.button] : undefined), + (data.button + ? MOUSE_BUTTONS[data.button] : undefined), true); break; } - case EVENT_TYPES.mousescroll:{ + case EVENTS.mousescroll:{ //FIXME: implement horizontal scrolling - if(event.y !== 0) { + if(data.y !== 0) { robot.scrollMouse( - Math.abs(event.y), - event.y > 0 ? "down" : "up" + Math.abs(data.y), + data.y > 0 ? "down" : "up" ); } break; } - case EVENT_TYPES.keydown: - case EVENT_TYPES.keyup: { - robot.keyToggle(event.key, - KEY_ACTIONS_FROM_EVENT_TYPE[event.type], event.modifiers); + case EVENTS.keydown: + case EVENTS.keyup: { + robot.keyToggle( + data.key, + KEY_ACTIONS_FROM_EVENT_TYPE[data.type], + data.modifiers); break; } - case EVENT_TYPES.permissions: { - if(event.action === PERMISSIONS_ACTIONS.request - && this.handleAuthorization) { - // Open Dialog and answer - this.handlePermissionRequest({ - userId: event.userId, - userJID: event.userJID, - displayName: event.displayName, - screenSharing: event.screenSharing - }); - } + case REQUESTS.start: { + this.start(id, data.sourceId); break; } - case EVENT_TYPES.stop: { + case EVENTS.stop: { this.stop(); break; } @@ -204,18 +214,28 @@ class RemoteControl { /** * Sends remote control event to the controlled participant. + * * @param {Object} event the remote control event. */ sendEvent(event) { const remoteControlEvent = Object.assign( - { name: REMOTE_CONTROL_EVENT_NAME }, + { name: REMOTE_CONTROL_MESSAGE_NAME }, event ); + this.sendMessage({ data: remoteControlEvent }); + } + + /** + * Sends a message to Jitsi Meet. + * + * @param {Object} message the message to be sent. + */ + sendMessage(message) { this.channel.send({ method: 'message', - params: { data: remoteControlEvent } + params: message }); } } -module.exports = new RemoteControl(); +module.exports = RemoteControl; diff --git a/node_addons/sourceId2Coordinates/binding.gyp b/node_addons/sourceId2Coordinates/binding.gyp new file mode 100644 index 0000000..4f3d0f6 --- /dev/null +++ b/node_addons/sourceId2Coordinates/binding.gyp @@ -0,0 +1,27 @@ +{ + 'targets': [{ + 'target_name': 'sourceId2Coordinates', + 'include_dirs': [ + "|undefined} - The x and y coordinates of the + * top left corner of the desktop. Currently works only for windows. Returns + * undefined for Mac OS, Linux. + */ +module.exports = function(sourceID) { + // On windows the source id will have the following format "0:desktop_id". + // we need the "desktop_id" only to get the coordinates. + const idArr = sourceID.split(":"); + const id = Number(idArr.length > 1 ? idArr[1] : sourceID); + if(id) { + return sourceId2Coordinates(id); + } + return undefined; +}; diff --git a/node_addons/sourceId2Coordinates/package.json b/node_addons/sourceId2Coordinates/package.json new file mode 100644 index 0000000..fce1579 --- /dev/null +++ b/node_addons/sourceId2Coordinates/package.json @@ -0,0 +1,17 @@ +{ + "name": "sourceId2Coordinates", + "version": "0.0.3", + "description": "Native addon that returns the coordinates of desktop using the passed source id", + "main": "index.js", + "scripts": { + "install": "node-gyp rebuild" + }, + "keywords": [ + "Util", + "WebRTC" + ], + "gypfile": true, + "dependencies": { + "nan": "^2.2.1" + } +} diff --git a/node_addons/sourceId2Coordinates/src/index.cc b/node_addons/sourceId2Coordinates/src/index.cc new file mode 100644 index 0000000..5820d4c --- /dev/null +++ b/node_addons/sourceId2Coordinates/src/index.cc @@ -0,0 +1,37 @@ +#include +#include +#include +#include +#include "sourceId2Coordinates.h" + +using namespace v8; + +NAN_METHOD(sourceId2Coordinates) +{ + const int sourceID = info[0]->Int32Value(); + Local obj = Nan::New(); + Point coordinates; + if(!sourceId2Coordinates(sourceID, &coordinates)) + { // return undefined if sourceId2Coordinates function fail. + info.GetReturnValue().Set(Nan::Undefined()); + } + else + { // return the coordinates if sourceId2Coordinates function succeed. + Nan::Set(obj, Nan::New("x").ToLocalChecked(), Nan::New(coordinates.x)); + Nan::Set(obj, Nan::New("y").ToLocalChecked(), Nan::New(coordinates.y)); + info.GetReturnValue().Set(obj); + } +} + +NAN_MODULE_INIT(Init) +{ + Nan::Set( + target, + Nan::New("sourceId2Coordinates").ToLocalChecked(), + Nan::GetFunction(Nan::New(sourceId2Coordinates)) + .ToLocalChecked() + ); + NAN_EXPORT(target, sourceId2Coordinates); +} + +NODE_MODULE(sourceId2CoordinatesModule, Init) diff --git a/node_addons/sourceId2Coordinates/src/sourceId2Coordinates.cc b/node_addons/sourceId2Coordinates/src/sourceId2Coordinates.cc new file mode 100644 index 0000000..8f2d1f3 --- /dev/null +++ b/node_addons/sourceId2Coordinates/src/sourceId2Coordinates.cc @@ -0,0 +1,45 @@ +#if defined(IS_WINDOWS) +#include +#endif + +#include "sourceId2Coordinates.h" + +/** + * Tries to get the coordinates of a desktop from passed sourceId + * (which identifies a desktop sharing source). Used to match the source id to a + * screen in Electron. + * + * Returns true on success and false on failure. + * + * NOTE: Works on windows only because on the other platforms there is an easier + * way to match the source id and the screen. + */ +bool sourceId2Coordinates(int sourceId, Point* res) +{ +#if defined(IS_WINDOWS) + DISPLAY_DEVICE device; + device.cb = sizeof(device); + + if (!EnumDisplayDevices(NULL, sourceId, &device, 0) // device not found + || !(device.StateFlags & DISPLAY_DEVICE_ACTIVE))// device is not active + { + return false; + } + + DEVMODE deviceSettings; + deviceSettings.dmSize = sizeof(deviceSettings); + deviceSettings.dmDriverExtra = 0; + if(!EnumDisplaySettingsEx(device.DeviceName, ENUM_CURRENT_SETTINGS, + &deviceSettings, 0)) + { + return false; + } + + res->x = deviceSettings.dmPosition.x; + res->y = deviceSettings.dmPosition.y; + + return true; +#else + return false; +#endif +} diff --git a/node_addons/sourceId2Coordinates/src/sourceId2Coordinates.h b/node_addons/sourceId2Coordinates/src/sourceId2Coordinates.h new file mode 100644 index 0000000..5425b75 --- /dev/null +++ b/node_addons/sourceId2Coordinates/src/sourceId2Coordinates.h @@ -0,0 +1,7 @@ +struct Point { + int x; + int y; + Point(): x(0), y(0) {}; +}; + +bool sourceId2Coordinates(int sourceId, Point* res); diff --git a/package.json b/package.json index e187535..0d1085c 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "readmeFilename": "README.md", "license": "Apache-2.0", "dependencies": { - "robotjs": "hristoterezov/robotjs", + "sourceId2Coordinates": "file:./node_addons/sourceId2Coordinates", + "robotjs": "jitsi/robotjs#jitsi", "postis": "^2.2.0" }, "devDependencies": { diff --git a/windows/jitsi-meet/render.js b/windows/jitsi-meet/render.js index a4246b9..9b67aa2 100644 --- a/windows/jitsi-meet/render.js +++ b/windows/jitsi-meet/render.js @@ -1,15 +1,19 @@ /* global process */ -const remoteControl = require("../../modules/remotecontrol"); +const RemoteControl = require("../../modules/remotecontrol"); let postis = require("postis"); const setupScreenSharingForWindow = require("../../modules/screensharing"); const config = require("../../config.js"); -const {dialog} = require('electron').remote; /** * The postis channel. */ let channel; +/** + * The remote control instance. + */ +let remoteControl; + /** * Cteates the iframe that will load Jitsi Meet. */ @@ -19,52 +23,6 @@ iframe.allowFullscreen = true; iframe.onload = onload; document.body.appendChild(iframe); -/** - * Factory for dialogs. - */ -class DialogFactory { - /** - * Creates new instance - * @constructor - */ - constructor() { } - - /** - * Shows message box dialog for request for remote control permissions - * @param {object} userInfo - information about the user that has sent the - * request: - * @param {string} userInfo.displayName - display name - * @param {string} userInfo.userJID - the JID of the user. - * @param {boolean} userInfo.screenSharing - true if the screen sharing - * is started. - */ - requestRemoteControlPermissions(userInfo) { - return new Promise( resolve => - dialog.showMessageBox({ - type: "question", - buttons: [ - "Yes", - "No" - ], - defaultId: 0, - title: "Request for permission for remote control", - message: "Would you like to allow " + userInfo.displayName - + " to remotely control your desktop?" - + (userInfo.screenSharing ? "" - : "\nNote: If you press \"Yes\" the screen sharing " - + "will start!"), - detail: "userId: " + userInfo.userJID, - cancelId: 1 - }, response => resolve(response === 0? true : false)) - ); - } -} - -/** - * Dialog factory instance. - */ -const dialogFactory = new DialogFactory(); - /** * Handles loaded event for iframe: * Enables screen sharing functionality to the iframe webpage. @@ -78,10 +36,7 @@ function onload() { window: iframe.contentWindow, windowForEventListening: window }); - remoteControl.init( - channel, - dialogFactory, - config.handleRemoteControlAuthorization); + remoteControl = new RemoteControl(channel); } /**