From aa1fd7cd805ed0a0f04cda415902a77e03852158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20D=C3=B6ring?= <simon.doering@stud.hs-bochum.de> Date: Wed, 6 Jan 2021 23:42:08 +0100 Subject: [PATCH] Add annotation and name logic --- README.md | 29 +++++- .../io-interface/handlers/input-handlers.ts | 44 ++++++++- .../io-interface/handlers/output-handlers.ts | 4 + .../src/socket-io/handlers/common-handlers.ts | 15 ++- .../src/socket-io/handlers/sender-handlers.ts | 23 ++++- camera-server/src/state/camera-slot-state.ts | 1 + novnc/app/camera-receiver.js | 95 +++++++++++++++---- novnc/app/styles/camera-receiver.css | 14 +-- sender/camera-sender.html | 46 ++++++--- sender/camera-sender.js | 29 ++++-- 10 files changed, 243 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index debe2d5..8175468 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,30 @@ Note that the project and its documentation are still in development. In this documentation clients that publish a feed will be referred to as *senders* and clients who receive a feed will be referred to as *receivers* or *viewers*. +# Sender Web Page + +CVH-Camera provides a web page with which users can transmit camera feeds. + +## Query Parameters + +The behavior of the sender page can be controlled using its query parameters. + +Providing query parameters can be done by appending a question mark to the url, followed by key-value pairs which are divided by and symbols. +Example: `http://www.mywebpage.com/somePage.html?param1=value1¶m2=value2`. In the example the parameters `param1` and `param2` are provided +with the values `value1` and `value2`. + +The following list explains the usage of the parameters: + +* `room`: The number of the janus room. Defaults to `1000`. + +* `slot`: The camera slot used in the room. Defaults to `0`. + +* `token`: The token required for authentication on the camera server (see below). Defaults to an empty string, which will yield to the user not being able to transmit his camera feed. + +* `pin` *optional*: The pin for the janus room. If the janus room has no pin, provide the value `none`. If this parameter is not provided, an input field for the pin is shown. + +* `customNameAllowed` *optional*: If this parameter is present (even when holding no value), an input field for a custom name is shown. This name is then required to start the transmission. + # Camera Server CVH-Camera uses a Nodejs server with the socket.io library to send events regarding the camera feeds out to the receivers in realtime. The socket connection is also used to verify feeds that are published in the Janus rooms and only send those to the receivers. This way no unwanted feeds can be shown to the receivers while still having a public password for the Janus room itself. @@ -57,9 +81,11 @@ This is the list of the available commands: | Command | Description | --------------------------------- | ----------- -| `activate_slot` | Activates a slot and sets its token. To set a new token use `refresh_token`. <br/><br/> **Usage**: `activate_slot <slot> <token>` +| `activate_slot` | Activates a slot and sets its token. To set a new token use `refresh_token`. <br/><br/> **Usage**: `activate_slot <slot> <token> [annotation]` <br/><br/> See `set_annotation` for an explanation of the annotation. | `refresh_token` | Sets a new token for a slot. <br/><br/> **Usage**: `refresh_token <slot> <new_token>` | `deactivate_slot` | Deactivates a slot. Also ensures that the feed is removed for the receivers. <br/><br/> **Usage**: `deactivate_slot <slot>` +| `set_annotation` | Sets the annotation of a slot. A simple use case would be displaying the user's name below his feed. <br/><br/> **Usage**: `set_annotation <slot> <annotation>` <br/><br/> `annotation` can be any HTML snippet which will annotate the camera feed of the slot. This is done by appending the snippet to the container which contains the video element of the feed. <br/> If this example snippet is provided, the text *Hello noVNC* will be displayed at the bottom of the feed container: `<div style="box-sizing: border-box; position: absolute; bottom: 0; width: 100%; background: rgba(0, 0, 0, 0.75); color: white; text-align: center; padding: 4px;">Hello noVNC</div>`. <br/><br/> **Important**: The HTML snippet has to have only one parent element. The container uses the CSS declaration `position: fixed`. Thus, working with `position: absolute` is possible and definitely advised. +| `remove_annotation` | Removes the annotation of a slot. <br/><br/> **Usage**: `remove_annotation <slot>` ### Camera Control Commands @@ -87,6 +113,7 @@ This is a list of all sent messages. Note that a newline character `\n` is appen | ---------------------------------- | ----------- | `new_feed <slot>` | Sent after a sender on a slot has started transmitting a feed. | `remove_feed <slot>` | Sent after a sender on a slot has stopped transmitting a feed or the slot is deactivated (which also removes the feed). +| `custom_name <slot> <custom_name>` | Sent after a sender on a slot has started transmitting a feed and has set a custom name. The name is a string which is guaranteed to be escaped to prevent Cross-Site-Scripting (XSS) attacks. Note that the name can contain spaces. <br/> The controller should wrap the name into a HTML snippet and send it back to the camera server using the `set_annotation` command. ## Socket Traffic diff --git a/camera-server/src/io-interface/handlers/input-handlers.ts b/camera-server/src/io-interface/handlers/input-handlers.ts index 5a2a097..216d115 100644 --- a/camera-server/src/io-interface/handlers/input-handlers.ts +++ b/camera-server/src/io-interface/handlers/input-handlers.ts @@ -1,13 +1,31 @@ import { socketIO } from '../../socket-io/socket-io'; import { cameraSlotState } from '../../state/camera-slot-state'; -import { emitRemoveFeed } from '../../socket-io/handlers/common-handlers'; +import { + emitRemoveFeed, + emitSetAnnotation, + emitRemoveAnnotation +} from '../../socket-io/handlers/common-handlers'; const visibilityCommands = ['hide', 'show']; const geometryCommands = [ 'set_geometry_relative_to_window', 'set_geometry_relative_to_canvas' ]; -const internalCommands = ['activate_slot', 'deactivate_slot', 'refresh_token']; +const internalCommands = [ + 'activate_slot', + 'deactivate_slot', + 'refresh_token', + 'set_annotation', + 'remove_annotation' +]; + +const setAnnotation = (slot: number, annotation: string) => { + console.log(`Setting annotation of slot ${slot} to ${annotation}`); + cameraSlotState[slot].annotation = annotation; + if (cameraSlotState[slot].feedActive) { + emitSetAnnotation(slot, annotation); + } +}; const handleInternalCommand = ( command: string, @@ -29,7 +47,10 @@ const handleInternalCommand = ( ); return; } - currentCameraState.token = params[0]; + currentCameraState.token = params.shift()!; + if (params.length > 0) { + setAnnotation(slot, params.join(' ')); + } currentCameraState.slotActive = true; break; case 'deactivate_slot': @@ -45,6 +66,7 @@ const handleInternalCommand = ( currentCameraState.feedActive = false; currentCameraState.feedId = null; currentCameraState.senderSocketId = null; + currentCameraState.annotation = null; break; case 'refresh_token': if (!currentCameraState.slotActive) { @@ -65,6 +87,22 @@ const handleInternalCommand = ( console.log('Refreshing token for slot ' + slot); currentCameraState.token = params[0]; break; + case 'set_annotation': + if (params.length === 0) { + console.log( + 'Error: Tried to set annotation without providing one' + ); + return; + } + setAnnotation(slot, params.join(' ')); + break; + case 'remove_annotation': + console.log(`Removing annotation for slot ${slot}`); + currentCameraState.annotation = null; + if (currentCameraState.feedActive) { + emitRemoveAnnotation(slot); + } + break; default: console.log( 'Error: handleInternalCommand got unknown command ' + command diff --git a/camera-server/src/io-interface/handlers/output-handlers.ts b/camera-server/src/io-interface/handlers/output-handlers.ts index ad81cd3..36a043a 100644 --- a/camera-server/src/io-interface/handlers/output-handlers.ts +++ b/camera-server/src/io-interface/handlers/output-handlers.ts @@ -53,3 +53,7 @@ export const notifyNewFeed = (slot: number) => { export const notifyRemoveFeed = (slot: number) => { notifyController(`remove_feed ${slot}`); }; + +export const notifyCustomName = (slot: number, name: string) => { + notifyController(`custom_name ${slot} ${name}`); +}; diff --git a/camera-server/src/socket-io/handlers/common-handlers.ts b/camera-server/src/socket-io/handlers/common-handlers.ts index 39d4e53..32fd535 100644 --- a/camera-server/src/socket-io/handlers/common-handlers.ts +++ b/camera-server/src/socket-io/handlers/common-handlers.ts @@ -12,7 +12,8 @@ export const emitNewFeed = (slot: number) => { slot, feedId: slotState.feedId, visibility: slotState.visibility, - geometry: slotState.geometry + geometry: slotState.geometry, + annotation: slotState.annotation }); notifyNewFeed(slot); }; @@ -22,6 +23,14 @@ export const emitRemoveFeed = (slot: number) => { notifyRemoveFeed(slot); }; +export const emitSetAnnotation = (slot: number, annotation: string) => { + socketIO.emit('set_annotation', { slot, annotation }); +}; + +export const emitRemoveAnnotation = (slot: number) => { + socketIO.emit('remove_annotation', { slot }); +}; + export const handleQueryState = (fn: Function) => { console.log('Got state query from socket'); let response: { @@ -29,6 +38,7 @@ export const handleQueryState = (fn: Function) => { feedId: string | null; visibility: CommandDescriptor; geometry: CommandDescriptor; + annotation: string | null; }; } = {}; for (let i = 0; i < cameraSlotState.length; i++) { @@ -37,7 +47,8 @@ export const handleQueryState = (fn: Function) => { response[i] = { feedId: slotState.feedId, visibility: slotState.visibility, - geometry: slotState.geometry + geometry: slotState.geometry, + annotation: slotState.annotation }; } } diff --git a/camera-server/src/socket-io/handlers/sender-handlers.ts b/camera-server/src/socket-io/handlers/sender-handlers.ts index ae4b7fb..5fbcc69 100644 --- a/camera-server/src/socket-io/handlers/sender-handlers.ts +++ b/camera-server/src/socket-io/handlers/sender-handlers.ts @@ -2,10 +2,11 @@ import { cameraSlotState } from '../../state/camera-slot-state'; import { emitNewFeed, emitRemoveFeed } from './common-handlers'; import { SenderSocket } from '../../models/sender-socket'; import { ValidationError } from '../../models/validation-error'; +import { notifyCustomName } from '../../io-interface/handlers/output-handlers'; const handleSetFeedId = ( socket: SenderSocket, - data: null | { feedId?: string }, + data: null | { feedId?: string; customName?: string }, fn: Function ) => { let success = true; @@ -56,6 +57,26 @@ const handleSetFeedId = ( throw new ValidationError('No feed id was provided'); } + const unescapedCustomName = data.customName; + if (unescapedCustomName != null) { + console.log( + `Got custom name from slot ${slot}: ${unescapedCustomName}` + ); + const customName = unescapedCustomName + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + if (unescapedCustomName !== customName) { + console.log( + `Warning: Escaped custom name (${customName}) does not equal the unescaped custom name` + + `(${unescapedCustomName}) on slot ${slot} - this could mean that the user tried a Cross-Site-Scripting (XSS) attack` + ); + } + notifyCustomName(slot, customName); + } + console.log('Setting feed id of slot ' + slot + ' to ' + feedId); message = 'Successfully set feed id - you are now using this slot'; diff --git a/camera-server/src/state/camera-slot-state.ts b/camera-server/src/state/camera-slot-state.ts index d450175..e923eb4 100644 --- a/camera-server/src/state/camera-slot-state.ts +++ b/camera-server/src/state/camera-slot-state.ts @@ -9,6 +9,7 @@ class SingleCameraSlotState { feedActive = false; feedId: NullableString = null; senderSocketId: NullableString = null; + annotation: NullableString = null; visibility: CommandDescriptor = { command: 'show', params: [] diff --git a/novnc/app/camera-receiver.js b/novnc/app/camera-receiver.js index 679f905..823ad4d 100644 --- a/novnc/app/camera-receiver.js +++ b/novnc/app/camera-receiver.js @@ -83,7 +83,8 @@ document.addEventListener('DOMContentLoaded', function() { var state = cameraStates[slot]; newRemoteFeed(slot, state.feedId, { geometry: state.geometry, - visibility: state.visibility + visibility: state.visibility, + annotation: state.annotation }); }); } @@ -111,12 +112,16 @@ document.addEventListener('DOMContentLoaded', function() { }); }}); + function getVideoContainer(slot) { + return document.getElementById(`camera-feed-${slot}-container`); + } + function removeRemoteFeed(slot) { delete videoActiveGeometry[slot]; delete videoGeometryParams[slot]; delete source[slot]; - var video = document.getElementById('camera-feed-' + slot); - video.remove(); + var videoContainer = getVideoContainer(slot); + videoContainer.remove(); janusPluginHandles[slot].detach(); delete janusPluginHandles[slot]; } @@ -127,6 +132,10 @@ document.addEventListener('DOMContentLoaded', function() { var video = document.getElementById(cameraElementId); if (video == null) { + var videoContainer = document.createElement('div'); + videoContainer.setAttribute('id', cameraElementId + '-container'); + videoContainer.classList.add('camera-feed-container'); + video = document.createElement('video'); video.setAttribute('id', cameraElementId); video.setAttribute('muted', ''); @@ -138,7 +147,9 @@ document.addEventListener('DOMContentLoaded', function() { video.play(); } video.classList.add('camera-feed'); - document.body.appendChild(video); + + document.body.appendChild(videoContainer); + videoContainer.appendChild(video); } var remoteFeedHandle = null; @@ -173,9 +184,11 @@ document.addEventListener('DOMContentLoaded', function() { } }); - handleCommand(slot, initialState.geometry.command, initialState.geometry.params); handleCommand(slot, initialState.visibility.command, initialState.visibility.params); + if (initialState.annotation) { + setAnnotation(slot, initialState.annotation); + } } function handleMessageSubscriber(slot, msg, jsep) { @@ -250,7 +263,8 @@ document.addEventListener('DOMContentLoaded', function() { console.log('new_feed', data); newRemoteFeed(data.slot, data.feedId, { geometry: data.geometry, - visibility: data.visibility + visibility: data.visibility, + annotation: data.annotation }); }); @@ -264,15 +278,60 @@ document.addEventListener('DOMContentLoaded', function() { removeRemoteFeed(slot); }); }); + + socket.on('set_annotation', function(data) { + console.log('set_annotation', data); + setAnnotation(data.slot, data.annotation); + }); + + socket.on('remove_annotation', function(data) { + console.log('remove_annotation', data); + removeAnnotation(data.slot); + }); + } + + function setAnnotation(slot, annotationHtml) { + var videoContainer = getVideoContainer(slot); + + if (videoContainer) { + var template = document.createElement('template'); + template.innerHTML = annotationHtml.trim(); + var annotationEl = template.content.firstChild; + if (annotationEl && annotationEl.classList) { + annotationEl.classList.add('annotation'); + var prevAnnotation = videoContainer.querySelector('.annotation'); + if (prevAnnotation) { + prevAnnotation.remove(); + } + videoContainer.appendChild(annotationEl); + } else { + console.error(`setAnnotation called for slot ${slot} with invalid HTML ${annotationHtml}`); + } + } else { + console.error(`setAnnotation called for slot ${slot} which has no video container`); + } + } + + function removeAnnotation(slot) { + var videoContainer = getVideoContainer(slot); + + if (videoContainer) { + var annotationEl = videoContainer.querySelector('.annotation'); + if (annotationEl) { + annotationEl.remove(); + } + } else { + console.error(`removeAnnotation called for slot ${slot} which has no video container`); + } } function handleCommand(slot, command, params) { console.log('Got command:', command); console.log('For slot:', slot); console.log('With params:', params); - var video = document.getElementById('camera-feed-' + slot); - if (video == null) { - console.log('handleCommand video element is null'); + var videoContainer = getVideoContainer(slot); + if (videoContainer == null) { + console.log('handleCommand videoContainer element is null'); } switch(command) { case 'set_geometry_relative_to_window': @@ -286,7 +345,7 @@ document.addEventListener('DOMContentLoaded', function() { z += parseInt(params[5]); } - setFixedPosition(video, origin, x, y, w, h, z); + setFixedPosition(videoContainer, origin, x, y, w, h, z); videoGeometryParams[slot] = { origin, x, y, w, h, z }; videoActiveGeometry[slot] = command; @@ -302,14 +361,14 @@ document.addEventListener('DOMContentLoaded', function() { z += parseInt(params[5]); } - handleSetGeometryRelativeToCanvas(video, slot, origin, x, y, w, h, z); + handleSetGeometryRelativeToCanvas(videoContainer, slot, origin, x, y, w, h, z); break; case 'show': - video.classList.remove('visually-hidden'); + videoContainer.classList.remove('visually-hidden'); break; case 'hide': - video.classList.add('visually-hidden'); + videoContainer.classList.add('visually-hidden'); break; default: console.log(`Socket got unknown command '${command}'`); @@ -317,7 +376,7 @@ document.addEventListener('DOMContentLoaded', function() { } } - function handleSetGeometryRelativeToCanvas(video, slot, origin, x, y, w, h, z) { + function handleSetGeometryRelativeToCanvas(videoContainer, slot, origin, x, y, w, h, z) { // Site contains only one canvas - the vnc viewer var canvas = document.querySelector('canvas'); videoGeometryParams[slot] = { origin, x, y, w, h, z }; @@ -360,7 +419,7 @@ document.addEventListener('DOMContentLoaded', function() { y += (canvasRect.bottom - canvasRect.height); } - setFixedPosition(video, origin, x, y, w, h, z); + setFixedPosition(videoContainer, origin, x, y, w, h, z); videoActiveGeometry[slot] = 'set_geometry_relative_to_canvas'; previousCanvasGeometryState = { vncWidth, @@ -372,7 +431,7 @@ document.addEventListener('DOMContentLoaded', function() { }; } - function setFixedPosition(video, origin, x, y, w, h, z) { + function setFixedPosition(videoContainer, origin, x, y, w, h, z) { var style = ( 'position: fixed;' + `width: ${w}px;` + @@ -399,7 +458,7 @@ document.addEventListener('DOMContentLoaded', function() { return; } - video.setAttribute('style', style); + videoContainer.setAttribute('style', style); } function adjustVideoGeometry() { @@ -424,7 +483,7 @@ document.addEventListener('DOMContentLoaded', function() { if (videoActiveGeometry[slot] === 'set_geometry_relative_to_canvas') { var params = videoGeometryParams[slot]; handleSetGeometryRelativeToCanvas( - document.getElementById('camera-feed-' + slot), + getVideoContainer(slot), slot, params.origin, params.x, diff --git a/novnc/app/styles/camera-receiver.css b/novnc/app/styles/camera-receiver.css index 8f2b11a..82e89f2 100644 --- a/novnc/app/styles/camera-receiver.css +++ b/novnc/app/styles/camera-receiver.css @@ -1,14 +1,8 @@ -.camera-feed.hidden { - display: none; -} - -.camera-feed.visually-hidden { +.camera-feed-container.visually-hidden { visibility: hidden; } -.camera-feed.fullscreen { - max-width: none; - max-height: none; - width: 100%; - height: 100%; +.camera-feed { + width: 100%; + height: 100%; } diff --git a/sender/camera-sender.html b/sender/camera-sender.html index 64129ed..8044478 100644 --- a/sender/camera-sender.html +++ b/sender/camera-sender.html @@ -6,25 +6,45 @@ <script type="text/javascript" src="socket.io.min.js"></script> <script type="text/javascript" src="camera-sender.js"></script> <link rel="stylesheet" href="camera-sender.css"> + <style> + /* temporary */ + div { + margin-bottom: 8px; + } + </style> </head> <body> <h3 id="room-indicator"></h3> + <div id="pin-hint"> Leave the pin empty if the rooms has none set. </div> - <p> - <select id="res-select"> - <option value="lowres">320x240</option> - <option value="lowres-16:9">320x180</option> - <option value="stdres" selected>640x480</option> - <option value="stdres-16:9">640x360</option> - <option value="hires-4:3">960x720</option> - <option value="hires-16:9">1280x720</option> - </select> - <input type="password" id="pin-input" placeholder="Room pin" /> - <button id="start" disabled>Start</button> - <button id="stop" disabled>Stop</button> - </p> + + <form id="room-form"> + <div> + <select id="res-select"> + <option value="lowres">320x240</option> + <option value="lowres-16:9">320x180</option> + <option value="stdres" selected>640x480</option> + <option value="stdres-16:9">640x360</option> + <option value="hires-4:3">960x720</option> + <option value="hires-16:9">1280x720</option> + </select> + </div> + + <div id="name-container"> + <input required type="text" id="name-input" placeholder="Display name" /> + </div> + + <div id="pin-container"> + <input type="password" id="pin-input" placeholder="Room pin" /> + </div> + + <div> + <button type="submit" id="start" disabled>Start</button> + <button id="stop" disabled>Stop</button> + </div> + </form> <p> <form id="bandwidth-form"> diff --git a/sender/camera-sender.js b/sender/camera-sender.js index 287cb99..eb57b9a 100644 --- a/sender/camera-sender.js +++ b/sender/camera-sender.js @@ -5,23 +5,26 @@ document.addEventListener('DOMContentLoaded', function() { var videoroomHandle = null; var sendResolution = 'stdres'; + var roomForm = document.getElementById('room-form'); var startButton = document.getElementById('start'); var stopButton = document.getElementById('stop'); var roomIndicator = document.getElementById('room-indicator'); var gotSocketInitResponse = false; var transmitting = false; - var room = 1006; + var room = 1000; var slot = 0; var token = ''; var pin = ''; var useUserPin = true; + var customNameAllowed = false; var feedId = null; parseRoomFromURL(); parseSlotFromURL(); parsePinFromURL(); parseTokenFromURL(); + parseCustomNameAllowed(); roomIndicator.innerText = `Channel ${room - 1000}, Camera ${slot + 1}`; @@ -101,7 +104,8 @@ document.addEventListener('DOMContentLoaded', function() { videoroomHandle = pluginHandle; Janus.log('Plugin attached! (' + videoroomHandle.getPlugin() + ', id=' + videoroomHandle.getId() + ')'); - startButton.onclick = function() { + roomForm.onsubmit = function(event) { + event.preventDefault(); var resSelect = document.getElementById('res-select'); startButton.setAttribute('disabled', ''); resSelect.setAttribute('disabled', ''); @@ -120,7 +124,11 @@ document.addEventListener('DOMContentLoaded', function() { }, webrtcState: function(on) { if (on) { - socket.emit('set_feed_id', { feedId }, handleSetFeedIdResponse); + var data = { feedId }; + if (customNameAllowed) { + data.customName = document.getElementById('name-input').value; + } + socket.emit('set_feed_id', data, handleSetFeedIdResponse); // Sharing camera successful, when the set_feed_id request is successful } else { janus.destroy(); @@ -138,11 +146,6 @@ document.addEventListener('DOMContentLoaded', function() { video.classList.add('visually-hidden'); } document.getElementById('preview-container').appendChild(video); - - // var noVncLink = 'https://simon-doering.com/novnc/vnc.html?room=' + room; - // var linkContainer = document.createElement('div'); - // linkContainer.innerHTML = `Camera feed can be viewed in noVNC at this link by clicking the connect button: <a href=${noVncLink}>${noVncLink}</a>`; - // document.body.appendChild(linkContainer); } Janus.attachMediaStream(document.getElementById('camera-preview'), stream); } @@ -261,7 +264,7 @@ document.addEventListener('DOMContentLoaded', function() { if (pin === 'none') { pin = ''; } - document.getElementById('pin-input').remove(); + document.getElementById('pin-container').remove(); document.getElementById('pin-hint').remove(); } else { console.log('Got no valid pin in URL search params'); @@ -276,7 +279,15 @@ document.addEventListener('DOMContentLoaded', function() { } else { console.log('Got no valid token in URL search params, using default token ' + token); } + } + function parseCustomNameAllowed() { + var urlParams = new URLSearchParams(window.location.search); + var param = urlParams.get('customNameAllowed'); + customNameAllowed = param != null; + if (!customNameAllowed) { + document.getElementById('name-container').remove(); + } } }, false); -- GitLab