/* * Copyright (C) 2019 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ 'use strict'; async function ConnectDevice(deviceId, serverConnector) { console.debug('Connect: ' + deviceId); // Prepare messages in case of connection failure let connectionAttemptDuration = 0; const intervalMs = 15000; let connectionInterval = setInterval(() => { connectionAttemptDuration += intervalMs; if (connectionAttemptDuration > 30000) { showError( 'Connection should have occurred by now. ' + 'Please attempt to restart the guest device.'); clearInterval(connectionInterval); } else if (connectionAttemptDuration > 15000) { showWarning('Connection is taking longer than expected'); } }, intervalMs); let module = await import('./cf_webrtc.js'); let deviceConnection = await module.Connect(deviceId, serverConnector); console.info('Connected to ' + deviceId); clearInterval(connectionInterval); return deviceConnection; } function setupMessages() { let closeBtn = document.querySelector('#error-message .close-btn'); closeBtn.addEventListener('click', evt => { evt.target.parentElement.className = 'hidden'; }); } function showMessage(msg, className) { let element = document.getElementById('error-message'); if (element.childNodes.length < 2) { // First time, no text node yet element.insertAdjacentText('afterBegin', msg); } else { element.childNodes[0].data = msg; } element.className = className; } function showWarning(msg) { showMessage(msg, 'warning'); } function showError(msg) { showMessage(msg, 'error'); } class DeviceDetailsUpdater { #element; constructor() { this.#element = document.getElementById('device-details-hardware'); } setHardwareDetailsText(text) { this.#element.dataset.hardwareDetailsText = text; return this; } setDeviceStateDetailsText(text) { this.#element.dataset.deviceStateDetailsText = text; return this; } update() { this.#element.textContent = [ this.#element.dataset.hardwareDetailsText, this.#element.dataset.deviceStateDetailsText, ].filter(e => e /*remove empty*/) .join('\n'); } } // DeviceDetailsUpdater class DeviceControlApp { #deviceConnection = {}; #currentRotation = 0; #displayDescriptions = []; #buttons = {}; #recording = {}; #phys = {}; #deviceCount = 0; constructor(deviceConnection) { this.#deviceConnection = deviceConnection; } start() { console.debug('Device description: ', this.#deviceConnection.description); this.#deviceConnection.onControlMessage(msg => this.#onControlMessage(msg)); createToggleControl( document.getElementById('keyboard-capture-control'), 'keyboard', enabled => this.#onKeyboardCaptureToggle(enabled)); createToggleControl( document.getElementById('mic-capture-control'), 'mic', enabled => this.#onMicCaptureToggle(enabled)); createToggleControl( document.getElementById('camera-control'), 'videocam', enabled => this.#onCameraCaptureToggle(enabled)); createToggleControl( document.getElementById('record-video-control'), 'movie_creation', enabled => this.#onVideoCaptureToggle(enabled)); const audioElm = document.getElementById('device-audio'); let audioPlaybackCtrl = createToggleControl( document.getElementById('audio-playback-control'), 'speaker', enabled => this.#onAudioPlaybackToggle(enabled), !audioElm.paused); // The audio element may start or stop playing at any time, this ensures the // audio control always show the right state. audioElm.onplay = () => audioPlaybackCtrl.Set(true); audioElm.onpause = () => audioPlaybackCtrl.Set(false); this.#showDeviceUI(); } #showDeviceUI() { window.onresize = evt => this.#resizeDeviceDisplays(); // Set up control panel buttons this.#buttons = {}; this.#buttons['power'] = createControlPanelButton( 'power', 'Power', 'power_settings_new', evt => this.#onControlPanelButton(evt)); this.#buttons['home'] = createControlPanelButton( 'home', 'Home', 'home', evt => this.#onControlPanelButton(evt)); this.#buttons['menu'] = createControlPanelButton( 'menu', 'Menu', 'menu', evt => this.#onControlPanelButton(evt)); this.#buttons['rotate'] = createControlPanelButton( 'rotate', 'Rotate', 'screen_rotation', evt => this.#onRotateButton(evt)); this.#buttons['rotate'].adb = true; this.#buttons['volumedown'] = createControlPanelButton( 'volumedown', 'Volume Down', 'volume_down', evt => this.#onControlPanelButton(evt)); this.#buttons['volumeup'] = createControlPanelButton( 'volumeup', 'Volume Up', 'volume_up', evt => this.#onControlPanelButton(evt)); createModalButton( 'device-details-button', 'device-details-modal', 'device-details-close'); createModalButton( 'bluetooth-modal-button', 'bluetooth-prompt', 'bluetooth-prompt-close'); createModalButton( 'bluetooth-prompt-wizard', 'bluetooth-wizard', 'bluetooth-wizard-close', 'bluetooth-prompt'); createModalButton( 'bluetooth-wizard-device', 'bluetooth-wizard-confirm', 'bluetooth-wizard-confirm-close', 'bluetooth-wizard'); createModalButton( 'bluetooth-wizard-another', 'bluetooth-wizard', 'bluetooth-wizard-close', 'bluetooth-wizard-confirm'); createModalButton( 'bluetooth-prompt-list', 'bluetooth-list', 'bluetooth-list-close', 'bluetooth-prompt'); createModalButton( 'bluetooth-prompt-console', 'bluetooth-console', 'bluetooth-console-close', 'bluetooth-prompt'); createModalButton( 'bluetooth-wizard-cancel', 'bluetooth-prompt', 'bluetooth-wizard-close', 'bluetooth-wizard'); positionModal('device-details-button', 'bluetooth-modal'); positionModal('device-details-button', 'bluetooth-prompt'); positionModal('device-details-button', 'bluetooth-wizard'); positionModal('device-details-button', 'bluetooth-wizard-confirm'); positionModal('device-details-button', 'bluetooth-list'); positionModal('device-details-button', 'bluetooth-console'); createButtonListener('bluetooth-prompt-list', null, this.#deviceConnection, evt => this.#onRootCanalCommand(this.#deviceConnection, "list", evt)); createButtonListener('bluetooth-wizard-device', null, this.#deviceConnection, evt => this.#onRootCanalCommand(this.#deviceConnection, "add", evt)); createButtonListener('bluetooth-list-trash', null, this.#deviceConnection, evt => this.#onRootCanalCommand(this.#deviceConnection, "del", evt)); createButtonListener('bluetooth-prompt-wizard', null, this.#deviceConnection, evt => this.#onRootCanalCommand(this.#deviceConnection, "list", evt)); createButtonListener('bluetooth-wizard-another', null, this.#deviceConnection, evt => this.#onRootCanalCommand(this.#deviceConnection, "list", evt)); if (this.#deviceConnection.description.custom_control_panel_buttons.length > 0) { document.getElementById('control-panel-custom-buttons').style.display = 'flex'; for (const button of this.#deviceConnection.description .custom_control_panel_buttons) { if (button.shell_command) { // This button's command is handled by sending an ADB shell command. this.#buttons[button.command] = createControlPanelButton( button.command, button.title, button.icon_name, e => this.#onCustomShellButton(button.shell_command, e), 'control-panel-custom-buttons'); this.#buttons[button.command].adb = true; } else if (button.device_states) { // This button corresponds to variable hardware device state(s). this.#buttons[button.command] = createControlPanelButton( button.command, button.title, button.icon_name, this.#getCustomDeviceStateButtonCb(button.device_states), 'control-panel-custom-buttons'); for (const device_state of button.device_states) { // hinge_angle is currently injected via an adb shell command that // triggers a guest binary. if ('hinge_angle_value' in device_state) { this.#buttons[button.command].adb = true; } } } else { // This button's command is handled by custom action server. this.#buttons[button.command] = createControlPanelButton( button.command, button.title, button.icon_name, evt => this.#onControlPanelButton(evt), 'control-panel-custom-buttons'); } } } // Set up displays this.#createDeviceDisplays(); // Set up audio const deviceAudio = document.getElementById('device-audio'); for (const audio_desc of this.#deviceConnection.description.audio_streams) { let stream_id = audio_desc.stream_id; this.#deviceConnection.getStream(stream_id) .then(stream => { deviceAudio.srcObject = stream; let playPromise = deviceAudio.play(); if (playPromise !== undefined) { playPromise.catch(error => { showWarning( 'Audio playback is disabled, click on the speaker control to activate it'); }); } }) .catch(e => console.error('Unable to get audio stream: ', e)); } // Set up touch input this.#startMouseTracking(); this.#updateDeviceHardwareDetails( this.#deviceConnection.description.hardware); // Show the error message and disable buttons when the WebRTC connection // fails. this.#deviceConnection.onConnectionStateChange(state => { if (state == 'disconnected' || state == 'failed') { this.#showWebrtcError(); } }); let bluetoothConsole = cmdConsole('bluetooth-console-view', 'bluetooth-console-input'); bluetoothConsole.addCommandListener(cmd => { let inputArr = cmd.split(' '); let command = inputArr[0]; inputArr.shift(); let args = inputArr; this.#deviceConnection.sendBluetoothMessage( createRootcanalMessage(command, args)); }); this.#deviceConnection.onBluetoothMessage(msg => { let decoded = decodeRootcanalMessage(msg); let deviceCount = btUpdateDeviceList(decoded); if (deviceCount > 0) { this.#deviceCount = deviceCount; createButtonListener('bluetooth-list-trash', null, this.#deviceConnection, evt => this.#onRootCanalCommand(this.#deviceConnection, "del", evt)); } btUpdateAdded(decoded); let phyList = btParsePhys(decoded); if (phyList) { this.#phys = phyList; } bluetoothConsole.addLine(decoded); }); } #onRootCanalCommand(deviceConnection, cmd, evt) { if (cmd == "list") { deviceConnection.sendBluetoothMessage(createRootcanalMessage("list", [])); } if (cmd == "del") { let id = evt.srcElement.getAttribute("data-device-id"); deviceConnection.sendBluetoothMessage(createRootcanalMessage("del", [id])); deviceConnection.sendBluetoothMessage(createRootcanalMessage("list", [])); } if (cmd == "add") { let name = document.getElementById('bluetooth-wizard-name').value; let type = document.getElementById('bluetooth-wizard-type').value; if (type == "remote_loopback") { deviceConnection.sendBluetoothMessage(createRootcanalMessage("add", [type])); } else { let mac = document.getElementById('bluetooth-wizard-mac').value; deviceConnection.sendBluetoothMessage(createRootcanalMessage("add", [type, mac])); } let phyId = this.#phys["LOW_ENERGY"].toString(); if (type == "remote_loopback") { phyId = this.#phys["BR_EDR"].toString(); } let devId = this.#deviceCount.toString(); this.#deviceCount++; deviceConnection.sendBluetoothMessage(createRootcanalMessage("add_device_to_phy", [devId, phyId])); } } #showWebrtcError() { document.getElementById('status-message').className = 'error'; document.getElementById('status-message').textContent = 'No connection to the guest device. ' + 'Please ensure the WebRTC process on the host machine is active.'; document.getElementById('status-message').style.visibility = 'visible'; const deviceDisplays = document.getElementById('device-displays'); deviceDisplays.style.display = 'none'; for (const [_, button] of Object.entries(this.#buttons)) { button.disabled = true; } } #takePhoto() { const imageCapture = this.#deviceConnection.imageCapture; if (imageCapture) { const photoSettings = { imageWidth: this.#deviceConnection.cameraWidth, imageHeight: this.#deviceConnection.cameraHeight }; imageCapture.takePhoto(photoSettings) .then(blob => blob.arrayBuffer()) .then(buffer => this.#deviceConnection.sendOrQueueCameraData(buffer)) .catch(error => console.error(error)); } } #getCustomDeviceStateButtonCb(device_states) { let states = device_states; let index = 0; return e => { if (e.type == 'mousedown') { // Reset any overridden device state. adbShell('cmd device_state state reset'); // Send a device_state message for the current state. let message = { command: 'device_state', ...states[index], }; this.#deviceConnection.sendControlMessage(JSON.stringify(message)); console.debug('Control message sent: ', JSON.stringify(message)); let lidSwitchOpen = null; if ('lid_switch_open' in states[index]) { lidSwitchOpen = states[index].lid_switch_open; } let hingeAngle = null; if ('hinge_angle_value' in states[index]) { hingeAngle = states[index].hinge_angle_value; // TODO(b/181157794): Use a custom Sensor HAL for hinge_angle // injection instead of this guest binary. adbShell( '/vendor/bin/cuttlefish_sensor_injection hinge_angle ' + states[index].hinge_angle_value); } // Update the Device Details view. this.#updateDeviceStateDetails(lidSwitchOpen, hingeAngle); // Cycle to the next state. index = (index + 1) % states.length; } } } #resizeDeviceDisplays() { // Padding between displays. const deviceDisplayWidthPadding = 10; // Padding for the display info above each display video. const deviceDisplayHeightPadding = 38; let deviceDisplayList = document.getElementsByClassName('device-display'); let deviceDisplayVideoList = document.getElementsByClassName('device-display-video'); let deviceDisplayInfoList = document.getElementsByClassName('device-display-info'); const deviceDisplays = document.getElementById('device-displays'); const rotationDegrees = this.#getTransformRotation(deviceDisplays); const rotationRadians = rotationDegrees * Math.PI / 180; // Auto-scale the screen based on window size. let availableWidth = deviceDisplays.clientWidth; let availableHeight = deviceDisplays.clientHeight - deviceDisplayHeightPadding; // Reserve space for padding between the displays. availableWidth = availableWidth - (this.#displayDescriptions.length * deviceDisplayWidthPadding); // Loop once over all of the displays to compute the total space needed. let neededWidth = 0; let neededHeight = 0; for (let i = 0; i < deviceDisplayList.length; i++) { let deviceDisplayDescription = this.#displayDescriptions[i]; let deviceDisplayVideo = deviceDisplayVideoList[i]; const originalDisplayWidth = deviceDisplayDescription.x_res; const originalDisplayHeight = deviceDisplayDescription.y_res; const neededBoundingBoxWidth = Math.abs(Math.cos(rotationRadians) * originalDisplayWidth) + Math.abs(Math.sin(rotationRadians) * originalDisplayHeight); const neededBoundingBoxHeight = Math.abs(Math.sin(rotationRadians) * originalDisplayWidth) + Math.abs(Math.cos(rotationRadians) * originalDisplayHeight); neededWidth = neededWidth + neededBoundingBoxWidth; neededHeight = Math.max(neededHeight, neededBoundingBoxHeight); } const scaling = Math.min(availableWidth / neededWidth, availableHeight / neededHeight); // Loop again over all of the displays to set the sizes and positions. let deviceDisplayLeftOffset = 0; for (let i = 0; i < deviceDisplayList.length; i++) { let deviceDisplay = deviceDisplayList[i]; let deviceDisplayVideo = deviceDisplayVideoList[i]; let deviceDisplayInfo = deviceDisplayInfoList[i]; let deviceDisplayDescription = this.#displayDescriptions[i]; let rotated = this.#currentRotation == 1 ? ' (Rotated)' : ''; deviceDisplayInfo.textContent = `Display ${i} - ` + `${deviceDisplayDescription.x_res}x` + `${deviceDisplayDescription.y_res} ` + `(${deviceDisplayDescription.dpi} DPI)${rotated}`; const originalDisplayWidth = deviceDisplayDescription.x_res; const originalDisplayHeight = deviceDisplayDescription.y_res; const scaledDisplayWidth = originalDisplayWidth * scaling; const scaledDisplayHeight = originalDisplayHeight * scaling; const neededBoundingBoxWidth = Math.abs(Math.cos(rotationRadians) * originalDisplayWidth) + Math.abs(Math.sin(rotationRadians) * originalDisplayHeight); const neededBoundingBoxHeight = Math.abs(Math.sin(rotationRadians) * originalDisplayWidth) + Math.abs(Math.cos(rotationRadians) * originalDisplayHeight); const scaledBoundingBoxWidth = neededBoundingBoxWidth * scaling; const scaledBoundingBoxHeight = neededBoundingBoxHeight * scaling; const offsetX = (scaledBoundingBoxWidth - scaledDisplayWidth) / 2; const offsetY = (scaledBoundingBoxHeight - scaledDisplayHeight) / 2; deviceDisplayVideo.style.width = scaledDisplayWidth; deviceDisplayVideo.style.height = scaledDisplayHeight; deviceDisplayVideo.style.transform = `translateX(${offsetX}px) ` + `translateY(${offsetY}px) ` + `rotateZ(${rotationDegrees}deg) `; deviceDisplay.style.left = `${deviceDisplayLeftOffset}px`; deviceDisplay.style.width = scaledBoundingBoxWidth; deviceDisplay.style.height = scaledBoundingBoxHeight; deviceDisplayLeftOffset = deviceDisplayLeftOffset + deviceDisplayWidthPadding + scaledBoundingBoxWidth; } } #getTransformRotation(element) { if (!element.style.textIndent) { return 0; } // Remove 'px' and convert to float. return parseFloat(element.style.textIndent.slice(0, -2)); } #onControlMessage(message) { let message_data = JSON.parse(message.data); console.debug('Control message received: ', message_data) let metadata = message_data.metadata; if (message_data.event == 'VIRTUAL_DEVICE_BOOT_STARTED') { // Start the adb connection after receiving the BOOT_STARTED message. // (This is after the adbd start message. Attempting to connect // immediately after adbd starts causes issues.) this.#initializeAdb(); } if (message_data.event == 'VIRTUAL_DEVICE_SCREEN_CHANGED') { if (metadata.rotation != this.#currentRotation) { // Animate the screen rotation. const targetRotation = metadata.rotation == 0 ? 0 : -90; $('#device-displays') .animate( { textIndent: targetRotation, }, { duration: 1000, step: (now, tween) => { this.#resizeDeviceDisplays(); }, }); } this.#currentRotation = metadata.rotation; } if (message_data.event == 'VIRTUAL_DEVICE_CAPTURE_IMAGE') { if (this.#deviceConnection.cameraEnabled) { this.#takePhoto(); } } if (message_data.event == 'VIRTUAL_DEVICE_DISPLAY_POWER_MODE_CHANGED') { this.#updateDisplayVisibility(metadata.display, metadata.mode); } } #updateDeviceStateDetails(lidSwitchOpen, hingeAngle) { let deviceStateDetailsTextLines = []; if (lidSwitchOpen != null) { let state = lidSwitchOpen ? 'Opened' : 'Closed'; deviceStateDetailsTextLines.push(`Lid Switch - ${state}`); } if (hingeAngle != null) { deviceStateDetailsTextLines.push(`Hinge Angle - ${hingeAngle}`); } let deviceStateDetailsText = deviceStateDetailsTextLines.join('\n'); new DeviceDetailsUpdater() .setDeviceStateDetailsText(deviceStateDetailsText) .update(); } #updateDeviceHardwareDetails(hardware) { let hardwareDetailsTextLines = []; Object.keys(hardware).forEach((key) => { let value = hardware[key]; hardwareDetailsTextLines.push(`${key} - ${value}`); }); let hardwareDetailsText = hardwareDetailsTextLines.join('\n'); new DeviceDetailsUpdater() .setHardwareDetailsText(hardwareDetailsText) .update(); } // Creates a