From 8dc1368bf1c11a80523893175bb427cc66d7ffc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?A=CC=81lvaro=20Velad=20Galva=CC=81n?= Date: Tue, 16 Apr 2024 17:19:39 +0200 Subject: [PATCH] feat(UI): UI support for VR content --- build/types/ui | 6 + demo/common/assets.js | 27 ++ demo/main.js | 8 + demo/search.js | 2 + docs/tutorials/ui.md | 19 + externs/device_sensor_event.js | 18 + lib/util/dom_utils.js | 11 + roadmap.md | 1 + shaka-player.uncompiled.js | 5 + test/test/util/ui_utils.js | 17 +- test/text/text_displayer_layout_unit.js | 3 +- test/ui/ad_ui_unit.js | 5 +- test/ui/ui_customization_unit.js | 27 +- test/ui/ui_integration.js | 6 +- test/ui/ui_unit.js | 30 +- ui/controls.js | 102 +++- ui/externs/ui.js | 6 +- ui/gl_matrix/matrix_4x4.js | 581 ++++++++++++++++++++++ ui/gl_matrix/matrix_quaternion.js | 45 ++ ui/less/containers.less | 11 + ui/ui.js | 81 +++- ui/vr_manager.js | 496 +++++++++++++++++++ ui/vr_utils.js | 115 +++++ ui/vr_webgl.js | 611 ++++++++++++++++++++++++ 24 files changed, 2185 insertions(+), 48 deletions(-) create mode 100644 externs/device_sensor_event.js create mode 100644 ui/gl_matrix/matrix_4x4.js create mode 100644 ui/gl_matrix/matrix_quaternion.js create mode 100644 ui/vr_manager.js create mode 100644 ui/vr_utils.js create mode 100644 ui/vr_webgl.js diff --git a/build/types/ui b/build/types/ui index 547498ad49..cff656634a 100644 --- a/build/types/ui +++ b/build/types/ui @@ -41,3 +41,9 @@ +../../ui/ui.js +../../ui/ui_utils.js +../../ui/volume_bar.js ++../../ui/vr_manager.js ++../../ui/vr_utils.js ++../../ui/vr_webgl.js + ++../../ui/gl_matrix/matrix_4x4.js ++../../ui/gl_matrix/matrix_quaternion.js diff --git a/demo/common/assets.js b/demo/common/assets.js index 11d8428d9f..49e460ffe5 100644 --- a/demo/common/assets.js +++ b/demo/common/assets.js @@ -43,6 +43,7 @@ shakaAssets.Source = { BRIGHTCOVE: 'Brightcove', BROADPEAK: 'Broadpeak', EZDRM: 'EZDRM', + THEO_PLAYER: 'THEOplayer', }; @@ -165,6 +166,9 @@ shakaAssets.Feature = { // Set if the asset has Content Steering. CONTENT_STEERING: 'Content Steering', + + // Set if the asset is VR. + VR: 'VR', }; @@ -1062,6 +1066,15 @@ shakaAssets.testAssets = [ .addFeature(shakaAssets.Feature.MP4) .addFeature(shakaAssets.Feature.THUMBNAILS) .addExtraThumbnail('https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/thumbnails/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.vtt'), + new ShakaDemoAssetInfo( + /* name= */ 'VR Playhouse (DASH, VR)', + /* iconUri= */ 'https://cdn.bitmovin.com/content/assets/playhouse-vr/poster.jpg', + /* manifestUri= */ 'https://cdn.bitmovin.com/content/assets/playhouse-vr/mpds/105560.mpd', + /* source= */ shakaAssets.Source.BITCODIN) + .addFeature(shakaAssets.Feature.DASH) + .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION) + .addFeature(shakaAssets.Feature.MP4) + .addFeature(shakaAssets.Feature.VR), // End bitcodin assets }}} // MetaCDN assets {{{ @@ -1679,5 +1692,19 @@ shakaAssets.testAssets = [ }, }), // }}} + + // THEOplayer assets {{{ + /* THEOplayer Contents */ + new ShakaDemoAssetInfo( + /* name= */ 'National Geographic (HLS, VR)', + /* iconUri= */ 'https://demo.theoplayer.com/hubfs/videos/natgeo/poster.jpg', + /* manifestUri= */ 'https://demo.theoplayer.com/hubfs/videos/natgeo/playlist.m3u8', + /* source= */ shakaAssets.Source.THEO_PLAYER) + .addFeature(shakaAssets.Feature.HLS) + .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION) + .addFeature(shakaAssets.Feature.MP2TS) + .addFeature(shakaAssets.Feature.VR) + .addFeature(shakaAssets.Feature.OFFLINE), + // }}} ]; /* eslint-enable max-len */ diff --git a/demo/main.js b/demo/main.js index da16b0ae0d..edf86e871d 100644 --- a/demo/main.js +++ b/demo/main.js @@ -1294,6 +1294,14 @@ shakaDemo.Main = class { await this.drmConfiguration_(asset); this.controls_.getCastProxy().setAppData({'asset': asset}); + const uiConfig = { + displayInVrMode: false, + }; + if (asset.features.includes(shakaAssets.Feature.VR)) { + uiConfig.displayInVrMode = true; + } + const ui = this.video_['ui']; + ui.configure(uiConfig); // Finally, the asset can be loaded. if (asset.preloadManager) { diff --git a/demo/search.js b/demo/search.js index 4ea846b63a..9d8d59f3d1 100644 --- a/demo/search.js +++ b/demo/search.js @@ -397,6 +397,8 @@ shakaDemo.Search = class { 'Filters for assets that have an LCEVC enhancement layer.'); this.makeBooleanInput_(specialContainer, Feature.CONTENT_STEERING, FEATURE, 'Filters for assets that use Content Steering.'); + this.makeBooleanInput_(specialContainer, Feature.VR, FEATURE, + 'Filters for assets that are VR.'); container.appendChild(this.resultsDiv_); } diff --git a/docs/tutorials/ui.md b/docs/tutorials/ui.md index fde77b3942..619f53761a 100644 --- a/docs/tutorials/ui.md +++ b/docs/tutorials/ui.md @@ -100,6 +100,25 @@ document.addEventListener('shaka-ui-load-failed', initFailed); ``` +#### Enabling VR + +To enable the playback of VR content, there are two possibilities: + +1. Enable via UI config: +```js +const config = { + 'displayInVrMode': true +} +ui.configure(config); +``` + +2. Using HLS or DASH with fMP4 segments and in the init segment exists 'prji' + and 'hfov' boxes. + + +Note: VR is only supported for DASH/HLS clear streams or HLS-AES stream. + + #### Enabling Chromecast support If you'd like to take advantage of Shaka's built-in Chromecast support, diff --git a/externs/device_sensor_event.js b/externs/device_sensor_event.js new file mode 100644 index 0000000000..15d1de2c33 --- /dev/null +++ b/externs/device_sensor_event.js @@ -0,0 +1,18 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @fileoverview Externs for DeviceMotionEvent which were missing in the + * Closure compiler. + * + * @externs + */ + + +/** + * @return {!Promise.} + */ +DeviceMotionEvent.requestPermission = function() {}; diff --git a/lib/util/dom_utils.js b/lib/util/dom_utils.js index 593e1d0c97..eb85fb669c 100644 --- a/lib/util/dom_utils.js +++ b/lib/util/dom_utils.js @@ -53,6 +53,17 @@ shaka.util.Dom = class { } + /** + * Cast a Node/Element to an HTMLCanvasElement + * + * @param {!Node|!Element} original + * @return {!HTMLCanvasElement} + */ + static asHTMLCanvasElement(original) { + return /** @type {!HTMLCanvasElement}*/ (original); + } + + /** * Cast a Node/Element to an HTMLMediaElement * diff --git a/roadmap.md b/roadmap.md index 01c8c23493..4b23b5eee2 100644 --- a/roadmap.md +++ b/roadmap.md @@ -25,6 +25,7 @@ v5.0 - 2024 Q1 - AES-256 and AES-256-CTR (HLS) https://github.com/shaka-project/shaka-player/issues/6001 - Detect maximum HW resolution automatically on some platforms + - UI support for VR content ===== diff --git a/shaka-player.uncompiled.js b/shaka-player.uncompiled.js index 3450848302..be8f0668db 100644 --- a/shaka-player.uncompiled.js +++ b/shaka-player.uncompiled.js @@ -95,6 +95,8 @@ goog.require('shaka.ui.FastForwardButton'); goog.require('shaka.ui.FullscreenButton'); goog.require('shaka.ui.Localization'); goog.require('shaka.ui.LoopButton'); +goog.require('shaka.ui.Matrix4x4'); +goog.require('shaka.ui.MatrixQuaternion'); goog.require('shaka.ui.MuteButton'); goog.require('shaka.ui.Overlay'); goog.require('shaka.ui.PipButton'); @@ -109,6 +111,9 @@ goog.require('shaka.ui.Spacer'); goog.require('shaka.ui.StatisticsButton'); goog.require('shaka.ui.TextSelection'); goog.require('shaka.ui.VolumeBar'); +goog.require('shaka.ui.VRManager'); +goog.require('shaka.ui.VRUtils'); +goog.require('shaka.ui.VRWebgl'); goog.require('shaka.util.Dom'); goog.require('shaka.util.Error'); goog.require('shaka.util.FairPlayUtils'); diff --git a/test/test/util/ui_utils.js b/test/test/util/ui_utils.js index c413a93c90..9cc17f3a2f 100644 --- a/test/test/util/ui_utils.js +++ b/test/test/util/ui_utils.js @@ -8,15 +8,16 @@ shaka.test.UiUtils = class { /** * @param {!HTMLElement} videoContainer * @param {!HTMLMediaElement} video + * @param {!HTMLCanvasElement} canvas * @param {!Object=} config * @return {!Promise.} */ - static async createUIThroughAPI(videoContainer, video, config) { + static async createUIThroughAPI(videoContainer, video, canvas, config) { const player = new shaka.Player(); await player.attach(video); // Create UI config = config || {}; - const ui = new shaka.ui.Overlay(player, videoContainer, video); + const ui = new shaka.ui.Overlay(player, videoContainer, video, canvas); // TODO: generate externs automatically from @event types // This event should be a shaka.Player.ErrorEvent ui.getControls().addEventListener('error', (e) => fail(e['detail'])); @@ -180,4 +181,16 @@ shaka.test.UiUtils = class { return video; } + + /** + * Creates a canvas element for testing. + * + * @return {!HTMLCanvasElement} + */ + static createCanvasElement() { + const canvas = /** @type {!HTMLCanvasElement} */(document.createElement( + 'canvas')); + + return canvas; + } }; diff --git a/test/text/text_displayer_layout_unit.js b/test/text/text_displayer_layout_unit.js index b6e1d01a7b..20fc72aed9 100644 --- a/test/text/text_displayer_layout_unit.js +++ b/test/text/text_displayer_layout_unit.js @@ -40,7 +40,8 @@ filterDescribe('Cue layout', shaka.test.TextLayoutTests.supported, () => { const player = new shaka.Player(); ui = new shaka.ui.Overlay( player, /** @type {!HTMLElement} */(helper.videoContainer), - shaka.test.UiUtils.createVideoElement()); + shaka.test.UiUtils.createVideoElement(), + shaka.test.UiUtils.createCanvasElement()); // Turn off every part of the UI that we can, so that the screenshot is // less likey to change because of something unrelated to text // rendering. diff --git a/test/ui/ad_ui_unit.js b/test/ui/ad_ui_unit.js index bd51150656..e10ec570b1 100644 --- a/test/ui/ad_ui_unit.js +++ b/test/ui/ad_ui_unit.js @@ -12,6 +12,8 @@ describe('Ad UI', () => { let container; /** @type {!HTMLMediaElement} */ let video; + /** @type {!HTMLCanvasElement} */ + let canvas; /** @type {!shaka.test.FakeAd} */ let ad; /** @type {!shaka.test.FakeAdManager} */ @@ -32,7 +34,8 @@ describe('Ad UI', () => { video = shaka.test.UiUtils.createVideoElement(); container.appendChild(video); - await UiUtils.createUIThroughAPI(container, video); + canvas = shaka.test.UiUtils.createCanvasElement(); + await UiUtils.createUIThroughAPI(container, video, canvas); adManager = video['ui'].getControls().getPlayer().getAdManager(); }); diff --git a/test/ui/ui_customization_unit.js b/test/ui/ui_customization_unit.js index e580d59748..7433d58137 100644 --- a/test/ui/ui_customization_unit.js +++ b/test/ui/ui_customization_unit.js @@ -12,6 +12,8 @@ describe('UI Customization', () => { let container; /** @type {!HTMLMediaElement} */ let video; + /** @type {!HTMLCanvasElement} */ + let canvas; beforeAll(async () => { // Add css file @@ -34,11 +36,13 @@ describe('UI Customization', () => { video = shaka.test.UiUtils.createVideoElement(); container.appendChild(video); + canvas = shaka.test.UiUtils.createCanvasElement(); + container.appendChild(canvas); }); it('only the specified controls are created', async () => { const config = {controlPanelElements: ['time_and_duration', 'mute']}; - await UiUtils.createUIThroughAPI(container, video, config); + await UiUtils.createUIThroughAPI(container, video, canvas, config); // Only current time and mute button should've been created UiUtils.confirmElementFound(container, 'shaka-current-time'); @@ -51,7 +55,7 @@ describe('UI Customization', () => { it('only the specified overflow menu buttons are created', async () => { const config = {overflowMenuButtons: ['cast']}; - await UiUtils.createUIThroughAPI(container, video, config); + await UiUtils.createUIThroughAPI(container, video, canvas, config); UiUtils.confirmElementFound(container, 'shaka-cast-button'); @@ -60,30 +64,31 @@ describe('UI Customization', () => { it('seek bar only created when configured', async () => { const ui = await UiUtils.createUIThroughAPI( - container, video, {addSeekBar: false}); + container, video, canvas, {addSeekBar: false}); UiUtils.confirmElementMissing(container, 'shaka-seek-bar'); await ui.destroy(); - await UiUtils.createUIThroughAPI(container, video, {addSeekBar: true}); + await UiUtils.createUIThroughAPI( + container, video, canvas, {addSeekBar: true}); UiUtils.confirmElementFound(container, 'shaka-seek-bar'); }); it('big play button only created when configured', async () => { const ui = await UiUtils.createUIThroughAPI( - container, video, {addBigPlayButton: false}); + container, video, canvas, {addBigPlayButton: false}); UiUtils.confirmElementMissing(container, 'shaka-play-button-container'); UiUtils.confirmElementMissing(container, 'shaka-play-button'); await ui.destroy(); await UiUtils.createUIThroughAPI( - container, video, {addBigPlayButton: true}); + container, video, canvas, {addBigPlayButton: true}); UiUtils.confirmElementFound(container, 'shaka-play-button-container'); UiUtils.confirmElementFound(container, 'shaka-play-button'); }); it('settings menus are lower when seek bar is absent', async () => { const config = {addSeekBar: false}; - await UiUtils.createUIThroughAPI(container, video, config); + await UiUtils.createUIThroughAPI(container, video, canvas, config); function confirmLowPosition(className) { const elements = @@ -111,7 +116,7 @@ describe('UI Customization', () => { ], }; - await UiUtils.createUIThroughAPI(container, video, config); + await UiUtils.createUIThroughAPI(container, video, canvas, config); const controlsButtonPanels = container.getElementsByClassName('shaka-controls-button-panel'); @@ -132,7 +137,8 @@ describe('UI Customization', () => { it('layout can be re-configured after the creation', async () => { const config = {controlPanelElements: ['time_and_duration', 'mute']}; - const ui = await UiUtils.createUIThroughAPI(container, video, config); + const ui = await UiUtils.createUIThroughAPI( + container, video, canvas, config); // Only current time and mute button should've been created UiUtils.confirmElementFound(container, 'shaka-current-time'); @@ -178,7 +184,8 @@ describe('UI Customization', () => { it('cast proxy and controls are unchanged by reconfiguration', async () => { const config = {controlPanelElements: ['time_and_duration', 'mute']}; /** @type {!shaka.ui.Overlay} */ - const ui = await UiUtils.createUIThroughAPI(container, video, config); + const ui = await UiUtils.createUIThroughAPI( + container, video, canvas, config); const eventManager = new shaka.util.EventManager(); const waiter = new shaka.test.Waiter(eventManager); diff --git a/test/ui/ui_integration.js b/test/ui/ui_integration.js index 8b29b16cf2..eeba3e8c58 100644 --- a/test/ui/ui_integration.js +++ b/test/ui/ui_integration.js @@ -13,6 +13,8 @@ describe('UI', () => { /** @type {!HTMLVideoElement} */ let video; + /** @type {!HTMLCanvasElement} */ + let canvas; /** @type {!HTMLElement} */ let videoContainer; /** @type {!shaka.Player} */ @@ -41,9 +43,11 @@ describe('UI', () => { beforeEach(async () => { video = shaka.test.UiUtils.createVideoElement(); + canvas = shaka.test.UiUtils.createCanvasElement(); videoContainer = shaka.util.Dom.createHTMLElement('div'); videoContainer.appendChild(video); + videoContainer.appendChild(canvas); document.body.appendChild(videoContainer); player = new compiledShaka.Player(); await player.attach(video); @@ -70,7 +74,7 @@ describe('UI', () => { // TODO: Cast receiver id to test chromecast integration }; - ui = new compiledShaka.ui.Overlay(player, videoContainer, video); + ui = new compiledShaka.ui.Overlay(player, videoContainer, video, canvas); ui.configure(config); // Grab event manager from the uncompiled library: diff --git a/test/ui/ui_unit.js b/test/ui/ui_unit.js index d993000c95..25d984108a 100644 --- a/test/ui/ui_unit.js +++ b/test/ui/ui_unit.js @@ -35,6 +35,8 @@ describe('UI', () => { let videoContainer; /** @type {!HTMLVideoElement} */ let video; + /** @type {!HTMLCanvasElement} */ + let canvas; beforeEach(async () => { videoContainer = @@ -43,7 +45,10 @@ describe('UI', () => { video = shaka.test.UiUtils.createVideoElement(); videoContainer.appendChild(video); - await UiUtils.createUIThroughAPI(videoContainer, video); + + canvas = shaka.test.UiUtils.createCanvasElement(); + videoContainer.appendChild(canvas); + await UiUtils.createUIThroughAPI(videoContainer, video, canvas); }); it('has all the basic elements', () => { @@ -169,6 +174,8 @@ describe('UI', () => { let videoContainer; /** @type {!HTMLVideoElement} */ let video; + /** @type {!HTMLCanvasElement} */ + let canvas; beforeEach(() => { videoContainer = @@ -177,6 +184,8 @@ describe('UI', () => { video = shaka.test.UiUtils.createVideoElement(); videoContainer.appendChild(video); + canvas = shaka.test.UiUtils.createCanvasElement(); + videoContainer.appendChild(canvas); }); it('goes into fullscreen on double click', async () => { @@ -194,7 +203,7 @@ describe('UI', () => { doubleClickForFullscreen: false, }; const ui = await UiUtils.createUIThroughAPI( - videoContainer, video, config); + videoContainer, video, canvas, config); const controls = ui.getControls(); const spy = spyOn(controls, 'toggleFullScreen'); @@ -218,7 +227,8 @@ describe('UI', () => { let controlsContainer; beforeEach(async () => { - const ui = await UiUtils.createUIThroughAPI(videoContainer, video); + const ui = await UiUtils.createUIThroughAPI( + videoContainer, video, canvas); player = ui.getControls().getLocalPlayer(); const controlsContainers = videoContainer.getElementsByClassName('shaka-controls-container'); @@ -254,7 +264,7 @@ describe('UI', () => { ], }; const ui = await UiUtils.createUIThroughAPI( - videoContainer, video, config); + videoContainer, video, canvas, config); player = ui.getControls().getLocalPlayer(); const overflowMenus = @@ -326,7 +336,7 @@ describe('UI', () => { let controlsButtonPanel; it('has default elements', async () => { - await UiUtils.createUIThroughAPI(videoContainer, video); + await UiUtils.createUIThroughAPI(videoContainer, video, canvas); const controlsButtonPanels = videoContainer.getElementsByClassName( 'shaka-controls-button-panel'); @@ -378,7 +388,7 @@ describe('UI', () => { ], }; - await UiUtils.createUIThroughAPI(videoContainer, video, config); + await UiUtils.createUIThroughAPI(videoContainer, video, canvas, config); const controlsButtonPanels = videoContainer.getElementsByClassName( 'shaka-controls-button-panel'); expect(controlsButtonPanels.length).toBe(1); @@ -413,7 +423,7 @@ describe('UI', () => { ], }; const ui = await UiUtils.createUIThroughAPI( - videoContainer, video, config); + videoContainer, video, canvas, config); player = ui.getControls().getLocalPlayer(); const resolutionsMenus = @@ -486,7 +496,7 @@ describe('UI', () => { ], }; const ui = await UiUtils.createUIThroughAPI( - videoContainer, video, config); + videoContainer, video, canvas, config); controls = ui.getControls(); player = controls.getLocalPlayer(); @@ -710,7 +720,7 @@ describe('UI', () => { ], }; const ui = await UiUtils.createUIThroughAPI( - videoContainer, video, config); + videoContainer, video, canvas, config); controlsContainer = ui.getControls().getControlsContainer(); @@ -759,7 +769,7 @@ describe('UI', () => { statisticsList: Object.keys(new shaka.util.Stats().getBlob()), }; const ui = await UiUtils.createUIThroughAPI( - videoContainer, video, config); + videoContainer, video, canvas, config); player = ui.getControls().getLocalPlayer(); const statisticsButtons = diff --git a/ui/controls.js b/ui/controls.js index 4456c73f21..538a088d0c 100644 --- a/ui/controls.js +++ b/ui/controls.js @@ -22,6 +22,7 @@ goog.require('shaka.ui.Locales'); goog.require('shaka.ui.Localization'); goog.require('shaka.ui.SeekBar'); goog.require('shaka.ui.Utils'); +goog.require('shaka.ui.VRManager'); goog.require('shaka.util.Dom'); goog.require('shaka.util.EventManager'); goog.require('shaka.util.FakeEvent'); @@ -42,9 +43,10 @@ shaka.ui.Controls = class extends shaka.util.FakeEventTarget { * @param {!shaka.Player} player * @param {!HTMLElement} videoContainer * @param {!HTMLMediaElement} video + * @param {!HTMLCanvasElement} vrCanvas * @param {shaka.extern.UIConfiguration} config */ - constructor(player, videoContainer, video, config) { + constructor(player, videoContainer, video, vrCanvas, config) { super(); /** @private {boolean} */ @@ -76,6 +78,8 @@ shaka.ui.Controls = class extends shaka.util.FakeEventTarget { /** @private {!HTMLElement} */ this.videoContainer_ = videoContainer; + this.vrCanvas_ = vrCanvas; + /** @private {shaka.extern.IAdManager} */ this.adManager_ = this.player_.getAdManager(); @@ -173,6 +177,12 @@ shaka.ui.Controls = class extends shaka.util.FakeEventTarget { this.configure(this.config_); this.addEventListeners_(); + goog.asserts.assert( + this.controlsContainer_, 'Controls container must be created!'); + + this.vr_ = new shaka.ui.VRManager(this.controlsContainer_, this.vrCanvas_, + this.localVideo_, this.player_, this.config_); + /** * The pressed keys set is used to record which keys are currently pressed * down, so we can know what keys are pressed at the same time. @@ -257,6 +267,11 @@ shaka.ui.Controls = class extends shaka.util.FakeEventTarget { this.localization_ = null; this.pressedKeys_.clear(); + if (this.vr_) { + this.vr_.release(); + this.vr_ = null; + } + // FakeEventTarget implements IReleasable super.release(); } @@ -334,6 +349,10 @@ shaka.ui.Controls = class extends shaka.util.FakeEventTarget { this.contextMenu_ = null; } + if (this.vr_) { + this.vr_.configure(config); + } + if (this.controlsContainer_) { shaka.util.Dom.removeAllChildren(this.controlsContainer_); this.releaseChildElements_(); @@ -1384,7 +1403,7 @@ shaka.ui.Controls = class extends shaka.util.FakeEventTarget { /** @private */ onContainerClick_() { - if (!this.enabled_) { + if (!this.enabled_ || this.vr_.isPlayingVR()) { return; } @@ -1717,6 +1736,85 @@ shaka.ui.Controls = class extends shaka.util.FakeEventTarget { this.onMouseLeave_(); } + /** + * Returns if a VR is supported. + * + * @return {boolean} + * @export + */ + isPlayingVR() { + return this.vr_.isPlayingVR(); + } + + /** + * Get the angle of the north. + * + * @return {?number} + * @export + */ + getVRNorth() { + return this.vr_.getNorth(); + } + + /** + * Returns the field view. + * + * @return {?number} + * @export + */ + getVRFieldView() { + return this.vr_.getFieldView(); + } + + /** + * Set the field view. + * + * @param {number} fieldView + * @export + */ + setVRFieldView(fieldView) { + this.vr_.setFieldView(fieldView); + } + + /** + * Toggle stereoscopic mode. + * + * @export + */ + toggleStereoscopicMode() { + this.vr_.toggleStereoscopicMode(); + } + + /** + * Increment the yaw in X angle in degrees. + * + * @param {number} angle + * @export + */ + incrementYaw(angle) { + this.vr_.incrementYaw(angle); + } + + /** + * Increment the pitch in X angle in degrees. + * + * @param {number} angle + * @export + */ + incrementPitch(angle) { + this.vr_.incrementPitch(angle); + } + + /** + * Increment the roll in X angle in degrees. + * + * @param {number} angle + * @export + */ + incrementRoll(angle) { + this.vr_.incrementRoll(angle); + } + /** * Create a localization instance already pre-loaded with all the locales that * we support. diff --git a/ui/externs/ui.js b/ui/externs/ui.js index 6de7e016bb..1084360a75 100644 --- a/ui/externs/ui.js +++ b/ui/externs/ui.js @@ -93,7 +93,8 @@ shaka.extern.UIVolumeBarColors; * showAudioChannelCountVariants: boolean, * seekOnTaps: boolean, * tapSeekDistance: number, - * refreshTickInSeconds: number + * refreshTickInSeconds: number, + * displayInVrMode: boolean * }} * * @property {!Array.} controlPanelElements @@ -232,6 +233,9 @@ shaka.extern.UIVolumeBarColors; * @property {number} refreshTickInSeconds * The time interval, in seconds, to update the seek bar. * Defaults to 0.125 seconds. + * @property {boolean} displayInVrMode + * Indicates whether or not the content should be rendered as VR. + * Defaults to false. * @exportDoc */ shaka.extern.UIConfiguration; diff --git a/ui/gl_matrix/matrix_4x4.js b/ui/gl_matrix/matrix_4x4.js new file mode 100644 index 0000000000..ca67b90b4d --- /dev/null +++ b/ui/gl_matrix/matrix_4x4.js @@ -0,0 +1,581 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + + +goog.provide('shaka.ui.Matrix4x4'); + +/** + * 4x4 Matrix + * Format: column-major, when typed out it looks like row-major + * The matrices are being post multiplied. + */ +shaka.ui.Matrix4x4 = class { + /** + * Creates a new identity 4x4 Matrix + * + * @return {!Float32Array} a new 4x4 matrix + */ + static create() { + const out = new Float32Array(16); + out[0] = 1; + out[5] = 1; + out[10] = 1; + out[15] = 1; + return out; + } + + /** + * Generates a look-at matrix with the given eye position, focal point, and + * up axis. + * + * @param {!Float32Array} out 4x4 matrix frustum matrix will be written into + * @param {!Array.} eye Position of the viewer + * @param {!Array.} center Point the viewer is looking at + * @param {!Array.} up Vector pointing up + */ + static lookAt(out, eye, center, up) { + let x0; + let x1; + let x2; + let y0; + let y1; + let y2; + let z0; + let z1; + let z2; + let len; + const eyex = eye[0]; + const eyey = eye[1]; + const eyez = eye[2]; + const upx = up[0]; + const upy = up[1]; + const upz = up[2]; + const centerx = center[0]; + const centery = center[1]; + const centerz = center[2]; + + if (Math.abs(eyex - centerx) < shaka.ui.Matrix4x4.EPSILON_ && + Math.abs(eyey - centery) < shaka.ui.Matrix4x4.EPSILON_ && + Math.abs(eyez - centerz) < shaka.ui.Matrix4x4.EPSILON_) { + return shaka.ui.Matrix4x4.identity_(out); + } + + z0 = eyex - centerx; + z1 = eyey - centery; + z2 = eyez - centerz; + + len = 1 / Math.sqrt(z0 * z0 + z1 * z1 + z2 * z2); + z0 *= len; + z1 *= len; + z2 *= len; + + x0 = upy * z2 - upz * z1; + x1 = upz * z0 - upx * z2; + x2 = upx * z1 - upy * z0; + len = Math.sqrt(x0 * x0 + x1 * x1 + x2 * x2); + if (!len) { + x0 = 0; + x1 = 0; + x2 = 0; + } else { + len = 1 / len; + x0 *= len; + x1 *= len; + x2 *= len; + } + + y0 = z1 * x2 - z2 * x1; + y1 = z2 * x0 - z0 * x2; + y2 = z0 * x1 - z1 * x0; + + len = Math.sqrt(y0 * y0 + y1 * y1 + y2 * y2); + if (!len) { + y0 = 0; + y1 = 0; + y2 = 0; + } else { + len = 1 / len; + y0 *= len; + y1 *= len; + y2 *= len; + } + + out[0] = x0; + out[1] = y0; + out[2] = z0; + out[3] = 0; + out[4] = x1; + out[5] = y1; + out[6] = z1; + out[7] = 0; + out[8] = x2; + out[9] = y2; + out[10] = z2; + out[11] = 0; + out[12] = -(x0 * eyex + x1 * eyey + x2 * eyez); + out[13] = -(y0 * eyex + y1 * eyey + y2 * eyez); + out[14] = -(z0 * eyex + z1 * eyey + z2 * eyez); + out[15] = 1; + + return out; + } + + /** + * Scales the 4x4 matrix by the dimensions in the given vector not using + * vectorization + * + * @param {!Float32Array} out the receiving matrix + * @param {!Float32Array} a the matrix to scale + * @param {!Array.} v the vector to scale the matrix by + */ + static scale(out, a, v) { + const x = v[0]; + const y = v[1]; + const z = v[2]; + + out[0] = a[0] * x; + out[1] = a[1] * x; + out[2] = a[2] * x; + out[3] = a[3] * x; + out[4] = a[4] * y; + out[5] = a[5] * y; + out[6] = a[6] * y; + out[7] = a[7] * y; + out[8] = a[8] * z; + out[9] = a[9] * z; + out[10] = a[10] * z; + out[11] = a[11] * z; + out[12] = a[12]; + out[13] = a[13]; + out[14] = a[14]; + out[15] = a[15]; + } + + /** + * Generates a perspective projection matrix with the given bounds. + * The near/far clip planes correspond to a normalized device coordinate Z + * range of [-1, 1], which matches WebGL's clip volume. + * Passing null/undefined/no value for far will generate infinite projection + * matrix. + * + * @param {!Float32Array} out 4x4 matrix frustum matrix will be written into + * @param {number} fovy Vertical field of view in radians + * @param {number} aspect Aspect ratio. typically viewport width/height + * @param {number} near Near bound of the frustum + * @param {number} far Far bound of the frustum, can be null or Infinity + */ + static perspective(out, fovy, aspect, near, far) { + const f = 1.0 / Math.tan(fovy / 2); + out[0] = f / aspect; + out[1] = 0; + out[2] = 0; + out[3] = 0; + out[4] = 0; + out[5] = f; + out[6] = 0; + out[7] = 0; + out[8] = 0; + out[9] = 0; + out[11] = -1; + out[12] = 0; + out[13] = 0; + out[15] = 0; + if (far != null && far !== Infinity) { + const nf = 1 / (near - far); + out[10] = (far + near) * nf; + out[14] = 2 * far * near * nf; + } else { + out[10] = -1; + out[14] = -2 * near; + } + return out; + } + + /** + * Multiplies two 4x4 matrix + * + * @param {!Float32Array} out the receiving matrix + * @param {!Float32Array} a the first operand + * @param {!Float32Array} b the second operand + */ + static multiply(out, a, b) { + const a00 = a[0]; + const a01 = a[1]; + const a02 = a[2]; + const a03 = a[3]; + const a10 = a[4]; + const a11 = a[5]; + const a12 = a[6]; + const a13 = a[7]; + const a20 = a[8]; + const a21 = a[9]; + const a22 = a[10]; + const a23 = a[11]; + const a30 = a[12]; + const a31 = a[13]; + const a32 = a[14]; + const a33 = a[15]; + + // Cache only the current line of the second matrix + let b0 = b[0]; + let b1 = b[1]; + let b2 = b[2]; + let b3 = b[3]; + out[0] = b0 * a00 + b1 * a10 + b2 * a20 + b3 * a30; + out[1] = b0 * a01 + b1 * a11 + b2 * a21 + b3 * a31; + out[2] = b0 * a02 + b1 * a12 + b2 * a22 + b3 * a32; + out[3] = b0 * a03 + b1 * a13 + b2 * a23 + b3 * a33; + + b0 = b[4]; + b1 = b[5]; + b2 = b[6]; + b3 = b[7]; + out[4] = b0 * a00 + b1 * a10 + b2 * a20 + b3 * a30; + out[5] = b0 * a01 + b1 * a11 + b2 * a21 + b3 * a31; + out[6] = b0 * a02 + b1 * a12 + b2 * a22 + b3 * a32; + out[7] = b0 * a03 + b1 * a13 + b2 * a23 + b3 * a33; + + b0 = b[8]; + b1 = b[9]; + b2 = b[10]; + b3 = b[11]; + out[8] = b0 * a00 + b1 * a10 + b2 * a20 + b3 * a30; + out[9] = b0 * a01 + b1 * a11 + b2 * a21 + b3 * a31; + out[10] = b0 * a02 + b1 * a12 + b2 * a22 + b3 * a32; + out[11] = b0 * a03 + b1 * a13 + b2 * a23 + b3 * a33; + + b0 = b[12]; + b1 = b[13]; + b2 = b[14]; + b3 = b[15]; + out[12] = b0 * a00 + b1 * a10 + b2 * a20 + b3 * a30; + out[13] = b0 * a01 + b1 * a11 + b2 * a21 + b3 * a31; + out[14] = b0 * a02 + b1 * a12 + b2 * a22 + b3 * a32; + out[15] = b0 * a03 + b1 * a13 + b2 * a23 + b3 * a33; + } + + /** + * Generates a frustum matrix with the given bounds + * + * @param {!Float32Array} out 4x4 matrix frustum matrix will be written into + * @param {number} left Left bound of the frustum + * @param {number} right Right bound of the frustum + * @param {number} bottom Bottom bound of the frustum + * @param {number} top Top bound of the frustum + * @param {number} near Near bound of the frustum + * @param {number} far Far bound of the frustum + */ + static frustum(out, left, right, bottom, top, near, far) { + const rl = 1 / (right - left); + const tb = 1 / (top - bottom); + const nf = 1 / (near - far); + out[0] = near * 2 * rl; + out[1] = 0; + out[2] = 0; + out[3] = 0; + out[4] = 0; + out[5] = near * 2 * tb; + out[6] = 0; + out[7] = 0; + out[8] = (right + left) * rl; + out[9] = (top + bottom) * tb; + out[10] = (far + near) * nf; + out[11] = -1; + out[12] = 0; + out[13] = 0; + out[14] = far * near * 2 * nf; + out[15] = 0; + return out; + } + + /** + * Rotates a matrix by the given angle around the X axis + * + * @param {!Float32Array} out the receiving matrix + * @param {!Float32Array} a the matrix to rotate + * @param {number} rad the angle to rotate the matrix by + */ + static rotateX(out, a, rad) { + const s = Math.sin(rad); + const c = Math.cos(rad); + const a10 = a[4]; + const a11 = a[5]; + const a12 = a[6]; + const a13 = a[7]; + const a20 = a[8]; + const a21 = a[9]; + const a22 = a[10]; + const a23 = a[11]; + + if (a !== out) { + // If the source and destination differ, copy the unchanged rows + out[0] = a[0]; + out[1] = a[1]; + out[2] = a[2]; + out[3] = a[3]; + out[12] = a[12]; + out[13] = a[13]; + out[14] = a[14]; + out[15] = a[15]; + } + + // Perform axis-specific matrix multiplication + out[4] = a10 * c + a20 * s; + out[5] = a11 * c + a21 * s; + out[6] = a12 * c + a22 * s; + out[7] = a13 * c + a23 * s; + out[8] = a20 * c - a10 * s; + out[9] = a21 * c - a11 * s; + out[10] = a22 * c - a12 * s; + out[11] = a23 * c - a13 * s; + } + + /** + * Rotates a matrix by the given angle around the Y axis + * + * @param {!Float32Array} out the receiving matrix + * @param {!Float32Array} a the matrix to rotate + * @param {number} rad the angle to rotate the matrix by + */ + static rotateY(out, a, rad) { + const s = Math.sin(rad); + const c = Math.cos(rad); + const a00 = a[0]; + const a01 = a[1]; + const a02 = a[2]; + const a03 = a[3]; + const a20 = a[8]; + const a21 = a[9]; + const a22 = a[10]; + const a23 = a[11]; + + if (a !== out) { + // If the source and destination differ, copy the unchanged rows + out[4] = a[4]; + out[5] = a[5]; + out[6] = a[6]; + out[7] = a[7]; + out[12] = a[12]; + out[13] = a[13]; + out[14] = a[14]; + out[15] = a[15]; + } + + // Perform axis-specific matrix multiplication + out[0] = a00 * c - a20 * s; + out[1] = a01 * c - a21 * s; + out[2] = a02 * c - a22 * s; + out[3] = a03 * c - a23 * s; + out[8] = a00 * s + a20 * c; + out[9] = a01 * s + a21 * c; + out[10] = a02 * s + a22 * c; + out[11] = a03 * s + a23 * c; + } + + /** + * Rotates a matrix by the given angle around the Z axis + * + * @param {!Float32Array} out the receiving matrix + * @param {!Float32Array} a the matrix to rotate + * @param {number} rad the angle to rotate the matrix by + */ + static rotateZ(out, a, rad) { + const s = Math.sin(rad); + const c = Math.cos(rad); + const a00 = a[0]; + const a01 = a[1]; + const a02 = a[2]; + const a03 = a[3]; + const a10 = a[4]; + const a11 = a[5]; + const a12 = a[6]; + const a13 = a[7]; + + if (a !== out) { + // If the source and destination differ, copy the unchanged last row + out[8] = a[8]; + out[9] = a[9]; + out[10] = a[10]; + out[11] = a[11]; + out[12] = a[12]; + out[13] = a[13]; + out[14] = a[14]; + out[15] = a[15]; + } + + // Perform axis-specific matrix multiplication + out[0] = a00 * c + a10 * s; + out[1] = a01 * c + a11 * s; + out[2] = a02 * c + a12 * s; + out[3] = a03 * c + a13 * s; + out[4] = a10 * c - a00 * s; + out[5] = a11 * c - a01 * s; + out[6] = a12 * c - a02 * s; + out[7] = a13 * c - a03 * s; + } + + /** + * Returns a quaternion representing the rotational component + * of a transformation matrix. If a matrix is built with + * fromRotationTranslation, the returned quaternion will be the + * same as the quaternion originally supplied. + * @param {!Float32Array} out Quaternion to receive the rotation component + * @param {!Float32Array} mat Matrix to be decomposed (input) + */ + static getRotation(out, mat) { + const scaling = new Float32Array(3); + shaka.ui.Matrix4x4.getScaling_(scaling, mat); + + const is1 = 1 / scaling[0]; + const is2 = 1 / scaling[1]; + const is3 = 1 / scaling[2]; + + const sm11 = mat[0] * is1; + const sm12 = mat[1] * is2; + const sm13 = mat[2] * is3; + const sm21 = mat[4] * is1; + const sm22 = mat[5] * is2; + const sm23 = mat[6] * is3; + const sm31 = mat[8] * is1; + const sm32 = mat[9] * is2; + const sm33 = mat[10] * is3; + + const trace = sm11 + sm22 + sm33; + let S = 0; + + if (trace > 0) { + S = Math.sqrt(trace + 1.0) * 2; + out[3] = 0.25 * S; + out[0] = (sm23 - sm32) / S; + out[1] = (sm31 - sm13) / S; + out[2] = (sm12 - sm21) / S; + } else if (sm11 > sm22 && sm11 > sm33) { + S = Math.sqrt(1.0 + sm11 - sm22 - sm33) * 2; + out[3] = (sm23 - sm32) / S; + out[0] = 0.25 * S; + out[1] = (sm12 + sm21) / S; + out[2] = (sm31 + sm13) / S; + } else if (sm22 > sm33) { + S = Math.sqrt(1.0 + sm22 - sm11 - sm33) * 2; + out[3] = (sm31 - sm13) / S; + out[0] = (sm12 + sm21) / S; + out[1] = 0.25 * S; + out[2] = (sm23 + sm32) / S; + } else { + S = Math.sqrt(1.0 + sm33 - sm11 - sm22) * 2; + out[3] = (sm12 - sm21) / S; + out[0] = (sm31 + sm13) / S; + out[1] = (sm23 + sm32) / S; + out[2] = 0.25 * S; + } + + return out; + } + + /** + * Calculates a 4x4 matrix from the given quaternion + * + * @param {!Float32Array} out 4x4 matrix receiving operation result + * @param {!Float32Array} q Quaternion to create matrix from + */ + static fromQuat(out, q) { + const x = q[0]; + const y = q[1]; + const z = q[2]; + const w = q[3]; + const x2 = x + x; + const y2 = y + y; + const z2 = z + z; + + const xx = x * x2; + const yx = y * x2; + const yy = y * y2; + const zx = z * x2; + const zy = z * y2; + const zz = z * z2; + const wx = w * x2; + const wy = w * y2; + const wz = w * z2; + + out[0] = 1 - yy - zz; + out[1] = yx + wz; + out[2] = zx - wy; + out[3] = 0; + + out[4] = yx - wz; + out[5] = 1 - xx - zz; + out[6] = zy + wx; + out[7] = 0; + + out[8] = zx + wy; + out[9] = zy - wx; + out[10] = 1 - xx - yy; + out[11] = 0; + + out[12] = 0; + out[13] = 0; + out[14] = 0; + out[15] = 1; + } + + + /** + * Set a 4x4 matrix to the identity matrix + * + * @param {!Float32Array} out the receiving matrix + * @private + */ + static identity_(out) { + out[0] = 1; + out[1] = 0; + out[2] = 0; + out[3] = 0; + out[4] = 0; + out[5] = 1; + out[6] = 0; + out[7] = 0; + out[8] = 0; + out[9] = 0; + out[10] = 1; + out[11] = 0; + out[12] = 0; + out[13] = 0; + out[14] = 0; + out[15] = 1; + } + + /** + * Returns the scaling factor component of a transformation + * matrix. If a matrix is built with fromRotationTranslationScale + * with a normalized Quaternion paramter, the returned vector will be + * the same as the scaling vector + * originally supplied. + * @param {!Float32Array} out Vector to receive scaling factor component + * @param {!Float32Array} mat Matrix to be decomposed (input) + * @private + */ + static getScaling_(out, mat) { + const m11 = mat[0]; + const m12 = mat[1]; + const m13 = mat[2]; + const m21 = mat[4]; + const m22 = mat[5]; + const m23 = mat[6]; + const m31 = mat[8]; + const m32 = mat[9]; + const m33 = mat[10]; + + out[0] = Math.sqrt(m11 * m11 + m12 * m12 + m13 * m13); + out[1] = Math.sqrt(m21 * m21 + m22 * m22 + m23 * m23); + out[2] = Math.sqrt(m31 * m31 + m32 * m32 + m33 * m33); + } +}; + +/** + * @const {number} + * @private + */ +shaka.ui.Matrix4x4.EPSILON_ = 0.000001; diff --git a/ui/gl_matrix/matrix_quaternion.js b/ui/gl_matrix/matrix_quaternion.js new file mode 100644 index 0000000000..732178f456 --- /dev/null +++ b/ui/gl_matrix/matrix_quaternion.js @@ -0,0 +1,45 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + + +goog.provide('shaka.ui.MatrixQuaternion'); + +/** + * Quaternion in the format XYZW + */ +shaka.ui.MatrixQuaternion = class { + /** + * Creates a new identity quaternion + * + * @return {!Float32Array} a new quaternion + */ + static create() { + const out = new Float32Array(4); + out[3] = 1; + return out; + } + + /** + * Normalize a quaternion + * + * @param {!Float32Array} out the receiving quaternion + * @param {!Float32Array} a quaternion to normalize + */ + static normalize(out, a) { + const x = a[0]; + const y = a[1]; + const z = a[2]; + const w = a[3]; + let len = x * x + y * y + z * z + w * w; + if (len > 0) { + len = 1 / Math.sqrt(len); + } + out[0] = x * len; + out[1] = y * len; + out[2] = z * len; + out[3] = w * len; + } +}; diff --git a/ui/less/containers.less b/ui/less/containers.less index 039a28e5f6..b797ba1018 100644 --- a/ui/less/containers.less +++ b/ui/less/containers.less @@ -110,6 +110,17 @@ pointer-events: none; } +/* A container for VR + * Sits inside .shaka-video-container, on top of (Z axis) .shaka-video, and + * below (Y axis) .shaka-play-button-container. */ +.shaka-vr-canvas-container { + .overlay-child(); + + /* Make sure pointer events can reach the video element in the case of native + * controls. */ + pointer-events: none; +} + /* Container for controls positioned at the bottom of the video container: * controls button panel and the seek bar. */ .shaka-bottom-controls { diff --git a/ui/ui.js b/ui/ui.js index 29662bd66a..622ea70a3b 100644 --- a/ui/ui.js +++ b/ui/ui.js @@ -29,8 +29,9 @@ shaka.ui.Overlay = class { * @param {!shaka.Player} player * @param {!HTMLElement} videoContainer * @param {!HTMLMediaElement} video + * @param {!HTMLCanvasElement} vrCanvas */ - constructor(player, videoContainer, video) { + constructor(player, videoContainer, video, vrCanvas) { /** @private {shaka.Player} */ this.player_ = player; @@ -52,7 +53,7 @@ shaka.ui.Overlay = class { /** @private {shaka.ui.Controls} */ this.controls_ = new shaka.ui.Controls( - player, videoContainer, video, this.config_); + player, videoContainer, video, vrCanvas, this.config_); // Run the initial setup so that no configure() call is required for default // settings. @@ -251,6 +252,7 @@ shaka.ui.Overlay = class { seekOnTaps: true, tapSeekDistance: 10, refreshTickInSeconds: 0.125, + displayInVrMode: false, }; // eslint-disable-next-line no-restricted-syntax @@ -309,6 +311,12 @@ shaka.ui.Overlay = class { const canvases = document.querySelectorAll( '[data-shaka-player-canvas]'); + // Look for elements marked 'data-shaka-player-vr-canvas' + // on the page. These will be used to create our default + // UI. + const vrCanvases = document.querySelectorAll( + '[data-shaka-player-vr-canvas]'); + if (!videos.length && !containers.length) { // No elements have been tagged with shaka attributes. } else if (videos.length && !containers.length) { @@ -326,21 +334,37 @@ shaka.ui.Overlay = class { const videoParent = video.parentElement; videoParent.replaceChild(container, video); container.appendChild(video); - let currentCanvas = null; + let lcevcCanvas = null; for (const canvas of canvases) { goog.asserts.assert(canvas.tagName.toLowerCase() == 'canvas', 'Should be a canvas element!'); if (canvas.parentElement == container) { - currentCanvas = canvas; + lcevcCanvas = canvas; break; } } - if (!currentCanvas) { - currentCanvas = document.createElement('canvas'); - currentCanvas.classList.add('shaka-canvas-container'); - container.appendChild(currentCanvas); + if (!lcevcCanvas) { + lcevcCanvas = document.createElement('canvas'); + lcevcCanvas.classList.add('shaka-canvas-container'); + container.appendChild(lcevcCanvas); + } + + let vrCanvas = null; + for (const canvas of vrCanvases) { + goog.asserts.assert(canvas.tagName.toLowerCase() == 'canvas', + 'Should be a canvas element!'); + if (canvas.parentElement == container) { + vrCanvas = canvas; + break; + } } - shaka.ui.Overlay.setupUIandAutoLoad_(container, video, currentCanvas); + if (!vrCanvas) { + vrCanvas = document.createElement('canvas'); + vrCanvas.classList.add('shaka-vr-canvas-container'); + container.appendChild(vrCanvas); + } + shaka.ui.Overlay.setupUIandAutoLoad_( + container, video, lcevcCanvas, vrCanvas); } } else { for (const container of containers) { @@ -368,25 +392,40 @@ shaka.ui.Overlay = class { container.appendChild(currentVideo); } - let currentCanvas = null; + let lcevcCanvas = null; for (const canvas of canvases) { goog.asserts.assert(canvas.tagName.toLowerCase() == 'canvas', 'Should be a canvas element!'); if (canvas.parentElement == container) { - currentCanvas = canvas; + lcevcCanvas = canvas; + break; + } + } + if (!lcevcCanvas) { + lcevcCanvas = document.createElement('canvas'); + lcevcCanvas.classList.add('shaka-canvas-container'); + container.appendChild(lcevcCanvas); + } + + let vrCanvas = null; + for (const canvas of vrCanvases) { + goog.asserts.assert(canvas.tagName.toLowerCase() == 'canvas', + 'Should be a canvas element!'); + if (canvas.parentElement == container) { + vrCanvas = canvas; break; } } - if (!currentCanvas) { - currentCanvas = document.createElement('canvas'); - currentCanvas.classList.add('shaka-canvas-container'); - container.appendChild(currentCanvas); + if (!vrCanvas) { + vrCanvas = document.createElement('canvas'); + vrCanvas.classList.add('shaka-vr-canvas-container'); + container.appendChild(vrCanvas); } try { // eslint-disable-next-line no-await-in-loop await shaka.ui.Overlay.setupUIandAutoLoad_( - container, currentVideo, currentCanvas); + container, currentVideo, lcevcCanvas, vrCanvas); } catch (e) { // This can fail if, for example, not every player file has loaded. // Ad-block is a likely cause for this sort of failure. @@ -425,18 +464,20 @@ shaka.ui.Overlay = class { /** * @param {!Element} container * @param {!Element} video - * @param {!Element} canvas + * @param {!Element} lcevcCanvas + * @param {!Element} vrCanvas * @private */ - static async setupUIandAutoLoad_(container, video, canvas) { + static async setupUIandAutoLoad_(container, video, lcevcCanvas, vrCanvas) { // Create the UI const player = new shaka.Player(); const ui = new shaka.ui.Overlay(player, shaka.util.Dom.asHTMLElement(container), - shaka.util.Dom.asHTMLMediaElement(video)); + shaka.util.Dom.asHTMLMediaElement(video), + shaka.util.Dom.asHTMLCanvasElement(vrCanvas)); // Attach Canvas used for LCEVC Decoding - player.attachCanvas(/** @type {HTMLCanvasElement} */(canvas)); + player.attachCanvas(/** @type {HTMLCanvasElement} */(lcevcCanvas)); // Get and configure cast app id. let castAppId = ''; diff --git a/ui/vr_manager.js b/ui/vr_manager.js new file mode 100644 index 0000000000..ebaca6d010 --- /dev/null +++ b/ui/vr_manager.js @@ -0,0 +1,496 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + + +goog.provide('shaka.ui.VRManager'); + +goog.require('shaka.log'); +goog.require('shaka.ui.VRUtils'); +goog.require('shaka.ui.VRWebgl'); +goog.require('shaka.util.EventManager'); +goog.require('shaka.util.IReleasable'); + +goog.requireType('shaka.Player'); + + +/** + * @implements {shaka.util.IReleasable} + */ +shaka.ui.VRManager = class { + /** + * @param {!HTMLElement} container + * @param {!HTMLCanvasElement} canvas + * @param {!HTMLMediaElement} video + * @param {!shaka.Player} player + * @param {shaka.extern.UIConfiguration} config + */ + constructor(container, canvas, video, player, config) { + /** @private {!HTMLElement} */ + this.container_ = container; + + /** @private {!HTMLCanvasElement} */ + this.canvas_ = canvas; + + /** @private {!HTMLMediaElement} */ + this.video_ = video; + + /** @private {!shaka.Player} */ + this.player_ = player; + + /** @private {shaka.extern.UIConfiguration} */ + this.config_ = config; + + /** @private {shaka.util.EventManager} */ + this.loadEventManager_ = new shaka.util.EventManager(); + + /** @private {shaka.util.EventManager} */ + this.eventManager_ = new shaka.util.EventManager(); + + /** @private {?shaka.ui.VRWebgl} */ + this.vrWebgl_ = null; + + /** @private {boolean} */ + this.onGesture_ = false; + + /** @private {number} */ + this.prevX_ = 0; + + /** @private {number} */ + this.prevY_ = 0; + + /** @private {number} */ + this.prevAlpha_ = 0; + + /** @private {number} */ + this.prevBeta_ = 0; + + /** @private {number} */ + this.prevGamma_ = 0; + + /** @private {boolean} */ + this.vrAsset_ = false; + + this.loadEventManager_.listen(player, 'spatialvideoinfo', (event) => { + /** @type {shaka.extern.SpatialVideoInfo} */ + const spatialInfo = event['detail']; + let unsupported = false; + switch (spatialInfo.projection) { + case 'hequ': + unsupported = spatialInfo.hfov != 360; + this.vrAsset_ = true; + break; + case 'fish': + this.vrAsset_ = true; + unsupported = true; + break; + default: + this.vrAsset_ = false; + break; + } + if (unsupported) { + shaka.log.warning('Unsupported VR projection or hfov', spatialInfo); + } + this.checkVrStatus_(); + }); + + this.loadEventManager_.listen(player, 'nospatialvideoinfo', () => { + this.vrAsset_ = false; + this.checkVrStatus_(); + }); + + this.loadEventManager_.listen(player, 'unloading', () => { + this.vrAsset_ = false; + this.checkVrStatus_(); + }); + + this.checkVrStatus_(); + } + + /** + * @override + */ + release() { + if (this.loadEventManager_) { + this.loadEventManager_.release(); + this.loadEventManager_ = null; + } + if (this.eventManager_) { + this.eventManager_.release(); + this.eventManager_ = null; + } + if (this.vrWebgl_) { + this.vrWebgl_.release(); + this.vrWebgl_ = null; + } + } + + /** + * @param {!shaka.extern.UIConfiguration} config + */ + configure(config) { + this.config_ = config; + this.checkVrStatus_(); + } + + /** + * Returns if a VR is supported. + * + * @return {boolean} + */ + isPlayingVR() { + return !!this.vrWebgl_; + } + + /** + * Get the angle of the north. + * + * @return {?number} + */ + getNorth() { + if (!this.vrWebgl_) { + shaka.log.alwaysWarn('Not playing VR content'); + return null; + } + return this.vrWebgl_.getNorth(); + } + + /** + * Returns the field view. + * + * @return {?number} + */ + getFieldView() { + if (!this.vrWebgl_) { + shaka.log.alwaysWarn('Not playing VR content'); + return null; + } + return this.vrWebgl_.getFieldView(); + } + + /** + * Set the field view. + * + * @param {number} fieldView + */ + setFieldView(fieldView) { + if (!this.vrWebgl_) { + shaka.log.alwaysWarn('Not playing VR content'); + return; + } + if (fieldView < 0) { + shaka.log.alwaysWarn('Field view should be greater than 0'); + fieldView = 0; + } else if (fieldView > 100) { + shaka.log.alwaysWarn('Field view should be less than 100'); + fieldView = 100; + } + this.vrWebgl_.setFieldView(fieldView); + } + + /** + * Toggle stereoscopic mode. + */ + toggleStereoscopicMode() { + if (!this.vrWebgl_) { + shaka.log.alwaysWarn('Not playing VR content'); + return; + } + this.vrWebgl_.togglestereoscopicMode(); + } + + /** + * Increment the yaw in X angle in degrees. + * + * @param {number} angle + */ + incrementYaw(angle) { + if (!this.vrWebgl_) { + shaka.log.alwaysWarn('Not playing VR content'); + return; + } + if (angle < -359) { + shaka.log.alwaysWarn('Yaw angle should be greater than -360'); + angle = -359; + } else if (angle > 359) { + shaka.log.alwaysWarn('Yaw angle should be less than 360'); + angle = 359; + } + this.vrWebgl_.rotateViewGlobal(angle * shaka.ui.VRUtils.TO_RADIANS, 0, 0); + } + + /** + * Increment the pitch in X angle in degrees. + * + * @param {number} angle + */ + incrementPitch(angle) { + if (!this.vrWebgl_) { + shaka.log.alwaysWarn('Not playing VR content'); + return; + } + if (angle < -89) { + shaka.log.alwaysWarn('Pitch angle should be greater than -90'); + angle = -89; + } else if (angle > 89) { + shaka.log.alwaysWarn('Pitch angle should be less than 90'); + angle = 89; + } + this.vrWebgl_.rotateViewGlobal(0, angle * shaka.ui.VRUtils.TO_RADIANS, 0); + } + + /** + * Increment the roll in X angle in degrees. + * + * @param {number} angle + */ + incrementRoll(angle) { + if (!this.vrWebgl_) { + shaka.log.alwaysWarn('Not playing VR content'); + return; + } + if (angle < -359) { + shaka.log.alwaysWarn('Roll angle should be greater than -360'); + angle = -359; + } else if (angle > 359) { + shaka.log.alwaysWarn('Roll angle should be less than 360'); + angle = 359; + } + this.vrWebgl_.rotateViewGlobal(0, 0, angle * shaka.ui.VRUtils.TO_RADIANS); + } + + /** + * @private + */ + checkVrStatus_() { + if ((this.config_.displayInVrMode || this.vrAsset_) && !this.vrWebgl_) { + this.canvas_.style.display = ''; + this.init_(); + } else if (!this.config_.displayInVrMode && !this.vrAsset_ && + this.vrWebgl_) { + this.canvas_.style.display = 'none'; + this.eventManager_.removeAll(); + this.vrWebgl_.release(); + this.vrWebgl_ = null; + } + } + + /** + * @private + */ + init_() { + const gl = this.getGL_(); + if (gl) { + this.vrWebgl_ = new shaka.ui.VRWebgl( + this.video_, this.player_, this.canvas_, gl); + this.setupVRListerners_(); + } + } + + /** + * @return {?WebGLRenderingContext} + * @private + */ + getGL_() { + const webglContexts = [ + 'webgl2', + 'webgl', + ]; + for (const webgl of webglContexts) { + const gl = this.canvas_.getContext(webgl); + if (gl) { + return /** @type {!WebGLRenderingContext} */(gl); + } + } + return null; + } + + /** + * @private + */ + setupVRListerners_() { + // Start + this.eventManager_.listen(this.container_, 'mousedown', (event) => { + if (!this.onGesture_) { + this.gestureStart_(event.clientX, event.clientY); + } + }); + if (navigator.maxTouchPoints > 0) { + this.eventManager_.listen(this.container_, 'touchstart', (e) => { + if (!this.onGesture_) { + const event = /** @type {!TouchEvent} */(e); + this.gestureStart_( + event.touches[0].clientX, event.touches[0].clientY); + } + }); + } + + // Zoom + this.eventManager_.listen(this.container_, 'wheel', (e) => { + if (!this.onGesture_) { + const event = /** @type {!WheelEvent} */(e); + this.vrWebgl_.zoom(event.deltaY); + event.preventDefault(); + event.stopPropagation(); + } + }); + + // Move + this.eventManager_.listen(this.container_, 'mousemove', (event) => { + if (this.onGesture_) { + this.gestureMove_(event.clientX, event.clientY); + } + }); + if (navigator.maxTouchPoints > 0) { + this.eventManager_.listen(this.container_, 'touchmove', (e) => { + if (this.onGesture_) { + const event = /** @type {!TouchEvent} */(e); + this.gestureMove_( + event.touches[0].clientX, event.touches[0].clientY); + } + e.preventDefault(); + }); + } + + // End + this.eventManager_.listen(this.container_, 'mouseleave', () => { + this.onGesture_ = false; + }); + this.eventManager_.listen(this.container_, 'mouseup', () => { + this.onGesture_ = false; + }); + if (navigator.maxTouchPoints > 0) { + this.eventManager_.listen(this.container_, 'touchend', () => { + this.onGesture_ = false; + }); + } + + // Detect device movement + let deviceOrientationListener = false; + if (window.DeviceOrientationEvent) { + // See: https://dev.to/li/how-to-requestpermission-for-devicemotion-and-deviceorientation-events-in-ios-13-46g2 + if (typeof DeviceMotionEvent.requestPermission == 'function') { + const userGestureListener = () => { + DeviceMotionEvent.requestPermission().then((newPermissionState) => { + if (newPermissionState !== 'granted' || + deviceOrientationListener) { + return; + } + deviceOrientationListener = true; + this.setupDeviceOrientationListener_(); + }); + }; + DeviceMotionEvent.requestPermission().then((permissionState) => { + this.eventManager_.unlisten( + this.container_, 'click', userGestureListener); + this.eventManager_.unlisten( + this.container_, 'mouseup', userGestureListener); + if (navigator.maxTouchPoints > 0) { + this.eventManager_.unlisten( + this.container_, 'touchend', userGestureListener); + } + if (permissionState !== 'granted') { + this.eventManager_.listenOnce( + this.container_, 'click', userGestureListener); + this.eventManager_.listenOnce( + this.container_, 'mouseup', userGestureListener); + if (navigator.maxTouchPoints > 0) { + this.eventManager_.listenOnce( + this.container_, 'touchend', userGestureListener); + } + return; + } + deviceOrientationListener = true; + this.setupDeviceOrientationListener_(); + }).catch(() => { + this.eventManager_.unlisten( + this.container_, 'click', userGestureListener); + this.eventManager_.unlisten( + this.container_, 'mouseup', userGestureListener); + if (navigator.maxTouchPoints > 0) { + this.eventManager_.unlisten( + this.container_, 'touchend', userGestureListener); + } + this.eventManager_.listenOnce( + this.container_, 'click', userGestureListener); + this.eventManager_.listenOnce( + this.container_, 'mouseup', userGestureListener); + if (navigator.maxTouchPoints > 0) { + this.eventManager_.listenOnce( + this.container_, 'touchend', userGestureListener); + } + }); + } else { + deviceOrientationListener = true; + this.setupDeviceOrientationListener_(); + } + } + } + + /** + * @private + */ + setupDeviceOrientationListener_() { + this.eventManager_.listen(window, 'deviceorientation', (e) => { + const event = /** @type {!DeviceOrientationEvent} */(e); + let alphaDif = (event.alpha || 0) - this.prevAlpha_; + let betaDif = (event.beta || 0) - this.prevBeta_; + let gammaDif = (event.gamma || 0) - this.prevGamma_; + + if (Math.abs(alphaDif) > 10) { + alphaDif = 0; + } + if (Math.abs(gammaDif) > 10) { + gammaDif = 0; + } + if (Math.abs(betaDif) > 10) { + betaDif = 0; + } + + this.prevAlpha_ = event.alpha || 0; + this.prevBeta_ = event.beta || 0; + this.prevGamma_ = event.gamma || 0; + + const toRadians = shaka.ui.VRUtils.TO_RADIANS; + + const orientation = screen.orientation.angle; + if (orientation == 90 || orientation == -90) { + this.vrWebgl_.rotateViewGlobal( + alphaDif * toRadians * -1, gammaDif * toRadians * -1, 0); + } else { + this.vrWebgl_.rotateViewGlobal( + alphaDif * toRadians * -1, betaDif * toRadians, 0); + } + }); + } + + /** + * @param {number} x + * @param {number} y + * @private + */ + gestureStart_(x, y) { + this.onGesture_ = true; + this.prevX_ = x; + this.prevY_ = y; + } + + /** + * @param {number} x + * @param {number} y + * @private + */ + gestureMove_(x, y) { + const touchScaleFactor = -0.60 * Math.PI / 180; + this.vrWebgl_.rotateViewGlobal((x - this.prevX_) * touchScaleFactor, + (y - this.prevY_) * -1 * touchScaleFactor, 0); + + this.prevX_ = x; + this.prevY_ = y; + } +}; + + +shaka.ui.VRManager.TO_RADIANS = Math.PI / 180; diff --git a/ui/vr_utils.js b/ui/vr_utils.js new file mode 100644 index 0000000000..40292b9ed2 --- /dev/null +++ b/ui/vr_utils.js @@ -0,0 +1,115 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + + +goog.provide('shaka.ui.VRUtils'); + + +shaka.ui.VRUtils = class { + /** + * @param {number} resolution + * @param {boolean=} semisphere + * @return {{vertices: !Array., texCoords: !Array., + * indices: !Array.}} + */ + static generateSphere(resolution, semisphere = false) { + /** @type {!Array.} */ + const vertices = []; + /** @type {!Array.} */ + const texCoords = []; + /** @type {!Array.} */ + const indices = []; + + let phiMax = Math.PI; + if (semisphere) { + phiMax = Math.PI / 2; + } + + for (let i = 0; i <= resolution; i++) { + const v = i / resolution; + const phi = v * phiMax; + const sinPhi = Math.sin(phi); + const cosPhi = Math.cos(phi); + + for (let j = 0; j <= resolution; j++) { + const u = j / resolution; + const theta = u * Math.PI * 2; + + const sinTheta = Math.sin(theta); + const cosTheta = Math.cos(theta); + + const x = -1 * cosTheta * sinPhi; + const y = cosPhi; + const z = sinTheta * sinPhi; + + vertices.push(x, y, z); + + texCoords.push(u); + texCoords.push(v); + } + } + + for (let i = 0; i < resolution; i++) { + for (let j = 0; j < resolution; j++) { + const a = i * (resolution + 1) + j; + const b = a + 1; + const c = (i + 1) * (resolution + 1) + j; + const d = c + 1; + + indices.push(a, c, b); + indices.push(b, c, d); + } + } + + return {vertices, texCoords, indices}; + } +}; + +/** + * Sphere vertex shader. + * + * @constant {string} + */ +shaka.ui.VRUtils.VERTEX_SPHERE_SHADER = +`attribute vec4 a_vPosition; +// Per-vertex texture coordinate info +attribute vec2 a_TexCoordinate; +uniform mat4 u_VPMatrix; +// Passed into the fragment shader. +varying vec2 v_TexCoordinate; +varying vec3 pass_position; +void main() +{ + gl_Position = u_VPMatrix * a_vPosition; + // Pass through texture coord + v_TexCoordinate = a_TexCoordinate; + pass_position = a_vPosition.xyz; +}`; + +/** + * Sphere fragment shader. + * + * @constant {string} + */ +shaka.ui.VRUtils.FRAGMENT_SPHERE_SHADER = +`precision highp float; +#define PI 3.141592653589793238462643383279 +varying vec2 v_TexCoordinate; +varying vec3 pass_position; +uniform sampler2D uSampler; +void main(void) { +highp float xValue = + (PI + atan(pass_position.z, pass_position.x)) / (2.0 * PI); + vec2 tc = vec2(xValue, v_TexCoordinate.t); + tc = vec2(tc.x , tc.y); +highp vec4 texelColor = + texture2D(uSampler, tc); + gl_FragColor = vec4(texelColor.rgb, texelColor.a); +}`; + + +shaka.ui.VRUtils.TO_RADIANS = Math.PI / 180; + diff --git a/ui/vr_webgl.js b/ui/vr_webgl.js new file mode 100644 index 0000000000..2c25825875 --- /dev/null +++ b/ui/vr_webgl.js @@ -0,0 +1,611 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + + +goog.provide('shaka.ui.VRWebgl'); + +goog.require('goog.asserts'); +goog.require('shaka.log'); +goog.require('shaka.Player'); +goog.require('shaka.ui.Matrix4x4'); +goog.require('shaka.ui.MatrixQuaternion'); +goog.require('shaka.ui.VRUtils'); +goog.require('shaka.util.EventManager'); +goog.require('shaka.util.IReleasable'); +goog.require('shaka.util.Timer'); + + +/** + * @implements {shaka.util.IReleasable} + */ +shaka.ui.VRWebgl = class { + /** + * @param {!HTMLMediaElement} video + * @param {!shaka.Player} player + * @param {!HTMLCanvasElement} canvas + * @param {WebGLRenderingContext} gl + */ + constructor(video, player, canvas, gl) { + /** @private {HTMLVideoElement} */ + this.video_ = /** @type {!HTMLVideoElement} */ (video); + + /** @private {shaka.Player} */ + this.player_ = player; + + /** @private {HTMLCanvasElement} */ + this.canvas_ = canvas; + + /** @private {WebGLRenderingContext} */ + this.gl_ = gl; + + /** @private {shaka.util.EventManager} */ + this.eventManager_ = new shaka.util.EventManager(); + + /** @private {!Float32Array} */ + this.originalQuaternion_ = shaka.ui.MatrixQuaternion.create(); + + /** @private {!Float32Array} */ + this.currentQuaternion_ = shaka.ui.MatrixQuaternion.create(); + + this.shaderProgram_ = null; + this.verticesBuffer_ = null; + this.normalsBuffer_ = null; + this.verticesTextureCoordBuffer_ = null; + this.verticesIndexBuffer_ = null; + + /** @private {!Float32Array} */ + this.viewMatrix_ = shaka.ui.Matrix4x4.create(); + + /** @private {!Float32Array} */ + this.projectionMatrix_ = shaka.ui.Matrix4x4.create(); + + /** @private {!Float32Array} */ + this.normalMatrix_ = shaka.ui.Matrix4x4.create(); + + /** @private {!Float32Array} */ + this.viewProjectionMatrix_ = shaka.ui.Matrix4x4.create(); + + /** @private {!Float32Array} */ + this.identityMatrix_ = shaka.ui.Matrix4x4.create(); + + /** @private {?Float32Array} */ + this.diff_ = null; + + /** @private {boolean} */ + this.stereoscopicMode_ = false; + + /** @private {?shaka.util.Timer} */ + this.activeTimer_ = null; + + /** @private {number} */ + this.previousCanvasWidth_ = 0; + + /** @private {number} */ + this.previousCanvasHeight_ = 0; + + /** + * @private {?{vertices: !Array., texCoords: !Array., + * indices: !Array.}} + */ + this.sphere_ = null; + + /** @private {?number} */ + this.vertexPositionAttribute_ = null; + + /** @private {?number} */ + this.textureCoordAttribute_ = null; + + /** @private {?WebGLTexture} */ + this.texture_ = null; + + /** @private {number} */ + this.positionY_ = 0; + + /** @private {number} */ + this.fieldOfView_ = 75; + + /** @private {number} */ + this.animDuration_ = 0.5; + + /** @private {number} */ + this.cont_ = 0; + + this.init_(); + } + + /** + * @override + */ + release() { + if (this.eventManager_) { + this.eventManager_.release(); + this.eventManager_ = null; + } + if (this.activeTimer_) { + this.activeTimer_.stop(); + this.activeTimer_ = null; + } + } + + /** + * @param {!Float32Array} quat + * @private + */ + toEulerAngles_(quat) { + const angles = {}; + + const sinr = 2.0 * (quat[3] * quat[0] + quat[1] * quat[2]); + const cosr = 1.0 - 2.0 * (quat[0] * quat[0] + quat[1] * quat[1]); + + angles.roll = Math.atan2(sinr, cosr); + + const sinp = 2.0 * (quat[3] * quat[1] - quat[2] * quat[0]); + + if (Math.abs(sinp) >= 1) { + switch (Math.sign(sinp)) { + case 1: + angles.pitch = Math.PI / 2; + break; + case -1: + angles.pitch = (-1) * Math.PI / 2; + break; + } + } else { + angles.pitch = Math.asin(sinp); + } + + const siny = 2.0 * (quat[3] * quat[2] + quat[0] * quat[1]); + const cosy = 1.0 - 2.0 * (quat[1] * quat[1] + quat[2] * quat[2]); + + angles.yaw = Math.atan2(siny, cosy); + + return angles; + } + + /** + * Toogle stereoscopic mode + */ + togglestereoscopicMode() { + this.stereoscopicMode_ = !this.stereoscopicMode_; + if (!this.stereoscopicMode_) { + this.gl_.viewport(0, 0, this.canvas_.width, this.canvas_.height); + } + this.renderGL_(false); + } + + /** + * @private + */ + init_() { + this.initMatrices_(); + this.initGL_(); + this.initGLShaders_(); + this.initGLBuffers_(); + this.initGLTexture_(); + + this.eventManager_.listenOnce(this.video_, 'loadeddata', () => { + let frameRate; + this.eventManager_.listen(this.video_, 'canplaythrough', () => { + this.renderGL_(); + }); + this.eventManager_.listen(this.video_, 'playing', () => { + if (this.activeTimer_) { + this.activeTimer_.stop(); + } + if (!frameRate) { + const variants = this.player_.getVariantTracks(); + for (const variant of variants) { + const variantFrameRate = variant.frameRate; + if (variantFrameRate && + (!frameRate || frameRate < variantFrameRate)) { + frameRate = variantFrameRate; + } + } + } + if (!frameRate) { + frameRate = 60; + } + this.renderGL_(); + this.activeTimer_ = new shaka.util.Timer(() => { + this.renderGL_(); + }).tickNow().tickEvery(1 / frameRate); + }); + this.eventManager_.listen(this.video_, 'pause', () => { + this.activeTimer_.stop(); + this.activeTimer_ = null; + this.renderGL_(); + }); + this.eventManager_.listen(this.video_, 'seeked', () => { + this.renderGL_(); + }); + + this.eventManager_.listen(document, 'visibilitychange', () => { + this.renderGL_(); + }); + }); + } + + /** + * @private + */ + initMatrices_() { + shaka.ui.Matrix4x4.lookAt( + this.viewMatrix_, [0, 0, 0], [1, 0, 0], [0, 1, 0]); + shaka.ui.Matrix4x4.getRotation( + this.originalQuaternion_, this.viewMatrix_); + shaka.ui.Matrix4x4.scale( + this.identityMatrix_, this.identityMatrix_, [4.0, 4.0, 4.0]); + } + + /** + * @private + */ + initGL_() { + this.updateViewPort_(); + this.gl_.viewport( + 0, 0, this.gl_.drawingBufferWidth, this.gl_.drawingBufferHeight); + this.gl_.clearColor(0.0, 0.0, 0.0, 1.0); + this.gl_.enable(this.gl_.CULL_FACE); + this.gl_.cullFace(this.gl_.FRONT); + // Clear the context with the newly set color. This is + // the function call that actually does the drawing. + this.gl_.clear(this.gl_.COLOR_BUFFER_BIT); + } + + /** + * @private + */ + initGLShaders_() { + const vertexShader = this.getGLShader_(this.gl_.VERTEX_SHADER); + const fragmentShader = this.getGLShader_(this.gl_.FRAGMENT_SHADER); + + // Create program + this.shaderProgram_ = this.gl_.createProgram(); + this.gl_.attachShader(this.shaderProgram_, vertexShader); + this.gl_.attachShader(this.shaderProgram_, fragmentShader); + this.gl_.linkProgram(this.shaderProgram_); + + // If creating the shader program failed, alert + if (!this.gl_.getProgramParameter( + this.shaderProgram_, this.gl_.LINK_STATUS)) { + shaka.log.error('Unable to initialize the shader program: ', + this.gl_.getProgramInfoLog(this.shaderProgram_)); + } + + // Bind data + this.vertexPositionAttribute_ = this.gl_.getAttribLocation( + this.shaderProgram_, 'a_vPosition'); + this.gl_.enableVertexAttribArray(this.vertexPositionAttribute_); + this.textureCoordAttribute_ = this.gl_.getAttribLocation( + this.shaderProgram_, 'a_TexCoordinate'); + this.gl_.enableVertexAttribArray(this.textureCoordAttribute_); + } + + /** + * Read and generate WebGL shader + * + * @param {number} glType Type of shader requested. + * @return {?WebGLShader} + * @private + */ + getGLShader_(glType) { + let source; + + switch (glType) { + case this.gl_.VERTEX_SHADER: + source = shaka.ui.VRUtils.VERTEX_SPHERE_SHADER; + break; + case this.gl_.FRAGMENT_SHADER: + source = shaka.ui.VRUtils.FRAGMENT_SPHERE_SHADER; + break; + default: + return null; + } + + const shader = this.gl_.createShader(glType); + + this.gl_.shaderSource(shader, source); + + this.gl_.compileShader(shader); + + if (!this.gl_.getShaderParameter(shader, this.gl_.COMPILE_STATUS)) { + shaka.log.warning( + 'Error in ' + glType + ' shader' + this.gl_.getShaderInfoLog(shader)); + return null; + } + + return shader; + } + + /** + * @private + */ + initGLBuffers_() { + this.sphere_ = shaka.ui.VRUtils.generateSphere(100); + this.verticesBuffer_ = this.gl_.createBuffer(); + this.gl_.bindBuffer(this.gl_.ARRAY_BUFFER, this.verticesBuffer_); + this.gl_.bufferData(this.gl_.ARRAY_BUFFER, + new Float32Array(this.sphere_.vertices), this.gl_.STATIC_DRAW); + this.verticesTextureCoordBuffer_ = this.gl_.createBuffer(); + this.gl_.bindBuffer( + this.gl_.ARRAY_BUFFER, this.verticesTextureCoordBuffer_); + this.gl_.bufferData(this.gl_.ARRAY_BUFFER, + new Float32Array(this.sphere_.texCoords), this.gl_.STATIC_DRAW); + this.verticesIndexBuffer_ = this.gl_.createBuffer(); + this.gl_.bindBuffer( + this.gl_.ELEMENT_ARRAY_BUFFER, this.verticesIndexBuffer_); + this.gl_.bufferData(this.gl_.ELEMENT_ARRAY_BUFFER, + new Uint16Array(this.sphere_.indices), this.gl_.STATIC_DRAW); + } + + /** + * @private + */ + initGLTexture_() { + this.texture_ = this.gl_.createTexture(); + this.gl_.bindTexture(this.gl_.TEXTURE_2D, this.texture_); + + this.gl_.texParameteri(this.gl_.TEXTURE_2D, + this.gl_.TEXTURE_WRAP_S, this.gl_.CLAMP_TO_EDGE); + this.gl_.texParameteri(this.gl_.TEXTURE_2D, + this.gl_.TEXTURE_WRAP_T, this.gl_.CLAMP_TO_EDGE); + this.gl_.texParameteri(this.gl_.TEXTURE_2D, + this.gl_.TEXTURE_MIN_FILTER, this.gl_.NEAREST); + this.gl_.texParameteri(this.gl_.TEXTURE_2D, + this.gl_.TEXTURE_MAG_FILTER, this.gl_.NEAREST); + } + + /** + * @param {boolean=} textureUpdate + * @private + */ + renderGL_(textureUpdate = true) { + const loadMode = this.player_.getLoadMode(); + const isMSE = loadMode == shaka.Player.LoadMode.MEDIA_SOURCE; + if (!this.video_ || this.video_.readyState < 2 || + (!isMSE && this.video_.playbackRate == 0)) { + return; + } + shaka.ui.Matrix4x4.perspective(this.projectionMatrix_, + this.fieldOfView_ * Math.PI / 180, 5 / 3.2, 0.1, 100.0); + + this.gl_.useProgram(this.shaderProgram_); + + this.gl_.clear(this.gl_.COLOR_BUFFER_BIT); + this.updateViewPort_(); + + if (textureUpdate) { + this.gl_.activeTexture(this.gl_.TEXTURE0); + this.gl_.bindTexture(this.gl_.TEXTURE_2D, this.texture_); + this.gl_.pixelStorei(this.gl_.UNPACK_PREMULTIPLY_ALPHA_WEBGL, 0); + this.gl_.texImage2D(this.gl_.TEXTURE_2D, 0, this.gl_.RGBA, + this.gl_.RGBA, this.gl_.UNSIGNED_BYTE, this.video_); + } + + // Update matrix + shaka.ui.Matrix4x4.multiply(this.viewProjectionMatrix_, + this.viewMatrix_, this.identityMatrix_); + shaka.ui.Matrix4x4.multiply(this.viewProjectionMatrix_, + this.projectionMatrix_, this.viewProjectionMatrix_); + + // Plumbing + // Vertices + this.gl_.bindBuffer(this.gl_.ARRAY_BUFFER, this.verticesBuffer_); + goog.asserts.assert(this.vertexPositionAttribute_ != null, + 'Should have a texture attribute!'); + this.gl_.vertexAttribPointer( + this.vertexPositionAttribute_, 3, this.gl_.FLOAT, false, 0, 0); + this.gl_.enableVertexAttribArray(this.vertexPositionAttribute_); + + // UVs + this.gl_.bindBuffer( + this.gl_.ARRAY_BUFFER, this.verticesTextureCoordBuffer_); + goog.asserts.assert(this.textureCoordAttribute_ != null, + 'Should have a texture attribute!'); + this.gl_.vertexAttribPointer( + this.textureCoordAttribute_, 2, this.gl_.FLOAT, false, 0, 0); + this.gl_.enableVertexAttribArray(this.textureCoordAttribute_); + + this.gl_.bindBuffer( + this.gl_.ELEMENT_ARRAY_BUFFER, this.verticesIndexBuffer_); + + this.setMatrixUniforms_(); + + this.gl_.uniform1i( + this.gl_.getUniformLocation(this.shaderProgram_, 'uSampler'), 0); + + if (this.stereoscopicMode_) { + this.gl_.viewport(0, 0, this.canvas_.width / 2, this.canvas_.height); + } + + // Draw + this.gl_.drawElements(this.gl_.TRIANGLES, + this.sphere_.indices.length, this.gl_.UNSIGNED_SHORT, 0); + + if (this.stereoscopicMode_) { + this.gl_.viewport(this.canvas_.width / 2, 0, + this.canvas_.width / 2, this.canvas_.height); + this.gl_.drawElements(this.gl_.TRIANGLES, + this.sphere_.indices.length, this.gl_.UNSIGNED_SHORT, 0); + } + } + + /** + * @private + */ + setMatrixUniforms_() { + const pUniform = + this.gl_.getUniformLocation(this.shaderProgram_, 'u_VPMatrix'); + this.gl_.uniformMatrix4fv(pUniform, false, this.viewProjectionMatrix_); + } + + /** + * @private + */ + updateViewPort_() { + const scale = 1.8; + let currentWidth = this.video_.videoWidth; + if (!currentWidth) { + currentWidth = this.canvas_.scrollWidth; + } + let currentHeight = this.video_.videoHeight; + if (!currentHeight) { + currentHeight = this.canvas_.scrollHeight; + } + + if (this.previousCanvasWidth_ !== currentWidth || + this.previousCanvasHeight_ !== currentHeight) { + this.canvas_.width = currentWidth; + this.canvas_.height = currentHeight; + + this.previousCanvasWidth_ = currentWidth; + this.previousCanvasHeight_ = currentHeight; + + const ratio = currentWidth / currentHeight; + + this.projectionMatrix_ = shaka.ui.Matrix4x4.frustum( + this.projectionMatrix_, -ratio, ratio, -1, 1, scale, 50.5); + + this.gl_.viewport(0, 0, currentWidth, currentHeight); + } + } + + /** + * Rotate the view matrix global + * + * @param {!number} yaw Yaw. + * @param {!number} pitch Pitch. + * @param {!number} roll Roll. + */ + rotateViewGlobal(yaw, pitch, roll) { + const pitchBoundary = 90.0 * Math.PI / 180; + const matrix = this.viewMatrix_; + + // Rotate global axis + shaka.ui.Matrix4x4.rotateY(matrix, matrix, yaw); + + // Variable to limit the pitch movement + this.positionY_ += pitch; + + if (this.positionY_ < pitchBoundary && + this.positionY_ > -pitchBoundary) { + const out = shaka.ui.Matrix4x4.create(); + shaka.ui.Matrix4x4.rotateX(out, shaka.ui.Matrix4x4.create(), -1 * pitch); + // Rotate local axis + shaka.ui.Matrix4x4.multiply(matrix, out, matrix); + } else { + // Doing this we restart the value to the previous position, + // to not mantain a value over 90º or under -90º. + this.positionY_ -= pitch; + } + + const out2 = shaka.ui.Matrix4x4.create(); + shaka.ui.Matrix4x4.rotateZ(out2, shaka.ui.Matrix4x4.create(), roll); + + // Rotate local axis + shaka.ui.Matrix4x4.multiply(matrix, out2, matrix); + + this.renderGL_(false); + } + + /** + * @param {number} amount + */ + zoom(amount) { + const zoomMin = 20; + const zoomMax = 100; + amount /= 50; + if (this.fieldOfView_ >= zoomMin && this.fieldOfView_ <= zoomMax) { + this.fieldOfView_ += amount; + } + if (this.fieldOfView_ < zoomMin) { + this.fieldOfView_ = zoomMin; + } else if (this.fieldOfView_ > zoomMax) { + this.fieldOfView_ = zoomMax; + } + this.renderGL_(false); + } + + /** + * @return {number} + */ + getFieldView() { + return this.fieldOfView_; + } + + /** + * @param {number} fieldView + */ + setFieldView(fieldView) { + this.fieldOfView_ = fieldView; + this.renderGL_(false); + } + + /** + * @return {number} + */ + getNorth() { + shaka.ui.Matrix4x4.getRotation(this.currentQuaternion_, this.viewMatrix_); + + const angles = this.toEulerAngles_(this.currentQuaternion_); + + let result = ((angles.yaw * 180) / Math.PI); + if (result < 0) { + result = 360 + result; + } + return result; + } + + /** + * @param {boolean=} firstTime + */ + reset(firstTime = true) { + const steps = 20; + + if (firstTime) { + shaka.ui.Matrix4x4.getRotation( + this.currentQuaternion_, this.viewMatrix_); + this.cont_ = 0; + this.diff_ = shaka.ui.MatrixQuaternion.create(); + this.diff_[0] = + (this.currentQuaternion_[0] - this.originalQuaternion_[0]) / steps; + this.diff_[1] = + (this.currentQuaternion_[1] - this.originalQuaternion_[1]) / steps; + this.diff_[2] = + (this.currentQuaternion_[2] - this.originalQuaternion_[2]) / steps; + this.diff_[3] = + (this.currentQuaternion_[3] - this.originalQuaternion_[3]) / steps; + } + + this.currentQuaternion_[0] -= this.diff_[0]; + this.currentQuaternion_[1] -= this.diff_[1]; + this.currentQuaternion_[2] -= this.diff_[2]; + this.currentQuaternion_[3] -= this.diff_[3]; + + // Set the view to the original matrix + const out = shaka.ui.Matrix4x4.create(); + + shaka.ui.MatrixQuaternion.normalize( + this.currentQuaternion_, this.currentQuaternion_); + + shaka.ui.Matrix4x4.fromQuat(out, this.currentQuaternion_); + + this.viewMatrix_ = out; + + if (this.cont_ < steps) { + new shaka.util.Timer(() => { + this.reset(false); + this.positionY_ = 0; + this.cont_++; + this.renderGL_(false); + }).tickAfter(this.animDuration_ / steps); + } else { + shaka.ui.Matrix4x4.fromQuat(out, this.originalQuaternion_); + this.viewMatrix_ = out; + this.renderGL_(false); + } + } +};