feat(UI): UI support for VR content

build/types/ui

@@ -41,3 +41,9 @@
 +../../ui/ui.js
 +../../ui/ui_utils.js
 +../../ui/volume_bar.js
++../../ui/vr_manager.js
++../../ui/vr_utils.js
++../../ui/vr_webgl.js
+
++../../ui/gl_matrix/matrix_4x4.js
++../../ui/gl_matrix/matrix_quaternion.js

demo/common/assets.js

@@ -43,6 +43,7 @@ shakaAssets.Source = {
   BRIGHTCOVE: 'Brightcove',
   BROADPEAK: 'Broadpeak',
   EZDRM: 'EZDRM',
+  THEO_PLAYER: 'THEOplayer',
 };
 
 
@@ -165,6 +166,9 @@ shakaAssets.Feature = {
 
   // Set if the asset has Content Steering.
   CONTENT_STEERING: 'Content Steering',
+
+  // Set if the asset is VR.
+  VR: 'VR',
 };
 
 
@@ -1062,6 +1066,15 @@ shakaAssets.testAssets = [
       .addFeature(shakaAssets.Feature.MP4)
       .addFeature(shakaAssets.Feature.THUMBNAILS)
       .addExtraThumbnail('https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/thumbnails/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.vtt'),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'VR Playhouse (DASH, VR)',
+      /* iconUri= */ 'https://cdn.bitmovin.com/content/assets/playhouse-vr/poster.jpg',
+      /* manifestUri= */ 'https://cdn.bitmovin.com/content/assets/playhouse-vr/mpds/105560.mpd',
+      /* source= */ shakaAssets.Source.BITCODIN)
+      .addFeature(shakaAssets.Feature.DASH)
+      .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.VR),
   // End bitcodin assets }}}
 
   // MetaCDN assets {{{
@@ -1679,5 +1692,19 @@ shakaAssets.testAssets = [
         },
       }),
   // }}}
+
+  // THEOplayer assets {{{
+  /* THEOplayer Contents */
+  new ShakaDemoAssetInfo(
+      /* name= */ 'National Geographic (HLS, VR)',
+      /* iconUri= */ 'https://demo.theoplayer.com/hubfs/videos/natgeo/poster.jpg',
+      /* manifestUri= */ 'https://demo.theoplayer.com/hubfs/videos/natgeo/playlist.m3u8',
+      /* source= */ shakaAssets.Source.THEO_PLAYER)
+      .addFeature(shakaAssets.Feature.HLS)
+      .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP2TS)
+      .addFeature(shakaAssets.Feature.VR)
+      .addFeature(shakaAssets.Feature.OFFLINE),
+  // }}}
 ];
 /* eslint-enable max-len */

demo/main.js

@@ -1294,6 +1294,14 @@ shakaDemo.Main = class {
 
       await this.drmConfiguration_(asset);
       this.controls_.getCastProxy().setAppData({'asset': asset});
+      const uiConfig = {
+        displayInVrMode: false,
+      };
+      if (asset.features.includes(shakaAssets.Feature.VR)) {
+        uiConfig.displayInVrMode = true;
+      }
+      const ui = this.video_['ui'];
+      ui.configure(uiConfig);
 
       // Finally, the asset can be loaded.
       if (asset.preloadManager) {

demo/search.js

@@ -397,6 +397,8 @@ shakaDemo.Search = class {
         'Filters for assets that have an LCEVC enhancement layer.');
     this.makeBooleanInput_(specialContainer, Feature.CONTENT_STEERING, FEATURE,
         'Filters for assets that use Content Steering.');
+    this.makeBooleanInput_(specialContainer, Feature.VR, FEATURE,
+        'Filters for assets that are VR.');
 
     container.appendChild(this.resultsDiv_);
   }

docs/tutorials/ui.md

@@ -100,6 +100,25 @@ document.addEventListener('shaka-ui-load-failed', initFailed);
 ```
 
 
+#### Enabling VR
+
+To enable the playback of VR content, there are two possibilities:
+
+1. Enable via UI config:
+```js
+const config = {
+  'displayInVrMode': true
+}
+ui.configure(config);
+```
+
+2. Using HLS or DASH with fMP4 segments and in the init segment exists 'prji'
+   and 'hfov' boxes.
+
+
+Note: VR is only supported for DASH/HLS clear streams or HLS-AES stream.
+
+
 #### Enabling Chromecast support
 
 If you'd like to take advantage of Shaka's built-in Chromecast support,

externs/device_sensor_event.js

@@ -0,0 +1,18 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * @fileoverview Externs for DeviceMotionEvent which were missing in the
+ * Closure compiler.
+ *
+ * @externs
+ */
+
+
+/**
+ * @return {!Promise.<string>}
+ */
+DeviceMotionEvent.requestPermission = function() {};

lib/util/dom_utils.js

@@ -53,6 +53,17 @@ shaka.util.Dom = class {
   }
 
 
+  /**
+   * Cast a Node/Element to an HTMLCanvasElement
+   *
+   * @param {!Node|!Element} original
+   * @return {!HTMLCanvasElement}
+   */
+  static asHTMLCanvasElement(original) {
+    return /** @type {!HTMLCanvasElement}*/ (original);
+  }
+
+
   /**
    * Cast a Node/Element to an HTMLMediaElement
    *

roadmap.md

@@ -25,6 +25,7 @@ v5.0 - 2024 Q1
  - AES-256 and AES-256-CTR (HLS)
    https://github.com/shaka-project/shaka-player/issues/6001
  - Detect maximum HW resolution automatically on some platforms
+ - UI support for VR content
 
 =====
 

shaka-player.uncompiled.js

@@ -95,6 +95,8 @@ goog.require('shaka.ui.FastForwardButton');
 goog.require('shaka.ui.FullscreenButton');
 goog.require('shaka.ui.Localization');
 goog.require('shaka.ui.LoopButton');
+goog.require('shaka.ui.Matrix4x4');
+goog.require('shaka.ui.MatrixQuaternion');
 goog.require('shaka.ui.MuteButton');
 goog.require('shaka.ui.Overlay');
 goog.require('shaka.ui.PipButton');
@@ -109,6 +111,9 @@ goog.require('shaka.ui.Spacer');
 goog.require('shaka.ui.StatisticsButton');
 goog.require('shaka.ui.TextSelection');
 goog.require('shaka.ui.VolumeBar');
+goog.require('shaka.ui.VRManager');
+goog.require('shaka.ui.VRUtils');
+goog.require('shaka.ui.VRWebgl');
 goog.require('shaka.util.Dom');
 goog.require('shaka.util.Error');
 goog.require('shaka.util.FairPlayUtils');

test/test/util/ui_utils.js

@@ -8,15 +8,16 @@ shaka.test.UiUtils = class {
   /**
    * @param {!HTMLElement} videoContainer
    * @param {!HTMLMediaElement} video
+   * @param {!HTMLCanvasElement} canvas
    * @param {!Object=} config
    * @return {!Promise.<!shaka.ui.Overlay>}
    */
-  static async createUIThroughAPI(videoContainer, video, config) {
+  static async createUIThroughAPI(videoContainer, video, canvas, config) {
     const player = new shaka.Player();
     await player.attach(video);
     // Create UI
     config = config || {};
-    const ui = new shaka.ui.Overlay(player, videoContainer, video);
+    const ui = new shaka.ui.Overlay(player, videoContainer, video, canvas);
     // TODO: generate externs automatically from @event types
     // This event should be a shaka.Player.ErrorEvent
     ui.getControls().addEventListener('error', (e) => fail(e['detail']));
@@ -180,4 +181,16 @@ shaka.test.UiUtils = class {
 
     return video;
   }
+
+  /**
+   * Creates a canvas element for testing.
+   *
+   * @return {!HTMLCanvasElement}
+   */
+  static createCanvasElement() {
+    const canvas = /** @type {!HTMLCanvasElement} */(document.createElement(
+        'canvas'));
+
+    return canvas;
+  }
 };

test/text/text_displayer_layout_unit.js

@@ -40,7 +40,8 @@ filterDescribe('Cue layout', shaka.test.TextLayoutTests.supported, () => {
         const player = new shaka.Player();
         ui = new shaka.ui.Overlay(
             player, /** @type {!HTMLElement} */(helper.videoContainer),
-            shaka.test.UiUtils.createVideoElement());
+            shaka.test.UiUtils.createVideoElement(),
+            shaka.test.UiUtils.createCanvasElement());
         // Turn off every part of the UI that we can, so that the screenshot is
         // less likey to change because of something unrelated to text
         // rendering.

test/ui/ad_ui_unit.js

@@ -12,6 +12,8 @@ describe('Ad UI', () => {
   let container;
   /** @type {!HTMLMediaElement} */
   let video;
+  /** @type {!HTMLCanvasElement} */
+  let canvas;
   /** @type {!shaka.test.FakeAd} */
   let ad;
   /** @type {!shaka.test.FakeAdManager} */
@@ -32,7 +34,8 @@ describe('Ad UI', () => {
 
     video = shaka.test.UiUtils.createVideoElement();
     container.appendChild(video);
-    await UiUtils.createUIThroughAPI(container, video);
+    canvas = shaka.test.UiUtils.createCanvasElement();
+    await UiUtils.createUIThroughAPI(container, video, canvas);
     adManager = video['ui'].getControls().getPlayer().getAdManager();
   });
 

test/ui/ui_customization_unit.js

@@ -12,6 +12,8 @@ describe('UI Customization', () => {
   let container;
   /** @type {!HTMLMediaElement} */
   let video;
+  /** @type {!HTMLCanvasElement} */
+  let canvas;
 
   beforeAll(async () => {
     // Add css file
@@ -34,11 +36,13 @@ describe('UI Customization', () => {
 
     video = shaka.test.UiUtils.createVideoElement();
     container.appendChild(video);
+    canvas = shaka.test.UiUtils.createCanvasElement();
+    container.appendChild(canvas);
   });
 
   it('only the specified controls are created', async () => {
     const config = {controlPanelElements: ['time_and_duration', 'mute']};
-    await UiUtils.createUIThroughAPI(container, video, config);
+    await UiUtils.createUIThroughAPI(container, video, canvas, config);
 
     // Only current time and mute button should've been created
     UiUtils.confirmElementFound(container, 'shaka-current-time');
@@ -51,7 +55,7 @@ describe('UI Customization', () => {
 
   it('only the specified overflow menu buttons are created', async () => {
     const config = {overflowMenuButtons: ['cast']};
-    await UiUtils.createUIThroughAPI(container, video, config);
+    await UiUtils.createUIThroughAPI(container, video, canvas, config);
 
     UiUtils.confirmElementFound(container, 'shaka-cast-button');
 
@@ -60,30 +64,31 @@ describe('UI Customization', () => {
 
   it('seek bar only created when configured', async () => {
     const ui = await UiUtils.createUIThroughAPI(
-        container, video, {addSeekBar: false});
+        container, video, canvas, {addSeekBar: false});
     UiUtils.confirmElementMissing(container, 'shaka-seek-bar');
     await ui.destroy();
 
-    await UiUtils.createUIThroughAPI(container, video, {addSeekBar: true});
+    await UiUtils.createUIThroughAPI(
+        container, video, canvas, {addSeekBar: true});
     UiUtils.confirmElementFound(container, 'shaka-seek-bar');
   });
 
   it('big play button only created when configured', async () => {
     const ui = await UiUtils.createUIThroughAPI(
-        container, video, {addBigPlayButton: false});
+        container, video, canvas, {addBigPlayButton: false});
     UiUtils.confirmElementMissing(container, 'shaka-play-button-container');
     UiUtils.confirmElementMissing(container, 'shaka-play-button');
     await ui.destroy();
 
     await UiUtils.createUIThroughAPI(
-        container, video, {addBigPlayButton: true});
+        container, video, canvas, {addBigPlayButton: true});
     UiUtils.confirmElementFound(container, 'shaka-play-button-container');
     UiUtils.confirmElementFound(container, 'shaka-play-button');
   });
 
   it('settings menus are lower when seek bar is absent', async () => {
     const config = {addSeekBar: false};
-    await UiUtils.createUIThroughAPI(container, video, config);
+    await UiUtils.createUIThroughAPI(container, video, canvas, config);
 
     function confirmLowPosition(className) {
       const elements =
@@ -111,7 +116,7 @@ describe('UI Customization', () => {
       ],
     };
 
-    await UiUtils.createUIThroughAPI(container, video, config);
+    await UiUtils.createUIThroughAPI(container, video, canvas, config);
 
     const controlsButtonPanels =
         container.getElementsByClassName('shaka-controls-button-panel');
@@ -132,7 +137,8 @@ describe('UI Customization', () => {
 
   it('layout can be re-configured after the creation', async () => {
     const config = {controlPanelElements: ['time_and_duration', 'mute']};
-    const ui = await UiUtils.createUIThroughAPI(container, video, config);
+    const ui = await UiUtils.createUIThroughAPI(
+        container, video, canvas, config);
 
     // Only current time and mute button should've been created
     UiUtils.confirmElementFound(container, 'shaka-current-time');
@@ -178,7 +184,8 @@ describe('UI Customization', () => {
   it('cast proxy and controls are unchanged by reconfiguration', async () => {
     const config = {controlPanelElements: ['time_and_duration', 'mute']};
     /** @type {!shaka.ui.Overlay} */
-    const ui = await UiUtils.createUIThroughAPI(container, video, config);
+    const ui = await UiUtils.createUIThroughAPI(
+        container, video, canvas, config);
 
     const eventManager = new shaka.util.EventManager();
     const waiter = new shaka.test.Waiter(eventManager);

test/ui/ui_integration.js

@@ -13,6 +13,8 @@ describe('UI', () => {
 
   /** @type {!HTMLVideoElement} */
   let video;
+  /** @type {!HTMLCanvasElement} */
+  let canvas;
   /** @type {!HTMLElement} */
   let videoContainer;
   /** @type {!shaka.Player} */
@@ -41,9 +43,11 @@ describe('UI', () => {
 
   beforeEach(async () => {
     video = shaka.test.UiUtils.createVideoElement();
+    canvas = shaka.test.UiUtils.createCanvasElement();
 
     videoContainer = shaka.util.Dom.createHTMLElement('div');
     videoContainer.appendChild(video);
+    videoContainer.appendChild(canvas);
     document.body.appendChild(videoContainer);
     player = new compiledShaka.Player();
     await player.attach(video);
@@ -70,7 +74,7 @@ describe('UI', () => {
       // TODO: Cast receiver id to test chromecast integration
     };
 
-    ui = new compiledShaka.ui.Overlay(player, videoContainer, video);
+    ui = new compiledShaka.ui.Overlay(player, videoContainer, video, canvas);
     ui.configure(config);
 
     // Grab event manager from the uncompiled library: