MediaCapabilities enhancements (#7354)

robwalch · web-flow · commit 8727c8d68ffd · 2025-07-01T07:59:29.000-07:00
* Combine audio and video into MediaCapabilities queries 
* Add bitrate to MediaCapabilities decodingInfo AudioConfiguration
* Fix leaky support info cache
diff --git a/src/controller/abr-controller.ts b/src/controller/abr-controller.ts
@@ -8,7 +8,6 @@ import {
   requiresMediaCapabilitiesDecodingInfo,
   SUPPORTED_INFO_DEFAULT,
 } from '../utils/mediacapabilities-helper';
-import { isHEVC } from '../utils/mp4-tools';
 import {
   type AudioTracksByGroup,
   type CodecSetTier,
@@ -47,6 +46,10 @@ class AbrController extends Logger implements AbrComponentAPI {
   private partCurrent: Part | null = null;
   private bitrateTestDelay: number = 0;
   private rebufferNotice: number = -1;
+  private supportedCache: Record<
+    string,
+    Promise<MediaCapabilitiesDecodingInfo>
+  > = {};
 
   public bwEstimator: EwmaBandWidthEstimator;
 
@@ -108,7 +111,7 @@ class AbrController extends Logger implements AbrComponentAPI {
     this.unregisterListeners();
     this.clearTimer();
     // @ts-ignore
-    this.hls = this._abandonRulesCheck = null;
+    this.hls = this._abandonRulesCheck = this.supportedCache = null;
     this.fragCurrent = this.partCurrent = null;
   }
 
@@ -119,6 +122,7 @@ class AbrController extends Logger implements AbrComponentAPI {
     this.lastLoadedFragLevel = -1;
     this.firstSelection = -1;
     this.lastLevelLoadSec = 0;
+    this.supportedCache = {};
     this.fragCurrent = this.partCurrent = null;
     this.onLevelsUpdated();
     this.clearTimer();
@@ -826,20 +830,20 @@ class AbrController extends Logger implements AbrComponentAPI {
           | undefined;
         if (
           typeof mediaCapabilities?.decodingInfo === 'function' &&
-          (requiresMediaCapabilitiesDecodingInfo(
+          requiresMediaCapabilitiesDecodingInfo(
             levelInfo,
             audioTracksByGroup,
             currentVideoRange,
             currentFrameRate,
             currentBw,
             audioPreference,
-          ) ||
-            isHEVC(levelInfo.videoCodec)) // Force media capabilities check for HEVC to avoid failure on Windows
+          )
         ) {
           levelInfo.supportedPromise = getMediaDecodingInfoPromise(
             levelInfo,
             audioTracksByGroup,
             mediaCapabilities,
+            this.supportedCache,
           );
           levelInfo.supportedPromise.then((decodingInfo) => {
             if (!this.hls) {
@@ -867,6 +871,15 @@ class AbrController extends Logger implements AbrComponentAPI {
                   this.hls.nextLoadLevel = 0;
                 }
               }
+            } else if (
+              decodingInfo.decodingInfoResults.some(
+                (info) =>
+                  info.smooth === false || info.powerEfficient === false,
+              )
+            ) {
+              this.log(
+                `MediaCapabilities decodingInfo for level ${index} not smooth or powerEfficient: ${stringify(decodingInfo)}`,
+              );
             }
           });
         } else {
@@ -883,8 +896,9 @@ class AbrController extends Logger implements AbrComponentAPI {
         (!upSwitch &&
           currentFrameRate > 0 &&
           currentFrameRate < levelInfo.frameRate) ||
-        (levelInfo.supportedResult &&
-          !levelInfo.supportedResult.decodingInfoResults?.[0].smooth)
+        levelInfo.supportedResult?.decodingInfoResults?.some(
+          (info) => info.smooth === false,
+        )
       ) {
         if (!firstSelection || i !== minStartIndex) {
           levelsSkipped.push(i);
diff --git a/src/utils/mediacapabilities-helper.ts b/src/utils/mediacapabilities-helper.ts
@@ -16,8 +16,6 @@ export type MediaDecodingInfo = {
   error?: Error;
 };
 
-type BaseVideoConfiguration = Omit<VideoConfiguration, 'contentType'>;
-
 export const SUPPORTED_INFO_DEFAULT: MediaDecodingInfo = {
   supported: true,
   configurations: [] as MediaDecodingConfiguration[],
@@ -48,11 +46,6 @@ export function getUnsupportedResult(
   };
 }
 
-export const SUPPORTED_INFO_CACHE: Record<
-  string,
-  Promise<MediaCapabilitiesDecodingInfo>
-> = {};
-
 export function requiresMediaCapabilitiesDecodingInfo(
   level: Level,
   audioTracksByGroup: AudioTracksByGroup,
@@ -62,6 +55,7 @@ export function requiresMediaCapabilitiesDecodingInfo(
   audioPreference: AudioSelectionOption | undefined,
 ): boolean {
   // Only test support when configuration is exceeds minimum options
+  const videoCodecs = level.videoCodec;
   const audioGroups = level.audioCodec ? level.audioGroups : null;
   const audioCodecPreference = audioPreference?.audioCodec;
   const channelsPreference = audioPreference?.channels;
@@ -98,8 +92,10 @@ export function requiresMediaCapabilitiesDecodingInfo(
     }
   }
   return (
-    (level.videoCodec !== undefined &&
-      ((level.width > 1920 && level.height > 1088) ||
+    (videoCodecs !== undefined &&
+      // Force media capabilities check for HEVC to avoid failure on Windows
+      (videoCodecs.split(',').some((videoCodec) => isHEVC(videoCodec)) ||
+        (level.width > 1920 && level.height > 1088) ||
         (level.height > 1920 && level.width > 1088) ||
         level.frameRate > Math.max(currentFrameRate, 30) ||
         (level.videoRange !== 'SDR' &&
@@ -117,95 +113,65 @@ export function getMediaDecodingInfoPromise(
   level: Level,
   audioTracksByGroup: AudioTracksByGroup,
   mediaCapabilities: MediaCapabilities | undefined,
+  cache: Record<string, Promise<MediaCapabilitiesDecodingInfo>> = {},
 ): Promise<MediaDecodingInfo> {
   const videoCodecs = level.videoCodec;
-  const audioCodecs = level.audioCodec;
-  if ((!videoCodecs && !audioCodecs) || !mediaCapabilities) {
+  if ((!videoCodecs && !level.audioCodec) || !mediaCapabilities) {
     return Promise.resolve(SUPPORTED_INFO_DEFAULT);
   }
 
   const configurations: MediaDecodingConfiguration[] = [];
 
-  if (videoCodecs) {
-    const baseVideoConfiguration: BaseVideoConfiguration = {
-      width: level.width,
-      height: level.height,
-      bitrate: Math.ceil(Math.max(level.bitrate * 0.9, level.averageBitrate)),
-      // Assume a framerate of 30fps since MediaCapabilities will not accept Level default of 0.
-      framerate: level.frameRate || 30,
+  const videoDecodeList = makeVideoConfigurations(level);
+  const videoCount = videoDecodeList.length;
+  const audioDecodeList = makeAudioConfigurations(
+    level,
+    audioTracksByGroup,
+    videoCount > 0,
+  );
+  const audioCount = audioDecodeList.length;
+  for (let i = videoCount || 1 * audioCount || 1; i--; ) {
+    const configuration: MediaDecodingConfiguration = {
+      type: 'media-source',
     };
-    const videoRange = level.videoRange;
-    if (videoRange !== 'SDR') {
-      baseVideoConfiguration.transferFunction =
-        videoRange.toLowerCase() as TransferFunction;
+    if (videoCount) {
+      configuration.video = videoDecodeList[i % videoCount];
+    }
+    if (audioCount) {
+      configuration.audio = audioDecodeList[i % audioCount];
+      const audioBitrate = configuration.audio.bitrate;
+      if (configuration.video && audioBitrate) {
+        configuration.video.bitrate -= audioBitrate;
+      }
     }
-    const videoCodecsArray = videoCodecs.split(',');
+    configurations.push(configuration);
+  }
+
+  if (videoCodecs) {
     // Override Windows Firefox HEVC MediaCapabilities result (https://github.com/video-dev/hls.js/issues/7046)
     const ua = navigator.userAgent;
     if (
-      videoCodecsArray.some((videoCodec) => isHEVC(videoCodec)) &&
+      videoCodecs.split(',').some((videoCodec) => isHEVC(videoCodec)) &&
       userAgentHevcSupportIsInaccurate()
     ) {
       return Promise.resolve(
         getUnsupportedResult(
           new Error(
-            `Overriding Windows Firefox HEVC MediaCapabilities result based on user-agent sting: (${ua})`,
+            `Overriding Windows Firefox HEVC MediaCapabilities result based on user-agent string: (${ua})`,
           ),
           configurations,
         ),
       );
     }
-    configurations.push.apply(
-      configurations,
-      videoCodecsArray.map((videoCodec) => ({
-        type: 'media-source',
-        video: {
-          ...baseVideoConfiguration,
-          contentType: mimeTypeForCodec(
-            fillInMissingAV01Params(videoCodec),
-            'video',
-          ),
-        },
-      })),
-    );
-  }
-
-  if (audioCodecs && level.audioGroups) {
-    level.audioGroups.forEach((audioGroupId) => {
-      if (!audioGroupId) {
-        return;
-      }
-      audioTracksByGroup.groups[audioGroupId]?.tracks.forEach((audioTrack) => {
-        if (audioTrack.groupId === audioGroupId) {
-          const channels = audioTrack.channels || '';
-          const channelsNumber = parseFloat(channels);
-          if (Number.isFinite(channelsNumber) && channelsNumber > 2) {
-            configurations.push.apply(
-              configurations,
-              audioCodecs.split(',').map((audioCodec) => ({
-                type: 'media-source',
-                audio: {
-                  contentType: mimeTypeForCodec(audioCodec, 'audio'),
-                  channels: '' + channelsNumber,
-                  // spatialRendering:
-                  //   audioCodec === 'ec-3' && channels.indexOf('JOC'),
-                },
-              })),
-            );
-          }
-        }
-      });
-    });
   }
 
   return Promise.all(
     configurations.map((configuration) => {
       // Cache MediaCapabilities promises
       const decodingInfoKey = getMediaDecodingInfoKey(configuration);
       return (
-        SUPPORTED_INFO_CACHE[decodingInfoKey] ||
-        (SUPPORTED_INFO_CACHE[decodingInfoKey] =
-          mediaCapabilities.decodingInfo(configuration))
+        cache[decodingInfoKey] ||
+        (cache[decodingInfoKey] = mediaCapabilities.decodingInfo(configuration))
       );
     }),
   )
@@ -222,19 +188,107 @@ export function getMediaDecodingInfoPromise(
     }));
 }
 
+function makeVideoConfigurations(level: Level): VideoConfiguration[] {
+  const videoCodecs = level.videoCodec?.split(',');
+  const bitrate = getVariantDecodingBitrate(level);
+  const width = level.width || 640;
+  const height = level.height || 480;
+  // Assume a framerate of 30fps since MediaCapabilities will not accept Level default of 0.
+  const framerate = level.frameRate || 30;
+  const videoRange = level.videoRange.toLowerCase() as 'sdr' | 'pq' | 'hlg';
+  return videoCodecs
+    ? videoCodecs.map((videoCodec: string) => {
+        const videoConfiguration: VideoConfiguration = {
+          contentType: mimeTypeForCodec(
+            fillInMissingAV01Params(videoCodec),
+            'video',
+          ),
+          width,
+          height,
+          bitrate,
+          framerate,
+        };
+        if (videoRange !== 'sdr') {
+          videoConfiguration.transferFunction = videoRange as TransferFunction;
+        }
+        return videoConfiguration;
+      })
+    : [];
+}
+
+function makeAudioConfigurations(
+  level: Level,
+  audioTracksByGroup: AudioTracksByGroup,
+  hasVideo: boolean,
+): AudioConfiguration[] {
+  const audioCodecs = level.audioCodec?.split(',');
+  const combinedBitrate = getVariantDecodingBitrate(level);
+  if (audioCodecs && level.audioGroups) {
+    return level.audioGroups.reduce((configurations, audioGroupId) => {
+      const tracks = audioGroupId
+        ? audioTracksByGroup.groups[audioGroupId]?.tracks
+        : null;
+      if (tracks) {
+        return tracks.reduce((configs, audioTrack) => {
+          if (audioTrack.groupId === audioGroupId) {
+            const channelsNumber = parseFloat(audioTrack.channels || '');
+            audioCodecs.forEach((audioCodec) => {
+              const audioConfiguration: AudioConfiguration = {
+                contentType: mimeTypeForCodec(audioCodec, 'audio'),
+                bitrate: hasVideo
+                  ? estimatedAudioBitrate(audioCodec, combinedBitrate)
+                  : combinedBitrate,
+              };
+              if (channelsNumber) {
+                audioConfiguration.channels = '' + channelsNumber;
+              }
+              configs.push(audioConfiguration);
+            });
+          }
+          return configs;
+        }, configurations);
+      }
+      return configurations;
+    }, [] as AudioConfiguration[]);
+  }
+  return [];
+}
+
+function estimatedAudioBitrate(
+  audioCodec: string,
+  levelBitrate: number,
+): number {
+  if (levelBitrate <= 1) {
+    return 1;
+  }
+  let audioBitrate = 128000;
+  if (audioCodec === 'ec-3') {
+    audioBitrate = 768000;
+  } else if (audioCodec === 'ac-3') {
+    audioBitrate = 640000;
+  }
+  return Math.min(levelBitrate / 2, audioBitrate); // Don't exceed some % of level bitrate
+}
+
+function getVariantDecodingBitrate(level: Level): number {
+  return (
+    Math.ceil(Math.max(level.bitrate * 0.9, level.averageBitrate) / 1000) *
+      1000 || 1
+  );
+}
+
 function getMediaDecodingInfoKey(config: MediaDecodingConfiguration): string {
+  let key = '';
   const { audio, video } = config;
-  const mediaConfig = video || audio;
-  if (mediaConfig) {
-    const codec = getCodecsForMimeType(mediaConfig.contentType);
-    if (video) {
-      return `r${video.height}x${video.width}f${Math.ceil(video.framerate)}${
-        video.transferFunction || 'sd'
-      }_${codec}_${Math.ceil(video.bitrate / 1e5)}`;
-    }
-    if (audio) {
-      return `c${audio.channels}${audio.spatialRendering ? 's' : 'n'}_${codec}`;
-    }
+  if (video) {
+    const codec = getCodecsForMimeType(video.contentType);
+    key += `${codec}_r${video.height}x${video.width}f${Math.ceil(video.framerate)}${
+      video.transferFunction || 'sd'
+    }_${Math.ceil(video.bitrate / 1e5)}`;
+  }
+  if (audio) {
+    const codec = getCodecsForMimeType(audio.contentType);
+    key += `${video ? '_' : ''}${codec}_c${audio.channels}`;
   }
-  return '';
+  return key;
 }
diff --git a/tests/unit/utils/mediacapabilities-helper.ts b/tests/unit/utils/mediacapabilities-helper.ts
