Source: lib/text/mp4_vtt_parser.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.text.Mp4VttParser');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.log');
  11. goog.require('shaka.text.Cue');
  12. goog.require('shaka.text.TextEngine');
  13. goog.require('shaka.text.VttTextParser');
  14. goog.require('shaka.util.DataViewReader');
  15. goog.require('shaka.util.Error');
  16. goog.require('shaka.util.Functional');
  17. goog.require('shaka.util.Mp4Parser');
  18. goog.require('shaka.util.Mp4BoxParsers');
  19. goog.require('shaka.util.StringUtils');
  20. goog.require('shaka.util.TextParser');
  23. /**
  24.  * @implements {shaka.extern.TextParser}
  25.  * @export
  26.  */
  27. shaka.text.Mp4VttParser = class {
  28.   /** */
  29.   constructor() {
  30.     /**
  31.      * The current time scale used by the VTT parser.
  32.      *
  33.      * @type {?number}
  34.      * @private
  35.      */
  36.     this.timescale_ = null;
  37.   }
  39.   /**
  40.    * @override
  41.    * @export
  42.    */
  43.   parseInit(data) {
  44.     const Mp4Parser = shaka.util.Mp4Parser;
  46.     let sawWVTT = false;
  48.     new Mp4Parser()
  49.         .box('moov', Mp4Parser.children)
  50.         .box('trak', Mp4Parser.children)
  51.         .box('mdia', Mp4Parser.children)
  52.         .fullBox('mdhd', (box) => {
  53.           goog.asserts.assert(
  54.               box.version == 0 || box.version == 1,
  55.               'MDHD version can only be 0 or 1');
  57.           const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
  58.               box.reader, box.version);
  59.           this.timescale_ = parsedMDHDBox.timescale;
  60.         })
  61.         .box('minf', Mp4Parser.children)
  62.         .box('stbl', Mp4Parser.children)
  63.         .fullBox('stsd', Mp4Parser.sampleDescription)
  64.         .box('wvtt', (box) => {
  65.           // A valid vtt init segment, though we have no actual subtitles yet.
  66.           sawWVTT = true;
  67.         }).parse(data);
  69.     if (!this.timescale_) {
  70.       // Missing timescale for VTT content. It should be located in the MDHD.
  71.       throw new shaka.util.Error(
  72.           shaka.util.Error.Severity.CRITICAL,
  73.           shaka.util.Error.Category.TEXT,
  74.           shaka.util.Error.Code.INVALID_MP4_VTT);
  75.     }
  77.     if (!sawWVTT) {
  78.       // A WVTT box should have been seen (a valid vtt init segment with no
  79.       // actual subtitles).
  80.       throw new shaka.util.Error(
  81.           shaka.util.Error.Severity.CRITICAL,
  82.           shaka.util.Error.Category.TEXT,
  83.           shaka.util.Error.Code.INVALID_MP4_VTT);
  84.     }
  85.   }
  87.   /**
  88.    * @override
  89.    * @export
  90.    */
  91.   setSequenceMode(sequenceMode) {
  92.     // Unused.
  93.   }
  95.   /**
  96.    * @override
  97.    * @export
  98.    */
  99.   setManifestType(manifestType) {
  100.     // Unused.
  101.   }
  103.   /**
  104.    * @override
  105.    * @export
  106.    */
  107.   parseMedia(data, time) {
  108.     if (!data.length) {
  109.       return [];
  110.     }
  112.     if (!this.timescale_) {
  113.       // Missing timescale for VTT content. We should have seen the init
  114.       // segment.
  115.       shaka.log.error('No init segment for MP4+VTT!');
  116.       throw new shaka.util.Error(
  117.           shaka.util.Error.Severity.CRITICAL,
  118.           shaka.util.Error.Category.TEXT,
  119.           shaka.util.Error.Code.INVALID_MP4_VTT);
  120.     }
  122.     const Mp4Parser = shaka.util.Mp4Parser;
  124.     let baseTime = 0;
  125.     /** @type {!Array<shaka.util.ParsedTRUNSample>} */
  126.     let presentations = [];
  127.     /** @type {!Uint8Array} */
  128.     let rawPayload;
  129.     /** @type {!Array<shaka.text.Cue>} */
  130.     const cues = [];
  132.     let sawTFDT = false;
  133.     let sawTRUN = false;
  134.     let sawMDAT = false;
  135.     let defaultDuration = null;
  137.     const parser = new Mp4Parser()
  138.         .box('moof', Mp4Parser.children)
  139.         .box('traf', Mp4Parser.children)
  140.         .fullBox('tfdt', (box) => {
  141.           sawTFDT = true;
  142.           goog.asserts.assert(
  143.               box.version == 0 || box.version == 1,
  144.               'TFDT version can only be 0 or 1');
  146.           const parsedTFDTBox = shaka.util.Mp4BoxParsers.parseTFDTInaccurate(
  147.               box.reader, box.version);
  148.           baseTime = parsedTFDTBox.baseMediaDecodeTime;
  149.         })
  150.         .fullBox('tfhd', (box) => {
  151.           goog.asserts.assert(
  152.               box.flags != null,
  153.               'A TFHD box should have a valid flags value');
  154.           const parsedTFHDBox = shaka.util.Mp4BoxParsers.parseTFHD(
  155.               box.reader, box.flags);
  156.           defaultDuration = parsedTFHDBox.defaultSampleDuration;
  157.         })
  158.         .fullBox('trun', (box) => {
  159.           sawTRUN = true;
  160.           goog.asserts.assert(
  161.               box.version != null,
  162.               'A TRUN box should have a valid version value');
  163.           goog.asserts.assert(
  164.               box.flags != null,
  165.               'A TRUN box should have a valid flags value');
  167.           const parsedTRUNBox = shaka.util.Mp4BoxParsers.parseTRUN(
  168.               box.reader, box.version, box.flags);
  169.           presentations = parsedTRUNBox.sampleData;
  170.         })
  171.         .box('mdat', Mp4Parser.allData((data) => {
  172.           goog.asserts.assert(
  173.               !sawMDAT,
  174.               'VTT cues in mp4 with multiple MDAT are not currently supported');
  175.           sawMDAT = true;
  176.           rawPayload = data;
  177.         }));
  178.     parser.parse(data, /* partialOkay= */ false);
  180.     if (!sawMDAT && !sawTFDT && !sawTRUN) {
  181.       // A required box is missing.
  182.       throw new shaka.util.Error(
  183.           shaka.util.Error.Severity.CRITICAL,
  184.           shaka.util.Error.Category.TEXT,
  185.           shaka.util.Error.Code.INVALID_MP4_VTT);
  186.     }
  188.     let currentTime = baseTime;
  190.     /** @type {!shaka.util.DataViewReader} */
  191.     const reader = new shaka.util.DataViewReader(
  192.         rawPayload, shaka.util.DataViewReader.Endianness.BIG_ENDIAN);
  194.     for (const presentation of presentations) {
  195.       // If one presentation corresponds to multiple payloads, it is assumed
  196.       // that all of those payloads have the same start time and duration.
  197.       const duration = presentation.sampleDuration || defaultDuration;
  198.       const startTime = presentation.sampleCompositionTimeOffset ?
  199.                       baseTime + presentation.sampleCompositionTimeOffset :
  200.                       currentTime;
  201.       currentTime = startTime + (duration || 0);
  203.       // Read samples until it adds up to the given size.
  204.       let totalSize = 0;
  205.       do {
  206.         // Read the payload size.
  207.         const payloadSize = reader.readUint32();
  208.         totalSize += payloadSize;
  210.         // Skip the type.
  211.         const payloadType = reader.readUint32();
  212.         const payloadName = shaka.util.Mp4Parser.typeToString(payloadType);
  214.         // Read the data payload.
  215.         /** @type {Uint8Array} */
  216.         let payload = null;
  217.         if (payloadName == 'vttc') {
  218.           if (payloadSize > 8) {
  219.             payload = reader.readBytes(payloadSize - 8);
  220.           }
  221.         } else if (payloadName == 'vtte') {
  222.           // It's a vtte, which is a vtt cue that is empty. Ignore any data that
  223.           // does exist.
  224.           reader.skip(payloadSize - 8);
  225.         } else {
  226.           shaka.log.error('Unknown box ' + payloadName + '! Skipping!');
  227.           reader.skip(payloadSize - 8);
  228.         }
  230.         if (duration) {
  231.           if (payload) {
  232.             goog.asserts.assert(
  233.                 this.timescale_ != null, 'Timescale should not be null!');
  234.             const cue = shaka.text.Mp4VttParser.parseVTTC_(
  235.                 payload,
  236.                 time.periodStart + startTime / this.timescale_,
  237.                 time.periodStart + currentTime / this.timescale_);
  238.             cues.push(cue);
  239.           }
  240.         } else {
  241.           shaka.log.error(
  242.               'WVTT sample duration unknown, and no default found!');
  243.         }
  245.         goog.asserts.assert(
  246.             !presentation.sampleSize || totalSize <= presentation.sampleSize,
  247.             'The samples do not fit evenly into the sample sizes given in ' +
  248.             'the TRUN box!');
  250.         // If no sampleSize was specified, it's assumed that this presentation
  251.         // corresponds to only a single cue.
  252.       } while (presentation.sampleSize &&
  253.                (totalSize < presentation.sampleSize));
  254.     }
  256.     goog.asserts.assert(
  257.         !reader.hasMoreData(),
  258.         'MDAT which contain VTT cues and non-VTT data are not currently ' +
  259.         'supported!');
  261.     return /** @type {!Array<!shaka.text.Cue>} */ (
  262.       cues.filter(shaka.util.Functional.isNotNull));
  263.   }
  265.   /**
  266.    * Parses a vttc box into a cue.
  267.    *
  268.    * @param {!Uint8Array} data
  269.    * @param {number} startTime
  270.    * @param {number} endTime
  271.    * @return {shaka.text.Cue}
  272.    * @private
  273.    */
  274.   static parseVTTC_(data, startTime, endTime) {
  275.     let payload;
  276.     let id;
  277.     let settings;
  279.     new shaka.util.Mp4Parser()
  280.         .box('payl', shaka.util.Mp4Parser.allData((data) => {
  281.           payload = shaka.util.StringUtils.fromUTF8(data);
  282.         }))
  283.         .box('iden', shaka.util.Mp4Parser.allData((data) => {
  284.           id = shaka.util.StringUtils.fromUTF8(data);
  285.         }))
  286.         .box('sttg', shaka.util.Mp4Parser.allData((data) => {
  287.           settings = shaka.util.StringUtils.fromUTF8(data);
  288.         }))
  289.         .parse(data);
  291.     if (payload) {
  292.       return shaka.text.Mp4VttParser.assembleCue_(
  293.           payload, id, settings, startTime, endTime);
  294.     } else {
  295.       return null;
  296.     }
  297.   }
  299.   /**
  300.    * Take the individual components that make a cue and create a vttc cue.
  301.    *
  302.    * @param {string} payload
  303.    * @param {?string} id
  304.    * @param {?string} settings
  305.    * @param {number} startTime
  306.    * @param {number} endTime
  307.    * @return {!shaka.text.Cue}
  308.    * @private
  309.    */
  310.   static assembleCue_(payload, id, settings, startTime, endTime) {
  311.     const cue = new shaka.text.Cue(startTime, endTime, payload);
  312.     shaka.text.Cue.parseCuePayload(cue);
  314.     if (id) {
  315.       cue.id = id;
  316.     }
  318.     if (settings) {
  319.       const parser = new shaka.util.TextParser(settings);
  321.       let word = parser.readWord();
  323.       while (word) {
  324.         // TODO: Check WebVTTConfigurationBox for region info.
  325.         if (!shaka.text.VttTextParser.parseCueSetting(
  326.             cue, word, /* VTTRegions= */[])) {
  327.           shaka.log.warning(
  328.               'VTT parser encountered an invalid VTT setting: ', word,
  329.               ' The setting will be ignored.');
  330.         }
  332.         parser.skipWhitespace();
  333.         word = parser.readWord();
  334.       }
  335.     }
  337.     return cue;
  338.   }
  339. };
  341. shaka.text.TextEngine.registerParser(
  342.     'application/mp4; codecs="wvtt"', () => new shaka.text.Mp4VttParser());