From dd9a66ee8543c5f5d6f79d34ae22810ebd54bbf1 Mon Sep 17 00:00:00 2001 From: bruefire Date: Wed, 13 Dec 2023 01:35:15 +0900 Subject: [PATCH] added a new node for loading mp4 file. --- SequentialImageLoader.py | 41 +++-- __init__.py | 11 +- js/getVideoFrames_js/LICENSE | 21 +++ js/getVideoFrames_js/README.md | 47 ++++++ js/getVideoFrames_js/mod.js | 161 ++++++++++++++++++ js/main.js | 290 +++++++++++++++++++++------------ 6 files changed, 455 insertions(+), 116 deletions(-) create mode 100644 js/getVideoFrames_js/LICENSE create mode 100644 js/getVideoFrames_js/README.md create mode 100644 js/getVideoFrames_js/mod.js diff --git a/SequentialImageLoader.py b/SequentialImageLoader.py index 3a1c45b..2fda858 100644 --- a/SequentialImageLoader.py +++ b/SequentialImageLoader.py @@ -5,18 +5,7 @@ import numpy as np from PIL import Image, ImageOps -class LoadImagesFromDirInpaint: - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "sequence_id": ("STRING", {"frames_upload": True}), - }, - "optional": { - "start_index": ("INT", {"default": 0, "min": 0, "step": 1}), - "end_index": ("INT", {"default": 0, "min": 0, "step": 1}), - } - } +class LoadImagesInpaint: RETURN_TYPES = ("IMAGE", "IMAGE", "INT") RETURN_NAMES = ("images", "mask_images", "image_count",) @@ -131,3 +120,31 @@ class LoadImagesFromDirInpaint: return img_files + +class LoadImagesFromDirInpaint(LoadImagesInpaint): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "sequence_id": ("STRING", {"frames_upload": True}), + }, + "optional": { + "start_index": ("INT", {"default": 0, "min": 0, "step": 1}), + "end_index": ("INT", {"default": 0, "min": 0, "step": 1}), + } + } + + +class LoadImagesFromFileInpaint(LoadImagesInpaint): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "sequence_id": ("STRING", {"video_upload": True}), + }, + "optional": { + "start_index": ("INT", {"default": 0, "min": 0, "step": 1}), + "end_index": ("INT", {"default": 0, "min": 0, "step": 1}), + } + } + diff --git a/__init__.py b/__init__.py index 4f47eff..2fb9ec7 100644 --- a/__init__.py +++ b/__init__.py @@ -1,10 +1,17 @@ import os, shutil import folder_paths from .SequentialImageLoader import LoadImagesFromDirInpaint +from .SequentialImageLoader import LoadImagesFromFileInpaint -NODE_CLASS_MAPPINGS = { "VFrame Loader With Mask Editor" : LoadImagesFromDirInpaint } -NODE_DISPLAY_NAME_MAPPINGS = { "VFrame Loader With Mask Editor" : "SequentialImageLoader" } +NODE_CLASS_MAPPINGS = { + "VFrame Loader With Mask Editor" : LoadImagesFromDirInpaint, + "Video Loader With Mask Editor" : LoadImagesFromFileInpaint +} +NODE_DISPLAY_NAME_MAPPINGS = { + "VFrame Loader With Mask Editor" : "SequentialImageLoader", + "Video Loader With Mask Editor" : "VideoExtractor", +} __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] diff --git a/js/getVideoFrames_js/LICENSE b/js/getVideoFrames_js/LICENSE new file mode 100644 index 0000000..e8a4019 --- /dev/null +++ b/js/getVideoFrames_js/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 josephrocca + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/js/getVideoFrames_js/README.md b/js/getVideoFrames_js/README.md new file mode 100644 index 0000000..953bcc5 --- /dev/null +++ b/js/getVideoFrames_js/README.md @@ -0,0 +1,47 @@ +Simple JavaScript library to performantly and reliably break a video down into its individual frames using the new WebCodecs API. + + * Currently only supports mp4 files. + * Check browser support: https://caniuse.com/webcodecs + * All credit for this code goes to authors of [official WebCodecs demos](https://w3c.github.io/webcodecs/samples/video-decode-display/) and of [mp4box.js](https://github.com/gpac/mp4box.js/). + +### Usage +```html + +
+ + +``` + + * Simple demo: https://jsbin.com/gehaqureca/edit?html,output + * Decode mp4 and re-encode as webm: https://josephrocca.github.io/createWebMFromFrames.js/demo/ diff --git a/js/getVideoFrames_js/mod.js b/js/getVideoFrames_js/mod.js new file mode 100644 index 0000000..2a3a912 --- /dev/null +++ b/js/getVideoFrames_js/mod.js @@ -0,0 +1,161 @@ +// This is basically a copy-paste of code from here - just repackaging it into a simple library/module. +// https://w3c.github.io/webcodecs/samples/video-decode-display/ + +export default function getVideoFrames(opts={}) { + + let onFinishResolver; + let onFinishPromise = new Promise(r => onFinishResolver=r); + + const decoder = new VideoDecoder({ + output: opts.onFrame, + error: function(e) { + console.error(e); + setStatus("decode", e); + }, + }); + + // Fetch and demux the media data. + const demuxer = new MP4Demuxer(opts.videoUrl, { + onConfig: function(config) { + if(opts.onConfig) opts.onConfig(config); + decoder.configure(config); + }, + onFinish: function() { + if(opts.onFinish) opts.onFinish(); + onFinishResolver(); + }, + onChunk: function(chunk) { + decoder.decode(chunk); + }, + setStatus: function(a, b) { + // console.log("status:", a, b); + }, + videoDecoder: decoder, + }); + + return onFinishPromise; +} + +// Demuxes the first video track of an MP4 file using MP4Box, calling +// `onConfig()` and `onChunk()` with appropriate WebCodecs objects. +class MP4Demuxer { + #onConfig = null; + #onChunk = null; + #onFinish = null; + #setStatus = null; + #file = null; + #videoDecoder = null; + + constructor(uri, {onConfig, onChunk, onFinish, setStatus, videoDecoder}) { + this.#onConfig = onConfig; + this.#onChunk = onChunk; + this.#onFinish = onFinish; + this.#setStatus = setStatus; + this.#videoDecoder = videoDecoder; + + // Configure an MP4Box File for demuxing. + this.#file = MP4Box.createFile(); + this.#file.onError = error => { + console.error(error); + setStatus("demux", error); + } + this.#file.onReady = this.#onReady.bind(this); + this.#file.onSamples = this.#onSamples.bind(this); + + // Fetch the file and pipe the data through. + const fileSink = new MP4FileSink(this.#file, setStatus); + fetch(uri).then(async response => { + // highWaterMark should be large enough for smooth streaming, but lower is better for memory usage. + await response.body.pipeTo(new WritableStream(fileSink, {highWaterMark: 2})); + + await this.#videoDecoder.flush(); + if(this.#onFinish) this.#onFinish(); + }); + } + + // Get the appropriate `description` for a specific track. Assumes that the + // track is H.264 or H.265. + #description(track) { + const trak = this.#file.getTrackById(track.id); + for (const entry of trak.mdia.minf.stbl.stsd.entries) { + if (entry.avcC || entry.hvcC) { + const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN); + if (entry.avcC) { + entry.avcC.write(stream); + } else { + entry.hvcC.write(stream); + } + return new Uint8Array(stream.buffer, 8); // Remove the box header. + } + } + throw "avcC or hvcC not found"; + } + + #onReady(info) { + this.#setStatus("demux", "Ready"); + const track = info.videoTracks[0]; + + // Generate and emit an appropriate VideoDecoderConfig. + this.#onConfig({ + codec: track.codec, + codedHeight: track.video.height, + codedWidth: track.video.width, + description: this.#description(track), + mp4boxFile: this.#file, + info: info, + }); + + // Start demuxing. + this.#file.setExtractionOptions(track.id); + this.#file.start(); + } + + async #onSamples(track_id, ref, samples) { + // Generate and emit an EncodedVideoChunk for each demuxed sample. + for (const sample of samples) { + this.#onChunk(new EncodedVideoChunk({ + type: sample.is_sync ? "key" : "delta", + timestamp: 1e6 * sample.cts / sample.timescale, + duration: 1e6 * sample.duration / sample.timescale, + data: sample.data + })); + } + } +} + + +// Wraps an MP4Box File as a WritableStream underlying sink. +class MP4FileSink { + #setStatus = null; + #file = null; + #offset = 0; + + constructor(file, setStatus) { + this.#file = file; + this.#setStatus = setStatus; + } + + write(chunk) { + // MP4Box.js requires buffers to be ArrayBuffers, but we have a Uint8Array. + const buffer = new ArrayBuffer(chunk.byteLength); + new Uint8Array(buffer).set(chunk); + + // Inform MP4Box where in the file this chunk is from. + buffer.fileStart = this.#offset; + this.#offset += buffer.byteLength; + + // Append chunk. + this.#setStatus("fetch", (this.#offset / (1024 ** 2)).toFixed(1) + " MiB"); + this.#file.appendBuffer(buffer); + } + + close() { + this.#setStatus("fetch", "Done"); + this.#file.flush(); + } +} + + + +/*! mp4box 23-09-2022 */ +var Log=function(){var i=new Date,r=4;return{setLogLevel:function(t){r=t==this.debug?1:t==this.info?2:t==this.warn?3:(this.error,4)},debug:function(t,e){void 0===console.debug&&(console.debug=console.log),r<=1&&console.debug("["+Log.getDurationString(new Date-i,1e3)+"]","["+t+"]",e)},log:function(t,e){this.debug(t.msg)},info:function(t,e){r<=2&&console.info("["+Log.getDurationString(new Date-i,1e3)+"]","["+t+"]",e)},warn:function(t,e){r<=3&&console.warn("["+Log.getDurationString(new Date-i,1e3)+"]","["+t+"]",e)},error:function(t,e){r<=4&&console.error("["+Log.getDurationString(new Date-i,1e3)+"]","["+t+"]",e)}}}();Log.getDurationString=function(t,e){var i;function r(t,e){for(var i=(""+t).split(".");i[0].length=this.getEndPosition()},MP4BoxStream.prototype.readAnyInt=function(t,e){var i=0;if(this.position+t<=this.buffer.byteLength){switch(t){case 1:i=e?this.dataview.getInt8(this.position):this.dataview.getUint8(this.position);break;case 2:i=e?this.dataview.getInt16(this.position):this.dataview.getUint16(this.position);break;case 3:if(e)throw"No method for reading signed 24 bits values";i=this.dataview.getUint8(this.position)<<16,i|=this.dataview.getUint8(this.position+1)<<8,i|=this.dataview.getUint8(this.position+2);break;case 4:i=e?this.dataview.getInt32(this.position):this.dataview.getUint32(this.position);break;case 8:if(e)throw"No method for reading signed 64 bits values";i=this.dataview.getUint32(this.position)<<32,i|=this.dataview.getUint32(this.position+4);break;default:throw"readInt method not implemented for size: "+t}return this.position+=t,i}throw"Not enough bytes in buffer"},MP4BoxStream.prototype.readUint8=function(){return this.readAnyInt(1,!1)},MP4BoxStream.prototype.readUint16=function(){return this.readAnyInt(2,!1)},MP4BoxStream.prototype.readUint24=function(){return this.readAnyInt(3,!1)},MP4BoxStream.prototype.readUint32=function(){return this.readAnyInt(4,!1)},MP4BoxStream.prototype.readUint64=function(){return this.readAnyInt(8,!1)},MP4BoxStream.prototype.readString=function(t){if(this.position+t<=this.buffer.byteLength){for(var e="",i=0;ithis._byteLength&&(this._byteLength=e);else{for(i<1&&(i=1);i=this._byteLength},DataStream.prototype.mapUint8Array=function(t){this._realloc(+t);var e=new Uint8Array(this._buffer,this.byteOffset+this.position,t);return this.position+=+t,e},DataStream.prototype.readInt32Array=function(t,e){t=null==t?this.byteLength-this.position/4:t;var i=new Int32Array(t);return DataStream.memcpy(i.buffer,0,this.buffer,this.byteOffset+this.position,t*i.BYTES_PER_ELEMENT),DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=i.byteLength,i},DataStream.prototype.readInt16Array=function(t,e){t=null==t?this.byteLength-this.position/2:t;var i=new Int16Array(t);return DataStream.memcpy(i.buffer,0,this.buffer,this.byteOffset+this.position,t*i.BYTES_PER_ELEMENT),DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=i.byteLength,i},DataStream.prototype.readInt8Array=function(t){t=null==t?this.byteLength-this.position:t;var e=new Int8Array(t);return DataStream.memcpy(e.buffer,0,this.buffer,this.byteOffset+this.position,t*e.BYTES_PER_ELEMENT),this.position+=e.byteLength,e},DataStream.prototype.readUint32Array=function(t,e){t=null==t?this.byteLength-this.position/4:t;var i=new Uint32Array(t);return DataStream.memcpy(i.buffer,0,this.buffer,this.byteOffset+this.position,t*i.BYTES_PER_ELEMENT),DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=i.byteLength,i},DataStream.prototype.readUint16Array=function(t,e){t=null==t?this.byteLength-this.position/2:t;var i=new Uint16Array(t);return DataStream.memcpy(i.buffer,0,this.buffer,this.byteOffset+this.position,t*i.BYTES_PER_ELEMENT),DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=i.byteLength,i},DataStream.prototype.readUint8Array=function(t){t=null==t?this.byteLength-this.position:t;var e=new Uint8Array(t);return DataStream.memcpy(e.buffer,0,this.buffer,this.byteOffset+this.position,t*e.BYTES_PER_ELEMENT),this.position+=e.byteLength,e},DataStream.prototype.readFloat64Array=function(t,e){t=null==t?this.byteLength-this.position/8:t;var i=new Float64Array(t);return DataStream.memcpy(i.buffer,0,this.buffer,this.byteOffset+this.position,t*i.BYTES_PER_ELEMENT),DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=i.byteLength,i},DataStream.prototype.readFloat32Array=function(t,e){t=null==t?this.byteLength-this.position/4:t;var i=new Float32Array(t);return DataStream.memcpy(i.buffer,0,this.buffer,this.byteOffset+this.position,t*i.BYTES_PER_ELEMENT),DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=i.byteLength,i},DataStream.prototype.readInt32=function(t){t=this._dataView.getInt32(this.position,null==t?this.endianness:t);return this.position+=4,t},DataStream.prototype.readInt16=function(t){t=this._dataView.getInt16(this.position,null==t?this.endianness:t);return this.position+=2,t},DataStream.prototype.readInt8=function(){var t=this._dataView.getInt8(this.position);return this.position+=1,t},DataStream.prototype.readUint32=function(t){t=this._dataView.getUint32(this.position,null==t?this.endianness:t);return this.position+=4,t},DataStream.prototype.readUint16=function(t){t=this._dataView.getUint16(this.position,null==t?this.endianness:t);return this.position+=2,t},DataStream.prototype.readUint8=function(){var t=this._dataView.getUint8(this.position);return this.position+=1,t},DataStream.prototype.readFloat32=function(t){t=this._dataView.getFloat32(this.position,null==t?this.endianness:t);return this.position+=4,t},DataStream.prototype.readFloat64=function(t){t=this._dataView.getFloat64(this.position,null==t?this.endianness:t);return this.position+=8,t},DataStream.endianness=0>16),this.writeUint8((65280&t)>>8),this.writeUint8(255&t)},DataStream.prototype.adjustUint32=function(t,e){var i=this.position;this.seek(t),this.writeUint32(e),this.seek(i)},DataStream.prototype.mapInt32Array=function(t,e){this._realloc(4*t);var i=new Int32Array(this._buffer,this.byteOffset+this.position,t);return DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=4*t,i},DataStream.prototype.mapInt16Array=function(t,e){this._realloc(2*t);var i=new Int16Array(this._buffer,this.byteOffset+this.position,t);return DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=2*t,i},DataStream.prototype.mapInt8Array=function(t){this._realloc(+t);var e=new Int8Array(this._buffer,this.byteOffset+this.position,t);return this.position+=+t,e},DataStream.prototype.mapUint32Array=function(t,e){this._realloc(4*t);var i=new Uint32Array(this._buffer,this.byteOffset+this.position,t);return DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=4*t,i},DataStream.prototype.mapUint16Array=function(t,e){this._realloc(2*t);var i=new Uint16Array(this._buffer,this.byteOffset+this.position,t);return DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=2*t,i},DataStream.prototype.mapFloat64Array=function(t,e){this._realloc(8*t);var i=new Float64Array(this._buffer,this.byteOffset+this.position,t);return DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=8*t,i},DataStream.prototype.mapFloat32Array=function(t,e){this._realloc(4*t);var i=new Float32Array(this._buffer,this.byteOffset+this.position,t);return DataStream.arrayToNative(i,null==e?this.endianness:e),this.position+=4*t,i};var MultiBufferStream=function(t){this.buffers=[],this.bufferIndex=-1,t&&(this.insertBuffer(t),this.bufferIndex=0)};MultiBufferStream.prototype=new DataStream(new ArrayBuffer,0,DataStream.BIG_ENDIAN),MultiBufferStream.prototype.initialized=function(){var t;return-1r.byteLength){this.buffers.splice(i,1),i--;continue}Log.warn("MultiBufferStream","Buffer (fileStart: "+t.fileStart+" - Length: "+t.byteLength+") already appended, ignoring")}else t.fileStart+t.byteLength<=r.fileStart||(t=this.reduceBuffer(t,0,r.fileStart-t.fileStart)),Log.debug("MultiBufferStream","Appending new buffer (fileStart: "+t.fileStart+" - Length: "+t.byteLength+")"),this.buffers.splice(i,0,t),0===i&&(this.buffer=t);e=!1;break}if(t.fileStart"+this.buffer.byteLength+")"),!0}return!1},MultiBufferStream.prototype.findPosition=function(t,e,i){for(var r=null,s=-1,a=!0===t?0:this.bufferIndex;a=e?(Log.debug("MultiBufferStream","Found position in existing buffer #"+s),s):-1},MultiBufferStream.prototype.findEndContiguousBuf=function(t){var e,i,t=void 0!==t?t:this.bufferIndex,r=this.buffers[t];if(this.buffers.length>t+1)for(e=t+1;e>3;return 31===e&&2<=i.data.length&&(e=32+((7&i.data[0])<<3)+((224&i.data[1])>>5)),e}return null},a.DecoderConfigDescriptor=function(t){a.Descriptor.call(this,4,t)},a.DecoderConfigDescriptor.prototype=new a.Descriptor,a.DecoderConfigDescriptor.prototype.parse=function(t){this.oti=t.readUint8(),this.streamType=t.readUint8(),this.bufferSize=t.readUint24(),this.maxBitrate=t.readUint32(),this.avgBitrate=t.readUint32(),this.size-=13,this.parseRemainingDescriptors(t)},a.DecoderSpecificInfo=function(t){a.Descriptor.call(this,5,t)},a.DecoderSpecificInfo.prototype=new a.Descriptor,a.SLConfigDescriptor=function(t){a.Descriptor.call(this,6,t)},a.SLConfigDescriptor.prototype=new a.Descriptor,this};"undefined"!=typeof exports&&(exports.MPEG4DescriptorParser=MPEG4DescriptorParser);var BoxParser={ERR_INVALID_DATA:-1,ERR_NOT_ENOUGH_DATA:0,OK:1,BASIC_BOXES:["mdat","idat","free","skip","meco","strk"],FULL_BOXES:["hmhd","nmhd","iods","xml ","bxml","ipro","mere"],CONTAINER_BOXES:[["moov",["trak","pssh"]],["trak"],["edts"],["mdia"],["minf"],["dinf"],["stbl",["sgpd","sbgp"]],["mvex",["trex"]],["moof",["traf"]],["traf",["trun","sgpd","sbgp"]],["vttc"],["tref"],["iref"],["mfra",["tfra"]],["meco"],["hnti"],["hinf"],["strk"],["strd"],["sinf"],["rinf"],["schi"],["trgr"],["udta",["kind"]],["iprp",["ipma"]],["ipco"]],boxCodes:[],fullBoxCodes:[],containerBoxCodes:[],sampleEntryCodes:{},sampleGroupEntryCodes:[],trackGroupTypes:[],UUIDBoxes:{},UUIDs:[],initialize:function(){BoxParser.FullBox.prototype=new BoxParser.Box,BoxParser.ContainerBox.prototype=new BoxParser.Box,BoxParser.SampleEntry.prototype=new BoxParser.Box,BoxParser.TrackGroupTypeBox.prototype=new BoxParser.FullBox,BoxParser.BASIC_BOXES.forEach(function(t){BoxParser.createBoxCtor(t)}),BoxParser.FULL_BOXES.forEach(function(t){BoxParser.createFullBoxCtor(t)}),BoxParser.CONTAINER_BOXES.forEach(function(t){BoxParser.createContainerBoxCtor(t[0],null,t[1])})},Box:function(t,e,i){this.type=t,this.size=e,this.uuid=i},FullBox:function(t,e,i){BoxParser.Box.call(this,t,e,i),this.flags=0,this.version=0},ContainerBox:function(t,e,i){BoxParser.Box.call(this,t,e,i),this.boxes=[]},SampleEntry:function(t,e,i,r){BoxParser.ContainerBox.call(this,t,e),this.hdr_size=i,this.start=r},SampleGroupEntry:function(t){this.grouping_type=t},TrackGroupTypeBox:function(t,e){BoxParser.FullBox.call(this,t,e)},createBoxCtor:function(e,t){BoxParser.boxCodes.push(e),BoxParser[e+"Box"]=function(t){BoxParser.Box.call(this,e,t)},BoxParser[e+"Box"].prototype=new BoxParser.Box,t&&(BoxParser[e+"Box"].prototype.parse=t)},createFullBoxCtor:function(e,i){BoxParser[e+"Box"]=function(t){BoxParser.FullBox.call(this,e,t)},BoxParser[e+"Box"].prototype=new BoxParser.FullBox,BoxParser[e+"Box"].prototype.parse=function(t){this.parseFullHeader(t),i&&i.call(this,t)}},addSubBoxArrays:function(t){if(t)for(var e=(this.subBoxNames=t).length,i=0;it.getEndPosition()?(t.seek(a),Log.info("BoxParser","Not enough data in stream to parse the entire '"+h+"' box"),{code:BoxParser.ERR_NOT_ENOUGH_DATA,type:h,size:o,hdr_size:n,start:a}):e?{code:BoxParser.OK,type:h,size:o,hdr_size:n,start:a}:(BoxParser[h+"Box"]?r=new BoxParser[h+"Box"](o):"uuid"!==h?(Log.warn("BoxParser","Unknown box type: '"+h+"'"),(r=new BoxParser.Box(h,o)).has_unparsed_data=!0):BoxParser.UUIDBoxes[s]?r=new BoxParser.UUIDBoxes[s](o):(Log.warn("BoxParser","Unknown uuid type: '"+s+"'"),(r=new BoxParser.Box(h,o)).uuid=s,r.has_unparsed_data=!0),r.hdr_size=n,r.start=a,r.write===BoxParser.Box.prototype.write&&"mdat"!==r.type&&(Log.info("BoxParser","'"+d+"' box writing not yet implemented, keeping unparsed data in memory for later write"),r.parseDataAndRewind(t)),r.parse(t),(a=t.getPosition()-(r.start+r.size))<0?(Log.warn("BoxParser","Parsing of box '"+d+"' did not read the entire indicated box data size (missing "+-a+" bytes), seeking forward"),t.seek(r.start+r.size)):0>10&31,t[1]=this.language>>5&31,t[2]=31&this.language,this.languageString=String.fromCharCode(t[0]+96,t[1]+96,t[2]+96)},BoxParser.SAMPLE_ENTRY_TYPE_VISUAL="Visual",BoxParser.SAMPLE_ENTRY_TYPE_AUDIO="Audio",BoxParser.SAMPLE_ENTRY_TYPE_HINT="Hint",BoxParser.SAMPLE_ENTRY_TYPE_METADATA="Metadata",BoxParser.SAMPLE_ENTRY_TYPE_SUBTITLE="Subtitle",BoxParser.SAMPLE_ENTRY_TYPE_SYSTEM="System",BoxParser.SAMPLE_ENTRY_TYPE_TEXT="Text",BoxParser.SampleEntry.prototype.parseHeader=function(t){t.readUint8Array(6),this.data_reference_index=t.readUint16(),this.hdr_size+=8},BoxParser.SampleEntry.prototype.parse=function(t){this.parseHeader(t),this.data=t.readUint8Array(this.size-this.hdr_size)},BoxParser.SampleEntry.prototype.parseDataAndRewind=function(t){this.parseHeader(t),this.data=t.readUint8Array(this.size-this.hdr_size),this.hdr_size-=8,t.position-=this.size-this.hdr_size},BoxParser.SampleEntry.prototype.parseFooter=function(t){BoxParser.ContainerBox.prototype.parse.call(this,t)},BoxParser.createMediaSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_HINT),BoxParser.createMediaSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_METADATA),BoxParser.createMediaSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_SUBTITLE),BoxParser.createMediaSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_SYSTEM),BoxParser.createMediaSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_TEXT),BoxParser.createMediaSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,function(t){var e;this.parseHeader(t),t.readUint16(),t.readUint16(),t.readUint32Array(3),this.width=t.readUint16(),this.height=t.readUint16(),this.horizresolution=t.readUint32(),this.vertresolution=t.readUint32(),t.readUint32(),this.frame_count=t.readUint16(),e=Math.min(31,t.readUint8()),this.compressorname=t.readString(e),e<31&&t.readString(31-e),this.depth=t.readUint16(),t.readUint16(),this.parseFooter(t)}),BoxParser.createMediaSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,function(t){this.parseHeader(t),t.readUint32Array(2),this.channel_count=t.readUint16(),this.samplesize=t.readUint16(),t.readUint16(),t.readUint16(),this.samplerate=t.readUint32()/65536,this.parseFooter(t)}),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"avc1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"avc2"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"avc3"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"avc4"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"av01"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"dav1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"hvc1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"hev1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"hvt1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"lhe1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"dvh1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"dvhe"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"vvc1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"vvi1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"vvs1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"vvcN"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"vp08"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"vp09"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"avs3"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"j2ki"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"mjp2"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"mjpg"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"mp4a"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"ac-3"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"ac-4"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"ec-3"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"Opus"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"mha1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"mha2"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"mhm1"),BoxParser.createSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"mhm2"),BoxParser.createEncryptedSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_VISUAL,"encv"),BoxParser.createEncryptedSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_AUDIO,"enca"),BoxParser.createEncryptedSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_SUBTITLE,"encu"),BoxParser.createEncryptedSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_SYSTEM,"encs"),BoxParser.createEncryptedSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_TEXT,"enct"),BoxParser.createEncryptedSampleEntryCtor(BoxParser.SAMPLE_ENTRY_TYPE_METADATA,"encm"),BoxParser.createBoxCtor("a1lx",function(t){var e=16*(1+(1&(1&t.readUint8())));this.layer_size=[];for(var i=0;i<3;i++)this.layer_size[i]=16==e?t.readUint16():t.readUint32()}),BoxParser.createBoxCtor("a1op",function(t){this.op_index=t.readUint8()}),BoxParser.createFullBoxCtor("auxC",function(t){this.aux_type=t.readCString();var e=this.size-this.hdr_size-(this.aux_type.length+1);this.aux_subtype=t.readUint8Array(e)}),BoxParser.createBoxCtor("av1C",function(t){var e=t.readUint8();if(e>>7&!1)Log.error("av1C marker problem");else if(this.version=127&e,1===this.version)if(e=t.readUint8(),this.seq_profile=e>>5&7,this.seq_level_idx_0=31&e,e=t.readUint8(),this.seq_tier_0=e>>7&1,this.high_bitdepth=e>>6&1,this.twelve_bit=e>>5&1,this.monochrome=e>>4&1,this.chroma_subsampling_x=e>>3&1,this.chroma_subsampling_y=e>>2&1,this.chroma_sample_position=3&e,e=t.readUint8(),this.reserved_1=e>>5&7,0===this.reserved_1){if(this.initial_presentation_delay_present=e>>4&1,1===this.initial_presentation_delay_present)this.initial_presentation_delay_minus_one=15&e;else if(this.reserved_2=15&e,0!==this.reserved_2)return void Log.error("av1C reserved_2 parsing problem");e=this.size-this.hdr_size-4;this.configOBUs=t.readUint8Array(e)}else Log.error("av1C reserved_1 parsing problem");else Log.error("av1C version "+this.version+" not supported")}),BoxParser.createBoxCtor("avcC",function(t){var e,i;for(this.configurationVersion=t.readUint8(),this.AVCProfileIndication=t.readUint8(),this.profile_compatibility=t.readUint8(),this.AVCLevelIndication=t.readUint8(),this.lengthSizeMinusOne=3&t.readUint8(),this.nb_SPS_nalus=31&t.readUint8(),i=this.size-this.hdr_size-6,this.SPS=[],e=0;e>7):"rICC"!==this.colour_type&&"prof"!==this.colour_type||(this.ICC_profile=t.readUint8Array(this.size-4))}),BoxParser.createFullBoxCtor("cprt",function(t){this.parseLanguage(t),this.notice=t.readCString()}),BoxParser.createFullBoxCtor("cslg",function(t){0===this.version&&(this.compositionToDTSShift=t.readInt32(),this.leastDecodeToDisplayDelta=t.readInt32(),this.greatestDecodeToDisplayDelta=t.readInt32(),this.compositionStartTime=t.readInt32(),this.compositionEndTime=t.readInt32())}),BoxParser.createFullBoxCtor("ctts",function(t){var e,i=t.readUint32();if(this.sample_counts=[],this.sample_offsets=[],0===this.version)for(e=0;e>6,this.bsid=e>>1&31,this.bsmod=(1&e)<<2|i>>6&3,this.acmod=i>>3&7,this.lfeon=i>>2&1,this.bit_rate_code=3&i|t>>5&7}),BoxParser.createBoxCtor("dec3",function(t){var e=t.readUint16();this.data_rate=e>>3,this.num_ind_sub=7&e,this.ind_subs=[];for(var i=0;i>6,r.bsid=s>>1&31,r.bsmod=(1&s)<<4|a>>4&15,r.acmod=a>>1&7,r.lfeon=1&a,r.num_dep_sub=n>>1&15,0>12,t.readUint8Array(20)),e.push(i[s]),128&r)break}this.numMetadataBlocks=e.length+" ("+e.join(", ")+")"}),BoxParser.createBoxCtor("dimm",function(t){this.bytessent=t.readUint64()}),BoxParser.createBoxCtor("dmax",function(t){this.time=t.readUint32()}),BoxParser.createBoxCtor("dmed",function(t){this.bytessent=t.readUint64()}),BoxParser.createBoxCtor("dOps",function(t){if(this.Version=t.readUint8(),this.OutputChannelCount=t.readUint8(),this.PreSkip=t.readUint16(),this.InputSampleRate=t.readUint32(),this.OutputGain=t.readInt16(),this.ChannelMappingFamily=t.readUint8(),0!==this.ChannelMappingFamily){this.StreamCount=t.readUint8(),this.CoupledCount=t.readUint8(),this.ChannelMapping=[];for(var e=0;e>6,this.general_tier_flag=(32&i)>>5,this.general_profile_idc=31&i,this.general_profile_compatibility=t.readUint32(),this.general_constraint_indicator=t.readUint8Array(6),this.general_level_idc=t.readUint8(),this.min_spatial_segmentation_idc=4095&t.readUint16(),this.parallelismType=3&t.readUint8(),this.chroma_format_idc=3&t.readUint8(),this.bit_depth_luma_minus8=7&t.readUint8(),this.bit_depth_chroma_minus8=7&t.readUint8(),this.avgFrameRate=t.readUint16(),i=t.readUint8(),this.constantFrameRate=i>>6,this.numTemporalLayers=(13&i)>>3,this.temporalIdNested=(4&i)>>2,this.lengthSizeMinusOne=3&i,this.nalu_arrays=[];for(var r=t.readUint8(),s=0;s>7,a.nalu_type=63&i;for(var n=t.readUint16(),o=0;o>4&15,this.length_size=15&e,e=t.readUint8(),this.base_offset_size=e>>4&15,1===this.version||2===this.version?this.index_size=15&e:this.index_size=0,this.items=[];var i=0;if(this.version<2)i=t.readUint16();else{if(2!==this.version)throw"version of iloc box not supported";i=t.readUint32()}for(var r=0;r>7,this.axis=1&t}),BoxParser.createFullBoxCtor("infe",function(t){return 0!==this.version&&1!==this.version||(this.item_ID=t.readUint16(),this.item_protection_index=t.readUint16(),this.item_name=t.readCString(),this.content_type=t.readCString(),this.content_encoding=t.readCString()),1===this.version?(this.extension_type=t.readString(4),Log.warn("BoxParser","Cannot parse extension type"),void t.seek(this.start+this.size)):void(2<=this.version&&(2===this.version?this.item_ID=t.readUint16():3===this.version&&(this.item_ID=t.readUint32()),this.item_protection_index=t.readUint16(),this.item_type=t.readString(4),this.item_name=t.readCString(),"mime"===this.item_type?(this.content_type=t.readCString(),this.content_encoding=t.readCString()):"uri "===this.item_type&&(this.item_uri_type=t.readCString())))}),BoxParser.createFullBoxCtor("ipma",function(t){var e,i;for(entry_count=t.readUint32(),this.associations=[],e=0;e>7==1,1&this.flags?n.property_index=(127&a)<<8|t.readUint8():n.property_index=127&a}}}),BoxParser.createFullBoxCtor("iref",function(t){var e;for(this.references=[];t.getPosition()>7,r.assignment_type=127&s,r.assignment_type){case 0:r.grouping_type=t.readString(4);break;case 1:r.grouping_type=t.readString(4),r.grouping_type_parameter=t.readUint32();break;case 2:case 3:break;case 4:r.sub_track_id=t.readUint32();break;default:Log.warn("BoxParser","Unknown leva assignement type")}}}),BoxParser.createBoxCtor("lsel",function(t){this.layer_id=t.readUint16()}),BoxParser.createBoxCtor("maxr",function(t){this.period=t.readUint32(),this.bytes=t.readUint32()}),BoxParser.createBoxCtor("mdcv",function(t){this.display_primaries=[],this.display_primaries[0]={},this.display_primaries[0].x=t.readUint16(),this.display_primaries[0].y=t.readUint16(),this.display_primaries[1]={},this.display_primaries[1].x=t.readUint16(),this.display_primaries[1].y=t.readUint16(),this.display_primaries[2]={},this.display_primaries[2].x=t.readUint16(),this.display_primaries[2].y=t.readUint16(),this.white_point={},this.white_point.x=t.readUint16(),this.white_point.y=t.readUint16(),this.max_display_mastering_luminance=t.readUint32(),this.min_display_mastering_luminance=t.readUint32()}),BoxParser.createFullBoxCtor("mdhd",function(t){1==this.version?(this.creation_time=t.readUint64(),this.modification_time=t.readUint64(),this.timescale=t.readUint32(),this.duration=t.readUint64()):(this.creation_time=t.readUint32(),this.modification_time=t.readUint32(),this.timescale=t.readUint32(),this.duration=t.readUint32()),this.parseLanguage(t),t.readUint16()}),BoxParser.createFullBoxCtor("mehd",function(t){1&this.flags&&(Log.warn("BoxParser","mehd box incorrectly uses flags set to 1, converting version to 1"),this.version=1),1==this.version?this.fragment_duration=t.readUint64():this.fragment_duration=t.readUint32()}),BoxParser.createFullBoxCtor("meta",function(t){this.boxes=[],BoxParser.ContainerBox.prototype.parse.call(this,t)}),BoxParser.createFullBoxCtor("mfhd",function(t){this.sequence_number=t.readUint32()}),BoxParser.createFullBoxCtor("mfro",function(t){this._size=t.readUint32()}),BoxParser.createFullBoxCtor("mvhd",function(t){1==this.version?(this.creation_time=t.readUint64(),this.modification_time=t.readUint64(),this.timescale=t.readUint32(),this.duration=t.readUint64()):(this.creation_time=t.readUint32(),this.modification_time=t.readUint32(),this.timescale=t.readUint32(),this.duration=t.readUint32()),this.rate=t.readUint32(),this.volume=t.readUint16()>>8,t.readUint16(),t.readUint32Array(2),this.matrix=t.readUint32Array(9),t.readUint32Array(6),this.next_track_id=t.readUint32()}),BoxParser.createBoxCtor("npck",function(t){this.packetssent=t.readUint32()}),BoxParser.createBoxCtor("nump",function(t){this.packetssent=t.readUint64()}),BoxParser.createFullBoxCtor("padb",function(t){var e=t.readUint32();this.padbits=[];for(var i=0;i>7,this.avgRateFlag=e>>6&1,this.durationFlag&&(this.duration=t.readUint32()),this.avgRateFlag&&(this.accurateStatisticsFlag=t.readUint8(),this.avgBitRate=t.readUint16(),this.avgFrameRate=t.readUint16()),this.dependency=[];for(var i=t.readUint8(),r=0;r>7,this.num_leading_samples=127&t}),BoxParser.createSampleGroupCtor("rash",function(t){if(this.operation_point_count=t.readUint16(),this.description_length!==2+(1===this.operation_point_count?2:6*this.operation_point_count)+9)Log.warn("BoxParser","Mismatch in "+this.grouping_type+" sample group length"),this.data=t.readUint8Array(this.description_length-2);else{if(1===this.operation_point_count)this.target_rate_share=t.readUint16();else{this.target_rate_share=[],this.available_bitrate=[];for(var e=0;e>4,this.skip_byte_block=15&e,this.isProtected=t.readUint8(),this.Per_Sample_IV_Size=t.readUint8(),this.KID=BoxParser.parseHex16(t),this.constant_IV_size=0,this.constant_IV=0,1===this.isProtected&&0===this.Per_Sample_IV_Size&&(this.constant_IV_size=t.readUint8(),this.constant_IV=t.readUint8Array(this.constant_IV_size))}),BoxParser.createSampleGroupCtor("stsa",function(t){Log.warn("BoxParser","Sample Group type: "+this.grouping_type+" not fully parsed")}),BoxParser.createSampleGroupCtor("sync",function(t){t=t.readUint8();this.NAL_unit_type=63&t}),BoxParser.createSampleGroupCtor("tele",function(t){t=t.readUint8();this.level_independently_decodable=t>>7}),BoxParser.createSampleGroupCtor("tsas",function(t){Log.warn("BoxParser","Sample Group type: "+this.grouping_type+" not fully parsed")}),BoxParser.createSampleGroupCtor("tscl",function(t){Log.warn("BoxParser","Sample Group type: "+this.grouping_type+" not fully parsed")}),BoxParser.createSampleGroupCtor("vipr",function(t){Log.warn("BoxParser","Sample Group type: "+this.grouping_type+" not fully parsed")}),BoxParser.createFullBoxCtor("sbgp",function(t){this.grouping_type=t.readString(4),1===this.version?this.grouping_type_parameter=t.readUint32():this.grouping_type_parameter=0,this.entries=[];for(var e=t.readUint32(),i=0;i>6,this.sample_depends_on[r]=e>>4&3,this.sample_is_depended_on[r]=e>>2&3,this.sample_has_redundancy[r]=3&e}),BoxParser.createFullBoxCtor("senc"),BoxParser.createFullBoxCtor("sgpd",function(t){this.grouping_type=t.readString(4),Log.debug("BoxParser","Found Sample Groups of type "+this.grouping_type),1===this.version?this.default_length=t.readUint32():this.default_length=0,2<=this.version&&(this.default_group_description_index=t.readUint32()),this.entries=[];for(var e=t.readUint32(),i=0;i>31&1,r.referenced_size=2147483647&s,r.subsegment_duration=t.readUint32(),s=t.readUint32(),r.starts_with_SAP=s>>31&1,r.SAP_type=s>>28&7,r.SAP_delta_time=268435455&s}}),BoxParser.SingleItemTypeReferenceBox=function(t,e,i,r){BoxParser.Box.call(this,t,e),this.hdr_size=i,this.start=r},BoxParser.SingleItemTypeReferenceBox.prototype=new BoxParser.Box,BoxParser.SingleItemTypeReferenceBox.prototype.parse=function(t){this.from_item_ID=t.readUint16();var e=t.readUint16();this.references=[];for(var i=0;i>4&15,this.sample_sizes[e+1]=15&r}else if(8===this.field_size)for(e=0;e>4&15,this.default_skip_byte_block=15&e),this.default_isProtected=t.readUint8(),this.default_Per_Sample_IV_Size=t.readUint8(),this.default_KID=BoxParser.parseHex16(t),1===this.default_isProtected&&0===this.default_Per_Sample_IV_Size&&(this.default_constant_IV_size=t.readUint8(),this.default_constant_IV=t.readUint8Array(this.default_constant_IV_size))}),BoxParser.createFullBoxCtor("tfdt",function(t){1==this.version?this.baseMediaDecodeTime=t.readUint64():this.baseMediaDecodeTime=t.readUint32()}),BoxParser.createFullBoxCtor("tfhd",function(t){var e=0;this.track_id=t.readUint32(),this.size-this.hdr_size>e&&this.flags&BoxParser.TFHD_FLAG_BASE_DATA_OFFSET?(this.base_data_offset=t.readUint64(),e+=8):this.base_data_offset=0,this.size-this.hdr_size>e&&this.flags&BoxParser.TFHD_FLAG_SAMPLE_DESC?(this.default_sample_description_index=t.readUint32(),e+=4):this.default_sample_description_index=0,this.size-this.hdr_size>e&&this.flags&BoxParser.TFHD_FLAG_SAMPLE_DUR?(this.default_sample_duration=t.readUint32(),e+=4):this.default_sample_duration=0,this.size-this.hdr_size>e&&this.flags&BoxParser.TFHD_FLAG_SAMPLE_SIZE?(this.default_sample_size=t.readUint32(),e+=4):this.default_sample_size=0,this.size-this.hdr_size>e&&this.flags&BoxParser.TFHD_FLAG_SAMPLE_FLAGS?(this.default_sample_flags=t.readUint32(),e+=4):this.default_sample_flags=0}),BoxParser.createFullBoxCtor("tfra",function(t){this.track_ID=t.readUint32(),t.readUint24();var e=t.readUint8();this.length_size_of_traf_num=e>>4&3,this.length_size_of_trun_num=e>>2&3,this.length_size_of_sample_num=3&e,this.entries=[];for(var i=t.readUint32(),r=0;r>8,t.readUint16(),this.matrix=t.readInt32Array(9),this.width=t.readUint32(),this.height=t.readUint32()}),BoxParser.createBoxCtor("tmax",function(t){this.time=t.readUint32()}),BoxParser.createBoxCtor("tmin",function(t){this.time=t.readUint32()}),BoxParser.createBoxCtor("totl",function(t){this.bytessent=t.readUint32()}),BoxParser.createBoxCtor("tpay",function(t){this.bytessent=t.readUint32()}),BoxParser.createBoxCtor("tpyl",function(t){this.bytessent=t.readUint64()}),BoxParser.TrackGroupTypeBox.prototype.parse=function(t){this.parseFullHeader(t),this.track_group_id=t.readUint32()},BoxParser.createTrackGroupCtor("msrc"),BoxParser.TrackReferenceTypeBox=function(t,e,i,r){BoxParser.Box.call(this,t,e),this.hdr_size=i,this.start=r},BoxParser.TrackReferenceTypeBox.prototype=new BoxParser.Box,BoxParser.TrackReferenceTypeBox.prototype.parse=function(t){this.track_ids=t.readUint32Array((this.size-this.hdr_size)/4)},BoxParser.trefBox.prototype.parse=function(t){for(var e;t.getPosition()e&&this.flags&BoxParser.TRUN_FLAGS_DATA_OFFSET?(this.data_offset=t.readInt32(),e+=4):this.data_offset=0,this.size-this.hdr_size>e&&this.flags&BoxParser.TRUN_FLAGS_FIRST_FLAG?(this.first_sample_flags=t.readUint32(),e+=4):this.first_sample_flags=0,this.sample_duration=[],this.sample_size=[],this.sample_flags=[],this.sample_composition_time_offset=[],this.size-this.hdr_size>e)for(var i=0;i/g,">").replace(/"/g,""").replace(/'/g,"'")}),BoxParser.createUUIDBox("d08a4f1810f34a82b6c832d8aba183d3",!0,!1,function(t){this.system_id=BoxParser.parseHex16(t);var e=t.readUint32();0>4,this.chromaSubsampling=e>>1&7,this.videoFullRangeFlag=1&e,this.colourPrimaries=t.readUint8(),this.transferCharacteristics=t.readUint8(),this.matrixCoefficients=t.readUint8()):(this.profile=t.readUint8(),this.level=t.readUint8(),e=t.readUint8(),this.bitDepth=e>>4&15,this.colorSpace=15&e,e=t.readUint8(),this.chromaSubsampling=e>>4&15,this.transferFunction=e>>1&7,this.videoFullRangeFlag=1&e),this.codecIntializationDataSize=t.readUint16(),this.codecIntializationData=t.readUint8Array(this.codecIntializationDataSize)}),BoxParser.createBoxCtor("vttC",function(t){this.text=t.readString(this.size-this.hdr_size)}),BoxParser.createFullBoxCtor("vvcC",function(t){var e,i={held_bits:void 0,num_held_bits:0,stream_read_1_bytes:function(t){this.held_bits=t.readUint8(),this.num_held_bits=8},stream_read_2_bytes:function(t){this.held_bits=t.readUint16(),this.num_held_bits=16},extract_bits:function(t){var e=this.held_bits>>this.num_held_bits-t&(1<>=1;t+=BoxParser.decimalToHex(i,0),t+=".",0===this.hvcC.general_tier_flag?t+="L":t+="H",t+=this.hvcC.general_level_idc;var s=!1,a="";for(r=5;0<=r;r--)(this.hvcC.general_constraint_indicator[r]||s)&&(a="."+BoxParser.decimalToHex(this.hvcC.general_constraint_indicator[r],0)+a,s=!0);t+=a}return t},BoxParser.vvc1SampleEntry.prototype.getCodec=BoxParser.vvi1SampleEntry.prototype.getCodec=function(){var t=BoxParser.SampleEntry.prototype.getCodec.call(this);if(this.vvcC){t+="."+this.vvcC.general_profile_idc,this.vvcC.general_tier_flag?t+=".H":t+=".L",t+=this.vvcC.general_level_idc;var e="";if(this.vvcC.general_constraint_info){var i,r=[],s=0;for(s|=this.vvcC.ptl_frame_only_constraint<<7,s|=this.vvcC.ptl_multilayer_enabled<<6,h=0;h>2&63,r.push(s),s&&(i=h),s=this.vvcC.general_constraint_info[h]>>2&3;if(void 0===i)e=".CA";else{e=".C";for(var a="ABCDEFGHIJKLMNOPQRSTUVWXYZ234567",n=0,o=0,h=0;h<=i;++h)for(n=n<<8|r[h],o+=8;5<=o;)e+=a[n>>o-5&31],n&=(1<<(o-=5))-1;o&&(e+=a[31&(n<<=5-o)])}}t+=e}return t},BoxParser.mp4aSampleEntry.prototype.getCodec=function(){var t=BoxParser.SampleEntry.prototype.getCodec.call(this);if(this.esds&&this.esds.esd){var e=this.esds.esd.getOTI(),i=this.esds.esd.getAudioConfig();return t+"."+BoxParser.decimalToHex(e)+(i?"."+i:"")}return t},BoxParser.stxtSampleEntry.prototype.getCodec=function(){var t=BoxParser.SampleEntry.prototype.getCodec.call(this);return this.mime_format?t+"."+this.mime_format:t},BoxParser.vp08SampleEntry.prototype.getCodec=BoxParser.vp09SampleEntry.prototype.getCodec=function(){var t=BoxParser.SampleEntry.prototype.getCodec.call(this),e=this.vpcC.level;0==e&&(e="00");var i=this.vpcC.bitDepth;return 8==i&&(i="08"),t+".0"+this.vpcC.profile+"."+e+"."+i},BoxParser.av01SampleEntry.prototype.getCodec=function(){var t,e=BoxParser.SampleEntry.prototype.getCodec.call(this),i=this.av1C.seq_level_idx_0;return i<10&&(i="0"+i),2===this.av1C.seq_profile&&1===this.av1C.high_bitdepth?t=1===this.av1C.twelve_bit?"12":"10":this.av1C.seq_profile<=2&&(t=1===this.av1C.high_bitdepth?"10":"08"),e+"."+this.av1C.seq_profile+"."+i+(this.av1C.seq_tier_0?"H":"M")+"."+t},BoxParser.Box.prototype.writeHeader=function(t,e){this.size+=8,this.size>MAX_SIZE&&(this.size+=8),"uuid"===this.type&&(this.size+=16),Log.debug("BoxWriter","Writing box "+this.type+" of size: "+this.size+" at position "+t.getPosition()+(e||"")),this.size>MAX_SIZE?t.writeUint32(1):(this.sizePosition=t.getPosition(),t.writeUint32(this.size)),t.writeString(this.type,null,4),"uuid"===this.type&&t.writeUint8Array(this.uuid),this.size>MAX_SIZE&&t.writeUint64(this.size)},BoxParser.FullBox.prototype.writeHeader=function(t){this.size+=4,BoxParser.Box.prototype.writeHeader.call(this,t," v="+this.version+" f="+this.flags),t.writeUint8(this.version),t.writeUint24(this.flags)},BoxParser.Box.prototype.write=function(t){"mdat"===this.type?this.data&&(this.size=this.data.length,this.writeHeader(t),t.writeUint8Array(this.data)):(this.size=this.data?this.data.length:0,this.writeHeader(t),this.data&&t.writeUint8Array(this.data))},BoxParser.ContainerBox.prototype.write=function(t){this.size=0,this.writeHeader(t);for(var e=0;ee?1:0,this.flags=0,this.size=4,1===this.version&&(this.size+=4),this.writeHeader(t),1===this.version?t.writeUint64(this.baseMediaDecodeTime):t.writeUint32(this.baseMediaDecodeTime)},BoxParser.tfhdBox.prototype.write=function(t){this.version=0,this.size=4,this.flags&BoxParser.TFHD_FLAG_BASE_DATA_OFFSET&&(this.size+=8),this.flags&BoxParser.TFHD_FLAG_SAMPLE_DESC&&(this.size+=4),this.flags&BoxParser.TFHD_FLAG_SAMPLE_DUR&&(this.size+=4),this.flags&BoxParser.TFHD_FLAG_SAMPLE_SIZE&&(this.size+=4),this.flags&BoxParser.TFHD_FLAG_SAMPLE_FLAGS&&(this.size+=4),this.writeHeader(t),t.writeUint32(this.track_id),this.flags&BoxParser.TFHD_FLAG_BASE_DATA_OFFSET&&t.writeUint64(this.base_data_offset),this.flags&BoxParser.TFHD_FLAG_SAMPLE_DESC&&t.writeUint32(this.default_sample_description_index),this.flags&BoxParser.TFHD_FLAG_SAMPLE_DUR&&t.writeUint32(this.default_sample_duration),this.flags&BoxParser.TFHD_FLAG_SAMPLE_SIZE&&t.writeUint32(this.default_sample_size),this.flags&BoxParser.TFHD_FLAG_SAMPLE_FLAGS&&t.writeUint32(this.default_sample_flags)},BoxParser.tkhdBox.prototype.write=function(t){this.version=0,this.size=80,this.writeHeader(t),t.writeUint32(this.creation_time),t.writeUint32(this.modification_time),t.writeUint32(this.track_id),t.writeUint32(0),t.writeUint32(this.duration),t.writeUint32(0),t.writeUint32(0),t.writeInt16(this.layer),t.writeInt16(this.alternate_group),t.writeInt16(this.volume<<8),t.writeUint16(0),t.writeInt32Array(this.matrix),t.writeUint32(this.width),t.writeUint32(this.height)},BoxParser.trexBox.prototype.write=function(t){this.version=0,this.flags=0,this.size=20,this.writeHeader(t),t.writeUint32(this.track_id),t.writeUint32(this.default_sample_description_index),t.writeUint32(this.default_sample_duration),t.writeUint32(this.default_sample_size),t.writeUint32(this.default_sample_flags)},BoxParser.trunBox.prototype.write=function(t){this.version=0,this.size=4,this.flags&BoxParser.TRUN_FLAGS_DATA_OFFSET&&(this.size+=4),this.flags&BoxParser.TRUN_FLAGS_FIRST_FLAG&&(this.size+=4),this.flags&BoxParser.TRUN_FLAGS_DURATION&&(this.size+=4*this.sample_duration.length),this.flags&BoxParser.TRUN_FLAGS_SIZE&&(this.size+=4*this.sample_size.length),this.flags&BoxParser.TRUN_FLAGS_FLAGS&&(this.size+=4*this.sample_flags.length),this.flags&BoxParser.TRUN_FLAGS_CTS_OFFSET&&(this.size+=4*this.sample_composition_time_offset.length),this.writeHeader(t),t.writeUint32(this.sample_count),this.flags&BoxParser.TRUN_FLAGS_DATA_OFFSET&&(this.data_offset_position=t.getPosition(),t.writeInt32(this.data_offset)),this.flags&BoxParser.TRUN_FLAGS_FIRST_FLAG&&t.writeUint32(this.first_sample_flags);for(var e=0;e=e?t:new Array(e-t.length+1).join(i)+t}function r(t){var e=Math.floor(t/3600),i=Math.floor((t-3600*e)/60),r=Math.floor(t-3600*e-60*i),t=Math.floor(1e3*(t-3600*e-60*i-r));return s(e,2)+":"+s(i,2)+":"+s(r,2)+"."+s(t,3)}for(var a=this.parseSample(i),n="",o=0;o=r.samples.length)&&(Log.info("ISOFile","Sending fragmented data on track #"+i.id+" for samples ["+Math.max(0,r.nextSample-i.nb_samples)+","+(r.nextSample-1)+"]"),Log.info("ISOFile","Sample data size in memory: "+this.getAllocatedSampleDataSize()),this.onSegment&&this.onSegment(i.id,i.user,i.segmentStream.buffer,r.nextSample,t||r.nextSample>=r.samples.length),i.segmentStream=null,i!==this.fragmentedTracks[e]))break}if(null!==this.onSamples)for(e=0;e=r.samples.length)&&(Log.debug("ISOFile","Sending samples on track #"+a.id+" for sample "+r.nextSample),this.onSamples&&this.onSamples(a.id,a.user,a.samples),a.samples=[],a!==this.extractedTracks[e]))break}}}},ISOFile.prototype.getBox=function(t){t=this.getBoxes(t,!0);return t.length?t[0]:null},ISOFile.prototype.getBoxes=function(t,e){var i=[];return ISOFile._sweep.call(this,t,i,e),i},ISOFile._sweep=function(t,e,i){for(var r in this.type&&this.type==t&&e.push(this),this.boxes){if(e.length&&i)return;ISOFile._sweep.call(this.boxes[r],t,e,i)}},ISOFile.prototype.getTrackSamplesInfo=function(t){t=this.getTrackById(t);if(t)return t.samples},ISOFile.prototype.getTrackSample=function(t,e){t=this.getTrackById(t);return this.getSample(t,e)},ISOFile.prototype.releaseUsedSamples=function(t,e){var i=0,r=this.getTrackById(t);r.lastValidSample||(r.lastValidSample=0);for(var s=r.lastValidSample;st*s.timescale){h=r-1;break}e&&s.is_sync&&(o=r)}for(e&&(h=o),t=i.samples[h].cts,i.nextSample=h;i.samples[h].alreadyRead===i.samples[h].size&&i.samples[h+1];)h++;return a=i.samples[h].offset+i.samples[h].alreadyRead,Log.info("ISOFile","Seeking to "+(e?"RAP":"")+" sample #"+i.nextSample+" on track "+i.tkhd.track_id+", time "+Log.getDurationString(t,n)+" and offset: "+a),{offset:a,time:t/n}},ISOFile.prototype.seek=function(t,e){var i,r,s=this.moov,a={offset:1/0,time:1/0};if(this.moov){for(r=0;r=r[s].last_sample_in_run&&(r[s].last_sample_in_run<0&&(r[s].last_sample_in_run=0),r[s].entry_index++,r[s].entry_index<=r[s].sbgp.entries.length-1&&(r[s].last_sample_in_run+=r[s].sbgp.entries[r[s].entry_index].sample_count)),r[s].entry_index<=r[s].sbgp.entries.length-1?e.sample_groups[s].group_description_index=r[s].sbgp.entries[r[s].entry_index].group_description_index:e.sample_groups[s].group_description_index=-1,0!==e.sample_groups[s].group_description_index&&(n=r[s].fragment_description||r[s].description,0>16)-1:e.sample_groups[s].group_description_index-1,n&&0<=a&&(e.sample_groups[s].description=n.entries[a])):n&&2<=n.version&&0>16&1),p.is_leading=g>>26&3,p.depends_on=g>>24&3,p.is_depended_on=g>>22&3,p.has_redundancy=g>>20&3,p.degradation_priority=65535&g;var _=!!(h.tfhd.flags&BoxParser.TFHD_FLAG_BASE_DATA_OFFSET),c=!!(h.tfhd.flags&BoxParser.TFHD_FLAG_DEFAULT_BASE_IS_MOOF),m=!!(f.flags&BoxParser.TRUN_FLAGS_DATA_OFFSET),g=0,g=_?h.tfhd.base_data_offset:c||0===y?o.start:a;p.offset=0===y&&0===u?m?g+f.data_offset:g:a,a=p.offset+p.size,(0MAX_SIZE&&(this.size+=8),"uuid"===this.type&&(this.size+=16),t.log(t.indent+"size:"+this.size),t.log(t.indent+"type:"+this.type)},BoxParser.FullBox.prototype.printHeader=function(t){this.size+=4,BoxParser.Box.prototype.printHeader.call(this,t),t.log(t.indent+"version:"+this.version),t.log(t.indent+"flags:"+this.flags)},BoxParser.Box.prototype.print=function(t){this.printHeader(t)},BoxParser.ContainerBox.prototype.print=function(t){this.printHeader(t);for(var e,i=0;i>8)),t.log(t.indent+"matrix: "+this.matrix.join(", ")),t.log(t.indent+"next_track_id: "+this.next_track_id)},BoxParser.tkhdBox.prototype.print=function(t){BoxParser.FullBox.prototype.printHeader.call(this,t),t.log(t.indent+"creation_time: "+this.creation_time),t.log(t.indent+"modification_time: "+this.modification_time),t.log(t.indent+"track_id: "+this.track_id),t.log(t.indent+"duration: "+this.duration),t.log(t.indent+"volume: "+(this.volume>>8)),t.log(t.indent+"matrix: "+this.matrix.join(", ")),t.log(t.indent+"layer: "+this.layer),t.log(t.indent+"alternate_group: "+this.alternate_group),t.log(t.indent+"width: "+this.width),t.log(t.indent+"height: "+this.height)};var MP4Box={createFile:function(t,e){t=void 0===t||t,e=new ISOFile(e);return e.discardMdatData=!t,e}};"undefined"!=typeof exports&&(exports.createFile=MP4Box.createFile); diff --git a/js/main.js b/js/main.js index 15cf9fc..01e852b 100644 --- a/js/main.js +++ b/js/main.js @@ -3,8 +3,113 @@ import { app } from "../../../scripts/app.js"; import { ComfyApp } from "../../../scripts/app.js"; import { api } from "../../../scripts/api.js"; import MaskEditorForVframes from "./MaskEditorForVframes.js"; +import getVideoFrames from "./getVideoFrames_js/mod.js" + +let sequentialimageloader_getExtraMenuOptionsOverrided = []; + + + +function showImage(name, node) { + const img = new Image(); + img.onload = () => { + node.extVframes = [img]; + app.graph.setDirtyCanvas(true); + }; + let folder_separator = name.lastIndexOf("/"); + let subfolder = ""; + if (folder_separator > -1) { + subfolder = name.substring(0, folder_separator); + name = name.substring(folder_separator + 1); + } + img.src = api.apiURL(`/view?filename=${encodeURIComponent(name)}&type=input&subfolder=${subfolder}${app.getPreviewFormatParam()}`); + node.setSizeForImage?.(); +} + +async function uploadFile(file, updateNode, vframeId, node) { + if (!file.type.startsWith("image/")) + return; + + try { + // Wrap file in formdata so it includes filename + const body = new FormData(); + body.append("image", file, file.name); + body.append("subfolder", "extVideoFrame" + String(vframeId)); + const resp = await api.fetchApi("/upload/image", { + method: "POST", + body, + }); + + if (resp.status === 200) { + const data = await resp.json(); + // update the widget value + let path = data.name; + if (data.subfolder) path = data.subfolder + "/" + path; + + if (updateNode) { + showImage(path, node); + } + } else { + alert(resp.status + " - " + resp.statusText); + } + } catch (error) { + alert(error); + } +} + +async function uplaodSequentialDataCore(node, inputName, inputData, app, fileInput) { + const idWidget = node.widgets.find((w) => w.name === "sequence_id"); + + // Add our own callback to the combo widget to render an image when it changes + idWidget.callback = function () { + // let imageData = idWidget.value.split(":"); + // let dirIds = imageData[imageData.length - 1].split("/"); + // showImage("extVideoFrame" + dirIds[0] + "/" + imageData[0]); + }; + + // On load if we have a value then render the image + requestAnimationFrame(() => { + if (idWidget.value) { + let imageData = idWidget.value.split(":"); + let dirIds = imageData[imageData.length - 1].split("/"); + showImage("extVideoFrame" + dirIds[0] + "/" + imageData[0], node); + } + }); + + document.body.append(fileInput); + + // Create the button widget for selecting the files + let uploadWidget = node.addWidget("button", "choose image directory to upload", "image", () => { + fileInput.click(); + }); + uploadWidget.serialize = false; + + // Add handler to check if an image is being dragged over our node + node.onDragOver = function (e) { + if (e.dataTransfer && e.dataTransfer.items) { + const image = [...e.dataTransfer.items].find((f) => f.kind === "file"); + return !!image; + } + + return false; + }; + + // On drop upload files + node.onDragDrop = function (e) { + // console.log("onDragDrop called"); + // let handled = false; + // for (const file of e.dataTransfer.files) { + // if (file.type.startsWith("image/")) { + // uploadFile(file, !handled); // Dont await these, any order is fine, only update on first one + // handled = true; + // } + // } + + return handled; + }; + + return { widget: uploadWidget }; +} -let sequentialimageloader_getExtraMenuOptionsOverrided = false; app.registerExtension({ @@ -38,10 +143,11 @@ app.registerExtension({ }, async beforeRegisterNodeDef(nodeType, nodeData, app) { - if (nodeData.name !== "VFrame Loader With Mask Editor") + if (nodeData.name !== "VFrame Loader With Mask Editor" + && nodeData.name !== "Video Loader With Mask Editor") return; - if (!sequentialimageloader_getExtraMenuOptionsOverrided) { + if (!sequentialimageloader_getExtraMenuOptionsOverrided.includes(nodeData.name)) { // menu initializing const originalProto = nodeType.prototype.getExtraMenuOptions; nodeType.prototype.getExtraMenuOptions = function (_, options){ @@ -61,7 +167,7 @@ app.registerExtension({ }); } } - sequentialimageloader_getExtraMenuOptionsOverrided = true; + sequentialimageloader_getExtraMenuOptionsOverrided.push(nodeData.name); } // head image drawing @@ -118,76 +224,16 @@ app.registerExtension({ if (nodeData?.input?.required?.sequence_id?.[1]?.frames_upload === true) { nodeData.input.required.upload = ["VFRAMESUPLOAD"]; } + else if (nodeData?.input?.required?.sequence_id?.[1]?.video_upload === true) { + nodeData.input.required.upload = ["VIDEOUPLOAD"]; + } }, async getCustomWidgets() { return { VFRAMESUPLOAD(node, inputName, inputData, app) { const idWidget = node.widgets.find((w) => w.name === "sequence_id"); - - function showImage(name) { - const img = new Image(); - img.onload = () => { - node.extVframes = [img]; - app.graph.setDirtyCanvas(true); - }; - let folder_separator = name.lastIndexOf("/"); - let subfolder = ""; - if (folder_separator > -1) { - subfolder = name.substring(0, folder_separator); - name = name.substring(folder_separator + 1); - } - img.src = api.apiURL(`/view?filename=${encodeURIComponent(name)}&type=input&subfolder=${subfolder}${app.getPreviewFormatParam()}`); - node.setSizeForImage?.(); - } - - // Add our own callback to the combo widget to render an image when it changes - idWidget.callback = function () { - // let imageData = idWidget.value.split(":"); - // let dirIds = imageData[imageData.length - 1].split("/"); - // showImage("extVideoFrame" + dirIds[0] + "/" + imageData[0]); - }; - - // On load if we have a value then render the image - requestAnimationFrame(() => { - if (idWidget.value) { - let imageData = idWidget.value.split(":"); - let dirIds = imageData[imageData.length - 1].split("/"); - showImage("extVideoFrame" + dirIds[0] + "/" + imageData[0]); - } - }); - - async function uploadFile(file, updateNode, vframeId) { - if (!file.type.startsWith("image/")) - return; - - try { - // Wrap file in formdata so it includes filename - const body = new FormData(); - body.append("image", file, file.name); - body.append("subfolder", "extVideoFrame" + String(vframeId)); - const resp = await api.fetchApi("/upload/image", { - method: "POST", - body, - }); - - if (resp.status === 200) { - const data = await resp.json(); - // update the widget value - let path = data.name; - if (data.subfolder) path = data.subfolder + "/" + path; - - if (updateNode) { - showImage(path); - } - } else { - alert(resp.status + " - " + resp.statusText); - } - } catch (error) { - alert(error); - } - } - + const fileInput = document.createElement("input"); Object.assign(fileInput, { type: "file", @@ -197,50 +243,90 @@ app.registerExtension({ if (fileInput.files.length) { const vframeId = Date.now(); let vframeData = ""; - + for (let i = 0; i < fileInput.files.length; i++) { vframeData += fileInput.files[i].name + ":"; await uploadFile( - fileInput.files[i], i === 0, vframeId); + fileInput.files[i], i === 0, vframeId, node); } vframeData += String(vframeId); idWidget.value = vframeData; } }, }); - document.body.append(fileInput); - - // Create the button widget for selecting the files - let uploadWidget = node.addWidget("button", "choose image directory to upload", "image", () => { - fileInput.click(); - }); - uploadWidget.serialize = false; - - // Add handler to check if an image is being dragged over our node - node.onDragOver = function (e) { - if (e.dataTransfer && e.dataTransfer.items) { - const image = [...e.dataTransfer.items].find((f) => f.kind === "file"); - return !!image; - } - - return false; - }; - - // On drop upload files - node.onDragDrop = function (e) { - console.log("onDragDrop called"); - let handled = false; - for (const file of e.dataTransfer.files) { - if (file.type.startsWith("image/")) { - uploadFile(file, !handled); // Dont await these, any order is fine, only update on first one - handled = true; + + return uplaodSequentialDataCore(node, inputName, inputData, app, fileInput); + }, + VIDEOUPLOAD(node, inputName, inputData, app) { + const idWidget = node.widgets.find((w) => w.name === "sequence_id"); + + async function getFramesFromVideo(file) { + const imageType = 'image/png'; + let frameCount = 0; + let tempCanvas = document.createElement('canvas'); + let ctx = tempCanvas.getContext("2d"); + let videoUrl = URL.createObjectURL(file); + let vframes = []; + + await getVideoFrames({ + videoUrl, + async onFrame(vframe) { // `frame` is a VideoFrame object + vframes.push(vframe); + }, + onConfig(config) { + tempCanvas.width = config.codedWidth; + tempCanvas.height = config.codedHeight; + }, + onFinish() { + }, + }); + + let frames = await Promise.all( + vframes.map(vframe => + new Promise((resolve, reject) => { + ctx.drawImage(vframe, 0, 0, tempCanvas.width, tempCanvas.height); + vframe.close(); + + tempCanvas.toBlob(blob => { + if (blob) { + blob.name = ("00000000" + frameCount++).slice(-8) + ".png"; + resolve(blob); + } else { + reject(new Error('blob creating failed')); + } + }, imageType); + }))); + + URL.revokeObjectURL(videoUrl); // revoke URL to prevent memory leak + console.log("Frame extraction completed with " + frameCount + " frames"); + + return frames; + } + + const fileInput = document.createElement("input"); + Object.assign(fileInput, { + type: "file", + style: "display: none", + accept: "video/*", + onchange: async () => { + if (fileInput.files.length) { + const file = fileInput.files[0]; + const frames = await getFramesFromVideo(file); + + const vframeId = Date.now(); + let vframeData = ""; + + for (let i = 0; i < frames.length; i++) { + vframeData += frames[i].name + ":"; + await uploadFile(frames[i], i === 0, vframeId, node); + } + vframeData += String(vframeId); + idWidget.value = vframeData; } - } - - return handled; - }; - - return { widget: uploadWidget }; + }, + }); + + return uplaodSequentialDataCore(node, inputName, inputData, app, fileInput); } } }