MediaSource, video buffering

Question

I would like to make a low latency video streaming service and open source (to improve my web development). I started the project last February, but I didn’t have much time to move on it. The project The project is called twitch-with-nodejs. Anyway. I dont understand clearly how works the MediaSources (I think).

My code use the MediaRecorded in one client and stop it to send every 2 seconds the video flux. Then, the server puts it in memory for 30s, then stands ready to return it as a stream.

Another client want to watch the stream, so he makes requests to the server every 2 seconds to have the video flux. Then, it work for the first call but not the following.

This is an exemple of my code : 1rst Client (streamer) :

this.media = await navigator.mediaDevices[what]({
            audio: this.config.audio,
            video: this.config.video ? {
                width: this.config.width,
                height: this.config.height,
                frameRate: this.config.fps,
                facingMode: (this.config.front ? "user" : "environment")
            } : false
        }).catch(e => {
            console.log(e.name + ": " + e.message);
            this.can = false;
        });

        let chunks = [];

        const video = document.querySelector('video');
        video.srcObject = this.media;
        video.onloadedmetadata = () => {
            video.play()
        }


        let mediaRecorder = new MediaRecorder(this.media, {"mimeType": "video/webm;"});
        mediaRecorder.ondataavailable = function(ev) {
            chunks.push(ev.data);
        }

        setInterval(() => {
            mediaRecorder.stop();
            mediaRecorder.start();
        }, 2000);

        mediaRecorder.onstop = () => {
            let blob = new Blob(chunks, {'type': 'video/webm;'});
            chunks = [];
            const data = new FormData();
            data.append('file', blob);
            fetch("/postStream", {
                "method": "post",
                "body": data
            });
        }

The server side :

app.post("/postStream", async (req, res) => {
            const fileProperty = req?.files?.file;
            if(!fileProperty?.data || fileProperty.mimetype !== 'video/webm') return res.json(false);
            this.chunks.push({ data: fileProperty?.data, id: i++, date: Date.now() });
            return res.json(true);
        });

        app.get("/playVideo", (req, res) => {
            if(this?.chunks?.length < 1) return res.json(false);
            const buffer = new Buffer.from(this.chunks?.reverse()?.[0]?.data, 'base64')
            const stream = this.bufferToStream(buffer);
            res.setHeader("content-type", "video/webm");
            stream.pipe(res);
        });

And my 2nd client, he wants to watch the video :

const mediaSource = new MediaSource();
            videoPlaying.src = URL.createObjectURL(mediaSource);

            mediaSource.addEventListener('sourceopen', sourceOpen);
            var sourceBuffer;
            const mime = "video/webm; codecs=\"vp8, vorbis\"";

            function fetchSegment(){
                return fetch("/playVideo").then(res => res.arrayBuffer());
            }

            async function sourceOpen(){
                let data = await fetchSegment();
                sourceBuffer = mediaSource.addSourceBuffer(mime);
                sourceBuffer.appendBuffer(data);
                sourceBuffer.addEventListener('updateend', onUpdateEnd);
                videoPlaying.play();
            }

            function onUpdateEnd(){
                //mediaSource.endOfStream();
                //clearInterval(intervalSegment);
            }

            var intervalSegment = setInterval(async () => {
                /* Here I catch the following error :
An attempt was made to use an object that is not, or is no longer, usable
*/
                console.log(mediaSource.readyState, mediaSource.duration); // => Open, Infinit
                let data = await fetchSegment();
                sourceBuffer.appendBuffer(data)

                console.log("Clearing memory...")
                sourceBuffer.remove(0, 2);
            }, 2000)

So, why do I catch the error "An attempt was made to use an object that is not, or is no longer, usable" ? How can the stream be read correctly ?

Answer 1

Okay, so for anyone want to know the correct answer because It's cool to know the answer.

I tried to change how my code work. So, the 2nd client who watch the video have this code (now):

function fetchSegment(){
                return fetch("/playVideo").then(res => res.arrayBuffer());
            }

            var intervalSegment = setInterval(async () => {
                if(sourceBuffer.updating) return;
                let data = await fetchSegment();
                if (queue.length > 0) {
                    queue.push(data);
                } else {
                    sourceBuffer.appendBuffer(data);
                    videoPlaying.play();
                }
            }, 2000)

            async function sourceOpen(){
                let data = await fetchSegment();
                queue.push(data);
                sourceBuffer = mediaSource.addSourceBuffer(mime);
                sourceBuffer.mode = 'sequence';
                sourceBuffer.addEventListener('updateend', onUpdateEnd);
                videoPlaying.play();
                sourceBuffer.appendBuffer(queue.shift());
            }

            function onUpdateEnd(){
                if (queue.length && !sourceBuffer.updating) {
                    sourceBuffer.appendBuffer(queue.shift())
                }else{
                    setTimeout(() => {
                        onUpdateEnd();
                    }, 500)
                }
            }

So now, the sourceBuffer have the mode sequence and not segments, if the user updating the chunck, the video isn't push. And, if the queue is empty, I don't push in it, BUT I add the segment directly to the sourceBuffer.