how to read video Frames directly into memory with Nodejs?

Question

What i am trying to do is taking a video and diving it to frames and passing this frames to a Model to detect objects in each frame but the problem is the extraction process cost so much time and i don't need the frames on my disk.

Answer 1

fmpeg-stream offers stream capabilities. So there is no need to write to a file.

It is also possible to use directly ffmpeg and spawn a new child process. Its .stdout property is a readable stream. On the event data, the chunk can be read.

const fs = require("fs");
const tf = require("@tensorflow/tfjs-node")

const logStream = fs.createWriteStream('./logFile.log');


const spawnProcess = require('child_process').spawn,
    ffmpeg = spawnProcess('ffmpeg', [
        '-i', 'videfile.mp4',
        '-vcodec', 'png',
        '-f', 'rawvideo',
        '-s', 'h*w', // size of one frame
        'pipe:1'
    ]);

ffmpeg.stderr.pipe(logStream); // for debugging

let i = 0

ffmpeg.stdout.on('data', (data) => {
    try {
        console.log(tf.node.decodeImage(data).shape)
        console.log(`${++i} frames read`)
        // dispose all tensors
    } catch(e) {
        console.log(e)
    } 
})


ffmpeg.on('close', function (code) {
    console.log('child process exited with code ' + code);
});

Decoding the image is in a try catch block to prevent error raised when the chunk does not match a frame.

A more robust code to prevent decoding chunks that do not correspond to images will be the following:

const { Transform } = require("stream")

class ExtractFrames extends Transform {
    constructor(delimiter) {
        super({ readableObjectMode: true })
        this.delimiter = Buffer.from(delimiter, "hex")
        this.buffer = Buffer.alloc(0)
    }

    _transform(data, enc, cb) {
        // Add new data to buffer
        this.buffer = Buffer.concat([this.buffer, data])
        const start = this.buffer.indexOf(this.delimiter)
        if (start < 0) return // there's no frame data at all
        const end = this.buffer.indexOf(
            this.delimiter,
            start + this.delimiter.length,
        )
        if (end < 0) return // we haven't got the whole frame yet
        this.push(this.buffer.slice(start, end)) // emit a frame
        this.buffer = this.buffer.slice(end) // remove frame data from buffer

        if (start > 0) console.error(`Discarded ${start} bytes of invalid data`)
        cb()
    }
    _flush(callback) {
        // push remaining buffer to readable stream
        callback(null, this.buffer);
    }
}

const fs = require("fs");
const tf = require("@tensorflow/tfjs-node")

const logStream = fs.createWriteStream('./logFile.log');


const spawnProcess = require('child_process').spawn,
    ffmpeg = spawnProcess('ffmpeg', [
        '-i', 'generique.mp4',
        '-vcodec', 'mjpeg',
        '-f', 'rawvideo',
        '-s', '420x360', // size of one frame
        'pipe:1'
    ]);

ffmpeg.stderr.pipe(logStream); // for debugging

let i = 0

ffmpeg.stdout
.pipe(new ExtractFrames("FFD8FF")).on('data', (data) => {
    try {
        console.log(tf.node.decodeImage(data).shape)
        console.log(`${++i} frames read`)
        // dispose all tensors
    } catch(e) {
        console.log(e)
    } 
})


ffmpeg.on('close', function (code) {
    console.log('child process exited with code ' + code);
});

Though, the above code works, it will still fill up quickly the memory. Separating the frame extraction from the data processing itself will help.

async function* frames() {
        let resolve;
        let promise = new Promise(r => resolve = r);
        let bool = true;
        ls.stdout.pipe(new ExtractFrames("FFD8FF")).on('data', data => {
            resolve(data); 

            promise = new Promise(r => resolve = r); 
        });

        ls.on('close', function (code) {
            bool = false
            console.log('code')
        });

        while (bool) {
            const data = await promise; 
            yield data;
        }
}

(async() => {
     // data processing
     // possibly create tf.dataset for training
     for await (const data of stream()) {
         console.log(tf.node.decodeImage(data).shape)
         console.log(data);
    }
})()

Answer 2

Here's a simpler example inspired by the previous answer that allows you to work with a raw RGB buffer for each frame. It's fast and memory efficient and the only dependency you need is a ffmpeg binary.

const fs = require('fs')
const { Writable } = require('stream')
const { spawn } = require('child_process')

const ffmpeg = spawn('ffmpeg', [
    '-i', 'example.mp4',
    '-f', 'rawvideo',
    '-pix_fmt', 'rgb24',
    'pipe:1', // ffmpeg will output the data to stdout
])

const frameSize = 1920*1080*3 // full HD in RGB (24bpp)
const buffer = Buffer.alloc(frameSize + 1024*1024) // frameSize + 1MB headroom
let bufPos = 0
let frame = 0

const outStream = new Writable({
    write(chunk, encoding, next) {
        chunk.copy(buffer, bufPos)
        bufPos += chunk.length
        
        // we have a complete frame (and possibly a bit of the next frame) in the buffer
        if (bufPos >= frameSize) {
            const rawPixels = buffer.subarray(0, frameSize)
            // do something with the data
            fs.writeFileSync(`frame_${frame}.raw`, rawPixels )

            // copy the overflowing part of the chunk to the beginning of the buffer
            buffer.copy(buffer, 0, frameSize, bufPos - frameSize)
            bufPos = bufPos - frameSize

            frame++
        }

        // abort after processing 5 frames
        if (frame >= 5) {
            ffmpeg.stdin.pause()
            ffmpeg.kill()
            return
        }

        next()
    },
})

ffmpeg.stderr.pipe(fs.createWriteStream('stderr.log'))
ffmpeg.stdout.pipe(outStream)