Search code examples
node.jsffmpegstdoutspawn

How to pipe sequence of thumbnails/screenshots from ffmpeg to nodejs for further processing?

What I want to achieve is to take screenshots continuously and pass them to nodejs application in order to process each one separately and do some other stuff. I need this for Linux environment only. I picked ffmpeg with x11grab as a screenshots provider. The following command works just fine:

ffmpeg -t 10 -s 1366x768 -f x11grab -i :0.0+0,0 -vf fps=30 output_%d.png -y

It creates 300 consequent frames of my screen during 10 seconds period. Then I want to redirect the output to my nodejs app rather than just to write files on the hard drive. So I'm calling ffmpeg from node:

var spawn = require('child_process').spawn,
    fps = 30,
    duration = 10,
    screenSize = {w: 1366, h: 768},
    args = [
        '-t', 
        duration, 
        '-s', 
        screenSize.w + 'x' + screenSize.h, 
        '-f', 
        'x11grab', 
        '-i', 
        ':0.0', 
        '-vf',
        'fps=' + fps,
        '-f',
        'mjpeg',
        'pipe:1'
    ],
    ff = spawn('ffmpeg', args);

ff.stdout.on('data', function (data) {
    console.log('Data size: ' + data.length);
});

ff.stdout.on('end', function (data) {
    console.log('Stream end');
});

ff.stderr.on('data', function (data) {
    console.log('ff error: ' + data);
});

I apologize for a long log, but it's important:

ff error: ffmpeg version N-77455-g4707497 Copyright (c) 2000-2015 the FFmpeg developers
  built with gcc 4.8 (Ubuntu 4.8.4-2ubuntu1~14.04)
  configuration: --extra-libs=-ldl --prefix=/opt/ffmpeg --mandir=/usr/share/man --enable-avresample --disable-debug --enable-nonfree --enable-gpl --enable-version3 --enable-libopencore-amrnb --enable-libopencore-amrwb --disable-decoder=amrnb --disable-decoder=amrwb --enable-libpulse --enable-libdcadec --enable-libfreetype --enable-libx264 --enable-libx265 --enable-libfdk-aac --enable-libvorbis --enable-libmp3lame --enable-libopus --enable-libvpx --enable-libspeex --enable-libass --enable-avisynth --enable-libsoxr --enable-libxvid --enable-libvo-aacenc --enable-libvidstab
  libavutil      55. 11.100 / 55. 11.100
  libavcodec     57. 20.100 / 57. 20.100
  libavformat    57. 20.100 / 57. 20.100
  libavdevice    57.  0.100 / 57.  0.100
  libavfilter     6. 21.101 /  6. 21.101
  libavresample   3.  0.  0 /  3.  0.  0
  libswscale      4.  0.100 /  4.  0.100
  libswresample   2.  0.101 /  2.  0.101
  libpostproc    54.  0.100 / 54.  0.100

ff error: Input #0, x11grab, from ':0.0':
  Duration: N/A, start: 1451414448.216650, bitrate: N/A
    Stream #0:0: Video: rawvideo (BGR[0] / 0x524742), bgr0, 1366x768, 29.97 fps, 29.97 tbr, 1000k tbn, 
ff error: 29.97 tbc

ff error: [swscaler @ 0x34238a0] deprecated pixel format used, make sure you did set range correctly

ff error: Output #0, mjpeg, to 'pipe:1':
  Metadata:
    encoder         : 
ff error: Lavf57.20.100
    Stream #0:0: Video: mjpeg, yuvj444p(pc), 1366x768, q=2-31, 200 kb/s, 30 fps, 30 tbn, 30 tbc
    Metadata:
      encoder         : Lavc57.20.100 mjpeg
    Side data:
      unknown side data type 10 (24 bytes)
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> mjpeg (native))
Press [q] to stop, [?] for help

ff error: [swscaler @ 0x34238a0] Warning: data is not aligned! This can lead to a speedloss

Data size: 65536
Data size: 65536
Data size: 21413
Data size: 65536
Data size: 65536
Data size: 45581
Data size: 65536
Data size: 65536
Data size: 62377
Data size: 65536
Data size: 65536
Data size: 45581
Data size: 65536
Data size: 65536
Data size: 21413
Data size: 65536
Data size: 60933
Data size: 65536
Data size: 49550
Data size: 65536
Data size: 36709
Data size: 65536
Data size: 27035
Data size: 65536
Data size: 20131
Data size: 65536
Data size: 15887
Data size: 65536
Data size: 15887
Data size: 65536
Data size: 15887
Data size: 65536
Data size: 15911
Data size: 65536
Data size: 15911
Data size: 65536
Data size: 15911
ff error: frame=   16 fps=0.0 q=24.8 size=    1819kB time=00:00:00.53 bitrate=27935.6kbits/s speed=1.04x    
Data size: 65536
Data size: 15911
Data size: 65536
Data size: 15911
Data size: 65536
Data size: 15919
Data size: 65536
Data size: 15919
Data size: 65536
Data size: 15919
Data size: 65536
Data size: 15919
Data size: 65536
Data size: 15919
Data size: 65536
Data size: 15911
Data size: 65536
Data size: 15911
Data size: 65536
Data size: 15906
Data size: 65536
Data size: 15906
Data size: 65536
Data size: 15917
Data size: 65536
Data size: 15999
Data size: 65536
Data size: 15949
Data size: 65536
Data size: 15997
Data size: 65536
Data size: 15965
ff error: frame=   32 fps= 30 q=24.8 size=    3092kB time=00:00:01.06 bitrate=23743.7kbits/s speed=1.01x    
Data size: 65536
Data size: 16025
Data size: 65536
Data size: 15978
Data size: 65536
Data size: 15963
Data size: 65536
Data size: 16028
Data size: 65536
Data size: 15976
Data size: 65536
Data size: 15958
Data size: 65536
Data size: 15940
Data size: 65536
Data size: 15992
Data size: 65536
Data size: 15962
Data size: 65536
Data size: 16010
Data size: 65536
Data size: 15941
Data size: 65536
Data size: 15941
Data size: 65536
Data size: 15973
Data size: 65536
Data size: 15943
Data size: 65536
Data size: 15947
Data size: 65536
Data size: 15947
ff error: frame=   48 fps= 30 q=24.8 size=    4365kB time=00:00:01.60 bitrate=22349.6kbits/s speed=1.01x    
Data size: 65536
Data size: 15982
Data size: 65536
Data size: 15982
Data size: 65536
Data size: 15956
Data size: 65536
Data size: 15956
Data size: 65536
Data size: 15956
Data size: 65536
Data size: 16001
Data size: 65536
Data size: 15930
Data size: 65536
Data size: 15922
Data size: 65536
Data size: 15924
Data size: 65536
Data size: 15924
Data size: 65536
Data size: 15924
Data size: 65536
Data size: 15924
Data size: 65536
Data size: 15911
Data size: 65536
Data size: 15924
Data size: 65536
Data size: 15985
Data size: 65536
Data size: 15985
ff error: frame=   64 fps= 30 q=24.8 size=    5638kB time=00:00:02.13 bitrate=21651.3kbits/s speed=1.01x    
Data size: 65536
Data size: 15985
Data size: 65536
Data size: 15924
Data size: 65536
Data size: 15976
Data size: 65536
Data size: 15976
Data size: 65536
Data size: 15958
Data size: 65536
Data size: 16319
Data size: 65536
Data size: 16558
Data size: 65536
Data size: 16576
Data size: 65536
Data size: 16564
Data size: 65536
Data size: 16582
Data size: 65536
Data size: 16589
Data size: 65536
Data size: 16587
Data size: 65536
Data size: 16446
Data size: 65536
Data size: 16450
Data size: 65536
Data size: 16450
Data size: 65536
Data size: 16450
ff error: frame=   80 fps= 30 q=24.8 size=    6918kB time=00:00:02.66 bitrate=21251.0kbits/s speed=1.01x    
Data size: 65536
Data size: 16568
Data size: 65536
Data size: 16575
Data size: 65536
Data size: 16585
Data size: 65536
Data size: 18182
Data size: 65536
Data size: 17203
Data size: 65536
Data size: 16769
Data size: 65536
Data size: 16734
Data size: 65536
Data size: 16823
Data size: 65536
Data size: 16338
Data size: 65536
Data size: 16455
Data size: 65536
Data size: 16406
Data size: 65536
Data size: 16645
Data size: 65536
Data size: 16800
Data size: 65536
Data size: 16800
Data size: 65536
Data size: 16800
Data size: 65536
Data size: 16806
ff error: frame=   96 fps= 30 q=24.8 size=    8204kB time=00:00:03.20 bitrate=21001.8kbits/s speed=   1x    
Data size: 65536
Data size: 16795
Data size: 65536
Data size: 16804
Data size: 65536
Data size: 16770
Data size: 65536
Data size: 16760
Data size: 65536
Data size: 16813
Data size: 65536
Data size: 16445
Data size: 65536
Data size: 16259
Data size: 65536
Data size: 16260
Data size: 65536
Data size: 16265
Data size: 65536
Data size: 16284
Data size: 65536
Data size: 16233
Data size: 65536
Data size: 16233
Data size: 65536
Data size: 16182
Data size: 65536
Data size: 16058
Data size: 60561
ff error: frame=  111 fps= 30 q=24.8 size=    9384kB time=00:00:03.70 bitrate=20776.1kbits/s speed=   1x    
Data size: 61813
Data size: 61813
Data size: 61813
Data size: 61813
Data size: 61813
Data size: 61781
Data size: 61784
Data size: 61796
Data size: 61842
Data size: 61839
Data size: 61793
Data size: 61810
Data size: 61844
Data size: 61844
Data size: 61850
Data size: 61841
ff error: frame=  127 fps= 30 q=24.8 size=   10350kB time=00:00:04.23 bitrate=20027.8kbits/s speed=   1x    
Data size: 61858
Data size: 61853
Data size: 61833
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
ff error: frame=  142 fps= 30 q=24.8 size=   11256kB time=00:00:04.73 bitrate=19480.5kbits/s speed=   1x    
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
ff error: frame=  158 fps= 30 q=24.8 size=   12223kB time=00:00:05.26 bitrate=19011.4kbits/s speed=   1x    
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61867
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
ff error: frame=  173 fps= 30 q=24.8 size=   13129kB time=00:00:05.76 bitrate=18650.3kbits/s speed=   1x    
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
ff error: frame=  188 fps= 30 q=24.8 size=   14035kB time=00:00:06.26 bitrate=18346.6kbits/s speed=   1x    
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
ff error: frame=  203 fps= 30 q=24.8 size=   14941kB time=00:00:06.76 bitrate=18087.8kbits/s speed=   1x    
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
ff error: frame=  219 fps= 30 q=24.8 size=   15907kB time=00:00:07.30 bitrate=17850.8kbits/s speed=   1x    
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
ff error: frame=  234 fps= 30 q=24.8 size=   16813kB time=00:00:07.80 bitrate=17658.1kbits/s speed=   1x    
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
Data size: 61850
ff error: frame=  250 fps= 30 q=24.8 size=   17780kB time=00:00:08.33 bitrate=17478.0kbits/s speed=   1x    
Data size: 61850
Data size: 61712
Data size: 61712
Data size: 61712
Data size: 61712
Data size: 61712
Data size: 61712
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
ff error: frame=  265 fps= 30 q=24.8 size=   18675kB time=00:00:08.83 bitrate=17319.4kbits/s speed=   1x    
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
ff error: frame=  280 fps= 30 q=24.8 size=   19563kB time=00:00:09.33 bitrate=17171.2kbits/s speed=   1x    
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
ff error: frame=  295 fps= 30 q=24.8 size=   20452kB time=00:00:09.83 bitrate=17038.0kbits/s speed=   1x    
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
Data size: 60638
ff error: frame=  300 fps= 30 q=24.8 Lsize=   20748kB time=00:00:10.00 bitrate=16996.6kbits/s speed=0.998x    
video:20748kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%

Stream end

The error messages are weird too, but not the biggest problem for now, so let's omit them.

Looking at all the data coming from ffmpeg I assume that the images are coming as a single stream without any separation. I don't think that looking into binary data and seeking for certain flag bytes is a good idea to solve the problem and separate the images. The command line I mentioned in the top of the post does separate writes to hard drive that means it produces separate writable streams per each screenshot. And it doesn't do so for a spawned process from nodejs.

What am I missing?

Solution

  • The solution is to use png-streamer and tune up ffmpeg args a little. png-streamer parses the incoming stream and recognizes the PNG images by PNG binary header. Here's the final code:

    var ffmpegArgs = [
        '-t', 
        options.duration, 
        '-s', 
        options.width + 'x' + options.height, 
        '-f', 
        'x11grab', 
        '-i', 
        ':' + options.display + '+' + options.offsetX + ',' + options.offsetY, 
        '-vf',
        'fps=' + options.fps,
        '-f',         //<<
        'image2pipe', //<<
        '-vcodec',    //<<
        'png',        //<<
        'pipe:1'
    ], 
    ffmpeg = spawn('ffmpeg', ffmpegArgs);

new pngStreamer(ffmpeg, callback);

    Along with helpful notes from comments there was a trick that -vcodec png needs to be used as well. Otherwise the buffer comes in LAVC format.