How to fetch both live video frame and timestamp from ffmpeg to python on Windows

Question

Searching for an alternative as OpenCV would not provide timestamps for live camera stream (on Windows), which are required in my computer vision algorithm, I found ffmpeg and this excellent article https://zulko.github.io/blog/2013/09/27/read-and-write-video-frames-in-python-using-ffmpeg/ The solution uses ffmpeg, accessing its standard output (stdout) stream. I extended it to read the standard error (stderr) stream as well.

Working up the python code on windows, while I received the video frames from ffmpeg stdout, but the stderr freezes after delivering the showinfo videofilter details (timestamp) for first frame.

I recollected seeing on ffmpeg forum somewhere that the video filters like showinfo are bypassed when redirected. Is this why the following code does not work as expected?

Expected: It should write video frames to disk as well as print timestamp details.
Actual: It writes video files but does not get the timestamp (showinfo) details.

Here's the code I tried:

import subprocess as sp
import numpy
import cv2

command = [ 'ffmpeg', 
            '-i', 'e:\sample.wmv',
            '-pix_fmt', 'rgb24',
            '-vcodec', 'rawvideo',
            '-vf', 'showinfo', # video filter - showinfo will provide frame timestamps
            '-an','-sn', #-an, -sn disables audio and sub-title processing respectively
            '-f', 'image2pipe', '-'] # we need to output to a pipe

pipe = sp.Popen(command, stdout = sp.PIPE, stderr = sp.PIPE) # TODO someone on ffmpeg forum said video filters (e.g. showinfo) are bypassed when stdout is redirected to pipes??? 

for i in range(10):
    raw_image = pipe.stdout.read(1280*720*3)
    img_info = pipe.stderr.read(244) # 244 characters is the current output of showinfo video filter
    print "showinfo output", img_info
    image1 =  numpy.fromstring(raw_image, dtype='uint8')
    image2 = image1.reshape((720,1280,3))  

    # write video frame to file just to verify
    videoFrameName = 'Video_Frame{0}.png'.format(i)
    cv2.imwrite(videoFrameName,image2)

    # throw away the data in the pipe's buffer.
    pipe.stdout.flush()
    pipe.stderr.flush()

So how to still get the frame timestamps from ffmpeg into python code so that it can be used in my computer vision algorithm...

Answer 1

Redirecting stderr works in python.
So instead of this pipe = sp.Popen(command, stdout = sp.PIPE, stderr = sp.PIPE)
do this pipe = sp.Popen(command, stdout = sp.PIPE, stderr = sp.STDOUT)

We could avoid redirection by adding an asynchronous call to read both the standard streams (stdout and stderr) of ffmpeg. This would avoid any mixing of the video frame and timestamp and thus the error prone seperation. So modifying the original code to use threading module would look like this:

# Python script to read video frames and timestamps using ffmpeg
import subprocess as sp
import threading

import matplotlib.pyplot as plt
import numpy
import cv2

ffmpeg_command = [ 'ffmpeg',
                   '-nostats', # do not print extra statistics
                    #'-debug_ts', # -debug_ts could provide timestamps avoiding showinfo filter (-vcodec copy). Need to check by providing expected fps TODO
                    '-r', '30', # output 30 frames per second
                    '-i', 'e:\sample.wmv',
                    '-an','-sn', #-an, -sn disables audio and sub-title processing respectively
                    '-pix_fmt', 'rgb24',
                    '-vcodec', 'rawvideo', 
                    #'-vcodec', 'copy', # very fast!, direct copy - Note: No Filters, No Decode/Encode, no quality loss
                    #'-vframes', '20', # process n video frames only. For Debugging
                    '-vf', 'showinfo', # showinfo videofilter provides frame timestamps as pts_time
                    '-f', 'image2pipe', 'pipe:1' ] # outputs to stdout pipe. can also use '-' which is redirected to pipe


# seperate method to read images on stdout asynchronously
def AppendProcStdout(proc, nbytes, AppendList):
    while proc.poll() is None: # continue while the process is alive
        AppendList.append(proc.stdout.read(nbytes)) # read image bytes at a time

# seperate method to read image info. on stderr asynchronously
def AppendProcStderr(proc, AppendList):
    while proc.poll() is None: # continue while the process is alive
        try: AppendList.append(proc.stderr.next()) # read stderr until empty
        except StopIteration: continue # ignore stderr empty exception and continue


if __name__ == '__main__':
    # run ffmpeg command
    pipe = sp.Popen(ffmpeg_command, stdout=sp.PIPE, stderr=sp.PIPE) 

    # 2 threads to talk with ffmpeg stdout and stderr pipes
    framesList = [];
    frameDetailsList = []
    appendFramesThread = threading.Thread(group=None, target=AppendProcStdout, name='FramesThread', args=(pipe, 1280*720*3, framesList), kwargs=None, verbose=None) # assuming rgb video frame with size 1280*720 
    appendInfoThread = threading.Thread(group=None, target=AppendProcStderr, name='InfoThread', args=(pipe, frameDetailsList), kwargs=None, verbose=None) 

    # start threads to capture ffmpeg frames and info.
    appendFramesThread.start()
    appendInfoThread.start()

    # wait for few seconds and close - simulating cancel
    import time; time.sleep(2) 
    pipe.terminate() 

    # check if threads finished and close
    appendFramesThread.join() 
    appendInfoThread.join() 

    # save an image per 30 frames to disk 
    savedList = []
    for cnt,raw_image in enumerate(framesList):
        if (cnt%30 != 0): continue
        image1 =  numpy.fromstring(raw_image, dtype='uint8')
        image2 = image1.reshape((720,1280,3))  # assuming rgb image with size 1280 X 720
        # write video frame to file just to verify
        videoFrameName = 'video_frame{0}.png'.format(cnt)
        cv2.imwrite(videoFrameName,image2)
        savedList.append('{} {}'.format(videoFrameName, image2.shape))

    print '### Results ###'
    print 'Images captured: ({}) \nImages saved to disk:{}\n'.format(len(framesList), savedList) # framesList contains all the video frames got from the ffmpeg
    print 'Images info captured: \n', ''.join(frameDetailsList) # this contains all the timestamp details got from the ffmpeg showinfo videofilter and some initial noise text which can be easily removed while parsing

Answer 2

You can use MoviePy:

import moviepy.editor as mpy

vid = mpy.VideoFileClip('e:\\sample.wmv')
for timestamp, raw_img in vid.iter_frames(with_times=True):
    # do stuff