Python subprocess hangs with named pipes

Question

I'm struggled in trying to emulate this simple piece of bash:

$ cat /tmp/fifo.tub &                                                            
[1] 24027
$ gunzip -c /tmp/filedat.dat.gz > /tmp/fifo.tub                                                              
line 01
line 02
line 03
line 04
line 05
line 06
line 07
line 08
line 09
line 10
[1]+  Done                    cat /tmp/fifo.tub

Basically I tried this subprocess approach:

# -*- coding: utf-8 -*-

import os
import sys
import shlex
import pprint
import subprocess

def main():

    fifo = '/tmp/fifo.tub'
    filedat = '/tmp/filedat.dat.gz '
    os.mkfifo(fifo,0777)
    cat  = "cat %s" % fifo
    args_cat = shlex.split(cat)
    pprint.pprint(args_cat)

    cat = subprocess.Popen( args_cat,
                            close_fds=True,
                            preexec_fn=os.setsid)

    print "PID cat: %s" % cat.pid

    f = os.open(fifo ,os.O_WRONLY)

    gunzip = 'gunzip -c  %s' %  (filedat)
    args_gunzip = shlex.split(gunzip)
    pprint.pprint(args_gunzip)

    gunzip = subprocess.Popen( args_gunzip,
                               stdout = f,
                               close_fds=True,
                               preexec_fn=os.setsid)

    print "PID gunzip: %s" % gunzip.pid

    while not cat.poll():
        # hangs for ever
        pass
    return True

if __name__=="__main__":
    main()

cat process never ends.

Alternatively I tried to bypass the problem with threads but I get the same result.

import os
import sys
import shlex
import pprint
import subprocess
import threading

class Th(threading.Thread):
    def __init__(self,cmd,stdout_h=None):
        self.stdout = None
        self.stderr = None
        self.cmd = cmd
        self.stdout_h = stdout_h

        self.proceso = None
        self.pid = None

        threading.Thread.__init__(self)

    def run(self):
        if self.stdout_h:
            self.proceso = subprocess.Popen(self.cmd,
                                 shell=False,
                                 close_fds=True,
                                 stdout=self.stdout_h)

        else:
            self.proceso = subprocess.Popen( self.cmd,
                                  close_fds=True,
                                  shell=False)
        print "PID: %d" % self.proceso.pid   


def main():

    fifo = '/tmp/fifo.tub'
    filedat = '/tmp/filedat.dat.gz '

    try:
        os.unlink(fifo)
    except:
        pass
    try:
       os.mkfifo(fifo,0777)
    except Exception , err:
       print "Error '%s' tub %s." % (err,fifo)
       sys.exit(5)

    cat  = "cat %s" % fifo
    args_cat = shlex.split(cat)
    pprint.pprint(args_cat)

    cat = Th(cmd=args_cat)
    cat.start()

    try:
        f = os.open(fifo ,os.O_WRONLY)
    except Exception, err:
        print  "Error '%s' when open fifo %s " % (err,fifo)
        sys.exit(5)

    gunzip = 'gunzip -c  %s ' %  (filedat)
    args_gunzip = shlex.split(gunzip)
    pprint.pprint(args_gunzip)

    gunzip = Th(cmd=args_gunzip,stdout_h=f)
    gunzip.start()
    gunzip.join()
    cat.join()

    while  gunzip.proceso.poll() is None:
        pass

    if  cat.proceso.poll() is None:
        print "Why?"
        cat.proceso.terminate() 
    return True

if __name__=="__main__":
    main()

I'm clearly missing something, any help will be really welcome.

Answer 1

You are not closing the FIFO file descriptor so cat is just hanging there thinking there is more to come.

I think you can use the .wait() method as well to do the same thing as your while loop.

# -*- coding: utf-8 -*-

import os
import sys
import shlex
import pprint
import subprocess

def main():

    fifo = '/tmp/fifo.tub'
    filedat = '/tmp/filedat.dat.gz '
    os.mkfifo(fifo,0777)
    cat  = "cat %s" % fifo
    args_cat = shlex.split(cat)
    pprint.pprint(args_cat)

    cat = subprocess.Popen( args_cat,
                            close_fds=True,
                            preexec_fn=os.setsid)

    print "PID cat: %s" % cat.pid

    f = os.open(fifo ,os.O_WRONLY)

    gunzip = 'gunzip -c  %s' %  (filedat)
    args_gunzip = shlex.split(gunzip)
    pprint.pprint(args_gunzip)

    gunzip = subprocess.Popen( args_gunzip,
                               stdout = f,
                               close_fds=True,
                               preexec_fn=os.setsid)

    print "PID gunzip: %s" % gunzip.pid

    gunzip.wait()
    print "gunzip finished"
    os.close(f)
    cat.wait()
    print "cat finished"

    return True

if __name__=="__main__":
    main()