python subprocess communicate freezes

Question

I have the following python code that hangs :

cmd = ["ssh", "-tt", "-vvv"] + self.common_args
cmd += [self.host]
cmd += ["cat > %s" % (out_path)]
p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate(in_string)

It is supposed to save a string (in_string) into a remote file over ssh.

The file is correctly saved but then the process hangs. If I use

cmd += ["echo"] instead of
cmd += ["cat > %s" % (out_path)]

the process does not hang so I am pretty sure that I misunderstand something about the way communicate considers that the process has exited.

do you know how I should write the command so the the "cat > file" does not make communicate hang ?

Answer 1

-tt option allocates tty that prevents the child process to exit when .communicate() closes p.stdin (EOF is ignored). This works:

import pipes
from subprocess import Popen, PIPE

cmd = ["ssh", self.host, "cat > " + pipes.quote(out_path)] # no '-tt'
p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(in_string)

You could use paramiko -- pure Python ssh library, to write data to a remote file via ssh:

#!/usr/bin/env python
import os
import posixpath
import sys
from contextlib import closing

from paramiko import SSHConfig, SSHClient

hostname, out_path, in_string = sys.argv[1:] # get from command-line 

# load parameters to setup ssh connection
config = SSHConfig()
with open(os.path.expanduser('~/.ssh/config')) as config_file:
    config.parse(config_file)
d = config.lookup(hostname)

# connect
with closing(SSHClient()) as ssh:
    ssh.load_system_host_keys()
    ssh.connect(d['hostname'], username=d.get('user'))
    with closing(ssh.open_sftp()) as sftp:
        makedirs_exists_ok(sftp, posixpath.dirname(out_path))
        with sftp.open(out_path, 'wb') as remote_file:
            remote_file.write(in_string)

where makedirs_exists_ok() function mimics os.makedirs():

from functools import partial
from stat import S_ISDIR

def isdir(ftp, path):
    try:
        return S_ISDIR(ftp.stat(path).st_mode)
    except EnvironmentError:
        return None

def makedirs_exists_ok(ftp, path):
    def exists_ok(mkdir, name):
        """Don't raise an error if name is already a directory."""
        try:
            mkdir(name)
        except EnvironmentError:
            if not isdir(ftp, name):
                raise

    # from os.makedirs()
    head, tail = posixpath.split(path)
    if not tail:
        assert path.endswith(posixpath.sep)
        head, tail = posixpath.split(head)

    if head and tail and not isdir(ftp, head):
        exists_ok(partial(makedirs_exists_ok, ftp), head)  # recursive call

    # do create directory
    assert isdir(ftp, head)
    exists_ok(ftp.mkdir, path)