ValueError("I/O operation on closed pipe") when using asyncio with subprocess

Question

A simplified version of my code looks like this:

def run_async(spawn_coro, timeout):
    async def _read_print_line(process):
        line = await process.stdout.readline()
        line = line.decode('utf-8', errors='replace')
        sys.stdout.write(line)

    process = await spawn_coro
    try:
        while not process.stdout.at_eof():
            # We should cancel if no output has been received for a certain timeout
            await asyncio.wait_for(_read_print_line(process), timeout)
    except asyncio.TimeoutError:
        print('Timed out executing "{}".  No output received for {} seconds'.format(cmd, timeout))
        process.kill()
    await process.wait()

processes = [asyncio.create_subprocess_shell(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for cmd in cmds]

loop = asyncio.ProactorEventLoop()   # This is windows
asyncio.set_event_loop(loop)
all_jobs = asyncio.gather(
    *(run_async(process, timeout) for process in processes)
)
loop.run_until_complete(all_jobs)
loop.close()

When the process exits, I get a bunch of error output like this:

Exception ignored in: <function _ProactorBasePipeTransport.__del__ at 0x000001CD9B582AF8>
Traceback (most recent call last):
  File "C:\Users\divis\AppData\Local\Programs\Python\Python37\lib\asyncio\proactor_events.py", line 93, in __del__
    warnings.warn(f"unclosed transport {self!r}", ResourceWarning,
  File "C:\Users\divis\AppData\Local\Programs\Python\Python37\lib\asyncio\proactor_events.py", line 57, in __repr__
    info.append(f'fd={self._sock.fileno()}')
  File "C:\Users\divis\AppData\Local\Programs\Python\Python37\lib\asyncio\windows_utils.py", line 102, in fileno
    raise ValueError("I/O operation on closed pipe")
ValueError: I/O operation on closed pipe
Exception ignored in: <function BaseSubprocessTransport.__del__ at 0x000001CD9B571DC8>
Traceback (most recent call last):
    warnings.warn(f"unclosed transport {self!r}", ResourceWarning,
  File "C:\Users\divis\AppData\Local\Programs\Python\Python37\lib\asyncio\base_subprocess.py", line 78, in __repr__
    info.append(f'stdout={stdout.pipe}')
  File "C:\Users\divis\AppData\Local\Programs\Python\Python37\lib\asyncio\proactor_events.py", line 57, in __repr__
    info.append(f'fd={self._sock.fileno()}')
  File "C:\Users\divis\AppData\Local\Programs\Python\Python37\lib\asyncio\windows_utils.py", line 102, in fileno
    raise ValueError("I/O operation on closed pipe")
ValueError: I/O operation on closed pipe

If I remove the loop.close(), then instead of errors the process just hangs. I'm properly awaiting every subprocess before I close the loop, so I'm not sure what the problem is here.

Answer 1

I know this question was asked a while ago, but I just had the same problem, and I was able to solve it like this:

try:
    process = asyncio.create_subprocess_exec(cmd)
    ...
    await process.wait()
finally:
    process._transport.close()

Taking a look at the code, this seems to close the piped file handles from the child process. I don't know why asyncio doesn't expose this as part of the returned Process object.