Python: redirect_stdout doesn't catch logger

Question

I'm trying to redirect some logging output.

import sys
import logging
from contextlib import redirect_stdout

logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger()

with open("out.txt", "w") as f:
    with redirect_stdout(f):
        print("STDOUT")
        logger.warning("LOGGER")

I would expect both of these to output to out.txt but instead STDOUT goes to the file, and WARNING:root:LOGGER displays in the terminal.

I've confirmed that that logger is outputting to stdout by removing the redirect stuff and running python script.py > stdout.txt. Then everything gets to the file, as it should.

So the logger must be doing something weird that causes it to not get pick up by redirect_stdout?

(Note: The behaviour is the same with redirect_stderr).

Answer 1

This is a solution/workaround that defines a custom function that replaces stdout and stderr with a different handler like the original redirect_stdout() and redirect_stderr() functions. However, it also adds an extra StreamHandler to the logger with the same target. This way the stdout, stderr and logger all write to the same stream. In the __exit__ method we reset the original streams, so this behaviour only applies to the code within the with block.

import logging
import sys
from types import TracebackType
from typing import TextIO


class redirect_std:
    """Context manager to redirect stdout and stderr to a new target.

    This is an alternative implementation to the function from
    contextlib, which also adds a log handler to the root logger to
    capture log messages.

    Args:
        new_target: The new target to redirect stdout and stderr to.
    """

    def __init__(self, new_target: TextIO) -> None:
        self._new_target = new_target
        self._old_stdout: TextIO | None = None
        self._old_stderr: TextIO | None = None
        self._handler: logging.Handler | None = None

    def __enter__(self) -> TextIO:
        # Store the original stdout and stderr
        self._old_stdout = getattr(sys, "stdout")
        self._old_stderr = getattr(sys, "stderr")

        # Replace stdout and stderr with the new target
        setattr(sys, "stdout", self._new_target)
        setattr(sys, "stderr", self._new_target)

        # Add a handler to the root logger to capture the log messages
        formatter = logging.getLogger().handlers[0].formatter
        handler = logging.StreamHandler(stream=self._new_target)
        handler.setFormatter(formatter)
        logging.getLogger().addHandler(handler)

        # Return the new target so the caller can use it
        return self._new_target

    def __exit__(
        self,
        type_: type[BaseException] | None,
        value: BaseException | None,
        traceback: TracebackType | None,
    ) -> None:
        # Restore the original stdout and stderr
        setattr(sys, "stdout", self._old_stdout)
        setattr(sys, "stderr", self._old_stderr)

        # Remove the handler from the root logger
        if self._handler is not None:
            logging.getLogger().removeHandler(self._handler)


logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

with open("out.txt", "w") as f:
    with redirect_std(f):
        print("STDOUT")
        logger.debug("DEBUG")
        logger.info("INFO")
        logger.warning("LOGGER")
        logger.error("ERROR")