Why don't the signals emit?

Question

The Application

I'm trying to build a python shell for my PyQt5 application using the stdlib InteractiveConsole so I can let users script live plots. I'm using a QTextEdit to display the stdout from the shell.

The Problem

When I do for loops in the shell, the application freezes because the insertPlainText() to the QTextEdit is too fast. So I wrote a buffer that would delay the inserts by a few milliseconds. However, I noticed that as soon as I ran any blocking functions like time.sleep() in the for loops, it would freeze. So the prints inside the for loops will only be displayed after the loop is done. This does not happen if the buffer is disabled.

For eg, if i do this in the shell:

>>>for i in range(10):
...    time.sleep(1)
...    print(i)
...

This will only print after 10 seconds.

Code

This is the most minimal version I could write according to MVCE guidelines.

Here is the main.ui file:

<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
 <class>main_window</class>
 <widget class="QMainWindow" name="main_window">
  <property name="geometry">
   <rect>
    <x>0</x>
    <y>0</y>
    <width>800</width>
    <height>600</height>
   </rect>
  </property>
  <property name="sizePolicy">
   <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
    <horstretch>0</horstretch>
    <verstretch>0</verstretch>
   </sizepolicy>
  </property>
  <property name="windowTitle">
   <string>MainWindow</string>
  </property>
  <property name="tabShape">
   <enum>QTabWidget::Rounded</enum>
  </property>
  <widget class="QWidget" name="central_widget">
   <layout class="QHBoxLayout" name="horizontalLayout">
    <item>
     <layout class="QVBoxLayout" name="console_layout">
      <item>
       <widget class="QTextEdit" name="console_log">
        <property name="undoRedoEnabled">
         <bool>false</bool>
        </property>
       </widget>
      </item>
      <item>
       <layout class="QHBoxLayout" name="horizontalLayout_4">
        <item>
         <widget class="QLabel" name="console_prompt">
          <property name="text">
           <string/>
          </property>
         </widget>
        </item>
        <item>
         <widget class="QLineEdit" name="console_input">
          <property name="frame">
           <bool>true</bool>
          </property>
         </widget>
        </item>
       </layout>
      </item>
     </layout>
    </item>
   </layout>
  </widget>
  <widget class="QMenuBar" name="menu_bar">
   <property name="geometry">
    <rect>
     <x>0</x>
     <y>0</y>
     <width>800</width>
     <height>26</height>
    </rect>
   </property>
  </widget>
  <widget class="QStatusBar" name="status_bar"/>
 </widget>
 <resources/>
 <connections/>
</ui>

Here is themain.py file:

import sys
from code import InteractiveConsole
from io import StringIO
from queue import Queue, Empty

from PyQt5 import uic
from PyQt5.QtCore import pyqtSlot, QThread, QObject, pyqtSignal, QTimer
from PyQt5.QtGui import QTextOption, QTextCursor
from PyQt5.QtWidgets import QApplication

__author__ = "daegontaven"
__copyright__ = "daegontaven"
__license__ = "gpl3"


class BaseSignals(QObject):
    """
    Standard set of pyqtSignals.
    """
    signal_str = pyqtSignal(str)
    signal_int = pyqtSignal(int)
    signal_float = pyqtSignal(float)
    signal_list = pyqtSignal(list)
    signal_tuple = pyqtSignal(tuple)
    signal_dict = pyqtSignal(dict)
    signal_object = pyqtSignal(object)

    def __init__(self):
        QObject.__init__(self)


class DelayedBuffer(QObject):
    """
    A buffer that uses a queue to store strings. It removes the
    first appended string first in a constant interval.
    """
    written = pyqtSignal(str)

    def __init__(self, output, delay):
        """
        :param output: used to access BaseSignals
        :param delay: delay for emitting
        """
        super().__init__()
        self.output = output

        # Set Delay
        self.delay = delay
        self.queue = Queue()
        self.timer = QTimer()
        self.timer.timeout.connect(self.process)
        self.timer.start(self.delay)

    def write(self, string):
        self.queue.put(string)

    def process(self):
        """
        Try to send the data to the stream
        """
        try:
            data = self.queue.get(block=False)
            self.written.emit(data)
        except Empty:
            pass

    def emit(self, string):
        """
        Force emit of string.
        """
        self.output.signal_str.emit(string)


class ConsoleStream(StringIO):
    """
    Custom StreamIO class that emits a signal on each write.
    """
    def __init__(self, enabled=True, *args, **kwargs):
        """
        Starts a delayed buffer to store writes due to UI
        refresh limitations.

        :param enabled: set False to bypass the buffer
        """
        StringIO.__init__(self, *args, **kwargs)
        self.enabled = enabled
        self.output = BaseSignals()

        # Buffer
        self.thread = QThread()
        self.buffer = DelayedBuffer(self.output, delay=5)
        self.buffer.moveToThread(self.thread)
        self.buffer.written.connect(self.get)
        self.thread.start()

    def write(self, string):
        """
        Overrides the parent write method and emits a signal
        meant to be received by interpreters.

        :param string: single write output from stdout
        """
        if self.enabled:
            self.buffer.write(string)
        else:
            self.output.signal_str.emit(string)

    def get(self, string):
        self.output.signal_str.emit(string)


class PythonInterpreter(QObject, InteractiveConsole):
    """
    A reimplementation of the builtin InteractiveConsole to
    work with threads.
    """
    output = pyqtSignal(str)
    push_command = pyqtSignal(str)
    multi_line = pyqtSignal(bool)

    def __init__(self):
        QObject.__init__(self)
        self.l = {}
        InteractiveConsole.__init__(self, self.l)
        self.stream = ConsoleStream()
        self.stream.output.signal_str.connect(self.console)
        self.push_command.connect(self.command)

    def write(self, string):
        self.output.emit(string)

    def runcode(self, code):
        """
        Overrides and captures stdout and stdin from
        InteractiveConsole.
        """
        sys.stdout = self.stream
        sys.stderr = self.stream
        sys.excepthook = sys.__excepthook__
        result = InteractiveConsole.runcode(self, code)
        sys.stdout = sys.__stdout__
        sys.stderr = sys.__stderr__
        return result

    @pyqtSlot(str)
    def command(self, command):
        """
        :param command: line retrieved from console_input on
                        returnPressed
        """
        result = self.push(command)
        self.multi_line.emit(result)

    @pyqtSlot(str)
    def console(self, string):
        """
        :param string: processed output from a stream
        """
        self.output.emit(string)


class MainWindow:
    """
    The main GUI window. Opens maximized.
    """
    def __init__(self):

        self.ui = uic.loadUi("main.ui")
        self.ui.showMaximized()

        # Console Properties
        self.ui.console_log.document().setMaximumBlockCount(1000)
        self.ui.console_log.setWordWrapMode(QTextOption.WrapAnywhere)

        self.ps1 = '>>>'
        self.ps2 = '...'
        self.ui.console_prompt.setText(self.ps1)

        # Spawn Interpreter
        self.thread = QThread()
        self.thread.start()

        self.interpreter = PythonInterpreter()
        self.interpreter.moveToThread(self.thread)

        # Interpreter Signals
        self.ui.console_input.returnPressed.connect(self.send_console_input)
        self.interpreter.output.connect(self.send_console_log)
        self.interpreter.multi_line.connect(self.prompt)

    def prompt(self, multi_line):
        """
        Sets what prompt to use.
        """
        if multi_line:
            self.ui.console_prompt.setText(self.ps2)
        else:
            self.ui.console_prompt.setText(self.ps1)

    def send_console_input(self):
        """
        Send input grabbed from the QLineEdit prompt to the console.
        """
        command = self.ui.console_input.text()
        self.ui.console_input.clear()
        self.interpreter.push_command.emit(str(command))

    def send_console_log(self, command):
        """
        Set the output from InteractiveConsole in the QTextEdit.
        Auto scroll scrollbar.
        """
        # Checks if scrolled
        old_cursor = self.ui.console_log.textCursor()
        old_scrollbar = self.ui.console_log.verticalScrollBar().value()
        new_scrollbar = self.ui.console_log.verticalScrollBar().maximum()
        if old_scrollbar == new_scrollbar:
            scrolled = True
        else:
            scrolled = False

        # Sets the text
        self.ui.console_log.insertPlainText(command)

        # Scrolls/Moves cursor based on available data
        if old_cursor.hasSelection() or not scrolled:
            self.ui.console_log.setTextCursor(old_cursor)
            self.ui.console_log.verticalScrollBar().setValue(old_scrollbar)
        else:
            self.ui.console_log.moveCursor(QTextCursor.End)
            self.ui.console_log.verticalScrollBar().setValue(
                self.ui.console_log.verticalScrollBar().maximum()
            )


def main():
    app = QApplication(sys.argv)
    window = MainWindow()
    sys.exit(app.exec_())

if __name__ == "__main__":
    main()

The class BaseSignals is needed for communication between the main thread and the interpreter. Here is a transcript as to why this was implemented.

What I know

This line is responsible for inserting the plain text self.output.signal_str.emit(data). This emit() happens inside a QThread. So until the multiple self.buffer.write() is finished the emit() won't be processed. I thought adding a QApplication.processEvents() in DelayedBuffer.process() would help. It doesn't. I admit I could however be wrong about this.

Any help appreciated. Thanks in advance.

Answer 1

Your interpreter thread is blocking on the InteractiveConsole.runcode() call. It will not be able to process any signals until this call completes. That is why you see the delayed output.

You can get the effect you're after by changing

self.interpreter.output.connect(self.send_console_log)

to

self.interpreter.stream.output.signal_str.connect(self.send_console_log)

For some old school debugging, disconnect you're stderr handling and sprinkle some print statements around like...

print('runcode after', file=sys.stderr)