Gtk3 TextBuffer.serialize() returns text with format tags, even when there is visually none

Question

I'm working with a Gtk TextView/TextBuffer in my project where the user is able to type in rich text (bold/italic/underline) by selecting the correct toggle buttons.

The problem is that if I apply the underline or italic Pango flag to text within the TextView, then turn off italic/underline and type some more, and then get the text with those flags via TextBuffer.serialize(), the unformatted text (visibly unformatted in the TextView) is returned with the Underline/Italic tags around it.

You can see this here: (Note, I simplified the tags to their HTML counterparts with BeautifulSoup for readability, but the actual positions/type haven't been edited at all.)

Here's the code (requires Gtk3 and BS4 for Python3 to be installed):

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, Gdk, Pango

import smtplib, mimetypes
from bs4 import BeautifulSoup

class Handler():

    def __init__(self):
        global html
        self.state = 0

    def onDeleteWindow(self, *args):
        Gtk.main_quit(*args)

    def onSendClicked(self, button):
        start, end = textBodyBuffer.get_bounds()
        self.content = textBodyBuffer.get_text(start, end, True)

        # Below is the serialization code for exporting with format tags
        format = textBodyBuffer.register_serialize_tagset() 
        exported = textBodyBuffer.serialize(textBodyBuffer, format, start, end)

        exported = exported.decode("latin-1")

        exported = exported.split('<text_view_markup>', 1)
        del exported[0]
        exported[0] = '<text_view_markup>' + str(exported[0])

        exported = exported[0].split('</tags>', 1)
        del exported[0]

        exported = exported[0].split('</text_view_markup>', 1)
        exported = str(exported[0]).replace('\n', ' ')

        soup = BeautifulSoup(exported)

        soupTags = soup.find_all('apply_tag')

        for tag in soupTags:

            if tag['name'] == 'bold':
                tag.name = 'b'
                del tag['name']
            elif tag['name'] == 'italic':
                tag.name = 'em'
                del tag['name']
            elif tag['name'] == 'underline':
                tag.name = 'u'
                del tag['name']

        print (soup)

    def bold(self, button):
        global tags_on
        name = button.get_name()
        if button.get_active():             # Button is "down"/enabled
            tags_on.append('bold')
        elif button.get_active() != True:   # Button is "up"/disabled
            del tags_on[tags_on.index('bold')]

    def italic(self, button):
        global tags_on
        name = button.get_name()
        if button.get_active():             # Button is "down"/enabled
            tags_on.append('italic')
        elif button.get_active() != True:   # Button is "up"/disabled
            del tags_on[tags_on.index('italic')]

    def underline(self, button):
        global tags_on
        name = button.get_name()
        if button.get_active():             # Button is "down"/enabled
            tags_on.append('underline')
        elif button.get_active() != True:   # Button is "up"/disabled
            del tags_on[tags_on.index('underline')]

    def alignToggled(self, radiobutton):
        pass

    def undo(self, button):
        pass

    def redo(self, button):
        pass

    def keyHandler(self, widget, event):
        global html
        if Gdk.ModifierType.CONTROL_MASK & event.state:
            if Gdk.keyval_name(event.keyval) == 'q':    # Quit the program
                w.destroy()
                Gtk.main_quit()

def get_iter_position(buffer):
    return buffer.get_iter_at_mark(buffer.get_insert())

def text_inserted(buffer, iter, char, length):

    global tags_on

    if len(tags_on) >= 0:
        iter.backward_chars(length)

        for tag in tags_on:
            w.queue_draw()
            if tag == 'bold':
                buffer.apply_tag(tag_bold, get_iter_position(buffer), iter)
            elif tag == 'italic':
                buffer.apply_tag(tag_italic, get_iter_position(buffer), iter)
            elif tag == 'underline':
                buffer.apply_tag(tag_underline, get_iter_position(buffer), iter)


if __name__ == '__main__':

    global text, html
    # Gtk tag globals
    global tag_bold, tag_italic, tag_underline, tags_on

    tags_on = []

    text = ''
    html = '<html><body><p>'

    builder = Gtk.Builder()
    builder.add_from_file('editor.glade')
    builder.connect_signals(Handler())

    buttonSend = builder.get_object('buttonSend')

    textBody = builder.get_object('textviewBody')
    textBodyBuffer = textBody.get_buffer()

    textBodyBuffer.connect_after('insert-text', text_inserted)
    tag_bold = textBodyBuffer.create_tag("bold", weight=Pango.Weight.BOLD)
    tag_italic = textBodyBuffer.create_tag("italic", style=Pango.Style.ITALIC)
    tag_underline = textBodyBuffer.create_tag("underline", underline=Pango.Underline.SINGLE)

    w = builder.get_object('window1')
    w.show_all()

    Gtk.main()

Here's the editor.glade file:

<?xml version="1.0" encoding="UTF-8"?>
<!-- Generated with glade 3.16.1 -->
<interface>
  <requires lib="gtk+" version="3.10"/>
  <object class="GtkImage" id="image1">
    <property name="visible">True</property>
    <property name="can_focus">False</property>
    <property name="icon_name">mail-send</property>
  </object>
  <object class="GtkWindow" id="window1">
    <property name="can_focus">False</property>
    <property name="title" translatable="yes">Keyboard Mail - Edit Message</property>
    <property name="modal">True</property>
    <property name="type_hint">dialog</property>
    <signal name="delete-event" handler="onDeleteWindow" swapped="no"/>
    <signal name="key-press-event" handler="keyHandler" swapped="no"/>
    <child>
      <object class="GtkBox" id="box2">
        <property name="visible">True</property>
        <property name="can_focus">False</property>
        <property name="orientation">vertical</property>
        <child>
          <object class="GtkBox" id="box1">
            <property name="visible">True</property>
            <property name="can_focus">False</property>
            <property name="orientation">vertical</property>
            <child>
              <object class="GtkButton" id="buttonSend">
                <property name="label" translatable="yes">Send</property>
                <property name="visible">True</property>
                <property name="can_focus">True</property>
                <property name="receives_default">True</property>
                <property name="image">image1</property>
                <property name="relief">none</property>
                <property name="image_position">top</property>
                <property name="always_show_image">True</property>
                <signal name="clicked" handler="onSendClicked" swapped="no"/>
              </object>
              <packing>
                <property name="expand">False</property>
                <property name="fill">False</property>
                <property name="position">1</property>
              </packing>
            </child>
          </object>
          <packing>
            <property name="expand">False</property>
            <property name="fill">False</property>
            <property name="position">1</property>
          </packing>
        </child>
        <child>
          <object class="GtkToolbar" id="toolbar1">
            <property name="visible">True</property>
            <property name="can_focus">False</property>
            <property name="toolbar_style">icons</property>
            <property name="show_arrow">False</property>
            <child>
              <object class="GtkToggleToolButton" id="buttonBold">
                <property name="name">bold</property>
                <property name="visible">True</property>
                <property name="can_focus">False</property>
                <property name="tooltip_text" translatable="yes">Make the selected text bold</property>
                <property name="icon_name">format-text-bold</property>
                <signal name="toggled" handler="bold" swapped="no"/>
              </object>
              <packing>
                <property name="expand">False</property>
                <property name="homogeneous">True</property>
              </packing>
            </child>
            <child>
              <object class="GtkToggleToolButton" id="buttonItalic">
                <property name="name">italic</property>
                <property name="visible">True</property>
                <property name="can_focus">False</property>
                <property name="icon_name">format-text-italic</property>
                <signal name="toggled" handler="italic" swapped="no"/>
              </object>
              <packing>
                <property name="expand">False</property>
                <property name="homogeneous">True</property>
              </packing>
            </child>
            <child>
              <object class="GtkToggleToolButton" id="buttonUnderline">
                <property name="name">underline</property>
                <property name="visible">True</property>
                <property name="can_focus">False</property>
                <property name="icon_name">format-text-underline</property>
                <signal name="toggled" handler="underline" swapped="no"/>
              </object>
              <packing>
                <property name="expand">False</property>
                <property name="homogeneous">True</property>
              </packing>
            </child>
          </object>
          <packing>
            <property name="expand">False</property>
            <property name="fill">True</property>
            <property name="position">3</property>
          </packing>
        </child>
        <child>
          <object class="GtkBox" id="box5">
            <property name="visible">True</property>
            <property name="can_focus">False</property>
            <property name="orientation">vertical</property>
            <child>
              <object class="GtkTextView" id="textviewBody">
                <property name="width_request">500</property>
                <property name="height_request">100</property>
                <property name="visible">True</property>
                <property name="can_focus">True</property>
                <property name="hexpand">True</property>
                <property name="vexpand">True</property>
                <property name="wrap_mode">word</property>
                <property name="left_margin">5</property>
                <property name="right_margin">5</property>
              </object>
              <packing>
                <property name="expand">True</property>
                <property name="fill">True</property>
                <property name="position">0</property>
              </packing>
            </child>
          </object>
          <packing>
            <property name="expand">False</property>
            <property name="fill">True</property>
            <property name="position">3</property>
          </packing>
        </child>
      </object>
    </child>
  </object>
</interface>

Does anyone know why the TextBuffer.serialize() statement (line 23) is returning the visibly non-formatted characters within the underline/italic tags?

I can't seem to find any pattern in when this is occurring, it seems to randomly decide to return the text with the tags or not.

Edit: I've gone over the code step by step with pdb (Python debugger) and still haven't seen anything that would cause this.

Edit: Interesting pattern I have figured out - if you turn on both italic and underline at once, then type some, then turn them both off at once, and type, the serialize() call returns the proper string.

However, if you apply them one at a time, typing a bit for each new tag, it's returned incorrectly.

Answer 1

I have been able to recreate the problem with a minimal example in C. I therefore strongly suspect that this is a bug in GTK. The questioner hence opened a bugreport upstream.

As a workaround, I'd suggest you try to implement serialization on your own. forward_to_tag_toggle should be helpful to this end. Note that you cannot use the obvious path of using register_serialize_format to register your own serializer and then call serialize, because while a GtkTextBufferSerializeFunc should return a string, in the GObject repository it is apparently recorded as a function returning a single integer. (Due to another bug.) Instead, do the serialization entirely from your code, i.e. grab a start iter and then walk through the text buffer.