Python json encoding with classes that need custom encoders but which may themselves have nested data structures

Question

This is obviously a simplified version of what I am trying do but ... let's say I have a

class X(object):
    pass

x = X()
y = X()
x.val = {1:2,3:4}
y.val = {1:2,3:x}

How do I write a custom json encoder so that it recurses through the encoding loop naturally ? I do not need the json to demonstrate that the class is of type X, (a dot dict will be fine). The actual example of this may have data structures nested 10 deep.

Obviously I could just override the default() method, but this does not seem to allow recursive calls - ie the best I have is some mess like this (and I need to json.loads otherwise the thing gets double quoted / escaped):

class XEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, X):
            return json.loads(self.encode({x:getattr(obj,x) for x in dir(obj) if "__" not in x}))
        return json.JSONEncoder.default(self, obj)

Answer 1

Maybe over kill but I used to used a Mixin class as follow:

def _default_json_encoder(obj):
    """ Default encoder, encountered must have to_dict method to be serialized.  """
    if hasattr(obj, "to_dict"):
        return obj.to_dict()
    else:
        raise TypeError('Object of type %s with value of %s is not JSON serializable' % (type(obj), repr(obj)))

class MixinJSONable(object):
    """
    Mixin pattern to add a capability to an object to be jsonable.

    If object have to_dict method it will be used to produce the dict, otherwise
    MixinJSONable.to_dict will be used.

    Only "public" attributes will be dump and instance of self._jsonable tuple.

    Attributes could be ignored by passing ignored_keys parameter to to_json method.
    Thus _ignored_json_keys (class attribute) will be update and spread over all class using this Mixin.

    """
    # _ignored_json_keys = list()

    def to_dict(self):
        """
        to_dict method to dump public attributes.
        """

        self._jsonable = (int, list, str, dict)

        _dict = dict()
        for attr in dir(self):
            value = getattr(self, attr)
            if attr.startswith("_") or attr in getattr(MixinJSONable, "_ignored_json_keys", []):
                continue
            elif isinstance(value, self._jsonable) or value is None or hasattr(value, 'to_dict'):
                # to_dict method is used as serialization method.
                value = value
            else:
                continue
            _dict[attr] = value
        return _dict

    def to_json(self, **kw):
        """
        Dump object as Json.

        Accept the same keys than :func json.dumps:. If ignored_keys (list) is passed,
        the keys will not be dumped in the json (filter over all sons)

        """
        indent = kw.pop("indent", 4)  # use indent key if passed otherwise 4.
        _ignored_json_keys = kw.pop("ignored_keys", [])
        if _ignored_json_keys:
            MixinJSONable._ignored_json_keys = _ignored_json_keys
        return json.dumps(self, indent=indent, default=_default_json_encoder, **kw)


class X(MixinJSONable):
    pass

x = X()
y = X()
setattr(x,"val",{1:2,3:4})
setattr(y,"val",{1:2,3:x})
y.to_json()

will print:

{
    "val": {
        "1": 2,
        "3": {
            "val": {
                "1": 2,
                "3": 4
            }
        }
    }
}

Answer 2

Maybe I'm not understanding the question, but the json.dumps() default option handles recursing through an object graph just fine.

If don't care about preserving the type of the originating object:

def jsondefault(obj):
    return dict(obj.__dict__)

If you want to remember the type of the object you're encoding:

def jsondefault_types(obj):
    result = dict(obj.__dict__)
    result['__type__'] = ".".join([
        obj.__class__.__module__, obj.__class__.__name__])
    return result

Then you can dump either way:

class X(object): pass
x, y, z = X(), X(), X()
x.val = {1:2, 3:4}
y.val = {1:2, 3:x}
z.val = {1:2, 3:y}

print json.dumps(z, default=jsondefault)
print json.dumps(z, default=jsondefault_types)

resulting in:

'{"val": {"1": 2, "3": {"val": {"1": 2, "3": {"val": {"1": 2, "3": 4}}}}}}'

'{"__type__": "__main__.X", "val": {"1": 2, "3": {"__type__": "__main__.X", "val": {"1": 2, "3": {"__type__": "__main__.X", "val": {"1": 2, "3": 4}}}}}}'