customise simplejson output

Question

I convert python dicts with simplejson, but I'd like to customise the output for some defined keys.

for example, I'd like the keys callback and scope to be always rendered with no surrounding quotes so the javascript can interpret the data and not read it as a string.

example desired output :

"data":{
    "name":"Julien"
    ,"callback":function() {alert('hello, world');}
    ,"params":{
       "target":"div2"
       ,"scope":this
     }
}

Notice the callback and scope keys have no surrounding quotes in their values.

I've tried create a custom class and subclass JSONencoder with no luck.

class JsonSpecialKey(object):
    def __init__(self, data):
        self.data = data

class JsonSpecialEncoder(simplejson.JSONEncoder):
     def default(self, obj):
        if isinstance (obj, JsonSpecialKey):
            # how to remove quotes ??
            return obj.data
        return simplejson.JSONEncoder.default(self, obj)

d = {'testKey':JsonSpecialKey('function() {alert(123);}')}
print simplejson.dumps(d, cls=JsonSpecialEncoder, ensure_ascii=False, indent=4)

I know the resulting JSON may be invalid in the JSON recommendations but it is important to some JS applications.

I've tried some regex workarounds but it's getting complex for multilines and inline functions with data inside.

Thank you !

Answer 1

I succeed by modifying json code

import json
from json.encoder import encode_basestring_ascii ,encode_basestring,FLOAT_REPR,INFINITY,c_make_encoder
class JsonSpecialKey(object):
    def __init__(self, data):
        self.data = data
def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
        ## HACK: hand-optimized bytecode; turn globals into locals
        ValueError=ValueError,
        dict=dict,
        float=float,
        id=id,
        int=int,
        isinstance=isinstance,
        list=list,
        str=str,
        tuple=tuple,
    ):

    if _indent is not None and not isinstance(_indent, str):
        _indent = ' ' * _indent

    def _iterencode_list(lst, _current_indent_level):
        if not lst:
            yield '[]'
            return
        if markers is not None:
            markerid = id(lst)
            if markerid in markers:
                raise ValueError("Circular reference detected")
            markers[markerid] = lst
        buf = '['
        if _indent is not None:
            _current_indent_level += 1
            newline_indent = '\n' + _indent * _current_indent_level
            separator = _item_separator + newline_indent
            buf += newline_indent
        else:
            newline_indent = None
            separator = _item_separator
        first = True
        for value in lst:
            if first:
                first = False
            else:
                buf = separator
            if isinstance(value, str):
                yield buf + _encoder(value)
            elif value is None:
                yield buf + 'null'
            elif value is True:
                yield buf + 'true'
            elif value is False:
                yield buf + 'false'
            elif isinstance(value, int):
                yield buf + str(value)
            elif isinstance(value, float):
                yield buf + _floatstr(value)
            elif isinstance(value, JsonSpecialKey):
                yield buf + value.data

            else:
                yield buf
                if isinstance(value, (list, tuple)):
                    chunks = _iterencode_list(value, _current_indent_level)
                elif isinstance(value, dict):
                    chunks = _iterencode_dict(value, _current_indent_level)
                else:
                    chunks = _iterencode(value, _current_indent_level)
                for chunk in chunks:
                    yield chunk
        if newline_indent is not None:
            _current_indent_level -= 1
            yield '\n' + _indent * _current_indent_level
        yield ']'
        if markers is not None:
            del markers[markerid]

    def _iterencode_dict(dct, _current_indent_level):
        if not dct:
            yield '{}'
            return
        if markers is not None:
            markerid = id(dct)
            if markerid in markers:
                raise ValueError("Circular reference detected")
            markers[markerid] = dct
        yield '{'
        if _indent is not None:
            _current_indent_level += 1
            newline_indent = '\n' + _indent * _current_indent_level
            item_separator = _item_separator + newline_indent
            yield newline_indent
        else:
            newline_indent = None
            item_separator = _item_separator
        first = True
        if _sort_keys:
            items = sorted(dct.items(), key=lambda kv: kv[0])
        else:
            items = dct.items()
        for key, value in items:
            if isinstance(key, str):
                pass
            # JavaScript is weakly typed for these, so it makes sense to
            # also allow them.  Many encoders seem to do something like this.
            elif isinstance(key, float):
                key = _floatstr(key)
            elif key is True:
                key = 'true'
            elif key is False:
                key = 'false'
            elif key is None:
                key = 'null'
            elif isinstance(key, int):
                key = str(key)
            elif _skipkeys:
                continue
            else:
                raise TypeError("key " + repr(key) + " is not a string")
            if first:
                first = False
            else:
                yield item_separator
            yield _encoder(key)
            yield _key_separator
            if isinstance(value, str):
                yield _encoder(value)
            elif value is None:
                yield 'null'
            elif value is True:
                yield 'true'
            elif value is False:
                yield 'false'
            elif isinstance(value, int):
                yield str(value)
            elif isinstance(value, float):
                yield _floatstr(value)
            elif isinstance(value, JsonSpecialKey):
                yield value.data
            else:
                if isinstance(value, (list, tuple)):
                    chunks = _iterencode_list(value, _current_indent_level)
                elif isinstance(value, dict):
                    chunks = _iterencode_dict(value, _current_indent_level)
                else:
                    chunks = _iterencode(value, _current_indent_level)
                for chunk in chunks:
                    yield chunk
        if newline_indent is not None:
            _current_indent_level -= 1
            yield '\n' + _indent * _current_indent_level
        yield '}'
        if markers is not None:
            del markers[markerid]

    def _iterencode(o, _current_indent_level):
        if isinstance(o, str):
            yield _encoder(o)
        elif o is None:
            yield 'null'
        elif o is True:
            yield 'true'
        elif o is False:
            yield 'false'
        elif isinstance(o, int):
            yield str(o)
        elif isinstance(o, float):
            yield _floatstr(o)
        elif isinstance(o, JsonSpecialKey):
            yield o.data
        elif isinstance(o, (list, tuple)):
            for chunk in _iterencode_list(o, _current_indent_level):
                yield chunk
        elif isinstance(o, dict):
            for chunk in _iterencode_dict(o, _current_indent_level):
                yield chunk
        else:
            if markers is not None:
                markerid = id(o)
                if markerid in markers:
                    raise ValueError("Circular reference detected")
                markers[markerid] = o
            o = _default(o)
            for chunk in _iterencode(o, _current_indent_level):
                yield chunk
            if markers is not None:
                del markers[markerid]
    return _iterencode
class JsonSpecialEncoder(json.JSONEncoder):


     def iterencode(self, o, _one_shot=False):
        """Encode the given object and yield each string
        representation as available.

        For example::

            for chunk in JSONEncoder().iterencode(bigobject):
                mysocket.write(chunk)

        """
        if self.check_circular:
            markers = {}
        else:
            markers = None
        if self.ensure_ascii:
            _encoder = encode_basestring_ascii
        else:
            _encoder = encode_basestring
        def floatstr(o, allow_nan=self.allow_nan,
                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
            # Check for specials.  Note that this type of test is processor
            # and/or platform-specific, so do tests which don't depend on the
            # internals.

            if o != o:
                text = 'NaN'
            elif o == _inf:
                text = 'Infinity'
            elif o == _neginf:
                text = '-Infinity'
            else:
                return _repr(o)

            if not allow_nan:
                raise ValueError(
                    "Out of range float values are not JSON compliant: " +
                    repr(o))

            return text



        _iterencode = _make_iterencode(
            markers, self.default, _encoder, self.indent, floatstr,
            self.key_separator, self.item_separator, self.sort_keys,
            self.skipkeys, _one_shot)
        return _iterencode(o, 0)
d = {'testKey':JsonSpecialKey('function() {alert(123);}')}
print (json.dumps(d, cls=JsonSpecialEncoder, ensure_ascii=False, indent=4))

EDITED:precision of the code I modified

from json.encode I took the function _make_iterencode

adding something like

       elif isinstance(value, JsonSpecialKey):
            yield buf + value.data

at three places

from JsonEncoder I took the method iterencode but I just forced the _iterencode to be my custom function _make_iterencode

 _iterencode = _make_iterencode(
        markers, self.default, _encoder, self.indent, floatstr,
        self.key_separator, self.item_separator, self.sort_keys,
        self.skipkeys, _one_shot)

I hope it's clear

Answer 2

I changed the problem a little bit and assumed that my client would encode a function as a dictionary {"__js_func__": "function() { return 10; }"} since I don't know exactly what my functions will be called.

Basically I wrote a custom decoder that turns a dict with the __js_func__ key into a JSFunction object, and then monkey-patched the encode_basestring_ascii function to not add the quotes for this kind of object. My JSFunction object extends str which means encode_basestring_ascii gets called both for normal strings as well as JSFunction objects.

import json
import json.encoder

class JSFunction(str):
    def __repr__(self):
        return '<JSFunction: self>'

    @staticmethod
    def decode_js_func(dct):
        """Turns any dictionary that contains a __js_func__ key into a JSFunction object.                                            

        Used when loads()'ing the json sent to us by the webserver.                                                                  
        """
        if '__js_func__' in dct:
            return JSFunction(dct['__js_func__'])
        return dct

    @staticmethod
    def encode_basestring(s):
        """A function we use to monkeypatch json.encoder.encode_basestring_ascii that dumps JSFunction objects without quotes."""
        if isinstance(s, JSFunction):
            return str(s)
        else:
            return _original_encode_basestring_ascii(s)

_original_encode_basestring_ascii = json.encoder.encode_basestring_ascii
json.encoder.encode_basestring_ascii = JSFunction.encode_basestring

So when you run the code with your example:

>>> data = '{"name": "Julien", "callback": {"__js_func__": "function() { return 10; }"}}'
>>> json.loads(data, object_hook=JSFunction.decode_js_func)
{u'callback': <JSFunction: self>, u'name': u'Julien'}

And to turn this into a nicely serialized json string, just json.dumps() it:

>>> json.dumps(json.loads(data, object_hook=JSFunction.decode_js_func))
'{"callback": function() { return 10; }, "name": "Julien"}'

Now this gets pretty messy since we've monkey-patched the behavior of the base json module (so it will impact everything app-wide), but it gets the job done.

I'd love to see a cleaner way of doing it!