diff -r 27dda39db119 Lib/json/decoder.py --- a/Lib/json/decoder.py Fri Dec 04 12:30:16 2009 +0100 +++ b/Lib/json/decoder.py Sat Dec 05 02:00:29 2009 +0100 @@ -147,7 +147,12 @@ WHITESPACE = re.compile(r'[ \t\n\r]*', F WHITESPACE_STR = ' \t\n\r' def JSONObject((s, end), encoding, strict, scan_once, object_hook, - object_pairs_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR): + object_pairs_hook, memo=None, + _w=WHITESPACE.match, _ws=WHITESPACE_STR): + # Backwards compatibility + if memo is None: + memo = {} + memo_get = memo.setdefault pairs = [] pairs_append = pairs.append # Use a slice to prevent IndexError from being raised, the following @@ -166,6 +171,7 @@ def JSONObject((s, end), encoding, stric end += 1 while True: key, end = scanstring(s, end, encoding, strict) + key = memo_get(key, key) # To skip some function call overhead we optimize the fast paths where # the JSON key separator is ": " or just ":". @@ -334,6 +340,7 @@ class JSONDecoder(object): self.parse_object = JSONObject self.parse_array = JSONArray self.parse_string = scanstring + self.memo = {} self.scan_once = make_scanner(self) def decode(self, s, _w=WHITESPACE.match): diff -r 27dda39db119 Lib/json/scanner.py --- a/Lib/json/scanner.py Fri Dec 04 12:30:16 2009 +0100 +++ b/Lib/json/scanner.py Sat Dec 05 02:00:29 2009 +0100 @@ -24,6 +24,7 @@ def py_make_scanner(context): parse_constant = context.parse_constant object_hook = context.object_hook object_pairs_hook = context.object_pairs_hook + memo = context.memo def _scan_once(string, idx): try: @@ -35,7 +36,7 @@ def py_make_scanner(context): return parse_string(string, idx + 1, encoding, strict) elif nextchar == '{': return parse_object((string, idx + 1), encoding, strict, - _scan_once, object_hook, object_pairs_hook) + _scan_once, object_hook, object_pairs_hook, memo) elif nextchar == '[': return parse_array((string, idx + 1), _scan_once) elif nextchar == 'n' and string[idx:idx + 4] == 'null': diff -r 27dda39db119 Lib/json/tests/test_decode.py --- a/Lib/json/tests/test_decode.py Fri Dec 04 12:30:16 2009 +0100 +++ b/Lib/json/tests/test_decode.py Sat Dec 05 02:00:29 2009 +0100 @@ -1,10 +1,25 @@ import decimal from unittest import TestCase from StringIO import StringIO +from contextlib import contextmanager import json +import json.decoder +import json.scanner from collections import OrderedDict + +@contextmanager +def use_python_scanner(): + py_scanner = json.scanner.py_make_scanner + old_scanner = json.decoder.make_scanner + json.decoder.make_scanner = py_scanner + try: + yield + finally: + json.decoder.make_scanner = old_scanner + + class TestDecode(TestCase): def test_decimal(self): rval = json.loads('1.1', parse_float=decimal.Decimal) @@ -39,3 +54,22 @@ class TestDecode(TestCase): object_pairs_hook=OrderedDict, object_hook=lambda x: None), OrderedDict(p)) + + def check_keys_reuse(self, source, loads): + rval = loads(source) + (a, b), (c, d) = sorted(rval[0]), sorted(rval[1]) + self.assertIs(a, c) + self.assertIs(b, d) + + def test_keys_reuse_str(self): + s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'.encode('utf8') + self.check_keys_reuse(s, json.loads) + with use_python_scanner(): + self.check_keys_reuse(s, json.decoder.JSONDecoder().decode) + + def test_keys_reuse_unicode(self): + s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]' + self.check_keys_reuse(s, json.loads) + with use_python_scanner(): + self.check_keys_reuse(s, json.decoder.JSONDecoder().decode) + diff -r 27dda39db119 Modules/_json.c --- a/Modules/_json.c Fri Dec 04 12:30:16 2009 +0100 +++ b/Modules/_json.c Sat Dec 05 02:00:29 2009 +0100 @@ -39,6 +39,7 @@ typedef struct _PyScannerObject { PyObject *parse_float; PyObject *parse_int; PyObject *parse_constant; + PyObject *memo; } PyScannerObject; static PyMemberDef scanner_members[] = { @@ -424,6 +425,21 @@ _build_rval_index_tuple(PyObject *rval, return tpl; } +#define APPEND_OLD_CHUNK \ + if (chunk != NULL) { \ + if (chunks == NULL) { \ + chunks = PyList_New(0); \ + if (chunks == NULL) { \ + goto bail; \ + } \ + } \ + if (PyList_Append(chunks, chunk)) { \ + Py_DECREF(chunk); \ + goto bail; \ + } \ + Py_CLEAR(chunk); \ + } + static PyObject * scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) { @@ -442,10 +458,9 @@ scanstring_str(PyObject *pystr, Py_ssize Py_ssize_t next = begin; int has_unicode = 0; char *buf = PyString_AS_STRING(pystr); - PyObject *chunks = PyList_New(0); - if (chunks == NULL) { - goto bail; - } + PyObject *chunks = NULL; + PyObject *chunk = NULL; + if (end < 0 || len <= end) { PyErr_SetString(PyExc_ValueError, "end is out of bounds"); goto bail; @@ -453,7 +468,6 @@ scanstring_str(PyObject *pystr, Py_ssize while (1) { /* Find the end of the string or the next escape */ Py_UNICODE c = 0; - PyObject *chunk = NULL; for (next = end; next < len; next++) { c = (unsigned char)buf[next]; if (c == '"' || c == '\\') { @@ -473,6 +487,7 @@ scanstring_str(PyObject *pystr, Py_ssize } /* Pick up this chunk if it's not zero length */ if (next != end) { + APPEND_OLD_CHUNK PyObject *strchunk = PyString_FromStringAndSize(&buf[end], next - end); if (strchunk == NULL) { goto bail; @@ -487,11 +502,6 @@ scanstring_str(PyObject *pystr, Py_ssize else { chunk = strchunk; } - if (PyList_Append(chunks, chunk)) { - Py_DECREF(chunk); - goto bail; - } - Py_DECREF(chunk); } next++; if (c == '"') { @@ -596,6 +606,7 @@ scanstring_str(PyObject *pystr, Py_ssize if (c > 0x7f) { has_unicode = 1; } + APPEND_OLD_CHUNK if (has_unicode) { chunk = PyUnicode_FromUnicode(&c, 1); if (chunk == NULL) { @@ -609,22 +620,27 @@ scanstring_str(PyObject *pystr, Py_ssize goto bail; } } - if (PyList_Append(chunks, chunk)) { - Py_DECREF(chunk); + } + + if (chunks == NULL) { + if (chunk != NULL) + rval = chunk; + else + rval = PyString_FromStringAndSize("", 0); + } + else { + APPEND_OLD_CHUNK + rval = join_list_string(chunks); + if (rval == NULL) { goto bail; } - Py_DECREF(chunk); + Py_CLEAR(chunks); } - - rval = join_list_string(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); *next_end_ptr = end; return rval; bail: *next_end_ptr = -1; + Py_XDECREF(chunk); Py_XDECREF(chunks); return NULL; } @@ -646,10 +662,9 @@ scanstring_unicode(PyObject *pystr, Py_s Py_ssize_t begin = end - 1; Py_ssize_t next = begin; const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr); - PyObject *chunks = PyList_New(0); - if (chunks == NULL) { - goto bail; - } + PyObject *chunks = NULL; + PyObject *chunk = NULL; + if (end < 0 || len <= end) { PyErr_SetString(PyExc_ValueError, "end is out of bounds"); goto bail; @@ -657,7 +672,6 @@ scanstring_unicode(PyObject *pystr, Py_s while (1) { /* Find the end of the string or the next escape */ Py_UNICODE c = 0; - PyObject *chunk = NULL; for (next = end; next < len; next++) { c = buf[next]; if (c == '"' || c == '\\') { @@ -674,15 +688,11 @@ scanstring_unicode(PyObject *pystr, Py_s } /* Pick up this chunk if it's not zero length */ if (next != end) { + APPEND_OLD_CHUNK chunk = PyUnicode_FromUnicode(&buf[end], next - end); if (chunk == NULL) { goto bail; } - if (PyList_Append(chunks, chunk)) { - Py_DECREF(chunk); - goto bail; - } - Py_DECREF(chunk); } next++; if (c == '"') { @@ -784,26 +794,32 @@ scanstring_unicode(PyObject *pystr, Py_s } #endif } + APPEND_OLD_CHUNK chunk = PyUnicode_FromUnicode(&c, 1); if (chunk == NULL) { goto bail; } - if (PyList_Append(chunks, chunk)) { - Py_DECREF(chunk); + } + + if (chunks == NULL) { + if (chunk != NULL) + rval = chunk; + else + rval = PyUnicode_FromStringAndSize("", 0); + } + else { + APPEND_OLD_CHUNK + rval = join_list_unicode(chunks); + if (rval == NULL) { goto bail; } - Py_DECREF(chunk); + Py_CLEAR(chunks); } - - rval = join_list_unicode(chunks); - if (rval == NULL) { - goto bail; - } - Py_DECREF(chunks); *next_end_ptr = end; return rval; bail: *next_end_ptr = -1; + Py_XDECREF(chunk); Py_XDECREF(chunks); return NULL; } @@ -913,6 +929,7 @@ scanner_clear(PyObject *self) Py_CLEAR(s->parse_float); Py_CLEAR(s->parse_int); Py_CLEAR(s->parse_constant); + Py_CLEAR(s->memo); return 0; } @@ -927,18 +944,26 @@ _parse_object_str(PyScannerObject *s, Py */ char *str = PyString_AS_STRING(pystr); Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *rval; - PyObject *pairs; + PyObject *rval = NULL; + PyObject *pairs = NULL; PyObject *item; PyObject *key = NULL; PyObject *val = NULL; char *encoding = PyString_AS_STRING(s->encoding); int strict = PyObject_IsTrue(s->strict); + int has_pairs_hook = (s->pairs_hook != Py_None); Py_ssize_t next_idx; - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; + if (has_pairs_hook) { + pairs = PyList_New(0); + if (pairs == NULL) + return NULL; + } + else { + rval = PyDict_New(); + if (rval == NULL) + return NULL; + } /* skip whitespace after { */ while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; @@ -946,6 +971,8 @@ _parse_object_str(PyScannerObject *s, Py /* only loop if the object is non-empty */ if (idx <= end_idx && str[idx] != '}') { while (idx <= end_idx) { + PyObject *memokey; + /* read key */ if (str[idx] != '"') { raise_errmsg("Expecting property name", pystr, idx); @@ -954,6 +981,16 @@ _parse_object_str(PyScannerObject *s, Py key = scanstring_str(pystr, idx + 1, encoding, strict, &next_idx); if (key == NULL) goto bail; + memokey = PyDict_GetItem(s->memo, key); + if (memokey != NULL) { + Py_INCREF(memokey); + Py_DECREF(key); + key = memokey; + } + else { + if (PyDict_SetItem(s->memo, key, key) < 0) + goto bail; + } idx = next_idx; /* skip whitespace between key and : delimiter, read :, skip whitespace */ @@ -970,16 +1007,24 @@ _parse_object_str(PyScannerObject *s, Py if (val == NULL) goto bail; - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { + if (has_pairs_hook) { + item = PyTuple_Pack(2, key, val); + if (item == NULL) + goto bail; + Py_CLEAR(key); + Py_CLEAR(val); + if (PyList_Append(pairs, item) == -1) { + Py_DECREF(item); + goto bail; + } Py_DECREF(item); - goto bail; } - Py_DECREF(item); + else { + if (PyDict_SetItem(rval, key, val) < 0) + goto bail; + Py_CLEAR(key); + Py_CLEAR(val); + } idx = next_idx; /* skip whitespace before } or , */ @@ -1016,12 +1061,6 @@ _parse_object_str(PyScannerObject *s, Py return val; } - rval = PyObject_CallFunctionObjArgs((PyObject *)(&PyDict_Type), - pairs, NULL); - if (rval == NULL) - goto bail; - Py_CLEAR(pairs); - /* if object_hook is not None: rval = object_hook(rval) */ if (s->object_hook != Py_None) { val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); @@ -1034,6 +1073,7 @@ _parse_object_str(PyScannerObject *s, Py *next_idx_ptr = idx + 1; return rval; bail: + Py_XDECREF(rval); Py_XDECREF(key); Py_XDECREF(val); Py_XDECREF(pairs); @@ -1057,11 +1097,19 @@ _parse_object_unicode(PyScannerObject *s PyObject *key = NULL; PyObject *val = NULL; int strict = PyObject_IsTrue(s->strict); + int has_pairs_hook = (s->pairs_hook != Py_None); Py_ssize_t next_idx; - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; + if (has_pairs_hook) { + pairs = PyList_New(0); + if (pairs == NULL) + return NULL; + } + else { + rval = PyDict_New(); + if (rval == NULL) + return NULL; + } /* skip whitespace after { */ while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; @@ -1069,6 +1117,8 @@ _parse_object_unicode(PyScannerObject *s /* only loop if the object is non-empty */ if (idx <= end_idx && str[idx] != '}') { while (idx <= end_idx) { + PyObject *memokey; + /* read key */ if (str[idx] != '"') { raise_errmsg("Expecting property name", pystr, idx); @@ -1077,6 +1127,16 @@ _parse_object_unicode(PyScannerObject *s key = scanstring_unicode(pystr, idx + 1, strict, &next_idx); if (key == NULL) goto bail; + memokey = PyDict_GetItem(s->memo, key); + if (memokey != NULL) { + Py_INCREF(memokey); + Py_DECREF(key); + key = memokey; + } + else { + if (PyDict_SetItem(s->memo, key, key) < 0) + goto bail; + } idx = next_idx; /* skip whitespace between key and : delimiter, read :, skip whitespace */ @@ -1093,16 +1153,24 @@ _parse_object_unicode(PyScannerObject *s if (val == NULL) goto bail; - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { + if (has_pairs_hook) { + item = PyTuple_Pack(2, key, val); + if (item == NULL) + goto bail; + Py_CLEAR(key); + Py_CLEAR(val); + if (PyList_Append(pairs, item) == -1) { + Py_DECREF(item); + goto bail; + } Py_DECREF(item); - goto bail; } - Py_DECREF(item); + else { + if (PyDict_SetItem(rval, key, val) < 0) + goto bail; + Py_CLEAR(key); + Py_CLEAR(val); + } idx = next_idx; /* skip whitespace before } or , */ @@ -1140,12 +1208,6 @@ _parse_object_unicode(PyScannerObject *s return val; } - rval = PyObject_CallFunctionObjArgs((PyObject *)(&PyDict_Type), - pairs, NULL); - if (rval == NULL) - goto bail; - Py_CLEAR(pairs); - /* if object_hook is not None: rval = object_hook(rval) */ if (s->object_hook != Py_None) { val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); @@ -1158,6 +1220,7 @@ _parse_object_unicode(PyScannerObject *s *next_idx_ptr = idx + 1; return rval; bail: + Py_XDECREF(rval); Py_XDECREF(key); Py_XDECREF(val); Py_XDECREF(pairs); @@ -1729,6 +1792,12 @@ scanner_init(PyObject *self, PyObject *a if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) return -1; + if (s->memo == NULL) { + s->memo = PyDict_New(); + if (s->memo == NULL) + goto bail; + } + /* PyString_AS_STRING is used on encoding */ s->encoding = PyObject_GetAttrString(ctx, "encoding"); if (s->encoding == Py_None) {