[issue4136] merge json library with simplejson 2.0.3

Lib/json/decoder.py

 """Implementation of JSONDecoder
 """
 
 import re
 import sys
+import struct
 
-from json.scanner import Scanner, pattern
+from json.scanner import make_scanner
 try:
     from _json import scanstring as c_scanstring
 except ImportError:
     c_scanstring = None
 
 __all__ = ['JSONDecoder']
 
 FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
 
 NaN, PosInf, NegInf = float('nan'), float('inf'), float('-inf')
@@ skipping 15 matching lines @@
         return fmt.format(msg, lineno, colno, pos)
     endlineno, endcolno = linecol(doc, end)
     fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
     return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
 
 
 _CONSTANTS = {
     '-Infinity': NegInf,
     'Infinity': PosInf,
     'NaN': NaN,
-    'true': True,
-    'false': False,
-    'null': None,
 }
-
-
-def JSONConstant(match, context, c=_CONSTANTS):
-    s = match.group(0)
-    fn = getattr(context, 'parse_constant', None)
-    if fn is None:
-        rval = c[s]
-    else:
-        rval = fn(s)
-    return rval, None
-pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
-
-
-def JSONNumber(match, context):
-    match = JSONNumber.regex.match(match.string, *match.span())
-    integer, frac, exp = match.groups()
-    if frac or exp:
-        fn = getattr(context, 'parse_float', None) or float
-        res = fn(integer + (frac or '') + (exp or ''))
-    else:
-        fn = getattr(context, 'parse_int', None) or int
-        res = fn(integer)
-    return res, None
-pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
-
 
 STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
 BACKSLASH = {
     '"': u'"', '\\': u'\\', '/': u'/',
     'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
 }
 
 DEFAULT_ENCODING = "utf-8"
 
 
 def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):

[Martin v. Löwis, 2009/01/04 13:22:29]

This function should get some comments what all the various cases are
(preferably speaking with the terms of JSON spec, i.e. chars, char, ...)

[bob.ippolito, 2009/01/05 01:28:19]

Commented in the next patch.

     if encoding is None:
         encoding = DEFAULT_ENCODING
     chunks = []
     _append = chunks.append
     begin = end - 1
     while 1:
         chunk = _m(s, end)
         if chunk is None:
             raise ValueError(
                 errmsg("Unterminated string starting at", s, begin))
         end = chunk.end()
         content, terminator = chunk.groups()
         if content:
             if not isinstance(content, unicode):
                 content = unicode(content, encoding)
             _append(content)

[Martin v. Löwis, 2009/01/04 13:22:29]

# 3 cases: end of string, control character, escape sequence

[bob.ippolito, 2009/01/05 01:28:19]

Commented in the next patch

         if terminator == '"':
             break
         elif terminator != '\\':
             if strict:
-                msg = "Invalid control character {0!r} at".format(terminator)
+                msg = "Invalid control character {0!r} at".format(esc)

[Martin v. Löwis, 2009/01/04 13:22:29]

esc isn't assigned until a few lines later. Is this really correct?

[bob.ippolito, 2009/01/05 01:28:19]

This is a bug in the exception handling code, fixed in the next patch.

                 raise ValueError(errmsg(msg, s, end))
             else:
                 _append(terminator)
                 continue
         try:
             esc = s[end]
         except IndexError:
             raise ValueError(
                 errmsg("Unterminated string starting at", s, begin))
         if esc != 'u':
             try:
                 m = _b[esc]
             except KeyError:
-                msg = "Invalid \\escape: {0!r}".format(esc)
-                raise ValueError(errmsg(msg, s, end))
+                raise ValueError(
+                    errmsg("Invalid \\escape: {0!r}".format(esc), s, end))
             end += 1
         else:
             esc = s[end + 1:end + 5]
             next_end = end + 5
             msg = "Invalid \\uXXXX escape"
             try:
                 if len(esc) != 4:
                     raise ValueError
                 uni = int(esc, 16)
                 if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
                     msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
                     if not s[end + 5:end + 7] == '\\u':
                         raise ValueError

[Martin v. Löwis, 2009/01/04 13:22:29]

No message?

[bob.ippolito, 2009/01/05 01:28:19]

Exception is caught at the except block and re-raised with a message. Next patch
unrolls this so it's not confusing.

                     esc2 = s[end + 7:end + 11]
                     if len(esc2) != 4:
                         raise ValueError

[Martin v. Löwis, 2009/01/04 13:22:29]

No message?

[bob.ippolito, 2009/01/05 01:28:19]

Exception is caught at the except block and re-raised with a message. Next patch
unrolls this so it's not confusing.

                     uni2 = int(esc2, 16)
                     uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
                     next_end += 6
                 m = unichr(uni)

[Martin v. Löwis, 2009/01/04 13:22:29]

What's the purpose of m?

[bob.ippolito, 2009/01/05 01:28:19]

Renamed m to char in the next patch.

             except ValueError:
                 raise ValueError(errmsg(msg, s, end))
             end = next_end
         _append(m)
     return u''.join(chunks), end
 
 
-# Use speedup
-if c_scanstring is not None:
-    scanstring = c_scanstring
-else:
-    scanstring = py_scanstring
+# Use speedup if available
+scanstring = c_scanstring or py_scanstring
 
-def JSONString(match, context):
-    encoding = getattr(context, 'encoding', None)
-    strict = getattr(context, 'strict', True)
-    return scanstring(match.string, match.end(), encoding, strict)
-pattern(r'"')(JSONString)
+WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
+WHITESPACE_STR = ' \t\n\r'
 
-
-WHITESPACE = re.compile(r'\s*', FLAGS)
-
-
-def JSONObject(match, context, _w=WHITESPACE.match):
+def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
     pairs = {}
-    s = match.string
-    end = _w(s, match.end()).end()
     nextchar = s[end:end + 1]

[Martin v. Löwis, 2009/01/04 13:22:29]

Why not s[end]? Add comment if this is necessary.

[bob.ippolito, 2009/01/05 01:28:19]

commented in next patch (only once). s[end] can raise an IndexError with bad
input, s[end:end+1] returns an empty string on underflow, which is caught later
with a more helpful error message.

-    # Trivial empty object
-    if nextchar == '}':
-        return pairs, end + 1
+    # Normally we expect nextchar == '"'
     if nextchar != '"':
-        raise ValueError(errmsg("Expecting property name", s, end))
+        if nextchar in _ws:
+            end = _w(s, end).end()
+            nextchar = s[end:end + 1]

[Martin v. Löwis, 2009/01/04 13:22:29]

Likewise. There are more places where it does slicing, but also places where it
does indexing, in this function.

[bob.ippolito, 2009/01/05 01:28:19]

commented in next patch (only once). s[end] can raise an IndexError with bad
input, s[end:end+1] returns an empty string on underflow, which is caught later
with a more helpful error message.

+        # Trivial empty object
+        if nextchar == '}':
+            return pairs, end + 1
+        elif nextchar != '"':
+            raise ValueError(errmsg("Expecting property name", s, end))
     end += 1
-    encoding = getattr(context, 'encoding', None)
-    strict = getattr(context, 'strict', True)
-    iterscan = JSONScanner.iterscan
     while True:
         key, end = scanstring(s, end, encoding, strict)
-        end = _w(s, end).end()
+
+        # To skip some function call overhead we optimize the fast paths where
+        # the JSON key separator is ": " or just ":".
         if s[end:end + 1] != ':':
-            raise ValueError(errmsg("Expecting : delimiter", s, end))
-        end = _w(s, end + 1).end()
+            end = _w(s, end).end()
+            if s[end:end + 1] != ':':
+                raise ValueError(errmsg("Expecting : delimiter", s, end))
+
+        end += 1
+
         try:
-            value, end = iterscan(s, idx=end, context=context).next()
+            if s[end] in _ws:
+                end += 1
+                if s[end] in _ws:
+                    end = _w(s, end + 1).end()
+        except IndexError:
+            pass
+
+        try:
+            value, end = scan_once(s, end)
         except StopIteration:
             raise ValueError(errmsg("Expecting object", s, end))
         pairs[key] = value
-        end = _w(s, end).end()
-        nextchar = s[end:end + 1]
+
+        try:
+            nextchar = s[end]
+            if nextchar in _ws:
+                end = _w(s, end + 1).end()
+                nextchar = s[end]
+        except IndexError:
+            nextchar = ''
         end += 1
+
         if nextchar == '}':
             break
-        if nextchar != ',':
+        elif nextchar != ',':
             raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
-        end = _w(s, end).end()
-        nextchar = s[end:end + 1]
+
+        try:
+            nextchar = s[end]
+            if nextchar in _ws:
+                end += 1
+                nextchar = s[end]
+                if nextchar in _ws:
+                    end = _w(s, end + 1).end()
+                    nextchar = s[end]
+        except IndexError:
+            nextchar = ''
+
         end += 1
         if nextchar != '"':
             raise ValueError(errmsg("Expecting property name", s, end - 1))
-    object_hook = getattr(context, 'object_hook', None)
+
     if object_hook is not None:
         pairs = object_hook(pairs)
     return pairs, end
-pattern(r'{')(JSONObject)
 
-
-def JSONArray(match, context, _w=WHITESPACE.match):
+def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
     values = []
-    s = match.string
-    end = _w(s, match.end()).end()
+    nextchar = s[end:end + 1]
+    if nextchar in _ws:
+        end = _w(s, end + 1).end()
+        nextchar = s[end:end + 1]
     # Look-ahead for trivial empty array
-    nextchar = s[end:end + 1]
     if nextchar == ']':
         return values, end + 1
-    iterscan = JSONScanner.iterscan
+    _append = values.append
     while True:
         try:
-            value, end = iterscan(s, idx=end, context=context).next()
+            value, end = scan_once(s, end)
         except StopIteration:
             raise ValueError(errmsg("Expecting object", s, end))
-        values.append(value)
-        end = _w(s, end).end()
+        _append(value)
         nextchar = s[end:end + 1]
+        if nextchar in _ws:
+            end = _w(s, end + 1).end()
+            nextchar = s[end:end + 1]
         end += 1
         if nextchar == ']':
             break
-        if nextchar != ',':
+        elif nextchar != ',':
             raise ValueError(errmsg("Expecting , delimiter", s, end))
-        end = _w(s, end).end()
+
+        try:
+            if s[end] in _ws:
+                end += 1
+                if s[end] in _ws:
+                    end = _w(s, end + 1).end()
+        except IndexError:
+            pass
+
     return values, end
-pattern(r'\[')(JSONArray)
-
-
-ANYTHING = [
-    JSONObject,
-    JSONArray,
-    JSONString,
-    JSONConstant,
-    JSONNumber,
-]
-
-JSONScanner = Scanner(ANYTHING)
-
 
 class JSONDecoder(object):
     """Simple JSON <http://json.org> decoder
 
     Performs the following translations in decoding by default:
 
     +---------------+-------------------+
     | JSON          | Python            |
     +===============+===================+
     | object        | dict              |
@@ skipping 10 matching lines @@
     +---------------+-------------------+
     | false         | False             |
     +---------------+-------------------+
     | null          | None              |
     +---------------+-------------------+
 
     It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
     their corresponding ``float`` values, which is outside the JSON spec.
     """
 
-    _scanner = Scanner(ANYTHING)
     __all__ = ['__init__', 'decode', 'raw_decode']
 
     def __init__(self, encoding=None, object_hook=None, parse_float=None,
             parse_int=None, parse_constant=None, strict=True):
         """``encoding`` determines the encoding used to interpret any ``str``
         objects decoded by this instance (utf-8 by default).  It has no
         effect when decoding ``unicode`` objects.
 
         Note that currently only encodings that are a superset of ASCII work,
         strings of other encodings should be passed in as ``unicode``.
 
-        ``object_hook``, if specified, will be called with the result of
-        every JSON object decoded and its return value will be used in
+        ``object_hook``, if specified, will be called with the result
+        of every JSON object decoded and its return value will be used in
         place of the given ``dict``.  This can be used to provide custom
         deserializations (e.g. to support JSON-RPC class hinting).
 
         ``parse_float``, if specified, will be called with the string
         of every JSON float to be decoded. By default this is equivalent to
         float(num_str). This can be used to use another datatype or parser
         for JSON floats (e.g. decimal.Decimal).
 
         ``parse_int``, if specified, will be called with the string
         of every JSON int to be decoded. By default this is equivalent to
         int(num_str). This can be used to use another datatype or parser
         for JSON integers (e.g. float).
 
         ``parse_constant``, if specified, will be called with one of the
-        following strings: -Infinity, Infinity, NaN, null, true, false.
+        following strings: -Infinity, Infinity, NaN.

[Martin v. Löwis, 2009/01/04 13:22:29]

This sounds like an incompatible change.

[bob.ippolito, 2009/01/05 01:28:19]

Not practically speaking. The documented purpose of this callback is "This can
be used to raise an exception if invalid JSON numbers are encountered.". I've
never seen it used to handle None, True, False in a different manner. That was
more of an implementation detail than anything else, and that is fixed by this
patch. Existing implementations of this callback will simply have dead code
since they will never be called with "null", "true" or "false" anymore.

         This can be used to raise an exception if invalid JSON numbers
         are encountered.
 
         """
         self.encoding = encoding
         self.object_hook = object_hook
-        self.parse_float = parse_float
-        self.parse_int = parse_int
-        self.parse_constant = parse_constant
+        self.parse_float = parse_float or float
+        self.parse_int = parse_int or int
+        self.parse_constant = parse_constant or _CONSTANTS.__getitem__
         self.strict = strict
+        self.parse_object = JSONObject
+        self.parse_array = JSONArray
+        self.parse_string = scanstring
+        self.scan_once = make_scanner(self)
 
     def decode(self, s, _w=WHITESPACE.match):
-        """
-        Return the Python representation of ``s`` (a ``str`` or ``unicode``
+        """Return the Python representation of ``s`` (a ``str`` or ``unicode``
         instance containing a JSON document)
 
         """
         obj, end = self.raw_decode(s, idx=_w(s, 0).end())
         end = _w(s, end).end()
         if end != len(s):
             raise ValueError(errmsg("Extra data", s, end, len(s)))
         return obj
 
-    def raw_decode(self, s, **kw):
+    def raw_decode(self, s, idx=0):

[Martin v. Löwis, 2009/01/04 13:22:29]

That looks like an incompatible change

[bob.ippolito, 2009/01/05 01:28:19]

It is a compatible change.

         """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
         with a JSON document) and return a 2-tuple of the Python
         representation and the index in ``s`` where the document ended.
 
         This can be used to decode a JSON document from a string that may
         have extraneous data at the end.
 
         """
-        kw.setdefault('context', self)
         try:
-            obj, end = self._scanner.iterscan(s, **kw).next()
+            obj, end = self.scan_once(s, idx)
         except StopIteration:
             raise ValueError("No JSON object could be decoded")
         return obj, end