[issue3672] Ill-formed surrogates not treated as errors during encoding/decoding

Lib/test/test_codecs.py

 from test import support
 import unittest
 import codecs
 import sys, _testcapi, io
 
 class Queue(object):
     """
     queue: write bytes at one end, read bytes from the other end
     """
     def __init__(self, buffer):
@@ skipping 522 matching lines @@
                 "\x00\xff\u07ff\u0800",
                 "\x00\xff\u07ff\u0800",
                 "\x00\xff\u07ff\u0800\uffff",
             ]
         )
 
     def test_decoder_state(self):
         u = "\x00\x7f\x80\xff\u0100\u07ff\u0800\uffff\U0010ffff"
         self.check_state_handling_decode(self.encoding,
                                          u, u.encode(self.encoding))
+
+    def test_surrogates(self):

[Benjamin, 2009/05/01 21:25:44]

I think this should be split into 2 tests. "test_lone_surrogates" and
"test_surrogate_handler".

[Martin v. Löwis, 2009/05/02 09:44:03]

Done.

+        self.assertRaises(UnicodeEncodeError, "\ud800".encode, "utf-8")
+        self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "utf-8")
+        self.assertEquals("\ud800".encode("utf-8", "surrogates"), b"\xed\xa0\x80")
+        self.assertEquals(b"\xed\xa0\x80".decode("utf-8", "surrogates"), "\ud800")
+        self.assertTrue(codecs.lookup_error("surrogates"))
 
 class UTF7Test(ReadTest):
     encoding = "utf-7"
 
     def test_partial(self):
         self.check_partial(
             "a+-b",
             [
                 "a",
                 "a",
@@ skipping 463 matching lines @@
 
 
 class NameprepTest(unittest.TestCase):
     def test_nameprep(self):
         from encodings.idna import nameprep
         for pos, (orig, prepped) in enumerate(nameprep_tests):
             if orig is None:
                 # Skipped
                 continue
             # The Unicode strings are given in UTF-8
-            orig = str(orig, "utf-8")
+            orig = str(orig, "utf-8", "surrogates")
             if prepped is None:
                 # Input contains prohibited characters
                 self.assertRaises(UnicodeError, nameprep, orig)
             else:
-                prepped = str(prepped, "utf-8")
+                prepped = str(prepped, "utf-8", "surrogates")
                 try:
                     self.assertEquals(nameprep(orig), prepped)
                 except Exception as e:
                     raise support.TestFailed("Test 3.%d: %s" % (pos+1, str(e)))
 
 class IDNACodecTest(unittest.TestCase):
     def test_builtin_decode(self):
         self.assertEquals(str(b"python.org", "idna"), "python.org")
         self.assertEquals(str(b"python.org.", "idna"), "python.org.")
         self.assertEquals(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org")
@@ skipping 488 matching lines @@
         EncodedFileTest,
         BasicUnicodeTest,
         CharmapTest,
         WithStmtTest,
         TypesTest,
     )
 
 
 if __name__ == "__main__":
     test_main()