traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-38043: Move unicodedata.normalize tests into test_unicodedata. (GH-15712) 

Having these in a separate file from the one that's named after the
module in the usual way makes it very easy to miss them when looking
for tests for these two functions.

(In fact when working recently on is_normalized, I'd been surprised to
see no tests for it here and concluded the function had evaded being
tested at all.  I'd gone as far as to write up some tests myself
before I spotted this other file.)

Mostly this just means moving all the one file's code into the other,
and moving code from the module toplevel to inside the test class to
keep it tidily separate from the rest of the file's code.

There's one substantive change, which reduces by a bit the amount of
code to be moved: we drop the `x > sys.maxunicode` conditional and all
the `RangeError` logic behind it.  Now if that condition ever occurs
it will cause an error at `chr(x)`, and a test failure.  That's the
right result because, since PEP 393 in Python 3.3, there is no longer
such a thing as an "unsupported character".
This commit is contained in:
Greg Price 2019-09-10 02:29:26 -07:00 committed by Benjamin Peterson
parent 5b00dd8fa8
commit 1ad0c776cb
4 changed files with 102 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
Lib/test/test_normalization.py
View File

@ -1,117 +0,0 @@
from test.support import open_urlresource
import unittest
from http.client import HTTPException
import sys
from unicodedata import normalize, is_normalized, unidata_version
TESTDATAFILE = "NormalizationTest.txt"
TESTDATAURL = "http://www.pythontest.net/unicode/" + unidata_version + "/" + TESTDATAFILE
def check_version(testfile):
hdr = testfile.readline()
return unidata_version in hdr
class RangeError(Exception):
pass
def NFC(str):
return normalize("NFC", str)
def NFKC(str):
return normalize("NFKC", str)
def NFD(str):
return normalize("NFD", str)
def NFKD(str):
return normalize("NFKD", str)
def unistr(data):
data = [int(x, 16) for x in data.split(" ")]
for x in data:
if x > sys.maxunicode:
raise RangeError
return "".join([chr(x) for x in data])
class NormalizationTest(unittest.TestCase):
def test_main(self):
# Hit the exception early
try:
testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
check=check_version)
except PermissionError:
self.skipTest(f"Permission error when downloading {TESTDATAURL} "
f"into the test data directory")
except (OSError, HTTPException):
self.fail(f"Could not retrieve {TESTDATAURL}")
with testdata:
self.run_normalization_tests(testdata)
def run_normalization_tests(self, testdata):
part = None
part1_data = {}
for line in testdata:
if '#' in line:
line = line.split('#')[0]
line = line.strip()
if not line:
continue
if line.startswith("@Part"):
part = line.split()[0]
continue
try:
c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
except RangeError:
# Skip unsupported characters;
# try at least adding c1 if we are in part1
if part == "@Part1":
try:
c1 = unistr(line.split(';')[0])
except RangeError:
pass
else:
part1_data[c1] = 1
continue
# Perform tests
self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
self.assertTrue(c4 == NFC(c4) == NFC(c5), line)
self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
self.assertTrue(c5 == NFD(c4) == NFD(c5), line)
self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
NFKC(c3) == NFKC(c4) == NFKC(c5),
line)
self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
NFKD(c3) == NFKD(c4) == NFKD(c5),
line)
self.assertTrue(is_normalized("NFC", c2))
self.assertTrue(is_normalized("NFC", c4))
self.assertTrue(is_normalized("NFD", c3))
self.assertTrue(is_normalized("NFD", c5))
self.assertTrue(is_normalized("NFKC", c4))
self.assertTrue(is_normalized("NFKD", c5))
# Record part 1 data
if part == "@Part1":
part1_data[c1] = 1
# Perform tests for all other data
for c in range(sys.maxunicode+1):
X = chr(c)
if X in part1_data:
continue
self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)
def test_bug_834676(self):
# Check for bug 834676
normalize('NFC', '\ud55c\uae00')
if __name__ == "__main__":
unittest.main()

4
Lib/test/test_ucn.py
View File

@ -12,7 +12,6 @@ import unicodedata
from test import support
from http.client import HTTPException
from test.test_normalization import check_version
try:
from _testcapi import INT_MAX, PY_SSIZE_T_MAX, UINT_MAX
@ -172,6 +171,9 @@ class UnicodeNamesTest(unittest.TestCase):
def test_named_sequences_full(self):
# Check all the named sequences
def check_version(testfile):
hdr = testfile.readline()
return unicodedata.unidata_version in hdr
url = ("http://www.pythontest.net/unicode/%s/NamedSequences.txt" %
unicodedata.unidata_version)
try:

110
Lib/test/test_unicodedata.py
View File

@ -7,10 +7,11 @@
"""
import hashlib
from http.client import HTTPException
import sys
import unicodedata
import unittest
from test.support import script_helper
from test.support import open_urlresource, script_helper
class UnicodeMethodsTest(unittest.TestCase):
@ -171,13 +172,6 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
self.assertRaises(TypeError, self.db.combining)
self.assertRaises(TypeError, self.db.combining, 'xx')
def test_normalize(self):
self.assertRaises(TypeError, self.db.normalize)
self.assertRaises(ValueError, self.db.normalize, 'unknown', 'xx')
self.assertEqual(self.db.normalize('NFKC', ''), '')
# The rest can be found in test_normalization.py
# which requires an external file.
def test_pr29(self):
# http://www.unicode.org/review/pr-29.html
# See issues #1054943 and #10254.
@ -208,9 +202,6 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b)
self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b)
# For tests of unicodedata.is_normalized / self.db.is_normalized ,
# see test_normalization.py .
def test_east_asian_width(self):
eaw = self.db.east_asian_width
self.assertRaises(TypeError, eaw, b'a')
@ -315,5 +306,102 @@ class UnicodeMiscTest(UnicodeDatabaseTest):
self.assertEqual(len(lines), 1,
r"\u%.4x should not be a linebreak" % i)
class NormalizationTest(unittest.TestCase):
@staticmethod
def check_version(testfile):
hdr = testfile.readline()
return unicodedata.unidata_version in hdr
@staticmethod
def unistr(data):
data = [int(x, 16) for x in data.split(" ")]
return "".join([chr(x) for x in data])
def test_normalization(self):
TESTDATAFILE = "NormalizationTest.txt"
TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}"
# Hit the exception early
try:
testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
check=self.check_version)
except PermissionError:
self.skipTest(f"Permission error when downloading {TESTDATAURL} "
f"into the test data directory")
except (OSError, HTTPException):
self.fail(f"Could not retrieve {TESTDATAURL}")
with testdata:
self.run_normalization_tests(testdata)
def run_normalization_tests(self, testdata):
part = None
part1_data = {}
def NFC(str):
return unicodedata.normalize("NFC", str)
def NFKC(str):
return unicodedata.normalize("NFKC", str)
def NFD(str):
return unicodedata.normalize("NFD", str)
def NFKD(str):
return unicodedata.normalize("NFKD", str)
for line in testdata:
if '#' in line:
line = line.split('#')[0]
line = line.strip()
if not line:
continue
if line.startswith("@Part"):
part = line.split()[0]
continue
c1,c2,c3,c4,c5 = [self.unistr(x) for x in line.split(';')[:-1]]
# Perform tests
self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
self.assertTrue(c4 == NFC(c4) == NFC(c5), line)
self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
self.assertTrue(c5 == NFD(c4) == NFD(c5), line)
self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
NFKC(c3) == NFKC(c4) == NFKC(c5),
line)
self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
NFKD(c3) == NFKD(c4) == NFKD(c5),
line)
self.assertTrue(unicodedata.is_normalized("NFC", c2))
self.assertTrue(unicodedata.is_normalized("NFC", c4))
self.assertTrue(unicodedata.is_normalized("NFD", c3))
self.assertTrue(unicodedata.is_normalized("NFD", c5))
self.assertTrue(unicodedata.is_normalized("NFKC", c4))
self.assertTrue(unicodedata.is_normalized("NFKD", c5))
# Record part 1 data
if part == "@Part1":
part1_data[c1] = 1
# Perform tests for all other data
for c in range(sys.maxunicode+1):
X = chr(c)
if X in part1_data:
continue
self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)
def test_edge_cases(self):
self.assertRaises(TypeError, unicodedata.normalize)
self.assertRaises(ValueError, unicodedata.normalize, 'unknown', 'xx')
self.assertEqual(unicodedata.normalize('NFKC', ''), '')
def test_bug_834676(self):
# Check for bug 834676
unicodedata.normalize('NFC', '\ud55c\uae00')
if __name__ == "__main__":
unittest.main()

1
PCbuild/lib.pyproj
View File

@ -1191,7 +1191,6 @@
<Compile Include="test\test_netrc.py" />
<Compile Include="test\test_nis.py" />
<Compile Include="test\test_nntplib.py" />
<Compile Include="test\test_normalization.py" />
<Compile Include="test\test_ntpath.py" />
<Compile Include="test\test_numeric_tower.py" />
<Compile Include="test\test_opcodes.py" />