mirror of https://github.com/python/cpython
gh-53203: Fix strptime() for %c, %x and %X formats on many locales (GH-125406)
Fixed most locales that use non-ASCII digits, like Persian, Burmese, Odia and Shan.
This commit is contained in:
Serhiy Storchaka
GitHub
parent
06ca33020e
commit
5f4e5b598c
|
|
@ -15,6 +15,7 @@ import time
|
|||
import locale
|
||||
import calendar
|
||||
from re import compile as re_compile
|
||||
from re import sub as re_sub
|
||||
from re import IGNORECASE
|
||||
from re import escape as re_escape
|
||||
from datetime import (date as datetime_date,
|
||||
|
|
@ -129,11 +130,23 @@ class LocaleTime(object):
|
|||
time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
|
||||
time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0))
|
||||
replacement_pairs = [
|
||||
('1999', '%Y'), ('99', '%y'), ('22', '%H'),
|
||||
('44', '%M'), ('55', '%S'), ('76', '%j'),
|
||||
('17', '%d'), ('03', '%m'), ('3', '%m'),
|
||||
# '3' needed for when no leading zero.
|
||||
('2', '%w'), ('10', '%I')]
|
||||
('1999', '%Y'), ('99', '%y'), ('22', '%H'),
|
||||
('44', '%M'), ('55', '%S'), ('76', '%j'),
|
||||
('17', '%d'), ('03', '%m'), ('3', '%m'),
|
||||
# '3' needed for when no leading zero.
|
||||
('2', '%w'), ('10', '%I'),
|
||||
# Non-ASCII digits
|
||||
('\u0661\u0669\u0669\u0669', '%Y'),
|
||||
('\u0669\u0669', '%Oy'),
|
||||
('\u0662\u0662', '%OH'),
|
||||
('\u0664\u0664', '%OM'),
|
||||
('\u0665\u0665', '%OS'),
|
||||
('\u0661\u0667', '%Od'),
|
||||
('\u0660\u0663', '%Om'),
|
||||
('\u0663', '%Om'),
|
||||
('\u0662', '%Ow'),
|
||||
('\u0661\u0660', '%OI'),
|
||||
]
|
||||
date_time = []
|
||||
for directive in ('%c', '%x', '%X'):
|
||||
current_format = time.strftime(directive, time_tuple).lower()
|
||||
|
|
@ -158,6 +171,10 @@ class LocaleTime(object):
|
|||
for tz in tz_values:
|
||||
if tz:
|
||||
current_format = current_format.replace(tz, "%Z")
|
||||
# Transform all non-ASCII digits to digits in range U+0660 to U+0669.
|
||||
current_format = re_sub(r'\d(?<![0-9])',
|
||||
lambda m: chr(0x0660 + int(m[0])),
|
||||
current_format)
|
||||
for old, new in replacement_pairs:
|
||||
current_format = current_format.replace(old, new)
|
||||
# If %W is used, then Sunday, 2005-01-03 will fall on week 0 since
|
||||
|
|
@ -267,7 +284,7 @@ class TimeRE(dict):
|
|||
else:
|
||||
self.locale_time = LocaleTime()
|
||||
base = super()
|
||||
base.__init__({
|
||||
mapping = {
|
||||
# The " [1-9]" part of the regex is to make %c from ANSI C work
|
||||
'd': r"(?P<d>3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
|
||||
'f': r"(?P<f>[0-9]{1,6})",
|
||||
|
|
@ -296,11 +313,15 @@ class TimeRE(dict):
|
|||
'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
|
||||
for tz in tz_names),
|
||||
'Z'),
|
||||
'%': '%'})
|
||||
base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
|
||||
base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
|
||||
base.__setitem__('x', self.pattern(self.locale_time.LC_date))
|
||||
'%': '%'}
|
||||
for d in 'dmyHIMS':
|
||||
mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d
|
||||
mapping['Ow'] = r'(?P<w>\d)'
|
||||
mapping['W'] = mapping['U'].replace('U', 'W')
|
||||
base.__init__(mapping)
|
||||
base.__setitem__('X', self.pattern(self.locale_time.LC_time))
|
||||
base.__setitem__('x', self.pattern(self.locale_time.LC_date))
|
||||
base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
|
||||
|
||||
def __seqToRE(self, to_convert, directive):
|
||||
"""Convert a list to a regex string for matching a directive.
|
||||
|
|
@ -328,28 +349,25 @@ class TimeRE(dict):
|
|||
regex syntax are escaped.
|
||||
|
||||
"""
|
||||
processed_format = ''
|
||||
# The sub() call escapes all characters that might be misconstrued
|
||||
# as regex syntax. Cannot use re.escape since we have to deal with
|
||||
# format directives (%m, etc.).
|
||||
regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
|
||||
format = regex_chars.sub(r"\\\1", format)
|
||||
whitespace_replacement = re_compile(r'\s+')
|
||||
format = whitespace_replacement.sub(r'\\s+', format)
|
||||
format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format)
|
||||
format = re_sub(r'\s+', r'\\s+', format)
|
||||
format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR
|
||||
year_in_format = False
|
||||
day_of_month_in_format = False
|
||||
while '%' in format:
|
||||
directive_index = format.index('%')+1
|
||||
format_char = format[directive_index]
|
||||
processed_format = "%s%s%s" % (processed_format,
|
||||
format[:directive_index-1],
|
||||
self[format_char])
|
||||
format = format[directive_index+1:]
|
||||
def repl(m):
|
||||
format_char = m[1]
|
||||
match format_char:
|
||||
case 'Y' | 'y' | 'G':
|
||||
nonlocal year_in_format
|
||||
year_in_format = True
|
||||
case 'd':
|
||||
nonlocal day_of_month_in_format
|
||||
day_of_month_in_format = True
|
||||
return self[format_char]
|
||||
format = re_sub(r'%(O?.)', repl, format)
|
||||
if day_of_month_in_format and not year_in_format:
|
||||
import warnings
|
||||
warnings.warn("""\
|
||||
|
|
@ -360,7 +378,7 @@ To avoid trouble, add a specific year to the input & format.
|
|||
See https://github.com/python/cpython/issues/70647.""",
|
||||
DeprecationWarning,
|
||||
skip_file_prefixes=(os.path.dirname(__file__),))
|
||||
return "%s%s" % (processed_format, format)
|
||||
return format
|
||||
|
||||
def compile(self, format):
|
||||
"""Return a compiled re object for the format string."""
|
||||
|
|
@ -434,8 +452,8 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
|
|||
_regex_cache[format] = format_regex
|
||||
found = format_regex.match(data_string)
|
||||
if not found:
|
||||
raise ValueError("time data %r does not match format %r :: /%s/" %
|
||||
(data_string, format, format_regex.pattern))
|
||||
raise ValueError("time data %r does not match format %r" %
|
||||
(data_string, format))
|
||||
if len(data_string) != found.end():
|
||||
raise ValueError("unconverted data remains: %s" %
|
||||
data_string[found.end():])
|
||||
|
|
|
|||
|
|
@ -292,7 +292,7 @@ class StrptimeTests(unittest.TestCase):
|
|||
# additional check for IndexError branch (issue #19545)
|
||||
with self.assertRaises(ValueError) as e:
|
||||
_strptime._strptime_time('19', '%Y %')
|
||||
self.assertIs(e.exception.__suppress_context__, True)
|
||||
self.assertIsNone(e.exception.__context__)
|
||||
|
||||
def test_unconverteddata(self):
|
||||
# Check ValueError is raised when there is unconverted data
|
||||
|
|
@ -485,12 +485,14 @@ class StrptimeTests(unittest.TestCase):
|
|||
# id_ID, ms_MY.
|
||||
# * Year is not included: ha_NG.
|
||||
# * Use non-Gregorian calendar: lo_LA, thai, th_TH.
|
||||
# On Windows: ar_IN, ar_SA, fa_IR, ps_AF.
|
||||
#
|
||||
# BUG: Generates regexp that does not match the current date and time
|
||||
# for az_IR, fa_IR, lzh_TW, my_MM, or_IN, shn_MM.
|
||||
# for lzh_TW.
|
||||
@run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
|
||||
'he_IL', 'eu_ES', 'ar_AE', 'mfe_MU', 'yo_NG',
|
||||
'csb_PL', 'br_FR', 'gez_ET', 'brx_IN')
|
||||
'csb_PL', 'br_FR', 'gez_ET', 'brx_IN',
|
||||
'my_MM', 'or_IN', 'shn_MM', 'az_IR')
|
||||
def test_date_time_locale(self):
|
||||
# Test %c directive
|
||||
loc = locale.getlocale(locale.LC_TIME)[0]
|
||||
|
|
@ -512,20 +514,23 @@ class StrptimeTests(unittest.TestCase):
|
|||
self.roundtrip('%c', slice(0, 6), time.localtime(now - 366*24*3600))
|
||||
|
||||
# NB: Dates before 1969 do not roundtrip on some locales:
|
||||
# bo_CN, bo_IN, dz_BT, eu_ES, eu_FR.
|
||||
# az_IR, bo_CN, bo_IN, dz_BT, eu_ES, eu_FR, fa_IR, or_IN.
|
||||
@run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
|
||||
'he_IL', 'ar_AE', 'mfe_MU', 'yo_NG',
|
||||
'csb_PL', 'br_FR', 'gez_ET', 'brx_IN')
|
||||
'csb_PL', 'br_FR', 'gez_ET', 'brx_IN',
|
||||
'my_MM', 'shn_MM')
|
||||
def test_date_time_locale2(self):
|
||||
# Test %c directive
|
||||
self.roundtrip('%c', slice(0, 6), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
|
||||
self.roundtrip('%c', slice(0, 6), (1800, 1, 1, 0, 0, 0, 0, 1, 0))
|
||||
|
||||
# NB: Does not roundtrip because use non-Gregorian calendar:
|
||||
# lo_LA, thai, th_TH.
|
||||
# lo_LA, thai, th_TH. On Windows: ar_IN, ar_SA, fa_IR, ps_AF.
|
||||
# BUG: Generates regexp that does not match the current date
|
||||
# for az_IR, fa_IR, lzh_TW, my_MM, or_IN, shn_MM.
|
||||
# for lzh_TW.
|
||||
@run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
|
||||
'he_IL', 'eu_ES', 'ar_AE')
|
||||
'he_IL', 'eu_ES', 'ar_AE',
|
||||
'az_IR', 'my_MM', 'or_IN', 'shn_MM')
|
||||
def test_date_locale(self):
|
||||
# Test %x directive
|
||||
now = time.time()
|
||||
|
|
@ -545,10 +550,11 @@ class StrptimeTests(unittest.TestCase):
|
|||
"musl libc issue on Emscripten, bpo-46390"
|
||||
)
|
||||
@run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
|
||||
'eu_ES', 'ar_AE')
|
||||
'eu_ES', 'ar_AE', 'my_MM', 'shn_MM')
|
||||
def test_date_locale2(self):
|
||||
# Test %x directive
|
||||
self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
|
||||
self.roundtrip('%x', slice(0, 3), (1800, 1, 1, 0, 0, 0, 0, 1, 0))
|
||||
|
||||
# NB: Does not roundtrip in some locales due to the ambiguity of
|
||||
# the time representation (bugs in locales?):
|
||||
|
|
@ -556,11 +562,11 @@ class StrptimeTests(unittest.TestCase):
|
|||
# norwegian, nynorsk.
|
||||
# * Hours are in 12-hour notation without AM/PM indication: hy_AM,
|
||||
# ms_MY, sm_WS.
|
||||
# BUG: Generates regexp that does not match the current time for
|
||||
# aa_DJ, aa_ER, aa_ET, am_ET, az_IR, byn_ER, fa_IR, gez_ER, gez_ET,
|
||||
# lzh_TW, my_MM, om_ET, om_KE, or_IN, shn_MM, sid_ET, so_DJ, so_ET,
|
||||
# so_SO, ti_ER, ti_ET, tig_ER, wal_ET.
|
||||
@run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP')
|
||||
# BUG: Generates regexp that does not match the current time for lzh_TW.
|
||||
@run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
|
||||
'aa_ET', 'am_ET', 'az_IR', 'byn_ER', 'fa_IR', 'gez_ET',
|
||||
'my_MM', 'om_ET', 'or_IN', 'shn_MM', 'sid_ET', 'so_SO',
|
||||
'ti_ET', 'tig_ER', 'wal_ET')
|
||||
def test_time_locale(self):
|
||||
# Test %X directive
|
||||
now = time.time()
|
||||
|
|
|
|||
|
|
@ -298,7 +298,7 @@ class TimeTestCase(unittest.TestCase):
|
|||
# additional check for IndexError branch (issue #19545)
|
||||
with self.assertRaises(ValueError) as e:
|
||||
time.strptime('19', '%Y %')
|
||||
self.assertIs(e.exception.__suppress_context__, True)
|
||||
self.assertIsNone(e.exception.__context__)
|
||||
|
||||
def test_strptime_leap_year(self):
|
||||
# GH-70647: warns if parsing a format with a day and no year.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
Fix :func:`time.strptime` for ``%c``, ``%x`` and ``%X`` formats in many
|
||||
locales that use non-ASCII digits, like Persian, Burmese, Odia and Shan.
|
||||
Loading…
Reference in New Issue