Issue 7846: fnmatch cache can grow without bound

Updated to solution to use the functools.lru_cache().
Restores the API so that purge() is not needed
(because the cache never gets big).
This commit is contained in:
Raymond Hettinger 2010-08-09 02:07:15 +00:00
parent 44c6c155d6
commit 5b0c1e07ca
3 changed files with 14 additions and 60 deletions

View File

@ -84,13 +84,6 @@ patterns.
<_sre.SRE_Match object at 0x...>
.. function:: purge()
Clear the internal pattern cache.
.. versionadded:: 3.2
.. seealso::
Module :mod:`glob`

View File

@ -12,19 +12,9 @@ corresponding to PATTERN. (It does not compile it.)
import os
import posixpath
import re
import functools
__all__ = ["filter", "fnmatch", "fnmatchcase", "purge", "translate"]
_cache = {} # Maps text patterns to compiled regexen.
_cacheb = {} # Ditto for bytes patterns.
_MAXCACHE = 100 # Maximum size of caches.
def purge():
"""Clear the pattern cache."""
_cache.clear()
_cacheb.clear()
__all__ = ["filter", "fnmatch", "fnmatchcase", "translate"]
def fnmatch(name, pat):
"""Test whether FILENAME matches PATTERN.
@ -45,28 +35,21 @@ def fnmatch(name, pat):
pat = os.path.normcase(pat)
return fnmatchcase(name, pat)
def _compile_pattern(pat):
cache = _cacheb if isinstance(pat, bytes) else _cache
regex = cache.get(pat)
if regex is None:
if isinstance(pat, bytes):
pat_str = str(pat, 'ISO-8859-1')
res_str = translate(pat_str)
res = bytes(res_str, 'ISO-8859-1')
else:
res = translate(pat)
if len(cache) >= _MAXCACHE:
cache.clear()
cache[pat] = regex = re.compile(res)
return regex.match
@functools.lru_cache(maxsize=250)
def _compile_pattern(pat, is_bytes=False):
if is_bytes:
pat_str = str(pat, 'ISO-8859-1')
res_str = translate(pat_str)
res = bytes(res_str, 'ISO-8859-1')
else:
res = translate(pat)
return re.compile(res).match
def filter(names, pat):
"""Return the subset of the list NAMES that match PAT."""
result = []
pat = os.path.normcase(pat)
match = _compile_pattern(pat)
match = _compile_pattern(pat, isinstance(pat, bytes))
if os.path is posixpath:
# normcase on posix is NOP. Optimize it away from the loop.
for name in names:
@ -78,14 +61,13 @@ def filter(names, pat):
result.append(name)
return result
def fnmatchcase(name, pat):
"""Test whether FILENAME matches PATTERN, including case.
This is a version of fnmatch() which doesn't case-normalize
its arguments.
"""
match = _compile_pattern(pat)
match = _compile_pattern(pat, isinstance(pat, bytes))
return match(name) is not None

View File

@ -3,15 +3,10 @@
from test import support
import unittest
from fnmatch import (fnmatch, fnmatchcase, _MAXCACHE, _cache, _cacheb, purge,
translate, filter)
from fnmatch import fnmatch, fnmatchcase, translate, filter
class FnmatchTestCase(unittest.TestCase):
def tearDown(self):
purge()
def check_match(self, filename, pattern, should_match=1, fn=fnmatch):
if should_match:
self.assertTrue(fn(filename, pattern),
@ -65,22 +60,6 @@ class FnmatchTestCase(unittest.TestCase):
self.check_match(b'test\xff', b'te*\xff')
self.check_match(b'foo\nbar', b'foo*')
def test_cache_clearing(self):
# check that caches do not grow too large
# http://bugs.python.org/issue7846
# string pattern cache
for i in range(_MAXCACHE + 1):
fnmatch('foo', '?' * i)
self.assertLessEqual(len(_cache), _MAXCACHE)
# bytes pattern cache
for i in range(_MAXCACHE + 1):
fnmatch(b'foo', b'?' * i)
self.assertLessEqual(len(_cacheb), _MAXCACHE)
class TranslateTestCase(unittest.TestCase):
def test_translate(self):