[3.13] gh-119310: Fix encoding when reading old history file (GH-121779) (#123784)

(cherry picked from commit e95984826e)

Co-authored-by: aorcajo <589252+aorcajo@users.noreply.github.com>
Co-authored-by: Łukasz Langa <lukasz@langa.pl>
This commit is contained in:
Miss Islington (bot) 2024-09-06 16:23:47 +02:00 committed by GitHub
parent c46ad20d41
commit 2cc1899b93
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 59 additions and 6 deletions

View File

@ -427,12 +427,16 @@ class _ReadlineWrapper:
history = self.get_reader().history
with open(os.path.expanduser(filename), 'rb') as f:
lines = [line.decode('utf-8', errors='replace') for line in f.read().split(b'\n')]
is_editline = f.readline().startswith(b"_HiStOrY_V2_")
if is_editline:
encoding = "unicode-escape"
else:
f.seek(0)
encoding = "utf-8"
lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')]
buffer = []
for line in lines:
# Ignore readline history file header
if line.startswith("_HiStOrY_V2_"):
continue
if line.endswith("\r"):
buffer.append(line+'\n')
else:

View File

@ -1216,12 +1216,34 @@ class TestMain(TestCase):
*,
cmdline_args: list[str] | None = None,
cwd: str | None = None,
) -> tuple[str, int]:
temp_dir = None
if cwd is None:
temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
cwd = temp_dir.name
try:
return self._run_repl(
repl_input, env=env, cmdline_args=cmdline_args, cwd=cwd
)
finally:
if temp_dir is not None:
temp_dir.cleanup()
def _run_repl(
self,
repl_input: str | list[str],
*,
env: dict | None,
cmdline_args: list[str] | None,
cwd: str,
) -> tuple[str, int]:
assert pty
master_fd, slave_fd = pty.openpty()
cmd = [sys.executable, "-i", "-u"]
if env is None:
cmd.append("-I")
elif "PYTHON_HISTORY" not in env:
env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history")
if cmdline_args is not None:
cmd.extend(cmdline_args)
process = subprocess.Popen(
@ -1260,3 +1282,26 @@ class TestMain(TestCase):
process.kill()
exit_code = process.wait()
return "".join(output), exit_code
def test_readline_history_file(self):
# skip, if readline module is not available
readline = import_module('readline')
if readline.backend != "editline":
self.skipTest("GNU readline is not affected by this issue")
hfile = tempfile.NamedTemporaryFile()
self.addCleanup(unlink, hfile.name)
env = os.environ.copy()
env["PYTHON_HISTORY"] = hfile.name
env["PYTHON_BASIC_REPL"] = "1"
output, exit_code = self.run_repl("spam \nexit()\n", env=env)
self.assertEqual(exit_code, 0)
self.assertIn("spam ", output)
self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0)
self.assertIn("spam\\040", pathlib.Path(hfile.name).read_text())
env.pop("PYTHON_BASIC_REPL", None)
output, exit_code = self.run_repl("exit\n", env=env)
self.assertEqual(exit_code, 0)
self.assertNotIn("\\040", pathlib.Path(hfile.name).read_text())

View File

@ -41,7 +41,7 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
# path may be used by Py_GetPath() to build the default module search
# path.
stdin_fname = os.path.join(os.path.dirname(sys.executable), "<stdin>")
cmd_line = [stdin_fname, '-E', '-i']
cmd_line = [stdin_fname, '-I', '-i']
cmd_line.extend(args)
# Set TERM=vt100, for the rationale see the comments in spawn_python() of
@ -228,6 +228,7 @@ class TestInteractiveInterpreter(unittest.TestCase):
f.write("exit(0)" + os.linesep)
env = os.environ.copy()
env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".asyncio_history")
env["PYTHONSTARTUP"] = script
subprocess.check_call(
[sys.executable, "-m", "asyncio"],
@ -240,7 +241,7 @@ class TestInteractiveInterpreter(unittest.TestCase):
@unittest.skipUnless(pty, "requires pty")
def test_asyncio_repl_is_ok(self):
m, s = pty.openpty()
cmd = [sys.executable, "-m", "asyncio"]
cmd = [sys.executable, "-I", "-m", "asyncio"]
proc = subprocess.Popen(
cmd,
stdin=s,

View File

@ -0,0 +1,3 @@
Allow the new interactive shell to read history files written with the
editline library that use unicode-escaped entries. Patch by aorcajo and
Łukasz Langa.