From 1813d318fd4e517042415fa4f59fe8668c17a235 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 24 Jun 2020 09:45:38 -0700 Subject: [PATCH] bpo-41094: Fix decoding errors with audit when open files. (GH-21095) (cherry picked from commit 6c6810d98979add7a89391c3c38990d0859f7a29) Co-authored-by: Serhiy Storchaka --- Lib/test/test_embed.py | 4 +-- .../2020-06-23-23-26-42.bpo-41094.zEIJse.rst | 2 ++ Modules/_ctypes/callproc.c | 7 ++--- Modules/main.c | 9 ++++++- Python/fileutils.c | 27 ++++++++++++++----- 5 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-06-23-23-26-42.bpo-41094.zEIJse.rst diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index da79d7af050..886ccc5a125 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1309,7 +1309,7 @@ class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): self.run_embedded_interpreter("test_audit_run_file", timeout=3, returncode=1) def test_audit_run_interactivehook(self): - startup = os.path.join(self.oldcwd, support.TESTFN) + ".py" + startup = os.path.join(self.oldcwd, support.TESTFN) + (support.TESTFN or '') + ".py" with open(startup, "w", encoding="utf-8") as f: print("import sys", file=f) print("sys.__interactivehook__ = lambda: None", file=f) @@ -1321,7 +1321,7 @@ class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): os.unlink(startup) def test_audit_run_startup(self): - startup = os.path.join(self.oldcwd, support.TESTFN) + ".py" + startup = os.path.join(self.oldcwd, support.TESTFN) + (support.TESTFN or '') + ".py" with open(startup, "w", encoding="utf-8") as f: print("pass", file=f) try: diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-23-23-26-42.bpo-41094.zEIJse.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-23-23-26-42.bpo-41094.zEIJse.rst new file mode 100644 index 00000000000..6dd45e21d17 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-23-23-26-42.bpo-41094.zEIJse.rst @@ -0,0 +1,2 @@ +Fix decoding errors with audit when open files with non-ASCII names on non-UTF-8 +locale. diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 55fc226ca12..a9b8675cd95 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1416,15 +1416,12 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) if (name != Py_None) { if (PyUnicode_FSConverter(name, &name2) == 0) return NULL; - if (PyBytes_Check(name2)) - name_str = PyBytes_AS_STRING(name2); - else - name_str = PyByteArray_AS_STRING(name2); + name_str = PyBytes_AS_STRING(name2); } else { name_str = NULL; name2 = NULL; } - if (PySys_Audit("ctypes.dlopen", "s", name_str) < 0) { + if (PySys_Audit("ctypes.dlopen", "O", name) < 0) { return NULL; } handle = ctypes_dlopen(name_str, mode); diff --git a/Modules/main.c b/Modules/main.c index 2a360b58efa..788bc119095 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -391,13 +391,20 @@ pymain_run_startup(PyConfig *config, PyCompilerFlags *cf, int *exitcode) if (startup == NULL) { return 0; } - if (PySys_Audit("cpython.run_startup", "s", startup) < 0) { + PyObject *startup_obj = PyUnicode_DecodeFSDefault(startup); + if (startup_obj == NULL) { return pymain_err_print(exitcode); } + if (PySys_Audit("cpython.run_startup", "O", startup_obj) < 0) { + Py_DECREF(startup_obj); + return pymain_err_print(exitcode); + } + Py_DECREF(startup_obj); FILE *fp = _Py_fopen(startup, "r"); if (fp == NULL) { int save_errno = errno; + PyErr_Clear(); PySys_WriteStderr("Could not open PYTHONSTARTUP\n"); errno = save_errno; diff --git a/Python/fileutils.c b/Python/fileutils.c index 1021ddb5885..b274116745e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1274,7 +1274,12 @@ _Py_open_impl(const char *pathname, int flags, int gil_held) #endif if (gil_held) { - if (PySys_Audit("open", "sOi", pathname, Py_None, flags) < 0) { + PyObject *pathname_obj = PyUnicode_DecodeFSDefault(pathname); + if (pathname_obj == NULL) { + return -1; + } + if (PySys_Audit("open", "OOi", pathname_obj, Py_None, flags) < 0) { + Py_DECREF(pathname_obj); return -1; } @@ -1284,12 +1289,16 @@ _Py_open_impl(const char *pathname, int flags, int gil_held) Py_END_ALLOW_THREADS } while (fd < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); - if (async_err) - return -1; - if (fd < 0) { - PyErr_SetFromErrnoWithFilename(PyExc_OSError, pathname); + if (async_err) { + Py_DECREF(pathname_obj); return -1; } + if (fd < 0) { + PyErr_SetFromErrnoWithFilenameObjects(PyExc_OSError, pathname_obj, NULL); + Py_DECREF(pathname_obj); + return -1; + } + Py_DECREF(pathname_obj); } else { fd = open(pathname, flags); @@ -1385,9 +1394,15 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode) FILE* _Py_fopen(const char *pathname, const char *mode) { - if (PySys_Audit("open", "ssi", pathname, mode, 0) < 0) { + PyObject *pathname_obj = PyUnicode_DecodeFSDefault(pathname); + if (pathname_obj == NULL) { return NULL; } + if (PySys_Audit("open", "Osi", pathname_obj, mode, 0) < 0) { + Py_DECREF(pathname_obj); + return NULL; + } + Py_DECREF(pathname_obj); FILE *f = fopen(pathname, mode); if (f == NULL)