Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-46303: Move fileutils.h private functions to internal C API (GH-30484) 

Move almost all private functions of Include/cpython/fileutils.h to
the internal C API Include/internal/pycore_fileutils.h.

Only keep _Py_fopen_obj() in Include/cpython/fileutils.h, since it's
used by _testcapi which must not use the internal C API.

Move EncodeLocaleEx() and DecodeLocaleEx() functions from _testcapi
to _testinternalcapi, since the C API moved to the internal C API.
This commit is contained in:
Victor Stinner 2022-01-11 11:56:16 +01:00 committed by GitHub
parent fc75bfb8be
commit ea1a54506b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 289 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

160
Include/cpython/fileutils.h
View File

@ -2,165 +2,7 @@
# error "this header file must not be included directly"
#endif
typedef enum {
_Py_ERROR_UNKNOWN=0,
_Py_ERROR_STRICT,
_Py_ERROR_SURROGATEESCAPE,
_Py_ERROR_REPLACE,
_Py_ERROR_IGNORE,
_Py_ERROR_BACKSLASHREPLACE,
_Py_ERROR_SURROGATEPASS,
_Py_ERROR_XMLCHARREFREPLACE,
_Py_ERROR_OTHER
} _Py_error_handler;
PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors);
PyAPI_FUNC(int) _Py_DecodeLocaleEx(
const char *arg,
wchar_t **wstr,
size_t *wlen,
const char **reason,
int current_locale,
_Py_error_handler errors);
PyAPI_FUNC(int) _Py_EncodeLocaleEx(
const wchar_t *text,
char **str,
size_t *error_pos,
const char **reason,
int current_locale,
_Py_error_handler errors);
PyAPI_FUNC(char*) _Py_EncodeLocaleRaw(
const wchar_t *text,
size_t *error_pos);
PyAPI_FUNC(PyObject *) _Py_device_encoding(int);
#if defined(MS_WINDOWS) || defined(__APPLE__)
/* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611).
On macOS 10.13, read() and write() with more than INT_MAX bytes
fail with EINVAL (bpo-24658). */
# define _PY_READ_MAX INT_MAX
# define _PY_WRITE_MAX INT_MAX
#else
/* write() should truncate the input to PY_SSIZE_T_MAX bytes,
but it's safer to do it ourself to have a portable behaviour */
# define _PY_READ_MAX PY_SSIZE_T_MAX
# define _PY_WRITE_MAX PY_SSIZE_T_MAX
#endif
#ifdef MS_WINDOWS
struct _Py_stat_struct {
unsigned long st_dev;
uint64_t st_ino;
unsigned short st_mode;
int st_nlink;
int st_uid;
int st_gid;
unsigned long st_rdev;
__int64 st_size;
time_t st_atime;
int st_atime_nsec;
time_t st_mtime;
int st_mtime_nsec;
time_t st_ctime;
int st_ctime_nsec;
unsigned long st_file_attributes;
unsigned long st_reparse_tag;
};
#else
# define _Py_stat_struct stat
#endif
PyAPI_FUNC(int) _Py_fstat(
int fd,
struct _Py_stat_struct *status);
PyAPI_FUNC(int) _Py_fstat_noraise(
int fd,
struct _Py_stat_struct *status);
PyAPI_FUNC(int) _Py_stat(
PyObject *path,
struct stat *status);
PyAPI_FUNC(int) _Py_open(
const char *pathname,
int flags);
PyAPI_FUNC(int) _Py_open_noraise(
const char *pathname,
int flags);
PyAPI_FUNC(FILE *) _Py_wfopen(
const wchar_t *path,
const wchar_t *mode);
// Used by _testcapi which must not use the internal C API
PyAPI_FUNC(FILE*) _Py_fopen_obj(
PyObject *path,
const char *mode);
PyAPI_FUNC(Py_ssize_t) _Py_read(
int fd,
void *buf,
size_t count);
PyAPI_FUNC(Py_ssize_t) _Py_write(
int fd,
const void *buf,
size_t count);
PyAPI_FUNC(Py_ssize_t) _Py_write_noraise(
int fd,
const void *buf,
size_t count);
#ifdef HAVE_READLINK
PyAPI_FUNC(int) _Py_wreadlink(
const wchar_t *path,
wchar_t *buf,
/* Number of characters of 'buf' buffer
including the trailing NUL character */
size_t buflen);
#endif
#ifdef HAVE_REALPATH
PyAPI_FUNC(wchar_t*) _Py_wrealpath(
const wchar_t *path,
wchar_t *resolved_path,
/* Number of characters of 'resolved_path' buffer
including the trailing NUL character */
size_t resolved_path_len);
#endif
PyAPI_FUNC(wchar_t*) _Py_wgetcwd(
wchar_t *buf,
/* Number of characters of 'buf' buffer
including the trailing NUL character */
size_t buflen);
PyAPI_FUNC(int) _Py_get_inheritable(int fd);
PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable,
int *atomic_flag_works);
PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable,
int *atomic_flag_works);
PyAPI_FUNC(int) _Py_dup(int fd);
#ifndef MS_WINDOWS
PyAPI_FUNC(int) _Py_get_blocking(int fd);
PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking);
#else /* MS_WINDOWS */
PyAPI_FUNC(void*) _Py_get_osfhandle_noraise(int fd);
PyAPI_FUNC(void*) _Py_get_osfhandle(int fd);
PyAPI_FUNC(int) _Py_open_osfhandle_noraise(void *handle, int flags);
PyAPI_FUNC(int) _Py_open_osfhandle(void *handle, int flags);
#endif /* MS_WINDOWS */

159
Include/internal/pycore_fileutils.h
View File

@ -10,6 +10,165 @@ extern "C" {
#include <locale.h> /* struct lconv */
typedef enum {
_Py_ERROR_UNKNOWN=0,
_Py_ERROR_STRICT,
_Py_ERROR_SURROGATEESCAPE,
_Py_ERROR_REPLACE,
_Py_ERROR_IGNORE,
_Py_ERROR_BACKSLASHREPLACE,
_Py_ERROR_SURROGATEPASS,
_Py_ERROR_XMLCHARREFREPLACE,
_Py_ERROR_OTHER
} _Py_error_handler;
PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors);
PyAPI_FUNC(int) _Py_DecodeLocaleEx(
const char *arg,
wchar_t **wstr,
size_t *wlen,
const char **reason,
int current_locale,
_Py_error_handler errors);
PyAPI_FUNC(int) _Py_EncodeLocaleEx(
const wchar_t *text,
char **str,
size_t *error_pos,
const char **reason,
int current_locale,
_Py_error_handler errors);
PyAPI_FUNC(char*) _Py_EncodeLocaleRaw(
const wchar_t *text,
size_t *error_pos);
PyAPI_FUNC(PyObject *) _Py_device_encoding(int);
#if defined(MS_WINDOWS) || defined(__APPLE__)
/* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611).
On macOS 10.13, read() and write() with more than INT_MAX bytes
fail with EINVAL (bpo-24658). */
# define _PY_READ_MAX INT_MAX
# define _PY_WRITE_MAX INT_MAX
#else
/* write() should truncate the input to PY_SSIZE_T_MAX bytes,
but it's safer to do it ourself to have a portable behaviour */
# define _PY_READ_MAX PY_SSIZE_T_MAX
# define _PY_WRITE_MAX PY_SSIZE_T_MAX
#endif
#ifdef MS_WINDOWS
struct _Py_stat_struct {
unsigned long st_dev;
uint64_t st_ino;
unsigned short st_mode;
int st_nlink;
int st_uid;
int st_gid;
unsigned long st_rdev;
__int64 st_size;
time_t st_atime;
int st_atime_nsec;
time_t st_mtime;
int st_mtime_nsec;
time_t st_ctime;
int st_ctime_nsec;
unsigned long st_file_attributes;
unsigned long st_reparse_tag;
};
#else
# define _Py_stat_struct stat
#endif
PyAPI_FUNC(int) _Py_fstat(
int fd,
struct _Py_stat_struct *status);
PyAPI_FUNC(int) _Py_fstat_noraise(
int fd,
struct _Py_stat_struct *status);
PyAPI_FUNC(int) _Py_stat(
PyObject *path,
struct stat *status);
PyAPI_FUNC(int) _Py_open(
const char *pathname,
int flags);
PyAPI_FUNC(int) _Py_open_noraise(
const char *pathname,
int flags);
PyAPI_FUNC(FILE *) _Py_wfopen(
const wchar_t *path,
const wchar_t *mode);
PyAPI_FUNC(Py_ssize_t) _Py_read(
int fd,
void *buf,
size_t count);
PyAPI_FUNC(Py_ssize_t) _Py_write(
int fd,
const void *buf,
size_t count);
PyAPI_FUNC(Py_ssize_t) _Py_write_noraise(
int fd,
const void *buf,
size_t count);
#ifdef HAVE_READLINK
PyAPI_FUNC(int) _Py_wreadlink(
const wchar_t *path,
wchar_t *buf,
/* Number of characters of 'buf' buffer
including the trailing NUL character */
size_t buflen);
#endif
#ifdef HAVE_REALPATH
PyAPI_FUNC(wchar_t*) _Py_wrealpath(
const wchar_t *path,
wchar_t *resolved_path,
/* Number of characters of 'resolved_path' buffer
including the trailing NUL character */
size_t resolved_path_len);
#endif
PyAPI_FUNC(wchar_t*) _Py_wgetcwd(
wchar_t *buf,
/* Number of characters of 'buf' buffer
including the trailing NUL character */
size_t buflen);
PyAPI_FUNC(int) _Py_get_inheritable(int fd);
PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable,
int *atomic_flag_works);
PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable,
int *atomic_flag_works);
PyAPI_FUNC(int) _Py_dup(int fd);
#ifndef MS_WINDOWS
PyAPI_FUNC(int) _Py_get_blocking(int fd);
PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking);
#else /* MS_WINDOWS */
PyAPI_FUNC(void*) _Py_get_osfhandle_noraise(int fd);
PyAPI_FUNC(void*) _Py_get_osfhandle(int fd);
PyAPI_FUNC(int) _Py_open_osfhandle_noraise(void *handle, int flags);
PyAPI_FUNC(int) _Py_open_osfhandle(void *handle, int flags);
#endif /* MS_WINDOWS */
// This is used after getting NULL back from Py_DecodeLocale().
#define DECODE_LOCALE_ERR(NAME, LEN) \
((LEN) == (size_t)-2) \

2
Include/internal/pycore_unicodeobject.h
View File

@ -8,6 +8,8 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
#include "pycore_fileutils.h" // _Py_error_handler
/* runtime lifecycle */

10
Lib/test/test_codecs.py
View File

@ -15,6 +15,10 @@ try:
import _testcapi
except ImportError:
_testcapi = None
try:
import _testinternalcapi
except ImportError:
_testinternalcapi = None
try:
import ctypes
@ -3345,7 +3349,7 @@ class StreamRecoderTest(unittest.TestCase):
self.assertEqual(sr.readline(), b'789\n')
@unittest.skipIf(_testcapi is None, 'need _testcapi module')
@unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module')
class LocaleCodecTest(unittest.TestCase):
"""
Test indirectly _Py_DecodeUTF8Ex() and _Py_EncodeUTF8Ex().
@ -3359,7 +3363,7 @@ class LocaleCodecTest(unittest.TestCase):
SURROGATES = "\uDC80\uDCFF"
def encode(self, text, errors="strict"):
return _testcapi.EncodeLocaleEx(text, 0, errors)
return _testinternalcapi.EncodeLocaleEx(text, 0, errors)
def check_encode_strings(self, errors):
for text in self.STRINGS:
@ -3399,7 +3403,7 @@ class LocaleCodecTest(unittest.TestCase):
self.assertEqual(str(cm.exception), 'unsupported error handler')
def decode(self, encoded, errors="strict"):
return _testcapi.DecodeLocaleEx(encoded, 0, errors)
return _testinternalcapi.DecodeLocaleEx(encoded, 0, errors)
def check_decode_strings(self, errors):
is_utf8 = (self.ENCODING == "utf-8")

94
Modules/_testcapimodule.c
View File

@ -5417,98 +5417,6 @@ bad_get(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
}
static PyObject *
encode_locale_ex(PyObject *self, PyObject *args)
{
PyObject *unicode;
int current_locale = 0;
wchar_t *wstr;
PyObject *res = NULL;
const char *errors = NULL;
if (!PyArg_ParseTuple(args, "U|is", &unicode, &current_locale, &errors)) {
return NULL;
}
wstr = PyUnicode_AsWideCharString(unicode, NULL);
if (wstr == NULL) {
return NULL;
}
_Py_error_handler error_handler = _Py_GetErrorHandler(errors);
char *str = NULL;
size_t error_pos;
const char *reason = NULL;
int ret = _Py_EncodeLocaleEx(wstr,
&str, &error_pos, &reason,
current_locale, error_handler);
PyMem_Free(wstr);
switch(ret) {
case 0:
res = PyBytes_FromString(str);
PyMem_RawFree(str);
break;
case -1:
PyErr_NoMemory();
break;
case -2:
PyErr_Format(PyExc_RuntimeError, "encode error: pos=%zu, reason=%s",
error_pos, reason);
break;
case -3:
PyErr_SetString(PyExc_ValueError, "unsupported error handler");
break;
default:
PyErr_SetString(PyExc_ValueError, "unknown error code");
break;
}
return res;
}
static PyObject *
decode_locale_ex(PyObject *self, PyObject *args)
{
char *str;
int current_locale = 0;
PyObject *res = NULL;
const char *errors = NULL;
if (!PyArg_ParseTuple(args, "y|is", &str, &current_locale, &errors)) {
return NULL;
}
_Py_error_handler error_handler = _Py_GetErrorHandler(errors);
wchar_t *wstr = NULL;
size_t wlen = 0;
const char *reason = NULL;
int ret = _Py_DecodeLocaleEx(str,
&wstr, &wlen, &reason,
current_locale, error_handler);
switch(ret) {
case 0:
res = PyUnicode_FromWideChar(wstr, wlen);
PyMem_RawFree(wstr);
break;
case -1:
PyErr_NoMemory();
break;
case -2:
PyErr_Format(PyExc_RuntimeError, "decode error: pos=%zu, reason=%s",
wlen, reason);
break;
case -3:
PyErr_SetString(PyExc_ValueError, "unsupported error handler");
break;
default:
PyErr_SetString(PyExc_ValueError, "unknown error code");
break;
}
return res;
}
#ifdef Py_REF_DEBUG
static PyObject *
negative_refcount(PyObject *self, PyObject *Py_UNUSED(args))
@ -6125,8 +6033,6 @@ static PyMethodDef TestMethods[] = {
{"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS},
{"hamt", new_hamt, METH_NOARGS},
{"bad_get", (PyCFunction)(void(*)(void))bad_get, METH_FASTCALL},
{"EncodeLocaleEx", encode_locale_ex, METH_VARARGS},
{"DecodeLocaleEx", decode_locale_ex, METH_VARARGS},
#ifdef Py_REF_DEBUG
{"negative_refcount", negative_refcount, METH_NOARGS},
#endif

94
Modules/_testinternalcapi.c
View File

@ -399,6 +399,98 @@ get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) {
}
static PyObject *
encode_locale_ex(PyObject *self, PyObject *args)
{
PyObject *unicode;
int current_locale = 0;
wchar_t *wstr;
PyObject *res = NULL;
const char *errors = NULL;
if (!PyArg_ParseTuple(args, "U|is", &unicode, &current_locale, &errors)) {
return NULL;
}
wstr = PyUnicode_AsWideCharString(unicode, NULL);
if (wstr == NULL) {
return NULL;
}
_Py_error_handler error_handler = _Py_GetErrorHandler(errors);
char *str = NULL;
size_t error_pos;
const char *reason = NULL;
int ret = _Py_EncodeLocaleEx(wstr,
&str, &error_pos, &reason,
current_locale, error_handler);
PyMem_Free(wstr);
switch(ret) {
case 0:
res = PyBytes_FromString(str);
PyMem_RawFree(str);
break;
case -1:
PyErr_NoMemory();
break;
case -2:
PyErr_Format(PyExc_RuntimeError, "encode error: pos=%zu, reason=%s",
error_pos, reason);
break;
case -3:
PyErr_SetString(PyExc_ValueError, "unsupported error handler");
break;
default:
PyErr_SetString(PyExc_ValueError, "unknown error code");
break;
}
return res;
}
static PyObject *
decode_locale_ex(PyObject *self, PyObject *args)
{
char *str;
int current_locale = 0;
PyObject *res = NULL;
const char *errors = NULL;
if (!PyArg_ParseTuple(args, "y|is", &str, &current_locale, &errors)) {
return NULL;
}
_Py_error_handler error_handler = _Py_GetErrorHandler(errors);
wchar_t *wstr = NULL;
size_t wlen = 0;
const char *reason = NULL;
int ret = _Py_DecodeLocaleEx(str,
&wstr, &wlen, &reason,
current_locale, error_handler);
switch(ret) {
case 0:
res = PyUnicode_FromWideChar(wstr, wlen);
PyMem_RawFree(wstr);
break;
case -1:
PyErr_NoMemory();
break;
case -2:
PyErr_Format(PyExc_RuntimeError, "decode error: pos=%zu, reason=%s",
wlen, reason);
break;
case -3:
PyErr_SetString(PyExc_ValueError, "unsupported error handler");
break;
default:
PyErr_SetString(PyExc_ValueError, "unknown error code");
break;
}
return res;
}
static PyMethodDef TestMethods[] = {
{"get_configs", get_configs, METH_NOARGS},
{"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@ -413,6 +505,8 @@ static PyMethodDef TestMethods[] = {
{"test_edit_cost", test_edit_cost, METH_NOARGS},
{"normalize_path", normalize_path, METH_O, NULL},
{"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL},
{"EncodeLocaleEx", encode_locale_ex, METH_VARARGS},
{"DecodeLocaleEx", decode_locale_ex, METH_VARARGS},
{NULL, NULL} /* sentinel */
};

3
Modules/_tracemalloc.c
View File

@ -1,8 +1,9 @@
#include "Python.h"
#include "pycore_fileutils.h" // _Py_write_noraise()
#include "pycore_gc.h" // PyGC_Head
#include "pycore_hashtable.h" // _Py_hashtable_t
#include "pycore_pymem.h" // _Py_tracemalloc_config
#include "pycore_traceback.h"
#include "pycore_hashtable.h"
#include <pycore_frame.h>
#include <stdlib.h> // malloc()

5
Modules/mmapmodule.c
View File

@ -18,8 +18,13 @@
/ ftp://squirl.nightmare.com/pub/python/python-ext.
*/
#ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1
#endif
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "pycore_fileutils.h" // _Py_stat_struct
#include "structmember.h" // PyMemberDef
#include <stddef.h> // offsetof()

5
Modules/ossaudiodev.c
View File

@ -17,8 +17,13 @@
* $Id$
*/
#ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1
#endif
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "pycore_fileutils.h" // _Py_write()
#include "structmember.h" // PyMemberDef
#include <stdlib.h> // getenv()

7
Modules/selectmodule.c
View File

@ -4,11 +4,16 @@
have any value except INVALID_SOCKET.
*/
#ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1
#endif
#if defined(HAVE_POLL_H) && !defined(_GNU_SOURCE)
#define _GNU_SOURCE
# define _GNU_SOURCE
#endif
#include "Python.h"
#include "pycore_fileutils.h" // _Py_set_inheritable()
#include "structmember.h" // PyMemberDef
#ifdef HAVE_SYS_DEVPOLL_H

5
Modules/socketmodule.c
View File

@ -85,6 +85,10 @@ Local naming conventions:
*/
#ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1
#endif
#ifdef __APPLE__
// Issue #35569: Expose RFC 3542 socket options.
#define __APPLE_USE_RFC_3542 1
@ -103,6 +107,7 @@ Local naming conventions:
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "pycore_fileutils.h" // _Py_set_inheritable()
#include "structmember.h" // PyMemberDef
#ifdef _Py_MEMORY_SANITIZER

1
Programs/_freeze_module.c
View File

@ -11,6 +11,7 @@
#include <Python.h>
#include <marshal.h>
#include "pycore_fileutils.h" // _Py_stat_struct
#include <pycore_import.h>
#include <stdio.h>

2
Python/bootstrap_hash.c
View File

@ -1,5 +1,7 @@
#include "Python.h"
#include "pycore_initconfig.h"
#include "pycore_fileutils.h" // _Py_fstat_noraise()
#ifdef MS_WINDOWS
# include <windows.h>
# include <bcrypt.h>