diff --git a/Include/pyport.h b/Include/pyport.h index 284e6da7470..baa04fee17b 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -865,4 +865,18 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *); #endif #endif +/* + * Convenient macros to deal with endianess of the platform. WORDS_BIGENDIAN is + * detected by configure and defined in pyconfig.h. The code in pyconfig.h + * also also takes care of Apple's universal builds. + */ + +#ifdef WORDS_BIGENDIAN +#define PY_BIG_ENDIAN 1 +#define PY_LITTLE_ENDIAN 0 +#else +#define PY_BIG_ENDIAN 0 +#define PY_LITTLE_ENDIAN 1 +#endif + #endif /* Py_PYPORT_H */ diff --git a/Misc/NEWS b/Misc/NEWS index 5ee9729622d..d939a638b36 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.4.0 Alpha 1? Core and Builtins ----------------- +- Issue #16166: Add PY_LITTLE_ENDIAN and PY_BIG_ENDIAN macros and unified + endianess detection and handling. + - Issue #15958: bytes.join and bytearray.join now accept arbitrary buffer objects. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 96434a81b74..8344d43767e 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -747,7 +747,7 @@ utf16_encode(textio *self, PyObject *text) { if (!self->encoding_start_of_stream) { /* Skip the BOM and use native byte ordering */ -#if defined(WORDS_BIGENDIAN) +#if PY_BIG_ENDIAN return utf16be_encode(self, text); #else return utf16le_encode(self, text); @@ -776,7 +776,7 @@ utf32_encode(textio *self, PyObject *text) { if (!self->encoding_start_of_stream) { /* Skip the BOM and use native byte ordering */ -#if defined(WORDS_BIGENDIAN) +#if PY_BIG_ENDIAN return utf32be_encode(self, text); #else return utf32le_encode(self, text); @@ -1913,10 +1913,7 @@ typedef struct { #define COOKIE_BUF_LEN (sizeof(Py_off_t) + 3 * sizeof(int) + sizeof(char)) -#if defined(WORDS_BIGENDIAN) - -# define IS_LITTLE_ENDIAN 0 - +#if PY_BIG_ENDIAN /* We want the least significant byte of start_pos to also be the least significant byte of the cookie, which means that in big-endian mode we must copy the fields in reverse order. */ @@ -1928,9 +1925,6 @@ typedef struct { # define OFF_NEED_EOF 0 #else - -# define IS_LITTLE_ENDIAN 1 - /* Little-endian mode: the least significant byte of start_pos will naturally end up the least significant byte of the cookie. */ @@ -1951,7 +1945,7 @@ textiowrapper_parse_cookie(cookie_type *cookie, PyObject *cookieObj) return -1; if (_PyLong_AsByteArray(cookieLong, buffer, sizeof(buffer), - IS_LITTLE_ENDIAN, 0) < 0) { + PY_LITTLE_ENDIAN, 0) < 0) { Py_DECREF(cookieLong); return -1; } @@ -1977,9 +1971,9 @@ textiowrapper_build_cookie(cookie_type *cookie) memcpy(buffer + OFF_CHARS_TO_SKIP, &cookie->chars_to_skip, sizeof(cookie->chars_to_skip)); memcpy(buffer + OFF_NEED_EOF, &cookie->need_eof, sizeof(cookie->need_eof)); - return _PyLong_FromByteArray(buffer, sizeof(buffer), IS_LITTLE_ENDIAN, 0); + return _PyLong_FromByteArray(buffer, sizeof(buffer), + PY_LITTLE_ENDIAN, 0); } -#undef IS_LITTLE_ENDIAN static int _textiowrapper_decoder_setstate(textio *self, cookie_type *cookie) diff --git a/Modules/_sha3/cleanup.py b/Modules/_sha3/cleanup.py index 5238ab3ca9d..aabcb0442ce 100755 --- a/Modules/_sha3/cleanup.py +++ b/Modules/_sha3/cleanup.py @@ -32,10 +32,10 @@ def cleanup(f): if line.startswith("typedef unsigned long long int"): buf.append("/* %s */\n" % line.strip()) continue - ## remove #include "brg_endian.h" - #if "brg_endian.h" in line: - # buf.append("/* %s */\n" % line.strip()) - # continue + # remove #include "brg_endian.h" + if "brg_endian.h" in line: + buf.append("/* %s */\n" % line.strip()) + continue # transform C++ comments into ANSI C comments line = CPP1.sub(r"/* \1 */", line) line = CPP2.sub(r" /* \1 */", line) diff --git a/Modules/_sha3/keccak/KeccakF-1600-opt32.c b/Modules/_sha3/keccak/KeccakF-1600-opt32.c index 473dde5d9a7..dba6d59f134 100644 --- a/Modules/_sha3/keccak/KeccakF-1600-opt32.c +++ b/Modules/_sha3/keccak/KeccakF-1600-opt32.c @@ -12,7 +12,7 @@ http://creativecommons.org/publicdomain/zero/1.0/ */ #include -#include "brg_endian.h" +/* #include "brg_endian.h" */ #include "KeccakF-1600-opt32-settings.h" #include "KeccakF-1600-interface.h" diff --git a/Modules/_sha3/keccak/KeccakF-1600-opt64.c b/Modules/_sha3/keccak/KeccakF-1600-opt64.c index 57e2aa45009..f19b18b36ac 100644 --- a/Modules/_sha3/keccak/KeccakF-1600-opt64.c +++ b/Modules/_sha3/keccak/KeccakF-1600-opt64.c @@ -12,7 +12,7 @@ http://creativecommons.org/publicdomain/zero/1.0/ */ #include -#include "brg_endian.h" +/* #include "brg_endian.h" */ #include "KeccakF-1600-opt64-settings.h" #include "KeccakF-1600-interface.h" diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index 08f683934b1..4c3c6db1322 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -124,11 +124,14 @@ #define UseInterleaveTables #endif -/* replacement for brg_endian.h -#define IS_BIG_ENDIAN BIG_ENDIAN -#define IS_LITTLE_ENDIAN LITTLE_ENDIAN -#define PLATFORM_BYTE_ORDER BYTE_ORDER -*/ +/* replacement for brg_endian.h */ +#define IS_BIG_ENDIAN 4321 +#define IS_LITTLE_ENDIAN 1234 +#if PY_BIG_ENDIAN +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +#else +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +#endif /* inline all Keccak dependencies */ #include "keccak/KeccakNISTInterface.h" diff --git a/Modules/_struct.c b/Modules/_struct.c index 0b20e26e6bf..0f501447b66 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1199,12 +1199,11 @@ whichtable(char **pfmt) case '!': /* Network byte order is big-endian */ return bigendian_table; case '=': { /* Host byte order -- different from native in alignment! */ - int n = 1; - char *p = (char *) &n; - if (*p == 1) - return lilendian_table; - else - return bigendian_table; +#if PY_LITTLE_ENDIAN + return lilendian_table; +#else + return bigendian_table; +#endif } default: --*pfmt; /* Back out of pointer increment */ @@ -2088,13 +2087,13 @@ PyInit__struct(void) /* Check endian and swap in faster functions */ { - int one = 1; formatdef *native = native_table; formatdef *other, *ptr; - if ((int)*(unsigned char*)&one) - other = lilendian_table; - else - other = bigendian_table; +#if PY_LITTLE_ENDIAN + other = lilendian_table; +#else + other = bigendian_table; +#endif /* Scan through the native table, find a matching entry in the endian table and swap in the native implementations whenever possible diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 3f5aa8b361e..4d4c2aee5ae 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1664,11 +1664,8 @@ static const struct mformatdescr { static enum machine_format_code typecode_to_mformat_code(char typecode) { -#ifdef WORDS_BIGENDIAN - const int is_big_endian = 1; -#else - const int is_big_endian = 0; -#endif + const int is_big_endian = PY_BIG_ENDIAN; + size_t intsize; int is_signed; diff --git a/Modules/sha256module.c b/Modules/sha256module.c index 76d91afda37..4abd1cd43a8 100644 --- a/Modules/sha256module.c +++ b/Modules/sha256module.c @@ -21,13 +21,6 @@ #include "hashlib.h" -/* Endianness testing and definitions */ -#define TestEndianness(variable) {int i=1; variable=PCT_BIG_ENDIAN;\ - if (*((char*)&i)==1) variable=PCT_LITTLE_ENDIAN;} - -#define PCT_LITTLE_ENDIAN 1 -#define PCT_BIG_ENDIAN 0 - /* Some useful types */ typedef unsigned char SHA_BYTE; @@ -50,7 +43,6 @@ typedef struct { SHA_INT32 digest[8]; /* Message digest */ SHA_INT32 count_lo, count_hi; /* 64-bit bit count */ SHA_BYTE data[SHA_BLOCKSIZE]; /* SHA data buffer */ - int Endianness; int local; /* unprocessed amount in data */ int digestsize; } SHAobject; @@ -58,13 +50,11 @@ typedef struct { /* When run on a little-endian CPU we need to perform byte reversal on an array of longwords. */ -static void longReverse(SHA_INT32 *buffer, int byteCount, int Endianness) +#if PY_LITTLE_ENDIAN +static void longReverse(SHA_INT32 *buffer, int byteCount) { SHA_INT32 value; - if ( Endianness == PCT_BIG_ENDIAN ) - return; - byteCount /= sizeof(*buffer); while (byteCount--) { value = *buffer; @@ -73,10 +63,10 @@ static void longReverse(SHA_INT32 *buffer, int byteCount, int Endianness) *buffer++ = ( value << 16 ) | ( value >> 16 ); } } +#endif static void SHAcopy(SHAobject *src, SHAobject *dest) { - dest->Endianness = src->Endianness; dest->local = src->local; dest->digestsize = src->digestsize; dest->count_lo = src->count_lo; @@ -131,7 +121,9 @@ sha_transform(SHAobject *sha_info) SHA_INT32 S[8], W[64], t0, t1; memcpy(W, sha_info->data, sizeof(sha_info->data)); - longReverse(W, (int)sizeof(sha_info->data), sha_info->Endianness); +#if PY_LITTLE_ENDIAN + longReverse(W, (int)sizeof(sha_info->data)); +#endif for (i = 16; i < 64; ++i) { W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; @@ -228,7 +220,6 @@ sha_transform(SHAobject *sha_info) static void sha_init(SHAobject *sha_info) { - TestEndianness(sha_info->Endianness) sha_info->digest[0] = 0x6A09E667L; sha_info->digest[1] = 0xBB67AE85L; sha_info->digest[2] = 0x3C6EF372L; @@ -246,7 +237,6 @@ sha_init(SHAobject *sha_info) static void sha224_init(SHAobject *sha_info) { - TestEndianness(sha_info->Endianness) sha_info->digest[0] = 0xc1059ed8L; sha_info->digest[1] = 0x367cd507L; sha_info->digest[2] = 0x3070dd17L; diff --git a/Modules/sha512module.c b/Modules/sha512module.c index 88f8a64d062..bfb022c3277 100644 --- a/Modules/sha512module.c +++ b/Modules/sha512module.c @@ -22,13 +22,6 @@ #ifdef PY_LONG_LONG /* If no PY_LONG_LONG, don't compile anything! */ -/* Endianness testing and definitions */ -#define TestEndianness(variable) {int i=1; variable=PCT_BIG_ENDIAN;\ - if (*((char*)&i)==1) variable=PCT_LITTLE_ENDIAN;} - -#define PCT_LITTLE_ENDIAN 1 -#define PCT_BIG_ENDIAN 0 - /* Some useful types */ typedef unsigned char SHA_BYTE; @@ -52,7 +45,6 @@ typedef struct { SHA_INT64 digest[8]; /* Message digest */ SHA_INT32 count_lo, count_hi; /* 64-bit bit count */ SHA_BYTE data[SHA_BLOCKSIZE]; /* SHA data buffer */ - int Endianness; int local; /* unprocessed amount in data */ int digestsize; } SHAobject; @@ -60,13 +52,11 @@ typedef struct { /* When run on a little-endian CPU we need to perform byte reversal on an array of longwords. */ -static void longReverse(SHA_INT64 *buffer, int byteCount, int Endianness) +#if PY_LITTLE_ENDIAN +static void longReverse(SHA_INT64 *buffer, int byteCount) { SHA_INT64 value; - if ( Endianness == PCT_BIG_ENDIAN ) - return; - byteCount /= sizeof(*buffer); while (byteCount--) { value = *buffer; @@ -83,10 +73,10 @@ static void longReverse(SHA_INT64 *buffer, int byteCount, int Endianness) buffer++; } } +#endif static void SHAcopy(SHAobject *src, SHAobject *dest) { - dest->Endianness = src->Endianness; dest->local = src->local; dest->digestsize = src->digestsize; dest->count_lo = src->count_lo; @@ -141,7 +131,9 @@ sha512_transform(SHAobject *sha_info) SHA_INT64 S[8], W[80], t0, t1; memcpy(W, sha_info->data, sizeof(sha_info->data)); - longReverse(W, (int)sizeof(sha_info->data), sha_info->Endianness); +#if PY_LITTLE_ENDIAN + longReverse(W, (int)sizeof(sha_info->data)); +#endif for (i = 16; i < 80; ++i) { W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; @@ -254,7 +246,6 @@ sha512_transform(SHAobject *sha_info) static void sha512_init(SHAobject *sha_info) { - TestEndianness(sha_info->Endianness) sha_info->digest[0] = Py_ULL(0x6a09e667f3bcc908); sha_info->digest[1] = Py_ULL(0xbb67ae8584caa73b); sha_info->digest[2] = Py_ULL(0x3c6ef372fe94f82b); @@ -272,7 +263,6 @@ sha512_init(SHAobject *sha_info) static void sha384_init(SHAobject *sha_info) { - TestEndianness(sha_info->Endianness) sha_info->digest[0] = Py_ULL(0xcbbb9d5dc1059ed8); sha_info->digest[1] = Py_ULL(0x629a292a367cd507); sha_info->digest[2] = Py_ULL(0x9159015a3070dd17); diff --git a/Objects/longobject.c b/Objects/longobject.c index 2aac8e40515..ead468025be 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -988,7 +988,6 @@ PyLong_AsVoidPtr(PyObject *vv) * rewritten to use the newer PyLong_{As,From}ByteArray API. */ -#define IS_LITTLE_ENDIAN (int)*(unsigned char*)&one #define PY_ABS_LLONG_MIN (0-(unsigned PY_LONG_LONG)PY_LLONG_MIN) /* Create a new long int object from a C PY_LONG_LONG int. */ @@ -1141,7 +1140,6 @@ PyLong_AsLongLong(PyObject *vv) { PyLongObject *v; PY_LONG_LONG bytes; - int one = 1; int res; if (vv == NULL) { @@ -1176,7 +1174,7 @@ PyLong_AsLongLong(PyObject *vv) case 1: return v->ob_digit[0]; } res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes, - SIZEOF_LONG_LONG, IS_LITTLE_ENDIAN, 1); + SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 1); /* Plan 9 can't handle PY_LONG_LONG in ? : expressions */ if (res < 0) @@ -1193,7 +1191,6 @@ PyLong_AsUnsignedLongLong(PyObject *vv) { PyLongObject *v; unsigned PY_LONG_LONG bytes; - int one = 1; int res; if (vv == NULL) { @@ -1212,7 +1209,7 @@ PyLong_AsUnsignedLongLong(PyObject *vv) } res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes, - SIZEOF_LONG_LONG, IS_LITTLE_ENDIAN, 0); + SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 0); /* Plan 9 can't handle PY_LONG_LONG in ? : expressions */ if (res < 0) @@ -1288,7 +1285,6 @@ PyLong_AsUnsignedLongLongMask(register PyObject *op) return (unsigned PY_LONG_LONG)-1; } } -#undef IS_LITTLE_ENDIAN /* Get a C long long int from a long int object or any object that has an __int__ method. diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 2a01089c0fc..cbf35084cdb 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -47,7 +47,7 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end, unsigned long value = *(unsigned long *) _s; if (value & ASCII_CHAR_MASK) break; -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN _p[0] = (STRINGLIB_CHAR)(value & 0xFFu); _p[1] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); _p[2] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); @@ -454,7 +454,7 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e, const unsigned char *q = *inptr; STRINGLIB_CHAR *p = dest + *outpos; /* Offsets from q for retrieving byte pairs in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN int ihi = !!native_ordering, ilo = !native_ordering; #else int ihi = !native_ordering, ilo = !!native_ordering; @@ -485,7 +485,7 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e, block = SWAB(block); #endif } -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN # if SIZEOF_LONG == 4 p[0] = (STRINGLIB_CHAR)(block & 0xFFFFu); p[1] = (STRINGLIB_CHAR)(block >> 16); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index b4c7ecf9002..9461563f99d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -47,14 +47,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include #endif -/* Endianness switches; defaults to little endian */ - -#ifdef WORDS_BIGENDIAN -# define BYTEORDER_IS_BIG_ENDIAN -#else -# define BYTEORDER_IS_LITTLE_ENDIAN -#endif - /* --- Globals ------------------------------------------------------------ The globals are initialized by the _PyUnicode_Init() API and should @@ -4813,7 +4805,7 @@ PyUnicode_DecodeUTF32Stateful(const char *s, int bo = 0; /* assume native ordering by default */ const char *errmsg = ""; /* Offsets from q for retrieving bytes in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN int iorder[] = {0, 1, 2, 3}; #else int iorder[] = {3, 2, 1, 0}; @@ -4835,7 +4827,7 @@ PyUnicode_DecodeUTF32Stateful(const char *s, if (size >= 4) { const Py_UCS4 bom = (q[iorder[3]] << 24) | (q[iorder[2]] << 16) | (q[iorder[1]] << 8) | q[iorder[0]]; -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN if (bom == 0x0000FEFF) { q += 4; bo = -1; @@ -4949,7 +4941,7 @@ _PyUnicode_EncodeUTF32(PyObject *str, unsigned char *p; Py_ssize_t nsize, i; /* Offsets from p for storing byte pairs in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN int iorder[] = {0, 1, 2, 3}; #else int iorder[] = {3, 2, 1, 0}; @@ -5092,7 +5084,7 @@ PyUnicode_DecodeUTF16Stateful(const char *s, return unicode_empty; } -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN native_ordering = bo <= 0; #else native_ordering = bo >= 0; @@ -5209,7 +5201,7 @@ _PyUnicode_EncodeUTF16(PyObject *str, unsigned short *out; Py_ssize_t bytesize; Py_ssize_t pairs; -#ifdef WORDS_BIGENDIAN +#if PY_BIG_ENDIAN int native_ordering = byteorder >= 0; #else int native_ordering = byteorder <= 0; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 20bfa555b3d..cafbb58667b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1561,7 +1561,6 @@ PyObject * _PySys_Init(void) { PyObject *m, *v, *sysdict, *version_info; - char *s; m = PyModule_Create(&sysmodule); if (m == NULL) @@ -1638,20 +1637,14 @@ _PySys_Init(void) PyLong_FromLong(0x10FFFF)); SET_SYS_FROM_STRING("builtin_module_names", list_builtin_module_names()); - { - /* Assumes that longs are at least 2 bytes long. - Should be safe! */ - unsigned long number = 1; - char *value; +#if PY_BIG_ENDIAN + SET_SYS_FROM_STRING("byteorder", + PyUnicode_FromString("big")); +#else + SET_SYS_FROM_STRING("byteorder", + PyUnicode_FromString("little")); +#endif - s = (char *) &number; - if (s[0] == 0) - value = "big"; - else - value = "little"; - SET_SYS_FROM_STRING("byteorder", - PyUnicode_FromString(value)); - } #ifdef MS_COREDLL SET_SYS_FROM_STRING("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule));