From 9f5fe7910f4a1bf5a425837d4915e332b945eb7b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 17 Apr 2020 19:05:35 +0200 Subject: [PATCH] bpo-40286: Add randbytes() method to random.Random (GH-19527) Add random.randbytes() function and random.Random.randbytes() method to generate random bytes. Modify secrets.token_bytes() to use SystemRandom.randbytes() rather than calling directly os.urandom(). Rename also genrand_int32() to genrand_uint32(), since it returns an unsigned 32-bit integer, not a signed integer. The _random module is now built with Py_BUILD_CORE_MODULE defined. --- Doc/library/random.rst | 7 ++ Doc/whatsnew/3.9.rst | 6 ++ Lib/random.py | 7 ++ Lib/secrets.py | 3 +- Lib/test/test_random.py | 51 +++++++++++++++ .../2020-04-15-00-39-25.bpo-40286.ai80FA.rst | 2 + Modules/Setup | 2 +- Modules/_randommodule.c | 65 +++++++++++++++++-- Modules/clinic/_randommodule.c.h | 43 +++++++++++- setup.py | 3 +- 10 files changed, 177 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-04-15-00-39-25.bpo-40286.ai80FA.rst diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 1eb39bbda42..51242cb0e95 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -112,6 +112,13 @@ Bookkeeping functions :meth:`randrange` to handle arbitrarily large ranges. +.. function:: randbytes(n) + + Generate *n* random bytes. + + .. versionadded:: 3.9 + + Functions for integers ---------------------- diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index aae8e5b0c97..2b36b0f154b 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -353,6 +353,12 @@ The documentation string is now shown not only for class, function, method etc, but for any object that has its own ``__doc__`` attribute. (Contributed by Serhiy Storchaka in :issue:`40257`.) +random +------ + +Add a new :attr:`random.Random.randbytes` method: generate random bytes. +(Contributed by Victor Stinner in :issue:`40286`.) + signal ------ diff --git a/Lib/random.py b/Lib/random.py index e24737d4508..82345fab921 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -739,6 +739,12 @@ class SystemRandom(Random): x = int.from_bytes(_urandom(numbytes), 'big') return x >> (numbytes * 8 - k) # trim excess bits + def randbytes(self, n): + """Generate n random bytes.""" + # os.urandom(n) fails with ValueError for n < 0 + # and returns an empty bytes string for n == 0. + return _urandom(n) + def seed(self, *args, **kwds): "Stub method. Not used for a system random number generator." return None @@ -819,6 +825,7 @@ weibullvariate = _inst.weibullvariate getstate = _inst.getstate setstate = _inst.setstate getrandbits = _inst.getrandbits +randbytes = _inst.randbytes if hasattr(_os, "fork"): _os.register_at_fork(after_in_child=_inst.seed) diff --git a/Lib/secrets.py b/Lib/secrets.py index 130434229e9..a546efbdd42 100644 --- a/Lib/secrets.py +++ b/Lib/secrets.py @@ -14,7 +14,6 @@ __all__ = ['choice', 'randbelow', 'randbits', 'SystemRandom', import base64 import binascii -import os from hmac import compare_digest from random import SystemRandom @@ -44,7 +43,7 @@ def token_bytes(nbytes=None): """ if nbytes is None: nbytes = DEFAULT_ENTROPY - return os.urandom(nbytes) + return _sysrand.randbytes(nbytes) def token_hex(nbytes=None): """Return a random text string, in hexadecimal. diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 548af706dbe..f709e52ecb8 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -291,6 +291,22 @@ class TestBasicOps: k = sum(randrange(6755399441055744) % 3 == 2 for i in range(n)) self.assertTrue(0.30 < k/n < .37, (k/n)) + def test_randbytes(self): + # Verify ranges + for n in range(1, 10): + data = self.gen.randbytes(n) + self.assertEqual(type(data), bytes) + self.assertEqual(len(data), n) + + self.assertEqual(self.gen.randbytes(0), b'') + + # Verify argument checking + self.assertRaises(TypeError, self.gen.randbytes) + self.assertRaises(TypeError, self.gen.randbytes, 1, 2) + self.assertRaises(ValueError, self.gen.randbytes, -1) + self.assertRaises(TypeError, self.gen.randbytes, 1.0) + + try: random.SystemRandom().random() except NotImplementedError: @@ -747,6 +763,41 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): c = self.gen.choices(population, cum_weights=cum_weights, k=10000) self.assertEqual(a, c) + def test_randbytes(self): + super().test_randbytes() + + # Mersenne Twister randbytes() is deterministic + # and does not depend on the endian and bitness. + seed = 8675309 + expected = b'f\xf9\xa836\xd0\xa4\xf4\x82\x9f\x8f\x19\xf0eo\x02' + + self.gen.seed(seed) + self.assertEqual(self.gen.randbytes(16), expected) + + # randbytes(0) must not consume any entropy + self.gen.seed(seed) + self.assertEqual(self.gen.randbytes(0), b'') + self.assertEqual(self.gen.randbytes(16), expected) + + # Four randbytes(4) calls give the same output than randbytes(16) + self.gen.seed(seed) + self.assertEqual(b''.join([self.gen.randbytes(4) for _ in range(4)]), + expected) + + # Each randbytes(2) or randbytes(3) call consumes 4 bytes of entropy + self.gen.seed(seed) + expected2 = b''.join(expected[i:i + 2] + for i in range(0, len(expected), 4)) + self.assertEqual(b''.join(self.gen.randbytes(2) for _ in range(4)), + expected2) + + self.gen.seed(seed) + expected3 = b''.join(expected[i:i + 3] + for i in range(0, len(expected), 4)) + self.assertEqual(b''.join(self.gen.randbytes(3) for _ in range(4)), + expected3) + + def gamma(z, sqrt2pi=(2.0*pi)**0.5): # Reflection to right half of complex plane if z < 0.5: diff --git a/Misc/NEWS.d/next/Library/2020-04-15-00-39-25.bpo-40286.ai80FA.rst b/Misc/NEWS.d/next/Library/2020-04-15-00-39-25.bpo-40286.ai80FA.rst new file mode 100644 index 00000000000..69c9cff10aa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-04-15-00-39-25.bpo-40286.ai80FA.rst @@ -0,0 +1,2 @@ +Add :func:`random.randbytes` function and +:meth:`random.Random.randbytes` method to generate random bytes. diff --git a/Modules/Setup b/Modules/Setup index 9dcca131000..6f0374a2063 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -174,7 +174,7 @@ _symtable symtablemodule.c #_weakref _weakref.c # basic weak reference support #_testcapi _testcapimodule.c # Python C API test module #_testinternalcapi _testinternalcapi.c -I$(srcdir)/Include/internal -DPy_BUILD_CORE_MODULE # Python internal C API test module -#_random _randommodule.c # Random number generator +#_random _randommodule.c -DPy_BUILD_CORE_MODULE # Random number generator #_elementtree -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI _elementtree.c # elementtree accelerator #_pickle _pickle.c # pickle accelerator #_datetime _datetimemodule.c # datetime accelerator diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 90762758f93..560460b9a44 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -11,7 +11,7 @@ * renamed genrand_res53() to random_random() and wrapped in python calling/return code. - * genrand_int32() and the helper functions, init_genrand() + * genrand_uint32() and the helper functions, init_genrand() and init_by_array(), were declared static, wrapped in Python calling/return code. also, their global data references were replaced with structure references. @@ -67,9 +67,9 @@ /* ---------------------------------------------------------------*/ #include "Python.h" -#include /* for seeding to current time */ +#include "pycore_byteswap.h" // _Py_bswap32() #ifdef HAVE_PROCESS_H -# include /* needed for getpid() */ +# include // getpid() #endif /* Period parameters -- These are all magic. Don't change. */ @@ -116,7 +116,7 @@ class _random.Random "RandomObject *" "&Random_Type" /* generates a random number on [0,0xffffffff]-interval */ static uint32_t -genrand_int32(RandomObject *self) +genrand_uint32(RandomObject *self) { uint32_t y; static const uint32_t mag01[2] = {0x0U, MATRIX_A}; @@ -171,7 +171,7 @@ static PyObject * _random_Random_random_impl(RandomObject *self) /*[clinic end generated code: output=117ff99ee53d755c input=afb2a59cbbb00349]*/ { - uint32_t a=genrand_int32(self)>>5, b=genrand_int32(self)>>6; + uint32_t a=genrand_uint32(self)>>5, b=genrand_uint32(self)>>6; return PyFloat_FromDouble((a*67108864.0+b)*(1.0/9007199254740992.0)); } @@ -481,7 +481,7 @@ _random_Random_getrandbits_impl(RandomObject *self, int k) } if (k <= 32) /* Fast path */ - return PyLong_FromUnsignedLong(genrand_int32(self) >> (32 - k)); + return PyLong_FromUnsignedLong(genrand_uint32(self) >> (32 - k)); words = (k - 1) / 32 + 1; wordarray = (uint32_t *)PyMem_Malloc(words * 4); @@ -498,7 +498,7 @@ _random_Random_getrandbits_impl(RandomObject *self, int k) for (i = words - 1; i >= 0; i--, k -= 32) #endif { - r = genrand_int32(self); + r = genrand_uint32(self); if (k < 32) r >>= (32 - k); /* Drop least significant bits */ wordarray[i] = r; @@ -510,6 +510,56 @@ _random_Random_getrandbits_impl(RandomObject *self, int k) return result; } +/*[clinic input] + +_random.Random.randbytes + + self: self(type="RandomObject *") + n: Py_ssize_t + / + +Generate n random bytes. +[clinic start generated code]*/ + +static PyObject * +_random_Random_randbytes_impl(RandomObject *self, Py_ssize_t n) +/*[clinic end generated code: output=67a28548079a17ea input=7ba658a24150d233]*/ +{ + if (n < 0) { + PyErr_SetString(PyExc_ValueError, + "number of bytes must be non-negative"); + return NULL; + } + + if (n == 0) { + /* Don't consume any entropy */ + return PyBytes_FromStringAndSize(NULL, 0); + } + + PyObject *bytes = PyBytes_FromStringAndSize(NULL, n); + if (bytes == NULL) { + return NULL; + } + uint8_t *ptr = (uint8_t *)PyBytes_AS_STRING(bytes); + + do { + uint32_t word = genrand_uint32(self); +#if PY_LITTLE_ENDIAN + /* Convert to big endian */ + word = _Py_bswap32(word); +#endif + if (n < 4) { + memcpy(ptr, &word, n); + break; + } + memcpy(ptr, &word, 4); + ptr += 4; + n -= 4; + } while (n); + + return bytes; +} + static PyObject * random_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -539,6 +589,7 @@ static PyMethodDef random_methods[] = { _RANDOM_RANDOM_GETSTATE_METHODDEF _RANDOM_RANDOM_SETSTATE_METHODDEF _RANDOM_RANDOM_GETRANDBITS_METHODDEF + _RANDOM_RANDOM_RANDBYTES_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index a467811d93b..dda78f6013c 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -114,4 +114,45 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=a7feb0c9c8d1b627 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_random_Random_randbytes__doc__, +"randbytes($self, n, /)\n" +"--\n" +"\n" +"Generate n random bytes."); + +#define _RANDOM_RANDOM_RANDBYTES_METHODDEF \ + {"randbytes", (PyCFunction)_random_Random_randbytes, METH_O, _random_Random_randbytes__doc__}, + +static PyObject * +_random_Random_randbytes_impl(RandomObject *self, Py_ssize_t n); + +static PyObject * +_random_Random_randbytes(RandomObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_ssize_t n; + + if (PyFloat_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float" ); + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + n = ival; + } + return_value = _random_Random_randbytes_impl(self, n); + +exit: + return return_value; +} +/*[clinic end generated code: output=e515c651860c4001 input=a9049054013a1b77]*/ diff --git a/setup.py b/setup.py index 65a1cfab078..d241dc0b4b4 100644 --- a/setup.py +++ b/setup.py @@ -808,7 +808,8 @@ class PyBuildExt(build_ext): self.add(Extension('_datetime', ['_datetimemodule.c'], libraries=['m'])) # random number generator implemented in C - self.add(Extension("_random", ["_randommodule.c"])) + self.add(Extension("_random", ["_randommodule.c"], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # bisect self.add(Extension("_bisect", ["_bisectmodule.c"])) # heapq