[Python-checkins] cpython: os.urandom() now blocks on Linux
victor.stinner
python-checkins at python.org
Tue Sep 6 19:37:43 EDT 2016
https://hg.python.org/cpython/rev/45fc0c83ed42
changeset: 103172:45fc0c83ed42
user: Victor Stinner <victor.stinner at gmail.com>
date: Tue Sep 06 16:33:52 2016 -0700
summary:
os.urandom() now blocks on Linux
Issue #27776: The os.urandom() function does now block on Linux 3.17 and newer
until the system urandom entropy pool is initialized to increase the security.
This change is part of the PEP 524.
files:
Doc/library/os.rst | 29 ++++++++---
Doc/whatsnew/3.6.rst | 12 ++++
Include/pylifecycle.h | 3 +-
Lib/random.py | 9 ---
Misc/NEWS | 4 +
Modules/_randommodule.c | 56 +++++++++++++++++---
Modules/posixmodule.c | 3 +-
Python/random.c | 75 +++++++++++++++++-----------
8 files changed, 131 insertions(+), 60 deletions(-)
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -3968,14 +3968,27 @@
returned data should be unpredictable enough for cryptographic applications,
though its exact quality depends on the OS implementation.
- On Linux, the ``getrandom()`` syscall is used if available and the urandom
- entropy pool is initialized (``getrandom()`` does not block).
- On a Unix-like system this will query ``/dev/urandom``. On Windows, it
- will use ``CryptGenRandom()``. If a randomness source is not found,
- :exc:`NotImplementedError` will be raised.
-
- For an easy-to-use interface to the random number generator
- provided by your platform, please see :class:`random.SystemRandom`.
+ On Linux, if the ``getrandom()`` syscall is available, it is used in
+ blocking mode: block until the system urandom entropy pool is initialized
+ (128 bits of entropy are collected by the kernel). See the :pep:`524` for
+ the rationale. On Linux, the :func:`getrandom` function can be used to get
+ random bytes in non-blocking mode (using the :data:`GRND_NONBLOCK` flag) or
+ to poll until the system urandom entropy pool is initialized.
+
+ On a Unix-like system, random bytes are read from the ``/dev/urandom``
+ device. If the ``/dev/urandom`` device is not available or not readable, the
+ :exc:`NotImplementedError` exception is raised.
+
+ On Windows, it will use ``CryptGenRandom()``.
+
+ .. seealso::
+ The :mod:`secrets` module provides higher level functions. For an
+ easy-to-use interface to the random number generator provided by your
+ platform, please see :class:`random.SystemRandom`.
+
+ .. versionchanged:: 3.6.0
+ On Linux, ``getrandom()`` is now used in blocking mode to increase the
+ security.
.. versionchanged:: 3.5.2
On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -70,6 +70,12 @@
* PEP 519: :ref:`Adding a file system path protocol <pep-519>`
+Security improvements:
+
+* On Linux, :func:`os.urandom` now blocks until the system urandom entropy pool
+ is initialized to increase the security. See the :pep:`524` for the
+ rationale.
+
Windows improvements:
* The ``py.exe`` launcher, when used interactively, no longer prefers
@@ -345,6 +351,9 @@
Improved Modules
================
+On Linux, :func:`os.urandom` now blocks until the system urandom entropy pool
+is initialized to increase the security. See the :pep:`524` for the rationale.
+
asyncio
-------
@@ -913,6 +922,9 @@
Changes in the Python API
-------------------------
+* On Linux, :func:`os.urandom` now blocks until the system urandom entropy pool
+ is initialized to increase the security.
+
* When :meth:`importlib.abc.Loader.exec_module` is defined,
:meth:`importlib.abc.Loader.create_module` must also be defined.
diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h
--- a/Include/pylifecycle.h
+++ b/Include/pylifecycle.h
@@ -117,7 +117,8 @@
PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t);
/* Random */
-PyAPI_FUNC(int) _PyOS_URandom (void *buffer, Py_ssize_t size);
+PyAPI_FUNC(int) _PyOS_URandom(void *buffer, Py_ssize_t size);
+PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size);
#ifdef __cplusplus
}
diff --git a/Lib/random.py b/Lib/random.py
--- a/Lib/random.py
+++ b/Lib/random.py
@@ -105,15 +105,6 @@
"""
- if a is None:
- try:
- # Seed with enough bytes to span the 19937 bit
- # state space for the Mersenne Twister
- a = int.from_bytes(_urandom(2500), 'big')
- except NotImplementedError:
- import time
- a = int(time.time() * 256) # use fractional seconds
-
if version == 1 and isinstance(a, (str, bytes)):
x = ord(a[0]) << 7 if a else 0
for c in a:
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -89,6 +89,10 @@
Library
-------
+- Issue #27776: The :func:`os.urandom` function does now block on Linux 3.17
+ and newer until the system urandom entropy pool is initialized to increase
+ the security. This change is part of the :pep:`524`.
+
- Issue #27778: Expose the Linux ``getrandom()`` syscall as a new
:func:`os.getrandom` function. This change is part of the :pep:`524`.
diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c
--- a/Modules/_randommodule.c
+++ b/Modules/_randommodule.c
@@ -165,7 +165,7 @@
/* initialize by an array with array-length */
/* init_key is the array for initializing keys */
/* key_length is its length */
-static PyObject *
+static void
init_by_array(RandomObject *self, uint32_t init_key[], size_t key_length)
{
size_t i, j, k; /* was signed in the original code. RDH 12/16/2002 */
@@ -190,8 +190,6 @@
}
mt[0] = 0x80000000U; /* MSB is 1; assuring non-zero initial array */
- Py_INCREF(Py_None);
- return Py_None;
}
/*
@@ -199,6 +197,37 @@
* Twister download.
*/
+static int
+random_seed_urandom(RandomObject *self)
+{
+ PY_UINT32_T key[N];
+
+ if (_PyOS_URandomNonblock(key, sizeof(key)) < 0) {
+ return -1;
+ }
+ init_by_array(self, key, Py_ARRAY_LENGTH(key));
+ return 0;
+}
+
+static void
+random_seed_time_pid(RandomObject *self)
+{
+ _PyTime_t now;
+ uint32_t key[5];
+
+ now = _PyTime_GetSystemClock();
+ key[0] = (PY_UINT32_T)(now & 0xffffffffU);
+ key[1] = (PY_UINT32_T)(now >> 32);
+
+ key[2] = (PY_UINT32_T)getpid();
+
+ now = _PyTime_GetMonotonicClock();
+ key[3] = (PY_UINT32_T)(now & 0xffffffffU);
+ key[4] = (PY_UINT32_T)(now >> 32);
+
+ init_by_array(self, key, Py_ARRAY_LENGTH(key));
+}
+
static PyObject *
random_seed(RandomObject *self, PyObject *args)
{
@@ -212,14 +241,17 @@
if (!PyArg_UnpackTuple(args, "seed", 0, 1, &arg))
return NULL;
- if (arg == NULL || arg == Py_None) {
- time_t now;
+ if (arg == NULL || arg == Py_None) {
+ if (random_seed_urandom(self) >= 0) {
+ PyErr_Clear();
- time(&now);
- init_genrand(self, (uint32_t)now);
- Py_INCREF(Py_None);
- return Py_None;
+ /* Reading system entropy failed, fall back on the worst entropy:
+ use the current time and process identifier. */
+ random_seed_time_pid(self);
+ }
+ Py_RETURN_NONE;
}
+
/* This algorithm relies on the number being unsigned.
* So: if the arg is a PyLong, use its absolute value.
* Otherwise use its hash value, cast to unsigned.
@@ -269,7 +301,11 @@
}
}
#endif
- result = init_by_array(self, key, keyused);
+ init_by_array(self, key, keyused);
+
+ Py_INCREF(Py_None);
+ result = Py_None;
+
Done:
Py_XDECREF(n);
PyMem_Free(key);
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -11168,8 +11168,7 @@
if (bytes == NULL)
return NULL;
- result = _PyOS_URandom(PyBytes_AS_STRING(bytes),
- PyBytes_GET_SIZE(bytes));
+ result = _PyOS_URandom(PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes));
if (result == -1) {
Py_DECREF(bytes);
return NULL;
diff --git a/Python/random.c b/Python/random.c
--- a/Python/random.c
+++ b/Python/random.c
@@ -77,7 +77,7 @@
}
/* Issue #25003: Don't use getentropy() on Solaris (available since
- Solaris 11.3), it is blocking whereas os.urandom() should not block. */
+ * Solaris 11.3), it is blocking whereas os.urandom() should not block. */
#elif defined(HAVE_GETENTROPY) && !defined(sun)
#define PY_GETENTROPY 1
@@ -121,24 +121,20 @@
/* Call getrandom()
- Return 1 on success
- - Return 0 if getrandom() syscall is not available (fails with ENOSYS).
+ - Return 0 if getrandom() syscall is not available (fails with ENOSYS)
+ or if getrandom(GRND_NONBLOCK) fails with EAGAIN (blocking=0 and system
+ urandom not initialized yet) and raise=0.
- Raise an exception (if raise is non-zero) and return -1 on error:
getrandom() failed with EINTR and the Python signal handler raised an
exception, or getrandom() failed with a different error. */
static int
-py_getrandom(void *buffer, Py_ssize_t size, int raise)
+py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise)
{
- /* Is getrandom() supported by the running kernel?
- Need Linux kernel 3.17 or newer, or Solaris 11.3 or newer */
+ /* Is getrandom() supported by the running kernel? Set to 0 if getrandom()
+ fails with ENOSYS. Need Linux kernel 3.17 or newer, or Solaris 11.3
+ or newer */
static int getrandom_works = 1;
-
- /* getrandom() on Linux will block if called before the kernel has
- initialized the urandom entropy pool. This will cause Python
- to hang on startup if called very early in the boot process -
- see https://bugs.python.org/issue26839. To avoid this, use the
- GRND_NONBLOCK flag. */
- const int flags = GRND_NONBLOCK;
-
+ int flags;
char *dest;
long n;
@@ -146,6 +142,7 @@
return 0;
}
+ flags = blocking ? 0 : GRND_NONBLOCK;
dest = buffer;
while (0 < size) {
#ifdef sun
@@ -185,15 +182,12 @@
getrandom_works = 0;
return 0;
}
- if (errno == EAGAIN) {
- /* If we failed with EAGAIN, the entropy pool was
- uninitialized. In this case, we return failure to fall
- back to reading from /dev/urandom.
- Note: In this case the data read will not be random so
- should not be used for cryptographic purposes. Retaining
- the existing semantics for practical purposes. */
- getrandom_works = 0;
+ /* getrandom(GRND_NONBLOCK) fails with EAGAIN if the system urandom
+ is not initialiazed yet. For _PyRandom_Init(), we ignore their
+ error and fall back on reading /dev/urandom which never blocks,
+ even if the system urandom is not initialized yet. */
+ if (errno == EAGAIN && !raise && !blocking) {
return 0;
}
@@ -228,13 +222,13 @@
} urandom_cache = { -1 };
-/* Read 'size' random bytes from getrandom(). Fall back on reading from
+/* Read 'size' random bytes from py_getrandom(). Fall back on reading from
/dev/urandom if getrandom() is not available.
Return 0 on success. Raise an exception (if raise is non-zero) and return -1
on error. */
static int
-dev_urandom(char *buffer, Py_ssize_t size, int raise)
+dev_urandom(char *buffer, Py_ssize_t size, int blocking, int raise)
{
int fd;
Py_ssize_t n;
@@ -245,7 +239,7 @@
assert(size > 0);
#ifdef PY_GETRANDOM
- res = py_getrandom(buffer, size, raise);
+ res = py_getrandom(buffer, size, blocking, raise);
if (res < 0) {
return -1;
}
@@ -381,7 +375,7 @@
syscall)
- Don't release the GIL to call syscalls. */
static int
-pyurandom(void *buffer, Py_ssize_t size, int raise)
+pyurandom(void *buffer, Py_ssize_t size, int blocking, int raise)
{
if (size < 0) {
if (raise) {
@@ -400,7 +394,7 @@
#elif defined(PY_GETENTROPY)
return py_getentropy(buffer, size, raise);
#else
- return dev_urandom(buffer, size, raise);
+ return dev_urandom(buffer, size, blocking, raise);
#endif
}
@@ -408,11 +402,29 @@
number generator (RNG). It is suitable for most cryptographic purposes
except long living private keys for asymmetric encryption.
- Return 0 on success, raise an exception and return -1 on error. */
+ On Linux 3.17 and newer, the getrandom() syscall is used in blocking mode:
+ block until the system urandom entropy pool is initialized (128 bits are
+ collected by the kernel).
+
+ Return 0 on success. Raise an exception and return -1 on error. */
int
_PyOS_URandom(void *buffer, Py_ssize_t size)
{
- return pyurandom(buffer, size, 1);
+ return pyurandom(buffer, size, 1, 1);
+}
+
+/* Fill buffer with size pseudo-random bytes from the operating system random
+ number generator (RNG). It is not suitable for cryptographic purpose.
+
+ On Linux 3.17 and newer (when getrandom() syscall is used), if the system
+ urandom is not initialized yet, the function returns "weak" entropy read
+ from /dev/urandom.
+
+ Return 0 on success. Raise an exception and return -1 on error. */
+int
+_PyOS_URandomNonblock(void *buffer, Py_ssize_t size)
+{
+ return pyurandom(buffer, size, 0, 1);
}
void
@@ -456,8 +468,11 @@
int res;
/* _PyRandom_Init() is called very early in the Python initialization
- and so exceptions cannot be used (use raise=0). */
- res = pyurandom(secret, secret_size, 0);
+ and so exceptions cannot be used (use raise=0).
+
+ _PyRandom_Init() must not block Python initialization: call
+ pyurandom() is non-blocking mode (blocking=0): see the PEP 524. */
+ res = pyurandom(secret, secret_size, 0, 0);
if (res < 0) {
Py_FatalError("failed to get random numbers to initialize Python");
}
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list