changeset: 106024:ca2c3aaeac77 branch: 3.5 tag: tip parent: 106010:999d636c72b4 user: Victor Stinner date: Sat Jan 07 00:26:17 2017 +0100 files: Python/random.c description: Issue #29157: Prefer getrandom() over getentropy() Copy and then adapt Python/random.c from default branch. Difference between 3.5 and default branches: * Python 3.5 only uses getrandom() in non-blocking mode: flags=GRND_NONBLOCK * If getrandom() fails with EAGAIN: py_getrandom() immediately fails and remembers that getrandom() doesn't work. * Python 3.5 has no _PyOS_URandomNonblock() function: _PyOS_URandom() works in non-blocking mode on Python 3.5 diff -r 999d636c72b4 -r ca2c3aaeac77 Python/random.c --- a/Python/random.c Fri Jan 06 09:50:27 2017 +0100 +++ b/Python/random.c Sat Jan 07 00:26:17 2017 +0100 @@ -1,6 +1,9 @@ #include "Python.h" #ifdef MS_WINDOWS # include +/* All sample MSDN wincrypt programs include the header below. It is at least + * required with Min GW. */ +# include #else # include # ifdef HAVE_SYS_STAT_H @@ -37,10 +40,9 @@ win32_urandom_init(int raise) return 0; error: - if (raise) + if (raise) { PyErr_SetFromWindowsErr(0); - else - Py_FatalError("Failed to initialize Windows random API (CryptoGen)"); + } return -1; } @@ -53,8 +55,9 @@ win32_urandom(unsigned char *buffer, Py_ if (hCryptProv == 0) { - if (win32_urandom_init(raise) == -1) + if (win32_urandom_init(raise) == -1) { return -1; + } } while (size > 0) @@ -63,11 +66,9 @@ win32_urandom(unsigned char *buffer, Py_ if (!CryptGenRandom(hCryptProv, (DWORD)chunk, buffer)) { /* CryptGenRandom() failed */ - if (raise) + if (raise) { PyErr_SetFromWindowsErr(0); - else - Py_FatalError("Failed to initialized the randomized hash " - "secret using CryptoGen)"); + } return -1; } buffer += chunk; @@ -76,58 +77,23 @@ win32_urandom(unsigned char *buffer, Py_ return 0; } -/* Issue #25003: Don't use getentropy() on Solaris (available since - * Solaris 11.3), it is blocking whereas os.urandom() should not block. */ -#elif defined(HAVE_GETENTROPY) && !defined(sun) -#define PY_GETENTROPY 1 - -/* Fill buffer with size pseudo-random bytes generated by getentropy(). - Return 0 on success, or raise an exception and return -1 on error. - - If fatal is nonzero, call Py_FatalError() instead of raising an exception - on error. */ -static int -py_getentropy(unsigned char *buffer, Py_ssize_t size, int fatal) -{ - while (size > 0) { - Py_ssize_t len = Py_MIN(size, 256); - int res; - - if (!fatal) { - Py_BEGIN_ALLOW_THREADS - res = getentropy(buffer, len); - Py_END_ALLOW_THREADS - - if (res < 0) { - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - } - else { - res = getentropy(buffer, len); - if (res < 0) - Py_FatalError("getentropy() failed"); - } - - buffer += len; - size -= len; - } - return 0; -} - -#else +#else /* !MS_WINDOWS */ #if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL) #define PY_GETRANDOM 1 -/* Call getrandom() +/* Call getrandom() to get random bytes: + - Return 1 on success - - Return 0 if getrandom() syscall is not available (failed with ENOSYS or - EPERM) or if getrandom(GRND_NONBLOCK) failed with EAGAIN (system urandom - not initialized yet) and raise=0. + - Return 0 if getrandom() is not available (failed with ENOSYS or EPERM), + or if getrandom(GRND_NONBLOCK) failed with EAGAIN (system urandom not + initialized yet). - Raise an exception (if raise is non-zero) and return -1 on error: - getrandom() failed with EINTR and the Python signal handler raised an - exception, or getrandom() failed with a different error. */ + if getrandom() failed with EINTR, raise is non-zero and the Python signal + handler raised an exception, or if getrandom() failed with a different + error. + + getrandom() is retried if it failed with EINTR: interrupted by a signal. */ static int py_getrandom(void *buffer, Py_ssize_t size, int raise) { @@ -142,16 +108,19 @@ py_getrandom(void *buffer, Py_ssize_t si * see https://bugs.python.org/issue26839. To avoid this, use the * GRND_NONBLOCK flag. */ const int flags = GRND_NONBLOCK; + char *dest; long n; if (!getrandom_works) { return 0; } + dest = buffer; while (0 < size) { #ifdef sun /* Issue #26735: On Solaris, getrandom() is limited to returning up - to 1024 bytes */ + to 1024 bytes. Call it multiple times if more bytes are + requested. */ n = Py_MIN(size, 1024); #else n = Py_MIN(size, LONG_MAX); @@ -161,34 +130,35 @@ py_getrandom(void *buffer, Py_ssize_t si #ifdef HAVE_GETRANDOM if (raise) { Py_BEGIN_ALLOW_THREADS - n = getrandom(buffer, n, flags); + n = getrandom(dest, n, flags); Py_END_ALLOW_THREADS } else { - n = getrandom(buffer, n, flags); + n = getrandom(dest, n, flags); } #else /* On Linux, use the syscall() function because the GNU libc doesn't - * expose the Linux getrandom() syscall yet. See: - * https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */ + expose the Linux getrandom() syscall yet. See: + https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */ if (raise) { Py_BEGIN_ALLOW_THREADS - n = syscall(SYS_getrandom, buffer, n, flags); + n = syscall(SYS_getrandom, dest, n, flags); Py_END_ALLOW_THREADS } else { - n = syscall(SYS_getrandom, buffer, n, flags); + n = syscall(SYS_getrandom, dest, n, flags); } #endif if (n < 0) { - /* ENOSYS: getrandom() syscall not supported by the kernel (but - * maybe supported by the host which built Python). EPERM: - * getrandom() syscall blocked by SECCOMP or something else. */ + /* ENOSYS: the syscall is not supported by the kernel. + EPERM: the syscall is blocked by a security policy (ex: SECCOMP) + or something else. */ if (errno == ENOSYS || errno == EPERM) { getrandom_works = 0; return 0; } + if (errno == EAGAIN) { /* getrandom(GRND_NONBLOCK) fails with EAGAIN if the system urandom is not initialiazed yet. In this case, fall back on @@ -202,32 +172,101 @@ py_getrandom(void *buffer, Py_ssize_t si } if (errno == EINTR) { - if (PyErr_CheckSignals()) { - if (!raise) { - Py_FatalError("getrandom() interrupted by a signal"); + if (raise) { + if (PyErr_CheckSignals()) { + return -1; } - return -1; } - /* retry getrandom() */ + /* retry getrandom() if it was interrupted by a signal */ continue; } if (raise) { PyErr_SetFromErrno(PyExc_OSError); } - else { - Py_FatalError("getrandom() failed"); + return -1; + } + + dest += n; + size -= n; + } + return 1; +} + +#elif defined(HAVE_GETENTROPY) +#define PY_GETENTROPY 1 + +/* Fill buffer with size pseudo-random bytes generated by getentropy(): + + - Return 1 on success + - Return 0 if getentropy() syscall is not available (failed with ENOSYS or + EPERM). + - Raise an exception (if raise is non-zero) and return -1 on error: + if getentropy() failed with EINTR, raise is non-zero and the Python signal + handler raised an exception, or if getentropy() failed with a different + error. + + getentropy() is retried if it failed with EINTR: interrupted by a signal. */ +static int +py_getentropy(char *buffer, Py_ssize_t size, int raise) +{ + /* Is getentropy() supported by the running kernel? Set to 0 if + getentropy() failed with ENOSYS or EPERM. */ + static int getentropy_works = 1; + + if (!getentropy_works) { + return 0; + } + + while (size > 0) { + /* getentropy() is limited to returning up to 256 bytes. Call it + multiple times if more bytes are requested. */ + Py_ssize_t len = Py_MIN(size, 256); + int res; + + if (raise) { + Py_BEGIN_ALLOW_THREADS + res = getentropy(buffer, len); + Py_END_ALLOW_THREADS + } + else { + res = getentropy(buffer, len); + } + + if (res < 0) { + /* ENOSYS: the syscall is not supported by the running kernel. + EPERM: the syscall is blocked by a security policy (ex: SECCOMP) + or something else. */ + if (errno == ENOSYS || errno == EPERM) { + getentropy_works = 0; + return 0; + } + + if (errno == EINTR) { + if (raise) { + if (PyErr_CheckSignals()) { + return -1; + } + } + + /* retry getentropy() if it was interrupted by a signal */ + continue; + } + + if (raise) { + PyErr_SetFromErrno(PyExc_OSError); } return -1; } - buffer += n; - size -= n; + buffer += len; + size -= len; } return 1; } -#endif +#endif /* defined(HAVE_GETENTROPY) && !defined(sun) */ + static struct { int fd; @@ -235,136 +274,123 @@ static struct { ino_t st_ino; } urandom_cache = { -1 }; +/* Read random bytes from the /dev/urandom device: -/* Read 'size' random bytes from py_getrandom(). Fall back on reading from - /dev/urandom if getrandom() is not available. + - Return 0 on success + - Raise an exception (if raise is non-zero) and return -1 on error - Call Py_FatalError() on error. */ -static void -dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size) + Possible causes of errors: + + - open() failed with ENOENT, ENXIO, ENODEV, EACCES: the /dev/urandom device + was not found. For example, it was removed manually or not exposed in a + chroot or container. + - open() failed with a different error + - fstat() failed + - read() failed or returned 0 + + read() is retried if it failed with EINTR: interrupted by a signal. + + The file descriptor of the device is kept open between calls to avoid using + many file descriptors when run in parallel from multiple threads: + see the issue #18756. + + st_dev and st_ino fields of the file descriptor (from fstat()) are cached to + check if the file descriptor was replaced by a different file (which is + likely a bug in the application): see the issue #21207. + + If the file descriptor was closed or replaced, open a new file descriptor + but don't close the old file descriptor: it probably points to something + important for some third-party code. */ +static int +dev_urandom(char *buffer, Py_ssize_t size, int raise) { int fd; Py_ssize_t n; - assert (0 < size); - -#ifdef PY_GETRANDOM - if (py_getrandom(buffer, size, 0) == 1) { - return; - } - /* getrandom() failed with ENOSYS or EPERM, - fall back on reading /dev/urandom */ -#endif - - fd = _Py_open_noraise("/dev/urandom", O_RDONLY); - if (fd < 0) { - Py_FatalError("Failed to open /dev/urandom"); - } - - while (0 < size) - { - do { - n = read(fd, buffer, (size_t)size); - } while (n < 0 && errno == EINTR); - - if (n <= 0) { - /* read() failed or returned 0 bytes */ - Py_FatalError("Failed to read bytes from /dev/urandom"); - break; - } - buffer += n; - size -= n; - } - close(fd); -} - -/* Read 'size' random bytes from py_getrandom(). Fall back on reading from - /dev/urandom if getrandom() is not available. + if (raise) { + struct _Py_stat_struct st; - Return 0 on success. Raise an exception and return -1 on error. */ -static int -dev_urandom_python(char *buffer, Py_ssize_t size) -{ - int fd; - Py_ssize_t n; - struct _Py_stat_struct st; -#ifdef PY_GETRANDOM - int res; -#endif + if (urandom_cache.fd >= 0) { + /* Does the fd point to the same thing as before? (issue #21207) */ + if (_Py_fstat_noraise(urandom_cache.fd, &st) + || st.st_dev != urandom_cache.st_dev + || st.st_ino != urandom_cache.st_ino) { + /* Something changed: forget the cached fd (but don't close it, + since it probably points to something important for some + third-party code). */ + urandom_cache.fd = -1; + } + } + if (urandom_cache.fd >= 0) + fd = urandom_cache.fd; + else { + fd = _Py_open("/dev/urandom", O_RDONLY); + if (fd < 0) { + if (errno == ENOENT || errno == ENXIO || + errno == ENODEV || errno == EACCES) { + PyErr_SetString(PyExc_NotImplementedError, + "/dev/urandom (or equivalent) not found"); + } + /* otherwise, keep the OSError exception raised by _Py_open() */ + return -1; + } + if (urandom_cache.fd >= 0) { + /* urandom_fd was initialized by another thread while we were + not holding the GIL, keep it. */ + close(fd); + fd = urandom_cache.fd; + } + else { + if (_Py_fstat(fd, &st)) { + close(fd); + return -1; + } + else { + urandom_cache.fd = fd; + urandom_cache.st_dev = st.st_dev; + urandom_cache.st_ino = st.st_ino; + } + } + } - if (size <= 0) - return 0; + do { + n = _Py_read(fd, buffer, (size_t)size); + if (n == -1) + return -1; + if (n == 0) { + PyErr_Format(PyExc_RuntimeError, + "Failed to read %zi bytes from /dev/urandom", + size); + return -1; + } -#ifdef PY_GETRANDOM - res = py_getrandom(buffer, size, 1); - if (res < 0) { - return -1; - } - if (res == 1) { - return 0; + buffer += n; + size -= n; + } while (0 < size); } - /* getrandom() failed with ENOSYS or EPERM, - fall back on reading /dev/urandom */ -#endif - - if (urandom_cache.fd >= 0) { - /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat_noraise(urandom_cache.fd, &st) - || st.st_dev != urandom_cache.st_dev - || st.st_ino != urandom_cache.st_ino) { - /* Something changed: forget the cached fd (but don't close it, - since it probably points to something important for some - third-party code). */ - urandom_cache.fd = -1; - } - } - if (urandom_cache.fd >= 0) - fd = urandom_cache.fd; else { - fd = _Py_open("/dev/urandom", O_RDONLY); + fd = _Py_open_noraise("/dev/urandom", O_RDONLY); if (fd < 0) { - if (errno == ENOENT || errno == ENXIO || - errno == ENODEV || errno == EACCES) - PyErr_SetString(PyExc_NotImplementedError, - "/dev/urandom (or equivalent) not found"); - /* otherwise, keep the OSError exception raised by _Py_open() */ return -1; } - if (urandom_cache.fd >= 0) { - /* urandom_fd was initialized by another thread while we were - not holding the GIL, keep it. */ - close(fd); - fd = urandom_cache.fd; - } - else { - if (_Py_fstat(fd, &st)) { + + while (0 < size) + { + do { + n = read(fd, buffer, (size_t)size); + } while (n < 0 && errno == EINTR); + + if (n <= 0) { + /* stop on error or if read(size) returned 0 */ close(fd); return -1; } - else { - urandom_cache.fd = fd; - urandom_cache.st_dev = st.st_dev; - urandom_cache.st_ino = st.st_ino; - } - } - } - do { - n = _Py_read(fd, buffer, (size_t)size); - if (n == -1) { - return -1; + buffer += n; + size -= n; } - if (n == 0) { - PyErr_Format(PyExc_RuntimeError, - "Failed to read %zi bytes from /dev/urandom", - size); - return -1; - } - - buffer += n; - size -= n; - } while (0 < size); - + close(fd); + } return 0; } @@ -376,8 +402,8 @@ dev_urandom_close(void) urandom_cache.fd = -1; } } +#endif /* !MS_WINDOWS */ -#endif /* Fill buffer with pseudo-random bytes generated by a linear congruent generator (LCG): @@ -400,29 +426,98 @@ lcg_urandom(unsigned int x0, unsigned ch } } +/* Read random bytes: + + - Return 0 on success + - Raise an exception (if raise is non-zero) and return -1 on error + + Used sources of entropy ordered by preference, preferred source first: + + - CryptGenRandom() on Windows + - getrandom() function (ex: Linux and Solaris): call py_getrandom() + - getentropy() function (ex: OpenBSD): call py_getentropy() + - /dev/urandom device + + Read from the /dev/urandom device if getrandom() or getentropy() function + is not available or does not work. + + Prefer getrandom() over getentropy() because getrandom() supports blocking + and non-blocking mode and Python requires non-blocking RNG at startup to + initialize its hash secret: see the PEP 524. + + Prefer getrandom() and getentropy() over reading directly /dev/urandom + because these functions don't need file descriptors and so avoid ENFILE or + EMFILE errors (too many open files): see the issue #18756. + + Only use RNG running in the kernel. They are more secure because it is + harder to get the internal state of a RNG running in the kernel land than a + RNG running in the user land. The kernel has a direct access to the hardware + and has access to hardware RNG, they are used as entropy sources. + + Note: the OpenSSL RAND_pseudo_bytes() function does not automatically reseed + its RNG on fork(), two child processes (with the same pid) generate the same + random numbers: see issue #18747. Kernel RNGs don't have this issue, + they have access to good quality entropy sources. + + If raise is zero: + + - Don't raise an exception on error + - Don't call the Python signal handler (don't call PyErr_CheckSignals()) if + a function fails with EINTR: retry directly the interrupted function + - Don't release the GIL to call functions. +*/ +static int +pyurandom(void *buffer, Py_ssize_t size, int raise) +{ +#if defined(PY_GETRANDOM) || defined(PY_GETENTROPY) + int res; +#endif + + if (size < 0) { + if (raise) { + PyErr_Format(PyExc_ValueError, + "negative argument not allowed"); + } + return -1; + } + + if (size == 0) { + return 0; + } + +#ifdef MS_WINDOWS + return win32_urandom((unsigned char *)buffer, size, raise); +#else + +#if defined(PY_GETRANDOM) || defined(PY_GETENTROPY) +#ifdef PY_GETRANDOM + res = py_getrandom(buffer, size, raise); +#else + res = py_getentropy(buffer, size, raise); +#endif + if (res < 0) { + return -1; + } + if (res == 1) { + return 0; + } + /* getrandom() or getentropy() function is not available: failed with + ENOSYS, EPERM or EAGAIN. Fall back on reading from /dev/urandom. */ +#endif + + return dev_urandom(buffer, size, raise); +#endif +} + /* Fill buffer with size pseudo-random bytes from the operating system random number generator (RNG). It is suitable for most cryptographic purposes except long living private keys for asymmetric encryption. - Return 0 on success, raise an exception and return -1 on error. */ + Return 0 on success. Raise an exception and return -1 on error. */ int _PyOS_URandom(void *buffer, Py_ssize_t size) { - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "negative argument not allowed"); - return -1; - } - if (size == 0) - return 0; - -#ifdef MS_WINDOWS - return win32_urandom((unsigned char *)buffer, size, 1); -#elif defined(PY_GETENTROPY) - return py_getentropy(buffer, size, 0); -#else - return dev_urandom_python((char*)buffer, size); -#endif + return pyurandom(buffer, size, 1); } void @@ -463,13 +558,14 @@ void } } else { -#ifdef MS_WINDOWS - (void)win32_urandom(secret, secret_size, 0); -#elif defined(PY_GETENTROPY) - (void)py_getentropy(secret, secret_size, 1); -#else - dev_urandom_noraise(secret, secret_size); -#endif + int res; + + /* _PyRandom_Init() is called very early in the Python initialization + and so exceptions cannot be used (use raise=0). */ + res = pyurandom(secret, secret_size, 0); + if (res < 0) { + Py_FatalError("failed to get random numbers to initialize Python"); + } } } @@ -481,8 +577,6 @@ void CryptReleaseContext(hCryptProv, 0); hCryptProv = 0; } -#elif defined(PY_GETENTROPY) - /* nothing to clean */ #else dev_urandom_close(); #endif