diff -r baae900289b5 -r de3ca4461b91 Include/pyport.h --- a/Include/pyport.h Sun Mar 21 11:03:36 2010 +0100 +++ b/Include/pyport.h Sun Mar 21 10:58:22 2010 +0000 @@ -126,6 +126,20 @@ #endif #endif +/* Parameters used for the numeric hash implementation. See notes for + _PyHash_Double in Objects/object.c. Numeric hashes are based on + reduction modulo the prime 2**_PyHASH_BITS - 1. */ + +#if SIZEOF_LONG >= 8 +#define _PyHASH_BITS 61 +#else +#define _PyHASH_BITS 31 +#endif +#define _PyHASH_MASK ((1UL << _PyHASH_BITS) - 1) +#define _PyHASH_INF 314159 +#define _PyHASH_NINF -271828 +#define _PyHASH_NAN 15858 + /* uintptr_t is the C9X name for an unsigned integral type such that a * legitimate void* can be cast to uintptr_t and then back to void* again * without loss of information. Similarly for intptr_t, wrt a signed diff -r baae900289b5 -r de3ca4461b91 Lib/decimal.py --- a/Lib/decimal.py Sun Mar 21 11:03:36 2010 +0100 +++ b/Lib/decimal.py Sun Mar 21 10:58:22 2010 +0000 @@ -928,33 +928,19 @@ def __hash__(self): """x.__hash__() <==> hash(x)""" - # Decimal integers must hash the same as the ints - # - # The hash of a nonspecial noninteger Decimal must depend only - # on the value of that Decimal, and not on its representation. - # For example: hash(Decimal('100E-1')) == hash(Decimal('10')). if self._is_special: if self._isnan(): raise TypeError('Cannot hash a NaN value.') - return hash(str(self)) + return _PyHASH_INF if self > 0 else _PyHASH_NINF if not self: return 0 - if self._isinteger(): - op = _WorkRep(self.to_integral_value()) - # to make computation feasible for Decimals with large - # exponent, we use the fact that hash(n) == hash(m) for - # any two nonzero integers n and m such that (i) n and m - # have the same sign, and (ii) n is congruent to m modulo - # 2**64-1. So we can replace hash((-1)**s*c*10**e) with - # hash((-1)**s*c*pow(10, e, 2**64-1). - return hash((-1)**op.sign*op.int*pow(10, op.exp, 2**64-1)) - # The value of a nonzero nonspecial Decimal instance is - # faithfully represented by the triple consisting of its sign, - # its adjusted exponent, and its coefficient with trailing - # zeros removed. - return hash((self._sign, - self._exp+len(self._int), - self._int.rstrip('0'))) + + if self._exp >= 0: + exp_hash = pow(10, self._exp, _PyHASH_MASK) + else: + exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MASK) + hash_ = 1 + (int(self._int) * exp_hash - 1) % _PyHASH_MASK + return hash_ if self > 0 else -hash_ def as_tuple(self): """Represents the number as a triple tuple. @@ -6119,6 +6105,18 @@ # _SignedInfinity[sign] is infinity w/ that sign _SignedInfinity = (_Infinity, _NegativeInfinity) +# Constants related to the hash implementation; hash(x) is based +# on the reduction of x modulo _PyHASH_MASK +import sys +_PyHASH_MASK = sys._hash_info.modulus +# hash values to use for positive and negative infinities, and nans +_PyHASH_INF = sys._hash_info.inf +_PyHASH_NINF = sys._hash_info.ninf +_PyHASH_NAN = sys._hash_info.nan +del sys + +# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MASK +_PyHASH_10INV = pow(10, _PyHASH_MASK - 2, _PyHASH_MASK) if __name__ == '__main__': diff -r baae900289b5 -r de3ca4461b91 Lib/fractions.py --- a/Lib/fractions.py Sun Mar 21 11:03:36 2010 +0100 +++ b/Lib/fractions.py Sun Mar 21 10:58:22 2010 +0000 @@ -7,6 +7,7 @@ import numbers import operator import re +import sys __all__ = ['Fraction', 'gcd'] @@ -22,6 +23,12 @@ a, b = b, a%b return a +# Constants related to the hash implementation; hash(x) is based +# on the reduction of x modulo the prime _PyHASH_MASK. +_PyHASH_MASK = sys._hash_info.modulus +# Value to be used for rationals that reduce to infinity modulo +# _PyHASH_MASK. +_PyHASH_INF = sys._hash_info.inf _RATIONAL_FORMAT = re.compile(r""" \A\s* # optional whitespace at the start, then @@ -482,16 +489,18 @@ """ # XXX since this method is expensive, consider caching the result - if self._denominator == 1: - # Get integers right. - return hash(self._numerator) - # Expensive check, but definitely correct. - if self == float(self): - return hash(float(self)) - else: - # Use tuple's hash to avoid a high collision rate on - # simple fractions. - return hash((self._numerator, self._denominator)) + + if not self: + return 0 + + # dinv is the inverse of self._denominator modulo the prime + # _PyHASH_MASK, or 0 if self._denominator is divisible by + # _PyHASH_MASK. + dinv = pow(self._denominator, _PyHASH_MASK - 2, _PyHASH_MASK) + if not dinv: + return _PyHASH_INF + hash_ = 1 + (abs(self._numerator) * dinv - 1) % _PyHASH_MASK + return hash_ if self > 0 else -hash_ def __eq__(a, b): """a == b""" diff -r baae900289b5 -r de3ca4461b91 Lib/test/test_numeric_tower.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/test_numeric_tower.py Sun Mar 21 10:58:22 2010 +0000 @@ -0,0 +1,148 @@ +# test interactions betwen int, float, Decimal and Fraction + +import unittest +import random +import math +import sys +from test.support import run_unittest + +from decimal import Decimal as D +from fractions import Fraction as F + +# Constants related to the hash implementation; hash(x) is based +# on the reduction of x modulo the prime _PyHASH_MASK. +_PyHASH_MASK = sys._hash_info.modulus + +class HashTest(unittest.TestCase): + def check_equal_hash(self, x, y): + self.assertEqual(hash(x), hash(y), + "got different hashes for {!r} and {!r}".format(x, y)) + + def test_bools(self): + self.check_equal_hash(False, 0) + self.check_equal_hash(True, 1) + + def test_integers(self): + # check that equal values hash equal + + # exact integers + for i in range(-1000, 1000): + self.check_equal_hash(i, float(i)) + self.check_equal_hash(i, D(i)) + self.check_equal_hash(i, F(i)) + + # the current hash is based on reduction modulo 2**n-1 for some + # n, so pay special attention to numbers of the form 2**n and 2**n-1. + for i in range(100): + n = 2**i - 1 + if n == int(float(n)): + self.check_equal_hash(n, float(n)) + self.check_equal_hash(-n, -float(n)) + self.check_equal_hash(n, D(n)) + self.check_equal_hash(n, F(n)) + self.check_equal_hash(-n, D(-n)) + self.check_equal_hash(-n, F(-n)) + + n = 2**i + self.check_equal_hash(n, float(n)) + self.check_equal_hash(-n, -float(n)) + self.check_equal_hash(n, D(n)) + self.check_equal_hash(n, F(n)) + self.check_equal_hash(-n, D(-n)) + self.check_equal_hash(-n, F(-n)) + + # random values of various sizes + for _ in range(1000): + e = random.randrange(300) + n = random.randrange(-10**e, 10**e) + self.check_equal_hash(n, D(n)) + self.check_equal_hash(n, F(n)) + if n == int(float(n)): + self.check_equal_hash(n, float(n)) + + def test_binary_floats(self): + # check that floats hash equal to corresponding Fractions and Decimals + + # floats that are distinct but numerically equal should hash the same + self.check_equal_hash(0.0, -0.0) + + # zeros + self.check_equal_hash(0.0, D(0)) + self.check_equal_hash(-0.0, D(0)) + self.check_equal_hash(-0.0, D('-0.0')) + self.check_equal_hash(0.0, F(0)) + + # infinities and nans + self.check_equal_hash(float('inf'), D('inf')) + self.check_equal_hash(float('-inf'), D('-inf')) + + for _ in range(1000): + x = random.random() * math.exp(random.random()*200.0 - 100.0) + self.check_equal_hash(x, D.from_float(x)) + self.check_equal_hash(x, F.from_float(x)) + + def test_complex(self): + # complex numbers with zero imaginary part should hash equal to + # the corresponding float + + test_values = [0.0, -0.0, 1.0, -1.0, 0.40625, -5136.5, + float('inf'), float('-inf'), float('nan'), + float('-nan')] + + for zero in -0.0, 0.0: + for value in test_values: + self.check_equal_hash(value, complex(value, zero)) + + def test_decimals(self): + # check that Decimal instances that have different representations + # but equal values give the same hash + zeros = ['0', '-0', '0.0', '-0.0e10', '000e-10'] + for zero in zeros: + self.check_equal_hash(D(zero), D(0)) + + self.check_equal_hash(D('1.00'), D(1)) + self.check_equal_hash(D('1.00000'), D(1)) + self.check_equal_hash(D('-1.00'), D(-1)) + self.check_equal_hash(D('-1.00000'), D(-1)) + self.check_equal_hash(D('123e2'), D(12300)) + self.check_equal_hash(D('1230e1'), D(12300)) + self.check_equal_hash(D('12300'), D(12300)) + self.check_equal_hash(D('12300.0'), D(12300)) + self.check_equal_hash(D('12300.00'), D(12300)) + self.check_equal_hash(D('12300.000'), D(12300)) + + def test_fractions(self): + # hashes are based on reduction modulo _PyHASH_MASK, so just check + # that we can still compute a hash without error for Fractions + # whose reduction modulo _PyHASH_MASK is infinite. + hash(F(1, _PyHASH_MASK)) + hash(F(-12345, _PyHASH_MASK)) + hash(F(_PyHASH_MASK, 5*_PyHASH_MASK)) + + def test_hash_normalization(self): + # Test for a bug encountered while changing long_hash. + # + # Given objects x and y, it should be possible for y's + # __hash__ method to return hash(x) in order to ensure that + # hash(x) == hash(y). But hash(x) is not exactly equal to the + # result of x.__hash__(): there's some internal normalization + # to make sure that the result fits in a C long, and is not + # equal to the invalid hash value -1. This internal + # normalization must therefore not change the result of + # hash(x) for any x. + + class HalibutProxy: + def __hash__(self): + return hash('halibut') + def __eq__(self, other): + return other == 'halibut' + + x = {'halibut', HalibutProxy()} + self.assertEqual(len(x), 1) + + +def test_main(): + run_unittest(HashTest) + +if __name__ == '__main__': + test_main() diff -r baae900289b5 -r de3ca4461b91 Objects/longobject.c --- a/Objects/longobject.c Sun Mar 21 11:03:36 2010 +0100 +++ b/Objects/longobject.c Sun Mar 21 10:58:22 2010 +0000 @@ -2572,18 +2572,17 @@ sign = -1; i = -(i); } - /* The following loop produces a C unsigned long x such that x is - congruent to the absolute value of v modulo ULONG_MAX. The - resulting x is nonzero if and only if v is. */ while (--i >= 0) { - /* Force a native long #-bits (32 or 64) circular shift */ - x = (x >> (8*SIZEOF_LONG-PyLong_SHIFT)) | (x << PyLong_SHIFT); + /* Rotate bottom 31 bits left by PyLong_SHIFT bits; in effect, + this multiplies by 2**PyLong_SHIFT modulo 2**31 - 1. */ + x = ((x << PyLong_SHIFT) & _PyHASH_MASK) | + (x >> (_PyHASH_BITS - PyLong_SHIFT)); x += v->ob_digit[i]; /* If the addition above overflowed we compensate by incrementing. This preserves the value modulo ULONG_MAX. */ - if (x < v->ob_digit[i]) - x++; + if (x > _PyHASH_MASK) + x -= _PyHASH_MASK; } x = x * sign; if (x == (unsigned long)-1) diff -r baae900289b5 -r de3ca4461b91 Objects/object.c --- a/Objects/object.c Sun Mar 21 11:03:36 2010 +0100 +++ b/Objects/object.c Sun Mar 21 10:58:22 2010 +0000 @@ -644,60 +644,100 @@ All the utility functions (_Py_Hash*()) return "-1" to signify an error. */ +/* For numeric types, the hash of a number x is based on the reduction + of x modulo the prime P = 2**_PyHASH_BITS - 1. It's designed so that + hash(x) == hash(y) whenever x and y are numerically equal, even if + x and y have different types. + + A quick summary of the hashing strategy: + + (1) First define the 'reduction of x modulo P' for any rational + number x; this is a standard extension of the usual notion of + reduction modulo P for integers. If x == p/q (written in lowest + terms), the reduction is interpreted as the reduction of p times + the inverse of the reduction of q, all modulo P; if q is exactly + divisible by P then define the reduction to be infinity. So we've + got a well-defined map + + reduce : { rational numbers } -> { 0, 1, 2, ..., P-1, infinity }. + + (2) Now for a rational number x, define hash(x) by: + + 0 if x == 0 + 1 + reduce(x-1) if x > 0 + -hash(-x) if x < 0 + + If reduce(x-1) is infinity then use the predefined hash value + _PyHASH_INF instead. _PyHASH_INF, _PyHASH_NINF and _PyHASH_NAN are + also used for the hashes of float and Decimal infinities and nans. + + A selling point for the above strategy is that it makes it possible + to compute hashes of decimal and binary floating-point numbers + efficiently, even if the exponent of the binary or decimal number + is large. The key point is that + + reduce(x * y) == reduce(reduce(x) * reduce(y)) + + provided that {reduce(x), reduce(y)} != {0, infinity}. For binary + and decimal floats the reduction is never infinity, since the + denominator is a power of 2 (for binary) or a divisor of a power of + 10 (for decimal). So we have: + + reduce(x * 2**e) == reduce(reduce(x) * reduce(2**e)) + + reduce(x * 10**e) == reduce(reduce(x) * reduce(10**e)) + + and reduce(10**e) can be computed efficiently by the usual modular + exponentiation algorithm. For reduce(2**e) it's even better: since + P is of the form 2**n-1, reduce(2**e) is 2**(e mod n), and multiplication + by 2**(e mod n) modulo 2**n-1 just amounts to a rotation of bits. + + */ + long _Py_HashDouble(double v) { - double intpart, fractpart; - int expo; - long hipart; - long x; /* the final hash value */ - /* This is designed so that Python numbers of different types - * that compare equal hash to the same value; otherwise comparisons - * of mapping keys will turn out weird. - */ + int e, sign; + double m; + unsigned long x, y; - fractpart = modf(v, &intpart); - if (fractpart == 0.0) { - /* This must return the same hash as an equal int or long. */ - if (intpart > LONG_MAX/2 || -intpart > LONG_MAX/2) { - /* Convert to long and use its hash. */ - PyObject *plong; /* converted to Python long */ - if (Py_IS_INFINITY(intpart)) - /* can't convert to long int -- arbitrary */ - v = v < 0 ? -271828.0 : 314159.0; - plong = PyLong_FromDouble(v); - if (plong == NULL) - return -1; - x = PyObject_Hash(plong); - Py_DECREF(plong); - return x; - } - /* Fits in a C long == a Python int, so is its own hash. */ - x = (long)intpart; - if (x == -1) - x = -2; - return x; + if (!Py_IS_FINITE(v)) { + if (Py_IS_INFINITY(v)) + return v > 0 ? _PyHASH_INF : _PyHASH_NINF; + else + return _PyHASH_NAN; } - /* The fractional part is non-zero, so we don't have to worry about - * making this match the hash of some other type. - * Use frexp to get at the bits in the double. - * Since the VAX D double format has 56 mantissa bits, which is the - * most of any double format in use, each of these parts may have as - * many as (but no more than) 56 significant bits. - * So, assuming sizeof(long) >= 4, each part can be broken into two - * longs; frexp and multiplication are used to do that. - * Also, since the Cray double format has 15 exponent bits, which is - * the most of any double format in use, shifting the exponent field - * left by 15 won't overflow a long (again assuming sizeof(long) >= 4). - */ - v = frexp(v, &expo); - v *= 2147483648.0; /* 2**31 */ - hipart = (long)v; /* take the top 32 bits */ - v = (v - (double)hipart) * 2147483648.0; /* get the next 32 bits */ - x = hipart + (long)v + (expo << 15); - if (x == -1) - x = -2; - return x; + + m = frexp(v, &e); + + sign = 1; + if (m < 0) { + sign = -1; + m = -m; + } + + /* process 28 bits at a time; this should work well both for binary + and hexadecimal floating point. */ + x = 0; + while (m) { + x = ((x << 28) & _PyHASH_MASK) | x >> (_PyHASH_BITS - 28); + m *= 268435456.0; /* 2**28 */ + e -= 28; + y = (unsigned long)m; /* pull out integer part */ + m -= y; + x += y; + if (x > _PyHASH_MASK) + x -= _PyHASH_MASK; + } + + /* adjust for the exponent; first reduce it modulo _PyHASH_BITS */ + e = e >= 0 ? e % _PyHASH_BITS : _PyHASH_BITS-1-((-1-e) % _PyHASH_BITS); + x = ((x << e) & _PyHASH_MASK) | x >> (_PyHASH_BITS - e); + + x = x * sign; + if (x == (unsigned long)-1) + x = (unsigned long)-2; + return (long)x; } long diff -r baae900289b5 -r de3ca4461b91 Objects/typeobject.c --- a/Objects/typeobject.c Sun Mar 21 11:03:36 2010 +0100 +++ b/Objects/typeobject.c Sun Mar 21 10:58:22 2010 +0000 @@ -4911,30 +4911,39 @@ PyObject *func, *res; static PyObject *hash_str; long h; + int overflow; func = lookup_method(self, "__hash__", &hash_str); - if (func == Py_None) { + if (func == Py_None) { Py_DECREF(func); func = NULL; } if (func == NULL) { return PyObject_HashNotImplemented(self); - } + } res = PyEval_CallObject(func, NULL); Py_DECREF(func); if (res == NULL) return -1; - if (PyLong_Check(res)) - h = PyLong_Type.tp_hash(res); - else - h = PyLong_AsLong(res); + /* It's important that any value that can come out of hash(x) + for a Python object x is left unchanged by this function, + so that an object y can ensure hash(x) == hash(y) by having + its __hash__ method return hash(x). hash(x) can be any + value that fits in a C long, with the exception of -1. */ + h = PyLong_AsLongAndOverflow(res, &overflow); + if (overflow) { + if (PyLong_Check(res)) + h = PyLong_Type.tp_hash(res); + else + h = PyLong_AsLong(res); + } Py_DECREF(res); - if (h == -1 && !PyErr_Occurred()) - h = -2; - return h; + if (h == -1 && !PyErr_Occurred()) + h = -2; + return h; } static PyObject * diff -r baae900289b5 -r de3ca4461b91 Python/sysmodule.c --- a/Python/sysmodule.c Sun Mar 21 11:03:36 2010 +0100 +++ b/Python/sysmodule.c Sun Mar 21 10:58:22 2010 +0000 @@ -570,6 +570,57 @@ return Py_None; } +static PyTypeObject _Hash_InfoType; + +PyDoc_STRVAR(_hash_info_doc, +"_hash_info\n\ +\n\ +A private struct sequence providing parameters used for computing\n\ +numeric hashes. The attributes are read only."); + +static PyStructSequence_Field _hash_info_fields[] = { + {"bits", "hash width"}, + {"modulus", "prime number giving the modulus on which the hash " + "function is based"}, + {"inf", "value to be used for hash of a positive or unsigned infinity"}, + {"ninf", "value to be used for hash of a negative infinity"}, + {"nan", "value to be used for hash of a nan"}, + {NULL, NULL} +}; + +static PyStructSequence_Desc _hash_info_desc = { + "sys._hash_info", + _hash_info_doc, + _hash_info_fields, + 5, +}; + +PyObject * +get_hash_info(void) +{ + PyObject *hash_info; + int field = 0; + hash_info = PyStructSequence_New(&_Hash_InfoType); + if (hash_info == NULL) + return NULL; + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromLong(_PyHASH_BITS)); + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromLong(_PyHASH_MASK)); + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromLong(_PyHASH_INF)); + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromLong(_PyHASH_NINF)); + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromLong(_PyHASH_NAN)); + if (PyErr_Occurred()) { + Py_CLEAR(hash_info); + return NULL; + } + return hash_info; +} + + PyDoc_STRVAR(setrecursionlimit_doc, "setrecursionlimit(n)\n\ \n\ @@ -1477,6 +1528,11 @@ PyFloat_GetInfo()); SET_SYS_FROM_STRING("int_info", PyLong_GetInfo()); + /* initialize _hash_info */ + if (_Hash_InfoType.tp_name == 0) + PyStructSequence_InitType(&_Hash_InfoType, &_hash_info_desc); + SET_SYS_FROM_STRING("_hash_info", + get_hash_info()); SET_SYS_FROM_STRING("maxunicode", PyLong_FromLong(PyUnicode_GetMax())); SET_SYS_FROM_STRING("builtin_module_names",