diff -r 84c8127453cf Include/abstract.h --- a/Include/abstract.h Sat Oct 06 03:40:10 2012 +0200 +++ b/Include/abstract.h Sat Oct 06 04:35:02 2012 -0700 @@ -403,9 +403,8 @@ PyAPI_FUNC(Py_ssize_t) PyObject_Length(PyObject *o); #define PyObject_Length PyObject_Size -#ifndef Py_LIMITED_API - PyAPI_FUNC(Py_ssize_t) _PyObject_LengthHint(PyObject *o, Py_ssize_t); -#endif +PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); +PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t); /* Guess the size of object o using len(o) or o.__length_hint__(). diff -r 84c8127453cf Lib/test/test_enumerate.py --- a/Lib/test/test_enumerate.py Sat Oct 06 03:40:10 2012 +0200 +++ b/Lib/test/test_enumerate.py Sat Oct 06 04:35:02 2012 -0700 @@ -1,4 +1,5 @@ import unittest +import operator import sys import pickle @@ -168,15 +169,13 @@ x = range(1) self.assertEqual(type(reversed(x)), type(iter(x))) - @support.cpython_only def test_len(self): # This is an implementation detail, not an interface requirement - from test.test_iterlen import len for s in ('hello', tuple('hello'), list('hello'), range(5)): - self.assertEqual(len(reversed(s)), len(s)) + self.assertEqual(operator.length_hint(reversed(s)), len(s)) r = reversed(s) list(r) - self.assertEqual(len(r), 0) + self.assertEqual(operator.length_hint(r), 0) class SeqWithWeirdLen: called = False def __len__(self): @@ -187,7 +186,7 @@ def __getitem__(self, index): return index r = reversed(SeqWithWeirdLen()) - self.assertRaises(ZeroDivisionError, len, r) + self.assertRaises(ZeroDivisionError, operator.length_hint, r) def test_gc(self): diff -r 84c8127453cf Lib/test/test_iterlen.py --- a/Lib/test/test_iterlen.py Sat Oct 06 03:40:10 2012 +0200 +++ b/Lib/test/test_iterlen.py Sat Oct 06 04:35:02 2012 -0700 @@ -45,31 +45,21 @@ from test import support from itertools import repeat from collections import deque -from builtins import len as _len +from operator import length_hint n = 10 -def len(obj): - try: - return _len(obj) - except TypeError: - try: - # note: this is an internal undocumented API, - # don't rely on it in your own programs - return obj.__length_hint__() - except AttributeError: - raise TypeError class TestInvariantWithoutMutations(unittest.TestCase): def test_invariant(self): it = self.it for i in reversed(range(1, n+1)): - self.assertEqual(len(it), i) + self.assertEqual(length_hint(it), i) next(it) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) self.assertRaises(StopIteration, next, it) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) class TestTemporarilyImmutable(TestInvariantWithoutMutations): @@ -78,12 +68,12 @@ # length immutability during iteration it = self.it - self.assertEqual(len(it), n) + self.assertEqual(length_hint(it), n) next(it) - self.assertEqual(len(it), n-1) + self.assertEqual(length_hint(it), n-1) self.mutate() self.assertRaises(RuntimeError, next, it) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) ## ------- Concrete Type Tests ------- @@ -92,10 +82,6 @@ def setUp(self): self.it = repeat(None, n) - def test_no_len_for_infinite_repeat(self): - # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) - class TestXrange(TestInvariantWithoutMutations): def setUp(self): @@ -167,14 +153,15 @@ it = iter(d) next(it) next(it) - self.assertEqual(len(it), n-2) + self.assertEqual(length_hint(it), n - 2) d.append(n) - self.assertEqual(len(it), n-1) # grow with append + self.assertEqual(length_hint(it), n - 1) # grow with append d[1:] = [] - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) self.assertEqual(list(it), []) d.extend(range(20)) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) + class TestListReversed(TestInvariantWithoutMutations): @@ -186,32 +173,41 @@ it = reversed(d) next(it) next(it) - self.assertEqual(len(it), n-2) + self.assertEqual(length_hint(it), n - 2) d.append(n) - self.assertEqual(len(it), n-2) # ignore append + self.assertEqual(length_hint(it), n - 2) # ignore append d[1:] = [] - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) self.assertEqual(list(it), []) # confirm invariant d.extend(range(20)) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) ## -- Check to make sure exceptions are not suppressed by __length_hint__() class BadLen(object): - def __iter__(self): return iter(range(10)) + def __iter__(self): + return iter(range(10)) + def __len__(self): raise RuntimeError('hello') + class BadLengthHint(object): - def __iter__(self): return iter(range(10)) + def __iter__(self): + return iter(range(10)) + def __length_hint__(self): raise RuntimeError('hello') + class NoneLengthHint(object): - def __iter__(self): return iter(range(10)) + def __iter__(self): + return iter(range(10)) + def __length_hint__(self): - return None + return NotImplemented + class TestLengthHintExceptions(unittest.TestCase): diff -r 84c8127453cf Lib/test/test_itertools.py --- a/Lib/test/test_itertools.py Sat Oct 06 03:40:10 2012 +0200 +++ b/Lib/test/test_itertools.py Sat Oct 06 04:35:02 2012 -0700 @@ -1723,9 +1723,8 @@ class LengthTransparency(unittest.TestCase): def test_repeat(self): - from test.test_iterlen import len - self.assertEqual(len(repeat(None, 50)), 50) - self.assertRaises(TypeError, len, repeat(None)) + self.assertEqual(operator.length_hint(repeat(None, 50)), 50) + self.assertEqual(operator.length_hint(repeat(None), 12), 12) class RegressionTests(unittest.TestCase): diff -r 84c8127453cf Lib/test/test_operator.py --- a/Lib/test/test_operator.py Sat Oct 06 03:40:10 2012 +0200 +++ b/Lib/test/test_operator.py Sat Oct 06 04:35:02 2012 -0700 @@ -410,6 +410,31 @@ self.assertEqual(operator.__ixor__ (c, 5), "ixor") self.assertEqual(operator.__iconcat__ (c, c), "iadd") + def test_length_hint(self): + class X(object): + def __init__(self, value): + self.value = value + + def __length_hint__(self): + if type(self.value) is type: + raise self.value + else: + return self.value + + self.assertEqual(operator.length_hint([], 2), 0) + self.assertEqual(operator.length_hint(iter([1, 2, 3])), 3) + + self.assertEqual(operator.length_hint(X(2)), 2) + self.assertEqual(operator.length_hint(X(NotImplemented), 4), 4) + self.assertEqual(operator.length_hint(X(TypeError), 12), 12) + with self.assertRaises(TypeError): + operator.length_hint(X("abc")) + with self.assertRaises(ValueError): + operator.length_hint(X(-2)) + with self.assertRaises(LookupError): + operator.length_hint(X(LookupError)) + + def test_main(verbose=None): import sys test_classes = ( diff -r 84c8127453cf Lib/test/test_set.py --- a/Lib/test/test_set.py Sat Oct 06 03:40:10 2012 +0200 +++ b/Lib/test/test_set.py Sat Oct 06 04:35:02 2012 -0700 @@ -848,8 +848,6 @@ for v in self.set: self.assertIn(v, self.values) setiter = iter(self.set) - # note: __length_hint__ is an internal undocumented API, - # don't rely on it in your own programs self.assertEqual(setiter.__length_hint__(), len(self.set)) def test_pickling(self): diff -r 84c8127453cf Modules/operator.c --- a/Modules/operator.c Sat Oct 06 03:40:10 2012 +0200 +++ b/Modules/operator.c Sat Oct 06 04:35:02 2012 -0700 @@ -208,6 +208,31 @@ return (result == 0); } +PyDoc_STRVAR(length_hint__doc__, +"length_hint(obj, default=0) -> int\n" +"Return an estimate of the number of items in obj.\n" +"This is useful for presizing containers when building from an\n" +"iterable.\n" +"\n" +"If the object supports len(), the result will be\n" +"exact. Otherwise, it may over- or under-estimate by an\n" +"arbitrary amount. The result will be an integer >= 0."); + +static PyObject *length_hint(PyObject *self, PyObject *args) +{ + PyObject *obj; + Py_ssize_t defaultvalue = 0, res; + if (!PyArg_ParseTuple(args, "O|n:length_hint", &obj, &defaultvalue)) { + return NULL; + } + res = PyObject_LengthHint(obj, defaultvalue); + if (res == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromSsize_t(res); +} + + PyDoc_STRVAR(compare_digest__doc__, "compare_digest(a, b) -> bool\n" "\n" @@ -366,6 +391,8 @@ {"_compare_digest", (PyCFunction)compare_digest, METH_VARARGS, compare_digest__doc__}, + {"length_hint", (PyCFunction)length_hint, METH_VARARGS, + length_hint__doc__}, {NULL, NULL} /* sentinel */ }; diff -r 84c8127453cf Objects/abstract.c --- a/Objects/abstract.c Sat Oct 06 03:40:10 2012 +0200 +++ b/Objects/abstract.c Sat Oct 06 04:35:02 2012 -0700 @@ -64,49 +64,67 @@ } #define PyObject_Length PyObject_Size +int +_PyObject_HasLen(PyObject *o) { + return (Py_TYPE(o)->tp_as_sequence && Py_TYPE(o)->tp_as_sequence->sq_length) || + (Py_TYPE(o)->tp_as_mapping && Py_TYPE(o)->tp_as_mapping->mp_length); +} /* The length hint function returns a non-negative value from o.__len__() - or o.__length_hint__(). If those methods aren't found or return a negative - value, then the defaultvalue is returned. If one of the calls fails, - this function returns -1. + or o.__length_hint__(). If those methods aren't found. If one of the calls + fails this function returns -1. */ Py_ssize_t -_PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue) +PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue) { _Py_IDENTIFIER(__length_hint__); - PyObject *ro, *hintmeth; - Py_ssize_t rv; - - /* try o.__len__() */ - rv = PyObject_Size(o); - if (rv >= 0) - return rv; - if (PyErr_Occurred()) { - if (!PyErr_ExceptionMatches(PyExc_TypeError)) + Py_ssize_t res = PyObject_Length(o); + if (res < 0 && PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_TypeError)) { return -1; + } PyErr_Clear(); } - - /* try o.__length_hint__() */ - hintmeth = _PyObject_LookupSpecial(o, &PyId___length_hint__); - if (hintmeth == NULL) { - if (PyErr_Occurred()) + else { + return res; + } + PyObject *hint = _PyObject_LookupSpecial(o, &PyId___length_hint__); + if (hint == NULL) { + if (PyErr_Occurred()) { return -1; - else - return defaultvalue; - } - ro = PyObject_CallFunctionObjArgs(hintmeth, NULL); - Py_DECREF(hintmeth); - if (ro == NULL) { - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return -1; - PyErr_Clear(); + } return defaultvalue; } - rv = PyLong_Check(ro) ? PyLong_AsSsize_t(ro) : defaultvalue; - Py_DECREF(ro); - return rv; + PyObject *result = PyObject_CallFunctionObjArgs(hint, NULL); + Py_DECREF(hint); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + return defaultvalue; + } + return -1; + } + else if (result == Py_NotImplemented) { + Py_DECREF(result); + return defaultvalue; + } + if (!PyLong_Check(result)) { + PyErr_Format(PyExc_TypeError, "Length hint must be an integer, not %s", + Py_TYPE(result)->tp_name); + Py_DECREF(result); + return -1; + } + defaultvalue = PyLong_AsSsize_t(result); + Py_DECREF(result); + if (defaultvalue < 0 && PyErr_Occurred()) { + return -1; + } + if (defaultvalue < 0) { + PyErr_Format(PyExc_ValueError, "__length_hint__() should return >= 0"); + return -1; + } + return defaultvalue; } PyObject * @@ -1687,7 +1705,7 @@ return NULL; /* Guess result size and allocate space. */ - n = _PyObject_LengthHint(v, 10); + n = PyObject_LengthHint(v, 10); if (n == -1) goto Fail; result = PyTuple_New(n); diff -r 84c8127453cf Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c Sat Oct 06 03:40:10 2012 +0200 +++ b/Objects/bytearrayobject.c Sat Oct 06 04:35:02 2012 -0700 @@ -2282,7 +2282,7 @@ return NULL; /* Try to determine the length of the argument. 32 is arbitrary. */ - buf_size = _PyObject_LengthHint(arg, 32); + buf_size = PyObject_LengthHint(arg, 32); if (buf_size == -1) { Py_DECREF(it); return NULL; diff -r 84c8127453cf Objects/bytesobject.c --- a/Objects/bytesobject.c Sat Oct 06 03:40:10 2012 +0200 +++ b/Objects/bytesobject.c Sat Oct 06 04:35:02 2012 -0700 @@ -2651,7 +2651,7 @@ } /* For iterator version, create a string object and resize as needed */ - size = _PyObject_LengthHint(x, 64); + size = PyObject_LengthHint(x, 64); if (size == -1 && PyErr_Occurred()) return NULL; /* Allocate an extra byte to prevent PyBytes_FromStringAndSize() from diff -r 84c8127453cf Objects/iterobject.c --- a/Objects/iterobject.c Sat Oct 06 03:40:10 2012 +0200 +++ b/Objects/iterobject.c Sat Oct 06 04:35:02 2012 -0700 @@ -76,9 +76,14 @@ Py_ssize_t seqsize, len; if (it->it_seq) { - seqsize = PySequence_Size(it->it_seq); - if (seqsize == -1) - return NULL; + if (_PyObject_HasLen(it->it_seq)) { + seqsize = PySequence_Size(it->it_seq); + if (seqsize == -1) + return NULL; + } + else { + return Py_NotImplemented; + } len = seqsize - it->it_index; if (len >= 0) return PyLong_FromSsize_t(len); diff -r 84c8127453cf Objects/listobject.c --- a/Objects/listobject.c Sat Oct 06 03:40:10 2012 +0200 +++ b/Objects/listobject.c Sat Oct 06 04:35:02 2012 -0700 @@ -826,7 +826,7 @@ iternext = *it->ob_type->tp_iternext; /* Guess a result list size. */ - n = _PyObject_LengthHint(b, 8); + n = PyObject_LengthHint(b, 8); if (n == -1) { Py_DECREF(it); return NULL;