This appears to be a bug in Python 3.6 that I hit while trying to add type hints to my bidirectional mapping library (https://bidict.rtfd.io).

Pasting a working, minimal repro below for easier inline viewing, and also attaching for easier downloading and running.

Please let me know if there is a workaround that would allow me to continue to support Python 3.6 after adding type hints without having to remove the use of slots and weak references. I spent a while trying to find one first but was then encouraged to report this by @ethanhs.

Thanks in advance for any pointers you may have.

#!/usr/bin/env python3
"""Repro for Python 3.6 slots + weakref + typing.Generic bug."""

from typing import Iterator, Mapping, MutableMapping, TypeVar
from weakref import ref


KT1 = TypeVar("KT1")
KT2 = TypeVar("KT2")


class Invertible(Mapping[KT1, KT2]):
    """A one-element mapping that is generic in two key types with a reference to its inverse.

    ...which in turn holds a (weak) reference back to it.

        >>> element = Invertible("H", 1)
        >>> element
        <Invertible key1='H' key2=1>
        >>> element.inverse
        <Invertible key1=1 key2='H'>
        >>> element.inverse.inverse
        <Invertible key1='H' key2=1>
        >>> element.inverse.inverse is element
        True

        >>> dict(element.items())
        {'H': 1}
        >>> dict(element.inverse.items())
        {1: 'H'}
        >>> list(element)
        ['H']

    Uses the __slots__ optimization, and uses weakrefs for references in one direction
    to avoid strong reference cycles. And still manages to support pickling to boot!

        >>> from pickle import dumps, loads
        >>> pickled = dumps(element)
        >>> roundtripped = loads(pickled)
        >>> roundtripped
        <Invertible key1='H' key2=1>

    """

    # Each instance has (either a strong or a weak) reference to its
    # inverse instance, which has a (weak or strong) reference back.
    __slots__ = ("_inverse_strong", "_inverse_weak", "__weakref__", "key1", "key2")

    def __init__(self, key1: KT1, key2: KT2) -> None:
        self._inverse_weak = None
        self._inverse_strong = inverse = self.__class__.__new__(self.__class__)
        self.key1 = inverse.key2 = key1
        self.key2 = inverse.key1 = key2
        inverse._inverse_strong = None
        inverse._inverse_weak = ref(self)

    def __len__(self) -> int:
        return 1

    def __iter__(self) -> Iterator[KT1]:
        yield self.key1

    def __getitem__(self, key: KT1) -> KT2:
        if key == self.key1:
            return self.key2
        raise KeyError(key)

    def __repr__(self) -> str:
        return f"<{self.__class__.__name__} key1={self.key1!r} key2={self.key2!r}>"

    @property
    def inverse(self) -> "Invertible[KT2, KT1]":
        """The inverse instance."""
        if self._inverse_strong is not None:
            return self._inverse_strong
        inverse = self._inverse_weak()
        if inverse is not None:
            return inverse
        # Refcount of referent must have dropped to zero,
        # as in `Invertible().inverse.inverse`, so init a new one.
        self._inverse_weak = None
        self._inverse_strong = inverse = self.__class__.__new__(self.__class__)
        inverse.key2 = self.key1
        inverse.key1 = self.key2
        inverse._inverse_strong = None
        inverse._inverse_weak = ref(self)
        return inverse

    def __getstate__(self) -> dict:
        """Needed to enable pickling due to use of __slots__ and weakrefs."""
        state = {}
        for cls in self.__class__.__mro__:
            slots = getattr(cls, '__slots__', ())
            for slot in slots:
                if hasattr(self, slot):
                    state[slot] = getattr(self, slot)
        # weakrefs can't be pickled.
        state.pop('_inverse_weak', None)  # Added back in __setstate__ below.
        state.pop('__weakref__', None)  # Not added back in __setstate__. Python manages this one.
        return state

    def __setstate__(self, state) -> None:
        """Needed because use of __slots__ would prevent unpickling otherwise."""
        for slot, value in state.items():
            setattr(self, slot, value)
        self._inverse_weak = None
        self._inverse_strong = inverse = self.__class__.__new__(self.__class__)
        inverse.key2 = self.key1
        inverse.key1 = self.key2
        inverse._inverse_strong = None
        inverse._inverse_weak = ref(self)


# So far so good, but now let's make a mutable version.
# 
# The following class definition works on Python > 3.6, but fails on 3.6 with
# TypeError: __weakref__ slot disallowed: either we already got one, or __itemsize__ != 0

class MutableInvertible(Invertible[KT1, KT2], MutableMapping[KT1, KT2]):
    """Works on > 3.6, but we don't even get this far on Python 3.6:

        >>> MutableInvertible("H", 1)
        <MutableInvertible key1='H' key2=1>

    """

    __slots__ = ()

    def __setitem__(self, key1: KT1, key2: KT2) -> None:
        self.key1 = self.inverse.key2 = key1
        self.key2 = self.inverse.key1 = key2

    def __delitem__(self, key: KT1) -> None:
        raise KeyError(key)


if __name__ == "__main__":
    import doctest
    doctest.testmod()

It turns out that this bug reproduces with any subclass of the generic type with a weakref slot, even without any multiple inheritance going on.

For example:

class Invertible2(Invertible[KT1, KT2]): pass

is enough to trigger this bug along with the Invertible class in my previous example. Attaching the more minimal repro with this comment, and renaming the issue to remove the reference to multiple inheritance.

Whittled this down to an even more minimal repro:


"""Repro for Python 3.6 slots + weakref + typing.Generic subclass bug."""

from typing import Generic, TypeVar
from weakref import ref


T = TypeVar("T")

class MyGeneric(Generic[T]):
    """MyGeneric works as expected.

        >>> example = MyGeneric()
        >>> from pickle import dumps, loads
        >>> pickled = dumps(example)
        >>> roundtripped = loads(pickled)
        >>> roundtripped
        <__main__.MyGeneric object at ...>

    """

    __slots__ = ("_other", "__weakref__")

    def __init__(self) -> None:
        self._init_other()

    def _init_other(self) -> None:
        other = self.__class__.__new__(self.__class__)
        other._other = self
        self._other = ref(other)

    def __getstate__(self) -> dict:
        """Needed to enable pickling due to use of __slots__ and weakrefs."""
        return {}

    def __setstate__(self, _) -> None:
        """Needed because use of __slots__ would prevent unpickling otherwise."""
        self._init_other()


# So far so good, but now let's make a subclass.
# The following class definition works on Python > 3.6, but fails on 3.6 with
# TypeError: __weakref__ slot disallowed: either we already got one, or __itemsize__ != 0
class FailsInPy36(MyGeneric[T]):
    """Minimal repro.

        >>> repro = FailsInPy36()
        >>> repro
        <__main__.FailsInPy36 object at ...>

    """


if __name__ == "__main__":
    import doctest
    doctest.testmod(optionflags=doctest.ELLIPSIS)

The problem appears to be occurring when the base class is subscripted, not when it's inherited. I can reproduce this issue on Python 3.6.10 by just evaluating Base[T].

'del Base.__slots__' after Base is constructed seems to work around the issue, and allow Base[T] to be evaluated. Of course, Base is still slotted at this point, since __slots__ are consulted only when initially building the class.

Thanks so much, @oremanj! Indeed, merely subscripting the class triggers the bug, and your 'del slots' workaround does the trick!

For completeness, there is an updated (yet more minimal) repro below/attached.


"""Repro for Python 3.6 slots + weakref + typing.Generic subclass bug."""

from typing import Generic, TypeVar
from weakref import ref


T = TypeVar("T")

class MyGeneric(Generic[T]):
    """MyGeneric works as expected.

        >>> example = MyGeneric()
        >>> example
        <__main__.MyGeneric object at ...>
        >>> example._other
        <__main__.MyGeneric object at ...>
        >>> example._other._other
        <weakref at ...; to 'MyGeneric' at ...>
        >>> from pickle import dumps, loads
        >>> pickled = dumps(example)
        >>> roundtripped = loads(pickled)
        >>> roundtripped
        <__main__.MyGeneric object at ...>

    """

    __slots__ = ("_other", "__weakref__")

    def __init__(self) -> None:
        self._init_other()

    def _init_other(self) -> None:
        other = self.__class__.__new__(self.__class__)
        other._other = ref(self)
        self._other = other

    def __getstate__(self) -> dict:
        """Needed to enable pickling due to use of __slots__ and weakrefs."""
        return {}

    def __setstate__(self, _) -> None:
        """Needed because use of __slots__ would prevent unpickling otherwise."""
        self._init_other()


# Merely the following is enough to trigger the bug on Python 3.6:
MyGeneric[T]

# This works around the issue if run first (thanks @oremanj):
del MyGeneric.__slots__  # does not actually 'unslot' the class


if __name__ == "__main__":
    import doctest
    doctest.testmod(optionflags=doctest.ELLIPSIS)

Thank you for the report and for the analysis. As you probably know, Python 3.6 is now in the security phase of its life cycle so generally only fixes for security-related issues are provided at this point. This issue doesn't seem to fall into that category. If you have a workaround for 3.6.x, documenting it is probably the best available option. I'm going to close this as "wont fix".

https://www.python.org/downloads/