Windows: Use 32-bit distribution of python

1 files changed, 425 insertions, 113 deletions

diff --git a/python/gevent/local.py b/python/gevent/local.py
index 955c320..f29bb23 100644
--- a/python/gevent/local.py
+++ b/python/gevent/local.py
@@ -1,3 +1,4 @@
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
 """
 Greenlet-local objects.
 
@@ -133,161 +134,472 @@ affects what we see:
    This results in locals being eligible for garbage collection as soon
    as their greenlet exits.
 
+.. versionchanged:: 1.2.3
+   Use a weak-reference to clear the greenlet link we establish in case
+   the local object dies before the greenlet does.
+
+.. versionchanged:: 1.3a1
+   Implement the methods for attribute access directly, handling
+   descriptors directly here. This allows removing the use of a lock
+   and facilitates greatly improved performance.
+
+.. versionchanged:: 1.3a1
+   The ``__init__`` method of subclasses of ``local`` is no longer
+   called with a lock held. CPython does not use such a lock in its
+   native implementation. This could potentially show as a difference
+   if code that uses multiple dependent attributes in ``__slots__``
+   (which are shared across all greenlets) switches during ``__init__``.
+
 """
+from __future__ import print_function
 
 from copy import copy
 from weakref import ref
-from contextlib import contextmanager
-from gevent.hub import getcurrent
-from gevent._compat import PYPY
-from gevent.lock import RLock
 
-__all__ = ["local"]
+
+locals()['getcurrent'] = __import__('greenlet').getcurrent
+locals()['greenlet_init'] = lambda: None
+
+__all__ = [
+    "local",
+]
+
+# The key used in the Thread objects' attribute dicts.
+# We keep it a string for speed but make it unlikely to clash with
+# a "real" attribute.
+key_prefix = '_gevent_local_localimpl_'
+
+# The overall structure is as follows:
+# For each local() object:
+# greenlet.__dict__[key_prefix + str(id(local))]
+#    => _localimpl.dicts[id(greenlet)] => (ref(greenlet), {})
+
+# That final tuple is actually a localimpl_dict_entry object.
+
+def all_local_dicts_for_greenlet(greenlet):
+    """
+    Internal debug helper for getting the local values associated
+    with a greenlet. This is subject to change or removal at any time.
+
+    :return: A list of ((type, id), {}) pairs, where the first element
+      is the type and id of the local object and the second object is its
+      instance dictionary, as seen from this greenlet.
+
+    .. versionadded:: 1.3a2
+    """
+
+    result = []
+    id_greenlet = id(greenlet)
+    greenlet_dict = greenlet.__dict__
+    for k, v in greenlet_dict.items():
+        if not k.startswith(key_prefix):
+            continue
+        local_impl = v()
+        if local_impl is None:
+            continue
+        entry = local_impl.dicts.get(id_greenlet)
+        if entry is None:
+            # Not yet used in this greenlet.
+            continue
+        assert entry.wrgreenlet() is greenlet
+        result.append((local_impl.localtypeid, entry.localdict))
+
+    return result
 
 
 class _wrefdict(dict):
     """A dict that can be weak referenced"""
 
+class _greenlet_deleted(object):
+    """
+    A weakref callback for when the greenlet
+    is deleted.
+
+    If the greenlet is a `gevent.greenlet.Greenlet` and
+    supplies ``rawlink``, that will be used instead of a
+    weakref.
+    """
+    __slots__ = ('idt', 'wrdicts')
+
+    def __init__(self, idt, wrdicts):
+        self.idt = idt
+        self.wrdicts = wrdicts
+
+    def __call__(self, _unused):
+        dicts = self.wrdicts()
+        if dicts:
+            dicts.pop(self.idt, None)
+
+class _local_deleted(object):
+    __slots__ = ('key', 'wrthread', 'greenlet_deleted')
+
+    def __init__(self, key, wrthread, greenlet_deleted):
+        self.key = key
+        self.wrthread = wrthread
+        self.greenlet_deleted = greenlet_deleted
+
+    def __call__(self, _unused):
+        thread = self.wrthread()
+        if thread is not None:
+            try:
+                unlink = thread.unlink
+            except AttributeError:
+                pass
+            else:
+                unlink(self.greenlet_deleted)
+            del thread.__dict__[self.key]
 
 class _localimpl(object):
     """A class managing thread-local dicts"""
-    __slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__'
-
-    def __init__(self):
-        # The key used in the Thread objects' attribute dicts.
-        # We keep it a string for speed but make it unlikely to clash with
-        # a "real" attribute.
-        self.key = '_threading_local._localimpl.' + str(id(self))
-        # { id(Thread) -> (ref(Thread), thread-local dict) }
+    __slots__ = ('key', 'dicts',
+                 'localargs', 'localkwargs',
+                 'localtypeid',
+                 '__weakref__',)
+
+    def __init__(self, args, kwargs, local_type, id_local):
+        self.key = key_prefix + str(id(self))
+        # { id(greenlet) -> _localimpl_dict_entry(ref(greenlet), greenlet-local dict) }
         self.dicts = _wrefdict()
+        self.localargs = args
+        self.localkwargs = kwargs
+        self.localtypeid = local_type, id_local
 
-    def get_dict(self):
-        """Return the dict for the current thread. Raises KeyError if none
-        defined."""
-        thread = getcurrent()
-        return self.dicts[id(thread)][1]
-
-    def create_dict(self):
-        """Create a new dict for the current thread, and return it."""
-        localdict = {}
-        key = self.key
-        thread = getcurrent()
-        idt = id(thread)
-
-        # If we are working with a gevent.greenlet.Greenlet, we can
-        # pro-actively clear out with a link. Use rawlink to avoid
-        # spawning any more greenlets
-        try:
-            rawlink = thread.rawlink
-        except AttributeError:
-            # Otherwise we need to do it with weak refs
-            def local_deleted(_, key=key):
-                # When the localimpl is deleted, remove the thread attribute.
-                thread = wrthread()
-                if thread is not None:
-                    del thread.__dict__[key]
-
-            def thread_deleted(_, idt=idt):
-                # When the thread is deleted, remove the local dict.
-                # Note that this is suboptimal if the thread object gets
-                # caught in a reference loop. We would like to be called
-                # as soon as the OS-level thread ends instead.
-                _local = wrlocal()
-                if _local is not None:
-                    _local.dicts.pop(idt, None)
-            wrlocal = ref(self, local_deleted)
-            wrthread = ref(thread, thread_deleted)
-            thread.__dict__[key] = wrlocal
-        else:
-            wrdicts = ref(self.dicts)
+        # We need to create the thread dict in anticipation of
+        # __init__ being called, to make sure we don't call it
+        # again ourselves. MUST do this before setting any attributes.
+        greenlet = getcurrent() # pylint:disable=undefined-variable
+        _localimpl_create_dict(self, greenlet, id(greenlet))
+
+class _localimpl_dict_entry(object):
+    """
+    The object that goes in the ``dicts`` of ``_localimpl``
+    object for each thread.
+    """
+    # This is a class, not just a tuple, so that cython can optimize
+    # attribute access
+    __slots__ = ('wrgreenlet', 'localdict')
+
+    def __init__(self, wrgreenlet, localdict):
+        self.wrgreenlet = wrgreenlet
+        self.localdict = localdict
+
+# We use functions instead of methods so that they can be cdef'd in
+# local.pxd; if they were cdef'd as methods, they would cause
+# the creation of a pointer and a vtable. This happens
+# even if we declare the class @cython.final. functions thus save memory overhead
+# (but not pointer chasing overhead; the vtable isn't used when we declare
+# the class final).
+
+
+def _localimpl_create_dict(self, greenlet, id_greenlet):
+    """Create a new dict for the current thread, and return it."""
+    localdict = {}
+    key = self.key
+
+    wrdicts = ref(self.dicts)
 
-            def clear(_):
-                dicts = wrdicts()
-                if dicts:
-                    dicts.pop(idt, None)
-            rawlink(clear)
-            wrthread = None
+    # When the greenlet is deleted, remove the local dict.
+    # Note that this is suboptimal if the greenlet object gets
+    # caught in a reference loop. We would like to be called
+    # as soon as the OS-level greenlet ends instead.
 
-        self.dicts[idt] = wrthread, localdict
-        return localdict
+    # If we are working with a gevent.greenlet.Greenlet, we
+    # can pro-actively clear out with a link, avoiding the
+    # issue described above. Use rawlink to avoid spawning any
+    # more greenlets.
+    greenlet_deleted = _greenlet_deleted(id_greenlet, wrdicts)
 
+    rawlink = getattr(greenlet, 'rawlink', None)
+    if rawlink is not None:
+        rawlink(greenlet_deleted)
+        wrthread = ref(greenlet)
+    else:
+        wrthread = ref(greenlet, greenlet_deleted)
 
-@contextmanager
-def _patch(self):
-    impl = object.__getattribute__(self, '_local__impl')
-    orig_dct = object.__getattribute__(self, '__dict__')
+
+    # When the localimpl is deleted, remove the thread attribute.
+    local_deleted = _local_deleted(key, wrthread, greenlet_deleted)
+
+
+    wrlocal = ref(self, local_deleted)
+    greenlet.__dict__[key] = wrlocal
+
+    self.dicts[id_greenlet] = _localimpl_dict_entry(wrthread, localdict)
+    return localdict
+
+
+_marker = object()
+
+def _local_get_dict(self):
+    impl = self._local__impl
+    # Cython can optimize dict[], but not dict.get()
+    greenlet = getcurrent() # pylint:disable=undefined-variable
+    idg = id(greenlet)
     try:
-        dct = impl.get_dict()
+        entry = impl.dicts[idg]
+        dct = entry.localdict
     except KeyError:
-        # it's OK to acquire the lock here and not earlier, because the above code won't switch out
-        # however, subclassed __init__ might switch, so we do need to acquire the lock here
-        dct = impl.create_dict()
-        args, kw = impl.localargs
-        with impl.locallock:
-            self.__init__(*args, **kw)
-    with impl.locallock:
-        object.__setattr__(self, '__dict__', dct)
-        yield
-        object.__setattr__(self, '__dict__', orig_dct)
-
+        dct = _localimpl_create_dict(impl, greenlet, idg)
+        self.__init__(*impl.localargs, **impl.localkwargs)
+    return dct
+
+def _init():
+    greenlet_init() # pylint:disable=undefined-variable
+
+_local_attrs = {
+    '_local__impl',
+    '_local_type_get_descriptors',
+    '_local_type_set_or_del_descriptors',
+    '_local_type_del_descriptors',
+    '_local_type_set_descriptors',
+    '_local_type',
+    '_local_type_vars',
+    '__class__',
+    '__cinit__',
+}
 
 class local(object):
     """
     An object whose attributes are greenlet-local.
     """
-    __slots__ = '_local__impl', '__dict__'
+    __slots__ = tuple(_local_attrs - {'__class__', '__cinit__'})
 
-    def __new__(cls, *args, **kw):
+    def __cinit__(self, *args, **kw):
         if args or kw:
-            if (PYPY and cls.__init__ == object.__init__) or (not PYPY and cls.__init__ is object.__init__):
-                raise TypeError("Initialization arguments are not supported")
-        self = object.__new__(cls)
-        impl = _localimpl()
-        impl.localargs = (args, kw)
-        impl.locallock = RLock()
-        object.__setattr__(self, '_local__impl', impl)
-        # We need to create the thread dict in anticipation of
-        # __init__ being called, to make sure we don't call it
-        # again ourselves.
-        impl.create_dict()
-        return self
-
-    def __getattribute__(self, name):
-        with _patch(self):
+            if type(self).__init__ == object.__init__:
+                raise TypeError("Initialization arguments are not supported", args, kw)
+        impl = _localimpl(args, kw, type(self), id(self))
+        # pylint:disable=attribute-defined-outside-init
+        self._local__impl = impl
+        get, dels, sets_or_dels, sets = _local_find_descriptors(self)
+        self._local_type_get_descriptors = get
+        self._local_type_set_or_del_descriptors = sets_or_dels
+        self._local_type_del_descriptors = dels
+        self._local_type_set_descriptors = sets
+        self._local_type = type(self)
+        self._local_type_vars = set(dir(self._local_type))
+
+    def __getattribute__(self, name): # pylint:disable=too-many-return-statements
+        if name in _local_attrs:
+            # The _local__impl,  __cinit__, etc, won't be hit by the
+            # Cython version, if we've done things right. If we haven't,
+            # they will be, and this will produce an error.
             return object.__getattribute__(self, name)
 
+        dct = _local_get_dict(self)
+
+        if name == '__dict__':
+            return dct
+        # If there's no possible way we can switch, because this
+        # attribute is *not* found in the class where it might be a
+        # data descriptor (property), and it *is* in the dict
+        # then we don't need to swizzle the dict and take the lock.
+
+        # We don't have to worry about people overriding __getattribute__
+        # because if they did, the dict-swizzling would only last as
+        # long as we were in here anyway.
+        # Similarly, a __getattr__ will still be called by _oga() if needed
+        # if it's not in the dict.
+
+        # Optimization: If we're not subclassed, then
+        # there can be no descriptors except for methods, which will
+        # never need to use __dict__.
+        if self._local_type is local:
+            return dct[name] if name in dct else object.__getattribute__(self, name)
+
+        # NOTE: If this is a descriptor, this will invoke its __get__.
+        # A broken descriptor that doesn't return itself when called with
+        # a None for the instance argument could mess us up here.
+        # But this is faster than a loop over mro() checking each class __dict__
+        # manually.
+        if name in dct:
+            if name not in self._local_type_vars:
+                # If there is a dict value, and nothing in the type,
+                # it can't possibly be a descriptor, so it is just returned.
+                return dct[name]
+
+            # It's in the type *and* in the dict. If the type value is
+            # a data descriptor (defines __get__ *and* either __set__ or
+            # __delete__), then the type wins. If it's a non-data descriptor
+            # (defines just __get__), then the instance wins. If it's not a
+            # descriptor at all (doesn't have __get__), the instance wins.
+            # NOTE that the docs for descriptors say that these methods must be
+            # defined on the *class* of the object in the type.
+            if name not in self._local_type_get_descriptors:
+                # Entirely not a descriptor. Instance wins.
+                return dct[name]
+            if name in self._local_type_set_or_del_descriptors:
+                # A data descriptor.
+                # arbitrary code execution while these run. If they touch self again,
+                # they'll call back into us and we'll repeat the dance.
+                type_attr = getattr(self._local_type, name)
+                return type(type_attr).__get__(type_attr, self, self._local_type)
+            # Last case is a non-data descriptor. Instance wins.
+            return dct[name]
+
+        if name in self._local_type_vars:
+            # Not in the dictionary, but is found in the type. It could be
+            # a non-data descriptor still. Some descriptors, like @staticmethod,
+            # return objects (functions, in this case), that are *themselves*
+            # descriptors, which when invoked, again, would do the wrong thing.
+            # So we can't rely on getattr() on the type for them, we have to
+            # look through the MRO dicts ourself.
+            if name not in self._local_type_get_descriptors:
+                # Not a descriptor, can't execute code. So all we need is
+                # the return value of getattr() on our type.
+                return getattr(self._local_type, name)
+
+            for base in self._local_type.mro():
+                bd = base.__dict__
+                if name in bd:
+                    attr_on_type = bd[name]
+                    result = type(attr_on_type).__get__(attr_on_type, self, self._local_type)
+                    return result
+
+        # It wasn't in the dict and it wasn't in the type.
+        # So the next step is to invoke type(self)__getattr__, if it
+        # exists, otherwise raise an AttributeError.
+        # we will invoke type(self).__getattr__ or raise an attribute error.
+        if hasattr(self._local_type, '__getattr__'):
+            return self._local_type.__getattr__(self, name)
+        raise AttributeError("%r object has no attribute '%s'"
+                             % (self._local_type.__name__, name))
+
     def __setattr__(self, name, value):
         if name == '__dict__':
             raise AttributeError(
                 "%r object attribute '__dict__' is read-only"
-                % self.__class__.__name__)
-        with _patch(self):
-            return object.__setattr__(self, name, value)
+                % type(self))
+
+        if name in _local_attrs:
+            object.__setattr__(self, name, value)
+            return
+
+        dct = _local_get_dict(self)
+
+        if self._local_type is local:
+            # Optimization: If we're not subclassed, we can't
+            # have data descriptors, so this goes right in the dict.
+            dct[name] = value
+            return
+
+        if name in self._local_type_vars:
+            if name in self._local_type_set_descriptors:
+                type_attr = getattr(self._local_type, name, _marker)
+                # A data descriptor, like a property or a slot.
+                type(type_attr).__set__(type_attr, self, value)
+                return
+        # Otherwise it goes directly in the dict
+        dct[name] = value
 
     def __delattr__(self, name):
         if name == '__dict__':
             raise AttributeError(
                 "%r object attribute '__dict__' is read-only"
                 % self.__class__.__name__)
-        with _patch(self):
-            return object.__delattr__(self, name)
 
-    def __copy__(self):
-        impl = object.__getattribute__(self, '_local__impl')
-        current = getcurrent()
-        currentId = id(current)
-        d = impl.get_dict()
-        duplicate = copy(d)
+        if name in self._local_type_vars:
+            if name in self._local_type_del_descriptors:
+                # A data descriptor, like a property or a slot.
+                type_attr = getattr(self._local_type, name, _marker)
+                type(type_attr).__delete__(type_attr, self)
+                return
+        # Otherwise it goes directly in the dict
 
-        cls = type(self)
-        if (PYPY and cls.__init__ != object.__init__) or (not PYPY and cls.__init__ is not object.__init__):
-            args, kw = impl.localargs
-            instance = cls(*args, **kw)
-        else:
-            instance = cls()
+        # Begin inlined function _get_dict()
+        dct = _local_get_dict(self)
 
-        new_impl = object.__getattribute__(instance, '_local__impl')
-        tpl = new_impl.dicts[currentId]
-        new_impl.dicts[currentId] = (tpl[0], duplicate)
+        try:
+            del dct[name]
+        except KeyError:
+            raise AttributeError(name)
+
+    def __copy__(self):
+        impl = self._local__impl
+        entry = impl.dicts[id(getcurrent())]  # pylint:disable=undefined-variable
+
+        dct = entry.localdict
+        duplicate = copy(dct)
 
+        cls = type(self)
+        instance = cls(*impl.localargs, **impl.localkwargs)
+        _local__copy_dict_from(instance, impl, duplicate)
         return instance
+
+def _local__copy_dict_from(self, impl, duplicate):
+    current = getcurrent() # pylint:disable=undefined-variable
+    currentId = id(current)
+    new_impl = self._local__impl
+    assert new_impl is not impl
+    entry = new_impl.dicts[currentId]
+    new_impl.dicts[currentId] = _localimpl_dict_entry(entry.wrgreenlet, duplicate)
+
+def _local_find_descriptors(self):
+    type_self = type(self)
+    gets = set()
+    dels = set()
+    set_or_del = set()
+    sets = set()
+    mro = list(type_self.mro())
+
+    for attr_name in dir(type_self):
+        # Conventionally, descriptors when called on a class
+        # return themself, but not all do. Notable exceptions are
+        # in the zope.interface package, where things like __provides__
+        # return other class attributes. So we can't use getattr, and instead
+        # walk up the dicts
+        for base in mro:
+            bd = base.__dict__
+            if attr_name in bd:
+                attr = bd[attr_name]
+                break
+        else:
+            raise AttributeError(attr_name)
+
+        type_attr = type(attr)
+        if hasattr(type_attr, '__get__'):
+            gets.add(attr_name)
+        if hasattr(type_attr, '__delete__'):
+            dels.add(attr_name)
+            set_or_del.add(attr_name)
+        if hasattr(type_attr, '__set__'):
+            sets.add(attr_name)
+
+    return (gets, dels, set_or_del, sets)
+
+# Cython doesn't let us use __new__, it requires
+# __cinit__. But we need __new__ if we're not compiled
+# (e.g., on PyPy). So we set it at runtime. Cython
+# will raise an error if we're compiled.
+def __new__(cls, *args, **kw):
+    self = super(local, cls).__new__(cls)
+    # We get the cls in *args for some reason
+    # too when we do it this way....except on PyPy3, which does
+    # not *unless* it's wrapped in a classmethod (which it is)
+    self.__cinit__(*args[1:], **kw)
+    return self
+
+try:
+    # PyPy2/3 and CPython handle adding a __new__ to the class
+    # in different ways. In CPython and PyPy3, it must be wrapped with classmethod;
+    # in PyPy2, it must not. In either case, the args that get passed to
+    # it are stil wrong.
+    local.__new__ = 'None'
+except TypeError: # pragma: no cover
+    # Must be compiled
+    pass
+else:
+    from gevent._compat import PYPY
+    from gevent._compat import PY2
+    if PYPY and PY2:
+        local.__new__ = __new__
+    else:
+        local.__new__ = classmethod(__new__)
+
+    del PYPY
+    del PY2
+
+_init()
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent._local')