diff options
Diffstat (limited to 'python/gevent/local.py')
| -rw-r--r-- | python/gevent/local.py | 538 |
1 files changed, 425 insertions, 113 deletions
diff --git a/python/gevent/local.py b/python/gevent/local.py index 955c320..f29bb23 100644 --- a/python/gevent/local.py +++ b/python/gevent/local.py @@ -1,3 +1,4 @@ +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False """ Greenlet-local objects. @@ -133,161 +134,472 @@ affects what we see: This results in locals being eligible for garbage collection as soon as their greenlet exits. +.. versionchanged:: 1.2.3 + Use a weak-reference to clear the greenlet link we establish in case + the local object dies before the greenlet does. + +.. versionchanged:: 1.3a1 + Implement the methods for attribute access directly, handling + descriptors directly here. This allows removing the use of a lock + and facilitates greatly improved performance. + +.. versionchanged:: 1.3a1 + The ``__init__`` method of subclasses of ``local`` is no longer + called with a lock held. CPython does not use such a lock in its + native implementation. This could potentially show as a difference + if code that uses multiple dependent attributes in ``__slots__`` + (which are shared across all greenlets) switches during ``__init__``. + """ +from __future__ import print_function from copy import copy from weakref import ref -from contextlib import contextmanager -from gevent.hub import getcurrent -from gevent._compat import PYPY -from gevent.lock import RLock -__all__ = ["local"] + +locals()['getcurrent'] = __import__('greenlet').getcurrent +locals()['greenlet_init'] = lambda: None + +__all__ = [ + "local", +] + +# The key used in the Thread objects' attribute dicts. +# We keep it a string for speed but make it unlikely to clash with +# a "real" attribute. +key_prefix = '_gevent_local_localimpl_' + +# The overall structure is as follows: +# For each local() object: +# greenlet.__dict__[key_prefix + str(id(local))] +# => _localimpl.dicts[id(greenlet)] => (ref(greenlet), {}) + +# That final tuple is actually a localimpl_dict_entry object. + +def all_local_dicts_for_greenlet(greenlet): + """ + Internal debug helper for getting the local values associated + with a greenlet. This is subject to change or removal at any time. + + :return: A list of ((type, id), {}) pairs, where the first element + is the type and id of the local object and the second object is its + instance dictionary, as seen from this greenlet. + + .. versionadded:: 1.3a2 + """ + + result = [] + id_greenlet = id(greenlet) + greenlet_dict = greenlet.__dict__ + for k, v in greenlet_dict.items(): + if not k.startswith(key_prefix): + continue + local_impl = v() + if local_impl is None: + continue + entry = local_impl.dicts.get(id_greenlet) + if entry is None: + # Not yet used in this greenlet. + continue + assert entry.wrgreenlet() is greenlet + result.append((local_impl.localtypeid, entry.localdict)) + + return result class _wrefdict(dict): """A dict that can be weak referenced""" +class _greenlet_deleted(object): + """ + A weakref callback for when the greenlet + is deleted. + + If the greenlet is a `gevent.greenlet.Greenlet` and + supplies ``rawlink``, that will be used instead of a + weakref. + """ + __slots__ = ('idt', 'wrdicts') + + def __init__(self, idt, wrdicts): + self.idt = idt + self.wrdicts = wrdicts + + def __call__(self, _unused): + dicts = self.wrdicts() + if dicts: + dicts.pop(self.idt, None) + +class _local_deleted(object): + __slots__ = ('key', 'wrthread', 'greenlet_deleted') + + def __init__(self, key, wrthread, greenlet_deleted): + self.key = key + self.wrthread = wrthread + self.greenlet_deleted = greenlet_deleted + + def __call__(self, _unused): + thread = self.wrthread() + if thread is not None: + try: + unlink = thread.unlink + except AttributeError: + pass + else: + unlink(self.greenlet_deleted) + del thread.__dict__[self.key] class _localimpl(object): """A class managing thread-local dicts""" - __slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__' - - def __init__(self): - # The key used in the Thread objects' attribute dicts. - # We keep it a string for speed but make it unlikely to clash with - # a "real" attribute. - self.key = '_threading_local._localimpl.' + str(id(self)) - # { id(Thread) -> (ref(Thread), thread-local dict) } + __slots__ = ('key', 'dicts', + 'localargs', 'localkwargs', + 'localtypeid', + '__weakref__',) + + def __init__(self, args, kwargs, local_type, id_local): + self.key = key_prefix + str(id(self)) + # { id(greenlet) -> _localimpl_dict_entry(ref(greenlet), greenlet-local dict) } self.dicts = _wrefdict() + self.localargs = args + self.localkwargs = kwargs + self.localtypeid = local_type, id_local - def get_dict(self): - """Return the dict for the current thread. Raises KeyError if none - defined.""" - thread = getcurrent() - return self.dicts[id(thread)][1] - - def create_dict(self): - """Create a new dict for the current thread, and return it.""" - localdict = {} - key = self.key - thread = getcurrent() - idt = id(thread) - - # If we are working with a gevent.greenlet.Greenlet, we can - # pro-actively clear out with a link. Use rawlink to avoid - # spawning any more greenlets - try: - rawlink = thread.rawlink - except AttributeError: - # Otherwise we need to do it with weak refs - def local_deleted(_, key=key): - # When the localimpl is deleted, remove the thread attribute. - thread = wrthread() - if thread is not None: - del thread.__dict__[key] - - def thread_deleted(_, idt=idt): - # When the thread is deleted, remove the local dict. - # Note that this is suboptimal if the thread object gets - # caught in a reference loop. We would like to be called - # as soon as the OS-level thread ends instead. - _local = wrlocal() - if _local is not None: - _local.dicts.pop(idt, None) - wrlocal = ref(self, local_deleted) - wrthread = ref(thread, thread_deleted) - thread.__dict__[key] = wrlocal - else: - wrdicts = ref(self.dicts) + # We need to create the thread dict in anticipation of + # __init__ being called, to make sure we don't call it + # again ourselves. MUST do this before setting any attributes. + greenlet = getcurrent() # pylint:disable=undefined-variable + _localimpl_create_dict(self, greenlet, id(greenlet)) + +class _localimpl_dict_entry(object): + """ + The object that goes in the ``dicts`` of ``_localimpl`` + object for each thread. + """ + # This is a class, not just a tuple, so that cython can optimize + # attribute access + __slots__ = ('wrgreenlet', 'localdict') + + def __init__(self, wrgreenlet, localdict): + self.wrgreenlet = wrgreenlet + self.localdict = localdict + +# We use functions instead of methods so that they can be cdef'd in +# local.pxd; if they were cdef'd as methods, they would cause +# the creation of a pointer and a vtable. This happens +# even if we declare the class @cython.final. functions thus save memory overhead +# (but not pointer chasing overhead; the vtable isn't used when we declare +# the class final). + + +def _localimpl_create_dict(self, greenlet, id_greenlet): + """Create a new dict for the current thread, and return it.""" + localdict = {} + key = self.key + + wrdicts = ref(self.dicts) - def clear(_): - dicts = wrdicts() - if dicts: - dicts.pop(idt, None) - rawlink(clear) - wrthread = None + # When the greenlet is deleted, remove the local dict. + # Note that this is suboptimal if the greenlet object gets + # caught in a reference loop. We would like to be called + # as soon as the OS-level greenlet ends instead. - self.dicts[idt] = wrthread, localdict - return localdict + # If we are working with a gevent.greenlet.Greenlet, we + # can pro-actively clear out with a link, avoiding the + # issue described above. Use rawlink to avoid spawning any + # more greenlets. + greenlet_deleted = _greenlet_deleted(id_greenlet, wrdicts) + rawlink = getattr(greenlet, 'rawlink', None) + if rawlink is not None: + rawlink(greenlet_deleted) + wrthread = ref(greenlet) + else: + wrthread = ref(greenlet, greenlet_deleted) -@contextmanager -def _patch(self): - impl = object.__getattribute__(self, '_local__impl') - orig_dct = object.__getattribute__(self, '__dict__') + + # When the localimpl is deleted, remove the thread attribute. + local_deleted = _local_deleted(key, wrthread, greenlet_deleted) + + + wrlocal = ref(self, local_deleted) + greenlet.__dict__[key] = wrlocal + + self.dicts[id_greenlet] = _localimpl_dict_entry(wrthread, localdict) + return localdict + + +_marker = object() + +def _local_get_dict(self): + impl = self._local__impl + # Cython can optimize dict[], but not dict.get() + greenlet = getcurrent() # pylint:disable=undefined-variable + idg = id(greenlet) try: - dct = impl.get_dict() + entry = impl.dicts[idg] + dct = entry.localdict except KeyError: - # it's OK to acquire the lock here and not earlier, because the above code won't switch out - # however, subclassed __init__ might switch, so we do need to acquire the lock here - dct = impl.create_dict() - args, kw = impl.localargs - with impl.locallock: - self.__init__(*args, **kw) - with impl.locallock: - object.__setattr__(self, '__dict__', dct) - yield - object.__setattr__(self, '__dict__', orig_dct) - + dct = _localimpl_create_dict(impl, greenlet, idg) + self.__init__(*impl.localargs, **impl.localkwargs) + return dct + +def _init(): + greenlet_init() # pylint:disable=undefined-variable + +_local_attrs = { + '_local__impl', + '_local_type_get_descriptors', + '_local_type_set_or_del_descriptors', + '_local_type_del_descriptors', + '_local_type_set_descriptors', + '_local_type', + '_local_type_vars', + '__class__', + '__cinit__', +} class local(object): """ An object whose attributes are greenlet-local. """ - __slots__ = '_local__impl', '__dict__' + __slots__ = tuple(_local_attrs - {'__class__', '__cinit__'}) - def __new__(cls, *args, **kw): + def __cinit__(self, *args, **kw): if args or kw: - if (PYPY and cls.__init__ == object.__init__) or (not PYPY and cls.__init__ is object.__init__): - raise TypeError("Initialization arguments are not supported") - self = object.__new__(cls) - impl = _localimpl() - impl.localargs = (args, kw) - impl.locallock = RLock() - object.__setattr__(self, '_local__impl', impl) - # We need to create the thread dict in anticipation of - # __init__ being called, to make sure we don't call it - # again ourselves. - impl.create_dict() - return self - - def __getattribute__(self, name): - with _patch(self): + if type(self).__init__ == object.__init__: + raise TypeError("Initialization arguments are not supported", args, kw) + impl = _localimpl(args, kw, type(self), id(self)) + # pylint:disable=attribute-defined-outside-init + self._local__impl = impl + get, dels, sets_or_dels, sets = _local_find_descriptors(self) + self._local_type_get_descriptors = get + self._local_type_set_or_del_descriptors = sets_or_dels + self._local_type_del_descriptors = dels + self._local_type_set_descriptors = sets + self._local_type = type(self) + self._local_type_vars = set(dir(self._local_type)) + + def __getattribute__(self, name): # pylint:disable=too-many-return-statements + if name in _local_attrs: + # The _local__impl, __cinit__, etc, won't be hit by the + # Cython version, if we've done things right. If we haven't, + # they will be, and this will produce an error. return object.__getattribute__(self, name) + dct = _local_get_dict(self) + + if name == '__dict__': + return dct + # If there's no possible way we can switch, because this + # attribute is *not* found in the class where it might be a + # data descriptor (property), and it *is* in the dict + # then we don't need to swizzle the dict and take the lock. + + # We don't have to worry about people overriding __getattribute__ + # because if they did, the dict-swizzling would only last as + # long as we were in here anyway. + # Similarly, a __getattr__ will still be called by _oga() if needed + # if it's not in the dict. + + # Optimization: If we're not subclassed, then + # there can be no descriptors except for methods, which will + # never need to use __dict__. + if self._local_type is local: + return dct[name] if name in dct else object.__getattribute__(self, name) + + # NOTE: If this is a descriptor, this will invoke its __get__. + # A broken descriptor that doesn't return itself when called with + # a None for the instance argument could mess us up here. + # But this is faster than a loop over mro() checking each class __dict__ + # manually. + if name in dct: + if name not in self._local_type_vars: + # If there is a dict value, and nothing in the type, + # it can't possibly be a descriptor, so it is just returned. + return dct[name] + + # It's in the type *and* in the dict. If the type value is + # a data descriptor (defines __get__ *and* either __set__ or + # __delete__), then the type wins. If it's a non-data descriptor + # (defines just __get__), then the instance wins. If it's not a + # descriptor at all (doesn't have __get__), the instance wins. + # NOTE that the docs for descriptors say that these methods must be + # defined on the *class* of the object in the type. + if name not in self._local_type_get_descriptors: + # Entirely not a descriptor. Instance wins. + return dct[name] + if name in self._local_type_set_or_del_descriptors: + # A data descriptor. + # arbitrary code execution while these run. If they touch self again, + # they'll call back into us and we'll repeat the dance. + type_attr = getattr(self._local_type, name) + return type(type_attr).__get__(type_attr, self, self._local_type) + # Last case is a non-data descriptor. Instance wins. + return dct[name] + + if name in self._local_type_vars: + # Not in the dictionary, but is found in the type. It could be + # a non-data descriptor still. Some descriptors, like @staticmethod, + # return objects (functions, in this case), that are *themselves* + # descriptors, which when invoked, again, would do the wrong thing. + # So we can't rely on getattr() on the type for them, we have to + # look through the MRO dicts ourself. + if name not in self._local_type_get_descriptors: + # Not a descriptor, can't execute code. So all we need is + # the return value of getattr() on our type. + return getattr(self._local_type, name) + + for base in self._local_type.mro(): + bd = base.__dict__ + if name in bd: + attr_on_type = bd[name] + result = type(attr_on_type).__get__(attr_on_type, self, self._local_type) + return result + + # It wasn't in the dict and it wasn't in the type. + # So the next step is to invoke type(self)__getattr__, if it + # exists, otherwise raise an AttributeError. + # we will invoke type(self).__getattr__ or raise an attribute error. + if hasattr(self._local_type, '__getattr__'): + return self._local_type.__getattr__(self, name) + raise AttributeError("%r object has no attribute '%s'" + % (self._local_type.__name__, name)) + def __setattr__(self, name, value): if name == '__dict__': raise AttributeError( "%r object attribute '__dict__' is read-only" - % self.__class__.__name__) - with _patch(self): - return object.__setattr__(self, name, value) + % type(self)) + + if name in _local_attrs: + object.__setattr__(self, name, value) + return + + dct = _local_get_dict(self) + + if self._local_type is local: + # Optimization: If we're not subclassed, we can't + # have data descriptors, so this goes right in the dict. + dct[name] = value + return + + if name in self._local_type_vars: + if name in self._local_type_set_descriptors: + type_attr = getattr(self._local_type, name, _marker) + # A data descriptor, like a property or a slot. + type(type_attr).__set__(type_attr, self, value) + return + # Otherwise it goes directly in the dict + dct[name] = value def __delattr__(self, name): if name == '__dict__': raise AttributeError( "%r object attribute '__dict__' is read-only" % self.__class__.__name__) - with _patch(self): - return object.__delattr__(self, name) - def __copy__(self): - impl = object.__getattribute__(self, '_local__impl') - current = getcurrent() - currentId = id(current) - d = impl.get_dict() - duplicate = copy(d) + if name in self._local_type_vars: + if name in self._local_type_del_descriptors: + # A data descriptor, like a property or a slot. + type_attr = getattr(self._local_type, name, _marker) + type(type_attr).__delete__(type_attr, self) + return + # Otherwise it goes directly in the dict - cls = type(self) - if (PYPY and cls.__init__ != object.__init__) or (not PYPY and cls.__init__ is not object.__init__): - args, kw = impl.localargs - instance = cls(*args, **kw) - else: - instance = cls() + # Begin inlined function _get_dict() + dct = _local_get_dict(self) - new_impl = object.__getattribute__(instance, '_local__impl') - tpl = new_impl.dicts[currentId] - new_impl.dicts[currentId] = (tpl[0], duplicate) + try: + del dct[name] + except KeyError: + raise AttributeError(name) + + def __copy__(self): + impl = self._local__impl + entry = impl.dicts[id(getcurrent())] # pylint:disable=undefined-variable + + dct = entry.localdict + duplicate = copy(dct) + cls = type(self) + instance = cls(*impl.localargs, **impl.localkwargs) + _local__copy_dict_from(instance, impl, duplicate) return instance + +def _local__copy_dict_from(self, impl, duplicate): + current = getcurrent() # pylint:disable=undefined-variable + currentId = id(current) + new_impl = self._local__impl + assert new_impl is not impl + entry = new_impl.dicts[currentId] + new_impl.dicts[currentId] = _localimpl_dict_entry(entry.wrgreenlet, duplicate) + +def _local_find_descriptors(self): + type_self = type(self) + gets = set() + dels = set() + set_or_del = set() + sets = set() + mro = list(type_self.mro()) + + for attr_name in dir(type_self): + # Conventionally, descriptors when called on a class + # return themself, but not all do. Notable exceptions are + # in the zope.interface package, where things like __provides__ + # return other class attributes. So we can't use getattr, and instead + # walk up the dicts + for base in mro: + bd = base.__dict__ + if attr_name in bd: + attr = bd[attr_name] + break + else: + raise AttributeError(attr_name) + + type_attr = type(attr) + if hasattr(type_attr, '__get__'): + gets.add(attr_name) + if hasattr(type_attr, '__delete__'): + dels.add(attr_name) + set_or_del.add(attr_name) + if hasattr(type_attr, '__set__'): + sets.add(attr_name) + + return (gets, dels, set_or_del, sets) + +# Cython doesn't let us use __new__, it requires +# __cinit__. But we need __new__ if we're not compiled +# (e.g., on PyPy). So we set it at runtime. Cython +# will raise an error if we're compiled. +def __new__(cls, *args, **kw): + self = super(local, cls).__new__(cls) + # We get the cls in *args for some reason + # too when we do it this way....except on PyPy3, which does + # not *unless* it's wrapped in a classmethod (which it is) + self.__cinit__(*args[1:], **kw) + return self + +try: + # PyPy2/3 and CPython handle adding a __new__ to the class + # in different ways. In CPython and PyPy3, it must be wrapped with classmethod; + # in PyPy2, it must not. In either case, the args that get passed to + # it are stil wrong. + local.__new__ = 'None' +except TypeError: # pragma: no cover + # Must be compiled + pass +else: + from gevent._compat import PYPY + from gevent._compat import PY2 + if PYPY and PY2: + local.__new__ = __new__ + else: + local.__new__ = classmethod(__new__) + + del PYPY + del PY2 + +_init() + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent._local') |