diff options
Diffstat (limited to 'python/gevent/util.py')
| -rw-r--r-- | python/gevent/util.py | 536 |
1 files changed, 534 insertions, 2 deletions
diff --git a/python/gevent/util.py b/python/gevent/util.py index 1438688..f15bd07 100644 --- a/python/gevent/util.py +++ b/python/gevent/util.py @@ -3,12 +3,39 @@ Low-level utilities. """ -from __future__ import absolute_import +from __future__ import absolute_import, print_function, division import functools +import gc +import pprint +import sys +import traceback -__all__ = ['wrap_errors'] +from greenlet import getcurrent +from greenlet import greenlet as RawGreenlet +from gevent._compat import PYPY +from gevent._compat import thread_mod_name +from gevent._util import _NONE + +__all__ = [ + 'format_run_info', + 'print_run_info', + 'GreenletTree', + 'wrap_errors', + 'assert_switches', +] + +# PyPy is very slow at formatting stacks +# for some reason. +_STACK_LIMIT = 20 if PYPY else None + + +def _noop(): + return None + +def _ready(): + return False class wrap_errors(object): """ @@ -58,3 +85,508 @@ class wrap_errors(object): def __getattr__(self, name): return getattr(self.__func, name) + + +def print_run_info(thread_stacks=True, greenlet_stacks=True, limit=_NONE, file=None): + """ + Call `format_run_info` and print the results to *file*. + + If *file* is not given, `sys.stderr` will be used. + + .. versionadded:: 1.3b1 + """ + lines = format_run_info(thread_stacks=thread_stacks, + greenlet_stacks=greenlet_stacks, + limit=limit) + file = sys.stderr if file is None else file + for l in lines: + print(l, file=file) + + +def format_run_info(thread_stacks=True, + greenlet_stacks=True, + limit=_NONE, + current_thread_ident=None): + """ + format_run_info(thread_stacks=True, greenlet_stacks=True, limit=None) -> [str] + + Request information about the running threads of the current process. + + This is a debugging utility. Its output has no guarantees other than being + intended for human consumption. + + :keyword bool thread_stacks: If true, then include the stacks for + running threads. + :keyword bool greenlet_stacks: If true, then include the stacks for + running greenlets. (Spawning stacks will always be printed.) + Setting this to False can reduce the output volume considerably + without reducing the overall information if *thread_stacks* is true + and you can associate a greenlet to a thread (using ``thread_ident`` + printed values). + :keyword int limit: If given, passed directly to `traceback.format_stack`. + If not given, this defaults to the whole stack under CPython, and a + smaller stack under PyPy. + + :return: A sequence of text lines detailing the stacks of running + threads and greenlets. (One greenlet will duplicate one thread, + the current thread and greenlet. If there are multiple running threads, + the stack for the current greenlet may be incorrectly duplicated in multiple + greenlets.) + Extra information about + :class:`gevent.Greenlet` object will also be returned. + + .. versionadded:: 1.3a1 + .. versionchanged:: 1.3a2 + Renamed from ``dump_stacks`` to reflect the fact that this + prints additional information about greenlets, including their + spawning stack, parent, locals, and any spawn tree locals. + .. versionchanged:: 1.3b1 + Added the *thread_stacks*, *greenlet_stacks*, and *limit* params. + """ + if current_thread_ident is None: + from gevent import monkey + current_thread_ident = monkey.get_original(thread_mod_name, 'get_ident')() + + lines = [] + + limit = _STACK_LIMIT if limit is _NONE else limit + _format_thread_info(lines, thread_stacks, limit, current_thread_ident) + _format_greenlet_info(lines, greenlet_stacks, limit) + return lines + + +def _format_thread_info(lines, thread_stacks, limit, current_thread_ident): + import threading + + threads = {th.ident: th for th in threading.enumerate()} + + lines.append('*' * 80) + lines.append('* Threads') + + thread = None + frame = None + for thread_ident, frame in sys._current_frames().items(): + lines.append("*" * 80) + thread = threads.get(thread_ident) + name = thread.name if thread else None + if getattr(thread, 'gevent_monitoring_thread', None): + name = repr(thread.gevent_monitoring_thread()) + if current_thread_ident == thread_ident: + name = '%s) (CURRENT' % (name,) + lines.append('Thread 0x%x (%s)\n' % (thread_ident, name)) + if thread_stacks: + lines.append(''.join(traceback.format_stack(frame, limit))) + else: + lines.append('\t...stack elided...') + + # We may have captured our own frame, creating a reference + # cycle, so clear it out. + del thread + del frame + del lines + del threads + +def _format_greenlet_info(lines, greenlet_stacks, limit): + # Use the gc module to inspect all objects to find the greenlets + # since there isn't a global registry + lines.append('*' * 80) + lines.append('* Greenlets') + lines.append('*' * 80) + for tree in GreenletTree.forest(): + lines.extend(tree.format_lines(details={ + 'running_stacks': greenlet_stacks, + 'running_stack_limit': limit, + })) + + del lines + +dump_stacks = format_run_info + +def _line(f): + @functools.wraps(f) + def w(self, *args, **kwargs): + r = f(self, *args, **kwargs) + self.lines.append(r) + + return w + +class _TreeFormatter(object): + UP_AND_RIGHT = '+' + HORIZONTAL = '-' + VERTICAL = '|' + VERTICAL_AND_RIGHT = '+' + DATA = ':' + + label_space = 1 + horiz_width = 3 + indent = 1 + + def __init__(self, details, depth=0): + self.lines = [] + self.depth = depth + self.details = details + if not details: + self.child_data = lambda *args, **kwargs: None + + def deeper(self): + return type(self)(self.details, self.depth + 1) + + @_line + def node_label(self, text): + return text + + @_line + def child_head(self, label, right=VERTICAL_AND_RIGHT): + return ( + ' ' * self.indent + + right + + self.HORIZONTAL * self.horiz_width + + ' ' * self.label_space + + label + ) + + def last_child_head(self, label): + return self.child_head(label, self.UP_AND_RIGHT) + + @_line + def child_tail(self, line, vertical=VERTICAL): + return ( + ' ' * self.indent + + vertical + + ' ' * self.horiz_width + + line + ) + + def last_child_tail(self, line): + return self.child_tail(line, vertical=' ' * len(self.VERTICAL)) + + @_line + def child_data(self, data, data_marker=DATA): # pylint:disable=method-hidden + return (( + ' ' * self.indent + + (data_marker if not self.depth else ' ') + + ' ' * self.horiz_width + + ' ' * self.label_space + + data + ),) + + def last_child_data(self, data): + return self.child_data(data, ' ') + + def child_multidata(self, data): + # Remove embedded newlines + for l in data.splitlines(): + self.child_data(l) + + +class GreenletTree(object): + """ + Represents a tree of greenlets. + + In gevent, the *parent* of a greenlet is usually the hub, so this + tree is primarily arganized along the *spawning_greenlet* dimension. + + This object has a small str form showing this hierarchy. The `format` + method can output more details. The exact output is unspecified but is + intended to be human readable. + + Use the `forest` method to get the root greenlet trees for + all threads, and the `current_tree` to get the root greenlet tree for + the current thread. + """ + + #: The greenlet this tree represents. + greenlet = None + + + def __init__(self, greenlet): + self.greenlet = greenlet + self.child_trees = [] + + def add_child(self, tree): + if tree is self: + return + self.child_trees.append(tree) + + @property + def root(self): + return self.greenlet.parent is None + + def __getattr__(self, name): + return getattr(self.greenlet, name) + + DEFAULT_DETAILS = { + 'running_stacks': True, + 'running_stack_limit': _STACK_LIMIT, + 'spawning_stacks': True, + 'locals': True, + } + + def format_lines(self, details=True): + """ + Return a sequence of lines for the greenlet tree. + + :keyword bool details: If true (the default), + then include more informative details in the output. + """ + if not isinstance(details, dict): + if not details: + details = {} + else: + details = self.DEFAULT_DETAILS.copy() + else: + params = details + details = self.DEFAULT_DETAILS.copy() + details.update(params) + tree = _TreeFormatter(details, depth=0) + lines = [l[0] if isinstance(l, tuple) else l + for l in self._render(tree)] + return lines + + def format(self, details=True): + """ + Like `format_lines` but returns a string. + """ + lines = self.format_lines(details) + return '\n'.join(lines) + + def __str__(self): + return self.format(False) + + @staticmethod + def __render_tb(tree, label, frame, limit): + tree.child_data(label) + tb = ''.join(traceback.format_stack(frame, limit)) + tree.child_multidata(tb) + + @staticmethod + def __spawning_parent(greenlet): + return (getattr(greenlet, 'spawning_greenlet', None) or _noop)() + + def __render_locals(self, tree): + # Defer the import to avoid cycles + from gevent.local import all_local_dicts_for_greenlet + + gr_locals = all_local_dicts_for_greenlet(self.greenlet) + if gr_locals: + tree.child_data("Greenlet Locals:") + for (kind, idl), vals in gr_locals: + tree.child_data(" Local %s at %s" % (kind, hex(idl))) + tree.child_multidata(" " + pprint.pformat(vals)) + + def _render(self, tree): + label = repr(self.greenlet) + if not self.greenlet: # Not running or dead + # raw greenlets do not have ready + if getattr(self.greenlet, 'ready', _ready)(): + label += '; finished' + if self.greenlet.value is not None: + label += ' with value ' + repr(self.greenlet.value)[:30] + elif getattr(self.greenlet, 'exception', None) is not None: + label += ' with exception ' + repr(self.greenlet.exception) + else: + label += '; not running' + tree.node_label(label) + + tree.child_data('Parent: ' + repr(self.greenlet.parent)) + + if getattr(self.greenlet, 'gevent_monitoring_thread', None) is not None: + tree.child_data('Monitoring Thread:' + repr(self.greenlet.gevent_monitoring_thread())) + + if self.greenlet and tree.details and tree.details['running_stacks']: + self.__render_tb(tree, 'Running:', self.greenlet.gr_frame, + tree.details['running_stack_limit']) + + + spawning_stack = getattr(self.greenlet, 'spawning_stack', None) + if spawning_stack and tree.details and tree.details['spawning_stacks']: + # We already placed a limit on the spawning stack when we captured it. + self.__render_tb(tree, 'Spawned at:', spawning_stack, None) + + spawning_parent = self.__spawning_parent(self.greenlet) + tree_locals = getattr(self.greenlet, 'spawn_tree_locals', None) + if tree_locals and tree_locals is not getattr(spawning_parent, 'spawn_tree_locals', None): + tree.child_data('Spawn Tree Locals') + tree.child_multidata(pprint.pformat(tree_locals)) + + self.__render_locals(tree) + self.__render_children(tree) + return tree.lines + + def __render_children(self, tree): + children = sorted(self.child_trees, + key=lambda c: ( + # raw greenlets first + getattr(c, 'minimal_ident', -1), + # running greenlets first + getattr(c, 'ready', _ready)(), + id(c.parent))) + for n, child in enumerate(children): + child_tree = child._render(tree.deeper()) + + head = tree.child_head + tail = tree.child_tail + data = tree.child_data + + if n == len(children) - 1: + # last child does not get the line drawn + head = tree.last_child_head + tail = tree.last_child_tail + data = tree.last_child_data + + head(child_tree.pop(0)) + for child_data in child_tree: + if isinstance(child_data, tuple): + data(child_data[0]) + else: + tail(child_data) + + return tree.lines + + + @staticmethod + def _root_greenlet(greenlet): + while greenlet.parent is not None and not getattr(greenlet, 'greenlet_tree_is_root', False): + greenlet = greenlet.parent + return greenlet + + @classmethod + def _forest(cls): + main_greenlet = cls._root_greenlet(getcurrent()) + + trees = {} + roots = {} + current_tree = roots[main_greenlet] = trees[main_greenlet] = cls(main_greenlet) + + + + for ob in gc.get_objects(): + if not isinstance(ob, RawGreenlet): + continue + if getattr(ob, 'greenlet_tree_is_ignored', False): + continue + + spawn_parent = cls.__spawning_parent(ob) + + if spawn_parent is None: + root = cls._root_greenlet(ob) + try: + tree = roots[root] + except KeyError: # pragma: no cover + tree = GreenletTree(root) + roots[root] = trees[root] = tree + else: + try: + tree = trees[spawn_parent] + except KeyError: # pragma: no cover + tree = trees[spawn_parent] = cls(spawn_parent) + + try: + child_tree = trees[ob] + except KeyError: + trees[ob] = child_tree = cls(ob) + tree.add_child(child_tree) + + return roots, current_tree + + @classmethod + def forest(cls): + """ + forest() -> sequence + + Return a sequence of `GreenletTree`, one for each running + native thread. + """ + + return list(cls._forest()[0].values()) + + @classmethod + def current_tree(cls): + """ + current_tree() -> GreenletTree + + Returns the `GreenletTree` for the current thread. + """ + return cls._forest()[1] + +class _FailedToSwitch(AssertionError): + pass + +class assert_switches(object): + """ + A context manager for ensuring a block of code switches greenlets. + + This performs a similar function as the :doc:`monitoring thread + </monitoring>`, but the scope is limited to the body of the with + statement. If the code within the body doesn't yield to the hub + (and doesn't raise an exception), then upon exiting the + context manager an :exc:`AssertionError` will be raised. + + This is useful in unit tests and for debugging purposes. + + :keyword float max_blocking_time: If given, the body is allowed + to block for up to this many fractional seconds before + an error is raised. + :keyword bool hub_only: If True, then *max_blocking_time* only + refers to the amount of time spent between switches into the + hub. If False, then it refers to the maximum time between + *any* switches. If *max_blocking_time* is not given, has no + effect. + + Example:: + + # This will always raise an exception: nothing switched + with assert_switches(): + pass + + # This will never raise an exception; nothing switched, + # but it happened very fast + with assert_switches(max_blocking_time=1.0): + pass + + .. versionadded:: 1.3 + """ + + hub = None + tracer = None + + + def __init__(self, max_blocking_time=None, hub_only=False): + self.max_blocking_time = max_blocking_time + self.hub_only = hub_only + + def __enter__(self): + from gevent import get_hub + from gevent import _tracer + + self.hub = hub = get_hub() + + # TODO: We could optimize this to use the GreenletTracer + # installed by the monitoring thread, if there is one. + # As it is, we will chain trace calls back to it. + if not self.max_blocking_time: + self.tracer = _tracer.GreenletTracer() + elif self.hub_only: + self.tracer = _tracer.HubSwitchTracer(hub, self.max_blocking_time) + else: + self.tracer = _tracer.MaxSwitchTracer(hub, self.max_blocking_time) + + self.tracer.monitor_current_greenlet_blocking() + return self + + def __exit__(self, t, v, tb): + self.tracer.kill() + hub = self.hub; self.hub = None + tracer = self.tracer; self.tracer = None + + # Only check if there was no exception raised, we + # don't want to hide anything + if t is not None: + return + + + did_block = tracer.did_block_hub(hub) + if did_block: + active_greenlet = did_block[1] + report_lines = tracer.did_block_hub_report(hub, active_greenlet, {}) + raise _FailedToSwitch('\n'.join(report_lines)) |