diff options
Diffstat (limited to 'python/jinja2/ext.py')
| -rw-r--r-- | python/jinja2/ext.py | 627 |
1 files changed, 0 insertions, 627 deletions
diff --git a/python/jinja2/ext.py b/python/jinja2/ext.py deleted file mode 100644 index 0734a84..0000000 --- a/python/jinja2/ext.py +++ /dev/null @@ -1,627 +0,0 @@ -# -*- coding: utf-8 -*- -""" - jinja2.ext - ~~~~~~~~~~ - - Jinja extensions allow to add custom tags similar to the way django custom - tags work. By default two example extensions exist: an i18n and a cache - extension. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD. -""" -import re - -from jinja2 import nodes -from jinja2.defaults import BLOCK_START_STRING, \ - BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ - COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ - LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ - KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS -from jinja2.environment import Environment -from jinja2.runtime import concat -from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError -from jinja2.utils import contextfunction, import_string, Markup -from jinja2._compat import with_metaclass, string_types, iteritems - - -# the only real useful gettext functions for a Jinja template. Note -# that ugettext must be assigned to gettext as Jinja doesn't support -# non unicode strings. -GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext') - - -class ExtensionRegistry(type): - """Gives the extension an unique identifier.""" - - def __new__(cls, name, bases, d): - rv = type.__new__(cls, name, bases, d) - rv.identifier = rv.__module__ + '.' + rv.__name__ - return rv - - -class Extension(with_metaclass(ExtensionRegistry, object)): - """Extensions can be used to add extra functionality to the Jinja template - system at the parser level. Custom extensions are bound to an environment - but may not store environment specific data on `self`. The reason for - this is that an extension can be bound to another environment (for - overlays) by creating a copy and reassigning the `environment` attribute. - - As extensions are created by the environment they cannot accept any - arguments for configuration. One may want to work around that by using - a factory function, but that is not possible as extensions are identified - by their import name. The correct way to configure the extension is - storing the configuration values on the environment. Because this way the - environment ends up acting as central configuration storage the - attributes may clash which is why extensions have to ensure that the names - they choose for configuration are not too generic. ``prefix`` for example - is a terrible name, ``fragment_cache_prefix`` on the other hand is a good - name as includes the name of the extension (fragment cache). - """ - - #: if this extension parses this is the list of tags it's listening to. - tags = set() - - #: the priority of that extension. This is especially useful for - #: extensions that preprocess values. A lower value means higher - #: priority. - #: - #: .. versionadded:: 2.4 - priority = 100 - - def __init__(self, environment): - self.environment = environment - - def bind(self, environment): - """Create a copy of this extension bound to another environment.""" - rv = object.__new__(self.__class__) - rv.__dict__.update(self.__dict__) - rv.environment = environment - return rv - - def preprocess(self, source, name, filename=None): - """This method is called before the actual lexing and can be used to - preprocess the source. The `filename` is optional. The return value - must be the preprocessed source. - """ - return source - - def filter_stream(self, stream): - """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used - to filter tokens returned. This method has to return an iterable of - :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a - :class:`~jinja2.lexer.TokenStream`. - - In the `ext` folder of the Jinja2 source distribution there is a file - called `inlinegettext.py` which implements a filter that utilizes this - method. - """ - return stream - - def parse(self, parser): - """If any of the :attr:`tags` matched this method is called with the - parser as first argument. The token the parser stream is pointing at - is the name token that matched. This method has to return one or a - list of multiple nodes. - """ - raise NotImplementedError() - - def attr(self, name, lineno=None): - """Return an attribute node for the current extension. This is useful - to pass constants on extensions to generated template code. - - :: - - self.attr('_my_attribute', lineno=lineno) - """ - return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno) - - def call_method(self, name, args=None, kwargs=None, dyn_args=None, - dyn_kwargs=None, lineno=None): - """Call a method of the extension. This is a shortcut for - :meth:`attr` + :class:`jinja2.nodes.Call`. - """ - if args is None: - args = [] - if kwargs is None: - kwargs = [] - return nodes.Call(self.attr(name, lineno=lineno), args, kwargs, - dyn_args, dyn_kwargs, lineno=lineno) - - -@contextfunction -def _gettext_alias(__context, *args, **kwargs): - return __context.call(__context.resolve('gettext'), *args, **kwargs) - - -def _make_new_gettext(func): - @contextfunction - def gettext(__context, __string, **variables): - rv = __context.call(func, __string) - if __context.eval_ctx.autoescape: - rv = Markup(rv) - return rv % variables - return gettext - - -def _make_new_ngettext(func): - @contextfunction - def ngettext(__context, __singular, __plural, __num, **variables): - variables.setdefault('num', __num) - rv = __context.call(func, __singular, __plural, __num) - if __context.eval_ctx.autoescape: - rv = Markup(rv) - return rv % variables - return ngettext - - -class InternationalizationExtension(Extension): - """This extension adds gettext support to Jinja2.""" - tags = set(['trans']) - - # TODO: the i18n extension is currently reevaluating values in a few - # situations. Take this example: - # {% trans count=something() %}{{ count }} foo{% pluralize - # %}{{ count }} fooss{% endtrans %} - # something is called twice here. One time for the gettext value and - # the other time for the n-parameter of the ngettext function. - - def __init__(self, environment): - Extension.__init__(self, environment) - environment.globals['_'] = _gettext_alias - environment.extend( - install_gettext_translations=self._install, - install_null_translations=self._install_null, - install_gettext_callables=self._install_callables, - uninstall_gettext_translations=self._uninstall, - extract_translations=self._extract, - newstyle_gettext=False - ) - - def _install(self, translations, newstyle=None): - gettext = getattr(translations, 'ugettext', None) - if gettext is None: - gettext = translations.gettext - ngettext = getattr(translations, 'ungettext', None) - if ngettext is None: - ngettext = translations.ngettext - self._install_callables(gettext, ngettext, newstyle) - - def _install_null(self, newstyle=None): - self._install_callables( - lambda x: x, - lambda s, p, n: (n != 1 and (p,) or (s,))[0], - newstyle - ) - - def _install_callables(self, gettext, ngettext, newstyle=None): - if newstyle is not None: - self.environment.newstyle_gettext = newstyle - if self.environment.newstyle_gettext: - gettext = _make_new_gettext(gettext) - ngettext = _make_new_ngettext(ngettext) - self.environment.globals.update( - gettext=gettext, - ngettext=ngettext - ) - - def _uninstall(self, translations): - for key in 'gettext', 'ngettext': - self.environment.globals.pop(key, None) - - def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS): - if isinstance(source, string_types): - source = self.environment.parse(source) - return extract_from_ast(source, gettext_functions) - - def parse(self, parser): - """Parse a translatable tag.""" - lineno = next(parser.stream).lineno - num_called_num = False - - # find all the variables referenced. Additionally a variable can be - # defined in the body of the trans block too, but this is checked at - # a later state. - plural_expr = None - plural_expr_assignment = None - variables = {} - trimmed = None - while parser.stream.current.type != 'block_end': - if variables: - parser.stream.expect('comma') - - # skip colon for python compatibility - if parser.stream.skip_if('colon'): - break - - name = parser.stream.expect('name') - if name.value in variables: - parser.fail('translatable variable %r defined twice.' % - name.value, name.lineno, - exc=TemplateAssertionError) - - # expressions - if parser.stream.current.type == 'assign': - next(parser.stream) - variables[name.value] = var = parser.parse_expression() - elif trimmed is None and name.value in ('trimmed', 'notrimmed'): - trimmed = name.value == 'trimmed' - continue - else: - variables[name.value] = var = nodes.Name(name.value, 'load') - - if plural_expr is None: - if isinstance(var, nodes.Call): - plural_expr = nodes.Name('_trans', 'load') - variables[name.value] = plural_expr - plural_expr_assignment = nodes.Assign( - nodes.Name('_trans', 'store'), var) - else: - plural_expr = var - num_called_num = name.value == 'num' - - parser.stream.expect('block_end') - - plural = None - have_plural = False - referenced = set() - - # now parse until endtrans or pluralize - singular_names, singular = self._parse_block(parser, True) - if singular_names: - referenced.update(singular_names) - if plural_expr is None: - plural_expr = nodes.Name(singular_names[0], 'load') - num_called_num = singular_names[0] == 'num' - - # if we have a pluralize block, we parse that too - if parser.stream.current.test('name:pluralize'): - have_plural = True - next(parser.stream) - if parser.stream.current.type != 'block_end': - name = parser.stream.expect('name') - if name.value not in variables: - parser.fail('unknown variable %r for pluralization' % - name.value, name.lineno, - exc=TemplateAssertionError) - plural_expr = variables[name.value] - num_called_num = name.value == 'num' - parser.stream.expect('block_end') - plural_names, plural = self._parse_block(parser, False) - next(parser.stream) - referenced.update(plural_names) - else: - next(parser.stream) - - # register free names as simple name expressions - for var in referenced: - if var not in variables: - variables[var] = nodes.Name(var, 'load') - - if not have_plural: - plural_expr = None - elif plural_expr is None: - parser.fail('pluralize without variables', lineno) - - if trimmed is None: - trimmed = self.environment.policies['ext.i18n.trimmed'] - if trimmed: - singular = self._trim_whitespace(singular) - if plural: - plural = self._trim_whitespace(plural) - - node = self._make_node(singular, plural, variables, plural_expr, - bool(referenced), - num_called_num and have_plural) - node.set_lineno(lineno) - if plural_expr_assignment is not None: - return [plural_expr_assignment, node] - else: - return node - - def _trim_whitespace(self, string, _ws_re=re.compile(r'\s*\n\s*')): - return _ws_re.sub(' ', string.strip()) - - def _parse_block(self, parser, allow_pluralize): - """Parse until the next block tag with a given name.""" - referenced = [] - buf = [] - while 1: - if parser.stream.current.type == 'data': - buf.append(parser.stream.current.value.replace('%', '%%')) - next(parser.stream) - elif parser.stream.current.type == 'variable_begin': - next(parser.stream) - name = parser.stream.expect('name').value - referenced.append(name) - buf.append('%%(%s)s' % name) - parser.stream.expect('variable_end') - elif parser.stream.current.type == 'block_begin': - next(parser.stream) - if parser.stream.current.test('name:endtrans'): - break - elif parser.stream.current.test('name:pluralize'): - if allow_pluralize: - break - parser.fail('a translatable section can have only one ' - 'pluralize section') - parser.fail('control structures in translatable sections are ' - 'not allowed') - elif parser.stream.eos: - parser.fail('unclosed translation block') - else: - assert False, 'internal parser error' - - return referenced, concat(buf) - - def _make_node(self, singular, plural, variables, plural_expr, - vars_referenced, num_called_num): - """Generates a useful node from the data provided.""" - # no variables referenced? no need to escape for old style - # gettext invocations only if there are vars. - if not vars_referenced and not self.environment.newstyle_gettext: - singular = singular.replace('%%', '%') - if plural: - plural = plural.replace('%%', '%') - - # singular only: - if plural_expr is None: - gettext = nodes.Name('gettext', 'load') - node = nodes.Call(gettext, [nodes.Const(singular)], - [], None, None) - - # singular and plural - else: - ngettext = nodes.Name('ngettext', 'load') - node = nodes.Call(ngettext, [ - nodes.Const(singular), - nodes.Const(plural), - plural_expr - ], [], None, None) - - # in case newstyle gettext is used, the method is powerful - # enough to handle the variable expansion and autoescape - # handling itself - if self.environment.newstyle_gettext: - for key, value in iteritems(variables): - # the function adds that later anyways in case num was - # called num, so just skip it. - if num_called_num and key == 'num': - continue - node.kwargs.append(nodes.Keyword(key, value)) - - # otherwise do that here - else: - # mark the return value as safe if we are in an - # environment with autoescaping turned on - node = nodes.MarkSafeIfAutoescape(node) - if variables: - node = nodes.Mod(node, nodes.Dict([ - nodes.Pair(nodes.Const(key), value) - for key, value in variables.items() - ])) - return nodes.Output([node]) - - -class ExprStmtExtension(Extension): - """Adds a `do` tag to Jinja2 that works like the print statement just - that it doesn't print the return value. - """ - tags = set(['do']) - - def parse(self, parser): - node = nodes.ExprStmt(lineno=next(parser.stream).lineno) - node.node = parser.parse_tuple() - return node - - -class LoopControlExtension(Extension): - """Adds break and continue to the template engine.""" - tags = set(['break', 'continue']) - - def parse(self, parser): - token = next(parser.stream) - if token.value == 'break': - return nodes.Break(lineno=token.lineno) - return nodes.Continue(lineno=token.lineno) - - -class WithExtension(Extension): - pass - - -class AutoEscapeExtension(Extension): - pass - - -def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, - babel_style=True): - """Extract localizable strings from the given template node. Per - default this function returns matches in babel style that means non string - parameters as well as keyword arguments are returned as `None`. This - allows Babel to figure out what you really meant if you are using - gettext functions that allow keyword arguments for placeholder expansion. - If you don't want that behavior set the `babel_style` parameter to `False` - which causes only strings to be returned and parameters are always stored - in tuples. As a consequence invalid gettext calls (calls without a single - string parameter or string parameters after non-string parameters) are - skipped. - - This example explains the behavior: - - >>> from jinja2 import Environment - >>> env = Environment() - >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}') - >>> list(extract_from_ast(node)) - [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))] - >>> list(extract_from_ast(node, babel_style=False)) - [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))] - - For every string found this function yields a ``(lineno, function, - message)`` tuple, where: - - * ``lineno`` is the number of the line on which the string was found, - * ``function`` is the name of the ``gettext`` function used (if the - string was extracted from embedded Python code), and - * ``message`` is the string itself (a ``unicode`` object, or a tuple - of ``unicode`` objects for functions with multiple string arguments). - - This extraction function operates on the AST and is because of that unable - to extract any comments. For comment support you have to use the babel - extraction interface or extract comments yourself. - """ - for node in node.find_all(nodes.Call): - if not isinstance(node.node, nodes.Name) or \ - node.node.name not in gettext_functions: - continue - - strings = [] - for arg in node.args: - if isinstance(arg, nodes.Const) and \ - isinstance(arg.value, string_types): - strings.append(arg.value) - else: - strings.append(None) - - for arg in node.kwargs: - strings.append(None) - if node.dyn_args is not None: - strings.append(None) - if node.dyn_kwargs is not None: - strings.append(None) - - if not babel_style: - strings = tuple(x for x in strings if x is not None) - if not strings: - continue - else: - if len(strings) == 1: - strings = strings[0] - else: - strings = tuple(strings) - yield node.lineno, node.node.name, strings - - -class _CommentFinder(object): - """Helper class to find comments in a token stream. Can only - find comments for gettext calls forwards. Once the comment - from line 4 is found, a comment for line 1 will not return a - usable value. - """ - - def __init__(self, tokens, comment_tags): - self.tokens = tokens - self.comment_tags = comment_tags - self.offset = 0 - self.last_lineno = 0 - - def find_backwards(self, offset): - try: - for _, token_type, token_value in \ - reversed(self.tokens[self.offset:offset]): - if token_type in ('comment', 'linecomment'): - try: - prefix, comment = token_value.split(None, 1) - except ValueError: - continue - if prefix in self.comment_tags: - return [comment.rstrip()] - return [] - finally: - self.offset = offset - - def find_comments(self, lineno): - if not self.comment_tags or self.last_lineno > lineno: - return [] - for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]): - if token_lineno > lineno: - return self.find_backwards(self.offset + idx) - return self.find_backwards(len(self.tokens)) - - -def babel_extract(fileobj, keywords, comment_tags, options): - """Babel extraction method for Jinja templates. - - .. versionchanged:: 2.3 - Basic support for translation comments was added. If `comment_tags` - is now set to a list of keywords for extraction, the extractor will - try to find the best preceeding comment that begins with one of the - keywords. For best results, make sure to not have more than one - gettext call in one line of code and the matching comment in the - same line or the line before. - - .. versionchanged:: 2.5.1 - The `newstyle_gettext` flag can be set to `True` to enable newstyle - gettext calls. - - .. versionchanged:: 2.7 - A `silent` option can now be provided. If set to `False` template - syntax errors are propagated instead of being ignored. - - :param fileobj: the file-like object the messages should be extracted from - :param keywords: a list of keywords (i.e. function names) that should be - recognized as translation functions - :param comment_tags: a list of translator tags to search for and include - in the results. - :param options: a dictionary of additional options (optional) - :return: an iterator over ``(lineno, funcname, message, comments)`` tuples. - (comments will be empty currently) - """ - extensions = set() - for extension in options.get('extensions', '').split(','): - extension = extension.strip() - if not extension: - continue - extensions.add(import_string(extension)) - if InternationalizationExtension not in extensions: - extensions.add(InternationalizationExtension) - - def getbool(options, key, default=False): - return options.get(key, str(default)).lower() in \ - ('1', 'on', 'yes', 'true') - - silent = getbool(options, 'silent', True) - environment = Environment( - options.get('block_start_string', BLOCK_START_STRING), - options.get('block_end_string', BLOCK_END_STRING), - options.get('variable_start_string', VARIABLE_START_STRING), - options.get('variable_end_string', VARIABLE_END_STRING), - options.get('comment_start_string', COMMENT_START_STRING), - options.get('comment_end_string', COMMENT_END_STRING), - options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX, - options.get('line_comment_prefix') or LINE_COMMENT_PREFIX, - getbool(options, 'trim_blocks', TRIM_BLOCKS), - getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS), - NEWLINE_SEQUENCE, - getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE), - frozenset(extensions), - cache_size=0, - auto_reload=False - ) - - if getbool(options, 'trimmed'): - environment.policies['ext.i18n.trimmed'] = True - if getbool(options, 'newstyle_gettext'): - environment.newstyle_gettext = True - - source = fileobj.read().decode(options.get('encoding', 'utf-8')) - try: - node = environment.parse(source) - tokens = list(environment.lex(environment.preprocess(source))) - except TemplateSyntaxError as e: - if not silent: - raise - # skip templates with syntax errors - return - - finder = _CommentFinder(tokens, comment_tags) - for lineno, func, message in extract_from_ast(node, keywords): - yield lineno, func, message, finder.find_comments(lineno) - - -#: nicer import names -i18n = InternationalizationExtension -do = ExprStmtExtension -loopcontrols = LoopControlExtension -with_ = WithExtension -autoescape = AutoEscapeExtension |