diff options
Diffstat (limited to 'python/jinja2/lexer.py')
| -rw-r--r-- | python/jinja2/lexer.py | 739 |
1 files changed, 739 insertions, 0 deletions
diff --git a/python/jinja2/lexer.py b/python/jinja2/lexer.py new file mode 100644 index 0000000..6fd135d --- /dev/null +++ b/python/jinja2/lexer.py @@ -0,0 +1,739 @@ +# -*- coding: utf-8 -*- +""" + jinja2.lexer + ~~~~~~~~~~~~ + + This module implements a Jinja / Python combination lexer. The + `Lexer` class provided by this module is used to do some preprocessing + for Jinja. + + On the one hand it filters out invalid operators like the bitshift + operators we don't allow in templates. On the other hand it separates + template code and python code in expressions. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import re +from collections import deque +from operator import itemgetter + +from jinja2._compat import implements_iterator, intern, iteritems, text_type +from jinja2.exceptions import TemplateSyntaxError +from jinja2.utils import LRUCache + +# cache for the lexers. Exists in order to be able to have multiple +# environments with the same lexer +_lexer_cache = LRUCache(50) + +# static regular expressions +whitespace_re = re.compile(r'\s+', re.U) +string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'" + r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) +integer_re = re.compile(r'\d+') + +try: + # check if this Python supports Unicode identifiers + compile('föö', '<unknown>', 'eval') +except SyntaxError: + # no Unicode support, use ASCII identifiers + name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*') + check_ident = False +else: + # Unicode support, build a pattern to match valid characters, and set flag + # to use str.isidentifier to validate during lexing + from jinja2 import _identifier + name_re = re.compile(r'[\w{0}]+'.format(_identifier.pattern)) + check_ident = True + # remove the pattern from memory after building the regex + import sys + del sys.modules['jinja2._identifier'] + import jinja2 + del jinja2._identifier + del _identifier + +float_re = re.compile(r'(?<!\.)\d+\.\d+') +newline_re = re.compile(r'(\r\n|\r|\n)') + +# internal the tokens and keep references to them +TOKEN_ADD = intern('add') +TOKEN_ASSIGN = intern('assign') +TOKEN_COLON = intern('colon') +TOKEN_COMMA = intern('comma') +TOKEN_DIV = intern('div') +TOKEN_DOT = intern('dot') +TOKEN_EQ = intern('eq') +TOKEN_FLOORDIV = intern('floordiv') +TOKEN_GT = intern('gt') +TOKEN_GTEQ = intern('gteq') +TOKEN_LBRACE = intern('lbrace') +TOKEN_LBRACKET = intern('lbracket') +TOKEN_LPAREN = intern('lparen') +TOKEN_LT = intern('lt') +TOKEN_LTEQ = intern('lteq') +TOKEN_MOD = intern('mod') +TOKEN_MUL = intern('mul') +TOKEN_NE = intern('ne') +TOKEN_PIPE = intern('pipe') +TOKEN_POW = intern('pow') +TOKEN_RBRACE = intern('rbrace') +TOKEN_RBRACKET = intern('rbracket') +TOKEN_RPAREN = intern('rparen') +TOKEN_SEMICOLON = intern('semicolon') +TOKEN_SUB = intern('sub') +TOKEN_TILDE = intern('tilde') +TOKEN_WHITESPACE = intern('whitespace') +TOKEN_FLOAT = intern('float') +TOKEN_INTEGER = intern('integer') +TOKEN_NAME = intern('name') +TOKEN_STRING = intern('string') +TOKEN_OPERATOR = intern('operator') +TOKEN_BLOCK_BEGIN = intern('block_begin') +TOKEN_BLOCK_END = intern('block_end') +TOKEN_VARIABLE_BEGIN = intern('variable_begin') +TOKEN_VARIABLE_END = intern('variable_end') +TOKEN_RAW_BEGIN = intern('raw_begin') +TOKEN_RAW_END = intern('raw_end') +TOKEN_COMMENT_BEGIN = intern('comment_begin') +TOKEN_COMMENT_END = intern('comment_end') +TOKEN_COMMENT = intern('comment') +TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin') +TOKEN_LINESTATEMENT_END = intern('linestatement_end') +TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin') +TOKEN_LINECOMMENT_END = intern('linecomment_end') +TOKEN_LINECOMMENT = intern('linecomment') +TOKEN_DATA = intern('data') +TOKEN_INITIAL = intern('initial') +TOKEN_EOF = intern('eof') + +# bind operators to token types +operators = { + '+': TOKEN_ADD, + '-': TOKEN_SUB, + '/': TOKEN_DIV, + '//': TOKEN_FLOORDIV, + '*': TOKEN_MUL, + '%': TOKEN_MOD, + '**': TOKEN_POW, + '~': TOKEN_TILDE, + '[': TOKEN_LBRACKET, + ']': TOKEN_RBRACKET, + '(': TOKEN_LPAREN, + ')': TOKEN_RPAREN, + '{': TOKEN_LBRACE, + '}': TOKEN_RBRACE, + '==': TOKEN_EQ, + '!=': TOKEN_NE, + '>': TOKEN_GT, + '>=': TOKEN_GTEQ, + '<': TOKEN_LT, + '<=': TOKEN_LTEQ, + '=': TOKEN_ASSIGN, + '.': TOKEN_DOT, + ':': TOKEN_COLON, + '|': TOKEN_PIPE, + ',': TOKEN_COMMA, + ';': TOKEN_SEMICOLON +} + +reverse_operators = dict([(v, k) for k, v in iteritems(operators)]) +assert len(operators) == len(reverse_operators), 'operators dropped' +operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in + sorted(operators, key=lambda x: -len(x)))) + +ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT, + TOKEN_COMMENT_END, TOKEN_WHITESPACE, + TOKEN_LINECOMMENT_BEGIN, TOKEN_LINECOMMENT_END, + TOKEN_LINECOMMENT]) +ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA, + TOKEN_COMMENT, TOKEN_LINECOMMENT]) + + +def _describe_token_type(token_type): + if token_type in reverse_operators: + return reverse_operators[token_type] + return { + TOKEN_COMMENT_BEGIN: 'begin of comment', + TOKEN_COMMENT_END: 'end of comment', + TOKEN_COMMENT: 'comment', + TOKEN_LINECOMMENT: 'comment', + TOKEN_BLOCK_BEGIN: 'begin of statement block', + TOKEN_BLOCK_END: 'end of statement block', + TOKEN_VARIABLE_BEGIN: 'begin of print statement', + TOKEN_VARIABLE_END: 'end of print statement', + TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement', + TOKEN_LINESTATEMENT_END: 'end of line statement', + TOKEN_DATA: 'template data / text', + TOKEN_EOF: 'end of template' + }.get(token_type, token_type) + + +def describe_token(token): + """Returns a description of the token.""" + if token.type == 'name': + return token.value + return _describe_token_type(token.type) + + +def describe_token_expr(expr): + """Like `describe_token` but for token expressions.""" + if ':' in expr: + type, value = expr.split(':', 1) + if type == 'name': + return value + else: + type = expr + return _describe_token_type(type) + + +def count_newlines(value): + """Count the number of newline characters in the string. This is + useful for extensions that filter a stream. + """ + return len(newline_re.findall(value)) + + +def compile_rules(environment): + """Compiles all the rules from the environment into a list of rules.""" + e = re.escape + rules = [ + (len(environment.comment_start_string), 'comment', + e(environment.comment_start_string)), + (len(environment.block_start_string), 'block', + e(environment.block_start_string)), + (len(environment.variable_start_string), 'variable', + e(environment.variable_start_string)) + ] + + if environment.line_statement_prefix is not None: + rules.append((len(environment.line_statement_prefix), 'linestatement', + r'^[ \t\v]*' + e(environment.line_statement_prefix))) + if environment.line_comment_prefix is not None: + rules.append((len(environment.line_comment_prefix), 'linecomment', + r'(?:^|(?<=\S))[^\S\r\n]*' + + e(environment.line_comment_prefix))) + + return [x[1:] for x in sorted(rules, reverse=True)] + + +class Failure(object): + """Class that raises a `TemplateSyntaxError` if called. + Used by the `Lexer` to specify known errors. + """ + + def __init__(self, message, cls=TemplateSyntaxError): + self.message = message + self.error_class = cls + + def __call__(self, lineno, filename): + raise self.error_class(self.message, lineno, filename) + + +class Token(tuple): + """Token class.""" + __slots__ = () + lineno, type, value = (property(itemgetter(x)) for x in range(3)) + + def __new__(cls, lineno, type, value): + return tuple.__new__(cls, (lineno, intern(str(type)), value)) + + def __str__(self): + if self.type in reverse_operators: + return reverse_operators[self.type] + elif self.type == 'name': + return self.value + return self.type + + def test(self, expr): + """Test a token against a token expression. This can either be a + token type or ``'token_type:token_value'``. This can only test + against string values and types. + """ + # here we do a regular string equality check as test_any is usually + # passed an iterable of not interned strings. + if self.type == expr: + return True + elif ':' in expr: + return expr.split(':', 1) == [self.type, self.value] + return False + + def test_any(self, *iterable): + """Test against multiple token expressions.""" + for expr in iterable: + if self.test(expr): + return True + return False + + def __repr__(self): + return 'Token(%r, %r, %r)' % ( + self.lineno, + self.type, + self.value + ) + + +@implements_iterator +class TokenStreamIterator(object): + """The iterator for tokenstreams. Iterate over the stream + until the eof token is reached. + """ + + def __init__(self, stream): + self.stream = stream + + def __iter__(self): + return self + + def __next__(self): + token = self.stream.current + if token.type is TOKEN_EOF: + self.stream.close() + raise StopIteration() + next(self.stream) + return token + + +@implements_iterator +class TokenStream(object): + """A token stream is an iterable that yields :class:`Token`\\s. The + parser however does not iterate over it but calls :meth:`next` to go + one token ahead. The current active token is stored as :attr:`current`. + """ + + def __init__(self, generator, name, filename): + self._iter = iter(generator) + self._pushed = deque() + self.name = name + self.filename = filename + self.closed = False + self.current = Token(1, TOKEN_INITIAL, '') + next(self) + + def __iter__(self): + return TokenStreamIterator(self) + + def __bool__(self): + return bool(self._pushed) or self.current.type is not TOKEN_EOF + __nonzero__ = __bool__ # py2 + + eos = property(lambda x: not x, doc="Are we at the end of the stream?") + + def push(self, token): + """Push a token back to the stream.""" + self._pushed.append(token) + + def look(self): + """Look at the next token.""" + old_token = next(self) + result = self.current + self.push(result) + self.current = old_token + return result + + def skip(self, n=1): + """Got n tokens ahead.""" + for x in range(n): + next(self) + + def next_if(self, expr): + """Perform the token test and return the token if it matched. + Otherwise the return value is `None`. + """ + if self.current.test(expr): + return next(self) + + def skip_if(self, expr): + """Like :meth:`next_if` but only returns `True` or `False`.""" + return self.next_if(expr) is not None + + def __next__(self): + """Go one token ahead and return the old one. + + Use the built-in :func:`next` instead of calling this directly. + """ + rv = self.current + if self._pushed: + self.current = self._pushed.popleft() + elif self.current.type is not TOKEN_EOF: + try: + self.current = next(self._iter) + except StopIteration: + self.close() + return rv + + def close(self): + """Close the stream.""" + self.current = Token(self.current.lineno, TOKEN_EOF, '') + self._iter = None + self.closed = True + + def expect(self, expr): + """Expect a given token type and return it. This accepts the same + argument as :meth:`jinja2.lexer.Token.test`. + """ + if not self.current.test(expr): + expr = describe_token_expr(expr) + if self.current.type is TOKEN_EOF: + raise TemplateSyntaxError('unexpected end of template, ' + 'expected %r.' % expr, + self.current.lineno, + self.name, self.filename) + raise TemplateSyntaxError("expected token %r, got %r" % + (expr, describe_token(self.current)), + self.current.lineno, + self.name, self.filename) + try: + return self.current + finally: + next(self) + + +def get_lexer(environment): + """Return a lexer which is probably cached.""" + key = (environment.block_start_string, + environment.block_end_string, + environment.variable_start_string, + environment.variable_end_string, + environment.comment_start_string, + environment.comment_end_string, + environment.line_statement_prefix, + environment.line_comment_prefix, + environment.trim_blocks, + environment.lstrip_blocks, + environment.newline_sequence, + environment.keep_trailing_newline) + lexer = _lexer_cache.get(key) + if lexer is None: + lexer = Lexer(environment) + _lexer_cache[key] = lexer + return lexer + + +class Lexer(object): + """Class that implements a lexer for a given environment. Automatically + created by the environment class, usually you don't have to do that. + + Note that the lexer is not automatically bound to an environment. + Multiple environments can share the same lexer. + """ + + def __init__(self, environment): + # shortcuts + c = lambda x: re.compile(x, re.M | re.S) + e = re.escape + + # lexing rules for tags + tag_rules = [ + (whitespace_re, TOKEN_WHITESPACE, None), + (float_re, TOKEN_FLOAT, None), + (integer_re, TOKEN_INTEGER, None), + (name_re, TOKEN_NAME, None), + (string_re, TOKEN_STRING, None), + (operator_re, TOKEN_OPERATOR, None) + ] + + # assemble the root lexing rule. because "|" is ungreedy + # we have to sort by length so that the lexer continues working + # as expected when we have parsing rules like <% for block and + # <%= for variables. (if someone wants asp like syntax) + # variables are just part of the rules if variable processing + # is required. + root_tag_rules = compile_rules(environment) + + # block suffix if trimming is enabled + block_suffix_re = environment.trim_blocks and '\\n?' or '' + + # strip leading spaces if lstrip_blocks is enabled + prefix_re = {} + if environment.lstrip_blocks: + # use '{%+' to manually disable lstrip_blocks behavior + no_lstrip_re = e('+') + # detect overlap between block and variable or comment strings + block_diff = c(r'^%s(.*)' % e(environment.block_start_string)) + # make sure we don't mistake a block for a variable or a comment + m = block_diff.match(environment.comment_start_string) + no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' + m = block_diff.match(environment.variable_start_string) + no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' + + # detect overlap between comment and variable strings + comment_diff = c(r'^%s(.*)' % e(environment.comment_start_string)) + m = comment_diff.match(environment.variable_start_string) + no_variable_re = m and r'(?!%s)' % e(m.group(1)) or '' + + lstrip_re = r'^[ \t]*' + block_prefix_re = r'%s%s(?!%s)|%s\+?' % ( + lstrip_re, + e(environment.block_start_string), + no_lstrip_re, + e(environment.block_start_string), + ) + comment_prefix_re = r'%s%s%s|%s\+?' % ( + lstrip_re, + e(environment.comment_start_string), + no_variable_re, + e(environment.comment_start_string), + ) + prefix_re['block'] = block_prefix_re + prefix_re['comment'] = comment_prefix_re + else: + block_prefix_re = '%s' % e(environment.block_start_string) + + self.newline_sequence = environment.newline_sequence + self.keep_trailing_newline = environment.keep_trailing_newline + + # global lexing rules + self.rules = { + 'root': [ + # directives + (c('(.*?)(?:%s)' % '|'.join( + [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % ( + e(environment.block_start_string), + block_prefix_re, + e(environment.block_end_string), + e(environment.block_end_string) + )] + [ + r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r)) + for n, r in root_tag_rules + ])), (TOKEN_DATA, '#bygroup'), '#bygroup'), + # data + (c('.+'), TOKEN_DATA, None) + ], + # comments + TOKEN_COMMENT_BEGIN: [ + (c(r'(.*?)((?:\-%s\s*|%s)%s)' % ( + e(environment.comment_end_string), + e(environment.comment_end_string), + block_suffix_re + )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'), + (c('(.)'), (Failure('Missing end of comment tag'),), None) + ], + # blocks + TOKEN_BLOCK_BEGIN: [ + (c(r'(?:\-%s\s*|%s)%s' % ( + e(environment.block_end_string), + e(environment.block_end_string), + block_suffix_re + )), TOKEN_BLOCK_END, '#pop'), + ] + tag_rules, + # variables + TOKEN_VARIABLE_BEGIN: [ + (c(r'\-%s\s*|%s' % ( + e(environment.variable_end_string), + e(environment.variable_end_string) + )), TOKEN_VARIABLE_END, '#pop') + ] + tag_rules, + # raw block + TOKEN_RAW_BEGIN: [ + (c(r'(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( + e(environment.block_start_string), + block_prefix_re, + e(environment.block_end_string), + e(environment.block_end_string), + block_suffix_re + )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'), + (c('(.)'), (Failure('Missing end of raw directive'),), None) + ], + # line statements + TOKEN_LINESTATEMENT_BEGIN: [ + (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop') + ] + tag_rules, + # line comments + TOKEN_LINECOMMENT_BEGIN: [ + (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT, + TOKEN_LINECOMMENT_END), '#pop') + ] + } + + def _normalize_newlines(self, value): + """Called for strings and template data to normalize it to unicode.""" + return newline_re.sub(self.newline_sequence, value) + + def tokenize(self, source, name=None, filename=None, state=None): + """Calls tokeniter + tokenize and wraps it in a token stream. + """ + stream = self.tokeniter(source, name, filename, state) + return TokenStream(self.wrap(stream, name, filename), name, filename) + + def wrap(self, stream, name=None, filename=None): + """This is called with the stream as returned by `tokenize` and wraps + every token in a :class:`Token` and converts the value. + """ + for lineno, token, value in stream: + if token in ignored_tokens: + continue + elif token == 'linestatement_begin': + token = 'block_begin' + elif token == 'linestatement_end': + token = 'block_end' + # we are not interested in those tokens in the parser + elif token in ('raw_begin', 'raw_end'): + continue + elif token == 'data': + value = self._normalize_newlines(value) + elif token == 'keyword': + token = value + elif token == 'name': + value = str(value) + if check_ident and not value.isidentifier(): + raise TemplateSyntaxError( + 'Invalid character in identifier', + lineno, name, filename) + elif token == 'string': + # try to unescape string + try: + value = self._normalize_newlines(value[1:-1]) \ + .encode('ascii', 'backslashreplace') \ + .decode('unicode-escape') + except Exception as e: + msg = str(e).split(':')[-1].strip() + raise TemplateSyntaxError(msg, lineno, name, filename) + elif token == 'integer': + value = int(value) + elif token == 'float': + value = float(value) + elif token == 'operator': + token = operators[value] + yield Token(lineno, token, value) + + def tokeniter(self, source, name, filename=None, state=None): + """This method tokenizes the text and returns the tokens in a + generator. Use this method if you just want to tokenize a template. + """ + source = text_type(source) + lines = source.splitlines() + if self.keep_trailing_newline and source: + for newline in ('\r\n', '\r', '\n'): + if source.endswith(newline): + lines.append('') + break + source = '\n'.join(lines) + pos = 0 + lineno = 1 + stack = ['root'] + if state is not None and state != 'root': + assert state in ('variable', 'block'), 'invalid state' + stack.append(state + '_begin') + else: + state = 'root' + statetokens = self.rules[stack[-1]] + source_length = len(source) + + balancing_stack = [] + + while 1: + # tokenizer loop + for regex, tokens, new_state in statetokens: + m = regex.match(source, pos) + # if no match we try again with the next rule + if m is None: + continue + + # we only match blocks and variables if braces / parentheses + # are balanced. continue parsing with the lower rule which + # is the operator rule. do this only if the end tags look + # like operators + if balancing_stack and \ + tokens in ('variable_end', 'block_end', + 'linestatement_end'): + continue + + # tuples support more options + if isinstance(tokens, tuple): + for idx, token in enumerate(tokens): + # failure group + if token.__class__ is Failure: + raise token(lineno, filename) + # bygroup is a bit more complex, in that case we + # yield for the current token the first named + # group that matched + elif token == '#bygroup': + for key, value in iteritems(m.groupdict()): + if value is not None: + yield lineno, key, value + lineno += value.count('\n') + break + else: + raise RuntimeError('%r wanted to resolve ' + 'the token dynamically' + ' but no group matched' + % regex) + # normal group + else: + data = m.group(idx + 1) + if data or token not in ignore_if_empty: + yield lineno, token, data + lineno += data.count('\n') + + # strings as token just are yielded as it. + else: + data = m.group() + # update brace/parentheses balance + if tokens == 'operator': + if data == '{': + balancing_stack.append('}') + elif data == '(': + balancing_stack.append(')') + elif data == '[': + balancing_stack.append(']') + elif data in ('}', ')', ']'): + if not balancing_stack: + raise TemplateSyntaxError('unexpected \'%s\'' % + data, lineno, name, + filename) + expected_op = balancing_stack.pop() + if expected_op != data: + raise TemplateSyntaxError('unexpected \'%s\', ' + 'expected \'%s\'' % + (data, expected_op), + lineno, name, + filename) + # yield items + if data or tokens not in ignore_if_empty: + yield lineno, tokens, data + lineno += data.count('\n') + + # fetch new position into new variable so that we can check + # if there is a internal parsing error which would result + # in an infinite loop + pos2 = m.end() + + # handle state changes + if new_state is not None: + # remove the uppermost state + if new_state == '#pop': + stack.pop() + # resolve the new state by group checking + elif new_state == '#bygroup': + for key, value in iteritems(m.groupdict()): + if value is not None: + stack.append(key) + break + else: + raise RuntimeError('%r wanted to resolve the ' + 'new state dynamically but' + ' no group matched' % + regex) + # direct state name given + else: + stack.append(new_state) + statetokens = self.rules[stack[-1]] + # we are still at the same position and no stack change. + # this means a loop without break condition, avoid that and + # raise error + elif pos2 == pos: + raise RuntimeError('%r yielded empty string without ' + 'stack change' % regex) + # publish new function and start again + pos = pos2 + break + # if loop terminated without break we haven't found a single match + # either we are at the end of the file or we have a problem + else: + # end of text + if pos >= source_length: + return + # something went wrong + raise TemplateSyntaxError('unexpected char %r at %d' % + (source[pos], pos), lineno, + name, filename) |