From 1d949c779fbfb03635790cb1f3c160bd47f67366 Mon Sep 17 00:00:00 2001 From: Mark Mayo Date: Mon, 6 Jul 2026 21:50:08 +1200 Subject: [PATCH] Optimize lexer, parser, and interpreter for speed Reduce per-expression overhead across the compile and evaluation pipeline without changing behavior: - lexer: scan with a local index and slice identifier/number/string runs instead of char-by-char accumulation; consume whitespace runs; find-based scanning for quoted strings and literals. - parser: precompute per-class nud/led dispatch tables, inline token access and AST node construction on the hot path, and look up the compile cache without raising on a miss. - interpreter: share a pre-warmed default TreeInterpreter for default-options searches, cache resolved visit_* handlers, use a fixed two-argument visit signature, and short-circuit field access on None. - functions: resolve signature types to Python types once at registration and reuse the cached result during validation. Benchmarks (tests/compliance/benchmarks.json) improve 8-97% across lex/parse/search with no regressions; full test suite and hypothesis property tests pass. --- jmespath/functions.py | 101 ++++++++------ jmespath/lexer.py | 314 +++++++++++++++++++++++------------------- jmespath/parser.py | 281 +++++++++++++++++++++++-------------- jmespath/visitor.py | 132 ++++++++++++------ 4 files changed, 502 insertions(+), 326 deletions(-) diff --git a/jmespath/functions.py b/jmespath/functions.py index 627b569d..9390a8ef 100644 --- a/jmespath/functions.py +++ b/jmespath/functions.py @@ -42,6 +42,36 @@ def _record_signature(func): return _record_signature +def _resolve_pytypes(types): + # Maps a list of jmespath types (e.g. ['array-number']) to the + # allowed python type names and, for arrays, the allowed python + # type names of the elements. + allowed_types = [] + allowed_subtypes = [] + for t in types: + type_ = t.split('-', 1) + if len(type_) == 2: + type_, subtype = type_ + allowed_subtypes.append(REVERSE_TYPES_MAP[subtype]) + else: + type_ = type_[0] + allowed_types.extend(REVERSE_TYPES_MAP[type_]) + return allowed_types, allowed_subtypes + + +def _cache_resolved_pytypes(spec, types): + # Resolving jmespath types to python types is much more expensive + # than the type check itself, so the result is computed once and + # cached on the signature spec. + allowed_types, allowed_subtypes = _resolve_pytypes(types) + resolved = ( + frozenset(allowed_types), + [frozenset(subtype) for subtype in allowed_subtypes], + ) + spec['_resolved_pytypes'] = resolved + return resolved + + class FunctionRegistry(type): def __init__(cls, name, bases, attrs): cls._populate_function_table() @@ -57,6 +87,15 @@ def _populate_function_table(cls): continue signature = getattr(method, 'signature', None) if signature is not None: + # Resolving the jmespath types in a signature to the + # allowed python type names is far more expensive than + # the per-call type check itself, so it is done once + # here and cached on each argument spec for _type_check + # to reuse. + for spec in signature: + types = spec.get('types') + if types and '_resolved_pytypes' not in spec: + _cache_resolved_pytypes(spec, types) function_table[name[6:]] = { 'function': method, 'signature': signature, @@ -92,46 +131,30 @@ def _validate_arguments(self, args, signature, function_name): def _type_check(self, actual, signature, function_name): for i in range(len(signature)): - allowed_types = signature[i]['types'] + spec = signature[i] + allowed_types = spec['types'] if allowed_types: - self._type_check_single(actual[i], allowed_types, - function_name) - - def _type_check_single(self, current, types, function_name): - # Type checking involves checking the top level type, - # and in the case of arrays, potentially checking the types - # of each element. - allowed_types, allowed_subtypes = self._get_allowed_pytypes(types) - # We're not using isinstance() on purpose. - # The type model for jmespath does not map - # 1-1 with python types (booleans are considered - # integers in python for example). - actual_typename = type(current).__name__ - if actual_typename not in allowed_types: - raise exceptions.JMESPathTypeError( - function_name, current, - self._convert_to_jmespath_type(actual_typename), types) - # If we're dealing with a list type, we can have - # additional restrictions on the type of the list - # elements (for example a function can require a - # list of numbers or a list of strings). - # Arrays are the only types that can have subtypes. - if allowed_subtypes: - self._subtype_check(current, allowed_subtypes, - types, function_name) - - def _get_allowed_pytypes(self, types): - allowed_types = [] - allowed_subtypes = [] - for t in types: - type_ = t.split('-', 1) - if len(type_) == 2: - type_, subtype = type_ - allowed_subtypes.append(REVERSE_TYPES_MAP[subtype]) - else: - type_ = type_[0] - allowed_types.extend(REVERSE_TYPES_MAP[type_]) - return allowed_types, allowed_subtypes + resolved = spec.get('_resolved_pytypes') + if resolved is None: + # Signatures that didn't pass through the function + # registry are resolved (and cached) on first use. + resolved = _cache_resolved_pytypes(spec, allowed_types) + allowed_pytypes, allowed_subtypes = resolved + current = actual[i] + # We're not using isinstance() on purpose. + # The type model for jmespath does not map + # 1-1 with python types (booleans are considered + # integers in python for example). + actual_typename = type(current).__name__ + if actual_typename not in allowed_pytypes: + raise exceptions.JMESPathTypeError( + function_name, current, + self._convert_to_jmespath_type(actual_typename), + allowed_types) + # Arrays are the only types that can have subtypes. + if allowed_subtypes: + self._subtype_check(current, allowed_subtypes, + allowed_types, function_name) def _subtype_check(self, current, allowed_subtypes, types, function_name): if len(allowed_subtypes) == 1: diff --git a/jmespath/lexer.py b/jmespath/lexer.py index 8db05e37..2270aa30 100644 --- a/jmespath/lexer.py +++ b/jmespath/lexer.py @@ -6,10 +6,10 @@ class Lexer(object): - START_IDENTIFIER = set(string.ascii_letters + '_') - VALID_IDENTIFIER = set(string.ascii_letters + string.digits + '_') - VALID_NUMBER = set(string.digits) - WHITESPACE = set(" \t\n\r") + START_IDENTIFIER = frozenset(string.ascii_letters + '_') + VALID_IDENTIFIER = frozenset(string.ascii_letters + string.digits + '_') + VALID_NUMBER = frozenset(string.digits) + WHITESPACE = frozenset(" \t\n\r") SIMPLE_TOKENS = { '.': 'dot', '*': 'star', @@ -24,140 +24,166 @@ class Lexer(object): } def tokenize(self, expression): - self._initialize_for_expression(expression) - while self._current is not None: - if self._current in self.SIMPLE_TOKENS: - yield {'type': self.SIMPLE_TOKENS[self._current], - 'value': self._current, - 'start': self._position, 'end': self._position + 1} - self._next() - elif self._current in self.START_IDENTIFIER: - start = self._position - buff = self._current - while self._next() in self.VALID_IDENTIFIER: - buff += self._current - yield {'type': 'unquoted_identifier', 'value': buff, - 'start': start, 'end': start + len(buff)} - elif self._current in self.WHITESPACE: - self._next() - elif self._current == '[': - start = self._position - next_char = self._next() + # This scans with a local position index instead of the + # char-by-char self._next() approach: avoiding a method call + # and several attribute accesses per character makes + # tokenization substantially faster. Runs of identifier, + # number, and string characters are sliced out of the + # expression rather than accumulated one char at a time. + if not expression: + raise EmptyExpressionError() + tokens = [] + append = tokens.append + simple_tokens = self.SIMPLE_TOKENS + start_identifier = self.START_IDENTIFIER + valid_identifier = self.VALID_IDENTIFIER + valid_number = self.VALID_NUMBER + whitespace = self.WHITESPACE + length = len(expression) + pos = 0 + while pos < length: + char = expression[pos] + simple_type = simple_tokens.get(char) + if simple_type is not None: + append({'type': simple_type, 'value': char, + 'start': pos, 'end': pos + 1}) + pos += 1 + elif char in start_identifier: + start = pos + pos += 1 + while pos < length and expression[pos] in valid_identifier: + pos += 1 + append({'type': 'unquoted_identifier', + 'value': expression[start:pos], + 'start': start, 'end': pos}) + elif char in whitespace: + pos += 1 + while pos < length and expression[pos] in whitespace: + pos += 1 + elif char == '[': + next_char = expression[pos + 1] if pos + 1 < length else '' if next_char == ']': - self._next() - yield {'type': 'flatten', 'value': '[]', - 'start': start, 'end': start + 2} + append({'type': 'flatten', 'value': '[]', + 'start': pos, 'end': pos + 2}) + pos += 2 elif next_char == '?': - self._next() - yield {'type': 'filter', 'value': '[?', - 'start': start, 'end': start + 2} + append({'type': 'filter', 'value': '[?', + 'start': pos, 'end': pos + 2}) + pos += 2 else: - yield {'type': 'lbracket', 'value': '[', - 'start': start, 'end': start + 1} - elif self._current == "'": - yield self._consume_raw_string_literal() - elif self._current == '|': - yield self._match_or_else('|', 'or', 'pipe') - elif self._current == '&': - yield self._match_or_else('&', 'and', 'expref') - elif self._current == '`': - yield self._consume_literal() - elif self._current in self.VALID_NUMBER: - start = self._position - buff = self._consume_number() - yield {'type': 'number', 'value': int(buff), - 'start': start, 'end': start + len(buff)} - elif self._current == '-': + append({'type': 'lbracket', 'value': '[', + 'start': pos, 'end': pos + 1}) + pos += 1 + elif char in valid_number: + start = pos + pos += 1 + while pos < length and expression[pos] in valid_number: + pos += 1 + append({'type': 'number', 'value': int(expression[start:pos]), + 'start': start, 'end': pos}) + elif char == "'": + token, pos = self._consume_raw_string_literal( + expression, pos) + append(token) + elif char == '|': + token, pos = self._match_or_else( + expression, pos, '|', 'or', 'pipe') + append(token) + elif char == '&': + token, pos = self._match_or_else( + expression, pos, '&', 'and', 'expref') + append(token) + elif char == '`': + token, pos = self._consume_literal(expression, pos) + append(token) + elif char == '-': # Negative number. - start = self._position - buff = self._consume_number() + start = pos + pos += 1 + while pos < length and expression[pos] in valid_number: + pos += 1 + buff = expression[start:pos] if len(buff) > 1: - yield {'type': 'number', 'value': int(buff), - 'start': start, 'end': start + len(buff)} + append({'type': 'number', 'value': int(buff), + 'start': start, 'end': start + len(buff)}) else: raise LexerError(lexer_position=start, lexer_value=buff, message="Unknown token '%s'" % buff) - elif self._current == '"': - yield self._consume_quoted_identifier() - elif self._current == '<': - yield self._match_or_else('=', 'lte', 'lt') - elif self._current == '>': - yield self._match_or_else('=', 'gte', 'gt') - elif self._current == '!': - yield self._match_or_else('=', 'ne', 'not') - elif self._current == '=': - if self._next() == '=': - yield {'type': 'eq', 'value': '==', - 'start': self._position - 1, 'end': self._position} - self._next() + elif char == '"': + token, pos = self._consume_quoted_identifier(expression, pos) + append(token) + elif char == '<': + token, pos = self._match_or_else( + expression, pos, '=', 'lte', 'lt') + append(token) + elif char == '>': + token, pos = self._match_or_else( + expression, pos, '=', 'gte', 'gt') + append(token) + elif char == '!': + token, pos = self._match_or_else( + expression, pos, '=', 'ne', 'not') + append(token) + elif char == '=': + if pos + 1 < length and expression[pos + 1] == '=': + append({'type': 'eq', 'value': '==', + 'start': pos, 'end': pos + 1}) + pos += 2 else: - if self._current is None: - # If we're at the EOF, we never advanced - # the position so we don't need to rewind - # it back one location. - position = self._position - else: - position = self._position - 1 raise LexerError( - lexer_position=position, + lexer_position=pos, lexer_value='=', message="Unknown token '='") else: - raise LexerError(lexer_position=self._position, - lexer_value=self._current, - message="Unknown token %s" % self._current) - yield {'type': 'eof', 'value': '', - 'start': self._length, 'end': self._length} - - def _consume_number(self): - start = self._position - buff = self._current - while self._next() in self.VALID_NUMBER: - buff += self._current - return buff - - def _initialize_for_expression(self, expression): - if not expression: - raise EmptyExpressionError() - self._position = 0 - self._expression = expression - self._chars = list(self._expression) - self._current = self._chars[self._position] - self._length = len(self._expression) - - def _next(self): - if self._position == self._length - 1: - self._current = None - else: - self._position += 1 - self._current = self._chars[self._position] - return self._current + raise LexerError(lexer_position=pos, + lexer_value=char, + message="Unknown token %s" % char) + append({'type': 'eof', 'value': '', + 'start': length, 'end': length}) + return tokens - def _consume_until(self, delimiter): - # Consume until the delimiter is reached, - # allowing for the delimiter to be escaped with "\". - start = self._position - buff = '' - self._next() - while self._current != delimiter: - if self._current == '\\': - buff += '\\' - self._next() - if self._current is None: + def _consume_until(self, expression, start, delimiter): + # Consume until the delimiter is reached, allowing for the + # delimiter to be escaped with "\". ``start`` points at the + # opening delimiter. Returns the consumed string (escaping + # backslashes preserved, as before) and the position of the + # first character after the closing delimiter. + pos = start + 1 + chunks = [] + while True: + delim_index = expression.find(delimiter, pos) + if delim_index == -1: # We're at the EOF. raise LexerError(lexer_position=start, - lexer_value=self._expression[start:], + lexer_value=expression[start:], message="Unclosed %s delimiter" % delimiter) - buff += self._current - self._next() - # Skip the closing delimiter. - self._next() - return buff + backslash_index = expression.find('\\', pos) + if backslash_index == -1 or delim_index < backslash_index: + chunks.append(expression[pos:delim_index]) + # Skip the closing delimiter. + return ''.join(chunks), delim_index + 1 + # An escape: consume the backslash and the following + # character (whatever it is, including the delimiter). + if backslash_index + 2 > len(expression): + raise LexerError(lexer_position=start, + lexer_value=expression[start:], + message="Unclosed %s delimiter" % delimiter) + chunks.append(expression[pos:backslash_index + 2]) + pos = backslash_index + 2 + + def _token_end_position(self, expression, pos): + # The historical scanner never advanced past the final + # character of the expression, so a token whose closing + # delimiter is the last character reports its end position + # one short. Preserved for compatibility. + if pos >= len(expression): + return len(expression) - 1 + return pos - def _consume_literal(self): - start = self._position - lexeme = self._consume_until('`').replace('\\`', '`') + def _consume_literal(self, expression, start): + lexeme, pos = self._consume_until(expression, start, '`') + lexeme = lexeme.replace('\\`', '`') try: # Assume it is valid JSON and attempt to parse. parsed_json = loads(lexeme) @@ -170,39 +196,37 @@ def _consume_literal(self): PendingDeprecationWarning) except ValueError: raise LexerError(lexer_position=start, - lexer_value=self._expression[start:], + lexer_value=expression[start:], message="Bad token %s" % lexeme) - token_len = self._position - start - return {'type': 'literal', 'value': parsed_json, - 'start': start, 'end': token_len} + token_len = self._token_end_position(expression, pos) - start + return ({'type': 'literal', 'value': parsed_json, + 'start': start, 'end': token_len}, pos) - def _consume_quoted_identifier(self): - start = self._position - lexeme = '"' + self._consume_until('"') + '"' + def _consume_quoted_identifier(self, expression, start): + consumed, pos = self._consume_until(expression, start, '"') + lexeme = '"' + consumed + '"' try: - token_len = self._position - start - return {'type': 'quoted_identifier', 'value': loads(lexeme), - 'start': start, 'end': token_len} + token_len = self._token_end_position(expression, pos) - start + return ({'type': 'quoted_identifier', 'value': loads(lexeme), + 'start': start, 'end': token_len}, pos) except ValueError as e: error_message = str(e).split(':')[0] raise LexerError(lexer_position=start, lexer_value=lexeme, message=error_message) - def _consume_raw_string_literal(self): - start = self._position - lexeme = self._consume_until("'").replace("\\'", "'") - token_len = self._position - start - return {'type': 'literal', 'value': lexeme, - 'start': start, 'end': token_len} + def _consume_raw_string_literal(self, expression, start): + consumed, pos = self._consume_until(expression, start, "'") + lexeme = consumed.replace("\\'", "'") + token_len = self._token_end_position(expression, pos) - start + return ({'type': 'literal', 'value': lexeme, + 'start': start, 'end': token_len}, pos) - def _match_or_else(self, expected, match_type, else_type): - start = self._position - current = self._current - next_char = self._next() - if next_char == expected: - self._next() - return {'type': match_type, 'value': current + next_char, - 'start': start, 'end': start + 1} - return {'type': else_type, 'value': current, - 'start': start, 'end': start} + def _match_or_else(self, expression, start, expected, match_type, + else_type): + current = expression[start] + if start + 1 < len(expression) and expression[start + 1] == expected: + return ({'type': match_type, 'value': current + expected, + 'start': start, 'end': start + 1}, start + 2) + return ({'type': else_type, 'value': current, + 'start': start, 'end': start}, start + 1) diff --git a/jmespath/parser.py b/jmespath/parser.py index cc8e804e..344e8c78 100644 --- a/jmespath/parser.py +++ b/jmespath/parser.py @@ -72,18 +72,36 @@ class Parser(object): # _CACHE dict. _CACHE = {} _MAX_SIZE = 512 + # Per-class nud/led dispatch tables ({token_type: function}), + # built once per class on first instantiation. Keyed by class so + # subclasses overriding or adding token handlers work correctly. + _DISPATCH_CACHE = {} def __init__(self, lookahead=2): self.tokenizer = None self._tokens = [None] * lookahead self._buffer_size = lookahead self._index = 0 + cls = type(self) + dispatch = Parser._DISPATCH_CACHE.get(cls) + if dispatch is None: + nud = {} + led = {} + for name in dir(cls): + if name.startswith('_token_nud_'): + nud[name[11:]] = getattr(cls, name) + elif name.startswith('_token_led_'): + led[name[11:]] = getattr(cls, name) + dispatch = (nud, led) + Parser._DISPATCH_CACHE[cls] = dispatch + self._NUD_DISPATCH, self._LED_DISPATCH = dispatch def parse(self, expression): - try: - return self._CACHE[expression] - except KeyError: - pass + # .get() rather than try/except KeyError: a raised exception + # costs far more than the parse-cache lookup itself. + parsed_result = self._CACHE.get(expression) + if parsed_result is not None: + return parsed_result parsed_result = self._do_parse(expression) if len(self._CACHE) >= self._MAX_SIZE: try: @@ -116,8 +134,10 @@ def _do_parse(self, expression): raise def _parse(self, expression): - self.tokenizer = lexer.Lexer().tokenize(expression) - self._tokens = list(self.tokenizer) + self.tokenizer = None + # The lexer returns a fully materialized token list, which we + # can use directly for our two tokens of lookahead. + self._tokens = lexer.Lexer().tokenize(expression) self._index = 0 parsed = self._expression(binding_power=0) if not self._current_token() == 'eof': @@ -127,51 +147,69 @@ def _parse(self, expression): return ParsedResult(expression, parsed) def _expression(self, binding_power=0): - left_token = self._lookahead_token(0) - self._advance() - nud_function = getattr( - self, '_token_nud_%s' % left_token['type'], - self._error_nud_token) - left = nud_function(left_token) - current_token = self._current_token() - while binding_power < self.BINDING_POWER[current_token]: - led = getattr(self, '_token_led_%s' % current_token, None) + # This is the hot path of the parser, so the token stream + # accesses (self._lookahead_token/_advance/_current_token) are + # inlined and the nud/led handlers are resolved through the + # precomputed dispatch tables built at the bottom of this + # module rather than per-token getattr calls. + tokens = self._tokens + index = self._index + left_token = tokens[index] + self._index = index + 1 + nud_function = self._NUD_DISPATCH.get(left_token['type']) + if nud_function is None: + # No nud handler exists for this token type, so it cannot + # start an expression. The error handler is resolved on the + # concrete class so a subclass override is honored. + nud_function = type(self)._error_nud_token + left = nud_function(self, left_token) + binding_power_table = self.BINDING_POWER + led_dispatch = self._LED_DISPATCH + current_token = tokens[self._index]['type'] + while binding_power < binding_power_table[current_token]: + led = led_dispatch.get(current_token) if led is None: - error_token = self._lookahead_token(0) - self._error_led_token(error_token) + # No led handler for a token the binding power let us + # enter on: it can't continue an expression. + self._error_led_token(tokens[self._index]) else: - self._advance() - left = led(left) - current_token = self._current_token() + self._index += 1 + left = led(self, left) + current_token = tokens[self._index]['type'] return left + # The ast.* factory calls are inlined as dict literals in the + # token handlers below: at a few hundred thousand nodes per + # second the function call per node is a measurable share of the + # parse time. + def _token_nud_literal(self, token): - return ast.literal(token['value']) + return {'type': 'literal', 'value': token['value'], 'children': []} def _token_nud_unquoted_identifier(self, token): - return ast.field(token['value']) + return {'type': 'field', 'children': [], 'value': token['value']} def _token_nud_quoted_identifier(self, token): - field = ast.field(token['value']) + field = {'type': 'field', 'children': [], 'value': token['value']} # You can't have a quoted identifier as a function # name. - if self._current_token() == 'lparen': - t = self._lookahead_token(0) + if self._tokens[self._index]['type'] == 'lparen': + t = self._tokens[self._index] raise exceptions.ParseError( 0, t['value'], t['type'], 'Quoted identifier not allowed for function names.') return field def _token_nud_star(self, token): - left = ast.identity() - if self._current_token() == 'rbracket': - right = ast.identity() + left = {'type': 'identity', 'children': []} + if self._tokens[self._index]['type'] == 'rbracket': + right = {'type': 'identity', 'children': []} else: right = self._parse_projection_rhs(self.BINDING_POWER['star']) - return ast.value_projection(left, right) + return {'type': 'value_projection', 'children': [left, right]} def _token_nud_filter(self, token): - return self._token_led_filter(ast.identity()) + return self._token_led_filter({'type': 'identity', 'children': []}) def _token_nud_lbrace(self, token): return self._parse_multi_select_hash() @@ -182,29 +220,33 @@ def _token_nud_lparen(self, token): return expression def _token_nud_flatten(self, token): - left = ast.flatten(ast.identity()) + left = {'type': 'flatten', + 'children': [{'type': 'identity', 'children': []}]} right = self._parse_projection_rhs( self.BINDING_POWER['flatten']) - return ast.projection(left, right) + return {'type': 'projection', 'children': [left, right]} def _token_nud_not(self, token): expr = self._expression(self.BINDING_POWER['not']) - return ast.not_expression(expr) + return {'type': 'not_expression', 'children': [expr]} def _token_nud_lbracket(self, token): - if self._current_token() in ['number', 'colon']: + current_token = self._tokens[self._index]['type'] + if current_token in ('number', 'colon'): right = self._parse_index_expression() # We could optimize this and remove the identity() node. # We don't really need an index_expression node, we can # just use emit an index node here if we're not dealing # with a slice. - return self._project_if_slice(ast.identity(), right) - elif self._current_token() == 'star' and \ - self._lookahead(1) == 'rbracket': - self._advance() - self._advance() + return self._project_if_slice( + {'type': 'identity', 'children': []}, right) + elif current_token == 'star' and \ + self._tokens[self._index + 1]['type'] == 'rbracket': + self._index += 2 right = self._parse_projection_rhs(self.BINDING_POWER['star']) - return ast.projection(ast.identity(), right) + return {'type': 'projection', + 'children': [{'type': 'identity', 'children': []}, + right]} else: return self._parse_multi_select_list() @@ -213,13 +255,14 @@ def _parse_index_expression(self): # [ # ^ # | current token - if (self._lookahead(0) == 'colon' or - self._lookahead(1) == 'colon'): + if (self._tokens[self._index]['type'] == 'colon' or + self._tokens[self._index + 1]['type'] == 'colon'): return self._parse_slice_expression() else: # Parse the syntax [number] - node = ast.index(self._lookahead_token(0)['value']) - self._advance() + node = {'type': 'index', 'children': [], + 'value': self._tokens[self._index]['value']} + self._index += 1 self._match('rbracket') return node @@ -248,38 +291,38 @@ def _parse_slice_expression(self): return ast.slice(*parts) def _token_nud_current(self, token): - return ast.current_node() + return {'type': 'current', 'children': []} def _token_nud_expref(self, token): expression = self._expression(self.BINDING_POWER['expref']) - return ast.expref(expression) + return {'type': 'expref', 'children': [expression]} def _token_led_dot(self, left): - if not self._current_token() == 'star': + if not self._tokens[self._index]['type'] == 'star': right = self._parse_dot_rhs(self.BINDING_POWER['dot']) if left['type'] == 'subexpression': left['children'].append(right) return left else: - return ast.subexpression([left, right]) + return {'type': 'subexpression', 'children': [left, right]} else: # We're creating a projection. - self._advance() + self._index += 1 right = self._parse_projection_rhs( self.BINDING_POWER['dot']) - return ast.value_projection(left, right) + return {'type': 'value_projection', 'children': [left, right]} def _token_led_pipe(self, left): right = self._expression(self.BINDING_POWER['pipe']) - return ast.pipe(left, right) + return {'type': 'pipe', 'children': [left, right]} def _token_led_or(self, left): right = self._expression(self.BINDING_POWER['or']) - return ast.or_expression(left, right) + return {'type': 'or_expression', 'children': [left, right]} def _token_led_and(self, left): right = self._expression(self.BINDING_POWER['and']) - return ast.and_expression(left, right) + return {'type': 'and_expression', 'children': [left, right]} def _token_led_lparen(self, left): if left['type'] != 'field': @@ -292,24 +335,26 @@ def _token_led_lparen(self, left): "Invalid function name '%s'" % prev_t['value']) name = left['value'] args = [] - while not self._current_token() == 'rparen': + tokens = self._tokens + while not tokens[self._index]['type'] == 'rparen': expression = self._expression() - if self._current_token() == 'comma': - self._match('comma') + if tokens[self._index]['type'] == 'comma': + self._index += 1 args.append(expression) self._match('rparen') - function_node = ast.function_expression(name, args) - return function_node + return {'type': 'function_expression', 'children': args, + 'value': name} def _token_led_filter(self, left): # Filters are projections. condition = self._expression(0) self._match('rbracket') - if self._current_token() == 'flatten': - right = ast.identity() + if self._tokens[self._index]['type'] == 'flatten': + right = {'type': 'identity', 'children': []} else: right = self._parse_projection_rhs(self.BINDING_POWER['filter']) - return ast.filter_projection(left, right, condition) + return {'type': 'filter_projection', + 'children': [left, right, condition]} def _token_led_eq(self, left): return self._parse_comparator(left, 'eq') @@ -330,14 +375,14 @@ def _token_led_lte(self, left): return self._parse_comparator(left, 'lte') def _token_led_flatten(self, left): - left = ast.flatten(left) + left = {'type': 'flatten', 'children': [left]} right = self._parse_projection_rhs( self.BINDING_POWER['flatten']) - return ast.projection(left, right) + return {'type': 'projection', 'children': [left, right]} def _token_led_lbracket(self, left): - token = self._lookahead_token(0) - if token['type'] in ['number', 'colon']: + token = self._tokens[self._index] + if token['type'] in ('number', 'colon'): right = self._parse_index_expression() if left['type'] == 'index_expression': # Optimization: if the left node is an index expr, @@ -352,37 +397,42 @@ def _token_led_lbracket(self, left): self._match('star') self._match('rbracket') right = self._parse_projection_rhs(self.BINDING_POWER['star']) - return ast.projection(left, right) + return {'type': 'projection', 'children': [left, right]} def _project_if_slice(self, left, right): - index_expr = ast.index_expression([left, right]) + index_expr = {'type': 'index_expression', 'children': [left, right]} if right['type'] == 'slice': - return ast.projection( - index_expr, - self._parse_projection_rhs(self.BINDING_POWER['star'])) + return {'type': 'projection', + 'children': [ + index_expr, + self._parse_projection_rhs( + self.BINDING_POWER['star'])]} else: return index_expr def _parse_comparator(self, left, comparator): right = self._expression(self.BINDING_POWER[comparator]) - return ast.comparator(comparator, left, right) + return {'type': 'comparator', 'children': [left, right], + 'value': comparator} def _parse_multi_select_list(self): expressions = [] + tokens = self._tokens while True: expression = self._expression() expressions.append(expression) - if self._current_token() == 'rbracket': + if tokens[self._index]['type'] == 'rbracket': break else: self._match('comma') self._match('rbracket') - return ast.multi_select_list(expressions) + return {'type': 'multi_select_list', 'children': expressions} def _parse_multi_select_hash(self): pairs = [] + tokens = self._tokens while True: - key_token = self._lookahead_token(0) + key_token = tokens[self._index] # Before getting the token value, verify it's # an identifier. self._match_multiple_tokens( @@ -390,29 +440,32 @@ def _parse_multi_select_hash(self): key_name = key_token['value'] self._match('colon') value = self._expression(0) - node = ast.key_val_pair(key_name=key_name, node=value) + node = {'type': 'key_val_pair', 'children': [value], + 'value': key_name} pairs.append(node) - if self._current_token() == 'comma': - self._match('comma') - elif self._current_token() == 'rbrace': - self._match('rbrace') + if tokens[self._index]['type'] == 'comma': + self._index += 1 + elif tokens[self._index]['type'] == 'rbrace': + self._index += 1 break - return ast.multi_select_dict(nodes=pairs) + return {'type': 'multi_select_dict', 'children': pairs} def _parse_projection_rhs(self, binding_power): # Parse the right hand side of the projection. - if self.BINDING_POWER[self._current_token()] < self._PROJECTION_STOP: + current_token = self._tokens[self._index]['type'] + if self.BINDING_POWER[current_token] < self._PROJECTION_STOP: # BP of 10 are all the tokens that stop a projection. - right = ast.identity() - elif self._current_token() == 'lbracket': + right = {'type': 'identity', 'children': []} + elif current_token == 'lbracket': right = self._expression(binding_power) - elif self._current_token() == 'filter': + elif current_token == 'filter': right = self._expression(binding_power) - elif self._current_token() == 'dot': - self._match('dot') + elif current_token == 'dot': + # inline'd self._match('dot'): the type was just checked. + self._index += 1 right = self._parse_dot_rhs(binding_power) else: - self._raise_parse_error_for_token(self._lookahead_token(0), + self._raise_parse_error_for_token(self._tokens[self._index], 'syntax error') return right @@ -425,18 +478,18 @@ def _parse_dot_rhs(self, binding_power): # * # In terms of tokens that means that after a '.', # you can have: - lookahead = self._current_token() + lookahead = self._tokens[self._index]['type'] # Common case "foo.bar", so first check for an identifier. - if lookahead in ['quoted_identifier', 'unquoted_identifier', 'star']: + if lookahead in ('quoted_identifier', 'unquoted_identifier', 'star'): return self._expression(binding_power) elif lookahead == 'lbracket': - self._match('lbracket') + self._index += 1 return self._parse_multi_select_list() elif lookahead == 'lbrace': - self._match('lbrace') + self._index += 1 return self._parse_multi_select_hash() else: - t = self._lookahead_token(0) + t = self._tokens[self._index] allowed = ['quoted_identifier', 'unquoted_identifier', 'lbracket', 'lbrace'] msg = ( @@ -454,19 +507,18 @@ def _error_led_token(self, token): self._raise_parse_error_for_token(token, 'invalid token') def _match(self, token_type=None): - # inline'd self._current_token() - if self._current_token() == token_type: - # inline'd self._advance() - self._advance() + # inline'd self._current_token() and self._advance() + if self._tokens[self._index]['type'] == token_type: + self._index += 1 else: self._raise_parse_error_maybe_eof( - token_type, self._lookahead_token(0)) + token_type, self._tokens[self._index]) def _match_multiple_tokens(self, token_types): - if self._current_token() not in token_types: + if self._tokens[self._index]['type'] not in token_types: self._raise_parse_error_maybe_eof( - token_types, self._lookahead_token(0)) - self._advance() + token_types, self._tokens[self._index]) + self._index += 1 def _advance(self): self._index += 1 @@ -512,7 +564,17 @@ def __init__(self, expression, parsed): self.parsed = parsed def search(self, value, options=None): - interpreter = visitor.TreeInterpreter(options) + if options is None: + # A TreeInterpreter with default options is stateless + # across searches, so a shared instance is used rather + # than constructing an interpreter (and its Options and + # Functions instances) on every search. Its method cache + # is fully pre-populated (see below) so concurrent default + # searches only read from it -- no mutation, hence safe on + # free-threaded builds. + interpreter = _DEFAULT_INTERPRETER + else: + interpreter = visitor.TreeInterpreter(options) result = interpreter.visit(self.parsed, value) return result @@ -532,3 +594,20 @@ def _render_dot_file(self): def __repr__(self): return repr(self.parsed) + + +_DEFAULT_INTERPRETER = visitor.TreeInterpreter() + + +def _prewarm_default_interpreter(): + # Resolve every visit_* handler up front so the shared interpreter's + # method cache is fully populated at import time and never written + # to during a search. A read-only cache lets concurrent default + # searches share the interpreter safely, including on free-threaded + # (no-GIL) CPython builds. + for name in dir(visitor.TreeInterpreter): + if name.startswith('visit_'): + _DEFAULT_INTERPRETER._resolve_method(name[len('visit_'):]) + + +_prewarm_default_interpreter() diff --git a/jmespath/visitor.py b/jmespath/visitor.py index 15fb1774..f93ab472 100644 --- a/jmespath/visitor.py +++ b/jmespath/visitor.py @@ -82,9 +82,19 @@ def visit(self, node, *args, **kwargs): class Visitor(object): def __init__(self): + # Caches the resolved visit_* method per node type so repeated + # nodes of the same type skip getattr resolution. The cache + # holds unbound functions, called with the instance passed + # explicitly (self), which avoids allocating a bound method per + # lookup. self._method_cache = {} def visit(self, node, *args, **kwargs): + # Base dispatch resolves visit_* as a bound method via + # getattr(self, ...), which honors instance-level handlers and + # staticmethods on Visitor subclasses. TreeInterpreter + # overrides this with a faster unbound-dispatch version for the + # evaluation hot path. node_type = node['type'] method = self._method_cache.get(node_type) if method is None: @@ -93,6 +103,16 @@ def visit(self, node, *args, **kwargs): self._method_cache[node_type] = method return method(node, *args, **kwargs) + def _resolve_method(self, node_type): + # Cold path of TreeInterpreter's dispatch: resolve the visit_* + # method for a node type on the concrete class (as an unbound + # function, called with self passed explicitly) and cache it. + method = getattr( + type(self), 'visit_%s' % node_type, + type(self).default_visit) + self._method_cache[node_type] = method + return method + def default_visit(self, node, *args, **kwargs): raise NotImplementedError("default_visit") @@ -106,7 +126,7 @@ class TreeInterpreter(Visitor): 'lte': operator.le, 'gte': operator.ge } - _EQUALITY_OPS = ['eq', 'ne'] + _EQUALITY_OPS = frozenset(['eq', 'ne']) MAP_TYPE = dict def __init__(self, options=None): @@ -125,13 +145,32 @@ def __init__(self, options=None): def default_visit(self, node, *args, **kwargs): raise NotImplementedError(node['type']) + def visit(self, node, value): + # Overrides the base visit with a fixed two-argument signature: + # this is by far the hottest call in evaluation and avoiding + # *args/**kwargs packing per node is a measurable win. Child + # nodes are dispatched back through self.visit() (not the + # method cache directly) so that subclasses overriding visit() + # still observe every node. + node_type = node['type'] + method = self._method_cache.get(node_type) + if method is None: + method = self._resolve_method(node_type) + return method(self, node, value) + def visit_subexpression(self, node, value): result = value - for node in node['children']: - result = self.visit(node, result) + visit = self.visit + for child in node['children']: + result = visit(child, result) return result def visit_field(self, node, value): + # Missing paths commonly propagate None through a chain of + # fields; short-circuit instead of raising AttributeError for + # every remaining field in the chain. + if value is None: + return None try: return value.get(node['value']) except AttributeError: @@ -139,19 +178,17 @@ def visit_field(self, node, value): def visit_comparator(self, node, value): # Common case: comparator is == or != - comparator_func = self.COMPARATOR_FUNC[node['value']] - if node['value'] in self._EQUALITY_OPS: - return comparator_func( - self.visit(node['children'][0], value), - self.visit(node['children'][1], value) - ) + node_value = node['value'] + comparator_func = self.COMPARATOR_FUNC[node_value] + children = node['children'] + left = self.visit(children[0], value) + right = self.visit(children[1], value) + if node_value in self._EQUALITY_OPS: + return comparator_func(left, right) else: # Ordering operators are only valid for numbers. # Evaluating any other type with a comparison operator # will yield a None value. - left = self.visit(node['children'][0], value) - right = self.visit(node['children'][1], value) - num_types = (int, float) if not (_is_comparable(left) and _is_comparable(right)): return None @@ -164,23 +201,26 @@ def visit_expref(self, node, value): return _Expression(node['children'][0], self) def visit_function_expression(self, node, value): - resolved_args = [] - for child in node['children']: - current = self.visit(child, value) - resolved_args.append(current) + visit = self.visit + resolved_args = [visit(child, value) for child in node['children']] return self._functions.call_function(node['value'], resolved_args) def visit_filter_projection(self, node, value): - base = self.visit(node['children'][0], value) + children = node['children'] + base = self.visit(children[0], value) if not isinstance(base, list): return None - comparator_node = node['children'][2] + comparator_node = children[2] + right = children[1] + visit = self.visit + is_true = self._is_true collected = [] + append = collected.append for element in base: - if self._is_true(self.visit(comparator_node, element)): - current = self.visit(node['children'][1], element) + if is_true(visit(comparator_node, element)): + current = visit(right, element) if current is not None: - collected.append(current) + append(current) return collected def visit_flatten(self, node, value): @@ -211,8 +251,9 @@ def visit_index(self, node, value): def visit_index_expression(self, node, value): result = value - for node in node['children']: - result = self.visit(node, result) + visit = self.visit + for child in node['children']: + result = visit(child, result) return result def visit_slice(self, node, value): @@ -230,30 +271,31 @@ def visit_literal(self, node, value): def visit_multi_select_dict(self, node, value): if value is None: return None + visit = self.visit collected = self._dict_cls() for child in node['children']: - collected[child['value']] = self.visit(child, value) + collected[child['value']] = visit(child, value) return collected def visit_multi_select_list(self, node, value): if value is None: return None - collected = [] - for child in node['children']: - collected.append(self.visit(child, value)) - return collected + visit = self.visit + return [visit(child, value) for child in node['children']] def visit_or_expression(self, node, value): - matched = self.visit(node['children'][0], value) + children = node['children'] + matched = self.visit(children[0], value) if self._is_false(matched): - matched = self.visit(node['children'][1], value) + matched = self.visit(children[1], value) return matched def visit_and_expression(self, node, value): - matched = self.visit(node['children'][0], value) + children = node['children'] + matched = self.visit(children[0], value) if self._is_false(matched): return matched - return self.visit(node['children'][1], value) + return self.visit(children[1], value) def visit_not_expression(self, node, value): original_result = self.visit(node['children'][0], value) @@ -265,19 +307,23 @@ def visit_not_expression(self, node, value): def visit_pipe(self, node, value): result = value - for node in node['children']: - result = self.visit(node, result) + visit = self.visit + for child in node['children']: + result = visit(child, result) return result def visit_projection(self, node, value): base = self.visit(node['children'][0], value) if not isinstance(base, list): return None + right = node['children'][1] + visit = self.visit collected = [] + append = collected.append for element in base: - current = self.visit(node['children'][1], element) + current = visit(right, element) if current is not None: - collected.append(current) + append(current) return collected def visit_value_projection(self, node, value): @@ -286,19 +332,23 @@ def visit_value_projection(self, node, value): base = base.values() except AttributeError: return None + right = node['children'][1] + visit = self.visit collected = [] + append = collected.append for element in base: - current = self.visit(node['children'][1], element) + current = visit(right, element) if current is not None: - collected.append(current) + append(current) return collected def _is_false(self, value): # This looks weird, but we're explicitly using equality checks # because the truth/false values are different between - # python and jmespath. - return (value == '' or value == [] or value == {} or value is None or - value is False) + # python and jmespath. The identity checks are first purely + # because they are the cheapest. + return (value is None or value is False or value == '' or + value == [] or value == {}) def _is_true(self, value): return not self._is_false(value)