diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 00000000..7bc397ff --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-08-05 - Avoid eager materialization of AST child nodes +**Learning:** Eagerly materializing AST node generators (e.g., `list(ast.iter_child_nodes(node))`) into lists creates unnecessary memory allocations and performance overhead, particularly during deep or recursive AST traversals in static analysis like `wardline`'s taint tracking. Functions in Python accept iterables perfectly fine without lists. +**Action:** When implementing or modifying AST traversal functions, pass generators directly and update type hints to accept `Iterable[ast.AST]` from `collections.abc` instead of eagerly converting them to lists. diff --git a/src/wardline/scanner/taint/variable_level.py b/src/wardline/scanner/taint/variable_level.py index a51a6989..6a3868cf 100644 --- a/src/wardline/scanner/taint/variable_level.py +++ b/src/wardline/scanner/taint/variable_level.py @@ -33,7 +33,7 @@ from wardline.core.taints import _PROVENANCE_CLASH, RAW_ZONE, TRUST_RANK, TaintState, combine if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Iterable, Iterator # Serialisation sinks — calls that cross the representation boundary. Their # output sheds validation provenance (raw bytes/str), so → UNKNOWN_RAW. This is @@ -2474,8 +2474,9 @@ def compute_return_callee( return None +# Optimization: Taking Iterable instead of list avoids eager materialization of generators def _assignment_callee( - nodes: list[ast.AST], + nodes: Iterable[ast.AST], name: str, worst: TaintState, function_taint: TaintState, @@ -2508,9 +2509,7 @@ def _assignment_callee( and _resolve_expr(node.value, function_taint, taint_map, var_taints) == worst ): result = callee - nested = _assignment_callee( - list(ast.iter_child_nodes(node)), name, worst, function_taint, taint_map, var_taints - ) + nested = _assignment_callee(ast.iter_child_nodes(node), name, worst, function_taint, taint_map, var_taints) if nested is not None: result = nested return result @@ -2526,8 +2525,9 @@ def _return_callee(node: ast.expr) -> str | None: return None +# Optimization: Taking Iterable instead of list avoids eager materialization of generators def _collect_return_paths( - nodes: list[ast.AST], + nodes: Iterable[ast.AST], function_taint: TaintState, taint_map: dict[str, TaintState], var_taints: dict[str, TaintState], @@ -2558,7 +2558,7 @@ def _collect_return_paths( taint = _resolve_expr(node.value, function_taint, taint_map, dict(snapshot or var_taints)) out.append((taint, _return_callee(node.value), node.value)) _collect_return_paths( - list(ast.iter_child_nodes(node)), + ast.iter_child_nodes(node), function_taint, taint_map, var_taints,