From c853e183088ba1953ee188d469be35c8f1bfefe7 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 7 Aug 2026 10:37:35 +0000 Subject: [PATCH] Implement analyzer-owned Bash loop state --- IMPLEMENTATION_PLAN.md | 16 +- SPEC.md | 4 +- .../v0-3-structured-shell-analysis/design.md | 14 +- .../specs/bounded-shell-analysis/spec.md | 10 + .../v0-3-structured-shell-analysis/tasks.md | 28 +- .../Bash/Parsing/BashAbstractStateAnalyzer.cs | 500 ++++++++++++++++-- .../Bash/Parsing/BashCommandParser.cs | 8 +- .../Internal/Bash/Parsing/BashLoopAnalysis.cs | 309 +++++++++-- .../Bash/Parsing/BashStructuralCoordinator.cs | 294 +++++++--- src/ShellSyntaxTree/ShellSyntaxProjection.cs | 12 + .../bash/205_v03_for_literal_finite.json | 4 +- .../206_v03_for_iterator_substitution.json | 4 +- .../bash/207_v03_for_static_pattern.json | 4 +- .../Corpus/bash/208_v03_for_dynamic_root.json | 4 +- .../bash/209_v03_for_option_injection.json | 4 +- .../bash/210_v03_for_nested_correlation.json | 4 +- .../bash/211_v03_for_pipeline_ancestry.json | 8 +- .../bash/213_v03_for_candidate_cap_32.json | 4 +- .../214_v03_for_candidate_overflow_33.json | 4 +- .../215_v03_for_pattern_parent_traversal.json | 4 +- .../216_v03_for_dot_glob_parent_escape.json | 4 +- ..._v03_for_ordered_duplicate_post_state.json | 57 ++ .../240_v03_for_empty_or_unreachable.json | 53 ++ .../241_v03_for_break_transfer_rejected.json | 10 + ...03_for_wrapped_exit_transfer_rejected.json | 10 + .../Corpus/bash/46_unparseable_for_loop.json | 2 +- .../Parsing/BashForInStructuralTests.cs | 163 +++++- 27 files changed, 1345 insertions(+), 193 deletions(-) create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/239_v03_for_ordered_duplicate_post_state.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/240_v03_for_empty_or_unreachable.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/241_v03_for_break_transfer_rejected.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/242_v03_for_wrapped_exit_transfer_rejected.json diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index b294443..47f3328 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -291,11 +291,15 @@ priorities. flow partitions, failure-aware `cd`, conservative `lastpipe` / `pipefail`, ordered duplicate-preserving loop plans, inherited but isolated decoded-wrapper state, and dynamic fail-closed compatibility attribution - whenever cwd joins to Unknown. Keep OpenSpec task 6.5 open, then add the - remaining loop cases and Netclaw approval matrix after implementation. - The non-loop state engine is now implemented for lists, pipelines, - substitutions, subshells, and decoded wrappers; loop iteration state and - removal of the temporary mutation rejection remain next. An adversarial + whenever cwd joins to Unknown. The analyzer now owns persistent loop + bindings, ordered and empty iteration, occurrence-fact joins, unreachable + flow partitions, substitution inheritance, and explicit decoded-wrapper + remapping of loop plans and argument provenance. Unknown-cardinality + loops use bounded fixed-point widening, and a 4096-transition global + budget fails nested cross-products atomically. Static bodies retain an + exact incoming cwd when no transfer can change it. Keep OpenSpec task 6.5 + open for full effective-argv transfer and removal of the temporary loop + mutation rejection, then add the Netclaw approval matrix. An adversarial pre-implementation review halted the first loop-state draft: parser-time binding frames could not model zero-iteration persistence, correlated nested iterables, special Bash variables, or candidate-derived `cd` @@ -307,7 +311,7 @@ priorities. cwd-changing loop body. Corpus-pin `HOME`, `RANDOM`, `LINENO`, `PATH`, `CDPATH`, `IFS`, 32/33 ordered visits, zero iterations, nested correlation, wrapped transfers, wrapper mapping, substitutions, and - pipelines in the same vertical slice. + pipelines across these two analyzer slices. - [ ] Complete PowerShell `$()` discovery in `foreach` expressions and add the Netclaw approval-matrix cases. The simple-command slice is delivered for ordinary, adjacent, quoted, here-string, redirect, standalone, diff --git a/SPEC.md b/SPEC.md index c240b53..5e53307 100644 --- a/SPEC.md +++ b/SPEC.md @@ -599,7 +599,9 @@ iteration plan: `a b a` performs three state transitions and leaves an exact final binding of `a`; 33 authored values use widening even when every value is the same. An iterable that depends on an outer binding is evaluated separately for each concrete outer visit so correlated nested state is not flattened into -an artificial cross-product. +an artificial cross-product. The analyzer permits at most 4096 total loop-body +transitions per parse; exceeding that resource budget makes the complete result +unparseable rather than returning a partial cross-product. Each concrete iteration assigns its candidate into the analyzer variable map, re-evaluates the complete effective argument vector for every body occurrence, diff --git a/openspec/changes/v0-3-structured-shell-analysis/design.md b/openspec/changes/v0-3-structured-shell-analysis/design.md index 0d2bc1a..305c2d5 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/design.md +++ b/openspec/changes/v0-3-structured-shell-analysis/design.md @@ -453,10 +453,11 @@ proved syntax tree, not by exposing the compatibility parser's mutable parse-order cwd attribution. Parse order is not execution-state order for pipelines or conditional lists, and one symbolic loop-body parse cannot prove the cwd of later iterations. The compatibility attribution path remains a -v0.2 leaf-construction detail. Until the abstract pass lands, loop cwd mutation -fails closed, recognized shell-state mutation before or inside a loop fails -closed, nested reuse of an active Bash binding name fails closed, and -occurrence `WorkingDirectory` stays `Unknown`. +v0.2 leaf-construction detail. The ordered-binding pass may retain exact cwd +for reached nonmutating loop occurrences. Loop cwd mutation, recognized shell- +state mutation before or inside a loop, and nested active-name reuse continue +to fail closed until complete effective-argv transfers and repeated mutable +state are modeled. Bash command substitution executes in an isolated subshell state. State changes affect later commands inside that substitution but never the containing command @@ -492,6 +493,11 @@ wrapped builtin forms such as `builtin break` and `command exit`. `eval`, `source` / `.`, and execution-bearing `trap` also fail the whole region closed unless every executable region and state transfer is discovered. +The analyzer also has a parse-wide budget of 4096 loop-body transitions. +Concrete nested products and fixed-point visits consume the same budget; an +overflow fails the complete parse atomically so an adversarial input cannot +force an unbounded cross-product or receive a partial security projection. + The internal loop plan is distinct from the public value-domain summary. It retains an executable word plan parameterized by the incoming analyzer binding map, plus a cardinality of `Never`, `OneOrMore`, or `ZeroOrMore`. Evaluating the diff --git a/openspec/changes/v0-3-structured-shell-analysis/specs/bounded-shell-analysis/spec.md b/openspec/changes/v0-3-structured-shell-analysis/specs/bounded-shell-analysis/spec.md index 555d866..0bfe22a 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/specs/bounded-shell-analysis/spec.md +++ b/openspec/changes/v0-3-structured-shell-analysis/specs/bounded-shell-analysis/spec.md @@ -362,6 +362,11 @@ partition merely to publish exact continuation facts. - **THEN** the internal iteration cardinality is `Never` - **THEN** the following `pwd` retains the exact incoming cwd +#### Scenario: Reached nonmutating loop body retains incoming cwd +- **WHEN** isolated-mode Bash parses a supported loop whose body cannot change cwd +- **THEN** each reached body occurrence retains the exact incoming cwd +- **THEN** a structurally present but unreachable body or continuation still receives conservative cwd facts + #### Scenario: Duplicate iteration values retain order - **WHEN** isolated-mode Bash parses `for f in a b a; do :; done; printf '%s' "$f"` - **THEN** the internal iteration plan retains `a`, `b`, `a` in that order @@ -372,6 +377,11 @@ partition merely to publish exact continuation facts. - **THEN** the internal plan exceeds the concrete-iteration cap - **THEN** it uses bounded fixed-point analysis instead of treating one distinct public value as one visit +#### Scenario: Nested loop analysis stays resource bounded +- **WHEN** nested concrete loops require more than 4096 total body transitions +- **THEN** the complete parse is unparseable +- **THEN** no partial occurrence or compatibility projection is published + #### Scenario: Loop-derived cd option is rebound from effective argv - **WHEN** isolated-mode Bash analyzes `for f in -P /tmp; do cd "$f"; done` - **THEN** the first visit treats `-P` as a `cd` option rather than a path operand diff --git a/openspec/changes/v0-3-structured-shell-analysis/tasks.md b/openspec/changes/v0-3-structured-shell-analysis/tasks.md index 48e8b4f..0280f68 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/tasks.md +++ b/openspec/changes/v0-3-structured-shell-analysis/tasks.md @@ -87,17 +87,23 @@ make the analyzer own persistent bindings, parameterized ordered plans, full argument provenance/effective-argv transfer, occurrence-fact joins, and unreachable exit partitions. - - [ ] 6.5c.2 Implement that corrected contract and corpus-pin `HOME`, - `RANDOM`, `LINENO`, `PATH`, `CDPATH`, `IFS`, 32/33 ordered visits, - zero-iteration state, loop-derived `cd` options, nested correlation, - wrapped transfers, wrapper mapping, substitutions, and pipelines. - - The first static-value slice deliberately leaves occurrence cwd Unknown - and rejects loop shell-state mutation, nested active-binding reuse, or - loops reached after recognized prior shell-state mutation. A separate - structure-aware abstract-state pass is required - before enabling cwd-changing bodies; - mutable parse-order attribution is unsound across pipelines, `&&` / `||`, - substitutions, and repeated iterations. + - [x] 6.5c.2 Make the analyzer own persistent bindings, ordered and empty + iteration, visit-joined effective arguments, unreachable exit partitions, + substitution inheritance, and explicit decoded-wrapper remapping. Pin + special-name rejection, 32/33 visit boundaries, duplicate order, + zero-iteration state, nested correlation, substitutions, pipelines, and + wrapper provenance in unit tests and the Bash corpus. Use bounded + fixed-point widening for unknown cardinality and fail atomically after + 4096 total loop-body transitions. + - [ ] 6.5c.3 Re-parse each visit's complete effective argv for state + transfers, including loop-derived `cd` options and wrapped dispatch; + carry those transfers through the bounded fixed point, then remove only + the temporary mutation rejections whose transfers are fully modeled. + - The analyzer now publishes exact incoming cwd for reached loop occurrences + when no modeled transfer can disagree. It still rejects loop shell-state + mutation, nested active-binding reuse, and loops reached after recognized + prior shell-state mutation until 6.5c.3 reclassifies complete effective argv + and models the corresponding repeated transfers. - [ ] 6.6 Cover empty iterables, separators, multiline bodies, redirects, pipelines, nested loops, and wrapper boundaries. - [ ] 6.7 Add adversarial cases for option injection, mutation, unquoted expansion, indirect expansion, substitutions, and cap overflow. - [ ] 6.8 Add sanitized Bash corpus entries and Netclaw allow/prompt/deny integration cases. diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashAbstractStateAnalyzer.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashAbstractStateAnalyzer.cs index b29fb12..93f3ddf 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashAbstractStateAnalyzer.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashAbstractStateAnalyzer.cs @@ -17,38 +17,69 @@ namespace ShellSyntaxTree.Internal.Bash.Parsing; /// internal sealed class BashAbstractStateAnalyzer { + private const int MaxLoopAnalysisTransitions = 4096; + private readonly BashParserOptions _options; private readonly Func _factsFactory; + private readonly Func _forInPlanFactory; private readonly Dictionary _inputs = new(ClauseReferenceComparer.Instance); + private readonly Dictionary> + _effectiveArguments = new(ClauseReferenceComparer.Instance); + private readonly List _rewrittenForInPlans = new(); + private bool _isComplete = true; + private int _remainingLoopAnalysisTransitions = MaxLoopAnalysisTransitions; private BashAbstractStateAnalyzer( BashParserOptions options, - Func factsFactory) + Func factsFactory, + Func forInPlanFactory) { _options = options; _factsFactory = factsFactory; + _forInPlanFactory = forInPlanFactory; } internal static bool TryAnalyze( ShellBlockSyntax syntax, BashParserOptions options, Func factsFactory, + Func forInPlanFactory, out ShellBlockSyntax analyzedSyntax, - out Func analyzedFacts) + out Func analyzedFacts, + out IReadOnlyList analyzedForInPlans) { - var analyzer = new BashAbstractStateAnalyzer(options, factsFactory); + var analyzer = new BashAbstractStateAnalyzer( + options, + factsFactory, + forInPlanFactory); var initial = new BashAbstractState( options.WorkingDirectory ?? Environment.CurrentDirectory, - hasCompatibilityAttribution: false); + hasCompatibilityAttribution: false, + new BashLoopBindingContext()); analyzer.AnalyzeBlock(syntax, initial); + if (!analyzer._isComplete) + { + analyzedSyntax = syntax; + analyzedFacts = factsFactory; + analyzedForInPlans = Array.Empty(); + return false; + } var facts = new Dictionary( ClauseReferenceComparer.Instance); analyzedSyntax = analyzer.RewriteBlock(syntax, facts); + if (!analyzer._isComplete) + { + analyzedFacts = factsFactory; + analyzedForInPlans = Array.Empty(); + return false; + } + analyzedFacts = simple => facts.TryGetValue(simple.Clause, out var value) ? value : new CommandOccurrenceFacts(); + analyzedForInPlans = analyzer._rewrittenForInPlans.ToArray(); return true; } @@ -63,8 +94,9 @@ private BashFlowResult AnalyzeNode(ShellSyntaxNode node, BashAbstractState input CommandSubstitutionSyntax substitution => AnalyzeIsolatedBody( substitution.Body, input, - resetCompatibilityAttribution: true), - ForEachSyntax => BashFlowResult.Both(input), + resetCompatibilityAttribution: true, + clearBindings: false), + ForEachSyntax forEach => AnalyzeForEach(forEach, input), ConditionLoopSyntax => BashFlowResult.Both(input.WithUnknownCwd()), ConditionalSyntax conditional => AnalyzeConditional(conditional, input), ConditionalBranchSyntax branch => AnalyzeBranch(branch, input), @@ -76,7 +108,13 @@ private BashFlowResult AnalyzeBlock(ShellBlockSyntax block, BashAbstractState in var flow = BashFlowResult.Both(input); foreach (var statement in block.Statements) { - flow = AnalyzeNode(statement, flow.JoinedState); + var joined = flow.JoinedState; + if (joined is null) + { + break; + } + + flow = AnalyzeNode(statement, joined.Value); } return flow; @@ -91,10 +129,12 @@ private BashFlowResult AnalyzeSimple( AnalyzeIsolatedBody( substitution.Body, input, - resetCompatibilityAttribution: true); + resetCompatibilityAttribution: true, + clearBindings: false); } RecordInput(simple.Clause, input); + RecordEffectiveArguments(simple, input); if (!TryGetCwdTransfer(simple.Clause, input, out var success)) { return BashFlowResult.Both(input); @@ -113,7 +153,12 @@ private BashFlowResult AnalyzeList(CommandListSyntax list, BashAbstractState inp { case CompoundOperator.AndIf: { - var right = AnalyzeNode(item.Command, flow.OnSuccess ?? flow.JoinedState); + if (flow.OnSuccess is null) + { + break; + } + + var right = AnalyzeNode(item.Command, flow.OnSuccess.Value); flow = new BashFlowResult( right.OnSuccess, BashAbstractState.JoinNullable(flow.OnFailure, right.OnFailure)); @@ -121,17 +166,28 @@ private BashFlowResult AnalyzeList(CommandListSyntax list, BashAbstractState inp } case CompoundOperator.OrIf: { - var right = AnalyzeNode(item.Command, flow.OnFailure ?? flow.JoinedState); + if (flow.OnFailure is null) + { + break; + } + + var right = AnalyzeNode(item.Command, flow.OnFailure.Value); flow = new BashFlowResult( BashAbstractState.JoinNullable(flow.OnSuccess, right.OnSuccess), right.OnFailure); break; } case CompoundOperator.Sequence: - flow = AnalyzeNode(item.Command, flow.JoinedState); + if (flow.JoinedState is BashAbstractState sequenceInput) + { + flow = AnalyzeNode(item.Command, sequenceInput); + } + break; default: - return BashFlowResult.Both(flow.JoinedState.WithUnknownCwd()); + return flow.JoinedState is BashAbstractState joined + ? BashFlowResult.Both(joined.WithUnknownCwd()) + : flow; } } @@ -152,7 +208,12 @@ private BashFlowResult AnalyzePipeline( // independently controls which exit partition receives it. With no // proved shell options, both partitions conservatively receive the // join of isolated and current-scope outcomes. - var joined = BashAbstractState.Join(input, last!.Value.JoinedState); + if (last?.JoinedState is not BashAbstractState lastState) + { + return new BashFlowResult(null, null); + } + + var joined = BashAbstractState.Join(input, lastState); return BashFlowResult.Both(joined); } @@ -166,17 +227,155 @@ private BashFlowResult AnalyzeGroup(GroupSyntax group, BashAbstractState input) return AnalyzeIsolatedBody( group.Body, input, - resetCompatibilityAttribution: IsDecodedWrapper(group.Body)); + resetCompatibilityAttribution: IsDecodedWrapper(group.Body), + clearBindings: IsDecodedWrapper(group.Body)); + } + + private BashFlowResult AnalyzeForEach( + ForEachSyntax forEach, + BashAbstractState input) + { + var iterator = AnalyzeBlock(forEach.IteratorCommands, input); + var loopInput = iterator.JoinedState; + var sourcePlan = _forInPlanFactory(forEach); + if (sourcePlan is null) + { + _isComplete = false; + return new BashFlowResult(null, null); + } + + if (loopInput is null) + { + return new BashFlowResult(null, null); + } + + var plan = loopInput.Value.Bindings.AnalyzeIterationPlan( + sourcePlan.Words, + OptionsFor(loopInput.Value), + workingDirectoryUnknown: loopInput.Value.WorkingDirectory is null); + if (plan.Cardinality == BashIterationCardinality.Never) + { + RecordUnvisitedBindingArguments(forEach.Body, sourcePlan.BindingName); + return BashFlowResult.Success(loopInput.Value); + } + + if (plan.RequiresFixedPoint) + { + return AnalyzeForEachFixedPoint( + forEach, + loopInput.Value, + sourcePlan, + plan); + } + + BashFlowResult? last = null; + var iterationInput = loopInput.Value; + foreach (var candidate in plan.OrderedCandidates) + { + if (!TryConsumeLoopAnalysisTransition()) + { + return new BashFlowResult(null, null); + } + + iterationInput = iterationInput.WithBinding( + sourcePlan.BindingName, + candidate); + last = AnalyzeBlock(forEach.Body, iterationInput); + if (last.Value.JoinedState is not BashAbstractState next) + { + return new BashFlowResult(null, null); + } + + iterationInput = next; + } + + return last ?? BashFlowResult.Success(loopInput.Value); + } + + private BashFlowResult AnalyzeForEachFixedPoint( + ForEachSyntax forEach, + BashAbstractState loopInput, + BashForInAnalysisPlan sourcePlan, + BashIterationPlan plan) + { + BashAbstractState? success = plan.Cardinality == BashIterationCardinality.ZeroOrMore + ? loopInput + : null; + BashAbstractState? failure = null; + var head = loopInput; + var wideningBase = loopInput; + var nextHead = loopInput; + for (var iteration = 0; + iteration <= ShellAnalysisLimits.MaxValueCandidates; + iteration++) + { + if (!TryConsumeLoopAnalysisTransition()) + { + return new BashFlowResult(null, null); + } + + var body = AnalyzeBlock( + forEach.Body, + head.WithBinding(sourcePlan.BindingName, plan.Summary)); + success = BashAbstractState.JoinNullable(success, body.OnSuccess); + failure = BashAbstractState.JoinNullable(failure, body.OnFailure); + if (body.JoinedState is not BashAbstractState bodyExit) + { + return new BashFlowResult(success, failure); + } + + wideningBase = head; + nextHead = BashAbstractState.Join(head, bodyExit); + if (head.StateEquals(nextHead)) + { + return new BashFlowResult(success, failure); + } + + head = nextHead; + } + + // Widen disagreement after the finite-domain budget, then run the body + // once more so occurrence facts also reflect the widened entry state. + if (!TryConsumeLoopAnalysisTransition()) + { + return new BashFlowResult(null, null); + } + + var widened = BashAbstractState.Widen(wideningBase, nextHead); + var widenedBody = AnalyzeBlock( + forEach.Body, + widened.WithBinding(sourcePlan.BindingName, plan.Summary)); + return new BashFlowResult( + BashAbstractState.JoinNullable(success, widenedBody.OnSuccess), + BashAbstractState.JoinNullable(failure, widenedBody.OnFailure)); + } + + private bool TryConsumeLoopAnalysisTransition() + { + if (_remainingLoopAnalysisTransitions == 0) + { + _isComplete = false; + return false; + } + + _remainingLoopAnalysisTransitions--; + return true; } private BashFlowResult AnalyzeIsolatedBody( ShellBlockSyntax body, BashAbstractState input, - bool resetCompatibilityAttribution) + bool resetCompatibilityAttribution, + bool clearBindings) { var childInput = resetCompatibilityAttribution ? input.WithoutCompatibilityAttribution() : input; + if (clearBindings) + { + childInput = childInput.WithoutBindings(); + } + var inner = AnalyzeBlock(body, childInput); return new BashFlowResult( inner.OnSuccess is null ? null : input, @@ -191,14 +390,19 @@ private BashFlowResult AnalyzeConditional( BashAbstractState? failure = input; foreach (var branch in conditional.Branches) { - var branchFlow = AnalyzeBranch(branch, failure ?? input.WithUnknownCwd()); + if (failure is null) + { + break; + } + + var branchFlow = AnalyzeBranch(branch, failure.Value); success = BashAbstractState.JoinNullable(success, branchFlow.OnSuccess); - failure = BashAbstractState.JoinNullable(failure, branchFlow.OnFailure); + failure = branchFlow.OnFailure; } - if (conditional.Else is not null) + if (conditional.Else is not null && failure is not null) { - var elseFlow = AnalyzeBlock(conditional.Else, failure ?? input.WithUnknownCwd()); + var elseFlow = AnalyzeBlock(conditional.Else, failure.Value); success = BashAbstractState.JoinNullable(success, elseFlow.OnSuccess); failure = elseFlow.OnFailure; } @@ -211,7 +415,12 @@ private BashFlowResult AnalyzeBranch( BashAbstractState input) { var condition = AnalyzeBlock(branch.Condition, input); - var body = AnalyzeBlock(branch.Body, condition.OnSuccess ?? condition.JoinedState); + if (condition.OnSuccess is null) + { + return new BashFlowResult(null, condition.OnFailure); + } + + var body = AnalyzeBlock(branch.Body, condition.OnSuccess.Value); return new BashFlowResult( body.OnSuccess, BashAbstractState.JoinNullable(condition.OnFailure, body.OnFailure)); @@ -229,6 +438,135 @@ private void RecordInput(Clause clause, BashAbstractState input) } } + private void RecordEffectiveArguments( + SimpleCommandSyntax simple, + BashAbstractState input) + { + var sourceFacts = _factsFactory(simple); + if (sourceFacts.ValueProvenance.Count == 0) + { + return; + } + + Dictionary? accumulated = null; + var evaluator = input.Bindings; + foreach (var provenance in sourceFacts.ValueProvenance) + { + if (!evaluator.TryAnalyzeEffectiveValue(provenance.Value, out var domain)) + { + continue; + } + + accumulated ??= GetEffectiveArguments(simple.Clause); + if (accumulated.TryGetValue(provenance.ClauseElementIndex, out var prior)) + { + accumulated[provenance.ClauseElementIndex] = + BashLoopBindingContext.JoinDomains(prior, domain); + } + else + { + accumulated.Add(provenance.ClauseElementIndex, domain); + } + } + } + + private void RecordUnvisitedBindingArguments( + ShellBlockSyntax block, + string bindingName) + { + foreach (var statement in block.Statements) + { + RecordUnvisitedBindingArguments(statement, bindingName); + } + } + + private void RecordUnvisitedBindingArguments( + ShellSyntaxNode node, + string bindingName) + { + switch (node) + { + case SimpleCommandSyntax simple: + var sourceFacts = _factsFactory(simple); + foreach (var provenance in sourceFacts.ValueProvenance) + { + if (BashLoopBindingContext.ReferencesBinding( + provenance.Value, + bindingName)) + { + GetEffectiveArguments(simple.Clause)[provenance.ClauseElementIndex] = + ShellValueDomain.Unknown; + } + } + + foreach (var substitution in simple.Substitutions) + { + RecordUnvisitedBindingArguments(substitution.Body, bindingName); + } + + break; + case ShellBlockSyntax nestedBlock: + RecordUnvisitedBindingArguments(nestedBlock, bindingName); + break; + case PipelineSyntax pipeline: + foreach (var stage in pipeline.Stages) + { + RecordUnvisitedBindingArguments(stage, bindingName); + } + + break; + case CommandListSyntax list: + foreach (var item in list.Items) + { + RecordUnvisitedBindingArguments(item.Command, bindingName); + } + + break; + case GroupSyntax group: + RecordUnvisitedBindingArguments(group.Body, bindingName); + break; + case ForEachSyntax forEach: + RecordUnvisitedBindingArguments(forEach.IteratorCommands, bindingName); + RecordUnvisitedBindingArguments(forEach.Body, bindingName); + break; + case ConditionLoopSyntax loop: + RecordUnvisitedBindingArguments(loop.Condition, bindingName); + RecordUnvisitedBindingArguments(loop.Body, bindingName); + break; + case ConditionalSyntax conditional: + foreach (var branch in conditional.Branches) + { + RecordUnvisitedBindingArguments(branch, bindingName); + } + + if (conditional.Else is not null) + { + RecordUnvisitedBindingArguments(conditional.Else, bindingName); + } + + break; + case ConditionalBranchSyntax branch: + RecordUnvisitedBindingArguments(branch.Condition, bindingName); + RecordUnvisitedBindingArguments(branch.Body, bindingName); + break; + case CommandSubstitutionSyntax substitution: + RecordUnvisitedBindingArguments(substitution.Body, bindingName); + break; + } + } + + private Dictionary GetEffectiveArguments(Clause clause) + { + if (_effectiveArguments.TryGetValue(clause, out var accumulated)) + { + return accumulated; + } + + accumulated = new Dictionary(); + _effectiveArguments.Add(clause, accumulated); + return accumulated; + } + private bool TryGetCwdTransfer( Clause clause, BashAbstractState input, @@ -279,7 +617,7 @@ private bool TryGetCwdTransfer( success = string.IsNullOrEmpty(_options.HomeDirectory) ? input.WithUnknownCwd() - : new BashAbstractState(_options.HomeDirectory, true); + : input.WithCwd(_options.HomeDirectory, true); return true; } @@ -291,7 +629,7 @@ private bool TryGetCwdTransfer( if (target.Kind == ArgKind.Tilde && target.Resolved is not null) { - success = new BashAbstractState(target.Resolved, true); + success = input.WithCwd(target.Resolved, true); return true; } @@ -311,7 +649,7 @@ private bool TryGetCwdTransfer( isLiteralBytes: true); success = resolved.Resolved is null ? input.WithUnknownCwd() - : new BashAbstractState(resolved.Resolved, true); + : input.WithCwd(resolved.Resolved, true); return true; } @@ -435,11 +773,7 @@ private ShellSyntaxNode RewriteNode( Items = RewriteItems(list.Items, facts), }, GroupSyntax group => group with { Body = RewriteBlock(group.Body, facts) }, - ForEachSyntax forEach => forEach with - { - IteratorCommands = RewriteBlock(forEach.IteratorCommands, facts), - Body = RewriteBlock(forEach.Body, facts), - }, + ForEachSyntax forEach => RewriteForEach(forEach, facts), ConditionLoopSyntax loop => loop with { Condition = RewriteBlock(loop.Condition, facts), @@ -462,6 +796,30 @@ private ShellSyntaxNode RewriteNode( _ => node, }; + private ForEachSyntax RewriteForEach( + ForEachSyntax forEach, + Dictionary facts) + { + var rewritten = forEach with + { + IteratorCommands = RewriteBlock(forEach.IteratorCommands, facts), + Body = RewriteBlock(forEach.Body, facts), + }; + var plan = _forInPlanFactory(forEach); + if (plan is null) + { + _isComplete = false; + } + else + { + _rewrittenForInPlans.Add(new BashForInAnalysisPlanReference( + rewritten, + plan)); + } + + return rewritten; + } + private SimpleCommandSyntax RewriteSimple( SimpleCommandSyntax simple, Dictionary facts) @@ -477,17 +835,20 @@ private SimpleCommandSyntax RewriteSimple( var sourceFacts = _factsFactory(simple); if (!_inputs.TryGetValue(simple.Clause, out var input)) { - facts.Add(simple.Clause, sourceFacts); - return simple with { Substitutions = substitutions }; + input = new BashAbstractState( + workingDirectory: null, + hasCompatibilityAttribution: true, + bindings: new BashLoopBindingContext()); } var clause = RewriteClause(simple.Clause, input, sourceFacts.CwdPathDependencies); facts.Add(clause, new CommandOccurrenceFacts { - EffectiveArguments = sourceFacts.EffectiveArguments, + EffectiveArguments = CreateEffectiveArguments(simple.Clause), WorkingDirectory = input.ToDomain(), Redirects = RewriteRedirectFacts(sourceFacts.Redirects, clause), CwdPathDependencies = sourceFacts.CwdPathDependencies, + ValueProvenance = sourceFacts.ValueProvenance, IsComplete = sourceFacts.IsComplete, }); return simple with @@ -497,6 +858,29 @@ private SimpleCommandSyntax RewriteSimple( }; } + private IReadOnlyList CreateEffectiveArguments(Clause clause) + { + if (!_effectiveArguments.TryGetValue(clause, out var accumulated) || + accumulated.Count == 0) + { + return Array.Empty(); + } + + var indices = new List(accumulated.Keys); + indices.Sort(); + var effective = new EffectiveArgument[indices.Count]; + for (var index = 0; index < effective.Length; index++) + { + effective[index] = new EffectiveArgument + { + ClauseElementIndex = indices[index], + Value = accumulated[indices[index]], + }; + } + + return effective; + } + private Clause RewriteClause( Clause clause, BashAbstractState input, @@ -1080,31 +1464,52 @@ internal BashFlowResult(BashAbstractState? onSuccess, BashAbstractState? onFailu internal BashAbstractState? OnFailure { get; } - internal BashAbstractState JoinedState => - BashAbstractState.JoinNullable(OnSuccess, OnFailure) ?? - new BashAbstractState(null, true); + internal BashAbstractState? JoinedState => + BashAbstractState.JoinNullable(OnSuccess, OnFailure); internal static BashFlowResult Both(BashAbstractState state) => new(state, state); + + internal static BashFlowResult Success(BashAbstractState state) => new(state, null); } private readonly struct BashAbstractState { internal BashAbstractState( string? workingDirectory, - bool hasCompatibilityAttribution) + bool hasCompatibilityAttribution, + BashLoopBindingContext bindings) { WorkingDirectory = workingDirectory; HasCompatibilityAttribution = hasCompatibilityAttribution; + Bindings = bindings; } internal string? WorkingDirectory { get; } internal bool HasCompatibilityAttribution { get; } - internal BashAbstractState WithUnknownCwd() => new(null, true); + internal BashLoopBindingContext Bindings { get; } + + internal BashAbstractState WithUnknownCwd() => new(null, true, Bindings); + + internal BashAbstractState WithBinding( + string name, + ShellValueDomain domain) => + new( + WorkingDirectory, + HasCompatibilityAttribution, + Bindings.WithBinding(name, domain)); + + internal BashAbstractState WithoutBindings() => + new(WorkingDirectory, HasCompatibilityAttribution, Bindings.WithoutBindings()); + + internal BashAbstractState WithCwd( + string? workingDirectory, + bool hasCompatibilityAttribution) => + new(workingDirectory, hasCompatibilityAttribution, Bindings); internal BashAbstractState WithoutCompatibilityAttribution() => - new(WorkingDirectory, WorkingDirectory is null); + new(WorkingDirectory, WorkingDirectory is null, Bindings); internal ShellValueDomain ToDomain() => WorkingDirectory is null @@ -1115,6 +1520,11 @@ WorkingDirectory is null Values = new[] { WorkingDirectory }, }; + internal bool StateEquals(BashAbstractState other) => + string.Equals(WorkingDirectory, other.WorkingDirectory, StringComparison.Ordinal) && + HasCompatibilityAttribution == other.HasCompatibilityAttribution && + Bindings.StateEquals(other.Bindings); + internal static BashAbstractState Join( BashAbstractState left, BashAbstractState right) => new( @@ -1124,7 +1534,20 @@ internal static BashAbstractState Join( StringComparison.Ordinal) ? left.WorkingDirectory : null, - left.HasCompatibilityAttribution || right.HasCompatibilityAttribution); + left.HasCompatibilityAttribution || right.HasCompatibilityAttribution, + BashLoopBindingContext.JoinState(left.Bindings, right.Bindings)); + + internal static BashAbstractState Widen( + BashAbstractState left, + BashAbstractState right) => new( + string.Equals( + left.WorkingDirectory, + right.WorkingDirectory, + StringComparison.Ordinal) + ? left.WorkingDirectory + : null, + left.HasCompatibilityAttribution || right.HasCompatibilityAttribution, + BashLoopBindingContext.WidenState(left.Bindings, right.Bindings)); internal static BashAbstractState? JoinNullable( BashAbstractState? left, @@ -1152,4 +1575,5 @@ private sealed class ClauseReferenceComparer : IEqualityComparer public int GetHashCode(Clause obj) => RuntimeHelpers.GetHashCode(obj); } + } diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs index c1c8bec..3e39eff 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs @@ -493,9 +493,15 @@ private readonly record struct CwdPathDependencySet( Clause Clause, IReadOnlyList Dependencies); + private readonly record struct ShellValueProvenanceSet( + Clause Clause, + IReadOnlyList Provenance); + private readonly record struct BashParseResult( ParsedCommand Command, - IReadOnlyList CwdPathDependencySets); + IReadOnlyList CwdPathDependencySets, + IReadOnlyList ValueProvenanceSets, + IReadOnlyList ForInPlans); private static ClauseResult ParseClauseSegment( Segment segment, string source, BashParserOptions options, bool workingDirectoryUnknown) diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashLoopAnalysis.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashLoopAnalysis.cs index 48309e9..a4b7a85 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashLoopAnalysis.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashLoopAnalysis.cs @@ -11,6 +11,31 @@ namespace ShellSyntaxTree.Internal.Bash.Parsing; +internal enum BashIterationCardinality +{ + Never, + OneOrMore, + ZeroOrMore, +} + +internal sealed record BashLoopWord( + ShellValue? Value, + bool HasUnmodeledBraceExpansion); + +internal sealed record BashForInAnalysisPlan( + string BindingName, + IReadOnlyList Words); + +internal sealed record BashForInAnalysisPlanReference( + ForEachSyntax Syntax, + BashForInAnalysisPlan Plan); + +internal sealed record BashIterationPlan( + IReadOnlyList OrderedCandidates, + BashIterationCardinality Cardinality, + bool RequiresFixedPoint, + ShellValueDomain Summary); + /// /// Preserves bounded loop-variable proofs while the Bash structural parser /// still owns lexer provenance. Compatibility leaves deliberately retain @@ -21,81 +46,257 @@ internal sealed class BashLoopBindingContext { private readonly List _bindings = new(); - internal int Count => _bindings.Count; - - internal bool Contains(string name) => FindExactBinding(name) is not null; - - internal void Push(string name, ShellValueDomain domain) => - _bindings.Add(new BindingFrame(name, domain)); - - internal void Pop() + internal BashLoopBindingContext WithBinding( + string name, + ShellValueDomain domain) { - if (_bindings.Count > 0) + var clone = Clone(); + for (var index = clone._bindings.Count - 1; index >= 0; index--) { - _bindings.RemoveAt(_bindings.Count - 1); + if (string.Equals(clone._bindings[index].Name, name, StringComparison.Ordinal)) + { + clone._bindings.RemoveAt(index); + } } + + clone._bindings.Add(new BindingFrame(name, domain)); + return clone; } - internal BashLoopBindingContext Clone() + private BashLoopBindingContext Clone() { var clone = new BashLoopBindingContext(); clone._bindings.AddRange(_bindings); return clone; } - internal ShellValueDomain AnalyzeIterable( - IReadOnlyList words, - BashParserOptions options, - bool workingDirectoryUnknown) + internal BashLoopBindingContext WithoutBindings() => new(); + + internal bool StateEquals(BashLoopBindingContext other) { - if (words.Count == 0) + if (_bindings.Count != other._bindings.Count) { - return ShellValueDomain.Unknown; + return false; } - if (words.Count == 1 && - words[0].ResolverValue is not null && - TryBuildStaticPattern( - words[0].ResolverValue!, - options, - workingDirectoryUnknown, - out var pattern)) + foreach (var binding in _bindings) { - return pattern; + var otherBinding = other.FindExactBinding(binding.Name); + if (otherBinding is null || !DomainEquals(binding.Domain, otherBinding.Domain)) + { + return false; + } } - var values = new List(words.Count); + return true; + } + + internal static BashLoopBindingContext JoinState( + BashLoopBindingContext left, + BashLoopBindingContext right) + { + var joined = new BashLoopBindingContext(); + var names = new HashSet(StringComparer.Ordinal); + foreach (var binding in left._bindings) + { + names.Add(binding.Name); + } + + foreach (var binding in right._bindings) + { + names.Add(binding.Name); + } + + foreach (var name in names) + { + var leftBinding = left.FindExactBinding(name); + var rightBinding = right.FindExactBinding(name); + joined._bindings.Add(new BindingFrame( + name, + leftBinding is null || rightBinding is null + ? ShellValueDomain.Unknown + : JoinDomains(leftBinding.Domain, rightBinding.Domain))); + } + + return joined; + } + + internal static BashLoopBindingContext WidenState( + BashLoopBindingContext left, + BashLoopBindingContext right) + { + var widened = new BashLoopBindingContext(); + var names = new HashSet(StringComparer.Ordinal); + foreach (var binding in left._bindings) + { + names.Add(binding.Name); + } + + foreach (var binding in right._bindings) + { + names.Add(binding.Name); + } + + foreach (var name in names) + { + var leftBinding = left.FindExactBinding(name); + var rightBinding = right.FindExactBinding(name); + widened._bindings.Add(new BindingFrame( + name, + leftBinding is not null && + rightBinding is not null && + DomainEquals(leftBinding.Domain, rightBinding.Domain) + ? leftBinding.Domain + : ShellValueDomain.Unknown)); + } + + return widened; + } + + internal static bool ReferencesBinding(ShellValue value, string bindingName) + { + foreach (var fragment in value.Fragments) + { + if (fragment.Kind == ShellValueFragmentKind.Expansion && + fragment.Expansion is ShellExpansionReference expansion && + expansion.Kind == ShellExpansionKind.Variable && + string.Equals(expansion.Name, bindingName, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + internal BashIterationPlan AnalyzeIterationPlan( + IReadOnlyList words, + BashParserOptions options, + bool workingDirectoryUnknown) + { + if (words.Count == 0) + { + return new BashIterationPlan( + Array.Empty(), + BashIterationCardinality.Never, + RequiresFixedPoint: false, + ShellValueDomain.Unknown); + } + + var ordered = new List(words.Count); + var summary = new List(words.Count); var distinct = new HashSet(StringComparer.Ordinal); foreach (var word in words) { - if (word.ResolverValue is null || HasUnmodeledBraceExpansion(word)) + if (word.Value is null || word.HasUnmodeledBraceExpansion) + { + return FixedPointPlan( + BashIterationCardinality.ZeroOrMore, + ShellValueDomain.Unknown); + } + + if (TryBuildStaticPattern( + word.Value, + options, + workingDirectoryUnknown, + out var pattern)) { - return ShellValueDomain.Unknown; + return FixedPointPlan(BashIterationCardinality.ZeroOrMore, pattern); } - ShellValueDomain wordDomain; - if (IsEntirelyLiteral(word.ResolverValue)) + ShellValueDomain domain; + if (IsEntirelyLiteral(word.Value)) { - wordDomain = new ShellValueDomain + domain = new ShellValueDomain { Kind = ShellValueDomainKind.Exact, - Values = new[] { word.ResolverValue.Decoded }, + Values = new[] { word.Value.Decoded }, }; } - else if (!TryAnalyzeEffectiveValue(word.ResolverValue, out wordDomain) || - wordDomain.Kind is not ( - ShellValueDomainKind.Exact or ShellValueDomainKind.FiniteSet)) + else if (!TryAnalyzeEffectiveValue(word.Value, out domain) || + domain.Kind != ShellValueDomainKind.Exact) { - return ShellValueDomain.Unknown; + return FixedPointPlan( + BashIterationCardinality.ZeroOrMore, + domain); } - foreach (var candidate in wordDomain.Values) + ordered.Add(domain); + var candidate = domain.Values[0]; + if (distinct.Add(candidate)) { - if (!distinct.Add(candidate)) - { - continue; - } + summary.Add(candidate); + } + + if (ordered.Count > ShellAnalysisLimits.MaxValueCandidates) + { + return FixedPointPlan( + BashIterationCardinality.OneOrMore, + ShellValueDomain.Unknown); + } + } + + return new BashIterationPlan( + ordered.ToArray(), + BashIterationCardinality.OneOrMore, + RequiresFixedPoint: false, + CreateFiniteDomain(summary)); + } + + internal static BashForInAnalysisPlan CapturePlan( + string bindingName, + IReadOnlyList words) + { + var captured = new BashLoopWord[words.Count]; + for (var index = 0; index < captured.Length; index++) + { + captured[index] = new BashLoopWord( + words[index].ResolverValue, + HasUnmodeledBraceExpansion(words[index])); + } + + return new BashForInAnalysisPlan(bindingName, captured); + } + + private static BashIterationPlan FixedPointPlan( + BashIterationCardinality cardinality, + ShellValueDomain summary) => new( + Array.Empty(), + cardinality, + RequiresFixedPoint: true, + summary); + + internal static ShellValueDomain JoinDomains( + ShellValueDomain left, + ShellValueDomain right) + { + if (DomainEquals(left, right)) + { + return left; + } + + if (left.Kind is not ( + ShellValueDomainKind.Exact or ShellValueDomainKind.FiniteSet) || + right.Kind is not ( + ShellValueDomainKind.Exact or ShellValueDomainKind.FiniteSet)) + { + return ShellValueDomain.Unknown; + } + var values = new List(left.Values.Count + right.Values.Count); + var distinct = new HashSet(StringComparer.Ordinal); + foreach (var candidate in left.Values) + { + if (distinct.Add(candidate)) + { + values.Add(candidate); + } + } + + foreach (var candidate in right.Values) + { + if (distinct.Add(candidate)) + { if (distinct.Count > ShellAnalysisLimits.MaxValueCandidates) { return ShellValueDomain.Unknown; @@ -108,6 +309,32 @@ wordDomain.Kind is not ( return CreateFiniteDomain(values); } + private static bool DomainEquals( + ShellValueDomain left, + ShellValueDomain right) + { + if (left.Kind != right.Kind || + !string.Equals(left.Pattern, right.Pattern, StringComparison.Ordinal) || + !string.Equals( + left.CoveringDirectory, + right.CoveringDirectory, + StringComparison.Ordinal) || + left.Values.Count != right.Values.Count) + { + return false; + } + + for (var index = 0; index < left.Values.Count; index++) + { + if (!string.Equals(left.Values[index], right.Values[index], StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + internal bool TryAnalyzeEffectiveValue( ShellValue value, out ShellValueDomain domain) diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs index 45e86a8..1ec68dc 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs @@ -40,8 +40,10 @@ private static BashParseResult ParseStructured( syntax, options, coordinator.GetFacts, + coordinator.GetForInPlan, out var analyzedSyntax, - out var analyzedFacts) || + out var analyzedFacts, + out var analyzedForInPlans) || !ShellSyntaxProjection.TryProject( analyzedSyntax, analyzedFacts, @@ -62,7 +64,9 @@ private static BashParseResult ParseStructured( }; return new BashParseResult( command, - CreateDependencySets(projection.Commands, analyzedFacts)); + CreateDependencySets(projection.Commands, analyzedFacts), + CreateValueProvenanceSets(projection.Commands, analyzedFacts), + analyzedForInPlans); } private static IReadOnlyList CreateDependencySets( @@ -82,6 +86,23 @@ private static IReadOnlyList CreateDependencySets( return sets; } + private static IReadOnlyList CreateValueProvenanceSets( + IReadOnlyList commands, + Func factsFactory) + { + var sets = new ShellValueProvenanceSet[commands.Count]; + for (var index = 0; index < sets.Length; index++) + { + var clause = commands[index].Clause; + var facts = factsFactory(new SimpleCommandSyntax { Clause = clause }); + sets[index] = new ShellValueProvenanceSet( + clause, + facts.ValueProvenance); + } + + return sets; + } + private static BashParseResult StructuralFailure( string source, string? reason, @@ -95,7 +116,9 @@ private static BashParseResult StructuralFailure( IsUnparseable = true, UnparseableReason = reason, }, - Array.Empty()); + Array.Empty(), + Array.Empty(), + Array.Empty()); private sealed class StructuralCoordinator { @@ -108,9 +131,11 @@ private sealed class StructuralCoordinator private readonly int _sourceStart; private readonly int _sourceLength; private readonly CdAttributionContext _attribution = new(); - private readonly BashLoopBindingContext _bindings; + private readonly List _activeLoopBindings; private readonly Dictionary _facts = new(ClauseReferenceComparer.Instance); + private readonly Dictionary _forInPlans = + new(ForEachReferenceComparer.Instance); private int _position; private int _subshellDepth; private int _loopDepth; @@ -126,7 +151,7 @@ internal StructuralCoordinator( bool markBashCWrapped, int sourceStart, int sourceLength, - BashLoopBindingContext? bindings = null, + IReadOnlyList? activeLoopBindings = null, bool hasUnmodeledShellStateMutation = false, bool hasUnmodeledVariableStateMutation = false) { @@ -138,7 +163,9 @@ internal StructuralCoordinator( _markBashCWrapped = markBashCWrapped; _sourceStart = sourceStart; _sourceLength = sourceLength; - _bindings = bindings ?? new BashLoopBindingContext(); + _activeLoopBindings = activeLoopBindings is null + ? new List() + : new List(activeLoopBindings); _hasUnmodeledShellStateMutation = hasUnmodeledShellStateMutation; _hasUnmodeledVariableStateMutation = hasUnmodeledVariableStateMutation; } @@ -148,6 +175,9 @@ internal CommandOccurrenceFacts GetFacts(SimpleCommandSyntax simple) => ? facts : CreateDefaultFacts(simple.Clause); + internal BashForInAnalysisPlan? GetForInPlan(ForEachSyntax forEach) => + _forInPlans.TryGetValue(forEach, out var plan) ? plan : null; + private CommandOccurrenceFacts CreateDefaultFacts(Clause clause) => new() { IsComplete = clause.Redirects.Count == 0 && @@ -447,7 +477,8 @@ private bool TryParseCommand( compatibilityOperator, _subshellDepth > 0, ref firstLeaf, - out var body)) + out var body, + out var referenceMap)) { error = "decoded bash -c syntax could not be lifted safely"; return false; @@ -463,11 +494,7 @@ private bool TryParseCommand( SourceLength = lastToken.SourceStart + lastToken.SourceLength - firstToken.SourceStart, }; - if (!TryRegisterDecodedFacts( - inner, - innerResult.CwdPathDependencySets, - body, - out error)) + if (!TryRegisterDecodedFacts(innerResult, referenceMap, out error)) { command = null; return false; @@ -516,9 +543,9 @@ private bool TryParseCommand( }; var emitted = AttachAttributionArg(clause, _attribution); var isPotentialStateMutation = IsPotentialBindingMutation(emitted); - if (_bindings.Count > 0 && isPotentialStateMutation) + if (_activeLoopBindings.Count > 0 && isPotentialStateMutation) { - error = "Bash loop binding mutation is not supported for bounded analysis"; + error = "Bash loop state mutation or control transfer is not supported for bounded analysis"; return false; } @@ -614,7 +641,7 @@ private bool TryParseForIn( return false; } - if (_bindings.Contains(bindingToken.Value)) + if (_activeLoopBindings.Contains(bindingToken.Value)) { error = "nested Bash for-in binding reuse requires state propagation"; return false; @@ -697,10 +724,9 @@ BashTokenKind.QuotedString or ? iterableStart : iterableWords[iterableWords.Count - 1].SourceStart + iterableWords[iterableWords.Count - 1].SourceLength; - var iterableDomain = _bindings.AnalyzeIterable( - iterableWords, - _options, - workingDirectoryUnknown: false); + var analysisPlan = BashLoopBindingContext.CapturePlan( + bindingToken.Value, + iterableWords); if (!TryParseIteratorSubstitutions( iterableWords, CurrentOptions(), @@ -712,7 +738,7 @@ BashTokenKind.QuotedString or return false; } - _bindings.Push(bindingToken.Value, iterableDomain); + _activeLoopBindings.Add(bindingToken.Value); _loopDepth++; var parsedBody = TryParseList( stopAtRightParen: false, @@ -721,7 +747,7 @@ BashTokenKind.QuotedString or out var bodyCommand, out error); _loopDepth--; - _bindings.Pop(); + _activeLoopBindings.RemoveAt(_activeLoopBindings.Count - 1); if (!parsedBody) { return false; @@ -747,7 +773,7 @@ BashTokenKind.QuotedString or SourceLength = doneToken.SourceStart - doToken.SourceStart - doToken.SourceLength, }; - command = new ForEachSyntax + var forEach = new ForEachSyntax { Binding = new LoopBindingSyntax { @@ -771,6 +797,8 @@ BashTokenKind.QuotedString or SourceLength = doneToken.SourceStart + doneToken.SourceLength - forToken.SourceStart, }; + _forInPlans.Add(forEach, analysisPlan); + command = forEach; return true; } @@ -1176,7 +1204,7 @@ private bool TryParseSubstitutionBody( _markBashCWrapped, sourceStart, sourceLength, - _bindings.Clone(), + _activeLoopBindings, _hasUnmodeledShellStateMutation, _hasUnmodeledVariableStateMutation); if (!coordinator.TryParse(out body, out error)) @@ -1194,7 +1222,7 @@ private void RegisterFacts( IReadOnlyList pathResolutions, BashParserOptions parseOptions) { - var effective = new List(); + var valueProvenance = new List(); var cwdPathDependencies = new List(); for (var elementIndex = 0; elementIndex < simple.Clause.Elements.Count; @@ -1206,14 +1234,11 @@ private void RegisterFacts( continue; } - if (element.Role == ClauseElementRole.Argument && - _bindings.TryAnalyzeEffectiveValue(value, out var domain)) + if (element.Role == ClauseElementRole.Argument) { - effective.Add(new EffectiveArgument - { - ClauseElementIndex = elementIndex, - Value = domain, - }); + valueProvenance.Add(new ShellValueElementProvenance( + elementIndex, + value)); } } @@ -1237,7 +1262,7 @@ private void RegisterFacts( _facts.Add(simple.Clause, new CommandOccurrenceFacts { - EffectiveArguments = effective.ToArray(), + ValueProvenance = valueProvenance.ToArray(), CwdPathDependencies = cwdPathDependencies.ToArray(), IsComplete = simple.Clause.Redirects.Count == 0 && !HasUnexpandedCommandString(simple.Clause), @@ -1292,53 +1317,103 @@ private void MergeFacts(StructuralCoordinator nested) _facts.Add(pair.Key, pair.Value); } + foreach (var pair in nested._forInPlans) + { + _forInPlans.Add(pair.Key, pair.Value); + } + } private bool TryRegisterDecodedFacts( - ParsedCommand inner, - IReadOnlyList dependencySets, - ShellBlockSyntax clonedBody, + BashParseResult innerResult, + DecodedReferenceMap referenceMap, out string? error) { - if (!ShellSyntaxProjection.TryProject(clonedBody, out var clonedProjection) || - clonedProjection.Commands.Count != inner.Commands.Count) + var inner = innerResult.Command; + foreach (var source in inner.Commands) { - error = "decoded bash -c facts could not be mapped safely"; - return false; - } + if (!referenceMap.TryGetClause(source.Clause, out var clonedClause)) + { + error = "decoded bash -c clause facts could not be mapped safely"; + return false; + } - for (var index = 0; index < inner.Commands.Count; index++) - { - var source = inner.Commands[index]; - _facts.Add(clonedProjection.Commands[index].Clause, new CommandOccurrenceFacts + if (!TryFindValueProvenance( + innerResult.ValueProvenanceSets, + source.Clause, + out var valueProvenance)) + { + error = "decoded bash -c argument provenance could not be mapped safely"; + return false; + } + + if (!TryFindDependencies( + innerResult.CwdPathDependencySets, + source.Clause, + out var cwdPathDependencies)) + { + error = "decoded bash -c cwd provenance could not be mapped safely"; + return false; + } + + _facts.Add(clonedClause, new CommandOccurrenceFacts { - EffectiveArguments = source.EffectiveArguments, - WorkingDirectory = source.WorkingDirectory, Redirects = source.Redirects, - CwdPathDependencies = FindDependencies( - dependencySets, - source.Clause), + CwdPathDependencies = cwdPathDependencies, + ValueProvenance = valueProvenance, IsComplete = source.IsComplete, }); } + foreach (var sourcePlan in innerResult.ForInPlans) + { + if (!referenceMap.TryGetForEach(sourcePlan.Syntax, out var clonedForEach)) + { + error = "decoded bash -c loop plan could not be mapped safely"; + return false; + } + + _forInPlans.Add(clonedForEach, sourcePlan.Plan); + } + error = null; return true; } - private static IReadOnlyList FindDependencies( + private static bool TryFindValueProvenance( + IReadOnlyList provenanceSets, + Clause clause, + out IReadOnlyList provenance) + { + foreach (var set in provenanceSets) + { + if (object.ReferenceEquals(set.Clause, clause)) + { + provenance = set.Provenance; + return true; + } + } + + provenance = Array.Empty(); + return false; + } + + private static bool TryFindDependencies( IReadOnlyList dependencySets, - Clause clause) + Clause clause, + out IReadOnlyList dependencies) { foreach (var set in dependencySets) { if (object.ReferenceEquals(set.Clause, clause)) { - return set.Dependencies; + dependencies = set.Dependencies; + return true; } } - return Array.Empty(); + dependencies = Array.Empty(); + return false; } private static bool IsPotentialBindingMutation(Clause clause) @@ -1352,7 +1427,8 @@ private static bool IsPotentialBindingMutation(Clause clause) if (verb is "unset" or "read" or "readarray" or "mapfile" or "declare" or "typeset" or "local" or "export" or "readonly" or "let" or "eval" or "." or "source" or "getopts" or "set" or - "cd" or "chdir" or "pushd" or "popd" or "trap") + "cd" or "chdir" or "pushd" or "popd" or "trap" or + "break" or "continue" or "return" or "exit" or "exec") { return true; } @@ -1440,6 +1516,16 @@ private sealed class ClauseReferenceComparer : IEqualityComparer public int GetHashCode(Clause obj) => RuntimeHelpers.GetHashCode(obj); } + + private sealed class ForEachReferenceComparer : IEqualityComparer + { + internal static ForEachReferenceComparer Instance { get; } = new(); + + public bool Equals(ForEachSyntax? x, ForEachSyntax? y) => + object.ReferenceEquals(x, y); + + public int GetHashCode(ForEachSyntax obj) => RuntimeHelpers.GetHashCode(obj); + } } private static IReadOnlyList ShiftTokens( @@ -1636,6 +1722,25 @@ private static bool TryCloneDecodedBlock( CompoundOperator firstOperator, bool outerSubshell, ref bool firstLeaf, + out ShellBlockSyntax clone, + out DecodedReferenceMap referenceMap) + { + referenceMap = new DecodedReferenceMap(); + return TryCloneDecodedBlock( + source, + firstOperator, + outerSubshell, + ref firstLeaf, + referenceMap, + out clone); + } + + private static bool TryCloneDecodedBlock( + ShellBlockSyntax source, + CompoundOperator firstOperator, + bool outerSubshell, + ref bool firstLeaf, + DecodedReferenceMap referenceMap, out ShellBlockSyntax clone) { if (source is null || source.Statements is null) @@ -1652,6 +1757,7 @@ private static bool TryCloneDecodedBlock( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var cloned)) { clone = new ShellBlockSyntax(); @@ -1670,6 +1776,7 @@ private static bool TryCloneDecodedNode( CompoundOperator firstOperator, bool outerSubshell, ref bool firstLeaf, + DecodedReferenceMap referenceMap, out ShellSyntaxNode? clone) { clone = null; @@ -1681,6 +1788,7 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var clonedBlock)) { return false; @@ -1697,6 +1805,7 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var clonedSubstitution) || clonedSubstitution is not CommandSubstitutionSyntax typedSubstitution) { @@ -1708,18 +1817,20 @@ private static bool TryCloneDecodedNode( var clauseOperator = firstLeaf ? firstOperator : simple.Clause.Operator; firstLeaf = false; + var clonedClause = simple.Clause with + { + Operator = clauseOperator, + IsSubshell = outerSubshell || simple.Clause.IsSubshell, + IsCommandStringWrapped = true, + Elements = ClauseElementProvenance.WithoutOuterSourceSpans( + simple.Clause.Elements), + }; clone = new SimpleCommandSyntax { - Clause = simple.Clause with - { - Operator = clauseOperator, - IsSubshell = outerSubshell || simple.Clause.IsSubshell, - IsCommandStringWrapped = true, - Elements = ClauseElementProvenance.WithoutOuterSourceSpans( - simple.Clause.Elements), - }, + Clause = clonedClause, Substitutions = substitutions, }; + referenceMap.Add(simple.Clause, clonedClause); return true; case PipelineSyntax pipeline: return TryCloneDecodedCollection( @@ -1727,6 +1838,7 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, stages => new PipelineSyntax { Stages = stages }, out clone); case CommandListSyntax list: @@ -1738,6 +1850,7 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var itemCommand)) { return false; @@ -1758,6 +1871,7 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var groupBody)) { return false; @@ -1771,18 +1885,20 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var iterator) || !TryCloneDecodedBlock( forEach.Body, firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var forBody)) { return false; } - clone = new ForEachSyntax + var clonedForEach = new ForEachSyntax { Binding = new LoopBindingSyntax { @@ -1793,6 +1909,8 @@ private static bool TryCloneDecodedNode( IteratorCommands = iterator, Body = forBody, }; + clone = clonedForEach; + referenceMap.Add(forEach, clonedForEach); return true; case ConditionLoopSyntax loop: if (!TryCloneDecodedBlock( @@ -1800,12 +1918,14 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var condition) || !TryCloneDecodedBlock( loop.Body, firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var loopBody)) { return false; @@ -1827,6 +1947,7 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var clonedBranch) || clonedBranch is not ConditionalBranchSyntax typedBranch) { @@ -1842,6 +1963,7 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out @else)) { return false; @@ -1855,12 +1977,14 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var branchCondition) || !TryCloneDecodedBlock( branch.Body, firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var branchBody)) { return false; @@ -1878,6 +2002,7 @@ private static bool TryCloneDecodedNode( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var substitutionBody)) { return false; @@ -1895,6 +2020,7 @@ private static bool TryCloneDecodedCollection( CompoundOperator firstOperator, bool outerSubshell, ref bool firstLeaf, + DecodedReferenceMap referenceMap, Func, ShellSyntaxNode> factory, out ShellSyntaxNode? clone) { @@ -1906,6 +2032,7 @@ private static bool TryCloneDecodedCollection( firstOperator, outerSubshell, ref firstLeaf, + referenceMap, out var clonedChild)) { clone = null; @@ -1923,4 +2050,45 @@ private static bool TryCloneDecodedCollection( { Raw = source.Raw, }; + + private sealed class DecodedReferenceMap + { + private readonly List<(Clause Source, Clause Clone)> _clauses = new(); + private readonly List<(ForEachSyntax Source, ForEachSyntax Clone)> _forEach = new(); + + internal void Add(Clause source, Clause clone) => _clauses.Add((source, clone)); + + internal void Add(ForEachSyntax source, ForEachSyntax clone) => + _forEach.Add((source, clone)); + + internal bool TryGetClause(Clause source, out Clause clone) + { + foreach (var pair in _clauses) + { + if (object.ReferenceEquals(pair.Source, source)) + { + clone = pair.Clone; + return true; + } + } + + clone = null!; + return false; + } + + internal bool TryGetForEach(ForEachSyntax source, out ForEachSyntax clone) + { + foreach (var pair in _forEach) + { + if (object.ReferenceEquals(pair.Source, source)) + { + clone = pair.Clone; + return true; + } + } + + clone = null!; + return false; + } + } } diff --git a/src/ShellSyntaxTree/ShellSyntaxProjection.cs b/src/ShellSyntaxTree/ShellSyntaxProjection.cs index 13220f2..6982289 100644 --- a/src/ShellSyntaxTree/ShellSyntaxProjection.cs +++ b/src/ShellSyntaxTree/ShellSyntaxProjection.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; +using ShellSyntaxTree.Internal.Resolving; namespace ShellSyntaxTree; @@ -26,9 +27,20 @@ internal sealed class CommandOccurrenceFacts internal IReadOnlyList CwdPathDependencies { get; init; } = Array.Empty(); + internal IReadOnlyList ValueProvenance { get; init; } = + Array.Empty(); + internal bool IsComplete { get; init; } } +/// +/// Retains the complete shell-owned value fragments for one authored clause +/// element so execution-state analysis can re-evaluate it per visit. +/// +internal readonly record struct ShellValueElementProvenance( + int ClauseElementIndex, + ShellValue Value); + /// /// Retains resolver-owned path provenance and its exact compatibility /// coordinates for outcome-sensitive rebasing. Public compatibility DTOs do diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/205_v03_for_literal_finite.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/205_v03_for_literal_finite.json index e3eb0d6..588f16e 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/205_v03_for_literal_finite.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/205_v03_for_literal_finite.json @@ -34,9 +34,9 @@ "effectiveArguments": [ { "clauseElementIndex": 2, "value": { "kind": "FiniteSet", "values": ["a.txt", "b.txt"], "pattern": null, "coveringDirectory": null } } ], - "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + "workingDirectory": { "kind": "Exact", "values": ["/work"], "pattern": null, "coveringDirectory": null } } ] }, - "notes": "Pins the full for-in structure, shared compatibility leaf, bounded candidates, and intentionally Unknown cwd." + "notes": "Pins the full for-in structure, shared compatibility leaf, bounded candidates, and analyzer-preserved cwd." } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/206_v03_for_iterator_substitution.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/206_v03_for_iterator_substitution.json index 54a0e71..804770a 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/206_v03_for_iterator_substitution.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/206_v03_for_iterator_substitution.json @@ -47,7 +47,7 @@ { "ancestorKind": "Block", "region": "Statement", "childIndex": 0, "sourceStart": 11, "sourceLength": 17 } ], "effectiveArguments": [], - "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + "workingDirectory": { "kind": "Exact", "values": ["/work"], "pattern": null, "coveringDirectory": null } }, { "clauseIndex": 1, @@ -61,7 +61,7 @@ "effectiveArguments": [ { "clauseElementIndex": 2, "value": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } } ], - "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + "workingDirectory": { "kind": "Exact", "values": ["/work"], "pattern": null, "coveringDirectory": null } } ] }, diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/207_v03_for_static_pattern.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/207_v03_for_static_pattern.json index 70a7d82..54dc844 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/207_v03_for_static_pattern.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/207_v03_for_static_pattern.json @@ -132,8 +132,8 @@ } ], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/208_v03_for_dynamic_root.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/208_v03_for_dynamic_root.json index 89033b9..6b9c531 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/208_v03_for_dynamic_root.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/208_v03_for_dynamic_root.json @@ -132,8 +132,8 @@ } ], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/209_v03_for_option_injection.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/209_v03_for_option_injection.json index 5606a40..080be38 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/209_v03_for_option_injection.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/209_v03_for_option_injection.json @@ -135,8 +135,8 @@ } ], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/210_v03_for_nested_correlation.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/210_v03_for_nested_correlation.json index d382fe4..a203e63 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/210_v03_for_nested_correlation.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/210_v03_for_nested_correlation.json @@ -184,8 +184,8 @@ } ], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/211_v03_for_pipeline_ancestry.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/211_v03_for_pipeline_ancestry.json index a964caa..0699e46 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/211_v03_for_pipeline_ancestry.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/211_v03_for_pipeline_ancestry.json @@ -174,8 +174,8 @@ } ], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } @@ -216,8 +216,8 @@ ], "effectiveArguments": [], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/213_v03_for_candidate_cap_32.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/213_v03_for_candidate_cap_32.json index a74df77..5074a69 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/213_v03_for_candidate_cap_32.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/213_v03_for_candidate_cap_32.json @@ -165,8 +165,8 @@ } ], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/214_v03_for_candidate_overflow_33.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/214_v03_for_candidate_overflow_33.json index ad7c0c3..9e77d03 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/214_v03_for_candidate_overflow_33.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/214_v03_for_candidate_overflow_33.json @@ -132,8 +132,8 @@ } ], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/215_v03_for_pattern_parent_traversal.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/215_v03_for_pattern_parent_traversal.json index 73d3839..0209652 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/215_v03_for_pattern_parent_traversal.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/215_v03_for_pattern_parent_traversal.json @@ -132,8 +132,8 @@ } ], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/216_v03_for_dot_glob_parent_escape.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/216_v03_for_dot_glob_parent_escape.json index a5eaec4..6f7ca7e 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/216_v03_for_dot_glob_parent_escape.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/216_v03_for_dot_glob_parent_escape.json @@ -132,8 +132,8 @@ } ], "workingDirectory": { - "kind": "Unknown", - "values": [], + "kind": "Exact", + "values": ["/work"], "pattern": null, "coveringDirectory": null } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/239_v03_for_ordered_duplicate_post_state.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/239_v03_for_ordered_duplicate_post_state.json new file mode 100644 index 0000000..afca31d --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/239_v03_for_ordered_duplicate_post_state.json @@ -0,0 +1,57 @@ +{ + "name": "v0.3 Bash ordered duplicate loop state", + "input": "for f in a b a; do printf \"%s\" \"$f\"; done; echo \"$f\"", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["printf"], + "args": [ + { "raw": "\"%s\"", "kind": "Literal", "isPath": false, "resolved": "__NULL__", "isFlag": false }, + { "raw": "\"$f\"", "kind": "EnvVar", "isPath": false, "resolved": "__NULL__", "isFlag": false } + ], + "redirects": [] + }, + { + "operator": "Sequence", + "verb": ["echo"], + "args": [ + { "raw": "\"$f\"", "kind": "EnvVar", "isPath": false, "resolved": "__NULL__", "isFlag": false } + ], + "redirects": [] + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 52 }, + { "ancestorKind": "CommandList", "region": "Statement", "childIndex": 0, "sourceStart": 0, "sourceLength": 52 }, + { "ancestorKind": "ForEach", "region": "LoopBody", "childIndex": null, "sourceStart": 0, "sourceLength": 41 }, + { "ancestorKind": "Block", "region": "Statement", "childIndex": 0, "sourceStart": 18, "sourceLength": 19 } + ], + "effectiveArguments": [ + { "clauseElementIndex": 2, "value": { "kind": "FiniteSet", "values": ["a", "b"], "pattern": null, "coveringDirectory": null } } + ], + "workingDirectory": { "kind": "Exact", "values": ["/work"], "pattern": null, "coveringDirectory": null } + }, + { + "clauseIndex": 1, + "immediateRole": "Ordinary", + "isComplete": true, + "ancestry": [ + { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 52 }, + { "ancestorKind": "CommandList", "region": "Statement", "childIndex": 1, "sourceStart": 0, "sourceLength": 52 } + ], + "effectiveArguments": [ + { "clauseElementIndex": 1, "value": { "kind": "Exact", "values": ["a"], "pattern": null, "coveringDirectory": null } } + ], + "workingDirectory": { "kind": "Exact", "values": ["/work"], "pattern": null, "coveringDirectory": null } + } + ] + }, + "notes": "Preserves authored a-b-a visit order: the body joins a and b while the post-loop binding remains exact a." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/240_v03_for_empty_or_unreachable.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/240_v03_for_empty_or_unreachable.json new file mode 100644 index 0000000..e1b7723 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/240_v03_for_empty_or_unreachable.json @@ -0,0 +1,53 @@ +{ + "name": "v0.3 Bash empty-loop failure continuation", + "input": "for f in; do false; done || cat relative.txt", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["false"], + "args": [ + { "raw": "\u003Cdynamic-cwd\u003E", "kind": "DynamicSkip", "isPath": false, "resolved": "__NULL__", "isFlag": false, "isCwdAttribution": true } + ], + "redirects": [] + }, + { + "operator": "OrIf", + "verb": ["cat"], + "args": [ + { "raw": "relative.txt", "kind": "Literal", "isPath": true, "resolved": "__NULL__", "isFlag": false }, + { "raw": "\u003Cdynamic-cwd\u003E", "kind": "DynamicSkip", "isPath": false, "resolved": "__NULL__", "isFlag": false, "isCwdAttribution": true } + ], + "redirects": [] + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 44 }, + { "ancestorKind": "CommandList", "region": "Statement", "childIndex": 0, "sourceStart": 0, "sourceLength": 44 }, + { "ancestorKind": "ForEach", "region": "LoopBody", "childIndex": null, "sourceStart": 0, "sourceLength": 24 }, + { "ancestorKind": "Block", "region": "Statement", "childIndex": 0, "sourceStart": 12, "sourceLength": 8 } + ], + "effectiveArguments": [], + "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + }, + { + "clauseIndex": 1, + "immediateRole": "Ordinary", + "isComplete": true, + "ancestry": [ + { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 44 }, + { "ancestorKind": "CommandList", "region": "Statement", "childIndex": 1, "sourceStart": 0, "sourceLength": 44 } + ], + "effectiveArguments": [], + "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + } + ] + }, + "notes": "An explicit empty loop has no failure exit; the authored OR continuation remains visible but receives no fabricated exact cwd." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/241_v03_for_break_transfer_rejected.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/241_v03_for_break_transfer_rejected.json new file mode 100644 index 0000000..3c92370 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/241_v03_for_break_transfer_rejected.json @@ -0,0 +1,10 @@ +{ + "name": "v0.3 Bash loop break transfer rejected", + "input": "for f in a b; do break; done", + "expected": { + "isUnparseable": true, + "unparseableReasonContains": "mutation or control transfer" + }, + "notes": "A valid Bash break changes loop reachability; it fails atomically until the analyzer models that transfer.", + "oracleExpectation": "OutOfScope" +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/242_v03_for_wrapped_exit_transfer_rejected.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/242_v03_for_wrapped_exit_transfer_rejected.json new file mode 100644 index 0000000..63b67b3 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/242_v03_for_wrapped_exit_transfer_rejected.json @@ -0,0 +1,10 @@ +{ + "name": "v0.3 Bash wrapped loop exit transfer rejected", + "input": "for f in a; do command exit 0; done", + "expected": { + "isUnparseable": true, + "unparseableReasonContains": "mutation or control transfer" + }, + "notes": "A statically wrapped exit cannot bypass the loop control-transfer guard.", + "oracleExpectation": "OutOfScope" +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/46_unparseable_for_loop.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/46_unparseable_for_loop.json index 480d2ea..abd0cd0 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/46_unparseable_for_loop.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/46_unparseable_for_loop.json @@ -33,7 +33,7 @@ "effectiveArguments": [ { "clauseElementIndex": 1, "value": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } } ], - "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + "workingDirectory": { "kind": "Exact", "values": ["/work"], "pattern": null, "coveringDirectory": null } } ] }, diff --git a/tests/ShellSyntaxTree.Tests/Parsing/BashForInStructuralTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/BashForInStructuralTests.cs index eb1249d..6b3cfb2 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/BashForInStructuralTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/BashForInStructuralTests.cs @@ -204,7 +204,7 @@ public void Body_substitution_inherits_the_loop_binding() "a", "b"); Assert.All(result.Commands, command => - Assert.Equal(ShellValueDomainKind.Unknown, command.WorkingDirectory.Kind)); + AssertDomain(command.WorkingDirectory, ShellValueDomainKind.Exact, "/work")); } [Fact] @@ -291,6 +291,140 @@ public void Empty_iterable_keeps_the_authored_body_with_unknown_binding() Assert.Single(Assert.Single(result.Commands).EffectiveArguments).Value.Kind); } + [Fact] + public void Ordered_duplicate_iterations_join_body_facts_and_preserve_final_binding() + { + var result = Parse( + "for f in a b a; do printf '%s' \"$f\"; done; echo \"$f\""); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal(2, result.Commands.Count); + AssertDomain( + Assert.Single(result.Commands[0].EffectiveArguments).Value, + ShellValueDomainKind.FiniteSet, + "a", + "b"); + AssertDomain( + Assert.Single(result.Commands[1].EffectiveArguments).Value, + ShellValueDomainKind.Exact, + "a"); + } + + [Fact] + public void Empty_loop_preserves_the_incoming_binding_value() + { + var result = Parse( + "for f in seed; do :; done; for f in; do :; done; echo \"$f\""); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + AssertDomain( + Assert.Single(result.Commands[2].EffectiveArguments).Value, + ShellValueDomainKind.Exact, + "seed"); + } + + [Fact] + public void Sequential_same_name_loop_replaces_the_prior_binding() + { + var result = Parse( + "for f in first; do :; done; for f in second; do :; done; echo \"$f\""); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + AssertDomain( + Assert.Single(result.Commands[2].EffectiveArguments).Value, + ShellValueDomainKind.Exact, + "second"); + } + + [Fact] + public void Empty_loop_failure_continuation_is_unreachable_and_conservative() + { + var result = Parse( + "for f in; do false; done || cat relative.txt"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal(2, result.Commands.Count); + var unreachable = result.Commands[1]; + Assert.Equal(ShellValueDomainKind.Unknown, unreachable.WorkingDirectory.Kind); + var relative = Assert.Single(unreachable.Clause.Args, argument => + argument.Raw == "relative.txt"); + Assert.Null(relative.Resolved); + } + + [Fact] + public void Empty_loop_success_continuation_keeps_exact_state() + { + var result = Parse( + "for f in; do false; done && cat relative.txt"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal(2, result.Commands.Count); + var reached = result.Commands[1]; + AssertDomain(reached.WorkingDirectory, ShellValueDomainKind.Exact, "/work"); + var relative = Assert.Single(reached.Clause.Args, argument => + argument.Raw == "relative.txt"); + Assert.Equal("/work/relative.txt", relative.Resolved); + } + + [Fact] + public void Thirty_two_visits_preserve_the_last_value_after_the_loop() + { + var values = Enumerable.Range(1, ShellAnalysisLimits.MaxValueCandidates) + .Select(index => $"v{index:00}") + .ToArray(); + var result = Parse( + $"for f in {string.Join(" ", values)}; do :; done; echo \"$f\""); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + AssertDomain( + Assert.Single(result.Commands[1].EffectiveArguments).Value, + ShellValueDomainKind.Exact, + "v32"); + } + + [Fact] + public void Thirty_three_duplicate_visits_widen_instead_of_deduplicating_the_plan() + { + var values = Enumerable.Repeat("same", ShellAnalysisLimits.MaxValueCandidates + 1); + var result = Parse( + $"for f in {string.Join(" ", values)}; do echo \"$f\"; done; printf '%s' \"$f\""); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.All(result.Commands, command => Assert.Equal( + ShellValueDomainKind.Unknown, + Assert.Single(command.EffectiveArguments).Value.Kind)); + } + + [Fact] + public void Zero_or_more_fixed_point_widens_recursive_binding_growth() + { + var result = Parse( + "for f in seed; do :; done; " + + "for d in \"$UNKNOWN\"; do for f in \"${f}x\"; do :; done; done; " + + "echo \"$f\""); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal( + ShellValueDomainKind.Unknown, + Assert.Single(result.Commands[2].EffectiveArguments).Value.Kind); + } + + [Fact] + public void Nested_concrete_loop_product_exceeding_analysis_budget_fails_atomically() + { + var values = string.Join( + " ", + Enumerable.Range(1, ShellAnalysisLimits.MaxValueCandidates)); + var result = Parse( + $"for a in {values}; do for b in {values}; do " + + $"for c in {values}; do :; done; done; done"); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + Assert.Contains("exceeded limits", result.UnparseableReason!); + } + [Fact] public void Pipelines_compose_with_loop_ancestry_without_synthetic_operators() { @@ -319,14 +453,21 @@ public void Pipelines_compose_with_loop_ancestry_without_synthetic_operators() [InlineData("for f in a b; do trap 'f=x' DEBUG; echo \"$f\"; done")] [InlineData("for f in a; do command unset f; echo \"$f\"; done")] [InlineData("for f in a; do builtin unset f; echo \"$f\"; done")] - public void Binding_mutation_fails_the_whole_parse(string source) + [InlineData("for f in a; do break; done")] + [InlineData("for f in a; do continue; done")] + [InlineData("for f in a; do return; done")] + [InlineData("for f in a; do exit 0; done")] + [InlineData("for f in a; do exec echo replaced; done")] + [InlineData("for f in a; do command exit 0; done")] + [InlineData("for f in a; do builtin break; done")] + public void Unsupported_loop_state_transfer_fails_the_whole_parse(string source) { var result = Parse(source); Assert.True(result.IsUnparseable); Assert.Empty(result.Commands); Assert.Empty(result.Clauses); - Assert.Contains("mutation", result.UnparseableReason!); + Assert.Contains("mutation or control transfer", result.UnparseableReason!); } [Theory] @@ -448,6 +589,22 @@ public void Static_bash_c_preserves_decoded_loop_facts_without_outer_source_span "b"); } + [Fact] + public void Static_bash_c_remaps_nested_loop_plans_and_value_provenance() + { + var result = Parse( + "bash -c 'for d in a b; do for f in x y; do echo \"$d/$f/$d\"; done; done'"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + AssertDomain( + Assert.Single(Assert.Single(result.Commands).EffectiveArguments).Value, + ShellValueDomainKind.FiniteSet, + "a/x/a", + "a/y/a", + "b/x/b", + "b/y/b"); + } + [Fact] public void Decoded_loop_fails_after_outer_variable_state_mutation() {