From b1b81512e7e52ff1340882d5d63b6c84957f106f Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 6 Aug 2026 18:52:40 +0000 Subject: [PATCH 1/2] Add v0.3 structural API skeleton --- IMPLEMENTATION_PLAN.md | 17 +- SPEC.POWERSHELL.md | 189 ++++-- SPEC.md | 580 ++++++++++++++++-- .../v0-3-structured-shell-analysis/design.md | 12 +- .../v0-3-structured-shell-analysis/tasks.md | 12 +- src/ShellSyntaxTree/CommandOccurrence.cs | 147 +++++ src/ShellSyntaxTree/ParsedCommand.cs | 26 +- src/ShellSyntaxTree/RedirectAnalysis.cs | 114 ++++ src/ShellSyntaxTree/ShellAnalysisLimits.cs | 19 + src/ShellSyntaxTree/ShellSyntaxNode.cs | 260 ++++++++ .../PublicApiSnapshotTests.cs | 54 ++ .../V03PublicApiSnapshotTests.cs | 532 ++++++++++++++++ 12 files changed, 1848 insertions(+), 114 deletions(-) create mode 100644 src/ShellSyntaxTree/CommandOccurrence.cs create mode 100644 src/ShellSyntaxTree/RedirectAnalysis.cs create mode 100644 src/ShellSyntaxTree/ShellAnalysisLimits.cs create mode 100644 src/ShellSyntaxTree/ShellSyntaxNode.cs create mode 100644 tests/ShellSyntaxTree.Tests/V03PublicApiSnapshotTests.cs diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index e9ada83..0482625 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -147,10 +147,12 @@ priorities. analysis limits, separate Bash and PowerShell grammar matrices, static pattern-cover rules, divergent-cwd fallback, deferred forms, project context, and the preimplementation consumer-guide migration contract. -- [ ] Complete OpenSpec tasks 1.6–1.7 in the public-API implementation PR: - synchronize the accepted shared and PowerShell contracts into - `SPEC.md` / `SPEC.POWERSHELL.md` together with source and snapshot tests - so the repository authority never intentionally drifts from the assembly. +- [x] Synchronize the accepted shared and PowerShell v0.3 contracts into + `SPEC.md` / `SPEC.POWERSHELL.md` in the public-API implementation change. + The canonical specs now lock the additive types and defaults, closed + record hierarchy, separate Bash and PowerShell grammars, occurrence and + compatibility projections, bounded values/state, explicit redirects, + resolver contexts, fail-closed consumer contract, and persistence caveat. - [x] Correct the lexer-to-resolver provenance boundary before issue #69. Paired Bash and PowerShell shell-oracle cases must distinguish escaped literal resolver syntax from expandable syntax even when both decode to @@ -214,6 +216,13 @@ priorities. their shell-specific implementations. Direct boundary tests pin the extracted helpers, while the complete resolver and corpus suites prove the refactor leaves both compatibility projections unchanged. +- [x] Add the inert v0.3 public API skeleton: the closed syntax-node family, + command occurrences and ancestry, value domains and fixed limits, + explicit redirect records, plus additive `ParsedCommand.Syntax` and + `Commands`. Public snapshot tests pin every member, enum order, default, + and assembly-only closure mechanism. Until the projection passes land, + `Syntax` is an empty block and `Commands` is empty, so early use remains + fail-closed while v0.2 `Clauses` behavior is unchanged. - [ ] Add the structural and command-occurrence projections for the existing grammar before enabling any control-flow construct. - [ ] Deliver Bash `for ... in` and PowerShell `foreach` as the first two diff --git a/SPEC.POWERSHELL.md b/SPEC.POWERSHELL.md index 296ebc1..2abe75a 100644 --- a/SPEC.POWERSHELL.md +++ b/SPEC.POWERSHELL.md @@ -1,23 +1,24 @@ -# ShellSyntaxTree — PowerShell Specification (v0.2.0) +# ShellSyntaxTree — PowerShell Specification (through v0.3) -**Status:** Shipped in the v0.2.0 prerelease line; stable promotion pending -downstream Netclaw validation. +**Status:** v0.2.0 shipped; the accepted v0.3 contract adds bounded PowerShell +`foreach`, `while`, and `if` structure plus shared command-occurrence and +explicit redirect analysis. **Audience:** Whoever (human or agent) implements, consumes, or maintains the ShellSyntaxTree PowerShell parser. **Read `SPEC.md` (the bash and shared-contract specification) end-to-end first — this document specifies only what differs for PowerShell.** -This document specifies the PowerShell parser shipping in ShellSyntaxTree -v0.2.0: its grammar, tokenization, cmdlet/verb tables, alias resolution, -resolver semantics, and corpus contract. PowerShell support reuses the -shared `IShellParser` interface and the shared AST (`ParsedCommand` / -`Clause` / `VerbChain` / `Arg` / `Redirect`) defined in `SPEC.md` §2–§3. +This document specifies the PowerShell parser through ShellSyntaxTree v0.3: +its grammar, tokenization, cmdlet/verb tables, alias resolution, resolver +semantics, bounded control-flow analysis, and corpus contract. PowerShell +reuses the shared public API defined in `SPEC.md` §2–§3. It is **not** a PowerShell interpreter. It does not execute, expand, or evaluate commands. It returns the same structured AST a consumer already walks for bash. The parsing scope is **Pipeline-aware** (§4): linear -command pipelines parse; control flow, definitions, and other script-level -constructs mark `IsUnparseable`. +command pipelines parse; stable v0.3 also supports only the explicitly bounded +`foreach`, `while`, and `if` subsets below. Other script-level constructs mark +`IsUnparseable`. `SPEC.md` is the canonical home of the shared public API, AST, sanitization workflow, and consumer contract. Where this spec says "see `SPEC.md` §N" the @@ -71,15 +72,25 @@ syntax (§5) are all PowerShell 7 semantics. The `pwsh` validation oracle - Command execution and variable expansion — the same non-goals as bash (`SPEC.md` §1). The library marks dynamic tokens; it never resolves them. +### v0.3 extension + +Stable v0.3 adds structured projection for bounded `foreach`, `while`, and +`if` statements; exposes iterator, condition, branch, and body commands +exactly once; derives exact or finite string values only from proved literal +iterables; and joins location and supported binding state conservatively. +Pipeline-produced objects and unsupported expressions remain unknown without +execution. `do`, `switch`, definitions, and arbitrary script evaluation stay +outside the supported grammar. + --- ## 2. Public API Surface The shared interface, AST records, and enums are defined in **`SPEC.md` §2**. The additive `Clause.Elements`, `ClauseElement`, and `ClauseElementRole` -provenance surface applies identically to both parsers. PowerShell adds the -following parser types to namespace `ShellSyntaxTree`; everything else is -internal. +provenance surface and the v0.3 syntax, occurrence, value-domain, and explicit +redirect types apply identically to both parsers. PowerShell adds the following +parser types to namespace `ShellSyntaxTree`; everything else is internal. ```csharp namespace ShellSyntaxTree; @@ -301,7 +312,8 @@ quoted_string := single_quoted | double_quoted creates **no scope and no working-directory boundary** (`$PWD` is runspace state, not a scoped variable). `Set-Location` attribution therefore **propagates through** `( ... )` rather than being isolated by it (§9). A - group containing control flow marks `IsUnparseable`. + group containing executable syntax outside the bounded v0.3 grammar marks + the whole result `IsUnparseable`. - `--%` is the stop-parsing token: the remainder of the **line** — to the next newline or end of input, including any `|`, `;`, `&&`, or `||`, which become literal text rather than operators — becomes one opaque @@ -311,9 +323,65 @@ quoted_string := single_quoted | double_quoted - Script blocks `{ ... }`, subexpressions `$( ... )`, array `@( ... )`, and hash `@{ ... }` literals are recognized as **opaque tokens** — the interior is not parsed; each becomes one `DynamicSkip` arg. -- Control-flow keywords, definition keywords, block keywords, `param()`, - assignment statements, bare `[type]::member` calls, and bare arithmetic at - statement position fall outside the grammar → `IsUnparseable` (§11). +- Under v0.2, control-flow keywords fall outside the grammar. Stable v0.3 owns + only the contextual statement forms below. Definition keywords, unsupported + block keywords, `param()`, assignment statements, bare `[type]::member` + calls, and bare arithmetic at statement position remain unparseable (§11). + +### v0.3 structured PowerShell grammar + +PowerShell retains a statement-versus-pipeline distinction. `foreach` is a +language keyword only at statement position when followed by `(`; +`Get-ChildItem | foreach { ... }` remains command/alias syntax and its ordinary +script-block argument remains opaque. + +```text +pwsh_script(stop) := pwsh_statement (statement_sep pwsh_statement)* +pwsh_statement := pwsh_foreach + | pwsh_while + | pwsh_if + | pwsh_pipeline + +pwsh_foreach := "foreach" "(" variable "in" foreach_expression ")" + script_block_body + +pwsh_while := "while" "(" condition_pipeline ")" + script_block_body + +pwsh_if := "if" "(" condition_pipeline ")" script_block_body + pwsh_elseif* pwsh_else? +pwsh_elseif := "elseif" "(" condition_pipeline ")" script_block_body +pwsh_else := "else" script_block_body + +foreach_expression := literal_value + | literal_array + | pipeline_expression +literal_array := "@(" literal_value ("," literal_value)* ")" +script_block_body := "{" pwsh_script(stop = "}") "}" +``` + +Literal scalar and literal-array iterables may produce exact or finite string +domains. A pipeline iterable exposes every producing command with role +`Iterator`, but its object values remain `Unknown`; the parser does not predict +PowerShell object-to-string conversion. The body is recursively parsed only +after the structural grammar proves that the `ScriptBlock` token is the body +of a recognized statement. An ordinary script-block argument remains one +opaque `DynamicSkip` value and does not invent child execution. + +`condition_pipeline` is limited to a pipeline the existing parser can delimit +completely. A subexpression, member invocation, script block, or other form +that may execute outside complete command discovery makes the whole result +unparseable. Missing delimiters and every unsupported executable region also +produce empty `Commands` and `Clauses`; partial `Syntax` is diagnostic only. + +PowerShell scope and location state remain shell-specific. Grouping `( ... )` +does not isolate location. Branch exits retain an exact cwd only when every +supported alternative agrees; disagreement becomes `Unknown`. Loop exits +include the zero-iteration state. The parser does not publish a finite cwd set. + +Stable v0.3 continues to defer `do`, `switch`, functions, definitions, +class/type bodies, and arbitrary execution-bearing expressions outside the +bounded forms above. --- @@ -798,6 +866,48 @@ implementers and consumers should expect a higher prompt rate. Privileging more `$env:` names is a deliberate non-goal: the value at parse time need not match the value when the command runs. +### v0.3 resolver provenance and consumer contexts + +The ordered v0.2 resolution steps describe compatibility results, but they do +not permit v0.3 to reconstruct expansion from decoded text. The PowerShell +front end retains each ordered literal, typed-expansion, or opaque fragment, +its exact-or-null source span, allowed lexical transforms, expansion identity, +cardinality, and opaque cause through decoding. Raw spelling, decoded logical +values, and source spans keep their v0.2 meanings. + +Resolution receives one explicit consumer context: + +- native argument; +- cmdlet `Path` binding; +- cmdlet `LiteralPath` binding; +- redirect target. + +Native arguments apply PowerShell lexical quote rules but never acquire +cmdlet provider or PSDrive semantics merely because their decoded text looks +provider-qualified. A quoted native `~`, `*.txt`, or +`FileSystem::C:\logs\x` remains literal. An unquoted native wildcard remains +unknown without filesystem enumeration. + +Cmdlet `Path` and redirect contexts apply tilde, wildcard, FileSystem +provider, and PSDrive semantics after value formation even when quoted. +`LiteralPath` suppresses wildcard interpretation but still applies quoted +tilde, provider, and PSDrive semantics. An unproved PSDrive mapping or wildcard +cardinality remains unknown. A drive-relative value such as `C:foo` is not an +absolute filesystem path proof and fails closed. + +Escaped interpolation starts remain exact literals. Recognized special, +numeric, scoped, braced, and Unicode-named variables retain typed expansion +identity while their value stays unknown without a bounded proof. An +unterminated `${...}` interpolation makes the entire result unparseable. +Adjacent fragments after a redirect operator form one target; the suffix is +never emitted as an unrelated argument. + +When every fragment, transformation, binding fact, cwd/home fact, and +consumer fact is exact, mixed literal and expandable fragments compose to one +exact compatibility result rather than becoming `DynamicSkip`. Any opaque or +incompletely mapped region, unknown required fact, or unsupported transform +fails closed. The resolver does not recursively rescan produced text. + ### Redirect stream → `RedirectDirection` mapping `RedirectDirection` (`SPEC.md` §3) has five members; PowerShell has more @@ -816,8 +926,13 @@ about which stream produced it: | `3>>`–`6>>`, `*>>` | `Append` (lossy) | | stream merge `N>&M` (`2>&1`, `3>&1`, ...) | `ErrOut` when `N` is `2`, else `Out`; `Target` carries `&M` verbatim with `IsDynamicSkip=true` | -Lossless stream identity is deferred (§18); v0.2.0 consumers gate on the -target path, not the originating stream. +The table above remains the v0.2 `Redirect` compatibility mapping. v0.3 also +populates `RedirectAnalysis`: `RedirectSourceKind.PowerShellAllStreams` +preserves `*`, `Descriptor` preserves numeric streams, and `Operation` +distinguishes file input/output/append from static descriptor duplication, +close, and move. Static descriptor operations are not path-relevant. A +variable, substitution, malformed suffix, or otherwise computed descriptor +target remains unknown or incomplete rather than becoming a static exemption. --- @@ -1144,13 +1259,15 @@ Clause 0: Operator=None, Verb=[Remove-Item], Args=[ {Raw="HKLM:\Software\X", Kind=Literal, IsPath=false} ] ``` -### Unparseable — a control-flow construct +### Unparseable — an unsupported control-flow construct -Input: `foreach ($f in $list) { Remove-Item $f }` +Input: `switch -Wildcard ($value) { '*.txt' { Remove-Item $_ } }` ``` IsUnparseable=true -UnparseableReason="control-flow keyword 'foreach' is not supported in v0.2" +UnparseableReason="PowerShell switch is unsupported in v0.3" +Commands=[] +Clauses=[] ``` --- @@ -1369,25 +1486,23 @@ v0.2.0 ships when **all** of these hold: --- -## 18. Out of Scope (deferred from v0.2.0) +## 18. Out of Scope -- PowerShell script-level constructs — control flow, `function`/`class`/ - `enum` definitions, `param()`/`begin`/`process`/`end` blocks, `trap`, - `DATA` (all `IsUnparseable`). +- PowerShell control flow outside stable v0.3's bounded `foreach`, `while`, + and `if` subsets — including `do` and `switch`. +- `function`/`filter`/`class`/`enum` definitions, + `param()`/`begin`/`process`/`end` blocks, `trap`, and `DATA`. - `.ps1` script-file parsing. -- PowerShell expression evaluation, `$_` / `$PSItem` semantics, .NET method - calls. +- General PowerShell expression evaluation, `$_` / `$PSItem` semantics, .NET + method calls, object-to-string prediction, and runtime pipeline evaluation. - Desired State Configuration (DSC). - A real `Push-Location` / `Pop-Location` directory-stack model (§9). -- Lossless redirect-stream identity — PowerShell streams 3–6 / `*` map - lossily onto `RedirectDirection` (§8); growing the public enum is a - candidate v0.2.x item. - Per-element path extraction from a comma-separated array (`-Path a,b,c`) — v0.2.0 marks the whole token `DynamicSkip` (§8); - splitting it into per-element path args is a candidate v0.2.x item. -- Extracting a shared lexer/parser core from the bash and PowerShell - implementations — deliberately deferred until two parsers exist so the - seam is designed from real duplication, not guessed. + splitting arbitrary parameter arrays remains independently gated. +- A shared lexer, structural parser base class, or false shared expression + grammar. Only post-parse machinery proven identical in both shells is + extracted through explicit adapters. - Windows `cmd` parsing — still deferred (`SPEC.md` §18). --- @@ -1395,8 +1510,8 @@ v0.2.0 ships when **all** of these hold: ## Appendix A: Consumer Contract The consumer contract is defined in **`SPEC.md` Appendix A** and is -shell-neutral — a consumer walks a PowerShell `ParsedCommand` exactly as it -walks a bash one. Two additions: +shell-neutral. Security consumers enumerate every `CommandOccurrence` and +evaluate every explicit redirect. PowerShell adds these identity rules: - When gating on verb identity, use the gate key `CanonicalVerb ?? (Tokens.Count > 0 ? Tokens[0] : null)` — the index is diff --git a/SPEC.md b/SPEC.md index 9cbcea3..1d34921 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,15 +1,16 @@ # ShellSyntaxTree — bash & shared-contract Specification -**Status:** Shipped. The bash parser is implemented (v0.1.x); v0.2.0 adds the -PowerShell parser and the shared multi-shell surface. +**Status:** v0.2.0 shipped; the accepted v0.3 contract adds structured syntax, +complete command occurrences, explicit redirect analysis, and bounded control +flow while retaining the v0.2 compatibility leaves. **Audience:** Whoever (human or agent) works on ShellSyntaxTree. **Read this end-to-end before writing any code.** **PowerShell support is specified separately in `SPEC.POWERSHELL.md` (v0.2.0); this document is the canonical home of the public API, AST, sanitization workflow, and consumer contract that PowerShell reuses.** -This document specifies the public API, AST, grammar, verb tables, resolver -semantics, and corpus contract for ShellSyntaxTree v0.1. The library is a +This document specifies the shared public API, AST, Bash grammar, verb tables, +resolver semantics, and corpus contract through ShellSyntaxTree v0.3. The library is a focused bash command parser designed for **security gate evaluators** — tools that inspect agent-emitted shell commands to decide whether to allow, prompt for, or deny execution. @@ -132,7 +133,7 @@ public abstract record ShellParserOptions public string? WorkingDirectory { get; init; } } -// AST records — see §3. +// v0.2 compatibility leaves — see §3. public sealed record ParsedCommand { ... } public sealed record Clause { ... } public sealed record ClauseElement { ... } @@ -143,6 +144,41 @@ public enum ClauseElementRole { Verb, Argument, Redirect } public enum ArgKind { Literal, EnvVar, Glob, Tilde, DynamicSkip } public enum RedirectDirection { In, Out, Append, ErrOut, ErrAppend } public enum CompoundOperator { None, AndIf, OrIf, Sequence, Pipe } + +// v0.3 authored structure — see §3. +public abstract record ShellSyntaxNode { ... } +public sealed record ShellBlockSyntax : ShellSyntaxNode { ... } +public sealed record SimpleCommandSyntax : ShellSyntaxNode { ... } +public sealed record PipelineSyntax : ShellSyntaxNode { ... } +public sealed record CommandListSyntax : ShellSyntaxNode { ... } +public sealed record CommandListItemSyntax { ... } +public sealed record GroupSyntax : ShellSyntaxNode { ... } +public sealed record ForEachSyntax : ShellSyntaxNode { ... } +public sealed record LoopBindingSyntax { ... } +public sealed record ShellSourceFragment { ... } +public sealed record ConditionLoopSyntax : ShellSyntaxNode { ... } +public sealed record ConditionalSyntax : ShellSyntaxNode { ... } +public sealed record ConditionalBranchSyntax : ShellSyntaxNode { ... } +public sealed record CommandSubstitutionSyntax : ShellSyntaxNode { ... } +public enum ShellSyntaxKind { ... } +public enum ShellGroupKind { ... } +public enum ConditionLoopKind { ... } + +// v0.3 authorization and bounded-analysis projections — see §3. +public sealed record CommandOccurrence { ... } +public sealed record CommandAncestryFrame { ... } +public sealed record EffectiveArgument { ... } +public sealed record ShellValueDomain { ... } +public sealed record RedirectAnalysis { ... } +public sealed record HereDocumentAnalysis { ... } +public sealed record RedirectSource { ... } +public enum CommandOccurrenceRole { ... } +public enum CommandAncestryRegion { ... } +public enum ShellValueDomainKind { ... } +public enum HereDocumentExpansionMode { ... } +public enum RedirectSourceKind { ... } +public enum RedirectOperation { ... } +public static class ShellAnalysisLimits { ... } ``` That's the entire public API. **Everything else is internal.** The lexer, @@ -163,15 +199,28 @@ public sealed record ParsedCommand public string Source { get; init; } = ""; /// - /// Top-level clauses, split on compound operators (&&, ||, ;, |). - /// For a simple command, exactly one clause with Operator=None. + /// Canonical authored nested structure. Direct-source nodes have exact + /// source ranges; decoded wrapper nodes report unavailable ranges unless + /// an exact outer mapping exists. + /// + public ShellBlockSyntax Syntax { get; init; } = new(); + + /// + /// Canonical authorization projection containing every authored simple + /// command that may execute exactly once, in deterministic source order. + /// + public IReadOnlyList Commands { get; init; } = []; + + /// + /// Conservative v0.2 compatibility projection. Existing simple-command + /// behavior remains available, but v0.3 security consumers use Commands. /// public IReadOnlyList Clauses { get; init; } = []; /// - /// True when the parser could not produce a clean AST (unbalanced - /// quotes, unparseable construct). When true, Clauses MAY be partial - /// or empty. Consumers should route to safe-fail. + /// True when the parser could not account for every executable region. + /// When true, Commands and Clauses are empty; Syntax may contain partial + /// diagnostic evidence only. Consumers must prompt or deny. /// public bool IsUnparseable { get; init; } @@ -182,6 +231,349 @@ public sealed record ParsedCommand } ``` +For a successful result, every authored simple command appears once in +`Syntax`, once in `Commands`, and once in `Clauses`, with all three projections +referencing the identical in-memory `Clause` instance. Serialization is not +required to preserve that reference identity. For an unparseable result, +`Commands` and `Clauses` are empty even when `Syntax` retains partial evidence +for diagnostics. + +### Authored structural nodes (v0.3) + +The syntax family is a closed, library-owned record hierarchy. The +`private protected` ordinary constructor is paired with an assembly-only +abstract ownership member because C# records also synthesize a protected copy +constructor; together they prevent an external concrete node implementation. +Later library versions may add node kinds, so +authorization code that inspects `Syntax` must fail closed on an unknown type +or enum value. Security consumers normally enumerate `Commands`; `Syntax` is +for structure, explanation, display, and specialized analysis. + +```csharp +public abstract record ShellSyntaxNode +{ + private protected ShellSyntaxNode() { } + + public abstract ShellSyntaxKind Kind { get; } + public int? SourceStart { get; init; } + public int? SourceLength { get; init; } +} + +public enum ShellSyntaxKind +{ + Unknown, + Block, + SimpleCommand, + Pipeline, + CommandList, + Group, + ForEach, + ConditionLoop, + Conditional, + ConditionalBranch, + CommandSubstitution, +} + +public sealed record ShellBlockSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.Block; + public IReadOnlyList Statements { get; init; } = []; +} + +public sealed record SimpleCommandSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.SimpleCommand; + public Clause Clause { get; init; } = new(); +} + +public sealed record PipelineSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.Pipeline; + public IReadOnlyList Stages { get; init; } = []; +} + +public sealed record CommandListSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.CommandList; + public IReadOnlyList Items { get; init; } = []; +} + +public sealed record CommandListItemSyntax +{ + public CompoundOperator Operator { get; init; } + public ShellSyntaxNode Command { get; init; } = new ShellBlockSyntax(); +} + +public sealed record GroupSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.Group; + public ShellGroupKind GroupKind { get; init; } + public ShellBlockSyntax Body { get; init; } = new(); +} + +public enum ShellGroupKind +{ + Unknown, + CurrentScope, + IsolatedScope, +} + +public sealed record ForEachSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.ForEach; + public LoopBindingSyntax Binding { get; init; } = new(); + public ShellSourceFragment Iterable { get; init; } = new(); + public ShellBlockSyntax IteratorCommands { get; init; } = new(); + public ShellBlockSyntax Body { get; init; } = new(); +} + +public sealed record LoopBindingSyntax +{ + public string Name { get; init; } = ""; + public ShellSourceFragment Source { get; init; } = new(); +} + +public sealed record ShellSourceFragment +{ + public string Raw { get; init; } = ""; + public int? SourceStart { get; init; } + public int? SourceLength { get; init; } +} + +public sealed record ConditionLoopSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.ConditionLoop; + public ConditionLoopKind LoopKind { get; init; } + public ShellBlockSyntax Condition { get; init; } = new(); + public ShellBlockSyntax Body { get; init; } = new(); +} + +public enum ConditionLoopKind +{ + Unknown, + While, + Until, +} + +public sealed record ConditionalSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.Conditional; + public IReadOnlyList Branches { get; init; } = []; + public ShellBlockSyntax? Else { get; init; } +} + +public sealed record ConditionalBranchSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.ConditionalBranch; + public ShellBlockSyntax Condition { get; init; } = new(); + public ShellBlockSyntax Body { get; init; } = new(); +} + +public sealed record CommandSubstitutionSyntax : ShellSyntaxNode +{ + public override ShellSyntaxKind Kind => ShellSyntaxKind.CommandSubstitution; + public ShellBlockSyntax Body { get; init; } = new(); +} +``` + +`ForEachSyntax` shares proved execution structure only. `Iterable.Raw` keeps +the shell-specific authored expression; it does not claim that Bash words and +PowerShell expressions share a grammar. Direct-source nodes have exact ranges +into `ParsedCommand.Source`. Nodes lifted from decoded, escaped, or encoded +wrapper content use null ranges unless an exact outer mapping exists. + +Every v0.3 enum reserves zero as `Unknown`. Consumers fail closed on +`Unknown` or an unrecognized numeric value when the fact affects policy. + +### Command occurrences and bounded values (v0.3) + +```csharp +public sealed record CommandOccurrence +{ + public Clause Clause { get; init; } = new(); + public CommandOccurrenceRole ImmediateRole { get; init; } + public IReadOnlyList Ancestry { get; init; } = []; + public IReadOnlyList EffectiveArguments { get; init; } = []; + public ShellValueDomain WorkingDirectory { get; init; } = ShellValueDomain.Unknown; + public IReadOnlyList Redirects { get; init; } = []; + public bool IsComplete { get; init; } +} + +public enum CommandOccurrenceRole +{ + Unknown, + Ordinary, + PipelineStage, + Condition, + Iterator, + LoopBody, + Branch, + Substitution, +} + +public sealed record CommandAncestryFrame +{ + public ShellSyntaxKind AncestorKind { get; init; } + public CommandAncestryRegion Region { get; init; } + public int? ChildIndex { get; init; } + public int? SourceStart { get; init; } + public int? SourceLength { get; init; } +} + +public enum CommandAncestryRegion +{ + Unknown, + Root, + Statement, + PipelineStage, + GroupBody, + Iterator, + LoopBody, + Condition, + Branch, + Substitution, +} + +public sealed record EffectiveArgument +{ + public int ClauseElementIndex { get; init; } = -1; + public ShellValueDomain Value { get; init; } = ShellValueDomain.Unknown; +} + +public sealed record ShellValueDomain +{ + public static ShellValueDomain Unknown { get; } = new(); + + public ShellValueDomainKind Kind { get; init; } + public IReadOnlyList Values { get; init; } = []; + public string? Pattern { get; init; } + public string? CoveringDirectory { get; init; } +} + +public enum ShellValueDomainKind +{ + Unknown, + Exact, + FiniteSet, + Pattern, +} + +public static class ShellAnalysisLimits +{ + public static int MaxValueCandidates => 32; + public static int MaxStructuralNesting => 16; + public static int MaxWrapperRecursionDepth => 5; +} +``` + +`Commands` contains one entry per authored simple command that may execute, +not one per predicted runtime iteration. `Ancestry` is ordered outermost to +innermost, excludes the simple-command leaf, and retains every enclosing +execution relation. `ImmediateRole` describes the nearest relation. + +`EffectiveArgument.ClauseElementIndex` is a stable authored coordinate. The +analysis never mutates a compatibility `Arg` to hold loop-specific values. +An occurrence can be structurally complete while one effective value remains +`Unknown`; completeness and value precision are independent. + +The parser emits only these value-domain combinations: + +- `Unknown`: no values, pattern, or covering directory; +- `Exact`: exactly one value and no pattern fields; +- `FiniteSet`: 2–32 distinct values and no pattern fields; +- `Pattern`: no values, a non-empty pattern, and a non-empty covering directory. + +The parser does not execute commands, inspect runtime variables, enumerate the +filesystem, or truncate an over-limit set and call it complete. A result with +33 or more candidates becomes `Unknown`. Structural depth starts at zero for +the root and increments on foreach loops, condition loops, conditionals, +groups, and command substitutions; blocks, lists, pipelines, branches, and +simple-command leaves do not independently increment it. Exceeding 16 +structural containers or 5 decoded-command wrapper recursions makes the entire +result unparseable. + +### Explicit redirect analysis (v0.3) + +Occurrence-specific redirect analysis is additive. The existing `Redirect` +record remains the v0.2 compatibility leaf and is not reinterpreted. + +```csharp +public sealed record RedirectAnalysis +{ + public int RedirectIndex { get; init; } = -1; + public RedirectSource Source { get; init; } = new(); + public RedirectOperation Operation { get; init; } + public int? TargetDescriptor { get; init; } + public ShellValueDomain Target { get; init; } = ShellValueDomain.Unknown; + public HereDocumentAnalysis? HereDocument { get; init; } + public bool IsPathRelevant { get; init; } + public bool IsComplete { get; init; } +} + +public sealed record HereDocumentAnalysis +{ + public ShellSourceFragment Delimiter { get; init; } = new(); + public ShellSourceFragment Body { get; init; } = new(); + public HereDocumentExpansionMode ExpansionMode { get; init; } + public bool StripLeadingTabs { get; init; } + public bool IsComplete { get; init; } +} + +public enum HereDocumentExpansionMode +{ + Unknown, + Literal, + Expand, +} + +public sealed record RedirectSource +{ + public RedirectSourceKind Kind { get; init; } + public int? Descriptor { get; init; } +} + +public enum RedirectSourceKind +{ + Unknown, + Default, + Descriptor, + PowerShellAllStreams, +} + +public enum RedirectOperation +{ + Unknown, + FileInput, + FileOutput, + FileAppend, + DescriptorDuplicate, + DescriptorClose, + DescriptorMove, + CombinedOutput, + CombinedOutputAppend, + HereDocument, + HereString, +} +``` + +`RedirectIndex` correlates to `Clause.Redirects`. `RedirectSource` preserves a +shell-default stream, an explicit numeric descriptor, or PowerShell's `*` +selector without erasing shell identity. Invalid source-kind/descriptor +combinations are incomplete and fail closed. File targets are path-relevant; +static descriptor duplicate, close, and move operations are not paths. + +`HereDocument` is non-null only for `HereDocument`. A quoted delimiter makes +the body `Literal`; an expanding body is complete only when every supported +execution-bearing substitution has been discovered as its own command +occurrence. Bash `HereString` data uses `Target`, includes the shell's trailing +newline in an exact value, and is not path-relevant. PowerShell here-strings +remain ordinary value tokens rather than redirect operations. + +The public records define an in-memory typed API, not a stable polymorphic JSON +wire format. Their generated equality, hashing, and `ToString()` behavior is +part of the normal record shape. Consumers that persist parser results own a +versioned DTO or explicit serializer mapping. + ### `Clause` One logical command within a compound. Each clause has its own verb chain, @@ -529,17 +921,71 @@ quoted_string := single-quoted | double-quoted handling" for boundary rules. - `\` before a metachar inside a double-quoted string escapes the metachar. - Single-quoted strings preserve all bytes literally — no escape processing. -- Heredocs (`<&1` from being incorrectly resolved to `/&1`. -- Function definitions, `for`/`while`/`do`/`done`/`then`/`fi`/`case`/`esac` - control-flow keywords, and arithmetic expansion `$(( ... ))` cause - `IsUnparseable = true`. We don't support these in v0.1. +- Function definitions, `case`/`esac`, C-style or implicit loops, arithmetic + execution, process substitution, and single-`&` background lists remain + unparseable in stable v0.3 because they can hide executable regions outside + the bounded grammar below. + +### v0.3 structured Bash grammar + +Contextual keywords match only in command position. `echo for` therefore +remains a simple command argument rather than starting a loop. + +```text +bash_script(stop) := bash_list_item (list_sep bash_list_item)* +bash_list_item := bash_and_or +bash_and_or := bash_pipeline (("&&" | "||") bash_pipeline)* +bash_pipeline := bash_command ("|" bash_command)* +bash_command := bash_for_in + | bash_condition_loop + | bash_if + | bash_group + | bash_subshell + | bash_c_wrapper + | bash_simple_command + +bash_for_in := "for" binding_name "in" iterable_word* + list_terminator "do" + bash_script(stop = "done") + "done" + +bash_condition_loop := ("while" | "until") + bash_script(stop = "do") "do" + bash_script(stop = "done") "done" + +bash_if := "if" bash_script(stop = "then") "then" + bash_script(stop = "elif" | "else" | "fi") + bash_elif* bash_else? "fi" +bash_elif := "elif" bash_script(stop = "then") "then" + bash_script(stop = "elif" | "else" | "fi") +bash_else := "else" bash_script(stop = "fi") + +list_sep := ";" | NEWLINE +list_terminator := ";" | NEWLINE+ +binding_name := shell_identifier +iterable_word := word | quoted_string | supported_substitution +``` + +The supported stable-v0.3 set is the existing simple-command grammar plus +`for name in words`, `while` / `until`, and `if` / `elif` / `else`. A fully +delimited command substitution in a supported iterable exposes its inner +commands but produces an `Unknown` value. A Bash path-shaped glob may produce +a `Pattern` only when its exact static covering directory is proved without +filesystem enumeration. Bash `<<<` is a non-path `HereString` redirect. + +Missing `do`, `done`, `then`, or `fi`; an unsupported substitution whose +commands cannot all be discovered; or any skipped executable region makes the +entire result unparseable. The parser may preserve a diagnostic syntax tree, +but it returns empty `Commands` and `Clauses` so consumers cannot authorize a +discovered subset. --- @@ -1382,14 +1828,16 @@ fallback "treat as one untrusted path = the raw token" prompt. ### Unparseable -Input: `for i in 1 2; do echo $i; done` +Input: `for ((i = $(next); i < 10; i++)); do run "$i"; done` ``` ParsedCommand { - Source = "for i in 1 2; do echo $i; done", + Source = "for ((i = $(next); i < 10; i++)); do run \"$i\"; done", IsUnparseable = true, - UnparseableReason = "control-flow keyword 'for' is not supported in v0.1", - Clauses = [] // or partial; consumer should not rely on contents + UnparseableReason = "C-style loops and arithmetic execution are unsupported", + Syntax = ShellBlockSyntax { ... } // optional diagnostic evidence only + Commands = [], + Clauses = [] } ``` @@ -1660,50 +2108,66 @@ v0.1.0-alpha ships when **all** of the following hold: --- -## 18. Out of Scope (deferred from v0.1) - -- PowerShell and cmd parsers. -- Variable expansion (any kind). -- Heredoc body extraction. -- Process substitution `<(cmd)`, `>(cmd)`. -- Function definitions. -- Arithmetic expansion `$((...))`. -- `for`/`while`/`case` control flow. +## 18. Out of Scope + +Stable v0.3 deliberately continues to exclude: + +- Windows `cmd` parsing. +- Command execution, filesystem glob enumeration, runtime variable lookup, or + live-shell evaluation. +- Executable-specific option, operand, object, revision, or subcommand + grammars; consumers own those semantics. +- Bash process substitution, single-`&` background lists, `case`, C-style or + implicit positional-parameter loops, arithmetic execution, functions, and + definitions until each hidden-execution and state boundary is specified. +- PowerShell `do`, `switch`, functions, definitions, class/type bodies, + arbitrary execution-bearing expressions, and `.ps1` file-content parsing. +- A stable serialized wire format for the polymorphic v0.3 records. +- Caller-configurable analysis limits, filesystem-dependent pattern + expansion, or unbounded value/state alternatives. +- Full IDE-style concrete syntax mapping. Exact-or-null source ranges exist + for security correlation, not lossless editing. - Performance optimization beyond "fast enough" (~1ms typical). -- Source-mapping (line/column for AST nodes — useful for IDEs, irrelevant - for security gates). -- Extensible verb table loading from config (v0.1 ships static tables; - consumers can layer their own knowledge on top via `BashParserOptions` - in a future version). +- Extensible verb-table loading from config. --- ## Appendix A: Consumer Contract (Netclaw) -What Netclaw expects from this library: - -1. `IShellParser` is registered in DI and `Parse(string)` returns a - `ParsedCommand` that Netclaw walks. -2. For each `Clause`, Netclaw extracts: - - `Verb.Tokens` for the verb-pattern gate evaluation. - - All `Args` where `IsPath = true` (excluding `IsCwdAttribution = true` - when the resolved path already appears in another arg) for the zone - gate evaluation. - - All `Redirects` where the target is path-shaped — the target is a - path the clause "operates on" for zone-gate purposes. -3. When `IsUnparseable = true`, Netclaw routes to safe-fail (prompt user; - offer Once / Deny only). -4. When any `Arg.Kind = DynamicSkip`, Netclaw treats that token as - "path unknown" — falls back to prompting on the raw command for the - zone gate. -5. Hard-deny rules in Netclaw evaluate against parsed `Clause` records, - not raw text (except the `rawText` escape-hatch rules — those operate - on the **rendered clause string**, recoverable via - `Clause.ToCommandString()` if we add it, or `string.Join(" ", verb + - args + redirects)` if we don't). - -The contract is stable — additive changes to AST records (new fields with -default values) are compatible; renaming or removing fields is breaking. +What a v0.3 Netclaw-style security consumer expects from this library: + +1. `IShellParser.Parse(string)` returns one `ParsedCommand`. If + `IsUnparseable` is true or `Commands` is empty, authorization prompts or + denies; neither partial `Syntax` nor raw-prefix inference can authorize. +2. The consumer evaluates every `CommandOccurrence`, including condition, + iterator, branch, body, substitution, and pipeline-stage occurrences. + `Syntax` may group the UI but is not the command-discovery API. +3. An incomplete occurrence, dynamic verb, unknown or unrecognized role, + ancestry kind, value kind, redirect kind, or policy-sensitive fact prompts + or denies. Unknown executable operands are never dropped to reuse a broader + approval. +4. For every exact or finite effective value, the consumer reapplies the + shell's binding rules and the complete executable-specific grammar at the + candidate's authored position. A finite shell proof is not authorization; + option-like candidates remain option-like. +5. Every `RedirectAnalysis` is evaluated. Static descriptor operations are + not paths. File targets are path-relevant. Complete heredoc/here-string + bodies remain data unless executable-specific stdin policy makes that data + sensitive; expanding-body substitutions appear as separate commands. +6. Hard-deny and protected-path rules still precede reusable grants. Stored + approval never bypasses a hard deny, and every command occurrence is + evaluated. +7. A v0.2 consumer may temporarily continue reading `Clauses`. The projection + includes every authored simple command from supported nested syntax and + retains dynamic authored operands, but it does not expose proved effective + loop values. Migration to `Commands` is required for bounded reuse. + +The contract is extend-only — additive records and members with safe defaults +are source and binary compatible; renaming, removing, or changing signatures +is breaking. Adding members changes generated record equality, hashing, +`ToString()`, and default reflection serialization. Consumers that persist +results own a versioned DTO or serializer mapping rather than treating the +in-memory hierarchy as a stable wire union. Before v1.0.0, while the library is in its `0.x` line, a breaking AST change MAY ship in a minor bump (e.g. the `Clause.IsCommandStringWrapped` → `IsCommandStringWrapped` rename in v0.2.0) provided `RELEASE_NOTES.md` diff --git a/openspec/changes/v0-3-structured-shell-analysis/design.md b/openspec/changes/v0-3-structured-shell-analysis/design.md index bd23d79..0321d28 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/design.md +++ b/openspec/changes/v0-3-structured-shell-analysis/design.md @@ -703,9 +703,15 @@ adapters remain shell-specific. Every enum introduced in v0.3 reserves zero as `Unknown`, except existing v0.2 enums whose zero values are already locked. Consumers fail closed on `Unknown` -or an unrecognized numeric value. The closed base constructor prevents external -syntax-node implementations; later library versions may add derived records, -so authorization code still needs a default fail-closed type-switch arm. +or an unrecognized numeric value. The source implementation pairs the shown +`private protected` ordinary base constructor with an assembly-only abstract +ownership member. Records synthesize a protected copy constructor, so the +ordinary constructor alone would still permit a specially constructed external +derived record. The non-public abstract member makes every external concrete +implementation fail compilation without adding to the public contract; every +library-owned sealed node implements it internally. Later library versions may +add derived records, so authorization code still needs a default fail-closed +type-switch arm. ### Command occurrence and bounded values diff --git a/openspec/changes/v0-3-structured-shell-analysis/tasks.md b/openspec/changes/v0-3-structured-shell-analysis/tasks.md index 5652f2a..639b023 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/tasks.md +++ b/openspec/changes/v0-3-structured-shell-analysis/tasks.md @@ -5,8 +5,8 @@ - [x] 1.3 Lock candidate-count, structural-nesting, and existing wrapper-recursion limits with boundary scenarios. - [x] 1.4 Review Appendix B, lock the v0.3 grammar separately for Bash and PowerShell, and park every deferred form explicitly. - [x] 1.5 Resolve the initial pattern and divergent-cwd exposure rules, updating the bounded-analysis specification. -- [ ] 1.6 Synchronize the accepted public API and shared requirements into `SPEC.md`. -- [ ] 1.7 Synchronize PowerShell grammar and analysis deltas into `SPEC.POWERSHELL.md`. +- [x] 1.6 Synchronize the accepted public API and shared requirements into `SPEC.md`. +- [x] 1.7 Synchronize PowerShell grammar and analysis deltas into `SPEC.POWERSHELL.md`. - [x] 1.8 Update `PROJECT_CONTEXT.md` and `IMPLEMENTATION_PLAN.md` with the accepted v0.3 scope and delivery slices. - [x] 1.9 Add a paired Bash and PowerShell design corpus that records current behavior, desired structure, command occurrences, bounded values, redirect facts, compatibility projections, and security invariants. - [ ] 1.10 Promote each design case into the executable corpus as its production parser slice lands. @@ -22,9 +22,9 @@ ## 3. Structural and Projection Skeleton -- [ ] 3.1 Add the locked public syntax-node hierarchy and defaults to the public API snapshot. -- [ ] 3.2 Add the locked command-occurrence, role, ancestry, completeness, and analysis records to the public API snapshot. -- [ ] 3.3 Add `ParsedCommand.Syntax` and `ParsedCommand.Commands` while retaining all v0.2 members. +- [x] 3.1 Add the locked public syntax-node hierarchy and defaults to the public API snapshot. +- [x] 3.2 Add the locked command-occurrence, role, ancestry, completeness, and analysis records to the public API snapshot. +- [x] 3.3 Add `ParsedCommand.Syntax` and `ParsedCommand.Commands` while retaining all v0.2 members. - [ ] 3.4 Build a library-owned traversal that emits each simple command occurrence exactly once in deterministic source order. - [ ] 3.5 Build the conservative `Clauses` compatibility flattener without inventing cross-structure compound operators. - [ ] 3.6 Adapt the existing Bash grammar to emit the structural model with no newly supported syntax. @@ -34,7 +34,7 @@ ## 4. Explicit Redirect Semantics -- [ ] 4.1 Add the locked redirect operation and target-analysis types while retaining compatibility redirect members. +- [x] 4.1 Add the locked redirect operation and target-analysis types while retaining compatibility redirect members. - [ ] 4.2 Classify Bash descriptor duplication, close, and move as static only for the complete literal descriptor grammar. - [ ] 4.3 Keep variable-driven and otherwise computed Bash descriptor targets unknown or incomplete. - [ ] 4.4 Lex and classify Bash `&>` and `&>>` independently from background-list operators. diff --git a/src/ShellSyntaxTree/CommandOccurrence.cs b/src/ShellSyntaxTree/CommandOccurrence.cs new file mode 100644 index 0000000..ba0f66b --- /dev/null +++ b/src/ShellSyntaxTree/CommandOccurrence.cs @@ -0,0 +1,147 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; + +namespace ShellSyntaxTree; + +/// One authored simple command that may execute. +public sealed record CommandOccurrence +{ + /// Gets the shared compatibility leaf. + public Clause Clause { get; init; } = new(); + + /// Gets the nearest structural execution role. + public CommandOccurrenceRole ImmediateRole { get; init; } + + /// Gets ancestry ordered from outermost to innermost. + public IReadOnlyList Ancestry { get; init; } = + Array.Empty(); + + /// Gets bounded effective values at authored argument coordinates. + public IReadOnlyList EffectiveArguments { get; init; } = + Array.Empty(); + + /// Gets the effective working-directory proof. + public ShellValueDomain WorkingDirectory { get; init; } = ShellValueDomain.Unknown; + + /// Gets explicit redirect analysis in compatibility redirect order. + public IReadOnlyList Redirects { get; init; } = + Array.Empty(); + + /// + /// Gets whether command identity, ancestry, and parser-owned shell analysis + /// are structurally complete. + /// + public bool IsComplete { get; init; } +} + +/// Identifies the nearest structural role of a command occurrence. +public enum CommandOccurrenceRole +{ + /// The role is unknown. + Unknown, + /// An ordinary command. + Ordinary, + /// A pipeline stage. + PipelineStage, + /// A condition command. + Condition, + /// An iterator-producing command. + Iterator, + /// A loop-body command. + LoopBody, + /// A conditional-branch command. + Branch, + /// A substitution command. + Substitution, +} + +/// One compositional structural ancestor of a command occurrence. +public sealed record CommandAncestryFrame +{ + /// Gets the ancestor's syntax kind. + public ShellSyntaxKind AncestorKind { get; init; } + + /// Gets the occurrence's region within the ancestor. + public CommandAncestryRegion Region { get; init; } + + /// Gets the child index for repeated regions, when applicable. + public int? ChildIndex { get; init; } + + /// Gets the ancestor source start, when exact. + public int? SourceStart { get; init; } + + /// Gets the ancestor source length, when exact. + public int? SourceLength { get; init; } +} + +/// Identifies an occurrence's region within a structural ancestor. +public enum CommandAncestryRegion +{ + /// The region is unknown. + Unknown, + /// The root region. + Root, + /// An ordinary statement region. + Statement, + /// A pipeline-stage region. + PipelineStage, + /// A group body. + GroupBody, + /// An iterator expression. + Iterator, + /// A loop body. + LoopBody, + /// A condition region. + Condition, + /// A conditional branch. + Branch, + /// A command substitution. + Substitution, +} + +/// A bounded effective value at one authored clause-element coordinate. +public sealed record EffectiveArgument +{ + /// Gets the index into . + public int ClauseElementIndex { get; init; } = -1; + + /// Gets the effective value proof. + public ShellValueDomain Value { get; init; } = ShellValueDomain.Unknown; +} + +/// A bounded, non-executing proof for a shell value. +public sealed record ShellValueDomain +{ + /// Gets the shared empty unknown-domain value. + public static ShellValueDomain Unknown { get; } = new(); + + /// Gets the proof kind. + public ShellValueDomainKind Kind { get; init; } + + /// Gets exact or finite values. + public IReadOnlyList Values { get; init; } = Array.Empty(); + + /// Gets a bounded symbolic pattern. + public string? Pattern { get; init; } + + /// Gets the pattern's conservative covering directory. + public string? CoveringDirectory { get; init; } +} + +/// Identifies the strength and shape of a shell-value proof. +public enum ShellValueDomainKind +{ + /// No bounded value is proved. + Unknown, + /// Exactly one value is proved. + Exact, + /// Two through 32 distinct values are proved. + FiniteSet, + /// A pattern and conservative covering directory are proved. + Pattern, +} diff --git a/src/ShellSyntaxTree/ParsedCommand.cs b/src/ShellSyntaxTree/ParsedCommand.cs index 585a0fe..4dda637 100644 --- a/src/ShellSyntaxTree/ParsedCommand.cs +++ b/src/ShellSyntaxTree/ParsedCommand.cs @@ -17,16 +17,30 @@ public sealed record ParsedCommand public string Source { get; init; } = ""; /// - /// Top-level clauses, split on compound operators - /// (&&, ||, ;, |). For a simple - /// command, exactly one clause with Operator=None. + /// Canonical authored nested structure. Direct-source nodes have exact + /// source ranges; decoded wrapper nodes use unavailable ranges unless an + /// exact outer mapping exists. + /// + public ShellBlockSyntax Syntax { get; init; } = new(); + + /// + /// Canonical authorization projection containing every authored simple + /// command that may execute exactly once in deterministic source order. + /// + public IReadOnlyList Commands { get; init; } = + Array.Empty(); + + /// + /// Conservative v0.2 compatibility projection. Existing simple-command + /// behavior remains available; v0.3 security consumers use + /// . /// public IReadOnlyList Clauses { get; init; } = Array.Empty(); /// - /// True when the parser could not produce a clean AST (unbalanced - /// quotes, unparseable construct). When true, Clauses MAY be partial - /// or empty. Consumers should route to safe-fail. + /// True when the parser could not account for every executable region. + /// When true, and are empty; + /// may contain partial diagnostic evidence only. /// public bool IsUnparseable { get; init; } diff --git a/src/ShellSyntaxTree/RedirectAnalysis.cs b/src/ShellSyntaxTree/RedirectAnalysis.cs new file mode 100644 index 0000000..2fa34d5 --- /dev/null +++ b/src/ShellSyntaxTree/RedirectAnalysis.cs @@ -0,0 +1,114 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +namespace ShellSyntaxTree; + +/// Explicit analysis for one compatibility redirect. +public sealed record RedirectAnalysis +{ + /// Gets the index into . + public int RedirectIndex { get; init; } = -1; + + /// Gets the redirect's source stream. + public RedirectSource Source { get; init; } = new(); + + /// Gets the redirect operation. + public RedirectOperation Operation { get; init; } + + /// Gets a static target descriptor, when proved. + public int? TargetDescriptor { get; init; } + + /// Gets the target path or data proof. + public ShellValueDomain Target { get; init; } = ShellValueDomain.Unknown; + + /// Gets heredoc-specific facts for a heredoc operation. + public HereDocumentAnalysis? HereDocument { get; init; } + + /// Gets whether the target is a filesystem policy input. + public bool IsPathRelevant { get; init; } + + /// Gets whether the parser-owned redirect analysis is complete. + public bool IsComplete { get; init; } +} + +/// Authored heredoc delimiter, body, and expansion facts. +public sealed record HereDocumentAnalysis +{ + /// Gets the authored delimiter. + public ShellSourceFragment Delimiter { get; init; } = new(); + + /// Gets the authored body. + public ShellSourceFragment Body { get; init; } = new(); + + /// Gets whether the body is literal or expanding. + public HereDocumentExpansionMode ExpansionMode { get; init; } + + /// Gets whether leading body tabs are stripped. + public bool StripLeadingTabs { get; init; } + + /// Gets whether delimiter, body, and substitutions are complete. + public bool IsComplete { get; init; } +} + +/// Identifies heredoc body expansion behavior. +public enum HereDocumentExpansionMode +{ + /// The expansion behavior is unknown. + Unknown, + /// The body is literal data. + Literal, + /// The body performs shell expansion. + Expand, +} + +/// A shell redirect's source stream. +public sealed record RedirectSource +{ + /// Gets the source-stream kind. + public RedirectSourceKind Kind { get; init; } + + /// Gets the numeric descriptor for a descriptor source. + public int? Descriptor { get; init; } +} + +/// Identifies the source side of a redirect. +public enum RedirectSourceKind +{ + /// The source is unknown. + Unknown, + /// The shell's default source stream applies. + Default, + /// An explicit numeric descriptor applies. + Descriptor, + /// PowerShell's all-streams selector applies. + PowerShellAllStreams, +} + +/// Identifies a redirect's shell operation. +public enum RedirectOperation +{ + /// The operation is unknown. + Unknown, + /// Read input from a file. + FileInput, + /// Write output to a file. + FileOutput, + /// Append output to a file. + FileAppend, + /// Duplicate one descriptor to another. + DescriptorDuplicate, + /// Close a descriptor. + DescriptorClose, + /// Move and close a descriptor. + DescriptorMove, + /// Write standard output and error to one file. + CombinedOutput, + /// Append standard output and error to one file. + CombinedOutputAppend, + /// Supply heredoc data. + HereDocument, + /// Supply Bash here-string data. + HereString, +} diff --git a/src/ShellSyntaxTree/ShellAnalysisLimits.cs b/src/ShellSyntaxTree/ShellAnalysisLimits.cs new file mode 100644 index 0000000..3ff0b45 --- /dev/null +++ b/src/ShellSyntaxTree/ShellAnalysisLimits.cs @@ -0,0 +1,19 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +namespace ShellSyntaxTree; + +/// Fixed resource and proof bounds for structured shell analysis. +public static class ShellAnalysisLimits +{ + /// Gets the maximum number of finite value candidates. + public static int MaxValueCandidates => 32; + + /// Gets the maximum structural container depth. + public static int MaxStructuralNesting => 16; + + /// Gets the maximum decoded command-string wrapper depth. + public static int MaxWrapperRecursionDepth => 5; +} diff --git a/src/ShellSyntaxTree/ShellSyntaxNode.cs b/src/ShellSyntaxTree/ShellSyntaxNode.cs new file mode 100644 index 0000000..dfbfb73 --- /dev/null +++ b/src/ShellSyntaxTree/ShellSyntaxNode.cs @@ -0,0 +1,260 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; + +namespace ShellSyntaxTree; + +/// +/// Base record for the library-owned authored syntax hierarchy. +/// +public abstract record ShellSyntaxNode +{ + private protected ShellSyntaxNode() + { + } + + private protected abstract bool IsLibraryOwnedNode { get; } + + /// Gets the stable syntax discriminant. + public abstract ShellSyntaxKind Kind { get; } + + /// Gets the zero-based start in , when exact. + public int? SourceStart { get; init; } + + /// Gets the source length, when exact. + public int? SourceLength { get; init; } +} + +/// Identifies a library-owned syntax-node shape. +public enum ShellSyntaxKind +{ + /// The kind is unknown to the consumer. + Unknown, + /// An ordered statement block. + Block, + /// An existing simple-command leaf. + SimpleCommand, + /// An ordered pipeline. + Pipeline, + /// A command list with authored operators. + CommandList, + /// A grouped command region. + Group, + /// A foreach-style loop. + ForEach, + /// A condition-controlled loop. + ConditionLoop, + /// A conditional statement. + Conditional, + /// One conditional branch. + ConditionalBranch, + /// A command substitution. + CommandSubstitution, +} + +/// An ordered block of authored statements. +public sealed record ShellBlockSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.Block; + + /// Gets the statements in authored order. + public IReadOnlyList Statements { get; init; } = + Array.Empty(); +} + +/// A simple command backed by the existing compatibility leaf. +public sealed record SimpleCommandSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.SimpleCommand; + + /// Gets the shared simple-command leaf. + public Clause Clause { get; init; } = new(); +} + +/// An ordered pipeline. +public sealed record PipelineSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.Pipeline; + + /// Gets the stages in authored order. + public IReadOnlyList Stages { get; init; } = + Array.Empty(); +} + +/// A command list preserving authored operator relationships. +public sealed record CommandListSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.CommandList; + + /// Gets the list items in authored order. + public IReadOnlyList Items { get; init; } = + Array.Empty(); +} + +/// One command-list item and its preceding authored operator. +public sealed record CommandListItemSyntax +{ + /// Gets the authored relationship to the preceding item. + public CompoundOperator Operator { get; init; } + + /// Gets the command structure for this item. + public ShellSyntaxNode Command { get; init; } = new ShellBlockSyntax(); +} + +/// A grouped command region. +public sealed record GroupSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.Group; + + /// Gets the group's shell-state behavior. + public ShellGroupKind GroupKind { get; init; } + + /// Gets the grouped body. + public ShellBlockSyntax Body { get; init; } = new(); +} + +/// Identifies whether a group shares or isolates shell state. +public enum ShellGroupKind +{ + /// The state behavior is unknown. + Unknown, + /// The group executes in the current shell scope. + CurrentScope, + /// The group executes in an isolated shell scope. + IsolatedScope, +} + +/// A foreach-style loop with a shell-specific iterable spelling. +public sealed record ForEachSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.ForEach; + + /// Gets the loop binding. + public LoopBindingSyntax Binding { get; init; } = new(); + + /// Gets the shell-specific authored iterable. + public ShellSourceFragment Iterable { get; init; } = new(); + + /// Gets commands discovered while producing iterator values. + public ShellBlockSyntax IteratorCommands { get; init; } = new(); + + /// Gets the loop body. + public ShellBlockSyntax Body { get; init; } = new(); +} + +/// A normalized loop binding and its authored source. +public sealed record LoopBindingSyntax +{ + /// Gets the normalized binding name. + public string Name { get; init; } = ""; + + /// Gets the authored binding source. + public ShellSourceFragment Source { get; init; } = new(); +} + +/// An authored source fragment with an exact-or-unavailable outer range. +public sealed record ShellSourceFragment +{ + /// Gets the authored raw spelling. + public string Raw { get; init; } = ""; + + /// Gets the zero-based outer source start, when exact. + public int? SourceStart { get; init; } + + /// Gets the outer source length, when exact. + public int? SourceLength { get; init; } +} + +/// A while- or until-style loop. +public sealed record ConditionLoopSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.ConditionLoop; + + /// Gets the loop kind. + public ConditionLoopKind LoopKind { get; init; } + + /// Gets the executable condition region. + public ShellBlockSyntax Condition { get; init; } = new(); + + /// Gets the loop body. + public ShellBlockSyntax Body { get; init; } = new(); +} + +/// Identifies the condition-loop behavior. +public enum ConditionLoopKind +{ + /// The loop kind is unknown. + Unknown, + /// The loop continues while its condition succeeds. + While, + /// The loop continues until its condition succeeds. + Until, +} + +/// A conditional statement with authored branches. +public sealed record ConditionalSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.Conditional; + + /// Gets the conditional branches in authored order. + public IReadOnlyList Branches { get; init; } = + Array.Empty(); + + /// Gets the optional else body. + public ShellBlockSyntax? Else { get; init; } +} + +/// One conditional condition and body pair. +public sealed record ConditionalBranchSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.ConditionalBranch; + + /// Gets the executable condition region. + public ShellBlockSyntax Condition { get; init; } = new(); + + /// Gets the branch body. + public ShellBlockSyntax Body { get; init; } = new(); +} + +/// A command substitution whose produced value is analyzed separately. +public sealed record CommandSubstitutionSyntax : ShellSyntaxNode +{ + private protected override bool IsLibraryOwnedNode => true; + + /// + public override ShellSyntaxKind Kind => ShellSyntaxKind.CommandSubstitution; + + /// Gets the commands inside the substitution. + public ShellBlockSyntax Body { get; init; } = new(); +} diff --git a/tests/ShellSyntaxTree.Tests/PublicApiSnapshotTests.cs b/tests/ShellSyntaxTree.Tests/PublicApiSnapshotTests.cs index d839f9d..0cdcbec 100644 --- a/tests/ShellSyntaxTree.Tests/PublicApiSnapshotTests.cs +++ b/tests/ShellSyntaxTree.Tests/PublicApiSnapshotTests.cs @@ -230,12 +230,35 @@ public void ParsedCommand_has_expected_shape() AssertIsRecord(t); AssertInitProperty(t, "Source", typeof(string)); + AssertInitProperty(t, "Syntax", typeof(ShellBlockSyntax)); + AssertInitProperty(t, "Commands", typeof(IReadOnlyList)); AssertInitProperty(t, "Clauses", typeof(IReadOnlyList)); AssertInitProperty(t, "IsUnparseable", typeof(bool)); AssertInitProperty(t, "UnparseableReason", typeof(string), nullable: true); + var declaredProps = DeclaredInstanceProps(t) + .Where(p => p.Name != "EqualityContract") + .Select(p => p.Name) + .OrderBy(n => n) + .ToArray(); + Assert.Equal( + new[] + { + "Clauses", + "Commands", + "IsUnparseable", + "Source", + "Syntax", + "UnparseableReason", + }, + declaredProps); + var instance = new ParsedCommand(); Assert.Equal("", instance.Source); + Assert.NotNull(instance.Syntax); + Assert.Equal(ShellSyntaxKind.Block, instance.Syntax.Kind); + Assert.Empty(instance.Syntax.Statements); + Assert.Empty(instance.Commands); Assert.Empty(instance.Clauses); Assert.False(instance.IsUnparseable); Assert.Null(instance.UnparseableReason); @@ -473,14 +496,45 @@ public void Public_namespace_contains_only_expected_types() nameof(Clause), nameof(ClauseElement), nameof(ClauseElementRole), + nameof(CommandAncestryFrame), + nameof(CommandAncestryRegion), + nameof(CommandListItemSyntax), + nameof(CommandListSyntax), + nameof(CommandOccurrence), + nameof(CommandOccurrenceRole), + nameof(CommandSubstitutionSyntax), nameof(CompoundOperator), + nameof(ConditionalBranchSyntax), + nameof(ConditionalSyntax), + nameof(ConditionLoopKind), + nameof(ConditionLoopSyntax), + nameof(EffectiveArgument), + nameof(ForEachSyntax), + nameof(GroupSyntax), + nameof(HereDocumentAnalysis), + nameof(HereDocumentExpansionMode), nameof(IShellParser), + nameof(LoopBindingSyntax), nameof(ParsedCommand), + nameof(PipelineSyntax), nameof(PwshParser), nameof(PwshParserOptions), nameof(Redirect), + nameof(RedirectAnalysis), nameof(RedirectDirection), + nameof(RedirectOperation), + nameof(RedirectSource), + nameof(RedirectSourceKind), + nameof(ShellAnalysisLimits), + nameof(ShellBlockSyntax), + nameof(ShellGroupKind), nameof(ShellParserOptions), + nameof(ShellSourceFragment), + nameof(ShellSyntaxKind), + nameof(ShellSyntaxNode), + nameof(ShellValueDomain), + nameof(ShellValueDomainKind), + nameof(SimpleCommandSyntax), nameof(VerbChain), }.OrderBy(n => n).ToArray(); diff --git a/tests/ShellSyntaxTree.Tests/V03PublicApiSnapshotTests.cs b/tests/ShellSyntaxTree.Tests/V03PublicApiSnapshotTests.cs new file mode 100644 index 0000000..abe7483 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/V03PublicApiSnapshotTests.cs @@ -0,0 +1,532 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Xunit; + +namespace ShellSyntaxTree.Tests; + +/// Locks the additive v0.3 public API and its safe defaults. +public class V03PublicApiSnapshotTests +{ + [Fact] + public void Syntax_node_base_is_record_shaped_and_closed_to_external_implementations() + { + var type = typeof(ShellSyntaxNode); + Assert.True(type.IsPublic); + Assert.True(type.IsAbstract); + AssertRecord(type); + + AssertGetProperty(type, nameof(ShellSyntaxNode.Kind), typeof(ShellSyntaxKind)); + AssertInitProperty(type, nameof(ShellSyntaxNode.SourceStart), typeof(int?)); + AssertInitProperty(type, nameof(ShellSyntaxNode.SourceLength), typeof(int?)); + Assert.Equal( + new[] + { + nameof(ShellSyntaxNode.Kind), + nameof(ShellSyntaxNode.SourceLength), + nameof(ShellSyntaxNode.SourceStart), + }, + DeclaredPublicProperties(type)); + + var constructor = type.GetConstructor( + BindingFlags.Instance | BindingFlags.NonPublic, + binder: null, + Type.EmptyTypes, + modifiers: null); + Assert.NotNull(constructor); + Assert.True(constructor!.IsFamilyAndAssembly); + + var ownership = type.GetProperty( + "IsLibraryOwnedNode", + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(ownership); + Assert.NotNull(ownership!.GetMethod); + Assert.True(ownership.GetMethod!.IsAbstract); + Assert.True(ownership.GetMethod.IsFamilyAndAssembly); + } + + [Fact] + public void Syntax_records_match_the_locked_members_and_defaults() + { + AssertNode( + new ShellBlockSyntax(), + ShellSyntaxKind.Block, + (nameof(ShellBlockSyntax.Statements), typeof(IReadOnlyList))); + AssertNode( + new SimpleCommandSyntax(), + ShellSyntaxKind.SimpleCommand, + (nameof(SimpleCommandSyntax.Clause), typeof(Clause))); + AssertNode( + new PipelineSyntax(), + ShellSyntaxKind.Pipeline, + (nameof(PipelineSyntax.Stages), typeof(IReadOnlyList))); + AssertNode( + new CommandListSyntax(), + ShellSyntaxKind.CommandList, + (nameof(CommandListSyntax.Items), typeof(IReadOnlyList))); + AssertNode( + new GroupSyntax(), + ShellSyntaxKind.Group, + (nameof(GroupSyntax.GroupKind), typeof(ShellGroupKind)), + (nameof(GroupSyntax.Body), typeof(ShellBlockSyntax))); + AssertNode( + new ForEachSyntax(), + ShellSyntaxKind.ForEach, + (nameof(ForEachSyntax.Binding), typeof(LoopBindingSyntax)), + (nameof(ForEachSyntax.Iterable), typeof(ShellSourceFragment)), + (nameof(ForEachSyntax.IteratorCommands), typeof(ShellBlockSyntax)), + (nameof(ForEachSyntax.Body), typeof(ShellBlockSyntax))); + AssertNode( + new ConditionLoopSyntax(), + ShellSyntaxKind.ConditionLoop, + (nameof(ConditionLoopSyntax.LoopKind), typeof(ConditionLoopKind)), + (nameof(ConditionLoopSyntax.Condition), typeof(ShellBlockSyntax)), + (nameof(ConditionLoopSyntax.Body), typeof(ShellBlockSyntax))); + AssertNode( + new ConditionalSyntax(), + ShellSyntaxKind.Conditional, + (nameof(ConditionalSyntax.Branches), typeof(IReadOnlyList)), + (nameof(ConditionalSyntax.Else), typeof(ShellBlockSyntax))); + AssertNode( + new ConditionalBranchSyntax(), + ShellSyntaxKind.ConditionalBranch, + (nameof(ConditionalBranchSyntax.Condition), typeof(ShellBlockSyntax)), + (nameof(ConditionalBranchSyntax.Body), typeof(ShellBlockSyntax))); + AssertNode( + new CommandSubstitutionSyntax(), + ShellSyntaxKind.CommandSubstitution, + (nameof(CommandSubstitutionSyntax.Body), typeof(ShellBlockSyntax))); + + var simple = new SimpleCommandSyntax(); + Assert.NotNull(simple.Clause); + Assert.Empty(new ShellBlockSyntax().Statements); + Assert.Empty(new PipelineSyntax().Stages); + Assert.Empty(new CommandListSyntax().Items); + var group = new GroupSyntax(); + Assert.Equal(ShellGroupKind.Unknown, group.GroupKind); + Assert.NotNull(group.Body); + Assert.Empty(group.Body.Statements); + + var forEach = new ForEachSyntax(); + Assert.NotNull(forEach.Binding); + Assert.NotNull(forEach.Iterable); + Assert.NotNull(forEach.IteratorCommands); + Assert.NotNull(forEach.Body); + Assert.Empty(forEach.IteratorCommands.Statements); + Assert.Empty(forEach.Body.Statements); + + var conditionLoop = new ConditionLoopSyntax(); + Assert.Equal(ConditionLoopKind.Unknown, conditionLoop.LoopKind); + Assert.NotNull(conditionLoop.Condition); + Assert.NotNull(conditionLoop.Body); + Assert.Empty(conditionLoop.Condition.Statements); + Assert.Empty(conditionLoop.Body.Statements); + + var conditional = new ConditionalSyntax(); + Assert.Empty(conditional.Branches); + Assert.Null(conditional.Else); + + var branch = new ConditionalBranchSyntax(); + Assert.NotNull(branch.Condition); + Assert.NotNull(branch.Body); + Assert.Empty(branch.Condition.Statements); + Assert.Empty(branch.Body.Statements); + + var substitution = new CommandSubstitutionSyntax(); + Assert.NotNull(substitution.Body); + Assert.Empty(substitution.Body.Statements); + } + + [Fact] + public void Supporting_syntax_records_match_the_locked_members_and_defaults() + { + AssertRecordWithInitProperties( + typeof(CommandListItemSyntax), + (nameof(CommandListItemSyntax.Operator), typeof(CompoundOperator)), + (nameof(CommandListItemSyntax.Command), typeof(ShellSyntaxNode))); + var item = new CommandListItemSyntax(); + Assert.Equal(CompoundOperator.None, item.Operator); + Assert.IsType(item.Command); + + AssertRecordWithInitProperties( + typeof(LoopBindingSyntax), + (nameof(LoopBindingSyntax.Name), typeof(string)), + (nameof(LoopBindingSyntax.Source), typeof(ShellSourceFragment))); + var binding = new LoopBindingSyntax(); + Assert.Equal("", binding.Name); + Assert.NotNull(binding.Source); + + AssertRecordWithInitProperties( + typeof(ShellSourceFragment), + (nameof(ShellSourceFragment.Raw), typeof(string)), + (nameof(ShellSourceFragment.SourceStart), typeof(int?)), + (nameof(ShellSourceFragment.SourceLength), typeof(int?))); + var fragment = new ShellSourceFragment(); + Assert.Equal("", fragment.Raw); + Assert.Null(fragment.SourceStart); + Assert.Null(fragment.SourceLength); + } + + [Fact] + public void Command_occurrence_records_match_the_locked_members_and_defaults() + { + AssertRecordWithInitProperties( + typeof(CommandOccurrence), + (nameof(CommandOccurrence.Clause), typeof(Clause)), + (nameof(CommandOccurrence.ImmediateRole), typeof(CommandOccurrenceRole)), + (nameof(CommandOccurrence.Ancestry), typeof(IReadOnlyList)), + (nameof(CommandOccurrence.EffectiveArguments), typeof(IReadOnlyList)), + (nameof(CommandOccurrence.WorkingDirectory), typeof(ShellValueDomain)), + (nameof(CommandOccurrence.Redirects), typeof(IReadOnlyList)), + (nameof(CommandOccurrence.IsComplete), typeof(bool))); + var occurrence = new CommandOccurrence(); + Assert.NotNull(occurrence.Clause); + Assert.Equal(CommandOccurrenceRole.Unknown, occurrence.ImmediateRole); + Assert.Empty(occurrence.Ancestry); + Assert.Empty(occurrence.EffectiveArguments); + Assert.Same(ShellValueDomain.Unknown, occurrence.WorkingDirectory); + Assert.Empty(occurrence.Redirects); + Assert.False(occurrence.IsComplete); + + AssertRecordWithInitProperties( + typeof(CommandAncestryFrame), + (nameof(CommandAncestryFrame.AncestorKind), typeof(ShellSyntaxKind)), + (nameof(CommandAncestryFrame.Region), typeof(CommandAncestryRegion)), + (nameof(CommandAncestryFrame.ChildIndex), typeof(int?)), + (nameof(CommandAncestryFrame.SourceStart), typeof(int?)), + (nameof(CommandAncestryFrame.SourceLength), typeof(int?))); + var ancestry = new CommandAncestryFrame(); + Assert.Equal(ShellSyntaxKind.Unknown, ancestry.AncestorKind); + Assert.Equal(CommandAncestryRegion.Unknown, ancestry.Region); + Assert.Null(ancestry.ChildIndex); + Assert.Null(ancestry.SourceStart); + Assert.Null(ancestry.SourceLength); + + AssertRecordWithInitProperties( + typeof(EffectiveArgument), + (nameof(EffectiveArgument.ClauseElementIndex), typeof(int)), + (nameof(EffectiveArgument.Value), typeof(ShellValueDomain))); + var argument = new EffectiveArgument(); + Assert.Equal(-1, argument.ClauseElementIndex); + Assert.Same(ShellValueDomain.Unknown, argument.Value); + } + + [Fact] + public void Shell_value_domain_matches_the_locked_members_and_defaults() + { + AssertRecordWithInitProperties( + typeof(ShellValueDomain), + (nameof(ShellValueDomain.Kind), typeof(ShellValueDomainKind)), + (nameof(ShellValueDomain.Values), typeof(IReadOnlyList)), + (nameof(ShellValueDomain.Pattern), typeof(string)), + (nameof(ShellValueDomain.CoveringDirectory), typeof(string))); + + var unknown = ShellValueDomain.Unknown; + Assert.Same(unknown, ShellValueDomain.Unknown); + Assert.Equal(ShellValueDomainKind.Unknown, unknown.Kind); + Assert.Empty(unknown.Values); + Assert.Null(unknown.Pattern); + Assert.Null(unknown.CoveringDirectory); + + var property = typeof(ShellValueDomain).GetProperty( + nameof(ShellValueDomain.Unknown), + BindingFlags.Public | BindingFlags.Static); + Assert.NotNull(property); + Assert.True(property!.CanRead); + Assert.False(property.CanWrite); + Assert.Equal( + new[] { nameof(ShellValueDomain.Unknown) }, + typeof(ShellValueDomain) + .GetProperties(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Select(candidate => candidate.Name) + .OrderBy(name => name)); + } + + [Fact] + public void Redirect_analysis_records_match_the_locked_members_and_defaults() + { + AssertRecordWithInitProperties( + typeof(RedirectAnalysis), + (nameof(RedirectAnalysis.RedirectIndex), typeof(int)), + (nameof(RedirectAnalysis.Source), typeof(RedirectSource)), + (nameof(RedirectAnalysis.Operation), typeof(RedirectOperation)), + (nameof(RedirectAnalysis.TargetDescriptor), typeof(int?)), + (nameof(RedirectAnalysis.Target), typeof(ShellValueDomain)), + (nameof(RedirectAnalysis.HereDocument), typeof(HereDocumentAnalysis)), + (nameof(RedirectAnalysis.IsPathRelevant), typeof(bool)), + (nameof(RedirectAnalysis.IsComplete), typeof(bool))); + var redirect = new RedirectAnalysis(); + Assert.Equal(-1, redirect.RedirectIndex); + Assert.NotNull(redirect.Source); + Assert.Equal(RedirectOperation.Unknown, redirect.Operation); + Assert.Null(redirect.TargetDescriptor); + Assert.Same(ShellValueDomain.Unknown, redirect.Target); + Assert.Null(redirect.HereDocument); + Assert.False(redirect.IsPathRelevant); + Assert.False(redirect.IsComplete); + + AssertRecordWithInitProperties( + typeof(HereDocumentAnalysis), + (nameof(HereDocumentAnalysis.Delimiter), typeof(ShellSourceFragment)), + (nameof(HereDocumentAnalysis.Body), typeof(ShellSourceFragment)), + (nameof(HereDocumentAnalysis.ExpansionMode), typeof(HereDocumentExpansionMode)), + (nameof(HereDocumentAnalysis.StripLeadingTabs), typeof(bool)), + (nameof(HereDocumentAnalysis.IsComplete), typeof(bool))); + var hereDocument = new HereDocumentAnalysis(); + Assert.NotNull(hereDocument.Delimiter); + Assert.NotNull(hereDocument.Body); + Assert.Equal(HereDocumentExpansionMode.Unknown, hereDocument.ExpansionMode); + Assert.False(hereDocument.StripLeadingTabs); + Assert.False(hereDocument.IsComplete); + + AssertRecordWithInitProperties( + typeof(RedirectSource), + (nameof(RedirectSource.Kind), typeof(RedirectSourceKind)), + (nameof(RedirectSource.Descriptor), typeof(int?))); + var source = new RedirectSource(); + Assert.Equal(RedirectSourceKind.Unknown, source.Kind); + Assert.Null(source.Descriptor); + } + + [Fact] + public void V03_enums_reserve_zero_for_unknown_and_match_the_locked_order() + { + AssertEnum( + "Unknown", "Block", "SimpleCommand", "Pipeline", "CommandList", + "Group", "ForEach", "ConditionLoop", "Conditional", + "ConditionalBranch", "CommandSubstitution"); + AssertEnum("Unknown", "CurrentScope", "IsolatedScope"); + AssertEnum("Unknown", "While", "Until"); + AssertEnum( + "Unknown", "Ordinary", "PipelineStage", "Condition", "Iterator", + "LoopBody", "Branch", "Substitution"); + AssertEnum( + "Unknown", "Root", "Statement", "PipelineStage", "GroupBody", + "Iterator", "LoopBody", "Condition", "Branch", "Substitution"); + AssertEnum("Unknown", "Exact", "FiniteSet", "Pattern"); + AssertEnum("Unknown", "Literal", "Expand"); + AssertEnum( + "Unknown", "Default", "Descriptor", "PowerShellAllStreams"); + AssertEnum( + "Unknown", "FileInput", "FileOutput", "FileAppend", + "DescriptorDuplicate", "DescriptorClose", "DescriptorMove", + "CombinedOutput", "CombinedOutputAppend", "HereDocument", "HereString"); + } + + [Fact] + public void V03_reference_property_nullability_matches_the_locked_contract() + { + var types = new[] + { + typeof(ParsedCommand), + typeof(ShellSyntaxNode), + typeof(ShellBlockSyntax), + typeof(SimpleCommandSyntax), + typeof(PipelineSyntax), + typeof(CommandListSyntax), + typeof(CommandListItemSyntax), + typeof(GroupSyntax), + typeof(ForEachSyntax), + typeof(LoopBindingSyntax), + typeof(ShellSourceFragment), + typeof(ConditionLoopSyntax), + typeof(ConditionalSyntax), + typeof(ConditionalBranchSyntax), + typeof(CommandSubstitutionSyntax), + typeof(CommandOccurrence), + typeof(CommandAncestryFrame), + typeof(EffectiveArgument), + typeof(ShellValueDomain), + typeof(RedirectAnalysis), + typeof(HereDocumentAnalysis), + typeof(RedirectSource), + }; + var nullableProperties = new HashSet(StringComparer.Ordinal) + { + $"{nameof(ParsedCommand)}.{nameof(ParsedCommand.UnparseableReason)}", + $"{nameof(ConditionalSyntax)}.{nameof(ConditionalSyntax.Else)}", + $"{nameof(ShellValueDomain)}.{nameof(ShellValueDomain.Pattern)}", + $"{nameof(ShellValueDomain)}.{nameof(ShellValueDomain.CoveringDirectory)}", + $"{nameof(RedirectAnalysis)}.{nameof(RedirectAnalysis.HereDocument)}", + }; + var context = new NullabilityInfoContext(); + var visited = new HashSet(StringComparer.Ordinal); + + foreach (var type in types) + { + var properties = type.GetProperties( + BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly); + foreach (var property in properties.Where( + property => property.Name != "EqualityContract" && + !property.PropertyType.IsValueType)) + { + var key = $"{type.Name}.{property.Name}"; + var expected = nullableProperties.Contains(key) + ? NullabilityState.Nullable + : NullabilityState.NotNull; + var nullability = context.Create(property); + Assert.Equal(expected, nullability.ReadState); + if (property.CanWrite) + { + Assert.Equal(expected, nullability.WriteState); + } + + AssertGenericArgumentsAreNotNull(key, nullability); + visited.Add(key); + } + } + + Assert.All(nullableProperties, property => Assert.Contains(property, visited)); + + var unknownProperty = typeof(ShellValueDomain).GetProperty( + nameof(ShellValueDomain.Unknown), + BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly); + Assert.NotNull(unknownProperty); + Assert.Equal( + NullabilityState.NotNull, + context.Create(unknownProperty!).ReadState); + } + + [Fact] + public void Analysis_limits_are_static_get_only_and_match_the_locked_values() + { + var type = typeof(ShellAnalysisLimits); + Assert.True(type.IsPublic); + Assert.True(type.IsAbstract); + Assert.True(type.IsSealed); + + Assert.Equal(32, ShellAnalysisLimits.MaxValueCandidates); + Assert.Equal(16, ShellAnalysisLimits.MaxStructuralNesting); + Assert.Equal(5, ShellAnalysisLimits.MaxWrapperRecursionDepth); + + var properties = type.GetProperties( + BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly); + Assert.Equal( + new[] + { + nameof(ShellAnalysisLimits.MaxStructuralNesting), + nameof(ShellAnalysisLimits.MaxValueCandidates), + nameof(ShellAnalysisLimits.MaxWrapperRecursionDepth), + }, + properties.Select(property => property.Name).OrderBy(name => name)); + + foreach (var property in properties) + { + Assert.True(property.GetMethod!.IsStatic); + Assert.False(property.CanWrite); + Assert.Equal(typeof(int), property.PropertyType); + } + } + + private static void AssertNode( + ShellSyntaxNode instance, + ShellSyntaxKind expectedKind, + params (string Name, Type Type)[] properties) + { + var type = instance.GetType(); + Assert.True(type.IsPublic); + Assert.True(type.IsSealed); + AssertRecord(type); + Assert.Equal(expectedKind, instance.Kind); + Assert.Null(instance.SourceStart); + Assert.Null(instance.SourceLength); + + AssertGetProperty(type, nameof(ShellSyntaxNode.Kind), typeof(ShellSyntaxKind)); + foreach (var property in properties) + { + AssertInitProperty(type, property.Name, property.Type); + } + + var expectedNames = properties + .Select(property => property.Name) + .Append(nameof(ShellSyntaxNode.Kind)) + .OrderBy(name => name) + .ToArray(); + Assert.Equal(expectedNames, DeclaredPublicProperties(type)); + } + + private static void AssertRecordWithInitProperties( + Type type, + params (string Name, Type Type)[] properties) + { + Assert.True(type.IsPublic); + Assert.True(type.IsSealed); + AssertRecord(type); + foreach (var property in properties) + { + AssertInitProperty(type, property.Name, property.Type); + } + + Assert.Equal( + properties.Select(property => property.Name).OrderBy(name => name), + DeclaredPublicProperties(type)); + } + + private static string[] DeclaredPublicProperties(Type type) => + type.GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly) + .Where(property => property.Name != "EqualityContract") + .Select(property => property.Name) + .OrderBy(name => name) + .ToArray(); + + private static void AssertRecord(Type type) + { + var clone = type.GetMethod( + "$", + BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly); + Assert.NotNull(clone); + } + + private static void AssertGenericArgumentsAreNotNull( + string propertyPath, + NullabilityInfo nullability) + { + foreach (var argument in nullability.GenericTypeArguments) + { + Assert.True( + argument.ReadState == NullabilityState.NotNull, + $"{propertyPath} has a nullable or oblivious generic argument."); + AssertGenericArgumentsAreNotNull(propertyPath, argument); + } + } + + private static void AssertGetProperty(Type type, string name, Type expectedType) + { + var property = type.GetProperty(name); + Assert.NotNull(property); + Assert.Equal(expectedType, property!.PropertyType); + Assert.True(property.CanRead); + Assert.False(property.CanWrite); + } + + private static void AssertInitProperty(Type type, string name, Type expectedType) + { + var property = type.GetProperty(name); + Assert.NotNull(property); + Assert.Equal(expectedType, property!.PropertyType); + Assert.True(property.CanRead); + Assert.True(property.CanWrite); + var setter = Assert.IsAssignableFrom(property.SetMethod); + Assert.Contains( + setter.ReturnParameter.GetRequiredCustomModifiers(), + modifier => modifier.FullName == "System.Runtime.CompilerServices.IsExternalInit"); + } + + private static void AssertEnum(params string[] expectedNames) + where T : struct, Enum + { + Assert.Equal(expectedNames, Enum.GetNames(typeof(T))); + var values = Enum.GetValues(typeof(T)); + Assert.Equal(expectedNames.Length, values.Length); + for (var index = 0; index < values.Length; index++) + { + Assert.Equal(index, Convert.ToInt32(values.GetValue(index))); + } + } +} From ce5624d8db24ce7f98b8e14c4a21b2bab3e27fa5 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 7 Aug 2026 02:51:48 +0000 Subject: [PATCH 2/2] Specify nested command substitutions --- IMPLEMENTATION_PLAN.md | 10 + SPEC.POWERSHELL.md | 124 ++++++--- SPEC.md | 61 ++++- .../v0-3-structured-shell-analysis/design.md | 67 ++++- .../specs/bounded-shell-analysis/spec.md | 23 ++ .../executable-command-projection/spec.md | 32 ++- .../specs/structured-shell-syntax/spec.md | 67 ++++- .../v0-3-structured-shell-analysis/tasks.md | 7 +- src/ShellSyntaxTree/ShellSyntaxNode.cs | 7 + src/ShellSyntaxTree/VerbChain.cs | 7 +- .../DesignCorpus/README.md | 18 ++ .../DesignCorpus/V03DesignCorpusTests.cs | 30 +++ .../DesignCorpus/v0.3/bash.json | 171 ++++++++++++- .../DesignCorpus/v0.3/powershell.json | 236 ++++++++++++++++++ .../V03PublicApiSnapshotTests.cs | 6 +- 15 files changed, 813 insertions(+), 53 deletions(-) diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 0482625..a70ab80 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -225,12 +225,22 @@ priorities. fail-closed while v0.2 `Clauses` behavior is unchanged. - [ ] Add the structural and command-occurrence projections for the existing grammar before enabling any control-flow construct. +- [ ] Deliver paired Bash and PowerShell `$()` substitution slices for all + locked executable value positions, including ordering, ancestry, + shell-specific cwd propagation, literal/escaped boundaries, dynamic + identities, fail-closed negatives, executable corpus, and Netclaw cases. - [ ] Deliver Bash `for ... in` and PowerShell `foreach` as the first two language-specific vertical slices, then extract only the shared analysis proven by both implementations. - [ ] Preserve the existing Bash heredoc grammar, fix quoted-delimiter adjacency, expose body/delimiter/expansion/completeness facts, and add a separately tested Bash `<<<` here-string redirect slice. +- [ ] Near the end of v0.3 delivery, expand the Web sample with curated complex + Bash and PowerShell inputs and deterministic Mermaid views of syntax, + occurrences, compatibility clauses, ancestry, redirects, and fail-closed + outcomes. Keep visualization downstream of the canonical projection so + it cannot become a second command-discovery implementation; snapshot the + rendering, escape arbitrary shell labels, and emit no raw HTML. --- diff --git a/SPEC.POWERSHELL.md b/SPEC.POWERSHELL.md index 2abe75a..9abfda9 100644 --- a/SPEC.POWERSHELL.md +++ b/SPEC.POWERSHELL.md @@ -221,9 +221,11 @@ value unambiguously signals "an alias was expanded." ```csharp /// /// True when the clause's command name is a dynamic token the parser -/// cannot statically identify — a variable (`& $exe`), a subexpression -/// (`& (Get-Thing)`), or a script block (`& { ... }`) at verb position. -/// Tokens still carries the verbatim token; CanonicalVerb is null. +/// cannot statically identify — a variable (`& $exe`), an interpolated name, +/// or a supported subexpression (`& $(Get-Thing)`) at verb position. Tokens +/// still carries the verbatim token; CanonicalVerb is null. An unsupported +/// executable identity expression makes the whole result unparseable with +/// empty command and compatibility projections instead. /// /// A consumer MUST treat a clause with IsDynamic=true as "the command being /// run is unknown" and route to safe-fail — the verb identity, and @@ -264,16 +266,21 @@ command := statement (statement_sep statement)* statement_sep := ";" | "&&" | "||" | NEWLINE statement := pipeline pipeline := pipeline_element ("|" pipeline_element)* -pipeline_element := call_op? command_name arg* redirect* +pipeline_element := static_invocation | dynamic_invocation + | supported_subexpression | grouped_pipeline +static_invocation := call_op? command_name arg* redirect* +dynamic_invocation := call_op dynamic_command_name arg* redirect* call_op := "&" // call operator at verb position -command_name := cmdlet | native_word | quoted_string | grouped_pipeline +command_name := cmdlet | native_word // statically identified; + // excludes variables, + // quoted expressions, and $() arg := parameter | value parameter := "-" param_name (":" value)? // -Name value | -Name:value | "-" param_name // switch parameter | "--" // end-of-parameters marker value := word | quoted_string | here_string | script_block // { ... } -> DynamicSkip Arg - | subexpression // $( ... ) -> DynamicSkip Arg + | supported_subexpression // $( ... ) -> child commands + DynamicSkip Arg | array_expression // @( ... ) -> DynamicSkip Arg | hash_literal // @{ ... } -> DynamicSkip Arg | splat // @var -> DynamicSkip Arg @@ -281,12 +288,19 @@ redirect := redirect_op target redirect_op := ">" | ">>" | "<" | STREAM ">" | STREAM ">>" // STREAM in {1..6, *} | STREAM ">&" STREAM // stream merge (2>&1) -target := word | quoted_string | "$null" +target := word | quoted_string | supported_subexpression | "$null" +supported_subexpression := "$(" command ")" +dynamic_command_name := variable | quoted_string | supported_subexpression +variable := "$" identifier | "${" variable_name "}" + | "$env:" identifier grouped_pipeline := "(" pipeline ")" // parenthesized sub-pipeline word := run of non-whitespace, non-operator, non-quote chars, honoring backtick escape; absorbs $var / ${name} / $env:NAME / drive-qualified path prefixes quoted_string := single_quoted | double_quoted + // double-quoted and expandable here-string values may + // contain supported_subexpression children; literal or + // backtick-escaped spellings do not ``` **Notes:** @@ -302,10 +316,13 @@ quoted_string := single_quoted | double_quoted - `&` at verb position is the **call operator** (`& git status`, `& $exe`, `& { ... }`). A *trailing* `&` (a PowerShell background job) marks `IsUnparseable` (§11). -- A clause whose command name (after an optional `&`) is a dynamic token — - a variable `$var`, a subexpression `$( ... )`, or a script block - `{ ... }` — still parses, but its `VerbChain.IsDynamic` is set true (§3). - Its args and redirects parse normally. +- Under v0.2 a variable, subexpression, quoted string, or script block at + command position could be retained as a dynamic clause. Stable v0.3 aligns + this with PowerShell invocation semantics: only `& ` + creates an outer dynamic command occurrence. Standalone `$()` is an + expression statement whose inner commands are exposed without inventing an + invocation of its produced value. Standalone `$()` followed by command-style + arguments is a syntax error and makes the whole result unparseable. - A parenthesized **pipeline** `( ... )` parses as a grouped sub-pipeline; its clauses carry `IsSubshell = true` as a *structural* marker only. Unlike a bash subshell, PowerShell's `( ... )` is a grouping operator — it @@ -320,9 +337,15 @@ quoted_string := single_quoted | double_quoted `DynamicSkip` arg. `--%` does **not** stop at a pipeline-element boundary; treating `| cmd` after `--%` as a new clause would invent a clause that does not exist. -- Script blocks `{ ... }`, subexpressions `$( ... )`, array `@( ... )`, and - hash `@{ ... }` literals are recognized as **opaque tokens** — the - interior is not parsed; each becomes one `DynamicSkip` arg. +- The lexer preserves script blocks `{ ... }`, subexpressions `$( ... )`, + array `@( ... )`, and hash `@{ ... }` literals as bounded tokens. Under + v0.2 each was one opaque `DynamicSkip` arg. Stable v0.3 recursively parses + every completely delimited executable `$()` in a supported value position + and exposes its commands while retaining the containing authored + `DynamicSkip` leaf. Ordinary script-block literals remain non-executing + opaque values. An `@()` or `@{}` value with execution-bearing content is + unparseable until that expression form has complete command discovery; + a non-executing literal form may remain an opaque value. - Under v0.2, control-flow keywords fall outside the grammar. Stable v0.3 owns only the contextual statement forms below. Definition keywords, unsupported block keywords, `param()`, assignment statements, bare `[type]::member` @@ -356,6 +379,7 @@ pwsh_else := "else" script_block_body foreach_expression := literal_value | literal_array | pipeline_expression + | supported_subexpression literal_array := "@(" literal_value ("," literal_value)* ")" script_block_body := "{" pwsh_script(stop = "}") "}" ``` @@ -368,6 +392,27 @@ after the structural grammar proves that the `ScriptBlock` token is the body of a recognized statement. An ordinary script-block argument remains one opaque `DynamicSkip` value and does not invent child execution. +A completely delimited `$()` used as an ordinary word, dynamic command +identity after `&`, redirect value, foreach expression, double-quoted interpolation, or +expandable here-string is recursively parsed as a command substitution. Its +commands are exposed before the containing command and its produced value is +`Unknown`. Single-quoted strings, literal here-strings, and backtick-escaped +`$()` text never create substitution nodes. An unsupported execution-bearing +expression makes the whole result unparseable rather than leaving hidden +commands inside a `DynamicSkip` value. + +A standalone `$()` statement has no containing simple command: the syntax block +contains the `CommandSubstitutionSyntax` directly and `Commands` contains only +commands from its body. `& $(...)` additionally retains one incomplete dynamic +outer occurrence after all substitution commands. Bash differs: an unquoted +command substitution in Bash command-name position contributes to the command +word, so stable v0.3 makes that runtime-dependent identity unparseable rather +than inventing a static or PowerShell-style dynamic clause. + +`& { ... }` executes a script block and remains unparseable in stable v0.3 +until its body, scope, and state propagation are modeled. This differs from an +ordinary script-block argument, which remains a non-executing opaque value. + `condition_pipeline` is limited to a pipeline the existing parser can delimit completely. A subexpression, member invocation, script block, or other form that may execute outside complete command discovery makes the whole result @@ -416,7 +461,10 @@ The `PwshLexer` produces tokens consumed by `PwshCommandParser`. Token kinds - **ScriptBlock** — a balanced `{ ... }` region, emitted whole. Parser → `DynamicSkip` arg. - **Subexpression** — a balanced `$( ... )`, `@( ... )`, or `@{ ... }` - region, emitted whole. Parser → `DynamicSkip` arg. + region, emitted whole. The v0.3 structural parser recursively lowers + supported `$()` regions and retains the outer compatibility arg as + `DynamicSkip`; unsupported execution-bearing array/hash expressions fail + closed. - **Splat** — `@identifier` (splatting). Parser → `DynamicSkip` arg. - **StopParsing** — the `--%` token; the pipeline-element remainder is opaque. @@ -432,14 +480,17 @@ The `PwshLexer` produces tokens consumed by `PwshCommandParser`. Token kinds `$var` / `${name}` / `$env:X` / `$( ... )` interpolation — but the parser **does not expand**; `$var` stays literal in the token value and the resolver (§8) classifies it. A `$( ... )` inside a double-quoted string - does not split the token. Backtick character escapes are decoded into the + does not split the compatibility token, but v0.3 recursively exposes its + commands through `SimpleCommandSyntax.Substitutions`. Backtick character + escapes are decoded into the same logical value PowerShell passes to a command, including `` `u{hex}`` Unicode scalar escapes. - **Here-strings** — `@"` + newline ... newline + `"@` (expandable) and `@'` + newline ... newline + `'@` (literal). The closing delimiter must start a line. Lexes to one `QuotedString` token with `IsHereString=true`. - Expandable here-strings decode backtick character escapes; literal - here-strings preserve their body bytes. + Expandable here-strings decode backtick character escapes and v0.3 discovers + every executable `$()` interpolation; literal here-strings preserve their + body bytes and never create substitution commands. - Unbalanced quotes or here-strings → `IsUnparseable` with a reason. ### Escape handling @@ -993,13 +1044,23 @@ pipeline elements within a statement, not separate statements. Script blocks `{ ... }`, subexpressions `$( ... )`, array subexpressions `@( ... )`, and hash literals `@{ ... }` are bounded by the shared -`OpaqueRegionScanner` and emitted as single tokens; the parser consumes each -as one `Arg { Kind=DynamicSkip, IsPath=false, Resolved=null }`, `Raw` being -the verbatim region slice. Splatting `@var` is likewise `DynamicSkip`. The -`--%` stop-parsing token makes the pipeline-element remainder one -`DynamicSkip` arg. The interior of these regions is **not** parsed — this is -the heart of the Pipeline-aware scope (§4): `gci | ? { ... } | rm` is a -clean three-clause pipeline whose middle clause carries an opaque arg. +`OpaqueRegionScanner` and emitted as single tokens. The compatibility parser +retains each as one `Arg { Kind=DynamicSkip, IsPath=false, Resolved=null }`, +`Raw` being the verbatim region slice. Stable v0.3 additionally parses every +supported executable `$()` interior into `SimpleCommandSyntax.Substitutions`. +Execution-bearing `@()` / `@{}` forms that cannot be completely discovered +make the whole result unparseable. Splatting `@var` remains `DynamicSkip`. +The `--%` stop-parsing token makes the pipeline-element remainder one +`DynamicSkip` arg. An ordinary script-block argument remains non-executing +opaque data: `gci | ? { ... } | rm` is a clean three-clause pipeline whose +middle clause carries an opaque arg; consumers decide whether that command +interprets the block. + +PowerShell `$()` runs in the current runspace scope. A `Set-Location` inside a +subexpression affects later inner commands, the containing command after value +evaluation, and following outer commands. Unknown location mutations propagate +as unknown. This differs from Bash command substitution, whose state is +isolated from the containing shell. `OpaqueRegionScanner` is grammar-agnostic but escapes on backslash; for PowerShell it is given a backtick-escape mode so `` { `} } `` scans @@ -1126,18 +1187,19 @@ location attribution because it can resolve to current-scope code that calls ## 11. Parser Anomaly Behavior -The safe-fail contract — set `IsUnparseable=true`, set `UnparseableReason`, -return whatever clauses parsed, never throw on a well-formed string — is -defined in **`SPEC.md` §11** and is unchanged. +The v0.3 safe-fail contract — set `IsUnparseable=true`, set +`UnparseableReason`, return empty `Commands` and `Clauses`, retain at most a +diagnostic `Syntax` tree, and never throw on a well-formed string — is defined +in **`SPEC.md` §11**. `PwshCommandParser` sets `ParsedCommand.IsUnparseable = true` for: 1. **Lexer sentinels** — unbalanced single/double quote, unterminated here-string, unterminated `<# ... #>` block comment, unbalanced `{ }` / `$( )` / `@( )` / `@{ }`, unbalanced grouping `( )`. -2. **Control-flow keywords at statement/verb position** — `if`, `elseif`, - `else`, `switch`, `foreach` (when followed by `(`), `for`, `while`, `do`, - `until`. +2. **Unsupported control-flow at statement/verb position** — `switch`, `for`, + `do`, `until`, or an `if`, `elseif`, `else`, `foreach`, or `while` form + outside the bounded stable-v0.3 grammar in §4. 3. **Definition keywords** — `function`, `filter`, `workflow`, `configuration`, `class`, `enum`. 4. **Block / trap / data keywords** — `param`, `begin`, `process`, `end`, diff --git a/SPEC.md b/SPEC.md index 1d34921..a08b599 100644 --- a/SPEC.md +++ b/SPEC.md @@ -284,6 +284,7 @@ public sealed record SimpleCommandSyntax : ShellSyntaxNode { public override ShellSyntaxKind Kind => ShellSyntaxKind.SimpleCommand; public Clause Clause { get; init; } = new(); + public IReadOnlyList Substitutions { get; init; } = []; } public sealed record PipelineSyntax : ShellSyntaxNode @@ -471,6 +472,24 @@ not one per predicted runtime iteration. `Ancestry` is ordered outermost to innermost, excludes the simple-command leaf, and retains every enclosing execution relation. `ImmediateRole` describes the nearest relation. +`SimpleCommandSyntax.Substitutions` owns each completely delimited executable +command substitution evaluated for that command's authored words and redirects, +including an expanding heredoc body. The collection is in authored order and +preserves nesting: a substitution inside an inner simple command belongs to +that inner command, not to the outer command or a side table. `Clause` remains +the unchanged v0.2 compatibility leaf and retains the authored dynamic value. + +The canonical `Commands` and compatibility `Clauses` projections use these +deterministic ordering rules: disjoint executable regions follow authored +source order; an enclosed substitution precedes its containing command; +nested substitutions are emitted innermost first; and nodes without comparable +outer source spans use their containing structural collection order. Thus +`rm "$(find /tmp)"` projects `find`, then `rm`, exactly once each. +Each substitution ancestry frame uses `Region=Substitution` and its authored +zero-based `ChildIndex` in the structural collection that owns the +`CommandSubstitutionSyntax`. For a simple command this is its `Substitutions` +collection; for an iterator it is the containing iterator-command collection. + `EffectiveArgument.ClauseElementIndex` is a stable authored coordinate. The analysis never mutates a compatibility `Arg` to hold loop-specific values. An occurrence can be structurally complete while one effective value remains @@ -753,7 +772,8 @@ public sealed record VerbChain /// /// True when the clause's command name is a dynamic token the parser /// cannot statically identify — `& $exe`, `& "tool-$name"`, - /// `& { ... }` (added v0.2.0). + /// or supported `& $(Get-Thing)` (added v0.2.0). An unsupported + /// executable identity expression makes the whole result unparseable. /// Always false for bash clauses. See SPEC.POWERSHELL.md §3. /// public bool IsDynamic { get; init; } @@ -894,13 +914,19 @@ verb_chain := verb_like_word (FW_pair? verb_like_word)* // the first path-shaped or non-verb-like // token. For // word_0 ∈ FileVerbs, exactly 1 token. -arg := word | flag | quoted_string +verb_like_word := static word satisfying §6.1; the initial command-name + element contains no supported_substitution +arg := word | flag | quoted_string | supported_substitution flag := "-" letter+ | "--" word redirect := redirect_op target redirect_op := ">" | ">>" | "<" | "2>" | "2>>" -target := word | quoted_string -word := non-whitespace, non-operator characters +target := word | quoted_string | supported_substitution +supported_substitution := "$(" command ")" +word := non-whitespace, non-operator fragments; may contain + supported_substitution children in v0.3 quoted_string := single-quoted | double-quoted + // double-quoted values may contain supported_substitution; + // single-quoted and escaped spellings remain literal ``` **Notes:** @@ -975,11 +1001,22 @@ iterable_word := word | quoted_string | supported_substitution ``` The supported stable-v0.3 set is the existing simple-command grammar plus -`for name in words`, `while` / `until`, and `if` / `elif` / `else`. A fully -delimited command substitution in a supported iterable exposes its inner -commands but produces an `Unknown` value. A Bash path-shaped glob may produce -a `Pattern` only when its exact static covering directory is proved without -filesystem enumeration. Bash `<<<` is a non-path `HereString` redirect. +`for name in words`, `while` / `until`, and `if` / `elif` / `else`. Every fully +delimited `$()` command substitution in a supported simple-command argument, +redirect value, iterable, or expanding heredoc body is recursively parsed and +exposes its inner commands; its produced value remains `Unknown`. A nested +substitution is recursively attached to the nearest containing simple command. +Legacy backtick substitution becomes unparseable in v0.3 until its distinct +escape and nesting rules can be mapped without guessing. A Bash path-shaped +glob may produce a `Pattern` only when its exact static covering directory is +proved without filesystem enumeration. Bash `<<<` is a non-path `HereString` +redirect. + +A `$()` fragment in Bash command-name position leaves the outer command +identity runtime-dependent. Stable v0.3 makes the whole result unparseable +rather than changing the v0.2 `VerbChain.IsDynamic` contract, which remains +PowerShell-specific. Diagnostic `Syntax` may retain the discovered substitution, +but `Commands` and `Clauses` are empty. Missing `do`, `done`, `then`, or `fi`; an unsupported substitution whose commands cannot all be discovered; or any skipped executable region makes the @@ -1622,8 +1659,10 @@ When the parser cannot produce a clean AST: 1. **Set `ParsedCommand.IsUnparseable = true`.** 2. **Set `UnparseableReason`** to a human-readable diagnostic. -3. **Return whatever clauses were successfully parsed** in `Clauses`. May - be empty. +3. **Return empty `Commands` and `Clauses`.** `Syntax` may retain partial + diagnostic structure, but it is never authorization evidence. Historical + v0.1/v0.2 parsers could retain partial clauses; v0.3 deliberately closes + that subset-authorization hazard. 4. **Never throw** on well-formed input strings (only throw on null). Conditions that produce `IsUnparseable = true`: diff --git a/openspec/changes/v0-3-structured-shell-analysis/design.md b/openspec/changes/v0-3-structured-shell-analysis/design.md index 0321d28..c540085 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/design.md +++ b/openspec/changes/v0-3-structured-shell-analysis/design.md @@ -320,6 +320,14 @@ A simple-command syntax node wraps the same `Clause` value exposed through the occurrence and compatibility projections. Existing `VerbChain`, `Arg`, `Redirect`, and `ClauseElement` facts are not replaced. +Executable command substitutions evaluated for the simple command's authored +words and redirects, including an expanding heredoc body, are owned by that +`SimpleCommandSyntax` through an authored-order `Substitutions` collection. +This is necessary because the unchanged `Clause` leaf records the +opaque/dynamic value but cannot retain a nested executable tree. A substitution +nested inside an inner command belongs to that inner simple command; it is not +promoted to a sibling or stored in a side table. + New structural nodes preserve their complete source range when it can be mapped exactly. Expanded wrapper content retains the current nullable-span rule rather than inventing offsets into escaped or encoded outer text. @@ -358,6 +366,20 @@ condition, branch, body, and substitution commands. It does not invent compound operators across structural boundaries. Existing `Clause.Operator` values are retained only for actual authored relationships. +Projection order is deterministic. Disjoint executable regions follow authored +source order. An enclosed substitution precedes its containing simple command, +and nested substitutions are emitted innermost first. When wrapper decoding +makes outer source spans unavailable, containing structural collection order +is the tie-breaker. The occurrence and compatibility projections use the same +order. + +Sibling substitution ancestry frames use `Region=Substitution` and the +authored zero-based index from the structural collection that owns each +`CommandSubstitutionSyntax`. This is `SimpleCommandSyntax.Substitutions` for an +embedded command value and the iterator-command collection for a direct +iterator substitution. It remains deterministic when decoded wrapper content +has no outer source span. + The compatibility clauses preserve authored arguments. They do not substitute loop variables into `Arg` and therefore retain dynamic markers that make v0.2-style security consumers prompt rather than silently authorize a broader @@ -400,6 +422,14 @@ scope-isolated groups do not leak state. Branches join their possible exit states; loops include the zero-iteration path unless shell semantics prove at least one iteration. +Bash command substitution executes in an isolated subshell state. State changes +affect later commands inside that substitution but never the containing command +or following outer commands. PowerShell `$()` evaluates in the current runspace +scope. Its sequential location changes affect later commands inside the +subexpression, then the containing command and outer continuation. An unknown +PowerShell location mutation therefore makes those later working-directory +facts unknown; the analyzer never falls back to the pre-subexpression cwd. + The first implementation may collapse any differing cwd states to `Unknown` rather than publish a finite cwd set. Selecting one branch's directory is never allowed. A later additive version may expose bounded cwd alternatives if @@ -602,6 +632,7 @@ public sealed record SimpleCommandSyntax : ShellSyntaxNode { public override ShellSyntaxKind Kind => ShellSyntaxKind.SimpleCommand; public Clause Clause { get; init; } = new(); + public IReadOnlyList Substitutions { get; init; } = []; } public sealed record PipelineSyntax : ShellSyntaxNode @@ -799,6 +830,10 @@ public static class ShellAnalysisLimits not the `SimpleCommandSyntax` leaf itself. `ChildIndex` disambiguates repeated regions such as a pipeline stage or conditional branch. `ImmediateRole` describes the nearest execution relation; ancestry retains outer relations. +An embedded substitution adds a substitution ancestry frame while its +containing simple-command leaf remains excluded. Iterator or other outer +frames are therefore preserved outside that substitution frame. Its +`ChildIndex` is the authored index within the owning structural collection. The contract uses a source-authored element coordinate rather than attaching derived values directly to `Arg`. This prevents a loop iteration from mutating @@ -1044,7 +1079,8 @@ existing lexer values and spans. | `for name in words; do ...; done` | Supported | | `while` / `until` command lists | Supported | | `if` / `elif` / `else` command lists | Supported | -| Completely delimited command substitution in a supported iterable | Inner commands visible; produced value `Unknown` | +| Completely delimited `$()` substitution in a supported word, redirect value, iterable, or expanding heredoc body | Inner commands visible; produced value `Unknown` | +| Legacy backtick command substitution | Whole result unparseable until its distinct escape and nesting rules are modeled | | Static path-shaped glob in a supported iterable | `Pattern` only under the locked covering-directory rule | | Existing `<<` / `<<-` heredocs | Supported; preserve delimiter, body, expansion mode, and completeness without treating body data as commands | | Bash `<<<` here strings | Supported with explicit here-string redirect facts | @@ -1109,6 +1145,13 @@ whole-stream top-level split; it does not replace verb extraction, native argument classification, redirects, resolver behavior, provenance, or wrapper recursion inside a simple-command leaf. +After the existing leaf is built, the Bash front end recursively parses each +completely delimited `$()` retained in an executable value position and attaches +the resulting `CommandSubstitutionSyntax` to the containing simple command (or +the foreach iterator region). Single-quoted or escaped `$()` text remains +literal. An unsupported inner region, a backtick substitution, or a failed +boundary makes the whole result unparseable. + All `Expect*` failures return one outer unparseable result. They do not skip to `done` and return a partial tree that could be mistaken for authorization evidence. @@ -1141,6 +1184,8 @@ pwsh_else := "else" script_block_body foreach_expression := literal_value | literal_array | pipeline_expression + | supported_subexpression +supported_subexpression := "$(" pwsh_script(stop = ")") ")" literal_array := "@(" literal_value ("," literal_value)* ")" script_block_body := "{" pwsh_script(stop = "}") "}" ``` @@ -1210,6 +1255,12 @@ complete command discovery makes the whole result unparseable. | PowerShell construct | Stable v0.3 status | |---|---| | Existing simple commands, pipelines, statement separators, grouping, and static wrapper / `Invoke-Expression` recursion | Supported and structurally projected | +| Completely delimited `$()` in a supported word, call-operator dynamic identity, redirect, foreach expression, double-quoted string, or expandable here-string | Inner commands visible; produced value `Unknown`; current-scope state propagates | +| Standalone `$()` expression statement | Inner commands visible; no outer invocation is invented | +| `& $(...)` dynamic invocation | Inner commands visible, followed by one incomplete dynamic outer occurrence | +| `& { ... }` script-block invocation | Whole result unparseable until body, scope, and state propagation are modeled | +| Single-quoted, literal-here-string, or backtick-escaped `$()` text | Literal/opaque data; no invented substitution occurrence | +| Execution-bearing `@()` / `@{}` outside a completely modeled foreach literal expression | Whole result unparseable until complete command discovery is modeled | | `foreach ($name in expression) { ... }` for literal scalar, literal array, or fully delimited pipeline iterables | Supported | | `while (condition_pipeline) { ... }` | Supported | | `if` / `elseif` / `else` with fully delimited condition pipelines | Supported | @@ -1217,6 +1268,17 @@ complete command discovery makes the whole result unparseable. | `ForEach-Object` / `foreach` alias script blocks and ordinary script-block arguments | Existing opaque argument; no invented child execution | | `do`, `switch`, functions, definitions, class/type bodies, or execution-bearing expressions outside the locked subset | Deferred; whole result unparseable when execution may be hidden | +PowerShell `$()` is a value-producing subexpression, not an invocation. +Standalone `$()` lowers directly to `CommandSubstitutionSyntax` and exposes +only its inner commands. Only a call-operator form such as `& $(...)` also +lowers an outer dynamic simple command, after the substitution. A standalone +subexpression followed by command-style arguments is unparseable. Bash command +word formation deliberately differs; a substitution in Bash command-name +position makes the whole result unparseable after diagnostic discovery rather +than changing the PowerShell-specific `VerbChain.IsDynamic` contract. +Call-operator script-block invocation remains unparseable; an ordinary +script-block argument remains non-executing opaque data. + ### Candidate internal nodes and lowering pipeline The internal tree may retain shell-specific syntax even if the reviewed public @@ -1275,7 +1337,8 @@ lattice. Executable-aware interpretation still occurs only in the consumer. |---|---|---| | Missing Bash `do` or `done` | Parse failure with the offending range | `IsUnparseable=true`; `Commands` and `Clauses` empty | | Bash keyword used as an argument | Existing simple-command leaf | No false control-flow node | -| Unsupported Bash substitution in an iterable | Inner commands surfaced only if completely parsed | Otherwise the entire result is unparseable | +| Unsupported Bash substitution in any executable value position | Inner commands surfaced only if completely parsed | Otherwise the entire result is unparseable | +| Supported PowerShell `$()` value | Inner commands surfaced before its consumer; current-scope state applied | Produced value remains unknown | | PowerShell `foreach` at statement position followed by `(` | `PwshForEachNode` | Iterator and body occurrences exposed | | PowerShell `foreach` in a pipeline command slot | Existing pipeline/simple-command path | Alias or command semantics preserved | | PowerShell statement body `ScriptBlock` | Interior recursively parsed with adjusted spans | Every body command exposed | diff --git a/openspec/changes/v0-3-structured-shell-analysis/specs/bounded-shell-analysis/spec.md b/openspec/changes/v0-3-structured-shell-analysis/specs/bounded-shell-analysis/spec.md index 1c9e566..bfb437b 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/specs/bounded-shell-analysis/spec.md +++ b/openspec/changes/v0-3-structured-shell-analysis/specs/bounded-shell-analysis/spec.md @@ -255,6 +255,14 @@ Working-directory and supported variable state SHALL be propagated through sequential regions and joined across branches and loop exits. Disagreement SHALL never be resolved by arbitrarily choosing one path. +Bash command substitution SHALL isolate its working-directory and variable +state from the containing command while retaining sequential state inside the +substitution. PowerShell `$()` SHALL evaluate in the current runspace scope; +its sequential location changes SHALL affect later commands inside the +subexpression, the containing command, and the following outer continuation. +An unknown mutation SHALL propagate as unknown wherever that shell's scope +rules make it observable. + #### Scenario: Branch-dependent cwd - **WHEN** one branch changes cwd to `/a` and another changes cwd to `/b` - **THEN** a following relative path is not resolved solely under `/a` or solely under `/b` @@ -272,6 +280,21 @@ SHALL never be resolved by arbitrarily choosing one path. - **WHEN** a supported subshell or scope-isolated group changes cwd - **THEN** that cwd does not leak into the enclosing continuation +#### Scenario: Bash substitution cwd is isolated +- **WHEN** Bash parses `printf '%s' "$(cd /tmp; pwd)"; cat relative.txt` +- **THEN** `pwd` uses `/tmp` inside the substitution +- **THEN** `printf` and `cat` retain the exact outer cwd + +#### Scenario: PowerShell subexpression cwd propagates +- **WHEN** PowerShell parses `Write-Output $(Set-Location /tmp; Get-Location); Get-Item relative.txt` +- **THEN** `Get-Location`, `Write-Output`, and `Get-Item` use `/tmp` +- **THEN** the analyzer does not restore the pre-subexpression cwd + +#### Scenario: Unknown PowerShell subexpression mutation propagates +- **WHEN** a PowerShell subexpression changes location to an unknown value +- **THEN** later commands inside the subexpression and in the containing outer continuation have unknown working-directory facts +- **THEN** no prior exact cwd is selected as a fallback + ### Requirement: Unknown analysis remains policy-sensitive An unknown value SHALL identify the occurrence and position it affects so a consumer can determine whether command identity, option parsing, path scope, diff --git a/openspec/changes/v0-3-structured-shell-analysis/specs/executable-command-projection/spec.md b/openspec/changes/v0-3-structured-shell-analysis/specs/executable-command-projection/spec.md index 480e894..aee727e 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/specs/executable-command-projection/spec.md +++ b/openspec/changes/v0-3-structured-shell-analysis/specs/executable-command-projection/spec.md @@ -77,7 +77,16 @@ be sufficient authorization evidence. ### Requirement: Source order is deterministic The occurrence collection SHALL be ordered by authored command occurrence, including commands nested in headers and bodies, with a documented tie-breaker -for enclosing and enclosed regions. +for enclosing and enclosed regions. Disjoint executable regions SHALL follow +authored source order. An enclosed substitution SHALL precede its containing +simple command. Nested substitutions SHALL be emitted innermost first. When +decoded wrapper content has no comparable outer spans, the containing +structural collection order SHALL be used. `ParsedCommand.Clauses` SHALL use +the same ordering. A substitution ancestry frame SHALL use +`Region=Substitution` and the authored zero-based child index in its containing +structural collection. For an embedded simple-command value this is +`SimpleCommandSyntax.Substitutions`; for a direct iterator substitution this is +the iterator-command collection. #### Scenario: Iterator precedes body - **WHEN** Bash parses `for f in $(find .); do rm "$f"; done` @@ -88,6 +97,27 @@ for enclosing and enclosed regions. - **THEN** condition commands precede then-body commands - **THEN** then-body commands precede else-body commands in the projection +#### Scenario: Ordinary command substitution precedes its consumer +- **WHEN** Bash parses `rm "$(find /tmp)"` +- **THEN** `find` precedes `rm` in both command and compatibility projections +- **THEN** each command appears exactly once + +#### Scenario: Multiple substitutions preserve authored order +- **WHEN** one command contains two sibling substitutions +- **THEN** commands from the first substitution precede commands from the second +- **THEN** the containing command follows both substitutions +- **THEN** the sibling substitution frames use child indices zero and one + +#### Scenario: Nested substitutions are innermost first +- **WHEN** a substitution contains a command with another substitution +- **THEN** the innermost command precedes its containing substitution command +- **THEN** both precede the outermost containing command + +#### Scenario: PowerShell expression output is not a command +- **WHEN** standalone PowerShell `$()` produces text shaped like a command name +- **THEN** the occurrence collection contains its inner commands only +- **THEN** an outer command occurrence exists only when the call operator invokes that value + ### Requirement: Command discovery is library-owned Security consumers SHALL be able to enumerate every potentially executable command without recursively matching syntax-node types. diff --git a/openspec/changes/v0-3-structured-shell-analysis/specs/structured-shell-syntax/spec.md b/openspec/changes/v0-3-structured-shell-analysis/specs/structured-shell-syntax/spec.md index 4b76be9..f29aeea 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/specs/structured-shell-syntax/spec.md +++ b/openspec/changes/v0-3-structured-shell-analysis/specs/structured-shell-syntax/spec.md @@ -28,13 +28,78 @@ conditional branch, and command substitution nodes. ### Requirement: Simple-command nodes preserve existing leaves A simple-command syntax node SHALL expose the existing `Clause` facts rather than replacing `VerbChain`, `Arg`, `Redirect`, or `ClauseElement` with a second -incompatible leaf model. +incompatible leaf model. It SHALL also own an authored-order collection of +completely delimited command substitutions evaluated for its words and +redirects, including expanding heredoc bodies. Nested substitutions SHALL +remain attached to the nearest containing simple command; they SHALL NOT be +promoted to unrelated siblings or stored only in a side table. #### Scenario: Clause provenance survives structural wrapping - **WHEN** Bash parses `git -C /repo status > status.txt` - **THEN** the simple-command node exposes the existing verb, arguments, redirect, and ordered elements - **THEN** their raw values and source spans retain the v0.2 meanings +#### Scenario: Substitution remains attached to its containing command +- **WHEN** Bash parses `rm "$(find /tmp)"` +- **THEN** the `rm` simple-command node retains its unchanged dynamic compatibility argument +- **THEN** its substitutions contain a command-substitution node whose body contains `find` + +#### Scenario: Nested substitutions preserve parentage +- **WHEN** a supported shell parses a command substitution inside another substitution +- **THEN** the inner substitution belongs to the simple command inside the outer substitution +- **THEN** the inner substitution is not flattened into the outer command's substitution collection + +### Requirement: Executable substitution boundaries are accounted for +Stable v0.3 SHALL recursively parse every completely delimited Bash `$()` or +PowerShell `$()` that can execute while forming a supported simple-command +argument word, redirect value, iterator, expanding heredoc or here-string, or +PowerShell call-operator dynamic identity. The produced value SHALL remain +unknown unless separately proved. If an executable substitution interior +cannot be completely parsed, the whole result SHALL be unparseable. + +Bash legacy backtick substitution and execution-bearing PowerShell `@()` or +`@{}` forms outside the locked literal-foreach subset SHALL remain unparseable +until their distinct semantics have complete command discovery. Literal or +escaped substitution-looking text SHALL NOT create syntax or occurrences. + +#### Scenario: Bash literal substitution spelling +- **WHEN** Bash parses single-quoted or backslash-escaped `$()` text +- **THEN** no substitution node or command occurrence is created for that text + +#### Scenario: PowerShell literal substitution spelling +- **WHEN** PowerShell parses single-quoted, literal-here-string, or backtick-escaped `$()` text +- **THEN** no substitution node or command occurrence is created for that text + +#### Scenario: Expandable value executes a substitution +- **WHEN** either shell parses a supported `$()` inside an expandable quoted value or redirect target +- **THEN** every inner command is exposed before the containing command +- **THEN** an unsupported inner executable region makes the whole result unparseable + +#### Scenario: Dynamic identity contains a substitution +- **WHEN** PowerShell `&` dynamic invocation contains supported executable `$()` syntax +- **THEN** every inner command remains visible +- **THEN** the containing dynamic command occurrence remains incomplete or the whole result is unparseable + +#### Scenario: Bash command-name substitution remains gated +- **WHEN** Bash command-name formation contains executable `$()` syntax +- **THEN** the whole result is unparseable because the outer command identity is runtime-dependent +- **THEN** diagnostic syntax may retain the substitution but command and compatibility projections are empty + +#### Scenario: Standalone PowerShell subexpression is not invocation +- **WHEN** PowerShell parses `$(Write-Output Get-Date)` as an expression statement +- **THEN** `Write-Output` is exposed as a substitution command +- **THEN** no outer `Get-Date` command is invented from the produced string + +#### Scenario: PowerShell call operator invokes subexpression output +- **WHEN** PowerShell parses `& $(Write-Output Get-Date)` +- **THEN** `Write-Output` is exposed before one incomplete dynamic outer invocation +- **THEN** the produced string is not assumed to equal a static command identity + +#### Scenario: PowerShell call operator script block remains gated +- **WHEN** PowerShell encounters `& { Remove-Item target.txt }` +- **THEN** the whole result is unparseable until script-block execution semantics are modeled +- **THEN** the body is not treated as an ordinary opaque argument + ### Requirement: Bash for-in loops preserve header and body structure The Bash parser SHALL represent a supported `for name in words; do body; done` loop as a typed loop node with its binding name, authored iterable expression, diff --git a/openspec/changes/v0-3-structured-shell-analysis/tasks.md b/openspec/changes/v0-3-structured-shell-analysis/tasks.md index 639b023..4cc8c91 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/tasks.md +++ b/openspec/changes/v0-3-structured-shell-analysis/tasks.md @@ -31,6 +31,10 @@ - [ ] 3.7 Adapt the existing PowerShell grammar to emit the structural model with no newly supported syntax. - [ ] 3.8 Add tests proving existing parser inputs retain their v0.2 leaf and compatibility results. - [ ] 3.9 Add corpus expectations for syntax shape, occurrences, roles, and completeness for existing constructs. +- [ ] 3.10 Implement Bash `$()` discovery in supported argument words, redirect values, iterables, and expanding heredoc bodies; retain literal/escaped spellings and fail closed on command-name substitutions, legacy backticks, or incomplete interiors. +- [ ] 3.11 Implement PowerShell `$()` discovery in supported words, redirect values, foreach expressions, call-operator dynamic identities, standalone expression statements, double-quoted strings, and expandable here-strings; never invent invocation from standalone output, retain literal/escaped spellings, and fail closed on trailing command-style arguments, call-operator script blocks, or unsupported execution-bearing `@()` / `@{}` forms. +- [ ] 3.12 Pin substitution parentage, authored sibling indices, innermost-first ordering, Bash-isolated versus PowerShell-current-scope state, unknown-state propagation, nesting/depth limits, and incomplete dynamic identities in direct tests. +- [ ] 3.13 Promote ordinary, multiple, nested, iterator, redirect, quoted, escaped, stateful, malformed, and hidden-execution substitution cases into both executable corpora and the Netclaw approval matrix. ## 4. Explicit Redirect Semantics @@ -114,4 +118,5 @@ - [ ] 11.5 Validate the public API field-for-field against the synchronized shared and PowerShell specifications. - [ ] 11.6 Validate Netclaw's ordinary-command, redirect, bounded-loop, and unknown-value approval matrices against the prerelease package. - [ ] 11.7 Update release notes and remove Netclaw's temporary descriptor workaround only after explicit redirect integration is live. -- [ ] 11.8 Promote stable 0.3.0 only after Linux and Windows CI, package publication, and downstream acceptance succeed. +- [ ] 11.8 Expand the Web sample with curated complex Bash and PowerShell inputs and snapshot-tested deterministic Mermaid diagrams produced only from canonical syntax, occurrence, and compatibility projections; cover ancestry, redirects, and fail-closed results, escape arbitrary shell labels safely, and emit no raw HTML. +- [ ] 11.9 Promote stable 0.3.0 only after Linux and Windows CI, package publication, and downstream acceptance succeed. diff --git a/src/ShellSyntaxTree/ShellSyntaxNode.cs b/src/ShellSyntaxTree/ShellSyntaxNode.cs index dfbfb73..947877d 100644 --- a/src/ShellSyntaxTree/ShellSyntaxNode.cs +++ b/src/ShellSyntaxTree/ShellSyntaxNode.cs @@ -79,6 +79,13 @@ public sealed record SimpleCommandSyntax : ShellSyntaxNode /// Gets the shared simple-command leaf. public Clause Clause { get; init; } = new(); + + /// + /// Gets executable command substitutions evaluated for this command's + /// authored words and redirects, in source order. + /// + public IReadOnlyList Substitutions { get; init; } = + Array.Empty(); } /// An ordered pipeline. diff --git a/src/ShellSyntaxTree/VerbChain.cs b/src/ShellSyntaxTree/VerbChain.cs index 6333e36..b9a2db6 100644 --- a/src/ShellSyntaxTree/VerbChain.cs +++ b/src/ShellSyntaxTree/VerbChain.cs @@ -43,9 +43,10 @@ public sealed record VerbChain /// True when the clause's command name is a dynamic token the parser /// cannot statically identify — a variable (& $exe), an /// interpolated name (& "tool-$name"), a subexpression - /// (& (Get-Thing)), or a script block - /// (& { ... }) at verb position. still - /// carries the verbatim token; is null. + /// (& $(Get-Thing)), or another supported value expression at + /// verb position. still carries the verbatim token; + /// is null. An unsupported executable + /// identity expression makes the whole result unparseable instead. /// /// A consumer MUST treat a clause with IsDynamic=true as "the /// command being run is unknown" and route to safe-fail. Always false diff --git a/tests/ShellSyntaxTree.Tests/DesignCorpus/README.md b/tests/ShellSyntaxTree.Tests/DesignCorpus/README.md index 6cfa03b..dc424a1 100644 --- a/tests/ShellSyntaxTree.Tests/DesignCorpus/README.md +++ b/tests/ShellSyntaxTree.Tests/DesignCorpus/README.md @@ -30,6 +30,24 @@ a pipeline stage and still carries the enclosing loop-body context. Likewise, `isComplete` describes command discovery and structure, not value precision; a complete occurrence may contain an `Unknown` effective value. +Executable substitutions embedded in a simple command are structural children +of that command. Projection order is security-significant: substitutions +precede their containing command, sibling substitutions retain authored order, +and nested substitutions are innermost first. The unchanged compatibility leaf +still records the authored opaque or dynamic operand. + +Substitution cases also record shell-specific working-directory facts. Bash +state changes remain isolated to the substitution, while PowerShell `$()` +changes runspace location before the containing command executes. Literal and +escaped substitution-looking text has no child commands. The executable corpus +will add the full quoted, redirect, dynamic-identity, malformed, and depth-limit +matrix as each shell's substitution slice lands. + +PowerShell cases distinguish a standalone `$()` expression from `& $(...)`. +The former exposes only commands inside the subexpression; the latter also has +one incomplete dynamic invocation. Rendering or authorization must never turn +subexpression output text into a synthetic command. + The v0.3 contract fixes the candidate cap at 32. Each shell has a case at the cap and a 33-value overflow case that collapses to `Unknown` rather than publishing a truncated finite set. Supported heredocs and Bash here strings diff --git a/tests/ShellSyntaxTree.Tests/DesignCorpus/V03DesignCorpusTests.cs b/tests/ShellSyntaxTree.Tests/DesignCorpus/V03DesignCorpusTests.cs index 231cfa2..7f49f90 100644 --- a/tests/ShellSyntaxTree.Tests/DesignCorpus/V03DesignCorpusTests.cs +++ b/tests/ShellSyntaxTree.Tests/DesignCorpus/V03DesignCorpusTests.cs @@ -172,6 +172,11 @@ private static void ValidateDesiredShape(DesignShell shell, V03DesignCase design ValidateValue(shell, designCase.Id, value); } + if (command.WorkingDirectory is not null) + { + ValidateWorkingDirectory(designCase.Id, command.WorkingDirectory); + } + foreach (var redirect in command.Redirects) { Assert.True(redirect.IsPathRelevant == (redirect.Operation is @@ -357,6 +362,20 @@ private static void ValidateValue( } } + private static void ValidateWorkingDirectory( + string caseId, + DesignWorkingDirectoryExpectation workingDirectory) + { + if (workingDirectory.Kind == DesignValueKind.Exact) + { + Assert.False(string.IsNullOrWhiteSpace(workingDirectory.Value)); + return; + } + + Assert.Equal(DesignValueKind.Unknown, workingDirectory.Kind); + Assert.Null(workingDirectory.Value); + } + private static IShellParser CreateParser(DesignShell shell) => shell switch { DesignShell.Bash => new BashParser(new BashParserOptions @@ -482,9 +501,18 @@ public sealed record DesignCommandExpectation public IReadOnlyList EffectiveValues { get; init; } = []; + public DesignWorkingDirectoryExpectation? WorkingDirectory { get; init; } + public IReadOnlyList Redirects { get; init; } = []; } +public sealed record DesignWorkingDirectoryExpectation +{ + public DesignValueKind Kind { get; init; } + + public string? Value { get; init; } +} + public sealed record DesignValueExpectation { public string SourceElement { get; init; } = ""; @@ -609,6 +637,7 @@ public enum DesignSyntaxKind ForEach, ConditionLoop, Conditional, + CommandSubstitution, SimpleCommand, OpaqueArgument, Unsupported, @@ -624,6 +653,7 @@ public enum DesignSyntaxSlot Then, Else, Stage, + Substitution, Argument, } diff --git a/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json b/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json index 79b68de..0acd950 100644 --- a/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json +++ b/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json @@ -1,6 +1,171 @@ { "shell": "Bash", "cases": [ + { + "id": "bash-simple-command-substitution", + "concern": "Substitution is attached to its containing simple command", + "input": "rm \"$(find /tmp)\"", + "current": { + "isUnparseable": false, + "argument": { "clauseIndex": 0, "argumentIndex": 0, "raw": "\"$(find /tmp)\"", "kind": "DynamicSkip", "isPath": false } + }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "rm", "kind": "SimpleCommand", "parent": "root", "slot": "Statement", "commandIndex": 1 }, + { "id": "sub", "kind": "CommandSubstitution", "parent": "rm", "slot": "Substitution" }, + { "id": "subBody", "kind": "Block", "parent": "sub", "slot": "Body" }, + { "id": "find", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 0 } + ], + "commands": [ + { "authoredVerb": "find", "immediateRole": "Substitution", "ancestry": ["root", "sub", "subBody"], "isComplete": true }, + { + "authoredVerb": "rm", + "immediateRole": "Ordinary", + "ancestry": ["root"], + "isComplete": true, + "effectiveValues": [ + { "sourceElement": "\"$(find /tmp)\"", "kind": "Unknown", "isPolicySensitive": true } + ] + } + ], + "argument": { "clauseIndex": 1, "argumentIndex": 0, "raw": "\"$(find /tmp)\"", "kind": "DynamicSkip", "isPath": false }, + "compatibility": { "verbs": ["find", "rm"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "UnknownPolicyValueFailsClosed", "NoSyntheticOperator"] + } + }, + { + "id": "bash-multiple-command-substitutions", + "concern": "Sibling substitutions preserve authored order before their consumer", + "input": "printf '%s %s' \"$(first)\" \"$(second)\"", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "printf", "kind": "SimpleCommand", "parent": "root", "slot": "Statement", "commandIndex": 2 }, + { "id": "firstSub", "kind": "CommandSubstitution", "parent": "printf", "slot": "Substitution" }, + { "id": "firstBody", "kind": "Block", "parent": "firstSub", "slot": "Body" }, + { "id": "first", "kind": "SimpleCommand", "parent": "firstBody", "slot": "Statement", "commandIndex": 0 }, + { "id": "secondSub", "kind": "CommandSubstitution", "parent": "printf", "slot": "Substitution" }, + { "id": "secondBody", "kind": "Block", "parent": "secondSub", "slot": "Body" }, + { "id": "second", "kind": "SimpleCommand", "parent": "secondBody", "slot": "Statement", "commandIndex": 1 } + ], + "commands": [ + { "authoredVerb": "first", "immediateRole": "Substitution", "ancestry": ["root", "firstSub", "firstBody"], "isComplete": true }, + { "authoredVerb": "second", "immediateRole": "Substitution", "ancestry": ["root", "secondSub", "secondBody"], "isComplete": true }, + { "authoredVerb": "printf", "immediateRole": "Ordinary", "ancestry": ["root"], "isComplete": true } + ], + "compatibility": { "verbs": ["first", "second", "printf"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "NoSyntheticOperator"] + } + }, + { + "id": "bash-nested-command-substitutions", + "concern": "Nested substitutions retain parentage and project innermost first", + "input": "printf '%s' \"$(echo \"$(whoami)\")\"", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "printf", "kind": "SimpleCommand", "parent": "root", "slot": "Statement", "commandIndex": 2 }, + { "id": "outerSub", "kind": "CommandSubstitution", "parent": "printf", "slot": "Substitution" }, + { "id": "outerBody", "kind": "Block", "parent": "outerSub", "slot": "Body" }, + { "id": "echo", "kind": "SimpleCommand", "parent": "outerBody", "slot": "Statement", "commandIndex": 1 }, + { "id": "innerSub", "kind": "CommandSubstitution", "parent": "echo", "slot": "Substitution" }, + { "id": "innerBody", "kind": "Block", "parent": "innerSub", "slot": "Body" }, + { "id": "whoami", "kind": "SimpleCommand", "parent": "innerBody", "slot": "Statement", "commandIndex": 0 } + ], + "commands": [ + { "authoredVerb": "whoami", "immediateRole": "Substitution", "ancestry": ["root", "outerSub", "outerBody", "innerSub", "innerBody"], "isComplete": true }, + { "authoredVerb": "echo", "immediateRole": "Substitution", "ancestry": ["root", "outerSub", "outerBody"], "isComplete": true }, + { "authoredVerb": "printf", "immediateRole": "Ordinary", "ancestry": ["root"], "isComplete": true } + ], + "compatibility": { "verbs": ["whoami", "echo", "printf"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "NoSyntheticOperator"] + } + }, + { + "id": "bash-literal-substitution-spellings", + "concern": "Quoted and escaped substitution-looking text does not execute", + "input": "printf '%s %s' '$(whoami)' \"\\$(id)\"", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "printf", "kind": "SimpleCommand", "parent": "root", "slot": "Statement", "commandIndex": 0 } + ], + "commands": [ + { "authoredVerb": "printf", "immediateRole": "Ordinary", "ancestry": ["root"], "isComplete": true } + ], + "compatibility": { "verbs": ["printf"], "preservesAuthoredDynamicValues": false }, + "securityInvariants": ["AllCommandsVisible", "LiteralExpansionProvenancePreserved"] + } + }, + { + "id": "bash-backtick-substitution-gated", + "concern": "Legacy backtick execution remains fail closed until escape semantics are modeled", + "input": "echo `whoami`", + "current": { + "isUnparseable": false, + "argument": { "clauseIndex": 0, "argumentIndex": 0, "raw": "`whoami`", "kind": "DynamicSkip", "isPath": false } + }, + "desired": { + "isUnparseable": true, + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "unsupported", "kind": "Unsupported", "parent": "root", "slot": "Argument" } + ], + "commands": [], + "compatibility": { "verbs": [], "preservesAuthoredDynamicValues": false }, + "securityInvariants": ["PartialTreeDiagnosticOnly", "UnknownPolicyValueFailsClosed"] + } + }, + { + "id": "bash-command-name-substitution-gated", + "concern": "Runtime-produced Bash command identity stays unparseable", + "input": "r$(printf m) file", + "current": { "isUnparseable": true, "reasonContains": "dynamic Bash command identity" }, + "desired": { + "isUnparseable": true, + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "outer", "kind": "Unsupported", "parent": "root", "slot": "Statement" }, + { "id": "sub", "kind": "CommandSubstitution", "parent": "outer", "slot": "Substitution" }, + { "id": "subBody", "kind": "Block", "parent": "sub", "slot": "Body" }, + { "id": "printf", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement" } + ], + "commands": [], + "compatibility": { "verbs": [], "preservesAuthoredDynamicValues": false }, + "securityInvariants": ["PartialTreeDiagnosticOnly", "UnknownPolicyValueFailsClosed"] + }, + "notes": "Diagnostic syntax may retain printf, but unparseable results expose no authorization projections." + }, + { + "id": "bash-command-substitution-isolated-cwd", + "concern": "Bash substitution state is sequential inside and isolated outside", + "input": "printf '%s' \"$(cd /tmp; pwd)\"; cat relative.txt", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "list", "kind": "CommandList", "parent": "root", "slot": "Statement" }, + { "id": "printf", "kind": "SimpleCommand", "parent": "list", "slot": "Statement", "commandIndex": 2 }, + { "id": "sub", "kind": "CommandSubstitution", "parent": "printf", "slot": "Substitution" }, + { "id": "subBody", "kind": "Block", "parent": "sub", "slot": "Body" }, + { "id": "cd", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 0 }, + { "id": "pwd", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 1 }, + { "id": "cat", "kind": "SimpleCommand", "parent": "list", "slot": "Statement", "commandIndex": 3 } + ], + "commands": [ + { "authoredVerb": "cd", "immediateRole": "Substitution", "ancestry": ["root", "list", "sub", "subBody"], "isComplete": true, "workingDirectory": { "kind": "Exact", "value": "/work" } }, + { "authoredVerb": "pwd", "immediateRole": "Substitution", "ancestry": ["root", "list", "sub", "subBody"], "isComplete": true, "workingDirectory": { "kind": "Exact", "value": "/tmp" } }, + { "authoredVerb": "printf", "immediateRole": "Ordinary", "ancestry": ["root", "list"], "isComplete": true, "workingDirectory": { "kind": "Exact", "value": "/work" } }, + { "authoredVerb": "cat", "immediateRole": "Ordinary", "ancestry": ["root", "list"], "isComplete": true, "workingDirectory": { "kind": "Exact", "value": "/work" } } + ], + "compatibility": { "verbs": ["cd", "pwd", "printf", "cat"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "StateJoinConservative", "NoSyntheticOperator"] + } + }, { "id": "bash-for-literal-finite", "concern": "Finite literal loop binding", @@ -92,12 +257,14 @@ { "id": "root", "kind": "Block", "slot": "Root" }, { "id": "loop", "kind": "ForEach", "parent": "root", "slot": "Statement", "binding": "f" }, { "id": "iterator", "kind": "Block", "parent": "loop", "slot": "Iterator" }, - { "id": "find", "kind": "SimpleCommand", "parent": "iterator", "slot": "Statement", "commandIndex": 0 }, + { "id": "sub", "kind": "CommandSubstitution", "parent": "iterator", "slot": "Substitution" }, + { "id": "subBody", "kind": "Block", "parent": "sub", "slot": "Body" }, + { "id": "find", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 0 }, { "id": "body", "kind": "Block", "parent": "loop", "slot": "Body" }, { "id": "rm", "kind": "SimpleCommand", "parent": "body", "slot": "Statement", "commandIndex": 1 } ], "commands": [ - { "authoredVerb": "find", "immediateRole": "Iterator", "ancestry": ["root", "loop", "iterator"], "isComplete": true }, + { "authoredVerb": "find", "immediateRole": "Substitution", "ancestry": ["root", "loop", "iterator", "sub", "subBody"], "isComplete": true }, { "authoredVerb": "rm", "immediateRole": "LoopBody", diff --git a/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/powershell.json b/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/powershell.json index 59e9527..fc731ba 100644 --- a/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/powershell.json +++ b/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/powershell.json @@ -1,6 +1,209 @@ { "shell": "PowerShell", "cases": [ + { + "id": "pwsh-simple-command-substitution", + "concern": "Subexpression command is attached to its containing simple command", + "input": "Remove-Item $(Get-Item target.txt)", + "current": { + "isUnparseable": false, + "argument": { "clauseIndex": 0, "argumentIndex": 0, "raw": "$(Get-Item target.txt)", "kind": "DynamicSkip", "isPath": false } + }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "remove", "kind": "SimpleCommand", "parent": "root", "slot": "Statement", "commandIndex": 1 }, + { "id": "sub", "kind": "CommandSubstitution", "parent": "remove", "slot": "Substitution" }, + { "id": "subBody", "kind": "Block", "parent": "sub", "slot": "Body" }, + { "id": "get", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 0 } + ], + "commands": [ + { "authoredVerb": "Get-Item", "immediateRole": "Substitution", "ancestry": ["root", "sub", "subBody"], "isComplete": true }, + { + "authoredVerb": "Remove-Item", + "immediateRole": "Ordinary", + "ancestry": ["root"], + "isComplete": true, + "effectiveValues": [ + { "sourceElement": "$(Get-Item target.txt)", "kind": "Unknown", "isPolicySensitive": true } + ] + } + ], + "argument": { "clauseIndex": 1, "argumentIndex": 0, "raw": "$(Get-Item target.txt)", "kind": "DynamicSkip", "isPath": false }, + "compatibility": { "verbs": ["Get-Item", "Remove-Item"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "UnknownPolicyValueFailsClosed", "NoSyntheticOperator"] + } + }, + { + "id": "pwsh-multiple-command-substitutions", + "concern": "Sibling subexpressions preserve authored order before their consumer", + "input": "Write-Output $(Get-Date) $(Get-Location)", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "write", "kind": "SimpleCommand", "parent": "root", "slot": "Statement", "commandIndex": 2 }, + { "id": "dateSub", "kind": "CommandSubstitution", "parent": "write", "slot": "Substitution" }, + { "id": "dateBody", "kind": "Block", "parent": "dateSub", "slot": "Body" }, + { "id": "date", "kind": "SimpleCommand", "parent": "dateBody", "slot": "Statement", "commandIndex": 0 }, + { "id": "locationSub", "kind": "CommandSubstitution", "parent": "write", "slot": "Substitution" }, + { "id": "locationBody", "kind": "Block", "parent": "locationSub", "slot": "Body" }, + { "id": "location", "kind": "SimpleCommand", "parent": "locationBody", "slot": "Statement", "commandIndex": 1 } + ], + "commands": [ + { "authoredVerb": "Get-Date", "immediateRole": "Substitution", "ancestry": ["root", "dateSub", "dateBody"], "isComplete": true }, + { "authoredVerb": "Get-Location", "immediateRole": "Substitution", "ancestry": ["root", "locationSub", "locationBody"], "isComplete": true }, + { "authoredVerb": "Write-Output", "immediateRole": "Ordinary", "ancestry": ["root"], "isComplete": true } + ], + "compatibility": { "verbs": ["Get-Date", "Get-Location", "Write-Output"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "NoSyntheticOperator"] + } + }, + { + "id": "pwsh-nested-command-substitutions", + "concern": "Nested subexpressions retain parentage and project innermost first", + "input": "Write-Output $(Get-Item $(Get-Location))", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "write", "kind": "SimpleCommand", "parent": "root", "slot": "Statement", "commandIndex": 2 }, + { "id": "outerSub", "kind": "CommandSubstitution", "parent": "write", "slot": "Substitution" }, + { "id": "outerBody", "kind": "Block", "parent": "outerSub", "slot": "Body" }, + { "id": "item", "kind": "SimpleCommand", "parent": "outerBody", "slot": "Statement", "commandIndex": 1 }, + { "id": "innerSub", "kind": "CommandSubstitution", "parent": "item", "slot": "Substitution" }, + { "id": "innerBody", "kind": "Block", "parent": "innerSub", "slot": "Body" }, + { "id": "location", "kind": "SimpleCommand", "parent": "innerBody", "slot": "Statement", "commandIndex": 0 } + ], + "commands": [ + { "authoredVerb": "Get-Location", "immediateRole": "Substitution", "ancestry": ["root", "outerSub", "outerBody", "innerSub", "innerBody"], "isComplete": true }, + { "authoredVerb": "Get-Item", "immediateRole": "Substitution", "ancestry": ["root", "outerSub", "outerBody"], "isComplete": true }, + { "authoredVerb": "Write-Output", "immediateRole": "Ordinary", "ancestry": ["root"], "isComplete": true } + ], + "compatibility": { "verbs": ["Get-Location", "Get-Item", "Write-Output"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "NoSyntheticOperator"] + } + }, + { + "id": "pwsh-literal-substitution-spellings", + "concern": "Single-quoted and escaped subexpression-looking text does not execute", + "input": "Write-Output '$(Get-Date)' \"literal `$(Get-Location)\"", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "write", "kind": "SimpleCommand", "parent": "root", "slot": "Statement", "commandIndex": 0 } + ], + "commands": [ + { "authoredVerb": "Write-Output", "immediateRole": "Ordinary", "ancestry": ["root"], "isComplete": true } + ], + "compatibility": { "verbs": ["Write-Output"], "preservesAuthoredDynamicValues": false }, + "securityInvariants": ["AllCommandsVisible", "LiteralExpansionProvenancePreserved"] + } + }, + { + "id": "pwsh-array-subexpression-execution-gated", + "concern": "Execution-bearing array subexpression fails closed until discovery is modeled", + "input": "Write-Output @(Get-Date)", + "current": { + "isUnparseable": false, + "argument": { "clauseIndex": 0, "argumentIndex": 0, "raw": "@(Get-Date)", "kind": "DynamicSkip", "isPath": false } + }, + "desired": { + "isUnparseable": true, + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "unsupported", "kind": "Unsupported", "parent": "root", "slot": "Argument" } + ], + "commands": [], + "compatibility": { "verbs": [], "preservesAuthoredDynamicValues": false }, + "securityInvariants": ["PartialTreeDiagnosticOnly", "UnknownPolicyValueFailsClosed"] + } + }, + { + "id": "pwsh-call-operator-script-block-gated", + "concern": "Invoked script block is not treated as an ordinary opaque argument", + "input": "& { Remove-Item target.txt }", + "current": { "isUnparseable": false }, + "desired": { + "isUnparseable": true, + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "unsupported", "kind": "Unsupported", "parent": "root", "slot": "Statement" } + ], + "commands": [], + "compatibility": { "verbs": [], "preservesAuthoredDynamicValues": false }, + "securityInvariants": ["PartialTreeDiagnosticOnly", "UnknownPolicyValueFailsClosed"] + } + }, + { + "id": "pwsh-standalone-subexpression-is-not-invocation", + "concern": "Standalone subexpression output does not invent an outer command", + "input": "$(Write-Output Get-Date)", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "sub", "kind": "CommandSubstitution", "parent": "root", "slot": "Statement" }, + { "id": "subBody", "kind": "Block", "parent": "sub", "slot": "Body" }, + { "id": "write", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 0 } + ], + "commands": [ + { "authoredVerb": "Write-Output", "immediateRole": "Substitution", "ancestry": ["root", "sub", "subBody"], "isComplete": true } + ], + "compatibility": { "verbs": ["Write-Output"], "preservesAuthoredDynamicValues": false }, + "securityInvariants": ["AllCommandsVisible", "NoSyntheticOperator"] + }, + "notes": "The produced string Get-Date is output data, not an invocation." + }, + { + "id": "pwsh-call-operator-subexpression-invocation", + "concern": "Call operator invokes unknown subexpression output after exposing its commands", + "input": "& $(Write-Output Get-Date)", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "invoke", "kind": "SimpleCommand", "parent": "root", "slot": "Statement", "commandIndex": 1 }, + { "id": "sub", "kind": "CommandSubstitution", "parent": "invoke", "slot": "Substitution" }, + { "id": "subBody", "kind": "Block", "parent": "sub", "slot": "Body" }, + { "id": "write", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 0 } + ], + "commands": [ + { "authoredVerb": "Write-Output", "immediateRole": "Substitution", "ancestry": ["root", "sub", "subBody"], "isComplete": true }, + { "authoredVerb": "$(Write-Output Get-Date)", "immediateRole": "Ordinary", "ancestry": ["root"], "isComplete": false } + ], + "compatibility": { "verbs": ["Write-Output", "$(Write-Output Get-Date)"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "UnknownPolicyValueFailsClosed", "NoSyntheticOperator"] + }, + "notes": "The parser does not evaluate the produced command name; the outer occurrence remains dynamic and incomplete." + }, + { + "id": "pwsh-command-substitution-current-scope-cwd", + "concern": "PowerShell subexpression location changes propagate to the consumer and continuation", + "input": "Write-Output $(Set-Location /tmp; Get-Location); Get-Item relative.txt", + "current": { "isUnparseable": false }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "list", "kind": "CommandList", "parent": "root", "slot": "Statement" }, + { "id": "write", "kind": "SimpleCommand", "parent": "list", "slot": "Statement", "commandIndex": 2 }, + { "id": "sub", "kind": "CommandSubstitution", "parent": "write", "slot": "Substitution" }, + { "id": "subBody", "kind": "Block", "parent": "sub", "slot": "Body" }, + { "id": "set", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 0 }, + { "id": "location", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 1 }, + { "id": "get", "kind": "SimpleCommand", "parent": "list", "slot": "Statement", "commandIndex": 3 } + ], + "commands": [ + { "authoredVerb": "Set-Location", "immediateRole": "Substitution", "ancestry": ["root", "list", "sub", "subBody"], "isComplete": true, "workingDirectory": { "kind": "Exact", "value": "C:/work" } }, + { "authoredVerb": "Get-Location", "immediateRole": "Substitution", "ancestry": ["root", "list", "sub", "subBody"], "isComplete": true, "workingDirectory": { "kind": "Exact", "value": "/tmp" } }, + { "authoredVerb": "Write-Output", "immediateRole": "Ordinary", "ancestry": ["root", "list"], "isComplete": true, "workingDirectory": { "kind": "Exact", "value": "/tmp" } }, + { "authoredVerb": "Get-Item", "immediateRole": "Ordinary", "ancestry": ["root", "list"], "isComplete": true, "workingDirectory": { "kind": "Exact", "value": "/tmp" } } + ], + "compatibility": { "verbs": ["Set-Location", "Get-Location", "Write-Output", "Get-Item"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "StateJoinConservative", "NoSyntheticOperator"] + } + }, { "id": "pwsh-foreach-literal-array", "concern": "Finite literal foreach binding", @@ -59,6 +262,39 @@ }, "notes": "The iterator emits filesystem objects, not proved literal strings." }, + { + "id": "pwsh-foreach-subexpression-iterator", + "concern": "Subexpression iterator preserves nearest substitution role and outer iterator ancestry", + "input": "foreach ($f in $(Get-ChildItem C:\\input)) { Remove-Item -LiteralPath $f }", + "current": { "isUnparseable": true, "reasonContains": "'foreach'" }, + "desired": { + "syntax": [ + { "id": "root", "kind": "Block", "slot": "Root" }, + { "id": "loop", "kind": "ForEach", "parent": "root", "slot": "Statement", "binding": "f" }, + { "id": "iterator", "kind": "Block", "parent": "loop", "slot": "Iterator" }, + { "id": "sub", "kind": "CommandSubstitution", "parent": "iterator", "slot": "Substitution" }, + { "id": "subBody", "kind": "Block", "parent": "sub", "slot": "Body" }, + { "id": "get", "kind": "SimpleCommand", "parent": "subBody", "slot": "Statement", "commandIndex": 0 }, + { "id": "body", "kind": "Block", "parent": "loop", "slot": "Body" }, + { "id": "remove", "kind": "SimpleCommand", "parent": "body", "slot": "Statement", "commandIndex": 1 } + ], + "commands": [ + { "authoredVerb": "Get-ChildItem", "immediateRole": "Substitution", "ancestry": ["root", "loop", "iterator", "sub", "subBody"], "isComplete": true }, + { + "authoredVerb": "Remove-Item", + "immediateRole": "LoopBody", + "ancestry": ["root", "loop", "body"], + "isComplete": true, + "effectiveValues": [ + { "sourceElement": "$f", "kind": "Unknown", "isPolicySensitive": true } + ] + } + ], + "compatibility": { "verbs": ["Get-ChildItem", "Remove-Item"], "preservesAuthoredDynamicValues": true }, + "securityInvariants": ["AllCommandsVisible", "IteratorCommandsVisible", "UnknownPolicyValueFailsClosed"] + }, + "notes": "The nearest role is Substitution; Iterator remains in ancestry and the produced objects remain unknown." + }, { "id": "pwsh-foreach-cmdlet-parameter-like-value", "concern": "Expanded cmdlet value is not a syntactic parameter token", diff --git a/tests/ShellSyntaxTree.Tests/V03PublicApiSnapshotTests.cs b/tests/ShellSyntaxTree.Tests/V03PublicApiSnapshotTests.cs index abe7483..48bc733 100644 --- a/tests/ShellSyntaxTree.Tests/V03PublicApiSnapshotTests.cs +++ b/tests/ShellSyntaxTree.Tests/V03PublicApiSnapshotTests.cs @@ -61,7 +61,9 @@ public void Syntax_records_match_the_locked_members_and_defaults() AssertNode( new SimpleCommandSyntax(), ShellSyntaxKind.SimpleCommand, - (nameof(SimpleCommandSyntax.Clause), typeof(Clause))); + (nameof(SimpleCommandSyntax.Clause), typeof(Clause)), + (nameof(SimpleCommandSyntax.Substitutions), + typeof(IReadOnlyList))); AssertNode( new PipelineSyntax(), ShellSyntaxKind.Pipeline, @@ -105,6 +107,8 @@ public void Syntax_records_match_the_locked_members_and_defaults() var simple = new SimpleCommandSyntax(); Assert.NotNull(simple.Clause); + Assert.NotNull(simple.Substitutions); + Assert.Empty(simple.Substitutions); Assert.Empty(new ShellBlockSyntax().Statements); Assert.Empty(new PipelineSyntax().Stages); Assert.Empty(new CommandListSyntax().Items);