diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 50d8122..d4a332b 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -258,10 +258,19 @@ priorities. unknown fields and always requires unparseable projections to be empty. The PowerShell manifest now owns all 310 entries and round-trips exactly; explicit false/null assertions remain opt-in and generator-preserved. -- [ ] Deliver paired Bash and PowerShell `$()` substitution slices for all - locked executable value positions, including ordering, ancestry, - shell-specific cwd propagation, literal/escaped boundaries, dynamic - identities, fail-closed negatives, executable corpus, and Netclaw cases. +- [x] Deliver the first Bash `$()` substitution slice for supported + simple-command arguments and redirect targets. Direct tests and corpus + entries pin multiple and nested ordering, exact ancestry/spans, isolated + cwd, decoded wrappers, literal boundaries, dynamic compatibility values, + depth limits, comment-safe delimiter scanning, and fail-closed command + identities, background lists, assignment prefixes, backticks, heredocs, + and malformed interiors. +- [ ] Extend Bash substitution discovery to iterables and expanding heredoc + bodies, then add the corresponding Netclaw approval-matrix cases. +- [ ] Deliver the PowerShell `$()` substitution slice for every locked value + and expression position, including current-scope state propagation, + literal boundaries, incomplete dynamic identities, executable corpus, + and Netclaw cases. - [ ] Deliver Bash `for ... in` and PowerShell `foreach` as the first two language-specific vertical slices, then extract only the shared analysis proven by both implementations. diff --git a/SPEC.md b/SPEC.md index 662f1e1..7f1ea01 100644 --- a/SPEC.md +++ b/SPEC.md @@ -53,12 +53,11 @@ command can consume it. - Variable expansion. We mark dynamic tokens, never resolve them. - Function definitions, here-docs body extraction, complex parameter expansion (`${var//pattern/replacement}`), arithmetic expansion. -- Command-substitution evaluation. `$(cmd)` and backtick `` `cmd` `` are - recognized at the lex level and collapsed into a single - `Kind=DynamicSkip, IsPath=false` arg per locked interpretation #2 (see - `openspec/changes/archive/.../v0.1-locked-interpretations`). The - surrounding clause stays parseable so hard-deny rules still fire on - visible parts. +- Command-substitution evaluation. The library never executes a substitution + or claims its produced value is known. Stable v0.3 recursively discovers + commands inside supported Bash `$()` positions while retaining the authored + `Kind=DynamicSkip, IsPath=false` compatibility value. Legacy backticks and + incomplete executable interiors fail closed. - Performance tuning beyond "fast enough to invoke per shell call without noticeable latency" (~1ms per typical input). @@ -966,7 +965,9 @@ quoted_string := single-quoted | double-quoted immediately following a compound operator all collapse: they never yield an empty clause. The newline after a heredoc terminator likewise separates the heredoc's clause from what follows. -- `\` followed by a newline is a line continuation (treat as whitespace). +- `\` followed by a newline is removed before word-boundary analysis. It joins + adjacent fragments (`r\` + newline + `m` is the command name `rm`); actual + surrounding spaces still separate words. - Bash line comments (`#` at a word boundary through end-of-line) are whitespace-equivalent at the lexer level — they emit a Comment token for source fidelity but are filtered alongside Whitespace by the @@ -982,10 +983,10 @@ quoted_string := single-quoted | double-quoted NOT path-resolved. The parser carries the raw token (e.g. `&1`) on `Redirect.Target` and sets `Redirect.IsDynamicSkip = true`. This prevents `2>&1` from being incorrectly resolved to `/&1`. -- Function definitions, `case`/`esac`, C-style or implicit loops, arithmetic - execution, process substitution, and single-`&` background lists remain - unparseable in stable v0.3 because they can hide executable regions outside - the bounded grammar below. +- Function definitions, assignment-prefix commands, `case`/`esac`, C-style or + implicit loops, arithmetic execution, process substitution, and single-`&` + background lists remain unparseable in stable v0.3 because they can hide + executable regions outside the bounded grammar below. ### v0.3 structured Bash grammar @@ -1072,7 +1073,8 @@ The lexer produces tokens consumed by the parser. Token kinds: is flagged as a **statement separator**; the parser retains those tokens past `FilterSignificant` and splits clauses on them per §4. A pure space/tab run carries no flag and is discarded after splitting. -- **CONTINUATION** — `\` + `\n`. Treated as whitespace. +- **CONTINUATION** — `\` + `\n` (or `\r\n`). Removed before word-boundary + analysis; adjacent lexical fragments remain one authored word. - **OPAQUE_SUBSTITUTION** — `$(cmd)` or backtick `` `cmd` ``. The full substitution slice (including delimiters) becomes a single token. Boundary tracking handles nested same-kind regions, nested quotes, diff --git a/openspec/changes/v0-3-structured-shell-analysis/tasks.md b/openspec/changes/v0-3-structured-shell-analysis/tasks.md index c6d032a..61539a2 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/tasks.md +++ b/openspec/changes/v0-3-structured-shell-analysis/tasks.md @@ -32,9 +32,12 @@ - [x] 3.8 Add tests proving existing parser inputs retain their v0.2 leaf and compatibility results, except for explicitly promoted v0.3 fail-closed cases. - [x] 3.9 Add corpus expectations for syntax shape, occurrences, roles, and completeness for existing constructs. - [ ] 3.10 Implement Bash `$()` discovery in supported argument words, redirect values, iterables, and expanding heredoc bodies; retain literal/escaped spellings and fail closed on command-name substitutions, legacy backticks, or incomplete interiors. + - [x] 3.10a Implement the simple-command argument and redirect-target slice, including comment-safe boundaries and fail-closed unsupported interiors. - [ ] 3.11 Implement PowerShell `$()` discovery in supported words, redirect values, foreach expressions, call-operator dynamic identities, standalone expression statements, double-quoted strings, and expandable here-strings; never invent invocation from standalone output, retain literal/escaped spellings, and fail closed on trailing command-style arguments, call-operator script blocks, or unsupported execution-bearing `@()` / `@{}` forms. - [ ] 3.12 Pin substitution parentage, authored sibling indices, innermost-first ordering, Bash-isolated versus PowerShell-current-scope state, unknown-state propagation, nesting/depth limits, and incomplete dynamic identities in direct tests. + - [x] 3.12a Pin the Bash argument/redirect slice, isolated cwd behavior, wrapper provenance, and the shared structural-depth budget. - [ ] 3.13 Promote ordinary, multiple, nested, iterator, redirect, quoted, escaped, stateful, malformed, and hidden-execution substitution cases into both executable corpora and the Netclaw approval matrix. + - [x] 3.13a Promote the Bash ordinary, multiple, nested, redirect, quoted, escaped, stateful, malformed, and hidden-execution cases into its executable corpus. ## 4. Explicit Redirect Semantics diff --git a/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs b/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs index 2540f97..66e86d6 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs @@ -21,10 +21,10 @@ namespace ShellSyntaxTree.Internal.Bash.Lexing; /// The lexer never expands variables. $VAR and /// ${VAR} stay literal inside a /// token; the resolver in PR 4 decides what to do with them. -/// Opaque regions ($(…) and backtick `…`) are -/// consumed by and emitted as -/// a single token -/// per the v0.1 locked interpretation. +/// Command substitutions use a Bash-specific boundary scan so line +/// comments cannot hide a closing parenthesis. Backtick regions use +/// . Both are emitted as a single +/// token. /// Constructs SPEC §1 calls non-goals — arithmetic expansion /// $((…)) and complex parameter expansion /// ${var//pat/repl} — emit a @@ -134,6 +134,24 @@ internal static IReadOnlyList Tokenize(string input) continue; } + if (c == '&') + { + if (TryConsumeFileDescriptorTarget(src, i, tokens, out var afterTarget)) + { + i = afterTarget; + continue; + } + + tokens.Add(new BashToken( + BashTokenKind.UnparseableSentinel, + src.Slice(i).ToString(), + null, + i, + src.Length - i, + "single-'&' background lists are not supported")); + return tokens; + } + // ---- quoted strings ---- if (c == '\'') { @@ -217,6 +235,93 @@ internal static IReadOnlyList Tokenize(string input) return tokens; } + private static bool TryConsumeFileDescriptorTarget( + ReadOnlySpan src, + int start, + List tokens, + out int afterTarget) + { + afterTarget = start; + var previousIndex = tokens.Count - 1; + while (previousIndex >= 0) + { + var previous = tokens[previousIndex]; + if (previous.Kind == BashTokenKind.Whitespace) + { + if (previous.IsStatementSeparator) + { + return false; + } + + previousIndex--; + continue; + } + + if (previous.Kind is BashTokenKind.Continuation or BashTokenKind.Comment) + { + previousIndex--; + continue; + } + + break; + } + + if (previousIndex < 0 || tokens[previousIndex].Kind != BashTokenKind.Operator || + tokens[previousIndex].OperatorText is not (">" or ">>" or "<" or "2>" or "2>>")) + { + return false; + } + + var end = start + 1; + if (end < src.Length && src[end] == '$') + { + afterTarget = ReadWord(src, start, tokens, allowLeadingAmpersand: true); + return afterTarget > start + 1; + } + + if (end < src.Length && src[end] == '-') + { + end++; + } + else + { + var digitStart = end; + while (end < src.Length && src[end] is >= '0' and <= '9') + { + end++; + } + + if (end == digitStart) + { + return false; + } + + if (end < src.Length && src[end] == '-') + { + end++; + } + } + + if (end < src.Length && !char.IsWhiteSpace(src[end]) && !IsOperatorStart(src, end)) + { + return false; + } + + var raw = src.Slice(start, end - start).ToString(); + tokens.Add(new BashToken( + BashTokenKind.Word, + raw, + null, + start, + end - start, + null) + { + ResolverValue = ShellValue.Literal(raw, start, end - start), + }); + afterTarget = end; + return true; + } + // ---------------------------------------------------------------- operators private static bool TryReadOperator( @@ -417,7 +522,7 @@ private static int ConsumeCommandSubstitution( { // src[start] = '$', src[start+1] = '(' var openParen = start + 1; - var scan = OpaqueRegionScanner.Scan(src, openParen, '(', ')'); + var scan = ScanCommandSubstitution(src, openParen); if (!scan.Closed) { tokens.Add(new BashToken( @@ -426,7 +531,7 @@ private static int ConsumeCommandSubstitution( null, start, src.Length - start, - "unbalanced '$(' command substitution")); + scan.Error ?? "unbalanced '$(' command substitution")); return src.Length; } @@ -450,6 +555,196 @@ private static int ConsumeCommandSubstitution( return start + length; } + private static CommandSubstitutionScan ScanCommandSubstitution( + ReadOnlySpan src, + int openParen) + { + if (openParen < 0 || openParen >= src.Length || src[openParen] != '(') + { + return new CommandSubstitutionScan(src.Length, false, null); + } + + var resumeDoubleQuote = new Stack(); + resumeDoubleQuote.Push(false); + var inDoubleQuote = false; + var atWordBoundary = true; + var i = openParen + 1; + while (i < src.Length) + { + var c = src[i]; + if (inDoubleQuote) + { + if (c == '\\' && i + 1 < src.Length) + { + if (src[i + 1] == '\r' && i + 2 < src.Length && src[i + 2] == '\n') + { + i += 3; + continue; + } + + i += 2; + continue; + } + + if (c == '"') + { + inDoubleQuote = false; + i++; + continue; + } + + if (c == '`') + { + return new CommandSubstitutionScan( + src.Length, + false, + "legacy backtick command substitution is not supported"); + } + + if (c == '$' && i + 1 < src.Length && src[i + 1] == '(') + { + if (resumeDoubleQuote.Count >= ShellAnalysisLimits.MaxStructuralNesting) + { + return StructuralNestingOverflow(src.Length); + } + + resumeDoubleQuote.Push(true); + inDoubleQuote = false; + atWordBoundary = true; + i += 2; + continue; + } + + i++; + continue; + } + + if (c == '\\' && i + 1 < src.Length) + { + if (src[i + 1] is '\n' or '\r') + { + i += src[i + 1] == '\r' && i + 2 < src.Length && src[i + 2] == '\n' + ? 3 + : 2; + continue; + } + + atWordBoundary = false; + i += 2; + continue; + } + + if (c == '\'') + { + i++; + while (i < src.Length && src[i] != '\'') + { + i++; + } + + if (i >= src.Length) + { + return new CommandSubstitutionScan(src.Length, false, null); + } + + atWordBoundary = false; + i++; + continue; + } + + if (c == '"') + { + inDoubleQuote = true; + atWordBoundary = false; + i++; + continue; + } + + if (c == '`') + { + return new CommandSubstitutionScan( + src.Length, + false, + "legacy backtick command substitution is not supported"); + } + + if (c == '#' && atWordBoundary) + { + while (i < src.Length && src[i] is not '\n' and not '\r') + { + i++; + } + + atWordBoundary = true; + continue; + } + + if (c == '<' && i + 1 < src.Length && src[i + 1] == '<') + { + return new CommandSubstitutionScan( + src.Length, + false, + "heredocs inside command substitution are not supported"); + } + + if (c == '$' && i + 1 < src.Length && src[i + 1] == '(') + { + if (resumeDoubleQuote.Count >= ShellAnalysisLimits.MaxStructuralNesting) + { + return StructuralNestingOverflow(src.Length); + } + + resumeDoubleQuote.Push(false); + atWordBoundary = true; + i += 2; + continue; + } + + if (c == '(') + { + if (resumeDoubleQuote.Count >= ShellAnalysisLimits.MaxStructuralNesting) + { + return StructuralNestingOverflow(src.Length); + } + + resumeDoubleQuote.Push(false); + atWordBoundary = true; + i++; + continue; + } + + if (c == ')') + { + var restoreDoubleQuote = resumeDoubleQuote.Pop(); + if (resumeDoubleQuote.Count == 0) + { + return new CommandSubstitutionScan(i, true, null); + } + + inDoubleQuote = restoreDoubleQuote; + atWordBoundary = false; + i++; + continue; + } + + atWordBoundary = char.IsWhiteSpace(c) || c is ';' or '|' or '&' or '<' or '>'; + i++; + } + + return new CommandSubstitutionScan(src.Length, false, null); + } + + private static CommandSubstitutionScan StructuralNestingOverflow(int endIndex) => + new( + endIndex, + false, + $"Bash structural nesting depth exceeded (>{ShellAnalysisLimits.MaxStructuralNesting})"); + + private readonly record struct CommandSubstitutionScan( + int EndIndex, + bool Closed, + string? Error); + private static int ConsumeBacktickSubstitution( ReadOnlySpan src, int start, List tokens) { @@ -607,7 +902,10 @@ private static bool TryConsumeComplexParamExpansion( // ---------------------------------------------------------------- words private static int ReadWord( - ReadOnlySpan src, int start, List tokens) + ReadOnlySpan src, + int start, + List tokens, + bool allowLeadingAmpersand = false) { // A word continues until we hit whitespace, an operator boundary, // a newline, a quote, a backtick, or the start of an opaque region @@ -631,7 +929,8 @@ private static int ReadWord( // Stop conditions. if (c == ' ' || c == '\t' || c == '\n' || c == '\r') break; if (c == '\'' || c == '"' || c == '`') break; - if (IsOperatorStart(src, i)) break; + if (IsOperatorStart(src, i) + && !(allowLeadingAmpersand && i == start && c == '&')) break; // Backslash escapes the next character (outside quotes). if (c == '\\') @@ -647,9 +946,12 @@ private static int ReadWord( var n = src[i + 1]; if (n == '\n' || n == '\r') { - // Continuation: terminate the current word; the outer - // loop will pick up the continuation token. - break; + // Bash removes a continuation before word-boundary + // analysis, so adjacent fragments remain one word. + i += n == '\r' && i + 2 < src.Length && src[i + 2] == '\n' + ? 3 + : 2; + continue; } value.AppendLiteral(n, i, 2); @@ -766,10 +1068,10 @@ private static bool TryAppendBashExpansion( return true; } - var scan = OpaqueRegionScanner.Scan(src, start + 1, '(', ')'); + var scan = ScanCommandSubstitution(src, start + 1); if (!scan.Closed) { - error = "unbalanced '$(' command substitution"; + error = scan.Error ?? "unbalanced '$(' command substitution"; index = src.Length; return true; } @@ -889,11 +1191,9 @@ private static bool IsOperatorStart(ReadOnlySpan src, int i) case ')': return true; case '&': - // Only `&&` is an operator in v0.1; bare `&` is unsupported - // background-job syntax. Treat `&` not followed by `&` as a - // word char to avoid silently splitting; consumers will see - // it in the Raw value. - return i + 1 < src.Length && src[i + 1] == '&'; + // Both `&&` and unsupported bare `&` terminate a word. The + // tokenizer emits a sentinel for the latter on its next pass. + return true; case '2': // `2>` and `2>>` start with '2' — only treat them as operator // starts when the immediate next char is '>'. diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs index 267c510..7a51462 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs @@ -51,13 +51,20 @@ internal static ParsedCommand Parse(string source, BashParserOptions options) throw new ArgumentNullException(nameof(options)); } - return ParseInternal(source, options, bashCDepth: 0, markBashCWrapped: false); + return ParseInternal( + source, + options, + bashCDepth: 0, + structuralDepth: 0, + markBashCWrapped: false); } /// /// Recursion entry point. counts how many /// bash -c wrappers we've unwrapped to reach this call; the - /// outer caller passes 0. sets + /// outer caller passes 0. shares the + /// bounded nesting budget across substitutions, subshells, and decoded + /// wrappers. sets /// on every emitted clause and /// fires only on recursive calls (the outer top-level command doesn't /// pretend to be wrapped). @@ -66,6 +73,7 @@ private static ParsedCommand ParseInternal( string source, BashParserOptions options, int bashCDepth, + int structuralDepth, bool markBashCWrapped) { if (source.Length == 0) @@ -75,6 +83,7 @@ private static ParsedCommand ParseInternal( Array.Empty(), options, bashCDepth, + structuralDepth, markBashCWrapped); } @@ -99,6 +108,7 @@ private static ParsedCommand ParseInternal( significant, options, bashCDepth, + structuralDepth, markBashCWrapped); } @@ -288,6 +298,8 @@ private static bool TryDetectBashCWrapper(Segment segment, string source, out st private static List FilterSignificant(IReadOnlyList tokens) { var filtered = new List(tokens.Count); + var joinsAcrossContinuation = false; + var continuationEnd = -1; foreach (var t in tokens) { // A newline-bearing Whitespace token is a statement separator @@ -295,17 +307,31 @@ private static List FilterSignificant(IReadOnlyList tokens // can treat it exactly like an explicit ';'. Plain // space/tab Whitespace, Continuation, and Comment carry no // structural signal and are dropped. + if (t.Kind == BashTokenKind.Continuation) + { + var followsJoinedFragment = joinsAcrossContinuation + ? t.SourceStart == continuationEnd + : filtered.Count > 0 + && IsNativeArgumentFragment(filtered[filtered.Count - 1]) + && filtered[filtered.Count - 1].SourceStart + + filtered[filtered.Count - 1].SourceLength == t.SourceStart; + joinsAcrossContinuation = followsJoinedFragment; + continuationEnd = t.SourceStart + t.SourceLength; + continue; + } + if ((t.Kind == BashTokenKind.Whitespace && !t.IsStatementSeparator) - || t.Kind == BashTokenKind.Continuation || t.Kind == BashTokenKind.Comment) { + joinsAcrossContinuation = false; continue; } if (filtered.Count > 0 && IsNativeArgumentFragment(filtered[filtered.Count - 1]) && IsNativeArgumentFragment(t) - && IsAdjacent(filtered[filtered.Count - 1], t) + && (IsAdjacent(filtered[filtered.Count - 1], t) + || joinsAcrossContinuation && t.SourceStart == continuationEnd) && !IsInlineNativeArgumentPrefix(filtered[filtered.Count - 1])) { var previous = filtered[filtered.Count - 1]; @@ -323,9 +349,11 @@ private static List FilterSignificant(IReadOnlyList tokens { ResolverValue = ShellValue.Concat(new[] { previousValue, currentValue }), }; + joinsAcrossContinuation = false; continue; } + joinsAcrossContinuation = false; filtered.Add(t); } diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs index 3a58081..a1c58b5 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs @@ -18,6 +18,7 @@ private static ParsedCommand ParseStructured( IReadOnlyList tokens, BashParserOptions options, int bashCDepth, + int structuralDepth, bool markBashCWrapped) { var coordinator = new StructuralCoordinator( @@ -25,7 +26,10 @@ private static ParsedCommand ParseStructured( tokens, options, bashCDepth, - markBashCWrapped); + structuralDepth, + markBashCWrapped, + sourceStart: 0, + source.Length); if (!coordinator.TryParse(out var syntax, out var error)) { return StructuralFailure(source, error, syntax); @@ -36,8 +40,7 @@ private static ParsedCommand ParseStructured( simple => new CommandOccurrenceFacts { IsComplete = simple.Clause.Redirects.Count == 0 && - !HasUnexpandedCommandString(simple.Clause) && - !HasUndiscoveredExecutableRegion(simple.Clause), + !HasUnexpandedCommandString(simple.Clause), }, out var projection)) { @@ -75,7 +78,10 @@ private sealed class StructuralCoordinator private readonly IReadOnlyList _tokens; private readonly BashParserOptions _options; private readonly int _bashCDepth; + private readonly int _structuralDepth; private readonly bool _markBashCWrapped; + private readonly int _sourceStart; + private readonly int _sourceLength; private readonly CdAttributionContext _attribution = new(); private int _position; private int _subshellDepth; @@ -85,13 +91,19 @@ internal StructuralCoordinator( IReadOnlyList tokens, BashParserOptions options, int bashCDepth, - bool markBashCWrapped) + int structuralDepth, + bool markBashCWrapped, + int sourceStart, + int sourceLength) { _source = source; _tokens = tokens; _options = options; _bashCDepth = bashCDepth; + _structuralDepth = structuralDepth; _markBashCWrapped = markBashCWrapped; + _sourceStart = sourceStart; + _sourceLength = sourceLength; } internal bool TryParse(out ShellBlockSyntax syntax, out string? error) @@ -101,8 +113,8 @@ internal bool TryParse(out ShellBlockSyntax syntax, out string? error) { syntax = new ShellBlockSyntax { - SourceStart = 0, - SourceLength = _source.Length, + SourceStart = _sourceStart, + SourceLength = _sourceLength, }; error = null; return true; @@ -132,8 +144,8 @@ internal bool TryParse(out ShellBlockSyntax syntax, out string? error) syntax = new ShellBlockSyntax { Statements = new[] { command }, - SourceStart = 0, - SourceLength = _source.Length, + SourceStart = _sourceStart, + SourceLength = _sourceLength, }; return true; } @@ -314,6 +326,20 @@ private bool TryParseCommand( Tokens = segmentTokens, }; + if (HasAssignmentPrefix(segmentTokens)) + { + error = "Bash assignment-prefix commands are not supported"; + return false; + } + + if (!TryCollectCommandSubstitutions( + segmentTokens, + out var substitutionFragments, + out error)) + { + return false; + } + if (TryDetectBashCWrapper(segment, _source, out var innerCommand)) { if (!IsExactStaticWrapper(segmentTokens)) @@ -332,6 +358,7 @@ private bool TryParseCommand( innerCommand!, _options, _bashCDepth + 1, + _structuralDepth + _subshellDepth + 1, markBashCWrapped: true); if (inner.IsUnparseable) { @@ -402,6 +429,15 @@ private bool TryParseCommand( IsCommandStringWrapped = _markBashCWrapped, }; var emitted = AttachAttributionArg(clause, _attribution); + if (!TryParseCommandSubstitutions( + substitutionFragments, + effectiveOptions, + out var substitutions, + out error)) + { + return false; + } + var firstVerbToken = clause.Verb.Tokens.Count > 0 ? clause.Verb.Tokens[0] : null; if (firstVerbToken is not null && (string.Equals(firstVerbToken, "cd", StringComparison.OrdinalIgnoreCase) || @@ -415,6 +451,7 @@ private bool TryParseCommand( command = new SimpleCommandSyntax { Clause = emitted, + Substitutions = substitutions, SourceStart = firstSource.SourceStart, SourceLength = lastSource.SourceStart + lastSource.SourceLength - firstSource.SourceStart, @@ -427,7 +464,8 @@ private bool TryParseSubshell( out ShellSyntaxNode? command, out string? error) { - if (_subshellDepth >= ShellAnalysisLimits.MaxStructuralNesting) + if (_structuralDepth + _subshellDepth >= + ShellAnalysisLimits.MaxStructuralNesting) { command = null; error = "Bash structural nesting depth exceeded (>16)"; @@ -454,7 +492,7 @@ private bool TryParseSubshell( if (_position == _tokens.Count || !IsOperator(")")) { command = null; - error = $"unbalanced parens at position {_source.Length}"; + error = $"unbalanced parens at position {_sourceStart + _sourceLength}"; return false; } @@ -527,6 +565,261 @@ private bool IsOperator(string value) => _position < _tokens.Count && _tokens[_position].Kind == BashTokenKind.Operator && string.Equals(_tokens[_position].OperatorText, value, StringComparison.Ordinal); + + private bool TryCollectCommandSubstitutions( + IReadOnlyList tokens, + out IReadOnlyList substitutions, + out string? error) + { + var discovered = new List(); + var commandNameEnd = tokens[0].SourceStart + tokens[0].SourceLength; + for (var index = 1; index < tokens.Count; index++) + { + var token = tokens[index]; + if (token.Kind == BashTokenKind.Operator || + token.SourceStart != commandNameEnd) + { + break; + } + + commandNameEnd = token.SourceStart + token.SourceLength; + } + + foreach (var token in tokens) + { + var value = token.ResolverValue; + if (value is null) + { + continue; + } + + foreach (var fragment in value.Fragments) + { + if (fragment.Kind != ShellValueFragmentKind.Opaque || + fragment.OpaqueCause != ShellOpaqueCause.CommandSubstitution) + { + continue; + } + + if (fragment.SourceStart is null || fragment.SourceLength is null || + fragment.SourceLength < 2 || + fragment.SourceStart < _sourceStart || + fragment.SourceStart + fragment.SourceLength > + _sourceStart + _sourceLength) + { + substitutions = Array.Empty(); + error = "Bash command substitution has invalid source provenance"; + return false; + } + + var raw = _source.Substring( + fragment.SourceStart.Value, + fragment.SourceLength.Value); + if (raw[0] == '`') + { + substitutions = Array.Empty(); + error = "legacy backtick command substitution is not supported"; + return false; + } + + if (!raw.StartsWith("$(", StringComparison.Ordinal) || raw[raw.Length - 1] != ')') + { + substitutions = Array.Empty(); + error = "unsupported Bash command substitution provenance"; + return false; + } + + if (fragment.SourceStart < commandNameEnd) + { + substitutions = Array.Empty(); + error = "Bash command-name substitution is not supported"; + return false; + } + + discovered.Add(fragment); + } + } + + substitutions = discovered; + error = null; + return true; + } + + private bool HasAssignmentPrefix(IReadOnlyList tokens) + { + var spelling = _source.Substring(tokens[0].SourceStart, tokens[0].SourceLength) + .Replace("\\\r\n", string.Empty) + .Replace("\\\n", string.Empty) + .Replace("\\\r", string.Empty); + var index = 0; + if (spelling.Length == 0 || !IsBashIdentifierStart(spelling[index])) + { + return false; + } + + index++; + while (index < spelling.Length && IsBashIdentifierContinuation(spelling[index])) + { + index++; + } + + if (index < spelling.Length && spelling[index] == '[') + { + while (index < spelling.Length && spelling[index] != ']') + { + index++; + } + + if (index >= spelling.Length) + { + return false; + } + + index++; + } + + if (index < spelling.Length && spelling[index] == '+') + { + index++; + } + + return index < spelling.Length && spelling[index] == '='; + } + + private static bool IsBashIdentifierStart(char value) => + value == '_' || value is >= 'A' and <= 'Z' or >= 'a' and <= 'z'; + + private static bool IsBashIdentifierContinuation(char value) => + IsBashIdentifierStart(value) || value is >= '0' and <= '9'; + + private bool TryParseCommandSubstitutions( + IReadOnlyList fragments, + BashParserOptions options, + out IReadOnlyList substitutions, + out string? error) + { + if (fragments.Count == 0) + { + substitutions = Array.Empty(); + error = null; + return true; + } + + if (_structuralDepth + _subshellDepth + 1 > + ShellAnalysisLimits.MaxStructuralNesting) + { + substitutions = Array.Empty(); + error = "Bash structural nesting depth exceeded (>16)"; + return false; + } + + var parsed = new List(fragments.Count); + foreach (var fragment in fragments) + { + var sourceStart = fragment.SourceStart!.Value; + var sourceLength = fragment.SourceLength!.Value; + var bodyStart = sourceStart + 2; + var bodyLength = sourceLength - 3; + if (!TryParseSubstitutionBody( + bodyStart, + bodyLength, + options, + out var body, + out error)) + { + substitutions = Array.Empty(); + return false; + } + + parsed.Add(new CommandSubstitutionSyntax + { + Body = body, + SourceStart = sourceStart, + SourceLength = sourceLength, + }); + } + + substitutions = parsed; + error = null; + return true; + } + + private bool TryParseSubstitutionBody( + int sourceStart, + int sourceLength, + BashParserOptions options, + out ShellBlockSyntax body, + out string? error) + { + var source = _source.Substring(sourceStart, sourceLength); + var relativeTokens = BashLexer.Tokenize(source); + for (var index = 0; index < relativeTokens.Count; index++) + { + if (relativeTokens[index].Kind == BashTokenKind.UnparseableSentinel) + { + body = new ShellBlockSyntax(); + error = relativeTokens[index].UnparseableReason; + return false; + } + } + + var significant = FilterSignificant(relativeTokens); + if (TryDetectAnomaly(significant, out error)) + { + body = new ShellBlockSyntax(); + return false; + } + + var shifted = ShiftTokens(significant, sourceStart); + var coordinator = new StructuralCoordinator( + _source, + shifted, + options, + _bashCDepth, + _structuralDepth + _subshellDepth + 1, + _markBashCWrapped, + sourceStart, + sourceLength); + return coordinator.TryParse(out body, out error); + } + } + + private static IReadOnlyList ShiftTokens( + IReadOnlyList tokens, + int sourceOffset) + { + var shifted = new BashToken[tokens.Count]; + for (var index = 0; index < tokens.Count; index++) + { + var token = tokens[index]; + shifted[index] = token with + { + SourceStart = token.SourceStart + sourceOffset, + ResolverValue = ShiftValue(token.ResolverValue, sourceOffset), + }; + } + + return shifted; + } + + private static ShellValue? ShiftValue(ShellValue? value, int sourceOffset) + { + if (value is null) + { + return null; + } + + var fragments = new ShellValueFragment[value.Fragments.Count]; + for (var index = 0; index < value.Fragments.Count; index++) + { + var fragment = value.Fragments[index]; + fragments[index] = fragment with + { + SourceStart = fragment.SourceStart + sourceOffset, + }; + } + + return new ShellValue(value.Decoded, fragments); } private static bool IsStructuralBoundary(BashToken token) => @@ -650,26 +943,6 @@ private static bool IsCommandStringOption(string value) return false; } - private static bool HasUndiscoveredExecutableRegion(Clause clause) - { - for (var index = 0; index < clause.Elements.Count; index++) - { - var element = clause.Elements[index]; - if (element.Kind != ArgKind.DynamicSkip) - { - continue; - } - - if (element.Raw.IndexOf("$(", StringComparison.Ordinal) >= 0 || - element.Raw.IndexOf('`') >= 0) - { - return true; - } - } - - return false; - } - private static int? CombinedStart(IReadOnlyList nodes) { if (nodes.Count == 0 || nodes[0].SourceStart is null) diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/174_double_quoted_backtick_substitution.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/174_double_quoted_backtick_substitution.json index cf8b56f..faadc3b 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/174_double_quoted_backtick_substitution.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/174_double_quoted_backtick_substitution.json @@ -1,14 +1,10 @@ { - "name": "Backtick substitution inside double quotes remains opaque", + "name": "Backtick substitution inside double quotes fails closed", "input": "cat \"`printf /etc/passwd`\"", "expected": { - "isUnparseable": false, - "clauses": [{ - "operator": "None", - "verb": ["cat"], - "args": [{ "raw": "\"`printf /etc/passwd`\"", "kind": "DynamicSkip", "isPath": false }], - "redirects": [] - }] + "isUnparseable": true, + "unparseableReasonContains": "backtick command substitution", + "clauses": [] }, - "notes": "Bash executes the backtick region before forming the quoted argument, so the parser cannot claim an exact path." + "notes": "Bash executes the backtick region before forming the quoted argument; stable v0.3 rejects it until its distinct grammar has complete command discovery." } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/180_dynamic_command_identity.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/180_dynamic_command_identity.json index 7acafa1..8df89f9 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/180_dynamic_command_identity.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/180_dynamic_command_identity.json @@ -3,7 +3,7 @@ "input": "r$(printf m) -rf /tmp/x", "expected": { "isUnparseable": true, - "unparseableReasonContains": "dynamic Bash command identity", + "unparseableReasonContains": "command-name substitution", "clauses": [] }, "notes": "Adjacent aggregation must not erase a command substitution from the executable identity." diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/187_v03_hidden_substitution_incomplete.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/187_v03_hidden_substitution_incomplete.json index 845b6f9..41ab137 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/187_v03_hidden_substitution_incomplete.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/187_v03_hidden_substitution_incomplete.json @@ -1,9 +1,19 @@ { - "name": "v0.3 hidden command substitution remains incomplete", + "name": "v0.3 command substitution is discovered", "input": "rm $(find /tmp)", "expected": { "isUnparseable": false, "clauses": [ + { + "operator": "None", + "verb": ["find"], + "args": [ + { "raw": "/tmp", "kind": "Literal", "isPath": true, "resolved": "/tmp" } + ], + "redirects": [], + "isSubshell": false, + "isCommandStringWrapped": false + }, { "operator": "None", "verb": ["rm"], @@ -17,18 +27,31 @@ ], "syntax": [ { "kind": "Block", "parentIndex": null, "region": "Unknown", "childIndex": null, "sourceStart": 0, "sourceLength": 15, "clauseIndex": null, "groupKind": null, "listOperator": null }, - { "kind": "SimpleCommand", "parentIndex": 0, "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 15, "clauseIndex": 0, "groupKind": null, "listOperator": null } + { "kind": "SimpleCommand", "parentIndex": 0, "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 15, "clauseIndex": 1, "groupKind": null, "listOperator": null }, + { "kind": "CommandSubstitution", "parentIndex": 1, "region": "Substitution", "childIndex": 0, "sourceStart": 3, "sourceLength": 12, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "Block", "parentIndex": 2, "region": "Substitution", "childIndex": 0, "sourceStart": 5, "sourceLength": 9, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "SimpleCommand", "parentIndex": 3, "region": "Statement", "childIndex": 0, "sourceStart": 5, "sourceLength": 9, "clauseIndex": 0, "groupKind": null, "listOperator": null } ], "commands": [ { "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 15 }, + { "ancestorKind": "CommandSubstitution", "region": "Substitution", "childIndex": 0, "sourceStart": 3, "sourceLength": 12 }, + { "ancestorKind": "Block", "region": "Statement", "childIndex": 0, "sourceStart": 5, "sourceLength": 9 } + ] + }, + { + "clauseIndex": 1, "immediateRole": "Ordinary", - "isComplete": false, + "isComplete": true, "ancestry": [ { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 15 } ] } ] }, - "notes": "Until task 3.10 discovers the inner find command, the authored outer leaf stays visible but cannot authorize execution." + "notes": "The inner find command is visible before rm while the authored DynamicSkip argument remains unchanged." } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/190_v03_multiple_substitutions.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/190_v03_multiple_substitutions.json new file mode 100644 index 0000000..f7d15fc --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/190_v03_multiple_substitutions.json @@ -0,0 +1,218 @@ +{ + "name": "v0.3 multiple command substitutions", + "input": "printf \u0027%s %s\u0027 \u0022$(whoami)\u0022 \u0022$(pwd)\u0022", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "whoami" + ], + "args": [], + "redirects": [] + }, + { + "operator": "None", + "verb": [ + "pwd" + ], + "args": [], + "redirects": [] + }, + { + "operator": "None", + "verb": [ + "printf" + ], + "args": [ + { + "raw": "\u0027%s %s\u0027", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + }, + { + "raw": "\u0022$(whoami)\u0022", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + }, + { + "raw": "\u0022$(pwd)\u0022", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 35, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 35, + "clauseIndex": 2, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 16, + "sourceLength": 9, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 2, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 18, + "sourceLength": 6, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 18, + "sourceLength": 6, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 1, + "sourceStart": 28, + "sourceLength": 6, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 5, + "region": "Substitution", + "childIndex": 1, + "sourceStart": 30, + "sourceLength": 3, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 6, + "region": "Statement", + "childIndex": 0, + "sourceStart": 30, + "sourceLength": 3, + "clauseIndex": 1, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 35 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 16, + "sourceLength": 9 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 18, + "sourceLength": 6 + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 35 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 1, + "sourceStart": 28, + "sourceLength": 6 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 30, + "sourceLength": 3 + } + ] + }, + { + "clauseIndex": 2, + "immediateRole": "Ordinary", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 35 + } + ] + } + ] + }, + "notes": "Both substitutions are ordered before their containing printf occurrence." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/191_v03_nested_substitution.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/191_v03_nested_substitution.json new file mode 100644 index 0000000..ed684e1 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/191_v03_nested_substitution.json @@ -0,0 +1,233 @@ +{ + "name": "v0.3 nested command substitution", + "input": "printf \u0027%s\u0027 \u0022$(echo \u0022$(whoami)\u0022)\u0022", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "whoami" + ], + "args": [], + "redirects": [] + }, + { + "operator": "None", + "verb": [ + "echo" + ], + "args": [ + { + "raw": "\u0022$(whoami)\u0022", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + }, + { + "operator": "None", + "verb": [ + "printf" + ], + "args": [ + { + "raw": "\u0027%s\u0027", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + }, + { + "raw": "\u0022$(echo \u0022$(whoami)\u0022)\u0022", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 33, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 33, + "clauseIndex": 2, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 13, + "sourceLength": 19, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 2, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 16, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 16, + "clauseIndex": 1, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 4, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 21, + "sourceLength": 9, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 5, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 23, + "sourceLength": 6, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 6, + "region": "Statement", + "childIndex": 0, + "sourceStart": 23, + "sourceLength": 6, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 33 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 13, + "sourceLength": 19 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 16 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 21, + "sourceLength": 9 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 23, + "sourceLength": 6 + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 33 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 13, + "sourceLength": 19 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 16 + } + ] + }, + { + "clauseIndex": 2, + "immediateRole": "Ordinary", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 33 + } + ] + } + ] + }, + "notes": "Nested substitutions retain exact parentage and emit inner commands first." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/192_v03_substitution_comment_boundary.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/192_v03_substitution_comment_boundary.json new file mode 100644 index 0000000..30eb7e6 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/192_v03_substitution_comment_boundary.json @@ -0,0 +1,208 @@ +{ + "name": "v0.3 comment-safe substitution boundary", + "input": "echo \u0022$(printf x # )\nid)\u0022", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "printf", + "x" + ], + "args": [], + "redirects": [] + }, + { + "operator": "Sequence", + "verb": [ + "id" + ], + "args": [], + "redirects": [] + }, + { + "operator": "None", + "verb": [ + "echo" + ], + "args": [ + { + "raw": "\u0022$(printf x # )\nid)\u0022", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 25, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 25, + "clauseIndex": 2, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 6, + "sourceLength": 18, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 2, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 8, + "sourceLength": 15, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandList", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 8, + "sourceLength": 15, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 4, + "region": "Statement", + "childIndex": 0, + "sourceStart": 8, + "sourceLength": 8, + "clauseIndex": 0, + "groupKind": null, + "listOperator": "None" + }, + { + "kind": "SimpleCommand", + "parentIndex": 4, + "region": "Statement", + "childIndex": 1, + "sourceStart": 21, + "sourceLength": 2, + "clauseIndex": 1, + "groupKind": null, + "listOperator": "Sequence" + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 25 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 6, + "sourceLength": 18 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 8, + "sourceLength": 15 + }, + { + "ancestorKind": "CommandList", + "region": "Statement", + "childIndex": 0, + "sourceStart": 8, + "sourceLength": 15 + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 25 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 6, + "sourceLength": 18 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 8, + "sourceLength": 15 + }, + { + "ancestorKind": "CommandList", + "region": "Statement", + "childIndex": 1, + "sourceStart": 8, + "sourceLength": 15 + } + ] + }, + { + "clauseIndex": 2, + "immediateRole": "Ordinary", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 25 + } + ] + } + ] + }, + "notes": "A parenthesis inside a Bash line comment cannot hide the following id command." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/193_v03_redirect_substitution.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/193_v03_redirect_substitution.json new file mode 100644 index 0000000..cfe2c34 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/193_v03_redirect_substitution.json @@ -0,0 +1,133 @@ +{ + "name": "v0.3 redirect command substitution", + "input": "cat \u003E \u0022$(mktemp)\u0022", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "mktemp" + ], + "args": [], + "redirects": [] + }, + { + "operator": "None", + "verb": [ + "cat" + ], + "args": [], + "redirects": [ + { + "direction": "Out", + "target": "\u0022$(mktemp)\u0022", + "isDynamicSkip": true + } + ] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 17, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 17, + "clauseIndex": 1, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 7, + "sourceLength": 9, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 2, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 9, + "sourceLength": 6, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 9, + "sourceLength": 6, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 17 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 7, + "sourceLength": 9 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 9, + "sourceLength": 6 + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Ordinary", + "isComplete": false, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 17 + } + ] + } + ] + }, + "notes": "The inner command is visible while the dynamic redirect keeps the outer occurrence incomplete." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/194_v03_substitution_cwd_isolation.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/194_v03_substitution_cwd_isolation.json new file mode 100644 index 0000000..3be6c9a --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/194_v03_substitution_cwd_isolation.json @@ -0,0 +1,311 @@ +{ + "name": "v0.3 command substitution cwd isolation", + "input": "printf \u0027%s\u0027 \u0022$(cd /tmp; pwd)\u0022; cat relative.txt", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "cd" + ], + "args": [ + { + "raw": "/tmp", + "kind": "Literal", + "isPath": true, + "resolved": "/tmp", + "isFlag": false + } + ], + "redirects": [] + }, + { + "operator": "Sequence", + "verb": [ + "pwd" + ], + "args": [ + { + "raw": "/tmp", + "kind": "Literal", + "isPath": true, + "resolved": "/tmp", + "isFlag": false, + "isCwdAttribution": true + } + ], + "redirects": [] + }, + { + "operator": "None", + "verb": [ + "printf" + ], + "args": [ + { + "raw": "\u0027%s\u0027", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + }, + { + "raw": "\u0022$(cd /tmp; pwd)\u0022", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + }, + { + "operator": "Sequence", + "verb": [ + "cat" + ], + "args": [ + { + "raw": "relative.txt", + "kind": "Literal", + "isPath": true, + "resolved": "/work/relative.txt", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 47, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandList", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 47, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 1, + "region": "Statement", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 29, + "clauseIndex": 2, + "groupKind": null, + "listOperator": "None" + }, + { + "kind": "CommandSubstitution", + "parentIndex": 2, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 13, + "sourceLength": 15, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 3, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 12, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandList", + "parentIndex": 4, + "region": "Statement", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 12, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 5, + "region": "Statement", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 7, + "clauseIndex": 0, + "groupKind": null, + "listOperator": "None" + }, + { + "kind": "SimpleCommand", + "parentIndex": 5, + "region": "Statement", + "childIndex": 1, + "sourceStart": 24, + "sourceLength": 3, + "clauseIndex": 1, + "groupKind": null, + "listOperator": "Sequence" + }, + { + "kind": "SimpleCommand", + "parentIndex": 1, + "region": "Statement", + "childIndex": 1, + "sourceStart": 31, + "sourceLength": 16, + "clauseIndex": 3, + "groupKind": null, + "listOperator": "Sequence" + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 47 + }, + { + "ancestorKind": "CommandList", + "region": "Statement", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 47 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 13, + "sourceLength": 15 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 12 + }, + { + "ancestorKind": "CommandList", + "region": "Statement", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 12 + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 47 + }, + { + "ancestorKind": "CommandList", + "region": "Statement", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 47 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 13, + "sourceLength": 15 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 15, + "sourceLength": 12 + }, + { + "ancestorKind": "CommandList", + "region": "Statement", + "childIndex": 1, + "sourceStart": 15, + "sourceLength": 12 + } + ] + }, + { + "clauseIndex": 2, + "immediateRole": "Ordinary", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 47 + }, + { + "ancestorKind": "CommandList", + "region": "Statement", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 47 + } + ] + }, + { + "clauseIndex": 3, + "immediateRole": "Ordinary", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 47 + }, + { + "ancestorKind": "CommandList", + "region": "Statement", + "childIndex": 1, + "sourceStart": 0, + "sourceLength": 47 + } + ] + } + ] + }, + "notes": "Directory changes inside a substitution do not leak to the following outer command." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/195_v03_outer_wrapper_substitution.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/195_v03_outer_wrapper_substitution.json new file mode 100644 index 0000000..2f5c333 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/195_v03_outer_wrapper_substitution.json @@ -0,0 +1,142 @@ +{ + "name": "v0.3 outer bash wrapper substitution", + "input": "bash -c \u0022echo $(id)\u0022", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "id" + ], + "args": [], + "redirects": [] + }, + { + "operator": "None", + "verb": [ + "bash" + ], + "args": [ + { + "raw": "-c", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": true + }, + { + "raw": "\u0022echo $(id)\u0022", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 20, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 20, + "clauseIndex": 1, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 14, + "sourceLength": 5, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 2, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 16, + "sourceLength": 2, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 16, + "sourceLength": 2, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 20 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 14, + "sourceLength": 5 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 16, + "sourceLength": 2 + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Ordinary", + "isComplete": false, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 20 + } + ] + } + ] + }, + "notes": "A substitution executed before the outer wrapper remains directly source-addressable." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/196_v03_decoded_wrapper_substitution.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/196_v03_decoded_wrapper_substitution.json new file mode 100644 index 0000000..d1c94fe --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/196_v03_decoded_wrapper_substitution.json @@ -0,0 +1,187 @@ +{ + "name": "v0.3 decoded bash wrapper substitution", + "input": "bash -c \u0027echo $(id)\u0027", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "id" + ], + "args": [], + "redirects": [], + "isCommandStringWrapped": true + }, + { + "operator": "None", + "verb": [ + "echo" + ], + "args": [ + { + "raw": "$(id)", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [], + "isCommandStringWrapped": true + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 20, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Group", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 20, + "clauseIndex": null, + "groupKind": "IsolatedScope", + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "GroupBody", + "childIndex": null, + "sourceStart": null, + "sourceLength": null, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 2, + "region": "Statement", + "childIndex": 0, + "sourceStart": null, + "sourceLength": null, + "clauseIndex": 1, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 3, + "region": "Substitution", + "childIndex": 0, + "sourceStart": null, + "sourceLength": null, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 4, + "region": "Substitution", + "childIndex": 0, + "sourceStart": null, + "sourceLength": null, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 5, + "region": "Statement", + "childIndex": 0, + "sourceStart": null, + "sourceLength": null, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 20 + }, + { + "ancestorKind": "Group", + "region": "GroupBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 20 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": null, + "sourceLength": null + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": null, + "sourceLength": null + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": null, + "sourceLength": null + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Ordinary", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 20 + }, + { + "ancestorKind": "Group", + "region": "GroupBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 20 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": null, + "sourceLength": null + } + ] + } + ] + }, + "notes": "Commands discovered only after decoding a wrapper carry null source spans." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/197_v03_background_list_unparseable.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/197_v03_background_list_unparseable.json new file mode 100644 index 0000000..66c4445 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/197_v03_background_list_unparseable.json @@ -0,0 +1,10 @@ +{ + "name": "v0.3 background list fails closed", + "input": "echo \u0022$(id \u0026 evil)\u0022", + "expected": { + "isUnparseable": true, + "unparseableReasonContains": "single-\u0027\u0026\u0027 background lists are not supported" + }, + "notes": "Single-ampersand concurrency is outside the bounded grammar and publishes no partial commands.", + "oracleExpectation": "OutOfScope" +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/198_v03_assignment_prefix_unparseable.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/198_v03_assignment_prefix_unparseable.json new file mode 100644 index 0000000..b5b3b57 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/198_v03_assignment_prefix_unparseable.json @@ -0,0 +1,10 @@ +{ + "name": "v0.3 assignment prefix fails closed", + "input": "echo \u0022$(X=1 rm -rf /tmp/x)\u0022", + "expected": { + "isUnparseable": true, + "unparseableReasonContains": "Bash assignment-prefix commands are not supported" + }, + "notes": "Assignment-prefix shell syntax is not modeled and therefore publishes no partial commands.", + "oracleExpectation": "OutOfScope" +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/199_v03_substitution_heredoc_unparseable.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/199_v03_substitution_heredoc_unparseable.json new file mode 100644 index 0000000..87fc073 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/199_v03_substitution_heredoc_unparseable.json @@ -0,0 +1,10 @@ +{ + "name": "v0.3 substitution heredoc fails closed", + "input": "rm \u0022$(cat \u003C\u003CEOF\nvalue\nEOF\n)\u0022", + "expected": { + "isUnparseable": true, + "unparseableReasonContains": "heredocs inside command substitution are not supported" + }, + "notes": "Heredoc bodies inside command substitution remain fail-closed until their boundaries are modeled.", + "oracleExpectation": "OutOfScope" +} diff --git a/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json b/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json index 0acd950..2fe237c 100644 --- a/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json +++ b/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json @@ -4,6 +4,7 @@ { "id": "bash-simple-command-substitution", "concern": "Substitution is attached to its containing simple command", + "compatibilityProjectionLanded": true, "input": "rm \"$(find /tmp)\"", "current": { "isUnparseable": false, @@ -37,6 +38,7 @@ { "id": "bash-multiple-command-substitutions", "concern": "Sibling substitutions preserve authored order before their consumer", + "compatibilityProjectionLanded": true, "input": "printf '%s %s' \"$(first)\" \"$(second)\"", "current": { "isUnparseable": false }, "desired": { @@ -62,6 +64,7 @@ { "id": "bash-nested-command-substitutions", "concern": "Nested substitutions retain parentage and project innermost first", + "compatibilityProjectionLanded": true, "input": "printf '%s' \"$(echo \"$(whoami)\")\"", "current": { "isUnparseable": false }, "desired": { @@ -87,6 +90,7 @@ { "id": "bash-literal-substitution-spellings", "concern": "Quoted and escaped substitution-looking text does not execute", + "compatibilityProjectionLanded": true, "input": "printf '%s %s' '$(whoami)' \"\\$(id)\"", "current": { "isUnparseable": false }, "desired": { @@ -104,6 +108,7 @@ { "id": "bash-backtick-substitution-gated", "concern": "Legacy backtick execution remains fail closed until escape semantics are modeled", + "compatibilityProjectionLanded": true, "input": "echo `whoami`", "current": { "isUnparseable": false, @@ -123,6 +128,7 @@ { "id": "bash-command-name-substitution-gated", "concern": "Runtime-produced Bash command identity stays unparseable", + "compatibilityProjectionLanded": true, "input": "r$(printf m) file", "current": { "isUnparseable": true, "reasonContains": "dynamic Bash command identity" }, "desired": { @@ -143,6 +149,7 @@ { "id": "bash-command-substitution-isolated-cwd", "concern": "Bash substitution state is sequential inside and isolated outside", + "compatibilityProjectionLanded": true, "input": "printf '%s' \"$(cd /tmp; pwd)\"; cat relative.txt", "current": { "isUnparseable": false }, "desired": { diff --git a/tests/ShellSyntaxTree.Tests/Parsing/BashCommandParserTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/BashCommandParserTests.cs index 2d99fdb..5e873da 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/BashCommandParserTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/BashCommandParserTests.cs @@ -401,7 +401,8 @@ public void Multiple_redirects_on_one_clause() public void Redirect_target_is_dynamic_when_opaque_substitution() { var result = Parse("cmd > $(date +log)"); - var clause = Assert.Single(result.Clauses); + Assert.Equal(new[] { "date", "cmd" }, result.Clauses.Select(clause => clause.Verb.Joined)); + var clause = result.Clauses[1]; var redirect = Assert.Single(clause.Redirects); Assert.True(redirect.IsDynamicSkip); } @@ -424,7 +425,8 @@ public void Command_substitution_becomes_DynamicSkip_arg() { var result = Parse("rm $(find /tmp)"); Assert.False(result.IsUnparseable); - var clause = Assert.Single(result.Clauses); + Assert.Equal(new[] { "find", "rm" }, result.Clauses.Select(clause => clause.Verb.Joined)); + var clause = result.Clauses[1]; Assert.Equal(new[] { "rm" }, clause.Verb.Tokens); var arg = Assert.Single(clause.Args); Assert.Equal(ArgKind.DynamicSkip, arg.Kind); @@ -433,13 +435,13 @@ public void Command_substitution_becomes_DynamicSkip_arg() } [Fact] - public void Backtick_substitution_becomes_DynamicSkip_arg() + public void Backtick_substitution_is_unparseable_until_its_distinct_grammar_is_supported() { var result = Parse("rm `find /tmp`"); - Assert.False(result.IsUnparseable); - var clause = Assert.Single(result.Clauses); - var arg = Assert.Single(clause.Args); - Assert.Equal(ArgKind.DynamicSkip, arg.Kind); + Assert.True(result.IsUnparseable); + Assert.Empty(result.Clauses); + Assert.Empty(result.Commands); + Assert.Contains("backtick", result.UnparseableReason!); } // ---------------- Unparseable ---------------- diff --git a/tests/ShellSyntaxTree.Tests/Parsing/BashStructuralProjectionTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/BashStructuralProjectionTests.cs index b66bdd5..8fecf7b 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/BashStructuralProjectionTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/BashStructuralProjectionTests.cs @@ -312,18 +312,352 @@ public void Noncanonical_command_string_options_remain_outer_and_incomplete(stri Assert.IsType(Assert.Single(result.Syntax.Statements)); } + [Fact] + public void Command_substitution_is_attached_and_projected_before_its_consumer() + { + const string source = "rm \"$(find /tmp)\""; + + var result = Parse(source); + + Assert.False(result.IsUnparseable); + var outer = Assert.IsType(Assert.Single(result.Syntax.Statements)); + var substitution = Assert.Single(outer.Substitutions); + Assert.Equal(4, substitution.SourceStart); + Assert.Equal(12, substitution.SourceLength); + Assert.Equal(6, substitution.Body.SourceStart); + Assert.Equal(9, substitution.Body.SourceLength); + var inner = Assert.IsType(Assert.Single(substitution.Body.Statements)); + Assert.Equal(6, inner.SourceStart); + Assert.Equal(9, inner.SourceLength); + + Assert.Equal(new[] { "find", "rm" }, result.Commands.Select(CommandVerb)); + Assert.Equal(result.Clauses, result.Commands.Select(command => command.Clause)); + Assert.Same(inner.Clause, result.Commands[0].Clause); + Assert.Same(outer.Clause, result.Commands[1].Clause); + Assert.Equal(CommandOccurrenceRole.Substitution, result.Commands[0].ImmediateRole); + Assert.Equal(CommandOccurrenceRole.Ordinary, result.Commands[1].ImmediateRole); + Assert.All(result.Commands, command => Assert.True(command.IsComplete)); + + Assert.Equal(3, result.Commands[0].Ancestry.Count); + Assert.Equal(CommandAncestryRegion.Root, result.Commands[0].Ancestry[0].Region); + Assert.Equal(ShellSyntaxKind.CommandSubstitution, result.Commands[0].Ancestry[1].AncestorKind); + Assert.Equal(CommandAncestryRegion.Substitution, result.Commands[0].Ancestry[1].Region); + Assert.Equal(0, result.Commands[0].Ancestry[1].ChildIndex); + Assert.Equal(ShellSyntaxKind.Block, result.Commands[0].Ancestry[2].AncestorKind); + Assert.Equal(CommandAncestryRegion.Statement, result.Commands[0].Ancestry[2].Region); + + var authored = Assert.Single(outer.Clause.Args); + Assert.Equal("\"$(find /tmp)\"", authored.Raw); + Assert.Equal(ArgKind.DynamicSkip, authored.Kind); + } + + [Fact] + public void Multiple_and_nested_substitutions_preserve_parentage_and_order() + { + var siblings = Parse("printf '%s %s' \"$(first)\" \"$(second)\""); + + var siblingOuter = Assert.IsType( + Assert.Single(siblings.Syntax.Statements)); + Assert.Equal(2, siblingOuter.Substitutions.Count); + Assert.Equal(new[] { "first", "second", "printf" }, + siblings.Commands.Select(CommandVerb)); + Assert.Equal(0, siblings.Commands[0].Ancestry[1].ChildIndex); + Assert.Equal(1, siblings.Commands[1].Ancestry[1].ChildIndex); + + var nested = Parse("printf '%s' \"$(echo \"$(whoami)\")\""); + + var nestedOuter = Assert.IsType( + Assert.Single(nested.Syntax.Statements)); + var outerSubstitution = Assert.Single(nestedOuter.Substitutions); + var echo = Assert.IsType( + Assert.Single(outerSubstitution.Body.Statements)); + Assert.Single(echo.Substitutions); + Assert.Equal(new[] { "whoami", "echo", "printf" }, + nested.Commands.Select(CommandVerb)); + Assert.Equal(5, nested.Commands[0].Ancestry.Count); + Assert.Equal(3, nested.Commands[1].Ancestry.Count); + } + + [Theory] + [InlineData("printf '%s' $(id)")] + [InlineData("printf '%s' pre$(id)post")] + [InlineData("printf '%s' \"pre$(id)post\"")] + public void Supported_argument_forms_discover_substitution_and_preserve_dynamic_outer_value( + string source) + { + var result = Parse(source); + + Assert.False(result.IsUnparseable); + Assert.Equal(new[] { "id", "printf" }, result.Commands.Select(CommandVerb)); + Assert.All(result.Commands, command => Assert.True(command.IsComplete)); + var outer = Assert.IsType(Assert.Single(result.Syntax.Statements)); + Assert.Single(outer.Substitutions); + Assert.Contains(outer.Clause.Args, argument => argument.Kind == ArgKind.DynamicSkip); + } + + [Fact] + public void Redirect_substitution_is_visible_while_redirect_analysis_remains_incomplete() + { + var result = Parse("cat > \"$(mktemp)\""); + + Assert.False(result.IsUnparseable); + Assert.Equal(new[] { "mktemp", "cat" }, result.Commands.Select(CommandVerb)); + Assert.True(result.Commands[0].IsComplete); + Assert.False(result.Commands[1].IsComplete); + Assert.Single(result.Clauses[1].Redirects); + Assert.True(result.Clauses[1].Redirects[0].IsDynamicSkip); + } + + [Theory] + [InlineData("echo \"$(printf x # )\nid)\"")] + [InlineData("echo \"$(printf x \\\n# )\nid)\"")] + [InlineData("echo \"$(printf x \\\r\n# )\r\nid)\"")] + public void Comment_parenthesis_does_not_close_command_substitution_early(string source) + { + var result = Parse(source); + + Assert.False(result.IsUnparseable); + Assert.Equal(new[] { "printf x", "id", "echo" }, result.Commands.Select(CommandVerb)); + Assert.Equal( + new[] { CompoundOperator.None, CompoundOperator.Sequence, CompoundOperator.None }, + result.Clauses.Select(clause => clause.Operator)); + Assert.All(result.Commands, command => Assert.True(command.IsComplete)); + } + + [Fact] + public void Empty_command_substitution_is_preserved_without_inventing_a_command() + { + var result = Parse("printf '%s' \"$()\""); + + Assert.False(result.IsUnparseable); + Assert.Equal("printf", CommandVerb(Assert.Single(result.Commands))); + var command = Assert.IsType(Assert.Single(result.Syntax.Statements)); + var substitution = Assert.Single(command.Substitutions); + Assert.Empty(substitution.Body.Statements); + Assert.Contains(command.Clause.Args, argument => argument.Kind == ArgKind.DynamicSkip); + } + + [Fact] + public void Substitution_state_is_sequential_inside_and_isolated_from_outer_commands() + { + var result = Parse("printf '%s' \"$(cd /tmp; pwd)\"; cat relative.txt"); + + Assert.False(result.IsUnparseable); + Assert.Equal(new[] { "cd", "pwd", "printf", "cat" }, + result.Commands.Select(CommandVerb)); + Assert.Equal( + new[] + { + CompoundOperator.None, + CompoundOperator.Sequence, + CompoundOperator.None, + CompoundOperator.Sequence, + }, + result.Clauses.Select(clause => clause.Operator)); + Assert.Contains(result.Clauses[1].Args, + argument => argument.IsCwdAttribution && argument.Resolved == "/tmp"); + Assert.DoesNotContain(result.Clauses[2].Args, argument => argument.IsCwdAttribution); + Assert.Equal("/work/relative.txt", Assert.Single(result.Clauses[3].Args).Resolved); + } + + [Fact] + public void Substitution_in_later_list_item_does_not_invent_a_compound_operator() + { + var result = Parse("echo ok && rm $(find /tmp)"); + + Assert.Equal(new[] { "echo ok", "find", "rm" }, result.Commands.Select(CommandVerb)); + Assert.Equal( + new[] { CompoundOperator.None, CompoundOperator.None, CompoundOperator.AndIf }, + result.Clauses.Select(clause => clause.Operator)); + } + + [Fact] + public void Outer_and_inner_wrapper_substitutions_retain_distinct_provenance() + { + var outer = Parse("bash -c \"echo $(id)\""); + + Assert.False(outer.IsUnparseable); + Assert.Equal(new[] { "id", "bash" }, outer.Commands.Select(CommandVerb)); + Assert.True(outer.Commands[0].IsComplete); + Assert.False(outer.Commands[1].IsComplete); + Assert.NotNull(outer.Commands[0].Clause.Elements[0].SourceStart); + Assert.False(outer.Commands[0].Clause.IsCommandStringWrapped); + + var inner = Parse("bash -c 'echo $(id)'"); + + Assert.False(inner.IsUnparseable); + Assert.Equal(new[] { "id", "echo" }, inner.Commands.Select(CommandVerb)); + Assert.All(inner.Commands, command => Assert.True(command.Clause.IsCommandStringWrapped)); + Assert.All(inner.Commands.SelectMany(command => command.Clause.Elements), + element => Assert.Null(element.SourceStart)); + } + + [Theory] + [InlineData("$(printf rm) target")] + [InlineData("r$(printf m) target")] + [InlineData("\"$(printf rm)\" target")] + public void Command_name_substitution_fails_closed(string source) + { + var result = Parse(source); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + Assert.Contains("command", result.UnparseableReason!, System.StringComparison.OrdinalIgnoreCase); + } + [Theory] - [InlineData("rm $(find /tmp)")] - [InlineData("printf '%s' \"$(whoami)\"")] [InlineData("echo `whoami`")] - public void Undiscovered_command_substitution_keeps_outer_leaf_incomplete(string source) + [InlineData("echo \"pre`whoami`post\"")] + [InlineData("echo pre`whoami`post")] + public void Executable_legacy_backticks_fail_closed(string source) + { + var result = Parse(source); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + Assert.Contains("backtick", result.UnparseableReason!); + } + + [Theory] + [InlineData("printf '%s %s' '$(whoami)' \"\\$(id)\"")] + [InlineData("printf '%s %s' '`whoami`' \"\\`id\\`\"")] + public void Literal_substitution_spellings_do_not_create_occurrences(string source) { var result = Parse(source); Assert.False(result.IsUnparseable); var command = Assert.Single(result.Commands); - Assert.False(command.IsComplete); - Assert.Same(Assert.Single(result.Clauses), command.Clause); + Assert.Equal("printf", CommandVerb(command)); + Assert.True(command.IsComplete); + Assert.Empty(Assert.IsType( + Assert.Single(result.Syntax.Statements)).Substitutions); + } + + [Theory] + [InlineData("id & evil")] + [InlineData("id&evil")] + [InlineData("echo \"$(id & evil)\"")] + [InlineData("echo \"$(id&evil)\"")] + public void Background_lists_fail_closed_without_partial_projections(string source) + { + var result = Parse(source); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + Assert.Contains("background", result.UnparseableReason!); + } + + [Theory] + [InlineData("printf '%s' '&' \\&")] + [InlineData("echo \"$(printf '%s' '&')\"")] + public void Quoted_and_escaped_ampersands_remain_literal_data(string source) + { + var result = Parse(source); + + Assert.False(result.IsUnparseable); + Assert.All(result.Commands, command => Assert.True(command.IsComplete)); + } + + [Theory] + [InlineData("X=1 rm -rf /tmp/x")] + [InlineData("X+=1 rm -rf /tmp/x")] + [InlineData("X[0]=1 rm -rf /tmp/x")] + [InlineData("X[key]+=1 rm -rf /tmp/x")] + [InlineData("echo \"$(X=1 rm -rf /tmp/x)\"")] + [InlineData("echo \"$(X\\\n=1 rm -rf /tmp/x)\"")] + public void Assignment_prefix_commands_fail_closed(string source) + { + var result = Parse(source); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + Assert.Contains("assignment", result.UnparseableReason!); + } + + [Fact] + public void Line_continuation_inside_command_name_is_removed_before_analysis() + { + var result = Parse("r\\\nm target"); + + Assert.False(result.IsUnparseable); + var command = Assert.Single(result.Commands); + Assert.Equal("rm", CommandVerb(command)); + Assert.Equal("/work/target", Assert.Single(command.Clause.Args).Resolved); + } + + [Fact] + public void Hash_after_command_name_continuation_remains_part_of_the_word() + { + var result = Parse("echo \"$(r\\\n#suffix)\""); + + Assert.False(result.IsUnparseable); + Assert.Equal(new[] { "r#suffix", "echo" }, result.Commands.Select(CommandVerb)); + } + + [Fact] + public void Escaped_equals_does_not_create_an_assignment_prefix() + { + var result = Parse("X\\=1 value"); + + Assert.False(result.IsUnparseable); + Assert.StartsWith("X=1", CommandVerb(Assert.Single(result.Commands))); + } + + [Theory] + [InlineData("rm \"$(echo ok &&)\"")] + [InlineData("rm \"$(diff <(id))\"")] + [InlineData("rm \"$(echo `id`)\"")] + [InlineData("rm \"$(cat < command.Clause.Verb.Joined; + + private static string NestedSubstitutions(int depth) + { + var command = "id"; + for (var index = 0; index < depth; index++) + { + command = "echo $(" + command + ")"; + } + + return command; + } } diff --git a/tests/ShellSyntaxTree.Tests/Parsing/ResolverProvenanceTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/ResolverProvenanceTests.cs index 1e5e469..6b1e9f5 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/ResolverProvenanceTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/ResolverProvenanceTests.cs @@ -66,14 +66,14 @@ public void Bash_redirect_aggregates_all_adjacent_fragments() } [Fact] - public void Bash_command_substitution_inside_double_quotes_is_opaque() + public void Bash_backtick_substitution_inside_double_quotes_fails_closed() { - var argument = Assert.Single( - Assert.Single(Bash.Parse("cat \"`printf /etc/passwd`\"").Clauses).Args); + var parsed = Bash.Parse("cat \"`printf /etc/passwd`\""); - Assert.Equal(ArgKind.DynamicSkip, argument.Kind); - Assert.False(argument.IsPath); - Assert.Null(argument.Resolved); + Assert.True(parsed.IsUnparseable); + Assert.Empty(parsed.Clauses); + Assert.Empty(parsed.Commands); + Assert.Contains("backtick", parsed.UnparseableReason!); } [Fact] diff --git a/tests/ShellSyntaxTree.Tests/Parsing/ShellValueOracleTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/ShellValueOracleTests.cs index f1eb89b..92644cd 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/ShellValueOracleTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/ShellValueOracleTests.cs @@ -552,6 +552,34 @@ public void Dynamic_command_fragments_are_executable_identity() } } + [Fact] + public void Bash_continuation_and_comment_boundary_samples_are_valid() + { + if (!IsNativeBashAvailable()) + { + return; + } + + var sources = new[] + { + "echo \"$(printf x # )\nid)\"", + "echo \"$(printf x \\\n# )\nid)\"", + "echo \"$(printf x \\\r\n# )\r\nid)\"", + "r\\\nm target", + "echo \"$(r\\\n#suffix)\"", + "echo \"$(X\\\n=1 rm -rf /tmp/x)\"", + }; + foreach (var source in sources) + { + var result = RunUnchecked("bash", "-n", "-c", source); + + Assert.Equal(0, result.ExitCode); + Assert.True( + string.IsNullOrEmpty(result.StandardError), + $"bash syntax oracle wrote to stderr: {result.StandardError}"); + } + } + private static bool IsAvailable(string executable) { try