diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 5c2c55c..1fcc015 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -194,11 +194,19 @@ priorities. Executable corpus cases preserve the corrected v0.2 compatibility projection; unknown facts remain `DynamicSkip` or unparseable, while completely proved mixed fragments resolve exactly. -- [ ] Implement [issue #69](https://github.com/Aaronontheweb/ShellSyntaxTree/issues/69) - against the corrected fragment contract. Preserve raw, decoded, and span - facts plus unaffected classifications; explicitly document only - shell-oracle-proved compatibility corrections to false exact, `Glob`, - `Tilde`, provider, or path claims and avoidable `DynamicSkip` results. +- [x] Implement [issue #69](https://github.com/Aaronontheweb/ShellSyntaxTree/issues/69) + against the corrected fragment contract. One shell-neutral classifier + now aggregates the complete raw span, decoded value, next-token index, + and ordered `ShellValue` provenance supplied by explicit Bash and + PowerShell adapters. It distinguishes literal-only, typed expansion, + and opaque/computed runs without rescanning decoded text; missing lexer + provenance fails closed as `Opaque` in both adapters. Direct adapter + tests pin spans, maximal consumption, expansion identity, opaque cause, + and fallback behavior. The full Bash and PowerShell corpora prove the + extraction leaves raw, decoded, span, path, and `DynamicSkip` results + unchanged. It introduces no new compatibility correction; the + shell-oracle-proved corrections remain the ones documented in the + preceding provenance item. - [ ] Add the structural and command-occurrence projections for the existing grammar before enabling any control-flow construct. - [ ] Deliver Bash `for ... in` and PowerShell `foreach` as the first two diff --git a/openspec/changes/v0-3-structured-shell-analysis/tasks.md b/openspec/changes/v0-3-structured-shell-analysis/tasks.md index be741cd..8c5bf69 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/tasks.md +++ b/openspec/changes/v0-3-structured-shell-analysis/tasks.md @@ -15,8 +15,8 @@ - [x] 2.1 Implement shell-specific lexical fragment provenance that distinguishes literal, typed recognized-expansion, and opaque resolver input; retains transform eligibility, expansion identity, cardinality, and opaque cause; aggregates complete argument and redirect-target fragment runs; and passes explicit Bash-argument, Bash-redirect, PowerShell-native, cmdlet-Path, cmdlet-LiteralPath, and PowerShell-redirect resolver context without changing the public API. - [x] 2.2 Add paired Bash and PowerShell shell-oracle regressions for standalone escapes, adjacent escaped values, all-static mixed quoting, within-token escapes, genuine literal-plus-expandable values, adjacent and wildcard redirect targets, runtime special/positional/numeric/Unicode variables, incomplete and escaped-literal braced interpolation, Bash provider-looking literals, and PowerShell native-versus-cmdlet, Path-versus-LiteralPath, and redirect-context divergence; require exact compatibility path results when every fragment, binding fact, and required resolver fact is exact, otherwise fail closed. -- [ ] 2.3 Implement issue #69's shell-neutral native argument-fragment classifier with explicit Bash and PowerShell adapters that preserve the new provenance. -- [ ] 2.4 Prove raw spelling, decoded logical values, source spans, and unaffected classifications remain unchanged; document each oracle-proved false exact, `Glob`, `Tilde`, provider, path, or avoidable `DynamicSkip` compatibility correction. +- [x] 2.3 Implement issue #69's shell-neutral native argument-fragment classifier with explicit Bash and PowerShell adapters that preserve the new provenance. +- [x] 2.4 Prove raw spelling, decoded logical values, source spans, and unaffected classifications remain unchanged; document each oracle-proved false exact, `Glob`, `Tilde`, provider, path, or avoidable `DynamicSkip` compatibility correction. - [ ] 2.5 Audit duplicated Bash and PowerShell path-normalization helpers and extract only rules with identical shell semantics. - [ ] 2.6 Run Release build, full tests, header verification, and the adversarial security corpus for the completed preparation. diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs index 83137a4..62d84c0 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs @@ -514,7 +514,8 @@ private static List FilterSignificant(IReadOnlyList tokens if (filtered.Count > 0 && IsNativeArgumentFragment(filtered[filtered.Count - 1]) && IsNativeArgumentFragment(t) - && IsAdjacent(filtered[filtered.Count - 1], t)) + && IsAdjacent(filtered[filtered.Count - 1], t) + && !IsInlineNativeArgumentPrefix(filtered[filtered.Count - 1])) { var previous = filtered[filtered.Count - 1]; var previousValue = previous.ResolverValue @@ -1200,48 +1201,19 @@ private static void ExtractRedirectsAndArgs( // `--data="@request file"` / `--data=$(generate)`. if (NativeFlagSyntax.TrySplitEqualsPrefix( t.Value, out var adjacentFlagPart, out var adjacentValuePrefix) - && i + 1 < segmentTokens.Count - && IsAdjacent(t, segmentTokens[i + 1]) - && segmentTokens[i + 1].Kind is BashTokenKind.QuotedString - or BashTokenKind.OpaqueSubstitution) + && NativeArgumentFragmentClassifier.TryClassify( + source, + t.SourceStart, + t.SourceStart + t.SourceLength, + t.SourceStart + sourceRaw.IndexOf('=') + 1, + adjacentFlagPart + "=", + GetResolverValue(t, adjacentValuePrefix), + segmentTokens, + i + 1, + new BashNativeArgumentFragmentAdapter(), + out var fragmentClassification)) { - var valueStart = i + 1; - var valueEnd = valueStart; - var valueBuilder = new StringBuilder(adjacentValuePrefix); - var hasOpaqueFragment = false; - var allFragmentsSingleQuoted = adjacentValuePrefix.Length == 0; - var hasSingleQuotedFragment = false; - var hasNonSingleQuotedFragment = adjacentValuePrefix.Length > 0; - var hasSensitiveLiteralFragment = false; - var previousFragment = t; - while (valueEnd < segmentTokens.Count - && IsAdjacent(previousFragment, segmentTokens[valueEnd]) - && IsNativeArgumentFragment(segmentTokens[valueEnd])) - { - var fragment = segmentTokens[valueEnd]; - valueBuilder.Append(fragment.Value); - hasOpaqueFragment |= fragment.Kind == BashTokenKind.OpaqueSubstitution; - hasSingleQuotedFragment |= fragment.Kind == BashTokenKind.QuotedString - && fragment.IsSingleQuoted; - hasNonSingleQuotedFragment |= fragment.Kind != BashTokenKind.QuotedString - || !fragment.IsSingleQuoted; - hasSensitiveLiteralFragment |= fragment.Kind == BashTokenKind.QuotedString - && fragment.IsSingleQuoted - && NativeFlagSyntax.ContainsResolverSensitiveLiteralSyntax(fragment.Value); - allFragmentsSingleQuoted &= fragment.Kind == BashTokenKind.QuotedString - && fragment.IsSingleQuoted; - previousFragment = fragment; - valueEnd++; - } - - var lastValueToken = segmentTokens[valueEnd - 1]; - var adjacentValue = valueBuilder.ToString(); - var equalsOffset = SourceSlice(source, t).IndexOf('='); - var adjacentRawStart = t.SourceStart + equalsOffset + 1; - var adjacentRaw = source.Substring( - adjacentRawStart, - lastValueToken.SourceStart + lastValueToken.SourceLength - - adjacentRawStart); + var adjacentValue = fragmentClassification.DecodedValue; argList.Add(new Arg { Raw = adjacentFlagPart, @@ -1251,17 +1223,14 @@ private static void ExtractRedirectsAndArgs( }); Arg valueArg; - if (hasOpaqueFragment - || (hasSingleQuotedFragment - && hasNonSingleQuotedFragment - && hasSensitiveLiteralFragment) + if (fragmentClassification.HasOpaqueFragment || (verbKeyForFlagValuePaths is not null && BashPerVerbRules.ValueOfFlagIsOpaqueCommand( verbKeyForFlagValuePaths, adjacentFlagPart))) { valueArg = new Arg { - Raw = adjacentRaw, + Raw = fragmentClassification.ValueRaw, Kind = ArgKind.DynamicSkip, IsPath = false, }; @@ -1276,7 +1245,7 @@ private static void ExtractRedirectsAndArgs( adjacentValue, out adjacentValueForResolution); var adjacentResolverValue = GetResolverValue( - t, + fragmentClassification.ResolverValue, adjacentValueForResolution); var (adjacentKind, adjacentResolved, adjacentIsPath) = BashResolver.Resolve( adjacentResolverValue, @@ -1286,7 +1255,7 @@ private static void ExtractRedirectsAndArgs( ShellResolutionConsumer.BashArgument); valueArg = new Arg { - Raw = adjacentRaw, + Raw = fragmentClassification.ValueRaw, Resolved = adjacentResolved, Kind = adjacentKind, IsPath = adjacentIsPath, @@ -1295,16 +1264,13 @@ private static void ExtractRedirectsAndArgs( argList.Add(valueArg); elementList.Add(CreateCombinedElement( - source, - t, - lastValueToken, - adjacentFlagPart + "=" + adjacentValue, + fragmentClassification, precedingVerbTokenCount, valueArg.Kind, isFlag: true, valueArg.IsPath, valueArg.Resolved)); - i = valueEnd; + i = fragmentClassification.NextTokenIndex; continue; } @@ -1695,34 +1661,6 @@ private static ClauseElement CreateElement( Resolved = resolved, }; - private static ClauseElement CreateCombinedElement( - string source, - BashToken first, - BashToken last, - string value, - int precedingVerbTokenCount, - ArgKind kind, - bool isFlag, - bool isPath, - string? resolved) - { - var sourceStart = first.SourceStart; - var sourceEnd = last.SourceStart + last.SourceLength; - return new ClauseElement - { - Raw = source.Substring(sourceStart, sourceEnd - sourceStart), - Value = value, - Role = ClauseElementRole.Argument, - SourceStart = sourceStart, - SourceLength = sourceEnd - sourceStart, - PrecedingVerbElementCount = precedingVerbTokenCount, - Kind = kind, - IsFlag = isFlag, - IsPath = isPath, - Resolved = resolved, - }; - } - private static ClauseElement CreateRedirectElement( string source, BashToken redirectOperator, @@ -1753,6 +1691,11 @@ private static ShellValue GetResolverValue(BashToken token, string logicalValue) { var value = token.ResolverValue ?? ShellValue.Literal(token.Value, token.SourceStart, token.SourceLength); + return GetResolverValue(value, logicalValue); + } + + private static ShellValue GetResolverValue(ShellValue value, string logicalValue) + { if (string.Equals(value.Decoded, logicalValue, StringComparison.Ordinal)) { return value; @@ -1768,6 +1711,26 @@ private static ShellValue GetResolverValue(BashToken token, string logicalValue) return ShellValue.Opaque(logicalValue, ShellOpaqueCause.Unsupported); } + private static ClauseElement CreateCombinedElement( + NativeArgumentFragmentClassification classification, + int precedingVerbTokenCount, + ArgKind kind, + bool isFlag, + bool isPath, + string? resolved) => new() + { + Raw = classification.Raw, + Value = classification.DecodedArgument, + Role = ClauseElementRole.Argument, + SourceStart = classification.SourceStart, + SourceLength = classification.SourceLength, + PrecedingVerbElementCount = precedingVerbTokenCount, + Kind = kind, + IsFlag = isFlag, + IsPath = isPath, + Resolved = resolved, + }; + private static bool TrySplitInlineFlag( BashToken token, out string flagPart, out string valuePart) { @@ -1799,6 +1762,10 @@ token.Kind is BashTokenKind.Word or BashTokenKind.QuotedString or BashTokenKind.OpaqueSubstitution; + private static bool IsInlineNativeArgumentPrefix(BashToken token) => + token.Kind == BashTokenKind.Word + && NativeFlagSyntax.TrySplitEqualsPrefix(token.Value, out _, out _); + private static bool IsFdDupTarget(string value) { // Recognized shapes (POSIX `[n]>&word` / `[n]<&word`): diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashNativeArgumentFragmentAdapter.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashNativeArgumentFragmentAdapter.cs new file mode 100644 index 0000000..ff50f12 --- /dev/null +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashNativeArgumentFragmentAdapter.cs @@ -0,0 +1,44 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using ShellSyntaxTree.Internal.Bash.Lexing; +using ShellSyntaxTree.Internal.Parsing; +using ShellSyntaxTree.Internal.Resolving; + +namespace ShellSyntaxTree.Internal.Bash.Parsing; + +internal readonly struct BashNativeArgumentFragmentAdapter + : INativeArgumentFragmentAdapter +{ + public bool CanStart(BashToken token) => token.Kind is + BashTokenKind.Word + or BashTokenKind.QuotedString + or BashTokenKind.OpaqueSubstitution; + + public bool TryAdapt(BashToken token, out NativeArgumentFragment fragment) + { + if (token.Kind is not (BashTokenKind.Word + or BashTokenKind.QuotedString + or BashTokenKind.OpaqueSubstitution)) + { + fragment = default; + return false; + } + + var value = token.ResolverValue + ?? ShellValue.Opaque( + token.Value, + token.Kind == BashTokenKind.OpaqueSubstitution + ? ShellOpaqueCause.CommandSubstitution + : ShellOpaqueCause.Unsupported, + token.SourceStart, + token.SourceLength); + fragment = new NativeArgumentFragment( + value, + token.SourceStart, + token.SourceLength); + return true; + } +} diff --git a/src/ShellSyntaxTree/Internal/Parsing/NativeArgumentFragmentClassifier.cs b/src/ShellSyntaxTree/Internal/Parsing/NativeArgumentFragmentClassifier.cs new file mode 100644 index 0000000..0f4a2c7 --- /dev/null +++ b/src/ShellSyntaxTree/Internal/Parsing/NativeArgumentFragmentClassifier.cs @@ -0,0 +1,132 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using System.Collections.Generic; +using ShellSyntaxTree.Internal.Resolving; + +namespace ShellSyntaxTree.Internal.Parsing; + +/// +/// Aggregates one native argument from a parser-owned prefix and its adjacent +/// shell fragments. Shell adapters define fragment membership and fallback +/// provenance; this type owns only source-contiguous aggregation. +/// +internal static class NativeArgumentFragmentClassifier +{ + internal static bool TryClassify( + string source, + int argumentSourceStart, + int argumentPrefixSourceEnd, + int valueSourceStart, + string decodedArgumentPrefix, + ShellValue decodedValuePrefix, + IReadOnlyList tokens, + int firstFragmentIndex, + TAdapter adapter, + out NativeArgumentFragmentClassification classification) + where TAdapter : struct, INativeArgumentFragmentAdapter + { + classification = default; + if (firstFragmentIndex < 0 + || firstFragmentIndex >= tokens.Count + || !adapter.CanStart(tokens[firstFragmentIndex])) + { + return false; + } + + var resolverValues = new List { decodedValuePrefix }; + var previousSourceEnd = argumentPrefixSourceEnd; + var nextTokenIndex = firstFragmentIndex; + while (nextTokenIndex < tokens.Count + && adapter.TryAdapt(tokens[nextTokenIndex], out var fragment) + && fragment.SourceStart == previousSourceEnd) + { + resolverValues.Add(fragment.Value); + previousSourceEnd = fragment.SourceStart + fragment.SourceLength; + nextTokenIndex++; + } + + if (nextTokenIndex == firstFragmentIndex) + { + return false; + } + + var resolverValue = ShellValue.Concat(resolverValues); + classification = new NativeArgumentFragmentClassification( + Raw: source.Substring( + argumentSourceStart, + previousSourceEnd - argumentSourceStart), + ValueRaw: source.Substring( + valueSourceStart, + previousSourceEnd - valueSourceStart), + DecodedArgument: decodedArgumentPrefix + resolverValue.Decoded, + ResolverValue: resolverValue, + SourceStart: argumentSourceStart, + SourceLength: previousSourceEnd - argumentSourceStart, + NextTokenIndex: nextTokenIndex); + return true; + } +} + +internal interface INativeArgumentFragmentAdapter +{ + bool CanStart(TToken token); + + bool TryAdapt(TToken token, out NativeArgumentFragment fragment); +} + +internal readonly record struct NativeArgumentFragment( + ShellValue Value, + int SourceStart, + int SourceLength); + +internal readonly record struct NativeArgumentFragmentClassification( + string Raw, + string ValueRaw, + string DecodedArgument, + ShellValue ResolverValue, + int SourceStart, + int SourceLength, + int NextTokenIndex) +{ + internal string DecodedValue => ResolverValue.Decoded; + + internal bool HasOpaqueFragment => ResolverValue.HasOpaqueFragment; + + internal bool HasExpansionFragment + { + get + { + foreach (var fragment in ResolverValue.Fragments) + { + if (fragment.Kind == ShellValueFragmentKind.Expansion) + { + return true; + } + } + + return false; + } + } + + internal bool HasOpaqueOrComputedFragment => + HasOpaqueFragment || HasExpansionFragment; + + internal bool AllValueFragmentsLiteral + { + get + { + foreach (var fragment in ResolverValue.Fragments) + { + if (fragment.Kind != ShellValueFragmentKind.Literal) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs b/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs index 49c9ec3..c64a6c0 100644 --- a/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs +++ b/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs @@ -75,25 +75,4 @@ internal static bool TrySplitEqualsPrefix( return true; } - /// - /// Characters whose meaning changes when a shell fragment is literal - /// rather than expandable. Mixed-fragment values containing these must - /// safe-fail unless fragment-level provenance is retained. - /// - internal static bool ContainsResolverSensitiveLiteralSyntax(string value) - { - var transformedValue = value.Length > 0 && value[0] == '@' - ? value.Substring(1) - : value; - return value.IndexOf('$') >= 0 - || value.IndexOf('*') >= 0 - || value.IndexOf('?') >= 0 - || value.IndexOf('[') >= 0 - || transformedValue.StartsWith("~", System.StringComparison.Ordinal) - || transformedValue.StartsWith( - "filesystem::", System.StringComparison.OrdinalIgnoreCase) - || transformedValue.StartsWith( - "Microsoft.PowerShell.Core\\FileSystem::", - System.StringComparison.OrdinalIgnoreCase); - } } diff --git a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs index 111db11..9e43599 100644 --- a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs @@ -1203,49 +1203,19 @@ private static ArgResult ExtractArgsAndRedirects( // opaque value together in the provenance view. if (NativeFlagSyntax.TrySplitEqualsPrefix( raw, out var adjacentFlagPart, out var adjacentValuePrefix) - && i + 1 < body.Count - && IsAdjacent(t, body[i + 1]) - && IsNativeArgumentFragment(body[i + 1])) + && NativeArgumentFragmentClassifier.TryClassify( + source, + t.SourceStart, + t.SourceStart + t.SourceLength, + t.SourceStart + SourceSlice(source, t).IndexOf('=') + 1, + adjacentFlagPart + "=", + GetResolverValue(t, adjacentValuePrefix), + body, + i + 1, + new PwshNativeArgumentFragmentAdapter(), + out var fragmentClassification)) { - var valueStart = i + 1; - var valueEnd = valueStart; - var valueBuilder = new StringBuilder(adjacentValuePrefix); - var hasOpaqueFragment = false; - var previousFragment = t; - while (valueEnd < body.Count - && IsAdjacent(previousFragment, body[valueEnd]) - && IsNativeArgumentFragment(body[valueEnd])) - { - var fragment = body[valueEnd]; - valueBuilder.Append(fragment.Value); - hasOpaqueFragment |= fragment.Kind != PwshTokenKind.Word - && fragment.Kind != PwshTokenKind.QuotedString; - previousFragment = fragment; - valueEnd++; - } - - var lastValueToken = body[valueEnd - 1]; - var adjacentValue = valueBuilder.ToString(); - var prefixResolverValue = GetResolverValue(t, adjacentValuePrefix); - var resolverFragments = new List { prefixResolverValue }; - for (var fragmentIndex = valueStart; fragmentIndex < valueEnd; fragmentIndex++) - { - var fragmentToken = body[fragmentIndex]; - resolverFragments.Add(fragmentToken.ResolverValue - ?? ShellValue.Opaque( - fragmentToken.Value, - ShellOpaqueCause.Unsupported, - fragmentToken.SourceStart, - fragmentToken.SourceLength)); - } - - var adjacentResolverValue = ShellValue.Concat(resolverFragments); - var equalsOffset = SourceSlice(source, t).IndexOf('='); - var adjacentRawStart = t.SourceStart + equalsOffset + 1; - var adjacentRaw = source.Substring( - adjacentRawStart, - lastValueToken.SourceStart + lastValueToken.SourceLength - - adjacentRawStart); + var adjacentValue = fragmentClassification.DecodedValue; args.Add(new Arg { Raw = adjacentFlagPart, @@ -1254,15 +1224,14 @@ private static ArgResult ExtractArgsAndRedirects( }); Arg valueArg; - if (hasOpaqueFragment - || adjacentResolverValue.HasOpaqueFragmentOtherThan( + if (fragmentClassification.ResolverValue.HasOpaqueFragmentOtherThan( ShellOpaqueCause.PowerShellExpressionSuffix) || BashPerVerbRules.ValueOfFlagIsOpaqueCommand( verbKey, adjacentFlagPart)) { valueArg = new Arg { - Raw = adjacentRaw, + Raw = fragmentClassification.ValueRaw, Kind = ArgKind.DynamicSkip, IsPath = false, }; @@ -1276,7 +1245,7 @@ private static ArgResult ExtractArgsAndRedirects( adjacentValue, out adjacentValueForResolution); var effectiveResolverValue = GetResolverValue( - adjacentResolverValue, + fragmentClassification.ResolverValue, adjacentValueForResolution, preserveLiteralPrefixBoundary: true); var (kind, resolved, isPath) = PwshResolver.Resolve( @@ -1287,7 +1256,7 @@ private static ArgResult ExtractArgsAndRedirects( ShellResolutionConsumer.PowerShellNativeArgument); valueArg = new Arg { - Raw = adjacentRaw, + Raw = fragmentClassification.ValueRaw, Kind = kind, Resolved = resolved, IsPath = isPath, @@ -1296,16 +1265,13 @@ private static ArgResult ExtractArgsAndRedirects( args.Add(valueArg); elements.Add(CreateCombinedElement( - source, - t, - lastValueToken, - adjacentFlagPart + "=" + adjacentValue, + fragmentClassification, precedingVerbTokenCount, valueArg.Kind, isFlag: true, valueArg.IsPath, valueArg.Resolved)); - i = valueEnd - 1; + i = fragmentClassification.NextTokenIndex - 1; continue; } @@ -1717,6 +1683,26 @@ private static ClauseElement CreateElement( Resolved = resolved, }; + private static ClauseElement CreateCombinedElement( + NativeArgumentFragmentClassification classification, + int precedingVerbTokenCount, + ArgKind kind, + bool isFlag, + bool isPath, + string? resolved) => new() + { + Raw = classification.Raw, + Value = classification.DecodedArgument, + Role = ClauseElementRole.Argument, + SourceStart = classification.SourceStart, + SourceLength = classification.SourceLength, + PrecedingVerbElementCount = precedingVerbTokenCount, + Kind = kind, + IsFlag = isFlag, + IsPath = isPath, + Resolved = resolved, + }; + private static ClauseElement CreateCombinedElement( string source, PwshToken first, diff --git a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshNativeArgumentFragmentAdapter.cs b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshNativeArgumentFragmentAdapter.cs new file mode 100644 index 0000000..dffb53c --- /dev/null +++ b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshNativeArgumentFragmentAdapter.cs @@ -0,0 +1,46 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using ShellSyntaxTree.Internal.Parsing; +using ShellSyntaxTree.Internal.Pwsh.Lexing; +using ShellSyntaxTree.Internal.Resolving; + +namespace ShellSyntaxTree.Internal.Pwsh.Parsing; + +internal readonly struct PwshNativeArgumentFragmentAdapter + : INativeArgumentFragmentAdapter +{ + public bool CanStart(PwshToken token) => token.Kind is + PwshTokenKind.Word + or PwshTokenKind.QuotedString + or PwshTokenKind.ScriptBlock + or PwshTokenKind.Subexpression + or PwshTokenKind.Splat; + + public bool TryAdapt(PwshToken token, out NativeArgumentFragment fragment) + { + if (token.Kind is not (PwshTokenKind.Word + or PwshTokenKind.QuotedString + or PwshTokenKind.ScriptBlock + or PwshTokenKind.Subexpression + or PwshTokenKind.Splat)) + { + fragment = default; + return false; + } + + var value = token.ResolverValue + ?? ShellValue.Opaque( + token.Value, + ShellOpaqueCause.Unsupported, + token.SourceStart, + token.SourceLength); + fragment = new NativeArgumentFragment( + value, + token.SourceStart, + token.SourceLength); + return true; + } +} diff --git a/tests/ShellSyntaxTree.Tests/Parsing/NativeArgumentFragmentClassifierTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/NativeArgumentFragmentClassifierTests.cs new file mode 100644 index 0000000..65a514a --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Parsing/NativeArgumentFragmentClassifierTests.cs @@ -0,0 +1,249 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using ShellSyntaxTree.Internal.Bash.Lexing; +using ShellSyntaxTree.Internal.Bash.Parsing; +using ShellSyntaxTree.Internal.Parsing; +using ShellSyntaxTree.Internal.Pwsh.Lexing; +using ShellSyntaxTree.Internal.Pwsh.Parsing; +using ShellSyntaxTree.Internal.Resolving; +using Xunit; + +namespace ShellSyntaxTree.Tests.Parsing; + +public class NativeArgumentFragmentClassifierTests +{ + [Fact] + public void Bash_adapter_preserves_complete_literal_run() + { + const string source = "--data=@request' file'\".json\" tail"; + var tokens = BashLexer.Tokenize(source); + var prefixIndex = FindToken(tokens, token => token.Value == "--data=@request"); + var prefix = tokens[prefixIndex]; + var equalsOffset = prefix.Value.IndexOf('='); + var prefixValue = Assert.IsType(prefix.ResolverValue) + .Slice(equalsOffset + 1); + + var classified = NativeArgumentFragmentClassifier.TryClassify( + source, + prefix.SourceStart, + prefix.SourceStart + prefix.SourceLength, + prefix.SourceStart + equalsOffset + 1, + "--data=", + prefixValue, + tokens, + prefixIndex + 1, + new BashNativeArgumentFragmentAdapter(), + out var result); + + Assert.True(classified); + Assert.Equal("--data=@request' file'\".json\"", result.Raw); + Assert.Equal("@request' file'\".json\"", result.ValueRaw); + Assert.Equal("--data=@request file.json", result.DecodedArgument); + Assert.Equal("@request file.json", result.DecodedValue); + Assert.Equal(0, result.SourceStart); + Assert.Equal(result.Raw.Length, result.SourceLength); + Assert.Equal(BashTokenKind.Whitespace, tokens[result.NextTokenIndex].Kind); + Assert.True(result.AllValueFragmentsLiteral); + Assert.False(result.HasExpansionFragment); + Assert.False(result.HasOpaqueFragment); + Assert.False(result.HasOpaqueOrComputedFragment); + } + + [Fact] + public void PowerShell_adapter_preserves_complete_literal_run() + { + const string source = "--data=@request' file'\".json\" tail"; + var tokens = PwshLexer.Tokenize(source); + var prefixIndex = FindToken(tokens, token => token.Value == "--data=@request"); + var prefix = tokens[prefixIndex]; + var equalsOffset = prefix.Value.IndexOf('='); + var prefixValue = Assert.IsType(prefix.ResolverValue) + .Slice(equalsOffset + 1); + + var classified = NativeArgumentFragmentClassifier.TryClassify( + source, + prefix.SourceStart, + prefix.SourceStart + prefix.SourceLength, + prefix.SourceStart + equalsOffset + 1, + "--data=", + prefixValue, + tokens, + prefixIndex + 1, + new PwshNativeArgumentFragmentAdapter(), + out var result); + + Assert.True(classified); + Assert.Equal("--data=@request' file'\".json\"", result.Raw); + Assert.Equal("@request' file'\".json\"", result.ValueRaw); + Assert.Equal("--data=@request file.json", result.DecodedArgument); + Assert.Equal("@request file.json", result.DecodedValue); + Assert.Equal(0, result.SourceStart); + Assert.Equal(result.Raw.Length, result.SourceLength); + Assert.Equal(PwshTokenKind.Whitespace, tokens[result.NextTokenIndex].Kind); + Assert.True(result.AllValueFragmentsLiteral); + Assert.False(result.HasExpansionFragment); + Assert.False(result.HasOpaqueFragment); + Assert.False(result.HasOpaqueOrComputedFragment); + } + + [Fact] + public void Bash_adapter_preserves_typed_expansion_and_opaque_cause() + { + const string expansionSource = "--output=$HOME\".json\" tail"; + var expansion = ClassifyBash(expansionSource, "--output=$HOME", "--output="); + + var home = Assert.Single( + expansion.ResolverValue.Fragments, + fragment => fragment.Kind == ShellValueFragmentKind.Expansion); + Assert.Equal( + new ShellExpansionReference(ShellExpansionKind.Variable, "HOME"), + home.Expansion); + Assert.False(expansion.AllValueFragmentsLiteral); + Assert.True(expansion.HasExpansionFragment); + Assert.False(expansion.HasOpaqueFragment); + Assert.True(expansion.HasOpaqueOrComputedFragment); + + const string opaqueSource = "--data=@$(generate) tail"; + var opaque = ClassifyBash(opaqueSource, "--data=@", "--data="); + + Assert.False(opaque.HasExpansionFragment); + Assert.True(opaque.HasOpaqueFragment); + Assert.True(opaque.HasOpaqueOrComputedFragment); + Assert.Contains( + opaque.ResolverValue.Fragments, + fragment => fragment.OpaqueCause == ShellOpaqueCause.CommandSubstitution); + } + + [Fact] + public void PowerShell_adapter_preserves_typed_expansion_and_opaque_cause() + { + const string expansionSource = "--output=$HOME\".json\" tail"; + var expansion = ClassifyPowerShell( + expansionSource, + "--output=$HOME", + "--output="); + + var home = Assert.Single( + expansion.ResolverValue.Fragments, + fragment => fragment.Kind == ShellValueFragmentKind.Expansion); + Assert.Equal( + new ShellExpansionReference(ShellExpansionKind.Variable, "HOME"), + home.Expansion); + Assert.False(expansion.AllValueFragmentsLiteral); + Assert.True(expansion.HasExpansionFragment); + Assert.False(expansion.HasOpaqueFragment); + Assert.True(expansion.HasOpaqueOrComputedFragment); + + const string opaqueSource = "--data=@$(Get-Item x) tail"; + var opaque = ClassifyPowerShell(opaqueSource, "--data=@", "--data="); + + Assert.False(opaque.HasExpansionFragment); + Assert.True(opaque.HasOpaqueFragment); + Assert.True(opaque.HasOpaqueOrComputedFragment); + Assert.Contains( + opaque.ResolverValue.Fragments, + fragment => fragment.OpaqueCause == ShellOpaqueCause.PowerShellSubexpression); + } + + [Fact] + public void Adapters_fail_closed_when_lexer_provenance_is_missing() + { + var bashToken = new BashToken( + BashTokenKind.Word, + "literal", + null, + 0, + 7, + null); + Assert.True(new BashNativeArgumentFragmentAdapter().TryAdapt( + bashToken, + out var bashFragment)); + var bashValue = Assert.Single(bashFragment.Value.Fragments); + Assert.Equal(ShellValueFragmentKind.Opaque, bashValue.Kind); + Assert.Equal(ShellOpaqueCause.Unsupported, bashValue.OpaqueCause); + + var powerShellToken = new PwshToken( + PwshTokenKind.Word, + "literal", + null, + 0, + 7, + null); + Assert.True(new PwshNativeArgumentFragmentAdapter().TryAdapt( + powerShellToken, + out var powerShellFragment)); + var powerShellValue = Assert.Single(powerShellFragment.Value.Fragments); + Assert.Equal(ShellValueFragmentKind.Opaque, powerShellValue.Kind); + Assert.Equal(ShellOpaqueCause.Unsupported, powerShellValue.OpaqueCause); + } + + private static NativeArgumentFragmentClassification ClassifyBash( + string source, + string prefixText, + string decodedArgumentPrefix) + { + var tokens = BashLexer.Tokenize(source); + var prefixIndex = FindToken(tokens, token => token.Value == prefixText); + var prefix = tokens[prefixIndex]; + var equalsOffset = prefix.Value.IndexOf('='); + var prefixValue = Assert.IsType(prefix.ResolverValue) + .Slice(equalsOffset + 1); + Assert.True(NativeArgumentFragmentClassifier.TryClassify( + source, + prefix.SourceStart, + prefix.SourceStart + prefix.SourceLength, + prefix.SourceStart + equalsOffset + 1, + decodedArgumentPrefix, + prefixValue, + tokens, + prefixIndex + 1, + new BashNativeArgumentFragmentAdapter(), + out var result)); + return result; + } + + private static NativeArgumentFragmentClassification ClassifyPowerShell( + string source, + string prefixText, + string decodedArgumentPrefix) + { + var tokens = PwshLexer.Tokenize(source); + var prefixIndex = FindToken(tokens, token => token.Value == prefixText); + var prefix = tokens[prefixIndex]; + var equalsOffset = prefix.Value.IndexOf('='); + var prefixValue = Assert.IsType(prefix.ResolverValue) + .Slice(equalsOffset + 1); + Assert.True(NativeArgumentFragmentClassifier.TryClassify( + source, + prefix.SourceStart, + prefix.SourceStart + prefix.SourceLength, + prefix.SourceStart + equalsOffset + 1, + decodedArgumentPrefix, + prefixValue, + tokens, + prefixIndex + 1, + new PwshNativeArgumentFragmentAdapter(), + out var result)); + return result; + } + + private static int FindToken( + IReadOnlyList tokens, + Func predicate) + { + for (var index = 0; index < tokens.Count; index++) + { + if (predicate(tokens[index])) + { + return index; + } + } + + throw new Xunit.Sdk.XunitException("Expected token was not emitted."); + } +}