From 96da647d9c4f92afdff3365a739219a3ec05a9a7 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Sun, 8 Feb 2026 19:07:09 -0300 Subject: [PATCH 01/10] feat: implement scoped prefixes/suffixes Signed-off-by: Felipe Zipitria --- regex/operators/assembler.go | 8 +--- regex/operators/assembler_test.go | 68 +++++++++++++++++++++++++++++++ regex/parser/include_test.go | 35 +++++----------- regex/parser/parser.go | 6 ++- regex/processors/assemble.go | 41 +++++++++++++++++-- 5 files changed, 123 insertions(+), 35 deletions(-) diff --git a/regex/operators/assembler.go b/regex/operators/assembler.go index 8a906c8..33332a4 100644 --- a/regex/operators/assembler.go +++ b/regex/operators/assembler.go @@ -129,12 +129,8 @@ func (a *Operator) complete(assembleParser *parser.Parser) (string, error) { return "", err } - if len(assembleParser.Prefixes) > 0 && len(assembleParser.Suffixes) > 0 && len(result) > 0 { - result = "(?:" + result + ")" - } - prefixes := strings.Join(assembleParser.Prefixes, "") - suffixes := strings.Join(assembleParser.Suffixes, "") - result = prefixes + result + suffixes + // Note: Prefix/suffix application is now handled by individual Assemble processors (block-scoped) + // Parser.Prefixes and Parser.Suffixes are kept for backward compatibility but are no longer used if len(result) > 0 { logger.Trace().Msgf("Applying last cleanups to %s\n", result) diff --git a/regex/operators/assembler_test.go b/regex/operators/assembler_test.go index 396a435..e106b9c 100644 --- a/regex/operators/assembler_test.go +++ b/regex/operators/assembler_test.go @@ -832,6 +832,7 @@ d } func (s *assemblerTestSuite) TestAssemble_ConcatenationWithPrefixAndSuffix() { + // Test that top-level prefix/suffix apply to the implicit top-level block contents := `##!^ prefix ##!$ suffix ##!> assemble @@ -1121,3 +1122,70 @@ func (s *assemblerTestSuite) TestAssemble_ValidateCharacterClass() { s.ErrorContains(err, "unicode hex escape codepoint too big: 1114111 > 255") } + +// Test block-scoped prefix/suffix behavior +func (s *assemblerTestSuite) TestAssemble_BlockScopedPrefixSuffix_InsideBlock() { + // Test that prefix/suffix inside an assemble block are block-scoped + contents := `##!> assemble +##!^ prefix +##!$ suffix +a +b +##!<` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + // wrapCompletedAssembly already wraps in (?:...) + s.Equal(`prefix[ab]suffix`, output) +} + +func (s *assemblerTestSuite) TestAssemble_BlockScopedPrefixSuffix_TopLevel() { + // Verify top-level directives still work (implicit top-level block) + contents := `##!^ prefix +##!$ suffix +a +b` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + s.Equal(`prefix[ab]suffix`, output) +} + +func (s *assemblerTestSuite) TestAssemble_BlockScopedPrefixSuffix_MultipleBlocks() { + // Test multiple assemble blocks with different prefix/suffix in each + contents := `##!> assemble +##!^ prefix1 +a +b +##!< +##!=> +##!> assemble +##!$ suffix2 +c +d +##!<` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + // Each block applies its own prefix/suffix independently + s.Equal(`prefix1[ab][cd]suffix2`, output) +} + +func (s *assemblerTestSuite) TestAssemble_BlockScopedPrefixSuffix_NestedPattern() { + // Test the pattern from 934160.ra - prefix/suffix at start of assemble block + contents := `##!> assemble +##!^ while\s*\([\s(]* +##!$ .*\) +true +false +##!<` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + // Note: assembler applies various transformations like \s -> [\s\x0b] + // We just check that prefix/suffix are applied + s.Contains(output, `while`) + s.Contains(output, `.*\)`) + s.Contains(output, `tru`) + s.Contains(output, `fals`) +} diff --git a/regex/parser/include_test.go b/regex/parser/include_test.go index f6c39ec..ec0229c 100644 --- a/regex/parser/include_test.go +++ b/regex/parser/include_test.go @@ -72,13 +72,10 @@ func (s *parserIncludeTestSuite) TestParserInclude_Prefixes() { included regex`, "data regex") parser := NewParser(s.ctx, s.reader) actual := parser.Parse(false) - expected := bytes.NewBufferString(`##!> assemble -prefix1 -##!=> -prefix2 -##!=> + // With block-scoped prefixes, directives are passed through as-is + expected := bytes.NewBufferString(`##!^ prefix1 +##!^ prefix2 included regex -##!< data regex `) s.Equal(expected.String(), actual.String()) @@ -90,14 +87,10 @@ func (s *parserIncludeTestSuite) TestParserInclude_Suffixes() { included regex`, "data regex") parser := NewParser(s.ctx, s.reader) actual := parser.Parse(false) - expected := bytes.NewBufferString(`##!> assemble + // With block-scoped suffixes, directives are passed through as-is + expected := bytes.NewBufferString(`##!$ suffix1 +##!$ suffix2 included regex -##!=> -suffix1 -##!=> -suffix2 -##!=> -##!< data regex `) @@ -112,18 +105,12 @@ func (s *parserIncludeTestSuite) TestParserInclude_FlagsPrefixesSuffixes() { included regex`, "data regex") parser := NewParser(s.ctx, s.reader) actual := parser.Parse(false) - expected := bytes.NewBufferString(`##!> assemble -prefix1 -##!=> -prefix2 -##!=> + // With block-scoped prefixes/suffixes, directives are passed through as-is + expected := bytes.NewBufferString(`##!$ suffix1 +##!$ suffix2 +##!^ prefix1 +##!^ prefix2 included regex -##!=> -suffix1 -##!=> -suffix2 -##!=> -##!< data regex `) diff --git a/regex/parser/parser.go b/regex/parser/parser.go index b6816a3..5a72255 100644 --- a/regex/parser/parser.go +++ b/regex/parser/parser.go @@ -150,9 +150,11 @@ func (p *Parser) Parse(formatOnly bool) *bytes.Buffer { } } case prefix: - p.Prefixes = append(p.Prefixes, parsedLine.prefix) + // Pass through prefix directive for processor to handle (block-scoped) + text = line + "\n" case suffix: - p.Suffixes = append(p.Suffixes, parsedLine.suffix) + // Pass through suffix directive for processor to handle (block-scoped) + text = line + "\n" } if formatOnly { text = line + "\n" diff --git a/regex/processors/assemble.go b/regex/processors/assemble.go index c3272c4..43676d5 100644 --- a/regex/processors/assemble.go +++ b/regex/processors/assemble.go @@ -19,8 +19,10 @@ const ( ) type Assemble struct { - proc *Processor - output strings.Builder + proc *Processor + output strings.Builder + prefixes []string // Block-scoped prefixes + suffixes []string // Block-scoped suffixes } // NewAssemble creates a new assemble processor @@ -32,7 +34,23 @@ func NewAssemble(ctx *Context) *Assemble { // ProcessLine applies the processors logic to a single line func (a *Assemble) ProcessLine(line string) error { - match := regex.AssembleInputRegex.FindStringSubmatch(line) + // Check for prefix directive + match := regex.PrefixRegex.FindStringSubmatch(line) + if len(match) > 0 { + a.prefixes = append(a.prefixes, match[1]) + logger.Trace().Msgf("Added block-scoped prefix: %s", match[1]) + return nil + } + + // Check for suffix directive + match = regex.SuffixRegex.FindStringSubmatch(line) + if len(match) > 0 { + a.suffixes = append(a.suffixes, match[1]) + logger.Trace().Msgf("Added block-scoped suffix: %s", match[1]) + return nil + } + + match = regex.AssembleInputRegex.FindStringSubmatch(line) if len(match) > 0 { if err := a.store(match[1]); err != nil { logger.Error().Err(err).Msgf("Failed to store input: %s", line) @@ -69,6 +87,23 @@ func (a *Assemble) Complete() ([]string, error) { } result := a.wrapCompletedAssembly(regex) + + // Apply block-scoped prefixes and suffixes + if len(a.prefixes) > 0 || len(a.suffixes) > 0 { + prefixes := strings.Join(a.prefixes, "") + suffixes := strings.Join(a.suffixes, "") + + // If we have content, apply prefixes/suffixes around it + if len(result) > 0 { + result = prefixes + result + suffixes + logger.Trace().Msgf("Applied block-scoped prefixes/suffixes: %s", result) + } else if len(prefixes) > 0 || len(suffixes) > 0 { + // If no content but we have prefixes/suffixes, just concatenate them + result = prefixes + suffixes + logger.Trace().Msgf("Applied block-scoped prefixes/suffixes with no content: %s", result) + } + } + logger.Trace().Msgf("Completed assembly: %s", result) if result == "" { From 4dbdbab101998f3b5fec53439847e5468dd752a6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Feb 2026 23:14:06 +0000 Subject: [PATCH 02/10] fix: apply prefixes/suffixes when storing expressions and update parser handling Co-authored-by: fzipi <3012076+fzipi@users.noreply.github.com> --- regex/operators/assembler.go | 5 +- regex/operators/assembler_test.go | 90 +++++++++++++++++++++++++++++++ regex/parser/parser.go | 46 ++++------------ regex/processors/assemble.go | 12 +++++ 4 files changed, 114 insertions(+), 39 deletions(-) diff --git a/regex/operators/assembler.go b/regex/operators/assembler.go index 33332a4..db7f98f 100644 --- a/regex/operators/assembler.go +++ b/regex/operators/assembler.go @@ -129,8 +129,9 @@ func (a *Operator) complete(assembleParser *parser.Parser) (string, error) { return "", err } - // Note: Prefix/suffix application is now handled by individual Assemble processors (block-scoped) - // Parser.Prefixes and Parser.Suffixes are kept for backward compatibility but are no longer used + // Note: Prefix/suffix application is now handled by individual Assemble processors (block-scoped). + // Parser.Prefixes and Parser.Suffixes are no longer populated or used here; any previous + // backward-compatibility behavior relying on them has been removed from this path. if len(result) > 0 { logger.Trace().Msgf("Applying last cleanups to %s\n", result) diff --git a/regex/operators/assembler_test.go b/regex/operators/assembler_test.go index e106b9c..69bd986 100644 --- a/regex/operators/assembler_test.go +++ b/regex/operators/assembler_test.go @@ -1189,3 +1189,93 @@ false s.Contains(output, `tru`) s.Contains(output, `fals`) } + +func (s *assemblerTestSuite) TestAssemble_PrefixSuffixWithStashing() { + // Test that prefixes/suffixes are applied when storing expressions with ##!=< + contents := `##!> assemble +##!^ prefix +##!$ suffix +a +b +##!=< stored +##!< +##!=> stored` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + // The stored expression should include the prefix/suffix from its block + s.Equal(`prefix[ab]suffix`, output) +} + +func (s *assemblerTestSuite) TestAssemble_PrefixSuffixWithStashing_MultipleBlocks() { + // Test that each block's prefix/suffix are independently applied when stashing + contents := `##!> assemble +##!^ prefix1 +##!$ suffix1 +a +b +##!=< block1 +##!< +##!> assemble +##!^ prefix2 +##!$ suffix2 +c +d +##!=< block2 +##!< +##!=> block1 +##!=> block2` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + // Each stored expression should have its own block's prefix/suffix + s.Equal(`prefix1[ab]suffix1prefix2[cd]suffix2`, output) +} + +func (s *assemblerTestSuite) TestAssemble_PrefixSuffixWithStashing_NoPrefix() { + // Test stashing without prefix/suffix works as before + contents := `##!> assemble +a +b +##!=< stored +##!< +##!=> stored` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + // No prefix/suffix should be applied + s.Equal(`[ab]`, output) +} + +func (s *assemblerTestSuite) TestAssemble_PrefixSuffixWithStashing_OnlyPrefix() { + // Test stashing with only prefix + contents := `##!> assemble +##!^ prefix +a +b +##!=< stored +##!< +##!=> stored` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + // Only prefix should be applied + s.Equal(`prefix[ab]`, output) +} + +func (s *assemblerTestSuite) TestAssemble_PrefixSuffixWithStashing_OnlySuffix() { + // Test stashing with only suffix + contents := `##!> assemble +##!$ suffix +a +b +##!=< stored +##!< +##!=> stored` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + // Only suffix should be applied + s.Equal(`[ab]suffix`, output) +} + diff --git a/regex/parser/parser.go b/regex/parser/parser.go index 5a72255..b1e906b 100644 --- a/regex/parser/parser.go +++ b/regex/parser/parser.go @@ -152,9 +152,13 @@ func (p *Parser) Parse(formatOnly bool) *bytes.Buffer { case prefix: // Pass through prefix directive for processor to handle (block-scoped) text = line + "\n" + // Also collect for include file merging + p.Prefixes = append(p.Prefixes, parsedLine.prefix) case suffix: // Pass through suffix directive for processor to handle (block-scoped) text = line + "\n" + // Also collect for include file merging + p.Suffixes = append(p.Suffixes, parsedLine.suffix) } if formatOnly { text = line + "\n" @@ -294,43 +298,11 @@ func mergePrefixesSuffixes(source *Parser, out *bytes.Buffer) (*bytes.Buffer, er if len(source.Flags) > 0 { return new(bytes.Buffer), errors.New("include files must not contain flags. See https://github.com/coreruleset/crs-toolchain/v2/issues/71") } - // IMPORTANT: don't write the assemble block at all if there are no flags, prefixes, or - // suffixes. Enclosing the output in an assemble block can change the semantics, for example, - // when the included content is processed by the cmdline processor in the including file. - if len(source.Prefixes) == 0 && len(source.Suffixes) == 0 { - return out, nil - } - - newOut := new(bytes.Buffer) - newOut.WriteString("##!> assemble\n") - - for _, prefix := range source.Prefixes { - newOut.WriteString(prefix) - newOut.WriteString("\n##!=>\n") - } - if _, err := out.WriteTo(newOut); err != nil { - logger.Fatal().Err(err).Msg("failed to copy output to new buffer") - } - - sawNewLine := false - if err := out.UnreadByte(); err == nil { - lastByte, err := out.ReadByte() - if err == nil { - sawNewLine = lastByte == 13 - } - } - if sawNewLine { - newOut.WriteString("\n") - } - if len(source.Suffixes) > 0 { - newOut.WriteString("##!=>\n") - } - for _, suffix := range source.Suffixes { - newOut.WriteString(suffix) - newOut.WriteString("\n##!=>\n") - } - newOut.WriteString("##!<\n") - return newOut, nil + // With block-scoped prefixes/suffixes, the directives are already in the content + // as raw lines, so we don't need to wrap them in an assemble block. + // We still collect them in Parser.Prefixes/Suffixes for compatibility but don't + // need to do anything special with them here. + return out, nil } func expandDefinitions(src *bytes.Buffer, variables map[string]string) *bytes.Buffer { diff --git a/regex/processors/assemble.go b/regex/processors/assemble.go index 43676d5..6a7f250 100644 --- a/regex/processors/assemble.go +++ b/regex/processors/assemble.go @@ -137,6 +137,18 @@ func (a *Assemble) store(identifier string) error { // the value we just stored a.output.Reset() + // Apply block-scoped prefixes and suffixes to stored expressions + if len(a.prefixes) > 0 || len(a.suffixes) > 0 { + prefixes := strings.Join(a.prefixes, "") + suffixes := strings.Join(a.suffixes, "") + outputString = prefixes + outputString + suffixes + logger.Trace().Msgf("Applied block-scoped prefixes/suffixes to stored expression: %s", outputString) + // Clear prefixes/suffixes after applying to stored expression + // so they won't be applied again in Complete() + a.prefixes = nil + a.suffixes = nil + } + logger.Debug().Msgf("Storing expression at %s: %s", identifier, outputString) a.proc.ctx.stash[identifier] = outputString return nil From ffd059a4adcc5bd1c6a5e17ac5ce6eaba543af49 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Feb 2026 23:15:22 +0000 Subject: [PATCH 03/10] docs: clarify comment about Parser.Prefixes/Suffixes usage Co-authored-by: fzipi <3012076+fzipi@users.noreply.github.com> --- regex/operators/assembler.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/regex/operators/assembler.go b/regex/operators/assembler.go index db7f98f..b122e91 100644 --- a/regex/operators/assembler.go +++ b/regex/operators/assembler.go @@ -130,8 +130,9 @@ func (a *Operator) complete(assembleParser *parser.Parser) (string, error) { } // Note: Prefix/suffix application is now handled by individual Assemble processors (block-scoped). - // Parser.Prefixes and Parser.Suffixes are no longer populated or used here; any previous - // backward-compatibility behavior relying on them has been removed from this path. + // While Parser.Prefixes and Parser.Suffixes are still populated by the parser for include file + // merging compatibility, they are not used in the operator path as prefix/suffix directives are + // passed through as raw lines and processed by each Assemble instance. if len(result) > 0 { logger.Trace().Msgf("Applying last cleanups to %s\n", result) From a4b492cf8a2a017acba13e61ea41fbb5f38d8a39 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 8 Feb 2026 23:17:15 +0000 Subject: [PATCH 04/10] fix: apply block-scoped prefixes/suffixes to stored expressions Co-authored-by: fzipi <3012076+fzipi@users.noreply.github.com> --- go.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/go.sum b/go.sum index 59071fd..5c2a625 100644 --- a/go.sum +++ b/go.sum @@ -202,6 +202,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= From 594ccd14b916b7947d7de8ea06e085e960bb5e12 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Sun, 8 Feb 2026 21:24:05 -0300 Subject: [PATCH 05/10] fix: apply go fmt Signed-off-by: Felipe Zipitria --- regex/operators/assembler_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/regex/operators/assembler_test.go b/regex/operators/assembler_test.go index 69bd986..bca1e25 100644 --- a/regex/operators/assembler_test.go +++ b/regex/operators/assembler_test.go @@ -1278,4 +1278,3 @@ b // Only suffix should be applied s.Equal(`[ab]suffix`, output) } - From 71a71a6fda551466e6e40d4ad3e3f4ffbcd9b0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20Zipitr=C3=ADa?= <3012076+fzipi@users.noreply.github.com> Date: Sat, 25 Apr 2026 06:32:30 -0300 Subject: [PATCH 06/10] Update regex/processors/assemble.go Co-authored-by: Max Leske <250711+theseion@users.noreply.github.com> --- regex/processors/assemble.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regex/processors/assemble.go b/regex/processors/assemble.go index 6a7f250..0932c6a 100644 --- a/regex/processors/assemble.go +++ b/regex/processors/assemble.go @@ -21,8 +21,8 @@ const ( type Assemble struct { proc *Processor output strings.Builder - prefixes []string // Block-scoped prefixes - suffixes []string // Block-scoped suffixes + prefixes []string + suffixes []string } // NewAssemble creates a new assemble processor From 39c24b7c4f4ec5899e03a8de40408a561d544da0 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Sat, 25 Apr 2026 06:59:44 -0300 Subject: [PATCH 07/10] fix: simplify prefix/suffix application and add nesting test Address review feedback from theseion: - Remove redundant else-if branch in Complete() since the outer guard already ensures at least one of prefixes/suffixes is non-empty - Add test for multiple nesting levels to verify block-scoped prefix/suffix isolation across nested assemble blocks Co-Authored-By: Claude Sonnet 4.6 --- regex/operators/assembler_test.go | 20 ++++++++++++++++++++ regex/processors/assemble.go | 14 ++------------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/regex/operators/assembler_test.go b/regex/operators/assembler_test.go index bca1e25..49a3814 100644 --- a/regex/operators/assembler_test.go +++ b/regex/operators/assembler_test.go @@ -1263,6 +1263,26 @@ b s.Equal(`prefix[ab]`, output) } +func (s *assemblerTestSuite) TestAssemble_BlockScopedPrefixSuffix_MultipleNestingLevels() { + // Test that prefix/suffix is correctly scoped when blocks are nested multiple levels deep. + // The outer block's prefix/suffix wraps the inner block's output, while the inner block's + // prefix/suffix only applies to the inner block's content. + contents := `##!> assemble +##!^ outer_prefix +##!$ outer_suffix +##!> assemble +##!^ inner_prefix +##!$ inner_suffix +a +b +##!< +##!<` + assembler := NewAssembler(s.ctx) + output, err := assembler.Run(contents) + s.Require().NoError(err) + s.Equal(`outer_prefixinner_prefix[ab]inner_suffixouter_suffix`, output) +} + func (s *assemblerTestSuite) TestAssemble_PrefixSuffixWithStashing_OnlySuffix() { // Test stashing with only suffix contents := `##!> assemble diff --git a/regex/processors/assemble.go b/regex/processors/assemble.go index 0932c6a..6c2c93e 100644 --- a/regex/processors/assemble.go +++ b/regex/processors/assemble.go @@ -90,18 +90,8 @@ func (a *Assemble) Complete() ([]string, error) { // Apply block-scoped prefixes and suffixes if len(a.prefixes) > 0 || len(a.suffixes) > 0 { - prefixes := strings.Join(a.prefixes, "") - suffixes := strings.Join(a.suffixes, "") - - // If we have content, apply prefixes/suffixes around it - if len(result) > 0 { - result = prefixes + result + suffixes - logger.Trace().Msgf("Applied block-scoped prefixes/suffixes: %s", result) - } else if len(prefixes) > 0 || len(suffixes) > 0 { - // If no content but we have prefixes/suffixes, just concatenate them - result = prefixes + suffixes - logger.Trace().Msgf("Applied block-scoped prefixes/suffixes with no content: %s", result) - } + result = strings.Join(a.prefixes, "") + result + strings.Join(a.suffixes, "") + logger.Trace().Msgf("Applied block-scoped prefixes/suffixes: %s", result) } logger.Trace().Msgf("Completed assembly: %s", result) From 43ac415e09ad913d772b41a37008f3be31ea17ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20Zipitr=C3=ADa?= <3012076+fzipi@users.noreply.github.com> Date: Sun, 7 Jun 2026 09:53:25 -0300 Subject: [PATCH 08/10] Update regex/parser/parser.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- regex/parser/parser.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/regex/parser/parser.go b/regex/parser/parser.go index b1e906b..dd32c0a 100644 --- a/regex/parser/parser.go +++ b/regex/parser/parser.go @@ -152,13 +152,9 @@ func (p *Parser) Parse(formatOnly bool) *bytes.Buffer { case prefix: // Pass through prefix directive for processor to handle (block-scoped) text = line + "\n" - // Also collect for include file merging - p.Prefixes = append(p.Prefixes, parsedLine.prefix) case suffix: // Pass through suffix directive for processor to handle (block-scoped) text = line + "\n" - // Also collect for include file merging - p.Suffixes = append(p.Suffixes, parsedLine.suffix) } if formatOnly { text = line + "\n" From eb3944a910edf4beadb99b8cca9ed4aa1ff94d5b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 7 Jun 2026 12:56:51 +0000 Subject: [PATCH 09/10] Rename no-op include merge helper --- go.sum | 1 - regex/parser/parser.go | 22 ++++++++-------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/go.sum b/go.sum index 5c2a625..59071fd 100644 --- a/go.sum +++ b/go.sum @@ -202,7 +202,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= diff --git a/regex/parser/parser.go b/regex/parser/parser.go index dd32c0a..542ce58 100644 --- a/regex/parser/parser.go +++ b/regex/parser/parser.go @@ -277,28 +277,22 @@ func parseFile(rootParser *Parser, filename string, definitions map[string]strin newP.variables = definitions } out := newP.Parse(false) - newOut, err := mergePrefixesSuffixes(newP, out) + err = validateIncludedFileDirectives(newP) if err != nil { logger.Fatal().Msgf("error parsing file: %v", err.Error()) } - logger.Trace().Msg(newOut.String()) - return newOut, newP.variables + logger.Trace().Msg(out.String()) + return out, newP.variables } -// Merge prefixes, and suffixes from include files into another parser. -// All of these need to be treated as local to the source parser. -// We removed flag merging because of https://github.com/coreruleset/crs-toolchain/issues/72 -func mergePrefixesSuffixes(source *Parser, out *bytes.Buffer) (*bytes.Buffer, error) { - logger.Trace().Msg("merging prefixes, suffixes from included file") +// validateIncludedFileDirectives validates directives that are not allowed in include files. +func validateIncludedFileDirectives(source *Parser) error { + logger.Trace().Msg("validating directives from included file") // If the included file has flags, this is an error if len(source.Flags) > 0 { - return new(bytes.Buffer), errors.New("include files must not contain flags. See https://github.com/coreruleset/crs-toolchain/v2/issues/71") + return errors.New("include files must not contain flags. See https://github.com/coreruleset/crs-toolchain/v2/issues/71") } - // With block-scoped prefixes/suffixes, the directives are already in the content - // as raw lines, so we don't need to wrap them in an assemble block. - // We still collect them in Parser.Prefixes/Suffixes for compatibility but don't - // need to do anything special with them here. - return out, nil + return nil } func expandDefinitions(src *bytes.Buffer, variables map[string]string) *bytes.Buffer { From cb88d62f8b616ba6bac33a1ba4f58dde0d1173ca Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Mon, 17 Aug 2026 20:00:02 -0300 Subject: [PATCH 10/10] fix: scope included prefixes and suffixes to the included content Passing `##!^` and `##!$` through as raw lines left them active for the rest of the including file. The Assemble processor holds a directive until store() or Complete(), so an include carrying a prefix wrapped the caller's own content as well, regardless of where the include appeared: include/inc.ra: ##!^ PRE_ alpha 932100.ra: ##!> include inc beta before: PRE_(?:alph|bet)a after: (?:PRE_alph|bet)a Wrap included content in its own assemble block when the included file contains prefix or suffix directives, which bounds them to that content. As before, the block is omitted when there are no such directives, because wrapping changes the semantics of content processed by the cmdline processor in the including file. This restores the local scope that mergePrefixesSuffixes used to provide. Parser.Prefixes and Parser.Suffixes are populated again, now only so parseFile can tell whether an included file needs the block, and the assembler comment describing them is corrected. The three include tests asserted the leaking output, so they could not catch this. Restore their expectations and add end-to-end tests covering an include before, after, and on both sides of the caller's own content. Co-Authored-By: Claude Opus 5 (1M context) --- regex/operators/assembler.go | 7 ++--- regex/operators/assembler_test.go | 40 ++++++++++++++++++++++++++++ regex/parser/include_test.go | 21 ++++++++++----- regex/parser/parser.go | 44 ++++++++++++++++++++++++------- 4 files changed, 93 insertions(+), 19 deletions(-) diff --git a/regex/operators/assembler.go b/regex/operators/assembler.go index b122e91..2540d64 100644 --- a/regex/operators/assembler.go +++ b/regex/operators/assembler.go @@ -130,9 +130,10 @@ func (a *Operator) complete(assembleParser *parser.Parser) (string, error) { } // Note: Prefix/suffix application is now handled by individual Assemble processors (block-scoped). - // While Parser.Prefixes and Parser.Suffixes are still populated by the parser for include file - // merging compatibility, they are not used in the operator path as prefix/suffix directives are - // passed through as raw lines and processed by each Assemble instance. + // Prefix/suffix directives are passed through as raw lines, so they are not applied here. + // Parser.Prefixes and Parser.Suffixes record which directives a file contained; the parser + // uses that to decide whether an included file needs its own assemble block, and the operator + // path does not read them. if len(result) > 0 { logger.Trace().Msgf("Applying last cleanups to %s\n", result) diff --git a/regex/operators/assembler_test.go b/regex/operators/assembler_test.go index 49a3814..b376687 100644 --- a/regex/operators/assembler_test.go +++ b/regex/operators/assembler_test.go @@ -4,7 +4,9 @@ package operators import ( + "io/fs" "os" + "path" "testing" "github.com/stretchr/testify/suite" @@ -1190,6 +1192,44 @@ false s.Contains(output, `fals`) } +// writeIncludeFile writes an include file that `##!> include name` will resolve. +func (s *assemblerTestSuite) writeIncludeFile(name string, contents string) { + s.T().Helper() + includeDir := path.Join(s.tempDir, "regex-assembly", "include") + s.Require().NoError(os.MkdirAll(includeDir, fs.ModePerm)) + s.Require().NoError(os.WriteFile(path.Join(includeDir, name+".ra"), []byte(contents), fs.ModePerm)) +} + +func (s *assemblerTestSuite) TestAssemble_IncludedPrefixIsScopedToIncludedContent() { + s.writeIncludeFile("scoped_prefix", "##!^ PRE_\nalpha\n") + + // `beta` belongs to the including file and must not pick up the include's prefix, + // no matter which side of the include it sits on. + for _, tt := range []struct { + name string + contents string + expected string + }{ + {"caller line after include", "##!> include scoped_prefix\nbeta", `(?:PRE_alph|bet)a`}, + {"caller line before include", "beta\n##!> include scoped_prefix", `(?:bet|PRE_alph)a`}, + {"caller lines on both sides", "gamma\n##!> include scoped_prefix\nbeta", `(?:gamm|PRE_alph|bet)a`}, + } { + s.Run(tt.name, func() { + output, err := NewAssembler(s.ctx).Run(tt.contents) + s.Require().NoError(err) + s.Equal(tt.expected, output) + }) + } +} + +func (s *assemblerTestSuite) TestAssemble_IncludedSuffixIsScopedToIncludedContent() { + s.writeIncludeFile("scoped_suffix", "##!$ _SUF\nalpha\n") + + output, err := NewAssembler(s.ctx).Run("##!> include scoped_suffix\nbeta") + s.Require().NoError(err) + s.Equal(`alpha_SUF|beta`, output) +} + func (s *assemblerTestSuite) TestAssemble_PrefixSuffixWithStashing() { // Test that prefixes/suffixes are applied when storing expressions with ##!=< contents := `##!> assemble diff --git a/regex/parser/include_test.go b/regex/parser/include_test.go index ec0229c..0b46024 100644 --- a/regex/parser/include_test.go +++ b/regex/parser/include_test.go @@ -72,10 +72,13 @@ func (s *parserIncludeTestSuite) TestParserInclude_Prefixes() { included regex`, "data regex") parser := NewParser(s.ctx, s.reader) actual := parser.Parse(false) - // With block-scoped prefixes, directives are passed through as-is - expected := bytes.NewBufferString(`##!^ prefix1 + // Directives are passed through as-is, wrapped in an assemble block that scopes + // them to the included content instead of the rest of the including file. + expected := bytes.NewBufferString(`##!> assemble +##!^ prefix1 ##!^ prefix2 included regex +##!< data regex `) s.Equal(expected.String(), actual.String()) @@ -87,10 +90,13 @@ func (s *parserIncludeTestSuite) TestParserInclude_Suffixes() { included regex`, "data regex") parser := NewParser(s.ctx, s.reader) actual := parser.Parse(false) - // With block-scoped suffixes, directives are passed through as-is - expected := bytes.NewBufferString(`##!$ suffix1 + // Directives are passed through as-is, wrapped in an assemble block that scopes + // them to the included content instead of the rest of the including file. + expected := bytes.NewBufferString(`##!> assemble +##!$ suffix1 ##!$ suffix2 included regex +##!< data regex `) @@ -105,12 +111,15 @@ func (s *parserIncludeTestSuite) TestParserInclude_FlagsPrefixesSuffixes() { included regex`, "data regex") parser := NewParser(s.ctx, s.reader) actual := parser.Parse(false) - // With block-scoped prefixes/suffixes, directives are passed through as-is - expected := bytes.NewBufferString(`##!$ suffix1 + // Directives are passed through as-is, wrapped in an assemble block that scopes + // them to the included content instead of the rest of the including file. + expected := bytes.NewBufferString(`##!> assemble +##!$ suffix1 ##!$ suffix2 ##!^ prefix1 ##!^ prefix2 included regex +##!< data regex `) diff --git a/regex/parser/parser.go b/regex/parser/parser.go index 542ce58..fb5fcbd 100644 --- a/regex/parser/parser.go +++ b/regex/parser/parser.go @@ -150,10 +150,14 @@ func (p *Parser) Parse(formatOnly bool) *bytes.Buffer { } } case prefix: - // Pass through prefix directive for processor to handle (block-scoped) + // Pass the directive through for the Assemble processor to apply (block-scoped). + // Prefixes is still recorded so that parseFile knows whether an included file + // needs its own assemble block to scope these directives to its content. + p.Prefixes = append(p.Prefixes, parsedLine.prefix) text = line + "\n" case suffix: - // Pass through suffix directive for processor to handle (block-scoped) + // See the prefix case above. + p.Suffixes = append(p.Suffixes, parsedLine.suffix) text = line + "\n" } if formatOnly { @@ -277,22 +281,42 @@ func parseFile(rootParser *Parser, filename string, definitions map[string]strin newP.variables = definitions } out := newP.Parse(false) - err = validateIncludedFileDirectives(newP) + newOut, err := scopeIncludedFileDirectives(newP, out) if err != nil { logger.Fatal().Msgf("error parsing file: %v", err.Error()) } - logger.Trace().Msg(out.String()) - return out, newP.variables + logger.Trace().Msg(newOut.String()) + return newOut, newP.variables } -// validateIncludedFileDirectives validates directives that are not allowed in include files. -func validateIncludedFileDirectives(source *Parser) error { - logger.Trace().Msg("validating directives from included file") +// Scope prefixes and suffixes from an included file to the content of that file. +// The directives themselves are passed through as raw lines for the Assemble processor +// to apply, so they need an assemble block of their own. Without it they stay active +// for the rest of the including file and wrap the caller's content as well. +// We removed flag merging because of https://github.com/coreruleset/crs-toolchain/issues/72 +func scopeIncludedFileDirectives(source *Parser, out *bytes.Buffer) (*bytes.Buffer, error) { + logger.Trace().Msg("scoping prefixes, suffixes from included file") // If the included file has flags, this is an error if len(source.Flags) > 0 { - return errors.New("include files must not contain flags. See https://github.com/coreruleset/crs-toolchain/v2/issues/71") + return new(bytes.Buffer), errors.New("include files must not contain flags. See https://github.com/coreruleset/crs-toolchain/v2/issues/71") } - return nil + // IMPORTANT: don't write the assemble block at all if there are no prefixes or + // suffixes. Enclosing the output in an assemble block can change the semantics, for example, + // when the included content is processed by the cmdline processor in the including file. + if len(source.Prefixes) == 0 && len(source.Suffixes) == 0 { + return out, nil + } + + newOut := new(bytes.Buffer) + newOut.WriteString("##!> assemble\n") + if _, err := out.WriteTo(newOut); err != nil { + logger.Fatal().Err(err).Msg("failed to copy output to new buffer") + } + if !bytes.HasSuffix(newOut.Bytes(), []byte("\n")) { + newOut.WriteString("\n") + } + newOut.WriteString("##!<\n") + return newOut, nil } func expandDefinitions(src *bytes.Buffer, variables map[string]string) *bytes.Buffer {