Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 65 additions & 6 deletions src/parse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,28 @@ pub fn parse(source: &str) -> Vec<Block> {
// ---- inline elements (accumulate into paragraph) ----
Expr::Space(_) => {
let text = node_text(&expr);
// Two consecutive Space("\n") nodes arise only when a LineComment
// was dropped between them (bare \n\n is always a Parbreak token,
// never two Space nodes). Skip the second \n to avoid producing
// \n\n in the reconstructed source, which Typst would treat as a
// paragraph break.
if !(text == "\n" && paragraph_buf.ends_with('\n')) {
// Two consecutive Space nodes spanning a newline arise only when
// a comment was dropped between them (bare \n\n is always a
// Parbreak token, never two Space nodes). Reconstructing them
// verbatim would leave a blank or whitespace-only line, which
// Typst treats as a paragraph break. Drop the buffer's trailing
// indent and the duplicate newline so the two lines stay in one
// paragraph. The buffer may end in \n plus indent when the
// dropped comment was itself indented.
let joined = match text.strip_prefix('\n') {
Some(rest) => {
let line_start = paragraph_buf.trim_end_matches([' ', '\t']);
if line_start.ends_with('\n') {
paragraph_buf.truncate(line_start.len());
paragraph_buf.push_str(rest);
true
} else {
false
}
}
None => false,
};
Comment on lines +79 to +91

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -eu
printf '%s\n' '--- candidate files ---'
git ls-files | rg '(^|/)(src/parse\.rs|Cargo\.toml|Cargo\.lock)$'
printf '%s\n' '--- parse.rs outline ---'
ast-grep outline src/parse.rs --lang rust
printf '%s\n' '--- relevant source ---'
sed -n '55,105p' src/parse.rs
sed -n '240,315p' src/parse.rs
printf '%s\n' '--- typst-syntax dependency declarations ---'
rg -n -C 3 'typst-syntax|typst_syntax' Cargo.toml Cargo.lock

Repository: sou1118/typdiff

Length of output: 6812


🏁 Script executed:

#!/bin/bash
set -eu
python3 - <<'PY'
from pathlib import Path
p = Path("src/parse.rs")
s = p.read_text()
for i, line in enumerate(s.splitlines(), 1):
    if "strip_prefix" in line or "trim_end_matches" in line or "paragraph_buf" in line:
        print(f"{i}: {line}")
PY

Repository: sou1118/typdiff

Length of output: 1629


🏁 Script executed:

#!/bin/bash
set -eu
printf '%s\n' '--- parser control flow ---'
sed -n '1,145p' src/parse.rs
printf '%s\n' '--- local dependency/cache candidates ---'
find "$HOME/.cargo" -type f \( -path '*typst-syntax*lexer.rs' -o -path '*typst-syntax*lib.rs' \) -print 2>/dev/null | head -20 || true
printf '%s\n' '--- typst-syntax 0.14.2 lexer source ---'
curl -LfsS https://raw.githubusercontent.com/typst/typst/v0.14.2/crates/typst-syntax/src/lexer.rs | sed -n '1,240p'

Repository: sou1118/typdiff

Length of output: 14243


🏁 Script executed:

#!/bin/bash
set -eu
curl -LfsS https://raw.githubusercontent.com/typst/typst/v0.14.2/crates/typst-syntax/src/lexer.rs \
  | rg -n -C 8 'newline|CRLF|\\r|Space|Parbreak'

Repository: sou1118/typdiff

Length of output: 13084


🏁 Script executed:

#!/bin/bash
set -eu
printf '%s\n' '--- public exports ---'
curl -LfsS https://raw.githubusercontent.com/typst/typst/v0.14.2/crates/typst-syntax/src/lib.rs \
  | rg -n -C 4 'mod lexer|pub use .*is_newline|is_newline'
printf '%s\n' '--- paragraph flush behavior ---'
sed -n '145,170p' src/parse.rs
printf '%s\n' '--- direct reconstruction probe ---'
python3 - <<'PY'
cases = {
    "LF": ("First.\n  ", "\n"),
    "CRLF": ("First.\r\n  ", "\r\n"),
    "CR": ("First.\r  ", "\r"),
    "LS": ("First.\u2028  ", "\u2028"),
}
for name, (buf, text) in cases.items():
    joined = text.startswith("\n")
    result = buf + text if not joined else buf.rstrip(" \t") + text[1:]
    print(name, {"joined": joined, "result_repr": repr(result)})
PY

Repository: sou1118/typdiff

Length of output: 1723


Handle all Typst newline forms.

When an indented comment occurs in a CRLF or other Typst newline sequence, strip_prefix('\n') fails. The paragraph then retains the comment indentation and duplicate newline. Use typst_syntax::is_newline, handle CRLF as one sequence, and add regression tests for supported newline forms.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/parse.rs` around lines 79 - 91, Update the newline handling in the joined
logic of the paragraph parser to recognize every Typst newline form using
typst_syntax::is_newline, including treating CRLF as a single sequence before
removing the prefix. Preserve the existing paragraph joining behavior while
preventing retained indentation and duplicate newlines, and add regression
coverage for each supported newline form.

if !joined {
paragraph_buf.push_str(&text);
}
}
Expand Down Expand Up @@ -240,6 +256,49 @@ mod tests {
}
}

#[test]
fn test_parse_line_comment_keeps_single_paragraph() {
let blocks = parse("First.\n// a comment\nSecond.\n");
assert_eq!(blocks.len(), 1, "expected 1 block, got: {blocks:?}");
assert!(
matches!(&blocks[0], Block::Paragraph { source_text } if source_text == "First.\nSecond.")
);
}

#[test]
fn test_parse_indented_line_comment_keeps_single_paragraph() {
let blocks = parse("First.\n // indented comment\nSecond.\n");
assert_eq!(blocks.len(), 1, "expected 1 block, got: {blocks:?}");
assert!(
matches!(&blocks[0], Block::Paragraph { source_text } if source_text == "First.\nSecond.")
);
}

#[test]
fn test_parse_consecutive_indented_comments_keep_single_paragraph() {
let blocks = parse("First.\n // c1\n // c2\nSecond.\n");
assert_eq!(blocks.len(), 1, "expected 1 block, got: {blocks:?}");
assert!(
matches!(&blocks[0], Block::Paragraph { source_text } if source_text == "First.\nSecond.")
);
}

#[test]
fn test_parse_blank_line_before_comment_keeps_paragraph_break() {
let blocks = parse("First.\n\n// a comment\nSecond.\n");
assert!(
blocks
.iter()
.any(|b| matches!(b, Block::Paragraph { source_text } if source_text == "First."))
);
assert!(blocks.iter().any(|b| matches!(b, Block::Parbreak)));
assert!(
blocks
.iter()
.any(|b| matches!(b, Block::Paragraph { source_text } if source_text == "Second."))
);
}

#[test]
fn test_parse_label_at_paragraph_start_as_own_block() {
let blocks = parse("= Sample\n\n<sample-anchor>\nAlpha beta.\n");
Expand Down
11 changes: 11 additions & 0 deletions tests/integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -198,3 +198,14 @@ after the field log was reconciled.
assert!(output.contains("#ref(<sec-bridge-ledger>, supplement: [])"));
assert!(!output.contains("#ref(\\<sec-bridge-ledger>"));
}

#[test]
fn test_indented_line_comment_does_not_split_paragraph() {
let old = "First sentence.\n // indented comment\nSecond sentence.\n";
let new = "First sentence.\n // indented comment\nSecond sentence changed.\n";
let output = run_diff(old, new);

assert!(output.contains("First sentence.\nSecond sentence"));
// A whitespace-only line would be treated as a paragraph break by Typst.
assert!(!output.contains("\n \n"));
}