diff --git a/ltml/samples/test_012_cjk_thai_grid.pdf b/ltml/samples/test_012_cjk_thai_grid.pdf index 42a30a74..a0d288df 100644 Binary files a/ltml/samples/test_012_cjk_thai_grid.pdf and b/ltml/samples/test_012_cjk_thai_grid.pdf differ diff --git a/ltml/samples/test_022_transforms.pdf b/ltml/samples/test_022_transforms.pdf index 719b03b9..e58487c3 100644 Binary files a/ltml/samples/test_022_transforms.pdf and b/ltml/samples/test_022_transforms.pdf differ diff --git a/rich_text/rich_text.go b/rich_text/rich_text.go index 90263ad3..c4359b1a 100644 --- a/rich_text/rich_text.go +++ b/rich_text/rich_text.go @@ -677,10 +677,14 @@ func (piece *RichText) TrimRightSpace() *RichText { return piece.TrimRightFunc(unicode.IsSpace) } -// TrimSpace trims the runes from both ends of the text where unicode.IsSpace returns true, +// TrimSpace trims Unicode whitespace and zero-width spaces from both ends of the text, // returning a new structure and leaving the original unchanged. func (piece *RichText) TrimSpace() *RichText { - return piece.TrimFunc(unicode.IsSpace) + return piece.TrimFunc(isLineEdgeSpace) +} + +func isLineEdgeSpace(r rune) bool { + return unicode.IsSpace(r) || r == wordbreaking.ZeroWidthSpace } // VisitAll invokes the callback function for each piece of text within the structure, in order. @@ -779,10 +783,10 @@ func decorationOverridesOption(opts options.Options, key string) *DecorationOver // WordsToWidth splits the text at the last breaking point before width is exceeded, making use of the previously-allocated and marked flags // and returning new structures representing both halves and the remaining flags. The original structure is unchanged. -// A minimum of one word is split off, even if width is exceeded, unless hardBreak is true, in which case the text is split along character -// boundaries. +// A minimum of one word is split off, even if width is exceeded, unless emergencyBreak is true, in which case the text is split along +// character boundaries. func (piece *RichText) WordsToWidth( - width float64, wordFlags []wordbreaking.Flags, hardBreak bool) ( + width float64, wordFlags []wordbreaking.Flags, emergencyBreak bool) ( line, remainder *RichText, remainderFlags []wordbreaking.Flags) { if width < 0.0 { width = 0.0 @@ -791,6 +795,7 @@ func (piece *RichText) WordsToWidth( currentWidth := 0.0 words := 0 wordWidth := 0.0 + trailingSpaceWidth := 0.0 extra := 0.0 lastOffset := 0 @@ -804,9 +809,20 @@ func (piece *RichText) WordsToWidth( var leafRuneIdx int fn := func(r rune, p *RichText, offset int) bool { - if words > 0 && currentWidth+extra+wordWidth > width { + if offset > 0 && (wordFlags[offset]&wordbreaking.MandatoryBreak) == wordbreaking.MandatoryBreak { + current = offset + return true + } + softBreak := offset > 0 && + wordFlags[offset]&wordbreaking.SoftBreak == wordbreaking.SoftBreak && + wordFlags[offset]&wordbreaking.NoBreak != wordbreaking.NoBreak + candidateWidth := currentWidth + extra + wordWidth + if softBreak { + candidateWidth -= trailingSpaceWidth + } + if words > 0 && candidateWidth > width { return true - } else if words == 0 && hardBreak && wordWidth > width { + } else if words == 0 && emergencyBreak && wordWidth > width { if lastOffset > 0 { current = lastOffset } else { @@ -814,9 +830,7 @@ func (piece *RichText) WordsToWidth( } return true } - if offset > 0 && - wordFlags[offset]&wordbreaking.SoftBreak == wordbreaking.SoftBreak && - wordFlags[offset]&wordbreaking.NoBreak != wordbreaking.NoBreak { + if softBreak { current = offset currentWidth += wordWidth wordWidth = 0.0 @@ -852,17 +866,24 @@ func (piece *RichText) WordsToWidth( shapedAdv = nil } } + runeWidth := 0.0 if r != wordbreaking.SoftHyphen { if shapedAdv != nil && leafRuneIdx < len(shapedAdv) { - wordWidth += shapedAdv[leafRuneIdx] + p.CharSpacing + runeWidth = shapedAdv[leafRuneIdx] + p.CharSpacing } else { - runeWidth, _ := metrics.AdvanceWidth(r) - wordWidth += (fsize * float64(runeWidth)) + p.CharSpacing + advanceWidth, _ := metrics.AdvanceWidth(r) + runeWidth = (fsize * float64(advanceWidth)) + p.CharSpacing } if unicode.IsSpace(r) { - wordWidth += p.WordSpacing + runeWidth += p.WordSpacing } } + wordWidth += runeWidth + if isLineEdgeSpace(r) { + trailingSpaceWidth += runeWidth + } else { + trailingSpaceWidth = 0.0 + } leafRuneIdx++ lastRune = r lastOffset = offset @@ -884,11 +905,11 @@ func (piece *RichText) WordsToWidth( } // WrapToWidth returns one or more lines of text resulting from repeatedly invoking WordsToWidth. -func (piece *RichText) WrapToWidth(width float64, wordFlags []wordbreaking.Flags, hardBreak bool) (lines []*RichText) { - line, remainder, remainderFlags := piece.WordsToWidth(width, wordFlags, hardBreak) +func (piece *RichText) WrapToWidth(width float64, wordFlags []wordbreaking.Flags, emergencyBreak bool) (lines []*RichText) { + line, remainder, remainderFlags := piece.WordsToWidth(width, wordFlags, emergencyBreak) for remainder != nil { lines = append(lines, line.TrimSpace()) - line, remainder, remainderFlags = remainder.WordsToWidth(width, remainderFlags, hardBreak) + line, remainder, remainderFlags = remainder.WordsToWidth(width, remainderFlags, emergencyBreak) } lines = append(lines, line.TrimSpace()) return diff --git a/rich_text/rich_text_test.go b/rich_text/rich_text_test.go index be7ebb5b..e7fbbcb9 100644 --- a/rich_text/rich_text_test.go +++ b/rich_text/rich_text_test.go @@ -7,6 +7,7 @@ import ( "errors" "io" "os" + "slices" "strings" "testing" "unicode" @@ -937,6 +938,13 @@ func TestRichText_TrimSpace(t *testing.T) { st.Equal(leadingAndTrailingWhitespaceTextTrimmed, t2.String()) } +func TestRichText_TrimSpace_IncludesZeroWidthSpace(t *testing.T) { + t1 := &RichText{Text: "\u200bcontent\u200b"} + if got := t1.TrimSpace().String(); got != "content" { + t.Fatalf("TrimSpace() = %q, want %q", got, "content") + } +} + func TestRichText_TrimRightFunc_complex(t *testing.T) { skipIfNoTTFFonts(t) st := SuperTest{t} @@ -1048,7 +1056,7 @@ func TestRichText_WordsToWidth_mixed(t *testing.T) { } } -func TestRichText_WordsToWidth_hardbreak(t *testing.T) { +func TestRichText_WordsToWidth_emergencyBreak(t *testing.T) { skipIfNoTTFFonts(t) st := SuperTest{t} p := arialText("Supercalifragilisticexpialidocious") @@ -1076,6 +1084,69 @@ func TestRichText_WordsToWidth_zero(t *testing.T) { st.Equal(33, len(remainderFlags)) } +func TestRichText_WordsToWidthAndWrapToWidth_WhitespaceContract(t *testing.T) { + font := minimalFixtureFont(t) + rt := (&RichText{Text: "alpha beta", Font: font, FontSize: 12}).measure() + prefix := (&RichText{Text: "alpha", Font: font, FontSize: 12}).measure() + flags := make([]wordbreaking.Flags, rt.Len()) + wordbreaking.MarkRuneAttributes(rt.String(), flags) + + line, remainder, _ := rt.WordsToWidth(prefix.Width()+0.01, flags, false) + if got := line.String(); got != "alpha " { + t.Fatalf("WordsToWidth line = %q, want %q", got, "alpha ") + } + if remainder == nil { + t.Fatal("WordsToWidth remainder is nil") + } + if got := remainder.String(); got != "beta" { + t.Fatalf("WordsToWidth remainder = %q, want %q", got, "beta") + } + + lines := rt.WrapToWidth(prefix.Width()+0.01, flags, false) + if len(lines) != 2 || lines[0].String() != "alpha" || lines[1].String() != "beta" { + got := make([]string, len(lines)) + for i, wrapped := range lines { + got[i] = wrapped.String() + } + t.Fatalf("WrapToWidth lines = %q, want [\"alpha\" \"beta\"]", got) + } +} + +func TestRichText_WrapToWidth_TrailingSpaceDoesNotCauseEarlyWrap(t *testing.T) { + font := minimalFixtureFont(t) + tests := []struct { + name string + text string + firstLine string + separator string + wordSpacing float64 + }{ + {name: "single space", text: "alpha beta gamma", firstLine: "alpha beta", separator: " "}, + {name: "consecutive spaces", text: "alpha beta gamma", firstLine: "alpha beta", separator: " "}, + {name: "word spacing", text: "alpha beta gamma", firstLine: "alpha beta", separator: " ", wordSpacing: 4}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rt := &RichText{Text: tt.text, Font: font, FontSize: 12, WordSpacing: tt.wordSpacing} + prefix := &RichText{Text: tt.firstLine, Font: font, FontSize: 12, WordSpacing: tt.wordSpacing} + trailing := &RichText{Text: tt.separator, Font: font, FontSize: 12, WordSpacing: tt.wordSpacing} + width := prefix.Width() + trailing.Width()/2 + flags := make([]wordbreaking.Flags, rt.Len()) + wordbreaking.MarkRuneAttributes(rt.String(), flags) + + lines := rt.WrapToWidth(width, flags, false) + got := make([]string, len(lines)) + for i, line := range lines { + got[i] = line.String() + } + if want := []string{tt.firstLine, "gamma"}; !slices.Equal(got, want) { + t.Fatalf("lines = %q, want %q (width %.3f)", got, want, width) + } + }) + } +} + func TestRichText_WordsToWidth_LogsShapingFailure(t *testing.T) { p := &RichText{ Text: "مرحبا بالعالم", @@ -1247,7 +1318,104 @@ func TestRichText_WrapToWidth_thai(t *testing.T) { } } -func TestRichText_WrapToWidth_hardBreak(t *testing.T) { +func TestRichText_WrapToWidth_MandatoryBreaks(t *testing.T) { + font := minimalFixtureFont(t) + tests := []struct { + name string + text string + want []string + }{ + {name: "LF", text: "alpha\nbeta", want: []string{"alpha", "beta"}}, + {name: "CRLF", text: "alpha\r\nbeta", want: []string{"alpha", "beta"}}, + {name: "NEL", text: "alpha\u0085beta", want: []string{"alpha", "beta"}}, + {name: "line separator", text: "alpha\u2028beta", want: []string{"alpha", "beta"}}, + {name: "paragraph separator", text: "alpha\u2029beta", want: []string{"alpha", "beta"}}, + {name: "leading", text: "\nalpha", want: []string{"", "alpha"}}, + {name: "consecutive", text: "alpha\n\nbeta", want: []string{"alpha", "", "beta"}}, + {name: "trailing", text: "alpha\n", want: []string{"alpha"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rt := (&RichText{Text: tt.text, Font: font, FontSize: 12}).measure() + flags := make([]wordbreaking.Flags, rt.Len()) + wordbreaking.MarkRuneAttributes(rt.String(), flags) + + lines := rt.WrapToWidth(rt.Width()+100, flags, false) + got := make([]string, len(lines)) + for i, line := range lines { + got[i] = line.String() + } + if !slices.Equal(got, tt.want) { + t.Fatalf("mandatory-break lines = %q, want %q", got, tt.want) + } + }) + } +} + +func TestRichText_WrapToWidth_MandatoryBreakOverridesNoBreak(t *testing.T) { + font := minimalFixtureFont(t) + rt := (&RichText{Text: "alpha\nbeta", Font: font, FontSize: 12, NoBreak: true}).measure() + flags := make([]wordbreaking.Flags, rt.Len()) + wordbreaking.MarkRuneAttributes(rt.String(), flags) + rt.MarkNoBreak(flags) + + lines := rt.WrapToWidth(rt.Width()+100, flags, false) + got := make([]string, len(lines)) + for i, line := range lines { + got[i] = line.String() + } + if want := []string{"alpha", "beta"}; !slices.Equal(got, want) { + t.Fatalf("mandatory-break lines = %q, want %q", got, want) + } +} + +func TestRichText_WrapToWidth_TrimsSelectedZeroWidthSpace(t *testing.T) { + font := minimalFixtureFont(t) + rt := (&RichText{Text: "alpha\u200bbeta", Font: font, FontSize: 12}).measure() + prefix := (&RichText{Text: "alpha", Font: font, FontSize: 12}).measure() + flags := make([]wordbreaking.Flags, rt.Len()) + wordbreaking.MarkRuneAttributes(rt.String(), flags) + + lines := rt.WrapToWidth(prefix.Width()+0.01, flags, false) + got := make([]string, len(lines)) + for i, line := range lines { + got[i] = line.String() + } + if want := []string{"alpha", "beta"}; !slices.Equal(got, want) { + t.Fatalf("zero-width-space lines = %q, want %q", got, want) + } +} + +func TestRichText_WrapToWidth_RespectsCJKPunctuation(t *testing.T) { + font := minimalFixtureFont(t) + const text = "中文，中文々中文（中文）中文" + rt := (&RichText{Text: text, Font: font, FontSize: 12}).measure() + twoIdeographs := (&RichText{Text: "中文", Font: font, FontSize: 12}).measure() + flags := make([]wordbreaking.Flags, rt.Len()) + wordbreaking.MarkRuneAttributes(rt.String(), flags) + + lines := rt.WrapToWidth(twoIdeographs.Width()+0.01, flags, false) + if len(lines) < 2 { + t.Fatalf("CJK text produced %d line, want multiple lines", len(lines)) + } + for _, line := range lines { + lineText := line.String() + lineRunes := []rune(lineText) + if len(lineRunes) == 0 { + t.Error("CJK wrapping produced an empty line") + continue + } + if strings.ContainsRune("，、。！？）】」』》〉々", lineRunes[0]) { + t.Errorf("line begins with prohibited punctuation: %q", lineText) + } + if strings.ContainsRune("（【「『《〈", lineRunes[len(lineRunes)-1]) { + t.Errorf("line ends with opening punctuation: %q", lineText) + } + } +} + +func TestRichText_WrapToWidth_emergencyBreak(t *testing.T) { skipIfNoTTFFonts(t) st := SuperTest{t} expected := []string{ diff --git a/wordbreaking/flags.go b/wordbreaking/flags.go index 124e3145..5041debd 100644 --- a/wordbreaking/flags.go +++ b/wordbreaking/flags.go @@ -6,10 +6,11 @@ package wordbreaking type Flags byte const ( - SoftBreak = Flags(1 << iota) // potential linebreak point - WhiteSpace = Flags(1 << iota) // a unicode whitespace character, except NBSP - CharStop = Flags(1 << iota) // valid cursor position - WordStop = Flags(1 << iota) // start of a word - Invalid = Flags(1 << iota) // invalid character sequence - NoBreak = Flags(1 << iota) // do not break here + SoftBreak = Flags(1 << iota) // potential linebreak point + WhiteSpace = Flags(1 << iota) // a unicode whitespace character, except NBSP + CharStop = Flags(1 << iota) // valid cursor position + WordStop = Flags(1 << iota) // start of a word + Invalid = Flags(1 << iota) // invalid character sequence + NoBreak = Flags(1 << iota) // do not break here + MandatoryBreak = Flags(1 << iota) // break here regardless of available width ) diff --git a/wordbreaking/mark_rune_attributes.go b/wordbreaking/mark_rune_attributes.go index ec6b4f44..fcf9f902 100644 --- a/wordbreaking/mark_rune_attributes.go +++ b/wordbreaking/mark_rune_attributes.go @@ -6,6 +6,8 @@ package wordbreaking import ( "fmt" "unicode" + + "github.com/go-text/typesetting/segmenter" ) const ( @@ -58,30 +60,62 @@ func isCJKBreakRune(r rune) bool { ) } +// MarkRuneAttributes augments flags with UTF-8 byte-indexed text attributes. +// Default line opportunities follow Unicode 17 UAX #14; Thai dictionary +// boundaries are added as a language-specific tailoring. func MarkRuneAttributes(text string, flags []Flags) { if len(flags) < len(text) { panic(fmt.Sprintf("flags (len: %d) is smaller than text (len: %d)", len(flags), len(text))) } + if text == "" { + return + } + + runes := make([]rune, 0, len(text)) + byteOffsets := make([]int, 0, len(text)+1) thaiBreaks := thaiBreakOffsets(text) var rc, last runeClass for i, r := range text { + runes = append(runes, r) + byteOffsets = append(byteOffsets, i) flags[i] |= CharStop rc = classifyRune(r) if i == 0 { flags[i] |= WordStop } else if _, ok := thaiBreaks[i]; ok { - flags[i] |= SoftBreak | WordStop + flags[i] |= WordStop } else if rc == rcWhiteSpace { - flags[i] |= SoftBreak | WhiteSpace + flags[i] |= WhiteSpace } else if last == rcWhiteSpace && (rc == rcHyphen || rc == rcOther || rc == rcCJK) { - flags[i] |= SoftBreak | WordStop + flags[i] |= WordStop } else if last == rcHyphen && (rc == rcOther || rc == rcCJK) { - flags[i] |= SoftBreak | WordStop + flags[i] |= WordStop } else if last == rcCJK && rc == rcCJK { - flags[i] |= SoftBreak | WordStop + flags[i] |= WordStop } else if (last == rcOther && rc == rcCJK) || (last == rcCJK && rc == rcOther) { - flags[i] |= SoftBreak | WordStop + flags[i] |= WordStop } last = rc } + byteOffsets = append(byteOffsets, len(text)) + + var seg segmenter.Segmenter + seg.Init(runes) + iter := seg.LineIterator() + for iter.Next() { + line := iter.Line() + runeOffset := line.Offset + len(line.Text) + if runeOffset >= len(runes) { + continue + } + byteOffset := byteOffsets[runeOffset] + flags[byteOffset] |= SoftBreak + if line.IsMandatoryBreak { + flags[byteOffset] |= MandatoryBreak + } + } + + for byteOffset := range thaiBreaks { + flags[byteOffset] |= SoftBreak | WordStop + } } diff --git a/wordbreaking/mark_rune_attributes_test.go b/wordbreaking/mark_rune_attributes_test.go index 54cd7c5b..4b86b151 100644 --- a/wordbreaking/mark_rune_attributes_test.go +++ b/wordbreaking/mark_rune_attributes_test.go @@ -3,20 +3,83 @@ package wordbreaking -import "testing" +import ( + "slices" + "strings" + "testing" + "unicode/utf8" + + "github.com/go-text/typesetting/segmenter" +) + +func markedRuneAttributes(text string) []Flags { + flags := make([]Flags, len(text)) + MarkRuneAttributes(text, flags) + return flags +} + +func offsetsWithFlag(flags []Flags, flag Flags) []int { + offsets := make([]int, 0) + for offset, attrs := range flags { + if attrs&flag != 0 { + offsets = append(offsets, offset) + } + } + return offsets +} + +func testRuneByteOffsets(text string) []int { + offsets := make([]int, 0, utf8.RuneCountInString(text)+1) + for offset := range text { + offsets = append(offsets, offset) + } + return append(offsets, len(text)) +} + +func testUAX14LineBreakOffsets(text string) []int { + byteOffsets := testRuneByteOffsets(text) + var seg segmenter.Segmenter + seg.InitWithString(text) + iter := seg.LineIterator() + offsets := make([]int, 0) + for iter.Next() { + line := iter.Line() + runeOffset := line.Offset + len(line.Text) + if runeOffset < len(byteOffsets)-1 { + offsets = append(offsets, byteOffsets[runeOffset]) + } + } + return offsets +} + +func textWithBreakMarkers(text string, offsets []int) string { + breaks := make(map[int]struct{}, len(offsets)) + for _, offset := range offsets { + breaks[offset] = struct{}{} + } + var marked strings.Builder + marked.Grow(len(text) + len(offsets)) + for offset, r := range text { + if _, ok := breaks[offset]; ok { + marked.WriteByte('|') + } + marked.WriteRune(r) + } + return marked.String() +} func TestMarkRuneAttributes(t *testing.T) { const quick = "The quick red fox jumps over the lazy brown dog." var quickFlags = []Flags{ - CharStop | WordStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // The - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // quick - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // red - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // fox - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // jumps - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // over - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // the - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // lazy - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // brown + CharStop | WordStop, CharStop, CharStop, CharStop | WhiteSpace, // The + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace, // quick + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop | WhiteSpace, // red + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop | WhiteSpace, // fox + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace, // jumps + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace, // over + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop | WhiteSpace, // the + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace, // lazy + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace, // brown CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, // dog. } var flags [len(quick)]Flags @@ -33,11 +96,11 @@ func TestMarkRuneAttributes_with_hyphens(t *testing.T) { const hyphenTest = "Word-breaking test with regular and soft\u00ADhyphens." var hyphenFlags = []Flags{ CharStop | WordStop, CharStop, CharStop, CharStop, CharStop, // Word- - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // breaking - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // test - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // with - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // regular - CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop | WhiteSpace | SoftBreak, // and + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace, // breaking + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace, // test + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace, // with + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop | WhiteSpace, // regular + CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop | WhiteSpace, // and CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, 0, // soft- CharStop | SoftBreak | WordStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop, CharStop, // hyphens. } @@ -70,16 +133,228 @@ func TestMarkRuneAttributes_cjkBreaks(t *testing.T) { } } -// Baseline: 310 ns -// 833 ns -// 931 ns with hyphen tests -// 977 ns go1.2.1 -// 586 ns go1.6.2 mbp -func BenchmarkMarkRunAttributes(b *testing.B) { - const quick = "The quick red fox jumps over the lazy brown dog." - var flags [len(quick)]Flags +func TestMarkRuneAttributes_UTF8ByteOffsets(t *testing.T) { + const text = "Aé中😀B" + flags := markedRuneAttributes(text) + + if got, want := offsetsWithFlag(flags, CharStop), []int{0, 1, 3, 6, 10}; !slices.Equal(got, want) { + t.Fatalf("CharStop byte offsets = %v, want %v", got, want) + } + if got, want := offsetsWithFlag(flags, SoftBreak), []int{3, 6, 10}; !slices.Equal(got, want) { + t.Fatalf("SoftBreak byte offsets = %v, want %v", got, want) + } + for _, offset := range []int{2, 4, 5, 7, 8, 9} { + if flags[offset] != 0 { + t.Errorf("UTF-8 continuation byte at offset %d has flags %08b, want zero", offset, flags[offset]) + } + } +} + +func TestMarkRuneAttributes_BreakControls(t *testing.T) { + tests := []struct { + name string + text string + softBreaks []int + }{ + {name: "space", text: "word break", softBreaks: []int{5}}, + {name: "hyphen-minus", text: "well-being", softBreaks: []int{5}}, + {name: "soft hyphen", text: "well\u00adbeing", softBreaks: []int{6}}, + {name: "non-breaking hyphen", text: "well\u2011being"}, + {name: "no-break space", text: "well\u00a0being"}, + {name: "zero-width space", text: "well\u200bbeing", softBreaks: []int{7}}, + {name: "mixed Latin and CJK", text: "ABC中文DEF", softBreaks: []int{3, 6, 9}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + flags := markedRuneAttributes(tt.text) + if got := offsetsWithFlag(flags, SoftBreak); !slices.Equal(got, tt.softBreaks) { + t.Fatalf("SoftBreak byte offsets = %v, want %v", got, tt.softBreaks) + } + }) + } +} + +func TestMarkRuneAttributes_ThaiDictionaryBoundary(t *testing.T) { + const text = "เขาไป" + flags := markedRuneAttributes(text) + breakOffset := strings.Index(text, "ไป") + if flags[breakOffset]&SoftBreak == 0 || flags[breakOffset]&WordStop == 0 { + t.Fatalf("Thai dictionary boundary at byte offset %d has flags %08b", breakOffset, flags[breakOffset]) + } +} + +func TestMarkRuneAttributes_AugmentsCallerFlags(t *testing.T) { + flags := make([]Flags, len("ab")) + flags[0] = NoBreak + flags[1] = Invalid + + MarkRuneAttributes("ab", flags) + + if flags[0]&NoBreak == 0 || flags[0]&CharStop == 0 || flags[0]&WordStop == 0 { + t.Fatalf("first byte flags = %08b, want caller NoBreak plus rune attributes", flags[0]) + } + if flags[1]&Invalid == 0 || flags[1]&CharStop == 0 { + t.Fatalf("second byte flags = %08b, want caller Invalid plus CharStop", flags[1]) + } +} + +func TestMarkRuneAttributes_KoreanUsesUAX14Boundaries(t *testing.T) { + const text = "한국어 예문 이른 오후가 되면" + want := testUAX14LineBreakOffsets(text) + if got := offsetsWithFlag(markedRuneAttributes(text), SoftBreak); !slices.Equal(got, want) { + t.Fatalf("Korean SoftBreak byte offsets = %v, want UAX #14 boundaries %v", got, want) + } +} + +func TestMarkRuneAttributes_KoreanBoundaryBaseline(t *testing.T) { + const text = "한국어 예문: 이른 오후가 되면 골목의 카페에는 사람들이 늘어나고, 창가에 앉습니다." + leadtypeOffsets := offsetsWithFlag(markedRuneAttributes(text), SoftBreak) + uaxOffsets := testUAX14LineBreakOffsets(text) + leadtypeMarked := textWithBreakMarkers(text, leadtypeOffsets) + uaxMarked := textWithBreakMarkers(text, uaxOffsets) + + const wantUAX14 = "한|국|어 |예|문: |이|른 |오|후|가 |되|면 |골|목|의 |카|페|에|는 |사|람|들|이 |늘|어|나|고, |창|가|에 |앉|습|니|다." + if leadtypeMarked != wantUAX14 { + t.Errorf("Leadtype Korean boundaries = %q, want %q", leadtypeMarked, wantUAX14) + } + if uaxMarked != wantUAX14 { + t.Errorf("UAX #14 Korean boundaries = %q; update pinned baseline", uaxMarked) + } + t.Logf("Leadtype: %s", leadtypeMarked) + t.Logf("UAX #14: %s", uaxMarked) +} + +func TestMarkRuneAttributes_MatchesUAX14Boundaries(t *testing.T) { + tests := []struct { + name string + text string + }{ + {name: "Latin", text: "Leadtype wraps words and punctuation."}, + {name: "kana", text: "かなカナ文章"}, + {name: "Bopomofo", text: "中文ㄅㄆㄇ中文"}, + {name: "numeric", text: "価格は123.45%です。"}, + {name: "emoji ZWJ", text: "中👩‍👩‍👧‍👦文"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := offsetsWithFlag(markedRuneAttributes(tt.text), SoftBreak) + want := testUAX14LineBreakOffsets(tt.text) + if !slices.Equal(got, want) { + t.Fatalf("SoftBreak byte offsets = %v, want UAX #14 boundaries %v", got, want) + } + }) + } +} + +func TestMarkRuneAttributes_ProhibitsBreakBeforeClosingPunctuationAndNonstarters(t *testing.T) { + for _, r := range "，、。！？）】」』》〉々" { + t.Run(string(r), func(t *testing.T) { + text := "中" + string(r) + "文" + offset := len("中") + flags := markedRuneAttributes(text) + if flags[offset]&SoftBreak != 0 { + t.Fatalf("SoftBreak before %q at byte offset %d must be prohibited", r, offset) + } + }) + } +} + +func TestMarkRuneAttributes_ProhibitsBreakAfterOpeningPunctuation(t *testing.T) { + for _, r := range "（【「『《〈" { + t.Run(string(r), func(t *testing.T) { + text := string(r) + "中文" + offset := len(string(r)) + flags := markedRuneAttributes(text) + if flags[offset]&SoftBreak != 0 { + t.Fatalf("SoftBreak after %q at byte offset %d must be prohibited", r, offset) + } + }) + } +} + +func TestMarkRuneAttributes_ProhibitsBreakBeforeCombiningMark(t *testing.T) { + const text = "中\u0301文" + offset := len("中") + flags := markedRuneAttributes(text) + if flags[offset]&SoftBreak != 0 { + t.Fatalf("SoftBreak before combining mark at byte offset %d must be prohibited", offset) + } +} + +func TestMarkRuneAttributes_HonorsWordJoiner(t *testing.T) { + const text = "中\u2060文" + flags := markedRuneAttributes(text) + for _, offset := range []int{len("中"), len("中\u2060")} { + if flags[offset]&SoftBreak != 0 { + t.Errorf("SoftBreak adjacent to WORD JOINER at byte offset %d must be prohibited", offset) + } + } +} + +func TestMarkRuneAttributes_HonorsNarrowNoBreakSpace(t *testing.T) { + const text = "中\u202f文" + flags := markedRuneAttributes(text) + for _, offset := range []int{len("中"), len("中\u202f")} { + if flags[offset]&SoftBreak != 0 { + t.Errorf("SoftBreak adjacent to NARROW NO-BREAK SPACE at byte offset %d must be prohibited", offset) + } + } +} + +func TestMarkRuneAttributes_MandatoryBreaks(t *testing.T) { + tests := []struct { + name string + text string + offset int + }{ + {name: "LF", text: "alpha\nbeta", offset: len("alpha\n")}, + {name: "CRLF", text: "alpha\r\nbeta", offset: len("alpha\r\n")}, + {name: "NEL", text: "alpha\u0085beta", offset: len("alpha\u0085")}, + {name: "line separator", text: "alpha\u2028beta", offset: len("alpha\u2028")}, + {name: "paragraph separator", text: "alpha\u2029beta", offset: len("alpha\u2029")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + flags := markedRuneAttributes(tt.text) + if got := offsetsWithFlag(flags, MandatoryBreak); !slices.Equal(got, []int{tt.offset}) { + t.Fatalf("MandatoryBreak byte offsets = %v, want [%d]", got, tt.offset) + } + if flags[tt.offset]&SoftBreak == 0 { + t.Fatalf("mandatory boundary at byte offset %d is missing SoftBreak", tt.offset) + } + }) + } +} + +func TestMarkRuneAttributes_TrailingMandatoryBreakHasNoEOFSlot(t *testing.T) { + const text = "alpha\n" + if got := offsetsWithFlag(markedRuneAttributes(text), MandatoryBreak); len(got) != 0 { + t.Fatalf("MandatoryBreak byte offsets = %v, want none before EOF", got) + } +} + +func BenchmarkMarkRuneAttributes(b *testing.B) { + corpora := map[string]string{ + "Latin": "The quick red fox jumps over the lazy brown dog.", + "Chinese": "汉语示例：清晨的街道刚刚苏醒，卖早点的小店升起热气，行人提着公文包快步穿过路口。", + "Japanese": "日本語の例文：朝の駅前では、通勤する人々が改札へ向かい、パン屋からは焼きたての香りが流れてきます。", + "Korean": "한국어 예문: 이른 오후가 되면 골목의 카페에는 사람들의 대화가 조금씩 늘어납니다.", + "Thai": "ตัวอย่างภาษาไทย: ในช่วงบ่ายอากาศเริ่มอ่อนลง ร้านกาแฟเล็ก ๆ ริมถนนมีผู้คนแวะมานั่งพัก", + "Mixed": "中\u2060文 A\u00a0B A\u202fB 👩‍👩‍👧‍👦 ภาษาไทย", + } - for range b.N { - MarkRuneAttributes(quick, flags[:]) + for name, text := range corpora { + b.Run(name, func(b *testing.B) { + flags := make([]Flags, len(text)) + MarkRuneAttributes(text, flags) // Warm lazy Thai dictionary initialization. + b.ReportAllocs() + b.ResetTimer() + for range b.N { + MarkRuneAttributes(text, flags) + } + }) } } diff --git a/wordbreaking/thai_test.go b/wordbreaking/thai_test.go index b447a77a..5ec7fe32 100644 --- a/wordbreaking/thai_test.go +++ b/wordbreaking/thai_test.go @@ -6,6 +6,7 @@ package wordbreaking import ( "strings" "testing" + "unicode/utf8" ) func TestSegmentThai_noThai(t *testing.T) { @@ -53,7 +54,7 @@ func TestSegmentThai_markRuneAttributes(t *testing.T) { flags := make([]Flags, len(segmented)) MarkRuneAttributes(segmented, flags) - // Find the ZWS byte offset in the segmented string. + // UAX #14 places the break after ZWS, at the following rune's byte offset. zwsOffset := -1 for i, r := range segmented { if r == ZeroWidthSpace { @@ -64,8 +65,9 @@ func TestSegmentThai_markRuneAttributes(t *testing.T) { if zwsOffset < 0 { t.Fatal("expected ZWS in segmented Thai text, found none") } - if flags[zwsOffset]&SoftBreak == 0 { - t.Errorf("ZWS at offset %d missing SoftBreak flag, got %08b", zwsOffset, flags[zwsOffset]) + breakOffset := zwsOffset + utf8.RuneLen(ZeroWidthSpace) + if flags[breakOffset]&SoftBreak == 0 { + t.Errorf("boundary after ZWS at offset %d missing SoftBreak flag, got %08b", breakOffset, flags[breakOffset]) } }