Integrate proper Unicode UAX #14 line breaking.

rich_text/rich_text.go

@@ -677,10 +677,14 @@ func (piece *RichText) TrimRightSpace() *RichText {
 	return piece.TrimRightFunc(unicode.IsSpace)
 }
 
-// TrimSpace trims the runes from both ends of the text where unicode.IsSpace returns true,
+// TrimSpace trims Unicode whitespace and zero-width spaces from both ends of the text,
 // returning a new structure and leaving the original unchanged.
 func (piece *RichText) TrimSpace() *RichText {
-	return piece.TrimFunc(unicode.IsSpace)
+	return piece.TrimFunc(isLineEdgeSpace)
+}
+
+func isLineEdgeSpace(r rune) bool {
+	return unicode.IsSpace(r) || r == wordbreaking.ZeroWidthSpace
 }
 
 // VisitAll invokes the callback function for each piece of text within the structure, in order.
@@ -779,10 +783,10 @@ func decorationOverridesOption(opts options.Options, key string) *DecorationOver
 
 // WordsToWidth splits the text at the last breaking point before width is exceeded, making use of the previously-allocated and marked flags
 // and returning new structures representing both halves and the remaining flags. The original structure is unchanged.
-// A minimum of one word is split off, even if width is exceeded, unless hardBreak is true, in which case the text is split along character
-// boundaries.
+// A minimum of one word is split off, even if width is exceeded, unless emergencyBreak is true, in which case the text is split along
+// character boundaries.
 func (piece *RichText) WordsToWidth(
-	width float64, wordFlags []wordbreaking.Flags, hardBreak bool) (
+	width float64, wordFlags []wordbreaking.Flags, emergencyBreak bool) (
 	line, remainder *RichText, remainderFlags []wordbreaking.Flags) {
 	if width < 0.0 {
 		width = 0.0
@@ -791,6 +795,7 @@ func (piece *RichText) WordsToWidth(
 	currentWidth := 0.0
 	words := 0
 	wordWidth := 0.0
+	trailingSpaceWidth := 0.0
 	extra := 0.0
 	lastOffset := 0
 
@@ -804,19 +809,28 @@ func (piece *RichText) WordsToWidth(
 	var leafRuneIdx int
 
 	fn := func(r rune, p *RichText, offset int) bool {
-		if words > 0 && currentWidth+extra+wordWidth > width {
+		if offset > 0 && (wordFlags[offset]&wordbreaking.MandatoryBreak) == wordbreaking.MandatoryBreak {
+			current = offset
+			return true
+		}
+		softBreak := offset > 0 &&
+			wordFlags[offset]&wordbreaking.SoftBreak == wordbreaking.SoftBreak &&
+			wordFlags[offset]&wordbreaking.NoBreak != wordbreaking.NoBreak
+		candidateWidth := currentWidth + extra + wordWidth
+		if softBreak {
+			candidateWidth -= trailingSpaceWidth
+		}
+		if words > 0 && candidateWidth > width {
 			return true
-		} else if words == 0 && hardBreak && wordWidth > width {
+		} else if words == 0 && emergencyBreak && wordWidth > width {
 			if lastOffset > 0 {
 				current = lastOffset
 			} else {
 				current = offset
 			}
 			return true
 		}
-		if offset > 0 &&
-			wordFlags[offset]&wordbreaking.SoftBreak == wordbreaking.SoftBreak &&
-			wordFlags[offset]&wordbreaking.NoBreak != wordbreaking.NoBreak {
+		if softBreak {
 			current = offset
 			currentWidth += wordWidth
 			wordWidth = 0.0
@@ -852,17 +866,24 @@ func (piece *RichText) WordsToWidth(
 				shapedAdv = nil
 			}
 		}
+		runeWidth := 0.0
 		if r != wordbreaking.SoftHyphen {
 			if shapedAdv != nil && leafRuneIdx < len(shapedAdv) {
-				wordWidth += shapedAdv[leafRuneIdx] + p.CharSpacing
+				runeWidth = shapedAdv[leafRuneIdx] + p.CharSpacing
 			} else {
-				runeWidth, _ := metrics.AdvanceWidth(r)
-				wordWidth += (fsize * float64(runeWidth)) + p.CharSpacing
+				advanceWidth, _ := metrics.AdvanceWidth(r)
+				runeWidth = (fsize * float64(advanceWidth)) + p.CharSpacing
 			}
 			if unicode.IsSpace(r) {
-				wordWidth += p.WordSpacing
+				runeWidth += p.WordSpacing
 			}
 		}
+		wordWidth += runeWidth
+		if isLineEdgeSpace(r) {
+			trailingSpaceWidth += runeWidth
+		} else {
+			trailingSpaceWidth = 0.0
+		}
 		leafRuneIdx++
 		lastRune = r
 		lastOffset = offset
@@ -884,11 +905,11 @@ func (piece *RichText) WordsToWidth(
 }
 
 // WrapToWidth returns one or more lines of text resulting from repeatedly invoking WordsToWidth.
-func (piece *RichText) WrapToWidth(width float64, wordFlags []wordbreaking.Flags, hardBreak bool) (lines []*RichText) {
-	line, remainder, remainderFlags := piece.WordsToWidth(width, wordFlags, hardBreak)
+func (piece *RichText) WrapToWidth(width float64, wordFlags []wordbreaking.Flags, emergencyBreak bool) (lines []*RichText) {
+	line, remainder, remainderFlags := piece.WordsToWidth(width, wordFlags, emergencyBreak)
 	for remainder != nil {
 		lines = append(lines, line.TrimSpace())
-		line, remainder, remainderFlags = remainder.WordsToWidth(width, remainderFlags, hardBreak)
+		line, remainder, remainderFlags = remainder.WordsToWidth(width, remainderFlags, emergencyBreak)
 	}
 	lines = append(lines, line.TrimSpace())
 	return

rich_text/rich_text_test.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"io"
 	"os"
+	"slices"
 	"strings"
 	"testing"
 	"unicode"
@@ -937,6 +938,13 @@ func TestRichText_TrimSpace(t *testing.T) {
 	st.Equal(leadingAndTrailingWhitespaceTextTrimmed, t2.String())
 }
 
+func TestRichText_TrimSpace_IncludesZeroWidthSpace(t *testing.T) {
+	t1 := &RichText{Text: "\u200bcontent\u200b"}
+	if got := t1.TrimSpace().String(); got != "content" {
+		t.Fatalf("TrimSpace() = %q, want %q", got, "content")
+	}
+}
+
 func TestRichText_TrimRightFunc_complex(t *testing.T) {
 	skipIfNoTTFFonts(t)
 	st := SuperTest{t}
@@ -1048,7 +1056,7 @@ func TestRichText_WordsToWidth_mixed(t *testing.T) {
 	}
 }
 
-func TestRichText_WordsToWidth_hardbreak(t *testing.T) {
+func TestRichText_WordsToWidth_emergencyBreak(t *testing.T) {
 	skipIfNoTTFFonts(t)
 	st := SuperTest{t}
 	p := arialText("Supercalifragilisticexpialidocious")
@@ -1076,6 +1084,69 @@ func TestRichText_WordsToWidth_zero(t *testing.T) {
 	st.Equal(33, len(remainderFlags))
 }
 
+func TestRichText_WordsToWidthAndWrapToWidth_WhitespaceContract(t *testing.T) {
+	font := minimalFixtureFont(t)
+	rt := (&RichText{Text: "alpha beta", Font: font, FontSize: 12}).measure()
+	prefix := (&RichText{Text: "alpha", Font: font, FontSize: 12}).measure()
+	flags := make([]wordbreaking.Flags, rt.Len())
+	wordbreaking.MarkRuneAttributes(rt.String(), flags)
+
+	line, remainder, _ := rt.WordsToWidth(prefix.Width()+0.01, flags, false)
+	if got := line.String(); got != "alpha " {
+		t.Fatalf("WordsToWidth line = %q, want %q", got, "alpha ")
+	}
+	if remainder == nil {
+		t.Fatal("WordsToWidth remainder is nil")
+	}
+	if got := remainder.String(); got != "beta" {
+		t.Fatalf("WordsToWidth remainder = %q, want %q", got, "beta")
+	}
+
+	lines := rt.WrapToWidth(prefix.Width()+0.01, flags, false)
+	if len(lines) != 2 || lines[0].String() != "alpha" || lines[1].String() != "beta" {
+		got := make([]string, len(lines))
+		for i, wrapped := range lines {
+			got[i] = wrapped.String()
+		}
+		t.Fatalf("WrapToWidth lines = %q, want [\"alpha\" \"beta\"]", got)
+	}
+}
+
+func TestRichText_WrapToWidth_TrailingSpaceDoesNotCauseEarlyWrap(t *testing.T) {
+	font := minimalFixtureFont(t)
+	tests := []struct {
+		name        string
+		text        string
+		firstLine   string
+		separator   string
+		wordSpacing float64
+	}{
+		{name: "single space", text: "alpha beta gamma", firstLine: "alpha beta", separator: " "},
+		{name: "consecutive spaces", text: "alpha  beta  gamma", firstLine: "alpha  beta", separator: "  "},
+		{name: "word spacing", text: "alpha beta gamma", firstLine: "alpha beta", separator: " ", wordSpacing: 4},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rt := &RichText{Text: tt.text, Font: font, FontSize: 12, WordSpacing: tt.wordSpacing}
+			prefix := &RichText{Text: tt.firstLine, Font: font, FontSize: 12, WordSpacing: tt.wordSpacing}
+			trailing := &RichText{Text: tt.separator, Font: font, FontSize: 12, WordSpacing: tt.wordSpacing}
+			width := prefix.Width() + trailing.Width()/2
+			flags := make([]wordbreaking.Flags, rt.Len())
+			wordbreaking.MarkRuneAttributes(rt.String(), flags)
+
+			lines := rt.WrapToWidth(width, flags, false)
+			got := make([]string, len(lines))
+			for i, line := range lines {
+				got[i] = line.String()
+			}
+			if want := []string{tt.firstLine, "gamma"}; !slices.Equal(got, want) {
+				t.Fatalf("lines = %q, want %q (width %.3f)", got, want, width)
+			}
+		})
+	}
+}
+
 func TestRichText_WordsToWidth_LogsShapingFailure(t *testing.T) {
 	p := &RichText{
 		Text:     "مرحبا بالعالم",
@@ -1247,7 +1318,104 @@ func TestRichText_WrapToWidth_thai(t *testing.T) {
 	}
 }
 
-func TestRichText_WrapToWidth_hardBreak(t *testing.T) {
+func TestRichText_WrapToWidth_MandatoryBreaks(t *testing.T) {
+	font := minimalFixtureFont(t)
+	tests := []struct {
+		name string
+		text string
+		want []string
+	}{
+		{name: "LF", text: "alpha\nbeta", want: []string{"alpha", "beta"}},
+		{name: "CRLF", text: "alpha\r\nbeta", want: []string{"alpha", "beta"}},
+		{name: "NEL", text: "alpha\u0085beta", want: []string{"alpha", "beta"}},
+		{name: "line separator", text: "alpha\u2028beta", want: []string{"alpha", "beta"}},
+		{name: "paragraph separator", text: "alpha\u2029beta", want: []string{"alpha", "beta"}},
+		{name: "leading", text: "\nalpha", want: []string{"", "alpha"}},
+		{name: "consecutive", text: "alpha\n\nbeta", want: []string{"alpha", "", "beta"}},
+		{name: "trailing", text: "alpha\n", want: []string{"alpha"}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rt := (&RichText{Text: tt.text, Font: font, FontSize: 12}).measure()
+			flags := make([]wordbreaking.Flags, rt.Len())
+			wordbreaking.MarkRuneAttributes(rt.String(), flags)
+
+			lines := rt.WrapToWidth(rt.Width()+100, flags, false)
+			got := make([]string, len(lines))
+			for i, line := range lines {
+				got[i] = line.String()
+			}
+			if !slices.Equal(got, tt.want) {
+				t.Fatalf("mandatory-break lines = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRichText_WrapToWidth_MandatoryBreakOverridesNoBreak(t *testing.T) {
+	font := minimalFixtureFont(t)
+	rt := (&RichText{Text: "alpha\nbeta", Font: font, FontSize: 12, NoBreak: true}).measure()
+	flags := make([]wordbreaking.Flags, rt.Len())
+	wordbreaking.MarkRuneAttributes(rt.String(), flags)
+	rt.MarkNoBreak(flags)
+
+	lines := rt.WrapToWidth(rt.Width()+100, flags, false)
+	got := make([]string, len(lines))
+	for i, line := range lines {
+		got[i] = line.String()
+	}
+	if want := []string{"alpha", "beta"}; !slices.Equal(got, want) {
+		t.Fatalf("mandatory-break lines = %q, want %q", got, want)
+	}
+}
+
+func TestRichText_WrapToWidth_TrimsSelectedZeroWidthSpace(t *testing.T) {
+	font := minimalFixtureFont(t)
+	rt := (&RichText{Text: "alpha\u200bbeta", Font: font, FontSize: 12}).measure()
+	prefix := (&RichText{Text: "alpha", Font: font, FontSize: 12}).measure()
+	flags := make([]wordbreaking.Flags, rt.Len())
+	wordbreaking.MarkRuneAttributes(rt.String(), flags)
+
+	lines := rt.WrapToWidth(prefix.Width()+0.01, flags, false)
+	got := make([]string, len(lines))
+	for i, line := range lines {
+		got[i] = line.String()
+	}
+	if want := []string{"alpha", "beta"}; !slices.Equal(got, want) {
+		t.Fatalf("zero-width-space lines = %q, want %q", got, want)
+	}
+}
+
+func TestRichText_WrapToWidth_RespectsCJKPunctuation(t *testing.T) {
+	font := minimalFixtureFont(t)
+	const text = "中文，中文々中文（中文）中文"
+	rt := (&RichText{Text: text, Font: font, FontSize: 12}).measure()
+	twoIdeographs := (&RichText{Text: "中文", Font: font, FontSize: 12}).measure()
+	flags := make([]wordbreaking.Flags, rt.Len())
+	wordbreaking.MarkRuneAttributes(rt.String(), flags)
+
+	lines := rt.WrapToWidth(twoIdeographs.Width()+0.01, flags, false)
+	if len(lines) < 2 {
+		t.Fatalf("CJK text produced %d line, want multiple lines", len(lines))
+	}
+	for _, line := range lines {
+		lineText := line.String()
+		lineRunes := []rune(lineText)
+		if len(lineRunes) == 0 {
+			t.Error("CJK wrapping produced an empty line")
+			continue
+		}
+		if strings.ContainsRune("，、。！？）】」』》〉々", lineRunes[0]) {
+			t.Errorf("line begins with prohibited punctuation: %q", lineText)
+		}
+		if strings.ContainsRune("（【「『《〈", lineRunes[len(lineRunes)-1]) {
+			t.Errorf("line ends with opening punctuation: %q", lineText)
+		}
+	}
+}
+
+func TestRichText_WrapToWidth_emergencyBreak(t *testing.T) {
 	skipIfNoTTFFonts(t)
 	st := SuperTest{t}
 	expected := []string{

wordbreaking/flags.go

@@ -6,10 +6,11 @@ package wordbreaking
 type Flags byte
 
 const (
-	SoftBreak  = Flags(1 << iota) // potential linebreak point
-	WhiteSpace = Flags(1 << iota) // a unicode whitespace character, except NBSP
-	CharStop   = Flags(1 << iota) // valid cursor position
-	WordStop   = Flags(1 << iota) // start of a word
-	Invalid    = Flags(1 << iota) // invalid character sequence
-	NoBreak    = Flags(1 << iota) // do not break here
+	SoftBreak      = Flags(1 << iota) // potential linebreak point
+	WhiteSpace     = Flags(1 << iota) // a unicode whitespace character, except NBSP
+	CharStop       = Flags(1 << iota) // valid cursor position
+	WordStop       = Flags(1 << iota) // start of a word
+	Invalid        = Flags(1 << iota) // invalid character sequence
+	NoBreak        = Flags(1 << iota) // do not break here
+	MandatoryBreak = Flags(1 << iota) // break here regardless of available width
 )