diff --git a/diffmatchpatch/patch.go b/diffmatchpatch/patch.go index 0dbe3bd..dfb45ed 100644 --- a/diffmatchpatch/patch.go +++ b/diffmatchpatch/patch.go @@ -16,6 +16,7 @@ import ( "regexp" "strconv" "strings" + "unicode/utf8" ) // Patch represents one patch operation. @@ -145,6 +146,16 @@ func (dmp *DiffMatchPatch) patchMake2(text1 string, diffs []Diff) []Patch { return patches // Get rid of the null case. } + // DiffMain replaces invalid UTF-8 with the Unicode replacement character. + // Use that normalized source when it is exactly what the diffs describe, + // while preserving raw bytes in caller-supplied byte-level diffs. + if !utf8.ValidString(text1) { + diffText1 := dmp.DiffText1(diffs) + if string([]rune(text1)) == diffText1 { + text1 = diffText1 + } + } + patch := Patch{} charCount1 := 0 // Number of characters into the text1 string. charCount2 := 0 // Number of characters into the text2 string. diff --git a/diffmatchpatch/patch_test.go b/diffmatchpatch/patch_test.go index c564f8c..066c386 100644 --- a/diffmatchpatch/patch_test.go +++ b/diffmatchpatch/patch_test.go @@ -362,3 +362,25 @@ func TestPatchMakeOutOfRangePanic(t *testing.T) { patches := dmp.PatchMake(text1, text2) assert.Equal(t, 6, len(patches), "TestPatchMakeOutOfRangePanic") } + +func TestPatchMakeInvalidUTF8(t *testing.T) { + text1 := string([]byte{0xe0}) + normalizedText1 := string([]rune(text1)) + dmp := New() + diffs := dmp.DiffMain(text1, "", true) + + for _, patches := range [][]Patch{ + dmp.PatchMake(text1, ""), + dmp.PatchMake(text1, diffs), + } { + actual, applied := dmp.PatchApply(patches, normalizedText1) + assert.Equal(t, "", actual) + assert.Equal(t, []bool{true}, applied) + } + + // Handcrafted byte-level diffs should continue to use the original bytes. + patches := dmp.PatchMake(text1, []Diff{{DiffDelete, text1}}) + actual, applied := dmp.PatchApply(patches, text1) + assert.Equal(t, "", actual) + assert.Equal(t, []bool{true}, applied) +}