This PR rewrites the invisible unicode detection algorithm to more closely match that of the Monaco editor on the system. It provides a technique for detecting ambiguous characters and relaxes the detection of combining marks. Control characters are in addition detected as invisible in this implementation whereas they are not on monaco but this is related to font issues. Close #19913
Showing the changes to services/gitdiff/gitdiff.go (1 of 29 files). Show all
Unified · Split
| @@ -32,6 +32,7 @@ import ( · 31 unmodified lines | |||
| 32 | "code.gitea.io/gitea/modules/lfs" | 32 | "code.gitea.io/gitea/modules/lfs" |
| 33 | "code.gitea.io/gitea/modules/log" | 33 | "code.gitea.io/gitea/modules/log" |
| 34 | "code.gitea.io/gitea/modules/setting" | 34 | "code.gitea.io/gitea/modules/setting" |
| 35 | "code.gitea.io/gitea/modules/translation" | ||
| 35 | 36 | ||
| 36 | "github.com/sergi/go-diff/diffmatchpatch" | 37 | "github.com/sergi/go-diff/diffmatchpatch" |
| 37 | stdcharset "golang.org/x/net/html/charset" | 38 | stdcharset "golang.org/x/net/html/charset" |
| @@ -169,11 +170,11 @@ func getDiffLineSectionInfo(treePath, line string, lastLeftIdx, lastRightIdx int · 131 unmodified lines | |||
| 169 | } | 170 | } |
| 170 | 171 | ||
| 171 | // escape a line's content or return <br> needed for copy/paste purposes | 172 | // escape a line's content or return <br> needed for copy/paste purposes |
| 172 | func getLineContent(content string) DiffInline { | 173 | func getLineContent(content string, locale translation.Locale) DiffInline { |
| 173 | if len(content) > 0 { | 174 | if len(content) > 0 { |
| 174 | return DiffInlineWithUnicodeEscape(template.HTML(html.EscapeString(content))) | 175 | return DiffInlineWithUnicodeEscape(template.HTML(html.EscapeString(content)), locale) |
| 175 | } | 176 | } |
| 176 | return DiffInline{Content: "<br>"} | 177 | return DiffInline{EscapeStatus: &charset.EscapeStatus{}, Content: "<br>"} |
| 177 | } | 178 | } |
| 178 | 179 | ||
| 179 | // DiffSection represents a section of a DiffFile. | 180 | // DiffSection represents a section of a DiffFile. |
| @@ -267,26 +268,26 @@ func init() { · 87 unmodified lines | |||
| 267 | 268 | ||
| 268 | // DiffInline is a struct that has a content and escape status | 269 | // DiffInline is a struct that has a content and escape status |
| 269 | type DiffInline struct { | 270 | type DiffInline struct { |
| 270 | EscapeStatus charset.EscapeStatus | 271 | EscapeStatus *charset.EscapeStatus |
| 271 | Content template.HTML | 272 | Content template.HTML |
| 272 | } | 273 | } |
| 273 | 274 | ||
| 274 | // DiffInlineWithUnicodeEscape makes a DiffInline with hidden unicode characters escaped | 275 | // DiffInlineWithUnicodeEscape makes a DiffInline with hidden unicode characters escaped |
| 275 | func DiffInlineWithUnicodeEscape(s template.HTML) DiffInline { | 276 | func DiffInlineWithUnicodeEscape(s template.HTML, locale translation.Locale) DiffInline { |
| 276 | status, content := charset.EscapeControlString(string(s)) | 277 | status, content := charset.EscapeControlHTML(string(s), locale) |
| 277 | return DiffInline{EscapeStatus: status, Content: template.HTML(content)} | 278 | return DiffInline{EscapeStatus: status, Content: template.HTML(content)} |
| 278 | } | 279 | } |
| 279 | 280 | ||
| 280 | // DiffInlineWithHighlightCode makes a DiffInline with code highlight and hidden unicode characters escaped | 281 | // DiffInlineWithHighlightCode makes a DiffInline with code highlight and hidden unicode characters escaped |
| 281 | func DiffInlineWithHighlightCode(fileName, language, code string) DiffInline { | 282 | func DiffInlineWithHighlightCode(fileName, language, code string, locale translation.Locale) DiffInline { |
| 282 | status, content := charset.EscapeControlString(highlight.Code(fileName, language, code)) | 283 | status, content := charset.EscapeControlHTML(highlight.Code(fileName, language, code), locale) |
| 283 | return DiffInline{EscapeStatus: status, Content: template.HTML(content)} | 284 | return DiffInline{EscapeStatus: status, Content: template.HTML(content)} |
| 284 | } | 285 | } |
| 285 | 286 | ||
| 286 | // GetComputedInlineDiffFor computes inline diff for the given line. | 287 | // GetComputedInlineDiffFor computes inline diff for the given line. |
| 287 | func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) DiffInline { | 288 | func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine, locale translation.Locale) DiffInline { |
| 288 | if setting.Git.DisableDiffHighlight { | 289 | if setting.Git.DisableDiffHighlight { |
| 289 | return getLineContent(diffLine.Content[1:]) | 290 | return getLineContent(diffLine.Content[1:], locale) |
| 290 | } | 291 | } |
| 291 | 292 | ||
| 292 | var ( | 293 | var ( |
| @@ -303,26 +304,26 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) Dif · 10 unmodified lines | |||
| 303 | // try to find equivalent diff line. ignore, otherwise | 304 | // try to find equivalent diff line. ignore, otherwise |
| 304 | switch diffLine.Type { | 305 | switch diffLine.Type { |
| 305 | case DiffLineSection: | 306 | case DiffLineSection: |
| 306 | return getLineContent(diffLine.Content[1:]) | 307 | return getLineContent(diffLine.Content[1:], locale) |
| 307 | case DiffLineAdd: | 308 | case DiffLineAdd: |
| 308 | compareDiffLine = diffSection.GetLine(DiffLineDel, diffLine.RightIdx) | 309 | compareDiffLine = diffSection.GetLine(DiffLineDel, diffLine.RightIdx) |
| 309 | if compareDiffLine == nil { | 310 | if compareDiffLine == nil { |
| 310 | return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:]) | 311 | return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:], locale) |
| 311 | } | 312 | } |
| 312 | diff1 = compareDiffLine.Content | 313 | diff1 = compareDiffLine.Content |
| 313 | diff2 = diffLine.Content | 314 | diff2 = diffLine.Content |
| 314 | case DiffLineDel: | 315 | case DiffLineDel: |
| 315 | compareDiffLine = diffSection.GetLine(DiffLineAdd, diffLine.LeftIdx) | 316 | compareDiffLine = diffSection.GetLine(DiffLineAdd, diffLine.LeftIdx) |
| 316 | if compareDiffLine == nil { | 317 | if compareDiffLine == nil { |
| 317 | return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:]) | 318 | return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:], locale) |
| 318 | } | 319 | } |
| 319 | diff1 = diffLine.Content | 320 | diff1 = diffLine.Content |
| 320 | diff2 = compareDiffLine.Content | 321 | diff2 = compareDiffLine.Content |
| 321 | default: | 322 | default: |
| 322 | if strings.IndexByte(" +-", diffLine.Content[0]) > -1 { | 323 | if strings.IndexByte(" +-", diffLine.Content[0]) > -1 { |
| 323 | return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:]) | 324 | return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:], locale) |
| 324 | } | 325 | } |
| 325 | return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content) | 326 | return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content, locale) |
| 326 | } | 327 | } |
| 327 | 328 | ||
| 328 | hcd := newHighlightCodeDiff() | 329 | hcd := newHighlightCodeDiff() |
| @@ -330,7 +331,7 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) Dif · 1 unmodified line | |||
| 330 | // it seems that Gitea doesn't need the line wrapper of Chroma, so do not add them back | 331 | // it seems that Gitea doesn't need the line wrapper of Chroma, so do not add them back |
| 331 | // if the line wrappers are still needed in the future, it can be added back by "diffToHTML(hcd.lineWrapperTags. ...)" | 332 | // if the line wrappers are still needed in the future, it can be added back by "diffToHTML(hcd.lineWrapperTags. ...)" |
| 332 | diffHTML := diffToHTML(nil, diffRecord, diffLine.Type) | 333 | diffHTML := diffToHTML(nil, diffRecord, diffLine.Type) |
| 333 | return DiffInlineWithUnicodeEscape(template.HTML(diffHTML)) | 334 | return DiffInlineWithUnicodeEscape(template.HTML(diffHTML), locale) |
| 334 | } | 335 | } |
| 335 | 336 | ||
| 336 | // DiffFile represents a file diff. | 337 | // DiffFile represents a file diff. |