History

Switch Unicode Escaping to a VSCode-like system (#19990)

This PR rewrites the invisible unicode detection algorithm to more closely match that of the Monaco editor on the system. It provides a technique for detecting ambiguous characters and relaxes the detection of combining marks. Control characters are in addition detected as invisible in this implementation whereas they are not on monaco but this is related to font issues. Close #19913

zeripath committed 4 years ago

Commit 99efa021 parent 11dc6df29 files · +2106 −370Browse files

Showing the changes to services/gitdiff/gitdiff.go (1 of 29 files). Show all

Unified · Split

services/gitdiff/gitdiff.go+17−16
@@ -32,6 +32,7 @@ import ( · 31 unmodified lines
3232 "code.gitea.io/gitea/modules/lfs"
3333 "code.gitea.io/gitea/modules/log"
3434 "code.gitea.io/gitea/modules/setting"
35+ "code.gitea.io/gitea/modules/translation"
3536
3637 "github.com/sergi/go-diff/diffmatchpatch"
3738 stdcharset "golang.org/x/net/html/charset"
@@ -169,11 +170,11 @@ func getDiffLineSectionInfo(treePath, line string, lastLeftIdx, lastRightIdx int · 131 unmodified lines
169170 }
170171
171172 // escape a line's content or return <br> needed for copy/paste purposes
172−func getLineContent(content string) DiffInline {
173+func getLineContent(content string, locale translation.Locale) DiffInline {
173174 if len(content) > 0 {
174− return DiffInlineWithUnicodeEscape(template.HTML(html.EscapeString(content)))
175+ return DiffInlineWithUnicodeEscape(template.HTML(html.EscapeString(content)), locale)
175176 }
176− return DiffInline{Content: "<br>"}
177+ return DiffInline{EscapeStatus: &charset.EscapeStatus{}, Content: "<br>"}
177178 }
178179
179180 // DiffSection represents a section of a DiffFile.
@@ -267,26 +268,26 @@ func init() { · 87 unmodified lines
267268
268269 // DiffInline is a struct that has a content and escape status
269270 type DiffInline struct {
270− EscapeStatus charset.EscapeStatus
271+ EscapeStatus *charset.EscapeStatus
271272 Content template.HTML
272273 }
273274
274275 // DiffInlineWithUnicodeEscape makes a DiffInline with hidden unicode characters escaped
275−func DiffInlineWithUnicodeEscape(s template.HTML) DiffInline {
276− status, content := charset.EscapeControlString(string(s))
276+func DiffInlineWithUnicodeEscape(s template.HTML, locale translation.Locale) DiffInline {
277+ status, content := charset.EscapeControlHTML(string(s), locale)
277278 return DiffInline{EscapeStatus: status, Content: template.HTML(content)}
278279 }
279280
280281 // DiffInlineWithHighlightCode makes a DiffInline with code highlight and hidden unicode characters escaped
281−func DiffInlineWithHighlightCode(fileName, language, code string) DiffInline {
282− status, content := charset.EscapeControlString(highlight.Code(fileName, language, code))
282+func DiffInlineWithHighlightCode(fileName, language, code string, locale translation.Locale) DiffInline {
283+ status, content := charset.EscapeControlHTML(highlight.Code(fileName, language, code), locale)
283284 return DiffInline{EscapeStatus: status, Content: template.HTML(content)}
284285 }
285286
286287 // GetComputedInlineDiffFor computes inline diff for the given line.
287−func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) DiffInline {
288+func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine, locale translation.Locale) DiffInline {
288289 if setting.Git.DisableDiffHighlight {
289− return getLineContent(diffLine.Content[1:])
290+ return getLineContent(diffLine.Content[1:], locale)
290291 }
291292
292293 var (
@@ -303,26 +304,26 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) Dif · 10 unmodified lines
303304 // try to find equivalent diff line. ignore, otherwise
304305 switch diffLine.Type {
305306 case DiffLineSection:
306− return getLineContent(diffLine.Content[1:])
307+ return getLineContent(diffLine.Content[1:], locale)
307308 case DiffLineAdd:
308309 compareDiffLine = diffSection.GetLine(DiffLineDel, diffLine.RightIdx)
309310 if compareDiffLine == nil {
310− return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:])
311+ return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:], locale)
311312 }
312313 diff1 = compareDiffLine.Content
313314 diff2 = diffLine.Content
314315 case DiffLineDel:
315316 compareDiffLine = diffSection.GetLine(DiffLineAdd, diffLine.LeftIdx)
316317 if compareDiffLine == nil {
317− return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:])
318+ return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:], locale)
318319 }
319320 diff1 = diffLine.Content
320321 diff2 = compareDiffLine.Content
321322 default:
322323 if strings.IndexByte(" +-", diffLine.Content[0]) > -1 {
323− return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:])
324+ return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:], locale)
324325 }
325− return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content)
326+ return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content, locale)
326327 }
327328
328329 hcd := newHighlightCodeDiff()
@@ -330,7 +331,7 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) Dif · 1 unmodified line
330331 // it seems that Gitea doesn't need the line wrapper of Chroma, so do not add them back
331332 // if the line wrappers are still needed in the future, it can be added back by "diffToHTML(hcd.lineWrapperTags. ...)"
332333 diffHTML := diffToHTML(nil, diffRecord, diffLine.Type)
333− return DiffInlineWithUnicodeEscape(template.HTML(diffHTML))
334+ return DiffInlineWithUnicodeEscape(template.HTML(diffHTML), locale)
334335 }
335336
336337 // DiffFile represents a file diff.