History

Add support for double-width cells and ideographs (CJK symbols)

- Add -dwfont option to specify the name of the double-width font. The default setting '18x18ja' matches the default main font '9x18', otherwise a font with matching geometry must be specified. - If the double-width font is not present or cannot be loaded (e.g., because its geometry does not match the main font), double-width characters will obviously not appear on screen, but spacing will still be correct and copy-pasting them will still work (as we have the unicode points, we just cannot render them). - Unicode display correctness has marginally improved in a few cases: U+0080 was drawn as a missing glyph (hinting the user that a better font would perhaps make it visible), whereas it is really a control character with no valid visual, hence the Unicode Replacement Char should be shown. The opposite change happened to U+10000 which is in fact a valid (if uncommon) character, but was rendered as RC. U+D7FF is now also correctly shown as a RC (as it belongs to a reserved range). Finally, the same correction was applied to the non-characters in the range U+FDD0..U+FDEF. - As a bonus, the Fontpack now releases all fonts from host memory (previously held for the rest of the program's life) once all font data is loaded into GL. This frees several megabytes of host RAM (the issue became more pronounced with CJK fonts having tens of thousands of glyphs).

Tom Szilagyi committed 5 years ago

Commit bf524791 parent 4035b3022 files · +660 −190Browse files

Showing the changes to src/charvdev.cc (1 of 22 files). Show all

Unified · Split

src/charvdev.cc+187−82
@@ -25,6 +25,8 @@ layout (local_size_x = 1, local_size_y = 1) in; · 24 unmodified lines
2525 layout (rgba32f, binding = 0) writeonly lowp uniform image2D imgOut;
2626 layout (binding = 1) uniform lowp sampler2DArray atlas;
2727 layout (binding = 2) uniform lowp sampler2D atlasMap;
28+layout (binding = 3) uniform lowp sampler2DArray atlas_dw;
29+layout (binding = 4) uniform lowp sampler2D atlasMap_dw;
2830 uniform lowp ivec2 glyphPixels;
2931 uniform lowp ivec2 sizeChars;
3032 uniform lowp ivec3 cursorColor;
@@ -35,6 +37,7 @@ uniform lowp int selectRectMode; · 4 unmodified lines
3537 uniform highp ivec2 selectDamage;
3638 uniform lowp int deltaFrame;
3739 uniform lowp int showWraps;
40+uniform lowp int hasDoubleWidth;
3841
3942 struct Cell
4043 {
@@ -56,25 +59,42 @@ void main () · 15 unmodified lines
5659
5760 if (deltaFrame == 1)
5861 {
59− uint dirty = bitfieldExtract (cell.charData, 21, 1);
62+ uint dirty = bitfieldExtract (cell.charData, 23, 1);
6063 if (dirty == 0u &&
6164 charPos != cursorPos.xy && charPos != cursorPos.zw &&
6265 (idx < selectDamage.x || idx >= selectDamage.y))
6366 return;
6467 }
65− vmem.cells[idx].charData = bitfieldInsert (cell.charData, 0u, 21, 1);
68+ vmem.cells[idx].charData = bitfieldInsert (cell.charData, 0u, 23, 1);
6669
6770 ivec2 charCode =
6871 ivec2 (bitfieldExtract (cell.charData, 0, 8), // Lowest byte
6972 bitfieldExtract (cell.charData, 8, 8)); // Next-lowest byte
7073
71− // fontIdx == 0 -> Normal; 1 -> Bold; 2 -> Italic; 3 -> BoldItalic
72− uint fontIdx = bitfieldExtract (cell.charData, 16, 2);
73− uint underline = bitfieldExtract (cell.charData, 18, 1);
74− uint inverse = bitfieldExtract (cell.charData, 19, 1);
75− uint wrap = bitfieldExtract (cell.charData, 20, 1);
74+ uint dwidth = bitfieldExtract (cell.charData, 16, 1);
75+ uint dwidth_cont = bitfieldExtract (cell.charData, 17, 1);
76+ if (dwidth_cont == 1u) // double-width cell continuation - drawn by left half
77+ return;
7678
77− ivec2 atlasPos = ivec2 (vec2 (256) * texelFetch (atlasMap, charCode, 0).zw);
79+ if (dwidth == 1u && charPos.x < sizeChars.x - 1)
80+ {
81+ // check validity (dwidth_cont marker in the cell to the right)
82+ if (bitfieldExtract (vmem.cells[idx + 1].charData, 17, 1) != 1u)
83+ dwidth = 0u;
84+ }
85+
86+ uint fontIdx = 0u; // 0 -> Normal; 1 -> Bold; 2 -> Italic; 3 -> BoldItalic
87+ if (dwidth == 0u)
88+ fontIdx = bitfieldExtract (cell.charData, 18, 2);
89+ uint underline = bitfieldExtract (cell.charData, 20, 1);
90+ uint inverse = bitfieldExtract (cell.charData, 21, 1);
91+ uint wrap = bitfieldExtract (cell.charData, 22, 1);
92+
93+ ivec2 atlasPos;
94+ if (dwidth == 0u)
95+ atlasPos = ivec2 (vec2 (256) * texelFetch (atlasMap, charCode, 0).zw);
96+ else
97+ atlasPos = ivec2 (vec2 (256) * texelFetch (atlasMap_dw, charCode, 0).zw);
7898
7999 vec3 fgColor = vec3 (float (bitfieldExtract (cell.fg, 0, 8)),
80100 float (bitfieldExtract (cell.fg, 8, 8)),
@@ -115,25 +135,66 @@ void main () · 34 unmodified lines
115135 bgColor = crColor;
116136 }
117137
118− for (int j = 0; j < glyphPixels.x; j++)
119− {
120− for (int k = 0; k < glyphPixels.y; k++)
138+ ivec2 srcGlyphPixels = glyphPixels;
139+ if (dwidth == 1u)
140+ srcGlyphPixels = ivec2 (2, 1) * glyphPixels;
141+
142+ if (dwidth == 0u)
143+ { // render regular cell
144+ for (int j = 0; j < glyphPixels.x; j++)
121145 {
122− ivec2 txCoords = atlasPos * glyphPixels + ivec2 (j, k);
123− ivec3 txc = ivec3 (txCoords, fontIdx);
124− float lumi = texelFetch (atlas, txc, 0).r;
125− vec4 pixel = vec4 (fgColor * lumi + bgColor * (1.0 - lumi), 1.0);
126− ivec2 pxCoords = charPos * glyphPixels + ivec2 (j, k);
127− imageStore (imgOut, pxCoords, pixel);
146+ for (int k = 0; k < glyphPixels.y; k++)
147+ {
148+ ivec2 txCoords = atlasPos * srcGlyphPixels + ivec2 (j, k);
149+ ivec3 txc = ivec3 (txCoords, fontIdx);
150+ float lumi = texelFetch (atlas, txc, 0).r;
151+ vec4 pixel = vec4 (fgColor * lumi + bgColor * (1.0 - lumi), 1.0);
152+ ivec2 pxCoords = charPos * glyphPixels + ivec2 (j, k);
153+ imageStore (imgOut, pxCoords, pixel);
154+ }
155+ }
156+ }
157+ else if (hasDoubleWidth == 1)
158+ { // render double-width cell
159+ for (int j = 0; j < srcGlyphPixels.x; j++)
160+ {
161+ for (int k = 0; k < srcGlyphPixels.y; k++)
162+ {
163+ ivec2 txCoords = atlasPos * srcGlyphPixels + ivec2 (j, k);
164+ ivec3 txc = ivec3 (txCoords, fontIdx);
165+ float lumi = texelFetch (atlas_dw, txc, 0).r;
166+ vec4 pixel = vec4 (fgColor * lumi + bgColor * (1.0 - lumi), 1.0);
167+ ivec2 pxCoords = charPos * glyphPixels + ivec2 (j, k);
168+ imageStore (imgOut, pxCoords, pixel);
169+ }
170+ }
171+ }
172+ else
173+ { // no double-width font -- draw an empty box
174+ for (int j = 0; j < srcGlyphPixels.x; j++)
175+ {
176+ for (int k = 0; k < srcGlyphPixels.y; k++)
177+ {
178+ float lumi = 0.0;
179+ if ((0 < j && j < srcGlyphPixels.x - 1) &&
180+ (0 < k && k < srcGlyphPixels.y - 1) &&
181+ (j == 1 || j == srcGlyphPixels.x - 2 ||
182+ k == 1 || k == srcGlyphPixels.y - 2))
183+ lumi = 0.7;
184+ vec4 pixel = vec4 (fgColor * lumi + bgColor * (1.0 - lumi), 1.0);
185+ ivec2 pxCoords = charPos * glyphPixels + ivec2 (j, k);
186+ imageStore (imgOut, pxCoords, pixel);
187+ }
128188 }
129189 }
130190
131191 if (underline == 1u)
132192 {
133− for (int j = 0; j < glyphPixels.x; j++)
193+ for (int j = 0; j < srcGlyphPixels.x; j++)
134194 {
135195 vec4 pixel = vec4 (fgColor, 1.0);
136− ivec2 pxCoords = charPos * glyphPixels + ivec2 (j, glyphPixels.y - 1);
196+ ivec2 pxCoords = charPos * glyphPixels +
197+ ivec2 (j, srcGlyphPixels.y - 1);
137198 imageStore (imgOut, pxCoords, pixel);
138199 }
139200 }
@@ -141,9 +202,10 @@ void main () · 1 unmodified line
141202 if (showWraps == 1 && wrap == 1u)
142203 {
143204 vec4 pixel = vec4 (fgColor, 1.0);
144− for (int k = 0; k < glyphPixels.y; k += 2)
205+ for (int k = 0; k < srcGlyphPixels.y; k += 2)
145206 {
146− ivec2 pxCoords = charPos * glyphPixels + ivec2 (glyphPixels.x - 1, k);
207+ ivec2 pxCoords = charPos * glyphPixels +
208+ ivec2 (srcGlyphPixels.x - 1, k);
147209 imageStore (imgOut, pxCoords, pixel);
148210 }
149211 }
@@ -151,18 +213,18 @@ void main () · 1 unmodified line
151213 if (charPos == cursorPos.xy && cursorStyle == 2)
152214 {
153215 vec4 pixel = vec4 (crColor, 1.0);
154− for (int j = 0; j < glyphPixels.x; j++)
216+ for (int j = 0; j < srcGlyphPixels.x; j++)
155217 {
156218 ivec2 pxCoords = charPos * glyphPixels + ivec2 (j, 0);
157219 imageStore (imgOut, pxCoords, pixel);
158− pxCoords += ivec2 (0, glyphPixels.y - 1);
220+ pxCoords += ivec2 (0, srcGlyphPixels.y - 1);
159221 imageStore (imgOut, pxCoords, pixel);
160222 }
161− for (int k = 1; k < glyphPixels.y - 1; k++)
223+ for (int k = 1; k < srcGlyphPixels.y - 1; k++)
162224 {
163225 ivec2 pxCoords = charPos * glyphPixels + ivec2 (0, k);
164226 imageStore (imgOut, pxCoords, pixel);
165− pxCoords += ivec2 (glyphPixels.x - 1, 0);
227+ pxCoords += ivec2 (srcGlyphPixels.x - 1, 0);
166228 imageStore (imgOut, pxCoords, pixel);
167229 }
168230 }
@@ -266,6 +328,52 @@ void main () · 97 unmodified lines
266328 glCheckError ();
267329 }
268330
331+ void
332+ setupAtlasMappingTexture (const zutty::Font& fnt,
333+ GLuint target, GLuint& texture)
334+ {
335+ auto atlasMap = std::vector <uint8_t> ();
336+ atlasMap.resize (2 * 256 * 256, 0);
337+
338+ // Pre-fill the mapping texture with references to "missing glyph"
339+ // and "replacement character" glyphs, if available in the font atlas.
340+ const auto itEnd = fnt.getAtlasMap ().end ();
341+
342+ zutty::Font::AtlasPos apRC {};
343+ {
344+ auto rcIt =
345+ fnt.getAtlasMap ().find (zutty::Unicode_Replacement_Character);
346+ if (rcIt != itEnd)
347+ apRC = rcIt->second;
348+ }
349+
350+ zutty::Font::AtlasPos apMG {};
351+ {
352+ auto mgIt = fnt.getAtlasMap ().find (zutty::Missing_Glyph_Marker);
353+ if (mgIt != itEnd)
354+ apMG = mgIt->second;
355+ }
356+
357+ for (int k = 0; k < 256 * 256; ++k)
358+ {
359+ const auto& apos = ((k >= 0xd800 && k < 0xe000) || k >= 0xfffe)
360+ ? apRC
361+ : apMG;
362+ atlasMap [2 * k] = apos.x;
363+ atlasMap [2 * k + 1] = apos.y;
364+ }
365+
366+ // Fill the mapping texture with supported characters
367+ for (auto it = fnt.getAtlasMap ().begin (); it != itEnd; ++it)
368+ {
369+ atlasMap [2 * it->first] = it->second.x;
370+ atlasMap [2 * it->first + 1] = it->second.y;
371+ }
372+ setupTexture (target, GL_TEXTURE_2D, texture);
373+ glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, 256, 256, 0,
374+ GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, atlasMap.data ());
375+ }
376+
269377 template <typename T> void
270378 setupStorageBuffer (GLuint index, GLuint& buffer, uint32_t n_items)
271379 {
@@ -284,8 +392,9 @@ void main () · 12 unmodified lines
284392
285393 namespace zutty
286394 {
287− CharVdev::CharVdev (const Fontpack* fontpk_)
288− : fontpk (* fontpk_)
395+ CharVdev::CharVdev (Fontpack* fontpk)
396+ : px (fontpk->getPx ())
397+ , py (fontpk->getPy ())
289398 {
290399 createShaders ();
291400
@@ -322,75 +431,62 @@ namespace zutty · 30 unmodified lines
322431 * Setup compute program
323432 */
324433 glUseProgram (P_compute);
325− glUniform2i (compU_glyphPixels, fontpk.getPx (), fontpk.getPy ());
434+ glUniform2i (compU_glyphPixels, px, py);
326435 glUniform2i (compU_sizeChars, nCols, nRows);
327436 glUniform1i (compU_showWraps, opts.showWraps ? 1 : 0);
328437
329438 // Setup atlas texture
330439 setupTexture (GL_TEXTURE1, GL_TEXTURE_2D_ARRAY, T_atlas);
331− const Font& reg = fontpk.getRegular ();
332− glTexStorage3D(GL_TEXTURE_2D_ARRAY, 1, GL_R8,
333− reg.getPx () * reg.getNx (),
334− reg.getPy () * reg.getNy (),
335− 4); // number of layers
440+ const Font& reg = fontpk->getRegular ();
441+ glTexStorage3D (GL_TEXTURE_2D_ARRAY, 1, GL_R8,
442+ reg.getPx () * reg.getNx (),
443+ reg.getPy () * reg.getNy (),
444+ 4); // number of layers
336445 glCheckError ();
337446
338447 setupAtlasTexture (reg, 0);
339448
340− if (fontpk.hasBold ())
341− setupAtlasTexture (fontpk.getBold (), 1);
449+ if (fontpk->hasBold ())
450+ setupAtlasTexture (fontpk->getBold (), 1);
342451 else
343− setupAtlasTexture (fontpk.getRegular (), 1);
452+ setupAtlasTexture (fontpk->getRegular (), 1);
344453
345− if (fontpk.hasItalic ())
346− setupAtlasTexture (fontpk.getItalic (), 2);
454+ if (fontpk->hasItalic ())
455+ setupAtlasTexture (fontpk->getItalic (), 2);
347456 else
348− setupAtlasTexture (fontpk.getRegular (), 2);
349−
350− if (fontpk.hasBoldItalic ())
351− setupAtlasTexture (fontpk.getBoldItalic (), 3);
352− else if (fontpk.hasItalic ())
353− setupAtlasTexture (fontpk.getItalic (), 3);
354− else if (fontpk.hasBold ())
355− setupAtlasTexture (fontpk.getBold (), 3);
457+ setupAtlasTexture (fontpk->getRegular (), 2);
458+
459+ if (fontpk->hasBoldItalic ())
460+ setupAtlasTexture (fontpk->getBoldItalic (), 3);
461+ else if (fontpk->hasItalic ())
462+ setupAtlasTexture (fontpk->getItalic (), 3);
463+ else if (fontpk->hasBold ())
464+ setupAtlasTexture (fontpk->getBold (), 3);
356465 else
357− setupAtlasTexture (fontpk.getRegular (), 3);
466+ setupAtlasTexture (fontpk->getRegular (), 3);
358467
359− // Setup atlas mapping texture
360− auto atlasMap = std::vector <uint8_t> ();
361− atlasMap.resize (2 * 256 * 256, 0);
468+ setupAtlasMappingTexture (reg, GL_TEXTURE2, T_atlasMap);
362469
363− // Pre-fill the mapping texture with references to "missing glyph"
364− // and "replacement character" glyphs, if available in the font atlas.
365− const auto itEnd = reg.getAtlasMap ().end ();
366− auto rcIt = reg.getAtlasMap ().find (Unicode_Replacement_Character);
367− if (rcIt == itEnd)
368− rcIt = reg.getAtlasMap ().find (' ');
369− assert (rcIt != itEnd);
370− auto mgIt = reg.getAtlasMap ().find (Missing_Glyph_Marker);
371− if (mgIt == itEnd)
372− mgIt = reg.getAtlasMap ().find (' ');
373− assert (mgIt != itEnd);
374− for (int k = 0; k < 256 * 256; ++k)
470+ // Setup atlas texture for double-width characters
471+ if (fontpk->hasDoubleWidth ())
375472 {
376− const auto& apos =
377− ((k >= 0xd800 && k < 0xe000) || k >= 0xfffe)
378− ? rcIt->second
379− : mgIt->second;
380− atlasMap [2 * k] = apos.x;
381− atlasMap [2 * k + 1] = apos.y;
473+ hasDoubleWidth = true;
474+
475+ setupTexture (GL_TEXTURE3, GL_TEXTURE_2D_ARRAY, T_atlas_dw);
476+ const Font& dw = fontpk->getDoubleWidth ();
477+ glTexStorage3D (GL_TEXTURE_2D_ARRAY, 1, GL_R8,
478+ dw.getPx () * dw.getNx (),
479+ dw.getPy () * dw.getNy (),
480+ 1); // number of layers
481+ glCheckError ();
482+
483+ setupAtlasTexture (dw, 0);
484+ setupAtlasMappingTexture (dw, GL_TEXTURE3, T_atlasMap_dw);
382485 }
486+ glUniform1i (compU_hasDoubleWidth, hasDoubleWidth ? 1 : 0);
383487
384− // Fill the mapping texture with supported characters
385− auto it = reg.getAtlasMap ().begin ();
386− for (; it != itEnd; ++it)
387− {
388− atlasMap [2 * it->first] = it->second.x;
389− atlasMap [2 * it->first + 1] = it->second.y;
390− }
391− setupTexture (GL_TEXTURE2, GL_TEXTURE_2D, T_atlasMap);
392− glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, 256, 256, 0,
393− GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, atlasMap.data ());
488+ // Now that it's all loaded into GL, no need to keep font data in-memory
489+ fontpk->releaseFonts ();
394490 }
395491
396492 CharVdev::~CharVdev ()
@@ -407,15 +503,15 @@ namespace zutty · 10 unmodified lines
407503
408504 pxWidth = pxWidth_;
409505 pxHeight = pxHeight_;
410− nCols = std::max (1, (pxWidth - 2 * opts.border) / fontpk.getPx ());
411− nRows = std::max (1, (pxHeight - 2 * opts.border) / fontpk.getPy ());
506+ nCols = std::max (1, (pxWidth - 2 * opts.border) / px);
507+ nRows = std::max (1, (pxHeight - 2 * opts.border) / py);
412508
413509 logI << "Resize to " << pxWidth << " x " << pxHeight
414510 << " pixels, " << nCols << " x " << nRows << " chars"
415511 << std::endl;
416512
417− GLint viewWidth = nCols * fontpk.getPx ();
418− GLint viewHeight = nRows * fontpk.getPy ();
513+ GLint viewWidth = nCols * px;
514+ GLint viewHeight = nRows * py;
419515 glViewport (opts.border, pxHeight - viewHeight - opts.border,
420516 viewWidth, viewHeight);
421517
@@ -487,6 +583,13 @@ namespace zutty · 65 unmodified lines
487583 glBindTexture (GL_TEXTURE_2D_ARRAY, T_atlas);
488584 glActiveTexture (GL_TEXTURE2);
489585 glBindTexture (GL_TEXTURE_2D, T_atlasMap);
586+ if (hasDoubleWidth)
587+ {
588+ glActiveTexture (GL_TEXTURE3);
589+ glBindTexture (GL_TEXTURE_2D_ARRAY, T_atlas_dw);
590+ glActiveTexture (GL_TEXTURE4);
591+ glBindTexture (GL_TEXTURE_2D, T_atlasMap_dw);
592+ }
490593 glCheckError ();
491594
492595 glDispatchCompute (nCols, nRows, 1);
@@ -562,6 +665,7 @@ namespace zutty · 69 unmodified lines
562665 compU_selectDamage = glGetUniformLocation (P_compute, "selectDamage");
563666 compU_deltaFrame = glGetUniformLocation (P_compute, "deltaFrame");
564667 compU_showWraps = glGetUniformLocation (P_compute, "showWraps");
668+ compU_hasDoubleWidth = glGetUniformLocation (P_compute, "hasDoubleWidth");
565669
566670 logT << "compute program:"
567671 << " uniform glyphPixels=" << compU_glyphPixels
@@ -574,6 +678,7 @@ namespace zutty · 6 unmodified lines
574678 << " selectDamage=" << compU_selectDamage
575679 << " deltaFrame=" << compU_deltaFrame
576680 << " showWraps=" << compU_showWraps
681+ << " hasDoubleWidth=" << compU_hasDoubleWidth
577682 << std::endl;
578683
579684 P_draw = glCreateProgram ();

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t