History

Initial public commit for Dis VM

arr committed 10 years ago

Commit f95902eNo parent (first commit)40 files · +9358 −0Browse files

Showing the changes to src/vm/utf8.cpp (1 of 40 files). Show all

Unified · Split

src/vm/utf8.cpp+136−0
@@ -0,0 +1,136 @@
1+//
2+// Dis VM
3+// File: utf8.cpp
4+// Author: arr
5+//
6+
7+// Copyright (c) 2008-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
8+// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
9+// Most code in this file falls under the above copyright. It was augmented only to be more C++ friendly.
10+
11+#include <cinttypes>
12+#include <utf8.h>
13+#include <exceptions.h>
14+
15+using namespace disvm::runtime;
16+
17+namespace
18+{
19+ const uint8_t utf8d[] =
20+ {
21+ // The first part of the table maps bytes to character classes that
22+ // to reduce the size of the transition table and create bitmasks.
23+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
24+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
25+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
26+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
27+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
28+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
29+ 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
30+ 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
31+
32+ // The second part is a transition table that maps a combination
33+ // of a state of the automaton and a character class to a state.
34+ 0, 12, 24, 36, 60, 96, 84, 12, 12, 12, 48, 72, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
35+ 12, 0, 12, 12, 12, 12, 12, 0, 12, 0, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 24, 12, 12,
36+ 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12,
37+ 12, 12, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12,
38+ 12, 36, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
39+ };
40+}
41+
42+// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
43+utf8::decode_state disvm::runtime::utf8::decode_step(utf8::decode_state &state, rune_t &codepoint, uint8_t byte)
44+{
45+ auto type = uint32_t{ utf8d[byte] };
46+
47+ codepoint = (state != utf8::decode_state::accept) ?
48+ (byte & 0x3fu) | (codepoint << 6) :
49+ (0xff >> type) & (byte);
50+
51+ state = static_cast<utf8::decode_state>(utf8d[256 + static_cast<uint32_t>(state) + type]);
52+ return state;
53+}
54+
55+// Adopted from 'countCodePoints(uint8_t* s, size_t* count)'
56+// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
57+std::size_t disvm::runtime::utf8::count_codepoints(const uint8_t *str, const std::size_t max_len)
58+{
59+ assert(str != nullptr);
60+ auto single_character = rune_t{};
61+ auto points = std::size_t{ 0 };
62+ auto state = utf8::decode_state::accept;
63+
64+ for (auto i = std::size_t{ 0 }; i < max_len; ++i)
65+ {
66+ if (utf8::decode_state::accept == utf8::decode_step(state, single_character, str[i]))
67+ points++;
68+ }
69+
70+ if (state != utf8::decode_state::accept)
71+ throw invalid_utf8{};
72+
73+ return points;
74+}
75+
76+std::size_t disvm::runtime::utf8::decode(const uint8_t *str, rune_t &codepoint)
77+{
78+ assert(str != nullptr);
79+ auto state = utf8::decode_state::accept;
80+
81+ codepoint = 0;
82+ auto begin = str;
83+ auto curr = str;
84+ auto c = uint8_t{};
85+ while (0 != (c = *curr))
86+ {
87+ curr++;
88+ state = utf8::decode_step(state, codepoint, c);
89+ if (state == utf8::decode_state::accept)
90+ return (curr - begin);
91+ }
92+
93+ // Empty string
94+ if (begin == curr)
95+ return 0;
96+
97+ throw invalid_utf8{};
98+}
99+
100+uint8_t *disvm::runtime::utf8::encode(const rune_t rune, uint8_t *str)
101+{
102+ assert(str != nullptr);
103+ if (rune <= utf8::constants::max_codepoint_ascii)
104+ {
105+ // Layout: 0xxxxxxx
106+ str[0] = static_cast<uint8_t>(rune);
107+ return (str + 1);
108+ }
109+ else if (rune <= utf8::constants::max_codepoint_2_byte)
110+ {
111+ // Layout: 110xxxxx 10xxxxxx
112+ str[0] = static_cast<uint8_t>(0xc0 + (rune >> 6));
113+ str[1] = static_cast<uint8_t>(0x80 + (rune & utf8::constants::multi_byte_value_mask));
114+ return (str + 2);
115+ }
116+ else if (rune <= utf8::constants::max_codepoint_3_byte)
117+ {
118+ // Layout: 1110xxxx 10xxxxxx 10xxxxxx
119+ str[0] = static_cast<uint8_t>(0xe0 + (rune >> 12));
120+ str[1] = static_cast<uint8_t>(0x80 + ((rune >> 6) & utf8::constants::multi_byte_value_mask));
121+ str[2] = static_cast<uint8_t>(0x80 + (rune & utf8::constants::multi_byte_value_mask));
122+ return (str + 3);
123+ }
124+ else if (rune <= utf8::constants::max_codepoint_4_byte)
125+ {
126+ // Layout: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
127+ str[0] = static_cast<uint8_t>(0xf0 + (rune >> 18));
128+ str[1] = static_cast<uint8_t>(0x80 + ((rune >> 12) & utf8::constants::multi_byte_value_mask));
129+ str[2] = static_cast<uint8_t>(0x80 + ((rune >> 6) & utf8::constants::multi_byte_value_mask));
130+ str[3] = static_cast<uint8_t>(0x80 + (rune & utf8::constants::multi_byte_value_mask));
131+ return (str + 4);
132+ }
133+
134+ assert(false && "Unsupported UTF-8 character");
135+ return nullptr;
136+}