History

Initial public commit for Dis VM

arr committed 10 years ago

Commit f95902eNo parent (first commit)40 files · +9358 −0Browse files

Showing the changes to src/vm/utf8.cpp (1 of 40 files). Show all

Unified · Split

src/vm/utf8.cpp+136−0
@@ -0,0 +1,136 @@
1//
2// Dis VM
3// File: utf8.cpp
4// Author: arr
5//
6
7// Copyright (c) 2008-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
8// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
9// Most code in this file falls under the above copyright. It was augmented only to be more C++ friendly.
10
11#include <cinttypes>
12#include <utf8.h>
13#include <exceptions.h>
14
15using namespace disvm::runtime;
16
17namespace
18{
19 const uint8_t utf8d[] =
20 {
21 // The first part of the table maps bytes to character classes that
22 // to reduce the size of the transition table and create bitmasks.
23 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
24 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
25 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
26 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
27 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
28 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
29 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
30 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
31
32 // The second part is a transition table that maps a combination
33 // of a state of the automaton and a character class to a state.
34 0, 12, 24, 36, 60, 96, 84, 12, 12, 12, 48, 72, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
35 12, 0, 12, 12, 12, 12, 12, 0, 12, 0, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 24, 12, 12,
36 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12,
37 12, 12, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12,
38 12, 36, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
39 };
40}
41
42// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
43utf8::decode_state disvm::runtime::utf8::decode_step(utf8::decode_state &state, rune_t &codepoint, uint8_t byte)
44{
45 auto type = uint32_t{ utf8d[byte] };
46
47 codepoint = (state != utf8::decode_state::accept) ?
48 (byte & 0x3fu) | (codepoint << 6) :
49 (0xff >> type) & (byte);
50
51 state = static_cast<utf8::decode_state>(utf8d[256 + static_cast<uint32_t>(state) + type]);
52 return state;
53}
54
55// Adopted from 'countCodePoints(uint8_t* s, size_t* count)'
56// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
57std::size_t disvm::runtime::utf8::count_codepoints(const uint8_t *str, const std::size_t max_len)
58{
59 assert(str != nullptr);
60 auto single_character = rune_t{};
61 auto points = std::size_t{ 0 };
62 auto state = utf8::decode_state::accept;
63
64 for (auto i = std::size_t{ 0 }; i < max_len; ++i)
65 {
66 if (utf8::decode_state::accept == utf8::decode_step(state, single_character, str[i]))
67 points++;
68 }
69
70 if (state != utf8::decode_state::accept)
71 throw invalid_utf8{};
72
73 return points;
74}
75
76std::size_t disvm::runtime::utf8::decode(const uint8_t *str, rune_t &codepoint)
77{
78 assert(str != nullptr);
79 auto state = utf8::decode_state::accept;
80
81 codepoint = 0;
82 auto begin = str;
83 auto curr = str;
84 auto c = uint8_t{};
85 while (0 != (c = *curr))
86 {
87 curr++;
88 state = utf8::decode_step(state, codepoint, c);
89 if (state == utf8::decode_state::accept)
90 return (curr - begin);
91 }
92
93 // Empty string
94 if (begin == curr)
95 return 0;
96
97 throw invalid_utf8{};
98}
99
100uint8_t *disvm::runtime::utf8::encode(const rune_t rune, uint8_t *str)
101{
102 assert(str != nullptr);
103 if (rune <= utf8::constants::max_codepoint_ascii)
104 {
105 // Layout: 0xxxxxxx
106 str[0] = static_cast<uint8_t>(rune);
107 return (str + 1);
108 }
109 else if (rune <= utf8::constants::max_codepoint_2_byte)
110 {
111 // Layout: 110xxxxx 10xxxxxx
112 str[0] = static_cast<uint8_t>(0xc0 + (rune >> 6));
113 str[1] = static_cast<uint8_t>(0x80 + (rune & utf8::constants::multi_byte_value_mask));
114 return (str + 2);
115 }
116 else if (rune <= utf8::constants::max_codepoint_3_byte)
117 {
118 // Layout: 1110xxxx 10xxxxxx 10xxxxxx
119 str[0] = static_cast<uint8_t>(0xe0 + (rune >> 12));
120 str[1] = static_cast<uint8_t>(0x80 + ((rune >> 6) & utf8::constants::multi_byte_value_mask));
121 str[2] = static_cast<uint8_t>(0x80 + (rune & utf8::constants::multi_byte_value_mask));
122 return (str + 3);
123 }
124 else if (rune <= utf8::constants::max_codepoint_4_byte)
125 {
126 // Layout: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
127 str[0] = static_cast<uint8_t>(0xf0 + (rune >> 18));
128 str[1] = static_cast<uint8_t>(0x80 + ((rune >> 12) & utf8::constants::multi_byte_value_mask));
129 str[2] = static_cast<uint8_t>(0x80 + ((rune >> 6) & utf8::constants::multi_byte_value_mask));
130 str[3] = static_cast<uint8_t>(0x80 + (rune & utf8::constants::multi_byte_value_mask));
131 return (str + 4);
132 }
133
134 assert(false && "Unsupported UTF-8 character");
135 return nullptr;
136}