Implement seek() utfbytes() in Sys module. - Updated tests for utfbytes() - Updated utf8 API surface to support the general length case (codepoint _and_ byte) - Testing of seek() still outstanding Comment added regarding stack frame pointer in stack frames.

arr committed 9 years ago

Commit a6da1dc1 parent 1af55286 files · +94 −18Browse files

Showing the changes to src (5 of 6 files). Show all

Unified · Split

5 files

src/exec/main.cpp +1 −1
src/include/utf8.h +9 −2
src/vm/string.cpp +8 −3
src/vm/sys/Sysmod.cpp +57 −4
src/vm/utf8.cpp +11 −8

@@ -32,7 +32,7 @@ namespace · 31 unmodified lines
32	{	32	{
33	word_t noret;	33	word_t noret;
34	pointer_t command_module_ref;	34	pointer_t command_module_ref;
35	pointer_t command_stack_frame; // Should not be marked as memory managed pointer field	35	pointer_t command_stack_frame; // [SPEC] Stack frames are not managed by the GC and as such should not be marked as a pointer field. See pointer map below.
36	};	36	};
37		37
38	const auto entry_frame_pointer_map = std::vector<byte_t>{ 0x04 };	38	const auto entry_frame_pointer_map = std::vector<byte_t>{ 0x04 };

src/include/utf8.h+9−2

                 const disvm::runtime::rune_t max_supported_codepoint = max_codepoint_4_byte;
             };
-            // Compute the number of code points in the supplied string
+            // Lengths of a UTF-8 string
+            struct utf8_length_t
+            {
+                const std::size_t codepoint_count;
+                const std::size_t byte_count;
+            };
+            // Compute the number of codepoints in the supplied string
             // See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
-            std::size_t count_codepoints(const uint8_t *str, const std::size_t max_len);
+            utf8_length_t count_codepoints(const uint8_t *str, const std::size_t max_len);
             enum class decode_state_t
             {

src/vm/string.cpp+8−3

     , _mem{}
 {
     assert((encoded_str_len > 0 && encoded_str != nullptr) || (encoded_str_len == 0 && encoded_str == nullptr));
-    auto character_count = utf8::count_codepoints(encoded_str, encoded_str_len);
+    const auto utf8_length = utf8::count_codepoints(encoded_str, encoded_str_len);
-    _character_size = (encoded_str_len != character_count) ? sizeof(rune_t) : sizeof(char);
-    _length = static_cast<word_t>(character_count);
+    // Check if the supplied string is completely valid UTF-8 - only valid UTF-8 strings can be DisVM strings.
+    if (encoded_str_len != utf8_length.byte_count)
+        throw invalid_utf8{};
+    _character_size = (encoded_str_len != utf8_length.codepoint_count) ? sizeof(rune_t) : sizeof(char);
+    _length = static_cast<word_t>(utf8_length.codepoint_count);
     _length_max = sizeof(_mem.local) / _character_size;
     // Allocate memory based on character count and size

src/vm/sys/Sysmod.cpp+57−4

@@ -188,6 +188,15 @@ namespace · 187 unmodified lines
188	throw vm_user_exception{ "Write operation error" };	188	throw vm_user_exception{ "Write operation error" };
189	}	189	}
190		190
		191	big_t seek(const std::ios::seekdir start, const big_t offset)
		192	{
		193	if (_stream == nullptr)
		194	throw vm_user_exception{ "Unable to seek on a fifo file descriptor (e.g. stdin, stdout, stderr)" };
		195
		196	auto new_offset = _stream->rdbuf()->pubseekoff(static_cast<std::streamoff>(offset), start, _fd_mode);
		197	return static_cast<big_t>(new_offset);
		198	}
		199
191	private:	200	private:
192	const word_t _fd_mode;	201	const word_t _fd_mode;
193	vm_string_t *_fd_path;	202	vm_string_t *_fd_path;
@@ -218,6 +227,21 @@ namespace · 24 unmodified lines
218		227
219	return om;	228	return om;
220	}	229	}
		230
		231	std::ios::seekdir convert_to_seekdir(const word_t start)
		232	{
		233	switch (start)
		234	{
		235	case Sys_SEEKSTART:
		236	return std::ios::beg;
		237	case Sys_SEEKRELA:
		238	return std::ios::cur;
		239	case Sys_SEEKEND:
		240	return std::ios::end;
		241	default:
		242	throw vm_user_exception{ "Invalid seek mode" };
		243	}
		244	}
221	}	245	}
222		246
223	void	247	void
@@ -612,7 +636,6 @@ Sys_read(vm_registers_t &r, vm_t &vm) · 388 unmodified lines
612	auto &fp = r.stack.peek_frame()->base<F_Sys_read>();	636	auto &fp = r.stack.peek_frame()->base<F_Sys_read>();
613		637
614	const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);	638	const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);
615	assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
616		639
617	auto n = fp.n;	640	auto n = fp.n;
618	if (buffer == nullptr \|\| n <= 0)	641	if (buffer == nullptr \|\| n <= 0)
@@ -621,11 +644,16 @@ Sys_read(vm_registers_t &r, vm_t &vm) · 2 unmodified lines
621	return;	644	return;
622	}	645	}
623		646
		647	assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
		648
624	// [SPEC] Supplying a size greater than the buffer length is	649	// [SPEC] Supplying a size greater than the buffer length is
625	// equivalent to indicating the entire buffer should be filled.	650	// equivalent to indicating the entire buffer should be filled.
626	n = std::min(n, buffer->get_length());	651	n = std::min(n, buffer->get_length());
627		652
628	auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);	653	auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);
		654	if (fd_alloc == nullptr)
		655	throw dereference_nil{ "Read from file descriptor" };
		656
629	assert(fd_alloc->alloc_type == T_FD);	657	assert(fd_alloc->alloc_type == T_FD);
630		658
631	auto fd = fd_alloc->get_allocation<vm_fd_t>();	659	auto fd = fd_alloc->get_allocation<vm_fd_t>();
@@ -650,7 +678,16 @@ void · 18 unmodified lines
650	Sys_seek(vm_registers_t &r, vm_t &vm)	678	Sys_seek(vm_registers_t &r, vm_t &vm)
651	{	679	{
652	auto &fp = r.stack.peek_frame()->base<F_Sys_seek>();	680	auto &fp = r.stack.peek_frame()->base<F_Sys_seek>();
653	throw vm_system_exception{ "Instruction not implemented" };	681
		682	auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);
		683	if (fd_alloc == nullptr)
		684	throw dereference_nil{ "Seek in file descriptor" };
		685
		686	assert(fd_alloc->alloc_type == T_FD);
		687	auto fd = fd_alloc->get_allocation<vm_fd_t>();
		688
		689	const auto start = convert_to_seekdir(fp.start);
		690	*fp.ret = fd->seek(start, fp.off);
654	}	691	}
655		692
656	void	693	void
@@ -814,7 +851,19 @@ void · 157 unmodified lines
814	Sys_utfbytes(vm_registers_t &r, vm_t &vm)	851	Sys_utfbytes(vm_registers_t &r, vm_t &vm)
815	{	852	{
816	auto &fp = r.stack.peek_frame()->base<F_Sys_utfbytes>();	853	auto &fp = r.stack.peek_frame()->base<F_Sys_utfbytes>();
817	throw vm_system_exception{ "Instruction not implemented" };	854	const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);
		855
		856	const auto n = fp.n;
		857	if (buffer == nullptr \|\| n > buffer->get_length())
		858	throw out_of_range_memory{};
		859
		860	assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
		861
		862	auto raw_buffer = reinterpret_cast<uint8_t *>(buffer->at(0));
		863
		864	const auto len = utf8::count_codepoints(raw_buffer, n);
		865	*fp.ret = static_cast<word_t>(len.byte_count);
		866	assert(*fp.ret <= n && "Code points should not exceed number of bytes processed");
818	}	867	}
819		868
820	void	869	void
@@ -830,7 +879,6 @@ Sys_write(vm_registers_t &r, vm_t &vm) · 9 unmodified lines
830	auto &fp = r.stack.peek_frame()->base<F_Sys_write>();	879	auto &fp = r.stack.peek_frame()->base<F_Sys_write>();
831		880
832	const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);	881	const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);
833	assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
834		882
835	auto n = fp.n;	883	auto n = fp.n;
836	if (buffer == nullptr \|\| n <= 0)	884	if (buffer == nullptr \|\| n <= 0)
@@ -839,11 +887,16 @@ Sys_write(vm_registers_t &r, vm_t &vm) · 2 unmodified lines
839	return;	887	return;
840	}	888	}
841		889
		890	assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
		891
842	// [SPEC] Supplying a size greater than the buffer length is	892	// [SPEC] Supplying a size greater than the buffer length is
843	// equivalent to indicating the entire buffer should be written.	893	// equivalent to indicating the entire buffer should be written.
844	n = std::min(n, buffer->get_length());	894	n = std::min(n, buffer->get_length());
845		895
846	auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);	896	auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);
		897	if (fd_alloc == nullptr)
		898	throw dereference_nil{ "Write to file descriptor" };
		899
847	assert(fd_alloc->alloc_type == T_FD);	900	assert(fd_alloc->alloc_type == T_FD);
848		901
849	auto fd = fd_alloc->get_allocation<vm_fd_t>();	902	auto fd = fd_alloc->get_allocation<vm_fd_t>();

src/vm/utf8.cpp+11−8

@@ -54,28 +54,31 @@ utf8::decode_state_t disvm::runtime::utf8::decode_step(utf8::decode_state_t &sta · 53 unmodified lines
54		54
55	// Adopted from 'countCodePoints(uint8_t* s, size_t* count)'	55	// Adopted from 'countCodePoints(uint8_t* s, size_t* count)'
56	// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.	56	// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
57	std::size_t disvm::runtime::utf8::count_codepoints(const uint8_t *str, const std::size_t max_len)	57	utf8::utf8_length_t disvm::runtime::utf8::count_codepoints(const uint8_t *str, const std::size_t max_len)
58	{	58	{
59	if (str == nullptr)	59	if (str == nullptr)
60	{	60	{
61	assert(max_len == 0);	61	assert(max_len == 0);
62	return std::size_t{ 0 };	62	return{ 0, 0 };
63	}	63	}
64		64
65	auto single_character = rune_t{};	65	auto single_character = rune_t{};
66	auto points = std::size_t{ 0 };	66	auto points = std::size_t{ 0 };
67	auto state = utf8::decode_state_t::accept;	67	auto state = utf8::decode_state_t::accept;
68		68
69	for (auto i = std::size_t{ 0 }; i < max_len; ++i)	69	auto last = str;
		70	for (auto curr = last; curr < (str + max_len); ++curr)
70	{	71	{
71	if (utf8::decode_state_t::accept == utf8::decode_step(state, single_character, str[i]))	72	if (utf8::decode_state_t::accept == utf8::decode_step(state, single_character, *curr))
		73	{
		74	last = curr;
72	points++;	75	points++;
		76	}
73	}	77	}
74		78
75	if (state != utf8::decode_state_t::accept)	79	assert(str <= last);
76	throw invalid_utf8{};	80	const auto byte_count = static_cast<std::size_t>(last - str + 1);
77		81	return{ points, byte_count };
78	return points;
79	}	82	}
80		83
81	std::size_t disvm::runtime::utf8::decode(const uint8_t *str, rune_t &codepoint)	84	std::size_t disvm::runtime::utf8::decode(const uint8_t *str, rune_t &codepoint)