Implement seek() utfbytes() in Sys module. - Updated tests for utfbytes() - Updated utf8 API surface to support the general length case (codepoint _and_ byte) - Testing of seek() still outstanding Comment added regarding stack frame pointer in stack frames.

arr committed 9 years ago

Commit a6da1dc1 parent 1af55286 files · +94 −18Browse files

Unified · Split

6 files

limbo/basic_limbo_tests.b +8 −0
src/exec/main.cpp +1 −1
src/include/utf8.h +9 −2
src/vm/string.cpp +8 −3
src/vm/sys/Sysmod.cpp +57 −4
src/vm/utf8.cpp +11 −8

@@ -666,6 +666,14 @@ test_string() · 665 unmodified lines
666	666	array19 := array of byte count7;
667	667	if (len array19 != 19) raise TEST_FAILED();
668	668
	669	+ # The above Devanagari text is used to test codepoint
	670	+ # counting below. The text needs 6 bytes for the first
	671	+ # two characters - 3 bytes each. Interpreting 5 bytes
	672	+ # will yield 3 since the last 2 bytes will represent an
	673	+ # incomplete codepoint.
	674	+ codepoint_bytes := sys->utfbytes(array19, 5);
	675	+ if (codepoint_bytes != 3) raise TEST_FAILED();
	676	+
669	677	count36 := "abcdefghijklmnopqrstuvwxyz0123456789";
670	678	if (len count36 != 36) raise TEST_FAILED();
671	679

src/exec/main.cpp+1−1

@@ -32,7 +32,7 @@ namespace · 31 unmodified lines
32	32	{
33	33	word_t noret;
34	34	pointer_t command_module_ref;
35		− pointer_t command_stack_frame; // Should not be marked as memory managed pointer field
	35	+ pointer_t command_stack_frame; // [SPEC] Stack frames are not managed by the GC and as such should not be marked as a pointer field. See pointer map below.
36	36	};
37	37
38	38	const auto entry_frame_pointer_map = std::vector<byte_t>{ 0x04 };

src/include/utf8.h+9−2

@@ -29,9 +29,16 @@ namespace disvm · 28 unmodified lines
29	29	const disvm::runtime::rune_t max_supported_codepoint = max_codepoint_4_byte;
30	30	};
31	31
32		− // Compute the number of code points in the supplied string
	32	+ // Lengths of a UTF-8 string
	33	+ struct utf8_length_t
	34	+ {
	35	+ const std::size_t codepoint_count;
	36	+ const std::size_t byte_count;
	37	+ };
	38	+
	39	+ // Compute the number of codepoints in the supplied string
33	40	// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
34		− std::size_t count_codepoints(const uint8_t *str, const std::size_t max_len);
	41	+ utf8_length_t count_codepoints(const uint8_t *str, const std::size_t max_len);
35	42
36	43	enum class decode_state_t
37	44	{

src/vm/string.cpp+8−3

@@ -191,10 +191,15 @@ vm_string_t::vm_string_t(std::size_t encoded_str_len, const uint8_t *encoded_str · 190 unmodified lines
191	191	, _mem{}
192	192	{
193	193	assert((encoded_str_len > 0 && encoded_str != nullptr) \|\| (encoded_str_len == 0 && encoded_str == nullptr));
194		− auto character_count = utf8::count_codepoints(encoded_str, encoded_str_len);
195		− _character_size = (encoded_str_len != character_count) ? sizeof(rune_t) : sizeof(char);
	194	+ const auto utf8_length = utf8::count_codepoints(encoded_str, encoded_str_len);
196	195
197		− _length = static_cast<word_t>(character_count);
	196	+ // Check if the supplied string is completely valid UTF-8 - only valid UTF-8 strings can be DisVM strings.
	197	+ if (encoded_str_len != utf8_length.byte_count)
	198	+ throw invalid_utf8{};
	199	+
	200	+ _character_size = (encoded_str_len != utf8_length.codepoint_count) ? sizeof(rune_t) : sizeof(char);
	201	+
	202	+ _length = static_cast<word_t>(utf8_length.codepoint_count);
198	203	_length_max = sizeof(_mem.local) / _character_size;
199	204
200	205	// Allocate memory based on character count and size

src/vm/sys/Sysmod.cpp+57−4

@@ -188,6 +188,15 @@ namespace · 187 unmodified lines
188	188	throw vm_user_exception{ "Write operation error" };
189	189	}
190	190
	191	+ big_t seek(const std::ios::seekdir start, const big_t offset)
	192	+ {
	193	+ if (_stream == nullptr)
	194	+ throw vm_user_exception{ "Unable to seek on a fifo file descriptor (e.g. stdin, stdout, stderr)" };
	195	+
	196	+ auto new_offset = _stream->rdbuf()->pubseekoff(static_cast<std::streamoff>(offset), start, _fd_mode);
	197	+ return static_cast<big_t>(new_offset);
	198	+ }
	199	+
191	200	private:
192	201	const word_t _fd_mode;
193	202	vm_string_t *_fd_path;
@@ -218,6 +227,21 @@ namespace · 24 unmodified lines
218	227
219	228	return om;
220	229	}
	230	+
	231	+ std::ios::seekdir convert_to_seekdir(const word_t start)
	232	+ {
	233	+ switch (start)
	234	+ {
	235	+ case Sys_SEEKSTART:
	236	+ return std::ios::beg;
	237	+ case Sys_SEEKRELA:
	238	+ return std::ios::cur;
	239	+ case Sys_SEEKEND:
	240	+ return std::ios::end;
	241	+ default:
	242	+ throw vm_user_exception{ "Invalid seek mode" };
	243	+ }
	244	+ }
221	245	}
222	246
223	247	void
@@ -612,7 +636,6 @@ Sys_read(vm_registers_t &r, vm_t &vm) · 388 unmodified lines
612	636	auto &fp = r.stack.peek_frame()->base<F_Sys_read>();
613	637
614	638	const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);
615		− assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
616	639
617	640	auto n = fp.n;
618	641	if (buffer == nullptr \|\| n <= 0)
@@ -621,11 +644,16 @@ Sys_read(vm_registers_t &r, vm_t &vm) · 2 unmodified lines
621	644	return;
622	645	}
623	646
	647	+ assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
	648	+
624	649	// [SPEC] Supplying a size greater than the buffer length is
625	650	// equivalent to indicating the entire buffer should be filled.
626	651	n = std::min(n, buffer->get_length());
627	652
628	653	auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);
	654	+ if (fd_alloc == nullptr)
	655	+ throw dereference_nil{ "Read from file descriptor" };
	656	+
629	657	assert(fd_alloc->alloc_type == T_FD);
630	658
631	659	auto fd = fd_alloc->get_allocation<vm_fd_t>();
@@ -650,7 +678,16 @@ void · 18 unmodified lines
650	678	Sys_seek(vm_registers_t &r, vm_t &vm)
651	679	{
652	680	auto &fp = r.stack.peek_frame()->base<F_Sys_seek>();
653		− throw vm_system_exception{ "Instruction not implemented" };
	681	+
	682	+ auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);
	683	+ if (fd_alloc == nullptr)
	684	+ throw dereference_nil{ "Seek in file descriptor" };
	685	+
	686	+ assert(fd_alloc->alloc_type == T_FD);
	687	+ auto fd = fd_alloc->get_allocation<vm_fd_t>();
	688	+
	689	+ const auto start = convert_to_seekdir(fp.start);
	690	+ *fp.ret = fd->seek(start, fp.off);
654	691	}
655	692
656	693	void
@@ -814,7 +851,19 @@ void · 157 unmodified lines
814	851	Sys_utfbytes(vm_registers_t &r, vm_t &vm)
815	852	{
816	853	auto &fp = r.stack.peek_frame()->base<F_Sys_utfbytes>();
817		− throw vm_system_exception{ "Instruction not implemented" };
	854	+ const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);
	855	+
	856	+ const auto n = fp.n;
	857	+ if (buffer == nullptr \|\| n > buffer->get_length())
	858	+ throw out_of_range_memory{};
	859	+
	860	+ assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
	861	+
	862	+ auto raw_buffer = reinterpret_cast<uint8_t *>(buffer->at(0));
	863	+
	864	+ const auto len = utf8::count_codepoints(raw_buffer, n);
	865	+ *fp.ret = static_cast<word_t>(len.byte_count);
	866	+ assert(*fp.ret <= n && "Code points should not exceed number of bytes processed");
818	867	}
819	868
820	869	void
@@ -830,7 +879,6 @@ Sys_write(vm_registers_t &r, vm_t &vm) · 9 unmodified lines
830	879	auto &fp = r.stack.peek_frame()->base<F_Sys_write>();
831	880
832	881	const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);
833		− assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
834	882
835	883	auto n = fp.n;
836	884	if (buffer == nullptr \|\| n <= 0)
@@ -839,11 +887,16 @@ Sys_write(vm_registers_t &r, vm_t &vm) · 2 unmodified lines
839	887	return;
840	888	}
841	889
	890	+ assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
	891	+
842	892	// [SPEC] Supplying a size greater than the buffer length is
843	893	// equivalent to indicating the entire buffer should be written.
844	894	n = std::min(n, buffer->get_length());
845	895
846	896	auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);
	897	+ if (fd_alloc == nullptr)
	898	+ throw dereference_nil{ "Write to file descriptor" };
	899	+
847	900	assert(fd_alloc->alloc_type == T_FD);
848	901
849	902	auto fd = fd_alloc->get_allocation<vm_fd_t>();

src/vm/utf8.cpp+11−8

@@ -54,28 +54,31 @@ utf8::decode_state_t disvm::runtime::utf8::decode_step(utf8::decode_state_t &sta · 53 unmodified lines
54	54
55	55	// Adopted from 'countCodePoints(uint8_t* s, size_t* count)'
56	56	// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
57		−std::size_t disvm::runtime::utf8::count_codepoints(const uint8_t *str, const std::size_t max_len)
	57	+utf8::utf8_length_t disvm::runtime::utf8::count_codepoints(const uint8_t *str, const std::size_t max_len)
58	58	{
59	59	if (str == nullptr)
60	60	{
61	61	assert(max_len == 0);
62		− return std::size_t{ 0 };
	62	+ return{ 0, 0 };
63	63	}
64	64
65	65	auto single_character = rune_t{};
66	66	auto points = std::size_t{ 0 };
67	67	auto state = utf8::decode_state_t::accept;
68	68
69		− for (auto i = std::size_t{ 0 }; i < max_len; ++i)
	69	+ auto last = str;
	70	+ for (auto curr = last; curr < (str + max_len); ++curr)
70	71	{
71		− if (utf8::decode_state_t::accept == utf8::decode_step(state, single_character, str[i]))
	72	+ if (utf8::decode_state_t::accept == utf8::decode_step(state, single_character, *curr))
	73	+ {
	74	+ last = curr;
72	75	points++;
	76	+ }
73	77	}
74	78
75		− if (state != utf8::decode_state_t::accept)
76		− throw invalid_utf8{};
77		−
78		− return points;
	79	+ assert(str <= last);
	80	+ const auto byte_count = static_cast<std::size_t>(last - str + 1);
	81	+ return{ points, byte_count };
79	82	}
80	83
81	84	std::size_t disvm::runtime::utf8::decode(const uint8_t *str, rune_t &codepoint)