History

Implement seek() utfbytes() in Sys module. - Updated tests for utfbytes() - Updated utf8 API surface to support the general length case (codepoint _and_ byte) - Testing of seek() still outstanding Comment added regarding stack frame pointer in stack frames.

arr committed 9 years ago

Commit a6da1dc1 parent 1af55286 files · +94 −18Browse files

Unified · Split

6 files
  1. limbo/basic_limbo_tests.b +8 −0
  2. src/exec/main.cpp +1 −1
  3. src/include/utf8.h +9 −2
  4. src/vm/string.cpp +8 −3
  5. src/vm/sys/Sysmod.cpp +57 −4
  6. src/vm/utf8.cpp +11 −8
limbo/basic_limbo_tests.b+8−0
@@ -666,6 +666,14 @@ test_string() · 665 unmodified lines
666 array19 := array of byte count7;666 array19 := array of byte count7;
667 if (len array19 != 19) raise TEST_FAILED();667 if (len array19 != 19) raise TEST_FAILED();
668 668
669 # The above Devanagari text is used to test codepoint
670 # counting below. The text needs 6 bytes for the first
671 # two characters - 3 bytes each. Interpreting 5 bytes
672 # will yield 3 since the last 2 bytes will represent an
673 # incomplete codepoint.
674 codepoint_bytes := sys->utfbytes(array19, 5);
675 if (codepoint_bytes != 3) raise TEST_FAILED();
676
669 count36 := "abcdefghijklmnopqrstuvwxyz0123456789";677 count36 := "abcdefghijklmnopqrstuvwxyz0123456789";
670 if (len count36 != 36) raise TEST_FAILED();678 if (len count36 != 36) raise TEST_FAILED();
671 679
src/exec/main.cpp+1−1
@@ -32,7 +32,7 @@ namespace · 31 unmodified lines
32 {32 {
33 word_t noret;33 word_t noret;
34 pointer_t command_module_ref;34 pointer_t command_module_ref;
35 pointer_t command_stack_frame; // Should not be marked as memory managed pointer field35 pointer_t command_stack_frame; // [SPEC] Stack frames are not managed by the GC and as such should not be marked as a pointer field. See pointer map below.
36 };36 };
37 37
38 const auto entry_frame_pointer_map = std::vector<byte_t>{ 0x04 };38 const auto entry_frame_pointer_map = std::vector<byte_t>{ 0x04 };
src/include/utf8.h+9−2
@@ -29,9 +29,16 @@ namespace disvm · 28 unmodified lines
29 const disvm::runtime::rune_t max_supported_codepoint = max_codepoint_4_byte;29 const disvm::runtime::rune_t max_supported_codepoint = max_codepoint_4_byte;
30 };30 };
31 31
32 // Compute the number of code points in the supplied string32 // Lengths of a UTF-8 string
33 struct utf8_length_t
34 {
35 const std::size_t codepoint_count;
36 const std::size_t byte_count;
37 };
38
39 // Compute the number of codepoints in the supplied string
33 // See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.40 // See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
34 std::size_t count_codepoints(const uint8_t *str, const std::size_t max_len);41 utf8_length_t count_codepoints(const uint8_t *str, const std::size_t max_len);
35 42
36 enum class decode_state_t43 enum class decode_state_t
37 {44 {
src/vm/string.cpp+8−3
@@ -191,10 +191,15 @@ vm_string_t::vm_string_t(std::size_t encoded_str_len, const uint8_t *encoded_str · 190 unmodified lines
191 , _mem{}191 , _mem{}
192{192{
193 assert((encoded_str_len > 0 && encoded_str != nullptr) || (encoded_str_len == 0 && encoded_str == nullptr));193 assert((encoded_str_len > 0 && encoded_str != nullptr) || (encoded_str_len == 0 && encoded_str == nullptr));
194 auto character_count = utf8::count_codepoints(encoded_str, encoded_str_len);194 const auto utf8_length = utf8::count_codepoints(encoded_str, encoded_str_len);
195 _character_size = (encoded_str_len != character_count) ? sizeof(rune_t) : sizeof(char);
196 195
197 _length = static_cast<word_t>(character_count);196 // Check if the supplied string is completely valid UTF-8 - only valid UTF-8 strings can be DisVM strings.
197 if (encoded_str_len != utf8_length.byte_count)
198 throw invalid_utf8{};
199
200 _character_size = (encoded_str_len != utf8_length.codepoint_count) ? sizeof(rune_t) : sizeof(char);
201
202 _length = static_cast<word_t>(utf8_length.codepoint_count);
198 _length_max = sizeof(_mem.local) / _character_size;203 _length_max = sizeof(_mem.local) / _character_size;
199 204
200 // Allocate memory based on character count and size205 // Allocate memory based on character count and size
src/vm/sys/Sysmod.cpp+57−4
@@ -188,6 +188,15 @@ namespace · 187 unmodified lines
188 throw vm_user_exception{ "Write operation error" };188 throw vm_user_exception{ "Write operation error" };
189 }189 }
190 190
191 big_t seek(const std::ios::seekdir start, const big_t offset)
192 {
193 if (_stream == nullptr)
194 throw vm_user_exception{ "Unable to seek on a fifo file descriptor (e.g. stdin, stdout, stderr)" };
195
196 auto new_offset = _stream->rdbuf()->pubseekoff(static_cast<std::streamoff>(offset), start, _fd_mode);
197 return static_cast<big_t>(new_offset);
198 }
199
191 private:200 private:
192 const word_t _fd_mode;201 const word_t _fd_mode;
193 vm_string_t *_fd_path;202 vm_string_t *_fd_path;
@@ -218,6 +227,21 @@ namespace · 24 unmodified lines
218 227
219 return om;228 return om;
220 }229 }
230
231 std::ios::seekdir convert_to_seekdir(const word_t start)
232 {
233 switch (start)
234 {
235 case Sys_SEEKSTART:
236 return std::ios::beg;
237 case Sys_SEEKRELA:
238 return std::ios::cur;
239 case Sys_SEEKEND:
240 return std::ios::end;
241 default:
242 throw vm_user_exception{ "Invalid seek mode" };
243 }
244 }
221}245}
222 246
223void247void
@@ -612,7 +636,6 @@ Sys_read(vm_registers_t &r, vm_t &vm) · 388 unmodified lines
612 auto &fp = r.stack.peek_frame()->base<F_Sys_read>();636 auto &fp = r.stack.peek_frame()->base<F_Sys_read>();
613 637
614 const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);638 const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);
615 assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
616 639
617 auto n = fp.n;640 auto n = fp.n;
618 if (buffer == nullptr || n <= 0)641 if (buffer == nullptr || n <= 0)
@@ -621,11 +644,16 @@ Sys_read(vm_registers_t &r, vm_t &vm) · 2 unmodified lines
621 return;644 return;
622 }645 }
623 646
647 assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
648
624 // [SPEC] Supplying a size greater than the buffer length is649 // [SPEC] Supplying a size greater than the buffer length is
625 // equivalent to indicating the entire buffer should be filled.650 // equivalent to indicating the entire buffer should be filled.
626 n = std::min(n, buffer->get_length());651 n = std::min(n, buffer->get_length());
627 652
628 auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);653 auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);
654 if (fd_alloc == nullptr)
655 throw dereference_nil{ "Read from file descriptor" };
656
629 assert(fd_alloc->alloc_type == T_FD);657 assert(fd_alloc->alloc_type == T_FD);
630 658
631 auto fd = fd_alloc->get_allocation<vm_fd_t>();659 auto fd = fd_alloc->get_allocation<vm_fd_t>();
@@ -650,7 +678,16 @@ void · 18 unmodified lines
650Sys_seek(vm_registers_t &r, vm_t &vm)678Sys_seek(vm_registers_t &r, vm_t &vm)
651{679{
652 auto &fp = r.stack.peek_frame()->base<F_Sys_seek>();680 auto &fp = r.stack.peek_frame()->base<F_Sys_seek>();
653 throw vm_system_exception{ "Instruction not implemented" };681
682 auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);
683 if (fd_alloc == nullptr)
684 throw dereference_nil{ "Seek in file descriptor" };
685
686 assert(fd_alloc->alloc_type == T_FD);
687 auto fd = fd_alloc->get_allocation<vm_fd_t>();
688
689 const auto start = convert_to_seekdir(fp.start);
690 *fp.ret = fd->seek(start, fp.off);
654}691}
655 692
656void693void
@@ -814,7 +851,19 @@ void · 157 unmodified lines
814Sys_utfbytes(vm_registers_t &r, vm_t &vm)851Sys_utfbytes(vm_registers_t &r, vm_t &vm)
815{852{
816 auto &fp = r.stack.peek_frame()->base<F_Sys_utfbytes>();853 auto &fp = r.stack.peek_frame()->base<F_Sys_utfbytes>();
817 throw vm_system_exception{ "Instruction not implemented" };854 const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);
855
856 const auto n = fp.n;
857 if (buffer == nullptr || n > buffer->get_length())
858 throw out_of_range_memory{};
859
860 assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
861
862 auto raw_buffer = reinterpret_cast<uint8_t *>(buffer->at(0));
863
864 const auto len = utf8::count_codepoints(raw_buffer, n);
865 *fp.ret = static_cast<word_t>(len.byte_count);
866 assert(*fp.ret <= n && "Code points should not exceed number of bytes processed");
818}867}
819 868
820void869void
@@ -830,7 +879,6 @@ Sys_write(vm_registers_t &r, vm_t &vm) · 9 unmodified lines
830 auto &fp = r.stack.peek_frame()->base<F_Sys_write>();879 auto &fp = r.stack.peek_frame()->base<F_Sys_write>();
831 880
832 const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);881 const auto buffer = vm_alloc_t::from_allocation<vm_array_t>(fp.buf);
833 assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
834 882
835 auto n = fp.n;883 auto n = fp.n;
836 if (buffer == nullptr || n <= 0)884 if (buffer == nullptr || n <= 0)
@@ -839,11 +887,16 @@ Sys_write(vm_registers_t &r, vm_t &vm) · 2 unmodified lines
839 return;887 return;
840 }888 }
841 889
890 assert(buffer->get_element_type()->size_in_bytes == intrinsic_type_desc::type<byte_t>()->size_in_bytes);
891
842 // [SPEC] Supplying a size greater than the buffer length is892 // [SPEC] Supplying a size greater than the buffer length is
843 // equivalent to indicating the entire buffer should be written.893 // equivalent to indicating the entire buffer should be written.
844 n = std::min(n, buffer->get_length());894 n = std::min(n, buffer->get_length());
845 895
846 auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);896 auto fd_alloc = vm_alloc_t::from_allocation(fp.fd);
897 if (fd_alloc == nullptr)
898 throw dereference_nil{ "Write to file descriptor" };
899
847 assert(fd_alloc->alloc_type == T_FD);900 assert(fd_alloc->alloc_type == T_FD);
848 901
849 auto fd = fd_alloc->get_allocation<vm_fd_t>();902 auto fd = fd_alloc->get_allocation<vm_fd_t>();
src/vm/utf8.cpp+11−8
@@ -54,28 +54,31 @@ utf8::decode_state_t disvm::runtime::utf8::decode_step(utf8::decode_state_t &sta · 53 unmodified lines
54 54
55// Adopted from 'countCodePoints(uint8_t* s, size_t* count)'55// Adopted from 'countCodePoints(uint8_t* s, size_t* count)'
56// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.56// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
57std::size_t disvm::runtime::utf8::count_codepoints(const uint8_t *str, const std::size_t max_len)57utf8::utf8_length_t disvm::runtime::utf8::count_codepoints(const uint8_t *str, const std::size_t max_len)
58{58{
59 if (str == nullptr)59 if (str == nullptr)
60 {60 {
61 assert(max_len == 0);61 assert(max_len == 0);
62 return std::size_t{ 0 };62 return{ 0, 0 };
63 }63 }
64 64
65 auto single_character = rune_t{};65 auto single_character = rune_t{};
66 auto points = std::size_t{ 0 };66 auto points = std::size_t{ 0 };
67 auto state = utf8::decode_state_t::accept;67 auto state = utf8::decode_state_t::accept;
68 68
69 for (auto i = std::size_t{ 0 }; i < max_len; ++i)69 auto last = str;
70 for (auto curr = last; curr < (str + max_len); ++curr)
70 {71 {
71 if (utf8::decode_state_t::accept == utf8::decode_step(state, single_character, str[i]))72 if (utf8::decode_state_t::accept == utf8::decode_step(state, single_character, *curr))
73 {
74 last = curr;
72 points++;75 points++;
76 }
73 }77 }
74 78
75 if (state != utf8::decode_state_t::accept)79 assert(str <= last);
76 throw invalid_utf8{};80 const auto byte_count = static_cast<std::size_t>(last - str + 1);
77 81 return{ points, byte_count };
78 return points;
79}82}
80 83
81std::size_t disvm::runtime::utf8::decode(const uint8_t *str, rune_t &codepoint)84std::size_t disvm::runtime::utf8::decode(const uint8_t *str, rune_t &codepoint)