History

Created multiple pointer field iterators based on application Fix bug in vm_string class related to appending null at the end of the current string - This was recently introduced because I didn't test the case :( Scheduler had an invalid usage of std::mutex that was found using the debug version of CRT Fix heinous perf issue in GC where it was taking longer than code execution - Switch from std::queue to std::stack -> perf gain of ~20%

arr committed 9 years ago

Commit 30c4c581 parent 6f29a816 files · +102 −39Browse files

Unified · Split

6 files
  1. src/include/vm_memory.h +7 −1
  2. src/vm/garbage_collector.cpp +37 −20
  3. src/vm/scheduler.cpp +3 −3
  4. src/vm/scheduler.h +1 −1
  5. src/vm/string.cpp +21 −10
  6. src/vm/vm_memory.cpp +33 −4
src/include/vm_memory.h+7−1
@@ -50,11 +50,17 @@ namespace disvm · 49 unmodified lines
50 void dec_ref_count_and_free(vm_alloc_t *alloc);50 void dec_ref_count_and_free(vm_alloc_t *alloc);
51 51
52 // Callback for a pointer field supplying the pointer52 // Callback for a pointer field supplying the pointer
53 using pointer_field_callback_t = std::function<void(pointer_t)>;53 using pointer_field_callback_t = void(*)(pointer_t);
54 54
55 // The callback will be called on all non-null pointer fields.55 // The callback will be called on all non-null pointer fields.
56 void enum_pointer_fields(const type_descriptor_t &type_desc, void *data, pointer_field_callback_t callback);56 void enum_pointer_fields(const type_descriptor_t &type_desc, void *data, pointer_field_callback_t callback);
57 57
58 // Callback for a pointer field supplying the pointer and a calling context
59 using pointer_field_callback_ext_t = void(*)(pointer_t, void *cxt);
60
61 // The callback will be called on all non-null pointer fields.
62 void enum_pointer_fields(const type_descriptor_t &type_desc, void *data, pointer_field_callback_ext_t callback, void *cxt);
63
58 // Callback for a pointer field supplying the pointer and byte offset64 // Callback for a pointer field supplying the pointer and byte offset
59 using pointer_field_offset_callback_t = std::function<void(pointer_t, std::size_t)>;65 using pointer_field_offset_callback_t = std::function<void(pointer_t, std::size_t)>;
60 66
src/vm/garbage_collector.cpp+37−20
@@ -6,7 +6,7 @@ · 5 unmodified lines
6 6
7#include <cinttypes>7#include <cinttypes>
8#include <array>8#include <array>
9#include <queue>9#include <stack>
10#include <debug.h>10#include <debug.h>
11#include <vm_memory.h>11#include <vm_memory.h>
12#include <exceptions.h>12#include <exceptions.h>
@@ -124,20 +124,37 @@ void default_garbage_collector_t::enum_tracked_allocations(vm_alloc_callback_t c · 111 unmodified lines
124 124
125namespace125namespace
126{126{
127 void mark(const std::vector<std::shared_ptr<const vm_thread_t>> &threads, const gc_colour_t curr_colour)127 struct mark_cxt_t final : public std::stack<vm_alloc_t *, std::vector<vm_alloc_t *>>
128 {128 {
129 std::queue<vm_alloc_t *> allocs;129 mark_cxt_t(const type_descriptor_t *string_type, const gc_colour_t curr_colour)
130 : std::stack<vm_alloc_t *, std::vector<vm_alloc_t *>>{}
131 , string_type{ string_type }
132 , curr_colour{ curr_colour }
133 { }
134
135 const type_descriptor_t *string_type;
136 const gc_colour_t curr_colour;
137 };
130 138
131 // Making this raw since it is a built-in type and has unlimited lifetime.139 void mark_pointers(pointer_t p, void *cxt)
132 const auto string_type_raw = intrinsic_type_desc::type<vm_string_t>().get();140 {
133 auto add_pointer_to_queue = std::function<void(pointer_t p)>{ [&allocs, string_type_raw, curr_colour](pointer_t p) -> void141 auto a = vm_alloc_t::from_allocation(p);
134 {142
135 auto a = vm_alloc_t::from_allocation(p);143 assert(cxt != nullptr);
144 auto c = reinterpret_cast<mark_cxt_t *>(cxt);
136 145
137 // Don't examine vm strings or allocations with the current colour146 // Don't examine vm strings or allocations with the current colour
138 if (curr_colour != get_gc_colour(a) && a->alloc_type.get() != string_type_raw)147 if (c->curr_colour != get_gc_colour(a) && a->alloc_type.get() != c->string_type)
139 allocs.push(a);148 c->push(a);
140 } };149 }
150
151 void mark(const std::vector<std::shared_ptr<const vm_thread_t>> &threads, const gc_colour_t curr_colour)
152 {
153 mark_cxt_t mark_cxt
154 {
155 intrinsic_type_desc::type<vm_string_t>().get(), // Using raw pointer since it is a built-in type and has unlimited lifetime
156 curr_colour
157 };
141 158
142 // Get roots from threads159 // Get roots from threads
143 for (auto &t : threads)160 for (auto &t : threads)
@@ -145,7 +162,7 @@ namespace · 1 unmodified line
145 auto &r = t->get_registers();162 auto &r = t->get_registers();
146 163
147 // Current MP164 // Current MP
148 allocs.push(r.mp_base);165 mark_cxt.push(r.mp_base);
149 166
150 // Traverse the stack for roots167 // Traverse the stack for roots
151 for (auto frame = r.stack.peek_frame(); frame != nullptr; frame = frame->prev_frame())168 for (auto frame = r.stack.peek_frame(); frame != nullptr; frame = frame->prev_frame())
@@ -154,29 +171,29 @@ namespace · 2 unmodified lines
154 if (frame->prev_module_ref() != nullptr)171 if (frame->prev_module_ref() != nullptr)
155 {172 {
156 auto prev_mp = frame->prev_module_ref()->mp_base;173 auto prev_mp = frame->prev_module_ref()->mp_base;
157 allocs.push(prev_mp);174 mark_cxt.push(prev_mp);
158 }175 }
159 176
160 if (frame->frame_type->map_in_bytes > 0)177 if (frame->frame_type->map_in_bytes > 0)
161 enum_pointer_fields(*frame->frame_type, frame->base(), add_pointer_to_queue);178 enum_pointer_fields(*frame->frame_type, frame->base(), mark_pointers, &mark_cxt);
162 }179 }
163 }180 }
164 181
165 const auto log_enabled = debug::is_component_tracing_enabled<debug::component_trace_t::garbage_collector>();182 const auto log_enabled = debug::is_component_tracing_enabled<debug::component_trace_t::garbage_collector>();
166 if (log_enabled)183 if (log_enabled)
167 debug::log_msg(debug::component_trace_t::garbage_collector, debug::log_level_t::debug, "gc: roots found: %" PRIuPTR, allocs.size());184 debug::log_msg(debug::component_trace_t::garbage_collector, debug::log_level_t::debug, "gc: roots found: %" PRIuPTR, mark_cxt.size());
168 185
169 // Traverse the graph186 // Traverse the graph
170 while (!allocs.empty())187 while (!mark_cxt.empty())
171 {188 {
172 auto curr = allocs.front();189 auto curr = mark_cxt.top();
173 allocs.pop();190 mark_cxt.pop();
174 191
175 if (log_enabled)192 if (log_enabled)
176 debug::log_msg(debug::component_trace_t::garbage_collector, debug::log_level_t::debug, "gc: mark: alloc: %#" PRIxPTR, curr);193 debug::log_msg(debug::component_trace_t::garbage_collector, debug::log_level_t::debug, "gc: mark: alloc: %#" PRIxPTR, curr);
177 194
178 if (curr->alloc_type->map_in_bytes > 0)195 if (curr->alloc_type->map_in_bytes > 0)
179 enum_pointer_fields(*curr->alloc_type, curr->get_allocation(), add_pointer_to_queue);196 enum_pointer_fields(*curr->alloc_type, curr->get_allocation(), mark_pointers, &mark_cxt);
180 197
181 set_gc_colour(curr, curr_colour);198 set_gc_colour(curr, curr_colour);
182 }199 }
src/vm/scheduler.cpp+3−3
@@ -254,7 +254,7 @@ void default_scheduler_t::enqueue_blocked_thread(uint32_t thread_id) · 253 unmodified lines
254 if (debug::is_component_tracing_enabled<debug::component_trace_t::scheduler>())254 if (debug::is_component_tracing_enabled<debug::component_trace_t::scheduler>())
255 debug::log_msg(debug::component_trace_t::scheduler, debug::log_level_t::debug, "scheduler: enqueue blocked: %d", thread_id);255 debug::log_msg(debug::component_trace_t::scheduler, debug::log_level_t::debug, "scheduler: enqueue blocked: %d", thread_id);
256 256
257 const bool runnable_thread = enqueue_thread_unsafe(std::move(thread_container), vm_thread_state_t::ready);257 const bool runnable_thread = enqueue_thread_unsafe(thread_container.get(), vm_thread_state_t::ready);
258 if (runnable_thread)258 if (runnable_thread)
259 {259 {
260 lock_all.unlock();260 lock_all.unlock();
@@ -294,7 +294,7 @@ std::shared_ptr<default_scheduler_t::thread_instance_t> default_scheduler_t::nex · 33 unmodified lines
294 294
295 std::unique_lock<std::mutex> lock_all{ _vm_threads_lock };295 std::unique_lock<std::mutex> lock_all{ _vm_threads_lock };
296 --_running_vm_thread_count;296 --_running_vm_thread_count;
297 const bool runnable_thread = enqueue_thread_unsafe(std::move(prev_thread), current_state);297 const bool runnable_thread = enqueue_thread_unsafe(prev_thread.get(), current_state);
298 if (runnable_thread)298 if (runnable_thread)
299 {299 {
300 lock_all.unlock();300 lock_all.unlock();
@@ -380,7 +380,7 @@ void default_scheduler_t::perform_gc(bool is_gc_thread, std::unique_lock<std::mu · 79 unmodified lines
380 }380 }
381}381}
382 382
383bool default_scheduler_t::enqueue_thread_unsafe(std::shared_ptr<thread_instance_t> thread_instance, runtime::vm_thread_state_t current_state)383bool default_scheduler_t::enqueue_thread_unsafe(thread_instance_t *thread_instance, runtime::vm_thread_state_t current_state)
384{384{
385 assert(thread_instance != nullptr);385 assert(thread_instance != nullptr);
386 386
src/vm/scheduler.h+1−1
@@ -73,7 +73,7 @@ namespace disvm · 72 unmodified lines
73 73
74 // Add the thread to the queue in a non-thread safe manner.74 // Add the thread to the queue in a non-thread safe manner.
75 // Returns 'true' if the runnable thread queue has been updated, otherwise 'false'.75 // Returns 'true' if the runnable thread queue has been updated, otherwise 'false'.
76 bool enqueue_thread_unsafe(std::shared_ptr<thread_instance_t> thread, runtime::vm_thread_state_t current_state);76 bool enqueue_thread_unsafe(thread_instance_t *thread, runtime::vm_thread_state_t current_state);
77 77
78 private:78 private:
79 vm_t &_vm;79 vm_t &_vm;
src/vm/string.cpp+21−10
@@ -123,14 +123,14 @@ namespace · 122 unmodified lines
123 template<>123 template<>
124 void copy_characters_to(char *dest, word_t dest_start, const char *source, word_t source_length)124 void copy_characters_to(char *dest, word_t dest_start, const char *source, word_t source_length)
125 {125 {
126 assert(dest != nullptr && source != nullptr);126 assert(dest != nullptr && source != nullptr && dest != source);
127 std::memcpy((dest + dest_start), source, source_length);127 std::memcpy((dest + dest_start), source, source_length);
128 }128 }
129 129
130 template<>130 template<>
131 void copy_characters_to(runtime::rune_t *dest, word_t dest_start, const runtime::rune_t *source, word_t source_length)131 void copy_characters_to(runtime::rune_t *dest, word_t dest_start, const runtime::rune_t *source, word_t source_length)
132 {132 {
133 assert(dest != nullptr && source != nullptr);133 assert(dest != nullptr && source != nullptr && dest != source);
134 std::memcpy((dest + dest_start), source, (source_length * sizeof(runtime::rune_t)));134 std::memcpy((dest + dest_start), source, (source_length * sizeof(runtime::rune_t)));
135 }135 }
136 136
@@ -286,17 +286,19 @@ vm_string_t::vm_string_t(const vm_string_t &other, word_t begin_index, word_t en · 149 unmodified lines
286 // Allocate memory based on character count and size286 // Allocate memory based on character count and size
287 auto src = other._mem.local;287 auto src = other._mem.local;
288 auto dest = _mem.local;288 auto dest = _mem.local;
289 const auto length_in_bytes = _length * _character_size;
290 if (_is_alloc)289 if (_is_alloc)
291 {290 {
292 src = other._mem.alloc;291 src = other._mem.alloc;
293 _mem.alloc = alloc_memory<uint8_t>(_length_max);292 _mem.alloc = alloc_memory<uint8_t>(_length_max * _character_size);
294 dest = _mem.alloc;293 dest = _mem.alloc;
295 }294 }
296 295
297 // Copy over string content starting at the correct index (char vs. rune_t)296 // Copy over string content starting at the correct index (char vs. rune_t)
297 const auto length_in_bytes = _length * _character_size;
298 std::memcpy(dest, (src + (begin_index * _character_size)), length_in_bytes);298 std::memcpy(dest, (src + (begin_index * _character_size)), length_in_bytes);
299 debug::log_msg(debug::component_trace_t::memory, debug::log_level_t::debug, "copy: vm string: %#" PRIxPTR " %d %d %#" PRIxPTR, &other, begin_index, end_index, this);299
300 if (debug::is_component_tracing_enabled<debug::component_trace_t::memory>())
301 debug::log_msg(debug::component_trace_t::memory, debug::log_level_t::debug, "copy: vm string: %#" PRIxPTR " %d %d %#" PRIxPTR, &other, begin_index, end_index, this);
300}302}
301 303
302vm_string_t::~vm_string_t()304vm_string_t::~vm_string_t()
@@ -309,10 +311,11 @@ vm_string_t::~vm_string_t() · 6 unmodified lines
309 if (_is_alloc)311 if (_is_alloc)
310 free_memory(source);312 free_memory(source);
311 313
312 debug::log_msg(debug::component_trace_t::memory, debug::log_level_t::debug, "destroy: vm string");
313
314 debug::assign_debug_pointer(&source);314 debug::assign_debug_pointer(&source);
315 debug::assign_debug_pointer(&_encoded_str);315 debug::assign_debug_pointer(&_encoded_str);
316
317 if (debug::is_component_tracing_enabled<debug::component_trace_t::memory>())
318 debug::log_msg(debug::component_trace_t::memory, debug::log_level_t::debug, "destroy: vm string");
316}319}
317 320
318vm_string_t &vm_string_t::append(const vm_string_t &other)321vm_string_t &vm_string_t::append(const vm_string_t &other)
@@ -324,6 +327,7 @@ vm_string_t &vm_string_t::append(const vm_string_t &other) · 5 unmodified lines
324 327
325 // Check if the new string has any length at all.328 // Check if the new string has any length at all.
326 const auto other_length = other._length;329 const auto other_length = other._length;
330 assert(other_length >= 0);
327 if (other_length == 0)331 if (other_length == 0)
328 return *this;332 return *this;
329 333
@@ -342,7 +346,7 @@ vm_string_t &vm_string_t::append(const vm_string_t &other) · 12 unmodified lines
342 346
343 // Define the values for the appended string347 // Define the values for the appended string
344 const auto new_length = local_length + other_length;348 const auto new_length = local_length + other_length;
345 const auto new_length_max = (new_length == 0) ? static_cast<word_t>(compute_max_length(_mem.local)) : compute_max_length(new_length);349 const auto new_length_max = compute_max_length(new_length);
346 const auto new_is_rune = local_is_rune || other_is_rune;350 const auto new_is_rune = local_is_rune || other_is_rune;
347 uint8_t *new_source = nullptr;351 uint8_t *new_source = nullptr;
348 352
@@ -497,8 +501,15 @@ const char *vm_string_t::str() const · 148 unmodified lines
497 // If the string is ASCII then return the memory501 // If the string is ASCII then return the memory
498 if (_character_size == sizeof(char))502 if (_character_size == sizeof(char))
499 {503 {
500 _encoded_str = (_is_alloc) ? reinterpret_cast<char *>(_mem.alloc) : reinterpret_cast<char *>(_mem.local);504 auto str = (_is_alloc) ? _mem.alloc : _mem.local;
501 _encoded_str[_length] = '\0';505 if (_length_max <= _length)
506 {
507 auto tmp = alloc_memory<char>(_length + 1); // +1 for null
508 str = static_cast<uint8_t *>(std::memcpy(tmp, str, _length));
509 }
510
511 _encoded_str = reinterpret_cast<char *>(str);
512 assert(_encoded_str[_length] == '\0');
502 return _encoded_str;513 return _encoded_str;
503 }514 }
504 515
src/vm/vm_memory.cpp+33−4
@@ -81,13 +81,17 @@ void disvm::runtime::destroy_memory(const type_descriptor_t &type_desc, void *da · 80 unmodified lines
81 81
82void disvm::runtime::inc_ref_count_in_memory(const type_descriptor_t &type_desc, void *data)82void disvm::runtime::inc_ref_count_in_memory(const type_descriptor_t &type_desc, void *data)
83{83{
84 assert(data != nullptr);84 if (type_desc.size_in_bytes == 0)
85 return;
86
85 enum_pointer_fields(type_desc, data, add_ref_pointer_field);87 enum_pointer_fields(type_desc, data, add_ref_pointer_field);
86}88}
87 89
88void disvm::runtime::dec_ref_count_in_memory(const type_descriptor_t &type_desc, void *data)90void disvm::runtime::dec_ref_count_in_memory(const type_descriptor_t &type_desc, void *data)
89{91{
90 assert(data != nullptr);92 if (type_desc.size_in_bytes == 0)
93 return;
94
91 enum_pointer_fields(type_desc, data, dec_ref_and_free_pointer_field);95 enum_pointer_fields(type_desc, data, dec_ref_and_free_pointer_field);
92}96}
93 97
@@ -104,8 +108,6 @@ void disvm::runtime::dec_ref_count_and_free(vm_alloc_t *alloc) · 10 unmodified lines
104void disvm::runtime::enum_pointer_fields(const type_descriptor_t &type_desc, void *data, pointer_field_callback_t callback)108void disvm::runtime::enum_pointer_fields(const type_descriptor_t &type_desc, void *data, pointer_field_callback_t callback)
105{109{
106 assert(data != nullptr && callback != nullptr);110 assert(data != nullptr && callback != nullptr);
107 if (type_desc.size_in_bytes == 0)
108 return;
109 111
110 auto offset_accum = std::size_t{ 0 };112 auto offset_accum = std::size_t{ 0 };
111 auto memory = reinterpret_cast<word_t *>(data);113 auto memory = reinterpret_cast<word_t *>(data);
@@ -130,6 +132,33 @@ void disvm::runtime::enum_pointer_fields(const type_descriptor_t &type_desc, voi · 18 unmodified lines
130 }132 }
131}133}
132 134
135void disvm::runtime::enum_pointer_fields(const type_descriptor_t &type_desc, void *data, pointer_field_callback_ext_t callback, void *cxt)
136{
137 assert(data != nullptr && callback != nullptr);
138
139 auto offset_accum = std::size_t{ 0 };
140 auto memory = reinterpret_cast<word_t *>(data);
141 for (auto i = word_t{ 0 }; i < type_desc.map_in_bytes; ++i, memory += 8, offset_accum += (8 * sizeof(pointer_t)))
142 {
143 const auto words8 = type_desc.pointer_map[i];
144 assert(sizeof(words8) == 1);
145 if (words8 == 0)
146 continue;
147
148 const auto flags = std::bitset<sizeof(words8) * 8>{ words8 };
149
150 // Enumerating the flags in reverse order so memory access is sequential.
151 if (flags[7] && (memory[0] != runtime_constants::nil)) callback(reinterpret_cast<pointer_t>(memory[0]), cxt);
152 if (flags[6] && (memory[1] != runtime_constants::nil)) callback(reinterpret_cast<pointer_t>(memory[1]), cxt);
153 if (flags[5] && (memory[2] != runtime_constants::nil)) callback(reinterpret_cast<pointer_t>(memory[2]), cxt);
154 if (flags[4] && (memory[3] != runtime_constants::nil)) callback(reinterpret_cast<pointer_t>(memory[3]), cxt);
155 if (flags[3] && (memory[4] != runtime_constants::nil)) callback(reinterpret_cast<pointer_t>(memory[4]), cxt);
156 if (flags[2] && (memory[5] != runtime_constants::nil)) callback(reinterpret_cast<pointer_t>(memory[5]), cxt);
157 if (flags[1] && (memory[6] != runtime_constants::nil)) callback(reinterpret_cast<pointer_t>(memory[6]), cxt);
158 if (flags[0] && (memory[7] != runtime_constants::nil)) callback(reinterpret_cast<pointer_t>(memory[7]), cxt);
159 }
160}
161
133void disvm::runtime::enum_pointer_fields_with_offset(const type_descriptor_t &type_desc, void *data, pointer_field_offset_callback_t callback)162void disvm::runtime::enum_pointer_fields_with_offset(const type_descriptor_t &type_desc, void *data, pointer_field_offset_callback_t callback)
134{163{
135 assert(data != nullptr && callback != nullptr);164 assert(data != nullptr && callback != nullptr);