History

incorporate dynamic chunking into 'less-time' algorithm

Sebastian Thiel committed 6 years ago

Commit 295aa2f1 parent a8422cf4 files · +66 −33Browse files

Unified · Split

4 files
  1. git-features/src/parallel.rs +1 −1
  2. git-features/tests/parallel.rs +3 −3
  3. git-odb/src/pack/graph.rs +1 −1
  4. git-odb/src/pack/index/verify/indexed.rs +61 −28
git-features/src/parallel.rs+1−1
@@ -122,7 +122,7 @@ pub fn optimize_chunk_size_and_thread_limit( · 121 unmodified lines
122 .map(|l| if l == 0 { available_threads } else { l })122 .map(|l| if l == 0 { available_threads } else { l })
123 .unwrap_or(available_threads);123 .unwrap_or(available_threads);
124 124
125 let (lower, upper) = (50, 500);125 let (lower, upper) = (50, 1000);
126 let (chunk_size, thread_limit) = num_items126 let (chunk_size, thread_limit) = num_items
127 .map(|num_items| {127 .map(|num_items| {
128 let desired_chunks_per_thread_at_least = 2;128 let desired_chunks_per_thread_at_least = 2;
git-features/tests/parallel.rs+3−3
@@ -98,7 +98,7 @@ mod optimize_chunk_size_and_thread_limit { · 97 unmodified lines
98 fn linux_kernel_pack_my_machine_lookup() {98 fn linux_kernel_pack_my_machine_lookup() {
99 assert_eq!(99 assert_eq!(
100 optimize_chunk_size_and_thread_limit(10000, Some(7_500_000), None, Some(4)),100 optimize_chunk_size_and_thread_limit(10000, Some(7_500_000), None, Some(4)),
101 (500, Some(4), 4),101 (1000, Some(4), 4),
102 "the bucket size is capped actually, somewhat arbitrarily"102 "the bucket size is capped actually, somewhat arbitrarily"
103 );103 );
104 }104 }
@@ -110,8 +110,8 @@ mod optimize_chunk_size_and_thread_limit { · 5 unmodified lines
110 "low values are raised to arbitrary value"110 "low values are raised to arbitrary value"
111 );111 );
112 assert_eq!(112 assert_eq!(
113 optimize_chunk_size_and_thread_limit(1000, None, None, Some(4)),113 optimize_chunk_size_and_thread_limit(10000, None, None, Some(4)),
114 (500, Some(4), 4),114 (1000, Some(4), 4),
115 "high values are capped"115 "high values are capped"
116 );116 );
117 }117 }
git-odb/src/pack/graph.rs+1−1
@@ -25,7 +25,7 @@ pub struct DeltaTree { · 24 unmodified lines
25 inner: DiGraph<PackOffset, (), u32>, // u32 = max amount of objects in pack25 inner: DiGraph<PackOffset, (), u32>, // u32 = max amount of objects in pack
26}26}
27 27
28#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]28#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
29pub struct Node {29pub struct Node {
30 pub pack_offset: PackOffset,30 pub pack_offset: PackOffset,
31 index: NodeIndex<u32>,31 index: NodeIndex<u32>,
git-odb/src/pack/index/verify/indexed.rs+61−28
@@ -47,20 +47,48 @@ impl index::File { · 46 unmodified lines
47 reduce_progress.lock().unwrap().add_child(format!("thread {}", index)), // per thread progress47 reduce_progress.lock().unwrap().add_child(format!("thread {}", index)), // per thread progress
48 )48 )
49 };49 };
50 struct Chunks<I> {
51 size: usize,
52 iter: I,
53 }
54 impl<I> Iterator for Chunks<I>
55 where
56 I: Iterator<Item = pack::graph::Node>,
57 {
58 type Item = Vec<pack::graph::Node>;
59
60 fn next(&mut self) -> Option<Self::Item> {
61 let mut res = Vec::with_capacity(self.size);
62 let mut items_left = self.size;
63 while let Some(item) = self.iter.next() {
64 res.push(item);
65 items_left -= 1;
66 if items_left == 0 {
67 break;
68 }
69 }
70 if res.is_empty() {
71 None
72 } else {
73 Some(res)
74 }
75 }
76 }
50 in_parallel_if(77 in_parallel_if(
51 if_there_are_enough_objects,78 if_there_are_enough_objects,
52 tree.bases(),79 Chunks {
80 size: 50,
81 iter: tree.bases(),
82 },
53 thread_limit,83 thread_limit,
54 state_per_thread,84 state_per_thread,
55 |node: pack::graph::Node,85 |input: Vec<pack::graph::Node>,
56 (buf, encode_buf, nodes, progress)|86 (buf, encode_buf, nodes, progress)|
57 -> Result<Vec<pack::data::decode::Outcome>, Error> {87 -> Result<Vec<pack::data::decode::Outcome>, Error> {
58 let mut stats = Vec::new();88 let mut stats = Vec::new();
59 let mut header_buf = [0u8; 64];89 let mut header_buf = [0u8; 64];
60 nodes.clear();
61 nodes.push((node, 0));
62 progress.init(None, Some("entries"));
63 let mut children = Vec::new();90 let mut children = Vec::new();
91 progress.init(None, Some("entries"));
64 92
65 struct CacheEntry {93 struct CacheEntry {
66 kind: Kind,94 kind: Kind,
@@ -92,33 +120,38 @@ impl index::File { · 25 unmodified lines
92 )120 )
93 }121 }
94 }122 }
95 let mut cache = BTreeMap::new();
96 123
97 while let Some((node, level)) = nodes.pop() {124 for node in input {
98 let pack_offset = node.pack_offset;125 nodes.clear();
99 let index_entry = sorted_entries126 nodes.push((node, 0));
100 .binary_search_by(|e| e.pack_offset.cmp(&pack_offset))127 let mut cache = BTreeMap::new();
101 .expect("tree created by our sorted entries");
102 let index_entry_of_node = &sorted_entries[index_entry];
103 128
104 tree.children(node, &mut children);129 while let Some((node, level)) = nodes.pop() {
130 let pack_offset = node.pack_offset;
131 let index_entry = sorted_entries
132 .binary_search_by(|e| e.pack_offset.cmp(&pack_offset))
133 .expect("tree created by our sorted entries");
134 let index_entry_of_node = &sorted_entries[index_entry];
105 135
106 let shared_cache = &mut SharedCache(&mut cache);136 tree.children(node, &mut children);
107 let mut stat = self.process_entry(
108 mode,
109 pack,
110 shared_cache,
111 buf,
112 encode_buf,
113 progress,
114 &mut header_buf,
115 index_entry_of_node,
116 )?;
117 stat.num_deltas = level;
118 stats.push(stat);
119 137
120 progress.inc();138 let shared_cache = &mut SharedCache(&mut cache);
121 nodes.extend(children.iter().cloned().map(|cn| (cn, level + 1)));139 let mut stat = self.process_entry(
140 mode,
141 pack,
142 shared_cache,
143 buf,
144 encode_buf,
145 progress,
146 &mut header_buf,
147 index_entry_of_node,
148 )?;
149 stat.num_deltas = level;
150 stats.push(stat);
151
152 progress.inc();
153 nodes.extend(children.iter().cloned().map(|cn| (cn, level + 1)));
154 }
122 }155 }
123 156
124 Ok(stats)157 Ok(stats)