| @@ -58,6 +58,7 @@ where · 57 unmodified lines |
| 58 | move |n| { | 58 | move |n| { |
| 59 | ( | 59 | ( |
| 60 | Vec::new(), // object data buffer | 60 | Vec::new(), // object data buffer |
| | 61 | Vec::new(), // object data buffer 2 to hold two objects at a time |
| 61 | make_cache(), // cache to speed up pack operations | 62 | make_cache(), // cache to speed up pack operations |
| 62 | { | 63 | { |
| 63 | let mut p = progress.lock().add_child(format!("thread {}", n)); | 64 | let mut p = progress.lock().add_child(format!("thread {}", n)); |
| @@ -69,17 +70,114 @@ where · 5 unmodified lines |
| 69 | }, | 70 | }, |
| 70 | { | 71 | { |
| 71 | let seen_objs = Arc::clone(&seen_objs); | 72 | let seen_objs = Arc::clone(&seen_objs); |
| 72 | move |oids: Vec<Oid>, (buf, cache, progress)| { | 73 | move |oids: Vec<Oid>, (buf1, buf2, cache, progress)| { |
| 73 | use ObjectExpansion::*; | 74 | use ObjectExpansion::*; |
| 74 | let mut out = Vec::new(); | 75 | let mut out = Vec::new(); |
| 75 | let mut tree_traversal_state = git_traverse::tree::breadthfirst::State::default(); | 76 | let mut tree_traversal_state = git_traverse::tree::breadthfirst::State::default(); |
| | 77 | let mut tree_diff_state = git_diff::tree::State::default(); |
| | 78 | let mut parent_commit_ids = Vec::new(); |
| 76 | let seen_objs = seen_objs.as_ref(); | 79 | let seen_objs = seen_objs.as_ref(); |
| 77 | let mut traverse_delegate = tree::traverse::AllUnseen::new(seen_objs); | 80 | let mut traverse_delegate = tree::traverse::AllUnseen::new(seen_objs); |
| | 81 | let mut changes_delegate = tree::changes::AllNew::new(seen_objs); |
| 78 | | 82 | |
| 79 | for id in oids.into_iter() { | 83 | for id in oids.into_iter() { |
| 80 | let id = id.as_ref(); | 84 | let id = id.as_ref(); |
| 81 | let obj = db.find_existing(id, buf, cache)?; | 85 | let obj = db.find_existing(id, buf1, cache)?; |
| 82 | match input_object_expansion { | 86 | match input_object_expansion { |
| | 87 | TreeAdditionsComparedToAncestor => { |
| | 88 | use git_object::Kind::*; |
| | 89 | let mut obj = obj; |
| | 90 | let mut id = id.to_owned(); |
| | 91 | |
| | 92 | loop { |
| | 93 | push_obj_count_unique(&mut out, seen_objs, &id, &obj, progress); |
| | 94 | match obj.kind { |
| | 95 | Tree | Blob => break, |
| | 96 | Tag => { |
| | 97 | id = immutable::TagIter::from_bytes(obj.data) |
| | 98 | .target_id() |
| | 99 | .expect("every tag has a target"); |
| | 100 | obj = db.find_existing(id, buf1, cache)?; |
| | 101 | continue; |
| | 102 | } |
| | 103 | Commit => { |
| | 104 | let current_tree_iter = { |
| | 105 | let mut commit_iter = immutable::CommitIter::from_bytes(obj.data); |
| | 106 | let tree_id = commit_iter.tree_id().expect("every commit has a tree"); |
| | 107 | parent_commit_ids.clear(); |
| | 108 | for token in commit_iter { |
| | 109 | match token { |
| | 110 | Ok(immutable::commit::iter::Token::Parent { id }) => { |
| | 111 | parent_commit_ids.push(id) |
| | 112 | } |
| | 113 | Ok(_) => break, |
| | 114 | Err(err) => return Err(Error::CommitDecode(err)), |
| | 115 | } |
| | 116 | } |
| | 117 | let obj = db.find_existing(tree_id, buf1, cache)?; |
| | 118 | push_obj_count_unique(&mut out, seen_objs, &tree_id, &obj, progress); |
| | 119 | immutable::TreeIter::from_bytes(obj.data) |
| | 120 | }; |
| | 121 | |
| | 122 | let objects = if parent_commit_ids.is_empty() { |
| | 123 | traverse_delegate.clear(); |
| | 124 | git_traverse::tree::breadthfirst( |
| | 125 | current_tree_iter, |
| | 126 | &mut tree_traversal_state, |
| | 127 | |oid, buf| db.find_existing_tree_iter(oid, buf, cache).ok(), |
| | 128 | &mut traverse_delegate, |
| | 129 | ) |
| | 130 | .map_err(Error::TreeTraverse)?; |
| | 131 | &traverse_delegate.objects |
| | 132 | } else { |
| | 133 | changes_delegate.clear(); |
| | 134 | for commit_id in &parent_commit_ids { |
| | 135 | let parent_tree_id = { |
| | 136 | let parent_commit_obj = db.find_existing(commit_id, buf2, cache)?; |
| | 137 | |
| | 138 | push_obj_count_unique( |
| | 139 | &mut out, |
| | 140 | seen_objs, |
| | 141 | &commit_id, |
| | 142 | &parent_commit_obj, |
| | 143 | progress, |
| | 144 | ); |
| | 145 | immutable::CommitIter::from_bytes(parent_commit_obj.data) |
| | 146 | .tree_id() |
| | 147 | .expect("every commit has a tree") |
| | 148 | }; |
| | 149 | let parent_tree = { |
| | 150 | let parent_tree_obj = |
| | 151 | db.find_existing(parent_tree_id, buf2, cache)?; |
| | 152 | push_obj_count_unique( |
| | 153 | &mut out, |
| | 154 | seen_objs, |
| | 155 | &parent_tree_id, |
| | 156 | &parent_tree_obj, |
| | 157 | progress, |
| | 158 | ); |
| | 159 | immutable::TreeIter::from_bytes(parent_tree_obj.data) |
| | 160 | }; |
| | 161 | |
| | 162 | git_diff::tree::Changes::from(Some(parent_tree)) |
| | 163 | .needed_to_obtain( |
| | 164 | current_tree_iter.clone(), |
| | 165 | &mut tree_diff_state, |
| | 166 | |oid, buf| db.find_existing_tree_iter(oid, buf, cache).ok(), |
| | 167 | &mut changes_delegate, |
| | 168 | ) |
| | 169 | .map_err(Error::TreeChanges)?; |
| | 170 | } |
| | 171 | &changes_delegate.objects |
| | 172 | }; |
| | 173 | for id in objects.iter() { |
| | 174 | out.push(id_to_count(&db, buf2, id, progress)); |
| | 175 | } |
| | 176 | break; |
| | 177 | } |
| | 178 | } |
| | 179 | } |
| | 180 | } |
| 83 | TreeContents => { | 181 | TreeContents => { |
| 84 | use git_object::Kind::*; | 182 | use git_object::Kind::*; |
| 85 | let mut id: ObjectId = id.into(); | 183 | let mut id: ObjectId = id.into(); |
| @@ -97,7 +195,7 @@ where · 11 unmodified lines |
| 97 | ) | 195 | ) |
| 98 | .map_err(Error::TreeTraverse)?; | 196 | .map_err(Error::TreeTraverse)?; |
| 99 | for id in traverse_delegate.objects.iter() { | 197 | for id in traverse_delegate.objects.iter() { |
| 100 | out.push(id_to_count(&db, buf, id, progress)); | 198 | out.push(id_to_count(&db, buf1, id, progress)); |
| 101 | } | 199 | } |
| 102 | break; | 200 | break; |
| 103 | } | 201 | } |
| @@ -105,7 +203,7 @@ where · 1 unmodified line |
| 105 | id = immutable::CommitIter::from_bytes(obj.data) | 203 | id = immutable::CommitIter::from_bytes(obj.data) |
| 106 | .tree_id() | 204 | .tree_id() |
| 107 | .expect("every commit has a tree"); | 205 | .expect("every commit has a tree"); |
| 108 | obj = db.find_existing(id, buf, cache)?; | 206 | obj = db.find_existing(id, buf1, cache)?; |
| 109 | continue; | 207 | continue; |
| 110 | } | 208 | } |
| 111 | Blob => break, | 209 | Blob => break, |
| @@ -113,7 +211,7 @@ where · 1 unmodified line |
| 113 | id = immutable::TagIter::from_bytes(obj.data) | 211 | id = immutable::TagIter::from_bytes(obj.data) |
| 114 | .target_id() | 212 | .target_id() |
| 115 | .expect("every tag has a target"); | 213 | .expect("every tag has a target"); |
| 116 | obj = db.find_existing(id, buf, cache)?; | 214 | obj = db.find_existing(id, buf1, cache)?; |
| 117 | continue; | 215 | continue; |
| 118 | } | 216 | } |
| 119 | } | 217 | } |
| @@ -130,6 +228,56 @@ where · 10 unmodified lines |
| 130 | } | 228 | } |
| 131 | | 229 | |
| 132 | mod tree { | 230 | mod tree { |
| | 231 | pub mod changes { |
| | 232 | use dashmap::DashSet; |
| | 233 | use git_diff::tree::{ |
| | 234 | visit::{Action, Change}, |
| | 235 | Visit, |
| | 236 | }; |
| | 237 | use git_hash::{bstr::BStr, ObjectId}; |
| | 238 | use std::collections::HashSet; |
| | 239 | |
| | 240 | pub struct AllNew<'a> { |
| | 241 | pub objects: HashSet<ObjectId>, |
| | 242 | all_seen: &'a DashSet<ObjectId>, |
| | 243 | } |
| | 244 | |
| | 245 | impl<'a> AllNew<'a> { |
| | 246 | pub fn new(all_seen: &'a DashSet<ObjectId>) -> Self { |
| | 247 | AllNew { |
| | 248 | objects: Default::default(), |
| | 249 | all_seen, |
| | 250 | } |
| | 251 | } |
| | 252 | pub fn clear(&mut self) { |
| | 253 | self.objects.clear(); |
| | 254 | } |
| | 255 | } |
| | 256 | |
| | 257 | impl<'a> Visit for AllNew<'a> { |
| | 258 | fn pop_front_tracked_path_and_set_current(&mut self) {} |
| | 259 | |
| | 260 | fn push_back_tracked_path_component(&mut self, _component: &BStr) {} |
| | 261 | |
| | 262 | fn push_path_component(&mut self, _component: &BStr) {} |
| | 263 | |
| | 264 | fn pop_path_component(&mut self) {} |
| | 265 | |
| | 266 | fn visit(&mut self, change: Change) -> Action { |
| | 267 | match change { |
| | 268 | Change::Addition { oid, .. } | Change::Modification { oid, .. } => { |
| | 269 | let inserted = self.all_seen.insert(oid); |
| | 270 | if inserted { |
| | 271 | self.objects.insert(oid); |
| | 272 | } |
| | 273 | } |
| | 274 | Change::Deletion { .. } => {} |
| | 275 | }; |
| | 276 | Action::Continue |
| | 277 | } |
| | 278 | } |
| | 279 | } |
| | 280 | |
| 133 | pub mod traverse { | 281 | pub mod traverse { |
| 134 | use dashmap::DashSet; | 282 | use dashmap::DashSet; |
| 135 | use git_hash::{bstr::BStr, ObjectId}; | 283 | use git_hash::{bstr::BStr, ObjectId}; |
| @@ -255,6 +403,13 @@ mod types { · 119 unmodified lines |
| 255 | /// | 403 | /// |
| 256 | /// This mode is useful if all reachable objects should be added, as in cloning a repository. | 404 | /// This mode is useful if all reachable objects should be added, as in cloning a repository. |
| 257 | TreeContents, | 405 | TreeContents, |
| | 406 | /// If the input is a commit, obtain its ancestors and turn them into pack entries. Obtain the ancestor trees along with the commits |
| | 407 | /// tree and turn them into pack entries. Finally obtain the added/changed objects when comparing the ancestor trees with the |
| | 408 | /// current tree and turn them into entries as well. |
| | 409 | /// Otherwise, the same as [`AsIs`][ObjectExpansion::AsIs]. |
| | 410 | /// |
| | 411 | /// This mode is useful to build a pack containing only new objects compared to a previous state. |
| | 412 | TreeAdditionsComparedToAncestor, |
| 258 | } | 413 | } |
| 259 | | 414 | |
| 260 | impl Default for ObjectExpansion { | 415 | impl Default for ObjectExpansion { |