History

lookup_prefix for ODB (#298)

Sebastian Thiel committed 4 years ago

Commit a4ccd181 parent 6244c064 files · +90 −57Browse files

Unified · Split

4 files
  1. git-odb/src/find.rs +1 −0
  2. git-odb/src/store_impls/dynamic/find.rs +51 −18
  3. git-odb/src/store_impls/dynamic/handle.rs +9 −0
  4. git-odb/tests/odb/store/dynamic.rs +29 −39
git-odb/src/find.rs+1−0
@@ -1,5 +1,6 @@
1/// A way to indicate if a lookup, despite successful, was ambiguous or yielded exactly1/// A way to indicate if a lookup, despite successful, was ambiguous or yielded exactly
2/// one result in the particular index.2/// one result in the particular index.
3// TODO: find better name, ambiguous with git_pack::index::PrefixLookupResult (entry_index inside)
3pub type PrefixLookupResult = Result<git_hash::ObjectId, ()>;4pub type PrefixLookupResult = Result<git_hash::ObjectId, ()>;
4 5
5///6///
git-odb/src/store_impls/dynamic/find.rs+51−18
@@ -15,6 +15,8 @@ mod error { · 14 unmodified lines
15 pub enum Error {15 pub enum Error {
16 #[error("An error occurred while obtaining an object from the loose object store")]16 #[error("An error occurred while obtaining an object from the loose object store")]
17 Loose(#[from] loose::find::Error),17 Loose(#[from] loose::find::Error),
18 #[error("An error occurred looking up a prefix which requires iteration")]
19 LooseWalkDir(#[from] loose::iter::Error),
18 #[error("An error occurred while obtaining an object from the packed object store")]20 #[error("An error occurred while obtaining an object from the packed object store")]
19 Pack(#[from] pack::data::decode_entry::Error),21 Pack(#[from] pack::data::decode_entry::Error),
20 #[error(transparent)]22 #[error(transparent)]
@@ -32,8 +34,44 @@ where · 11 unmodified lines
32 S: Deref<Target = super::Store> + Clone,34 S: Deref<Target = super::Store> + Clone,
33{35{
34 #[allow(missing_docs)] // TODO: docs36 #[allow(missing_docs)] // TODO: docs
35 pub fn lookup_prefix(&self, _prefix: git_hash::Prefix) -> Result<Option<crate::find::PrefixLookupResult>, Error> {37 pub fn lookup_prefix(&self, prefix: git_hash::Prefix) -> Result<Option<crate::find::PrefixLookupResult>, Error> {
36 todo!()38 let mut candidate: Option<git_hash::ObjectId> = None;
39 loop {
40 let snapshot = self.snapshot.borrow();
41 {
42 for index in snapshot.indices.iter() {
43 match (index.lookup_prefix(prefix), candidate) {
44 (Some(Ok(oid)), Some(candidate)) if candidate != oid => return Ok(Some(Err(()))),
45 (Some(Ok(_)), Some(_)) | (None, None) | (None, Some(_)) => continue,
46 (Some(Err(())), _) => return Ok(Some(Err(()))),
47 (Some(Ok(oid)), None) => {
48 candidate = Some(oid);
49 continue;
50 }
51 }
52 }
53 }
54
55 for lodb in snapshot.loose_dbs.iter() {
56 match (lodb.lookup_prefix(prefix)?, candidate) {
57 (Some(Ok(oid)), Some(candidate)) if candidate != oid => return Ok(Some(Err(()))),
58 (Some(Ok(_)), Some(_)) | (None, None) | (None, Some(_)) => continue,
59 (Some(Err(())), _) => return Ok(Some(Err(()))),
60 (Some(Ok(oid)), None) => {
61 candidate = Some(oid);
62 continue;
63 }
64 }
65 }
66
67 match self.store.load_one_index(self.refresh_mode, snapshot.marker)? {
68 Some(new_snapshot) => {
69 drop(snapshot);
70 *self.snapshot.borrow_mut() = new_snapshot;
71 }
72 None => return Ok(candidate.map(Ok)),
73 }
74 }
37 }75 }
38}76}
39 77
@@ -46,8 +84,8 @@ where · 6 unmodified lines
46 // TODO: probably make this method fallible, but that would mean its own error type.84 // TODO: probably make this method fallible, but that would mean its own error type.
47 fn contains(&self, id: impl AsRef<oid>) -> bool {85 fn contains(&self, id: impl AsRef<oid>) -> bool {
48 let id = id.as_ref();86 let id = id.as_ref();
87 let mut snapshot = self.snapshot.borrow_mut();
49 loop {88 loop {
50 let mut snapshot = self.snapshot.borrow_mut();
51 {89 {
52 for (idx, index) in snapshot.indices.iter().enumerate() {90 for (idx, index) in snapshot.indices.iter().enumerate() {
53 if index.contains(id) {91 if index.contains(id) {
@@ -67,8 +105,7 @@ where · 13 unmodified lines
67 105
68 match self.store.load_one_index(self.refresh_mode, snapshot.marker) {106 match self.store.load_one_index(self.refresh_mode, snapshot.marker) {
69 Ok(Some(new_snapshot)) => {107 Ok(Some(new_snapshot)) => {
70 drop(snapshot);108 *snapshot = new_snapshot;
71 *self.snapshot.borrow_mut() = new_snapshot;
72 }109 }
73 Ok(None) => return false, // nothing more to load, or our refresh mode doesn't allow disk refreshes110 Ok(None) => return false, // nothing more to load, or our refresh mode doesn't allow disk refreshes
74 Err(_) => return false, // something went wrong, nothing we can communicate here with this trait. TODO: Maybe that should change?111 Err(_) => return false, // something went wrong, nothing we can communicate here with this trait. TODO: Maybe that should change?
@@ -83,8 +120,8 @@ where · 8 unmodified lines
83 pack_cache: &mut impl DecodeEntry,120 pack_cache: &mut impl DecodeEntry,
84 ) -> Result<Option<(Data<'a>, Option<Location>)>, Self::Error> {121 ) -> Result<Option<(Data<'a>, Option<Location>)>, Self::Error> {
85 let id = id.as_ref();122 let id = id.as_ref();
123 let mut snapshot = self.snapshot.borrow_mut();
86 'outer: loop {124 'outer: loop {
87 let mut snapshot = self.snapshot.borrow_mut();
88 {125 {
89 let marker = snapshot.marker;126 let marker = snapshot.marker;
90 for (idx, index) in snapshot.indices.iter_mut().enumerate() {127 for (idx, index) in snapshot.indices.iter_mut().enumerate() {
@@ -105,8 +142,7 @@ where · 14 unmodified lines
105 // The pack wasn't available anymore so we are supposed to try another round with a fresh index142 // The pack wasn't available anymore so we are supposed to try another round with a fresh index
106 match self.store.load_one_index(self.refresh_mode, snapshot.marker)? {143 match self.store.load_one_index(self.refresh_mode, snapshot.marker)? {
107 Some(new_snapshot) => {144 Some(new_snapshot) => {
108 drop(snapshot);145 *snapshot = new_snapshot;
109 *self.snapshot.borrow_mut() = new_snapshot;
110 continue 'outer;146 continue 'outer;
111 }147 }
112 None => {148 None => {
@@ -166,8 +202,7 @@ where · 53 unmodified lines
166 202
167 match self.store.load_one_index(self.refresh_mode, snapshot.marker)? {203 match self.store.load_one_index(self.refresh_mode, snapshot.marker)? {
168 Some(new_snapshot) => {204 Some(new_snapshot) => {
169 drop(snapshot);205 *snapshot = new_snapshot;
170 *self.snapshot.borrow_mut() = new_snapshot;
171 }206 }
172 None => return Ok(None),207 None => return Ok(None),
173 }208 }
@@ -180,8 +215,8 @@ where · 6 unmodified lines
180 "BUG: handle must be configured to `prevent_pack_unload()` before using this method"215 "BUG: handle must be configured to `prevent_pack_unload()` before using this method"
181 );216 );
182 let id = id.as_ref();217 let id = id.as_ref();
218 let mut snapshot = self.snapshot.borrow_mut();
183 'outer: loop {219 'outer: loop {
184 let mut snapshot = self.snapshot.borrow_mut();
185 {220 {
186 let marker = snapshot.marker;221 let marker = snapshot.marker;
187 for (idx, index) in snapshot.indices.iter_mut().enumerate() {222 for (idx, index) in snapshot.indices.iter_mut().enumerate() {
@@ -202,8 +237,7 @@ where · 14 unmodified lines
202 // The pack wasn't available anymore so we are supposed to try another round with a fresh index237 // The pack wasn't available anymore so we are supposed to try another round with a fresh index
203 match self.store.load_one_index(self.refresh_mode, snapshot.marker).ok()? {238 match self.store.load_one_index(self.refresh_mode, snapshot.marker).ok()? {
204 Some(new_snapshot) => {239 Some(new_snapshot) => {
205 drop(snapshot);240 *snapshot = new_snapshot;
206 *self.snapshot.borrow_mut() = new_snapshot;
207 continue 'outer;241 continue 'outer;
208 }242 }
209 None => {243 None => {
@@ -239,8 +273,7 @@ where · 29 unmodified lines
239 273
240 match self.store.load_one_index(self.refresh_mode, snapshot.marker).ok()? {274 match self.store.load_one_index(self.refresh_mode, snapshot.marker).ok()? {
241 Some(new_snapshot) => {275 Some(new_snapshot) => {
242 drop(snapshot);276 *snapshot = new_snapshot;
243 *self.snapshot.borrow_mut() = new_snapshot;
244 }277 }
245 None => return None,278 None => return None,
246 }279 }
@@ -279,9 +312,9 @@ where · 32 unmodified lines
279 "BUG: handle must be configured to `prevent_pack_unload()` before using this method"312 "BUG: handle must be configured to `prevent_pack_unload()` before using this method"
280 );313 );
281 let pack_id = PackId::from_intrinsic_pack_id(location.pack_id);314 let pack_id = PackId::from_intrinsic_pack_id(location.pack_id);
315 let mut snapshot = self.snapshot.borrow_mut();
316 let marker = snapshot.marker;
282 loop {317 loop {
283 let mut snapshot = self.snapshot.borrow_mut();
284 let marker = snapshot.marker;
285 {318 {
286 for index in snapshot.indices.iter_mut() {319 for index in snapshot.indices.iter_mut() {
287 if let Some(possibly_pack) = index.pack(pack_id) {320 if let Some(possibly_pack) = index.pack(pack_id) {
@@ -289,7 +322,7 @@ where · 1 unmodified line
289 Some(pack) => pack,322 Some(pack) => pack,
290 None => {323 None => {
291 let pack = self.store.load_pack(pack_id, marker).ok()?.expect(324 let pack = self.store.load_pack(pack_id, marker).ok()?.expect(
292 "BUG: pack must exist from previous call to locaion_by_oid() and must not be unloaded",325 "BUG: pack must exist from previous call to location_by_oid() and must not be unloaded",
293 );326 );
294 *possibly_pack = Some(pack);327 *possibly_pack = Some(pack);
295 possibly_pack.as_deref().expect("just put it in")328 possibly_pack.as_deref().expect("just put it in")
git-odb/src/store_impls/dynamic/handle.rs+9−0
@@ -138,6 +138,15 @@ pub(crate) mod index_lookup { · 137 unmodified lines
138 }138 }
139 }139 }
140 140
141 #[allow(missing_docs)] // TODO: docs
142 pub(crate) fn lookup_prefix(&self, prefix: git_hash::Prefix) -> Option<crate::find::PrefixLookupResult> {
143 let res = match &self.file {
144 handle::SingleOrMultiIndex::Single { index, .. } => index.lookup_prefix(prefix),
145 handle::SingleOrMultiIndex::Multi { index, .. } => index.lookup_prefix(prefix),
146 }?;
147 Some(res.map(|entry_index| self.oid_at_index(entry_index).to_owned()))
148 }
149
141 /// See if the oid is contained in this index, and return its full id for lookup possibly alongside its data file if already150 /// See if the oid is contained in this index, and return its full id for lookup possibly alongside its data file if already
142 /// loaded.151 /// loaded.
143 /// Also return the index itself as it's needed to resolve intra-pack ref-delta objects. They are a possibility even though152 /// Also return the index itself as it's needed to resolve intra-pack ref-delta objects. They are a possibility even though
git-odb/tests/odb/store/dynamic.rs+29−39
@@ -381,53 +381,43 @@ fn lookup() { · 380 unmodified lines
381 381
382mod lookup_prefix {382mod lookup_prefix {
383 use crate::store::dynamic::db_with_all_object_sources;383 use crate::store::dynamic::db_with_all_object_sources;
384 use git_testtools::hex_to_id;
384 385
385 #[test]386 #[test]
386 #[should_panic]
387 fn returns_none_for_prefixes_without_any_match() {387 fn returns_none_for_prefixes_without_any_match() {
388 let (handle, _tmp) = db_with_all_object_sources().unwrap();388 let (handle, _tmp) = db_with_all_object_sources().unwrap();
389 let prefix = git_hash::Prefix::new(git_hash::ObjectId::null(git_hash::Kind::Sha1), 7).unwrap();389 let prefix = git_hash::Prefix::new(git_hash::ObjectId::null(git_hash::Kind::Sha1), 7).unwrap();
390 assert!(handle.lookup_prefix(prefix).unwrap().is_none());390 assert!(handle.lookup_prefix(prefix).unwrap().is_none());
391 }391 }
392 392
393 // #[test]393 #[test]
394 // fn returns_some_err_for_prefixes_with_more_than_one_match() {394 fn returns_some_err_for_prefixes_with_more_than_one_match() {
395 // let objects_dir = git_testtools::tempfile::tempdir().unwrap();395 let (store, _tmp) = db_with_all_object_sources().unwrap();
396 // git_testtools::copy_recursively_into_existing_dir(fixture_path("objects"), &objects_dir).unwrap();396 let prefix = git_hash::Prefix::new(hex_to_id("a7065b5e971a6d8b55875d8cf634a3a37202ab23"), 4).unwrap();
397 // std::fs::write(397 assert_eq!(
398 // objects_dir398 store.lookup_prefix(prefix).unwrap(),
399 // .path()399 Some(Err(())),
400 // .join("37")400 "there are two objects with that prefix"
401 // .join("d4ffffffffffffffffffffffffffffffffffff"),401 );
402 // b"fake",402 }
403 // )403
404 // .unwrap();404 #[test]
405 // let store = git_odb::loose::Store::at(objects_dir.path(), git_hash::Kind::Sha1);405 fn iterable_objects_can_be_looked_up_with_varying_prefix_lengths() {
406 // let prefix = git_hash::Prefix::new(hex_to_id("37d4e6c5c48ba0d245164c4e10d5f41140cab980"), 4).unwrap();406 let (store, _tmp) = db_with_all_object_sources().unwrap();
407 // assert_eq!(407 let hex_lengths = &[5, 7, 40];
408 // store.lookup_prefix(prefix).unwrap(),408 for (index, oid) in store.iter().unwrap().map(Result::unwrap).enumerate() {
409 // Some(Err(())),409 let hex_len = hex_lengths[index % hex_lengths.len()];
410 // "there are two objects with that prefix"410 let prefix = git_hash::Prefix::new(oid, hex_len).unwrap();
411 // );411 assert_eq!(
412 // }412 store
413 413 .lookup_prefix(prefix)
414 // #[test]414 .unwrap()
415 // fn iterable_objects_can_be_looked_up_with_varying_prefix_lengths() {415 .expect("object exists")
416 // let store = ldb();416 .expect("unambiguous"),
417 // let hex_lengths = &[4, 7, 40];417 oid
418 // for (index, oid) in store.iter().map(Result::unwrap).enumerate() {418 );
419 // let hex_len = hex_lengths[index % hex_lengths.len()];419 }
420 // let prefix = git_hash::Prefix::new(oid, hex_len).unwrap();420 }
421 // assert_eq!(
422 // store
423 // .lookup_prefix(prefix)
424 // .unwrap()
425 // .expect("object exists")
426 // .expect("unambiguous"),
427 // oid
428 // );
429 // }
430 // }
431}421}
432 422
433#[test]423#[test]