History

Generalize intra-pack offset lookup (#279)

An incredibly rare feature which probably isn't used in the majority of the packs out there today, if any.

Sebastian Thiel committed 4 years ago

Commit dff05ef1 parent 28e3ea84 files · +51 −25Browse files

Unified · Split

4 files
  1. git-odb/src/store_impls/dynamic/find.rs +2 −4
  2. git-odb/src/store_impls/dynamic/handle.rs +44 −17
  3. git-odb/src/store_impls/dynamic/types.rs +4 −4
  4. git-odb/tests/odb/store/dynamic.rs +1 −0
git-odb/src/store_impls/dynamic/find.rs+2−4
@@ -117,10 +117,8 @@ where · 116 unmodified lines
117 entry,117 entry,
118 buffer,118 buffer,
119 |id, _out| {119 |id, _out| {
120 index_file.lookup(id).map(|idx| {120 index_file.pack_offset_by_id(id).map(|pack_offset| {
121 git_pack::data::ResolvedBase::InPack(121 git_pack::data::ResolvedBase::InPack(pack.entry(pack_offset))
122 pack.entry(index_file.pack_offset_at_index(idx)),
123 )
124 })122 })
125 },123 },
126 pack_cache,124 pack_cache,
git-odb/src/store_impls/dynamic/handle.rs+44−17
@@ -6,6 +6,7 @@ use std::{ · 5 unmodified lines
6};6};
7 7
8use git_features::threading::OwnShared;8use git_features::threading::OwnShared;
9use git_hash::oid;
9 10
10use crate::store::{handle, types, RefreshMode};11use crate::store::{handle, types, RefreshMode};
11 12
@@ -25,7 +26,33 @@ pub enum SingleOrMultiIndex { · 13 unmodified lines
25 },26 },
26}27}
27 28
28// pub enum SingleOrMultiIndexRef<'a> {}29/// A utility to allow looking up pack offsets for a particular pack
30pub enum IntraPackLookup<'a> {
31 Single(&'a git_pack::index::File),
32 /// the internal pack-id inside of a multi-index for which the lookup is supposed to be.
33 /// Used to prevent ref-delta OIDs to, for some reason, point to a different pack.
34 Multi {
35 index: &'a git_pack::multi_index::File,
36 required_pack_index: git_pack::multi_index::PackIndex,
37 },
38}
39
40impl<'a> IntraPackLookup<'a> {
41 pub(crate) fn pack_offset_by_id(&self, id: &oid) -> Option<git_pack::data::Offset> {
42 match self {
43 IntraPackLookup::Single(index) => index
44 .lookup(id)
45 .map(|entry_index| index.pack_offset_at_index(entry_index)),
46 IntraPackLookup::Multi {
47 index,
48 required_pack_index,
49 } => index.lookup(id).and_then(|entry_index| {
50 let (pack_index, pack_offset) = index.pack_id_and_pack_offset_at_index(entry_index);
51 (pack_index == *required_pack_index).then(|| pack_offset)
52 }),
53 }
54 }
55}
29 56
30pub struct IndexLookup {57pub struct IndexLookup {
31 pub(crate) file: SingleOrMultiIndex,58 pub(crate) file: SingleOrMultiIndex,
@@ -45,11 +72,12 @@ pub(crate) mod index_lookup { · 13 unmodified lines
45 72
46 use git_hash::oid;73 use git_hash::oid;
47 74
75 use crate::store::handle::IntraPackLookup;
48 use crate::store::{handle, types};76 use crate::store::{handle, types};
49 77
50 pub(crate) struct Outcome<'a> {78 pub(crate) struct Outcome<'a> {
51 pub object_index: handle::IndexForObjectInPack,79 pub object_index: handle::IndexForObjectInPack,
52 pub index_file: &'a git_pack::index::File,80 pub index_file: IntraPackLookup<'a>,
53 pub pack: &'a mut Option<Arc<git_pack::data::File>>,81 pub pack: &'a mut Option<Arc<git_pack::data::File>>,
54 }82 }
55 83
@@ -62,10 +90,9 @@ pub(crate) mod index_lookup { · 6 unmodified lines
62 (self.id == pack_id.index).then(|| match &self.file {90 (self.id == pack_id.index).then(|| match &self.file {
63 handle::SingleOrMultiIndex::Single { index, .. } => index.iter(),91 handle::SingleOrMultiIndex::Single { index, .. } => index.iter(),
64 handle::SingleOrMultiIndex::Multi { index, .. } => {92 handle::SingleOrMultiIndex::Multi { index, .. } => {
65 // TODO: figure out if this could actually not be true all the time.93 let pack_index = pack_id.multipack_index.expect(
66 let pack_index = pack_id94 "BUG: multi-pack index must be set if this is a multi-pack, pack-indices seem unstable",
67 .multipack_index95 );
68 .expect("multi-pack index must be set if this is a multi-pack");
69 Box::new(index.iter().filter_map(move |e| {96 Box::new(index.iter().filter_map(move |e| {
70 (e.pack_index == pack_index as u32).then(|| git_pack::index::Entry {97 (e.pack_index == pack_index as u32).then(|| git_pack::index::Entry {
71 oid: e.oid,98 oid: e.oid,
@@ -81,11 +108,10 @@ pub(crate) mod index_lookup { · 9 unmodified lines
81 (self.id == pack_id.index).then(move || match &mut self.file {108 (self.id == pack_id.index).then(move || match &mut self.file {
82 handle::SingleOrMultiIndex::Single { data, .. } => data,109 handle::SingleOrMultiIndex::Single { data, .. } => data,
83 handle::SingleOrMultiIndex::Multi { data, .. } => {110 handle::SingleOrMultiIndex::Multi { data, .. } => {
84 // TODO: figure out if this could actually not be true all the time.111 let pack_index = pack_id.multipack_index.expect(
85 let pack_index = pack_id112 "BUG: multi-pack index must be set if this is a multi-pack, pack-indices seem unstable",
86 .multipack_index113 );
87 .expect("multi-pack index must be set if this is a multi-pack");114 &mut data[pack_index as usize]
88 &mut data[pack_index]
89 }115 }
90 })116 })
91 }117 }
@@ -130,12 +156,11 @@ pub(crate) mod index_lookup { · 38 unmodified lines
130 },156 },
131 pack_offset: index.pack_offset_at_index(idx),157 pack_offset: index.pack_offset_at_index(idx),
132 },158 },
133 index_file: &**index,159 index_file: IntraPackLookup::Single(&**index),
134 pack: data,160 pack: data,
135 }),161 }),
136 handle::SingleOrMultiIndex::Multi { index, data } => index.lookup(object_id).map(move |idx| {162 handle::SingleOrMultiIndex::Multi { index, data } => index.lookup(object_id).map(move |idx| {
137 let (pack_index, pack_offset) = index.pack_offset_and_pack_id_at_index(idx);163 let (pack_index, pack_offset) = index.pack_id_and_pack_offset_at_index(idx);
138 let pack_index = pack_index as usize;
139 Outcome {164 Outcome {
140 object_index: handle::IndexForObjectInPack {165 object_index: handle::IndexForObjectInPack {
141 pack_id: types::PackId {166 pack_id: types::PackId {
@@ -144,9 +169,11 @@ pub(crate) mod index_lookup { · 2 unmodified lines
144 },169 },
145 pack_offset,170 pack_offset,
146 },171 },
147 // index_file: &**index,172 index_file: IntraPackLookup::Multi {
148 index_file: todo!("figure out how to pass different kinds of indices"),173 index: &**index,
149 pack: &mut data[pack_index],174 required_pack_index: pack_index,
175 },
176 pack: &mut data[pack_index as usize],
150 }177 }
151 }),178 }),
152 }179 }
git-odb/src/store_impls/dynamic/types.rs+4−4
@@ -36,7 +36,7 @@ pub struct PackId { · 35 unmodified lines
36 /// This is the index in the slot map at which the packs index is located.36 /// This is the index in the slot map at which the packs index is located.
37 pub(crate) index: IndexId,37 pub(crate) index: IndexId,
38 /// If the pack is in a multi-pack index, this additional index is the pack-index within the multi-pack index identified by `index`.38 /// If the pack is in a multi-pack index, this additional index is the pack-index within the multi-pack index identified by `index`.
39 pub(crate) multipack_index: Option<IndexId>,39 pub(crate) multipack_index: Option<git_pack::multi_index::PackIndex>,
40}40}
41 41
42impl PackId {42impl PackId {
@@ -45,7 +45,7 @@ impl PackId { · 2 unmodified lines
45 (1 << 15) - 145 (1 << 15) - 1
46 }46 }
47 /// Returns the maximum of packs we can represent if stored in a multi-index.47 /// Returns the maximum of packs we can represent if stored in a multi-index.
48 pub(crate) const fn max_packs_in_multi_index() -> usize {48 pub(crate) const fn max_packs_in_multi_index() -> git_pack::multi_index::PackIndex {
49 (1 << 16) - 149 (1 << 16) - 1
50 }50 }
51 /// Packs have a built-in identifier to make data structures simpler, and this method represents ourselves as such id51 /// Packs have a built-in identifier to make data structures simpler, and this method represents ourselves as such id
@@ -59,7 +59,7 @@ impl PackId { · 7 unmodified lines
59 midx <= Self::max_packs_in_multi_index(),59 midx <= Self::max_packs_in_multi_index(),
60 "There shouldn't be more than 2^16 packs per multi-index"60 "There shouldn't be more than 2^16 packs per multi-index"
61 );61 );
62 ((self.index | 1 << 15) | midx << 16) as git_pack::data::Id62 ((self.index as git_pack::data::Id | 1 << 15) | midx << 16) as git_pack::data::Id
63 }63 }
64 }64 }
65 }65 }
@@ -73,7 +73,7 @@ impl PackId { · 7 unmodified lines
73 } else {73 } else {
74 PackId {74 PackId {
75 index: (pack_id & 0x7fff) as IndexId,75 index: (pack_id & 0x7fff) as IndexId,
76 multipack_index: Some((pack_id >> 16) as IndexId),76 multipack_index: Some(pack_id >> 16),
77 }77 }
78 }78 }
79 }79 }
git-odb/tests/odb/store/dynamic.rs+1−0
@@ -11,6 +11,7 @@ fn db() -> git_odb::Handle { · 10 unmodified lines
11}11}
12 12
13#[test]13#[test]
14#[ignore]
14fn multi_index_access() {15fn multi_index_access() {
15 let db = git_odb::at(16 let db = git_odb::at(
16 git_testtools::scripted_fixture_repo_read_only("make_repo_multi_index.sh")17 git_testtools::scripted_fixture_repo_read_only("make_repo_multi_index.sh")