History

[pack #179] refactor

Sebastian Thiel committed 5 years ago

Commit 620d8a51 parent 21025697 files · +249 −248Browse files

Unified · Split

7 files
  1. git-pack/CHANGELOG.md +4 −1
  2. git-pack/src/data/output/count/mod.rs +2 −15
  3. git-pack/src/data/output/entry/mod.rs +2 −20
  4. git-pack/src/data/output/mod.rs +33 −4
  5. git-pack/src/find.rs +0 −206
  6. git-pack/src/find_traits.rs +205 −0
  7. git-pack/src/lib.rs +3 −2
git-pack/CHANGELOG.md+4−1
@@ -1,3 +1,6 @@
1### 0.9.0 (2021-08-??)1### 0.9.0 (2021-08-??)
2 2
3- TBD3- **renames / moves**
4 - `find::Find` and `find::FindExt` only in `Find` and `FindExt` (not in `find` anymore)
5 - `data::output::count::Count` -> `data::output::Count`
6 - `data::output::entry::Entry` -> `data::output::Entry`
git-pack/src/data/output/count/mod.rs+2−15
@@ -1,19 +1,6 @@
1use git_hash::ObjectId;
2
3use crate::data;1use crate::data;
4 2use crate::data::output::Count;
5/// An item representing a future Entry in the leanest way possible.3use git_hash::ObjectId;
6///
7/// One can expect to have one of these in memory when building big objects, so smaller is better here.
8/// They should contain everything of importance to generate a pack as fast as possible.
9#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]
10#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
11pub struct Count {
12 /// The hash of the object to write
13 pub id: ObjectId,
14 /// A way to locate a pack entry in the object database, only available if the object is in a pack.
15 pub entry_pack_location: PackLocation,
16}
17 4
18/// Specifies how the pack location was handled during counting5/// Specifies how the pack location was handled during counting
19#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]6#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]
git-pack/src/data/output/entry/mod.rs+2−20
@@ -9,24 +9,6 @@ use crate::{data, data::output, find}; · 8 unmodified lines
9pub mod iter_from_counts;9pub mod iter_from_counts;
10pub use iter_from_counts::iter_from_counts;10pub use iter_from_counts::iter_from_counts;
11 11
12/// An entry to be written to a file.
13///
14/// Some of these will be in-flight and in memory while waiting to be written. Memory requirements depend on the amount of compressed
15/// data they hold.
16#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]
17#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
18pub struct Entry {
19 /// The hash of the object to write
20 pub id: ObjectId,
21 /// The kind of entry represented by `data`. It's used alongside with it to complete the pack entry
22 /// at rest or in transit.
23 pub kind: Kind,
24 /// The size in bytes needed once `data` gets decompressed
25 pub decompressed_size: usize,
26 /// The compressed data right behind the header
27 pub compressed_data: Vec<u8>,
28}
29
30/// The kind of pack entry to be written12/// The kind of pack entry to be written
31#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]13#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
32#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]14#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
@@ -62,8 +44,8 @@ pub enum Error { · 29 unmodified lines
62 44
63impl output::Entry {45impl output::Entry {
64 /// An object which can be identified as invalid easily which happens if objects didn't exist even if they were referred to.46 /// An object which can be identified as invalid easily which happens if objects didn't exist even if they were referred to.
65 pub fn invalid() -> Entry {47 pub fn invalid() -> output::Entry {
66 Entry {48 output::Entry {
67 id: ObjectId::null_sha1(),49 id: ObjectId::null_sha1(),
68 kind: Kind::Base(git_object::Kind::Blob),50 kind: Kind::Base(git_object::Kind::Blob),
69 decompressed_size: 0,51 decompressed_size: 0,
git-pack/src/data/output/mod.rs+33−4
@@ -1,12 +1,41 @@
1use git_hash::ObjectId;
2
1///3///
2pub mod count;4pub mod count;
3#[doc(inline)]5
4pub use count::Count;6/// An item representing a future Entry in the leanest way possible.
7///
8/// One can expect to have one of these in memory when building big objects, so smaller is better here.
9/// They should contain everything of importance to generate a pack as fast as possible.
10#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]
11#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
12pub struct Count {
13 /// The hash of the object to write
14 pub id: ObjectId,
15 /// A way to locate a pack entry in the object database, only available if the object is in a pack.
16 pub entry_pack_location: count::PackLocation,
17}
18
19/// An entry to be written to a file.
20///
21/// Some of these will be in-flight and in memory while waiting to be written. Memory requirements depend on the amount of compressed
22/// data they hold.
23#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]
24#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
25pub struct Entry {
26 /// The hash of the object to write
27 pub id: ObjectId,
28 /// The kind of entry represented by `data`. It's used alongside with it to complete the pack entry
29 /// at rest or in transit.
30 pub kind: entry::Kind,
31 /// The size in bytes needed once `data` gets decompressed
32 pub decompressed_size: usize,
33 /// The compressed data right behind the header
34 pub compressed_data: Vec<u8>,
35}
5 36
6///37///
7pub mod entry;38pub mod entry;
8#[doc(inline)]
9pub use entry::Entry;
10 39
11///40///
12pub mod bytes;41pub mod bytes;
git-pack/src/find.rs+0−206
@@ -1,145 +1,3 @@
1use crate::data;
2
3/// Describe how object can be located in an object store with built-in facilities to supports packs specifically.
4///
5/// ## Notes
6///
7/// Locate effectively needs [generic associated types][issue] to allow a trait for the returned object type.
8/// Until then, we will have to make due with explicit types and give them the potentially added features we want.
9///
10/// [issue]: https://github.com/rust-lang/rust/issues/44265
11pub trait Find {
12 /// The error returned by [`find()`][Find::find()]
13 type Error: std::error::Error + 'static;
14
15 /// Find an object matching `id` in the database while placing its raw, undecoded data into `buffer`.
16 /// A `pack_cache` can be used to speed up subsequent lookups, set it to [`crate::cache::Never`] if the
17 /// workload isn't suitable for caching.
18 ///
19 /// Returns `Some` object if it was present in the database, or the error that occurred during lookup or object
20 /// retrieval.
21 fn find<'a>(
22 &self,
23 id: impl AsRef<git_hash::oid>,
24 buffer: &'a mut Vec<u8>,
25 pack_cache: &mut impl crate::cache::DecodeEntry,
26 ) -> Result<Option<data::Object<'a>>, Self::Error>;
27
28 /// Find the packs location where an object with `id` can be found in the database, or `None` if there is no pack
29 /// holding the object.
30 ///
31 /// _Note_ that the object database may have no notion of packs and thus always returns `None`.
32 fn location_by_oid(&self, id: impl AsRef<git_hash::oid>, buf: &mut Vec<u8>) -> Option<crate::bundle::Location>;
33
34 /// Find the bundle matching `pack_id`, or `None` if there is no such pack.
35 ///
36 /// _Note_ that the object database may have no notion of packs and thus always returns `None`.
37 fn bundle_by_pack_id(&self, pack_id: u32) -> Option<&crate::Bundle>;
38
39 /// Return the [`Entry`] for `location` if it is backed by a pack.
40 ///
41 /// Note that this is only in the interest of avoiding duplicate work during pack generation.
42 /// Pack locations can be obtained from a [`data::Object`].
43 ///
44 /// # Notes
45 ///
46 /// Custom implementations might be interested in providing their own meta-data with `object`,
47 /// which currently isn't possible as the `Locate` trait requires GATs to work like that.
48 fn entry_by_location(&self, location: &crate::bundle::Location) -> Option<Entry<'_>>;
49}
50
51mod ext {
52 use git_object::{BlobRef, CommitRef, CommitRefIter, Kind, ObjectRef, TagRef, TreeRef, TreeRefIter};
53
54 use crate::{data, find};
55
56 macro_rules! make_obj_lookup {
57 ($method:ident, $object_variant:path, $object_kind:path, $object_type:ty) => {
58 /// Like [`find_existing(…)`][Self::find_existing()], but flattens the `Result<Option<_>>` into a single `Result` making a non-existing object an error
59 /// while returning the desired object type.
60 fn $method<'a>(
61 &self,
62 id: impl AsRef<git_hash::oid>,
63 buffer: &'a mut Vec<u8>,
64 pack_cache: &mut impl crate::cache::DecodeEntry,
65 ) -> Result<$object_type, find::existing_object::Error<Self::Error>> {
66 let id = id.as_ref();
67 self.find(id, buffer, pack_cache)
68 .map_err(find::existing_object::Error::Find)?
69 .ok_or_else(|| find::existing_object::Error::NotFound {
70 oid: id.as_ref().to_owned(),
71 })
72 .and_then(|o| o.decode().map_err(find::existing_object::Error::Decode))
73 .and_then(|o| match o {
74 $object_variant(o) => return Ok(o),
75 _other => Err(find::existing_object::Error::ObjectKind {
76 expected: $object_kind,
77 }),
78 })
79 }
80 };
81 }
82
83 macro_rules! make_iter_lookup {
84 ($method:ident, $object_kind:path, $object_type:ty, $into_iter:tt) => {
85 /// Like [`find_existing(…)`][Self::find_existing()], but flattens the `Result<Option<_>>` into a single `Result` making a non-existing object an error
86 /// while returning the desired iterator type.
87 fn $method<'a>(
88 &self,
89 id: impl AsRef<git_hash::oid>,
90 buffer: &'a mut Vec<u8>,
91 pack_cache: &mut impl crate::cache::DecodeEntry,
92 ) -> Result<$object_type, find::existing_iter::Error<Self::Error>> {
93 let id = id.as_ref();
94 self.find(id, buffer, pack_cache)
95 .map_err(find::existing_iter::Error::Find)?
96 .ok_or_else(|| find::existing_iter::Error::NotFound {
97 oid: id.as_ref().to_owned(),
98 })
99 .and_then(|o| {
100 o.$into_iter()
101 .ok_or_else(|| find::existing_iter::Error::ObjectKind {
102 expected: $object_kind,
103 })
104 })
105 }
106 };
107 }
108
109 /// An extension trait with convenience functions.
110 pub trait FindExt: super::Find {
111 /// Like [`find(…)`][super::Find::find()], but flattens the `Result<Option<_>>` into a single `Result` making a non-existing object an error.
112 fn find_existing<'a>(
113 &self,
114 id: impl AsRef<git_hash::oid>,
115 buffer: &'a mut Vec<u8>,
116 pack_cache: &mut impl crate::cache::DecodeEntry,
117 ) -> Result<data::Object<'a>, find::existing::Error<Self::Error>> {
118 let id = id.as_ref();
119 self.find(id, buffer, pack_cache)
120 .map_err(find::existing::Error::Find)?
121 .ok_or_else(|| find::existing::Error::NotFound {
122 oid: id.as_ref().to_owned(),
123 })
124 }
125
126 make_obj_lookup!(find_existing_commit, ObjectRef::Commit, Kind::Commit, CommitRef<'a>);
127 make_obj_lookup!(find_existing_tree, ObjectRef::Tree, Kind::Tree, TreeRef<'a>);
128 make_obj_lookup!(find_existing_tag, ObjectRef::Tag, Kind::Tag, TagRef<'a>);
129 make_obj_lookup!(find_existing_blob, ObjectRef::Blob, Kind::Blob, BlobRef<'a>);
130 make_iter_lookup!(
131 find_existing_commit_iter,
132 Kind::Blob,
133 CommitRefIter<'a>,
134 into_commit_iter
135 );
136 make_iter_lookup!(find_existing_tree_iter, Kind::Tree, TreeRefIter<'a>, into_tree_iter);
137 }
138
139 impl<T: super::Find> FindExt for T {}
140}
141pub use ext::FindExt;
142
143///1///
144pub mod existing {2pub mod existing {
145 use git_hash::ObjectId;3 use git_hash::ObjectId;
@@ -205,67 +63,3 @@ pub struct Entry<'a> { · 59 unmodified lines
205 /// The version of the pack file containing `data`63 /// The version of the pack file containing `data`
206 pub version: crate::data::Version,64 pub version: crate::data::Version,
207}65}
208
209mod find_impls {
210 use std::ops::Deref;
211
212 use git_hash::oid;
213
214 use crate::{bundle::Location, data::Object, find::Entry, Bundle};
215
216 impl<T> super::Find for std::sync::Arc<T>
217 where
218 T: super::Find,
219 {
220 type Error = T::Error;
221
222 fn find<'a>(
223 &self,
224 id: impl AsRef<oid>,
225 buffer: &'a mut Vec<u8>,
226 pack_cache: &mut impl crate::cache::DecodeEntry,
227 ) -> Result<Option<Object<'a>>, Self::Error> {
228 self.deref().find(id, buffer, pack_cache)
229 }
230
231 fn location_by_oid(&self, id: impl AsRef<oid>, buf: &mut Vec<u8>) -> Option<Location> {
232 self.deref().location_by_oid(id, buf)
233 }
234
235 fn bundle_by_pack_id(&self, pack_id: u32) -> Option<&Bundle> {
236 self.deref().bundle_by_pack_id(pack_id)
237 }
238
239 fn entry_by_location(&self, object: &crate::bundle::Location) -> Option<Entry<'_>> {
240 self.deref().entry_by_location(object)
241 }
242 }
243
244 impl<T> super::Find for Box<T>
245 where
246 T: super::Find,
247 {
248 type Error = T::Error;
249
250 fn find<'a>(
251 &self,
252 id: impl AsRef<oid>,
253 buffer: &'a mut Vec<u8>,
254 pack_cache: &mut impl crate::cache::DecodeEntry,
255 ) -> Result<Option<Object<'a>>, Self::Error> {
256 self.deref().find(id, buffer, pack_cache)
257 }
258
259 fn location_by_oid(&self, id: impl AsRef<oid>, buf: &mut Vec<u8>) -> Option<Location> {
260 self.deref().location_by_oid(id, buf)
261 }
262
263 fn bundle_by_pack_id(&self, pack_id: u32) -> Option<&Bundle> {
264 self.deref().bundle_by_pack_id(pack_id)
265 }
266
267 fn entry_by_location(&self, location: &crate::bundle::Location) -> Option<Entry<'_>> {
268 self.deref().entry_by_location(location)
269 }
270 }
271}
git-pack/src/find_traits.rs+205−0
@@ -0,0 +1,205 @@
1use crate::{data, find};
2
3/// Describe how object can be located in an object store with built-in facilities to supports packs specifically.
4///
5/// ## Notes
6///
7/// Locate effectively needs [generic associated types][issue] to allow a trait for the returned object type.
8/// Until then, we will have to make due with explicit types and give them the potentially added features we want.
9///
10/// [issue]: https://github.com/rust-lang/rust/issues/44265
11pub trait Find {
12 /// The error returned by [`find()`][Find::find()]
13 type Error: std::error::Error + 'static;
14
15 /// Find an object matching `id` in the database while placing its raw, undecoded data into `buffer`.
16 /// A `pack_cache` can be used to speed up subsequent lookups, set it to [`crate::cache::Never`] if the
17 /// workload isn't suitable for caching.
18 ///
19 /// Returns `Some` object if it was present in the database, or the error that occurred during lookup or object
20 /// retrieval.
21 fn find<'a>(
22 &self,
23 id: impl AsRef<git_hash::oid>,
24 buffer: &'a mut Vec<u8>,
25 pack_cache: &mut impl crate::cache::DecodeEntry,
26 ) -> Result<Option<data::Object<'a>>, Self::Error>;
27
28 /// Find the packs location where an object with `id` can be found in the database, or `None` if there is no pack
29 /// holding the object.
30 ///
31 /// _Note_ that the object database may have no notion of packs and thus always returns `None`.
32 fn location_by_oid(&self, id: impl AsRef<git_hash::oid>, buf: &mut Vec<u8>) -> Option<crate::bundle::Location>;
33
34 /// Find the bundle matching `pack_id`, or `None` if there is no such pack.
35 ///
36 /// _Note_ that the object database may have no notion of packs and thus always returns `None`.
37 fn bundle_by_pack_id(&self, pack_id: u32) -> Option<&crate::Bundle>;
38
39 /// Return the [`find::Entry`] for `location` if it is backed by a pack.
40 ///
41 /// Note that this is only in the interest of avoiding duplicate work during pack generation.
42 /// Pack locations can be obtained from a [`data::Object`].
43 ///
44 /// # Notes
45 ///
46 /// Custom implementations might be interested in providing their own meta-data with `object`,
47 /// which currently isn't possible as the `Locate` trait requires GATs to work like that.
48 fn entry_by_location(&self, location: &crate::bundle::Location) -> Option<find::Entry<'_>>;
49}
50
51mod ext {
52 use git_object::{BlobRef, CommitRef, CommitRefIter, Kind, ObjectRef, TagRef, TreeRef, TreeRefIter};
53
54 use crate::{data, find};
55
56 macro_rules! make_obj_lookup {
57 ($method:ident, $object_variant:path, $object_kind:path, $object_type:ty) => {
58 /// Like [`find_existing(…)`][Self::find_existing()], but flattens the `Result<Option<_>>` into a single `Result` making a non-existing object an error
59 /// while returning the desired object type.
60 fn $method<'a>(
61 &self,
62 id: impl AsRef<git_hash::oid>,
63 buffer: &'a mut Vec<u8>,
64 pack_cache: &mut impl crate::cache::DecodeEntry,
65 ) -> Result<$object_type, find::existing_object::Error<Self::Error>> {
66 let id = id.as_ref();
67 self.find(id, buffer, pack_cache)
68 .map_err(find::existing_object::Error::Find)?
69 .ok_or_else(|| find::existing_object::Error::NotFound {
70 oid: id.as_ref().to_owned(),
71 })
72 .and_then(|o| o.decode().map_err(find::existing_object::Error::Decode))
73 .and_then(|o| match o {
74 $object_variant(o) => return Ok(o),
75 _other => Err(find::existing_object::Error::ObjectKind {
76 expected: $object_kind,
77 }),
78 })
79 }
80 };
81 }
82
83 macro_rules! make_iter_lookup {
84 ($method:ident, $object_kind:path, $object_type:ty, $into_iter:tt) => {
85 /// Like [`find_existing(…)`][Self::find_existing()], but flattens the `Result<Option<_>>` into a single `Result` making a non-existing object an error
86 /// while returning the desired iterator type.
87 fn $method<'a>(
88 &self,
89 id: impl AsRef<git_hash::oid>,
90 buffer: &'a mut Vec<u8>,
91 pack_cache: &mut impl crate::cache::DecodeEntry,
92 ) -> Result<$object_type, find::existing_iter::Error<Self::Error>> {
93 let id = id.as_ref();
94 self.find(id, buffer, pack_cache)
95 .map_err(find::existing_iter::Error::Find)?
96 .ok_or_else(|| find::existing_iter::Error::NotFound {
97 oid: id.as_ref().to_owned(),
98 })
99 .and_then(|o| {
100 o.$into_iter()
101 .ok_or_else(|| find::existing_iter::Error::ObjectKind {
102 expected: $object_kind,
103 })
104 })
105 }
106 };
107 }
108
109 /// An extension trait with convenience functions.
110 pub trait FindExt: super::Find {
111 /// Like [`find(…)`][super::Find::find()], but flattens the `Result<Option<_>>` into a single `Result` making a non-existing object an error.
112 fn find_existing<'a>(
113 &self,
114 id: impl AsRef<git_hash::oid>,
115 buffer: &'a mut Vec<u8>,
116 pack_cache: &mut impl crate::cache::DecodeEntry,
117 ) -> Result<data::Object<'a>, find::existing::Error<Self::Error>> {
118 let id = id.as_ref();
119 self.find(id, buffer, pack_cache)
120 .map_err(find::existing::Error::Find)?
121 .ok_or_else(|| find::existing::Error::NotFound {
122 oid: id.as_ref().to_owned(),
123 })
124 }
125
126 make_obj_lookup!(find_existing_commit, ObjectRef::Commit, Kind::Commit, CommitRef<'a>);
127 make_obj_lookup!(find_existing_tree, ObjectRef::Tree, Kind::Tree, TreeRef<'a>);
128 make_obj_lookup!(find_existing_tag, ObjectRef::Tag, Kind::Tag, TagRef<'a>);
129 make_obj_lookup!(find_existing_blob, ObjectRef::Blob, Kind::Blob, BlobRef<'a>);
130 make_iter_lookup!(
131 find_existing_commit_iter,
132 Kind::Blob,
133 CommitRefIter<'a>,
134 into_commit_iter
135 );
136 make_iter_lookup!(find_existing_tree_iter, Kind::Tree, TreeRefIter<'a>, into_tree_iter);
137 }
138
139 impl<T: super::Find> FindExt for T {}
140}
141pub use ext::FindExt;
142
143mod find_impls {
144 use std::ops::Deref;
145
146 use git_hash::oid;
147
148 use crate::{bundle::Location, data::Object, find, Bundle};
149
150 impl<T> super::Find for std::sync::Arc<T>
151 where
152 T: super::Find,
153 {
154 type Error = T::Error;
155
156 fn find<'a>(
157 &self,
158 id: impl AsRef<oid>,
159 buffer: &'a mut Vec<u8>,
160 pack_cache: &mut impl crate::cache::DecodeEntry,
161 ) -> Result<Option<Object<'a>>, Self::Error> {
162 self.deref().find(id, buffer, pack_cache)
163 }
164
165 fn location_by_oid(&self, id: impl AsRef<oid>, buf: &mut Vec<u8>) -> Option<Location> {
166 self.deref().location_by_oid(id, buf)
167 }
168
169 fn bundle_by_pack_id(&self, pack_id: u32) -> Option<&Bundle> {
170 self.deref().bundle_by_pack_id(pack_id)
171 }
172
173 fn entry_by_location(&self, object: &crate::bundle::Location) -> Option<find::Entry<'_>> {
174 self.deref().entry_by_location(object)
175 }
176 }
177
178 impl<T> super::Find for Box<T>
179 where
180 T: super::Find,
181 {
182 type Error = T::Error;
183
184 fn find<'a>(
185 &self,
186 id: impl AsRef<oid>,
187 buffer: &'a mut Vec<u8>,
188 pack_cache: &mut impl crate::cache::DecodeEntry,
189 ) -> Result<Option<Object<'a>>, Self::Error> {
190 self.deref().find(id, buffer, pack_cache)
191 }
192
193 fn location_by_oid(&self, id: impl AsRef<oid>, buf: &mut Vec<u8>) -> Option<Location> {
194 self.deref().location_by_oid(id, buf)
195 }
196
197 fn bundle_by_pack_id(&self, pack_id: u32) -> Option<&Bundle> {
198 self.deref().bundle_by_pack_id(pack_id)
199 }
200
201 fn entry_by_location(&self, location: &crate::bundle::Location) -> Option<find::Entry<'_>> {
202 self.deref().entry_by_location(location)
203 }
204 }
205}
git-pack/src/lib.rs+3−2
@@ -19,8 +19,9 @@ pub use crate::bundle::Bundle; · 18 unmodified lines
19 19
20///20///
21pub mod find;21pub mod find;
22#[doc(inline)]22
23pub use find::{Find, FindExt};23mod find_traits;
24pub use find_traits::{Find, FindExt};
24 25
25///26///
26pub mod cache;27pub mod cache;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t