History

[pack] a quickly made iterator that writes input::Entries

Sebastian Thiel committed 5 years ago

Commit 116bdc41 parent a4d27643 files · +43 −79Browse files

Unified · Split

3 files
  1. git-pack/src/data/input/bytes.rs +40 −77
  2. git-pack/src/data/input/mod.rs +2 −1
  3. git-pack/src/data/output/bytes.rs +1 −1
git-pack/src/data/input/bytes.rs+40−77
@@ -1,25 +1,12 @@
1−use crate::data::output;
1+use crate::data::input;
22 use git_features::hash;
33 use std::io::Write;
44
5−/// The error returned by `next()` in the [`FromEntriesIter`] iterator.
6−#[allow(missing_docs)]
7−#[derive(Debug, thiserror::Error)]
8−pub enum Error<E>
9−where
10− E: std::error::Error + 'static,
11−{
12− #[error(transparent)]
13− Io(#[from] std::io::Error),
14− #[error(transparent)]
15− Input(E),
16−}
17−
185 /// An implementation of [`Iterator`] to write [encoded entries][output::Entry] to an inner implementation each time
196 /// `next()` is called.
207 pub struct FromEntriesIter<I, W> {
218 /// An iterator for input [`output::Entry`] instances
22− pub input: I,
9+ pub inner: I,
2310 /// A way of writing encoded bytes.
2411 output: hash::Write<W>,
2512 /// Our trailing hash when done writing all input entries
@@ -27,31 +14,19 @@ pub struct FromEntriesIter<I, W> { · 1 unmodified line
2714 /// The amount of objects in the iteration and the version of the packfile to be written.
2815 /// Will be `None` to signal the header was written already.
2916 header_info: Option<(crate::data::Version, u32)>,
30− /// The pack data version with which pack entries should be written.
31− entry_version: crate::data::Version,
32− /// The amount of written bytes thus far
33− written: u64,
34− /// Required to quickly find offsets by object IDs, as future objects may refer to those in the past to become a delta offset base.
35− /// It stores the pack offsets at which objects begin.
36− /// Additionally we store if an object was invalid, and if so we will not write it nor will we allow delta objects to it.
37− pack_offsets_and_validity: Vec<(u64, bool)>,
3817 /// If we are done, no additional writes will occour
3918 is_done: bool,
4019 }
4120
42−impl<I, W, E> FromEntriesIter<I, W>
21+impl<I, W> FromEntriesIter<I, W>
4322 where
44− I: Iterator<Item = Result<Vec<output::Entry>, E>>,
23+ I: Iterator<Item = Result<input::Entry, input::Error>>,
4524 W: std::io::Write,
46− E: std::error::Error + 'static,
4725 {
48− /// Create a new instance reading [entries][output::Entry] from an `input` iterator and write pack data bytes to
26+ /// Create a new instance reading [entries][input::Entry] from an `input` iterator and write pack data bytes to
4927 /// `output` writer, resembling a pack of `version` with exactly `num_entries` amount of objects contained in it.
5028 /// `hash_kind` is the kind of hash to use for the pack checksum and maybe other places, depending on the version.
5129 ///
52− /// The input chunks are expected to be sorted already. You can use the [InOrderIter][super::InOrderIter] to assure
53− /// this happens on the fly holding entire chunks in memory as long as needed for them to be dispensed in order.
54− ///
5530 /// # Panics
5631 ///
5732 /// Not all combinations of `hash_kind` and `version` are supported currently triggering assertion errors.
@@ -71,12 +46,9 @@ where · 13 unmodified lines
7146 "currently only Sha1 is supported, right now we don't know how other hashes are encoded",
7247 );
7348 FromEntriesIter {
74− input,
49+ inner: input,
7550 output: hash::Write::new(output, hash_kind),
7651 trailer: None,
77− entry_version: version,
78− pack_offsets_and_validity: Vec::with_capacity(num_entries as usize),
79− written: 0,
8052 header_info: Some((version, num_entries)),
8153 is_done: false,
8254 }
@@ -84,75 +56,66 @@ where · 1 unmodified line
8456
8557 /// Consume this instance and return the `output` implementation.
8658 ///
87− /// _Note_ that the input can be moved out of this instance beforehand.
59+ /// _Note_ that the `inner` iterator can be moved out of this instance beforehand.
8860 pub fn into_write(self) -> W {
8961 self.output.inner
9062 }
9163
92− /// Returns the trailing hash over all written entries once done.
64+ /// Returns the trailing hash over all ~ entries once done.
9365 /// It's `None` if we are not yet done writing.
9466 pub fn digest(&self) -> Option<git_hash::ObjectId> {
9567 self.trailer
9668 }
9769
98− fn next_inner(&mut self) -> Result<u64, Error<E>> {
99− let previous_written = self.written;
70+ fn next_inner(&mut self, entry: input::Entry) -> Result<input::Entry, input::Error> {
10071 if let Some((version, num_entries)) = self.header_info.take() {
10172 let header_bytes = crate::data::header::encode(version, num_entries);
10273 self.output.write_all(&header_bytes[..])?;
103− self.written += header_bytes.len() as u64;
10474 }
105− match self.input.next() {
106− Some(entries) => {
107− for entry in entries.map_err(Error::Input)? {
108− if entry.is_invalid() {
109− self.pack_offsets_and_validity.push((0, false));
110− continue;
111− };
112− self.pack_offsets_and_validity.push((self.written, true));
113− let header = entry.to_entry_header(self.entry_version, |index| {
114− let (base_offset, is_valid_object) = self.pack_offsets_and_validity[index];
115− if !is_valid_object {
116− unreachable!("if you see this the object database is correct as a delta refers to a non-existing object")
117− }
118− self.written - base_offset
119− });
120− self.written += header.write_to(entry.decompressed_size as u64, &mut self.output)? as u64;
121− self.written += std::io::copy(&mut &*entry.compressed_data, &mut self.output)? as u64;
122− }
123− }
124− None => {
125− let digest = self.output.hash.clone().digest();
126− self.output.inner.write_all(&digest[..])?;
127− self.written += digest.len() as u64;
128− self.output.inner.flush()?;
129− self.is_done = true;
130− self.trailer = Some(git_hash::ObjectId::from(digest));
131− }
132− };
133− Ok(self.written - previous_written)
75+ entry
76+ .header
77+ .write_to(entry.decompressed_size as u64, &mut self.output)? as u64;
78+ std::io::copy(
79+ &mut &*entry
80+ .compressed
81+ .as_ref()
82+ .map(|v| v.as_slice())
83+ .expect("caller must configure generator to keep compressed bytes"),
84+ &mut self.output,
85+ )?;
86+ Ok(entry)
87+ }
88+
89+ fn write_digest(&mut self) -> Result<(), input::Error> {
90+ let digest = self.output.hash.clone().digest();
91+ self.output.inner.write_all(&digest[..])?;
92+ self.output.inner.flush()?;
93+ self.is_done = true;
94+ self.trailer = Some(git_hash::ObjectId::from(digest));
95+ Ok(())
13496 }
13597 }
13698
137−impl<I, W, E> Iterator for FromEntriesIter<I, W>
99+impl<I, W> Iterator for FromEntriesIter<I, W>
138100 where
139− I: Iterator<Item = Result<Vec<output::Entry>, E>>,
101+ I: Iterator<Item = Result<input::Entry, input::Error>>,
140102 W: std::io::Write,
141− E: std::error::Error + 'static,
142103 {
143104 /// The amount of bytes written to `out` if `Ok` or the error `E` received from the input.
144− type Item = Result<u64, Error<E>>;
105+ type Item = Result<input::Entry, input::Error>;
145106
146107 fn next(&mut self) -> Option<Self::Item> {
147108 if self.is_done {
148109 return None;
149110 }
150− Some(match self.next_inner() {
151− Err(err) => {
111+
112+ match self.inner.next() {
113+ Some(Ok(entry)) => Some(self.next_inner(entry)),
114+ Some(Err(err)) => {
152115 self.is_done = true;
153− Err(err)
116+ Some(Err(err))
154117 }
155− Ok(written) => Ok(written),
156− })
118+ None => self.write_digest().err().map(Err),
119+ }
157120 }
158121 }
git-pack/src/data/input/mod.rs+2−1
@@ -37,4 +37,5 @@ pub use bytes_to_entries::BytesToEntriesIter; · 36 unmodified lines
3737 mod lookup_ref_delta_objects;
3838 pub use lookup_ref_delta_objects::LookupRefDeltaObjectsIter;
3939
40−mod bytes;
40+///
41+pub mod bytes;
git-pack/src/data/output/bytes.rs+1−1
@@ -84,7 +84,7 @@ where · 83 unmodified lines
8484
8585 /// Consume this instance and return the `output` implementation.
8686 ///
87− /// _Note_ that the input can be moved out of this instance beforehand.
87+ /// _Note_ that the `input` iterator can be moved out of this instance beforehand.
8888 pub fn into_write(self) -> W {
8989 self.output.inner
9090 }