write pack-ids and offsets (#279)

Sebastian Thiel committed 4 years ago

Commit bfc80691 parent 565a7ae3 files · +43 −5Browse files

Unified · Split

3 files

git-pack/src/index/write/encode.rs +3 −3
git-pack/src/multi_index/chunk.rs +30 −0
git-pack/src/multi_index/write.rs +10 −2

@@ -1,5 +1,8 @@
1	1	use std::{cmp::Ordering, collections::VecDeque, io};
2	2
	3	+pub(crate) const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff;
	4	+pub(crate) const HIGH_BIT: u32 = 0x8000_0000;
	5	+
3	6	use byteorder::{BigEndian, WriteBytesExt};
4	7	use git_features::{
5	8	hash,
@@ -34,9 +37,6 @@ pub(crate) fn write_to( · 28 unmodified lines
34	37	out.write_all(V2_SIGNATURE)?;
35	38	out.write_u32::<BigEndian>(kind as u32)?;
36	39
37		− const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff;
38		− const HIGH_BIT: u32 = 0x8000_0000;
39		−
40	40	let needs_64bit_offsets =
41	41	entries_sorted_by_oid.back().expect("at least one pack entry").offset > LARGE_OFFSET_THRESHOLD;
42	42	progress.init(Some(4), progress::steps());

git-pack/src/multi_index/chunk.rs+30−0

@@ -173,11 +173,41 @@ pub mod lookup { · 172 unmodified lines
173	173
174	174	/// Information about the offsets table.
175	175	pub mod offsets {
	176	+ use crate::multi_index;
	177	+ use byteorder::{BigEndian, WriteBytesExt};
176	178	use std::ops::Range;
177	179
178	180	/// The id uniquely identifying the offsets table.
179	181	pub const ID: git_chunk::Id = *b"OOFF";
180	182
	183	+ /// Return the amount of bytes needed to offset data for `entries`.
	184	+ pub fn storage_size(entries: usize) -> u64 {
	185	+ (entries * (4 /pack-id/ + 4/* pack offset */)) as u64
	186	+ }
	187	+
	188	+ /// Returns the amount of entries that need a u64 offset.
	189	+ pub(crate) fn write(
	190	+ sorted_entries: &[multi_index::write::Entry],
	191	+ mut out: impl std::io::Write,
	192	+ ) -> std::io::Result<u32> {
	193	+ use crate::index::write::encode::{HIGH_BIT, LARGE_OFFSET_THRESHOLD};
	194	+ let mut num_large_offsets = 0u32;
	195	+
	196	+ for entry in sorted_entries {
	197	+ out.write_u32::<BigEndian>(entry.pack_index)?;
	198	+
	199	+ let offset = if entry.pack_offset > LARGE_OFFSET_THRESHOLD {
	200	+ let res = num_large_offsets \| HIGH_BIT;
	201	+ num_large_offsets += 1;
	202	+ res
	203	+ } else {
	204	+ entry.pack_offset as u32
	205	+ };
	206	+ out.write_u32::<BigEndian>(offset)?;
	207	+ }
	208	+ Ok(num_large_offsets)
	209	+ }
	210	+
181	211	/// Returns true if the `offset` range seems to match the size required for `num_objects`.
182	212	pub fn is_valid(offset: &Range<usize>, num_objects: u32) -> bool {
183	213	let entry_size = 4 /* pack-id / + 4 / pack-offset */;

git-pack/src/multi_index/write.rs+10−2

@@ -26,8 +26,8 @@ pub use error::Error; · 25 unmodified lines
26	26	/// An entry suitable for sorting and writing
27	27	pub(crate) struct Entry {
28	28	pub(crate) id: git_hash::ObjectId,
29		− pack_index: u32,
30		− pack_offset: crate::data::Offset,
	29	+ pub(crate) pack_index: u32,
	30	+ pub(crate) pack_offset: crate::data::Offset,
31	31	/// Used for sorting in case of duplicates
32	32	index_mtime: SystemTime,
33	33	}
@@ -120,6 +120,10 @@ impl multi_index::File { · 86 unmodified lines
120	120	multi_index::chunk::lookup::ID,
121	121	multi_index::chunk::lookup::storage_size(entries.len(), object_hash),
122	122	);
	123	+ cf.plan_chunk(
	124	+ multi_index::chunk::offsets::ID,
	125	+ multi_index::chunk::offsets::storage_size(entries.len()),
	126	+ );
123	127
124	128	let bytes_written = Self::write_header(
125	129	&mut out,
@@ -128,6 +132,7 @@ impl multi_index::File { · 2 unmodified lines
128	132	object_hash,
129	133	)?;
130	134	let mut chunk_write = cf.into_write(&mut out, bytes_written)?;
	135	+ let mut num_large_offsets = None;
131	136	while let Some(chunk_to_write) = chunk_write.next_chunk() {
132	137	match chunk_to_write {
133	138	multi_index::chunk::index_names::ID => {
@@ -135,6 +140,9 @@ impl multi_index::File { · 1 unmodified line
135	140	}
136	141	multi_index::chunk::fanout::ID => multi_index::chunk::fanout::write(&entries, &mut chunk_write)?,
137	142	multi_index::chunk::lookup::ID => multi_index::chunk::lookup::write(&entries, &mut chunk_write)?,
	143	+ multi_index::chunk::offsets::ID => {
	144	+ num_large_offsets = multi_index::chunk::offsets::write(&entries, &mut chunk_write)?.into();
	145	+ }
138	146	unknown => unreachable!("BUG: forgot to implement chunk {:?}", std::str::from_utf8(&unknown)),
139	147	}
140	148	}