History

write pack-ids and offsets (#279)

Sebastian Thiel committed 4 years ago

Commit bfc80691 parent 565a7ae3 files · +43 −5Browse files

Unified · Split

3 files
  1. git-pack/src/index/write/encode.rs +3 −3
  2. git-pack/src/multi_index/chunk.rs +30 −0
  3. git-pack/src/multi_index/write.rs +10 −2
git-pack/src/index/write/encode.rs+3−3
@@ -1,5 +1,8 @@
1use std::{cmp::Ordering, collections::VecDeque, io};1use std::{cmp::Ordering, collections::VecDeque, io};
2 2
3pub(crate) const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff;
4pub(crate) const HIGH_BIT: u32 = 0x8000_0000;
5
3use byteorder::{BigEndian, WriteBytesExt};6use byteorder::{BigEndian, WriteBytesExt};
4use git_features::{7use git_features::{
5 hash,8 hash,
@@ -34,9 +37,6 @@ pub(crate) fn write_to( · 28 unmodified lines
34 out.write_all(V2_SIGNATURE)?;37 out.write_all(V2_SIGNATURE)?;
35 out.write_u32::<BigEndian>(kind as u32)?;38 out.write_u32::<BigEndian>(kind as u32)?;
36 39
37 const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff;
38 const HIGH_BIT: u32 = 0x8000_0000;
39
40 let needs_64bit_offsets =40 let needs_64bit_offsets =
41 entries_sorted_by_oid.back().expect("at least one pack entry").offset > LARGE_OFFSET_THRESHOLD;41 entries_sorted_by_oid.back().expect("at least one pack entry").offset > LARGE_OFFSET_THRESHOLD;
42 progress.init(Some(4), progress::steps());42 progress.init(Some(4), progress::steps());
git-pack/src/multi_index/chunk.rs+30−0
@@ -173,11 +173,41 @@ pub mod lookup { · 172 unmodified lines
173 173
174/// Information about the offsets table.174/// Information about the offsets table.
175pub mod offsets {175pub mod offsets {
176 use crate::multi_index;
177 use byteorder::{BigEndian, WriteBytesExt};
176 use std::ops::Range;178 use std::ops::Range;
177 179
178 /// The id uniquely identifying the offsets table.180 /// The id uniquely identifying the offsets table.
179 pub const ID: git_chunk::Id = *b"OOFF";181 pub const ID: git_chunk::Id = *b"OOFF";
180 182
183 /// Return the amount of bytes needed to offset data for `entries`.
184 pub fn storage_size(entries: usize) -> u64 {
185 (entries * (4 /*pack-id*/ + 4/* pack offset */)) as u64
186 }
187
188 /// Returns the amount of entries that need a u64 offset.
189 pub(crate) fn write(
190 sorted_entries: &[multi_index::write::Entry],
191 mut out: impl std::io::Write,
192 ) -> std::io::Result<u32> {
193 use crate::index::write::encode::{HIGH_BIT, LARGE_OFFSET_THRESHOLD};
194 let mut num_large_offsets = 0u32;
195
196 for entry in sorted_entries {
197 out.write_u32::<BigEndian>(entry.pack_index)?;
198
199 let offset = if entry.pack_offset > LARGE_OFFSET_THRESHOLD {
200 let res = num_large_offsets | HIGH_BIT;
201 num_large_offsets += 1;
202 res
203 } else {
204 entry.pack_offset as u32
205 };
206 out.write_u32::<BigEndian>(offset)?;
207 }
208 Ok(num_large_offsets)
209 }
210
181 /// Returns true if the `offset` range seems to match the size required for `num_objects`.211 /// Returns true if the `offset` range seems to match the size required for `num_objects`.
182 pub fn is_valid(offset: &Range<usize>, num_objects: u32) -> bool {212 pub fn is_valid(offset: &Range<usize>, num_objects: u32) -> bool {
183 let entry_size = 4 /* pack-id */ + 4 /* pack-offset */;213 let entry_size = 4 /* pack-id */ + 4 /* pack-offset */;
git-pack/src/multi_index/write.rs+10−2
@@ -26,8 +26,8 @@ pub use error::Error; · 25 unmodified lines
26/// An entry suitable for sorting and writing26/// An entry suitable for sorting and writing
27pub(crate) struct Entry {27pub(crate) struct Entry {
28 pub(crate) id: git_hash::ObjectId,28 pub(crate) id: git_hash::ObjectId,
29 pack_index: u32,29 pub(crate) pack_index: u32,
30 pack_offset: crate::data::Offset,30 pub(crate) pack_offset: crate::data::Offset,
31 /// Used for sorting in case of duplicates31 /// Used for sorting in case of duplicates
32 index_mtime: SystemTime,32 index_mtime: SystemTime,
33}33}
@@ -120,6 +120,10 @@ impl multi_index::File { · 86 unmodified lines
120 multi_index::chunk::lookup::ID,120 multi_index::chunk::lookup::ID,
121 multi_index::chunk::lookup::storage_size(entries.len(), object_hash),121 multi_index::chunk::lookup::storage_size(entries.len(), object_hash),
122 );122 );
123 cf.plan_chunk(
124 multi_index::chunk::offsets::ID,
125 multi_index::chunk::offsets::storage_size(entries.len()),
126 );
123 127
124 let bytes_written = Self::write_header(128 let bytes_written = Self::write_header(
125 &mut out,129 &mut out,
@@ -128,6 +132,7 @@ impl multi_index::File { · 2 unmodified lines
128 object_hash,132 object_hash,
129 )?;133 )?;
130 let mut chunk_write = cf.into_write(&mut out, bytes_written)?;134 let mut chunk_write = cf.into_write(&mut out, bytes_written)?;
135 let mut num_large_offsets = None;
131 while let Some(chunk_to_write) = chunk_write.next_chunk() {136 while let Some(chunk_to_write) = chunk_write.next_chunk() {
132 match chunk_to_write {137 match chunk_to_write {
133 multi_index::chunk::index_names::ID => {138 multi_index::chunk::index_names::ID => {
@@ -135,6 +140,9 @@ impl multi_index::File { · 1 unmodified line
135 }140 }
136 multi_index::chunk::fanout::ID => multi_index::chunk::fanout::write(&entries, &mut chunk_write)?,141 multi_index::chunk::fanout::ID => multi_index::chunk::fanout::write(&entries, &mut chunk_write)?,
137 multi_index::chunk::lookup::ID => multi_index::chunk::lookup::write(&entries, &mut chunk_write)?,142 multi_index::chunk::lookup::ID => multi_index::chunk::lookup::write(&entries, &mut chunk_write)?,
143 multi_index::chunk::offsets::ID => {
144 num_large_offsets = multi_index::chunk::offsets::write(&entries, &mut chunk_write)?.into();
145 }
138 unknown => unreachable!("BUG: forgot to implement chunk {:?}", std::str::from_utf8(&unknown)),146 unknown => unreachable!("BUG: forgot to implement chunk {:?}", std::str::from_utf8(&unknown)),
139 }147 }
140 }148 }