History

[clone] a piped iterator

Sebastian Thiel committed 6 years ago

Commit 5148c851 parent 9cfa9b74 files · +138 −100Browse files

Unified · Split

4 files
  1. git-features/src/pipe.rs +69 −47
  2. git-features/tests/pipe.rs +68 −53
  3. git-transport/tests/client/http.rs +0 −0
  4. tasks.md +1 −0
git-features/src/pipe.rs+69−47
@@ -1,60 +1,82 @@
1use bytes::{Buf, BufMut, BytesMut};1mod iter {
2use std::io;2 pub struct Iter<T> {
3 channel: std::sync::mpsc::Receiver<T>,
4 }
5 pub fn iter<T>(in_flight_writes: impl Into<Option<usize>>) -> (std::sync::mpsc::SyncSender<T>, Iter<T>) {
6 let (tx, rx) = std::sync::mpsc::sync_channel(in_flight_writes.into().unwrap_or(0));
7 (tx, Iter { channel: rx })
8 }
3 9
4pub struct Writer {10 impl<T> Iterator for Iter<T> {
5 channel: std::sync::mpsc::SyncSender<BytesMut>,11 type Item = T;
6 buf: BytesMut,
7}
8 12
9pub struct Reader {13 fn next(&mut self) -> Option<Self::Item> {
10 channel: std::sync::mpsc::Receiver<BytesMut>,14 self.channel.recv().ok()
11 buf: BytesMut,15 }
16 }
12}17}
18pub use iter::{iter, Iter};
19
20mod io {
21 use bytes::{Buf, BufMut, BytesMut};
22 use std::io;
23
24 pub struct Writer {
25 channel: std::sync::mpsc::SyncSender<BytesMut>,
26 buf: BytesMut,
27 }
13 28
14impl io::Read for Reader {29 pub struct Reader {
15 fn read(&mut self, mut out: &mut [u8]) -> io::Result<usize> {30 channel: std::sync::mpsc::Receiver<BytesMut>,
16 let mut written = 0;31 buf: BytesMut,
17 while !out.is_empty() {32 }
18 if self.buf.is_empty() {33
19 match self.channel.recv() {34 impl io::Read for Reader {
20 Ok(buf) => self.buf = buf,35 fn read(&mut self, mut out: &mut [u8]) -> io::Result<usize> {
21 Err(_) => break,36 let mut written = 0;
37 while !out.is_empty() {
38 if self.buf.is_empty() {
39 match self.channel.recv() {
40 Ok(buf) => self.buf = buf,
41 Err(_) => break,
42 }
22 }43 }
44 let bytes_to_write = self.buf.len().min(out.len());
45 let (to_write, rest) = out.split_at_mut(bytes_to_write);
46 self.buf.split_to(bytes_to_write).copy_to_slice(to_write);
47 out = rest;
48 written += bytes_to_write;
23 }49 }
24 let bytes_to_write = self.buf.len().min(out.len());50 Ok(written)
25 let (to_write, rest) = out.split_at_mut(bytes_to_write);
26 self.buf.split_to(bytes_to_write).copy_to_slice(to_write);
27 out = rest;
28 written += bytes_to_write;
29 }51 }
30 Ok(written)
31 }52 }
32}
33 53
34impl io::Write for Writer {54 impl io::Write for Writer {
35 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {55 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
36 self.buf.put_slice(buf);56 self.buf.put_slice(buf);
37 self.channel57 self.channel
38 .send(self.buf.split())58 .send(self.buf.split())
39 .map_err(|err| io::Error::new(io::ErrorKind::BrokenPipe, err))?;59 .map_err(|err| io::Error::new(io::ErrorKind::BrokenPipe, err))?;
40 Ok(buf.len())60 Ok(buf.len())
41 }61 }
42 62
43 fn flush(&mut self) -> io::Result<()> {63 fn flush(&mut self) -> io::Result<()> {
44 Ok(())64 Ok(())
65 }
45 }66 }
46}
47 67
48pub fn unidirectional(in_flight_writes: impl Into<Option<usize>>) -> (Writer, Reader) {68 pub fn unidirectional(in_flight_writes: impl Into<Option<usize>>) -> (Writer, Reader) {
49 let (tx, rx) = std::sync::mpsc::sync_channel(in_flight_writes.into().unwrap_or(0));69 let (tx, rx) = std::sync::mpsc::sync_channel(in_flight_writes.into().unwrap_or(0));
50 (70 (
51 Writer {71 Writer {
52 channel: tx,72 channel: tx,
53 buf: BytesMut::with_capacity(4096),73 buf: BytesMut::with_capacity(4096),
54 },74 },
55 Reader {75 Reader {
56 channel: rx,76 channel: rx,
57 buf: BytesMut::new(),77 buf: BytesMut::new(),
58 },78 },
59 )79 )
80 }
60}81}
82pub use io::{unidirectional, Reader, Writer};
git-features/tests/pipe.rs+68−53
@@ -1,64 +1,79 @@
1use git_features::pipe;1mod iter {
2use std::io::{ErrorKind, Read, Write};2 use git_features::pipe;
3 3
4#[test]4 #[test]
5fn threaded_read_to_end() {5 fn iterator_ends_when_sender_is_dropped() {
6 let (mut writer, mut reader) = git_features::pipe::unidirectional(0);6 let (send, mut iter) = pipe::iter(1);
7 send.send(1).expect("buffered send successful");
8 assert_eq!(iter.next(), Some(1), "it receives the item");
9 drop(send);
10 assert_eq!(iter.next(), None, "sender is dropped, so no more items for us");
11 }
12}
7 13
8 let message = "Hello, world!";14mod io {
9 std::thread::spawn(move || {15 use git_features::pipe;
10 writer16 use std::io::{ErrorKind, Read, Write};
11 .write_all(message.as_bytes())
12 .expect("writes to work if reader is present")
13 });
14 17
15 let mut received = String::new();18 #[test]
16 reader.read_to_string(&mut received).unwrap();19 fn threaded_read_to_end() {
20 let (mut writer, mut reader) = git_features::pipe::unidirectional(0);
17 21
18 assert_eq!(&received, message);22 let message = "Hello, world!";
19}23 std::thread::spawn(move || {
24 writer
25 .write_all(message.as_bytes())
26 .expect("writes to work if reader is present")
27 });
20 28
21#[test]29 let mut received = String::new();
22fn write_failure_propagates() {30 reader.read_to_string(&mut received).unwrap();
23 let (mut writer, _) = pipe::unidirectional(None);
24 assert_eq!(
25 writer.write_all(b"must fail").unwrap_err().kind(),
26 ErrorKind::BrokenPipe
27 );
28}
29 31
30#[test]32 assert_eq!(&received, message);
31fn continue_on_empty_writes() {33 }
32 let (mut writer, mut reader) = pipe::unidirectional(2);
33 writer.write(&[]).expect("write successful and non-blocking");
34 let input = b"hello";
35 writer
36 .write(input)
37 .expect("second write works as well as there is capacity");
38 let mut buf = vec![0u8; input.len()];
39 assert_eq!(reader.read(&mut buf).expect("read succeeds"), input.len());
40 assert_eq!(buf, &input[..]);
41}
42 34
43#[test]35 #[test]
44fn small_reads() {36 fn write_failure_propagates() {
45 const BLOCK_SIZE: usize = 20;37 let (mut writer, _) = pipe::unidirectional(None);
46 let block_count = 20;38 assert_eq!(
47 let (mut writer, mut reader) = pipe::unidirectional(Some(4));39 writer.write_all(b"must fail").unwrap_err().kind(),
48 std::thread::spawn(move || {40 ErrorKind::BrokenPipe
49 for _ in 0..block_count {41 );
50 let data = &[0; BLOCK_SIZE];42 }
51 writer.write_all(data).unwrap();43
52 }44 #[test]
53 });45 fn continue_on_empty_writes() {
46 let (mut writer, mut reader) = pipe::unidirectional(2);
47 writer.write(&[]).expect("write successful and non-blocking");
48 let input = b"hello";
49 writer
50 .write(input)
51 .expect("second write works as well as there is capacity");
52 let mut buf = vec![0u8; input.len()];
53 assert_eq!(reader.read(&mut buf).expect("read succeeds"), input.len());
54 assert_eq!(buf, &input[..]);
55 }
56
57 #[test]
58 fn small_reads() {
59 const BLOCK_SIZE: usize = 20;
60 let block_count = 20;
61 let (mut writer, mut reader) = pipe::unidirectional(Some(4));
62 std::thread::spawn(move || {
63 for _ in 0..block_count {
64 let data = &[0; BLOCK_SIZE];
65 writer.write_all(data).unwrap();
66 }
67 });
54 68
55 let mut small_read_buf = [0; BLOCK_SIZE / 2];69 let mut small_read_buf = [0; BLOCK_SIZE / 2];
56 let mut bytes_read = 0;70 let mut bytes_read = 0;
57 while let Ok(size) = reader.read(&mut small_read_buf) {71 while let Ok(size) = reader.read(&mut small_read_buf) {
58 if size == 0 {72 if size == 0 {
59 break;73 break;
74 }
75 bytes_read += size;
60 }76 }
61 bytes_read += size;77 assert_eq!(block_count * BLOCK_SIZE, bytes_read);
62 }78 }
63 assert_eq!(block_count * BLOCK_SIZE, bytes_read);
64}79}
git-transport/tests/client/http.rs+0−0

Empty file added.

tasks.md+1−0
@@ -24,6 +24,7 @@ · 23 unmodified lines
24 * [ ] http://git-upload-pack24 * [ ] http://git-upload-pack
25 * **prerequisites**25 * **prerequisites**
26 * [x] pipe to link writers with readers using bytes26 * [x] pipe to link writers with readers using bytes
27 * [x] piped iterator
27 * [ ] HTTP trait for simple gets and post implemented for Curl28 * [ ] HTTP trait for simple gets and post implemented for Curl
28 * [ ] V1 handshake29 * [ ] V1 handshake
29* **git-refs**30* **git-refs**