Somehow handle chunk size in absence of known chunk amount

Sebastian Thiel committed 6 years ago

Commit acfccad1 parent 973e6bb2 files · +75 −20Browse files

Unified · Split

2 files

git-features/src/parallel.rs +25 −20
git-features/tests/parallel.rs +50 −0

@@ -36,7 +36,7 @@ mod serial { · 35 unmodified lines
36	36
37	37	#[cfg(feature = "parallel")]
38	38	mod in_parallel {
39		− use crate::parallel::Reducer;
	39	+ use crate::parallel::{num_threads, Reducer};
40	40	use crossbeam_utils::thread;
41	41
42	42	pub fn join<O1: Send, O2: Send>(left: impl FnOnce() -> O1 + Send, right: impl FnOnce() -> O2 + Send) -> (O1, O2) {
@@ -48,13 +48,6 @@ mod in_parallel { · 5 unmodified lines
48	48	.unwrap()
49	49	}
50	50
51		− fn num_threads(thread_limit: Option<usize>) -> usize {
52		− let logical_cores = num_cpus::get();
53		− thread_limit
54		− .map(\|l\| if l == 0 { logical_cores } else { l })
55		− .unwrap_or(logical_cores)
56		− }
57		−
58	51	pub fn in_parallel<I, S, O, R>(
59	52	input: impl Iterator<Item = I> + Send,
60	53	thread_limit: Option<usize>,
@@ -107,16 +100,6 @@ mod in_parallel { · 46 unmodified lines
107	100	}
108	101	}
109	102
110		−pub fn num_threads(thread_limit: Option<usize>) -> usize {
111		− #[cfg(not(feature = "num_cpus"))]
112		− return 1;
113		−
114		− let logical_cores = num_cpus::get();
115		− thread_limit
116		− .map(\|l\| if l == 0 { logical_cores } else { l })
117		− .unwrap_or(logical_cores)
118		−}
119		−
120	103	pub fn optimize_chunk_size_and_thread_limit(
121	104	desired_chunk_size: usize,
122	105	num_chunks: Option<usize>,
@@ -144,7 +127,19 @@ pub fn optimize_chunk_size_and_thread_limit( · 21 unmodified lines
144	127	};
145	128	(chunk_size, thread_limit)
146	129	})
147		− .unwrap_or((desired_chunk_size, available_threads));
	130	+ .unwrap_or((
	131	+ if available_threads == 1 {
	132	+ desired_chunk_size
	133	+ } else {
	134	+ let arbitrary_desirable_chunk_size = 5;
	135	+ if desired_chunk_size < arbitrary_desirable_chunk_size {
	136	+ arbitrary_desirable_chunk_size
	137	+ } else {
	138	+ desired_chunk_size
	139	+ }
	140	+ },
	141	+ available_threads,
	142	+ ));
148	143	(chunk_size, Some(thread_limit))
149	144	}
150	145
@@ -154,6 +149,16 @@ pub use serial::*; · 3 unmodified lines
154	149	#[cfg(feature = "parallel")]
155	150	pub use in_parallel::*;
156	151
	152	+pub(crate) fn num_threads(thread_limit: Option<usize>) -> usize {
	153	+ #[cfg(not(feature = "num_cpus"))]
	154	+ return 1;
	155	+
	156	+ let logical_cores = num_cpus::get();
	157	+ thread_limit
	158	+ .map(\|l\| if l == 0 { logical_cores } else { l })
	159	+ .unwrap_or(logical_cores)
	160	+}
	161	+
157	162	pub fn in_parallel_if<I, S, O, R>(
158	163	condition: impl FnOnce() -> bool,
159	164	input: impl Iterator<Item = I> + Send,
@@ -167,7 +172,7 @@ where · 7 unmodified lines
167	172	I: Send,
168	173	O: Send,
169	174	{
170		− if condition() {
	175	+ if num_threads(thread_limit) > 1 && condition() {
171	176	in_parallel(input, thread_limit, new_thread_state, consume, reducer)
172	177	} else {
173	178	serial::in_parallel(input, thread_limit, new_thread_state, consume, reducer)

git-features/tests/parallel.rs+50−0

@@ -32,6 +32,11 @@ mod optimize_chunk_size_and_thread_limit { · 31 unmodified lines
32	32	optimize_chunk_size_and_thread_limit(1, Some(100), None, Some(10)),
33	33	(5, Some(10))
34	34	);
	35	+ assert_eq!(
	36	+ optimize_chunk_size_and_thread_limit(1, Some(100), Some(5), Some(10)),
	37	+ (10, Some(5)),
	38	+ "the thread limit is always respected"
	39	+ );
35	40	}
36	41
37	42	#[test]
@@ -40,5 +45,50 @@ mod optimize_chunk_size_and_thread_limit { · 2 unmodified lines
40	45	optimize_chunk_size_and_thread_limit(50, Some(2), None, Some(10)),
41	46	(5, Some(10))
42	47	);
	48	+ assert_eq!(
	49	+ optimize_chunk_size_and_thread_limit(50, Some(2), Some(5), Some(10)),
	50	+ (10, Some(5)),
	51	+ "the thread limit is always respected"
	52	+ );
	53	+ }
	54	+
	55	+ mod unknown_chunk_count {
	56	+ use git_features::parallel::optimize_chunk_size_and_thread_limit;
	57	+
	58	+ #[test]
	59	+ fn medium_chunk_size_many_threads() {
	60	+ assert_eq!(
	61	+ optimize_chunk_size_and_thread_limit(50, None, None, Some(4)),
	62	+ (50, Some(4)),
	63	+ "really, what do we know"
	64	+ );
	65	+ }
	66	+
	67	+ #[test]
	68	+ fn medium_chunk_size_single_thread() {
	69	+ assert_eq!(
	70	+ optimize_chunk_size_and_thread_limit(50, None, None, Some(1)),
	71	+ (50, Some(1)),
	72	+ "single threaded - we don't touch that"
	73	+ );
	74	+ }
	75	+
	76	+ #[test]
	77	+ fn small_chunk_size_single_thread() {
	78	+ assert_eq!(
	79	+ optimize_chunk_size_and_thread_limit(1, None, None, Some(1)),
	80	+ (1, Some(1)),
	81	+ "single threaded - we don't touch that"
	82	+ );
	83	+ }
	84	+
	85	+ #[test]
	86	+ fn small_chunk_size_many_threads() {
	87	+ assert_eq!(
	88	+ optimize_chunk_size_and_thread_limit(1, None, None, Some(4)),
	89	+ (5, Some(4)),
	90	+ "we prefer an arbitrary number, which should really be based on effort, but the caller has to adjust for that"
	91	+ );
	92	+ }
43	93	}
44	94	}