Somehow handle chunk size in absence of known chunk amount

Sebastian Thiel committed 6 years ago

Commit acfccad1 parent 973e6bb2 files · +75 −20Browse files

Unified · Split

2 files

git-features/src/parallel.rs +25 −20
git-features/tests/parallel.rs +50 −0

 #[cfg(feature = "parallel")]
 mod in_parallel {
-    use crate::parallel::Reducer;
+    use crate::parallel::{num_threads, Reducer};
     use crossbeam_utils::thread;
     pub fn join<O1: Send, O2: Send>(left: impl FnOnce() -> O1 + Send, right: impl FnOnce() -> O2 + Send) -> (O1, O2) {
         .unwrap()
     }
-    fn num_threads(thread_limit: Option<usize>) -> usize {
-        let logical_cores = num_cpus::get();
-        thread_limit
-            .map(|l| if l == 0 { logical_cores } else { l })
-            .unwrap_or(logical_cores)
-    }
     pub fn in_parallel<I, S, O, R>(
         input: impl Iterator<Item = I> + Send,
         thread_limit: Option<usize>,
     }
 }
-pub fn num_threads(thread_limit: Option<usize>) -> usize {
-    #[cfg(not(feature = "num_cpus"))]
-    return 1;
-    let logical_cores = num_cpus::get();
-    thread_limit
-        .map(|l| if l == 0 { logical_cores } else { l })
-        .unwrap_or(logical_cores)
-}
 pub fn optimize_chunk_size_and_thread_limit(
     desired_chunk_size: usize,
     num_chunks: Option<usize>,
             };
             (chunk_size, thread_limit)
         })
-        .unwrap_or((desired_chunk_size, available_threads));
+        .unwrap_or((
+            if available_threads == 1 {
+                desired_chunk_size
+            } else {
+                let arbitrary_desirable_chunk_size = 5;
+                if desired_chunk_size < arbitrary_desirable_chunk_size {
+                    arbitrary_desirable_chunk_size
+                } else {
+                    desired_chunk_size
+                }
+            },
+            available_threads,
+        ));
     (chunk_size, Some(thread_limit))
 }
 #[cfg(feature = "parallel")]
 pub use in_parallel::*;
+pub(crate) fn num_threads(thread_limit: Option<usize>) -> usize {
+    #[cfg(not(feature = "num_cpus"))]
+    return 1;
+    let logical_cores = num_cpus::get();
+    thread_limit
+        .map(|l| if l == 0 { logical_cores } else { l })
+        .unwrap_or(logical_cores)
+}
 pub fn in_parallel_if<I, S, O, R>(
     condition: impl FnOnce() -> bool,
     input: impl Iterator<Item = I> + Send,
     I: Send,
     O: Send,
 {
-    if condition() {
+    if num_threads(thread_limit) > 1 && condition() {
         in_parallel(input, thread_limit, new_thread_state, consume, reducer)
     } else {
         serial::in_parallel(input, thread_limit, new_thread_state, consume, reducer)

git-features/tests/parallel.rs+50−0

@@ -32,6 +32,11 @@ mod optimize_chunk_size_and_thread_limit { · 31 unmodified lines
32	optimize_chunk_size_and_thread_limit(1, Some(100), None, Some(10)),	32	optimize_chunk_size_and_thread_limit(1, Some(100), None, Some(10)),
33	(5, Some(10))	33	(5, Some(10))
34	);	34	);
		35	assert_eq!(
		36	optimize_chunk_size_and_thread_limit(1, Some(100), Some(5), Some(10)),
		37	(10, Some(5)),
		38	"the thread limit is always respected"
		39	);
35	}	40	}
36		41
37	#[test]	42	#[test]
@@ -40,5 +45,50 @@ mod optimize_chunk_size_and_thread_limit { · 2 unmodified lines
40	optimize_chunk_size_and_thread_limit(50, Some(2), None, Some(10)),	45	optimize_chunk_size_and_thread_limit(50, Some(2), None, Some(10)),
41	(5, Some(10))	46	(5, Some(10))
42	);	47	);
		48	assert_eq!(
		49	optimize_chunk_size_and_thread_limit(50, Some(2), Some(5), Some(10)),
		50	(10, Some(5)),
		51	"the thread limit is always respected"
		52	);
		53	}
		54
		55	mod unknown_chunk_count {
		56	use git_features::parallel::optimize_chunk_size_and_thread_limit;
		57
		58	#[test]
		59	fn medium_chunk_size_many_threads() {
		60	assert_eq!(
		61	optimize_chunk_size_and_thread_limit(50, None, None, Some(4)),
		62	(50, Some(4)),
		63	"really, what do we know"
		64	);
		65	}
		66
		67	#[test]
		68	fn medium_chunk_size_single_thread() {
		69	assert_eq!(
		70	optimize_chunk_size_and_thread_limit(50, None, None, Some(1)),
		71	(50, Some(1)),
		72	"single threaded - we don't touch that"
		73	);
		74	}
		75
		76	#[test]
		77	fn small_chunk_size_single_thread() {
		78	assert_eq!(
		79	optimize_chunk_size_and_thread_limit(1, None, None, Some(1)),
		80	(1, Some(1)),
		81	"single threaded - we don't touch that"
		82	);
		83	}
		84
		85	#[test]
		86	fn small_chunk_size_many_threads() {
		87	assert_eq!(
		88	optimize_chunk_size_and_thread_limit(1, None, None, Some(4)),
		89	(5, Some(4)),
		90	"we prefer an arbitrary number, which should really be based on effort, but the caller has to adjust for that"
		91	);
		92	}
43	}	93	}
44	}	94	}