// SPDX-License-Identifier: GPL-2.0-or-later/* * Copyright 2002 H. Peter Anvin - All Rights Reserved * * Algorithm list and algorithm selection for RAID-6 */ #include <linux/module.h>#include <linux/gfp.h>#include <linux/raid/pq.h>#include <linux/slab.h>#include <linux/static_call.h>#include <kunit/visibility.h>#include "algos.h" #define RAID6_MAX_ALGOS		16static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS];static unsigned int raid6_nr_algos;static const struct raid6_recov_calls *raid6_recov_algo; /* Selected algorithm */DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome);DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome);DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2);DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap); /** * raid6_gen_syndrome - generate RAID6 P/Q parity * @disks:	number of "disks" to operate on including parity * @bytes:	length in bytes of each vector * @ptrs:	@disks size array of memory pointers * * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to * @ptrs[@disks - 3]. * * @disks must be at least 4, and the memory pointed to by each member of @ptrs * must be at least 64-byte aligned.  @bytes must be non-zero and a multiple of * 512. * * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying * algorithm. */void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs){	WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());	WARN_ON_ONCE(bytes & 511);	WARN_ON_ONCE(disks < RAID6_MIN_DISKS); 	static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs);}EXPORT_SYMBOL_GPL(raid6_gen_syndrome); /** * raid6_xor_syndrome - update RAID6 P/Q parity * @disks:	number of "disks" to operate on including parity * @start:	first index into @disk to update * @stop:	last index into @disk to update * @bytes:	length in bytes of each vector * @ptrs:	@disks size array of memory pointers * * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and * @ptrs[@disks - 1] respectively for the memory pointed to by * @ptrs[@start..@stop]. * * This is used to update parity in place using the following sequence: * * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data. * 2) update the the data in @ptrs[@start..@stop]. * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data. * * Data between @start and @stop that is not changed should be filled * with a pointer to the kernel zero page. * * @disks must be at least 4, and the memory pointed to by each member of @ptrs * must be at least 64-byte aligned.  @bytes must be non-zero and a multiple of * 512.  @stop must be larger or equal to @start. */void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes,		void **ptrs){	WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());	WARN_ON_ONCE(bytes & 511);	WARN_ON_ONCE(disks < RAID6_MIN_DISKS);	WARN_ON_ONCE(stop < start); 	static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs);}EXPORT_SYMBOL_GPL(raid6_xor_syndrome); /* * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used * * Returns %true if raid6_can_xor_syndrome() can be used, else %false. */bool raid6_can_xor_syndrome(void){	return !!static_call_query(raid6_xor_syndrome_impl);}EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome); /** * raid6_recov_2data - recover two missing data disks * @disks:	number of "disks" to operate on including parity * @bytes:	length in bytes of each vector * @faila:	first failed data disk index * @failb:	second failed data disk index * @ptrs:	@disks size array of memory pointers * * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the * data in the remaining disks and the two parities pointed to by the other * indices between 0 and @disks - 1 in @ptrs.  @disks includes the data disks * and the two parities.  @faila must be smaller than @failb. * * Memory pointed to by each pointer in @ptrs must be page aligned and is * limited to %PAGE_SIZE. */void raid6_recov_2data(int disks, size_t bytes, int faila, int failb,		void **ptrs){	WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());	WARN_ON_ONCE(bytes & 511);	WARN_ON_ONCE(bytes > PAGE_SIZE);	WARN_ON_ONCE(failb <= faila); 	static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs);}EXPORT_SYMBOL_GPL(raid6_recov_2data); /** * raid6_recov_datap - recover a missing data disk and missing P-parity * @disks:	number of "disks" to operate on including parity * @bytes:	length in bytes of each vector * @faila:	failed data disk index * @ptrs:	@disks size array of memory pointers * * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity * pointed to by the other indices between 0 and @disks - 1 in @ptrs.  @disks * includes the data disks and the two parities. * * Memory pointed to by each pointer in @ptrs must be page aligned and is * limited to %PAGE_SIZE. */void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs){	WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());	WARN_ON_ONCE(bytes & 511);	WARN_ON_ONCE(bytes > PAGE_SIZE); 	static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs);}EXPORT_SYMBOL_GPL(raid6_recov_datap); #define BENCH_SIZE	SZ_4K#define NR_SRCS		8#define NR_DISKS	(NR_SRCS + 2)#define REPS		800U static int raid6_choose_gen(void *dptrs[NR_DISKS], const int disks){	const struct raid6_calls *best = NULL;	unsigned long bestgenperf = 0;	unsigned int i; 	for (i = 0; i < raid6_nr_algos; i++) {		const struct raid6_calls *algo = raid6_algos[i];		unsigned long perf = 0;		u64 t;		int i; 		preempt_disable();		t = ktime_get_ns();		for (i = 0; i < REPS; i++)			algo->gen_syndrome(disks, BENCH_SIZE, dptrs);		t = max(ktime_get_ns() - t, 1);		preempt_enable(); 		/* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */		perf = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t);		if (perf > bestgenperf) {			bestgenperf = perf;			best = algo;		}		pr_info("raid6: %-8s gen() %5lu MB/s\n", algo->name, perf);	} 	if (!best) {		pr_err("raid6: Yikes! No algorithm found!\n");		return -EINVAL;	} 	static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome);	static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); 	pr_info("raid6: using algorithm %s gen() %ld MB/s\n",		best->name, bestgenperf); 	if (best->xor_syndrome) {		/* work on the second half of the disks */		int start = (disks / 2) - 1, stop = disks - 3;		u64 t; 		preempt_disable();		t = ktime_get_ns();		for (i = 0; i < REPS; i++)			best->xor_syndrome(disks, start, stop, BENCH_SIZE,				dptrs);		t = max(ktime_get_ns() - t, 1);		preempt_enable(); 		pr_info("raid6: .... xor() %llu MB/s, rmw enabled\n",			div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS / 2 * 1000,				t));	} 	return 0;}  /* Try to pick the best algorithm *//* This code uses the gfmul table as convenient data set to abuse */ static int __init raid6_select_algo(void){	const int disks = NR_DISKS;	void *dptrs[NR_DISKS];	char *disk_ptr, *p;	int i, cycle;	int error; 	if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) {		pr_info("raid6: skipped pq benchmark and selected %s\n",			raid6_algos[raid6_nr_algos - 1]->name);		return 0;	} 	/* prepare the buffer and fill it circularly with gfmul table */	disk_ptr = kmalloc_array(NR_DISKS, BENCH_SIZE, GFP_KERNEL);	if (!disk_ptr) {		pr_err("raid6: Yikes!  No memory available.\n");		return -ENOMEM;	} 	p = disk_ptr;	for (i = 0; i < disks; i++)		dptrs[i] = p + BENCH_SIZE * i; 	cycle = ((disks - 2) * BENCH_SIZE) / 65536;	for (i = 0; i < cycle; i++) {		memcpy(p, raid6_gfmul, 65536);		p += 65536;	} 	if ((disks - 2) * BENCH_SIZE % 65536)		memcpy(p, raid6_gfmul, (disks - 2) * BENCH_SIZE % 65536); 	/* select raid gen_syndrome function */	error = raid6_choose_gen(dptrs, disks); 	kfree(disk_ptr); 	return error;} /* * Register a RAID6 P/Q generation algorithm.  The most optimized/unrolled * implementation should be registered last so it will be selected when the * boot-time benchmark is disabled. */void __init raid6_algo_add(const struct raid6_calls *algo){	if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS))		return;	raid6_algos[raid6_nr_algos++] = algo;} void __init raid6_algo_add_default(void){	raid6_algo_add(&raid6_intx1);	raid6_algo_add(&raid6_intx2);	raid6_algo_add(&raid6_intx4);	raid6_algo_add(&raid6_intx8);} void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo){	if (WARN_ON_ONCE(raid6_recov_algo))		return;	raid6_recov_algo = algo;} #ifdef CONFIG_RAID6_PQ_ARCH#include "pq_arch.h"#elsestatic inline void arch_raid6_init(void){	raid6_algo_add_default();}#endif /* CONFIG_RAID6_PQ_ARCH */ static int __init raid6_init(void){	/*	 * Architectures providing arch_raid6_init must add all PQ generation	 * algorithms they want to consider in arch_raid6_init(), including	 * the generic ones using raid6_algo_add_default() if wanted.	 */	arch_raid6_init(); 	/*	 * Architectures don't have to set a recovery algorithm, we'll just pick	 * the generic integer one if none was set.	 */	if (!raid6_recov_algo)		raid6_recov_algo = &raid6_recov_intx1;	static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2);	static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap);	pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); 	/*	 * Pick the last registered implementation as the temporary default until	 * calibration happens.	 */	static_call_update(raid6_gen_syndrome_impl,			raid6_algos[raid6_nr_algos - 1]->gen_syndrome);	static_call_update(raid6_xor_syndrome_impl,			raid6_algos[raid6_nr_algos - 1]->xor_syndrome); #ifdef MODULE	return raid6_select_algo();#else	return 0;#endif} static void __exit raid6_exit(void){} /* * When built-in we must register the default implementation before md * initializes, but we don't want calibration to run that early as that * would delay the boot process. */#ifndef MODULEdevice_initcall(raid6_select_algo);#endifsubsys_initcall(raid6_init);module_exit(raid6_exit);MODULE_LICENSE("GPL");MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); #if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST)const struct raid6_calls *raid6_algo_find(unsigned int idx){	if (idx >= raid6_nr_algos) {		/*		 * Always include the simplest generic integer implementation in		 * the unit tests as a baseline.		 */		if (idx == raid6_nr_algos &&		    raid6_algos[0] != &raid6_intx1)			return &raid6_intx1;		return NULL;	}	return raid6_algos[idx];}EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find); const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx){	switch (idx) {	case 0:		/* always test the generic integer implementation */		return &raid6_recov_intx1;	case 1:		/* test the optimized implementation if there is one */		if (raid6_recov_algo != &raid6_recov_intx1)			return raid6_recov_algo;		return NULL;	default:		return NULL;	}}EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find);#endif /* CONFIG_RAID6_PQ_KUNIT_TEST */