History

Merge tag 'md/4.9-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shli/md

Pull MD updates from Shaohua Li: "This update includes: - new AVX512 instruction based raid6 gen/recovery algorithm - a couple of md-cluster related bug fixes - fix a potential deadlock - set nonrotational bit for raid array with SSD - set correct max_hw_sectors for raid5/6, which hopefuly can improve performance a little bit - other minor fixes" * tag 'md/4.9-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shli/md: md: set rotational bit raid6/test/test.c: bug fix: Specify aligned(alignment) attributes to the char arrays raid5: handle register_shrinker failure raid5: fix to detect failure of register_shrinker md: fix a potential deadlock md/bitmap: fix wrong cleanup raid5: allow arbitrary max_hw_sectors lib/raid6: Add AVX512 optimized xor_syndrome functions lib/raid6/test/Makefile: Add avx512 gen_syndrome and recovery functions lib/raid6: Add AVX512 optimized recovery functions lib/raid6: Add AVX512 optimized gen_syndrome functions md-cluster: make resync lock also could be interruptted md-cluster: introduce dlm_lock_sync_interruptible to fix tasks hang md-cluster: convert the completion to wait queue md-cluster: protect md_find_rdev_nr_rcu with rcu lock md-cluster: clean related infos of cluster md: changes for MD_STILL_CLOSED flag md-cluster: remove some unnecessary dlm_unlock_sync md-cluster: use FORCEUNLOCK in lockres_free md-cluster: call md_kick_rdev_from_array once ack failed

Linus Torvalds committed 10 years ago

Commit c23112e2 parents 4dfddf5 bb086a814 files · +1111 −54Browse files

Unified · Split

14 files
  1. arch/x86/Makefile +3 −2
  2. drivers/md/bitmap.c +1 −3
  3. drivers/md/md-cluster.c +69 −30
  4. drivers/md/md.c +35 −9
  5. drivers/md/md.h +2 −3
  6. drivers/md/raid5.c +9 −2
  7. include/linux/raid/pq.h +4 −0
  8. lib/raid6/Makefile +1 −1
  9. lib/raid6/algos.c +12 −0
  10. lib/raid6/avx512.c +569 −0
  11. lib/raid6/recov_avx512.c +388 −0
  12. lib/raid6/test/Makefile +4 −1
  13. lib/raid6/test/test.c +4 −3
  14. lib/raid6/x86.h +10 −0
arch/x86/Makefile+3−2
@@ -163,11 +163,12 @@ asinstr += $(call as-instr,pshufb %xmm0$(comma)%xmm0,-DCONFIG_AS_SSSE3=1) · 162 unmodified lines
163asinstr += $(call as-instr,crc32l %eax$(comma)%eax,-DCONFIG_AS_CRC32=1)163asinstr += $(call as-instr,crc32l %eax$(comma)%eax,-DCONFIG_AS_CRC32=1)
164avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1)164avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1)
165avx2_instr :=$(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1)165avx2_instr :=$(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1)
166avx512_instr :=$(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1)
166sha1_ni_instr :=$(call as-instr,sha1msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA1_NI=1)167sha1_ni_instr :=$(call as-instr,sha1msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA1_NI=1)
167sha256_ni_instr :=$(call as-instr,sha256msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA256_NI=1)168sha256_ni_instr :=$(call as-instr,sha256msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA256_NI=1)
168 169
169KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(sha1_ni_instr) $(sha256_ni_instr)170KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr)
170KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(sha1_ni_instr) $(sha256_ni_instr)171KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr)
171 172
172LDFLAGS := -m elf_$(UTS_MACHINE)173LDFLAGS := -m elf_$(UTS_MACHINE)
173 174
drivers/md/bitmap.c+1−3
@@ -1903,10 +1903,8 @@ int bitmap_copy_from_slot(struct mddev *mddev, int slot, · 1902 unmodified lines
1903 struct bitmap_counts *counts;1903 struct bitmap_counts *counts;
1904 struct bitmap *bitmap = bitmap_create(mddev, slot);1904 struct bitmap *bitmap = bitmap_create(mddev, slot);
1905 1905
1906 if (IS_ERR(bitmap)) {1906 if (IS_ERR(bitmap))
1907 bitmap_free(bitmap);
1908 return PTR_ERR(bitmap);1907 return PTR_ERR(bitmap);
1909 }
1910 1908
1911 rv = bitmap_init_from_disk(bitmap, 0);1909 rv = bitmap_init_from_disk(bitmap, 0);
1912 if (rv)1910 if (rv)
drivers/md/md-cluster.c+69−30
@@ -10,6 +10,7 @@ · 9 unmodified lines
10 10
11 11
12#include <linux/module.h>12#include <linux/module.h>
13#include <linux/kthread.h>
13#include <linux/dlm.h>14#include <linux/dlm.h>
14#include <linux/sched.h>15#include <linux/sched.h>
15#include <linux/raid/md_p.h>16#include <linux/raid/md_p.h>
@@ -25,7 +26,8 @@ struct dlm_lock_resource { · 9 unmodified lines
25 struct dlm_lksb lksb;26 struct dlm_lksb lksb;
26 char *name; /* lock name. */27 char *name; /* lock name. */
27 uint32_t flags; /* flags to pass to dlm_lock() */28 uint32_t flags; /* flags to pass to dlm_lock() */
28 struct completion completion; /* completion for synchronized locking */29 wait_queue_head_t sync_locking; /* wait queue for synchronized locking */
30 bool sync_locking_done;
29 void (*bast)(void *arg, int mode); /* blocking AST function pointer*/31 void (*bast)(void *arg, int mode); /* blocking AST function pointer*/
30 struct mddev *mddev; /* pointing back to mddev. */32 struct mddev *mddev; /* pointing back to mddev. */
31 int mode;33 int mode;
@@ -118,7 +120,8 @@ static void sync_ast(void *arg) · 86 unmodified lines
118 struct dlm_lock_resource *res;120 struct dlm_lock_resource *res;
119 121
120 res = arg;122 res = arg;
121 complete(&res->completion);123 res->sync_locking_done = true;
124 wake_up(&res->sync_locking);
122}125}
123 126
124static int dlm_lock_sync(struct dlm_lock_resource *res, int mode)127static int dlm_lock_sync(struct dlm_lock_resource *res, int mode)
@@ -130,7 +133,8 @@ static int dlm_lock_sync(struct dlm_lock_resource *res, int mode) · 5 unmodified lines
130 0, sync_ast, res, res->bast);133 0, sync_ast, res, res->bast);
131 if (ret)134 if (ret)
132 return ret;135 return ret;
133 wait_for_completion(&res->completion);136 wait_event(res->sync_locking, res->sync_locking_done);
137 res->sync_locking_done = false;
134 if (res->lksb.sb_status == 0)138 if (res->lksb.sb_status == 0)
135 res->mode = mode;139 res->mode = mode;
136 return res->lksb.sb_status;140 return res->lksb.sb_status;
@@ -141,6 +145,44 @@ static int dlm_unlock_sync(struct dlm_lock_resource *res) · 4 unmodified lines
141 return dlm_lock_sync(res, DLM_LOCK_NL);145 return dlm_lock_sync(res, DLM_LOCK_NL);
142}146}
143 147
148/*
149 * An variation of dlm_lock_sync, which make lock request could
150 * be interrupted
151 */
152static int dlm_lock_sync_interruptible(struct dlm_lock_resource *res, int mode,
153 struct mddev *mddev)
154{
155 int ret = 0;
156
157 ret = dlm_lock(res->ls, mode, &res->lksb,
158 res->flags, res->name, strlen(res->name),
159 0, sync_ast, res, res->bast);
160 if (ret)
161 return ret;
162
163 wait_event(res->sync_locking, res->sync_locking_done
164 || kthread_should_stop()
165 || test_bit(MD_CLOSING, &mddev->flags));
166 if (!res->sync_locking_done) {
167 /*
168 * the convert queue contains the lock request when request is
169 * interrupted, and sync_ast could still be run, so need to
170 * cancel the request and reset completion
171 */
172 ret = dlm_unlock(res->ls, res->lksb.sb_lkid, DLM_LKF_CANCEL,
173 &res->lksb, res);
174 res->sync_locking_done = false;
175 if (unlikely(ret != 0))
176 pr_info("failed to cancel previous lock request "
177 "%s return %d\n", res->name, ret);
178 return -EPERM;
179 } else
180 res->sync_locking_done = false;
181 if (res->lksb.sb_status == 0)
182 res->mode = mode;
183 return res->lksb.sb_status;
184}
185
144static struct dlm_lock_resource *lockres_init(struct mddev *mddev,186static struct dlm_lock_resource *lockres_init(struct mddev *mddev,
145 char *name, void (*bastfn)(void *arg, int mode), int with_lvb)187 char *name, void (*bastfn)(void *arg, int mode), int with_lvb)
146{188{
@@ -151,7 +193,8 @@ static struct dlm_lock_resource *lockres_init(struct mddev *mddev, · 4 unmodified lines
151 res = kzalloc(sizeof(struct dlm_lock_resource), GFP_KERNEL);193 res = kzalloc(sizeof(struct dlm_lock_resource), GFP_KERNEL);
152 if (!res)194 if (!res)
153 return NULL;195 return NULL;
154 init_completion(&res->completion);196 init_waitqueue_head(&res->sync_locking);
197 res->sync_locking_done = false;
155 res->ls = cinfo->lockspace;198 res->ls = cinfo->lockspace;
156 res->mddev = mddev;199 res->mddev = mddev;
157 res->mode = DLM_LOCK_IV;200 res->mode = DLM_LOCK_IV;
@@ -194,25 +237,21 @@ out_err: · 36 unmodified lines
194 237
195static void lockres_free(struct dlm_lock_resource *res)238static void lockres_free(struct dlm_lock_resource *res)
196{239{
197 int ret;240 int ret = 0;
198 241
199 if (!res)242 if (!res)
200 return;243 return;
201 244
202 /* cancel a lock request or a conversion request that is blocked */245 /*
203 res->flags |= DLM_LKF_CANCEL;246 * use FORCEUNLOCK flag, so we can unlock even the lock is on the
204retry:247 * waiting or convert queue
205 ret = dlm_unlock(res->ls, res->lksb.sb_lkid, 0, &res->lksb, res);248 */
206 if (unlikely(ret != 0)) {249 ret = dlm_unlock(res->ls, res->lksb.sb_lkid, DLM_LKF_FORCEUNLOCK,
207 pr_info("%s: failed to unlock %s return %d\n", __func__, res->name, ret);250 &res->lksb, res);
208 251 if (unlikely(ret != 0))
209 /* if a lock conversion is cancelled, then the lock is put252 pr_err("failed to unlock %s return %d\n", res->name, ret);
210 * back to grant queue, need to ensure it is unlocked */253 else
211 if (ret == -DLM_ECANCEL)254 wait_event(res->sync_locking, res->sync_locking_done);
212 goto retry;
213 }
214 res->flags &= ~DLM_LKF_CANCEL;
215 wait_for_completion(&res->completion);
216 255
217 kfree(res->name);256 kfree(res->name);
218 kfree(res->lksb.sb_lvbptr);257 kfree(res->lksb.sb_lvbptr);
@@ -279,7 +318,7 @@ static void recover_bitmaps(struct md_thread *thread) · 60 unmodified lines
279 goto clear_bit;318 goto clear_bit;
280 }319 }
281 320
282 ret = dlm_lock_sync(bm_lockres, DLM_LOCK_PW);321 ret = dlm_lock_sync_interruptible(bm_lockres, DLM_LOCK_PW, mddev);
283 if (ret) {322 if (ret) {
284 pr_err("md-cluster: Could not DLM lock %s: %d\n",323 pr_err("md-cluster: Could not DLM lock %s: %d\n",
285 str, ret);324 str, ret);
@@ -288,7 +327,7 @@ static void recover_bitmaps(struct md_thread *thread) · 2 unmodified lines
288 ret = bitmap_copy_from_slot(mddev, slot, &lo, &hi, true);327 ret = bitmap_copy_from_slot(mddev, slot, &lo, &hi, true);
289 if (ret) {328 if (ret) {
290 pr_err("md-cluster: Could not copy data from bitmap %d\n", slot);329 pr_err("md-cluster: Could not copy data from bitmap %d\n", slot);
291 goto dlm_unlock;330 goto clear_bit;
292 }331 }
293 if (hi > 0) {332 if (hi > 0) {
294 if (lo < mddev->recovery_cp)333 if (lo < mddev->recovery_cp)
@@ -300,8 +339,6 @@ static void recover_bitmaps(struct md_thread *thread) · 5 unmodified lines
300 md_wakeup_thread(mddev->thread);339 md_wakeup_thread(mddev->thread);
301 }340 }
302 }341 }
303dlm_unlock:
304 dlm_unlock_sync(bm_lockres);
305clear_bit:342clear_bit:
306 lockres_free(bm_lockres);343 lockres_free(bm_lockres);
307 clear_bit(slot, &cinfo->recovery_map);344 clear_bit(slot, &cinfo->recovery_map);
@@ -495,9 +532,10 @@ static void process_metadata_update(struct mddev *mddev, struct cluster_msg *msg · 187 unmodified lines
495 532
496static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg)533static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg)
497{534{
498 struct md_rdev *rdev = md_find_rdev_nr_rcu(mddev,535 struct md_rdev *rdev;
499 le32_to_cpu(msg->raid_slot));
500 536
537 rcu_read_lock();
538 rdev = md_find_rdev_nr_rcu(mddev, le32_to_cpu(msg->raid_slot));
501 if (rdev) {539 if (rdev) {
502 set_bit(ClusterRemove, &rdev->flags);540 set_bit(ClusterRemove, &rdev->flags);
503 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);541 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
@@ -506,18 +544,21 @@ static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg) · 2 unmodified lines
506 else544 else
507 pr_warn("%s: %d Could not find disk(%d) to REMOVE\n",545 pr_warn("%s: %d Could not find disk(%d) to REMOVE\n",
508 __func__, __LINE__, le32_to_cpu(msg->raid_slot));546 __func__, __LINE__, le32_to_cpu(msg->raid_slot));
547 rcu_read_unlock();
509}548}
510 549
511static void process_readd_disk(struct mddev *mddev, struct cluster_msg *msg)550static void process_readd_disk(struct mddev *mddev, struct cluster_msg *msg)
512{551{
513 struct md_rdev *rdev = md_find_rdev_nr_rcu(mddev,552 struct md_rdev *rdev;
514 le32_to_cpu(msg->raid_slot));
515 553
554 rcu_read_lock();
555 rdev = md_find_rdev_nr_rcu(mddev, le32_to_cpu(msg->raid_slot));
516 if (rdev && test_bit(Faulty, &rdev->flags))556 if (rdev && test_bit(Faulty, &rdev->flags))
517 clear_bit(Faulty, &rdev->flags);557 clear_bit(Faulty, &rdev->flags);
518 else558 else
519 pr_warn("%s: %d Could not find disk(%d) which is faulty",559 pr_warn("%s: %d Could not find disk(%d) which is faulty",
520 __func__, __LINE__, le32_to_cpu(msg->raid_slot));560 __func__, __LINE__, le32_to_cpu(msg->raid_slot));
561 rcu_read_unlock();
521}562}
522 563
523static int process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)564static int process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
@@ -770,7 +811,6 @@ static int gather_all_resync_info(struct mddev *mddev, int total_slots) · 246 unmodified lines
770 md_check_recovery(mddev);811 md_check_recovery(mddev);
771 }812 }
772 813
773 dlm_unlock_sync(bm_lockres);
774 lockres_free(bm_lockres);814 lockres_free(bm_lockres);
775 }815 }
776out:816out:
@@ -1006,7 +1046,7 @@ static void metadata_update_cancel(struct mddev *mddev) · 229 unmodified lines
1006static int resync_start(struct mddev *mddev)1046static int resync_start(struct mddev *mddev)
1007{1047{
1008 struct md_cluster_info *cinfo = mddev->cluster_info;1048 struct md_cluster_info *cinfo = mddev->cluster_info;
1009 return dlm_lock_sync(cinfo->resync_lockres, DLM_LOCK_EX);1049 return dlm_lock_sync_interruptible(cinfo->resync_lockres, DLM_LOCK_EX, mddev);
1010}1050}
1011 1051
1012static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi)1052static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi)
@@ -1186,7 +1226,6 @@ static void unlock_all_bitmaps(struct mddev *mddev) · 173 unmodified lines
1186 if (cinfo->other_bitmap_lockres) {1226 if (cinfo->other_bitmap_lockres) {
1187 for (i = 0; i < mddev->bitmap_info.nodes - 1; i++) {1227 for (i = 0; i < mddev->bitmap_info.nodes - 1; i++) {
1188 if (cinfo->other_bitmap_lockres[i]) {1228 if (cinfo->other_bitmap_lockres[i]) {
1189 dlm_unlock_sync(cinfo->other_bitmap_lockres[i]);
1190 lockres_free(cinfo->other_bitmap_lockres[i]);1229 lockres_free(cinfo->other_bitmap_lockres[i]);
1191 }1230 }
1192 }1231 }
drivers/md/md.c+35−9
@@ -5297,6 +5297,21 @@ int md_run(struct mddev *mddev) · 5296 unmodified lines
5297 return err;5297 return err;
5298 }5298 }
5299 if (mddev->queue) {5299 if (mddev->queue) {
5300 bool nonrot = true;
5301
5302 rdev_for_each(rdev, mddev) {
5303 if (rdev->raid_disk >= 0 &&
5304 !blk_queue_nonrot(bdev_get_queue(rdev->bdev))) {
5305 nonrot = false;
5306 break;
5307 }
5308 }
5309 if (mddev->degraded)
5310 nonrot = false;
5311 if (nonrot)
5312 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mddev->queue);
5313 else
5314 queue_flag_clear_unlocked(QUEUE_FLAG_NONROT, mddev->queue);
5300 mddev->queue->backing_dev_info.congested_data = mddev;5315 mddev->queue->backing_dev_info.congested_data = mddev;
5301 mddev->queue->backing_dev_info.congested_fn = md_congested;5316 mddev->queue->backing_dev_info.congested_fn = md_congested;
5302 }5317 }
@@ -5454,12 +5469,14 @@ static void md_clean(struct mddev *mddev) · 151 unmodified lines
5454 mddev->degraded = 0;5469 mddev->degraded = 0;
5455 mddev->safemode = 0;5470 mddev->safemode = 0;
5456 mddev->private = NULL;5471 mddev->private = NULL;
5472 mddev->cluster_info = NULL;
5457 mddev->bitmap_info.offset = 0;5473 mddev->bitmap_info.offset = 0;
5458 mddev->bitmap_info.default_offset = 0;5474 mddev->bitmap_info.default_offset = 0;
5459 mddev->bitmap_info.default_space = 0;5475 mddev->bitmap_info.default_space = 0;
5460 mddev->bitmap_info.chunksize = 0;5476 mddev->bitmap_info.chunksize = 0;
5461 mddev->bitmap_info.daemon_sleep = 0;5477 mddev->bitmap_info.daemon_sleep = 0;
5462 mddev->bitmap_info.max_write_behind = 0;5478 mddev->bitmap_info.max_write_behind = 0;
5479 mddev->bitmap_info.nodes = 0;
5463}5480}
5464 5481
5465static void __md_stop_writes(struct mddev *mddev)5482static void __md_stop_writes(struct mddev *mddev)
@@ -5573,8 +5590,7 @@ static int md_set_readonly(struct mddev *mddev, struct block_device *bdev) · 107 unmodified lines
5573 mutex_lock(&mddev->open_mutex);5590 mutex_lock(&mddev->open_mutex);
5574 if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) ||5591 if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) ||
5575 mddev->sync_thread ||5592 mddev->sync_thread ||
5576 test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) ||5593 test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
5577 (bdev && !test_bit(MD_STILL_CLOSED, &mddev->flags))) {
5578 printk("md: %s still in use.\n",mdname(mddev));5594 printk("md: %s still in use.\n",mdname(mddev));
5579 if (did_freeze) {5595 if (did_freeze) {
5580 clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);5596 clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
@@ -5636,8 +5652,7 @@ static int do_md_stop(struct mddev *mddev, int mode, · 55 unmodified lines
5636 if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) ||5652 if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) ||
5637 mddev->sysfs_active ||5653 mddev->sysfs_active ||
5638 mddev->sync_thread ||5654 mddev->sync_thread ||
5639 test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) ||5655 test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
5640 (bdev && !test_bit(MD_STILL_CLOSED, &mddev->flags))) {
5641 printk("md: %s still in use.\n",mdname(mddev));5656 printk("md: %s still in use.\n",mdname(mddev));
5642 mutex_unlock(&mddev->open_mutex);5657 mutex_unlock(&mddev->open_mutex);
5643 if (did_freeze) {5658 if (did_freeze) {
@@ -6101,9 +6116,14 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info) · 457 unmodified lines
6101 export_rdev(rdev);6116 export_rdev(rdev);
6102 6117
6103 if (mddev_is_clustered(mddev)) {6118 if (mddev_is_clustered(mddev)) {
6104 if (info->state & (1 << MD_DISK_CANDIDATE))6119 if (info->state & (1 << MD_DISK_CANDIDATE)) {
6105 md_cluster_ops->new_disk_ack(mddev, (err == 0));6120 if (!err) {
6106 else {6121 err = md_cluster_ops->new_disk_ack(mddev,
6122 err == 0);
6123 if (err)
6124 md_kick_rdev_from_array(rdev);
6125 }
6126 } else {
6107 if (err)6127 if (err)
6108 md_cluster_ops->add_new_disk_cancel(mddev);6128 md_cluster_ops->add_new_disk_cancel(mddev);
6109 else6129 else
@@ -6821,7 +6841,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, · 711 unmodified lines
6821 err = -EBUSY;6841 err = -EBUSY;
6822 goto out;6842 goto out;
6823 }6843 }
6824 set_bit(MD_STILL_CLOSED, &mddev->flags);6844 set_bit(MD_CLOSING, &mddev->flags);
6825 mutex_unlock(&mddev->open_mutex);6845 mutex_unlock(&mddev->open_mutex);
6826 sync_blockdev(bdev);6846 sync_blockdev(bdev);
6827 }6847 }
@@ -7070,9 +7090,13 @@ static int md_open(struct block_device *bdev, fmode_t mode) · 242 unmodified lines
7070 if ((err = mutex_lock_interruptible(&mddev->open_mutex)))7090 if ((err = mutex_lock_interruptible(&mddev->open_mutex)))
7071 goto out;7091 goto out;
7072 7092
7093 if (test_bit(MD_CLOSING, &mddev->flags)) {
7094 mutex_unlock(&mddev->open_mutex);
7095 return -ENODEV;
7096 }
7097
7073 err = 0;7098 err = 0;
7074 atomic_inc(&mddev->openers);7099 atomic_inc(&mddev->openers);
7075 clear_bit(MD_STILL_CLOSED, &mddev->flags);
7076 mutex_unlock(&mddev->open_mutex);7100 mutex_unlock(&mddev->open_mutex);
7077 7101
7078 check_disk_change(bdev);7102 check_disk_change(bdev);
@@ -8873,7 +8897,9 @@ static void autostart_arrays(int part) · 1794 unmodified lines
8873 list_del(&node_detected_dev->list);8897 list_del(&node_detected_dev->list);
8874 dev = node_detected_dev->dev;8898 dev = node_detected_dev->dev;
8875 kfree(node_detected_dev);8899 kfree(node_detected_dev);
8900 mutex_unlock(&detected_devices_mutex);
8876 rdev = md_import_device(dev,0, 90);8901 rdev = md_import_device(dev,0, 90);
8902 mutex_lock(&detected_devices_mutex);
8877 if (IS_ERR(rdev))8903 if (IS_ERR(rdev))
8878 continue;8904 continue;
8879 8905
drivers/md/md.h+2−3
@@ -201,9 +201,8 @@ struct mddev { · 200 unmodified lines
201#define MD_CHANGE_PENDING 2 /* switch from 'clean' to 'active' in progress */201#define MD_CHANGE_PENDING 2 /* switch from 'clean' to 'active' in progress */
202#define MD_UPDATE_SB_FLAGS (1 | 2 | 4) /* If these are set, md_update_sb needed */202#define MD_UPDATE_SB_FLAGS (1 | 2 | 4) /* If these are set, md_update_sb needed */
203#define MD_ARRAY_FIRST_USE 3 /* First use of array, needs initialization */203#define MD_ARRAY_FIRST_USE 3 /* First use of array, needs initialization */
204#define MD_STILL_CLOSED 4 /* If set, then array has not been opened since204#define MD_CLOSING 4 /* If set, we are closing the array, do not open
205 * md_ioctl checked on it.205 * it then */
206 */
207#define MD_JOURNAL_CLEAN 5 /* A raid with journal is already clean */206#define MD_JOURNAL_CLEAN 5 /* A raid with journal is already clean */
208#define MD_HAS_JOURNAL 6 /* The raid array has journal feature set */207#define MD_HAS_JOURNAL 6 /* The raid array has journal feature set */
209#define MD_RELOAD_SB 7 /* Reload the superblock because another node208#define MD_RELOAD_SB 7 /* Reload the superblock because another node
drivers/md/raid5.c+9−2
@@ -6370,7 +6370,7 @@ static void free_conf(struct r5conf *conf) · 6369 unmodified lines
6370{6370{
6371 if (conf->log)6371 if (conf->log)
6372 r5l_exit_log(conf->log);6372 r5l_exit_log(conf->log);
6373 if (conf->shrinker.seeks)6373 if (conf->shrinker.nr_deferred)
6374 unregister_shrinker(&conf->shrinker);6374 unregister_shrinker(&conf->shrinker);
6375 6375
6376 free_thread_groups(conf);6376 free_thread_groups(conf);
@@ -6632,7 +6632,12 @@ static struct r5conf *setup_conf(struct mddev *mddev) · 255 unmodified lines
6632 conf->shrinker.count_objects = raid5_cache_count;6632 conf->shrinker.count_objects = raid5_cache_count;
6633 conf->shrinker.batch = 128;6633 conf->shrinker.batch = 128;
6634 conf->shrinker.flags = 0;6634 conf->shrinker.flags = 0;
6635 register_shrinker(&conf->shrinker);6635 if (register_shrinker(&conf->shrinker)) {
6636 printk(KERN_ERR
6637 "md/raid:%s: couldn't register shrinker.\n",
6638 mdname(mddev));
6639 goto abort;
6640 }
6636 6641
6637 sprintf(pers_name, "raid%d", mddev->new_level);6642 sprintf(pers_name, "raid%d", mddev->new_level);
6638 conf->thread = md_register_thread(raid5d, mddev, pers_name);6643 conf->thread = md_register_thread(raid5d, mddev, pers_name);
@@ -7028,6 +7033,8 @@ static int raid5_run(struct mddev *mddev) · 389 unmodified lines
7028 else7033 else
7029 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,7034 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
7030 mddev->queue);7035 mddev->queue);
7036
7037 blk_queue_max_hw_sectors(mddev->queue, UINT_MAX);
7031 }7038 }
7032 7039
7033 if (journal_dev) {7040 if (journal_dev) {
include/linux/raid/pq.h+4−0
@@ -102,6 +102,9 @@ extern const struct raid6_calls raid6_altivec8; · 101 unmodified lines
102extern const struct raid6_calls raid6_avx2x1;102extern const struct raid6_calls raid6_avx2x1;
103extern const struct raid6_calls raid6_avx2x2;103extern const struct raid6_calls raid6_avx2x2;
104extern const struct raid6_calls raid6_avx2x4;104extern const struct raid6_calls raid6_avx2x4;
105extern const struct raid6_calls raid6_avx512x1;
106extern const struct raid6_calls raid6_avx512x2;
107extern const struct raid6_calls raid6_avx512x4;
105extern const struct raid6_calls raid6_tilegx8;108extern const struct raid6_calls raid6_tilegx8;
106extern const struct raid6_calls raid6_s390vx8;109extern const struct raid6_calls raid6_s390vx8;
107 110
@@ -116,6 +119,7 @@ struct raid6_recov_calls { · 8 unmodified lines
116extern const struct raid6_recov_calls raid6_recov_intx1;119extern const struct raid6_recov_calls raid6_recov_intx1;
117extern const struct raid6_recov_calls raid6_recov_ssse3;120extern const struct raid6_recov_calls raid6_recov_ssse3;
118extern const struct raid6_recov_calls raid6_recov_avx2;121extern const struct raid6_recov_calls raid6_recov_avx2;
122extern const struct raid6_recov_calls raid6_recov_avx512;
119extern const struct raid6_recov_calls raid6_recov_s390xc;123extern const struct raid6_recov_calls raid6_recov_s390xc;
120 124
121extern const struct raid6_calls raid6_neonx1;125extern const struct raid6_calls raid6_neonx1;
lib/raid6/Makefile+1−1
@@ -3,7 +3,7 @@ obj-$(CONFIG_RAID6_PQ) += raid6_pq.o · 2 unmodified lines
3raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \3raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \
4 int8.o int16.o int32.o4 int8.o int16.o int32.o
5 5
6raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o6raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o
7raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o7raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o
8raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o8raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o
9raid6_pq-$(CONFIG_TILEGX) += tilegx8.o9raid6_pq-$(CONFIG_TILEGX) += tilegx8.o
lib/raid6/algos.c+12−0
@@ -49,6 +49,10 @@ const struct raid6_calls * const raid6_algos[] = { · 48 unmodified lines
49 &raid6_avx2x1,49 &raid6_avx2x1,
50 &raid6_avx2x2,50 &raid6_avx2x2,
51#endif51#endif
52#ifdef CONFIG_AS_AVX512
53 &raid6_avx512x1,
54 &raid6_avx512x2,
55#endif
52#endif56#endif
53#if defined(__x86_64__) && !defined(__arch_um__)57#if defined(__x86_64__) && !defined(__arch_um__)
54 &raid6_sse2x1,58 &raid6_sse2x1,
@@ -59,6 +63,11 @@ const struct raid6_calls * const raid6_algos[] = { · 4 unmodified lines
59 &raid6_avx2x2,63 &raid6_avx2x2,
60 &raid6_avx2x4,64 &raid6_avx2x4,
61#endif65#endif
66#ifdef CONFIG_AS_AVX512
67 &raid6_avx512x1,
68 &raid6_avx512x2,
69 &raid6_avx512x4,
70#endif
62#endif71#endif
63#ifdef CONFIG_ALTIVEC72#ifdef CONFIG_ALTIVEC
64 &raid6_altivec1,73 &raid6_altivec1,
@@ -92,6 +101,9 @@ void (*raid6_datap_recov)(int, size_t, int, void **); · 27 unmodified lines
92EXPORT_SYMBOL_GPL(raid6_datap_recov);101EXPORT_SYMBOL_GPL(raid6_datap_recov);
93 102
94const struct raid6_recov_calls *const raid6_recov_algos[] = {103const struct raid6_recov_calls *const raid6_recov_algos[] = {
104#ifdef CONFIG_AS_AVX512
105 &raid6_recov_avx512,
106#endif
95#ifdef CONFIG_AS_AVX2107#ifdef CONFIG_AS_AVX2
96 &raid6_recov_avx2,108 &raid6_recov_avx2,
97#endif109#endif
lib/raid6/avx512.c+569−0
@@ -0,0 +1,569 @@
1/* -*- linux-c -*- --------------------------------------------------------
2 *
3 * Copyright (C) 2016 Intel Corporation
4 *
5 * Author: Gayatri Kammela <gayatri.kammela@intel.com>
6 * Author: Megha Dey <megha.dey@linux.intel.com>
7 *
8 * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved
9 * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation, Inc., 53 Temple Place Ste 330,
14 * Boston MA 02111-1307, USA; either version 2 of the License, or
15 * (at your option) any later version; incorporated herein by reference.
16 *
17 * -----------------------------------------------------------------------
18 */
19
20/*
21 * AVX512 implementation of RAID-6 syndrome functions
22 *
23 */
24
25#ifdef CONFIG_AS_AVX512
26
27#include <linux/raid/pq.h>
28#include "x86.h"
29
30static const struct raid6_avx512_constants {
31 u64 x1d[8];
32} raid6_avx512_constants __aligned(512) = {
33 { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
34 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
35 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
36 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
37};
38
39static int raid6_have_avx512(void)
40{
41 return boot_cpu_has(X86_FEATURE_AVX2) &&
42 boot_cpu_has(X86_FEATURE_AVX) &&
43 boot_cpu_has(X86_FEATURE_AVX512F) &&
44 boot_cpu_has(X86_FEATURE_AVX512BW) &&
45 boot_cpu_has(X86_FEATURE_AVX512VL) &&
46 boot_cpu_has(X86_FEATURE_AVX512DQ);
47}
48
49static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
50{
51 u8 **dptr = (u8 **)ptrs;
52 u8 *p, *q;
53 int d, z, z0;
54
55 z0 = disks - 3; /* Highest data disk */
56 p = dptr[z0+1]; /* XOR parity */
57 q = dptr[z0+2]; /* RS syndrome */
58
59 kernel_fpu_begin();
60
61 asm volatile("vmovdqa64 %0,%%zmm0\n\t"
62 "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
63 :
64 : "m" (raid6_avx512_constants.x1d[0]));
65
66 for (d = 0; d < bytes; d += 64) {
67 asm volatile("prefetchnta %0\n\t"
68 "vmovdqa64 %0,%%zmm2\n\t" /* P[0] */
69 "prefetchnta %1\n\t"
70 "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
71 "vmovdqa64 %1,%%zmm6"
72 :
73 : "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
74 for (z = z0-2; z >= 0; z--) {
75 asm volatile("prefetchnta %0\n\t"
76 "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
77 "vpmovm2b %%k1,%%zmm5\n\t"
78 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
79 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
80 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
81 "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
82 "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
83 "vmovdqa64 %0,%%zmm6"
84 :
85 : "m" (dptr[z][d]));
86 }
87 asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
88 "vpmovm2b %%k1,%%zmm5\n\t"
89 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
90 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
91 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
92 "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
93 "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
94 "vmovntdq %%zmm2,%0\n\t"
95 "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
96 "vmovntdq %%zmm4,%1\n\t"
97 "vpxorq %%zmm4,%%zmm4,%%zmm4"
98 :
99 : "m" (p[d]), "m" (q[d]));
100 }
101
102 asm volatile("sfence" : : : "memory");
103 kernel_fpu_end();
104}
105
106static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
107 size_t bytes, void **ptrs)
108{
109 u8 **dptr = (u8 **)ptrs;
110 u8 *p, *q;
111 int d, z, z0;
112
113 z0 = stop; /* P/Q right side optimization */
114 p = dptr[disks-2]; /* XOR parity */
115 q = dptr[disks-1]; /* RS syndrome */
116
117 kernel_fpu_begin();
118
119 asm volatile("vmovdqa64 %0,%%zmm0"
120 : : "m" (raid6_avx512_constants.x1d[0]));
121
122 for (d = 0 ; d < bytes ; d += 64) {
123 asm volatile("vmovdqa64 %0,%%zmm4\n\t"
124 "vmovdqa64 %1,%%zmm2\n\t"
125 "vpxorq %%zmm4,%%zmm2,%%zmm2"
126 :
127 : "m" (dptr[z0][d]), "m" (p[d]));
128 /* P/Q data pages */
129 for (z = z0-1 ; z >= start ; z--) {
130 asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
131 "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
132 "vpmovm2b %%k1,%%zmm5\n\t"
133 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
134 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
135 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
136 "vmovdqa64 %0,%%zmm5\n\t"
137 "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
138 "vpxorq %%zmm5,%%zmm4,%%zmm4"
139 :
140 : "m" (dptr[z][d]));
141 }
142 /* P/Q left side optimization */
143 for (z = start-1 ; z >= 0 ; z--) {
144 asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
145 "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
146 "vpmovm2b %%k1,%%zmm5\n\t"
147 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
148 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
149 "vpxorq %%zmm5,%%zmm4,%%zmm4"
150 :
151 : );
152 }
153 asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
154 /* Don't use movntdq for r/w memory area < cache line */
155 "vmovdqa64 %%zmm4,%0\n\t"
156 "vmovdqa64 %%zmm2,%1"
157 :
158 : "m" (q[d]), "m" (p[d]));
159 }
160
161 asm volatile("sfence" : : : "memory");
162 kernel_fpu_end();
163}
164
165const struct raid6_calls raid6_avx512x1 = {
166 raid6_avx5121_gen_syndrome,
167 raid6_avx5121_xor_syndrome,
168 raid6_have_avx512,
169 "avx512x1",
170 1 /* Has cache hints */
171};
172
173/*
174 * Unrolled-by-2 AVX512 implementation
175 */
176static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
177{
178 u8 **dptr = (u8 **)ptrs;
179 u8 *p, *q;
180 int d, z, z0;
181
182 z0 = disks - 3; /* Highest data disk */
183 p = dptr[z0+1]; /* XOR parity */
184 q = dptr[z0+2]; /* RS syndrome */
185
186 kernel_fpu_begin();
187
188 asm volatile("vmovdqa64 %0,%%zmm0\n\t"
189 "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
190 :
191 : "m" (raid6_avx512_constants.x1d[0]));
192
193 /* We uniformly assume a single prefetch covers at least 64 bytes */
194 for (d = 0; d < bytes; d += 128) {
195 asm volatile("prefetchnta %0\n\t"
196 "prefetchnta %1\n\t"
197 "vmovdqa64 %0,%%zmm2\n\t" /* P[0] */
198 "vmovdqa64 %1,%%zmm3\n\t" /* P[1] */
199 "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
200 "vmovdqa64 %%zmm3,%%zmm6" /* Q[1] */
201 :
202 : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]));
203 for (z = z0-1; z >= 0; z--) {
204 asm volatile("prefetchnta %0\n\t"
205 "prefetchnta %1\n\t"
206 "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
207 "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
208 "vpmovm2b %%k1,%%zmm5\n\t"
209 "vpmovm2b %%k2,%%zmm7\n\t"
210 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
211 "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
212 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
213 "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
214 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
215 "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
216 "vmovdqa64 %0,%%zmm5\n\t"
217 "vmovdqa64 %1,%%zmm7\n\t"
218 "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
219 "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
220 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
221 "vpxorq %%zmm7,%%zmm6,%%zmm6"
222 :
223 : "m" (dptr[z][d]), "m" (dptr[z][d+64]));
224 }
225 asm volatile("vmovntdq %%zmm2,%0\n\t"
226 "vmovntdq %%zmm3,%1\n\t"
227 "vmovntdq %%zmm4,%2\n\t"
228 "vmovntdq %%zmm6,%3"
229 :
230 : "m" (p[d]), "m" (p[d+64]), "m" (q[d]),
231 "m" (q[d+64]));
232 }
233
234 asm volatile("sfence" : : : "memory");
235 kernel_fpu_end();
236}
237
238static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
239 size_t bytes, void **ptrs)
240{
241 u8 **dptr = (u8 **)ptrs;
242 u8 *p, *q;
243 int d, z, z0;
244
245 z0 = stop; /* P/Q right side optimization */
246 p = dptr[disks-2]; /* XOR parity */
247 q = dptr[disks-1]; /* RS syndrome */
248
249 kernel_fpu_begin();
250
251 asm volatile("vmovdqa64 %0,%%zmm0"
252 : : "m" (raid6_avx512_constants.x1d[0]));
253
254 for (d = 0 ; d < bytes ; d += 128) {
255 asm volatile("vmovdqa64 %0,%%zmm4\n\t"
256 "vmovdqa64 %1,%%zmm6\n\t"
257 "vmovdqa64 %2,%%zmm2\n\t"
258 "vmovdqa64 %3,%%zmm3\n\t"
259 "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
260 "vpxorq %%zmm6,%%zmm3,%%zmm3"
261 :
262 : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
263 "m" (p[d]), "m" (p[d+64]));
264 /* P/Q data pages */
265 for (z = z0-1 ; z >= start ; z--) {
266 asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
267 "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
268 "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
269 "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
270 "vpmovm2b %%k1,%%zmm5\n\t"
271 "vpmovm2b %%k2,%%zmm7\n\t"
272 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
273 "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
274 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
275 "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
276 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
277 "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
278 "vmovdqa64 %0,%%zmm5\n\t"
279 "vmovdqa64 %1,%%zmm7\n\t"
280 "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
281 "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
282 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
283 "vpxorq %%zmm7,%%zmm6,%%zmm6"
284 :
285 : "m" (dptr[z][d]), "m" (dptr[z][d+64]));
286 }
287 /* P/Q left side optimization */
288 for (z = start-1 ; z >= 0 ; z--) {
289 asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
290 "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
291 "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
292 "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
293 "vpmovm2b %%k1,%%zmm5\n\t"
294 "vpmovm2b %%k2,%%zmm7\n\t"
295 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
296 "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
297 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
298 "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
299 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
300 "vpxorq %%zmm7,%%zmm6,%%zmm6"
301 :
302 : );
303 }
304 asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
305 "vpxorq %1,%%zmm6,%%zmm6\n\t"
306 /* Don't use movntdq for r/w
307 * memory area < cache line
308 */
309 "vmovdqa64 %%zmm4,%0\n\t"
310 "vmovdqa64 %%zmm6,%1\n\t"
311 "vmovdqa64 %%zmm2,%2\n\t"
312 "vmovdqa64 %%zmm3,%3"
313 :
314 : "m" (q[d]), "m" (q[d+64]), "m" (p[d]),
315 "m" (p[d+64]));
316 }
317
318 asm volatile("sfence" : : : "memory");
319 kernel_fpu_end();
320}
321
322const struct raid6_calls raid6_avx512x2 = {
323 raid6_avx5122_gen_syndrome,
324 raid6_avx5122_xor_syndrome,
325 raid6_have_avx512,
326 "avx512x2",
327 1 /* Has cache hints */
328};
329
330#ifdef CONFIG_X86_64
331
332/*
333 * Unrolled-by-4 AVX2 implementation
334 */
335static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
336{
337 u8 **dptr = (u8 **)ptrs;
338 u8 *p, *q;
339 int d, z, z0;
340
341 z0 = disks - 3; /* Highest data disk */
342 p = dptr[z0+1]; /* XOR parity */
343 q = dptr[z0+2]; /* RS syndrome */
344
345 kernel_fpu_begin();
346
347 asm volatile("vmovdqa64 %0,%%zmm0\n\t"
348 "vpxorq %%zmm1,%%zmm1,%%zmm1\n\t" /* Zero temp */
349 "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t" /* P[0] */
350 "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t" /* P[1] */
351 "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t" /* Q[0] */
352 "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t" /* Q[1] */
353 "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t" /* P[2] */
354 "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t" /* P[3] */
355 "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t" /* Q[2] */
356 "vpxorq %%zmm14,%%zmm14,%%zmm14" /* Q[3] */
357 :
358 : "m" (raid6_avx512_constants.x1d[0]));
359
360 for (d = 0; d < bytes; d += 256) {
361 for (z = z0; z >= 0; z--) {
362 asm volatile("prefetchnta %0\n\t"
363 "prefetchnta %1\n\t"
364 "prefetchnta %2\n\t"
365 "prefetchnta %3\n\t"
366 "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
367 "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
368 "vpcmpgtb %%zmm12,%%zmm1,%%k3\n\t"
369 "vpcmpgtb %%zmm14,%%zmm1,%%k4\n\t"
370 "vpmovm2b %%k1,%%zmm5\n\t"
371 "vpmovm2b %%k2,%%zmm7\n\t"
372 "vpmovm2b %%k3,%%zmm13\n\t"
373 "vpmovm2b %%k4,%%zmm15\n\t"
374 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
375 "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
376 "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
377 "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
378 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
379 "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
380 "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
381 "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
382 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
383 "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
384 "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
385 "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
386 "vmovdqa64 %0,%%zmm5\n\t"
387 "vmovdqa64 %1,%%zmm7\n\t"
388 "vmovdqa64 %2,%%zmm13\n\t"
389 "vmovdqa64 %3,%%zmm15\n\t"
390 "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
391 "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
392 "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
393 "vpxorq %%zmm15,%%zmm11,%%zmm11\n"
394 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
395 "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
396 "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
397 "vpxorq %%zmm15,%%zmm14,%%zmm14"
398 :
399 : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
400 "m" (dptr[z][d+128]), "m" (dptr[z][d+192]));
401 }
402 asm volatile("vmovntdq %%zmm2,%0\n\t"
403 "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
404 "vmovntdq %%zmm3,%1\n\t"
405 "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"
406 "vmovntdq %%zmm10,%2\n\t"
407 "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"
408 "vmovntdq %%zmm11,%3\n\t"
409 "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"
410 "vmovntdq %%zmm4,%4\n\t"
411 "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"
412 "vmovntdq %%zmm6,%5\n\t"
413 "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"
414 "vmovntdq %%zmm12,%6\n\t"
415 "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"
416 "vmovntdq %%zmm14,%7\n\t"
417 "vpxorq %%zmm14,%%zmm14,%%zmm14"
418 :
419 : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
420 "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
421 "m" (q[d+128]), "m" (q[d+192]));
422 }
423
424 asm volatile("sfence" : : : "memory");
425 kernel_fpu_end();
426}
427
428static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
429 size_t bytes, void **ptrs)
430{
431 u8 **dptr = (u8 **)ptrs;
432 u8 *p, *q;
433 int d, z, z0;
434
435 z0 = stop; /* P/Q right side optimization */
436 p = dptr[disks-2]; /* XOR parity */
437 q = dptr[disks-1]; /* RS syndrome */
438
439 kernel_fpu_begin();
440
441 asm volatile("vmovdqa64 %0,%%zmm0"
442 :: "m" (raid6_avx512_constants.x1d[0]));
443
444 for (d = 0 ; d < bytes ; d += 256) {
445 asm volatile("vmovdqa64 %0,%%zmm4\n\t"
446 "vmovdqa64 %1,%%zmm6\n\t"
447 "vmovdqa64 %2,%%zmm12\n\t"
448 "vmovdqa64 %3,%%zmm14\n\t"
449 "vmovdqa64 %4,%%zmm2\n\t"
450 "vmovdqa64 %5,%%zmm3\n\t"
451 "vmovdqa64 %6,%%zmm10\n\t"
452 "vmovdqa64 %7,%%zmm11\n\t"
453 "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
454 "vpxorq %%zmm6,%%zmm3,%%zmm3\n\t"
455 "vpxorq %%zmm12,%%zmm10,%%zmm10\n\t"
456 "vpxorq %%zmm14,%%zmm11,%%zmm11"
457 :
458 : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
459 "m" (dptr[z0][d+128]), "m" (dptr[z0][d+192]),
460 "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
461 "m" (p[d+192]));
462 /* P/Q data pages */
463 for (z = z0-1 ; z >= start ; z--) {
464 asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
465 "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
466 "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
467 "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
468 "prefetchnta %0\n\t"
469 "prefetchnta %2\n\t"
470 "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
471 "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
472 "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
473 "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
474 "vpmovm2b %%k1,%%zmm5\n\t"
475 "vpmovm2b %%k2,%%zmm7\n\t"
476 "vpmovm2b %%k3,%%zmm13\n\t"
477 "vpmovm2b %%k4,%%zmm15\n\t"
478 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
479 "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
480 "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
481 "vpaddb %%Zmm14,%%zmm14,%%zmm14\n\t"
482 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
483 "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
484 "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
485 "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
486 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
487 "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
488 "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
489 "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
490 "vmovdqa64 %0,%%zmm5\n\t"
491 "vmovdqa64 %1,%%zmm7\n\t"
492 "vmovdqa64 %2,%%zmm13\n\t"
493 "vmovdqa64 %3,%%zmm15\n\t"
494 "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
495 "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
496 "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
497 "vpxorq %%zmm15,%%zmm11,%%zmm11\n\t"
498 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
499 "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
500 "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
501 "vpxorq %%zmm15,%%zmm14,%%zmm14"
502 :
503 : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
504 "m" (dptr[z][d+128]),
505 "m" (dptr[z][d+192]));
506 }
507 asm volatile("prefetchnta %0\n\t"
508 "prefetchnta %1\n\t"
509 :
510 : "m" (q[d]), "m" (q[d+128]));
511 /* P/Q left side optimization */
512 for (z = start-1 ; z >= 0 ; z--) {
513 asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
514 "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
515 "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
516 "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
517 "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
518 "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
519 "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
520 "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
521 "vpmovm2b %%k1,%%zmm5\n\t"
522 "vpmovm2b %%k2,%%zmm7\n\t"
523 "vpmovm2b %%k3,%%zmm13\n\t"
524 "vpmovm2b %%k4,%%zmm15\n\t"
525 "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
526 "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
527 "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
528 "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
529 "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
530 "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
531 "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
532 "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
533 "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
534 "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
535 "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
536 "vpxorq %%zmm15,%%zmm14,%%zmm14"
537 :
538 : );
539 }
540 asm volatile("vmovntdq %%zmm2,%0\n\t"
541 "vmovntdq %%zmm3,%1\n\t"
542 "vmovntdq %%zmm10,%2\n\t"
543 "vmovntdq %%zmm11,%3\n\t"
544 "vpxorq %4,%%zmm4,%%zmm4\n\t"
545 "vpxorq %5,%%zmm6,%%zmm6\n\t"
546 "vpxorq %6,%%zmm12,%%zmm12\n\t"
547 "vpxorq %7,%%zmm14,%%zmm14\n\t"
548 "vmovntdq %%zmm4,%4\n\t"
549 "vmovntdq %%zmm6,%5\n\t"
550 "vmovntdq %%zmm12,%6\n\t"
551 "vmovntdq %%zmm14,%7"
552 :
553 : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
554 "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
555 "m" (q[d+128]), "m" (q[d+192]));
556 }
557 asm volatile("sfence" : : : "memory");
558 kernel_fpu_end();
559}
560const struct raid6_calls raid6_avx512x4 = {
561 raid6_avx5124_gen_syndrome,
562 raid6_avx5124_xor_syndrome,
563 raid6_have_avx512,
564 "avx512x4",
565 1 /* Has cache hints */
566};
567#endif
568
569#endif /* CONFIG_AS_AVX512 */
lib/raid6/recov_avx512.c+388−0
@@ -0,0 +1,388 @@
1/*
2 * Copyright (C) 2016 Intel Corporation
3 *
4 * Author: Gayatri Kammela <gayatri.kammela@intel.com>
5 * Author: Megha Dey <megha.dey@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
10 * of the License.
11 *
12 */
13
14#ifdef CONFIG_AS_AVX512
15
16#include <linux/raid/pq.h>
17#include "x86.h"
18
19static int raid6_has_avx512(void)
20{
21 return boot_cpu_has(X86_FEATURE_AVX2) &&
22 boot_cpu_has(X86_FEATURE_AVX) &&
23 boot_cpu_has(X86_FEATURE_AVX512F) &&
24 boot_cpu_has(X86_FEATURE_AVX512BW) &&
25 boot_cpu_has(X86_FEATURE_AVX512VL) &&
26 boot_cpu_has(X86_FEATURE_AVX512DQ);
27}
28
29static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila,
30 int failb, void **ptrs)
31{
32 u8 *p, *q, *dp, *dq;
33 const u8 *pbmul; /* P multiplier table for B data */
34 const u8 *qmul; /* Q multiplier table (for both) */
35 const u8 x0f = 0x0f;
36
37 p = (u8 *)ptrs[disks-2];
38 q = (u8 *)ptrs[disks-1];
39
40 /*
41 * Compute syndrome with zero for the missing data pages
42 * Use the dead data pages as temporary storage for
43 * delta p and delta q
44 */
45
46 dp = (u8 *)ptrs[faila];
47 ptrs[faila] = (void *)raid6_empty_zero_page;
48 ptrs[disks-2] = dp;
49 dq = (u8 *)ptrs[failb];
50 ptrs[failb] = (void *)raid6_empty_zero_page;
51 ptrs[disks-1] = dq;
52
53 raid6_call.gen_syndrome(disks, bytes, ptrs);
54
55 /* Restore pointer table */
56 ptrs[faila] = dp;
57 ptrs[failb] = dq;
58 ptrs[disks-2] = p;
59 ptrs[disks-1] = q;
60
61 /* Now, pick the proper data tables */
62 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
63 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
64 raid6_gfexp[failb]]];
65
66 kernel_fpu_begin();
67
68 /* zmm0 = x0f[16] */
69 asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));
70
71 while (bytes) {
72#ifdef CONFIG_X86_64
73 asm volatile("vmovdqa64 %0, %%zmm1\n\t"
74 "vmovdqa64 %1, %%zmm9\n\t"
75 "vmovdqa64 %2, %%zmm0\n\t"
76 "vmovdqa64 %3, %%zmm8\n\t"
77 "vpxorq %4, %%zmm1, %%zmm1\n\t"
78 "vpxorq %5, %%zmm9, %%zmm9\n\t"
79 "vpxorq %6, %%zmm0, %%zmm0\n\t"
80 "vpxorq %7, %%zmm8, %%zmm8"
81 :
82 : "m" (q[0]), "m" (q[64]), "m" (p[0]),
83 "m" (p[64]), "m" (dq[0]), "m" (dq[64]),
84 "m" (dp[0]), "m" (dp[64]));
85
86 /*
87 * 1 = dq[0] ^ q[0]
88 * 9 = dq[64] ^ q[64]
89 * 0 = dp[0] ^ p[0]
90 * 8 = dp[64] ^ p[64]
91 */
92
93 asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
94 "vbroadcasti64x2 %1, %%zmm5"
95 :
96 : "m" (qmul[0]), "m" (qmul[16]));
97
98 asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t"
99 "vpsraw $4, %%zmm9, %%zmm12\n\t"
100 "vpandq %%zmm7, %%zmm1, %%zmm1\n\t"
101 "vpandq %%zmm7, %%zmm9, %%zmm9\n\t"
102 "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
103 "vpandq %%zmm7, %%zmm12, %%zmm12\n\t"
104 "vpshufb %%zmm9, %%zmm4, %%zmm14\n\t"
105 "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t"
106 "vpshufb %%zmm12, %%zmm5, %%zmm15\n\t"
107 "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t"
108 "vpxorq %%zmm14, %%zmm15, %%zmm15\n\t"
109 "vpxorq %%zmm4, %%zmm5, %%zmm5"
110 :
111 : );
112
113 /*
114 * 5 = qx[0]
115 * 15 = qx[64]
116 */
117
118 asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
119 "vbroadcasti64x2 %1, %%zmm1\n\t"
120 "vpsraw $4, %%zmm0, %%zmm2\n\t"
121 "vpsraw $4, %%zmm8, %%zmm6\n\t"
122 "vpandq %%zmm7, %%zmm0, %%zmm3\n\t"
123 "vpandq %%zmm7, %%zmm8, %%zmm14\n\t"
124 "vpandq %%zmm7, %%zmm2, %%zmm2\n\t"
125 "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
126 "vpshufb %%zmm14, %%zmm4, %%zmm12\n\t"
127 "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t"
128 "vpshufb %%zmm6, %%zmm1, %%zmm13\n\t"
129 "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t"
130 "vpxorq %%zmm4, %%zmm1, %%zmm1\n\t"
131 "vpxorq %%zmm12, %%zmm13, %%zmm13"
132 :
133 : "m" (pbmul[0]), "m" (pbmul[16]));
134
135 /*
136 * 1 = pbmul[px[0]]
137 * 13 = pbmul[px[64]]
138 */
139 asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t"
140 "vpxorq %%zmm15, %%zmm13, %%zmm13"
141 :
142 : );
143
144 /*
145 * 1 = db = DQ
146 * 13 = db[64] = DQ[64]
147 */
148 asm volatile("vmovdqa64 %%zmm1, %0\n\t"
149 "vmovdqa64 %%zmm13,%1\n\t"
150 "vpxorq %%zmm1, %%zmm0, %%zmm0\n\t"
151 "vpxorq %%zmm13, %%zmm8, %%zmm8"
152 :
153 : "m" (dq[0]), "m" (dq[64]));
154
155 asm volatile("vmovdqa64 %%zmm0, %0\n\t"
156 "vmovdqa64 %%zmm8, %1"
157 :
158 : "m" (dp[0]), "m" (dp[64]));
159
160 bytes -= 128;
161 p += 128;
162 q += 128;
163 dp += 128;
164 dq += 128;
165#else
166 asm volatile("vmovdqa64 %0, %%zmm1\n\t"
167 "vmovdqa64 %1, %%zmm0\n\t"
168 "vpxorq %2, %%zmm1, %%zmm1\n\t"
169 "vpxorq %3, %%zmm0, %%zmm0"
170 :
171 : "m" (*q), "m" (*p), "m"(*dq), "m" (*dp));
172
173 /* 1 = dq ^ q; 0 = dp ^ p */
174
175 asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
176 "vbroadcasti64x2 %1, %%zmm5"
177 :
178 : "m" (qmul[0]), "m" (qmul[16]));
179
180 /*
181 * 1 = dq ^ q
182 * 3 = dq ^ p >> 4
183 */
184 asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t"
185 "vpandq %%zmm7, %%zmm1, %%zmm1\n\t"
186 "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
187 "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t"
188 "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t"
189 "vpxorq %%zmm4, %%zmm5, %%zmm5"
190 :
191 : );
192
193 /* 5 = qx */
194
195 asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
196 "vbroadcasti64x2 %1, %%zmm1"
197 :
198 : "m" (pbmul[0]), "m" (pbmul[16]));
199
200 asm volatile("vpsraw $4, %%zmm0, %%zmm2\n\t"
201 "vpandq %%zmm7, %%zmm0, %%zmm3\n\t"
202 "vpandq %%zmm7, %%zmm2, %%zmm2\n\t"
203 "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t"
204 "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t"
205 "vpxorq %%zmm4, %%zmm1, %%zmm1"
206 :
207 : );
208
209 /* 1 = pbmul[px] */
210 asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t"
211 /* 1 = db = DQ */
212 "vmovdqa64 %%zmm1, %0\n\t"
213 :
214 : "m" (dq[0]));
215
216 asm volatile("vpxorq %%zmm1, %%zmm0, %%zmm0\n\t"
217 "vmovdqa64 %%zmm0, %0"
218 :
219 : "m" (dp[0]));
220
221 bytes -= 64;
222 p += 64;
223 q += 64;
224 dp += 64;
225 dq += 64;
226#endif
227 }
228
229 kernel_fpu_end();
230}
231
232static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,
233 void **ptrs)
234{
235 u8 *p, *q, *dq;
236 const u8 *qmul; /* Q multiplier table */
237 const u8 x0f = 0x0f;
238
239 p = (u8 *)ptrs[disks-2];
240 q = (u8 *)ptrs[disks-1];
241
242 /*
243 * Compute syndrome with zero for the missing data page
244 * Use the dead data page as temporary storage for delta q
245 */
246
247 dq = (u8 *)ptrs[faila];
248 ptrs[faila] = (void *)raid6_empty_zero_page;
249 ptrs[disks-1] = dq;
250
251 raid6_call.gen_syndrome(disks, bytes, ptrs);
252
253 /* Restore pointer table */
254 ptrs[faila] = dq;
255 ptrs[disks-1] = q;
256
257 /* Now, pick the proper data tables */
258 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
259
260 kernel_fpu_begin();
261
262 asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));
263
264 while (bytes) {
265#ifdef CONFIG_X86_64
266 asm volatile("vmovdqa64 %0, %%zmm3\n\t"
267 "vmovdqa64 %1, %%zmm8\n\t"
268 "vpxorq %2, %%zmm3, %%zmm3\n\t"
269 "vpxorq %3, %%zmm8, %%zmm8"
270 :
271 : "m" (dq[0]), "m" (dq[64]), "m" (q[0]),
272 "m" (q[64]));
273
274 /*
275 * 3 = q[0] ^ dq[0]
276 * 8 = q[64] ^ dq[64]
277 */
278 asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"
279 "vmovapd %%zmm0, %%zmm13\n\t"
280 "vbroadcasti64x2 %1, %%zmm1\n\t"
281 "vmovapd %%zmm1, %%zmm14"
282 :
283 : "m" (qmul[0]), "m" (qmul[16]));
284
285 asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t"
286 "vpsraw $4, %%zmm8, %%zmm12\n\t"
287 "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
288 "vpandq %%zmm7, %%zmm8, %%zmm8\n\t"
289 "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
290 "vpandq %%zmm7, %%zmm12, %%zmm12\n\t"
291 "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t"
292 "vpshufb %%zmm8, %%zmm13, %%zmm13\n\t"
293 "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t"
294 "vpshufb %%zmm12, %%zmm14, %%zmm14\n\t"
295 "vpxorq %%zmm0, %%zmm1, %%zmm1\n\t"
296 "vpxorq %%zmm13, %%zmm14, %%zmm14"
297 :
298 : );
299
300 /*
301 * 1 = qmul[q[0] ^ dq[0]]
302 * 14 = qmul[q[64] ^ dq[64]]
303 */
304 asm volatile("vmovdqa64 %0, %%zmm2\n\t"
305 "vmovdqa64 %1, %%zmm12\n\t"
306 "vpxorq %%zmm1, %%zmm2, %%zmm2\n\t"
307 "vpxorq %%zmm14, %%zmm12, %%zmm12"
308 :
309 : "m" (p[0]), "m" (p[64]));
310
311 /*
312 * 2 = p[0] ^ qmul[q[0] ^ dq[0]]
313 * 12 = p[64] ^ qmul[q[64] ^ dq[64]]
314 */
315
316 asm volatile("vmovdqa64 %%zmm1, %0\n\t"
317 "vmovdqa64 %%zmm14, %1\n\t"
318 "vmovdqa64 %%zmm2, %2\n\t"
319 "vmovdqa64 %%zmm12,%3"
320 :
321 : "m" (dq[0]), "m" (dq[64]), "m" (p[0]),
322 "m" (p[64]));
323
324 bytes -= 128;
325 p += 128;
326 q += 128;
327 dq += 128;
328#else
329 asm volatile("vmovdqa64 %0, %%zmm3\n\t"
330 "vpxorq %1, %%zmm3, %%zmm3"
331 :
332 : "m" (dq[0]), "m" (q[0]));
333
334 /* 3 = q ^ dq */
335
336 asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"
337 "vbroadcasti64x2 %1, %%zmm1"
338 :
339 : "m" (qmul[0]), "m" (qmul[16]));
340
341 asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t"
342 "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
343 "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
344 "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t"
345 "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t"
346 "vpxorq %%zmm0, %%zmm1, %%zmm1"
347 :
348 : );
349
350 /* 1 = qmul[q ^ dq] */
351
352 asm volatile("vmovdqa64 %0, %%zmm2\n\t"
353 "vpxorq %%zmm1, %%zmm2, %%zmm2"
354 :
355 : "m" (p[0]));
356
357 /* 2 = p ^ qmul[q ^ dq] */
358
359 asm volatile("vmovdqa64 %%zmm1, %0\n\t"
360 "vmovdqa64 %%zmm2, %1"
361 :
362 : "m" (dq[0]), "m" (p[0]));
363
364 bytes -= 64;
365 p += 64;
366 q += 64;
367 dq += 64;
368#endif
369 }
370
371 kernel_fpu_end();
372}
373
374const struct raid6_recov_calls raid6_recov_avx512 = {
375 .data2 = raid6_2data_recov_avx512,
376 .datap = raid6_datap_recov_avx512,
377 .valid = raid6_has_avx512,
378#ifdef CONFIG_X86_64
379 .name = "avx512x2",
380#else
381 .name = "avx512x1",
382#endif
383 .priority = 3,
384};
385
386#else
387#warning "your version of binutils lacks AVX512 support"
388#endif
lib/raid6/test/Makefile+4−1
@@ -32,10 +32,13 @@ ifeq ($(ARCH),arm64) · 31 unmodified lines
32endif32endif
33 33
34ifeq ($(IS_X86),yes)34ifeq ($(IS_X86),yes)
35 OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o35 OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o
36 CFLAGS += $(shell echo "vpbroadcastb %xmm0, %ymm1" | \36 CFLAGS += $(shell echo "vpbroadcastb %xmm0, %ymm1" | \
37 gcc -c -x assembler - >&/dev/null && \37 gcc -c -x assembler - >&/dev/null && \
38 rm ./-.o && echo -DCONFIG_AS_AVX2=1)38 rm ./-.o && echo -DCONFIG_AS_AVX2=1)
39 CFLAGS += $(shell echo "vpmovm2b %k1, %zmm5" | \
40 gcc -c -x assembler - >&/dev/null && \
41 rm ./-.o && echo -DCONFIG_AS_AVX512=1)
39else ifeq ($(HAS_NEON),yes)42else ifeq ($(HAS_NEON),yes)
40 OBJS += neon.o neon1.o neon2.o neon4.o neon8.o43 OBJS += neon.o neon1.o neon2.o neon4.o neon8.o
41 CFLAGS += -DCONFIG_KERNEL_MODE_NEON=144 CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1
lib/raid6/test/test.c+4−3
@@ -21,12 +21,13 @@ · 20 unmodified lines
21 21
22#define NDISKS 16 /* Including P and Q */22#define NDISKS 16 /* Including P and Q */
23 23
24const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256)));24const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
25struct raid6_calls raid6_call;25struct raid6_calls raid6_call;
26 26
27char *dataptrs[NDISKS];27char *dataptrs[NDISKS];
28char data[NDISKS][PAGE_SIZE];28char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
29char recovi[PAGE_SIZE], recovj[PAGE_SIZE];29char recovi[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
30char recovj[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
30 31
31static void makedata(int start, int stop)32static void makedata(int start, int stop)
32{33{
lib/raid6/x86.h+10−0
@@ -46,6 +46,16 @@ static inline void kernel_fpu_end(void) · 45 unmodified lines
46#define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */46#define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */
47#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */47#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */
48#define X86_FEATURE_AVX2 (9*32+ 5) /* AVX2 instructions */48#define X86_FEATURE_AVX2 (9*32+ 5) /* AVX2 instructions */
49#define X86_FEATURE_AVX512F (9*32+16) /* AVX-512 Foundation */
50#define X86_FEATURE_AVX512DQ (9*32+17) /* AVX-512 DQ (Double/Quad granular)
51 * Instructions
52 */
53#define X86_FEATURE_AVX512BW (9*32+30) /* AVX-512 BW (Byte/Word granular)
54 * Instructions
55 */
56#define X86_FEATURE_AVX512VL (9*32+31) /* AVX-512 VL (128/256 Vector Length)
57 * Extensions
58 */
49#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */59#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */
50 60
51/* Should work well enough on modern CPUs for testing */61/* Should work well enough on modern CPUs for testing */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t