Pull MD updates from Shaohua Li: "This update includes: - new AVX512 instruction based raid6 gen/recovery algorithm - a couple of md-cluster related bug fixes - fix a potential deadlock - set nonrotational bit for raid array with SSD - set correct max_hw_sectors for raid5/6, which hopefuly can improve performance a little bit - other minor fixes" * tag 'md/4.9-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shli/md: md: set rotational bit raid6/test/test.c: bug fix: Specify aligned(alignment) attributes to the char arrays raid5: handle register_shrinker failure raid5: fix to detect failure of register_shrinker md: fix a potential deadlock md/bitmap: fix wrong cleanup raid5: allow arbitrary max_hw_sectors lib/raid6: Add AVX512 optimized xor_syndrome functions lib/raid6/test/Makefile: Add avx512 gen_syndrome and recovery functions lib/raid6: Add AVX512 optimized recovery functions lib/raid6: Add AVX512 optimized gen_syndrome functions md-cluster: make resync lock also could be interruptted md-cluster: introduce dlm_lock_sync_interruptible to fix tasks hang md-cluster: convert the completion to wait queue md-cluster: protect md_find_rdev_nr_rcu with rcu lock md-cluster: clean related infos of cluster md: changes for MD_STILL_CLOSED flag md-cluster: remove some unnecessary dlm_unlock_sync md-cluster: use FORCEUNLOCK in lockres_free md-cluster: call md_kick_rdev_from_array once ack failed
Unified · Split
| @@ -163,11 +163,12 @@ asinstr += $(call as-instr,pshufb %xmm0$(comma)%xmm0,-DCONFIG_AS_SSSE3=1) · 162 unmodified lines | ||
| 163 | 163 | asinstr += $(call as-instr,crc32l %eax$(comma)%eax,-DCONFIG_AS_CRC32=1) |
| 164 | 164 | avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1) |
| 165 | 165 | avx2_instr :=$(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1) |
| 166 | +avx512_instr :=$(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1) | |
| 166 | 167 | sha1_ni_instr :=$(call as-instr,sha1msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA1_NI=1) |
| 167 | 168 | sha256_ni_instr :=$(call as-instr,sha256msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA256_NI=1) |
| 168 | 169 | |
| 169 | −KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(sha1_ni_instr) $(sha256_ni_instr) | |
| 170 | −KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(sha1_ni_instr) $(sha256_ni_instr) | |
| 170 | +KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) | |
| 171 | +KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) | |
| 171 | 172 | |
| 172 | 173 | LDFLAGS := -m elf_$(UTS_MACHINE) |
| 173 | 174 | |
| @@ -1903,10 +1903,8 @@ int bitmap_copy_from_slot(struct mddev *mddev, int slot, · 1902 unmodified lines | ||
| 1903 | 1903 | struct bitmap_counts *counts; |
| 1904 | 1904 | struct bitmap *bitmap = bitmap_create(mddev, slot); |
| 1905 | 1905 | |
| 1906 | − if (IS_ERR(bitmap)) { | |
| 1907 | − bitmap_free(bitmap); | |
| 1906 | + if (IS_ERR(bitmap)) | |
| 1908 | 1907 | return PTR_ERR(bitmap); |
| 1909 | − } | |
| 1910 | 1908 | |
| 1911 | 1909 | rv = bitmap_init_from_disk(bitmap, 0); |
| 1912 | 1910 | if (rv) |
| @@ -10,6 +10,7 @@ · 9 unmodified lines | ||
| 10 | 10 | |
| 11 | 11 | |
| 12 | 12 | #include <linux/module.h> |
| 13 | +#include <linux/kthread.h> | |
| 13 | 14 | #include <linux/dlm.h> |
| 14 | 15 | #include <linux/sched.h> |
| 15 | 16 | #include <linux/raid/md_p.h> |
| @@ -25,7 +26,8 @@ struct dlm_lock_resource { · 9 unmodified lines | ||
| 25 | 26 | struct dlm_lksb lksb; |
| 26 | 27 | char *name; /* lock name. */ |
| 27 | 28 | uint32_t flags; /* flags to pass to dlm_lock() */ |
| 28 | − struct completion completion; /* completion for synchronized locking */ | |
| 29 | + wait_queue_head_t sync_locking; /* wait queue for synchronized locking */ | |
| 30 | + bool sync_locking_done; | |
| 29 | 31 | void (*bast)(void *arg, int mode); /* blocking AST function pointer*/ |
| 30 | 32 | struct mddev *mddev; /* pointing back to mddev. */ |
| 31 | 33 | int mode; |
| @@ -118,7 +120,8 @@ static void sync_ast(void *arg) · 86 unmodified lines | ||
| 118 | 120 | struct dlm_lock_resource *res; |
| 119 | 121 | |
| 120 | 122 | res = arg; |
| 121 | − complete(&res->completion); | |
| 123 | + res->sync_locking_done = true; | |
| 124 | + wake_up(&res->sync_locking); | |
| 122 | 125 | } |
| 123 | 126 | |
| 124 | 127 | static int dlm_lock_sync(struct dlm_lock_resource *res, int mode) |
| @@ -130,7 +133,8 @@ static int dlm_lock_sync(struct dlm_lock_resource *res, int mode) · 5 unmodified lines | ||
| 130 | 133 | 0, sync_ast, res, res->bast); |
| 131 | 134 | if (ret) |
| 132 | 135 | return ret; |
| 133 | − wait_for_completion(&res->completion); | |
| 136 | + wait_event(res->sync_locking, res->sync_locking_done); | |
| 137 | + res->sync_locking_done = false; | |
| 134 | 138 | if (res->lksb.sb_status == 0) |
| 135 | 139 | res->mode = mode; |
| 136 | 140 | return res->lksb.sb_status; |
| @@ -141,6 +145,44 @@ static int dlm_unlock_sync(struct dlm_lock_resource *res) · 4 unmodified lines | ||
| 141 | 145 | return dlm_lock_sync(res, DLM_LOCK_NL); |
| 142 | 146 | } |
| 143 | 147 | |
| 148 | +/* | |
| 149 | + * An variation of dlm_lock_sync, which make lock request could | |
| 150 | + * be interrupted | |
| 151 | + */ | |
| 152 | +static int dlm_lock_sync_interruptible(struct dlm_lock_resource *res, int mode, | |
| 153 | + struct mddev *mddev) | |
| 154 | +{ | |
| 155 | + int ret = 0; | |
| 156 | + | |
| 157 | + ret = dlm_lock(res->ls, mode, &res->lksb, | |
| 158 | + res->flags, res->name, strlen(res->name), | |
| 159 | + 0, sync_ast, res, res->bast); | |
| 160 | + if (ret) | |
| 161 | + return ret; | |
| 162 | + | |
| 163 | + wait_event(res->sync_locking, res->sync_locking_done | |
| 164 | + || kthread_should_stop() | |
| 165 | + || test_bit(MD_CLOSING, &mddev->flags)); | |
| 166 | + if (!res->sync_locking_done) { | |
| 167 | + /* | |
| 168 | + * the convert queue contains the lock request when request is | |
| 169 | + * interrupted, and sync_ast could still be run, so need to | |
| 170 | + * cancel the request and reset completion | |
| 171 | + */ | |
| 172 | + ret = dlm_unlock(res->ls, res->lksb.sb_lkid, DLM_LKF_CANCEL, | |
| 173 | + &res->lksb, res); | |
| 174 | + res->sync_locking_done = false; | |
| 175 | + if (unlikely(ret != 0)) | |
| 176 | + pr_info("failed to cancel previous lock request " | |
| 177 | + "%s return %d\n", res->name, ret); | |
| 178 | + return -EPERM; | |
| 179 | + } else | |
| 180 | + res->sync_locking_done = false; | |
| 181 | + if (res->lksb.sb_status == 0) | |
| 182 | + res->mode = mode; | |
| 183 | + return res->lksb.sb_status; | |
| 184 | +} | |
| 185 | + | |
| 144 | 186 | static struct dlm_lock_resource *lockres_init(struct mddev *mddev, |
| 145 | 187 | char *name, void (*bastfn)(void *arg, int mode), int with_lvb) |
| 146 | 188 | { |
| @@ -151,7 +193,8 @@ static struct dlm_lock_resource *lockres_init(struct mddev *mddev, · 4 unmodified lines | ||
| 151 | 193 | res = kzalloc(sizeof(struct dlm_lock_resource), GFP_KERNEL); |
| 152 | 194 | if (!res) |
| 153 | 195 | return NULL; |
| 154 | − init_completion(&res->completion); | |
| 196 | + init_waitqueue_head(&res->sync_locking); | |
| 197 | + res->sync_locking_done = false; | |
| 155 | 198 | res->ls = cinfo->lockspace; |
| 156 | 199 | res->mddev = mddev; |
| 157 | 200 | res->mode = DLM_LOCK_IV; |
| @@ -194,25 +237,21 @@ out_err: · 36 unmodified lines | ||
| 194 | 237 | |
| 195 | 238 | static void lockres_free(struct dlm_lock_resource *res) |
| 196 | 239 | { |
| 197 | − int ret; | |
| 240 | + int ret = 0; | |
| 198 | 241 | |
| 199 | 242 | if (!res) |
| 200 | 243 | return; |
| 201 | 244 | |
| 202 | − /* cancel a lock request or a conversion request that is blocked */ | |
| 203 | − res->flags |= DLM_LKF_CANCEL; | |
| 204 | −retry: | |
| 205 | − ret = dlm_unlock(res->ls, res->lksb.sb_lkid, 0, &res->lksb, res); | |
| 206 | − if (unlikely(ret != 0)) { | |
| 207 | − pr_info("%s: failed to unlock %s return %d\n", __func__, res->name, ret); | |
| 208 | − | |
| 209 | − /* if a lock conversion is cancelled, then the lock is put | |
| 210 | − * back to grant queue, need to ensure it is unlocked */ | |
| 211 | − if (ret == -DLM_ECANCEL) | |
| 212 | − goto retry; | |
| 213 | − } | |
| 214 | − res->flags &= ~DLM_LKF_CANCEL; | |
| 215 | − wait_for_completion(&res->completion); | |
| 245 | + /* | |
| 246 | + * use FORCEUNLOCK flag, so we can unlock even the lock is on the | |
| 247 | + * waiting or convert queue | |
| 248 | + */ | |
| 249 | + ret = dlm_unlock(res->ls, res->lksb.sb_lkid, DLM_LKF_FORCEUNLOCK, | |
| 250 | + &res->lksb, res); | |
| 251 | + if (unlikely(ret != 0)) | |
| 252 | + pr_err("failed to unlock %s return %d\n", res->name, ret); | |
| 253 | + else | |
| 254 | + wait_event(res->sync_locking, res->sync_locking_done); | |
| 216 | 255 | |
| 217 | 256 | kfree(res->name); |
| 218 | 257 | kfree(res->lksb.sb_lvbptr); |
| @@ -279,7 +318,7 @@ static void recover_bitmaps(struct md_thread *thread) · 60 unmodified lines | ||
| 279 | 318 | goto clear_bit; |
| 280 | 319 | } |
| 281 | 320 | |
| 282 | − ret = dlm_lock_sync(bm_lockres, DLM_LOCK_PW); | |
| 321 | + ret = dlm_lock_sync_interruptible(bm_lockres, DLM_LOCK_PW, mddev); | |
| 283 | 322 | if (ret) { |
| 284 | 323 | pr_err("md-cluster: Could not DLM lock %s: %d\n", |
| 285 | 324 | str, ret); |
| @@ -288,7 +327,7 @@ static void recover_bitmaps(struct md_thread *thread) · 2 unmodified lines | ||
| 288 | 327 | ret = bitmap_copy_from_slot(mddev, slot, &lo, &hi, true); |
| 289 | 328 | if (ret) { |
| 290 | 329 | pr_err("md-cluster: Could not copy data from bitmap %d\n", slot); |
| 291 | − goto dlm_unlock; | |
| 330 | + goto clear_bit; | |
| 292 | 331 | } |
| 293 | 332 | if (hi > 0) { |
| 294 | 333 | if (lo < mddev->recovery_cp) |
| @@ -300,8 +339,6 @@ static void recover_bitmaps(struct md_thread *thread) · 5 unmodified lines | ||
| 300 | 339 | md_wakeup_thread(mddev->thread); |
| 301 | 340 | } |
| 302 | 341 | } |
| 303 | −dlm_unlock: | |
| 304 | − dlm_unlock_sync(bm_lockres); | |
| 305 | 342 | clear_bit: |
| 306 | 343 | lockres_free(bm_lockres); |
| 307 | 344 | clear_bit(slot, &cinfo->recovery_map); |
| @@ -495,9 +532,10 @@ static void process_metadata_update(struct mddev *mddev, struct cluster_msg *msg · 187 unmodified lines | ||
| 495 | 532 | |
| 496 | 533 | static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg) |
| 497 | 534 | { |
| 498 | − struct md_rdev *rdev = md_find_rdev_nr_rcu(mddev, | |
| 499 | − le32_to_cpu(msg->raid_slot)); | |
| 535 | + struct md_rdev *rdev; | |
| 500 | 536 | |
| 537 | + rcu_read_lock(); | |
| 538 | + rdev = md_find_rdev_nr_rcu(mddev, le32_to_cpu(msg->raid_slot)); | |
| 501 | 539 | if (rdev) { |
| 502 | 540 | set_bit(ClusterRemove, &rdev->flags); |
| 503 | 541 | set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); |
| @@ -506,18 +544,21 @@ static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg) · 2 unmodified lines | ||
| 506 | 544 | else |
| 507 | 545 | pr_warn("%s: %d Could not find disk(%d) to REMOVE\n", |
| 508 | 546 | __func__, __LINE__, le32_to_cpu(msg->raid_slot)); |
| 547 | + rcu_read_unlock(); | |
| 509 | 548 | } |
| 510 | 549 | |
| 511 | 550 | static void process_readd_disk(struct mddev *mddev, struct cluster_msg *msg) |
| 512 | 551 | { |
| 513 | − struct md_rdev *rdev = md_find_rdev_nr_rcu(mddev, | |
| 514 | − le32_to_cpu(msg->raid_slot)); | |
| 552 | + struct md_rdev *rdev; | |
| 515 | 553 | |
| 554 | + rcu_read_lock(); | |
| 555 | + rdev = md_find_rdev_nr_rcu(mddev, le32_to_cpu(msg->raid_slot)); | |
| 516 | 556 | if (rdev && test_bit(Faulty, &rdev->flags)) |
| 517 | 557 | clear_bit(Faulty, &rdev->flags); |
| 518 | 558 | else |
| 519 | 559 | pr_warn("%s: %d Could not find disk(%d) which is faulty", |
| 520 | 560 | __func__, __LINE__, le32_to_cpu(msg->raid_slot)); |
| 561 | + rcu_read_unlock(); | |
| 521 | 562 | } |
| 522 | 563 | |
| 523 | 564 | static int process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg) |
| @@ -770,7 +811,6 @@ static int gather_all_resync_info(struct mddev *mddev, int total_slots) · 246 unmodified lines | ||
| 770 | 811 | md_check_recovery(mddev); |
| 771 | 812 | } |
| 772 | 813 | |
| 773 | − dlm_unlock_sync(bm_lockres); | |
| 774 | 814 | lockres_free(bm_lockres); |
| 775 | 815 | } |
| 776 | 816 | out: |
| @@ -1006,7 +1046,7 @@ static void metadata_update_cancel(struct mddev *mddev) · 229 unmodified lines | ||
| 1006 | 1046 | static int resync_start(struct mddev *mddev) |
| 1007 | 1047 | { |
| 1008 | 1048 | struct md_cluster_info *cinfo = mddev->cluster_info; |
| 1009 | − return dlm_lock_sync(cinfo->resync_lockres, DLM_LOCK_EX); | |
| 1049 | + return dlm_lock_sync_interruptible(cinfo->resync_lockres, DLM_LOCK_EX, mddev); | |
| 1010 | 1050 | } |
| 1011 | 1051 | |
| 1012 | 1052 | static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi) |
| @@ -1186,7 +1226,6 @@ static void unlock_all_bitmaps(struct mddev *mddev) · 173 unmodified lines | ||
| 1186 | 1226 | if (cinfo->other_bitmap_lockres) { |
| 1187 | 1227 | for (i = 0; i < mddev->bitmap_info.nodes - 1; i++) { |
| 1188 | 1228 | if (cinfo->other_bitmap_lockres[i]) { |
| 1189 | − dlm_unlock_sync(cinfo->other_bitmap_lockres[i]); | |
| 1190 | 1229 | lockres_free(cinfo->other_bitmap_lockres[i]); |
| 1191 | 1230 | } |
| 1192 | 1231 | } |
| @@ -5297,6 +5297,21 @@ int md_run(struct mddev *mddev) · 5296 unmodified lines | ||
| 5297 | 5297 | return err; |
| 5298 | 5298 | } |
| 5299 | 5299 | if (mddev->queue) { |
| 5300 | + bool nonrot = true; | |
| 5301 | + | |
| 5302 | + rdev_for_each(rdev, mddev) { | |
| 5303 | + if (rdev->raid_disk >= 0 && | |
| 5304 | + !blk_queue_nonrot(bdev_get_queue(rdev->bdev))) { | |
| 5305 | + nonrot = false; | |
| 5306 | + break; | |
| 5307 | + } | |
| 5308 | + } | |
| 5309 | + if (mddev->degraded) | |
| 5310 | + nonrot = false; | |
| 5311 | + if (nonrot) | |
| 5312 | + queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mddev->queue); | |
| 5313 | + else | |
| 5314 | + queue_flag_clear_unlocked(QUEUE_FLAG_NONROT, mddev->queue); | |
| 5300 | 5315 | mddev->queue->backing_dev_info.congested_data = mddev; |
| 5301 | 5316 | mddev->queue->backing_dev_info.congested_fn = md_congested; |
| 5302 | 5317 | } |
| @@ -5454,12 +5469,14 @@ static void md_clean(struct mddev *mddev) · 151 unmodified lines | ||
| 5454 | 5469 | mddev->degraded = 0; |
| 5455 | 5470 | mddev->safemode = 0; |
| 5456 | 5471 | mddev->private = NULL; |
| 5472 | + mddev->cluster_info = NULL; | |
| 5457 | 5473 | mddev->bitmap_info.offset = 0; |
| 5458 | 5474 | mddev->bitmap_info.default_offset = 0; |
| 5459 | 5475 | mddev->bitmap_info.default_space = 0; |
| 5460 | 5476 | mddev->bitmap_info.chunksize = 0; |
| 5461 | 5477 | mddev->bitmap_info.daemon_sleep = 0; |
| 5462 | 5478 | mddev->bitmap_info.max_write_behind = 0; |
| 5479 | + mddev->bitmap_info.nodes = 0; | |
| 5463 | 5480 | } |
| 5464 | 5481 | |
| 5465 | 5482 | static void __md_stop_writes(struct mddev *mddev) |
| @@ -5573,8 +5590,7 @@ static int md_set_readonly(struct mddev *mddev, struct block_device *bdev) · 107 unmodified lines | ||
| 5573 | 5590 | mutex_lock(&mddev->open_mutex); |
| 5574 | 5591 | if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) || |
| 5575 | 5592 | mddev->sync_thread || |
| 5576 | − test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) || | |
| 5577 | − (bdev && !test_bit(MD_STILL_CLOSED, &mddev->flags))) { | |
| 5593 | + test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) { | |
| 5578 | 5594 | printk("md: %s still in use.\n",mdname(mddev)); |
| 5579 | 5595 | if (did_freeze) { |
| 5580 | 5596 | clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery); |
| @@ -5636,8 +5652,7 @@ static int do_md_stop(struct mddev *mddev, int mode, · 55 unmodified lines | ||
| 5636 | 5652 | if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) || |
| 5637 | 5653 | mddev->sysfs_active || |
| 5638 | 5654 | mddev->sync_thread || |
| 5639 | − test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) || | |
| 5640 | − (bdev && !test_bit(MD_STILL_CLOSED, &mddev->flags))) { | |
| 5655 | + test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) { | |
| 5641 | 5656 | printk("md: %s still in use.\n",mdname(mddev)); |
| 5642 | 5657 | mutex_unlock(&mddev->open_mutex); |
| 5643 | 5658 | if (did_freeze) { |
| @@ -6101,9 +6116,14 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info) · 457 unmodified lines | ||
| 6101 | 6116 | export_rdev(rdev); |
| 6102 | 6117 | |
| 6103 | 6118 | if (mddev_is_clustered(mddev)) { |
| 6104 | − if (info->state & (1 << MD_DISK_CANDIDATE)) | |
| 6105 | − md_cluster_ops->new_disk_ack(mddev, (err == 0)); | |
| 6106 | − else { | |
| 6119 | + if (info->state & (1 << MD_DISK_CANDIDATE)) { | |
| 6120 | + if (!err) { | |
| 6121 | + err = md_cluster_ops->new_disk_ack(mddev, | |
| 6122 | + err == 0); | |
| 6123 | + if (err) | |
| 6124 | + md_kick_rdev_from_array(rdev); | |
| 6125 | + } | |
| 6126 | + } else { | |
| 6107 | 6127 | if (err) |
| 6108 | 6128 | md_cluster_ops->add_new_disk_cancel(mddev); |
| 6109 | 6129 | else |
| @@ -6821,7 +6841,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, · 711 unmodified lines | ||
| 6821 | 6841 | err = -EBUSY; |
| 6822 | 6842 | goto out; |
| 6823 | 6843 | } |
| 6824 | − set_bit(MD_STILL_CLOSED, &mddev->flags); | |
| 6844 | + set_bit(MD_CLOSING, &mddev->flags); | |
| 6825 | 6845 | mutex_unlock(&mddev->open_mutex); |
| 6826 | 6846 | sync_blockdev(bdev); |
| 6827 | 6847 | } |
| @@ -7070,9 +7090,13 @@ static int md_open(struct block_device *bdev, fmode_t mode) · 242 unmodified lines | ||
| 7070 | 7090 | if ((err = mutex_lock_interruptible(&mddev->open_mutex))) |
| 7071 | 7091 | goto out; |
| 7072 | 7092 | |
| 7093 | + if (test_bit(MD_CLOSING, &mddev->flags)) { | |
| 7094 | + mutex_unlock(&mddev->open_mutex); | |
| 7095 | + return -ENODEV; | |
| 7096 | + } | |
| 7097 | + | |
| 7073 | 7098 | err = 0; |
| 7074 | 7099 | atomic_inc(&mddev->openers); |
| 7075 | − clear_bit(MD_STILL_CLOSED, &mddev->flags); | |
| 7076 | 7100 | mutex_unlock(&mddev->open_mutex); |
| 7077 | 7101 | |
| 7078 | 7102 | check_disk_change(bdev); |
| @@ -8873,7 +8897,9 @@ static void autostart_arrays(int part) · 1794 unmodified lines | ||
| 8873 | 8897 | list_del(&node_detected_dev->list); |
| 8874 | 8898 | dev = node_detected_dev->dev; |
| 8875 | 8899 | kfree(node_detected_dev); |
| 8900 | + mutex_unlock(&detected_devices_mutex); | |
| 8876 | 8901 | rdev = md_import_device(dev,0, 90); |
| 8902 | + mutex_lock(&detected_devices_mutex); | |
| 8877 | 8903 | if (IS_ERR(rdev)) |
| 8878 | 8904 | continue; |
| 8879 | 8905 | |
| @@ -201,9 +201,8 @@ struct mddev { · 200 unmodified lines | ||
| 201 | 201 | #define MD_CHANGE_PENDING 2 /* switch from 'clean' to 'active' in progress */ |
| 202 | 202 | #define MD_UPDATE_SB_FLAGS (1 | 2 | 4) /* If these are set, md_update_sb needed */ |
| 203 | 203 | #define MD_ARRAY_FIRST_USE 3 /* First use of array, needs initialization */ |
| 204 | −#define MD_STILL_CLOSED 4 /* If set, then array has not been opened since | |
| 205 | − * md_ioctl checked on it. | |
| 206 | − */ | |
| 204 | +#define MD_CLOSING 4 /* If set, we are closing the array, do not open | |
| 205 | + * it then */ | |
| 207 | 206 | #define MD_JOURNAL_CLEAN 5 /* A raid with journal is already clean */ |
| 208 | 207 | #define MD_HAS_JOURNAL 6 /* The raid array has journal feature set */ |
| 209 | 208 | #define MD_RELOAD_SB 7 /* Reload the superblock because another node |
| @@ -6370,7 +6370,7 @@ static void free_conf(struct r5conf *conf) · 6369 unmodified lines | ||
| 6370 | 6370 | { |
| 6371 | 6371 | if (conf->log) |
| 6372 | 6372 | r5l_exit_log(conf->log); |
| 6373 | − if (conf->shrinker.seeks) | |
| 6373 | + if (conf->shrinker.nr_deferred) | |
| 6374 | 6374 | unregister_shrinker(&conf->shrinker); |
| 6375 | 6375 | |
| 6376 | 6376 | free_thread_groups(conf); |
| @@ -6632,7 +6632,12 @@ static struct r5conf *setup_conf(struct mddev *mddev) · 255 unmodified lines | ||
| 6632 | 6632 | conf->shrinker.count_objects = raid5_cache_count; |
| 6633 | 6633 | conf->shrinker.batch = 128; |
| 6634 | 6634 | conf->shrinker.flags = 0; |
| 6635 | − register_shrinker(&conf->shrinker); | |
| 6635 | + if (register_shrinker(&conf->shrinker)) { | |
| 6636 | + printk(KERN_ERR | |
| 6637 | + "md/raid:%s: couldn't register shrinker.\n", | |
| 6638 | + mdname(mddev)); | |
| 6639 | + goto abort; | |
| 6640 | + } | |
| 6636 | 6641 | |
| 6637 | 6642 | sprintf(pers_name, "raid%d", mddev->new_level); |
| 6638 | 6643 | conf->thread = md_register_thread(raid5d, mddev, pers_name); |
| @@ -7028,6 +7033,8 @@ static int raid5_run(struct mddev *mddev) · 389 unmodified lines | ||
| 7028 | 7033 | else |
| 7029 | 7034 | queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, |
| 7030 | 7035 | mddev->queue); |
| 7036 | + | |
| 7037 | + blk_queue_max_hw_sectors(mddev->queue, UINT_MAX); | |
| 7031 | 7038 | } |
| 7032 | 7039 | |
| 7033 | 7040 | if (journal_dev) { |
| @@ -102,6 +102,9 @@ extern const struct raid6_calls raid6_altivec8; · 101 unmodified lines | ||
| 102 | 102 | extern const struct raid6_calls raid6_avx2x1; |
| 103 | 103 | extern const struct raid6_calls raid6_avx2x2; |
| 104 | 104 | extern const struct raid6_calls raid6_avx2x4; |
| 105 | +extern const struct raid6_calls raid6_avx512x1; | |
| 106 | +extern const struct raid6_calls raid6_avx512x2; | |
| 107 | +extern const struct raid6_calls raid6_avx512x4; | |
| 105 | 108 | extern const struct raid6_calls raid6_tilegx8; |
| 106 | 109 | extern const struct raid6_calls raid6_s390vx8; |
| 107 | 110 | |
| @@ -116,6 +119,7 @@ struct raid6_recov_calls { · 8 unmodified lines | ||
| 116 | 119 | extern const struct raid6_recov_calls raid6_recov_intx1; |
| 117 | 120 | extern const struct raid6_recov_calls raid6_recov_ssse3; |
| 118 | 121 | extern const struct raid6_recov_calls raid6_recov_avx2; |
| 122 | +extern const struct raid6_recov_calls raid6_recov_avx512; | |
| 119 | 123 | extern const struct raid6_recov_calls raid6_recov_s390xc; |
| 120 | 124 | |
| 121 | 125 | extern const struct raid6_calls raid6_neonx1; |
| @@ -3,7 +3,7 @@ obj-$(CONFIG_RAID6_PQ) += raid6_pq.o · 2 unmodified lines | ||
| 3 | 3 | raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ |
| 4 | 4 | int8.o int16.o int32.o |
| 5 | 5 | |
| 6 | −raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o | |
| 6 | +raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o | |
| 7 | 7 | raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o |
| 8 | 8 | raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o |
| 9 | 9 | raid6_pq-$(CONFIG_TILEGX) += tilegx8.o |
| @@ -49,6 +49,10 @@ const struct raid6_calls * const raid6_algos[] = { · 48 unmodified lines | ||
| 49 | 49 | &raid6_avx2x1, |
| 50 | 50 | &raid6_avx2x2, |
| 51 | 51 | #endif |
| 52 | +#ifdef CONFIG_AS_AVX512 | |
| 53 | + &raid6_avx512x1, | |
| 54 | + &raid6_avx512x2, | |
| 55 | +#endif | |
| 52 | 56 | #endif |
| 53 | 57 | #if defined(__x86_64__) && !defined(__arch_um__) |
| 54 | 58 | &raid6_sse2x1, |
| @@ -59,6 +63,11 @@ const struct raid6_calls * const raid6_algos[] = { · 4 unmodified lines | ||
| 59 | 63 | &raid6_avx2x2, |
| 60 | 64 | &raid6_avx2x4, |
| 61 | 65 | #endif |
| 66 | +#ifdef CONFIG_AS_AVX512 | |
| 67 | + &raid6_avx512x1, | |
| 68 | + &raid6_avx512x2, | |
| 69 | + &raid6_avx512x4, | |
| 70 | +#endif | |
| 62 | 71 | #endif |
| 63 | 72 | #ifdef CONFIG_ALTIVEC |
| 64 | 73 | &raid6_altivec1, |
| @@ -92,6 +101,9 @@ void (*raid6_datap_recov)(int, size_t, int, void **); · 27 unmodified lines | ||
| 92 | 101 | EXPORT_SYMBOL_GPL(raid6_datap_recov); |
| 93 | 102 | |
| 94 | 103 | const struct raid6_recov_calls *const raid6_recov_algos[] = { |
| 104 | +#ifdef CONFIG_AS_AVX512 | |
| 105 | + &raid6_recov_avx512, | |
| 106 | +#endif | |
| 95 | 107 | #ifdef CONFIG_AS_AVX2 |
| 96 | 108 | &raid6_recov_avx2, |
| 97 | 109 | #endif |
| @@ -0,0 +1,569 @@ | ||
| 1 | +/* -*- linux-c -*- -------------------------------------------------------- | |
| 2 | + * | |
| 3 | + * Copyright (C) 2016 Intel Corporation | |
| 4 | + * | |
| 5 | + * Author: Gayatri Kammela <gayatri.kammela@intel.com> | |
| 6 | + * Author: Megha Dey <megha.dey@linux.intel.com> | |
| 7 | + * | |
| 8 | + * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved | |
| 9 | + * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved | |
| 10 | + * | |
| 11 | + * This program is free software; you can redistribute it and/or modify | |
| 12 | + * it under the terms of the GNU General Public License as published by | |
| 13 | + * the Free Software Foundation, Inc., 53 Temple Place Ste 330, | |
| 14 | + * Boston MA 02111-1307, USA; either version 2 of the License, or | |
| 15 | + * (at your option) any later version; incorporated herein by reference. | |
| 16 | + * | |
| 17 | + * ----------------------------------------------------------------------- | |
| 18 | + */ | |
| 19 | + | |
| 20 | +/* | |
| 21 | + * AVX512 implementation of RAID-6 syndrome functions | |
| 22 | + * | |
| 23 | + */ | |
| 24 | + | |
| 25 | +#ifdef CONFIG_AS_AVX512 | |
| 26 | + | |
| 27 | +#include <linux/raid/pq.h> | |
| 28 | +#include "x86.h" | |
| 29 | + | |
| 30 | +static const struct raid6_avx512_constants { | |
| 31 | + u64 x1d[8]; | |
| 32 | +} raid6_avx512_constants __aligned(512) = { | |
| 33 | + { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL, | |
| 34 | + 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL, | |
| 35 | + 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL, | |
| 36 | + 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,}, | |
| 37 | +}; | |
| 38 | + | |
| 39 | +static int raid6_have_avx512(void) | |
| 40 | +{ | |
| 41 | + return boot_cpu_has(X86_FEATURE_AVX2) && | |
| 42 | + boot_cpu_has(X86_FEATURE_AVX) && | |
| 43 | + boot_cpu_has(X86_FEATURE_AVX512F) && | |
| 44 | + boot_cpu_has(X86_FEATURE_AVX512BW) && | |
| 45 | + boot_cpu_has(X86_FEATURE_AVX512VL) && | |
| 46 | + boot_cpu_has(X86_FEATURE_AVX512DQ); | |
| 47 | +} | |
| 48 | + | |
| 49 | +static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs) | |
| 50 | +{ | |
| 51 | + u8 **dptr = (u8 **)ptrs; | |
| 52 | + u8 *p, *q; | |
| 53 | + int d, z, z0; | |
| 54 | + | |
| 55 | + z0 = disks - 3; /* Highest data disk */ | |
| 56 | + p = dptr[z0+1]; /* XOR parity */ | |
| 57 | + q = dptr[z0+2]; /* RS syndrome */ | |
| 58 | + | |
| 59 | + kernel_fpu_begin(); | |
| 60 | + | |
| 61 | + asm volatile("vmovdqa64 %0,%%zmm0\n\t" | |
| 62 | + "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */ | |
| 63 | + : | |
| 64 | + : "m" (raid6_avx512_constants.x1d[0])); | |
| 65 | + | |
| 66 | + for (d = 0; d < bytes; d += 64) { | |
| 67 | + asm volatile("prefetchnta %0\n\t" | |
| 68 | + "vmovdqa64 %0,%%zmm2\n\t" /* P[0] */ | |
| 69 | + "prefetchnta %1\n\t" | |
| 70 | + "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */ | |
| 71 | + "vmovdqa64 %1,%%zmm6" | |
| 72 | + : | |
| 73 | + : "m" (dptr[z0][d]), "m" (dptr[z0-1][d])); | |
| 74 | + for (z = z0-2; z >= 0; z--) { | |
| 75 | + asm volatile("prefetchnta %0\n\t" | |
| 76 | + "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t" | |
| 77 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 78 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 79 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 80 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 81 | + "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t" | |
| 82 | + "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t" | |
| 83 | + "vmovdqa64 %0,%%zmm6" | |
| 84 | + : | |
| 85 | + : "m" (dptr[z][d])); | |
| 86 | + } | |
| 87 | + asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t" | |
| 88 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 89 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 90 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 91 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 92 | + "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t" | |
| 93 | + "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t" | |
| 94 | + "vmovntdq %%zmm2,%0\n\t" | |
| 95 | + "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t" | |
| 96 | + "vmovntdq %%zmm4,%1\n\t" | |
| 97 | + "vpxorq %%zmm4,%%zmm4,%%zmm4" | |
| 98 | + : | |
| 99 | + : "m" (p[d]), "m" (q[d])); | |
| 100 | + } | |
| 101 | + | |
| 102 | + asm volatile("sfence" : : : "memory"); | |
| 103 | + kernel_fpu_end(); | |
| 104 | +} | |
| 105 | + | |
| 106 | +static void raid6_avx5121_xor_syndrome(int disks, int start, int stop, | |
| 107 | + size_t bytes, void **ptrs) | |
| 108 | +{ | |
| 109 | + u8 **dptr = (u8 **)ptrs; | |
| 110 | + u8 *p, *q; | |
| 111 | + int d, z, z0; | |
| 112 | + | |
| 113 | + z0 = stop; /* P/Q right side optimization */ | |
| 114 | + p = dptr[disks-2]; /* XOR parity */ | |
| 115 | + q = dptr[disks-1]; /* RS syndrome */ | |
| 116 | + | |
| 117 | + kernel_fpu_begin(); | |
| 118 | + | |
| 119 | + asm volatile("vmovdqa64 %0,%%zmm0" | |
| 120 | + : : "m" (raid6_avx512_constants.x1d[0])); | |
| 121 | + | |
| 122 | + for (d = 0 ; d < bytes ; d += 64) { | |
| 123 | + asm volatile("vmovdqa64 %0,%%zmm4\n\t" | |
| 124 | + "vmovdqa64 %1,%%zmm2\n\t" | |
| 125 | + "vpxorq %%zmm4,%%zmm2,%%zmm2" | |
| 126 | + : | |
| 127 | + : "m" (dptr[z0][d]), "m" (p[d])); | |
| 128 | + /* P/Q data pages */ | |
| 129 | + for (z = z0-1 ; z >= start ; z--) { | |
| 130 | + asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t" | |
| 131 | + "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t" | |
| 132 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 133 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 134 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 135 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 136 | + "vmovdqa64 %0,%%zmm5\n\t" | |
| 137 | + "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t" | |
| 138 | + "vpxorq %%zmm5,%%zmm4,%%zmm4" | |
| 139 | + : | |
| 140 | + : "m" (dptr[z][d])); | |
| 141 | + } | |
| 142 | + /* P/Q left side optimization */ | |
| 143 | + for (z = start-1 ; z >= 0 ; z--) { | |
| 144 | + asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t" | |
| 145 | + "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t" | |
| 146 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 147 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 148 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 149 | + "vpxorq %%zmm5,%%zmm4,%%zmm4" | |
| 150 | + : | |
| 151 | + : ); | |
| 152 | + } | |
| 153 | + asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t" | |
| 154 | + /* Don't use movntdq for r/w memory area < cache line */ | |
| 155 | + "vmovdqa64 %%zmm4,%0\n\t" | |
| 156 | + "vmovdqa64 %%zmm2,%1" | |
| 157 | + : | |
| 158 | + : "m" (q[d]), "m" (p[d])); | |
| 159 | + } | |
| 160 | + | |
| 161 | + asm volatile("sfence" : : : "memory"); | |
| 162 | + kernel_fpu_end(); | |
| 163 | +} | |
| 164 | + | |
| 165 | +const struct raid6_calls raid6_avx512x1 = { | |
| 166 | + raid6_avx5121_gen_syndrome, | |
| 167 | + raid6_avx5121_xor_syndrome, | |
| 168 | + raid6_have_avx512, | |
| 169 | + "avx512x1", | |
| 170 | + 1 /* Has cache hints */ | |
| 171 | +}; | |
| 172 | + | |
| 173 | +/* | |
| 174 | + * Unrolled-by-2 AVX512 implementation | |
| 175 | + */ | |
| 176 | +static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs) | |
| 177 | +{ | |
| 178 | + u8 **dptr = (u8 **)ptrs; | |
| 179 | + u8 *p, *q; | |
| 180 | + int d, z, z0; | |
| 181 | + | |
| 182 | + z0 = disks - 3; /* Highest data disk */ | |
| 183 | + p = dptr[z0+1]; /* XOR parity */ | |
| 184 | + q = dptr[z0+2]; /* RS syndrome */ | |
| 185 | + | |
| 186 | + kernel_fpu_begin(); | |
| 187 | + | |
| 188 | + asm volatile("vmovdqa64 %0,%%zmm0\n\t" | |
| 189 | + "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */ | |
| 190 | + : | |
| 191 | + : "m" (raid6_avx512_constants.x1d[0])); | |
| 192 | + | |
| 193 | + /* We uniformly assume a single prefetch covers at least 64 bytes */ | |
| 194 | + for (d = 0; d < bytes; d += 128) { | |
| 195 | + asm volatile("prefetchnta %0\n\t" | |
| 196 | + "prefetchnta %1\n\t" | |
| 197 | + "vmovdqa64 %0,%%zmm2\n\t" /* P[0] */ | |
| 198 | + "vmovdqa64 %1,%%zmm3\n\t" /* P[1] */ | |
| 199 | + "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */ | |
| 200 | + "vmovdqa64 %%zmm3,%%zmm6" /* Q[1] */ | |
| 201 | + : | |
| 202 | + : "m" (dptr[z0][d]), "m" (dptr[z0][d+64])); | |
| 203 | + for (z = z0-1; z >= 0; z--) { | |
| 204 | + asm volatile("prefetchnta %0\n\t" | |
| 205 | + "prefetchnta %1\n\t" | |
| 206 | + "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t" | |
| 207 | + "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t" | |
| 208 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 209 | + "vpmovm2b %%k2,%%zmm7\n\t" | |
| 210 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 211 | + "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t" | |
| 212 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 213 | + "vpandq %%zmm0,%%zmm7,%%zmm7\n\t" | |
| 214 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 215 | + "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t" | |
| 216 | + "vmovdqa64 %0,%%zmm5\n\t" | |
| 217 | + "vmovdqa64 %1,%%zmm7\n\t" | |
| 218 | + "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t" | |
| 219 | + "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t" | |
| 220 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 221 | + "vpxorq %%zmm7,%%zmm6,%%zmm6" | |
| 222 | + : | |
| 223 | + : "m" (dptr[z][d]), "m" (dptr[z][d+64])); | |
| 224 | + } | |
| 225 | + asm volatile("vmovntdq %%zmm2,%0\n\t" | |
| 226 | + "vmovntdq %%zmm3,%1\n\t" | |
| 227 | + "vmovntdq %%zmm4,%2\n\t" | |
| 228 | + "vmovntdq %%zmm6,%3" | |
| 229 | + : | |
| 230 | + : "m" (p[d]), "m" (p[d+64]), "m" (q[d]), | |
| 231 | + "m" (q[d+64])); | |
| 232 | + } | |
| 233 | + | |
| 234 | + asm volatile("sfence" : : : "memory"); | |
| 235 | + kernel_fpu_end(); | |
| 236 | +} | |
| 237 | + | |
| 238 | +static void raid6_avx5122_xor_syndrome(int disks, int start, int stop, | |
| 239 | + size_t bytes, void **ptrs) | |
| 240 | +{ | |
| 241 | + u8 **dptr = (u8 **)ptrs; | |
| 242 | + u8 *p, *q; | |
| 243 | + int d, z, z0; | |
| 244 | + | |
| 245 | + z0 = stop; /* P/Q right side optimization */ | |
| 246 | + p = dptr[disks-2]; /* XOR parity */ | |
| 247 | + q = dptr[disks-1]; /* RS syndrome */ | |
| 248 | + | |
| 249 | + kernel_fpu_begin(); | |
| 250 | + | |
| 251 | + asm volatile("vmovdqa64 %0,%%zmm0" | |
| 252 | + : : "m" (raid6_avx512_constants.x1d[0])); | |
| 253 | + | |
| 254 | + for (d = 0 ; d < bytes ; d += 128) { | |
| 255 | + asm volatile("vmovdqa64 %0,%%zmm4\n\t" | |
| 256 | + "vmovdqa64 %1,%%zmm6\n\t" | |
| 257 | + "vmovdqa64 %2,%%zmm2\n\t" | |
| 258 | + "vmovdqa64 %3,%%zmm3\n\t" | |
| 259 | + "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t" | |
| 260 | + "vpxorq %%zmm6,%%zmm3,%%zmm3" | |
| 261 | + : | |
| 262 | + : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]), | |
| 263 | + "m" (p[d]), "m" (p[d+64])); | |
| 264 | + /* P/Q data pages */ | |
| 265 | + for (z = z0-1 ; z >= start ; z--) { | |
| 266 | + asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t" | |
| 267 | + "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t" | |
| 268 | + "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t" | |
| 269 | + "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t" | |
| 270 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 271 | + "vpmovm2b %%k2,%%zmm7\n\t" | |
| 272 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 273 | + "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t" | |
| 274 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 275 | + "vpandq %%zmm0,%%zmm7,%%zmm7\n\t" | |
| 276 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 277 | + "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t" | |
| 278 | + "vmovdqa64 %0,%%zmm5\n\t" | |
| 279 | + "vmovdqa64 %1,%%zmm7\n\t" | |
| 280 | + "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t" | |
| 281 | + "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t" | |
| 282 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 283 | + "vpxorq %%zmm7,%%zmm6,%%zmm6" | |
| 284 | + : | |
| 285 | + : "m" (dptr[z][d]), "m" (dptr[z][d+64])); | |
| 286 | + } | |
| 287 | + /* P/Q left side optimization */ | |
| 288 | + for (z = start-1 ; z >= 0 ; z--) { | |
| 289 | + asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t" | |
| 290 | + "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t" | |
| 291 | + "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t" | |
| 292 | + "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t" | |
| 293 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 294 | + "vpmovm2b %%k2,%%zmm7\n\t" | |
| 295 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 296 | + "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t" | |
| 297 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 298 | + "vpandq %%zmm0,%%zmm7,%%zmm7\n\t" | |
| 299 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 300 | + "vpxorq %%zmm7,%%zmm6,%%zmm6" | |
| 301 | + : | |
| 302 | + : ); | |
| 303 | + } | |
| 304 | + asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t" | |
| 305 | + "vpxorq %1,%%zmm6,%%zmm6\n\t" | |
| 306 | + /* Don't use movntdq for r/w | |
| 307 | + * memory area < cache line | |
| 308 | + */ | |
| 309 | + "vmovdqa64 %%zmm4,%0\n\t" | |
| 310 | + "vmovdqa64 %%zmm6,%1\n\t" | |
| 311 | + "vmovdqa64 %%zmm2,%2\n\t" | |
| 312 | + "vmovdqa64 %%zmm3,%3" | |
| 313 | + : | |
| 314 | + : "m" (q[d]), "m" (q[d+64]), "m" (p[d]), | |
| 315 | + "m" (p[d+64])); | |
| 316 | + } | |
| 317 | + | |
| 318 | + asm volatile("sfence" : : : "memory"); | |
| 319 | + kernel_fpu_end(); | |
| 320 | +} | |
| 321 | + | |
| 322 | +const struct raid6_calls raid6_avx512x2 = { | |
| 323 | + raid6_avx5122_gen_syndrome, | |
| 324 | + raid6_avx5122_xor_syndrome, | |
| 325 | + raid6_have_avx512, | |
| 326 | + "avx512x2", | |
| 327 | + 1 /* Has cache hints */ | |
| 328 | +}; | |
| 329 | + | |
| 330 | +#ifdef CONFIG_X86_64 | |
| 331 | + | |
| 332 | +/* | |
| 333 | + * Unrolled-by-4 AVX2 implementation | |
| 334 | + */ | |
| 335 | +static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs) | |
| 336 | +{ | |
| 337 | + u8 **dptr = (u8 **)ptrs; | |
| 338 | + u8 *p, *q; | |
| 339 | + int d, z, z0; | |
| 340 | + | |
| 341 | + z0 = disks - 3; /* Highest data disk */ | |
| 342 | + p = dptr[z0+1]; /* XOR parity */ | |
| 343 | + q = dptr[z0+2]; /* RS syndrome */ | |
| 344 | + | |
| 345 | + kernel_fpu_begin(); | |
| 346 | + | |
| 347 | + asm volatile("vmovdqa64 %0,%%zmm0\n\t" | |
| 348 | + "vpxorq %%zmm1,%%zmm1,%%zmm1\n\t" /* Zero temp */ | |
| 349 | + "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t" /* P[0] */ | |
| 350 | + "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t" /* P[1] */ | |
| 351 | + "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t" /* Q[0] */ | |
| 352 | + "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t" /* Q[1] */ | |
| 353 | + "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t" /* P[2] */ | |
| 354 | + "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t" /* P[3] */ | |
| 355 | + "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t" /* Q[2] */ | |
| 356 | + "vpxorq %%zmm14,%%zmm14,%%zmm14" /* Q[3] */ | |
| 357 | + : | |
| 358 | + : "m" (raid6_avx512_constants.x1d[0])); | |
| 359 | + | |
| 360 | + for (d = 0; d < bytes; d += 256) { | |
| 361 | + for (z = z0; z >= 0; z--) { | |
| 362 | + asm volatile("prefetchnta %0\n\t" | |
| 363 | + "prefetchnta %1\n\t" | |
| 364 | + "prefetchnta %2\n\t" | |
| 365 | + "prefetchnta %3\n\t" | |
| 366 | + "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t" | |
| 367 | + "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t" | |
| 368 | + "vpcmpgtb %%zmm12,%%zmm1,%%k3\n\t" | |
| 369 | + "vpcmpgtb %%zmm14,%%zmm1,%%k4\n\t" | |
| 370 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 371 | + "vpmovm2b %%k2,%%zmm7\n\t" | |
| 372 | + "vpmovm2b %%k3,%%zmm13\n\t" | |
| 373 | + "vpmovm2b %%k4,%%zmm15\n\t" | |
| 374 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 375 | + "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t" | |
| 376 | + "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t" | |
| 377 | + "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t" | |
| 378 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 379 | + "vpandq %%zmm0,%%zmm7,%%zmm7\n\t" | |
| 380 | + "vpandq %%zmm0,%%zmm13,%%zmm13\n\t" | |
| 381 | + "vpandq %%zmm0,%%zmm15,%%zmm15\n\t" | |
| 382 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 383 | + "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t" | |
| 384 | + "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t" | |
| 385 | + "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t" | |
| 386 | + "vmovdqa64 %0,%%zmm5\n\t" | |
| 387 | + "vmovdqa64 %1,%%zmm7\n\t" | |
| 388 | + "vmovdqa64 %2,%%zmm13\n\t" | |
| 389 | + "vmovdqa64 %3,%%zmm15\n\t" | |
| 390 | + "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t" | |
| 391 | + "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t" | |
| 392 | + "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t" | |
| 393 | + "vpxorq %%zmm15,%%zmm11,%%zmm11\n" | |
| 394 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 395 | + "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t" | |
| 396 | + "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t" | |
| 397 | + "vpxorq %%zmm15,%%zmm14,%%zmm14" | |
| 398 | + : | |
| 399 | + : "m" (dptr[z][d]), "m" (dptr[z][d+64]), | |
| 400 | + "m" (dptr[z][d+128]), "m" (dptr[z][d+192])); | |
| 401 | + } | |
| 402 | + asm volatile("vmovntdq %%zmm2,%0\n\t" | |
| 403 | + "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t" | |
| 404 | + "vmovntdq %%zmm3,%1\n\t" | |
| 405 | + "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t" | |
| 406 | + "vmovntdq %%zmm10,%2\n\t" | |
| 407 | + "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t" | |
| 408 | + "vmovntdq %%zmm11,%3\n\t" | |
| 409 | + "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t" | |
| 410 | + "vmovntdq %%zmm4,%4\n\t" | |
| 411 | + "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 412 | + "vmovntdq %%zmm6,%5\n\t" | |
| 413 | + "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t" | |
| 414 | + "vmovntdq %%zmm12,%6\n\t" | |
| 415 | + "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t" | |
| 416 | + "vmovntdq %%zmm14,%7\n\t" | |
| 417 | + "vpxorq %%zmm14,%%zmm14,%%zmm14" | |
| 418 | + : | |
| 419 | + : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]), | |
| 420 | + "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]), | |
| 421 | + "m" (q[d+128]), "m" (q[d+192])); | |
| 422 | + } | |
| 423 | + | |
| 424 | + asm volatile("sfence" : : : "memory"); | |
| 425 | + kernel_fpu_end(); | |
| 426 | +} | |
| 427 | + | |
| 428 | +static void raid6_avx5124_xor_syndrome(int disks, int start, int stop, | |
| 429 | + size_t bytes, void **ptrs) | |
| 430 | +{ | |
| 431 | + u8 **dptr = (u8 **)ptrs; | |
| 432 | + u8 *p, *q; | |
| 433 | + int d, z, z0; | |
| 434 | + | |
| 435 | + z0 = stop; /* P/Q right side optimization */ | |
| 436 | + p = dptr[disks-2]; /* XOR parity */ | |
| 437 | + q = dptr[disks-1]; /* RS syndrome */ | |
| 438 | + | |
| 439 | + kernel_fpu_begin(); | |
| 440 | + | |
| 441 | + asm volatile("vmovdqa64 %0,%%zmm0" | |
| 442 | + :: "m" (raid6_avx512_constants.x1d[0])); | |
| 443 | + | |
| 444 | + for (d = 0 ; d < bytes ; d += 256) { | |
| 445 | + asm volatile("vmovdqa64 %0,%%zmm4\n\t" | |
| 446 | + "vmovdqa64 %1,%%zmm6\n\t" | |
| 447 | + "vmovdqa64 %2,%%zmm12\n\t" | |
| 448 | + "vmovdqa64 %3,%%zmm14\n\t" | |
| 449 | + "vmovdqa64 %4,%%zmm2\n\t" | |
| 450 | + "vmovdqa64 %5,%%zmm3\n\t" | |
| 451 | + "vmovdqa64 %6,%%zmm10\n\t" | |
| 452 | + "vmovdqa64 %7,%%zmm11\n\t" | |
| 453 | + "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t" | |
| 454 | + "vpxorq %%zmm6,%%zmm3,%%zmm3\n\t" | |
| 455 | + "vpxorq %%zmm12,%%zmm10,%%zmm10\n\t" | |
| 456 | + "vpxorq %%zmm14,%%zmm11,%%zmm11" | |
| 457 | + : | |
| 458 | + : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]), | |
| 459 | + "m" (dptr[z0][d+128]), "m" (dptr[z0][d+192]), | |
| 460 | + "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]), | |
| 461 | + "m" (p[d+192])); | |
| 462 | + /* P/Q data pages */ | |
| 463 | + for (z = z0-1 ; z >= start ; z--) { | |
| 464 | + asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t" | |
| 465 | + "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t" | |
| 466 | + "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t" | |
| 467 | + "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t" | |
| 468 | + "prefetchnta %0\n\t" | |
| 469 | + "prefetchnta %2\n\t" | |
| 470 | + "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t" | |
| 471 | + "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t" | |
| 472 | + "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t" | |
| 473 | + "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t" | |
| 474 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 475 | + "vpmovm2b %%k2,%%zmm7\n\t" | |
| 476 | + "vpmovm2b %%k3,%%zmm13\n\t" | |
| 477 | + "vpmovm2b %%k4,%%zmm15\n\t" | |
| 478 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 479 | + "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t" | |
| 480 | + "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t" | |
| 481 | + "vpaddb %%Zmm14,%%zmm14,%%zmm14\n\t" | |
| 482 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 483 | + "vpandq %%zmm0,%%zmm7,%%zmm7\n\t" | |
| 484 | + "vpandq %%zmm0,%%zmm13,%%zmm13\n\t" | |
| 485 | + "vpandq %%zmm0,%%zmm15,%%zmm15\n\t" | |
| 486 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 487 | + "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t" | |
| 488 | + "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t" | |
| 489 | + "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t" | |
| 490 | + "vmovdqa64 %0,%%zmm5\n\t" | |
| 491 | + "vmovdqa64 %1,%%zmm7\n\t" | |
| 492 | + "vmovdqa64 %2,%%zmm13\n\t" | |
| 493 | + "vmovdqa64 %3,%%zmm15\n\t" | |
| 494 | + "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t" | |
| 495 | + "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t" | |
| 496 | + "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t" | |
| 497 | + "vpxorq %%zmm15,%%zmm11,%%zmm11\n\t" | |
| 498 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 499 | + "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t" | |
| 500 | + "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t" | |
| 501 | + "vpxorq %%zmm15,%%zmm14,%%zmm14" | |
| 502 | + : | |
| 503 | + : "m" (dptr[z][d]), "m" (dptr[z][d+64]), | |
| 504 | + "m" (dptr[z][d+128]), | |
| 505 | + "m" (dptr[z][d+192])); | |
| 506 | + } | |
| 507 | + asm volatile("prefetchnta %0\n\t" | |
| 508 | + "prefetchnta %1\n\t" | |
| 509 | + : | |
| 510 | + : "m" (q[d]), "m" (q[d+128])); | |
| 511 | + /* P/Q left side optimization */ | |
| 512 | + for (z = start-1 ; z >= 0 ; z--) { | |
| 513 | + asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t" | |
| 514 | + "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t" | |
| 515 | + "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t" | |
| 516 | + "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t" | |
| 517 | + "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t" | |
| 518 | + "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t" | |
| 519 | + "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t" | |
| 520 | + "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t" | |
| 521 | + "vpmovm2b %%k1,%%zmm5\n\t" | |
| 522 | + "vpmovm2b %%k2,%%zmm7\n\t" | |
| 523 | + "vpmovm2b %%k3,%%zmm13\n\t" | |
| 524 | + "vpmovm2b %%k4,%%zmm15\n\t" | |
| 525 | + "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t" | |
| 526 | + "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t" | |
| 527 | + "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t" | |
| 528 | + "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t" | |
| 529 | + "vpandq %%zmm0,%%zmm5,%%zmm5\n\t" | |
| 530 | + "vpandq %%zmm0,%%zmm7,%%zmm7\n\t" | |
| 531 | + "vpandq %%zmm0,%%zmm13,%%zmm13\n\t" | |
| 532 | + "vpandq %%zmm0,%%zmm15,%%zmm15\n\t" | |
| 533 | + "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t" | |
| 534 | + "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t" | |
| 535 | + "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t" | |
| 536 | + "vpxorq %%zmm15,%%zmm14,%%zmm14" | |
| 537 | + : | |
| 538 | + : ); | |
| 539 | + } | |
| 540 | + asm volatile("vmovntdq %%zmm2,%0\n\t" | |
| 541 | + "vmovntdq %%zmm3,%1\n\t" | |
| 542 | + "vmovntdq %%zmm10,%2\n\t" | |
| 543 | + "vmovntdq %%zmm11,%3\n\t" | |
| 544 | + "vpxorq %4,%%zmm4,%%zmm4\n\t" | |
| 545 | + "vpxorq %5,%%zmm6,%%zmm6\n\t" | |
| 546 | + "vpxorq %6,%%zmm12,%%zmm12\n\t" | |
| 547 | + "vpxorq %7,%%zmm14,%%zmm14\n\t" | |
| 548 | + "vmovntdq %%zmm4,%4\n\t" | |
| 549 | + "vmovntdq %%zmm6,%5\n\t" | |
| 550 | + "vmovntdq %%zmm12,%6\n\t" | |
| 551 | + "vmovntdq %%zmm14,%7" | |
| 552 | + : | |
| 553 | + : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]), | |
| 554 | + "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]), | |
| 555 | + "m" (q[d+128]), "m" (q[d+192])); | |
| 556 | + } | |
| 557 | + asm volatile("sfence" : : : "memory"); | |
| 558 | + kernel_fpu_end(); | |
| 559 | +} | |
| 560 | +const struct raid6_calls raid6_avx512x4 = { | |
| 561 | + raid6_avx5124_gen_syndrome, | |
| 562 | + raid6_avx5124_xor_syndrome, | |
| 563 | + raid6_have_avx512, | |
| 564 | + "avx512x4", | |
| 565 | + 1 /* Has cache hints */ | |
| 566 | +}; | |
| 567 | +#endif | |
| 568 | + | |
| 569 | +#endif /* CONFIG_AS_AVX512 */ | |
| @@ -0,0 +1,388 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright (C) 2016 Intel Corporation | |
| 3 | + * | |
| 4 | + * Author: Gayatri Kammela <gayatri.kammela@intel.com> | |
| 5 | + * Author: Megha Dey <megha.dey@linux.intel.com> | |
| 6 | + * | |
| 7 | + * This program is free software; you can redistribute it and/or | |
| 8 | + * modify it under the terms of the GNU General Public License | |
| 9 | + * as published by the Free Software Foundation; version 2 | |
| 10 | + * of the License. | |
| 11 | + * | |
| 12 | + */ | |
| 13 | + | |
| 14 | +#ifdef CONFIG_AS_AVX512 | |
| 15 | + | |
| 16 | +#include <linux/raid/pq.h> | |
| 17 | +#include "x86.h" | |
| 18 | + | |
| 19 | +static int raid6_has_avx512(void) | |
| 20 | +{ | |
| 21 | + return boot_cpu_has(X86_FEATURE_AVX2) && | |
| 22 | + boot_cpu_has(X86_FEATURE_AVX) && | |
| 23 | + boot_cpu_has(X86_FEATURE_AVX512F) && | |
| 24 | + boot_cpu_has(X86_FEATURE_AVX512BW) && | |
| 25 | + boot_cpu_has(X86_FEATURE_AVX512VL) && | |
| 26 | + boot_cpu_has(X86_FEATURE_AVX512DQ); | |
| 27 | +} | |
| 28 | + | |
| 29 | +static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, | |
| 30 | + int failb, void **ptrs) | |
| 31 | +{ | |
| 32 | + u8 *p, *q, *dp, *dq; | |
| 33 | + const u8 *pbmul; /* P multiplier table for B data */ | |
| 34 | + const u8 *qmul; /* Q multiplier table (for both) */ | |
| 35 | + const u8 x0f = 0x0f; | |
| 36 | + | |
| 37 | + p = (u8 *)ptrs[disks-2]; | |
| 38 | + q = (u8 *)ptrs[disks-1]; | |
| 39 | + | |
| 40 | + /* | |
| 41 | + * Compute syndrome with zero for the missing data pages | |
| 42 | + * Use the dead data pages as temporary storage for | |
| 43 | + * delta p and delta q | |
| 44 | + */ | |
| 45 | + | |
| 46 | + dp = (u8 *)ptrs[faila]; | |
| 47 | + ptrs[faila] = (void *)raid6_empty_zero_page; | |
| 48 | + ptrs[disks-2] = dp; | |
| 49 | + dq = (u8 *)ptrs[failb]; | |
| 50 | + ptrs[failb] = (void *)raid6_empty_zero_page; | |
| 51 | + ptrs[disks-1] = dq; | |
| 52 | + | |
| 53 | + raid6_call.gen_syndrome(disks, bytes, ptrs); | |
| 54 | + | |
| 55 | + /* Restore pointer table */ | |
| 56 | + ptrs[faila] = dp; | |
| 57 | + ptrs[failb] = dq; | |
| 58 | + ptrs[disks-2] = p; | |
| 59 | + ptrs[disks-1] = q; | |
| 60 | + | |
| 61 | + /* Now, pick the proper data tables */ | |
| 62 | + pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]]; | |
| 63 | + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ | |
| 64 | + raid6_gfexp[failb]]]; | |
| 65 | + | |
| 66 | + kernel_fpu_begin(); | |
| 67 | + | |
| 68 | + /* zmm0 = x0f[16] */ | |
| 69 | + asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f)); | |
| 70 | + | |
| 71 | + while (bytes) { | |
| 72 | +#ifdef CONFIG_X86_64 | |
| 73 | + asm volatile("vmovdqa64 %0, %%zmm1\n\t" | |
| 74 | + "vmovdqa64 %1, %%zmm9\n\t" | |
| 75 | + "vmovdqa64 %2, %%zmm0\n\t" | |
| 76 | + "vmovdqa64 %3, %%zmm8\n\t" | |
| 77 | + "vpxorq %4, %%zmm1, %%zmm1\n\t" | |
| 78 | + "vpxorq %5, %%zmm9, %%zmm9\n\t" | |
| 79 | + "vpxorq %6, %%zmm0, %%zmm0\n\t" | |
| 80 | + "vpxorq %7, %%zmm8, %%zmm8" | |
| 81 | + : | |
| 82 | + : "m" (q[0]), "m" (q[64]), "m" (p[0]), | |
| 83 | + "m" (p[64]), "m" (dq[0]), "m" (dq[64]), | |
| 84 | + "m" (dp[0]), "m" (dp[64])); | |
| 85 | + | |
| 86 | + /* | |
| 87 | + * 1 = dq[0] ^ q[0] | |
| 88 | + * 9 = dq[64] ^ q[64] | |
| 89 | + * 0 = dp[0] ^ p[0] | |
| 90 | + * 8 = dp[64] ^ p[64] | |
| 91 | + */ | |
| 92 | + | |
| 93 | + asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t" | |
| 94 | + "vbroadcasti64x2 %1, %%zmm5" | |
| 95 | + : | |
| 96 | + : "m" (qmul[0]), "m" (qmul[16])); | |
| 97 | + | |
| 98 | + asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t" | |
| 99 | + "vpsraw $4, %%zmm9, %%zmm12\n\t" | |
| 100 | + "vpandq %%zmm7, %%zmm1, %%zmm1\n\t" | |
| 101 | + "vpandq %%zmm7, %%zmm9, %%zmm9\n\t" | |
| 102 | + "vpandq %%zmm7, %%zmm3, %%zmm3\n\t" | |
| 103 | + "vpandq %%zmm7, %%zmm12, %%zmm12\n\t" | |
| 104 | + "vpshufb %%zmm9, %%zmm4, %%zmm14\n\t" | |
| 105 | + "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t" | |
| 106 | + "vpshufb %%zmm12, %%zmm5, %%zmm15\n\t" | |
| 107 | + "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t" | |
| 108 | + "vpxorq %%zmm14, %%zmm15, %%zmm15\n\t" | |
| 109 | + "vpxorq %%zmm4, %%zmm5, %%zmm5" | |
| 110 | + : | |
| 111 | + : ); | |
| 112 | + | |
| 113 | + /* | |
| 114 | + * 5 = qx[0] | |
| 115 | + * 15 = qx[64] | |
| 116 | + */ | |
| 117 | + | |
| 118 | + asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t" | |
| 119 | + "vbroadcasti64x2 %1, %%zmm1\n\t" | |
| 120 | + "vpsraw $4, %%zmm0, %%zmm2\n\t" | |
| 121 | + "vpsraw $4, %%zmm8, %%zmm6\n\t" | |
| 122 | + "vpandq %%zmm7, %%zmm0, %%zmm3\n\t" | |
| 123 | + "vpandq %%zmm7, %%zmm8, %%zmm14\n\t" | |
| 124 | + "vpandq %%zmm7, %%zmm2, %%zmm2\n\t" | |
| 125 | + "vpandq %%zmm7, %%zmm6, %%zmm6\n\t" | |
| 126 | + "vpshufb %%zmm14, %%zmm4, %%zmm12\n\t" | |
| 127 | + "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t" | |
| 128 | + "vpshufb %%zmm6, %%zmm1, %%zmm13\n\t" | |
| 129 | + "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t" | |
| 130 | + "vpxorq %%zmm4, %%zmm1, %%zmm1\n\t" | |
| 131 | + "vpxorq %%zmm12, %%zmm13, %%zmm13" | |
| 132 | + : | |
| 133 | + : "m" (pbmul[0]), "m" (pbmul[16])); | |
| 134 | + | |
| 135 | + /* | |
| 136 | + * 1 = pbmul[px[0]] | |
| 137 | + * 13 = pbmul[px[64]] | |
| 138 | + */ | |
| 139 | + asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t" | |
| 140 | + "vpxorq %%zmm15, %%zmm13, %%zmm13" | |
| 141 | + : | |
| 142 | + : ); | |
| 143 | + | |
| 144 | + /* | |
| 145 | + * 1 = db = DQ | |
| 146 | + * 13 = db[64] = DQ[64] | |
| 147 | + */ | |
| 148 | + asm volatile("vmovdqa64 %%zmm1, %0\n\t" | |
| 149 | + "vmovdqa64 %%zmm13,%1\n\t" | |
| 150 | + "vpxorq %%zmm1, %%zmm0, %%zmm0\n\t" | |
| 151 | + "vpxorq %%zmm13, %%zmm8, %%zmm8" | |
| 152 | + : | |
| 153 | + : "m" (dq[0]), "m" (dq[64])); | |
| 154 | + | |
| 155 | + asm volatile("vmovdqa64 %%zmm0, %0\n\t" | |
| 156 | + "vmovdqa64 %%zmm8, %1" | |
| 157 | + : | |
| 158 | + : "m" (dp[0]), "m" (dp[64])); | |
| 159 | + | |
| 160 | + bytes -= 128; | |
| 161 | + p += 128; | |
| 162 | + q += 128; | |
| 163 | + dp += 128; | |
| 164 | + dq += 128; | |
| 165 | +#else | |
| 166 | + asm volatile("vmovdqa64 %0, %%zmm1\n\t" | |
| 167 | + "vmovdqa64 %1, %%zmm0\n\t" | |
| 168 | + "vpxorq %2, %%zmm1, %%zmm1\n\t" | |
| 169 | + "vpxorq %3, %%zmm0, %%zmm0" | |
| 170 | + : | |
| 171 | + : "m" (*q), "m" (*p), "m"(*dq), "m" (*dp)); | |
| 172 | + | |
| 173 | + /* 1 = dq ^ q; 0 = dp ^ p */ | |
| 174 | + | |
| 175 | + asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t" | |
| 176 | + "vbroadcasti64x2 %1, %%zmm5" | |
| 177 | + : | |
| 178 | + : "m" (qmul[0]), "m" (qmul[16])); | |
| 179 | + | |
| 180 | + /* | |
| 181 | + * 1 = dq ^ q | |
| 182 | + * 3 = dq ^ p >> 4 | |
| 183 | + */ | |
| 184 | + asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t" | |
| 185 | + "vpandq %%zmm7, %%zmm1, %%zmm1\n\t" | |
| 186 | + "vpandq %%zmm7, %%zmm3, %%zmm3\n\t" | |
| 187 | + "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t" | |
| 188 | + "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t" | |
| 189 | + "vpxorq %%zmm4, %%zmm5, %%zmm5" | |
| 190 | + : | |
| 191 | + : ); | |
| 192 | + | |
| 193 | + /* 5 = qx */ | |
| 194 | + | |
| 195 | + asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t" | |
| 196 | + "vbroadcasti64x2 %1, %%zmm1" | |
| 197 | + : | |
| 198 | + : "m" (pbmul[0]), "m" (pbmul[16])); | |
| 199 | + | |
| 200 | + asm volatile("vpsraw $4, %%zmm0, %%zmm2\n\t" | |
| 201 | + "vpandq %%zmm7, %%zmm0, %%zmm3\n\t" | |
| 202 | + "vpandq %%zmm7, %%zmm2, %%zmm2\n\t" | |
| 203 | + "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t" | |
| 204 | + "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t" | |
| 205 | + "vpxorq %%zmm4, %%zmm1, %%zmm1" | |
| 206 | + : | |
| 207 | + : ); | |
| 208 | + | |
| 209 | + /* 1 = pbmul[px] */ | |
| 210 | + asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t" | |
| 211 | + /* 1 = db = DQ */ | |
| 212 | + "vmovdqa64 %%zmm1, %0\n\t" | |
| 213 | + : | |
| 214 | + : "m" (dq[0])); | |
| 215 | + | |
| 216 | + asm volatile("vpxorq %%zmm1, %%zmm0, %%zmm0\n\t" | |
| 217 | + "vmovdqa64 %%zmm0, %0" | |
| 218 | + : | |
| 219 | + : "m" (dp[0])); | |
| 220 | + | |
| 221 | + bytes -= 64; | |
| 222 | + p += 64; | |
| 223 | + q += 64; | |
| 224 | + dp += 64; | |
| 225 | + dq += 64; | |
| 226 | +#endif | |
| 227 | + } | |
| 228 | + | |
| 229 | + kernel_fpu_end(); | |
| 230 | +} | |
| 231 | + | |
| 232 | +static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, | |
| 233 | + void **ptrs) | |
| 234 | +{ | |
| 235 | + u8 *p, *q, *dq; | |
| 236 | + const u8 *qmul; /* Q multiplier table */ | |
| 237 | + const u8 x0f = 0x0f; | |
| 238 | + | |
| 239 | + p = (u8 *)ptrs[disks-2]; | |
| 240 | + q = (u8 *)ptrs[disks-1]; | |
| 241 | + | |
| 242 | + /* | |
| 243 | + * Compute syndrome with zero for the missing data page | |
| 244 | + * Use the dead data page as temporary storage for delta q | |
| 245 | + */ | |
| 246 | + | |
| 247 | + dq = (u8 *)ptrs[faila]; | |
| 248 | + ptrs[faila] = (void *)raid6_empty_zero_page; | |
| 249 | + ptrs[disks-1] = dq; | |
| 250 | + | |
| 251 | + raid6_call.gen_syndrome(disks, bytes, ptrs); | |
| 252 | + | |
| 253 | + /* Restore pointer table */ | |
| 254 | + ptrs[faila] = dq; | |
| 255 | + ptrs[disks-1] = q; | |
| 256 | + | |
| 257 | + /* Now, pick the proper data tables */ | |
| 258 | + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; | |
| 259 | + | |
| 260 | + kernel_fpu_begin(); | |
| 261 | + | |
| 262 | + asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f)); | |
| 263 | + | |
| 264 | + while (bytes) { | |
| 265 | +#ifdef CONFIG_X86_64 | |
| 266 | + asm volatile("vmovdqa64 %0, %%zmm3\n\t" | |
| 267 | + "vmovdqa64 %1, %%zmm8\n\t" | |
| 268 | + "vpxorq %2, %%zmm3, %%zmm3\n\t" | |
| 269 | + "vpxorq %3, %%zmm8, %%zmm8" | |
| 270 | + : | |
| 271 | + : "m" (dq[0]), "m" (dq[64]), "m" (q[0]), | |
| 272 | + "m" (q[64])); | |
| 273 | + | |
| 274 | + /* | |
| 275 | + * 3 = q[0] ^ dq[0] | |
| 276 | + * 8 = q[64] ^ dq[64] | |
| 277 | + */ | |
| 278 | + asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t" | |
| 279 | + "vmovapd %%zmm0, %%zmm13\n\t" | |
| 280 | + "vbroadcasti64x2 %1, %%zmm1\n\t" | |
| 281 | + "vmovapd %%zmm1, %%zmm14" | |
| 282 | + : | |
| 283 | + : "m" (qmul[0]), "m" (qmul[16])); | |
| 284 | + | |
| 285 | + asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t" | |
| 286 | + "vpsraw $4, %%zmm8, %%zmm12\n\t" | |
| 287 | + "vpandq %%zmm7, %%zmm3, %%zmm3\n\t" | |
| 288 | + "vpandq %%zmm7, %%zmm8, %%zmm8\n\t" | |
| 289 | + "vpandq %%zmm7, %%zmm6, %%zmm6\n\t" | |
| 290 | + "vpandq %%zmm7, %%zmm12, %%zmm12\n\t" | |
| 291 | + "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t" | |
| 292 | + "vpshufb %%zmm8, %%zmm13, %%zmm13\n\t" | |
| 293 | + "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t" | |
| 294 | + "vpshufb %%zmm12, %%zmm14, %%zmm14\n\t" | |
| 295 | + "vpxorq %%zmm0, %%zmm1, %%zmm1\n\t" | |
| 296 | + "vpxorq %%zmm13, %%zmm14, %%zmm14" | |
| 297 | + : | |
| 298 | + : ); | |
| 299 | + | |
| 300 | + /* | |
| 301 | + * 1 = qmul[q[0] ^ dq[0]] | |
| 302 | + * 14 = qmul[q[64] ^ dq[64]] | |
| 303 | + */ | |
| 304 | + asm volatile("vmovdqa64 %0, %%zmm2\n\t" | |
| 305 | + "vmovdqa64 %1, %%zmm12\n\t" | |
| 306 | + "vpxorq %%zmm1, %%zmm2, %%zmm2\n\t" | |
| 307 | + "vpxorq %%zmm14, %%zmm12, %%zmm12" | |
| 308 | + : | |
| 309 | + : "m" (p[0]), "m" (p[64])); | |
| 310 | + | |
| 311 | + /* | |
| 312 | + * 2 = p[0] ^ qmul[q[0] ^ dq[0]] | |
| 313 | + * 12 = p[64] ^ qmul[q[64] ^ dq[64]] | |
| 314 | + */ | |
| 315 | + | |
| 316 | + asm volatile("vmovdqa64 %%zmm1, %0\n\t" | |
| 317 | + "vmovdqa64 %%zmm14, %1\n\t" | |
| 318 | + "vmovdqa64 %%zmm2, %2\n\t" | |
| 319 | + "vmovdqa64 %%zmm12,%3" | |
| 320 | + : | |
| 321 | + : "m" (dq[0]), "m" (dq[64]), "m" (p[0]), | |
| 322 | + "m" (p[64])); | |
| 323 | + | |
| 324 | + bytes -= 128; | |
| 325 | + p += 128; | |
| 326 | + q += 128; | |
| 327 | + dq += 128; | |
| 328 | +#else | |
| 329 | + asm volatile("vmovdqa64 %0, %%zmm3\n\t" | |
| 330 | + "vpxorq %1, %%zmm3, %%zmm3" | |
| 331 | + : | |
| 332 | + : "m" (dq[0]), "m" (q[0])); | |
| 333 | + | |
| 334 | + /* 3 = q ^ dq */ | |
| 335 | + | |
| 336 | + asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t" | |
| 337 | + "vbroadcasti64x2 %1, %%zmm1" | |
| 338 | + : | |
| 339 | + : "m" (qmul[0]), "m" (qmul[16])); | |
| 340 | + | |
| 341 | + asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t" | |
| 342 | + "vpandq %%zmm7, %%zmm3, %%zmm3\n\t" | |
| 343 | + "vpandq %%zmm7, %%zmm6, %%zmm6\n\t" | |
| 344 | + "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t" | |
| 345 | + "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t" | |
| 346 | + "vpxorq %%zmm0, %%zmm1, %%zmm1" | |
| 347 | + : | |
| 348 | + : ); | |
| 349 | + | |
| 350 | + /* 1 = qmul[q ^ dq] */ | |
| 351 | + | |
| 352 | + asm volatile("vmovdqa64 %0, %%zmm2\n\t" | |
| 353 | + "vpxorq %%zmm1, %%zmm2, %%zmm2" | |
| 354 | + : | |
| 355 | + : "m" (p[0])); | |
| 356 | + | |
| 357 | + /* 2 = p ^ qmul[q ^ dq] */ | |
| 358 | + | |
| 359 | + asm volatile("vmovdqa64 %%zmm1, %0\n\t" | |
| 360 | + "vmovdqa64 %%zmm2, %1" | |
| 361 | + : | |
| 362 | + : "m" (dq[0]), "m" (p[0])); | |
| 363 | + | |
| 364 | + bytes -= 64; | |
| 365 | + p += 64; | |
| 366 | + q += 64; | |
| 367 | + dq += 64; | |
| 368 | +#endif | |
| 369 | + } | |
| 370 | + | |
| 371 | + kernel_fpu_end(); | |
| 372 | +} | |
| 373 | + | |
| 374 | +const struct raid6_recov_calls raid6_recov_avx512 = { | |
| 375 | + .data2 = raid6_2data_recov_avx512, | |
| 376 | + .datap = raid6_datap_recov_avx512, | |
| 377 | + .valid = raid6_has_avx512, | |
| 378 | +#ifdef CONFIG_X86_64 | |
| 379 | + .name = "avx512x2", | |
| 380 | +#else | |
| 381 | + .name = "avx512x1", | |
| 382 | +#endif | |
| 383 | + .priority = 3, | |
| 384 | +}; | |
| 385 | + | |
| 386 | +#else | |
| 387 | +#warning "your version of binutils lacks AVX512 support" | |
| 388 | +#endif | |
| @@ -32,10 +32,13 @@ ifeq ($(ARCH),arm64) · 31 unmodified lines | ||
| 32 | 32 | endif |
| 33 | 33 | |
| 34 | 34 | ifeq ($(IS_X86),yes) |
| 35 | − OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o | |
| 35 | + OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o | |
| 36 | 36 | CFLAGS += $(shell echo "vpbroadcastb %xmm0, %ymm1" | \ |
| 37 | 37 | gcc -c -x assembler - >&/dev/null && \ |
| 38 | 38 | rm ./-.o && echo -DCONFIG_AS_AVX2=1) |
| 39 | + CFLAGS += $(shell echo "vpmovm2b %k1, %zmm5" | \ | |
| 40 | + gcc -c -x assembler - >&/dev/null && \ | |
| 41 | + rm ./-.o && echo -DCONFIG_AS_AVX512=1) | |
| 39 | 42 | else ifeq ($(HAS_NEON),yes) |
| 40 | 43 | OBJS += neon.o neon1.o neon2.o neon4.o neon8.o |
| 41 | 44 | CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1 |
| @@ -21,12 +21,13 @@ · 20 unmodified lines | ||
| 21 | 21 | |
| 22 | 22 | #define NDISKS 16 /* Including P and Q */ |
| 23 | 23 | |
| 24 | −const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256))); | |
| 24 | +const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); | |
| 25 | 25 | struct raid6_calls raid6_call; |
| 26 | 26 | |
| 27 | 27 | char *dataptrs[NDISKS]; |
| 28 | −char data[NDISKS][PAGE_SIZE]; | |
| 29 | −char recovi[PAGE_SIZE], recovj[PAGE_SIZE]; | |
| 28 | +char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); | |
| 29 | +char recovi[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); | |
| 30 | +char recovj[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); | |
| 30 | 31 | |
| 31 | 32 | static void makedata(int start, int stop) |
| 32 | 33 | { |
| @@ -46,6 +46,16 @@ static inline void kernel_fpu_end(void) · 45 unmodified lines | ||
| 46 | 46 | #define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */ |
| 47 | 47 | #define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */ |
| 48 | 48 | #define X86_FEATURE_AVX2 (9*32+ 5) /* AVX2 instructions */ |
| 49 | +#define X86_FEATURE_AVX512F (9*32+16) /* AVX-512 Foundation */ | |
| 50 | +#define X86_FEATURE_AVX512DQ (9*32+17) /* AVX-512 DQ (Double/Quad granular) | |
| 51 | + * Instructions | |
| 52 | + */ | |
| 53 | +#define X86_FEATURE_AVX512BW (9*32+30) /* AVX-512 BW (Byte/Word granular) | |
| 54 | + * Instructions | |
| 55 | + */ | |
| 56 | +#define X86_FEATURE_AVX512VL (9*32+31) /* AVX-512 VL (128/256 Vector Length) | |
| 57 | + * Extensions | |
| 58 | + */ | |
| 49 | 59 | #define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */ |
| 50 | 60 | |
| 51 | 61 | /* Should work well enough on modern CPUs for testing */ |