| @@ -0,0 +1,388 @@ |
| | 1 | /* |
| | 2 | * Copyright (C) 2016 Intel Corporation |
| | 3 | * |
| | 4 | * Author: Gayatri Kammela <gayatri.kammela@intel.com> |
| | 5 | * Author: Megha Dey <megha.dey@linux.intel.com> |
| | 6 | * |
| | 7 | * This program is free software; you can redistribute it and/or |
| | 8 | * modify it under the terms of the GNU General Public License |
| | 9 | * as published by the Free Software Foundation; version 2 |
| | 10 | * of the License. |
| | 11 | * |
| | 12 | */ |
| | 13 | |
| | 14 | #ifdef CONFIG_AS_AVX512 |
| | 15 | |
| | 16 | #include <linux/raid/pq.h> |
| | 17 | #include "x86.h" |
| | 18 | |
| | 19 | static int raid6_has_avx512(void) |
| | 20 | { |
| | 21 | return boot_cpu_has(X86_FEATURE_AVX2) && |
| | 22 | boot_cpu_has(X86_FEATURE_AVX) && |
| | 23 | boot_cpu_has(X86_FEATURE_AVX512F) && |
| | 24 | boot_cpu_has(X86_FEATURE_AVX512BW) && |
| | 25 | boot_cpu_has(X86_FEATURE_AVX512VL) && |
| | 26 | boot_cpu_has(X86_FEATURE_AVX512DQ); |
| | 27 | } |
| | 28 | |
| | 29 | static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, |
| | 30 | int failb, void **ptrs) |
| | 31 | { |
| | 32 | u8 *p, *q, *dp, *dq; |
| | 33 | const u8 *pbmul; /* P multiplier table for B data */ |
| | 34 | const u8 *qmul; /* Q multiplier table (for both) */ |
| | 35 | const u8 x0f = 0x0f; |
| | 36 | |
| | 37 | p = (u8 *)ptrs[disks-2]; |
| | 38 | q = (u8 *)ptrs[disks-1]; |
| | 39 | |
| | 40 | /* |
| | 41 | * Compute syndrome with zero for the missing data pages |
| | 42 | * Use the dead data pages as temporary storage for |
| | 43 | * delta p and delta q |
| | 44 | */ |
| | 45 | |
| | 46 | dp = (u8 *)ptrs[faila]; |
| | 47 | ptrs[faila] = (void *)raid6_empty_zero_page; |
| | 48 | ptrs[disks-2] = dp; |
| | 49 | dq = (u8 *)ptrs[failb]; |
| | 50 | ptrs[failb] = (void *)raid6_empty_zero_page; |
| | 51 | ptrs[disks-1] = dq; |
| | 52 | |
| | 53 | raid6_call.gen_syndrome(disks, bytes, ptrs); |
| | 54 | |
| | 55 | /* Restore pointer table */ |
| | 56 | ptrs[faila] = dp; |
| | 57 | ptrs[failb] = dq; |
| | 58 | ptrs[disks-2] = p; |
| | 59 | ptrs[disks-1] = q; |
| | 60 | |
| | 61 | /* Now, pick the proper data tables */ |
| | 62 | pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]]; |
| | 63 | qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ |
| | 64 | raid6_gfexp[failb]]]; |
| | 65 | |
| | 66 | kernel_fpu_begin(); |
| | 67 | |
| | 68 | /* zmm0 = x0f[16] */ |
| | 69 | asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f)); |
| | 70 | |
| | 71 | while (bytes) { |
| | 72 | #ifdef CONFIG_X86_64 |
| | 73 | asm volatile("vmovdqa64 %0, %%zmm1\n\t" |
| | 74 | "vmovdqa64 %1, %%zmm9\n\t" |
| | 75 | "vmovdqa64 %2, %%zmm0\n\t" |
| | 76 | "vmovdqa64 %3, %%zmm8\n\t" |
| | 77 | "vpxorq %4, %%zmm1, %%zmm1\n\t" |
| | 78 | "vpxorq %5, %%zmm9, %%zmm9\n\t" |
| | 79 | "vpxorq %6, %%zmm0, %%zmm0\n\t" |
| | 80 | "vpxorq %7, %%zmm8, %%zmm8" |
| | 81 | : |
| | 82 | : "m" (q[0]), "m" (q[64]), "m" (p[0]), |
| | 83 | "m" (p[64]), "m" (dq[0]), "m" (dq[64]), |
| | 84 | "m" (dp[0]), "m" (dp[64])); |
| | 85 | |
| | 86 | /* |
| | 87 | * 1 = dq[0] ^ q[0] |
| | 88 | * 9 = dq[64] ^ q[64] |
| | 89 | * 0 = dp[0] ^ p[0] |
| | 90 | * 8 = dp[64] ^ p[64] |
| | 91 | */ |
| | 92 | |
| | 93 | asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t" |
| | 94 | "vbroadcasti64x2 %1, %%zmm5" |
| | 95 | : |
| | 96 | : "m" (qmul[0]), "m" (qmul[16])); |
| | 97 | |
| | 98 | asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t" |
| | 99 | "vpsraw $4, %%zmm9, %%zmm12\n\t" |
| | 100 | "vpandq %%zmm7, %%zmm1, %%zmm1\n\t" |
| | 101 | "vpandq %%zmm7, %%zmm9, %%zmm9\n\t" |
| | 102 | "vpandq %%zmm7, %%zmm3, %%zmm3\n\t" |
| | 103 | "vpandq %%zmm7, %%zmm12, %%zmm12\n\t" |
| | 104 | "vpshufb %%zmm9, %%zmm4, %%zmm14\n\t" |
| | 105 | "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t" |
| | 106 | "vpshufb %%zmm12, %%zmm5, %%zmm15\n\t" |
| | 107 | "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t" |
| | 108 | "vpxorq %%zmm14, %%zmm15, %%zmm15\n\t" |
| | 109 | "vpxorq %%zmm4, %%zmm5, %%zmm5" |
| | 110 | : |
| | 111 | : ); |
| | 112 | |
| | 113 | /* |
| | 114 | * 5 = qx[0] |
| | 115 | * 15 = qx[64] |
| | 116 | */ |
| | 117 | |
| | 118 | asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t" |
| | 119 | "vbroadcasti64x2 %1, %%zmm1\n\t" |
| | 120 | "vpsraw $4, %%zmm0, %%zmm2\n\t" |
| | 121 | "vpsraw $4, %%zmm8, %%zmm6\n\t" |
| | 122 | "vpandq %%zmm7, %%zmm0, %%zmm3\n\t" |
| | 123 | "vpandq %%zmm7, %%zmm8, %%zmm14\n\t" |
| | 124 | "vpandq %%zmm7, %%zmm2, %%zmm2\n\t" |
| | 125 | "vpandq %%zmm7, %%zmm6, %%zmm6\n\t" |
| | 126 | "vpshufb %%zmm14, %%zmm4, %%zmm12\n\t" |
| | 127 | "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t" |
| | 128 | "vpshufb %%zmm6, %%zmm1, %%zmm13\n\t" |
| | 129 | "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t" |
| | 130 | "vpxorq %%zmm4, %%zmm1, %%zmm1\n\t" |
| | 131 | "vpxorq %%zmm12, %%zmm13, %%zmm13" |
| | 132 | : |
| | 133 | : "m" (pbmul[0]), "m" (pbmul[16])); |
| | 134 | |
| | 135 | /* |
| | 136 | * 1 = pbmul[px[0]] |
| | 137 | * 13 = pbmul[px[64]] |
| | 138 | */ |
| | 139 | asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t" |
| | 140 | "vpxorq %%zmm15, %%zmm13, %%zmm13" |
| | 141 | : |
| | 142 | : ); |
| | 143 | |
| | 144 | /* |
| | 145 | * 1 = db = DQ |
| | 146 | * 13 = db[64] = DQ[64] |
| | 147 | */ |
| | 148 | asm volatile("vmovdqa64 %%zmm1, %0\n\t" |
| | 149 | "vmovdqa64 %%zmm13,%1\n\t" |
| | 150 | "vpxorq %%zmm1, %%zmm0, %%zmm0\n\t" |
| | 151 | "vpxorq %%zmm13, %%zmm8, %%zmm8" |
| | 152 | : |
| | 153 | : "m" (dq[0]), "m" (dq[64])); |
| | 154 | |
| | 155 | asm volatile("vmovdqa64 %%zmm0, %0\n\t" |
| | 156 | "vmovdqa64 %%zmm8, %1" |
| | 157 | : |
| | 158 | : "m" (dp[0]), "m" (dp[64])); |
| | 159 | |
| | 160 | bytes -= 128; |
| | 161 | p += 128; |
| | 162 | q += 128; |
| | 163 | dp += 128; |
| | 164 | dq += 128; |
| | 165 | #else |
| | 166 | asm volatile("vmovdqa64 %0, %%zmm1\n\t" |
| | 167 | "vmovdqa64 %1, %%zmm0\n\t" |
| | 168 | "vpxorq %2, %%zmm1, %%zmm1\n\t" |
| | 169 | "vpxorq %3, %%zmm0, %%zmm0" |
| | 170 | : |
| | 171 | : "m" (*q), "m" (*p), "m"(*dq), "m" (*dp)); |
| | 172 | |
| | 173 | /* 1 = dq ^ q; 0 = dp ^ p */ |
| | 174 | |
| | 175 | asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t" |
| | 176 | "vbroadcasti64x2 %1, %%zmm5" |
| | 177 | : |
| | 178 | : "m" (qmul[0]), "m" (qmul[16])); |
| | 179 | |
| | 180 | /* |
| | 181 | * 1 = dq ^ q |
| | 182 | * 3 = dq ^ p >> 4 |
| | 183 | */ |
| | 184 | asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t" |
| | 185 | "vpandq %%zmm7, %%zmm1, %%zmm1\n\t" |
| | 186 | "vpandq %%zmm7, %%zmm3, %%zmm3\n\t" |
| | 187 | "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t" |
| | 188 | "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t" |
| | 189 | "vpxorq %%zmm4, %%zmm5, %%zmm5" |
| | 190 | : |
| | 191 | : ); |
| | 192 | |
| | 193 | /* 5 = qx */ |
| | 194 | |
| | 195 | asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t" |
| | 196 | "vbroadcasti64x2 %1, %%zmm1" |
| | 197 | : |
| | 198 | : "m" (pbmul[0]), "m" (pbmul[16])); |
| | 199 | |
| | 200 | asm volatile("vpsraw $4, %%zmm0, %%zmm2\n\t" |
| | 201 | "vpandq %%zmm7, %%zmm0, %%zmm3\n\t" |
| | 202 | "vpandq %%zmm7, %%zmm2, %%zmm2\n\t" |
| | 203 | "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t" |
| | 204 | "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t" |
| | 205 | "vpxorq %%zmm4, %%zmm1, %%zmm1" |
| | 206 | : |
| | 207 | : ); |
| | 208 | |
| | 209 | /* 1 = pbmul[px] */ |
| | 210 | asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t" |
| | 211 | /* 1 = db = DQ */ |
| | 212 | "vmovdqa64 %%zmm1, %0\n\t" |
| | 213 | : |
| | 214 | : "m" (dq[0])); |
| | 215 | |
| | 216 | asm volatile("vpxorq %%zmm1, %%zmm0, %%zmm0\n\t" |
| | 217 | "vmovdqa64 %%zmm0, %0" |
| | 218 | : |
| | 219 | : "m" (dp[0])); |
| | 220 | |
| | 221 | bytes -= 64; |
| | 222 | p += 64; |
| | 223 | q += 64; |
| | 224 | dp += 64; |
| | 225 | dq += 64; |
| | 226 | #endif |
| | 227 | } |
| | 228 | |
| | 229 | kernel_fpu_end(); |
| | 230 | } |
| | 231 | |
| | 232 | static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, |
| | 233 | void **ptrs) |
| | 234 | { |
| | 235 | u8 *p, *q, *dq; |
| | 236 | const u8 *qmul; /* Q multiplier table */ |
| | 237 | const u8 x0f = 0x0f; |
| | 238 | |
| | 239 | p = (u8 *)ptrs[disks-2]; |
| | 240 | q = (u8 *)ptrs[disks-1]; |
| | 241 | |
| | 242 | /* |
| | 243 | * Compute syndrome with zero for the missing data page |
| | 244 | * Use the dead data page as temporary storage for delta q |
| | 245 | */ |
| | 246 | |
| | 247 | dq = (u8 *)ptrs[faila]; |
| | 248 | ptrs[faila] = (void *)raid6_empty_zero_page; |
| | 249 | ptrs[disks-1] = dq; |
| | 250 | |
| | 251 | raid6_call.gen_syndrome(disks, bytes, ptrs); |
| | 252 | |
| | 253 | /* Restore pointer table */ |
| | 254 | ptrs[faila] = dq; |
| | 255 | ptrs[disks-1] = q; |
| | 256 | |
| | 257 | /* Now, pick the proper data tables */ |
| | 258 | qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; |
| | 259 | |
| | 260 | kernel_fpu_begin(); |
| | 261 | |
| | 262 | asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f)); |
| | 263 | |
| | 264 | while (bytes) { |
| | 265 | #ifdef CONFIG_X86_64 |
| | 266 | asm volatile("vmovdqa64 %0, %%zmm3\n\t" |
| | 267 | "vmovdqa64 %1, %%zmm8\n\t" |
| | 268 | "vpxorq %2, %%zmm3, %%zmm3\n\t" |
| | 269 | "vpxorq %3, %%zmm8, %%zmm8" |
| | 270 | : |
| | 271 | : "m" (dq[0]), "m" (dq[64]), "m" (q[0]), |
| | 272 | "m" (q[64])); |
| | 273 | |
| | 274 | /* |
| | 275 | * 3 = q[0] ^ dq[0] |
| | 276 | * 8 = q[64] ^ dq[64] |
| | 277 | */ |
| | 278 | asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t" |
| | 279 | "vmovapd %%zmm0, %%zmm13\n\t" |
| | 280 | "vbroadcasti64x2 %1, %%zmm1\n\t" |
| | 281 | "vmovapd %%zmm1, %%zmm14" |
| | 282 | : |
| | 283 | : "m" (qmul[0]), "m" (qmul[16])); |
| | 284 | |
| | 285 | asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t" |
| | 286 | "vpsraw $4, %%zmm8, %%zmm12\n\t" |
| | 287 | "vpandq %%zmm7, %%zmm3, %%zmm3\n\t" |
| | 288 | "vpandq %%zmm7, %%zmm8, %%zmm8\n\t" |
| | 289 | "vpandq %%zmm7, %%zmm6, %%zmm6\n\t" |
| | 290 | "vpandq %%zmm7, %%zmm12, %%zmm12\n\t" |
| | 291 | "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t" |
| | 292 | "vpshufb %%zmm8, %%zmm13, %%zmm13\n\t" |
| | 293 | "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t" |
| | 294 | "vpshufb %%zmm12, %%zmm14, %%zmm14\n\t" |
| | 295 | "vpxorq %%zmm0, %%zmm1, %%zmm1\n\t" |
| | 296 | "vpxorq %%zmm13, %%zmm14, %%zmm14" |
| | 297 | : |
| | 298 | : ); |
| | 299 | |
| | 300 | /* |
| | 301 | * 1 = qmul[q[0] ^ dq[0]] |
| | 302 | * 14 = qmul[q[64] ^ dq[64]] |
| | 303 | */ |
| | 304 | asm volatile("vmovdqa64 %0, %%zmm2\n\t" |
| | 305 | "vmovdqa64 %1, %%zmm12\n\t" |
| | 306 | "vpxorq %%zmm1, %%zmm2, %%zmm2\n\t" |
| | 307 | "vpxorq %%zmm14, %%zmm12, %%zmm12" |
| | 308 | : |
| | 309 | : "m" (p[0]), "m" (p[64])); |
| | 310 | |
| | 311 | /* |
| | 312 | * 2 = p[0] ^ qmul[q[0] ^ dq[0]] |
| | 313 | * 12 = p[64] ^ qmul[q[64] ^ dq[64]] |
| | 314 | */ |
| | 315 | |
| | 316 | asm volatile("vmovdqa64 %%zmm1, %0\n\t" |
| | 317 | "vmovdqa64 %%zmm14, %1\n\t" |
| | 318 | "vmovdqa64 %%zmm2, %2\n\t" |
| | 319 | "vmovdqa64 %%zmm12,%3" |
| | 320 | : |
| | 321 | : "m" (dq[0]), "m" (dq[64]), "m" (p[0]), |
| | 322 | "m" (p[64])); |
| | 323 | |
| | 324 | bytes -= 128; |
| | 325 | p += 128; |
| | 326 | q += 128; |
| | 327 | dq += 128; |
| | 328 | #else |
| | 329 | asm volatile("vmovdqa64 %0, %%zmm3\n\t" |
| | 330 | "vpxorq %1, %%zmm3, %%zmm3" |
| | 331 | : |
| | 332 | : "m" (dq[0]), "m" (q[0])); |
| | 333 | |
| | 334 | /* 3 = q ^ dq */ |
| | 335 | |
| | 336 | asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t" |
| | 337 | "vbroadcasti64x2 %1, %%zmm1" |
| | 338 | : |
| | 339 | : "m" (qmul[0]), "m" (qmul[16])); |
| | 340 | |
| | 341 | asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t" |
| | 342 | "vpandq %%zmm7, %%zmm3, %%zmm3\n\t" |
| | 343 | "vpandq %%zmm7, %%zmm6, %%zmm6\n\t" |
| | 344 | "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t" |
| | 345 | "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t" |
| | 346 | "vpxorq %%zmm0, %%zmm1, %%zmm1" |
| | 347 | : |
| | 348 | : ); |
| | 349 | |
| | 350 | /* 1 = qmul[q ^ dq] */ |
| | 351 | |
| | 352 | asm volatile("vmovdqa64 %0, %%zmm2\n\t" |
| | 353 | "vpxorq %%zmm1, %%zmm2, %%zmm2" |
| | 354 | : |
| | 355 | : "m" (p[0])); |
| | 356 | |
| | 357 | /* 2 = p ^ qmul[q ^ dq] */ |
| | 358 | |
| | 359 | asm volatile("vmovdqa64 %%zmm1, %0\n\t" |
| | 360 | "vmovdqa64 %%zmm2, %1" |
| | 361 | : |
| | 362 | : "m" (dq[0]), "m" (p[0])); |
| | 363 | |
| | 364 | bytes -= 64; |
| | 365 | p += 64; |
| | 366 | q += 64; |
| | 367 | dq += 64; |
| | 368 | #endif |
| | 369 | } |
| | 370 | |
| | 371 | kernel_fpu_end(); |
| | 372 | } |
| | 373 | |
| | 374 | const struct raid6_recov_calls raid6_recov_avx512 = { |
| | 375 | .data2 = raid6_2data_recov_avx512, |
| | 376 | .datap = raid6_datap_recov_avx512, |
| | 377 | .valid = raid6_has_avx512, |
| | 378 | #ifdef CONFIG_X86_64 |
| | 379 | .name = "avx512x2", |
| | 380 | #else |
| | 381 | .name = "avx512x1", |
| | 382 | #endif |
| | 383 | .priority = 3, |
| | 384 | }; |
| | 385 | |
| | 386 | #else |
| | 387 | #warning "your version of binutils lacks AVX512 support" |
| | 388 | #endif |