lib/raid6: fix the x1 SIMD gen_syndrome variants for a single data disk

The single-unrolled x86 syndrome generators (sse1x1, sse2x1, avx2x1 and
avx512x1) software-pipeline their data loads: before the loop over the
remaining data disks (z0-2 .. 0) they load dptr[z0-1], and after the
loop they fold that register in.  With exactly one data disk (disks == 3,
z0 == 0) the preload reads dptr[-1], one pointer BEFORE the caller's
array.  Whatever is stored there is then dereferenced for `bytes` bytes
and XORed into both outputs: a NULL oopses, anything else silently
produces P = D0 ^ X and Q = 2*D0 ^ X instead of P = Q = D0.

md never gets here (raid6 needs four devices there), but btrfs allows
raid6 on three devices, and a wider btrfs raid6 with unevenly sized
devices creates 3-wide chunks as soon as the small devices are full.
Which implementation runs is decided by the boot-time benchmark, so the
same filesystem is fine on a boot where x2 or x4 won and crashes on the
boot where x1 won:

  raid6: using algorithm avx2x1 gen() 29714 MB/s
  ...
  BUG: kernel NULL pointer dereference, address: 0000000000000000
  Workqueue: btrfs-rmw rmw_rbio_work
  RIP: 0010:raid6_avx21_gen_syndrome+0xaa/0x150
  Call Trace:
   rmw_rbio+0x9fe/0x15b0
   rmw_rbio_work+0x29b/0x400
   process_one_work+0x239/0x5d0

on a 9-device btrfs raid6 (6 x 512M + 3 x 6G) writing into its first
3-wide chunk.  lib/raid6/test with NDISKS set to 3 segfaults on the same
line (avx2.c:54) under AddressSanitizer, and every x1 variant fails the
same way.

Give the x1 variants the loop shape the x2 and x4 variants already use:
copy the highest data disk into P and Q, then loop over z0-1 .. 0
multiplying Q and XORing each disk in as it is loaded.  With one data
disk the loop body never runs and P = Q = D0 fall out of the initial
copy.  The generic integer variants and the other architectures already
loop from z0-1 and are not affected.

With the change lib/raid6/test passes for every algorithm at NDISKS 3
and 16 (AddressSanitizer clean); a 3-device btrfs raid6 written with the
forced avx2x1 implementation has P == Q == D0 on disk and reconstructs
after losing any device.

Fixes: 53b381b3abeb ("Btrfs: RAID-5/6 support") # first user of 3-disk raid6
Cc: stable@vger.kernel.org
Assisted-by: Claude:claude-fable-5-1

diff --git a/lib/raid6/avx2.c b/lib/raid6/avx2.c
index 059024234dce15bd826afd33428494870b8e93ab..a176c169a2607583cb1c4f6d1b5b088f841d1265 100644 (file)
--- a/lib/raid6/avx2.c
+++ b/lib/raid6/avx2.c
@@ -49,25 +49,17 @@ static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs)
        for (d = 0; d < bytes; d += 32) {
                asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
                asm volatile("vmovdqa %0,%%ymm2" : : "m" (dptr[z0][d]));/* P[0] */
-               asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
                asm volatile("vmovdqa %ymm2,%ymm4");/* Q[0] */
-               asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z0-1][d]));
-               for (z = z0-2; z >= 0; z--) {
+               for (z = z0-1; z >= 0; z--) {
                        asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
                        asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");
                        asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
                        asm volatile("vpand %ymm0,%ymm5,%ymm5");
                        asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+                       asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z][d]));
                        asm volatile("vpxor %ymm6,%ymm2,%ymm2");
                        asm volatile("vpxor %ymm6,%ymm4,%ymm4");
-                       asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z][d]));
                }
-               asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");
-               asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-               asm volatile("vpand %ymm0,%ymm5,%ymm5");
-               asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-               asm volatile("vpxor %ymm6,%ymm2,%ymm2");
-               asm volatile("vpxor %ymm6,%ymm4,%ymm4");
 
                asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
                asm volatile("vpxor %ymm2,%ymm2,%ymm2");

diff --git a/lib/raid6/avx512.c b/lib/raid6/avx512.c
index 009bd0adeebf0388353a3cfbc531d3abecea1add..1c55e2cc8c6060e37abff641e7b3ea6058b4f7a4 100644 (file)
--- a/lib/raid6/avx512.c
+++ b/lib/raid6/avx512.c
@@ -59,32 +59,23 @@ static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
        for (d = 0; d < bytes; d += 64) {
                asm volatile("prefetchnta %0\n\t"
                             "vmovdqa64 %0,%%zmm2\n\t"     /* P[0] */
-                            "prefetchnta %1\n\t"
-                            "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
-                            "vmovdqa64 %1,%%zmm6"
+                            "vmovdqa64 %%zmm2,%%zmm4"     /* Q[0] */
                             :
-                            : "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
-               for (z = z0-2; z >= 0; z--) {
+                            : "m" (dptr[z0][d]));
+               for (z = z0-1; z >= 0; z--) {
                        asm volatile("prefetchnta %0\n\t"
                                     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
                                     "vpmovm2b %%k1,%%zmm5\n\t"
                                     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
                                     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
                                     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+                                    "vmovdqa64 %0,%%zmm6\n\t"
                                     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
-                                    "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
-                                    "vmovdqa64 %0,%%zmm6"
+                                    "vpxorq %%zmm6,%%zmm4,%%zmm4"
                                     :
                                     : "m" (dptr[z][d]));
                }
-               asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
-                            "vpmovm2b %%k1,%%zmm5\n\t"
-                            "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-                            "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-                            "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-                            "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
-                            "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
-                            "vmovntdq %%zmm2,%0\n\t"
+               asm volatile("vmovntdq %%zmm2,%0\n\t"
                             "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
                             "vmovntdq %%zmm4,%1\n\t"
                             "vpxorq %%zmm4,%%zmm4,%%zmm4"

diff --git a/lib/raid6/sse1.c b/lib/raid6/sse1.c
index 692fa3a93bf0bf8b956a443e51792f2948074f8c..aa30b865ada080f71730d3d8339280cb406f1604 100644 (file)
--- a/lib/raid6/sse1.c
+++ b/lib/raid6/sse1.c
@@ -55,27 +55,18 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs)
        for ( d = 0 ; d < bytes ; d += 8 ) {
                asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
                asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
-               asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
                asm volatile("movq %mm2,%mm4"); /* Q[0] */
-               asm volatile("movq %0,%%mm6" : : "m" (dptr[z0-1][d]));
-               for ( z = z0-2 ; z >= 0 ; z-- ) {
+               for ( z = z0-1 ; z >= 0 ; z-- ) {
                        asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
                        asm volatile("pcmpgtb %mm4,%mm5");
                        asm volatile("paddb %mm4,%mm4");
                        asm volatile("pand %mm0,%mm5");
                        asm volatile("pxor %mm5,%mm4");
                        asm volatile("pxor %mm5,%mm5");
+                       asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
                        asm volatile("pxor %mm6,%mm2");
                        asm volatile("pxor %mm6,%mm4");
-                       asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
                }
-               asm volatile("pcmpgtb %mm4,%mm5");
-               asm volatile("paddb %mm4,%mm4");
-               asm volatile("pand %mm0,%mm5");
-               asm volatile("pxor %mm5,%mm4");
-               asm volatile("pxor %mm5,%mm5");
-               asm volatile("pxor %mm6,%mm2");
-               asm volatile("pxor %mm6,%mm4");
 
                asm volatile("movntq %%mm2,%0" : "=m" (p[d]));
                asm volatile("movntq %%mm4,%0" : "=m" (q[d]));

diff --git a/lib/raid6/sse2.c b/lib/raid6/sse2.c
index 2930220249c901e708eea76905fdcceaef29baf1..7288cbba13ebf3ede4834d94ad23032e9947a46c 100644 (file)
--- a/lib/raid6/sse2.c
+++ b/lib/raid6/sse2.c
@@ -51,27 +51,18 @@ static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
        for ( d = 0 ; d < bytes ; d += 16 ) {
                asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
                asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
-               asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
                asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
-               asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d]));
-               for ( z = z0-2 ; z >= 0 ; z-- ) {
+               for ( z = z0-1 ; z >= 0 ; z-- ) {
                        asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
                        asm volatile("pcmpgtb %xmm4,%xmm5");
                        asm volatile("paddb %xmm4,%xmm4");
                        asm volatile("pand %xmm0,%xmm5");
                        asm volatile("pxor %xmm5,%xmm4");
                        asm volatile("pxor %xmm5,%xmm5");
+                       asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
                        asm volatile("pxor %xmm6,%xmm2");
                        asm volatile("pxor %xmm6,%xmm4");
-                       asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
                }
-               asm volatile("pcmpgtb %xmm4,%xmm5");
-               asm volatile("paddb %xmm4,%xmm4");
-               asm volatile("pand %xmm0,%xmm5");
-               asm volatile("pxor %xmm5,%xmm4");
-               asm volatile("pxor %xmm5,%xmm5");
-               asm volatile("pxor %xmm6,%xmm2");
-               asm volatile("pxor %xmm6,%xmm4");
 
                asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
                asm volatile("pxor %xmm2,%xmm2");