drm/amdkfd: use iosys_map for CWSR buffer access

After moving TBA/TMA from GTT to VRAM for GFX9.4.2+ in commit
5088a1ba6d6d ("drm/amdkfd: move TBA/TMA from system to device memory"),
direct pointer dereferences to CWSR buffers became unsafe because VRAM
is accessed via MMIO (PCI BAR mappings).

Direct writes like 'tma[2] = enabled' and memcpy() can fail or
produce incorrect results on non-x86 architectures because:
- MMIO requires specific accessor functions (writeq/readq)
- Compiler optimizations may generate invalid instruction sequences
- No guarantee of proper memory barriers or atomic access

This patch converts CWSR buffer access to use struct iosys_map,
which automatically handles both system memory (GTT) and MMIO
(VRAM) correctly by:
- Using writeq/writel/memcpy_toio for MMIO regions
- Using WRITE_ONCE/memcpy for system memory
- Providing proper memory barriers and access guarantees

Changes:
- Replace void *cwsr_kaddr with struct iosys_map cwsr_map
- Detect MMIO vs system memory using TTM_BO_MAP_IOMEM_MASK
- Use iosys_map_wr() for writing trap handler addresses and flags
- Use iosys_map_memcpy_to() for copying CWSR ISA code

This ensures correct operation on all architectures while maintaining
backward compatibility with older GPUs and APUs that use GTT.

Assisted-by: Claude:Claude-Opus-4.6
Signed-off-by: James Zhu <James.Zhu@amd.com>
Acked-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 90f010cbe54e1ce76628eb3791a74d55edcb83df..86a944d36aaf9e6f51ab4f72bfc21d2a19c324a8 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -32,6 +32,7 @@
 #include <linux/atomic.h>
 #include <linux/workqueue.h>
 #include <linux/spinlock.h>
+#include <linux/iosys-map.h>
 #include <uapi/linux/kfd_ioctl.h>
 #include <linux/idr.h>
 #include <linux/kfifo.h>
@@ -710,7 +711,7 @@ struct qcm_process_device {
 
        /* CWSR memory */
        struct kgd_mem *cwsr_mem;
-       void *cwsr_kaddr;
+       struct iosys_map cwsr_map;
        uint64_t cwsr_base;
        uint64_t tba_addr;
        uint64_t tma_addr;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 767c2cc8e29ec331d1c507e5c9d7b2300596a0df..7a604b8c8886252fff8425bc4a197c50a109dd46 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -33,6 +33,7 @@
 #include <linux/mman.h>
 #include <linux/file.h>
 #include <linux/pm_runtime.h>
+#include <drm/ttm/ttm_bo.h>
 #include "amdgpu_amdkfd.h"
 #include "amdgpu.h"
 #include "amdgpu_reset.h"
@@ -745,6 +746,22 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem,
                                               NULL);
 }
 
+static void kfd_process_free_gpuvm_map(struct kgd_mem *mem,
+                                      struct kfd_process_device *pdd,
+                                      struct iosys_map *map)
+{
+       struct kfd_node *dev = pdd->dev;
+
+       if (map && !iosys_map_is_null(map)) {
+               amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
+               iosys_map_clear(map);
+       }
+
+       amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
+       amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
+                                              NULL);
+}
+
 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
  *     This function should be only called right after the process
  *     is created and when kfd_processes_mutex is still being held
@@ -1186,9 +1203,9 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
                if (pdd->drm_file)
                        fput(pdd->drm_file);
 
-               if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
-                       free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
-                               get_order(KFD_CWSR_TBA_TMA_SIZE));
+               if (!iosys_map_is_null(&pdd->qpd.cwsr_map) && !pdd->qpd.cwsr_base)
+                       free_pages((unsigned long)pdd->qpd.cwsr_map.vaddr,
+                                  get_order(KFD_CWSR_TBA_TMA_SIZE));
 
                idr_destroy(&pdd->alloc_idr);
 
@@ -1495,7 +1512,7 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
        void *kaddr;
        int ret;
 
-       if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
+       if (!dev->kfd->cwsr_enabled || !iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
                return 0;
 
        if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt)
@@ -1510,17 +1527,28 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
                return ret;
 
        qpd->cwsr_mem = mem;
-       qpd->cwsr_kaddr = kaddr;
+
+       /* Set up iosys_map based on whether memory is MMIO or system memory */
+       if (mem->bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK)
+               iosys_map_set_vaddr_iomem(&qpd->cwsr_map, kaddr);
+       else
+               iosys_map_set_vaddr(&qpd->cwsr_map, kaddr);
+
        qpd->tba_addr = qpd->cwsr_base;
 
-       memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
+       /* Copy CWSR ISA to buffer using appropriate accessor */
+       iosys_map_memcpy_to(&qpd->cwsr_map, 0, dev->kfd->cwsr_isa,
+                           dev->kfd->cwsr_isa_size);
 
        kfd_process_set_trap_debug_flag(&pdd->qpd,
                                        pdd->process->debug_trap_enabled);
 
        qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
-       pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
-                qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
+       pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_map:%s at %p for pqm.\n",
+                qpd->tba_addr, qpd->tma_addr,
+                qpd->cwsr_map.is_iomem ? "iomem" : "system",
+                qpd->cwsr_map.is_iomem ? (void *)qpd->cwsr_map.vaddr_iomem :
+                                         qpd->cwsr_map.vaddr);
 
        return 0;
 }
@@ -1530,24 +1558,24 @@ static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
        struct kfd_node *dev = pdd->dev;
        struct qcm_process_device *qpd = &pdd->qpd;
 
-       if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
+       if (!dev->kfd->cwsr_enabled || iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
                return;
 
-       kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
+       kfd_process_free_gpuvm_map(qpd->cwsr_mem, pdd, &qpd->cwsr_map);
 }
 
 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
                                  uint64_t tba_addr,
                                  uint64_t tma_addr)
 {
-       if (qpd->cwsr_kaddr) {
+       if (!iosys_map_is_null(&qpd->cwsr_map)) {
                /* KFD trap handler is bound, record as second-level TBA/TMA
                 * in first-level TMA. First-level trap will jump to second.
                 */
-               uint64_t *tma =
-                       (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-               tma[0] = tba_addr;
-               tma[1] = tma_addr;
+               iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET,
+                            uint64_t, tba_addr);
+               iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET + sizeof(uint64_t),
+                            uint64_t, tma_addr);
        } else {
                /* No trap handler bound, bind as first-level TBA/TMA. */
                qpd->tba_addr = tba_addr;
@@ -1613,10 +1641,10 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
                                     bool enabled)
 {
-       if (qpd->cwsr_kaddr) {
-               uint64_t *tma =
-                       (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-               tma[2] = enabled;
+       if (!iosys_map_is_null(&qpd->cwsr_map)) {
+               iosys_map_wr(&qpd->cwsr_map,
+                            KFD_CWSR_TMA_OFFSET + 2 * sizeof(uint64_t),
+                            uint64_t, enabled);
        }
 }