drm/v3d: Serialize jobs across queues when a perfmon is attached

A non-global perfmon is meant to count events generated by a specific
submission, but the scheduler can run jobs from different queues
concurrently on the same V3D core. Without explicit serialization, an
unrelated job running in parallel with a perfmon-carrying job pollutes
the counters and generates unusable results.

To address such issue, we must enforce cross-queue serialization when we
detect a perfmon-carrying submission. It's possible to implement
serialization by enforcing two rules:

  1. A job that carries a non-global perfmon must wait for every job
    currently in-flight across all HW queues to finish.

  2. While a perfmon-carrying job is still in-flight, all subsequently
    submitted jobs must wait for it.

Note that serialization is not needed in the global perfmon case, as the
global perfmon tracks activity from all jobs, so concurrency is desirable.

Therefore, check if serialization is needed during job submission and if
so, attach fence dependences to enforce cross-queue serialization.

Reviewed-by: Iago Toral Quiroga <itoral@igalia.com>
Link: https://patch.msgid.link/20260706-v3d-perfmon-lifetime-v4-2-d7b312ff2c83@igalia.com
Signed-off-by: Maíra Canal <mcanal@igalia.com>

diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index cf836c4f13ad56db8f9c22f16f1f99445a90f171..1b88b3dff75746fa30a9e103ba98909ef7d809f5 100644 (file)
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -75,11 +75,13 @@ struct v3d_queue_state {
        spinlock_t queue_lock;
 };
 
-/* Performance monitor object. The perform lifetime is controlled by userspace
- * using perfmon related ioctls. A perfmon can be attached to a submit_cl
- * request, and when this is the case, HW perf counters will be activated just
- * before the submit_cl is submitted to the GPU and disabled when the job is
- * done. This way, only events related to a specific job will be counted.
+/* Performance monitor object
+ *
+ * The performance monitor (perfmon) lifetime is controlled by userspace using
+ * perfmon related ioctls. A perfmon can be attached to a CL or CSD submission
+ * request, and when it is, HW performance counters will be activated just
+ * before the job is submitted to the GPU and disabled when the job is done.
+ * This way, only events related to a specific submission will be counted.
  */
 struct v3d_perfmon {
        /* Tracks the number of users of the perfmon, when this counter reaches
@@ -168,13 +170,31 @@ struct v3d_dev {
 
        struct v3d_queue_state queue[V3D_MAX_QUEUES];
 
-       /* Tracks the performance monitor state. */
+       /*
+        * Tracks the performance monitor state and consistency.
+        *
+        * When a non-global perfmon is attached to a job, the scheduler must
+        * not run any other job on the HW concurrently (otherwise, the
+        * counters would be polluted by unrelated work).
+        */
        struct {
                /* Protects @active. */
                spinlock_t lock;
 
                /* Perfmon currently programmed in HW (or NULL if none). */
                struct v3d_perfmon *active;
+
+               /* Finished fence of the most recently submitted job that
+                * opened a serialization window (i.e. a job with a non-global
+                * perfmon attached).
+                */
+               struct dma_fence *fence;
+
+               /* Finished fence of the most recently submitted job on each HW
+                * queue. Used so that a new perfmon-carrying job can depend on
+                * every job currently in-flight across all queues.
+                */
+               struct dma_fence *last_hw_fence[V3D_MAX_QUEUES];
        } perfmon_state;
 
        /* Protects bo_stats */

diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 7ad81b146f02ed6171c19643b32151d1cd66e291..18c941f15e2e4cdebf878a0fed7be8b456befb10 100644 (file)
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -371,8 +371,11 @@ v3d_gem_destroy(struct drm_device *dev)
        for (q = 0; q < V3D_MAX_QUEUES; q++) {
                WARN_ON(v3d->queue[q].active_job);
                v3d_stats_put(v3d->queue[q].stats);
+               dma_fence_put(v3d->perfmon_state.last_hw_fence[q]);
        }
 
+       dma_fence_put(v3d->perfmon_state.fence);
+
        drm_mm_takedown(&v3d->mm);
 
        dma_free_coherent(v3d->drm.dev, 4096 * 1024, (void *)v3d->pt,

diff --git a/drivers/gpu/drm/v3d/v3d_perfmon.c b/drivers/gpu/drm/v3d/v3d_perfmon.c
index 3ad0f022753c844ce9985de2b9bbc0bee7d95353..07dab7fb30607d2482529a1e62aec69dbcf2d029 100644 (file)
--- a/drivers/gpu/drm/v3d/v3d_perfmon.c
+++ b/drivers/gpu/drm/v3d/v3d_perfmon.c
@@ -275,6 +275,12 @@ void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
        if (!perfmon || v3d->global_perfmon)
                return;
 
+       /* Cross-queue serialization should have drained any previous perfmon
+        * job before this one runs.
+        */
+       if (WARN_ON_ONCE(v3d->perfmon_state.active))
+               return;
+
        if (!pm_runtime_get_if_active(v3d->drm.dev))
                return;
 

diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c
index 27cc98770b37e16c49571fea0ca62338cff9daa5..d32defa7da73a692fc14ac70be6664be74c4abcb 100644 (file)
--- a/drivers/gpu/drm/v3d/v3d_submit.c
+++ b/drivers/gpu/drm/v3d/v3d_submit.c
@@ -4,6 +4,8 @@
  * Copyright (C) 2023 Raspberry Pi
  */
 
+#include <linux/dma-fence-unwrap.h>
+
 #include <drm/drm_print.h>
 #include <drm/drm_syncobj.h>
 
@@ -320,6 +322,73 @@ v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id)
        return 0;
 }
 
+/*
+ * Prepare fences to enforce job serialization when a perfmon is active. A job
+ * that carries a non-global perfmon must wait for every job currently in-flight
+ * across all HW queues to finish, otherwise concurrent unrelated work on the
+ * same core would pollute the performance counters. Symmetrically, while such a
+ * job is still in-flight, all subsequently submitted jobs must wait for it.
+ *
+ * We don't serialize the jobs when using a global perfmon as it's expected to
+ * track concurrent activity from all jobs.
+ */
+static int
+v3d_serialize_for_perfmon(struct v3d_job *job)
+{
+       struct v3d_dev *v3d = job->v3d;
+       struct dma_fence *merged;
+       bool is_global_perfmon;
+       int ret;
+
+       lockdep_assert_held(&v3d->sched_lock);
+
+       scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock)
+               is_global_perfmon = !!v3d->global_perfmon;
+
+       if (is_global_perfmon)
+               goto publish;
+
+       if (job->perfmon) {
+               for (enum v3d_queue q = 0; q < V3D_MAX_QUEUES; q++) {
+                       struct dma_fence *f = v3d->perfmon_state.last_hw_fence[q];
+
+                       if (!f || dma_fence_is_signaled(f))
+                               continue;
+
+                       ret = drm_sched_job_add_dependency(&job->base, dma_fence_get(f));
+                       if (ret)
+                               return ret;
+               }
+       } else if (v3d->perfmon_state.fence &&
+                  !dma_fence_is_signaled(v3d->perfmon_state.fence)) {
+               ret = drm_sched_job_add_dependency(&job->base,
+                                                  dma_fence_get(v3d->perfmon_state.fence));
+               if (ret)
+                       return ret;
+       }
+
+publish:
+       /*
+        * Accumulate every in-flight job on this queue into one merged fence.
+        * A HW queue is fed by several scheduler entities (one per-fd), so jobs
+        * on it can complete out of order.
+        */
+       merged = dma_fence_unwrap_merge(v3d->perfmon_state.last_hw_fence[job->queue],
+                                       job->done_fence);
+       if (!merged)
+               return -ENOMEM;
+
+       dma_fence_put(v3d->perfmon_state.last_hw_fence[job->queue]);
+       v3d->perfmon_state.last_hw_fence[job->queue] = merged;
+
+       if (job->perfmon && !is_global_perfmon) {
+               dma_fence_put(v3d->perfmon_state.fence);
+               v3d->perfmon_state.fence = dma_fence_get(job->done_fence);
+       }
+
+       return 0;
+}
+
 static void
 v3d_submit_attach_object_fences(struct v3d_submit *submit)
 {
@@ -396,6 +465,12 @@ v3d_submit_jobs(struct v3d_submit *submit, struct drm_syncobj *sync_out,
                        goto err;
        }
 
+       for (int i = 0; i < submit->job_count; i++) {
+               ret = v3d_serialize_for_perfmon(submit->jobs[i]);
+               if (ret)
+                       goto err;
+       }
+
        for (int i = 0; i < submit->job_count; i++)
                drm_sched_entity_push_job(&submit->jobs[i]->base);