return nr;
}
-static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
+static __cold noinline bool io_queue_deferred(struct io_ring_ctx *ctx)
{
- bool drain_seen = false, first = true;
+ bool drain_seen = false, first = true, drain_queued = false;
lockdep_assert_held(&ctx->uring_lock);
__io_req_caches_free(ctx);
drain_seen |= de->req->flags & REQ_F_IO_DRAIN;
if ((drain_seen || first) && ctx->nr_req_allocated != ctx->nr_drained)
- return;
+ return drain_queued;
list_del_init(&de->list);
ctx->nr_drained -= io_linked_nr(de->req);
- /* it runs alone, track its completion unless it's a link */
- if ((de->req->flags & REQ_F_IO_DRAIN) && !de->req->link)
- ctx->drain_req = de->req;
+ if (de->req->flags & REQ_F_IO_DRAIN) {
+ drain_queued = true;
+ /* it runs alone, track its completion unless it's a link */
+ if (!de->req->link)
+ ctx->drain_req = de->req;
+ }
io_req_task_queue(de->req);
kfree(de);
first = false;
}
+ return drain_queued;
}
void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
}
if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) {
- io_queue_deferred(ctx);
+ /* a drain request started from the list keeps the drain going */
+ if (io_queue_deferred(ctx))
+ drain_done = false;
if (drain_done && list_empty(&ctx->defer_list))
ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
}