lockdep_assert_held(&ctx->uring_lock);
__io_req_caches_free(ctx);
+ /* nothing outside of ->defer_list is alive */
+ if (ctx->nr_req_allocated == ctx->nr_drained)
+ ctx->drain_req = NULL;
while (!list_empty(&ctx->defer_list)) {
struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
list_del_init(&de->list);
ctx->nr_drained -= io_linked_nr(de->req);
+ /* it runs alone, track its completion unless it's a link */
+ if ((de->req->flags & REQ_F_IO_DRAIN) && !de->req->link)
+ ctx->drain_req = de->req;
io_req_task_queue(de->req);
kfree(de);
first = false;
/*
* Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
- * the submitter task context, IOPOLL protects with uring_lock.
+ * the submitter task context, IOPOLL protects with uring_lock. A drain
+ * request ends the drain under uring_lock as its CQE is posted.
*/
- if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) || (req->flags & REQ_F_REISSUE)) {
+ if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) ||
+ (req->flags & (REQ_F_REISSUE | REQ_F_IO_DRAIN))) {
defer_complete:
req->io_task_work.func = io_req_task_complete;
io_req_task_work_add(req);
} while (node);
}
+/*
+ * The dispatched drain request ran alone, and anything submitted after it was
+ * deferred. Once it completed, new requests may run if nothing is waiting.
+ */
+static __cold bool io_drain_req_completed(struct io_ring_ctx *ctx)
+{
+ struct io_wq_work_node *node;
+
+ __wq_list_for_each(node, &ctx->submit_state.compl_reqs) {
+ struct io_kiocb *req = container_of(node, struct io_kiocb,
+ comp_list);
+
+ /* a requeued request hasn't completed */
+ if (req == ctx->drain_req && !(req->flags & REQ_F_REISSUE)) {
+ ctx->drain_req = NULL;
+ return true;
+ }
+ }
+ return false;
+}
+
void __io_submit_flush_completions(struct io_ring_ctx *ctx)
__must_hold(&ctx->uring_lock)
{
struct io_submit_state *state = &ctx->submit_state;
struct io_wq_work_node *node;
+ bool drain_done = false;
/*
* Drop the files before the CQEs are posted, so they're released by
}
__io_cq_unlock_post(ctx);
+ if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE) && ctx->drain_req)
+ drain_done = io_drain_req_completed(ctx);
+
if (!wq_list_empty(&state->compl_reqs)) {
io_free_batch_list(ctx, state->compl_reqs.first);
INIT_WQ_LIST(&state->compl_reqs);
}
- if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE))
+ if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) {
io_queue_deferred(ctx);
+ if (drain_done && list_empty(&ctx->defer_list))
+ ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
+ }
ctx->submit_state.cq_flush = false;
}