io_uring/waitid: use io_waitid_remove_wq() consistently

[ Upstream commit ab673c1bcaf20ac70352eeb6bf5b828462676693 ]

Use it everywhere that the wait_queue_entry is removed from the head,
and be a bit more cautious in zeroing out iw->head whenever the entry is
removed from the list.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
Stable-dep-of: b994ace83a2b ("io_uring/waitid: fix KCSAN warning on io_waitid->head")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/io_uring/waitid.c b/io_uring/waitid.c
index 41d5eed6c5723a948196e8dc746a8749e24341bf..1137e3a166d12b49822d63735eaebcf7d59cd87c 100644 (file)
--- a/io_uring/waitid.c
+++ b/io_uring/waitid.c
@@ -193,18 +193,18 @@ bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
 static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req)
 {
        struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
-       struct io_waitid_async *iwa = req->async_data;
 
        if (!atomic_sub_return(1, &iw->refs))
                return false;
 
+       io_waitid_remove_wq(req);
+
        /*
         * Wakeup triggered, racing with us. It was prevented from
         * completing because of that, queue up the tw to do that.
         */
        req->io_task_work.func = io_waitid_cb;
        io_req_task_work_add(req);
-       remove_wait_queue(iw->head, &iwa->wo.child_wait);
        return true;
 }
 
@@ -265,6 +265,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,
                return 0;
 
        list_del_init(&wait->entry);
+       iw->head = NULL;
 
        /* cancel is in progress */
        if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
@@ -291,6 +292,7 @@ int io_waitid_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
        iw->which = READ_ONCE(sqe->len);
        iw->upid = READ_ONCE(sqe->fd);
        iw->options = READ_ONCE(sqe->file_index);
+       iw->head = NULL;
        iw->infop = u64_to_user_ptr(READ_ONCE(sqe->addr2));
        memset(&iw->info, 0, sizeof(iw->info));
        return 0;
@@ -322,11 +324,16 @@ int io_waitid(struct io_kiocb *req, unsigned int issue_flags)
         * callback.
         */
        io_ring_submit_lock(ctx, issue_flags);
+
+       /*
+        * iw->head is valid under the ring lock, and as long as the request
+        * is on the waitid_list where cancelations may find it.
+        */
+       iw->head = &current->signal->wait_chldexit;
        hlist_add_head(&req->hash_node, &ctx->waitid_list);
 
        init_waitqueue_func_entry(&iwa->wo.child_wait, io_waitid_wait);
        iwa->wo.child_wait.private = req->tctx->task;
-       iw->head = &current->signal->wait_chldexit;
        add_wait_queue(iw->head, &iwa->wo.child_wait);
 
        ret = __do_wait(&iwa->wo);
@@ -349,7 +356,7 @@ int io_waitid(struct io_kiocb *req, unsigned int issue_flags)
        }
 
        hlist_del_init(&req->hash_node);
-       remove_wait_queue(iw->head, &iwa->wo.child_wait);
+       io_waitid_remove_wq(req);
        ret = io_waitid_finish(req, ret);
 
        io_ring_submit_unlock(ctx, issue_flags);