From 8d4388d1166faa5e16fd100157c9cb3c9e982397 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 24 May 2022 09:13:46 -0600 Subject: [PATCH] io_uring: convert poll path to use io_cmd_type Remove struct io_poll_iocb from io_kiocb, and convert the poll path to use the io_cmd_type approach instead. While at it, rename io_poll_iocb to io_poll which is consistent with the other request type private structures. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 53 ++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index db2880ccb227..7459a535060c 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -556,7 +556,7 @@ struct io_uring_task { * First field must be the file pointer in all the * iocb unions! See also 'struct kiocb' in */ -struct io_poll_iocb { +struct io_poll { struct file *file; struct wait_queue_head *head; __poll_t events; @@ -919,8 +919,8 @@ enum { }; struct async_poll { - struct io_poll_iocb poll; - struct io_poll_iocb *double_poll; + struct io_poll poll; + struct io_poll *double_poll; }; typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked); @@ -976,7 +976,6 @@ struct io_kiocb { */ struct file *file; struct io_cmd_data cmd; - struct io_poll_iocb poll; struct io_poll_update poll_update; struct io_accept accept; struct io_sync sync; @@ -6579,7 +6578,7 @@ static void io_poll_mark_cancelled(struct io_kiocb *req) atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs); } -static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req) +static struct io_poll *io_poll_get_double(struct io_kiocb *req) { /* pure poll stashes this in ->async_data, poll driven retry elsewhere */ if (req->opcode == IORING_OP_POLL_ADD) @@ -6587,10 +6586,10 @@ static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req) return req->apoll->double_poll; } -static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req) +static struct io_poll *io_poll_get_single(struct io_kiocb *req) { if (req->opcode == IORING_OP_POLL_ADD) - return &req->poll; + return io_kiocb_to_cmd(req); return &req->apoll->poll; } @@ -6603,7 +6602,7 @@ static void io_poll_req_insert(struct io_kiocb *req) hlist_add_head(&req->hash_node, list); } -static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, +static void io_init_poll_iocb(struct io_poll *poll, __poll_t events, wait_queue_func_t wake_func) { poll->head = NULL; @@ -6614,7 +6613,7 @@ static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, init_waitqueue_func_entry(&poll->wait, wake_func); } -static inline void io_poll_remove_entry(struct io_poll_iocb *poll) +static inline void io_poll_remove_entry(struct io_poll *poll) { struct wait_queue_head *head = smp_load_acquire(&poll->head); @@ -6740,7 +6739,9 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) return; if (!ret) { - req->cqe.res = mangle_poll(req->cqe.res & req->poll.events); + struct io_poll *poll = io_kiocb_to_cmd(req); + + req->cqe.res = mangle_poll(req->cqe.res & poll->events); } else { req->cqe.res = ret; req_set_fail(req); @@ -6816,8 +6817,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, void *key) { struct io_kiocb *req = wqe_to_req(wait); - struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb, - wait); + struct io_poll *poll = container_of(wait, struct io_poll, wait); __poll_t mask = key_to_poll(key); if (unlikely(mask & POLLFREE)) { @@ -6863,20 +6863,20 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, return 1; } -static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, +static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt, struct wait_queue_head *head, - struct io_poll_iocb **poll_ptr) + struct io_poll **poll_ptr) { struct io_kiocb *req = pt->req; unsigned long wqe_private = (unsigned long) req; /* * The file being polled uses multiple waitqueues for poll handling - * (e.g. one for read, one for write). Setup a separate io_poll_iocb + * (e.g. one for read, one for write). Setup a separate io_poll * if this happens. */ if (unlikely(pt->nr_entries)) { - struct io_poll_iocb *first = poll; + struct io_poll *first = poll; /* double add on the same waitqueue head, ignore */ if (first->head == head) @@ -6918,13 +6918,14 @@ static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, struct poll_table_struct *p) { struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); + struct io_poll *poll = io_kiocb_to_cmd(pt->req); - __io_queue_proc(&pt->req->poll, pt, head, - (struct io_poll_iocb **) &pt->req->async_data); + __io_queue_proc(poll, pt, head, + (struct io_poll **) &pt->req->async_data); } static int __io_arm_poll_handler(struct io_kiocb *req, - struct io_poll_iocb *poll, + struct io_poll *poll, struct io_poll_table *ipt, __poll_t mask) { struct io_ring_ctx *ctx = req->ctx; @@ -7207,7 +7208,7 @@ static int io_poll_remove_prep(struct io_kiocb *req, static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_poll_iocb *poll = &req->poll; + struct io_poll *poll = io_kiocb_to_cmd(req); u32 flags; if (sqe->buf_index || sqe->off || sqe->addr) @@ -7225,13 +7226,13 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) { - struct io_poll_iocb *poll = &req->poll; + struct io_poll *poll = io_kiocb_to_cmd(req); struct io_poll_table ipt; int ret; ipt.pt._qproc = io_poll_queue_proc; - ret = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events); + ret = __io_arm_poll_handler(req, poll, &ipt, poll->events); if (!ret && ipt.error) req_set_fail(req); ret = ret ?: ipt.error; @@ -7260,9 +7261,11 @@ static int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) if (req->poll_update.update_events || req->poll_update.update_user_data) { /* only mask one event flags, keep behavior flags */ if (req->poll_update.update_events) { - preq->poll.events &= ~0xffff; - preq->poll.events |= req->poll_update.events & 0xffff; - preq->poll.events |= IO_POLL_UNMASK; + struct io_poll *poll = io_kiocb_to_cmd(preq); + + poll->events &= ~0xffff; + poll->events |= req->poll_update.events & 0xffff; + poll->events |= IO_POLL_UNMASK; } if (req->poll_update.update_user_data) preq->cqe.user_data = req->poll_update.new_user_data;