uprobes: encapsulate preparation of uprobe args buffer

commit 3eaea21b4d27cff0017c20549aeb53034c58fc23 upstream.

Move the logic of fetching temporary per-CPU uprobe buffer and storing
uprobes args into it to a new helper function. Store data size as part
of this buffer, simplifying interfaces a bit, as now we only pass single
uprobe_cpu_buffer reference around, instead of pointer + dsize.

This logic was duplicated across uprobe_dispatcher and uretprobe_dispatcher,
and now will be centralized. All this is also in preparation to make
this uprobe_cpu_buffer handling logic optional in the next patch.

Link: https://lore.kernel.org/all/20240318181728.2795838-2-andrii@kernel.org/
[Masami: update for v6.9-rc3 kernel]

Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Reviewed-by: Jiri Olsa <jolsa@kernel.org>
Acked-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Stable-dep-of: 373b9338c972 ("uprobe: avoid out-of-bounds memory access of fetching args")
Signed-off-by: Sasha Levin <sashal@kernel.org>
Signed-off-by: Vamsi Krishna Brahmajosyula <vamsi-krishna.brahmajosyula@broadcom.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index 127c78aec17dbd9193b915574dae5412ecd5d081..e09eef65d32f62508cc6323975e0624e921938c8 100644 (file)
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -858,6 +858,7 @@ static const struct file_operations uprobe_profile_ops = {
 struct uprobe_cpu_buffer {
        struct mutex mutex;
        void *buf;
+       int dsize;
 };
 static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer;
 static int uprobe_buffer_refcnt;
@@ -947,9 +948,26 @@ static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)
        mutex_unlock(&ucb->mutex);
 }
 
+static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu,
+                                                      struct pt_regs *regs)
+{
+       struct uprobe_cpu_buffer *ucb;
+       int dsize, esize;
+
+       esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
+       dsize = __get_data_size(&tu->tp, regs);
+
+       ucb = uprobe_buffer_get();
+       ucb->dsize = tu->tp.size + dsize;
+
+       store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
+
+       return ucb;
+}
+
 static void __uprobe_trace_func(struct trace_uprobe *tu,
                                unsigned long func, struct pt_regs *regs,
-                               struct uprobe_cpu_buffer *ucb, int dsize,
+                               struct uprobe_cpu_buffer *ucb,
                                struct trace_event_file *trace_file)
 {
        struct uprobe_trace_entry_head *entry;
@@ -960,14 +978,14 @@ static void __uprobe_trace_func(struct trace_uprobe *tu,
 
        WARN_ON(call != trace_file->event_call);
 
-       if (WARN_ON_ONCE(tu->tp.size + dsize > PAGE_SIZE))
+       if (WARN_ON_ONCE(ucb->dsize > PAGE_SIZE))
                return;
 
        if (trace_trigger_soft_disabled(trace_file))
                return;
 
        esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
-       size = esize + tu->tp.size + dsize;
+       size = esize + ucb->dsize;
        entry = trace_event_buffer_reserve(&fbuffer, trace_file, size);
        if (!entry)
                return;
@@ -981,14 +999,14 @@ static void __uprobe_trace_func(struct trace_uprobe *tu,
                data = DATAOF_TRACE_ENTRY(entry, false);
        }
 
-       memcpy(data, ucb->buf, tu->tp.size + dsize);
+       memcpy(data, ucb->buf, ucb->dsize);
 
        trace_event_buffer_commit(&fbuffer);
 }
 
 /* uprobe handler */
 static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
-                            struct uprobe_cpu_buffer *ucb, int dsize)
+                            struct uprobe_cpu_buffer *ucb)
 {
        struct event_file_link *link;
 
@@ -997,7 +1015,7 @@ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
 
        rcu_read_lock();
        trace_probe_for_each_link_rcu(link, &tu->tp)
-               __uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file);
+               __uprobe_trace_func(tu, 0, regs, ucb, link->file);
        rcu_read_unlock();
 
        return 0;
@@ -1005,13 +1023,13 @@ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
 
 static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
                                 struct pt_regs *regs,
-                                struct uprobe_cpu_buffer *ucb, int dsize)
+                                struct uprobe_cpu_buffer *ucb)
 {
        struct event_file_link *link;
 
        rcu_read_lock();
        trace_probe_for_each_link_rcu(link, &tu->tp)
-               __uprobe_trace_func(tu, func, regs, ucb, dsize, link->file);
+               __uprobe_trace_func(tu, func, regs, ucb, link->file);
        rcu_read_unlock();
 }
 
@@ -1339,7 +1357,7 @@ static bool uprobe_perf_filter(struct uprobe_consumer *uc,
 
 static void __uprobe_perf_func(struct trace_uprobe *tu,
                               unsigned long func, struct pt_regs *regs,
-                              struct uprobe_cpu_buffer *ucb, int dsize)
+                              struct uprobe_cpu_buffer *ucb)
 {
        struct trace_event_call *call = trace_probe_event_call(&tu->tp);
        struct uprobe_trace_entry_head *entry;
@@ -1360,7 +1378,7 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
 
        esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
 
-       size = esize + tu->tp.size + dsize;
+       size = esize + ucb->dsize;
        size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);
        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
                return;
@@ -1383,13 +1401,10 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
                data = DATAOF_TRACE_ENTRY(entry, false);
        }
 
-       memcpy(data, ucb->buf, tu->tp.size + dsize);
-
-       if (size - esize > tu->tp.size + dsize) {
-               int len = tu->tp.size + dsize;
+       memcpy(data, ucb->buf, ucb->dsize);
 
-               memset(data + len, 0, size - esize - len);
-       }
+       if (size - esize > ucb->dsize)
+               memset(data + ucb->dsize, 0, size - esize - ucb->dsize);
 
        perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
                              head, NULL);
@@ -1399,21 +1414,21 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
 
 /* uprobe profile handler */
 static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs,
-                           struct uprobe_cpu_buffer *ucb, int dsize)
+                           struct uprobe_cpu_buffer *ucb)
 {
        if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))
                return UPROBE_HANDLER_REMOVE;
 
        if (!is_ret_probe(tu))
-               __uprobe_perf_func(tu, 0, regs, ucb, dsize);
+               __uprobe_perf_func(tu, 0, regs, ucb);
        return 0;
 }
 
 static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
                                struct pt_regs *regs,
-                               struct uprobe_cpu_buffer *ucb, int dsize)
+                               struct uprobe_cpu_buffer *ucb)
 {
-       __uprobe_perf_func(tu, func, regs, ucb, dsize);
+       __uprobe_perf_func(tu, func, regs, ucb);
 }
 
 int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
@@ -1479,10 +1494,8 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
        struct trace_uprobe *tu;
        struct uprobe_dispatch_data udd;
        struct uprobe_cpu_buffer *ucb;
-       int dsize, esize;
        int ret = 0;
 
-
        tu = container_of(con, struct trace_uprobe, consumer);
        tu->nhit++;
 
@@ -1494,18 +1507,14 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
        if (WARN_ON_ONCE(!uprobe_cpu_buffer))
                return 0;
 
-       dsize = __get_data_size(&tu->tp, regs);
-       esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
-
-       ucb = uprobe_buffer_get();
-       store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
+       ucb = prepare_uprobe_buffer(tu, regs);
 
        if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
-               ret |= uprobe_trace_func(tu, regs, ucb, dsize);
+               ret |= uprobe_trace_func(tu, regs, ucb);
 
 #ifdef CONFIG_PERF_EVENTS
        if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
-               ret |= uprobe_perf_func(tu, regs, ucb, dsize);
+               ret |= uprobe_perf_func(tu, regs, ucb);
 #endif
        uprobe_buffer_put(ucb);
        return ret;
@@ -1517,7 +1526,6 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con,
        struct trace_uprobe *tu;
        struct uprobe_dispatch_data udd;
        struct uprobe_cpu_buffer *ucb;
-       int dsize, esize;
 
        tu = container_of(con, struct trace_uprobe, consumer);
 
@@ -1529,18 +1537,13 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con,
        if (WARN_ON_ONCE(!uprobe_cpu_buffer))
                return 0;
 
-       dsize = __get_data_size(&tu->tp, regs);
-       esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
-
-       ucb = uprobe_buffer_get();
-       store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
-
+       ucb = prepare_uprobe_buffer(tu, regs);
        if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
-               uretprobe_trace_func(tu, func, regs, ucb, dsize);
+               uretprobe_trace_func(tu, func, regs, ucb);
 
 #ifdef CONFIG_PERF_EVENTS
        if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
-               uretprobe_perf_func(tu, func, regs, ucb, dsize);
+               uretprobe_perf_func(tu, func, regs, ucb);
 #endif
        uprobe_buffer_put(ucb);
        return 0;