History

sched_ext: Format bstr exit messages after claiming the exit

The bstr exit kfuncs format the message into a shared static buffer under a raw spinlock before initiating the exit. The lock can't be taken from NMI and needlessly serializes all bstr exits system-wide. Now that exit claiming is lock-free, reverse the order: claim the exit first and format directly into the exit_info message buffer which the claim winner owns exclusively. The new scx_exit_bstr() implements the sequence, replacing scx_bstr_format(), and the shared buffer and lock are deleted; the formatter itself is what bpf_trace_printk() already runs from NMI. scx_prog_sched() callers were relying on the lock for RCU protection, which is now provided explicitly. A malformed format no longer changes or fails the requested operation: scx_bpf_exit_bstr() keeps its graceful exit kind and scx_bpf_sub_kill_bstr() still kills the child, with a fallback message carrying the formatting errno, while the sched that supplied the bad format is aborted for its bug. Before this and the previous patch, an "any" category kfunc called from NMI context could trigger scx_error() and deadlock - e.g. a tracing prog attached to a function running in NMI calling scx_bpf_dsq_peek() on a non-existent DSQ would try to grab scx_sched_lock, which may be held by the interrupted CPU. This and the previous patch fix the deadlock: scx_error() and the bstr exit kfuncs, and thus scx_bpf_error() and scx_bpf_exit(), are now safe to call from any context including NMI.

Tejun Heo committed 3 months ago

Commit 1bf623e1 parent f883dbb3 files · +64 −40Browse files

Showing the changes to kernel (3 of 3 files). Show all

Unified · Split

3 files
  1. kernel/sched/ext/ext.c +60 −30
  2. kernel/sched/ext/internal.h +3 −4
  3. kernel/sched/ext/sub.c +1 −6
kernel/sched/ext/ext.c+60−30
@@ -196,9 +196,6 @@ static const struct rhashtable_params dsq_hash_params = { · 195 unmodified lines
196196
197197 static LLIST_HEAD(dsqs_to_free);
198198
199−DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock);
200−struct scx_bstr_buf scx_exit_bstr_buf;
201−
202199 /* ops debug dump */
203200 static DEFINE_RAW_SPINLOCK(scx_dump_lock);
204201
@@ -6760,17 +6757,12 @@ static void scx_disable_irq_workfn(struct irq_work *irq_work) · 6555 unmodified lines
67606757 kthread_queue_work(sch->helper, &sch->disable_work);
67616758 }
67626759
6763−bool scx_vexit(struct scx_sched *sch,
6764− enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu,
6765− const char *fmt, va_list args)
6760+/* finish exit_info and kick the disable work, ei->msg must already be set */
6761+static void scx_finish_exit(struct scx_sched *sch, enum scx_exit_kind kind,
6762+ s64 exit_code, s32 exit_cpu)
67666763 {
67676764 struct scx_exit_info *ei = sch->exit_info;
67686765
6769− guard(preempt)();
6770−
6771− if (!scx_claim_exit(sch, kind))
6772− return false;
6773−
67746766 ei->exit_code = exit_code;
67756767 #ifdef CONFIG_STACKTRACE
67766768 /*
@@ -6780,8 +6772,6 @@ bool scx_vexit(struct scx_sched *sch, · 3 unmodified lines
67806772 if (kind >= SCX_EXIT_ERROR && !in_nmi())
67816773 ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1);
67826774 #endif
6783− vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args);
6784−
67856775 /*
67866776 * Set ei->kind and ->reason for scx_dump_state(). They'll be set again
67876777 * in scx_disable_workfn().
@@ -6791,6 +6781,22 @@ bool scx_vexit(struct scx_sched *sch, · 3 unmodified lines
67916781 ei->exit_cpu = exit_cpu;
67926782
67936783 irq_work_queue(&sch->disable_irq_work);
6784+}
6785+
6786+bool scx_vexit(struct scx_sched *sch,
6787+ enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu,
6788+ const char *fmt, va_list args)
6789+{
6790+ struct scx_exit_info *ei = sch->exit_info;
6791+
6792+ guard(preempt)();
6793+
6794+ if (!scx_claim_exit(sch, kind))
6795+ return false;
6796+
6797+ vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args);
6798+
6799+ scx_finish_exit(sch, kind, exit_code, exit_cpu);
67946800 return true;
67956801 }
67966802
@@ -9604,12 +9610,38 @@ static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, · 2807 unmodified lines
96049610 return ret;
96059611 }
96069612
9607−__printf(3, 0)
9608−s32 scx_bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf,
9609− char *fmt, unsigned long long *data, u32 data__sz)
9613+/*
9614+ * Exit @sch with the reason formatted from a BPF-supplied bstr format. The exit
9615+ * is claimed first and the reason is formatted directly into the winner-owned
9616+ * exit_info buffer, which allows use from any context including NMI.
9617+ *
9618+ * @fmt_blame is the sched blamed for formatting failures through the
9619+ * scx_error() calls in __bstr_format() and differs from @sch when a parent
9620+ * supplies the kill reason for a child. A formatting failure doesn't revert the
9621+ * claim - @sch still exits with the claimed kind and a fallback message.
9622+ */
9623+__printf(5, 0)
9624+bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind,
9625+ s64 exit_code, struct scx_sched *fmt_blame, char *fmt,
9626+ unsigned long long *data, u32 data__sz)
96109627 {
9611− return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line),
9612− fmt, data, data__sz);
9628+ struct scx_exit_info *ei = sch->exit_info;
9629+ u64 data_buf[MAX_BPRINTF_VARARGS];
9630+ s32 ret;
9631+
9632+ guard(preempt)();
9633+
9634+ if (!scx_claim_exit(sch, kind))
9635+ return false;
9636+
9637+ ret = __bstr_format(fmt_blame, data_buf, ei->msg, SCX_EXIT_MSG_LEN,
9638+ fmt, data, data__sz);
9639+ if (ret < 0)
9640+ scnprintf(ei->msg, SCX_EXIT_MSG_LEN,
9641+ "exit message formatting failed (%d)", ret);
9642+
9643+ scx_finish_exit(sch, kind, exit_code, raw_smp_processor_id());
9644+ return true;
96139645 }
96149646
96159647 __bpf_kfunc_start_defs();
@@ -9631,14 +9663,13 @@ __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, · 15 unmodified lines
96319663 const struct bpf_prog_aux *aux)
96329664 {
96339665 struct scx_sched *sch;
9634− unsigned long flags;
96359666
9636− raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
9667+ guard(rcu)();
9668+
96379669 sch = scx_prog_sched(aux);
9638− if (likely(sch) &&
9639− scx_bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
9640− scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line);
9641− raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);
9670+ if (likely(sch))
9671+ scx_exit_bstr(sch, SCX_EXIT_UNREG_BPF, exit_code, sch, fmt,
9672+ data, data__sz);
96429673 }
96439674
96449675 /**
@@ -9656,14 +9687,13 @@ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, · 11 unmodified lines
96569687 u32 data__sz, const struct bpf_prog_aux *aux)
96579688 {
96589689 struct scx_sched *sch;
9659− unsigned long flags;
96609690
9661− raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
9691+ guard(rcu)();
9692+
96629693 sch = scx_prog_sched(aux);
9663− if (likely(sch) &&
9664− scx_bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
9665− scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line);
9666− raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);
9694+ if (likely(sch))
9695+ scx_exit_bstr(sch, SCX_EXIT_ERROR_BPF, 0, sch, fmt, data,
9696+ data__sz);
96679697 }
96689698
96699699 /**
kernel/sched/ext/internal.h+3−4
@@ -2004,15 +2004,14 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, · 2003 unmodified lines
20042004 int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops);
20052005 int scx_sched_sysfs_add(struct scx_sched *sch);
20062006 bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor);
2007−__printf(3, 0) s32 scx_bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf,
2008− char *fmt, unsigned long long *data, u32 data__sz);
2007+__printf(5, 0) bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind,
2008+ s64 exit_code, struct scx_sched *fmt_blame,
2009+ char *fmt, unsigned long long *data, u32 data__sz);
20092010
20102011 extern raw_spinlock_t scx_sched_lock;
20112012 extern struct mutex scx_enable_mutex;
20122013 extern struct percpu_rw_semaphore scx_fork_rwsem;
20132014 extern bool scx_cgroup_enabled;
2014−extern raw_spinlock_t scx_exit_bstr_buf_lock;
2015−extern struct scx_bstr_buf scx_exit_bstr_buf;
20162015 #ifdef CONFIG_EXT_SUB_SCHED
20172016 extern const struct rhashtable_params scx_sched_hash_params;
20182017 extern struct rhashtable scx_sched_hash;
kernel/sched/ext/sub.c+1−6
@@ -2155,7 +2155,6 @@ __bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, · 2154 unmodified lines
21552155 const struct bpf_prog_aux *aux)
21562156 {
21572157 struct scx_sched *parent, *child;
2158− s32 ret;
21592158
21602159 guard(rcu)();
21612160
@@ -2178,11 +2177,7 @@ __bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, · 16 unmodified lines
21782177 return -EINVAL;
21792178 }
21802179
2181− guard(raw_spinlock_irqsave)(&scx_exit_bstr_buf_lock);
2182− ret = scx_bstr_format(parent, &scx_exit_bstr_buf, fmt, data, data__sz);
2183− if (ret < 0)
2184− return ret;
2185− scx_exit(child, SCX_EXIT_PARENT_KILL, 0, "%s", scx_exit_bstr_buf.line);
2180+ scx_exit_bstr(child, SCX_EXIT_PARENT_KILL, 0, parent, fmt, data, data__sz);
21862181 return 0;
21872182 }
21882183