History

ftrace: Add multi direct register/unregister interface

Adding interface to register multiple direct functions within single call. Adding following functions: register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) The register_ftrace_direct_multi registers direct function (addr) with all functions in ops filter. The ops filter can be updated before with ftrace_set_filter_ip calls. All requested functions must not have direct function currently registered, otherwise register_ftrace_direct_multi will fail. The unregister_ftrace_direct_multi unregisters ops related direct functions. Link: https://lkml.kernel.org/r/20211008091336.33616-7-jolsa@kernel.org

Jiri Olsa committed 5 years ago

Commit f64dd461 parent 1904a812 files · +153 −0Browse files

Unified · Split

2 files
  1. include/linux/ftrace.h +11 −0
  2. kernel/trace/ftrace.c +142 −0
include/linux/ftrace.h+11−0
@@ -324,7 +324,10 @@ int ftrace_modify_direct_caller(struct ftrace_func_entry *entry, · 323 unmodified lines
324324 unsigned long old_addr,
325325 unsigned long new_addr);
326326 unsigned long ftrace_find_rec_direct(unsigned long ip);
327+int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
328+int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
327329 #else
330+struct ftrace_ops;
328331 # define ftrace_direct_func_count 0
329332 static inline int register_ftrace_direct(unsigned long ip, unsigned long addr)
330333 {
@@ -354,6 +357,14 @@ static inline unsigned long ftrace_find_rec_direct(unsigned long ip) · 23 unmodified lines
354357 {
355358 return 0;
356359 }
360+static inline int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
361+{
362+ return -ENODEV;
363+}
364+static inline int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
365+{
366+ return -ENODEV;
367+}
357368 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
358369
359370 #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
kernel/trace/ftrace.c+142−0
@@ -5401,6 +5401,148 @@ int modify_ftrace_direct(unsigned long ip, · 5400 unmodified lines
54015401 return ret;
54025402 }
54035403 EXPORT_SYMBOL_GPL(modify_ftrace_direct);
5404+
5405+#define MULTI_FLAGS (FTRACE_OPS_FL_IPMODIFY | FTRACE_OPS_FL_DIRECT | \
5406+ FTRACE_OPS_FL_SAVE_REGS)
5407+
5408+static int check_direct_multi(struct ftrace_ops *ops)
5409+{
5410+ if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
5411+ return -EINVAL;
5412+ if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS)
5413+ return -EINVAL;
5414+ return 0;
5415+}
5416+
5417+static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long addr)
5418+{
5419+ struct ftrace_func_entry *entry, *del;
5420+ int size, i;
5421+
5422+ size = 1 << hash->size_bits;
5423+ for (i = 0; i < size; i++) {
5424+ hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5425+ del = __ftrace_lookup_ip(direct_functions, entry->ip);
5426+ if (del && del->direct == addr) {
5427+ remove_hash_entry(direct_functions, del);
5428+ kfree(del);
5429+ }
5430+ }
5431+ }
5432+}
5433+
5434+/**
5435+ * register_ftrace_direct_multi - Call a custom trampoline directly
5436+ * for multiple functions registered in @ops
5437+ * @ops: The address of the struct ftrace_ops object
5438+ * @addr: The address of the trampoline to call at @ops functions
5439+ *
5440+ * This is used to connect a direct calls to @addr from the nop locations
5441+ * of the functions registered in @ops (with by ftrace_set_filter_ip
5442+ * function).
5443+ *
5444+ * The location that it calls (@addr) must be able to handle a direct call,
5445+ * and save the parameters of the function being traced, and restore them
5446+ * (or inject new ones if needed), before returning.
5447+ *
5448+ * Returns:
5449+ * 0 on success
5450+ * -EINVAL - The @ops object was already registered with this call or
5451+ * when there are no functions in @ops object.
5452+ * -EBUSY - Another direct function is already attached (there can be only one)
5453+ * -ENODEV - @ip does not point to a ftrace nop location (or not supported)
5454+ * -ENOMEM - There was an allocation failure.
5455+ */
5456+int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
5457+{
5458+ struct ftrace_hash *hash, *free_hash = NULL;
5459+ struct ftrace_func_entry *entry, *new;
5460+ int err = -EBUSY, size, i;
5461+
5462+ if (ops->func || ops->trampoline)
5463+ return -EINVAL;
5464+ if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
5465+ return -EINVAL;
5466+ if (ops->flags & FTRACE_OPS_FL_ENABLED)
5467+ return -EINVAL;
5468+
5469+ hash = ops->func_hash->filter_hash;
5470+ if (ftrace_hash_empty(hash))
5471+ return -EINVAL;
5472+
5473+ mutex_lock(&direct_mutex);
5474+
5475+ /* Make sure requested entries are not already registered.. */
5476+ size = 1 << hash->size_bits;
5477+ for (i = 0; i < size; i++) {
5478+ hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5479+ if (ftrace_find_rec_direct(entry->ip))
5480+ goto out_unlock;
5481+ }
5482+ }
5483+
5484+ /* ... and insert them to direct_functions hash. */
5485+ err = -ENOMEM;
5486+ for (i = 0; i < size; i++) {
5487+ hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5488+ new = ftrace_add_rec_direct(entry->ip, addr, &free_hash);
5489+ if (!new)
5490+ goto out_remove;
5491+ entry->direct = addr;
5492+ }
5493+ }
5494+
5495+ ops->func = call_direct_funcs;
5496+ ops->flags = MULTI_FLAGS;
5497+ ops->trampoline = FTRACE_REGS_ADDR;
5498+
5499+ err = register_ftrace_function(ops);
5500+
5501+ out_remove:
5502+ if (err)
5503+ remove_direct_functions_hash(hash, addr);
5504+
5505+ out_unlock:
5506+ mutex_unlock(&direct_mutex);
5507+
5508+ if (free_hash) {
5509+ synchronize_rcu_tasks();
5510+ free_ftrace_hash(free_hash);
5511+ }
5512+ return err;
5513+}
5514+EXPORT_SYMBOL_GPL(register_ftrace_direct_multi);
5515+
5516+/**
5517+ * unregister_ftrace_direct_multi - Remove calls to custom trampoline
5518+ * previously registered by register_ftrace_direct_multi for @ops object.
5519+ * @ops: The address of the struct ftrace_ops object
5520+ *
5521+ * This is used to remove a direct calls to @addr from the nop locations
5522+ * of the functions registered in @ops (with by ftrace_set_filter_ip
5523+ * function).
5524+ *
5525+ * Returns:
5526+ * 0 on success
5527+ * -EINVAL - The @ops object was not properly registered.
5528+ */
5529+int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
5530+{
5531+ struct ftrace_hash *hash = ops->func_hash->filter_hash;
5532+ int err;
5533+
5534+ if (check_direct_multi(ops))
5535+ return -EINVAL;
5536+ if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
5537+ return -EINVAL;
5538+
5539+ mutex_lock(&direct_mutex);
5540+ err = unregister_ftrace_function(ops);
5541+ remove_direct_functions_hash(hash, addr);
5542+ mutex_unlock(&direct_mutex);
5543+ return err;
5544+}
5545+EXPORT_SYMBOL_GPL(unregister_ftrace_direct_multi);
54045546 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
54055547
54065548 /**