Adding interface to register multiple direct functions within single call. Adding following functions: register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) The register_ftrace_direct_multi registers direct function (addr) with all functions in ops filter. The ops filter can be updated before with ftrace_set_filter_ip calls. All requested functions must not have direct function currently registered, otherwise register_ftrace_direct_multi will fail. The unregister_ftrace_direct_multi unregisters ops related direct functions. Link: https://lkml.kernel.org/r/20211008091336.33616-7-jolsa@kernel.org
Showing the changes to kernel (1 of 2 files). Show all
Unified · Split
| @@ -5401,6 +5401,148 @@ int modify_ftrace_direct(unsigned long ip, · 5400 unmodified lines | |||
| 5401 | return ret; | 5401 | return ret; |
| 5402 | } | 5402 | } |
| 5403 | EXPORT_SYMBOL_GPL(modify_ftrace_direct); | 5403 | EXPORT_SYMBOL_GPL(modify_ftrace_direct); |
| 5404 | |||
| 5405 | #define MULTI_FLAGS (FTRACE_OPS_FL_IPMODIFY | FTRACE_OPS_FL_DIRECT | \ | ||
| 5406 | FTRACE_OPS_FL_SAVE_REGS) | ||
| 5407 | |||
| 5408 | static int check_direct_multi(struct ftrace_ops *ops) | ||
| 5409 | { | ||
| 5410 | if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) | ||
| 5411 | return -EINVAL; | ||
| 5412 | if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS) | ||
| 5413 | return -EINVAL; | ||
| 5414 | return 0; | ||
| 5415 | } | ||
| 5416 | |||
| 5417 | static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long addr) | ||
| 5418 | { | ||
| 5419 | struct ftrace_func_entry *entry, *del; | ||
| 5420 | int size, i; | ||
| 5421 | |||
| 5422 | size = 1 << hash->size_bits; | ||
| 5423 | for (i = 0; i < size; i++) { | ||
| 5424 | hlist_for_each_entry(entry, &hash->buckets[i], hlist) { | ||
| 5425 | del = __ftrace_lookup_ip(direct_functions, entry->ip); | ||
| 5426 | if (del && del->direct == addr) { | ||
| 5427 | remove_hash_entry(direct_functions, del); | ||
| 5428 | kfree(del); | ||
| 5429 | } | ||
| 5430 | } | ||
| 5431 | } | ||
| 5432 | } | ||
| 5433 | |||
| 5434 | /** | ||
| 5435 | * register_ftrace_direct_multi - Call a custom trampoline directly | ||
| 5436 | * for multiple functions registered in @ops | ||
| 5437 | * @ops: The address of the struct ftrace_ops object | ||
| 5438 | * @addr: The address of the trampoline to call at @ops functions | ||
| 5439 | * | ||
| 5440 | * This is used to connect a direct calls to @addr from the nop locations | ||
| 5441 | * of the functions registered in @ops (with by ftrace_set_filter_ip | ||
| 5442 | * function). | ||
| 5443 | * | ||
| 5444 | * The location that it calls (@addr) must be able to handle a direct call, | ||
| 5445 | * and save the parameters of the function being traced, and restore them | ||
| 5446 | * (or inject new ones if needed), before returning. | ||
| 5447 | * | ||
| 5448 | * Returns: | ||
| 5449 | * 0 on success | ||
| 5450 | * -EINVAL - The @ops object was already registered with this call or | ||
| 5451 | * when there are no functions in @ops object. | ||
| 5452 | * -EBUSY - Another direct function is already attached (there can be only one) | ||
| 5453 | * -ENODEV - @ip does not point to a ftrace nop location (or not supported) | ||
| 5454 | * -ENOMEM - There was an allocation failure. | ||
| 5455 | */ | ||
| 5456 | int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) | ||
| 5457 | { | ||
| 5458 | struct ftrace_hash *hash, *free_hash = NULL; | ||
| 5459 | struct ftrace_func_entry *entry, *new; | ||
| 5460 | int err = -EBUSY, size, i; | ||
| 5461 | |||
| 5462 | if (ops->func || ops->trampoline) | ||
| 5463 | return -EINVAL; | ||
| 5464 | if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) | ||
| 5465 | return -EINVAL; | ||
| 5466 | if (ops->flags & FTRACE_OPS_FL_ENABLED) | ||
| 5467 | return -EINVAL; | ||
| 5468 | |||
| 5469 | hash = ops->func_hash->filter_hash; | ||
| 5470 | if (ftrace_hash_empty(hash)) | ||
| 5471 | return -EINVAL; | ||
| 5472 | |||
| 5473 | mutex_lock(&direct_mutex); | ||
| 5474 | |||
| 5475 | /* Make sure requested entries are not already registered.. */ | ||
| 5476 | size = 1 << hash->size_bits; | ||
| 5477 | for (i = 0; i < size; i++) { | ||
| 5478 | hlist_for_each_entry(entry, &hash->buckets[i], hlist) { | ||
| 5479 | if (ftrace_find_rec_direct(entry->ip)) | ||
| 5480 | goto out_unlock; | ||
| 5481 | } | ||
| 5482 | } | ||
| 5483 | |||
| 5484 | /* ... and insert them to direct_functions hash. */ | ||
| 5485 | err = -ENOMEM; | ||
| 5486 | for (i = 0; i < size; i++) { | ||
| 5487 | hlist_for_each_entry(entry, &hash->buckets[i], hlist) { | ||
| 5488 | new = ftrace_add_rec_direct(entry->ip, addr, &free_hash); | ||
| 5489 | if (!new) | ||
| 5490 | goto out_remove; | ||
| 5491 | entry->direct = addr; | ||
| 5492 | } | ||
| 5493 | } | ||
| 5494 | |||
| 5495 | ops->func = call_direct_funcs; | ||
| 5496 | ops->flags = MULTI_FLAGS; | ||
| 5497 | ops->trampoline = FTRACE_REGS_ADDR; | ||
| 5498 | |||
| 5499 | err = register_ftrace_function(ops); | ||
| 5500 | |||
| 5501 | out_remove: | ||
| 5502 | if (err) | ||
| 5503 | remove_direct_functions_hash(hash, addr); | ||
| 5504 | |||
| 5505 | out_unlock: | ||
| 5506 | mutex_unlock(&direct_mutex); | ||
| 5507 | |||
| 5508 | if (free_hash) { | ||
| 5509 | synchronize_rcu_tasks(); | ||
| 5510 | free_ftrace_hash(free_hash); | ||
| 5511 | } | ||
| 5512 | return err; | ||
| 5513 | } | ||
| 5514 | EXPORT_SYMBOL_GPL(register_ftrace_direct_multi); | ||
| 5515 | |||
| 5516 | /** | ||
| 5517 | * unregister_ftrace_direct_multi - Remove calls to custom trampoline | ||
| 5518 | * previously registered by register_ftrace_direct_multi for @ops object. | ||
| 5519 | * @ops: The address of the struct ftrace_ops object | ||
| 5520 | * | ||
| 5521 | * This is used to remove a direct calls to @addr from the nop locations | ||
| 5522 | * of the functions registered in @ops (with by ftrace_set_filter_ip | ||
| 5523 | * function). | ||
| 5524 | * | ||
| 5525 | * Returns: | ||
| 5526 | * 0 on success | ||
| 5527 | * -EINVAL - The @ops object was not properly registered. | ||
| 5528 | */ | ||
| 5529 | int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) | ||
| 5530 | { | ||
| 5531 | struct ftrace_hash *hash = ops->func_hash->filter_hash; | ||
| 5532 | int err; | ||
| 5533 | |||
| 5534 | if (check_direct_multi(ops)) | ||
| 5535 | return -EINVAL; | ||
| 5536 | if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) | ||
| 5537 | return -EINVAL; | ||
| 5538 | |||
| 5539 | mutex_lock(&direct_mutex); | ||
| 5540 | err = unregister_ftrace_function(ops); | ||
| 5541 | remove_direct_functions_hash(hash, addr); | ||
| 5542 | mutex_unlock(&direct_mutex); | ||
| 5543 | return err; | ||
| 5544 | } | ||
| 5545 | EXPORT_SYMBOL_GPL(unregister_ftrace_direct_multi); | ||
| 5404 | #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ | 5546 | #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ |
| 5405 | 5547 | ||
| 5406 | /** | 5548 | /** |