History

ftrace: Fix accounting of subop hashes

The function graph infrastructure uses ftrace to hook to functions. It has a single ftrace_ops to manage all the users of function graph. Each individual user (tracing, bpf, fprobes, etc) has its own ftrace_ops to track the functions it will have its callback called from. These ftrace_ops are "subops" to the main ftrace_ops of the function graph infrastructure. Each ftrace_ops has a filter_hash and a notrace_hash that is defined as: Only trace functions that are in the filter_hash but not in the notrace_hash. If the filter_hash is empty, it means to trace all functions. If the notrace_hash is empty, it means do not disable any function. The function graph main ftrace_ops needs to be a superset containing all the functions to be traced by all the subops it has. The algorithm to perform this merge was incorrect. When the first subops was added to the main ops, it simply made the main ops a copy of the subops (same filter_hash and notrace_hash). When a second ops was added, it joined the new subops filter_hash with the main ops filter_hash as a union of the two sets. The intersect between the new subops notrace_hash and the main ops notrace_hash was created as the new notrace_hash of the main ops. The issue here is that it would then start tracing functions than no subops were tracing. For example if you had two subops that had: subops 1: filter_hash = '*sched*' # trace all functions with "sched" in it notrace_hash = '*time*' # except do not trace functions with "time" subops 2: filter_hash = '*lock*' # trace all functions with "lock" in it notrace_hash = '*clock*' # except do not trace functions with "clock" The intersect of '*time*' functions with '*clock*' functions could be the empty set. That means the main ops will be tracing all functions with '*time*' and all "*clock*" in it! Instead, modify the algorithm to be a bit simpler and correct. First, when adding a new subops, even if it's the first one, do not add the notrace_hash if the filter_hash is not empty. Instead, just add the functions that are in the filter_hash of the subops but not in the notrace_hash of the subops into the main ops filter_hash. There's no reason to add anything to the main ops notrace_hash. The notrace_hash of the main ops should only be non empty iff all subops filter_hashes are empty (meaning to trace all functions) and all subops notrace_hashes include the same functions. That is, the main ops notrace_hash is empty if any subops filter_hash is non empty. The main ops notrace_hash only has content in it if all subops filter_hashes are empty, and the content are only functions that intersect all the subops notrace_hashes. If any subops notrace_hash is empty, then so is the main ops notrace_hash.

Steven Rostedt committed 1 year ago

Commit 0ae6b8c1 parent 04a80a31 file · +177 −137Browse files

Unified · Split

kernel/trace/ftrace.c+177−137
@@ -3255,6 +3255,31 @@ static int append_hash(struct ftrace_hash **hash, struct ftrace_hash *new_hash, · 3254 unmodified lines
32553255 return 0;
32563256 }
32573257
3258+/*
3259+ * Remove functions from @hash that are in @notrace_hash
3260+ */
3261+static void remove_hash(struct ftrace_hash *hash, struct ftrace_hash *notrace_hash)
3262+{
3263+ struct ftrace_func_entry *entry;
3264+ struct hlist_node *tmp;
3265+ int size;
3266+ int i;
3267+
3268+ /* If the notrace hash is empty, there's nothing to do */
3269+ if (ftrace_hash_empty(notrace_hash))
3270+ return;
3271+
3272+ size = 1 << hash->size_bits;
3273+ for (i = 0; i < size; i++) {
3274+ hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
3275+ if (!__ftrace_lookup_ip(notrace_hash, entry->ip))
3276+ continue;
3277+ remove_hash_entry(hash, entry);
3278+ kfree(entry);
3279+ }
3280+ }
3281+}
3282+
32583283 /*
32593284 * Add to @hash only those that are in both @new_hash1 and @new_hash2
32603285 *
@@ -3295,67 +3320,6 @@ static int intersect_hash(struct ftrace_hash **hash, struct ftrace_hash *new_has · 34 unmodified lines
32953320 return 0;
32963321 }
32973322
3298−/* Return a new hash that has a union of all @ops->filter_hash entries */
3299−static struct ftrace_hash *append_hashes(struct ftrace_ops *ops)
3300−{
3301− struct ftrace_hash *new_hash = NULL;
3302− struct ftrace_ops *subops;
3303− int size_bits;
3304− int ret;
3305−
3306− if (ops->func_hash->filter_hash)
3307− size_bits = ops->func_hash->filter_hash->size_bits;
3308− else
3309− size_bits = FTRACE_HASH_DEFAULT_BITS;
3310−
3311− list_for_each_entry(subops, &ops->subop_list, list) {
3312− ret = append_hash(&new_hash, subops->func_hash->filter_hash, size_bits);
3313− if (ret < 0) {
3314− free_ftrace_hash(new_hash);
3315− return NULL;
3316− }
3317− /* Nothing more to do if new_hash is empty */
3318− if (ftrace_hash_empty(new_hash))
3319− break;
3320− }
3321− /* Can't return NULL as that means this failed */
3322− return new_hash ? : EMPTY_HASH;
3323−}
3324−
3325−/* Make @ops trace evenything except what all its subops do not trace */
3326−static struct ftrace_hash *intersect_hashes(struct ftrace_ops *ops)
3327−{
3328− struct ftrace_hash *new_hash = NULL;
3329− struct ftrace_ops *subops;
3330− int size_bits;
3331− int ret;
3332−
3333− list_for_each_entry(subops, &ops->subop_list, list) {
3334− struct ftrace_hash *next_hash;
3335−
3336− if (!new_hash) {
3337− size_bits = subops->func_hash->notrace_hash->size_bits;
3338− new_hash = alloc_and_copy_ftrace_hash(size_bits, ops->func_hash->notrace_hash);
3339− if (!new_hash)
3340− return NULL;
3341− continue;
3342− }
3343− size_bits = new_hash->size_bits;
3344− next_hash = new_hash;
3345− new_hash = alloc_ftrace_hash(size_bits);
3346− ret = intersect_hash(&new_hash, next_hash, subops->func_hash->notrace_hash);
3347− free_ftrace_hash(next_hash);
3348− if (ret < 0) {
3349− free_ftrace_hash(new_hash);
3350− return NULL;
3351− }
3352− /* Nothing more to do if new_hash is empty */
3353− if (ftrace_hash_empty(new_hash))
3354− break;
3355− }
3356− return new_hash;
3357−}
3358−
33593323 static bool ops_equal(struct ftrace_hash *A, struct ftrace_hash *B)
33603324 {
33613325 struct ftrace_func_entry *entry;
@@ -3427,6 +3391,93 @@ static int ftrace_update_ops(struct ftrace_ops *ops, struct ftrace_hash *filter_ · 65 unmodified lines
34273391 return 0;
34283392 }
34293393
3394+static int add_first_hash(struct ftrace_hash **filter_hash, struct ftrace_hash **notrace_hash,
3395+ struct ftrace_ops_hash *func_hash)
3396+{
3397+ /* If the filter hash is not empty, simply remove the nohash from it */
3398+ if (!ftrace_hash_empty(func_hash->filter_hash)) {
3399+ *filter_hash = copy_hash(func_hash->filter_hash);
3400+ if (!*filter_hash)
3401+ return -ENOMEM;
3402+ remove_hash(*filter_hash, func_hash->notrace_hash);
3403+ *notrace_hash = EMPTY_HASH;
3404+
3405+ } else {
3406+ *notrace_hash = copy_hash(func_hash->notrace_hash);
3407+ if (!*notrace_hash)
3408+ return -ENOMEM;
3409+ *filter_hash = EMPTY_HASH;
3410+ }
3411+ return 0;
3412+}
3413+
3414+static int add_next_hash(struct ftrace_hash **filter_hash, struct ftrace_hash **notrace_hash,
3415+ struct ftrace_ops_hash *ops_hash, struct ftrace_ops_hash *subops_hash)
3416+{
3417+ int size_bits;
3418+ int ret;
3419+
3420+ /* If the subops trace all functions so must the main ops */
3421+ if (ftrace_hash_empty(ops_hash->filter_hash) ||
3422+ ftrace_hash_empty(subops_hash->filter_hash)) {
3423+ *filter_hash = EMPTY_HASH;
3424+ } else {
3425+ /*
3426+ * The main ops filter hash is not empty, so its
3427+ * notrace_hash had better be, as the notrace hash
3428+ * is only used for empty main filter hashes.
3429+ */
3430+ WARN_ON_ONCE(!ftrace_hash_empty(ops_hash->notrace_hash));
3431+
3432+ size_bits = max(ops_hash->filter_hash->size_bits,
3433+ subops_hash->filter_hash->size_bits);
3434+
3435+ /* Copy the subops hash */
3436+ *filter_hash = alloc_and_copy_ftrace_hash(size_bits, subops_hash->filter_hash);
3437+ if (!filter_hash)
3438+ return -ENOMEM;
3439+ /* Remove any notrace functions from the copy */
3440+ remove_hash(*filter_hash, subops_hash->notrace_hash);
3441+
3442+ ret = append_hash(filter_hash, ops_hash->filter_hash,
3443+ size_bits);
3444+ if (ret < 0) {
3445+ free_ftrace_hash(*filter_hash);
3446+ return ret;
3447+ }
3448+ }
3449+
3450+ /*
3451+ * Only process notrace hashes if the main filter hash is empty
3452+ * (tracing all functions), otherwise the filter hash will just
3453+ * remove the notrace hash functions, and the notrace hash is
3454+ * not needed.
3455+ */
3456+ if (ftrace_hash_empty(*filter_hash)) {
3457+ /*
3458+ * Intersect the notrace functions. That is, if two
3459+ * subops are not tracing a set of functions, the
3460+ * main ops will only not trace the functions that are
3461+ * in both subops, but has to trace the functions that
3462+ * are only notrace in one of the subops, for the other
3463+ * subops to be able to trace them.
3464+ */
3465+ size_bits = max(ops_hash->notrace_hash->size_bits,
3466+ subops_hash->notrace_hash->size_bits);
3467+ *notrace_hash = alloc_ftrace_hash(size_bits);
3468+ if (!*notrace_hash)
3469+ return -ENOMEM;
3470+
3471+ ret = intersect_hash(notrace_hash, ops_hash->notrace_hash,
3472+ subops_hash->notrace_hash);
3473+ if (ret < 0) {
3474+ free_ftrace_hash(*notrace_hash);
3475+ return ret;
3476+ }
3477+ }
3478+ return 0;
3479+}
3480+
34303481 /**
34313482 * ftrace_startup_subops - enable tracing for subops of an ops
34323483 * @ops: Manager ops (used to pick all the functions of its subops)
@@ -3443,7 +3494,6 @@ int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int · 10 unmodified lines
34433494 struct ftrace_hash *notrace_hash;
34443495 struct ftrace_hash *save_filter_hash;
34453496 struct ftrace_hash *save_notrace_hash;
3446− int size_bits;
34473497 int ret;
34483498
34493499 if (unlikely(ftrace_disabled))
@@ -3467,14 +3517,14 @@ int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int · 17 unmodified lines
34673517
34683518 /* For the first subops to ops just enable it normally */
34693519 if (list_empty(&ops->subop_list)) {
3470− /* Just use the subops hashes */
3471− filter_hash = copy_hash(subops->func_hash->filter_hash);
3472− notrace_hash = copy_hash(subops->func_hash->notrace_hash);
3473− if (!filter_hash || !notrace_hash) {
3474− free_ftrace_hash(filter_hash);
3475− free_ftrace_hash(notrace_hash);
3476− return -ENOMEM;
3477− }
3520+
3521+ /* The ops was empty, should have empty hashes */
3522+ WARN_ON_ONCE(!ftrace_hash_empty(ops->func_hash->filter_hash));
3523+ WARN_ON_ONCE(!ftrace_hash_empty(ops->func_hash->notrace_hash));
3524+
3525+ ret = add_first_hash(&filter_hash, &notrace_hash, subops->func_hash);
3526+ if (ret < 0)
3527+ return ret;
34783528
34793529 save_filter_hash = ops->func_hash->filter_hash;
34803530 save_notrace_hash = ops->func_hash->notrace_hash;
@@ -3500,48 +3550,16 @@ int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int · 19 unmodified lines
35003550
35013551 /*
35023552 * Here there's already something attached. Here are the rules:
3503− * o If either filter_hash is empty then the final stays empty
3504− * o Otherwise, the final is a superset of both hashes
3505− * o If either notrace_hash is empty then the final stays empty
3506− * o Otherwise, the final is an intersection between the hashes
3553+ * If the new subops and main ops filter hashes are not empty:
3554+ * o Make a copy of the subops filter hash
3555+ * o Remove all functions in the nohash from it.
3556+ * o Add in the main hash filter functions
3557+ * o Remove any of these functions from the main notrace hash
35073558 */
3508− if (ftrace_hash_empty(ops->func_hash->filter_hash) ||
3509− ftrace_hash_empty(subops->func_hash->filter_hash)) {
3510− filter_hash = EMPTY_HASH;
3511− } else {
3512− size_bits = max(ops->func_hash->filter_hash->size_bits,
3513− subops->func_hash->filter_hash->size_bits);
3514− filter_hash = alloc_and_copy_ftrace_hash(size_bits, ops->func_hash->filter_hash);
3515− if (!filter_hash)
3516− return -ENOMEM;
3517− ret = append_hash(&filter_hash, subops->func_hash->filter_hash,
3518− size_bits);
3519− if (ret < 0) {
3520− free_ftrace_hash(filter_hash);
3521− return ret;
3522− }
3523− }
3524−
3525− if (ftrace_hash_empty(ops->func_hash->notrace_hash) ||
3526− ftrace_hash_empty(subops->func_hash->notrace_hash)) {
3527− notrace_hash = EMPTY_HASH;
3528− } else {
3529− size_bits = max(ops->func_hash->notrace_hash->size_bits,
3530− subops->func_hash->notrace_hash->size_bits);
3531− notrace_hash = alloc_ftrace_hash(size_bits);
3532− if (!notrace_hash) {
3533− free_ftrace_hash(filter_hash);
3534− return -ENOMEM;
3535− }
35363559
3537− ret = intersect_hash(&notrace_hash, ops->func_hash->notrace_hash,
3538− subops->func_hash->notrace_hash);
3539− if (ret < 0) {
3540− free_ftrace_hash(filter_hash);
3541− free_ftrace_hash(notrace_hash);
3542− return ret;
3543− }
3544− }
3560+ ret = add_next_hash(&filter_hash, &notrace_hash, ops->func_hash, subops->func_hash);
3561+ if (ret < 0)
3562+ return ret;
35453563
35463564 list_add(&subops->list, &ops->subop_list);
35473565
@@ -3557,6 +3575,42 @@ int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int · 9 unmodified lines
35573575 return ret;
35583576 }
35593577
3578+static int rebuild_hashes(struct ftrace_hash **filter_hash, struct ftrace_hash **notrace_hash,
3579+ struct ftrace_ops *ops)
3580+{
3581+ struct ftrace_ops_hash temp_hash;
3582+ struct ftrace_ops *subops;
3583+ bool first = true;
3584+ int ret;
3585+
3586+ temp_hash.filter_hash = EMPTY_HASH;
3587+ temp_hash.notrace_hash = EMPTY_HASH;
3588+
3589+ list_for_each_entry(subops, &ops->subop_list, list) {
3590+ *filter_hash = EMPTY_HASH;
3591+ *notrace_hash = EMPTY_HASH;
3592+
3593+ if (first) {
3594+ ret = add_first_hash(filter_hash, notrace_hash, subops->func_hash);
3595+ if (ret < 0)
3596+ return ret;
3597+ first = false;
3598+ } else {
3599+ ret = add_next_hash(filter_hash, notrace_hash,
3600+ &temp_hash, subops->func_hash);
3601+ if (ret < 0) {
3602+ free_ftrace_hash(temp_hash.filter_hash);
3603+ free_ftrace_hash(temp_hash.notrace_hash);
3604+ return ret;
3605+ }
3606+ }
3607+
3608+ temp_hash.filter_hash = *filter_hash;
3609+ temp_hash.notrace_hash = *notrace_hash;
3610+ }
3611+ return 0;
3612+}
3613+
35603614 /**
35613615 * ftrace_shutdown_subops - Remove a subops from a manager ops
35623616 * @ops: A manager ops to remove @subops from
@@ -3605,14 +3659,9 @@ int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, in · 42 unmodified lines
36053659 }
36063660
36073661 /* Rebuild the hashes without subops */
3608− filter_hash = append_hashes(ops);
3609− notrace_hash = intersect_hashes(ops);
3610− if (!filter_hash || !notrace_hash) {
3611− free_ftrace_hash(filter_hash);
3612− free_ftrace_hash(notrace_hash);
3613− list_add(&subops->list, &ops->subop_list);
3614− return -ENOMEM;
3615− }
3662+ ret = rebuild_hashes(&filter_hash, &notrace_hash, ops);
3663+ if (ret < 0)
3664+ return ret;
36163665
36173666 ret = ftrace_update_ops(ops, filter_hash, notrace_hash);
36183667 if (ret < 0) {
@@ -3628,11 +3677,11 @@ int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, in · 9 unmodified lines
36283677
36293678 static int ftrace_hash_move_and_update_subops(struct ftrace_ops *subops,
36303679 struct ftrace_hash **orig_subhash,
3631− struct ftrace_hash *hash,
3632− int enable)
3680+ struct ftrace_hash *hash)
36333681 {
36343682 struct ftrace_ops *ops = subops->managed;
3635− struct ftrace_hash **orig_hash;
3683+ struct ftrace_hash *notrace_hash;
3684+ struct ftrace_hash *filter_hash;
36363685 struct ftrace_hash *save_hash;
36373686 struct ftrace_hash *new_hash;
36383687 int ret;
@@ -3649,24 +3698,15 @@ static int ftrace_hash_move_and_update_subops(struct ftrace_ops *subops, · 10 unmodified lines
36493698 return -ENOMEM;
36503699 }
36513700
3652− /* Create a new_hash to hold the ops new functions */
3653− if (enable) {
3654− orig_hash = &ops->func_hash->filter_hash;
3655− new_hash = append_hashes(ops);
3656− } else {
3657− orig_hash = &ops->func_hash->notrace_hash;
3658− new_hash = intersect_hashes(ops);
3659− }
3660−
3661− /* Move the hash over to the new hash */
3662− ret = __ftrace_hash_move_and_update_ops(ops, orig_hash, new_hash, enable);
3663−
3664− free_ftrace_hash(new_hash);
3701+ ret = rebuild_hashes(&filter_hash, &notrace_hash, ops);
3702+ if (!ret)
3703+ ret = ftrace_update_ops(ops, filter_hash, notrace_hash);
36653704
36663705 if (ret) {
36673706 /* Put back the original hash */
3668− free_ftrace_hash_rcu(*orig_subhash);
3707+ new_hash = *orig_subhash;
36693708 *orig_subhash = save_hash;
3709+ free_ftrace_hash_rcu(new_hash);
36703710 } else {
36713711 free_ftrace_hash_rcu(save_hash);
36723712 }
@@ -4890,7 +4930,7 @@ static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, · 1217 unmodified lines
48904930 int enable)
48914931 {
48924932 if (ops->flags & FTRACE_OPS_FL_SUBOP)
4893− return ftrace_hash_move_and_update_subops(ops, orig_hash, hash, enable);
4933+ return ftrace_hash_move_and_update_subops(ops, orig_hash, hash);
48944934
48954935 /*
48964936 * If this ops is not enabled, it could be sharing its filters
@@ -4909,7 +4949,7 @@ static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, · 12 unmodified lines
49094949 list_for_each_entry(subops, &op->subop_list, list) {
49104950 if ((subops->flags & FTRACE_OPS_FL_ENABLED) &&
49114951 subops->func_hash == ops->func_hash) {
4912− return ftrace_hash_move_and_update_subops(subops, orig_hash, hash, enable);
4952+ return ftrace_hash_move_and_update_subops(subops, orig_hash, hash);
49134953 }
49144954 }
49154955 } while_for_each_ftrace_op(op);