// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (c) 2021, Microsoft Corporation. * * Authors: * Beau Belgrave <beaub@linux.microsoft.com> */ #include <linux/bitmap.h>#include <linux/cdev.h>#include <linux/hashtable.h>#include <linux/list.h>#include <linux/io.h>#include <linux/uio.h>#include <linux/ioctl.h>#include <linux/jhash.h>#include <linux/refcount.h>#include <linux/trace_events.h>#include <linux/tracefs.h>#include <linux/types.h>#include <linux/uaccess.h>#include <linux/highmem.h>#include <linux/init.h>#include <linux/user_events.h>#include "trace_dynevent.h"#include "trace_output.h"#include "trace.h" #define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1) #define FIELD_DEPTH_TYPE 0#define FIELD_DEPTH_NAME 1#define FIELD_DEPTH_SIZE 2 /* Limit how long of an event name plus args within the subsystem. */#define MAX_EVENT_DESC 512#define EVENT_NAME(user_event) ((user_event)->reg_name)#define EVENT_TP_NAME(user_event) ((user_event)->tracepoint.name)#define MAX_FIELD_ARRAY_SIZE 1024 /* * Internal bits (kernel side only) to keep track of connected probes: * These are used when status is requested in text form about an event. These * bits are compared against an internal byte on the event to determine which * probes to print out to the user. * * These do not reflect the mapped bytes between the user and kernel space. */#define EVENT_STATUS_FTRACE BIT(0)#define EVENT_STATUS_PERF BIT(1)#define EVENT_STATUS_OTHER BIT(7) /* * Stores the system name, tables, and locks for a group of events. This * allows isolation for events by various means. */struct user_event_group { char *system_name; char *system_multi_name; struct hlist_node node; struct mutex reg_mutex; DECLARE_HASHTABLE(register_table, 8); /* ID that moves forward within the group for multi-event names */ u64 multi_id;}; /* Group for init_user_ns mapping, top-most group */static struct user_event_group *init_group; /* Max allowed events for the whole system */static unsigned int max_user_events = 32768; /* Current number of events on the whole system */static unsigned int current_user_events; /* * Stores per-event properties, as users register events * within a file a user_event might be created if it does not * already exist. These are globally used and their lifetime * is tied to the refcnt member. These cannot go away until the * refcnt reaches one. */struct user_event { struct user_event_group *group; char *reg_name; struct tracepoint tracepoint; struct trace_event_call call; struct trace_event_class class; struct dyn_event devent; struct hlist_node node; struct list_head fields; struct list_head validators; struct work_struct put_work; refcount_t refcnt; int min_size; int reg_flags; char status;}; /* * Stores per-mm/event properties that enable an address to be * updated properly for each task. As tasks are forked, we use * these to track enablement sites that are tied to an event. */struct user_event_enabler { struct list_head mm_enablers_link; struct user_event *event; unsigned long addr; /* Track enable bit, flags, etc. Aligned for bitops. */ unsigned long values; /* Defer the event put and enabler free past an RCU grace period. */ struct rcu_work put_rwork;}; /* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */#define ENABLE_VAL_BIT_MASK 0x3F /* Bit 6 is for faulting status of enablement */#define ENABLE_VAL_FAULTING_BIT 6 /* Bit 7 is for freeing status of enablement */#define ENABLE_VAL_FREEING_BIT 7 /* Bit 8 is for marking 32-bit on 64-bit */#define ENABLE_VAL_32_ON_64_BIT 8 #define ENABLE_VAL_COMPAT_MASK (1 << ENABLE_VAL_32_ON_64_BIT) /* Only duplicate the bit and compat values */#define ENABLE_VAL_DUP_MASK (ENABLE_VAL_BIT_MASK | ENABLE_VAL_COMPAT_MASK) #define ENABLE_BITOPS(e) (&(e)->values) #define ENABLE_BIT(e) ((int)((e)->values & ENABLE_VAL_BIT_MASK)) #define EVENT_MULTI_FORMAT(f) ((f) & USER_EVENT_REG_MULTI_FORMAT) /* Used for asynchronous faulting in of pages */struct user_event_enabler_fault { struct work_struct work; struct user_event_mm *mm; struct user_event_enabler *enabler; int attempt;}; static struct kmem_cache *fault_cache; /* Global list of memory descriptors using user_events */static LIST_HEAD(user_event_mms);static DEFINE_SPINLOCK(user_event_mms_lock); /* * Stores per-file events references, as users register events * within a file this structure is modified and freed via RCU. * The lifetime of this struct is tied to the lifetime of the file. * These are not shared and only accessible by the file that created it. */struct user_event_refs { struct rcu_head rcu; int count; struct user_event *events[];}; struct user_event_file_info { struct user_event_group *group; struct user_event_refs *refs;}; #define VALIDATOR_ENSURE_NULL (1 << 0)#define VALIDATOR_REL (1 << 1) struct user_event_validator { struct list_head user_event_link; int offset; int flags;}; static inline void align_addr_bit(unsigned long *addr, int *bit, unsigned long *flags){ if (IS_ALIGNED(*addr, sizeof(long))) {#ifdef __BIG_ENDIAN /* 32 bit on BE 64 bit requires a 32 bit offset when aligned. */ if (test_bit(ENABLE_VAL_32_ON_64_BIT, flags)) *bit += 32;#endif return; } *addr = ALIGN_DOWN(*addr, sizeof(long)); /* * We only support 32 and 64 bit values. The only time we need * to align is a 32 bit value on a 64 bit kernel, which on LE * is always 32 bits, and on BE requires no change when unaligned. */#ifdef __LITTLE_ENDIAN *bit += 32;#endif} typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i, void *tpdata, bool *faulted); static int user_event_parse(struct user_event_group *group, char *name, char *args, char *flags, struct user_event **newuser, int reg_flags); static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm);static struct user_event_mm *user_event_mm_get_all(struct user_event *user);static void user_event_mm_put(struct user_event_mm *mm);static int destroy_user_event(struct user_event *user);static bool user_fields_match(struct user_event *user, int argc, const char **argv); static u32 user_event_key(char *name){ return jhash(name, strlen(name), 0);} static bool user_event_capable(u16 reg_flags){ /* Persistent events require CAP_PERFMON / CAP_SYS_ADMIN */ if (reg_flags & USER_EVENT_REG_PERSIST) { if (!perfmon_capable()) return false; } return true;} static struct user_event *user_event_get(struct user_event *user){ refcount_inc(&user->refcnt); return user;} static void delayed_destroy_user_event(struct work_struct *work){ struct user_event *user = container_of( work, struct user_event, put_work); mutex_lock(&event_mutex); if (!refcount_dec_and_test(&user->refcnt)) goto out; if (destroy_user_event(user)) { /* * The only reason this would fail here is if we cannot * update the visibility of the event. In this case the * event stays in the hashtable, waiting for someone to * attempt to delete it later. */ pr_warn("user_events: Unable to delete event\n"); refcount_set(&user->refcnt, 1); }out: mutex_unlock(&event_mutex);} static void user_event_put(struct user_event *user, bool locked){ bool delete; if (unlikely(!user)) return; /* * When the event is not enabled for auto-delete there will always * be at least 1 reference to the event. During the event creation * we initially set the refcnt to 2 to achieve this. In those cases * the caller must acquire event_mutex and after decrement check if * the refcnt is 1, meaning this is the last reference. When auto * delete is enabled, there will only be 1 ref, IE: refcnt will be * only set to 1 during creation to allow the below checks to go * through upon the last put. The last put must always be done with * the event mutex held. */ if (!locked) { lockdep_assert_not_held(&event_mutex); delete = refcount_dec_and_mutex_lock(&user->refcnt, &event_mutex); } else { lockdep_assert_held(&event_mutex); delete = refcount_dec_and_test(&user->refcnt); } if (!delete) return; /* * We now have the event_mutex in all cases, which ensures that * no new references will be taken until event_mutex is released. * New references come through find_user_event(), which requires * the event_mutex to be held. */ if (user->reg_flags & USER_EVENT_REG_PERSIST) { /* We should not get here when persist flag is set */ pr_alert("BUG: Auto-delete engaged on persistent event\n"); goto out; } /* * Unfortunately we have to attempt the actual destroy in a work * queue. This is because not all cases handle a trace_event_call * being removed within the class->reg() operation for unregister. */ INIT_WORK(&user->put_work, delayed_destroy_user_event); /* * Since the event is still in the hashtable, we have to re-inc * the ref count to 1. This count will be decremented and checked * in the work queue to ensure it's still the last ref. This is * needed because a user-process could register the same event in * between the time of event_mutex release and the work queue * running the delayed destroy. If we removed the item now from * the hashtable, this would result in a timing window where a * user process would fail a register because the trace_event_call * register would fail in the tracing layers. */ refcount_set(&user->refcnt, 1); if (WARN_ON_ONCE(!schedule_work(&user->put_work))) { /* * If we fail we must wait for an admin to attempt delete or * another register/close of the event, whichever is first. */ pr_warn("user_events: Unable to queue delayed destroy\n"); }out: /* Ensure if we didn't have event_mutex before we unlock it */ if (!locked) mutex_unlock(&event_mutex);} static void user_event_group_destroy(struct user_event_group *group){ kfree(group->system_name); kfree(group->system_multi_name); kfree(group);} static char *user_event_group_system_name(void){ char *system_name; int len = sizeof(USER_EVENTS_SYSTEM) + 1; system_name = kmalloc(len, GFP_KERNEL); if (!system_name) return NULL; snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM); return system_name;} static char *user_event_group_system_multi_name(void){ return kstrdup(USER_EVENTS_MULTI_SYSTEM, GFP_KERNEL);} static struct user_event_group *current_user_event_group(void){ return init_group;} static struct user_event_group *user_event_group_create(void){ struct user_event_group *group; group = kzalloc_obj(*group); if (!group) return NULL; group->system_name = user_event_group_system_name(); if (!group->system_name) goto error; group->system_multi_name = user_event_group_system_multi_name(); if (!group->system_multi_name) goto error; mutex_init(&group->reg_mutex); hash_init(group->register_table); return group;error: if (group) user_event_group_destroy(group); return NULL;}; static void delayed_user_event_enabler_put(struct work_struct *work){ struct user_event_enabler *enabler = container_of(to_rcu_work(work), struct user_event_enabler, put_rwork); /* No longer tracking the event via the enabler */ user_event_put(enabler->event, false); /* Run from queue_rcu_work(), the RCU grace period has elapsed */ kfree(enabler);} static void user_event_enabler_destroy(struct user_event_enabler *enabler){ list_del_rcu(&enabler->mm_enablers_link); /* * The enabler is removed from an RCU-traversed list * (user_event_mm_dup() walks mm->enablers under rcu_read_lock() only), * and readers there dereference enabler->event and take a new ref on * it. Both the put of that event reference and the free of the enabler * therefore have to wait for a grace period so no reader can be looking * at the enabler or racing the last put of its event. * * The put itself must not run in RCU context: when it drops the last * reference user_event_put() takes event_mutex, which cannot be taken * from a softirq/RCU callback. Defer both to a work item scheduled * after a grace period via queue_rcu_work(). */ INIT_RCU_WORK(&enabler->put_rwork, delayed_user_event_enabler_put); queue_rcu_work(system_percpu_wq, &enabler->put_rwork);} static int user_event_mm_fault_in(struct user_event_mm *mm, unsigned long uaddr, int attempt){ bool unlocked; int ret; /* * Normally this is low, ensure that it cannot be taken advantage of by * bad user processes to cause excessive looping. */ if (attempt > 10) return -EFAULT; mmap_read_lock(mm->mm); /* Ensure MM has tasks, cannot use after exit_mm() */ if (refcount_read(&mm->tasks) == 0) { ret = -ENOENT; goto out; } ret = fixup_user_fault(mm->mm, uaddr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, &unlocked);out: mmap_read_unlock(mm->mm); return ret;} static int user_event_enabler_write(struct user_event_mm *mm, struct user_event_enabler *enabler, bool fixup_fault, int *attempt); static void user_event_enabler_fault_fixup(struct work_struct *work){ struct user_event_enabler_fault *fault = container_of( work, struct user_event_enabler_fault, work); struct user_event_enabler *enabler = fault->enabler; struct user_event_mm *mm = fault->mm; unsigned long uaddr = enabler->addr; int attempt = fault->attempt; int ret; ret = user_event_mm_fault_in(mm, uaddr, attempt); if (ret && ret != -ENOENT) { struct user_event *user = enabler->event; pr_warn("user_events: Fault for mm: 0x%p @ 0x%llx event: %s\n", mm->mm, (unsigned long long)uaddr, EVENT_NAME(user)); } /* Prevent state changes from racing */ mutex_lock(&event_mutex); /* User asked for enabler to be removed during fault */ if (test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))) { user_event_enabler_destroy(enabler); goto out; } /* * If we managed to get the page, re-issue the write. We do not * want to get into a possible infinite loop, which is why we only * attempt again directly if the page came in. If we couldn't get * the page here, then we will try again the next time the event isLines 1–500 of 2961. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2021, Microsoft Corporation. * * Authors: * Beau Belgrave <beaub@linux.microsoft.com> */ #include <linux/bitmap.h> #include <linux/cdev.h> #include <linux/hashtable.h> #include <linux/list.h> #include <linux/io.h> #include <linux/uio.h> #include <linux/ioctl.h> #include <linux/jhash.h> #include <linux/refcount.h> #include <linux/trace_events.h> #include <linux/tracefs.h> #include <linux/types.h> #include <linux/uaccess.h> #include <linux/highmem.h> #include <linux/init.h> #include <linux/user_events.h> #include "trace_dynevent.h" #include "trace_output.h" #include "trace.h" #define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1) #define FIELD_DEPTH_TYPE 0 #define FIELD_DEPTH_NAME 1 #define FIELD_DEPTH_SIZE 2 /* Limit how long of an event name plus args within the subsystem. */ #define MAX_EVENT_DESC 512 #define EVENT_NAME(user_event) ((user_event)->reg_name) #define EVENT_TP_NAME(user_event) ((user_event)->tracepoint.name) #define MAX_FIELD_ARRAY_SIZE 1024 /* * Internal bits (kernel side only) to keep track of connected probes: * These are used when status is requested in text form about an event. These * bits are compared against an internal byte on the event to determine which * probes to print out to the user. * * These do not reflect the mapped bytes between the user and kernel space. */ #define EVENT_STATUS_FTRACE BIT(0) #define EVENT_STATUS_PERF BIT(1) #define EVENT_STATUS_OTHER BIT(7) /* * Stores the system name, tables, and locks for a group of events. This * allows isolation for events by various means. */ struct user_event_group { char *system_name; char *system_multi_name; struct hlist_node node; struct mutex reg_mutex; DECLARE_HASHTABLE(register_table, 8); /* ID that moves forward within the group for multi-event names */ u64 multi_id; }; /* Group for init_user_ns mapping, top-most group */ static struct user_event_group *init_group; /* Max allowed events for the whole system */ static unsigned int max_user_events = 32768; /* Current number of events on the whole system */ static unsigned int current_user_events; /* * Stores per-event properties, as users register events * within a file a user_event might be created if it does not * already exist. These are globally used and their lifetime * is tied to the refcnt member. These cannot go away until the * refcnt reaches one. */ struct user_event { struct user_event_group *group; char *reg_name; struct tracepoint tracepoint; struct trace_event_call call; struct trace_event_class class; struct dyn_event devent; struct hlist_node node; struct list_head fields; struct list_head validators; struct work_struct put_work; refcount_t refcnt; int min_size; int reg_flags; char status; }; /* * Stores per-mm/event properties that enable an address to be * updated properly for each task. As tasks are forked, we use * these to track enablement sites that are tied to an event. */ struct user_event_enabler { struct list_head mm_enablers_link; struct user_event *event; unsigned long addr; /* Track enable bit, flags, etc. Aligned for bitops. */ unsigned long values; /* Defer the event put and enabler free past an RCU grace period. */ struct rcu_work put_rwork; }; /* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */ #define ENABLE_VAL_BIT_MASK 0x3F /* Bit 6 is for faulting status of enablement */ #define ENABLE_VAL_FAULTING_BIT 6 /* Bit 7 is for freeing status of enablement */ #define ENABLE_VAL_FREEING_BIT 7 /* Bit 8 is for marking 32-bit on 64-bit */ #define ENABLE_VAL_32_ON_64_BIT 8 #define ENABLE_VAL_COMPAT_MASK (1 << ENABLE_VAL_32_ON_64_BIT) /* Only duplicate the bit and compat values */ #define ENABLE_VAL_DUP_MASK (ENABLE_VAL_BIT_MASK | ENABLE_VAL_COMPAT_MASK) #define ENABLE_BITOPS(e) (&(e)->values) #define ENABLE_BIT(e) ((int)((e)->values & ENABLE_VAL_BIT_MASK)) #define EVENT_MULTI_FORMAT(f) ((f) & USER_EVENT_REG_MULTI_FORMAT) /* Used for asynchronous faulting in of pages */ struct user_event_enabler_fault { struct work_struct work; struct user_event_mm *mm; struct user_event_enabler *enabler; int attempt; }; static struct kmem_cache *fault_cache; /* Global list of memory descriptors using user_events */ static LIST_HEAD(user_event_mms); static DEFINE_SPINLOCK(user_event_mms_lock); /* * Stores per-file events references, as users register events * within a file this structure is modified and freed via RCU. * The lifetime of this struct is tied to the lifetime of the file. * These are not shared and only accessible by the file that created it. */ struct user_event_refs { struct rcu_head rcu; int count; struct user_event *events[]; }; struct user_event_file_info { struct user_event_group *group; struct user_event_refs *refs; }; #define VALIDATOR_ENSURE_NULL (1 << 0) #define VALIDATOR_REL (1 << 1) struct user_event_validator { struct list_head user_event_link; int offset; int flags; }; static inline void align_addr_bit(unsigned long *addr, int *bit, unsigned long *flags) { if (IS_ALIGNED(*addr, sizeof(long))) { #ifdef __BIG_ENDIAN /* 32 bit on BE 64 bit requires a 32 bit offset when aligned. */ if (test_bit(ENABLE_VAL_32_ON_64_BIT, flags)) *bit += 32; #endif return; } *addr = ALIGN_DOWN(*addr, sizeof(long)); /* * We only support 32 and 64 bit values. The only time we need * to align is a 32 bit value on a 64 bit kernel, which on LE * is always 32 bits, and on BE requires no change when unaligned. */ #ifdef __LITTLE_ENDIAN *bit += 32; #endif } typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i, void *tpdata, bool *faulted); static int user_event_parse(struct user_event_group *group, char *name, char *args, char *flags, struct user_event **newuser, int reg_flags); static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm); static struct user_event_mm *user_event_mm_get_all(struct user_event *user); static void user_event_mm_put(struct user_event_mm *mm); static int destroy_user_event(struct user_event *user); static bool user_fields_match(struct user_event *user, int argc, const char **argv); static u32 user_event_key(char *name) { return jhash(name, strlen(name), 0); } static bool user_event_capable(u16 reg_flags) { /* Persistent events require CAP_PERFMON / CAP_SYS_ADMIN */ if (reg_flags & USER_EVENT_REG_PERSIST) { if (!perfmon_capable()) return false; } return true; } static struct user_event *user_event_get(struct user_event *user) { refcount_inc(&user->refcnt); return user; } static void delayed_destroy_user_event(struct work_struct *work) { struct user_event *user = container_of( work, struct user_event, put_work); mutex_lock(&event_mutex); if (!refcount_dec_and_test(&user->refcnt)) goto out; if (destroy_user_event(user)) { /* * The only reason this would fail here is if we cannot * update the visibility of the event. In this case the * event stays in the hashtable, waiting for someone to * attempt to delete it later. */ pr_warn("user_events: Unable to delete event\n"); refcount_set(&user->refcnt, 1); } out: mutex_unlock(&event_mutex); } static void user_event_put(struct user_event *user, bool locked) { bool delete; if (unlikely(!user)) return; /* * When the event is not enabled for auto-delete there will always * be at least 1 reference to the event. During the event creation * we initially set the refcnt to 2 to achieve this. In those cases * the caller must acquire event_mutex and after decrement check if * the refcnt is 1, meaning this is the last reference. When auto * delete is enabled, there will only be 1 ref, IE: refcnt will be * only set to 1 during creation to allow the below checks to go * through upon the last put. The last put must always be done with * the event mutex held. */ if (!locked) { lockdep_assert_not_held(&event_mutex); delete = refcount_dec_and_mutex_lock(&user->refcnt, &event_mutex); } else { lockdep_assert_held(&event_mutex); delete = refcount_dec_and_test(&user->refcnt); } if (!delete) return; /* * We now have the event_mutex in all cases, which ensures that * no new references will be taken until event_mutex is released. * New references come through find_user_event(), which requires * the event_mutex to be held. */ if (user->reg_flags & USER_EVENT_REG_PERSIST) { /* We should not get here when persist flag is set */ pr_alert("BUG: Auto-delete engaged on persistent event\n"); goto out; } /* * Unfortunately we have to attempt the actual destroy in a work * queue. This is because not all cases handle a trace_event_call * being removed within the class->reg() operation for unregister. */ INIT_WORK(&user->put_work, delayed_destroy_user_event); /* * Since the event is still in the hashtable, we have to re-inc * the ref count to 1. This count will be decremented and checked * in the work queue to ensure it's still the last ref. This is * needed because a user-process could register the same event in * between the time of event_mutex release and the work queue * running the delayed destroy. If we removed the item now from * the hashtable, this would result in a timing window where a * user process would fail a register because the trace_event_call * register would fail in the tracing layers. */ refcount_set(&user->refcnt, 1); if (WARN_ON_ONCE(!schedule_work(&user->put_work))) { /* * If we fail we must wait for an admin to attempt delete or * another register/close of the event, whichever is first. */ pr_warn("user_events: Unable to queue delayed destroy\n"); } out: /* Ensure if we didn't have event_mutex before we unlock it */ if (!locked) mutex_unlock(&event_mutex); } static void user_event_group_destroy(struct user_event_group *group) { kfree(group->system_name); kfree(group->system_multi_name); kfree(group); } static char *user_event_group_system_name(void) { char *system_name; int len = sizeof(USER_EVENTS_SYSTEM) + 1; system_name = kmalloc(len, GFP_KERNEL); if (!system_name) return NULL; snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM); return system_name; } static char *user_event_group_system_multi_name(void) { return kstrdup(USER_EVENTS_MULTI_SYSTEM, GFP_KERNEL); } static struct user_event_group *current_user_event_group(void) { return init_group; } static struct user_event_group *user_event_group_create(void) { struct user_event_group *group; group = kzalloc_obj(*group); if (!group) return NULL; group->system_name = user_event_group_system_name(); if (!group->system_name) goto error; group->system_multi_name = user_event_group_system_multi_name(); if (!group->system_multi_name) goto error; mutex_init(&group->reg_mutex); hash_init(group->register_table); return group; error: if (group) user_event_group_destroy(group); return NULL; }; static void delayed_user_event_enabler_put(struct work_struct *work) { struct user_event_enabler *enabler = container_of(to_rcu_work(work), struct user_event_enabler, put_rwork); /* No longer tracking the event via the enabler */ user_event_put(enabler->event, false); /* Run from queue_rcu_work(), the RCU grace period has elapsed */ kfree(enabler); } static void user_event_enabler_destroy(struct user_event_enabler *enabler) { list_del_rcu(&enabler->mm_enablers_link); /* * The enabler is removed from an RCU-traversed list * (user_event_mm_dup() walks mm->enablers under rcu_read_lock() only), * and readers there dereference enabler->event and take a new ref on * it. Both the put of that event reference and the free of the enabler * therefore have to wait for a grace period so no reader can be looking * at the enabler or racing the last put of its event. * * The put itself must not run in RCU context: when it drops the last * reference user_event_put() takes event_mutex, which cannot be taken * from a softirq/RCU callback. Defer both to a work item scheduled * after a grace period via queue_rcu_work(). */ INIT_RCU_WORK(&enabler->put_rwork, delayed_user_event_enabler_put); queue_rcu_work(system_percpu_wq, &enabler->put_rwork); } static int user_event_mm_fault_in(struct user_event_mm *mm, unsigned long uaddr, int attempt) { bool unlocked; int ret; /* * Normally this is low, ensure that it cannot be taken advantage of by * bad user processes to cause excessive looping. */ if (attempt > 10) return -EFAULT; mmap_read_lock(mm->mm); /* Ensure MM has tasks, cannot use after exit_mm() */ if (refcount_read(&mm->tasks) == 0) { ret = -ENOENT; goto out; } ret = fixup_user_fault(mm->mm, uaddr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, &unlocked); out: mmap_read_unlock(mm->mm); return ret; } static int user_event_enabler_write(struct user_event_mm *mm, struct user_event_enabler *enabler, bool fixup_fault, int *attempt); static void user_event_enabler_fault_fixup(struct work_struct *work) { struct user_event_enabler_fault *fault = container_of( work, struct user_event_enabler_fault, work); struct user_event_enabler *enabler = fault->enabler; struct user_event_mm *mm = fault->mm; unsigned long uaddr = enabler->addr; int attempt = fault->attempt; int ret; ret = user_event_mm_fault_in(mm, uaddr, attempt); if (ret && ret != -ENOENT) { struct user_event *user = enabler->event; pr_warn("user_events: Fault for mm: 0x%p @ 0x%llx event: %s\n", mm->mm, (unsigned long long)uaddr, EVENT_NAME(user)); } /* Prevent state changes from racing */ mutex_lock(&event_mutex); /* User asked for enabler to be removed during fault */ if (test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))) { user_event_enabler_destroy(enabler); goto out; } /* * If we managed to get the page, re-issue the write. We do not * want to get into a possible infinite loop, which is why we only * attempt again directly if the page came in. If we couldn't get * the page here, then we will try again the next time the event is