// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (c) 2021, Microsoft Corporation. * * Authors: * Beau Belgrave <beaub@linux.microsoft.com> */ #include <linux/bitmap.h>#include <linux/cdev.h>#include <linux/hashtable.h>#include <linux/list.h>#include <linux/io.h>#include <linux/uio.h>#include <linux/ioctl.h>#include <linux/jhash.h>#include <linux/refcount.h>#include <linux/trace_events.h>#include <linux/tracefs.h>#include <linux/types.h>#include <linux/uaccess.h>/* Reminder to move to uapi when everything works */#ifdef CONFIG_COMPILE_TEST#include <linux/user_events.h>#else#include <uapi/linux/user_events.h>#endif#include "trace.h"#include "trace_dynevent.h" #define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1) #define FIELD_DEPTH_TYPE 0#define FIELD_DEPTH_NAME 1#define FIELD_DEPTH_SIZE 2 /* * Limits how many trace_event calls user processes can create: * Must be a power of two of PAGE_SIZE. */#define MAX_PAGE_ORDER 0#define MAX_PAGES (1 << MAX_PAGE_ORDER)#define MAX_BYTES (MAX_PAGES * PAGE_SIZE)#define MAX_EVENTS (MAX_BYTES * 8) /* Limit how long of an event name plus args within the subsystem. */#define MAX_EVENT_DESC 512#define EVENT_NAME(user_event) ((user_event)->tracepoint.name)#define MAX_FIELD_ARRAY_SIZE 1024 /* * The MAP_STATUS_* macros are used for taking a index and determining the * appropriate byte and the bit in the byte to set/reset for an event. * * The lower 3 bits of the index decide which bit to set. * The remaining upper bits of the index decide which byte to use for the bit. * * This is used when an event has a probe attached/removed to reflect live * status of the event wanting tracing or not to user-programs via shared * memory maps. */#define MAP_STATUS_BYTE(index) ((index) >> 3)#define MAP_STATUS_MASK(index) BIT((index) & 7) /* * Internal bits (kernel side only) to keep track of connected probes: * These are used when status is requested in text form about an event. These * bits are compared against an internal byte on the event to determine which * probes to print out to the user. * * These do not reflect the mapped bytes between the user and kernel space. */#define EVENT_STATUS_FTRACE BIT(0)#define EVENT_STATUS_PERF BIT(1)#define EVENT_STATUS_OTHER BIT(7) /* * Stores the pages, tables, and locks for a group of events. * Each logical grouping of events has its own group, with a * matching page for status checks within user programs. This * allows for isolation of events to user programs by various * means. */struct user_event_group { struct page *pages; char *register_page_data; char *system_name; struct hlist_node node; struct mutex reg_mutex; DECLARE_HASHTABLE(register_table, 8); DECLARE_BITMAP(page_bitmap, MAX_EVENTS);}; /* Group for init_user_ns mapping, top-most group */static struct user_event_group *init_group; /* * Stores per-event properties, as users register events * within a file a user_event might be created if it does not * already exist. These are globally used and their lifetime * is tied to the refcnt member. These cannot go away until the * refcnt reaches one. */struct user_event { struct user_event_group *group; struct tracepoint tracepoint; struct trace_event_call call; struct trace_event_class class; struct dyn_event devent; struct hlist_node node; struct list_head fields; struct list_head validators; refcount_t refcnt; int index; int flags; int min_size; char status;}; /* * Stores per-file events references, as users register events * within a file this structure is modified and freed via RCU. * The lifetime of this struct is tied to the lifetime of the file. * These are not shared and only accessible by the file that created it. */struct user_event_refs { struct rcu_head rcu; int count; struct user_event *events[];}; struct user_event_file_info { struct user_event_group *group; struct user_event_refs *refs;}; #define VALIDATOR_ENSURE_NULL (1 << 0)#define VALIDATOR_REL (1 << 1) struct user_event_validator { struct list_head link; int offset; int flags;}; typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i, void *tpdata, bool *faulted); static int user_event_parse(struct user_event_group *group, char *name, char *args, char *flags, struct user_event **newuser); static u32 user_event_key(char *name){ return jhash(name, strlen(name), 0);} static void set_page_reservations(char *pages, bool set){ int page; for (page = 0; page < MAX_PAGES; ++page) { void *addr = pages + (PAGE_SIZE * page); if (set) SetPageReserved(virt_to_page(addr)); else ClearPageReserved(virt_to_page(addr)); }} static void user_event_group_destroy(struct user_event_group *group){ if (group->register_page_data) set_page_reservations(group->register_page_data, false); if (group->pages) __free_pages(group->pages, MAX_PAGE_ORDER); kfree(group->system_name); kfree(group);} static char *user_event_group_system_name(struct user_namespace *user_ns){ char *system_name; int len = sizeof(USER_EVENTS_SYSTEM) + 1; if (user_ns != &init_user_ns) { /* * Unexpected at this point: * We only currently support init_user_ns. * When we enable more, this will trigger a failure so log. */ pr_warn("user_events: Namespace other than init_user_ns!\n"); return NULL; } system_name = kmalloc(len, GFP_KERNEL); if (!system_name) return NULL; snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM); return system_name;} static inline struct user_event_group*user_event_group_from_user_ns(struct user_namespace *user_ns){ if (user_ns == &init_user_ns) return init_group; return NULL;} static struct user_event_group *current_user_event_group(void){ struct user_namespace *user_ns = current_user_ns(); struct user_event_group *group = NULL; while (user_ns) { group = user_event_group_from_user_ns(user_ns); if (group) break; user_ns = user_ns->parent; } return group;} static struct user_event_group*user_event_group_create(struct user_namespace *user_ns){ struct user_event_group *group; group = kzalloc(sizeof(*group), GFP_KERNEL); if (!group) return NULL; group->system_name = user_event_group_system_name(user_ns); if (!group->system_name) goto error; group->pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER); if (!group->pages) goto error; group->register_page_data = page_address(group->pages); set_page_reservations(group->register_page_data, true); /* Zero all bits beside 0 (which is reserved for failures) */ bitmap_zero(group->page_bitmap, MAX_EVENTS); set_bit(0, group->page_bitmap); mutex_init(&group->reg_mutex); hash_init(group->register_table); return group;error: if (group) user_event_group_destroy(group); return NULL;}; static __always_inlinevoid user_event_register_set(struct user_event *user){ int i = user->index; user->group->register_page_data[MAP_STATUS_BYTE(i)] |= MAP_STATUS_MASK(i);} static __always_inlinevoid user_event_register_clear(struct user_event *user){ int i = user->index; user->group->register_page_data[MAP_STATUS_BYTE(i)] &= ~MAP_STATUS_MASK(i);} static __always_inline __must_checkbool user_event_last_ref(struct user_event *user){ return refcount_read(&user->refcnt) == 1;} static __always_inline __must_checksize_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i){ size_t ret; pagefault_disable(); ret = copy_from_iter_nocache(addr, bytes, i); pagefault_enable(); return ret;} static struct list_head *user_event_get_fields(struct trace_event_call *call){ struct user_event *user = (struct user_event *)call->data; return &user->fields;} /* * Parses a register command for user_events * Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]] * * Example event named 'test' with a 20 char 'msg' field with an unsigned int * 'id' field after: * test char[20] msg;unsigned int id * * NOTE: Offsets are from the user data perspective, they are not from the * trace_entry/buffer perspective. We automatically add the common properties * sizes to the offset for the user. * * Upon success user_event has its ref count increased by 1. */static int user_event_parse_cmd(struct user_event_group *group, char *raw_command, struct user_event **newuser){ char *name = raw_command; char *args = strpbrk(name, " "); char *flags; if (args) *args++ = '\0'; flags = strpbrk(name, ":"); if (flags) *flags++ = '\0'; return user_event_parse(group, name, args, flags, newuser);} static int user_field_array_size(const char *type){ const char *start = strchr(type, '['); char val[8]; char *bracket; int size = 0; if (start == NULL) return -EINVAL; if (strscpy(val, start + 1, sizeof(val)) <= 0) return -EINVAL; bracket = strchr(val, ']'); if (!bracket) return -EINVAL; *bracket = '\0'; if (kstrtouint(val, 0, &size)) return -EINVAL; if (size > MAX_FIELD_ARRAY_SIZE) return -EINVAL; return size;} static int user_field_size(const char *type){ /* long is not allowed from a user, since it's ambigious in size */ if (strcmp(type, "s64") == 0) return sizeof(s64); if (strcmp(type, "u64") == 0) return sizeof(u64); if (strcmp(type, "s32") == 0) return sizeof(s32); if (strcmp(type, "u32") == 0) return sizeof(u32); if (strcmp(type, "int") == 0) return sizeof(int); if (strcmp(type, "unsigned int") == 0) return sizeof(unsigned int); if (strcmp(type, "s16") == 0) return sizeof(s16); if (strcmp(type, "u16") == 0) return sizeof(u16); if (strcmp(type, "short") == 0) return sizeof(short); if (strcmp(type, "unsigned short") == 0) return sizeof(unsigned short); if (strcmp(type, "s8") == 0) return sizeof(s8); if (strcmp(type, "u8") == 0) return sizeof(u8); if (strcmp(type, "char") == 0) return sizeof(char); if (strcmp(type, "unsigned char") == 0) return sizeof(unsigned char); if (str_has_prefix(type, "char[")) return user_field_array_size(type); if (str_has_prefix(type, "unsigned char[")) return user_field_array_size(type); if (str_has_prefix(type, "__data_loc ")) return sizeof(u32); if (str_has_prefix(type, "__rel_loc ")) return sizeof(u32); /* Uknown basic type, error */ return -EINVAL;} static void user_event_destroy_validators(struct user_event *user){ struct user_event_validator *validator, *next; struct list_head *head = &user->validators; list_for_each_entry_safe(validator, next, head, link) { list_del(&validator->link); kfree(validator); }} static void user_event_destroy_fields(struct user_event *user){ struct ftrace_event_field *field, *next; struct list_head *head = &user->fields; list_for_each_entry_safe(field, next, head, link) { list_del(&field->link); kfree(field); }} static int user_event_add_field(struct user_event *user, const char *type, const char *name, int offset, int size, int is_signed, int filter_type){ struct user_event_validator *validator; struct ftrace_event_field *field; int validator_flags = 0; field = kmalloc(sizeof(*field), GFP_KERNEL); if (!field) return -ENOMEM; if (str_has_prefix(type, "__data_loc ")) goto add_validator; if (str_has_prefix(type, "__rel_loc ")) { validator_flags |= VALIDATOR_REL; goto add_validator; } goto add_field; add_validator: if (strstr(type, "char") != NULL) validator_flags |= VALIDATOR_ENSURE_NULL; validator = kmalloc(sizeof(*validator), GFP_KERNEL); if (!validator) { kfree(field); return -ENOMEM; } validator->flags = validator_flags; validator->offset = offset; /* Want sequential access when validating */ list_add_tail(&validator->link, &user->validators); add_field: field->type = type; field->name = name; field->offset = offset; field->size = size; field->is_signed = is_signed; field->filter_type = filter_type; list_add(&field->link, &user->fields); /* * Min size from user writes that are required, this does not include * the size of trace_entry (common fields). */ user->min_size = (offset + size) - sizeof(struct trace_entry); return 0;Lines 1–500 of 1915. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2021, Microsoft Corporation. * * Authors: * Beau Belgrave <beaub@linux.microsoft.com> */ #include <linux/bitmap.h> #include <linux/cdev.h> #include <linux/hashtable.h> #include <linux/list.h> #include <linux/io.h> #include <linux/uio.h> #include <linux/ioctl.h> #include <linux/jhash.h> #include <linux/refcount.h> #include <linux/trace_events.h> #include <linux/tracefs.h> #include <linux/types.h> #include <linux/uaccess.h> /* Reminder to move to uapi when everything works */ #ifdef CONFIG_COMPILE_TEST #include <linux/user_events.h> #else #include <uapi/linux/user_events.h> #endif #include "trace.h" #include "trace_dynevent.h" #define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1) #define FIELD_DEPTH_TYPE 0 #define FIELD_DEPTH_NAME 1 #define FIELD_DEPTH_SIZE 2 /* * Limits how many trace_event calls user processes can create: * Must be a power of two of PAGE_SIZE. */ #define MAX_PAGE_ORDER 0 #define MAX_PAGES (1 << MAX_PAGE_ORDER) #define MAX_BYTES (MAX_PAGES * PAGE_SIZE) #define MAX_EVENTS (MAX_BYTES * 8) /* Limit how long of an event name plus args within the subsystem. */ #define MAX_EVENT_DESC 512 #define EVENT_NAME(user_event) ((user_event)->tracepoint.name) #define MAX_FIELD_ARRAY_SIZE 1024 /* * The MAP_STATUS_* macros are used for taking a index and determining the * appropriate byte and the bit in the byte to set/reset for an event. * * The lower 3 bits of the index decide which bit to set. * The remaining upper bits of the index decide which byte to use for the bit. * * This is used when an event has a probe attached/removed to reflect live * status of the event wanting tracing or not to user-programs via shared * memory maps. */ #define MAP_STATUS_BYTE(index) ((index) >> 3) #define MAP_STATUS_MASK(index) BIT((index) & 7) /* * Internal bits (kernel side only) to keep track of connected probes: * These are used when status is requested in text form about an event. These * bits are compared against an internal byte on the event to determine which * probes to print out to the user. * * These do not reflect the mapped bytes between the user and kernel space. */ #define EVENT_STATUS_FTRACE BIT(0) #define EVENT_STATUS_PERF BIT(1) #define EVENT_STATUS_OTHER BIT(7) /* * Stores the pages, tables, and locks for a group of events. * Each logical grouping of events has its own group, with a * matching page for status checks within user programs. This * allows for isolation of events to user programs by various * means. */ struct user_event_group { struct page *pages; char *register_page_data; char *system_name; struct hlist_node node; struct mutex reg_mutex; DECLARE_HASHTABLE(register_table, 8); DECLARE_BITMAP(page_bitmap, MAX_EVENTS); }; /* Group for init_user_ns mapping, top-most group */ static struct user_event_group *init_group; /* * Stores per-event properties, as users register events * within a file a user_event might be created if it does not * already exist. These are globally used and their lifetime * is tied to the refcnt member. These cannot go away until the * refcnt reaches one. */ struct user_event { struct user_event_group *group; struct tracepoint tracepoint; struct trace_event_call call; struct trace_event_class class; struct dyn_event devent; struct hlist_node node; struct list_head fields; struct list_head validators; refcount_t refcnt; int index; int flags; int min_size; char status; }; /* * Stores per-file events references, as users register events * within a file this structure is modified and freed via RCU. * The lifetime of this struct is tied to the lifetime of the file. * These are not shared and only accessible by the file that created it. */ struct user_event_refs { struct rcu_head rcu; int count; struct user_event *events[]; }; struct user_event_file_info { struct user_event_group *group; struct user_event_refs *refs; }; #define VALIDATOR_ENSURE_NULL (1 << 0) #define VALIDATOR_REL (1 << 1) struct user_event_validator { struct list_head link; int offset; int flags; }; typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i, void *tpdata, bool *faulted); static int user_event_parse(struct user_event_group *group, char *name, char *args, char *flags, struct user_event **newuser); static u32 user_event_key(char *name) { return jhash(name, strlen(name), 0); } static void set_page_reservations(char *pages, bool set) { int page; for (page = 0; page < MAX_PAGES; ++page) { void *addr = pages + (PAGE_SIZE * page); if (set) SetPageReserved(virt_to_page(addr)); else ClearPageReserved(virt_to_page(addr)); } } static void user_event_group_destroy(struct user_event_group *group) { if (group->register_page_data) set_page_reservations(group->register_page_data, false); if (group->pages) __free_pages(group->pages, MAX_PAGE_ORDER); kfree(group->system_name); kfree(group); } static char *user_event_group_system_name(struct user_namespace *user_ns) { char *system_name; int len = sizeof(USER_EVENTS_SYSTEM) + 1; if (user_ns != &init_user_ns) { /* * Unexpected at this point: * We only currently support init_user_ns. * When we enable more, this will trigger a failure so log. */ pr_warn("user_events: Namespace other than init_user_ns!\n"); return NULL; } system_name = kmalloc(len, GFP_KERNEL); if (!system_name) return NULL; snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM); return system_name; } static inline struct user_event_group *user_event_group_from_user_ns(struct user_namespace *user_ns) { if (user_ns == &init_user_ns) return init_group; return NULL; } static struct user_event_group *current_user_event_group(void) { struct user_namespace *user_ns = current_user_ns(); struct user_event_group *group = NULL; while (user_ns) { group = user_event_group_from_user_ns(user_ns); if (group) break; user_ns = user_ns->parent; } return group; } static struct user_event_group *user_event_group_create(struct user_namespace *user_ns) { struct user_event_group *group; group = kzalloc(sizeof(*group), GFP_KERNEL); if (!group) return NULL; group->system_name = user_event_group_system_name(user_ns); if (!group->system_name) goto error; group->pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER); if (!group->pages) goto error; group->register_page_data = page_address(group->pages); set_page_reservations(group->register_page_data, true); /* Zero all bits beside 0 (which is reserved for failures) */ bitmap_zero(group->page_bitmap, MAX_EVENTS); set_bit(0, group->page_bitmap); mutex_init(&group->reg_mutex); hash_init(group->register_table); return group; error: if (group) user_event_group_destroy(group); return NULL; }; static __always_inline void user_event_register_set(struct user_event *user) { int i = user->index; user->group->register_page_data[MAP_STATUS_BYTE(i)] |= MAP_STATUS_MASK(i); } static __always_inline void user_event_register_clear(struct user_event *user) { int i = user->index; user->group->register_page_data[MAP_STATUS_BYTE(i)] &= ~MAP_STATUS_MASK(i); } static __always_inline __must_check bool user_event_last_ref(struct user_event *user) { return refcount_read(&user->refcnt) == 1; } static __always_inline __must_check size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i) { size_t ret; pagefault_disable(); ret = copy_from_iter_nocache(addr, bytes, i); pagefault_enable(); return ret; } static struct list_head *user_event_get_fields(struct trace_event_call *call) { struct user_event *user = (struct user_event *)call->data; return &user->fields; } /* * Parses a register command for user_events * Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]] * * Example event named 'test' with a 20 char 'msg' field with an unsigned int * 'id' field after: * test char[20] msg;unsigned int id * * NOTE: Offsets are from the user data perspective, they are not from the * trace_entry/buffer perspective. We automatically add the common properties * sizes to the offset for the user. * * Upon success user_event has its ref count increased by 1. */ static int user_event_parse_cmd(struct user_event_group *group, char *raw_command, struct user_event **newuser) { char *name = raw_command; char *args = strpbrk(name, " "); char *flags; if (args) *args++ = '\0'; flags = strpbrk(name, ":"); if (flags) *flags++ = '\0'; return user_event_parse(group, name, args, flags, newuser); } static int user_field_array_size(const char *type) { const char *start = strchr(type, '['); char val[8]; char *bracket; int size = 0; if (start == NULL) return -EINVAL; if (strscpy(val, start + 1, sizeof(val)) <= 0) return -EINVAL; bracket = strchr(val, ']'); if (!bracket) return -EINVAL; *bracket = '\0'; if (kstrtouint(val, 0, &size)) return -EINVAL; if (size > MAX_FIELD_ARRAY_SIZE) return -EINVAL; return size; } static int user_field_size(const char *type) { /* long is not allowed from a user, since it's ambigious in size */ if (strcmp(type, "s64") == 0) return sizeof(s64); if (strcmp(type, "u64") == 0) return sizeof(u64); if (strcmp(type, "s32") == 0) return sizeof(s32); if (strcmp(type, "u32") == 0) return sizeof(u32); if (strcmp(type, "int") == 0) return sizeof(int); if (strcmp(type, "unsigned int") == 0) return sizeof(unsigned int); if (strcmp(type, "s16") == 0) return sizeof(s16); if (strcmp(type, "u16") == 0) return sizeof(u16); if (strcmp(type, "short") == 0) return sizeof(short); if (strcmp(type, "unsigned short") == 0) return sizeof(unsigned short); if (strcmp(type, "s8") == 0) return sizeof(s8); if (strcmp(type, "u8") == 0) return sizeof(u8); if (strcmp(type, "char") == 0) return sizeof(char); if (strcmp(type, "unsigned char") == 0) return sizeof(unsigned char); if (str_has_prefix(type, "char[")) return user_field_array_size(type); if (str_has_prefix(type, "unsigned char[")) return user_field_array_size(type); if (str_has_prefix(type, "__data_loc ")) return sizeof(u32); if (str_has_prefix(type, "__rel_loc ")) return sizeof(u32); /* Uknown basic type, error */ return -EINVAL; } static void user_event_destroy_validators(struct user_event *user) { struct user_event_validator *validator, *next; struct list_head *head = &user->validators; list_for_each_entry_safe(validator, next, head, link) { list_del(&validator->link); kfree(validator); } } static void user_event_destroy_fields(struct user_event *user) { struct ftrace_event_field *field, *next; struct list_head *head = &user->fields; list_for_each_entry_safe(field, next, head, link) { list_del(&field->link); kfree(field); } } static int user_event_add_field(struct user_event *user, const char *type, const char *name, int offset, int size, int is_signed, int filter_type) { struct user_event_validator *validator; struct ftrace_event_field *field; int validator_flags = 0; field = kmalloc(sizeof(*field), GFP_KERNEL); if (!field) return -ENOMEM; if (str_has_prefix(type, "__data_loc ")) goto add_validator; if (str_has_prefix(type, "__rel_loc ")) { validator_flags |= VALIDATOR_REL; goto add_validator; } goto add_field; add_validator: if (strstr(type, "char") != NULL) validator_flags |= VALIDATOR_ENSURE_NULL; validator = kmalloc(sizeof(*validator), GFP_KERNEL); if (!validator) { kfree(field); return -ENOMEM; } validator->flags = validator_flags; validator->offset = offset; /* Want sequential access when validating */ list_add_tail(&validator->link, &user->validators); add_field: field->type = type; field->name = name; field->offset = offset; field->size = size; field->is_signed = is_signed; field->filter_type = filter_type; list_add(&field->link, &user->fields); /* * Min size from user writes that are required, this does not include * the size of trace_entry (common fields). */ user->min_size = (offset + size) - sizeof(struct trace_entry); return 0;