obj = container_of(kobj, struct klp_object, kobj); return sysfs_emit(buf, "%d\n", obj->patched);} static struct kobj_attribute patched_kobj_attr = __ATTR_RO(patched);static struct attribute *klp_object_attrs[] = { &patched_kobj_attr.attr, NULL,};ATTRIBUTE_GROUPS(klp_object); static void klp_free_object_dynamic(struct klp_object *obj){ kfree(obj->name); kfree(obj);} static void klp_init_func_early(struct klp_object *obj, struct klp_func *func);static void klp_init_object_early(struct klp_patch *patch, struct klp_object *obj); static struct klp_object *klp_alloc_object_dynamic(const char *name, struct klp_patch *patch){ struct klp_object *obj; obj = kzalloc_obj(*obj); if (!obj) return NULL; if (name) { obj->name = kstrdup(name, GFP_KERNEL); if (!obj->name) { kfree(obj); return NULL; } } klp_init_object_early(patch, obj); obj->dynamic = true; return obj;} static void klp_free_func_nop(struct klp_func *func){ kfree(func->old_name); kfree(func);} static struct klp_func *klp_alloc_func_nop(struct klp_func *old_func, struct klp_object *obj){ struct klp_func *func; func = kzalloc_obj(*func); if (!func) return NULL; if (old_func->old_name) { func->old_name = kstrdup(old_func->old_name, GFP_KERNEL); if (!func->old_name) { kfree(func); return NULL; } } klp_init_func_early(obj, func); /* * func->new_func is same as func->old_func. These addresses are * set when the object is loaded, see klp_init_object_loaded(). */ func->old_sympos = old_func->old_sympos; func->nop = true; return func;} static int klp_add_object_nops(struct klp_patch *patch, struct klp_object *old_obj){ struct klp_object *obj; struct klp_func *func, *old_func; obj = klp_find_object(patch, old_obj); if (!obj) { obj = klp_alloc_object_dynamic(old_obj->name, patch); if (!obj) return -ENOMEM; } klp_for_each_func(old_obj, old_func) { func = klp_find_func(obj, old_func); if (func) continue; func = klp_alloc_func_nop(old_func, obj); if (!func) return -ENOMEM; } return 0;} /* * Add 'nop' functions which simply return to the caller to run the * original function. * * They are added only when the atomic replace mode is used and only for * functions which are currently livepatched but are no longer included * in the new livepatch. */static int klp_add_nops(struct klp_patch *patch){ struct klp_patch *old_patch; struct klp_object *old_obj; klp_for_each_patch(old_patch) { klp_for_each_object(old_patch, old_obj) { int err; err = klp_add_object_nops(patch, old_obj); if (err) return err; } } return 0;} static void klp_kobj_release_patch(struct kobject *kobj){ struct klp_patch *patch; patch = container_of(kobj, struct klp_patch, kobj); complete(&patch->finish);} static const struct kobj_type klp_ktype_patch = { .release = klp_kobj_release_patch, .sysfs_ops = &kobj_sysfs_ops, .default_groups = klp_patch_groups,}; static void klp_kobj_release_object(struct kobject *kobj){ struct klp_object *obj; obj = container_of(kobj, struct klp_object, kobj); if (obj->dynamic) klp_free_object_dynamic(obj);} static const struct kobj_type klp_ktype_object = { .release = klp_kobj_release_object, .sysfs_ops = &kobj_sysfs_ops, .default_groups = klp_object_groups,}; static void klp_kobj_release_func(struct kobject *kobj){ struct klp_func *func; func = container_of(kobj, struct klp_func, kobj); if (func->nop) klp_free_func_nop(func);} static const struct kobj_type klp_ktype_func = { .release = klp_kobj_release_func, .sysfs_ops = &kobj_sysfs_ops,}; static void __klp_free_funcs(struct klp_object *obj, bool nops_only){ struct klp_func *func, *tmp_func; klp_for_each_func_safe(obj, func, tmp_func) { if (nops_only && !func->nop) continue; list_del(&func->node); kobject_put(&func->kobj); }} /* Clean up when a patched object is unloaded */static void klp_free_object_loaded(struct klp_object *obj){ struct klp_func *func; obj->mod = NULL; klp_for_each_func(obj, func) { func->old_func = NULL; if (func->nop) func->new_func = NULL; }} static void __klp_free_objects(struct klp_patch *patch, bool nops_only){ struct klp_object *obj, *tmp_obj; klp_for_each_object_safe(patch, obj, tmp_obj) { __klp_free_funcs(obj, nops_only); if (nops_only && !obj->dynamic) continue; list_del(&obj->node); kobject_put(&obj->kobj); }} static void klp_free_objects(struct klp_patch *patch){ __klp_free_objects(patch, false);} static void klp_free_objects_dynamic(struct klp_patch *patch){ __klp_free_objects(patch, true);} /* * This function implements the free operations that can be called safely * under klp_mutex. * * The operation must be completed by calling klp_free_patch_finish() * outside klp_mutex. */static void klp_free_patch_start(struct klp_patch *patch){ if (!list_empty(&patch->list)) list_del(&patch->list); klp_free_objects(patch);} /* * This function implements the free part that must be called outside * klp_mutex. * * It must be called after klp_free_patch_start(). And it has to be * the last function accessing the livepatch structures when the patch * gets disabled. */static void klp_free_patch_finish(struct klp_patch *patch){ /* * Avoid deadlock with enabled_store() sysfs callback by * calling this outside klp_mutex. It is safe because * this is called when the patch gets disabled and it * cannot get enabled again. */ kobject_put(&patch->kobj); wait_for_completion(&patch->finish); /* Put the module after the last access to struct klp_patch. */ if (!patch->forced) module_put(patch->mod);} /* * The livepatch might be freed from sysfs interface created by the patch. * This work allows to wait until the interface is destroyed in a separate * context. */static void klp_free_patch_work_fn(struct work_struct *work){ struct klp_patch *patch = container_of(work, struct klp_patch, free_work); klp_free_patch_finish(patch);} void klp_free_patch_async(struct klp_patch *patch){ klp_free_patch_start(patch); schedule_work(&patch->free_work);} void klp_free_replaced_patches_async(struct klp_patch *new_patch){ struct klp_patch *old_patch, *tmp_patch; klp_for_each_patch_safe(old_patch, tmp_patch) { if (old_patch == new_patch) return; klp_free_patch_async(old_patch); }} static int klp_init_func(struct klp_object *obj, struct klp_func *func){ if (!func->old_name) return -EINVAL; /* * NOPs get the address later. The patched module must be loaded, * see klp_init_object_loaded(). */ if (!func->new_func && !func->nop) return -EINVAL; if (strlen(func->old_name) >= KSYM_NAME_LEN) return -EINVAL; INIT_LIST_HEAD(&func->stack_node); func->patched = false; func->transition = false; /* The format for the sysfs directory is <function,sympos> where sympos * is the nth occurrence of this symbol in kallsyms for the patched * object. If the user selects 0 for old_sympos, then 1 will be used * since a unique symbol will be the first occurrence. */ return kobject_add(&func->kobj, &obj->kobj, "%s,%lu", func->old_name, func->old_sympos ? func->old_sympos : 1);} static int klp_write_object_relocs(struct klp_patch *patch, struct klp_object *obj, bool apply){ int i, ret; struct klp_modinfo *info = patch->mod->klp_info; for (i = 1; i < info->hdr.e_shnum; i++) { Elf_Shdr *sec = info->sechdrs + i; if (!(sec->sh_flags & SHF_RELA_LIVEPATCH)) continue; ret = klp_write_section_relocs(patch->mod, info->sechdrs, info->secstrings, patch->mod->core_kallsyms.strtab, info->symndx, i, obj->name, apply); if (ret) return ret; } return 0;} static int klp_apply_object_relocs(struct klp_patch *patch, struct klp_object *obj){ return klp_write_object_relocs(patch, obj, true);} static void klp_clear_object_relocs(struct klp_patch *patch, struct klp_object *obj){ klp_write_object_relocs(patch, obj, false);} /* parts of the initialization that is done only when the object is loaded */static int klp_init_object_loaded(struct klp_patch *patch, struct klp_object *obj){ struct klp_func *func; int ret; if (klp_is_module(obj)) { /* * Only write module-specific relocations here * (.klp.rela.{module}.*). vmlinux-specific relocations were * written earlier during the initialization of the klp module * itself. */ ret = klp_apply_object_relocs(patch, obj); if (ret) return ret; } klp_for_each_func(obj, func) { ret = klp_find_object_symbol(obj->name, func->old_name, func->old_sympos, (unsigned long *)&func->old_func); if (ret) return ret; ret = kallsyms_lookup_size_offset((unsigned long)func->old_func, &func->old_size, NULL); if (!ret) { pr_err("kallsyms size lookup failed for '%s'\n", func->old_name); return -ENOENT; } if (func->nop) func->new_func = func->old_func; ret = kallsyms_lookup_size_offset((unsigned long)func->new_func, &func->new_size, NULL); if (!ret) { pr_err("kallsyms size lookup failed for '%s' replacement\n", func->old_name); return -ENOENT; } } return 0;} static int klp_init_object(struct klp_patch *patch, struct klp_object *obj){ struct klp_func *func; int ret; const char *name; if (klp_is_module(obj) && strlen(obj->name) >= MODULE_NAME_LEN) return -EINVAL; obj->patched = false; obj->mod = NULL; klp_find_object_module(obj); name = klp_is_module(obj) ? obj->name : "vmlinux"; ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name); if (ret) return ret; klp_for_each_func(obj, func) { ret = klp_init_func(obj, func); if (ret) return ret; } if (klp_is_object_loaded(obj)) ret = klp_init_object_loaded(patch, obj); return ret;} static void klp_init_func_early(struct klp_object *obj, struct klp_func *func){ kobject_init(&func->kobj, &klp_ktype_func); list_add_tail(&func->node, &obj->func_list);} static void klp_init_object_early(struct klp_patch *patch, struct klp_object *obj){ INIT_LIST_HEAD(&obj->func_list); kobject_init(&obj->kobj, &klp_ktype_object); list_add_tail(&obj->node, &patch->obj_list);} static void klp_init_patch_early(struct klp_patch *patch){ struct klp_object *obj; struct klp_func *func; INIT_LIST_HEAD(&patch->list); INIT_LIST_HEAD(&patch->obj_list); kobject_init(&patch->kobj, &klp_ktype_patch); patch->enabled = false; patch->forced = false; INIT_WORK(&patch->free_work, klp_free_patch_work_fn); init_completion(&patch->finish); klp_for_each_object_static(patch, obj) { klp_init_object_early(patch, obj); klp_for_each_func_static(obj, func) { klp_init_func_early(obj, func); } }} static int klp_init_patch(struct klp_patch *patch){ struct klp_object *obj; int ret; ret = kobject_add(&patch->kobj, klp_root_kobj, "%s", patch->mod->name); if (ret) return ret; if (patch->replace) { ret = klp_add_nops(patch); if (ret) return ret; } klp_for_each_object(patch, obj) { ret = klp_init_object(patch, obj); if (ret) return ret;Lines 501–1000 of 1387. Previous lines Load more
obj = container_of(kobj, struct klp_object, kobj); return sysfs_emit(buf, "%d\n", obj->patched); } static struct kobj_attribute patched_kobj_attr = __ATTR_RO(patched); static struct attribute *klp_object_attrs[] = { &patched_kobj_attr.attr, NULL, }; ATTRIBUTE_GROUPS(klp_object); static void klp_free_object_dynamic(struct klp_object *obj) { kfree(obj->name); kfree(obj); } static void klp_init_func_early(struct klp_object *obj, struct klp_func *func); static void klp_init_object_early(struct klp_patch *patch, struct klp_object *obj); static struct klp_object *klp_alloc_object_dynamic(const char *name, struct klp_patch *patch) { struct klp_object *obj; obj = kzalloc_obj(*obj); if (!obj) return NULL; if (name) { obj->name = kstrdup(name, GFP_KERNEL); if (!obj->name) { kfree(obj); return NULL; } } klp_init_object_early(patch, obj); obj->dynamic = true; return obj; } static void klp_free_func_nop(struct klp_func *func) { kfree(func->old_name); kfree(func); } static struct klp_func *klp_alloc_func_nop(struct klp_func *old_func, struct klp_object *obj) { struct klp_func *func; func = kzalloc_obj(*func); if (!func) return NULL; if (old_func->old_name) { func->old_name = kstrdup(old_func->old_name, GFP_KERNEL); if (!func->old_name) { kfree(func); return NULL; } } klp_init_func_early(obj, func); /* * func->new_func is same as func->old_func. These addresses are * set when the object is loaded, see klp_init_object_loaded(). */ func->old_sympos = old_func->old_sympos; func->nop = true; return func; } static int klp_add_object_nops(struct klp_patch *patch, struct klp_object *old_obj) { struct klp_object *obj; struct klp_func *func, *old_func; obj = klp_find_object(patch, old_obj); if (!obj) { obj = klp_alloc_object_dynamic(old_obj->name, patch); if (!obj) return -ENOMEM; } klp_for_each_func(old_obj, old_func) { func = klp_find_func(obj, old_func); if (func) continue; func = klp_alloc_func_nop(old_func, obj); if (!func) return -ENOMEM; } return 0; } /* * Add 'nop' functions which simply return to the caller to run the * original function. * * They are added only when the atomic replace mode is used and only for * functions which are currently livepatched but are no longer included * in the new livepatch. */ static int klp_add_nops(struct klp_patch *patch) { struct klp_patch *old_patch; struct klp_object *old_obj; klp_for_each_patch(old_patch) { klp_for_each_object(old_patch, old_obj) { int err; err = klp_add_object_nops(patch, old_obj); if (err) return err; } } return 0; } static void klp_kobj_release_patch(struct kobject *kobj) { struct klp_patch *patch; patch = container_of(kobj, struct klp_patch, kobj); complete(&patch->finish); } static const struct kobj_type klp_ktype_patch = { .release = klp_kobj_release_patch, .sysfs_ops = &kobj_sysfs_ops, .default_groups = klp_patch_groups, }; static void klp_kobj_release_object(struct kobject *kobj) { struct klp_object *obj; obj = container_of(kobj, struct klp_object, kobj); if (obj->dynamic) klp_free_object_dynamic(obj); } static const struct kobj_type klp_ktype_object = { .release = klp_kobj_release_object, .sysfs_ops = &kobj_sysfs_ops, .default_groups = klp_object_groups, }; static void klp_kobj_release_func(struct kobject *kobj) { struct klp_func *func; func = container_of(kobj, struct klp_func, kobj); if (func->nop) klp_free_func_nop(func); } static const struct kobj_type klp_ktype_func = { .release = klp_kobj_release_func, .sysfs_ops = &kobj_sysfs_ops, }; static void __klp_free_funcs(struct klp_object *obj, bool nops_only) { struct klp_func *func, *tmp_func; klp_for_each_func_safe(obj, func, tmp_func) { if (nops_only && !func->nop) continue; list_del(&func->node); kobject_put(&func->kobj); } } /* Clean up when a patched object is unloaded */ static void klp_free_object_loaded(struct klp_object *obj) { struct klp_func *func; obj->mod = NULL; klp_for_each_func(obj, func) { func->old_func = NULL; if (func->nop) func->new_func = NULL; } } static void __klp_free_objects(struct klp_patch *patch, bool nops_only) { struct klp_object *obj, *tmp_obj; klp_for_each_object_safe(patch, obj, tmp_obj) { __klp_free_funcs(obj, nops_only); if (nops_only && !obj->dynamic) continue; list_del(&obj->node); kobject_put(&obj->kobj); } } static void klp_free_objects(struct klp_patch *patch) { __klp_free_objects(patch, false); } static void klp_free_objects_dynamic(struct klp_patch *patch) { __klp_free_objects(patch, true); } /* * This function implements the free operations that can be called safely * under klp_mutex. * * The operation must be completed by calling klp_free_patch_finish() * outside klp_mutex. */ static void klp_free_patch_start(struct klp_patch *patch) { if (!list_empty(&patch->list)) list_del(&patch->list); klp_free_objects(patch); } /* * This function implements the free part that must be called outside * klp_mutex. * * It must be called after klp_free_patch_start(). And it has to be * the last function accessing the livepatch structures when the patch * gets disabled. */ static void klp_free_patch_finish(struct klp_patch *patch) { /* * Avoid deadlock with enabled_store() sysfs callback by * calling this outside klp_mutex. It is safe because * this is called when the patch gets disabled and it * cannot get enabled again. */ kobject_put(&patch->kobj); wait_for_completion(&patch->finish); /* Put the module after the last access to struct klp_patch. */ if (!patch->forced) module_put(patch->mod); } /* * The livepatch might be freed from sysfs interface created by the patch. * This work allows to wait until the interface is destroyed in a separate * context. */ static void klp_free_patch_work_fn(struct work_struct *work) { struct klp_patch *patch = container_of(work, struct klp_patch, free_work); klp_free_patch_finish(patch); } void klp_free_patch_async(struct klp_patch *patch) { klp_free_patch_start(patch); schedule_work(&patch->free_work); } void klp_free_replaced_patches_async(struct klp_patch *new_patch) { struct klp_patch *old_patch, *tmp_patch; klp_for_each_patch_safe(old_patch, tmp_patch) { if (old_patch == new_patch) return; klp_free_patch_async(old_patch); } } static int klp_init_func(struct klp_object *obj, struct klp_func *func) { if (!func->old_name) return -EINVAL; /* * NOPs get the address later. The patched module must be loaded, * see klp_init_object_loaded(). */ if (!func->new_func && !func->nop) return -EINVAL; if (strlen(func->old_name) >= KSYM_NAME_LEN) return -EINVAL; INIT_LIST_HEAD(&func->stack_node); func->patched = false; func->transition = false; /* The format for the sysfs directory is <function,sympos> where sympos * is the nth occurrence of this symbol in kallsyms for the patched * object. If the user selects 0 for old_sympos, then 1 will be used * since a unique symbol will be the first occurrence. */ return kobject_add(&func->kobj, &obj->kobj, "%s,%lu", func->old_name, func->old_sympos ? func->old_sympos : 1); } static int klp_write_object_relocs(struct klp_patch *patch, struct klp_object *obj, bool apply) { int i, ret; struct klp_modinfo *info = patch->mod->klp_info; for (i = 1; i < info->hdr.e_shnum; i++) { Elf_Shdr *sec = info->sechdrs + i; if (!(sec->sh_flags & SHF_RELA_LIVEPATCH)) continue; ret = klp_write_section_relocs(patch->mod, info->sechdrs, info->secstrings, patch->mod->core_kallsyms.strtab, info->symndx, i, obj->name, apply); if (ret) return ret; } return 0; } static int klp_apply_object_relocs(struct klp_patch *patch, struct klp_object *obj) { return klp_write_object_relocs(patch, obj, true); } static void klp_clear_object_relocs(struct klp_patch *patch, struct klp_object *obj) { klp_write_object_relocs(patch, obj, false); } /* parts of the initialization that is done only when the object is loaded */ static int klp_init_object_loaded(struct klp_patch *patch, struct klp_object *obj) { struct klp_func *func; int ret; if (klp_is_module(obj)) { /* * Only write module-specific relocations here * (.klp.rela.{module}.*). vmlinux-specific relocations were * written earlier during the initialization of the klp module * itself. */ ret = klp_apply_object_relocs(patch, obj); if (ret) return ret; } klp_for_each_func(obj, func) { ret = klp_find_object_symbol(obj->name, func->old_name, func->old_sympos, (unsigned long *)&func->old_func); if (ret) return ret; ret = kallsyms_lookup_size_offset((unsigned long)func->old_func, &func->old_size, NULL); if (!ret) { pr_err("kallsyms size lookup failed for '%s'\n", func->old_name); return -ENOENT; } if (func->nop) func->new_func = func->old_func; ret = kallsyms_lookup_size_offset((unsigned long)func->new_func, &func->new_size, NULL); if (!ret) { pr_err("kallsyms size lookup failed for '%s' replacement\n", func->old_name); return -ENOENT; } } return 0; } static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) { struct klp_func *func; int ret; const char *name; if (klp_is_module(obj) && strlen(obj->name) >= MODULE_NAME_LEN) return -EINVAL; obj->patched = false; obj->mod = NULL; klp_find_object_module(obj); name = klp_is_module(obj) ? obj->name : "vmlinux"; ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name); if (ret) return ret; klp_for_each_func(obj, func) { ret = klp_init_func(obj, func); if (ret) return ret; } if (klp_is_object_loaded(obj)) ret = klp_init_object_loaded(patch, obj); return ret; } static void klp_init_func_early(struct klp_object *obj, struct klp_func *func) { kobject_init(&func->kobj, &klp_ktype_func); list_add_tail(&func->node, &obj->func_list); } static void klp_init_object_early(struct klp_patch *patch, struct klp_object *obj) { INIT_LIST_HEAD(&obj->func_list); kobject_init(&obj->kobj, &klp_ktype_object); list_add_tail(&obj->node, &patch->obj_list); } static void klp_init_patch_early(struct klp_patch *patch) { struct klp_object *obj; struct klp_func *func; INIT_LIST_HEAD(&patch->list); INIT_LIST_HEAD(&patch->obj_list); kobject_init(&patch->kobj, &klp_ktype_patch); patch->enabled = false; patch->forced = false; INIT_WORK(&patch->free_work, klp_free_patch_work_fn); init_completion(&patch->finish); klp_for_each_object_static(patch, obj) { klp_init_object_early(patch, obj); klp_for_each_func_static(obj, func) { klp_init_func_early(obj, func); } } } static int klp_init_patch(struct klp_patch *patch) { struct klp_object *obj; int ret; ret = kobject_add(&patch->kobj, klp_root_kobj, "%s", patch->mod->name); if (ret) return ret; if (patch->replace) { ret = klp_add_nops(patch); if (ret) return ret; } klp_for_each_object(patch, obj) { ret = klp_init_object(patch, obj); if (ret) return ret;