} list_add_tail(&patch->list, &klp_patches); return 0;} static int __klp_disable_patch(struct klp_patch *patch){ struct klp_object *obj; if (WARN_ON(!patch->enabled)) return -EINVAL; if (klp_transition_patch) return -EBUSY; klp_init_transition(patch, KLP_TRANSITION_UNPATCHED); klp_for_each_object(patch, obj) if (obj->patched) klp_pre_unpatch_callback(obj); /* * Enforce the order of the func->transition writes in * klp_init_transition() and the TIF_PATCH_PENDING writes in * klp_start_transition(). In the rare case where klp_ftrace_handler() * is called shortly after klp_update_patch_state() switches the task, * this ensures the handler sees that func->transition is set. */ smp_wmb(); klp_start_transition(); patch->enabled = false; klp_try_complete_transition(); return 0;} static int __klp_enable_patch(struct klp_patch *patch){ struct klp_object *obj; int ret; if (klp_transition_patch) return -EBUSY; if (WARN_ON(patch->enabled)) return -EINVAL; pr_notice("enabling patch '%s'\n", patch->mod->name); klp_init_transition(patch, KLP_TRANSITION_PATCHED); /* * Enforce the order of the func->transition writes in * klp_init_transition() and the ops->func_stack writes in * klp_patch_object(), so that klp_ftrace_handler() will see the * func->transition updates before the handler is registered and the * new funcs become visible to the handler. */ smp_wmb(); klp_for_each_object(patch, obj) { if (!klp_is_object_loaded(obj)) continue; ret = klp_pre_patch_callback(obj); if (ret) { pr_warn("pre-patch callback failed for object '%s'\n", klp_is_module(obj) ? obj->name : "vmlinux"); goto err; } ret = klp_patch_object(obj); if (ret) { pr_warn("failed to patch object '%s'\n", klp_is_module(obj) ? obj->name : "vmlinux"); goto err; } } klp_start_transition(); patch->enabled = true; klp_try_complete_transition(); return 0;err: pr_warn("failed to enable patch '%s'\n", patch->mod->name); klp_cancel_transition(); return ret;} /** * klp_enable_patch() - enable the livepatch * @patch: patch to be enabled * * Initializes the data structure associated with the patch, creates the sysfs * interface, performs the needed symbol lookups and code relocations, * registers the patched functions with ftrace. * * This function is supposed to be called from the livepatch module_init() * callback. * * Return: 0 on success, otherwise error */int klp_enable_patch(struct klp_patch *patch){ int ret; struct klp_object *obj; if (!patch || !patch->mod || !patch->objs) return -EINVAL; klp_for_each_object_static(patch, obj) { if (!obj->funcs) return -EINVAL; } if (!is_livepatch_module(patch->mod)) { pr_err("module %s is not marked as a livepatch module\n", patch->mod->name); return -EINVAL; } if (!klp_initialized()) return -ENODEV; if (!klp_have_reliable_stack()) { pr_warn("This architecture doesn't have support for the livepatch consistency model.\n"); pr_warn("The livepatch transition may never complete.\n"); } mutex_lock(&klp_mutex); if (!klp_is_patch_compatible(patch)) { pr_err("Livepatch patch (%s) is not compatible with the already installed livepatches.\n", patch->mod->name); mutex_unlock(&klp_mutex); return -EINVAL; } if (!try_module_get(patch->mod)) { mutex_unlock(&klp_mutex); return -ENODEV; } klp_init_patch_early(patch); ret = klp_init_patch(patch); if (ret) goto err; ret = __klp_enable_patch(patch); if (ret) goto err; mutex_unlock(&klp_mutex); return 0; err: klp_free_patch_start(patch); mutex_unlock(&klp_mutex); klp_free_patch_finish(patch); return ret;}EXPORT_SYMBOL_GPL(klp_enable_patch); /* * This function unpatches objects from the replaced livepatches. * * We could be pretty aggressive here. It is called in the situation where * these structures are no longer accessed from the ftrace handler. * All functions are redirected by the klp_transition_patch. They * use either a new code or they are in the original code because * of the special nop function patches. * * The only exception is when the transition was forced. In this case, * klp_ftrace_handler() might still see the replaced patch on the stack. * Fortunately, it is carefully designed to work with removed functions * thanks to RCU. We only have to keep the patches on the system. Also * this is handled transparently by patch->module_put. */void klp_unpatch_replaced_patches(struct klp_patch *new_patch){ struct klp_patch *old_patch; klp_for_each_patch(old_patch) { if (old_patch == new_patch) return; old_patch->enabled = false; klp_unpatch_objects(old_patch); }} /* * This function removes the dynamically allocated 'nop' functions. * * We could be pretty aggressive. NOPs do not change the existing * behavior except for adding unnecessary delay by the ftrace handler. * * It is safe even when the transition was forced. The ftrace handler * will see a valid ops->func_stack entry thanks to RCU. * * We could even free the NOPs structures. They must be the last entry * in ops->func_stack. Therefore unregister_ftrace_function() is called. * It does the same as klp_synchronize_transition() to make sure that * nobody is inside the ftrace handler once the operation finishes. * * IMPORTANT: It must be called right after removing the replaced patches! */void klp_discard_nops(struct klp_patch *new_patch){ klp_unpatch_objects_dynamic(klp_transition_patch); klp_free_objects_dynamic(klp_transition_patch);} /* * Remove parts of patches that touch a given kernel module. The list of * patches processed might be limited. When limit is NULL, all patches * will be handled. */static void klp_cleanup_module_patches_limited(struct module *mod, struct klp_patch *limit){ struct klp_patch *patch; struct klp_object *obj; klp_for_each_patch(patch) { if (patch == limit) break; klp_for_each_object(patch, obj) { if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) continue; if (patch != klp_transition_patch) klp_pre_unpatch_callback(obj); pr_notice("reverting patch '%s' on unloading module '%s'\n", patch->mod->name, obj->mod->name); klp_unpatch_object(obj); klp_post_unpatch_callback(obj); klp_clear_object_relocs(patch, obj); klp_free_object_loaded(obj); break; } }} int klp_module_coming(struct module *mod){ int ret; struct klp_patch *patch; struct klp_object *obj; if (WARN_ON(mod->state != MODULE_STATE_COMING)) return -EINVAL; if (!strcmp(mod->name, "vmlinux")) { pr_err("vmlinux.ko: invalid module name\n"); return -EINVAL; } mutex_lock(&klp_mutex); /* * Each module has to know that klp_module_coming() * has been called. We never know what module will * get patched by a new patch. */ mod->klp_alive = true; klp_for_each_patch(patch) { klp_for_each_object(patch, obj) { if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) continue; obj->mod = mod; ret = klp_init_object_loaded(patch, obj); if (ret) { pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n", patch->mod->name, obj->mod->name, ret); goto err; } pr_notice("applying patch '%s' to loading module '%s'\n", patch->mod->name, obj->mod->name); ret = klp_pre_patch_callback(obj); if (ret) { pr_warn("pre-patch callback failed for object '%s'\n", obj->name); goto err; } ret = klp_patch_object(obj); if (ret) { pr_warn("failed to apply patch '%s' to module '%s' (%d)\n", patch->mod->name, obj->mod->name, ret); klp_post_unpatch_callback(obj); goto err; } if (patch != klp_transition_patch) klp_post_patch_callback(obj); break; } } mutex_unlock(&klp_mutex); return 0; err: /* * If a patch is unsuccessfully applied, return * error to the module loader. */ pr_warn("patch '%s' failed for module '%s', refusing to load module '%s'\n", patch->mod->name, obj->mod->name, obj->mod->name); mod->klp_alive = false; obj->mod = NULL; klp_cleanup_module_patches_limited(mod, patch); mutex_unlock(&klp_mutex); return ret;} void klp_module_going(struct module *mod){ if (WARN_ON(mod->state != MODULE_STATE_GOING && mod->state != MODULE_STATE_COMING)) return; mutex_lock(&klp_mutex); /* * Each module has to know that klp_module_going() * has been called. We never know what module will * get patched by a new patch. */ mod->klp_alive = false; klp_cleanup_module_patches_limited(mod, NULL); mutex_unlock(&klp_mutex);} void *klp_find_section_by_name(const struct module *mod, const char *name, size_t *sec_size){ struct klp_modinfo *info = mod->klp_info; for (int i = 1; i < info->hdr.e_shnum; i++) { Elf_Shdr *shdr = &info->sechdrs[i]; if (!strcmp(info->secstrings + shdr->sh_name, name)) { *sec_size = shdr->sh_size; return (void *)shdr->sh_addr; } } *sec_size = 0; return NULL;}EXPORT_SYMBOL_GPL(klp_find_section_by_name); static int __init klp_init(void){ klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj); if (!klp_root_kobj) return -ENOMEM; return 0;} module_init(klp_init);Lines 1001–1387 of 1387. Previous lines
} list_add_tail(&patch->list, &klp_patches); return 0; } static int __klp_disable_patch(struct klp_patch *patch) { struct klp_object *obj; if (WARN_ON(!patch->enabled)) return -EINVAL; if (klp_transition_patch) return -EBUSY; klp_init_transition(patch, KLP_TRANSITION_UNPATCHED); klp_for_each_object(patch, obj) if (obj->patched) klp_pre_unpatch_callback(obj); /* * Enforce the order of the func->transition writes in * klp_init_transition() and the TIF_PATCH_PENDING writes in * klp_start_transition(). In the rare case where klp_ftrace_handler() * is called shortly after klp_update_patch_state() switches the task, * this ensures the handler sees that func->transition is set. */ smp_wmb(); klp_start_transition(); patch->enabled = false; klp_try_complete_transition(); return 0; } static int __klp_enable_patch(struct klp_patch *patch) { struct klp_object *obj; int ret; if (klp_transition_patch) return -EBUSY; if (WARN_ON(patch->enabled)) return -EINVAL; pr_notice("enabling patch '%s'\n", patch->mod->name); klp_init_transition(patch, KLP_TRANSITION_PATCHED); /* * Enforce the order of the func->transition writes in * klp_init_transition() and the ops->func_stack writes in * klp_patch_object(), so that klp_ftrace_handler() will see the * func->transition updates before the handler is registered and the * new funcs become visible to the handler. */ smp_wmb(); klp_for_each_object(patch, obj) { if (!klp_is_object_loaded(obj)) continue; ret = klp_pre_patch_callback(obj); if (ret) { pr_warn("pre-patch callback failed for object '%s'\n", klp_is_module(obj) ? obj->name : "vmlinux"); goto err; } ret = klp_patch_object(obj); if (ret) { pr_warn("failed to patch object '%s'\n", klp_is_module(obj) ? obj->name : "vmlinux"); goto err; } } klp_start_transition(); patch->enabled = true; klp_try_complete_transition(); return 0; err: pr_warn("failed to enable patch '%s'\n", patch->mod->name); klp_cancel_transition(); return ret; } /** * klp_enable_patch() - enable the livepatch * @patch: patch to be enabled * * Initializes the data structure associated with the patch, creates the sysfs * interface, performs the needed symbol lookups and code relocations, * registers the patched functions with ftrace. * * This function is supposed to be called from the livepatch module_init() * callback. * * Return: 0 on success, otherwise error */ int klp_enable_patch(struct klp_patch *patch) { int ret; struct klp_object *obj; if (!patch || !patch->mod || !patch->objs) return -EINVAL; klp_for_each_object_static(patch, obj) { if (!obj->funcs) return -EINVAL; } if (!is_livepatch_module(patch->mod)) { pr_err("module %s is not marked as a livepatch module\n", patch->mod->name); return -EINVAL; } if (!klp_initialized()) return -ENODEV; if (!klp_have_reliable_stack()) { pr_warn("This architecture doesn't have support for the livepatch consistency model.\n"); pr_warn("The livepatch transition may never complete.\n"); } mutex_lock(&klp_mutex); if (!klp_is_patch_compatible(patch)) { pr_err("Livepatch patch (%s) is not compatible with the already installed livepatches.\n", patch->mod->name); mutex_unlock(&klp_mutex); return -EINVAL; } if (!try_module_get(patch->mod)) { mutex_unlock(&klp_mutex); return -ENODEV; } klp_init_patch_early(patch); ret = klp_init_patch(patch); if (ret) goto err; ret = __klp_enable_patch(patch); if (ret) goto err; mutex_unlock(&klp_mutex); return 0; err: klp_free_patch_start(patch); mutex_unlock(&klp_mutex); klp_free_patch_finish(patch); return ret; } EXPORT_SYMBOL_GPL(klp_enable_patch); /* * This function unpatches objects from the replaced livepatches. * * We could be pretty aggressive here. It is called in the situation where * these structures are no longer accessed from the ftrace handler. * All functions are redirected by the klp_transition_patch. They * use either a new code or they are in the original code because * of the special nop function patches. * * The only exception is when the transition was forced. In this case, * klp_ftrace_handler() might still see the replaced patch on the stack. * Fortunately, it is carefully designed to work with removed functions * thanks to RCU. We only have to keep the patches on the system. Also * this is handled transparently by patch->module_put. */ void klp_unpatch_replaced_patches(struct klp_patch *new_patch) { struct klp_patch *old_patch; klp_for_each_patch(old_patch) { if (old_patch == new_patch) return; old_patch->enabled = false; klp_unpatch_objects(old_patch); } } /* * This function removes the dynamically allocated 'nop' functions. * * We could be pretty aggressive. NOPs do not change the existing * behavior except for adding unnecessary delay by the ftrace handler. * * It is safe even when the transition was forced. The ftrace handler * will see a valid ops->func_stack entry thanks to RCU. * * We could even free the NOPs structures. They must be the last entry * in ops->func_stack. Therefore unregister_ftrace_function() is called. * It does the same as klp_synchronize_transition() to make sure that * nobody is inside the ftrace handler once the operation finishes. * * IMPORTANT: It must be called right after removing the replaced patches! */ void klp_discard_nops(struct klp_patch *new_patch) { klp_unpatch_objects_dynamic(klp_transition_patch); klp_free_objects_dynamic(klp_transition_patch); } /* * Remove parts of patches that touch a given kernel module. The list of * patches processed might be limited. When limit is NULL, all patches * will be handled. */ static void klp_cleanup_module_patches_limited(struct module *mod, struct klp_patch *limit) { struct klp_patch *patch; struct klp_object *obj; klp_for_each_patch(patch) { if (patch == limit) break; klp_for_each_object(patch, obj) { if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) continue; if (patch != klp_transition_patch) klp_pre_unpatch_callback(obj); pr_notice("reverting patch '%s' on unloading module '%s'\n", patch->mod->name, obj->mod->name); klp_unpatch_object(obj); klp_post_unpatch_callback(obj); klp_clear_object_relocs(patch, obj); klp_free_object_loaded(obj); break; } } } int klp_module_coming(struct module *mod) { int ret; struct klp_patch *patch; struct klp_object *obj; if (WARN_ON(mod->state != MODULE_STATE_COMING)) return -EINVAL; if (!strcmp(mod->name, "vmlinux")) { pr_err("vmlinux.ko: invalid module name\n"); return -EINVAL; } mutex_lock(&klp_mutex); /* * Each module has to know that klp_module_coming() * has been called. We never know what module will * get patched by a new patch. */ mod->klp_alive = true; klp_for_each_patch(patch) { klp_for_each_object(patch, obj) { if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) continue; obj->mod = mod; ret = klp_init_object_loaded(patch, obj); if (ret) { pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n", patch->mod->name, obj->mod->name, ret); goto err; } pr_notice("applying patch '%s' to loading module '%s'\n", patch->mod->name, obj->mod->name); ret = klp_pre_patch_callback(obj); if (ret) { pr_warn("pre-patch callback failed for object '%s'\n", obj->name); goto err; } ret = klp_patch_object(obj); if (ret) { pr_warn("failed to apply patch '%s' to module '%s' (%d)\n", patch->mod->name, obj->mod->name, ret); klp_post_unpatch_callback(obj); goto err; } if (patch != klp_transition_patch) klp_post_patch_callback(obj); break; } } mutex_unlock(&klp_mutex); return 0; err: /* * If a patch is unsuccessfully applied, return * error to the module loader. */ pr_warn("patch '%s' failed for module '%s', refusing to load module '%s'\n", patch->mod->name, obj->mod->name, obj->mod->name); mod->klp_alive = false; obj->mod = NULL; klp_cleanup_module_patches_limited(mod, patch); mutex_unlock(&klp_mutex); return ret; } void klp_module_going(struct module *mod) { if (WARN_ON(mod->state != MODULE_STATE_GOING && mod->state != MODULE_STATE_COMING)) return; mutex_lock(&klp_mutex); /* * Each module has to know that klp_module_going() * has been called. We never know what module will * get patched by a new patch. */ mod->klp_alive = false; klp_cleanup_module_patches_limited(mod, NULL); mutex_unlock(&klp_mutex); } void *klp_find_section_by_name(const struct module *mod, const char *name, size_t *sec_size) { struct klp_modinfo *info = mod->klp_info; for (int i = 1; i < info->hdr.e_shnum; i++) { Elf_Shdr *shdr = &info->sechdrs[i]; if (!strcmp(info->secstrings + shdr->sh_name, name)) { *sec_size = shdr->sh_size; return (void *)shdr->sh_addr; } } *sec_size = 0; return NULL; } EXPORT_SYMBOL_GPL(klp_find_section_by_name); static int __init klp_init(void) { klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj); if (!klp_root_kobj) return -ENOMEM; return 0; } module_init(klp_init);