The kretprobe hash is mostly superfluous, replace it with a per-task variable. This gets rid of the task hash and it's related locking. Note that this may change the kprobes module-exported API for kretprobe handlers. If any out-of-tree kretprobe user uses ri->rp, use get_kretprobe(ri) instead.
Showing the changes to kernel (3 of 5 files). Show all
Unified · Split
| @@ -2161,6 +2161,10 @@ static __latent_entropy struct task_struct *copy_process( · 2160 unmodified lines | |||
| 2161 | INIT_LIST_HEAD(&p->thread_group); | 2161 | INIT_LIST_HEAD(&p->thread_group); |
| 2162 | p->task_works = NULL; | 2162 | p->task_works = NULL; |
| 2163 | 2163 | ||
| 2164 | #ifdef CONFIG_KRETPROBES | ||
| 2165 | p->kretprobe_instances.first = NULL; | ||
| 2166 | #endif | ||
| 2167 | |||
| 2164 | /* | 2168 | /* |
| 2165 | * Ensure that the cgroup subsystem policies allow the new process to be | 2169 | * Ensure that the cgroup subsystem policies allow the new process to be |
| 2166 | * forked. It should be noted the the new process's css_set can be changed | 2170 | * forked. It should be noted the the new process's css_set can be changed |
| @@ -53,7 +53,6 @@ static int kprobes_initialized; · 52 unmodified lines | |||
| 53 | * - RCU hlist traversal under disabling preempt (breakpoint handlers) | 53 | * - RCU hlist traversal under disabling preempt (breakpoint handlers) |
| 54 | */ | 54 | */ |
| 55 | static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; | 55 | static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; |
| 56 | static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; | ||
| 57 | 56 | ||
| 58 | /* NOTE: change this value only with kprobe_mutex held */ | 57 | /* NOTE: change this value only with kprobe_mutex held */ |
| 59 | static bool kprobes_all_disarmed; | 58 | static bool kprobes_all_disarmed; |
| @@ -61,9 +60,6 @@ static bool kprobes_all_disarmed; · 1 unmodified line | |||
| 61 | /* This protects kprobe_table and optimizing_list */ | 60 | /* This protects kprobe_table and optimizing_list */ |
| 62 | static DEFINE_MUTEX(kprobe_mutex); | 61 | static DEFINE_MUTEX(kprobe_mutex); |
| 63 | static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; | 62 | static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; |
| 64 | static struct { | ||
| 65 | raw_spinlock_t lock ____cacheline_aligned_in_smp; | ||
| 66 | } kretprobe_table_locks[KPROBE_TABLE_SIZE]; | ||
| 67 | 63 | ||
| 68 | kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, | 64 | kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, |
| 69 | unsigned int __unused) | 65 | unsigned int __unused) |
| @@ -71,11 +67,6 @@ kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, · 1 unmodified line | |||
| 71 | return ((kprobe_opcode_t *)(kallsyms_lookup_name(name))); | 67 | return ((kprobe_opcode_t *)(kallsyms_lookup_name(name))); |
| 72 | } | 68 | } |
| 73 | 69 | ||
| 74 | static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) | ||
| 75 | { | ||
| 76 | return &(kretprobe_table_locks[hash].lock); | ||
| 77 | } | ||
| 78 | |||
| 79 | /* Blacklist -- list of struct kprobe_blacklist_entry */ | 70 | /* Blacklist -- list of struct kprobe_blacklist_entry */ |
| 80 | static LIST_HEAD(kprobe_blacklist); | 71 | static LIST_HEAD(kprobe_blacklist); |
| 81 | 72 | ||
| @@ -1223,65 +1214,30 @@ void kprobes_inc_nmissed_count(struct kprobe *p) · 1141 unmodified lines | |||
| 1223 | } | 1214 | } |
| 1224 | NOKPROBE_SYMBOL(kprobes_inc_nmissed_count); | 1215 | NOKPROBE_SYMBOL(kprobes_inc_nmissed_count); |
| 1225 | 1216 | ||
| 1217 | static void free_rp_inst_rcu(struct rcu_head *head) | ||
| 1218 | { | ||
| 1219 | struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu); | ||
| 1220 | |||
| 1221 | if (refcount_dec_and_test(&ri->rph->ref)) | ||
| 1222 | kfree(ri->rph); | ||
| 1223 | kfree(ri); | ||
| 1224 | } | ||
| 1225 | NOKPROBE_SYMBOL(free_rp_inst_rcu); | ||
| 1226 | |||
| 1226 | static void recycle_rp_inst(struct kretprobe_instance *ri) | 1227 | static void recycle_rp_inst(struct kretprobe_instance *ri) |
| 1227 | { | 1228 | { |
| 1228 | struct kretprobe *rp = ri->rp; | 1229 | struct kretprobe *rp = get_kretprobe(ri); |
| 1229 | 1230 | ||
| 1230 | /* remove rp inst off the rprobe_inst_table */ | ||
| 1231 | hlist_del(&ri->hlist); | ||
| 1232 | INIT_HLIST_NODE(&ri->hlist); | 1231 | INIT_HLIST_NODE(&ri->hlist); |
| 1233 | if (likely(rp)) { | 1232 | if (likely(rp)) { |
| 1234 | raw_spin_lock(&rp->lock); | 1233 | raw_spin_lock(&rp->lock); |
| 1235 | hlist_add_head(&ri->hlist, &rp->free_instances); | 1234 | hlist_add_head(&ri->hlist, &rp->free_instances); |
| 1236 | raw_spin_unlock(&rp->lock); | 1235 | raw_spin_unlock(&rp->lock); |
| 1237 | } else | 1236 | } else |
| 1238 | kfree_rcu(ri, rcu); | 1237 | call_rcu(&ri->rcu, free_rp_inst_rcu); |
| 1239 | } | 1238 | } |
| 1240 | NOKPROBE_SYMBOL(recycle_rp_inst); | 1239 | NOKPROBE_SYMBOL(recycle_rp_inst); |
| 1241 | 1240 | ||
| 1242 | static void kretprobe_hash_lock(struct task_struct *tsk, | ||
| 1243 | struct hlist_head **head, unsigned long *flags) | ||
| 1244 | __acquires(hlist_lock) | ||
| 1245 | { | ||
| 1246 | unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); | ||
| 1247 | raw_spinlock_t *hlist_lock; | ||
| 1248 | |||
| 1249 | *head = &kretprobe_inst_table[hash]; | ||
| 1250 | hlist_lock = kretprobe_table_lock_ptr(hash); | ||
| 1251 | raw_spin_lock_irqsave(hlist_lock, *flags); | ||
| 1252 | } | ||
| 1253 | NOKPROBE_SYMBOL(kretprobe_hash_lock); | ||
| 1254 | |||
| 1255 | static void kretprobe_table_lock(unsigned long hash, | ||
| 1256 | unsigned long *flags) | ||
| 1257 | __acquires(hlist_lock) | ||
| 1258 | { | ||
| 1259 | raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); | ||
| 1260 | raw_spin_lock_irqsave(hlist_lock, *flags); | ||
| 1261 | } | ||
| 1262 | NOKPROBE_SYMBOL(kretprobe_table_lock); | ||
| 1263 | |||
| 1264 | static void kretprobe_hash_unlock(struct task_struct *tsk, | ||
| 1265 | unsigned long *flags) | ||
| 1266 | __releases(hlist_lock) | ||
| 1267 | { | ||
| 1268 | unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); | ||
| 1269 | raw_spinlock_t *hlist_lock; | ||
| 1270 | |||
| 1271 | hlist_lock = kretprobe_table_lock_ptr(hash); | ||
| 1272 | raw_spin_unlock_irqrestore(hlist_lock, *flags); | ||
| 1273 | } | ||
| 1274 | NOKPROBE_SYMBOL(kretprobe_hash_unlock); | ||
| 1275 | |||
| 1276 | static void kretprobe_table_unlock(unsigned long hash, | ||
| 1277 | unsigned long *flags) | ||
| 1278 | __releases(hlist_lock) | ||
| 1279 | { | ||
| 1280 | raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); | ||
| 1281 | raw_spin_unlock_irqrestore(hlist_lock, *flags); | ||
| 1282 | } | ||
| 1283 | NOKPROBE_SYMBOL(kretprobe_table_unlock); | ||
| 1284 | |||
| 1285 | static struct kprobe kprobe_busy = { | 1241 | static struct kprobe kprobe_busy = { |
| 1286 | .addr = (void *) get_kprobe, | 1242 | .addr = (void *) get_kprobe, |
| 1287 | }; | 1243 | }; |
| @@ -1311,24 +1267,21 @@ void kprobe_busy_end(void) · 23 unmodified lines | |||
| 1311 | void kprobe_flush_task(struct task_struct *tk) | 1267 | void kprobe_flush_task(struct task_struct *tk) |
| 1312 | { | 1268 | { |
| 1313 | struct kretprobe_instance *ri; | 1269 | struct kretprobe_instance *ri; |
| 1314 | struct hlist_head *head; | 1270 | struct llist_node *node; |
| 1315 | struct hlist_node *tmp; | ||
| 1316 | unsigned long hash, flags = 0; | ||
| 1317 | 1271 | ||
| 1272 | /* Early boot, not yet initialized. */ | ||
| 1318 | if (unlikely(!kprobes_initialized)) | 1273 | if (unlikely(!kprobes_initialized)) |
| 1319 | /* Early boot. kretprobe_table_locks not yet initialized. */ | ||
| 1320 | return; | 1274 | return; |
| 1321 | 1275 | ||
| 1322 | kprobe_busy_begin(); | 1276 | kprobe_busy_begin(); |
| 1323 | 1277 | ||
| 1324 | hash = hash_ptr(tk, KPROBE_HASH_BITS); | 1278 | node = __llist_del_all(&tk->kretprobe_instances); |
| 1325 | head = &kretprobe_inst_table[hash]; | 1279 | while (node) { |
| 1326 | kretprobe_table_lock(hash, &flags); | 1280 | ri = container_of(node, struct kretprobe_instance, llist); |
| 1327 | hlist_for_each_entry_safe(ri, tmp, head, hlist) { | 1281 | node = node->next; |
| 1328 | if (ri->task == tk) | 1282 | |
| 1329 | recycle_rp_inst(ri); | 1283 | recycle_rp_inst(ri); |
| 1330 | } | 1284 | } |
| 1331 | kretprobe_table_unlock(hash, &flags); | ||
| 1332 | 1285 | ||
| 1333 | kprobe_busy_end(); | 1286 | kprobe_busy_end(); |
| 1334 | } | 1287 | } |
| @@ -1338,36 +1291,19 @@ static inline void free_rp_inst(struct kretprobe *rp) · 3 unmodified lines | |||
| 1338 | { | 1291 | { |
| 1339 | struct kretprobe_instance *ri; | 1292 | struct kretprobe_instance *ri; |
| 1340 | struct hlist_node *next; | 1293 | struct hlist_node *next; |
| 1294 | int count = 0; | ||
| 1341 | 1295 | ||
| 1342 | hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) { | 1296 | hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) { |
| 1343 | hlist_del(&ri->hlist); | 1297 | hlist_del(&ri->hlist); |
| 1344 | kfree(ri); | 1298 | kfree(ri); |
| 1299 | count++; | ||
| 1345 | } | 1300 | } |
| 1346 | } | ||
| 1347 | |||
| 1348 | static void cleanup_rp_inst(struct kretprobe *rp) | ||
| 1349 | { | ||
| 1350 | unsigned long flags, hash; | ||
| 1351 | struct kretprobe_instance *ri; | ||
| 1352 | struct hlist_node *next; | ||
| 1353 | struct hlist_head *head; | ||
| 1354 | 1301 | ||
| 1355 | /* To avoid recursive kretprobe by NMI, set kprobe busy here */ | 1302 | if (refcount_sub_and_test(count, &rp->rph->ref)) { |
| 1356 | kprobe_busy_begin(); | 1303 | kfree(rp->rph); |
| 1357 | for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) { | 1304 | rp->rph = NULL; |
| 1358 | kretprobe_table_lock(hash, &flags); | ||
| 1359 | head = &kretprobe_inst_table[hash]; | ||
| 1360 | hlist_for_each_entry_safe(ri, next, head, hlist) { | ||
| 1361 | if (ri->rp == rp) | ||
| 1362 | ri->rp = NULL; | ||
| 1363 | } | ||
| 1364 | kretprobe_table_unlock(hash, &flags); | ||
| 1365 | } | 1305 | } |
| 1366 | kprobe_busy_end(); | ||
| 1367 | |||
| 1368 | free_rp_inst(rp); | ||
| 1369 | } | 1306 | } |
| 1370 | NOKPROBE_SYMBOL(cleanup_rp_inst); | ||
| 1371 | 1307 | ||
| 1372 | /* Add the new probe to ap->list */ | 1308 | /* Add the new probe to ap->list */ |
| 1373 | static int add_new_kprobe(struct kprobe *ap, struct kprobe *p) | 1309 | static int add_new_kprobe(struct kprobe *ap, struct kprobe *p) |
| @@ -1928,88 +1864,56 @@ unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs, · 554 unmodified lines | |||
| 1928 | void *trampoline_address, | 1864 | void *trampoline_address, |
| 1929 | void *frame_pointer) | 1865 | void *frame_pointer) |
| 1930 | { | 1866 | { |
| 1931 | struct kretprobe_instance *ri = NULL, *last = NULL; | ||
| 1932 | struct hlist_head *head; | ||
| 1933 | struct hlist_node *tmp; | ||
| 1934 | unsigned long flags; | ||
| 1935 | kprobe_opcode_t *correct_ret_addr = NULL; | 1867 | kprobe_opcode_t *correct_ret_addr = NULL; |
| 1936 | bool skipped = false; | 1868 | struct kretprobe_instance *ri = NULL; |
| 1869 | struct llist_node *first, *node; | ||
| 1870 | struct kretprobe *rp; | ||
| 1937 | 1871 | ||
| 1938 | kretprobe_hash_lock(current, &head, &flags); | 1872 | /* Find all nodes for this frame. */ |
| 1873 | first = node = current->kretprobe_instances.first; | ||
| 1874 | while (node) { | ||
| 1875 | ri = container_of(node, struct kretprobe_instance, llist); | ||
| 1939 | 1876 | ||
| 1940 | /* | 1877 | BUG_ON(ri->fp != frame_pointer); |
| 1941 | * It is possible to have multiple instances associated with a given | ||
| 1942 | * task either because multiple functions in the call path have | ||
| 1943 | * return probes installed on them, and/or more than one | ||
| 1944 | * return probe was registered for a target function. | ||
| 1945 | * | ||
| 1946 | * We can handle this because: | ||
| 1947 | * - instances are always pushed into the head of the list | ||
| 1948 | * - when multiple return probes are registered for the same | ||
| 1949 | * function, the (chronologically) first instance's ret_addr | ||
| 1950 | * will be the real return address, and all the rest will | ||
| 1951 | * point to kretprobe_trampoline. | ||
| 1952 | */ | ||
| 1953 | hlist_for_each_entry(ri, head, hlist) { | ||
| 1954 | if (ri->task != current) | ||
| 1955 | /* another task is sharing our hash bucket */ | ||
| 1956 | continue; | ||
| 1957 | /* | ||
| 1958 | * Return probes must be pushed on this hash list correct | ||
| 1959 | * order (same as return order) so that it can be popped | ||
| 1960 | * correctly. However, if we find it is pushed it incorrect | ||
| 1961 | * order, this means we find a function which should not be | ||
| 1962 | * probed, because the wrong order entry is pushed on the | ||
| 1963 | * path of processing other kretprobe itself. | ||
| 1964 | */ | ||
| 1965 | if (ri->fp != frame_pointer) { | ||
| 1966 | if (!skipped) | ||
| 1967 | pr_warn("kretprobe is stacked incorrectly. Trying to fixup.\n"); | ||
| 1968 | skipped = true; | ||
| 1969 | continue; | ||
| 1970 | } | ||
| 1971 | 1878 | ||
| 1972 | correct_ret_addr = ri->ret_addr; | 1879 | if (ri->ret_addr != trampoline_address) { |
| 1973 | if (skipped) | 1880 | correct_ret_addr = ri->ret_addr; |
| 1974 | pr_warn("%ps must be blacklisted because of incorrect kretprobe order\n", | ||
| 1975 | ri->rp->kp.addr); | ||
| 1976 | |||
| 1977 | if (correct_ret_addr != trampoline_address) | ||
| 1978 | /* | 1881 | /* |
| 1979 | * This is the real return address. Any other | 1882 | * This is the real return address. Any other |
| 1980 | * instances associated with this task are for | 1883 | * instances associated with this task are for |
| 1981 | * other calls deeper on the call stack | 1884 | * other calls deeper on the call stack |
| 1982 | */ | 1885 | */ |
| 1983 | break; | 1886 | goto found; |
| 1887 | } | ||
| 1888 | |||
| 1889 | node = node->next; | ||
| 1984 | } | 1890 | } |
| 1891 | pr_err("Oops! Kretprobe fails to find correct return address.\n"); | ||
| 1892 | BUG_ON(1); | ||
| 1985 | 1893 | ||
| 1986 | BUG_ON(!correct_ret_addr || (correct_ret_addr == trampoline_address)); | 1894 | found: |
| 1987 | last = ri; | 1895 | /* Unlink all nodes for this frame. */ |
| 1896 | current->kretprobe_instances.first = node->next; | ||
| 1897 | node->next = NULL; | ||
| 1988 | 1898 | ||
| 1989 | hlist_for_each_entry_safe(ri, tmp, head, hlist) { | 1899 | /* Run them.. */ |
| 1990 | if (ri->task != current) | 1900 | while (first) { |
| 1991 | /* another task is sharing our hash bucket */ | 1901 | ri = container_of(first, struct kretprobe_instance, llist); |
| 1992 | continue; | 1902 | first = first->next; |
| 1993 | if (ri->fp != frame_pointer) | ||
| 1994 | continue; | ||
| 1995 | 1903 | ||
| 1996 | if (ri->rp && ri->rp->handler) { | 1904 | rp = get_kretprobe(ri); |
| 1905 | if (rp && rp->handler) { | ||
| 1997 | struct kprobe *prev = kprobe_running(); | 1906 | struct kprobe *prev = kprobe_running(); |
| 1998 | 1907 | ||
| 1999 | __this_cpu_write(current_kprobe, &ri->rp->kp); | 1908 | __this_cpu_write(current_kprobe, &rp->kp); |
| 2000 | ri->ret_addr = correct_ret_addr; | 1909 | ri->ret_addr = correct_ret_addr; |
| 2001 | ri->rp->handler(ri, regs); | 1910 | rp->handler(ri, regs); |
| 2002 | __this_cpu_write(current_kprobe, prev); | 1911 | __this_cpu_write(current_kprobe, prev); |
| 2003 | } | 1912 | } |
| 2004 | 1913 | ||
| 2005 | recycle_rp_inst(ri); | 1914 | recycle_rp_inst(ri); |
| 2006 | |||
| 2007 | if (ri == last) | ||
| 2008 | break; | ||
| 2009 | } | 1915 | } |
| 2010 | 1916 | ||
| 2011 | kretprobe_hash_unlock(current, &flags); | ||
| 2012 | |||
| 2013 | return (unsigned long)correct_ret_addr; | 1917 | return (unsigned long)correct_ret_addr; |
| 2014 | } | 1918 | } |
| 2015 | NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) | 1919 | NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) |
| @@ -2021,11 +1925,10 @@ NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) · 5 unmodified lines | |||
| 2021 | static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) | 1925 | static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) |
| 2022 | { | 1926 | { |
| 2023 | struct kretprobe *rp = container_of(p, struct kretprobe, kp); | 1927 | struct kretprobe *rp = container_of(p, struct kretprobe, kp); |
| 2024 | unsigned long hash, flags = 0; | 1928 | unsigned long flags = 0; |
| 2025 | struct kretprobe_instance *ri; | 1929 | struct kretprobe_instance *ri; |
| 2026 | 1930 | ||
| 2027 | /* TODO: consider to only swap the RA after the last pre_handler fired */ | 1931 | /* TODO: consider to only swap the RA after the last pre_handler fired */ |
| 2028 | hash = hash_ptr(current, KPROBE_HASH_BITS); | ||
| 2029 | raw_spin_lock_irqsave(&rp->lock, flags); | 1932 | raw_spin_lock_irqsave(&rp->lock, flags); |
| 2030 | if (!hlist_empty(&rp->free_instances)) { | 1933 | if (!hlist_empty(&rp->free_instances)) { |
| 2031 | ri = hlist_entry(rp->free_instances.first, | 1934 | ri = hlist_entry(rp->free_instances.first, |
| @@ -2033,9 +1936,6 @@ static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) · 1 unmodified line | |||
| 2033 | hlist_del(&ri->hlist); | 1936 | hlist_del(&ri->hlist); |
| 2034 | raw_spin_unlock_irqrestore(&rp->lock, flags); | 1937 | raw_spin_unlock_irqrestore(&rp->lock, flags); |
| 2035 | 1938 | ||
| 2036 | ri->rp = rp; | ||
| 2037 | ri->task = current; | ||
| 2038 | |||
| 2039 | if (rp->entry_handler && rp->entry_handler(ri, regs)) { | 1939 | if (rp->entry_handler && rp->entry_handler(ri, regs)) { |
| 2040 | raw_spin_lock_irqsave(&rp->lock, flags); | 1940 | raw_spin_lock_irqsave(&rp->lock, flags); |
| 2041 | hlist_add_head(&ri->hlist, &rp->free_instances); | 1941 | hlist_add_head(&ri->hlist, &rp->free_instances); |
| @@ -2045,11 +1945,8 @@ static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) · 3 unmodified lines | |||
| 2045 | 1945 | ||
| 2046 | arch_prepare_kretprobe(ri, regs); | 1946 | arch_prepare_kretprobe(ri, regs); |
| 2047 | 1947 | ||
| 2048 | /* XXX(hch): why is there no hlist_move_head? */ | 1948 | __llist_add(&ri->llist, ¤t->kretprobe_instances); |
| 2049 | INIT_HLIST_NODE(&ri->hlist); | 1949 | |
| 2050 | kretprobe_table_lock(hash, &flags); | ||
| 2051 | hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]); | ||
| 2052 | kretprobe_table_unlock(hash, &flags); | ||
| 2053 | } else { | 1950 | } else { |
| 2054 | rp->nmissed++; | 1951 | rp->nmissed++; |
| 2055 | raw_spin_unlock_irqrestore(&rp->lock, flags); | 1952 | raw_spin_unlock_irqrestore(&rp->lock, flags); |
| @@ -2112,16 +2009,24 @@ int register_kretprobe(struct kretprobe *rp) · 56 unmodified lines | |||
| 2112 | } | 2009 | } |
| 2113 | raw_spin_lock_init(&rp->lock); | 2010 | raw_spin_lock_init(&rp->lock); |
| 2114 | INIT_HLIST_HEAD(&rp->free_instances); | 2011 | INIT_HLIST_HEAD(&rp->free_instances); |
| 2012 | rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL); | ||
| 2013 | if (!rp->rph) | ||
| 2014 | return -ENOMEM; | ||
| 2015 | |||
| 2016 | rp->rph->rp = rp; | ||
| 2115 | for (i = 0; i < rp->maxactive; i++) { | 2017 | for (i = 0; i < rp->maxactive; i++) { |
| 2116 | inst = kmalloc(sizeof(struct kretprobe_instance) + | 2018 | inst = kzalloc(sizeof(struct kretprobe_instance) + |
| 2117 | rp->data_size, GFP_KERNEL); | 2019 | rp->data_size, GFP_KERNEL); |
| 2118 | if (inst == NULL) { | 2020 | if (inst == NULL) { |
| 2021 | refcount_set(&rp->rph->ref, i); | ||
| 2119 | free_rp_inst(rp); | 2022 | free_rp_inst(rp); |
| 2120 | return -ENOMEM; | 2023 | return -ENOMEM; |
| 2121 | } | 2024 | } |
| 2025 | inst->rph = rp->rph; | ||
| 2122 | INIT_HLIST_NODE(&inst->hlist); | 2026 | INIT_HLIST_NODE(&inst->hlist); |
| 2123 | hlist_add_head(&inst->hlist, &rp->free_instances); | 2027 | hlist_add_head(&inst->hlist, &rp->free_instances); |
| 2124 | } | 2028 | } |
| 2029 | refcount_set(&rp->rph->ref, i); | ||
| 2125 | 2030 | ||
| 2126 | rp->nmissed = 0; | 2031 | rp->nmissed = 0; |
| 2127 | /* Establish function entry probe point */ | 2032 | /* Establish function entry probe point */ |
| @@ -2163,16 +2068,18 @@ void unregister_kretprobes(struct kretprobe **rps, int num) · 35 unmodified lines | |||
| 2163 | if (num <= 0) | 2068 | if (num <= 0) |
| 2164 | return; | 2069 | return; |
| 2165 | mutex_lock(&kprobe_mutex); | 2070 | mutex_lock(&kprobe_mutex); |
| 2166 | for (i = 0; i < num; i++) | 2071 | for (i = 0; i < num; i++) { |
| 2167 | if (__unregister_kprobe_top(&rps[i]->kp) < 0) | 2072 | if (__unregister_kprobe_top(&rps[i]->kp) < 0) |
| 2168 | rps[i]->kp.addr = NULL; | 2073 | rps[i]->kp.addr = NULL; |
| 2074 | rps[i]->rph->rp = NULL; | ||
| 2075 | } | ||
| 2169 | mutex_unlock(&kprobe_mutex); | 2076 | mutex_unlock(&kprobe_mutex); |
| 2170 | 2077 | ||
| 2171 | synchronize_rcu(); | 2078 | synchronize_rcu(); |
| 2172 | for (i = 0; i < num; i++) { | 2079 | for (i = 0; i < num; i++) { |
| 2173 | if (rps[i]->kp.addr) { | 2080 | if (rps[i]->kp.addr) { |
| 2174 | __unregister_kprobe_bottom(&rps[i]->kp); | 2081 | __unregister_kprobe_bottom(&rps[i]->kp); |
| 2175 | cleanup_rp_inst(rps[i]); | 2082 | free_rp_inst(rps[i]); |
| 2176 | } | 2083 | } |
| 2177 | } | 2084 | } |
| 2178 | } | 2085 | } |
| @@ -2535,11 +2442,8 @@ static int __init init_kprobes(void) · 356 unmodified lines | |||
| 2535 | 2442 | ||
| 2536 | /* FIXME allocate the probe table, currently defined statically */ | 2443 | /* FIXME allocate the probe table, currently defined statically */ |
| 2537 | /* initialize all list heads */ | 2444 | /* initialize all list heads */ |
| 2538 | for (i = 0; i < KPROBE_TABLE_SIZE; i++) { | 2445 | for (i = 0; i < KPROBE_TABLE_SIZE; i++) |
| 2539 | INIT_HLIST_HEAD(&kprobe_table[i]); | 2446 | INIT_HLIST_HEAD(&kprobe_table[i]); |
| 2540 | INIT_HLIST_HEAD(&kretprobe_inst_table[i]); | ||
| 2541 | raw_spin_lock_init(&(kretprobe_table_locks[i].lock)); | ||
| 2542 | } | ||
| 2543 | 2447 | ||
| 2544 | err = populate_kprobe_blacklist(__start_kprobe_blacklist, | 2448 | err = populate_kprobe_blacklist(__start_kprobe_blacklist, |
| 2545 | __stop_kprobe_blacklist); | 2449 | __stop_kprobe_blacklist); |
| @@ -1714,7 +1714,8 @@ NOKPROBE_SYMBOL(kprobe_dispatcher); · 1713 unmodified lines | |||
| 1714 | static int | 1714 | static int |
| 1715 | kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) | 1715 | kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) |
| 1716 | { | 1716 | { |
| 1717 | struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp); | 1717 | struct kretprobe *rp = get_kretprobe(ri); |
| 1718 | struct trace_kprobe *tk = container_of(rp, struct trace_kprobe, rp); | ||
| 1718 | 1719 | ||
| 1719 | raw_cpu_inc(*tk->nhit); | 1720 | raw_cpu_inc(*tk->nhit); |
| 1720 | 1721 | ||