The kretprobe hash is mostly superfluous, replace it with a per-task variable. This gets rid of the task hash and it's related locking. Note that this may change the kprobes module-exported API for kretprobe handlers. If any out-of-tree kretprobe user uses ri->rp, use get_kretprobe(ri) instead.
Showing the changes to kernel (3 of 5 files). Show all
Unified · Split
| @@ -2161,6 +2161,10 @@ static __latent_entropy struct task_struct *copy_process( · 2160 unmodified lines | ||
| 2161 | 2161 | INIT_LIST_HEAD(&p->thread_group); |
| 2162 | 2162 | p->task_works = NULL; |
| 2163 | 2163 | |
| 2164 | +#ifdef CONFIG_KRETPROBES | |
| 2165 | + p->kretprobe_instances.first = NULL; | |
| 2166 | +#endif | |
| 2167 | + | |
| 2164 | 2168 | /* |
| 2165 | 2169 | * Ensure that the cgroup subsystem policies allow the new process to be |
| 2166 | 2170 | * forked. It should be noted the the new process's css_set can be changed |
| @@ -53,7 +53,6 @@ static int kprobes_initialized; · 52 unmodified lines | ||
| 53 | 53 | * - RCU hlist traversal under disabling preempt (breakpoint handlers) |
| 54 | 54 | */ |
| 55 | 55 | static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; |
| 56 | −static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; | |
| 57 | 56 | |
| 58 | 57 | /* NOTE: change this value only with kprobe_mutex held */ |
| 59 | 58 | static bool kprobes_all_disarmed; |
| @@ -61,9 +60,6 @@ static bool kprobes_all_disarmed; · 1 unmodified line | ||
| 61 | 60 | /* This protects kprobe_table and optimizing_list */ |
| 62 | 61 | static DEFINE_MUTEX(kprobe_mutex); |
| 63 | 62 | static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; |
| 64 | −static struct { | |
| 65 | − raw_spinlock_t lock ____cacheline_aligned_in_smp; | |
| 66 | −} kretprobe_table_locks[KPROBE_TABLE_SIZE]; | |
| 67 | 63 | |
| 68 | 64 | kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, |
| 69 | 65 | unsigned int __unused) |
| @@ -71,11 +67,6 @@ kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, · 1 unmodified line | ||
| 71 | 67 | return ((kprobe_opcode_t *)(kallsyms_lookup_name(name))); |
| 72 | 68 | } |
| 73 | 69 | |
| 74 | −static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) | |
| 75 | −{ | |
| 76 | − return &(kretprobe_table_locks[hash].lock); | |
| 77 | −} | |
| 78 | − | |
| 79 | 70 | /* Blacklist -- list of struct kprobe_blacklist_entry */ |
| 80 | 71 | static LIST_HEAD(kprobe_blacklist); |
| 81 | 72 | |
| @@ -1223,65 +1214,30 @@ void kprobes_inc_nmissed_count(struct kprobe *p) · 1141 unmodified lines | ||
| 1223 | 1214 | } |
| 1224 | 1215 | NOKPROBE_SYMBOL(kprobes_inc_nmissed_count); |
| 1225 | 1216 | |
| 1217 | +static void free_rp_inst_rcu(struct rcu_head *head) | |
| 1218 | +{ | |
| 1219 | + struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu); | |
| 1220 | + | |
| 1221 | + if (refcount_dec_and_test(&ri->rph->ref)) | |
| 1222 | + kfree(ri->rph); | |
| 1223 | + kfree(ri); | |
| 1224 | +} | |
| 1225 | +NOKPROBE_SYMBOL(free_rp_inst_rcu); | |
| 1226 | + | |
| 1226 | 1227 | static void recycle_rp_inst(struct kretprobe_instance *ri) |
| 1227 | 1228 | { |
| 1228 | − struct kretprobe *rp = ri->rp; | |
| 1229 | + struct kretprobe *rp = get_kretprobe(ri); | |
| 1229 | 1230 | |
| 1230 | − /* remove rp inst off the rprobe_inst_table */ | |
| 1231 | − hlist_del(&ri->hlist); | |
| 1232 | 1231 | INIT_HLIST_NODE(&ri->hlist); |
| 1233 | 1232 | if (likely(rp)) { |
| 1234 | 1233 | raw_spin_lock(&rp->lock); |
| 1235 | 1234 | hlist_add_head(&ri->hlist, &rp->free_instances); |
| 1236 | 1235 | raw_spin_unlock(&rp->lock); |
| 1237 | 1236 | } else |
| 1238 | − kfree_rcu(ri, rcu); | |
| 1237 | + call_rcu(&ri->rcu, free_rp_inst_rcu); | |
| 1239 | 1238 | } |
| 1240 | 1239 | NOKPROBE_SYMBOL(recycle_rp_inst); |
| 1241 | 1240 | |
| 1242 | −static void kretprobe_hash_lock(struct task_struct *tsk, | |
| 1243 | − struct hlist_head **head, unsigned long *flags) | |
| 1244 | −__acquires(hlist_lock) | |
| 1245 | −{ | |
| 1246 | − unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); | |
| 1247 | − raw_spinlock_t *hlist_lock; | |
| 1248 | − | |
| 1249 | − *head = &kretprobe_inst_table[hash]; | |
| 1250 | − hlist_lock = kretprobe_table_lock_ptr(hash); | |
| 1251 | − raw_spin_lock_irqsave(hlist_lock, *flags); | |
| 1252 | −} | |
| 1253 | −NOKPROBE_SYMBOL(kretprobe_hash_lock); | |
| 1254 | − | |
| 1255 | −static void kretprobe_table_lock(unsigned long hash, | |
| 1256 | − unsigned long *flags) | |
| 1257 | −__acquires(hlist_lock) | |
| 1258 | −{ | |
| 1259 | − raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); | |
| 1260 | − raw_spin_lock_irqsave(hlist_lock, *flags); | |
| 1261 | −} | |
| 1262 | −NOKPROBE_SYMBOL(kretprobe_table_lock); | |
| 1263 | − | |
| 1264 | −static void kretprobe_hash_unlock(struct task_struct *tsk, | |
| 1265 | − unsigned long *flags) | |
| 1266 | −__releases(hlist_lock) | |
| 1267 | −{ | |
| 1268 | − unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); | |
| 1269 | − raw_spinlock_t *hlist_lock; | |
| 1270 | − | |
| 1271 | − hlist_lock = kretprobe_table_lock_ptr(hash); | |
| 1272 | − raw_spin_unlock_irqrestore(hlist_lock, *flags); | |
| 1273 | −} | |
| 1274 | −NOKPROBE_SYMBOL(kretprobe_hash_unlock); | |
| 1275 | − | |
| 1276 | −static void kretprobe_table_unlock(unsigned long hash, | |
| 1277 | − unsigned long *flags) | |
| 1278 | −__releases(hlist_lock) | |
| 1279 | −{ | |
| 1280 | − raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); | |
| 1281 | − raw_spin_unlock_irqrestore(hlist_lock, *flags); | |
| 1282 | −} | |
| 1283 | −NOKPROBE_SYMBOL(kretprobe_table_unlock); | |
| 1284 | − | |
| 1285 | 1241 | static struct kprobe kprobe_busy = { |
| 1286 | 1242 | .addr = (void *) get_kprobe, |
| 1287 | 1243 | }; |
| @@ -1311,24 +1267,21 @@ void kprobe_busy_end(void) · 23 unmodified lines | ||
| 1311 | 1267 | void kprobe_flush_task(struct task_struct *tk) |
| 1312 | 1268 | { |
| 1313 | 1269 | struct kretprobe_instance *ri; |
| 1314 | − struct hlist_head *head; | |
| 1315 | − struct hlist_node *tmp; | |
| 1316 | − unsigned long hash, flags = 0; | |
| 1270 | + struct llist_node *node; | |
| 1317 | 1271 | |
| 1272 | + /* Early boot, not yet initialized. */ | |
| 1318 | 1273 | if (unlikely(!kprobes_initialized)) |
| 1319 | − /* Early boot. kretprobe_table_locks not yet initialized. */ | |
| 1320 | 1274 | return; |
| 1321 | 1275 | |
| 1322 | 1276 | kprobe_busy_begin(); |
| 1323 | 1277 | |
| 1324 | − hash = hash_ptr(tk, KPROBE_HASH_BITS); | |
| 1325 | − head = &kretprobe_inst_table[hash]; | |
| 1326 | − kretprobe_table_lock(hash, &flags); | |
| 1327 | − hlist_for_each_entry_safe(ri, tmp, head, hlist) { | |
| 1328 | − if (ri->task == tk) | |
| 1329 | − recycle_rp_inst(ri); | |
| 1278 | + node = __llist_del_all(&tk->kretprobe_instances); | |
| 1279 | + while (node) { | |
| 1280 | + ri = container_of(node, struct kretprobe_instance, llist); | |
| 1281 | + node = node->next; | |
| 1282 | + | |
| 1283 | + recycle_rp_inst(ri); | |
| 1330 | 1284 | } |
| 1331 | − kretprobe_table_unlock(hash, &flags); | |
| 1332 | 1285 | |
| 1333 | 1286 | kprobe_busy_end(); |
| 1334 | 1287 | } |
| @@ -1338,36 +1291,19 @@ static inline void free_rp_inst(struct kretprobe *rp) · 3 unmodified lines | ||
| 1338 | 1291 | { |
| 1339 | 1292 | struct kretprobe_instance *ri; |
| 1340 | 1293 | struct hlist_node *next; |
| 1294 | + int count = 0; | |
| 1341 | 1295 | |
| 1342 | 1296 | hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) { |
| 1343 | 1297 | hlist_del(&ri->hlist); |
| 1344 | 1298 | kfree(ri); |
| 1299 | + count++; | |
| 1345 | 1300 | } |
| 1346 | −} | |
| 1347 | − | |
| 1348 | −static void cleanup_rp_inst(struct kretprobe *rp) | |
| 1349 | −{ | |
| 1350 | − unsigned long flags, hash; | |
| 1351 | − struct kretprobe_instance *ri; | |
| 1352 | − struct hlist_node *next; | |
| 1353 | − struct hlist_head *head; | |
| 1354 | 1301 | |
| 1355 | − /* To avoid recursive kretprobe by NMI, set kprobe busy here */ | |
| 1356 | − kprobe_busy_begin(); | |
| 1357 | − for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) { | |
| 1358 | − kretprobe_table_lock(hash, &flags); | |
| 1359 | − head = &kretprobe_inst_table[hash]; | |
| 1360 | − hlist_for_each_entry_safe(ri, next, head, hlist) { | |
| 1361 | − if (ri->rp == rp) | |
| 1362 | − ri->rp = NULL; | |
| 1363 | − } | |
| 1364 | − kretprobe_table_unlock(hash, &flags); | |
| 1302 | + if (refcount_sub_and_test(count, &rp->rph->ref)) { | |
| 1303 | + kfree(rp->rph); | |
| 1304 | + rp->rph = NULL; | |
| 1365 | 1305 | } |
| 1366 | − kprobe_busy_end(); | |
| 1367 | − | |
| 1368 | − free_rp_inst(rp); | |
| 1369 | 1306 | } |
| 1370 | −NOKPROBE_SYMBOL(cleanup_rp_inst); | |
| 1371 | 1307 | |
| 1372 | 1308 | /* Add the new probe to ap->list */ |
| 1373 | 1309 | static int add_new_kprobe(struct kprobe *ap, struct kprobe *p) |
| @@ -1928,88 +1864,56 @@ unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs, · 554 unmodified lines | ||
| 1928 | 1864 | void *trampoline_address, |
| 1929 | 1865 | void *frame_pointer) |
| 1930 | 1866 | { |
| 1931 | − struct kretprobe_instance *ri = NULL, *last = NULL; | |
| 1932 | − struct hlist_head *head; | |
| 1933 | − struct hlist_node *tmp; | |
| 1934 | − unsigned long flags; | |
| 1935 | 1867 | kprobe_opcode_t *correct_ret_addr = NULL; |
| 1936 | − bool skipped = false; | |
| 1868 | + struct kretprobe_instance *ri = NULL; | |
| 1869 | + struct llist_node *first, *node; | |
| 1870 | + struct kretprobe *rp; | |
| 1937 | 1871 | |
| 1938 | − kretprobe_hash_lock(current, &head, &flags); | |
| 1872 | + /* Find all nodes for this frame. */ | |
| 1873 | + first = node = current->kretprobe_instances.first; | |
| 1874 | + while (node) { | |
| 1875 | + ri = container_of(node, struct kretprobe_instance, llist); | |
| 1939 | 1876 | |
| 1940 | − /* | |
| 1941 | − * It is possible to have multiple instances associated with a given | |
| 1942 | − * task either because multiple functions in the call path have | |
| 1943 | − * return probes installed on them, and/or more than one | |
| 1944 | − * return probe was registered for a target function. | |
| 1945 | − * | |
| 1946 | − * We can handle this because: | |
| 1947 | − * - instances are always pushed into the head of the list | |
| 1948 | − * - when multiple return probes are registered for the same | |
| 1949 | − * function, the (chronologically) first instance's ret_addr | |
| 1950 | − * will be the real return address, and all the rest will | |
| 1951 | − * point to kretprobe_trampoline. | |
| 1952 | − */ | |
| 1953 | − hlist_for_each_entry(ri, head, hlist) { | |
| 1954 | − if (ri->task != current) | |
| 1955 | − /* another task is sharing our hash bucket */ | |
| 1956 | − continue; | |
| 1957 | − /* | |
| 1958 | − * Return probes must be pushed on this hash list correct | |
| 1959 | − * order (same as return order) so that it can be popped | |
| 1960 | − * correctly. However, if we find it is pushed it incorrect | |
| 1961 | − * order, this means we find a function which should not be | |
| 1962 | − * probed, because the wrong order entry is pushed on the | |
| 1963 | − * path of processing other kretprobe itself. | |
| 1964 | − */ | |
| 1965 | − if (ri->fp != frame_pointer) { | |
| 1966 | − if (!skipped) | |
| 1967 | − pr_warn("kretprobe is stacked incorrectly. Trying to fixup.\n"); | |
| 1968 | − skipped = true; | |
| 1969 | − continue; | |
| 1970 | − } | |
| 1877 | + BUG_ON(ri->fp != frame_pointer); | |
| 1971 | 1878 | |
| 1972 | − correct_ret_addr = ri->ret_addr; | |
| 1973 | − if (skipped) | |
| 1974 | − pr_warn("%ps must be blacklisted because of incorrect kretprobe order\n", | |
| 1975 | − ri->rp->kp.addr); | |
| 1976 | − | |
| 1977 | − if (correct_ret_addr != trampoline_address) | |
| 1879 | + if (ri->ret_addr != trampoline_address) { | |
| 1880 | + correct_ret_addr = ri->ret_addr; | |
| 1978 | 1881 | /* |
| 1979 | 1882 | * This is the real return address. Any other |
| 1980 | 1883 | * instances associated with this task are for |
| 1981 | 1884 | * other calls deeper on the call stack |
| 1982 | 1885 | */ |
| 1983 | − break; | |
| 1886 | + goto found; | |
| 1887 | + } | |
| 1888 | + | |
| 1889 | + node = node->next; | |
| 1984 | 1890 | } |
| 1891 | + pr_err("Oops! Kretprobe fails to find correct return address.\n"); | |
| 1892 | + BUG_ON(1); | |
| 1985 | 1893 | |
| 1986 | − BUG_ON(!correct_ret_addr || (correct_ret_addr == trampoline_address)); | |
| 1987 | − last = ri; | |
| 1894 | +found: | |
| 1895 | + /* Unlink all nodes for this frame. */ | |
| 1896 | + current->kretprobe_instances.first = node->next; | |
| 1897 | + node->next = NULL; | |
| 1988 | 1898 | |
| 1989 | − hlist_for_each_entry_safe(ri, tmp, head, hlist) { | |
| 1990 | − if (ri->task != current) | |
| 1991 | − /* another task is sharing our hash bucket */ | |
| 1992 | − continue; | |
| 1993 | − if (ri->fp != frame_pointer) | |
| 1994 | − continue; | |
| 1899 | + /* Run them.. */ | |
| 1900 | + while (first) { | |
| 1901 | + ri = container_of(first, struct kretprobe_instance, llist); | |
| 1902 | + first = first->next; | |
| 1995 | 1903 | |
| 1996 | − if (ri->rp && ri->rp->handler) { | |
| 1904 | + rp = get_kretprobe(ri); | |
| 1905 | + if (rp && rp->handler) { | |
| 1997 | 1906 | struct kprobe *prev = kprobe_running(); |
| 1998 | 1907 | |
| 1999 | − __this_cpu_write(current_kprobe, &ri->rp->kp); | |
| 1908 | + __this_cpu_write(current_kprobe, &rp->kp); | |
| 2000 | 1909 | ri->ret_addr = correct_ret_addr; |
| 2001 | − ri->rp->handler(ri, regs); | |
| 1910 | + rp->handler(ri, regs); | |
| 2002 | 1911 | __this_cpu_write(current_kprobe, prev); |
| 2003 | 1912 | } |
| 2004 | 1913 | |
| 2005 | 1914 | recycle_rp_inst(ri); |
| 2006 | − | |
| 2007 | − if (ri == last) | |
| 2008 | − break; | |
| 2009 | 1915 | } |
| 2010 | 1916 | |
| 2011 | − kretprobe_hash_unlock(current, &flags); | |
| 2012 | − | |
| 2013 | 1917 | return (unsigned long)correct_ret_addr; |
| 2014 | 1918 | } |
| 2015 | 1919 | NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) |
| @@ -2021,11 +1925,10 @@ NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) · 5 unmodified lines | ||
| 2021 | 1925 | static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) |
| 2022 | 1926 | { |
| 2023 | 1927 | struct kretprobe *rp = container_of(p, struct kretprobe, kp); |
| 2024 | − unsigned long hash, flags = 0; | |
| 1928 | + unsigned long flags = 0; | |
| 2025 | 1929 | struct kretprobe_instance *ri; |
| 2026 | 1930 | |
| 2027 | 1931 | /* TODO: consider to only swap the RA after the last pre_handler fired */ |
| 2028 | − hash = hash_ptr(current, KPROBE_HASH_BITS); | |
| 2029 | 1932 | raw_spin_lock_irqsave(&rp->lock, flags); |
| 2030 | 1933 | if (!hlist_empty(&rp->free_instances)) { |
| 2031 | 1934 | ri = hlist_entry(rp->free_instances.first, |
| @@ -2033,9 +1936,6 @@ static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) · 1 unmodified line | ||
| 2033 | 1936 | hlist_del(&ri->hlist); |
| 2034 | 1937 | raw_spin_unlock_irqrestore(&rp->lock, flags); |
| 2035 | 1938 | |
| 2036 | − ri->rp = rp; | |
| 2037 | − ri->task = current; | |
| 2038 | − | |
| 2039 | 1939 | if (rp->entry_handler && rp->entry_handler(ri, regs)) { |
| 2040 | 1940 | raw_spin_lock_irqsave(&rp->lock, flags); |
| 2041 | 1941 | hlist_add_head(&ri->hlist, &rp->free_instances); |
| @@ -2045,11 +1945,8 @@ static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) · 3 unmodified lines | ||
| 2045 | 1945 | |
| 2046 | 1946 | arch_prepare_kretprobe(ri, regs); |
| 2047 | 1947 | |
| 2048 | − /* XXX(hch): why is there no hlist_move_head? */ | |
| 2049 | − INIT_HLIST_NODE(&ri->hlist); | |
| 2050 | − kretprobe_table_lock(hash, &flags); | |
| 2051 | − hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]); | |
| 2052 | − kretprobe_table_unlock(hash, &flags); | |
| 1948 | + __llist_add(&ri->llist, ¤t->kretprobe_instances); | |
| 1949 | + | |
| 2053 | 1950 | } else { |
| 2054 | 1951 | rp->nmissed++; |
| 2055 | 1952 | raw_spin_unlock_irqrestore(&rp->lock, flags); |
| @@ -2112,16 +2009,24 @@ int register_kretprobe(struct kretprobe *rp) · 56 unmodified lines | ||
| 2112 | 2009 | } |
| 2113 | 2010 | raw_spin_lock_init(&rp->lock); |
| 2114 | 2011 | INIT_HLIST_HEAD(&rp->free_instances); |
| 2012 | + rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL); | |
| 2013 | + if (!rp->rph) | |
| 2014 | + return -ENOMEM; | |
| 2015 | + | |
| 2016 | + rp->rph->rp = rp; | |
| 2115 | 2017 | for (i = 0; i < rp->maxactive; i++) { |
| 2116 | − inst = kmalloc(sizeof(struct kretprobe_instance) + | |
| 2018 | + inst = kzalloc(sizeof(struct kretprobe_instance) + | |
| 2117 | 2019 | rp->data_size, GFP_KERNEL); |
| 2118 | 2020 | if (inst == NULL) { |
| 2021 | + refcount_set(&rp->rph->ref, i); | |
| 2119 | 2022 | free_rp_inst(rp); |
| 2120 | 2023 | return -ENOMEM; |
| 2121 | 2024 | } |
| 2025 | + inst->rph = rp->rph; | |
| 2122 | 2026 | INIT_HLIST_NODE(&inst->hlist); |
| 2123 | 2027 | hlist_add_head(&inst->hlist, &rp->free_instances); |
| 2124 | 2028 | } |
| 2029 | + refcount_set(&rp->rph->ref, i); | |
| 2125 | 2030 | |
| 2126 | 2031 | rp->nmissed = 0; |
| 2127 | 2032 | /* Establish function entry probe point */ |
| @@ -2163,16 +2068,18 @@ void unregister_kretprobes(struct kretprobe **rps, int num) · 35 unmodified lines | ||
| 2163 | 2068 | if (num <= 0) |
| 2164 | 2069 | return; |
| 2165 | 2070 | mutex_lock(&kprobe_mutex); |
| 2166 | − for (i = 0; i < num; i++) | |
| 2071 | + for (i = 0; i < num; i++) { | |
| 2167 | 2072 | if (__unregister_kprobe_top(&rps[i]->kp) < 0) |
| 2168 | 2073 | rps[i]->kp.addr = NULL; |
| 2074 | + rps[i]->rph->rp = NULL; | |
| 2075 | + } | |
| 2169 | 2076 | mutex_unlock(&kprobe_mutex); |
| 2170 | 2077 | |
| 2171 | 2078 | synchronize_rcu(); |
| 2172 | 2079 | for (i = 0; i < num; i++) { |
| 2173 | 2080 | if (rps[i]->kp.addr) { |
| 2174 | 2081 | __unregister_kprobe_bottom(&rps[i]->kp); |
| 2175 | − cleanup_rp_inst(rps[i]); | |
| 2082 | + free_rp_inst(rps[i]); | |
| 2176 | 2083 | } |
| 2177 | 2084 | } |
| 2178 | 2085 | } |
| @@ -2535,11 +2442,8 @@ static int __init init_kprobes(void) · 356 unmodified lines | ||
| 2535 | 2442 | |
| 2536 | 2443 | /* FIXME allocate the probe table, currently defined statically */ |
| 2537 | 2444 | /* initialize all list heads */ |
| 2538 | − for (i = 0; i < KPROBE_TABLE_SIZE; i++) { | |
| 2445 | + for (i = 0; i < KPROBE_TABLE_SIZE; i++) | |
| 2539 | 2446 | INIT_HLIST_HEAD(&kprobe_table[i]); |
| 2540 | − INIT_HLIST_HEAD(&kretprobe_inst_table[i]); | |
| 2541 | − raw_spin_lock_init(&(kretprobe_table_locks[i].lock)); | |
| 2542 | − } | |
| 2543 | 2447 | |
| 2544 | 2448 | err = populate_kprobe_blacklist(__start_kprobe_blacklist, |
| 2545 | 2449 | __stop_kprobe_blacklist); |
| @@ -1714,7 +1714,8 @@ NOKPROBE_SYMBOL(kprobe_dispatcher); · 1713 unmodified lines | ||
| 1714 | 1714 | static int |
| 1715 | 1715 | kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) |
| 1716 | 1716 | { |
| 1717 | − struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp); | |
| 1717 | + struct kretprobe *rp = get_kretprobe(ri); | |
| 1718 | + struct trace_kprobe *tk = container_of(rp, struct trace_kprobe, rp); | |
| 1718 | 1719 | |
| 1719 | 1720 | raw_cpu_inc(*tk->nhit); |
| 1720 | 1721 | |