| @@ -53,7 +53,6 @@ static int kprobes_initialized; · 52 unmodified lines |
| 53 | 53 | * - RCU hlist traversal under disabling preempt (breakpoint handlers) |
| 54 | 54 | */ |
| 55 | 55 | static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; |
| 56 | | −static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; |
| 57 | 56 | |
| 58 | 57 | /* NOTE: change this value only with kprobe_mutex held */ |
| 59 | 58 | static bool kprobes_all_disarmed; |
| @@ -61,9 +60,6 @@ static bool kprobes_all_disarmed; · 1 unmodified line |
| 61 | 60 | /* This protects kprobe_table and optimizing_list */ |
| 62 | 61 | static DEFINE_MUTEX(kprobe_mutex); |
| 63 | 62 | static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; |
| 64 | | −static struct { |
| 65 | | − raw_spinlock_t lock ____cacheline_aligned_in_smp; |
| 66 | | −} kretprobe_table_locks[KPROBE_TABLE_SIZE]; |
| 67 | 63 | |
| 68 | 64 | kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, |
| 69 | 65 | unsigned int __unused) |
| @@ -71,11 +67,6 @@ kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, · 1 unmodified line |
| 71 | 67 | return ((kprobe_opcode_t *)(kallsyms_lookup_name(name))); |
| 72 | 68 | } |
| 73 | 69 | |
| 74 | | −static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) |
| 75 | | −{ |
| 76 | | − return &(kretprobe_table_locks[hash].lock); |
| 77 | | −} |
| 78 | | − |
| 79 | 70 | /* Blacklist -- list of struct kprobe_blacklist_entry */ |
| 80 | 71 | static LIST_HEAD(kprobe_blacklist); |
| 81 | 72 | |
| @@ -1223,65 +1214,30 @@ void kprobes_inc_nmissed_count(struct kprobe *p) · 1141 unmodified lines |
| 1223 | 1214 | } |
| 1224 | 1215 | NOKPROBE_SYMBOL(kprobes_inc_nmissed_count); |
| 1225 | 1216 | |
| 1217 | +static void free_rp_inst_rcu(struct rcu_head *head) |
| 1218 | +{ |
| 1219 | + struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu); |
| 1220 | + |
| 1221 | + if (refcount_dec_and_test(&ri->rph->ref)) |
| 1222 | + kfree(ri->rph); |
| 1223 | + kfree(ri); |
| 1224 | +} |
| 1225 | +NOKPROBE_SYMBOL(free_rp_inst_rcu); |
| 1226 | + |
| 1226 | 1227 | static void recycle_rp_inst(struct kretprobe_instance *ri) |
| 1227 | 1228 | { |
| 1228 | | − struct kretprobe *rp = ri->rp; |
| 1229 | + struct kretprobe *rp = get_kretprobe(ri); |
| 1229 | 1230 | |
| 1230 | | − /* remove rp inst off the rprobe_inst_table */ |
| 1231 | | − hlist_del(&ri->hlist); |
| 1232 | 1231 | INIT_HLIST_NODE(&ri->hlist); |
| 1233 | 1232 | if (likely(rp)) { |
| 1234 | 1233 | raw_spin_lock(&rp->lock); |
| 1235 | 1234 | hlist_add_head(&ri->hlist, &rp->free_instances); |
| 1236 | 1235 | raw_spin_unlock(&rp->lock); |
| 1237 | 1236 | } else |
| 1238 | | − kfree_rcu(ri, rcu); |
| 1237 | + call_rcu(&ri->rcu, free_rp_inst_rcu); |
| 1239 | 1238 | } |
| 1240 | 1239 | NOKPROBE_SYMBOL(recycle_rp_inst); |
| 1241 | 1240 | |
| 1242 | | −static void kretprobe_hash_lock(struct task_struct *tsk, |
| 1243 | | − struct hlist_head **head, unsigned long *flags) |
| 1244 | | −__acquires(hlist_lock) |
| 1245 | | −{ |
| 1246 | | − unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); |
| 1247 | | − raw_spinlock_t *hlist_lock; |
| 1248 | | − |
| 1249 | | − *head = &kretprobe_inst_table[hash]; |
| 1250 | | − hlist_lock = kretprobe_table_lock_ptr(hash); |
| 1251 | | − raw_spin_lock_irqsave(hlist_lock, *flags); |
| 1252 | | −} |
| 1253 | | −NOKPROBE_SYMBOL(kretprobe_hash_lock); |
| 1254 | | − |
| 1255 | | −static void kretprobe_table_lock(unsigned long hash, |
| 1256 | | − unsigned long *flags) |
| 1257 | | −__acquires(hlist_lock) |
| 1258 | | −{ |
| 1259 | | − raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); |
| 1260 | | − raw_spin_lock_irqsave(hlist_lock, *flags); |
| 1261 | | −} |
| 1262 | | −NOKPROBE_SYMBOL(kretprobe_table_lock); |
| 1263 | | − |
| 1264 | | −static void kretprobe_hash_unlock(struct task_struct *tsk, |
| 1265 | | − unsigned long *flags) |
| 1266 | | −__releases(hlist_lock) |
| 1267 | | −{ |
| 1268 | | − unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); |
| 1269 | | − raw_spinlock_t *hlist_lock; |
| 1270 | | − |
| 1271 | | − hlist_lock = kretprobe_table_lock_ptr(hash); |
| 1272 | | − raw_spin_unlock_irqrestore(hlist_lock, *flags); |
| 1273 | | −} |
| 1274 | | −NOKPROBE_SYMBOL(kretprobe_hash_unlock); |
| 1275 | | − |
| 1276 | | −static void kretprobe_table_unlock(unsigned long hash, |
| 1277 | | − unsigned long *flags) |
| 1278 | | −__releases(hlist_lock) |
| 1279 | | −{ |
| 1280 | | − raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); |
| 1281 | | − raw_spin_unlock_irqrestore(hlist_lock, *flags); |
| 1282 | | −} |
| 1283 | | −NOKPROBE_SYMBOL(kretprobe_table_unlock); |
| 1284 | | − |
| 1285 | 1241 | static struct kprobe kprobe_busy = { |
| 1286 | 1242 | .addr = (void *) get_kprobe, |
| 1287 | 1243 | }; |
| @@ -1311,24 +1267,21 @@ void kprobe_busy_end(void) · 23 unmodified lines |
| 1311 | 1267 | void kprobe_flush_task(struct task_struct *tk) |
| 1312 | 1268 | { |
| 1313 | 1269 | struct kretprobe_instance *ri; |
| 1314 | | − struct hlist_head *head; |
| 1315 | | − struct hlist_node *tmp; |
| 1316 | | − unsigned long hash, flags = 0; |
| 1270 | + struct llist_node *node; |
| 1317 | 1271 | |
| 1272 | + /* Early boot, not yet initialized. */ |
| 1318 | 1273 | if (unlikely(!kprobes_initialized)) |
| 1319 | | − /* Early boot. kretprobe_table_locks not yet initialized. */ |
| 1320 | 1274 | return; |
| 1321 | 1275 | |
| 1322 | 1276 | kprobe_busy_begin(); |
| 1323 | 1277 | |
| 1324 | | − hash = hash_ptr(tk, KPROBE_HASH_BITS); |
| 1325 | | − head = &kretprobe_inst_table[hash]; |
| 1326 | | − kretprobe_table_lock(hash, &flags); |
| 1327 | | − hlist_for_each_entry_safe(ri, tmp, head, hlist) { |
| 1328 | | − if (ri->task == tk) |
| 1329 | | − recycle_rp_inst(ri); |
| 1278 | + node = __llist_del_all(&tk->kretprobe_instances); |
| 1279 | + while (node) { |
| 1280 | + ri = container_of(node, struct kretprobe_instance, llist); |
| 1281 | + node = node->next; |
| 1282 | + |
| 1283 | + recycle_rp_inst(ri); |
| 1330 | 1284 | } |
| 1331 | | − kretprobe_table_unlock(hash, &flags); |
| 1332 | 1285 | |
| 1333 | 1286 | kprobe_busy_end(); |
| 1334 | 1287 | } |
| @@ -1338,36 +1291,19 @@ static inline void free_rp_inst(struct kretprobe *rp) · 3 unmodified lines |
| 1338 | 1291 | { |
| 1339 | 1292 | struct kretprobe_instance *ri; |
| 1340 | 1293 | struct hlist_node *next; |
| 1294 | + int count = 0; |
| 1341 | 1295 | |
| 1342 | 1296 | hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) { |
| 1343 | 1297 | hlist_del(&ri->hlist); |
| 1344 | 1298 | kfree(ri); |
| 1299 | + count++; |
| 1345 | 1300 | } |
| 1346 | | −} |
| 1347 | | − |
| 1348 | | −static void cleanup_rp_inst(struct kretprobe *rp) |
| 1349 | | −{ |
| 1350 | | − unsigned long flags, hash; |
| 1351 | | − struct kretprobe_instance *ri; |
| 1352 | | − struct hlist_node *next; |
| 1353 | | − struct hlist_head *head; |
| 1354 | 1301 | |
| 1355 | | − /* To avoid recursive kretprobe by NMI, set kprobe busy here */ |
| 1356 | | − kprobe_busy_begin(); |
| 1357 | | − for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) { |
| 1358 | | − kretprobe_table_lock(hash, &flags); |
| 1359 | | − head = &kretprobe_inst_table[hash]; |
| 1360 | | − hlist_for_each_entry_safe(ri, next, head, hlist) { |
| 1361 | | − if (ri->rp == rp) |
| 1362 | | − ri->rp = NULL; |
| 1363 | | − } |
| 1364 | | − kretprobe_table_unlock(hash, &flags); |
| 1302 | + if (refcount_sub_and_test(count, &rp->rph->ref)) { |
| 1303 | + kfree(rp->rph); |
| 1304 | + rp->rph = NULL; |
| 1365 | 1305 | } |
| 1366 | | − kprobe_busy_end(); |
| 1367 | | − |
| 1368 | | − free_rp_inst(rp); |
| 1369 | 1306 | } |
| 1370 | | −NOKPROBE_SYMBOL(cleanup_rp_inst); |
| 1371 | 1307 | |
| 1372 | 1308 | /* Add the new probe to ap->list */ |
| 1373 | 1309 | static int add_new_kprobe(struct kprobe *ap, struct kprobe *p) |
| @@ -1928,88 +1864,56 @@ unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs, · 554 unmodified lines |
| 1928 | 1864 | void *trampoline_address, |
| 1929 | 1865 | void *frame_pointer) |
| 1930 | 1866 | { |
| 1931 | | − struct kretprobe_instance *ri = NULL, *last = NULL; |
| 1932 | | − struct hlist_head *head; |
| 1933 | | − struct hlist_node *tmp; |
| 1934 | | − unsigned long flags; |
| 1935 | 1867 | kprobe_opcode_t *correct_ret_addr = NULL; |
| 1936 | | − bool skipped = false; |
| 1868 | + struct kretprobe_instance *ri = NULL; |
| 1869 | + struct llist_node *first, *node; |
| 1870 | + struct kretprobe *rp; |
| 1937 | 1871 | |
| 1938 | | − kretprobe_hash_lock(current, &head, &flags); |
| 1872 | + /* Find all nodes for this frame. */ |
| 1873 | + first = node = current->kretprobe_instances.first; |
| 1874 | + while (node) { |
| 1875 | + ri = container_of(node, struct kretprobe_instance, llist); |
| 1939 | 1876 | |
| 1940 | | − /* |
| 1941 | | − * It is possible to have multiple instances associated with a given |
| 1942 | | − * task either because multiple functions in the call path have |
| 1943 | | − * return probes installed on them, and/or more than one |
| 1944 | | − * return probe was registered for a target function. |
| 1945 | | − * |
| 1946 | | − * We can handle this because: |
| 1947 | | − * - instances are always pushed into the head of the list |
| 1948 | | − * - when multiple return probes are registered for the same |
| 1949 | | − * function, the (chronologically) first instance's ret_addr |
| 1950 | | − * will be the real return address, and all the rest will |
| 1951 | | − * point to kretprobe_trampoline. |
| 1952 | | − */ |
| 1953 | | − hlist_for_each_entry(ri, head, hlist) { |
| 1954 | | − if (ri->task != current) |
| 1955 | | − /* another task is sharing our hash bucket */ |
| 1956 | | − continue; |
| 1957 | | − /* |
| 1958 | | − * Return probes must be pushed on this hash list correct |
| 1959 | | − * order (same as return order) so that it can be popped |
| 1960 | | − * correctly. However, if we find it is pushed it incorrect |
| 1961 | | − * order, this means we find a function which should not be |
| 1962 | | − * probed, because the wrong order entry is pushed on the |
| 1963 | | − * path of processing other kretprobe itself. |
| 1964 | | − */ |
| 1965 | | − if (ri->fp != frame_pointer) { |
| 1966 | | − if (!skipped) |
| 1967 | | − pr_warn("kretprobe is stacked incorrectly. Trying to fixup.\n"); |
| 1968 | | − skipped = true; |
| 1969 | | − continue; |
| 1970 | | − } |
| 1877 | + BUG_ON(ri->fp != frame_pointer); |
| 1971 | 1878 | |
| 1972 | | − correct_ret_addr = ri->ret_addr; |
| 1973 | | − if (skipped) |
| 1974 | | − pr_warn("%ps must be blacklisted because of incorrect kretprobe order\n", |
| 1975 | | − ri->rp->kp.addr); |
| 1976 | | − |
| 1977 | | − if (correct_ret_addr != trampoline_address) |
| 1879 | + if (ri->ret_addr != trampoline_address) { |
| 1880 | + correct_ret_addr = ri->ret_addr; |
| 1978 | 1881 | /* |
| 1979 | 1882 | * This is the real return address. Any other |
| 1980 | 1883 | * instances associated with this task are for |
| 1981 | 1884 | * other calls deeper on the call stack |
| 1982 | 1885 | */ |
| 1983 | | − break; |
| 1886 | + goto found; |
| 1887 | + } |
| 1888 | + |
| 1889 | + node = node->next; |
| 1984 | 1890 | } |
| 1891 | + pr_err("Oops! Kretprobe fails to find correct return address.\n"); |
| 1892 | + BUG_ON(1); |
| 1985 | 1893 | |
| 1986 | | − BUG_ON(!correct_ret_addr || (correct_ret_addr == trampoline_address)); |
| 1987 | | − last = ri; |
| 1894 | +found: |
| 1895 | + /* Unlink all nodes for this frame. */ |
| 1896 | + current->kretprobe_instances.first = node->next; |
| 1897 | + node->next = NULL; |
| 1988 | 1898 | |
| 1989 | | − hlist_for_each_entry_safe(ri, tmp, head, hlist) { |
| 1990 | | − if (ri->task != current) |
| 1991 | | − /* another task is sharing our hash bucket */ |
| 1992 | | − continue; |
| 1993 | | − if (ri->fp != frame_pointer) |
| 1994 | | − continue; |
| 1899 | + /* Run them.. */ |
| 1900 | + while (first) { |
| 1901 | + ri = container_of(first, struct kretprobe_instance, llist); |
| 1902 | + first = first->next; |
| 1995 | 1903 | |
| 1996 | | − if (ri->rp && ri->rp->handler) { |
| 1904 | + rp = get_kretprobe(ri); |
| 1905 | + if (rp && rp->handler) { |
| 1997 | 1906 | struct kprobe *prev = kprobe_running(); |
| 1998 | 1907 | |
| 1999 | | − __this_cpu_write(current_kprobe, &ri->rp->kp); |
| 1908 | + __this_cpu_write(current_kprobe, &rp->kp); |
| 2000 | 1909 | ri->ret_addr = correct_ret_addr; |
| 2001 | | − ri->rp->handler(ri, regs); |
| 1910 | + rp->handler(ri, regs); |
| 2002 | 1911 | __this_cpu_write(current_kprobe, prev); |
| 2003 | 1912 | } |
| 2004 | 1913 | |
| 2005 | 1914 | recycle_rp_inst(ri); |
| 2006 | | − |
| 2007 | | − if (ri == last) |
| 2008 | | − break; |
| 2009 | 1915 | } |
| 2010 | 1916 | |
| 2011 | | − kretprobe_hash_unlock(current, &flags); |
| 2012 | | − |
| 2013 | 1917 | return (unsigned long)correct_ret_addr; |
| 2014 | 1918 | } |
| 2015 | 1919 | NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) |
| @@ -2021,11 +1925,10 @@ NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) · 5 unmodified lines |
| 2021 | 1925 | static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) |
| 2022 | 1926 | { |
| 2023 | 1927 | struct kretprobe *rp = container_of(p, struct kretprobe, kp); |
| 2024 | | − unsigned long hash, flags = 0; |
| 1928 | + unsigned long flags = 0; |
| 2025 | 1929 | struct kretprobe_instance *ri; |
| 2026 | 1930 | |
| 2027 | 1931 | /* TODO: consider to only swap the RA after the last pre_handler fired */ |
| 2028 | | − hash = hash_ptr(current, KPROBE_HASH_BITS); |
| 2029 | 1932 | raw_spin_lock_irqsave(&rp->lock, flags); |
| 2030 | 1933 | if (!hlist_empty(&rp->free_instances)) { |
| 2031 | 1934 | ri = hlist_entry(rp->free_instances.first, |
| @@ -2033,9 +1936,6 @@ static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) · 1 unmodified line |
| 2033 | 1936 | hlist_del(&ri->hlist); |
| 2034 | 1937 | raw_spin_unlock_irqrestore(&rp->lock, flags); |
| 2035 | 1938 | |
| 2036 | | − ri->rp = rp; |
| 2037 | | − ri->task = current; |
| 2038 | | − |
| 2039 | 1939 | if (rp->entry_handler && rp->entry_handler(ri, regs)) { |
| 2040 | 1940 | raw_spin_lock_irqsave(&rp->lock, flags); |
| 2041 | 1941 | hlist_add_head(&ri->hlist, &rp->free_instances); |
| @@ -2045,11 +1945,8 @@ static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) · 3 unmodified lines |
| 2045 | 1945 | |
| 2046 | 1946 | arch_prepare_kretprobe(ri, regs); |
| 2047 | 1947 | |
| 2048 | | − /* XXX(hch): why is there no hlist_move_head? */ |
| 2049 | | − INIT_HLIST_NODE(&ri->hlist); |
| 2050 | | − kretprobe_table_lock(hash, &flags); |
| 2051 | | − hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]); |
| 2052 | | − kretprobe_table_unlock(hash, &flags); |
| 1948 | + __llist_add(&ri->llist, ¤t->kretprobe_instances); |
| 1949 | + |
| 2053 | 1950 | } else { |
| 2054 | 1951 | rp->nmissed++; |
| 2055 | 1952 | raw_spin_unlock_irqrestore(&rp->lock, flags); |
| @@ -2112,16 +2009,24 @@ int register_kretprobe(struct kretprobe *rp) · 56 unmodified lines |
| 2112 | 2009 | } |
| 2113 | 2010 | raw_spin_lock_init(&rp->lock); |
| 2114 | 2011 | INIT_HLIST_HEAD(&rp->free_instances); |
| 2012 | + rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL); |
| 2013 | + if (!rp->rph) |
| 2014 | + return -ENOMEM; |
| 2015 | + |
| 2016 | + rp->rph->rp = rp; |
| 2115 | 2017 | for (i = 0; i < rp->maxactive; i++) { |
| 2116 | | − inst = kmalloc(sizeof(struct kretprobe_instance) + |
| 2018 | + inst = kzalloc(sizeof(struct kretprobe_instance) + |
| 2117 | 2019 | rp->data_size, GFP_KERNEL); |
| 2118 | 2020 | if (inst == NULL) { |
| 2021 | + refcount_set(&rp->rph->ref, i); |
| 2119 | 2022 | free_rp_inst(rp); |
| 2120 | 2023 | return -ENOMEM; |
| 2121 | 2024 | } |
| 2025 | + inst->rph = rp->rph; |
| 2122 | 2026 | INIT_HLIST_NODE(&inst->hlist); |
| 2123 | 2027 | hlist_add_head(&inst->hlist, &rp->free_instances); |
| 2124 | 2028 | } |
| 2029 | + refcount_set(&rp->rph->ref, i); |
| 2125 | 2030 | |
| 2126 | 2031 | rp->nmissed = 0; |
| 2127 | 2032 | /* Establish function entry probe point */ |
| @@ -2163,16 +2068,18 @@ void unregister_kretprobes(struct kretprobe **rps, int num) · 35 unmodified lines |
| 2163 | 2068 | if (num <= 0) |
| 2164 | 2069 | return; |
| 2165 | 2070 | mutex_lock(&kprobe_mutex); |
| 2166 | | − for (i = 0; i < num; i++) |
| 2071 | + for (i = 0; i < num; i++) { |
| 2167 | 2072 | if (__unregister_kprobe_top(&rps[i]->kp) < 0) |
| 2168 | 2073 | rps[i]->kp.addr = NULL; |
| 2074 | + rps[i]->rph->rp = NULL; |
| 2075 | + } |
| 2169 | 2076 | mutex_unlock(&kprobe_mutex); |
| 2170 | 2077 | |
| 2171 | 2078 | synchronize_rcu(); |
| 2172 | 2079 | for (i = 0; i < num; i++) { |
| 2173 | 2080 | if (rps[i]->kp.addr) { |
| 2174 | 2081 | __unregister_kprobe_bottom(&rps[i]->kp); |
| 2175 | | − cleanup_rp_inst(rps[i]); |
| 2082 | + free_rp_inst(rps[i]); |
| 2176 | 2083 | } |
| 2177 | 2084 | } |
| 2178 | 2085 | } |
| @@ -2535,11 +2442,8 @@ static int __init init_kprobes(void) · 356 unmodified lines |
| 2535 | 2442 | |
| 2536 | 2443 | /* FIXME allocate the probe table, currently defined statically */ |
| 2537 | 2444 | /* initialize all list heads */ |
| 2538 | | − for (i = 0; i < KPROBE_TABLE_SIZE; i++) { |
| 2445 | + for (i = 0; i < KPROBE_TABLE_SIZE; i++) |
| 2539 | 2446 | INIT_HLIST_HEAD(&kprobe_table[i]); |
| 2540 | | − INIT_HLIST_HEAD(&kretprobe_inst_table[i]); |
| 2541 | | − raw_spin_lock_init(&(kretprobe_table_locks[i].lock)); |
| 2542 | | − } |
| 2543 | 2447 | |
| 2544 | 2448 | err = populate_kprobe_blacklist(__start_kprobe_blacklist, |
| 2545 | 2449 | __stop_kprobe_blacklist); |