History

bpf: Add lookup_and_delete_elem support to hashtab

Extend the existing bpf_map_lookup_and_delete_elem() functionality to hashtab map types, in addition to stacks and queues. Create a new hashtab bpf_map_ops function that does lookup and deletion of the element under the same bucket lock and add the created map_ops to bpf.h.

Denis Salopek committed 5 years ago

Commit 3e87f191 parent f9bceaa5 files · +156 −4Browse files

Unified · Split

5 files
  1. include/linux/bpf.h +2 −0
  2. include/uapi/linux/bpf.h +13 −0
  3. kernel/bpf/hashtab.c +98 −0
  4. kernel/bpf/syscall.c +30 −4
  5. tools/include/uapi/linux/bpf.h +13 −0
include/linux/bpf.h+2−0
@@ -70,6 +70,8 @@ struct bpf_map_ops { · 69 unmodified lines
70 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);70 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
71 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,71 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
72 union bpf_attr __user *uattr);72 union bpf_attr __user *uattr);
73 int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key,
74 void *value, u64 flags);
73 int (*map_lookup_and_delete_batch)(struct bpf_map *map,75 int (*map_lookup_and_delete_batch)(struct bpf_map *map,
74 const union bpf_attr *attr,76 const union bpf_attr *attr,
75 union bpf_attr __user *uattr);77 union bpf_attr __user *uattr);
include/uapi/linux/bpf.h+13−0
@@ -527,6 +527,15 @@ union bpf_iter_link_info { · 526 unmodified lines
527 * Look up an element with the given *key* in the map referred to527 * Look up an element with the given *key* in the map referred to
528 * by the file descriptor *fd*, and if found, delete the element.528 * by the file descriptor *fd*, and if found, delete the element.
529 *529 *
530 * For **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map
531 * types, the *flags* argument needs to be set to 0, but for other
532 * map types, it may be specified as:
533 *
534 * **BPF_F_LOCK**
535 * Look up and delete the value of a spin-locked map
536 * without returning the lock. This must be specified if
537 * the elements contain a spinlock.
538 *
530 * The **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map types539 * The **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map types
531 * implement this command as a "pop" operation, deleting the top540 * implement this command as a "pop" operation, deleting the top
532 * element rather than one corresponding to *key*.541 * element rather than one corresponding to *key*.
@@ -536,6 +545,10 @@ union bpf_iter_link_info { · 3 unmodified lines
536 * This command is only valid for the following map types:545 * This command is only valid for the following map types:
537 * * **BPF_MAP_TYPE_QUEUE**546 * * **BPF_MAP_TYPE_QUEUE**
538 * * **BPF_MAP_TYPE_STACK**547 * * **BPF_MAP_TYPE_STACK**
548 * * **BPF_MAP_TYPE_HASH**
549 * * **BPF_MAP_TYPE_PERCPU_HASH**
550 * * **BPF_MAP_TYPE_LRU_HASH**
551 * * **BPF_MAP_TYPE_LRU_PERCPU_HASH**
539 *552 *
540 * Return553 * Return
541 * Returns zero on success. On error, -1 is returned and *errno*554 * Returns zero on success. On error, -1 is returned and *errno*
kernel/bpf/hashtab.c+98−0
@@ -1401,6 +1401,100 @@ static void htab_map_seq_show_elem(struct bpf_map *map, void *key, · 1400 unmodified lines
1401 rcu_read_unlock();1401 rcu_read_unlock();
1402}1402}
1403 1403
1404static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
1405 void *value, bool is_lru_map,
1406 bool is_percpu, u64 flags)
1407{
1408 struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
1409 struct hlist_nulls_head *head;
1410 unsigned long bflags;
1411 struct htab_elem *l;
1412 u32 hash, key_size;
1413 struct bucket *b;
1414 int ret;
1415
1416 key_size = map->key_size;
1417
1418 hash = htab_map_hash(key, key_size, htab->hashrnd);
1419 b = __select_bucket(htab, hash);
1420 head = &b->head;
1421
1422 ret = htab_lock_bucket(htab, b, hash, &bflags);
1423 if (ret)
1424 return ret;
1425
1426 l = lookup_elem_raw(head, hash, key, key_size);
1427 if (!l) {
1428 ret = -ENOENT;
1429 } else {
1430 if (is_percpu) {
1431 u32 roundup_value_size = round_up(map->value_size, 8);
1432 void __percpu *pptr;
1433 int off = 0, cpu;
1434
1435 pptr = htab_elem_get_ptr(l, key_size);
1436 for_each_possible_cpu(cpu) {
1437 bpf_long_memcpy(value + off,
1438 per_cpu_ptr(pptr, cpu),
1439 roundup_value_size);
1440 off += roundup_value_size;
1441 }
1442 } else {
1443 u32 roundup_key_size = round_up(map->key_size, 8);
1444
1445 if (flags & BPF_F_LOCK)
1446 copy_map_value_locked(map, value, l->key +
1447 roundup_key_size,
1448 true);
1449 else
1450 copy_map_value(map, value, l->key +
1451 roundup_key_size);
1452 check_and_init_map_lock(map, value);
1453 }
1454
1455 hlist_nulls_del_rcu(&l->hash_node);
1456 if (!is_lru_map)
1457 free_htab_elem(htab, l);
1458 }
1459
1460 htab_unlock_bucket(htab, b, hash, bflags);
1461
1462 if (is_lru_map && l)
1463 bpf_lru_push_free(&htab->lru, &l->lru_node);
1464
1465 return ret;
1466}
1467
1468static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
1469 void *value, u64 flags)
1470{
1471 return __htab_map_lookup_and_delete_elem(map, key, value, false, false,
1472 flags);
1473}
1474
1475static int htab_percpu_map_lookup_and_delete_elem(struct bpf_map *map,
1476 void *key, void *value,
1477 u64 flags)
1478{
1479 return __htab_map_lookup_and_delete_elem(map, key, value, false, true,
1480 flags);
1481}
1482
1483static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
1484 void *value, u64 flags)
1485{
1486 return __htab_map_lookup_and_delete_elem(map, key, value, true, false,
1487 flags);
1488}
1489
1490static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map,
1491 void *key, void *value,
1492 u64 flags)
1493{
1494 return __htab_map_lookup_and_delete_elem(map, key, value, true, true,
1495 flags);
1496}
1497
1404static int1498static int
1405__htab_map_lookup_and_delete_batch(struct bpf_map *map,1499__htab_map_lookup_and_delete_batch(struct bpf_map *map,
1406 const union bpf_attr *attr,1500 const union bpf_attr *attr,
@@ -1934,6 +2028,7 @@ const struct bpf_map_ops htab_map_ops = { · 527 unmodified lines
1934 .map_free = htab_map_free,2028 .map_free = htab_map_free,
1935 .map_get_next_key = htab_map_get_next_key,2029 .map_get_next_key = htab_map_get_next_key,
1936 .map_lookup_elem = htab_map_lookup_elem,2030 .map_lookup_elem = htab_map_lookup_elem,
2031 .map_lookup_and_delete_elem = htab_map_lookup_and_delete_elem,
1937 .map_update_elem = htab_map_update_elem,2032 .map_update_elem = htab_map_update_elem,
1938 .map_delete_elem = htab_map_delete_elem,2033 .map_delete_elem = htab_map_delete_elem,
1939 .map_gen_lookup = htab_map_gen_lookup,2034 .map_gen_lookup = htab_map_gen_lookup,
@@ -1954,6 +2049,7 @@ const struct bpf_map_ops htab_lru_map_ops = { · 14 unmodified lines
1954 .map_free = htab_map_free,2049 .map_free = htab_map_free,
1955 .map_get_next_key = htab_map_get_next_key,2050 .map_get_next_key = htab_map_get_next_key,
1956 .map_lookup_elem = htab_lru_map_lookup_elem,2051 .map_lookup_elem = htab_lru_map_lookup_elem,
2052 .map_lookup_and_delete_elem = htab_lru_map_lookup_and_delete_elem,
1957 .map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys,2053 .map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys,
1958 .map_update_elem = htab_lru_map_update_elem,2054 .map_update_elem = htab_lru_map_update_elem,
1959 .map_delete_elem = htab_lru_map_delete_elem,2055 .map_delete_elem = htab_lru_map_delete_elem,
@@ -2077,6 +2173,7 @@ const struct bpf_map_ops htab_percpu_map_ops = { · 117 unmodified lines
2077 .map_free = htab_map_free,2173 .map_free = htab_map_free,
2078 .map_get_next_key = htab_map_get_next_key,2174 .map_get_next_key = htab_map_get_next_key,
2079 .map_lookup_elem = htab_percpu_map_lookup_elem,2175 .map_lookup_elem = htab_percpu_map_lookup_elem,
2176 .map_lookup_and_delete_elem = htab_percpu_map_lookup_and_delete_elem,
2080 .map_update_elem = htab_percpu_map_update_elem,2177 .map_update_elem = htab_percpu_map_update_elem,
2081 .map_delete_elem = htab_map_delete_elem,2178 .map_delete_elem = htab_map_delete_elem,
2082 .map_seq_show_elem = htab_percpu_map_seq_show_elem,2179 .map_seq_show_elem = htab_percpu_map_seq_show_elem,
@@ -2096,6 +2193,7 @@ const struct bpf_map_ops htab_lru_percpu_map_ops = { · 13 unmodified lines
2096 .map_free = htab_map_free,2193 .map_free = htab_map_free,
2097 .map_get_next_key = htab_map_get_next_key,2194 .map_get_next_key = htab_map_get_next_key,
2098 .map_lookup_elem = htab_lru_percpu_map_lookup_elem,2195 .map_lookup_elem = htab_lru_percpu_map_lookup_elem,
2196 .map_lookup_and_delete_elem = htab_lru_percpu_map_lookup_and_delete_elem,
2099 .map_update_elem = htab_lru_percpu_map_update_elem,2197 .map_update_elem = htab_lru_percpu_map_update_elem,
2100 .map_delete_elem = htab_lru_map_delete_elem,2198 .map_delete_elem = htab_lru_map_delete_elem,
2101 .map_seq_show_elem = htab_percpu_map_seq_show_elem,2199 .map_seq_show_elem = htab_percpu_map_seq_show_elem,
kernel/bpf/syscall.c+30−4
@@ -1483,7 +1483,7 @@ free_buf: · 1482 unmodified lines
1483 return err;1483 return err;
1484}1484}
1485 1485
1486#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value1486#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags
1487 1487
1488static int map_lookup_and_delete_elem(union bpf_attr *attr)1488static int map_lookup_and_delete_elem(union bpf_attr *attr)
1489{1489{
@@ -1499,6 +1499,9 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) · 9 unmodified lines
1499 if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))1499 if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1500 return -EINVAL;1500 return -EINVAL;
1501 1501
1502 if (attr->flags & ~BPF_F_LOCK)
1503 return -EINVAL;
1504
1502 f = fdget(ufd);1505 f = fdget(ufd);
1503 map = __bpf_map_get(f);1506 map = __bpf_map_get(f);
1504 if (IS_ERR(map))1507 if (IS_ERR(map))
@@ -1509,24 +1512,47 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) · 4 unmodified lines
1509 goto err_put;1512 goto err_put;
1510 }1513 }
1511 1514
1515 if (attr->flags &&
1516 (map->map_type == BPF_MAP_TYPE_QUEUE ||
1517 map->map_type == BPF_MAP_TYPE_STACK)) {
1518 err = -EINVAL;
1519 goto err_put;
1520 }
1521
1522 if ((attr->flags & BPF_F_LOCK) &&
1523 !map_value_has_spin_lock(map)) {
1524 err = -EINVAL;
1525 goto err_put;
1526 }
1527
1512 key = __bpf_copy_key(ukey, map->key_size);1528 key = __bpf_copy_key(ukey, map->key_size);
1513 if (IS_ERR(key)) {1529 if (IS_ERR(key)) {
1514 err = PTR_ERR(key);1530 err = PTR_ERR(key);
1515 goto err_put;1531 goto err_put;
1516 }1532 }
1517 1533
1518 value_size = map->value_size;1534 value_size = bpf_map_value_size(map);
1519 1535
1520 err = -ENOMEM;1536 err = -ENOMEM;
1521 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);1537 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1522 if (!value)1538 if (!value)
1523 goto free_key;1539 goto free_key;
1524 1540
1541 err = -ENOTSUPP;
1525 if (map->map_type == BPF_MAP_TYPE_QUEUE ||1542 if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1526 map->map_type == BPF_MAP_TYPE_STACK) {1543 map->map_type == BPF_MAP_TYPE_STACK) {
1527 err = map->ops->map_pop_elem(map, value);1544 err = map->ops->map_pop_elem(map, value);
1528 } else {1545 } else if (map->map_type == BPF_MAP_TYPE_HASH ||
1529 err = -ENOTSUPP;1546 map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1547 map->map_type == BPF_MAP_TYPE_LRU_HASH ||
1548 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
1549 if (!bpf_map_is_dev_bound(map)) {
1550 bpf_disable_instrumentation();
1551 rcu_read_lock();
1552 err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags);
1553 rcu_read_unlock();
1554 bpf_enable_instrumentation();
1555 }
1530 }1556 }
1531 1557
1532 if (err)1558 if (err)
tools/include/uapi/linux/bpf.h+13−0
@@ -527,6 +527,15 @@ union bpf_iter_link_info { · 526 unmodified lines
527 * Look up an element with the given *key* in the map referred to527 * Look up an element with the given *key* in the map referred to
528 * by the file descriptor *fd*, and if found, delete the element.528 * by the file descriptor *fd*, and if found, delete the element.
529 *529 *
530 * For **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map
531 * types, the *flags* argument needs to be set to 0, but for other
532 * map types, it may be specified as:
533 *
534 * **BPF_F_LOCK**
535 * Look up and delete the value of a spin-locked map
536 * without returning the lock. This must be specified if
537 * the elements contain a spinlock.
538 *
530 * The **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map types539 * The **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map types
531 * implement this command as a "pop" operation, deleting the top540 * implement this command as a "pop" operation, deleting the top
532 * element rather than one corresponding to *key*.541 * element rather than one corresponding to *key*.
@@ -536,6 +545,10 @@ union bpf_iter_link_info { · 3 unmodified lines
536 * This command is only valid for the following map types:545 * This command is only valid for the following map types:
537 * * **BPF_MAP_TYPE_QUEUE**546 * * **BPF_MAP_TYPE_QUEUE**
538 * * **BPF_MAP_TYPE_STACK**547 * * **BPF_MAP_TYPE_STACK**
548 * * **BPF_MAP_TYPE_HASH**
549 * * **BPF_MAP_TYPE_PERCPU_HASH**
550 * * **BPF_MAP_TYPE_LRU_HASH**
551 * * **BPF_MAP_TYPE_LRU_PERCPU_HASH**
539 *552 *
540 * Return553 * Return
541 * Returns zero on success. On error, -1 is returned and *errno*554 * Returns zero on success. On error, -1 is returned and *errno*

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t