Extend the existing bpf_map_lookup_and_delete_elem() functionality to hashtab map types, in addition to stacks and queues. Create a new hashtab bpf_map_ops function that does lookup and deletion of the element under the same bucket lock and add the created map_ops to bpf.h.
Showing the changes to kernel (2 of 5 files). Show all
Unified · Split
| @@ -1401,6 +1401,100 @@ static void htab_map_seq_show_elem(struct bpf_map *map, void *key, · 1400 unmodified lines | |||
| 1401 | rcu_read_unlock(); | 1401 | rcu_read_unlock(); |
| 1402 | } | 1402 | } |
| 1403 | 1403 | ||
| 1404 | static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, | ||
| 1405 | void *value, bool is_lru_map, | ||
| 1406 | bool is_percpu, u64 flags) | ||
| 1407 | { | ||
| 1408 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map); | ||
| 1409 | struct hlist_nulls_head *head; | ||
| 1410 | unsigned long bflags; | ||
| 1411 | struct htab_elem *l; | ||
| 1412 | u32 hash, key_size; | ||
| 1413 | struct bucket *b; | ||
| 1414 | int ret; | ||
| 1415 | |||
| 1416 | key_size = map->key_size; | ||
| 1417 | |||
| 1418 | hash = htab_map_hash(key, key_size, htab->hashrnd); | ||
| 1419 | b = __select_bucket(htab, hash); | ||
| 1420 | head = &b->head; | ||
| 1421 | |||
| 1422 | ret = htab_lock_bucket(htab, b, hash, &bflags); | ||
| 1423 | if (ret) | ||
| 1424 | return ret; | ||
| 1425 | |||
| 1426 | l = lookup_elem_raw(head, hash, key, key_size); | ||
| 1427 | if (!l) { | ||
| 1428 | ret = -ENOENT; | ||
| 1429 | } else { | ||
| 1430 | if (is_percpu) { | ||
| 1431 | u32 roundup_value_size = round_up(map->value_size, 8); | ||
| 1432 | void __percpu *pptr; | ||
| 1433 | int off = 0, cpu; | ||
| 1434 | |||
| 1435 | pptr = htab_elem_get_ptr(l, key_size); | ||
| 1436 | for_each_possible_cpu(cpu) { | ||
| 1437 | bpf_long_memcpy(value + off, | ||
| 1438 | per_cpu_ptr(pptr, cpu), | ||
| 1439 | roundup_value_size); | ||
| 1440 | off += roundup_value_size; | ||
| 1441 | } | ||
| 1442 | } else { | ||
| 1443 | u32 roundup_key_size = round_up(map->key_size, 8); | ||
| 1444 | |||
| 1445 | if (flags & BPF_F_LOCK) | ||
| 1446 | copy_map_value_locked(map, value, l->key + | ||
| 1447 | roundup_key_size, | ||
| 1448 | true); | ||
| 1449 | else | ||
| 1450 | copy_map_value(map, value, l->key + | ||
| 1451 | roundup_key_size); | ||
| 1452 | check_and_init_map_lock(map, value); | ||
| 1453 | } | ||
| 1454 | |||
| 1455 | hlist_nulls_del_rcu(&l->hash_node); | ||
| 1456 | if (!is_lru_map) | ||
| 1457 | free_htab_elem(htab, l); | ||
| 1458 | } | ||
| 1459 | |||
| 1460 | htab_unlock_bucket(htab, b, hash, bflags); | ||
| 1461 | |||
| 1462 | if (is_lru_map && l) | ||
| 1463 | bpf_lru_push_free(&htab->lru, &l->lru_node); | ||
| 1464 | |||
| 1465 | return ret; | ||
| 1466 | } | ||
| 1467 | |||
| 1468 | static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, | ||
| 1469 | void *value, u64 flags) | ||
| 1470 | { | ||
| 1471 | return __htab_map_lookup_and_delete_elem(map, key, value, false, false, | ||
| 1472 | flags); | ||
| 1473 | } | ||
| 1474 | |||
| 1475 | static int htab_percpu_map_lookup_and_delete_elem(struct bpf_map *map, | ||
| 1476 | void *key, void *value, | ||
| 1477 | u64 flags) | ||
| 1478 | { | ||
| 1479 | return __htab_map_lookup_and_delete_elem(map, key, value, false, true, | ||
| 1480 | flags); | ||
| 1481 | } | ||
| 1482 | |||
| 1483 | static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key, | ||
| 1484 | void *value, u64 flags) | ||
| 1485 | { | ||
| 1486 | return __htab_map_lookup_and_delete_elem(map, key, value, true, false, | ||
| 1487 | flags); | ||
| 1488 | } | ||
| 1489 | |||
| 1490 | static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, | ||
| 1491 | void *key, void *value, | ||
| 1492 | u64 flags) | ||
| 1493 | { | ||
| 1494 | return __htab_map_lookup_and_delete_elem(map, key, value, true, true, | ||
| 1495 | flags); | ||
| 1496 | } | ||
| 1497 | |||
| 1404 | static int | 1498 | static int |
| 1405 | __htab_map_lookup_and_delete_batch(struct bpf_map *map, | 1499 | __htab_map_lookup_and_delete_batch(struct bpf_map *map, |
| 1406 | const union bpf_attr *attr, | 1500 | const union bpf_attr *attr, |
| @@ -1934,6 +2028,7 @@ const struct bpf_map_ops htab_map_ops = { · 527 unmodified lines | |||
| 1934 | .map_free = htab_map_free, | 2028 | .map_free = htab_map_free, |
| 1935 | .map_get_next_key = htab_map_get_next_key, | 2029 | .map_get_next_key = htab_map_get_next_key, |
| 1936 | .map_lookup_elem = htab_map_lookup_elem, | 2030 | .map_lookup_elem = htab_map_lookup_elem, |
| 2031 | .map_lookup_and_delete_elem = htab_map_lookup_and_delete_elem, | ||
| 1937 | .map_update_elem = htab_map_update_elem, | 2032 | .map_update_elem = htab_map_update_elem, |
| 1938 | .map_delete_elem = htab_map_delete_elem, | 2033 | .map_delete_elem = htab_map_delete_elem, |
| 1939 | .map_gen_lookup = htab_map_gen_lookup, | 2034 | .map_gen_lookup = htab_map_gen_lookup, |
| @@ -1954,6 +2049,7 @@ const struct bpf_map_ops htab_lru_map_ops = { · 14 unmodified lines | |||
| 1954 | .map_free = htab_map_free, | 2049 | .map_free = htab_map_free, |
| 1955 | .map_get_next_key = htab_map_get_next_key, | 2050 | .map_get_next_key = htab_map_get_next_key, |
| 1956 | .map_lookup_elem = htab_lru_map_lookup_elem, | 2051 | .map_lookup_elem = htab_lru_map_lookup_elem, |
| 2052 | .map_lookup_and_delete_elem = htab_lru_map_lookup_and_delete_elem, | ||
| 1957 | .map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys, | 2053 | .map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys, |
| 1958 | .map_update_elem = htab_lru_map_update_elem, | 2054 | .map_update_elem = htab_lru_map_update_elem, |
| 1959 | .map_delete_elem = htab_lru_map_delete_elem, | 2055 | .map_delete_elem = htab_lru_map_delete_elem, |
| @@ -2077,6 +2173,7 @@ const struct bpf_map_ops htab_percpu_map_ops = { · 117 unmodified lines | |||
| 2077 | .map_free = htab_map_free, | 2173 | .map_free = htab_map_free, |
| 2078 | .map_get_next_key = htab_map_get_next_key, | 2174 | .map_get_next_key = htab_map_get_next_key, |
| 2079 | .map_lookup_elem = htab_percpu_map_lookup_elem, | 2175 | .map_lookup_elem = htab_percpu_map_lookup_elem, |
| 2176 | .map_lookup_and_delete_elem = htab_percpu_map_lookup_and_delete_elem, | ||
| 2080 | .map_update_elem = htab_percpu_map_update_elem, | 2177 | .map_update_elem = htab_percpu_map_update_elem, |
| 2081 | .map_delete_elem = htab_map_delete_elem, | 2178 | .map_delete_elem = htab_map_delete_elem, |
| 2082 | .map_seq_show_elem = htab_percpu_map_seq_show_elem, | 2179 | .map_seq_show_elem = htab_percpu_map_seq_show_elem, |
| @@ -2096,6 +2193,7 @@ const struct bpf_map_ops htab_lru_percpu_map_ops = { · 13 unmodified lines | |||
| 2096 | .map_free = htab_map_free, | 2193 | .map_free = htab_map_free, |
| 2097 | .map_get_next_key = htab_map_get_next_key, | 2194 | .map_get_next_key = htab_map_get_next_key, |
| 2098 | .map_lookup_elem = htab_lru_percpu_map_lookup_elem, | 2195 | .map_lookup_elem = htab_lru_percpu_map_lookup_elem, |
| 2196 | .map_lookup_and_delete_elem = htab_lru_percpu_map_lookup_and_delete_elem, | ||
| 2099 | .map_update_elem = htab_lru_percpu_map_update_elem, | 2197 | .map_update_elem = htab_lru_percpu_map_update_elem, |
| 2100 | .map_delete_elem = htab_lru_map_delete_elem, | 2198 | .map_delete_elem = htab_lru_map_delete_elem, |
| 2101 | .map_seq_show_elem = htab_percpu_map_seq_show_elem, | 2199 | .map_seq_show_elem = htab_percpu_map_seq_show_elem, |
| @@ -1483,7 +1483,7 @@ free_buf: · 1482 unmodified lines | |||
| 1483 | return err; | 1483 | return err; |
| 1484 | } | 1484 | } |
| 1485 | 1485 | ||
| 1486 | #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value | 1486 | #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags |
| 1487 | 1487 | ||
| 1488 | static int map_lookup_and_delete_elem(union bpf_attr *attr) | 1488 | static int map_lookup_and_delete_elem(union bpf_attr *attr) |
| 1489 | { | 1489 | { |
| @@ -1499,6 +1499,9 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) · 9 unmodified lines | |||
| 1499 | if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) | 1499 | if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) |
| 1500 | return -EINVAL; | 1500 | return -EINVAL; |
| 1501 | 1501 | ||
| 1502 | if (attr->flags & ~BPF_F_LOCK) | ||
| 1503 | return -EINVAL; | ||
| 1504 | |||
| 1502 | f = fdget(ufd); | 1505 | f = fdget(ufd); |
| 1503 | map = __bpf_map_get(f); | 1506 | map = __bpf_map_get(f); |
| 1504 | if (IS_ERR(map)) | 1507 | if (IS_ERR(map)) |
| @@ -1509,24 +1512,47 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) · 4 unmodified lines | |||
| 1509 | goto err_put; | 1512 | goto err_put; |
| 1510 | } | 1513 | } |
| 1511 | 1514 | ||
| 1515 | if (attr->flags && | ||
| 1516 | (map->map_type == BPF_MAP_TYPE_QUEUE || | ||
| 1517 | map->map_type == BPF_MAP_TYPE_STACK)) { | ||
| 1518 | err = -EINVAL; | ||
| 1519 | goto err_put; | ||
| 1520 | } | ||
| 1521 | |||
| 1522 | if ((attr->flags & BPF_F_LOCK) && | ||
| 1523 | !map_value_has_spin_lock(map)) { | ||
| 1524 | err = -EINVAL; | ||
| 1525 | goto err_put; | ||
| 1526 | } | ||
| 1527 | |||
| 1512 | key = __bpf_copy_key(ukey, map->key_size); | 1528 | key = __bpf_copy_key(ukey, map->key_size); |
| 1513 | if (IS_ERR(key)) { | 1529 | if (IS_ERR(key)) { |
| 1514 | err = PTR_ERR(key); | 1530 | err = PTR_ERR(key); |
| 1515 | goto err_put; | 1531 | goto err_put; |
| 1516 | } | 1532 | } |
| 1517 | 1533 | ||
| 1518 | value_size = map->value_size; | 1534 | value_size = bpf_map_value_size(map); |
| 1519 | 1535 | ||
| 1520 | err = -ENOMEM; | 1536 | err = -ENOMEM; |
| 1521 | value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); | 1537 | value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); |
| 1522 | if (!value) | 1538 | if (!value) |
| 1523 | goto free_key; | 1539 | goto free_key; |
| 1524 | 1540 | ||
| 1541 | err = -ENOTSUPP; | ||
| 1525 | if (map->map_type == BPF_MAP_TYPE_QUEUE || | 1542 | if (map->map_type == BPF_MAP_TYPE_QUEUE || |
| 1526 | map->map_type == BPF_MAP_TYPE_STACK) { | 1543 | map->map_type == BPF_MAP_TYPE_STACK) { |
| 1527 | err = map->ops->map_pop_elem(map, value); | 1544 | err = map->ops->map_pop_elem(map, value); |
| 1528 | } else { | 1545 | } else if (map->map_type == BPF_MAP_TYPE_HASH || |
| 1529 | err = -ENOTSUPP; | 1546 | map->map_type == BPF_MAP_TYPE_PERCPU_HASH || |
| 1547 | map->map_type == BPF_MAP_TYPE_LRU_HASH || | ||
| 1548 | map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { | ||
| 1549 | if (!bpf_map_is_dev_bound(map)) { | ||
| 1550 | bpf_disable_instrumentation(); | ||
| 1551 | rcu_read_lock(); | ||
| 1552 | err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags); | ||
| 1553 | rcu_read_unlock(); | ||
| 1554 | bpf_enable_instrumentation(); | ||
| 1555 | } | ||
| 1530 | } | 1556 | } |
| 1531 | 1557 | ||
| 1532 | if (err) | 1558 | if (err) |