Extend the existing bpf_map_lookup_and_delete_elem() functionality to hashtab map types, in addition to stacks and queues. Create a new hashtab bpf_map_ops function that does lookup and deletion of the element under the same bucket lock and add the created map_ops to bpf.h.
Showing the changes to kernel (2 of 5 files). Show all
Unified · Split
| @@ -1401,6 +1401,100 @@ static void htab_map_seq_show_elem(struct bpf_map *map, void *key, · 1400 unmodified lines | ||
| 1401 | 1401 | rcu_read_unlock(); |
| 1402 | 1402 | } |
| 1403 | 1403 | |
| 1404 | +static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, | |
| 1405 | + void *value, bool is_lru_map, | |
| 1406 | + bool is_percpu, u64 flags) | |
| 1407 | +{ | |
| 1408 | + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); | |
| 1409 | + struct hlist_nulls_head *head; | |
| 1410 | + unsigned long bflags; | |
| 1411 | + struct htab_elem *l; | |
| 1412 | + u32 hash, key_size; | |
| 1413 | + struct bucket *b; | |
| 1414 | + int ret; | |
| 1415 | + | |
| 1416 | + key_size = map->key_size; | |
| 1417 | + | |
| 1418 | + hash = htab_map_hash(key, key_size, htab->hashrnd); | |
| 1419 | + b = __select_bucket(htab, hash); | |
| 1420 | + head = &b->head; | |
| 1421 | + | |
| 1422 | + ret = htab_lock_bucket(htab, b, hash, &bflags); | |
| 1423 | + if (ret) | |
| 1424 | + return ret; | |
| 1425 | + | |
| 1426 | + l = lookup_elem_raw(head, hash, key, key_size); | |
| 1427 | + if (!l) { | |
| 1428 | + ret = -ENOENT; | |
| 1429 | + } else { | |
| 1430 | + if (is_percpu) { | |
| 1431 | + u32 roundup_value_size = round_up(map->value_size, 8); | |
| 1432 | + void __percpu *pptr; | |
| 1433 | + int off = 0, cpu; | |
| 1434 | + | |
| 1435 | + pptr = htab_elem_get_ptr(l, key_size); | |
| 1436 | + for_each_possible_cpu(cpu) { | |
| 1437 | + bpf_long_memcpy(value + off, | |
| 1438 | + per_cpu_ptr(pptr, cpu), | |
| 1439 | + roundup_value_size); | |
| 1440 | + off += roundup_value_size; | |
| 1441 | + } | |
| 1442 | + } else { | |
| 1443 | + u32 roundup_key_size = round_up(map->key_size, 8); | |
| 1444 | + | |
| 1445 | + if (flags & BPF_F_LOCK) | |
| 1446 | + copy_map_value_locked(map, value, l->key + | |
| 1447 | + roundup_key_size, | |
| 1448 | + true); | |
| 1449 | + else | |
| 1450 | + copy_map_value(map, value, l->key + | |
| 1451 | + roundup_key_size); | |
| 1452 | + check_and_init_map_lock(map, value); | |
| 1453 | + } | |
| 1454 | + | |
| 1455 | + hlist_nulls_del_rcu(&l->hash_node); | |
| 1456 | + if (!is_lru_map) | |
| 1457 | + free_htab_elem(htab, l); | |
| 1458 | + } | |
| 1459 | + | |
| 1460 | + htab_unlock_bucket(htab, b, hash, bflags); | |
| 1461 | + | |
| 1462 | + if (is_lru_map && l) | |
| 1463 | + bpf_lru_push_free(&htab->lru, &l->lru_node); | |
| 1464 | + | |
| 1465 | + return ret; | |
| 1466 | +} | |
| 1467 | + | |
| 1468 | +static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, | |
| 1469 | + void *value, u64 flags) | |
| 1470 | +{ | |
| 1471 | + return __htab_map_lookup_and_delete_elem(map, key, value, false, false, | |
| 1472 | + flags); | |
| 1473 | +} | |
| 1474 | + | |
| 1475 | +static int htab_percpu_map_lookup_and_delete_elem(struct bpf_map *map, | |
| 1476 | + void *key, void *value, | |
| 1477 | + u64 flags) | |
| 1478 | +{ | |
| 1479 | + return __htab_map_lookup_and_delete_elem(map, key, value, false, true, | |
| 1480 | + flags); | |
| 1481 | +} | |
| 1482 | + | |
| 1483 | +static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key, | |
| 1484 | + void *value, u64 flags) | |
| 1485 | +{ | |
| 1486 | + return __htab_map_lookup_and_delete_elem(map, key, value, true, false, | |
| 1487 | + flags); | |
| 1488 | +} | |
| 1489 | + | |
| 1490 | +static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, | |
| 1491 | + void *key, void *value, | |
| 1492 | + u64 flags) | |
| 1493 | +{ | |
| 1494 | + return __htab_map_lookup_and_delete_elem(map, key, value, true, true, | |
| 1495 | + flags); | |
| 1496 | +} | |
| 1497 | + | |
| 1404 | 1498 | static int |
| 1405 | 1499 | __htab_map_lookup_and_delete_batch(struct bpf_map *map, |
| 1406 | 1500 | const union bpf_attr *attr, |
| @@ -1934,6 +2028,7 @@ const struct bpf_map_ops htab_map_ops = { · 527 unmodified lines | ||
| 1934 | 2028 | .map_free = htab_map_free, |
| 1935 | 2029 | .map_get_next_key = htab_map_get_next_key, |
| 1936 | 2030 | .map_lookup_elem = htab_map_lookup_elem, |
| 2031 | + .map_lookup_and_delete_elem = htab_map_lookup_and_delete_elem, | |
| 1937 | 2032 | .map_update_elem = htab_map_update_elem, |
| 1938 | 2033 | .map_delete_elem = htab_map_delete_elem, |
| 1939 | 2034 | .map_gen_lookup = htab_map_gen_lookup, |
| @@ -1954,6 +2049,7 @@ const struct bpf_map_ops htab_lru_map_ops = { · 14 unmodified lines | ||
| 1954 | 2049 | .map_free = htab_map_free, |
| 1955 | 2050 | .map_get_next_key = htab_map_get_next_key, |
| 1956 | 2051 | .map_lookup_elem = htab_lru_map_lookup_elem, |
| 2052 | + .map_lookup_and_delete_elem = htab_lru_map_lookup_and_delete_elem, | |
| 1957 | 2053 | .map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys, |
| 1958 | 2054 | .map_update_elem = htab_lru_map_update_elem, |
| 1959 | 2055 | .map_delete_elem = htab_lru_map_delete_elem, |
| @@ -2077,6 +2173,7 @@ const struct bpf_map_ops htab_percpu_map_ops = { · 117 unmodified lines | ||
| 2077 | 2173 | .map_free = htab_map_free, |
| 2078 | 2174 | .map_get_next_key = htab_map_get_next_key, |
| 2079 | 2175 | .map_lookup_elem = htab_percpu_map_lookup_elem, |
| 2176 | + .map_lookup_and_delete_elem = htab_percpu_map_lookup_and_delete_elem, | |
| 2080 | 2177 | .map_update_elem = htab_percpu_map_update_elem, |
| 2081 | 2178 | .map_delete_elem = htab_map_delete_elem, |
| 2082 | 2179 | .map_seq_show_elem = htab_percpu_map_seq_show_elem, |
| @@ -2096,6 +2193,7 @@ const struct bpf_map_ops htab_lru_percpu_map_ops = { · 13 unmodified lines | ||
| 2096 | 2193 | .map_free = htab_map_free, |
| 2097 | 2194 | .map_get_next_key = htab_map_get_next_key, |
| 2098 | 2195 | .map_lookup_elem = htab_lru_percpu_map_lookup_elem, |
| 2196 | + .map_lookup_and_delete_elem = htab_lru_percpu_map_lookup_and_delete_elem, | |
| 2099 | 2197 | .map_update_elem = htab_lru_percpu_map_update_elem, |
| 2100 | 2198 | .map_delete_elem = htab_lru_map_delete_elem, |
| 2101 | 2199 | .map_seq_show_elem = htab_percpu_map_seq_show_elem, |
| @@ -1483,7 +1483,7 @@ free_buf: · 1482 unmodified lines | ||
| 1483 | 1483 | return err; |
| 1484 | 1484 | } |
| 1485 | 1485 | |
| 1486 | −#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value | |
| 1486 | +#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags | |
| 1487 | 1487 | |
| 1488 | 1488 | static int map_lookup_and_delete_elem(union bpf_attr *attr) |
| 1489 | 1489 | { |
| @@ -1499,6 +1499,9 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) · 9 unmodified lines | ||
| 1499 | 1499 | if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) |
| 1500 | 1500 | return -EINVAL; |
| 1501 | 1501 | |
| 1502 | + if (attr->flags & ~BPF_F_LOCK) | |
| 1503 | + return -EINVAL; | |
| 1504 | + | |
| 1502 | 1505 | f = fdget(ufd); |
| 1503 | 1506 | map = __bpf_map_get(f); |
| 1504 | 1507 | if (IS_ERR(map)) |
| @@ -1509,24 +1512,47 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) · 4 unmodified lines | ||
| 1509 | 1512 | goto err_put; |
| 1510 | 1513 | } |
| 1511 | 1514 | |
| 1515 | + if (attr->flags && | |
| 1516 | + (map->map_type == BPF_MAP_TYPE_QUEUE || | |
| 1517 | + map->map_type == BPF_MAP_TYPE_STACK)) { | |
| 1518 | + err = -EINVAL; | |
| 1519 | + goto err_put; | |
| 1520 | + } | |
| 1521 | + | |
| 1522 | + if ((attr->flags & BPF_F_LOCK) && | |
| 1523 | + !map_value_has_spin_lock(map)) { | |
| 1524 | + err = -EINVAL; | |
| 1525 | + goto err_put; | |
| 1526 | + } | |
| 1527 | + | |
| 1512 | 1528 | key = __bpf_copy_key(ukey, map->key_size); |
| 1513 | 1529 | if (IS_ERR(key)) { |
| 1514 | 1530 | err = PTR_ERR(key); |
| 1515 | 1531 | goto err_put; |
| 1516 | 1532 | } |
| 1517 | 1533 | |
| 1518 | − value_size = map->value_size; | |
| 1534 | + value_size = bpf_map_value_size(map); | |
| 1519 | 1535 | |
| 1520 | 1536 | err = -ENOMEM; |
| 1521 | 1537 | value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); |
| 1522 | 1538 | if (!value) |
| 1523 | 1539 | goto free_key; |
| 1524 | 1540 | |
| 1541 | + err = -ENOTSUPP; | |
| 1525 | 1542 | if (map->map_type == BPF_MAP_TYPE_QUEUE || |
| 1526 | 1543 | map->map_type == BPF_MAP_TYPE_STACK) { |
| 1527 | 1544 | err = map->ops->map_pop_elem(map, value); |
| 1528 | − } else { | |
| 1529 | − err = -ENOTSUPP; | |
| 1545 | + } else if (map->map_type == BPF_MAP_TYPE_HASH || | |
| 1546 | + map->map_type == BPF_MAP_TYPE_PERCPU_HASH || | |
| 1547 | + map->map_type == BPF_MAP_TYPE_LRU_HASH || | |
| 1548 | + map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { | |
| 1549 | + if (!bpf_map_is_dev_bound(map)) { | |
| 1550 | + bpf_disable_instrumentation(); | |
| 1551 | + rcu_read_lock(); | |
| 1552 | + err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags); | |
| 1553 | + rcu_read_unlock(); | |
| 1554 | + bpf_enable_instrumentation(); | |
| 1555 | + } | |
| 1530 | 1556 | } |
| 1531 | 1557 | |
| 1532 | 1558 | if (err) |