For double-checked locking in bpf_common_lru_push_free(), node->type is read outside the critical section and then re-checked under the lock. However, concurrent writes to node->type result in data races. For example, the following concurrent access was observed by KCSAN: write to 0xffff88801521bc22 of 1 bytes by task 10038 on cpu 1: __bpf_lru_node_move_in kernel/bpf/bpf_lru_list.c:91 __local_list_flush kernel/bpf/bpf_lru_list.c:298 ... read to 0xffff88801521bc22 of 1 bytes by task 10043 on cpu 0: bpf_common_lru_push_free kernel/bpf/bpf_lru_list.c:507 bpf_lru_push_free kernel/bpf/bpf_lru_list.c:555 ... Fix the data races where node->type is read outside the critical section (for double-checked locking) by marking the access with READ_ONCE() as well as ensuring the variable is only accessed once.
Unified · Split
| @@ -502,13 +502,14 @@ struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash) · 501 unmodified lines | |||
| 502 | static void bpf_common_lru_push_free(struct bpf_lru *lru, | 502 | static void bpf_common_lru_push_free(struct bpf_lru *lru, |
| 503 | struct bpf_lru_node *node) | 503 | struct bpf_lru_node *node) |
| 504 | { | 504 | { |
| 505 | u8 node_type = READ_ONCE(node->type); | ||
| 505 | unsigned long flags; | 506 | unsigned long flags; |
| 506 | 507 | ||
| 507 | if (WARN_ON_ONCE(node->type == BPF_LRU_LIST_T_FREE) || | 508 | if (WARN_ON_ONCE(node_type == BPF_LRU_LIST_T_FREE) || |
| 508 | WARN_ON_ONCE(node->type == BPF_LRU_LOCAL_LIST_T_FREE)) | 509 | WARN_ON_ONCE(node_type == BPF_LRU_LOCAL_LIST_T_FREE)) |
| 509 | return; | 510 | return; |
| 510 | 511 | ||
| 511 | if (node->type == BPF_LRU_LOCAL_LIST_T_PENDING) { | 512 | if (node_type == BPF_LRU_LOCAL_LIST_T_PENDING) { |
| 512 | struct bpf_lru_locallist *loc_l; | 513 | struct bpf_lru_locallist *loc_l; |
| 513 | 514 | ||
| 514 | loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu); | 515 | loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu); |