Simplify the logic that selects bpf_prog_pack_size, and always use (PMD_SIZE * num_possible_nodes()). This is a good tradeoff, as most of the performance benefit observed is from less direct map fragmentation [0]. Also, module_alloc(4MB) may not allocate 4MB aligned memory. Therefore, we cannot use (ptr & bpf_prog_pack_mask) to find the correct address of bpf_prog_pack. Fix this by checking the header address falls in the range of pack->ptr and (pack->ptr + bpf_prog_pack_size). [0] https://lore.kernel.org/bpf/20220707223546.4124919-1-song@kernel.org/
Unified · Split
| @@ -825,15 +825,6 @@ struct bpf_prog_pack { · 824 unmodified lines | |||
| 825 | 825 | ||
| 826 | #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) | 826 | #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) |
| 827 | 827 | ||
| 828 | static size_t bpf_prog_pack_size = -1; | ||
| 829 | static size_t bpf_prog_pack_mask = -1; | ||
| 830 | |||
| 831 | static int bpf_prog_chunk_count(void) | ||
| 832 | { | ||
| 833 | WARN_ON_ONCE(bpf_prog_pack_size == -1); | ||
| 834 | return bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE; | ||
| 835 | } | ||
| 836 | |||
| 837 | static DEFINE_MUTEX(pack_mutex); | 828 | static DEFINE_MUTEX(pack_mutex); |
| 838 | static LIST_HEAD(pack_list); | 829 | static LIST_HEAD(pack_list); |
| 839 | 830 | ||
| @@ -841,55 +832,33 @@ static LIST_HEAD(pack_list); · 1 unmodified line | |||
| 841 | * CONFIG_MMU=n. Use PAGE_SIZE in these cases. | 832 | * CONFIG_MMU=n. Use PAGE_SIZE in these cases. |
| 842 | */ | 833 | */ |
| 843 | #ifdef PMD_SIZE | 834 | #ifdef PMD_SIZE |
| 844 | #define BPF_HPAGE_SIZE PMD_SIZE | 835 | #define BPF_PROG_PACK_SIZE (PMD_SIZE * num_possible_nodes()) |
| 845 | #define BPF_HPAGE_MASK PMD_MASK | ||
| 846 | #else | 836 | #else |
| 847 | #define BPF_HPAGE_SIZE PAGE_SIZE | 837 | #define BPF_PROG_PACK_SIZE PAGE_SIZE |
| 848 | #define BPF_HPAGE_MASK PAGE_MASK | ||
| 849 | #endif | 838 | #endif |
| 850 | 839 | ||
| 851 | static size_t select_bpf_prog_pack_size(void) | 840 | #define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE) |
| 852 | { | ||
| 853 | size_t size; | ||
| 854 | void *ptr; | ||
| 855 | |||
| 856 | size = BPF_HPAGE_SIZE * num_online_nodes(); | ||
| 857 | ptr = module_alloc(size); | ||
| 858 | |||
| 859 | /* Test whether we can get huge pages. If not just use PAGE_SIZE | ||
| 860 | * packs. | ||
| 861 | */ | ||
| 862 | if (!ptr || !is_vm_area_hugepages(ptr)) { | ||
| 863 | size = PAGE_SIZE; | ||
| 864 | bpf_prog_pack_mask = PAGE_MASK; | ||
| 865 | } else { | ||
| 866 | bpf_prog_pack_mask = BPF_HPAGE_MASK; | ||
| 867 | } | ||
| 868 | |||
| 869 | vfree(ptr); | ||
| 870 | return size; | ||
| 871 | } | ||
| 872 | 841 | ||
| 873 | static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns) | 842 | static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns) |
| 874 | { | 843 | { |
| 875 | struct bpf_prog_pack *pack; | 844 | struct bpf_prog_pack *pack; |
| 876 | 845 | ||
| 877 | pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(bpf_prog_chunk_count())), | 846 | pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)), |
| 878 | GFP_KERNEL); | 847 | GFP_KERNEL); |
| 879 | if (!pack) | 848 | if (!pack) |
| 880 | return NULL; | 849 | return NULL; |
| 881 | pack->ptr = module_alloc(bpf_prog_pack_size); | 850 | pack->ptr = module_alloc(BPF_PROG_PACK_SIZE); |
| 882 | if (!pack->ptr) { | 851 | if (!pack->ptr) { |
| 883 | kfree(pack); | 852 | kfree(pack); |
| 884 | return NULL; | 853 | return NULL; |
| 885 | } | 854 | } |
| 886 | bpf_fill_ill_insns(pack->ptr, bpf_prog_pack_size); | 855 | bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); |
| 887 | bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE); | 856 | bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); |
| 888 | list_add_tail(&pack->list, &pack_list); | 857 | list_add_tail(&pack->list, &pack_list); |
| 889 | 858 | ||
| 890 | set_vm_flush_reset_perms(pack->ptr); | 859 | set_vm_flush_reset_perms(pack->ptr); |
| 891 | set_memory_ro((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); | 860 | set_memory_ro((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); |
| 892 | set_memory_x((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); | 861 | set_memory_x((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); |
| 893 | return pack; | 862 | return pack; |
| 894 | } | 863 | } |
| 895 | 864 | ||
| @@ -901,10 +870,7 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn · 5 unmodified lines | |||
| 901 | void *ptr = NULL; | 870 | void *ptr = NULL; |
| 902 | 871 | ||
| 903 | mutex_lock(&pack_mutex); | 872 | mutex_lock(&pack_mutex); |
| 904 | if (bpf_prog_pack_size == -1) | 873 | if (size > BPF_PROG_PACK_SIZE) { |
| 905 | bpf_prog_pack_size = select_bpf_prog_pack_size(); | ||
| 906 | |||
| 907 | if (size > bpf_prog_pack_size) { | ||
| 908 | size = round_up(size, PAGE_SIZE); | 874 | size = round_up(size, PAGE_SIZE); |
| 909 | ptr = module_alloc(size); | 875 | ptr = module_alloc(size); |
| 910 | if (ptr) { | 876 | if (ptr) { |
| @@ -916,9 +882,9 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn · 5 unmodified lines | |||
| 916 | goto out; | 882 | goto out; |
| 917 | } | 883 | } |
| 918 | list_for_each_entry(pack, &pack_list, list) { | 884 | list_for_each_entry(pack, &pack_list, list) { |
| 919 | pos = bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, | 885 | pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, |
| 920 | nbits, 0); | 886 | nbits, 0); |
| 921 | if (pos < bpf_prog_chunk_count()) | 887 | if (pos < BPF_PROG_CHUNK_COUNT) |
| 922 | goto found_free_area; | 888 | goto found_free_area; |
| 923 | } | 889 | } |
| 924 | 890 | ||
| @@ -942,18 +908,15 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 17 unmodified lines | |||
| 942 | struct bpf_prog_pack *pack = NULL, *tmp; | 908 | struct bpf_prog_pack *pack = NULL, *tmp; |
| 943 | unsigned int nbits; | 909 | unsigned int nbits; |
| 944 | unsigned long pos; | 910 | unsigned long pos; |
| 945 | void *pack_ptr; | ||
| 946 | 911 | ||
| 947 | mutex_lock(&pack_mutex); | 912 | mutex_lock(&pack_mutex); |
| 948 | if (hdr->size > bpf_prog_pack_size) { | 913 | if (hdr->size > BPF_PROG_PACK_SIZE) { |
| 949 | module_memfree(hdr); | 914 | module_memfree(hdr); |
| 950 | goto out; | 915 | goto out; |
| 951 | } | 916 | } |
| 952 | 917 | ||
| 953 | pack_ptr = (void *)((unsigned long)hdr & bpf_prog_pack_mask); | ||
| 954 | |||
| 955 | list_for_each_entry(tmp, &pack_list, list) { | 918 | list_for_each_entry(tmp, &pack_list, list) { |
| 956 | if (tmp->ptr == pack_ptr) { | 919 | if ((void *)hdr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > (void *)hdr) { |
| 957 | pack = tmp; | 920 | pack = tmp; |
| 958 | break; | 921 | break; |
| 959 | } | 922 | } |
| @@ -963,14 +926,14 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 3 unmodified lines | |||
| 963 | goto out; | 926 | goto out; |
| 964 | 927 | ||
| 965 | nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size); | 928 | nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size); |
| 966 | pos = ((unsigned long)hdr - (unsigned long)pack_ptr) >> BPF_PROG_CHUNK_SHIFT; | 929 | pos = ((unsigned long)hdr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT; |
| 967 | 930 | ||
| 968 | WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size), | 931 | WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size), |
| 969 | "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); | 932 | "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); |
| 970 | 933 | ||
| 971 | bitmap_clear(pack->bitmap, pos, nbits); | 934 | bitmap_clear(pack->bitmap, pos, nbits); |
| 972 | if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, | 935 | if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, |
| 973 | bpf_prog_chunk_count(), 0) == 0) { | 936 | BPF_PROG_CHUNK_COUNT, 0) == 0) { |
| 974 | list_del(&pack->list); | 937 | list_del(&pack->list); |
| 975 | module_memfree(pack->ptr); | 938 | module_memfree(pack->ptr); |
| 976 | kfree(pack); | 939 | kfree(pack); |