Simplify the logic that selects bpf_prog_pack_size, and always use (PMD_SIZE * num_possible_nodes()). This is a good tradeoff, as most of the performance benefit observed is from less direct map fragmentation [0]. Also, module_alloc(4MB) may not allocate 4MB aligned memory. Therefore, we cannot use (ptr & bpf_prog_pack_mask) to find the correct address of bpf_prog_pack. Fix this by checking the header address falls in the range of pack->ptr and (pack->ptr + bpf_prog_pack_size). [0] https://lore.kernel.org/bpf/20220707223546.4124919-1-song@kernel.org/
Unified · Split
| @@ -825,15 +825,6 @@ struct bpf_prog_pack { · 824 unmodified lines | ||
| 825 | 825 | |
| 826 | 826 | #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) |
| 827 | 827 | |
| 828 | −static size_t bpf_prog_pack_size = -1; | |
| 829 | −static size_t bpf_prog_pack_mask = -1; | |
| 830 | − | |
| 831 | −static int bpf_prog_chunk_count(void) | |
| 832 | −{ | |
| 833 | − WARN_ON_ONCE(bpf_prog_pack_size == -1); | |
| 834 | − return bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE; | |
| 835 | −} | |
| 836 | − | |
| 837 | 828 | static DEFINE_MUTEX(pack_mutex); |
| 838 | 829 | static LIST_HEAD(pack_list); |
| 839 | 830 | |
| @@ -841,55 +832,33 @@ static LIST_HEAD(pack_list); · 1 unmodified line | ||
| 841 | 832 | * CONFIG_MMU=n. Use PAGE_SIZE in these cases. |
| 842 | 833 | */ |
| 843 | 834 | #ifdef PMD_SIZE |
| 844 | −#define BPF_HPAGE_SIZE PMD_SIZE | |
| 845 | −#define BPF_HPAGE_MASK PMD_MASK | |
| 835 | +#define BPF_PROG_PACK_SIZE (PMD_SIZE * num_possible_nodes()) | |
| 846 | 836 | #else |
| 847 | −#define BPF_HPAGE_SIZE PAGE_SIZE | |
| 848 | −#define BPF_HPAGE_MASK PAGE_MASK | |
| 837 | +#define BPF_PROG_PACK_SIZE PAGE_SIZE | |
| 849 | 838 | #endif |
| 850 | 839 | |
| 851 | −static size_t select_bpf_prog_pack_size(void) | |
| 852 | −{ | |
| 853 | − size_t size; | |
| 854 | − void *ptr; | |
| 855 | − | |
| 856 | − size = BPF_HPAGE_SIZE * num_online_nodes(); | |
| 857 | − ptr = module_alloc(size); | |
| 858 | − | |
| 859 | − /* Test whether we can get huge pages. If not just use PAGE_SIZE | |
| 860 | − * packs. | |
| 861 | − */ | |
| 862 | − if (!ptr || !is_vm_area_hugepages(ptr)) { | |
| 863 | − size = PAGE_SIZE; | |
| 864 | − bpf_prog_pack_mask = PAGE_MASK; | |
| 865 | − } else { | |
| 866 | − bpf_prog_pack_mask = BPF_HPAGE_MASK; | |
| 867 | − } | |
| 868 | − | |
| 869 | − vfree(ptr); | |
| 870 | − return size; | |
| 871 | −} | |
| 840 | +#define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE) | |
| 872 | 841 | |
| 873 | 842 | static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns) |
| 874 | 843 | { |
| 875 | 844 | struct bpf_prog_pack *pack; |
| 876 | 845 | |
| 877 | − pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(bpf_prog_chunk_count())), | |
| 846 | + pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)), | |
| 878 | 847 | GFP_KERNEL); |
| 879 | 848 | if (!pack) |
| 880 | 849 | return NULL; |
| 881 | − pack->ptr = module_alloc(bpf_prog_pack_size); | |
| 850 | + pack->ptr = module_alloc(BPF_PROG_PACK_SIZE); | |
| 882 | 851 | if (!pack->ptr) { |
| 883 | 852 | kfree(pack); |
| 884 | 853 | return NULL; |
| 885 | 854 | } |
| 886 | − bpf_fill_ill_insns(pack->ptr, bpf_prog_pack_size); | |
| 887 | − bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE); | |
| 855 | + bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); | |
| 856 | + bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); | |
| 888 | 857 | list_add_tail(&pack->list, &pack_list); |
| 889 | 858 | |
| 890 | 859 | set_vm_flush_reset_perms(pack->ptr); |
| 891 | − set_memory_ro((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); | |
| 892 | − set_memory_x((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); | |
| 860 | + set_memory_ro((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); | |
| 861 | + set_memory_x((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); | |
| 893 | 862 | return pack; |
| 894 | 863 | } |
| 895 | 864 | |
| @@ -901,10 +870,7 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn · 5 unmodified lines | ||
| 901 | 870 | void *ptr = NULL; |
| 902 | 871 | |
| 903 | 872 | mutex_lock(&pack_mutex); |
| 904 | − if (bpf_prog_pack_size == -1) | |
| 905 | − bpf_prog_pack_size = select_bpf_prog_pack_size(); | |
| 906 | − | |
| 907 | − if (size > bpf_prog_pack_size) { | |
| 873 | + if (size > BPF_PROG_PACK_SIZE) { | |
| 908 | 874 | size = round_up(size, PAGE_SIZE); |
| 909 | 875 | ptr = module_alloc(size); |
| 910 | 876 | if (ptr) { |
| @@ -916,9 +882,9 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn · 5 unmodified lines | ||
| 916 | 882 | goto out; |
| 917 | 883 | } |
| 918 | 884 | list_for_each_entry(pack, &pack_list, list) { |
| 919 | − pos = bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, | |
| 885 | + pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, | |
| 920 | 886 | nbits, 0); |
| 921 | − if (pos < bpf_prog_chunk_count()) | |
| 887 | + if (pos < BPF_PROG_CHUNK_COUNT) | |
| 922 | 888 | goto found_free_area; |
| 923 | 889 | } |
| 924 | 890 | |
| @@ -942,18 +908,15 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 17 unmodified lines | ||
| 942 | 908 | struct bpf_prog_pack *pack = NULL, *tmp; |
| 943 | 909 | unsigned int nbits; |
| 944 | 910 | unsigned long pos; |
| 945 | − void *pack_ptr; | |
| 946 | 911 | |
| 947 | 912 | mutex_lock(&pack_mutex); |
| 948 | − if (hdr->size > bpf_prog_pack_size) { | |
| 913 | + if (hdr->size > BPF_PROG_PACK_SIZE) { | |
| 949 | 914 | module_memfree(hdr); |
| 950 | 915 | goto out; |
| 951 | 916 | } |
| 952 | 917 | |
| 953 | − pack_ptr = (void *)((unsigned long)hdr & bpf_prog_pack_mask); | |
| 954 | − | |
| 955 | 918 | list_for_each_entry(tmp, &pack_list, list) { |
| 956 | − if (tmp->ptr == pack_ptr) { | |
| 919 | + if ((void *)hdr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > (void *)hdr) { | |
| 957 | 920 | pack = tmp; |
| 958 | 921 | break; |
| 959 | 922 | } |
| @@ -963,14 +926,14 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 3 unmodified lines | ||
| 963 | 926 | goto out; |
| 964 | 927 | |
| 965 | 928 | nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size); |
| 966 | − pos = ((unsigned long)hdr - (unsigned long)pack_ptr) >> BPF_PROG_CHUNK_SHIFT; | |
| 929 | + pos = ((unsigned long)hdr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT; | |
| 967 | 930 | |
| 968 | 931 | WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size), |
| 969 | 932 | "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); |
| 970 | 933 | |
| 971 | 934 | bitmap_clear(pack->bitmap, pos, nbits); |
| 972 | − if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, | |
| 973 | − bpf_prog_chunk_count(), 0) == 0) { | |
| 935 | + if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, | |
| 936 | + BPF_PROG_CHUNK_COUNT, 0) == 0) { | |
| 974 | 937 | list_del(&pack->list); |
| 975 | 938 | module_memfree(pack->ptr); |
| 976 | 939 | kfree(pack); |