History

bpf: Simplify bpf_prog_pack_[size|mask]

Simplify the logic that selects bpf_prog_pack_size, and always use (PMD_SIZE * num_possible_nodes()). This is a good tradeoff, as most of the performance benefit observed is from less direct map fragmentation [0]. Also, module_alloc(4MB) may not allocate 4MB aligned memory. Therefore, we cannot use (ptr & bpf_prog_pack_mask) to find the correct address of bpf_prog_pack. Fix this by checking the header address falls in the range of pack->ptr and (pack->ptr + bpf_prog_pack_size). [0] https://lore.kernel.org/bpf/20220707223546.4124919-1-song@kernel.org/

Song Liu committed 4 years ago

Commit ea2baba1 parent 00963a21 file · +17 −54Browse files

Unified · Split

kernel/bpf/core.c+17−54
@@ -825,15 +825,6 @@ struct bpf_prog_pack { · 824 unmodified lines
825 825
826#define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE)826#define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE)
827 827
828static size_t bpf_prog_pack_size = -1;
829static size_t bpf_prog_pack_mask = -1;
830
831static int bpf_prog_chunk_count(void)
832{
833 WARN_ON_ONCE(bpf_prog_pack_size == -1);
834 return bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE;
835}
836
837static DEFINE_MUTEX(pack_mutex);828static DEFINE_MUTEX(pack_mutex);
838static LIST_HEAD(pack_list);829static LIST_HEAD(pack_list);
839 830
@@ -841,55 +832,33 @@ static LIST_HEAD(pack_list); · 1 unmodified line
841 * CONFIG_MMU=n. Use PAGE_SIZE in these cases.832 * CONFIG_MMU=n. Use PAGE_SIZE in these cases.
842 */833 */
843#ifdef PMD_SIZE834#ifdef PMD_SIZE
844#define BPF_HPAGE_SIZE PMD_SIZE835#define BPF_PROG_PACK_SIZE (PMD_SIZE * num_possible_nodes())
845#define BPF_HPAGE_MASK PMD_MASK
846#else836#else
847#define BPF_HPAGE_SIZE PAGE_SIZE837#define BPF_PROG_PACK_SIZE PAGE_SIZE
848#define BPF_HPAGE_MASK PAGE_MASK
849#endif838#endif
850 839
851static size_t select_bpf_prog_pack_size(void)840#define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE)
852{
853 size_t size;
854 void *ptr;
855
856 size = BPF_HPAGE_SIZE * num_online_nodes();
857 ptr = module_alloc(size);
858
859 /* Test whether we can get huge pages. If not just use PAGE_SIZE
860 * packs.
861 */
862 if (!ptr || !is_vm_area_hugepages(ptr)) {
863 size = PAGE_SIZE;
864 bpf_prog_pack_mask = PAGE_MASK;
865 } else {
866 bpf_prog_pack_mask = BPF_HPAGE_MASK;
867 }
868
869 vfree(ptr);
870 return size;
871}
872 841
873static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns)842static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns)
874{843{
875 struct bpf_prog_pack *pack;844 struct bpf_prog_pack *pack;
876 845
877 pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(bpf_prog_chunk_count())),846 pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)),
878 GFP_KERNEL);847 GFP_KERNEL);
879 if (!pack)848 if (!pack)
880 return NULL;849 return NULL;
881 pack->ptr = module_alloc(bpf_prog_pack_size);850 pack->ptr = module_alloc(BPF_PROG_PACK_SIZE);
882 if (!pack->ptr) {851 if (!pack->ptr) {
883 kfree(pack);852 kfree(pack);
884 return NULL;853 return NULL;
885 }854 }
886 bpf_fill_ill_insns(pack->ptr, bpf_prog_pack_size);855 bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE);
887 bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE);856 bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);
888 list_add_tail(&pack->list, &pack_list);857 list_add_tail(&pack->list, &pack_list);
889 858
890 set_vm_flush_reset_perms(pack->ptr);859 set_vm_flush_reset_perms(pack->ptr);
891 set_memory_ro((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE);860 set_memory_ro((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE);
892 set_memory_x((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE);861 set_memory_x((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE);
893 return pack;862 return pack;
894}863}
895 864
@@ -901,10 +870,7 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn · 5 unmodified lines
901 void *ptr = NULL;870 void *ptr = NULL;
902 871
903 mutex_lock(&pack_mutex);872 mutex_lock(&pack_mutex);
904 if (bpf_prog_pack_size == -1)873 if (size > BPF_PROG_PACK_SIZE) {
905 bpf_prog_pack_size = select_bpf_prog_pack_size();
906
907 if (size > bpf_prog_pack_size) {
908 size = round_up(size, PAGE_SIZE);874 size = round_up(size, PAGE_SIZE);
909 ptr = module_alloc(size);875 ptr = module_alloc(size);
910 if (ptr) {876 if (ptr) {
@@ -916,9 +882,9 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn · 5 unmodified lines
916 goto out;882 goto out;
917 }883 }
918 list_for_each_entry(pack, &pack_list, list) {884 list_for_each_entry(pack, &pack_list, list) {
919 pos = bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0,885 pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
920 nbits, 0);886 nbits, 0);
921 if (pos < bpf_prog_chunk_count())887 if (pos < BPF_PROG_CHUNK_COUNT)
922 goto found_free_area;888 goto found_free_area;
923 }889 }
924 890
@@ -942,18 +908,15 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 17 unmodified lines
942 struct bpf_prog_pack *pack = NULL, *tmp;908 struct bpf_prog_pack *pack = NULL, *tmp;
943 unsigned int nbits;909 unsigned int nbits;
944 unsigned long pos;910 unsigned long pos;
945 void *pack_ptr;
946 911
947 mutex_lock(&pack_mutex);912 mutex_lock(&pack_mutex);
948 if (hdr->size > bpf_prog_pack_size) {913 if (hdr->size > BPF_PROG_PACK_SIZE) {
949 module_memfree(hdr);914 module_memfree(hdr);
950 goto out;915 goto out;
951 }916 }
952 917
953 pack_ptr = (void *)((unsigned long)hdr & bpf_prog_pack_mask);
954
955 list_for_each_entry(tmp, &pack_list, list) {918 list_for_each_entry(tmp, &pack_list, list) {
956 if (tmp->ptr == pack_ptr) {919 if ((void *)hdr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > (void *)hdr) {
957 pack = tmp;920 pack = tmp;
958 break;921 break;
959 }922 }
@@ -963,14 +926,14 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 3 unmodified lines
963 goto out;926 goto out;
964 927
965 nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size);928 nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size);
966 pos = ((unsigned long)hdr - (unsigned long)pack_ptr) >> BPF_PROG_CHUNK_SHIFT;929 pos = ((unsigned long)hdr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT;
967 930
968 WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size),931 WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size),
969 "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n");932 "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n");
970 933
971 bitmap_clear(pack->bitmap, pos, nbits);934 bitmap_clear(pack->bitmap, pos, nbits);
972 if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0,935 if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
973 bpf_prog_chunk_count(), 0) == 0) {936 BPF_PROG_CHUNK_COUNT, 0) == 0) {
974 list_del(&pack->list);937 list_del(&pack->list);
975 module_memfree(pack->ptr);938 module_memfree(pack->ptr);
976 kfree(pack);939 kfree(pack);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t