History

bpf: Simplify bpf_prog_pack_[size|mask]

Simplify the logic that selects bpf_prog_pack_size, and always use (PMD_SIZE * num_possible_nodes()). This is a good tradeoff, as most of the performance benefit observed is from less direct map fragmentation [0]. Also, module_alloc(4MB) may not allocate 4MB aligned memory. Therefore, we cannot use (ptr & bpf_prog_pack_mask) to find the correct address of bpf_prog_pack. Fix this by checking the header address falls in the range of pack->ptr and (pack->ptr + bpf_prog_pack_size). [0] https://lore.kernel.org/bpf/20220707223546.4124919-1-song@kernel.org/

Song Liu committed 4 years ago

Commit ea2baba1 parent 00963a21 file · +17 −54Browse files

Unified · Split

kernel/bpf/core.c+17−54
@@ -825,15 +825,6 @@ struct bpf_prog_pack { · 824 unmodified lines
825825
826826 #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE)
827827
828−static size_t bpf_prog_pack_size = -1;
829−static size_t bpf_prog_pack_mask = -1;
830−
831−static int bpf_prog_chunk_count(void)
832−{
833− WARN_ON_ONCE(bpf_prog_pack_size == -1);
834− return bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE;
835−}
836−
837828 static DEFINE_MUTEX(pack_mutex);
838829 static LIST_HEAD(pack_list);
839830
@@ -841,55 +832,33 @@ static LIST_HEAD(pack_list); · 1 unmodified line
841832 * CONFIG_MMU=n. Use PAGE_SIZE in these cases.
842833 */
843834 #ifdef PMD_SIZE
844−#define BPF_HPAGE_SIZE PMD_SIZE
845−#define BPF_HPAGE_MASK PMD_MASK
835+#define BPF_PROG_PACK_SIZE (PMD_SIZE * num_possible_nodes())
846836 #else
847−#define BPF_HPAGE_SIZE PAGE_SIZE
848−#define BPF_HPAGE_MASK PAGE_MASK
837+#define BPF_PROG_PACK_SIZE PAGE_SIZE
849838 #endif
850839
851−static size_t select_bpf_prog_pack_size(void)
852−{
853− size_t size;
854− void *ptr;
855−
856− size = BPF_HPAGE_SIZE * num_online_nodes();
857− ptr = module_alloc(size);
858−
859− /* Test whether we can get huge pages. If not just use PAGE_SIZE
860− * packs.
861− */
862− if (!ptr || !is_vm_area_hugepages(ptr)) {
863− size = PAGE_SIZE;
864− bpf_prog_pack_mask = PAGE_MASK;
865− } else {
866− bpf_prog_pack_mask = BPF_HPAGE_MASK;
867− }
868−
869− vfree(ptr);
870− return size;
871−}
840+#define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE)
872841
873842 static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns)
874843 {
875844 struct bpf_prog_pack *pack;
876845
877− pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(bpf_prog_chunk_count())),
846+ pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)),
878847 GFP_KERNEL);
879848 if (!pack)
880849 return NULL;
881− pack->ptr = module_alloc(bpf_prog_pack_size);
850+ pack->ptr = module_alloc(BPF_PROG_PACK_SIZE);
882851 if (!pack->ptr) {
883852 kfree(pack);
884853 return NULL;
885854 }
886− bpf_fill_ill_insns(pack->ptr, bpf_prog_pack_size);
887− bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE);
855+ bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE);
856+ bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);
888857 list_add_tail(&pack->list, &pack_list);
889858
890859 set_vm_flush_reset_perms(pack->ptr);
891− set_memory_ro((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE);
892− set_memory_x((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE);
860+ set_memory_ro((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE);
861+ set_memory_x((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE);
893862 return pack;
894863 }
895864
@@ -901,10 +870,7 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn · 5 unmodified lines
901870 void *ptr = NULL;
902871
903872 mutex_lock(&pack_mutex);
904− if (bpf_prog_pack_size == -1)
905− bpf_prog_pack_size = select_bpf_prog_pack_size();
906−
907− if (size > bpf_prog_pack_size) {
873+ if (size > BPF_PROG_PACK_SIZE) {
908874 size = round_up(size, PAGE_SIZE);
909875 ptr = module_alloc(size);
910876 if (ptr) {
@@ -916,9 +882,9 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn · 5 unmodified lines
916882 goto out;
917883 }
918884 list_for_each_entry(pack, &pack_list, list) {
919− pos = bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0,
885+ pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
920886 nbits, 0);
921− if (pos < bpf_prog_chunk_count())
887+ if (pos < BPF_PROG_CHUNK_COUNT)
922888 goto found_free_area;
923889 }
924890
@@ -942,18 +908,15 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 17 unmodified lines
942908 struct bpf_prog_pack *pack = NULL, *tmp;
943909 unsigned int nbits;
944910 unsigned long pos;
945− void *pack_ptr;
946911
947912 mutex_lock(&pack_mutex);
948− if (hdr->size > bpf_prog_pack_size) {
913+ if (hdr->size > BPF_PROG_PACK_SIZE) {
949914 module_memfree(hdr);
950915 goto out;
951916 }
952917
953− pack_ptr = (void *)((unsigned long)hdr & bpf_prog_pack_mask);
954−
955918 list_for_each_entry(tmp, &pack_list, list) {
956− if (tmp->ptr == pack_ptr) {
919+ if ((void *)hdr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > (void *)hdr) {
957920 pack = tmp;
958921 break;
959922 }
@@ -963,14 +926,14 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 3 unmodified lines
963926 goto out;
964927
965928 nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size);
966− pos = ((unsigned long)hdr - (unsigned long)pack_ptr) >> BPF_PROG_CHUNK_SHIFT;
929+ pos = ((unsigned long)hdr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT;
967930
968931 WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size),
969932 "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n");
970933
971934 bitmap_clear(pack->bitmap, pos, nbits);
972− if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0,
973− bpf_prog_chunk_count(), 0) == 0) {
935+ if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
936+ BPF_PROG_CHUNK_COUNT, 0) == 0) {
974937 list_del(&pack->list);
975938 module_memfree(pack->ptr);
976939 kfree(pack);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t