commit f3c5c1bfd4 (netfilter: xtables: make ip_tables reentrant) introduced a performance regression, because stackptr array is shared by all cpus, adding cache line ping pongs. (16 cpus share a 64 bytes cache line) Fix this using alloc_percpu()
Unified · Split
| @@ -397,7 +397,7 @@ struct xt_table_info { · 396 unmodified lines | ||
| 397 | 397 | * @stacksize jumps (number of user chains) can possibly be made. |
| 398 | 398 | */ |
| 399 | 399 | unsigned int stacksize; |
| 400 | − unsigned int *stackptr; | |
| 400 | + unsigned int __percpu *stackptr; | |
| 401 | 401 | void ***jumpstack; |
| 402 | 402 | /* ipt_entry tables: one per CPU */ |
| 403 | 403 | /* Note : this field MUST be the last one, see XT_TABLE_INFO_SZ */ |
| @@ -336,7 +336,7 @@ ipt_do_table(struct sk_buff *skb, · 335 unmodified lines | ||
| 336 | 336 | cpu = smp_processor_id(); |
| 337 | 337 | table_base = private->entries[cpu]; |
| 338 | 338 | jumpstack = (struct ipt_entry **)private->jumpstack[cpu]; |
| 339 | − stackptr = &private->stackptr[cpu]; | |
| 339 | + stackptr = per_cpu_ptr(private->stackptr, cpu); | |
| 340 | 340 | origptr = *stackptr; |
| 341 | 341 | |
| 342 | 342 | e = get_entry(table_base, private->hook_entry[hook]); |
| @@ -363,7 +363,7 @@ ip6t_do_table(struct sk_buff *skb, · 362 unmodified lines | ||
| 363 | 363 | cpu = smp_processor_id(); |
| 364 | 364 | table_base = private->entries[cpu]; |
| 365 | 365 | jumpstack = (struct ip6t_entry **)private->jumpstack[cpu]; |
| 366 | − stackptr = &private->stackptr[cpu]; | |
| 366 | + stackptr = per_cpu_ptr(private->stackptr, cpu); | |
| 367 | 367 | origptr = *stackptr; |
| 368 | 368 | |
| 369 | 369 | e = get_entry(table_base, private->hook_entry[hook]); |
| @@ -699,10 +699,8 @@ void xt_free_table_info(struct xt_table_info *info) · 698 unmodified lines | ||
| 699 | 699 | vfree(info->jumpstack); |
| 700 | 700 | else |
| 701 | 701 | kfree(info->jumpstack); |
| 702 | − if (sizeof(unsigned int) * nr_cpu_ids > PAGE_SIZE) | |
| 703 | − vfree(info->stackptr); | |
| 704 | − else | |
| 705 | − kfree(info->stackptr); | |
| 702 | + | |
| 703 | + free_percpu(info->stackptr); | |
| 706 | 704 | |
| 707 | 705 | kfree(info); |
| 708 | 706 | } |
| @@ -753,14 +751,9 @@ static int xt_jumpstack_alloc(struct xt_table_info *i) · 44 unmodified lines | ||
| 753 | 751 | unsigned int size; |
| 754 | 752 | int cpu; |
| 755 | 753 | |
| 756 | − size = sizeof(unsigned int) * nr_cpu_ids; | |
| 757 | − if (size > PAGE_SIZE) | |
| 758 | − i->stackptr = vmalloc(size); | |
| 759 | − else | |
| 760 | − i->stackptr = kmalloc(size, GFP_KERNEL); | |
| 754 | + i->stackptr = alloc_percpu(unsigned int); | |
| 761 | 755 | if (i->stackptr == NULL) |
| 762 | 756 | return -ENOMEM; |
| 763 | − memset(i->stackptr, 0, size); | |
| 764 | 757 | |
| 765 | 758 | size = sizeof(void **) * nr_cpu_ids; |
| 766 | 759 | if (size > PAGE_SIZE) |