commit f3c5c1bfd4 (netfilter: xtables: make ip_tables reentrant) introduced a performance regression, because stackptr array is shared by all cpus, adding cache line ping pongs. (16 cpus share a 64 bytes cache line) Fix this using alloc_percpu()
Unified · Split
| @@ -397,7 +397,7 @@ struct xt_table_info { · 396 unmodified lines | |||
| 397 | * @stacksize jumps (number of user chains) can possibly be made. | 397 | * @stacksize jumps (number of user chains) can possibly be made. |
| 398 | */ | 398 | */ |
| 399 | unsigned int stacksize; | 399 | unsigned int stacksize; |
| 400 | unsigned int *stackptr; | 400 | unsigned int __percpu *stackptr; |
| 401 | void ***jumpstack; | 401 | void ***jumpstack; |
| 402 | /* ipt_entry tables: one per CPU */ | 402 | /* ipt_entry tables: one per CPU */ |
| 403 | /* Note : this field MUST be the last one, see XT_TABLE_INFO_SZ */ | 403 | /* Note : this field MUST be the last one, see XT_TABLE_INFO_SZ */ |
| @@ -336,7 +336,7 @@ ipt_do_table(struct sk_buff *skb, · 335 unmodified lines | |||
| 336 | cpu = smp_processor_id(); | 336 | cpu = smp_processor_id(); |
| 337 | table_base = private->entries[cpu]; | 337 | table_base = private->entries[cpu]; |
| 338 | jumpstack = (struct ipt_entry **)private->jumpstack[cpu]; | 338 | jumpstack = (struct ipt_entry **)private->jumpstack[cpu]; |
| 339 | stackptr = &private->stackptr[cpu]; | 339 | stackptr = per_cpu_ptr(private->stackptr, cpu); |
| 340 | origptr = *stackptr; | 340 | origptr = *stackptr; |
| 341 | 341 | ||
| 342 | e = get_entry(table_base, private->hook_entry[hook]); | 342 | e = get_entry(table_base, private->hook_entry[hook]); |
| @@ -363,7 +363,7 @@ ip6t_do_table(struct sk_buff *skb, · 362 unmodified lines | |||
| 363 | cpu = smp_processor_id(); | 363 | cpu = smp_processor_id(); |
| 364 | table_base = private->entries[cpu]; | 364 | table_base = private->entries[cpu]; |
| 365 | jumpstack = (struct ip6t_entry **)private->jumpstack[cpu]; | 365 | jumpstack = (struct ip6t_entry **)private->jumpstack[cpu]; |
| 366 | stackptr = &private->stackptr[cpu]; | 366 | stackptr = per_cpu_ptr(private->stackptr, cpu); |
| 367 | origptr = *stackptr; | 367 | origptr = *stackptr; |
| 368 | 368 | ||
| 369 | e = get_entry(table_base, private->hook_entry[hook]); | 369 | e = get_entry(table_base, private->hook_entry[hook]); |
| @@ -699,10 +699,8 @@ void xt_free_table_info(struct xt_table_info *info) · 698 unmodified lines | |||
| 699 | vfree(info->jumpstack); | 699 | vfree(info->jumpstack); |
| 700 | else | 700 | else |
| 701 | kfree(info->jumpstack); | 701 | kfree(info->jumpstack); |
| 702 | if (sizeof(unsigned int) * nr_cpu_ids > PAGE_SIZE) | 702 | |
| 703 | vfree(info->stackptr); | 703 | free_percpu(info->stackptr); |
| 704 | else | ||
| 705 | kfree(info->stackptr); | ||
| 706 | 704 | ||
| 707 | kfree(info); | 705 | kfree(info); |
| 708 | } | 706 | } |
| @@ -753,14 +751,9 @@ static int xt_jumpstack_alloc(struct xt_table_info *i) · 44 unmodified lines | |||
| 753 | unsigned int size; | 751 | unsigned int size; |
| 754 | int cpu; | 752 | int cpu; |
| 755 | 753 | ||
| 756 | size = sizeof(unsigned int) * nr_cpu_ids; | 754 | i->stackptr = alloc_percpu(unsigned int); |
| 757 | if (size > PAGE_SIZE) | ||
| 758 | i->stackptr = vmalloc(size); | ||
| 759 | else | ||
| 760 | i->stackptr = kmalloc(size, GFP_KERNEL); | ||
| 761 | if (i->stackptr == NULL) | 755 | if (i->stackptr == NULL) |
| 762 | return -ENOMEM; | 756 | return -ENOMEM; |
| 763 | memset(i->stackptr, 0, size); | ||
| 764 | 757 | ||
| 765 | size = sizeof(void **) * nr_cpu_ids; | 758 | size = sizeof(void **) * nr_cpu_ids; |
| 766 | if (size > PAGE_SIZE) | 759 | if (size > PAGE_SIZE) |