History

netfilter: x_table: speedup compat operations

One iptables invocation with 135000 rules takes 35 seconds of cpu time on a recent server, using a 32bit distro and a 64bit kernel. We eventually trigger NMI/RCU watchdog. INFO: rcu_sched_state detected stall on CPU 3 (t=6000 jiffies) COMPAT mode has quadratic behavior and consume 16 bytes of memory per rule. Switch the xt_compat algos to use an array instead of list, and use a binary search to locate an offset in the sorted array. This halves memory need (8 bytes per rule), and removes quadratic behavior [ O(N*N) -> O(N*log2(N)) ] Time of iptables goes from 35 s to 150 ms.

Eric Dumazet committed 15 years ago

Commit 255d0dc1 parent b0179006 files · +57 −35Browse files

Unified · Split

6 files
  1. include/linux/netfilter/x_tables.h +2 −1
  2. net/bridge/netfilter/ebtables.c +1 −0
  3. net/ipv4/netfilter/arp_tables.c +2 −0
  4. net/ipv4/netfilter/ip_tables.c +2 −0
  5. net/ipv6/netfilter/ip6_tables.c +2 −0
  6. net/netfilter/x_tables.c +48 −34
include/linux/netfilter/x_tables.h+2−1
@@ -611,8 +611,9 @@ struct _compat_xt_align { · 610 unmodified lines
611611 extern void xt_compat_lock(u_int8_t af);
612612 extern void xt_compat_unlock(u_int8_t af);
613613
614−extern int xt_compat_add_offset(u_int8_t af, unsigned int offset, short delta);
614+extern int xt_compat_add_offset(u_int8_t af, unsigned int offset, int delta);
615615 extern void xt_compat_flush_offsets(u_int8_t af);
616+extern void xt_compat_init_offsets(u_int8_t af, unsigned int number);
616617 extern int xt_compat_calc_jump(u_int8_t af, unsigned int offset);
617618
618619 extern int xt_compat_match_offset(const struct xt_match *match);
net/bridge/netfilter/ebtables.c+1−0
@@ -1764,6 +1764,7 @@ static int compat_table_info(const struct ebt_table_info *info, · 1763 unmodified lines
17641764
17651765 newinfo->entries_size = size;
17661766
1767+ xt_compat_init_offsets(AF_INET, info->nentries);
17671768 return EBT_ENTRY_ITERATE(entries, size, compat_calc_entry, info,
17681769 entries, newinfo);
17691770 }
net/ipv4/netfilter/arp_tables.c+2−0
@@ -883,6 +883,7 @@ static int compat_table_info(const struct xt_table_info *info, · 882 unmodified lines
883883 memcpy(newinfo, info, offsetof(struct xt_table_info, entries));
884884 newinfo->initial_entries = 0;
885885 loc_cpu_entry = info->entries[raw_smp_processor_id()];
886+ xt_compat_init_offsets(NFPROTO_ARP, info->number);
886887 xt_entry_foreach(iter, loc_cpu_entry, info->size) {
887888 ret = compat_calc_entry(iter, info, loc_cpu_entry, newinfo);
888889 if (ret != 0)
@@ -1350,6 +1351,7 @@ static int translate_compat_table(const char *name, · 461 unmodified lines
13501351 duprintf("translate_compat_table: size %u\n", info->size);
13511352 j = 0;
13521353 xt_compat_lock(NFPROTO_ARP);
1354+ xt_compat_init_offsets(NFPROTO_ARP, number);
13531355 /* Walk through entries, checking offsets. */
13541356 xt_entry_foreach(iter0, entry0, total_size) {
13551357 ret = check_compat_entry_size_and_hooks(iter0, info, &size,
net/ipv4/netfilter/ip_tables.c+2−0
@@ -1080,6 +1080,7 @@ static int compat_table_info(const struct xt_table_info *info, · 1079 unmodified lines
10801080 memcpy(newinfo, info, offsetof(struct xt_table_info, entries));
10811081 newinfo->initial_entries = 0;
10821082 loc_cpu_entry = info->entries[raw_smp_processor_id()];
1083+ xt_compat_init_offsets(AF_INET, info->number);
10831084 xt_entry_foreach(iter, loc_cpu_entry, info->size) {
10841085 ret = compat_calc_entry(iter, info, loc_cpu_entry, newinfo);
10851086 if (ret != 0)
@@ -1681,6 +1682,7 @@ translate_compat_table(struct net *net, · 595 unmodified lines
16811682 duprintf("translate_compat_table: size %u\n", info->size);
16821683 j = 0;
16831684 xt_compat_lock(AF_INET);
1685+ xt_compat_init_offsets(AF_INET, number);
16841686 /* Walk through entries, checking offsets. */
16851687 xt_entry_foreach(iter0, entry0, total_size) {
16861688 ret = check_compat_entry_size_and_hooks(iter0, info, &size,
net/ipv6/netfilter/ip6_tables.c+2−0
@@ -1093,6 +1093,7 @@ static int compat_table_info(const struct xt_table_info *info, · 1092 unmodified lines
10931093 memcpy(newinfo, info, offsetof(struct xt_table_info, entries));
10941094 newinfo->initial_entries = 0;
10951095 loc_cpu_entry = info->entries[raw_smp_processor_id()];
1096+ xt_compat_init_offsets(AF_INET6, info->number);
10961097 xt_entry_foreach(iter, loc_cpu_entry, info->size) {
10971098 ret = compat_calc_entry(iter, info, loc_cpu_entry, newinfo);
10981099 if (ret != 0)
@@ -1696,6 +1697,7 @@ translate_compat_table(struct net *net, · 597 unmodified lines
16961697 duprintf("translate_compat_table: size %u\n", info->size);
16971698 j = 0;
16981699 xt_compat_lock(AF_INET6);
1700+ xt_compat_init_offsets(AF_INET6, number);
16991701 /* Walk through entries, checking offsets. */
17001702 xt_entry_foreach(iter0, entry0, total_size) {
17011703 ret = check_compat_entry_size_and_hooks(iter0, info, &size,
net/netfilter/x_tables.c+48−34
@@ -38,9 +38,8 @@ MODULE_DESCRIPTION("{ip,ip6,arp,eb}_tables backend module"); · 37 unmodified lines
3838 #define SMP_ALIGN(x) (((x) + SMP_CACHE_BYTES-1) & ~(SMP_CACHE_BYTES-1))
3939
4040 struct compat_delta {
41− struct compat_delta *next;
42− unsigned int offset;
43− int delta;
41+ unsigned int offset; /* offset in kernel */
42+ int delta; /* delta in 32bit user land */
4443 };
4544
4645 struct xt_af {
@@ -49,7 +48,9 @@ struct xt_af { · 2 unmodified lines
4948 struct list_head target;
5049 #ifdef CONFIG_COMPAT
5150 struct mutex compat_mutex;
52− struct compat_delta *compat_offsets;
51+ struct compat_delta *compat_tab;
52+ unsigned int number; /* number of slots in compat_tab[] */
53+ unsigned int cur; /* number of used slots in compat_tab[] */
5354 #endif
5455 };
5556
@@ -414,54 +415,67 @@ int xt_check_match(struct xt_mtchk_param *par, · 358 unmodified lines
414415 EXPORT_SYMBOL_GPL(xt_check_match);
415416
416417 #ifdef CONFIG_COMPAT
417−int xt_compat_add_offset(u_int8_t af, unsigned int offset, short delta)
418+int xt_compat_add_offset(u_int8_t af, unsigned int offset, int delta)
418419 {
419− struct compat_delta *tmp;
420+ struct xt_af *xp = &xt[af];
420421
421− tmp = kmalloc(sizeof(struct compat_delta), GFP_KERNEL);
422− if (!tmp)
423− return -ENOMEM;
422+ if (!xp->compat_tab) {
423+ if (!xp->number)
424+ return -EINVAL;
425+ xp->compat_tab = vmalloc(sizeof(struct compat_delta) * xp->number);
426+ if (!xp->compat_tab)
427+ return -ENOMEM;
428+ xp->cur = 0;
429+ }
424430
425− tmp->offset = offset;
426− tmp->delta = delta;
431+ if (xp->cur >= xp->number)
432+ return -EINVAL;
427433
428− if (xt[af].compat_offsets) {
429− tmp->next = xt[af].compat_offsets->next;
430− xt[af].compat_offsets->next = tmp;
431− } else {
432− xt[af].compat_offsets = tmp;
433− tmp->next = NULL;
434− }
434+ if (xp->cur)
435+ delta += xp->compat_tab[xp->cur - 1].delta;
436+ xp->compat_tab[xp->cur].offset = offset;
437+ xp->compat_tab[xp->cur].delta = delta;
438+ xp->cur++;
435439 return 0;
436440 }
437441 EXPORT_SYMBOL_GPL(xt_compat_add_offset);
438442
439443 void xt_compat_flush_offsets(u_int8_t af)
440444 {
441− struct compat_delta *tmp, *next;
442−
443− if (xt[af].compat_offsets) {
444− for (tmp = xt[af].compat_offsets; tmp; tmp = next) {
445− next = tmp->next;
446− kfree(tmp);
447− }
448− xt[af].compat_offsets = NULL;
445+ if (xt[af].compat_tab) {
446+ vfree(xt[af].compat_tab);
447+ xt[af].compat_tab = NULL;
448+ xt[af].number = 0;
449449 }
450450 }
451451 EXPORT_SYMBOL_GPL(xt_compat_flush_offsets);
452452
453453 int xt_compat_calc_jump(u_int8_t af, unsigned int offset)
454454 {
455− struct compat_delta *tmp;
456− int delta;
457−
458− for (tmp = xt[af].compat_offsets, delta = 0; tmp; tmp = tmp->next)
459− if (tmp->offset < offset)
460− delta += tmp->delta;
461− return delta;
455+ struct compat_delta *tmp = xt[af].compat_tab;
456+ int mid, left = 0, right = xt[af].cur - 1;
457+
458+ while (left <= right) {
459+ mid = (left + right) >> 1;
460+ if (offset > tmp[mid].offset)
461+ left = mid + 1;
462+ else if (offset < tmp[mid].offset)
463+ right = mid - 1;
464+ else
465+ return mid ? tmp[mid - 1].delta : 0;
466+ }
467+ WARN_ON_ONCE(1);
468+ return 0;
462469 }
463470 EXPORT_SYMBOL_GPL(xt_compat_calc_jump);
464471
472+void xt_compat_init_offsets(u_int8_t af, unsigned int number)
473+{
474+ xt[af].number = number;
475+ xt[af].cur = 0;
476+}
477+EXPORT_SYMBOL(xt_compat_init_offsets);
478+
465479 int xt_compat_match_offset(const struct xt_match *match)
466480 {
467481 u_int16_t csize = match->compatsize ? : match->matchsize;
@@ -1337,7 +1351,7 @@ static int __init xt_init(void) · 869 unmodified lines
13371351 mutex_init(&xt[i].mutex);
13381352 #ifdef CONFIG_COMPAT
13391353 mutex_init(&xt[i].compat_mutex);
1340− xt[i].compat_offsets = NULL;
1354+ xt[i].compat_tab = NULL;
13411355 #endif
13421356 INIT_LIST_HEAD(&xt[i].target);
13431357 INIT_LIST_HEAD(&xt[i].match);