We store the rule blob per (possible) cpu. Unfortunately this means we can waste lot of memory on big smp machines. ipt_entry structure ('rule head') is 112 byte, so e.g. with maxcpu=64 one single rule eats close to 8k RAM. Since previous patch made counters percpu it appears there is nothing left in the rule blob that needs to be percpu. On my test system (144 possible cpus, 400k dummy rules) this change saves close to 9 Gigabyte of RAM.
Unified · Split
| @@ -224,9 +224,9 @@ struct xt_table_info { · 223 unmodified lines | ||
| 224 | 224 | unsigned int stacksize; |
| 225 | 225 | unsigned int __percpu *stackptr; |
| 226 | 226 | void ***jumpstack; |
| 227 | − /* ipt_entry tables: one per CPU */ | |
| 227 | + | |
| 228 | 228 | /* Note : this field MUST be the last one, see XT_TABLE_INFO_SZ */ |
| 229 | − void *entries[1]; | |
| 229 | + void *entries; | |
| 230 | 230 | }; |
| 231 | 231 | |
| 232 | 232 | #define XT_TABLE_INFO_SZ (offsetof(struct xt_table_info, entries) \ |
| @@ -275,7 +275,7 @@ unsigned int arpt_do_table(struct sk_buff *skb, · 274 unmodified lines | ||
| 275 | 275 | * pointer. |
| 276 | 276 | */ |
| 277 | 277 | smp_read_barrier_depends(); |
| 278 | − table_base = private->entries[smp_processor_id()]; | |
| 278 | + table_base = private->entries; | |
| 279 | 279 | |
| 280 | 280 | e = get_entry(table_base, private->hook_entry[hook]); |
| 281 | 281 | back = get_entry(table_base, private->underflow[hook]); |
| @@ -711,12 +711,6 @@ static int translate_table(struct xt_table_info *newinfo, void *entry0, · 429 unmodified lines | ||
| 711 | 711 | return ret; |
| 712 | 712 | } |
| 713 | 713 | |
| 714 | − /* And one copy for every other CPU */ | |
| 715 | − for_each_possible_cpu(i) { | |
| 716 | − if (newinfo->entries[i] && newinfo->entries[i] != entry0) | |
| 717 | − memcpy(newinfo->entries[i], entry0, newinfo->size); | |
| 718 | − } | |
| 719 | − | |
| 720 | 714 | return ret; |
| 721 | 715 | } |
| 722 | 716 | |
| @@ -731,7 +725,7 @@ static void get_counters(const struct xt_table_info *t, · 8 unmodified lines | ||
| 731 | 725 | seqcount_t *s = &per_cpu(xt_recseq, cpu); |
| 732 | 726 | |
| 733 | 727 | i = 0; |
| 734 | − xt_entry_foreach(iter, t->entries[cpu], t->size) { | |
| 728 | + xt_entry_foreach(iter, t->entries, t->size) { | |
| 735 | 729 | struct xt_counters *tmp; |
| 736 | 730 | u64 bcnt, pcnt; |
| 737 | 731 | unsigned int start; |
| @@ -785,7 +779,7 @@ static int copy_entries_to_user(unsigned int total_size, · 47 unmodified lines | ||
| 785 | 779 | if (IS_ERR(counters)) |
| 786 | 780 | return PTR_ERR(counters); |
| 787 | 781 | |
| 788 | − loc_cpu_entry = private->entries[raw_smp_processor_id()]; | |
| 782 | + loc_cpu_entry = private->entries; | |
| 789 | 783 | /* ... then copy entire thing ... */ |
| 790 | 784 | if (copy_to_user(userptr, loc_cpu_entry, total_size) != 0) { |
| 791 | 785 | ret = -EFAULT; |
| @@ -880,10 +874,10 @@ static int compat_table_info(const struct xt_table_info *info, · 88 unmodified lines | ||
| 880 | 874 | if (!newinfo || !info) |
| 881 | 875 | return -EINVAL; |
| 882 | 876 | |
| 883 | − /* we dont care about newinfo->entries[] */ | |
| 877 | + /* we dont care about newinfo->entries */ | |
| 884 | 878 | memcpy(newinfo, info, offsetof(struct xt_table_info, entries)); |
| 885 | 879 | newinfo->initial_entries = 0; |
| 886 | − loc_cpu_entry = info->entries[raw_smp_processor_id()]; | |
| 880 | + loc_cpu_entry = info->entries; | |
| 887 | 881 | xt_compat_init_offsets(NFPROTO_ARP, info->number); |
| 888 | 882 | xt_entry_foreach(iter, loc_cpu_entry, info->size) { |
| 889 | 883 | ret = compat_calc_entry(iter, info, loc_cpu_entry, newinfo); |
| @@ -1048,7 +1042,7 @@ static int __do_replace(struct net *net, const char *name, · 158 unmodified lines | ||
| 1048 | 1042 | get_counters(oldinfo, counters); |
| 1049 | 1043 | |
| 1050 | 1044 | /* Decrease module usage counts and free resource */ |
| 1051 | − loc_cpu_old_entry = oldinfo->entries[raw_smp_processor_id()]; | |
| 1045 | + loc_cpu_old_entry = oldinfo->entries; | |
| 1052 | 1046 | xt_entry_foreach(iter, loc_cpu_old_entry, oldinfo->size) |
| 1053 | 1047 | cleanup_entry(iter); |
| 1054 | 1048 | |
| @@ -1095,8 +1089,7 @@ static int do_replace(struct net *net, const void __user *user, · 40 unmodified lines | ||
| 1095 | 1089 | if (!newinfo) |
| 1096 | 1090 | return -ENOMEM; |
| 1097 | 1091 | |
| 1098 | − /* choose the copy that is on our node/cpu */ | |
| 1099 | − loc_cpu_entry = newinfo->entries[raw_smp_processor_id()]; | |
| 1092 | + loc_cpu_entry = newinfo->entries; | |
| 1100 | 1093 | if (copy_from_user(loc_cpu_entry, user + sizeof(tmp), |
| 1101 | 1094 | tmp.size) != 0) { |
| 1102 | 1095 | ret = -EFAULT; |
| @@ -1126,7 +1119,7 @@ static int do_replace(struct net *net, const void __user *user, · 23 unmodified lines | ||
| 1126 | 1119 | static int do_add_counters(struct net *net, const void __user *user, |
| 1127 | 1120 | unsigned int len, int compat) |
| 1128 | 1121 | { |
| 1129 | − unsigned int i, curcpu; | |
| 1122 | + unsigned int i; | |
| 1130 | 1123 | struct xt_counters_info tmp; |
| 1131 | 1124 | struct xt_counters *paddc; |
| 1132 | 1125 | unsigned int num_counters; |
| @@ -1136,7 +1129,6 @@ static int do_add_counters(struct net *net, const void __user *user, · 3 unmodified lines | ||
| 1136 | 1129 | struct xt_table *t; |
| 1137 | 1130 | const struct xt_table_info *private; |
| 1138 | 1131 | int ret = 0; |
| 1139 | − void *loc_cpu_entry; | |
| 1140 | 1132 | struct arpt_entry *iter; |
| 1141 | 1133 | unsigned int addend; |
| 1142 | 1134 | #ifdef CONFIG_COMPAT |
| @@ -1192,11 +1184,9 @@ static int do_add_counters(struct net *net, const void __user *user, · 49 unmodified lines | ||
| 1192 | 1184 | } |
| 1193 | 1185 | |
| 1194 | 1186 | i = 0; |
| 1195 | − /* Choose the copy that is on our node */ | |
| 1196 | − curcpu = smp_processor_id(); | |
| 1197 | − loc_cpu_entry = private->entries[curcpu]; | |
| 1187 | + | |
| 1198 | 1188 | addend = xt_write_recseq_begin(); |
| 1199 | − xt_entry_foreach(iter, loc_cpu_entry, private->size) { | |
| 1189 | + xt_entry_foreach(iter, private->entries, private->size) { | |
| 1200 | 1190 | struct xt_counters *tmp; |
| 1201 | 1191 | |
| 1202 | 1192 | tmp = xt_get_this_cpu_counter(&iter->counters); |
| @@ -1410,7 +1400,7 @@ static int translate_compat_table(const char *name, · 207 unmodified lines | ||
| 1410 | 1400 | newinfo->hook_entry[i] = info->hook_entry[i]; |
| 1411 | 1401 | newinfo->underflow[i] = info->underflow[i]; |
| 1412 | 1402 | } |
| 1413 | − entry1 = newinfo->entries[raw_smp_processor_id()]; | |
| 1403 | + entry1 = newinfo->entries; | |
| 1414 | 1404 | pos = entry1; |
| 1415 | 1405 | size = total_size; |
| 1416 | 1406 | xt_entry_foreach(iter0, entry0, total_size) { |
| @@ -1470,11 +1460,6 @@ static int translate_compat_table(const char *name, · 53 unmodified lines | ||
| 1470 | 1460 | return ret; |
| 1471 | 1461 | } |
| 1472 | 1462 | |
| 1473 | − /* And one copy for every other CPU */ | |
| 1474 | − for_each_possible_cpu(i) | |
| 1475 | − if (newinfo->entries[i] && newinfo->entries[i] != entry1) | |
| 1476 | − memcpy(newinfo->entries[i], entry1, newinfo->size); | |
| 1477 | − | |
| 1478 | 1463 | *pinfo = newinfo; |
| 1479 | 1464 | *pentry0 = entry1; |
| 1480 | 1465 | xt_free_table_info(info); |
| @@ -1533,8 +1518,7 @@ static int compat_do_replace(struct net *net, void __user *user, · 52 unmodified lines | ||
| 1533 | 1518 | if (!newinfo) |
| 1534 | 1519 | return -ENOMEM; |
| 1535 | 1520 | |
| 1536 | − /* choose the copy that is on our node/cpu */ | |
| 1537 | − loc_cpu_entry = newinfo->entries[raw_smp_processor_id()]; | |
| 1521 | + loc_cpu_entry = newinfo->entries; | |
| 1538 | 1522 | if (copy_from_user(loc_cpu_entry, user + sizeof(tmp), tmp.size) != 0) { |
| 1539 | 1523 | ret = -EFAULT; |
| 1540 | 1524 | goto free_newinfo; |
| @@ -1631,7 +1615,6 @@ static int compat_copy_entries_to_user(unsigned int total_size, · 90 unmodified lines | ||
| 1631 | 1615 | void __user *pos; |
| 1632 | 1616 | unsigned int size; |
| 1633 | 1617 | int ret = 0; |
| 1634 | − void *loc_cpu_entry; | |
| 1635 | 1618 | unsigned int i = 0; |
| 1636 | 1619 | struct arpt_entry *iter; |
| 1637 | 1620 | |
| @@ -1639,11 +1622,9 @@ static int compat_copy_entries_to_user(unsigned int total_size, · 1 unmodified line | ||
| 1639 | 1622 | if (IS_ERR(counters)) |
| 1640 | 1623 | return PTR_ERR(counters); |
| 1641 | 1624 | |
| 1642 | − /* choose the copy on our node/cpu */ | |
| 1643 | − loc_cpu_entry = private->entries[raw_smp_processor_id()]; | |
| 1644 | 1625 | pos = userptr; |
| 1645 | 1626 | size = total_size; |
| 1646 | − xt_entry_foreach(iter, loc_cpu_entry, total_size) { | |
| 1627 | + xt_entry_foreach(iter, private->entries, total_size) { | |
| 1647 | 1628 | ret = compat_copy_entry_to_user(iter, &pos, |
| 1648 | 1629 | &size, counters, i++); |
| 1649 | 1630 | if (ret != 0) |
| @@ -1812,8 +1793,7 @@ struct xt_table *arpt_register_table(struct net *net, · 162 unmodified lines | ||
| 1812 | 1793 | goto out; |
| 1813 | 1794 | } |
| 1814 | 1795 | |
| 1815 | − /* choose the copy on our node/cpu */ | |
| 1816 | − loc_cpu_entry = newinfo->entries[raw_smp_processor_id()]; | |
| 1796 | + loc_cpu_entry = newinfo->entries; | |
| 1817 | 1797 | memcpy(loc_cpu_entry, repl->entries, repl->size); |
| 1818 | 1798 | |
| 1819 | 1799 | ret = translate_table(newinfo, loc_cpu_entry, repl); |
| @@ -1844,7 +1824,7 @@ void arpt_unregister_table(struct xt_table *table) · 24 unmodified lines | ||
| 1844 | 1824 | private = xt_unregister_table(table); |
| 1845 | 1825 | |
| 1846 | 1826 | /* Decrease module usage counts and free resources */ |
| 1847 | − loc_cpu_entry = private->entries[raw_smp_processor_id()]; | |
| 1827 | + loc_cpu_entry = private->entries; | |
| 1848 | 1828 | xt_entry_foreach(iter, loc_cpu_entry, private->size) |
| 1849 | 1829 | cleanup_entry(iter); |
| 1850 | 1830 | if (private->number > private->initial_entries) |
| @@ -254,15 +254,13 @@ static void trace_packet(const struct sk_buff *skb, · 253 unmodified lines | ||
| 254 | 254 | const struct xt_table_info *private, |
| 255 | 255 | const struct ipt_entry *e) |
| 256 | 256 | { |
| 257 | − const void *table_base; | |
| 258 | 257 | const struct ipt_entry *root; |
| 259 | 258 | const char *hookname, *chainname, *comment; |
| 260 | 259 | const struct ipt_entry *iter; |
| 261 | 260 | unsigned int rulenum = 0; |
| 262 | 261 | struct net *net = dev_net(in ? in : out); |
| 263 | 262 | |
| 264 | − table_base = private->entries[smp_processor_id()]; | |
| 265 | − root = get_entry(table_base, private->hook_entry[hook]); | |
| 263 | + root = get_entry(private->entries, private->hook_entry[hook]); | |
| 266 | 264 | |
| 267 | 265 | hookname = chainname = hooknames[hook]; |
| 268 | 266 | comment = comments[NF_IP_TRACE_COMMENT_RULE]; |
| @@ -331,7 +329,7 @@ ipt_do_table(struct sk_buff *skb, · 62 unmodified lines | ||
| 331 | 329 | * pointer. |
| 332 | 330 | */ |
| 333 | 331 | smp_read_barrier_depends(); |
| 334 | − table_base = private->entries[cpu]; | |
| 332 | + table_base = private->entries; | |
| 335 | 333 | jumpstack = (struct ipt_entry **)private->jumpstack[cpu]; |
| 336 | 334 | stackptr = per_cpu_ptr(private->stackptr, cpu); |
| 337 | 335 | origptr = *stackptr; |
| @@ -877,12 +875,6 @@ translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0, · 539 unmodified lines | ||
| 877 | 875 | return ret; |
| 878 | 876 | } |
| 879 | 877 | |
| 880 | − /* And one copy for every other CPU */ | |
| 881 | − for_each_possible_cpu(i) { | |
| 882 | − if (newinfo->entries[i] && newinfo->entries[i] != entry0) | |
| 883 | − memcpy(newinfo->entries[i], entry0, newinfo->size); | |
| 884 | − } | |
| 885 | − | |
| 886 | 878 | return ret; |
| 887 | 879 | } |
| 888 | 880 | |
| @@ -898,7 +890,7 @@ get_counters(const struct xt_table_info *t, · 9 unmodified lines | ||
| 898 | 890 | seqcount_t *s = &per_cpu(xt_recseq, cpu); |
| 899 | 891 | |
| 900 | 892 | i = 0; |
| 901 | − xt_entry_foreach(iter, t->entries[cpu], t->size) { | |
| 893 | + xt_entry_foreach(iter, t->entries, t->size) { | |
| 902 | 894 | struct xt_counters *tmp; |
| 903 | 895 | u64 bcnt, pcnt; |
| 904 | 896 | unsigned int start; |
| @@ -946,17 +938,13 @@ copy_entries_to_user(unsigned int total_size, · 41 unmodified lines | ||
| 946 | 938 | struct xt_counters *counters; |
| 947 | 939 | const struct xt_table_info *private = table->private; |
| 948 | 940 | int ret = 0; |
| 949 | − const void *loc_cpu_entry; | |
| 941 | + void *loc_cpu_entry; | |
| 950 | 942 | |
| 951 | 943 | counters = alloc_counters(table); |
| 952 | 944 | if (IS_ERR(counters)) |
| 953 | 945 | return PTR_ERR(counters); |
| 954 | 946 | |
| 955 | − /* choose the copy that is on our node/cpu, ... | |
| 956 | − * This choice is lazy (because current thread is | |
| 957 | − * allowed to migrate to another cpu) | |
| 958 | − */ | |
| 959 | − loc_cpu_entry = private->entries[raw_smp_processor_id()]; | |
| 947 | + loc_cpu_entry = private->entries; | |
| 960 | 948 | if (copy_to_user(userptr, loc_cpu_entry, total_size) != 0) { |
| 961 | 949 | ret = -EFAULT; |
| 962 | 950 | goto free_counters; |
| @@ -1070,10 +1058,10 @@ static int compat_table_info(const struct xt_table_info *info, · 107 unmodified lines | ||
| 1070 | 1058 | if (!newinfo || !info) |
| 1071 | 1059 | return -EINVAL; |
| 1072 | 1060 | |
| 1073 | − /* we dont care about newinfo->entries[] */ | |
| 1061 | + /* we dont care about newinfo->entries */ | |
| 1074 | 1062 | memcpy(newinfo, info, offsetof(struct xt_table_info, entries)); |
| 1075 | 1063 | newinfo->initial_entries = 0; |
| 1076 | − loc_cpu_entry = info->entries[raw_smp_processor_id()]; | |
| 1064 | + loc_cpu_entry = info->entries; | |
| 1077 | 1065 | xt_compat_init_offsets(AF_INET, info->number); |
| 1078 | 1066 | xt_entry_foreach(iter, loc_cpu_entry, info->size) { |
| 1079 | 1067 | ret = compat_calc_entry(iter, info, loc_cpu_entry, newinfo); |
| @@ -1194,7 +1182,6 @@ __do_replace(struct net *net, const char *name, unsigned int valid_hooks, · 114 unmodified lines | ||
| 1194 | 1182 | struct xt_table *t; |
| 1195 | 1183 | struct xt_table_info *oldinfo; |
| 1196 | 1184 | struct xt_counters *counters; |
| 1197 | − void *loc_cpu_old_entry; | |
| 1198 | 1185 | struct ipt_entry *iter; |
| 1199 | 1186 | |
| 1200 | 1187 | ret = 0; |
| @@ -1237,8 +1224,7 @@ __do_replace(struct net *net, const char *name, unsigned int valid_hooks, · 36 unmodified lines | ||
| 1237 | 1224 | get_counters(oldinfo, counters); |
| 1238 | 1225 | |
| 1239 | 1226 | /* Decrease module usage counts and free resource */ |
| 1240 | − loc_cpu_old_entry = oldinfo->entries[raw_smp_processor_id()]; | |
| 1241 | − xt_entry_foreach(iter, loc_cpu_old_entry, oldinfo->size) | |
| 1227 | + xt_entry_foreach(iter, oldinfo->entries, oldinfo->size) | |
| 1242 | 1228 | cleanup_entry(iter, net); |
| 1243 | 1229 | |
| 1244 | 1230 | xt_free_table_info(oldinfo); |
| @@ -1284,8 +1270,7 @@ do_replace(struct net *net, const void __user *user, unsigned int len) · 39 unmodified lines | ||
| 1284 | 1270 | if (!newinfo) |
| 1285 | 1271 | return -ENOMEM; |
| 1286 | 1272 | |
| 1287 | − /* choose the copy that is on our node/cpu */ | |
| 1288 | − loc_cpu_entry = newinfo->entries[raw_smp_processor_id()]; | |
| 1273 | + loc_cpu_entry = newinfo->entries; | |
| 1289 | 1274 | if (copy_from_user(loc_cpu_entry, user + sizeof(tmp), |
| 1290 | 1275 | tmp.size) != 0) { |
| 1291 | 1276 | ret = -EFAULT; |
| @@ -1316,7 +1301,7 @@ static int · 24 unmodified lines | ||
| 1316 | 1301 | do_add_counters(struct net *net, const void __user *user, |
| 1317 | 1302 | unsigned int len, int compat) |
| 1318 | 1303 | { |
| 1319 | − unsigned int i, curcpu; | |
| 1304 | + unsigned int i; | |
| 1320 | 1305 | struct xt_counters_info tmp; |
| 1321 | 1306 | struct xt_counters *paddc; |
| 1322 | 1307 | unsigned int num_counters; |
| @@ -1326,7 +1311,6 @@ do_add_counters(struct net *net, const void __user *user, · 3 unmodified lines | ||
| 1326 | 1311 | struct xt_table *t; |
| 1327 | 1312 | const struct xt_table_info *private; |
| 1328 | 1313 | int ret = 0; |
| 1329 | − void *loc_cpu_entry; | |
| 1330 | 1314 | struct ipt_entry *iter; |
| 1331 | 1315 | unsigned int addend; |
| 1332 | 1316 | #ifdef CONFIG_COMPAT |
| @@ -1382,11 +1366,8 @@ do_add_counters(struct net *net, const void __user *user, · 49 unmodified lines | ||
| 1382 | 1366 | } |
| 1383 | 1367 | |
| 1384 | 1368 | i = 0; |
| 1385 | − /* Choose the copy that is on our node */ | |
| 1386 | − curcpu = smp_processor_id(); | |
| 1387 | − loc_cpu_entry = private->entries[curcpu]; | |
| 1388 | 1369 | addend = xt_write_recseq_begin(); |
| 1389 | − xt_entry_foreach(iter, loc_cpu_entry, private->size) { | |
| 1370 | + xt_entry_foreach(iter, private->entries, private->size) { | |
| 1390 | 1371 | struct xt_counters *tmp; |
| 1391 | 1372 | |
| 1392 | 1373 | tmp = xt_get_this_cpu_counter(&iter->counters); |
| @@ -1739,7 +1720,7 @@ translate_compat_table(struct net *net, · 346 unmodified lines | ||
| 1739 | 1720 | newinfo->hook_entry[i] = info->hook_entry[i]; |
| 1740 | 1721 | newinfo->underflow[i] = info->underflow[i]; |
| 1741 | 1722 | } |
| 1742 | − entry1 = newinfo->entries[raw_smp_processor_id()]; | |
| 1723 | + entry1 = newinfo->entries; | |
| 1743 | 1724 | pos = entry1; |
| 1744 | 1725 | size = total_size; |
| 1745 | 1726 | xt_entry_foreach(iter0, entry0, total_size) { |
| @@ -1791,11 +1772,6 @@ translate_compat_table(struct net *net, · 45 unmodified lines | ||
| 1791 | 1772 | return ret; |
| 1792 | 1773 | } |
| 1793 | 1774 | |
| 1794 | − /* And one copy for every other CPU */ | |
| 1795 | − for_each_possible_cpu(i) | |
| 1796 | − if (newinfo->entries[i] && newinfo->entries[i] != entry1) | |
| 1797 | − memcpy(newinfo->entries[i], entry1, newinfo->size); | |
| 1798 | − | |
| 1799 | 1775 | *pinfo = newinfo; |
| 1800 | 1776 | *pentry0 = entry1; |
| 1801 | 1777 | xt_free_table_info(info); |
| @@ -1842,8 +1818,7 @@ compat_do_replace(struct net *net, void __user *user, unsigned int len) · 40 unmodified lines | ||
| 1842 | 1818 | if (!newinfo) |
| 1843 | 1819 | return -ENOMEM; |
| 1844 | 1820 | |
| 1845 | − /* choose the copy that is on our node/cpu */ | |
| 1846 | − loc_cpu_entry = newinfo->entries[raw_smp_processor_id()]; | |
| 1821 | + loc_cpu_entry = newinfo->entries; | |
| 1847 | 1822 | if (copy_from_user(loc_cpu_entry, user + sizeof(tmp), |
| 1848 | 1823 | tmp.size) != 0) { |
| 1849 | 1824 | ret = -EFAULT; |
| @@ -1914,7 +1889,6 @@ compat_copy_entries_to_user(unsigned int total_size, struct xt_table *table, · 64 unmodified lines | ||
| 1914 | 1889 | void __user *pos; |
| 1915 | 1890 | unsigned int size; |
| 1916 | 1891 | int ret = 0; |
| 1917 | − const void *loc_cpu_entry; | |
| 1918 | 1892 | unsigned int i = 0; |
| 1919 | 1893 | struct ipt_entry *iter; |
| 1920 | 1894 | |
| @@ -1922,14 +1896,9 @@ compat_copy_entries_to_user(unsigned int total_size, struct xt_table *table, · 1 unmodified line | ||
| 1922 | 1896 | if (IS_ERR(counters)) |
| 1923 | 1897 | return PTR_ERR(counters); |
| 1924 | 1898 | |
| 1925 | − /* choose the copy that is on our node/cpu, ... | |
| 1926 | − * This choice is lazy (because current thread is | |
| 1927 | − * allowed to migrate to another cpu) | |
| 1928 | − */ | |
| 1929 | − loc_cpu_entry = private->entries[raw_smp_processor_id()]; | |
| 1930 | 1899 | pos = userptr; |
| 1931 | 1900 | size = total_size; |
| 1932 | − xt_entry_foreach(iter, loc_cpu_entry, total_size) { | |
| 1901 | + xt_entry_foreach(iter, private->entries, total_size) { | |
| 1933 | 1902 | ret = compat_copy_entry_to_user(iter, &pos, |
| 1934 | 1903 | &size, counters, i++); |
| 1935 | 1904 | if (ret != 0) |
| @@ -2104,8 +2073,7 @@ struct xt_table *ipt_register_table(struct net *net, · 168 unmodified lines | ||
| 2104 | 2073 | goto out; |
| 2105 | 2074 | } |
| 2106 | 2075 | |
| 2107 | − /* choose the copy on our node/cpu, but dont care about preemption */ | |
| 2108 | − loc_cpu_entry = newinfo->entries[raw_smp_processor_id()]; | |
| 2076 | + loc_cpu_entry = newinfo->entries; | |
| 2109 | 2077 | memcpy(loc_cpu_entry, repl->entries, repl->size); |
| 2110 | 2078 | |
| 2111 | 2079 | ret = translate_table(net, newinfo, loc_cpu_entry, repl); |
| @@ -2136,7 +2104,7 @@ void ipt_unregister_table(struct net *net, struct xt_table *table) · 24 unmodified lines | ||
| 2136 | 2104 | private = xt_unregister_table(table); |
| 2137 | 2105 | |
| 2138 | 2106 | /* Decrease module usage counts and free resources */ |
| 2139 | − loc_cpu_entry = private->entries[raw_smp_processor_id()]; | |
| 2107 | + loc_cpu_entry = private->entries; | |
| 2140 | 2108 | xt_entry_foreach(iter, loc_cpu_entry, private->size) |
| 2141 | 2109 | cleanup_entry(iter, net); |
| 2142 | 2110 | if (private->number > private->initial_entries) |
| @@ -283,15 +283,13 @@ static void trace_packet(const struct sk_buff *skb, · 282 unmodified lines | ||
| 283 | 283 | const struct xt_table_info *private, |
| 284 | 284 | const struct ip6t_entry *e) |
| 285 | 285 | { |
| 286 | − const void *table_base; | |
| 287 | 286 | const struct ip6t_entry *root; |
| 288 | 287 | const char *hookname, *chainname, *comment; |
| 289 | 288 | const struct ip6t_entry *iter; |
| 290 | 289 | unsigned int rulenum = 0; |
| 291 | 290 | struct net *net = dev_net(in ? in : out); |
| 292 | 291 | |
| 293 | − table_base = private->entries[smp_processor_id()]; | |
| 294 | − root = get_entry(table_base, private->hook_entry[hook]); | |
| 292 | + root = get_entry(private->entries, private->hook_entry[hook]); | |
| 295 | 293 | |
| 296 | 294 | hookname = chainname = hooknames[hook]; |
| 297 | 295 | comment = comments[NF_IP6_TRACE_COMMENT_RULE]; |
| @@ -357,7 +355,7 @@ ip6t_do_table(struct sk_buff *skb, · 59 unmodified lines | ||
| 357 | 355 | */ |
| 358 | 356 | smp_read_barrier_depends(); |
| 359 | 357 | cpu = smp_processor_id(); |
| 360 | − table_base = private->entries[cpu]; | |
| 358 | + table_base = private->entries; | |
| 361 | 359 | jumpstack = (struct ip6t_entry **)private->jumpstack[cpu]; |
| 362 | 360 | stackptr = per_cpu_ptr(private->stackptr, cpu); |
| 363 | 361 | origptr = *stackptr; |
| @@ -890,12 +888,6 @@ translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0, · 526 unmodified lines | ||
| 890 | 888 | return ret; |
| 891 | 889 | } |
| 892 | 890 | |
| 893 | − /* And one copy for every other CPU */ | |
| 894 | − for_each_possible_cpu(i) { | |
| 895 | − if (newinfo->entries[i] && newinfo->entries[i] != entry0) | |
| 896 | − memcpy(newinfo->entries[i], entry0, newinfo->size); | |
| 897 | − } | |
| 898 | − | |
| 899 | 891 | return ret; |
| 900 | 892 | } |
| 901 | 893 | |
| @@ -911,7 +903,7 @@ get_counters(const struct xt_table_info *t, · 9 unmodified lines | ||
| 911 | 903 | seqcount_t *s = &per_cpu(xt_recseq, cpu); |
| 912 | 904 | |
| 913 | 905 | i = 0; |
| 914 | − xt_entry_foreach(iter, t->entries[cpu], t->size) { | |
| 906 | + xt_entry_foreach(iter, t->entries, t->size) { | |
| 915 | 907 | struct xt_counters *tmp; |
| 916 | 908 | u64 bcnt, pcnt; |
| 917 | 909 | unsigned int start; |
| @@ -959,17 +951,13 @@ copy_entries_to_user(unsigned int total_size, · 41 unmodified lines | ||
| 959 | 951 | struct xt_counters *counters; |
| 960 | 952 | const struct xt_table_info *private = table->private; |
| 961 | 953 | int ret = 0; |
| 962 | − const void *loc_cpu_entry; | |
| 954 | + void *loc_cpu_entry; | |
| 963 | 955 | |
| 964 | 956 | counters = alloc_counters(table); |
| 965 | 957 | if (IS_ERR(counters)) |
| 966 | 958 | return PTR_ERR(counters); |
| 967 | 959 | |
| 968 | − /* choose the copy that is on our node/cpu, ... | |
| 969 | − * This choice is lazy (because current thread is | |
| 970 | − * allowed to migrate to another cpu) | |
| 971 | − */ | |
| 972 | − loc_cpu_entry = private->entries[raw_smp_processor_id()]; | |
| 960 | + loc_cpu_entry = private->entries; | |
| 973 | 961 | if (copy_to_user(userptr, loc_cpu_entry, total_size) != 0) { |
| 974 | 962 | ret = -EFAULT; |
| 975 | 963 | goto free_counters; |
| @@ -1083,10 +1071,10 @@ static int compat_table_info(const struct xt_table_info *info, · 107 unmodified lines | ||
| 1083 | 1071 | if (!newinfo || !info) |
| 1084 | 1072 | return -EINVAL; |
| 1085 | 1073 | |
| 1086 | − /* we dont care about newinfo->entries[] */ | |
| 1074 | + /* we dont care about newinfo->entries */ | |
| 1087 | 1075 | memcpy(newinfo, info, offsetof(struct xt_table_info, entries)); |
| 1088 | 1076 | newinfo->initial_entries = 0; |
| 1089 | − loc_cpu_entry = info->entries[raw_smp_processor_id()]; | |
| 1077 | + loc_cpu_entry = info->entries; | |
| 1090 | 1078 | xt_compat_init_offsets(AF_INET6, info->number); |
| 1091 | 1079 | xt_entry_foreach(iter, loc_cpu_entry, info->size) { |
| 1092 | 1080 | ret = compat_calc_entry(iter, info, loc_cpu_entry, newinfo); |
| @@ -1207,7 +1195,6 @@ __do_replace(struct net *net, const char *name, unsigned int valid_hooks, · 114 unmodified lines | ||
| 1207 | 1195 | struct xt_table *t; |
| 1208 | 1196 | struct xt_table_info *oldinfo; |
| 1209 | 1197 | struct xt_counters *counters; |
| 1210 | − const void *loc_cpu_old_entry; | |
| 1211 | 1198 | struct ip6t_entry *iter; |
| 1212 | 1199 | |
| 1213 | 1200 | ret = 0; |
| @@ -1250,8 +1237,7 @@ __do_replace(struct net *net, const char *name, unsigned int valid_hooks, · 36 unmodified lines | ||
| 1250 | 1237 | get_counters(oldinfo, counters); |
| 1251 | 1238 | |
| 1252 | 1239 | /* Decrease module usage counts and free resource */ |
| 1253 | − loc_cpu_old_entry = oldinfo->entries[raw_smp_processor_id()]; | |
| 1254 | − xt_entry_foreach(iter, loc_cpu_old_entry, oldinfo->size) | |
| 1240 | + xt_entry_foreach(iter, oldinfo->entries, oldinfo->size) | |
| 1255 | 1241 | cleanup_entry(iter, net); |
| 1256 | 1242 | |
| 1257 | 1243 | xt_free_table_info(oldinfo); |
| @@ -1297,8 +1283,7 @@ do_replace(struct net *net, const void __user *user, unsigned int len) · 39 unmodified lines | ||
| 1297 | 1283 | if (!newinfo) |
| 1298 | 1284 | return -ENOMEM; |
| 1299 | 1285 | |
| 1300 | − /* choose the copy that is on our node/cpu */ | |
| 1301 | − loc_cpu_entry = newinfo->entries[raw_smp_processor_id()]; | |
| 1286 | + loc_cpu_entry = newinfo->entries; | |
| 1302 | 1287 | if (copy_from_user(loc_cpu_entry, user + sizeof(tmp), |
| 1303 | 1288 | tmp.size) != 0) { |
| 1304 | 1289 | ret = -EFAULT; |
| @@ -1329,7 +1314,7 @@ static int · 24 unmodified lines | ||
| 1329 | 1314 | do_add_counters(struct net *net, const void __user *user, unsigned int len, |
| 1330 | 1315 | int compat) |
| 1331 | 1316 | { |
| 1332 | − unsigned int i, curcpu; | |
| 1317 | + unsigned int i; | |
| 1333 | 1318 | struct xt_counters_info tmp; |
| 1334 | 1319 | struct xt_counters *paddc; |
| 1335 | 1320 | unsigned int num_counters; |
| @@ -1339,7 +1324,6 @@ do_add_counters(struct net *net, const void __user *user, unsigned int len, · 3 unmodified lines | ||
| 1339 | 1324 | struct xt_table *t; |
| 1340 | 1325 | const struct xt_table_info *private; |
| 1341 | 1326 | int ret = 0; |
| 1342 | − const void *loc_cpu_entry; | |
| 1343 | 1327 | struct ip6t_entry *iter; |
| 1344 | 1328 | unsigned int addend; |
| 1345 | 1329 | #ifdef CONFIG_COMPAT |
| @@ -1395,11 +1379,8 @@ do_add_counters(struct net *net, const void __user *user, unsigned int len, · 49 unmodified lines | ||
| 1395 | 1379 | } |
| 1396 | 1380 | |
| 1397 | 1381 | i = 0; |
| 1398 | − /* Choose the copy that is on our node */ | |
| 1399 | − curcpu = smp_processor_id(); | |
| 1400 | 1382 | addend = xt_write_recseq_begin(); |
| 1401 | − loc_cpu_entry = private->entries[curcpu]; | |
| 1402 | − xt_entry_foreach(iter, loc_cpu_entry, private->size) { | |
| 1383 | + xt_entry_foreach(iter, private->entries, private->size) { | |
| 1403 | 1384 | struct xt_counters *tmp; |
| 1404 | 1385 | |
| 1405 | 1386 | tmp = xt_get_this_cpu_counter(&iter->counters); |
| @@ -1407,7 +1388,6 @@ do_add_counters(struct net *net, const void __user *user, unsigned int len, · 1 unmodified line | ||
| 1407 | 1388 | ++i; |
| 1408 | 1389 | } |
| 1409 | 1390 | xt_write_recseq_end(addend); |
| 1410 | − | |
| 1411 | 1391 | unlock_up_free: |
| 1412 | 1392 | local_bh_enable(); |
| 1413 | 1393 | xt_table_unlock(t); |
| @@ -1750,7 +1730,7 @@ translate_compat_table(struct net *net, · 336 unmodified lines | ||
| 1750 | 1730 | newinfo->hook_entry[i] = info->hook_entry[i]; |
| 1751 | 1731 | newinfo->underflow[i] = info->underflow[i]; |
| 1752 | 1732 | } |
| 1753 | − entry1 = newinfo->entries[raw_smp_processor_id()]; | |
| 1733 | + entry1 = newinfo->entries; | |
| 1754 | 1734 | pos = entry1; |
| 1755 | 1735 | size = total_size; |
| 1756 | 1736 | xt_entry_foreach(iter0, entry0, total_size) { |
| @@ -1802,11 +1782,6 @@ translate_compat_table(struct net *net, · 45 unmodified lines | ||
| 1802 | 1782 | return ret; |
| 1803 | 1783 | } |
| 1804 | 1784 | |
| 1805 | − /* And one copy for every other CPU */ | |
| 1806 | − for_each_possible_cpu(i) | |
| 1807 | − if (newinfo->entries[i] && newinfo->entries[i] != entry1) | |
| 1808 | − memcpy(newinfo->entries[i], entry1, newinfo->size); | |
| 1809 | − | |
| 1810 | 1785 | *pinfo = newinfo; |
| 1811 | 1786 | *pentry0 = entry1; |
| 1812 | 1787 | xt_free_table_info(info); |
| @@ -1853,8 +1828,7 @@ compat_do_replace(struct net *net, void __user *user, unsigned int len) · 40 unmodified lines | ||
| 1853 | 1828 | if (!newinfo) |
| 1854 | 1829 | return -ENOMEM; |
| 1855 | 1830 | |
| 1856 | − /* choose the copy that is on our node/cpu */ | |
| 1857 | − loc_cpu_entry = newinfo->entries[raw_smp_processor_id()]; | |
| 1831 | + loc_cpu_entry = newinfo->entries; | |
| 1858 | 1832 | if (copy_from_user(loc_cpu_entry, user + sizeof(tmp), |
| 1859 | 1833 | tmp.size) != 0) { |
| 1860 | 1834 | ret = -EFAULT; |
| @@ -1925,7 +1899,6 @@ compat_copy_entries_to_user(unsigned int total_size, struct xt_table *table, · 64 unmodified lines | ||
| 1925 | 1899 | void __user *pos; |
| 1926 | 1900 | unsigned int size; |
| 1927 | 1901 | int ret = 0; |
| 1928 | − const void *loc_cpu_entry; | |
| 1929 | 1902 | unsigned int i = 0; |
| 1930 | 1903 | struct ip6t_entry *iter; |
| 1931 | 1904 | |
| @@ -1933,14 +1906,9 @@ compat_copy_entries_to_user(unsigned int total_size, struct xt_table *table, · 1 unmodified line | ||
| 1933 | 1906 | if (IS_ERR(counters)) |
| 1934 | 1907 | return PTR_ERR(counters); |
| 1935 | 1908 | |
| 1936 | − /* choose the copy that is on our node/cpu, ... | |
| 1937 | − * This choice is lazy (because current thread is | |
| 1938 | − * allowed to migrate to another cpu) | |
| 1939 | − */ | |
| 1940 | − loc_cpu_entry = private->entries[raw_smp_processor_id()]; | |
| 1941 | 1909 | pos = userptr; |
| 1942 | 1910 | size = total_size; |
| 1943 | − xt_entry_foreach(iter, loc_cpu_entry, total_size) { | |
| 1911 | + xt_entry_foreach(iter, private->entries, total_size) { | |
| 1944 | 1912 | ret = compat_copy_entry_to_user(iter, &pos, |
| 1945 | 1913 | &size, counters, i++); |
| 1946 | 1914 | if (ret != 0) |
| @@ -2115,8 +2083,7 @@ struct xt_table *ip6t_register_table(struct net *net, · 168 unmodified lines | ||
| 2115 | 2083 | goto out; |
| 2116 | 2084 | } |
| 2117 | 2085 | |
| 2118 | − /* choose the copy on our node/cpu, but dont care about preemption */ | |
| 2119 | − loc_cpu_entry = newinfo->entries[raw_smp_processor_id()]; | |
| 2086 | + loc_cpu_entry = newinfo->entries; | |
| 2120 | 2087 | memcpy(loc_cpu_entry, repl->entries, repl->size); |
| 2121 | 2088 | |
| 2122 | 2089 | ret = translate_table(net, newinfo, loc_cpu_entry, repl); |
| @@ -2146,7 +2113,7 @@ void ip6t_unregister_table(struct net *net, struct xt_table *table) · 23 unmodified lines | ||
| 2146 | 2113 | private = xt_unregister_table(table); |
| 2147 | 2114 | |
| 2148 | 2115 | /* Decrease module usage counts and free resources */ |
| 2149 | − loc_cpu_entry = private->entries[raw_smp_processor_id()]; | |
| 2116 | + loc_cpu_entry = private->entries; | |
| 2150 | 2117 | xt_entry_foreach(iter, loc_cpu_entry, private->size) |
| 2151 | 2118 | cleanup_entry(iter, net); |
| 2152 | 2119 | if (private->number > private->initial_entries) |
| @@ -659,7 +659,6 @@ EXPORT_SYMBOL_GPL(xt_compat_target_to_user); · 658 unmodified lines | ||
| 659 | 659 | struct xt_table_info *xt_alloc_table_info(unsigned int size) |
| 660 | 660 | { |
| 661 | 661 | struct xt_table_info *newinfo; |
| 662 | − int cpu; | |
| 663 | 662 | |
| 664 | 663 | /* Pedantry: prevent them from hitting BUG() in vmalloc.c --RR */ |
| 665 | 664 | if ((SMP_ALIGN(size) >> PAGE_SHIFT) + 2 > totalram_pages) |
| @@ -671,19 +670,14 @@ struct xt_table_info *xt_alloc_table_info(unsigned int size) · 5 unmodified lines | ||
| 671 | 670 | |
| 672 | 671 | newinfo->size = size; |
| 673 | 672 | |
| 674 | − for_each_possible_cpu(cpu) { | |
| 675 | − if (size <= PAGE_SIZE) | |
| 676 | − newinfo->entries[cpu] = kmalloc_node(size, | |
| 677 | − GFP_KERNEL, | |
| 678 | − cpu_to_node(cpu)); | |
| 679 | − else | |
| 680 | − newinfo->entries[cpu] = vmalloc_node(size, | |
| 681 | − cpu_to_node(cpu)); | |
| 673 | + if (size <= PAGE_SIZE) | |
| 674 | + newinfo->entries = kmalloc(size, GFP_KERNEL); | |
| 675 | + else | |
| 676 | + newinfo->entries = vmalloc(size); | |
| 682 | 677 | |
| 683 | − if (newinfo->entries[cpu] == NULL) { | |
| 684 | − xt_free_table_info(newinfo); | |
| 685 | − return NULL; | |
| 686 | − } | |
| 678 | + if (newinfo->entries == NULL) { | |
| 679 | + xt_free_table_info(newinfo); | |
| 680 | + return NULL; | |
| 687 | 681 | } |
| 688 | 682 | |
| 689 | 683 | return newinfo; |
| @@ -694,8 +688,7 @@ void xt_free_table_info(struct xt_table_info *info) · 4 unmodified lines | ||
| 694 | 688 | { |
| 695 | 689 | int cpu; |
| 696 | 690 | |
| 697 | − for_each_possible_cpu(cpu) | |
| 698 | − kvfree(info->entries[cpu]); | |
| 691 | + kvfree(info->entries); | |
| 699 | 692 | |
| 700 | 693 | if (info->jumpstack != NULL) { |
| 701 | 694 | for_each_possible_cpu(cpu) |