History

net: ethernet: mtk_eth_soc: rework hardware flow table management

The hardware was designed to handle flow detection and creation of flow entries by itself, relying on the software primarily for filling in egress routing information. When there is a hash collision between multiple flows, this allows the hardware to maintain the entry for the most active flow. Additionally, the hardware only keeps offloading active for entries with at least 30 packets per second. With this rework, the code no longer creates a hardware entries directly. Instead, the hardware entry is only created when the PPE reports a matching unbound flow with the minimum target rate. In order to reduce CPU overhead, looking for flows belonging to a hash entry is rate limited to once every 100ms. This rework is also used as preparation for emulating bridge offload by managing L4 offload entries on demand.

Felix Fietkau committed 4 years ago

Commit c4f033d1 parent 1ccc7234 files · +170 −54Browse files

Unified · Split

4 files
  1. drivers/net/ethernet/mediatek/mtk_eth_soc.c +8 −2
  2. drivers/net/ethernet/mediatek/mtk_ppe.c +120 −14
  3. drivers/net/ethernet/mediatek/mtk_ppe.h +32 −6
  4. drivers/net/ethernet/mediatek/mtk_ppe_offload.c +10 −32
drivers/net/ethernet/mediatek/mtk_eth_soc.c+8−2
@@ -21,6 +21,7 @@ · 20 unmodified lines
21#include <linux/pinctrl/devinfo.h>21#include <linux/pinctrl/devinfo.h>
22#include <linux/phylink.h>22#include <linux/phylink.h>
23#include <linux/jhash.h>23#include <linux/jhash.h>
24#include <linux/bitfield.h>
24#include <net/dsa.h>25#include <net/dsa.h>
25 26
26#include "mtk_eth_soc.h"27#include "mtk_eth_soc.h"
@@ -1239,7 +1240,7 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, · 1212 unmodified lines
1239 struct net_device *netdev;1240 struct net_device *netdev;
1240 unsigned int pktlen;1241 unsigned int pktlen;
1241 dma_addr_t dma_addr;1242 dma_addr_t dma_addr;
1242 u32 hash;1243 u32 hash, reason;
1243 int mac;1244 int mac;
1244 1245
1245 ring = mtk_get_rx_ring(eth);1246 ring = mtk_get_rx_ring(eth);
@@ -1315,6 +1316,11 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, · 69 unmodified lines
1315 skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);1316 skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
1316 }1317 }
1317 1318
1319 reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4);
1320 if (reason == MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
1321 mtk_ppe_check_skb(eth->ppe, skb,
1322 trxd.rxd4 & MTK_RXD4_FOE_ENTRY);
1323
1318 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&1324 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
1319 (trxd.rxd2 & RX_DMA_VTAG))1325 (trxd.rxd2 & RX_DMA_VTAG))
1320 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),1326 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
@@ -3262,7 +3268,7 @@ static int mtk_probe(struct platform_device *pdev) · 1941 unmodified lines
3262 }3268 }
3263 3269
3264 if (eth->soc->offload_version) {3270 if (eth->soc->offload_version) {
3265 eth->ppe = mtk_ppe_init(eth->dev, eth->base + MTK_ETH_PPE_BASE, 2);3271 eth->ppe = mtk_ppe_init(eth, eth->base + MTK_ETH_PPE_BASE, 2);
3266 if (!eth->ppe) {3272 if (!eth->ppe) {
3267 err = -ENOMEM;3273 err = -ENOMEM;
3268 goto err_free_dev;3274 goto err_free_dev;
drivers/net/ethernet/mediatek/mtk_ppe.c+120−14
@@ -6,9 +6,12 @@ · 5 unmodified lines
6#include <linux/iopoll.h>6#include <linux/iopoll.h>
7#include <linux/etherdevice.h>7#include <linux/etherdevice.h>
8#include <linux/platform_device.h>8#include <linux/platform_device.h>
9#include "mtk_eth_soc.h"
9#include "mtk_ppe.h"10#include "mtk_ppe.h"
10#include "mtk_ppe_regs.h"11#include "mtk_ppe_regs.h"
11 12
13static DEFINE_SPINLOCK(ppe_lock);
14
12static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)15static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)
13{16{
14 writel(val, ppe->base + reg);17 writel(val, ppe->base + reg);
@@ -41,6 +44,11 @@ static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) · 26 unmodified lines
41 return ppe_m32(ppe, reg, val, 0);44 return ppe_m32(ppe, reg, val, 0);
42}45}
43 46
47static u32 mtk_eth_timestamp(struct mtk_eth *eth)
48{
49 return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP;
50}
51
44static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)52static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
45{53{
46 int ret;54 int ret;
@@ -353,26 +361,59 @@ static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry) · 306 unmodified lines
353 FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1) != MTK_FOE_STATE_BIND;361 FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1) != MTK_FOE_STATE_BIND;
354}362}
355 363
356int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,364static bool
357 u16 timestamp)365mtk_flow_entry_match(struct mtk_flow_entry *entry, struct mtk_foe_entry *data)
366{
367 int type, len;
368
369 if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP)
370 return false;
371
372 type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1);
373 if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE)
374 len = offsetof(struct mtk_foe_entry, ipv6._rsv);
375 else
376 len = offsetof(struct mtk_foe_entry, ipv4.ib2);
377
378 return !memcmp(&entry->data.data, &data->data, len - 4);
379}
380
381static void
382mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
358{383{
359 struct mtk_foe_entry *hwe;384 struct mtk_foe_entry *hwe;
360 u32 hash;385 struct mtk_foe_entry foe;
386
387 spin_lock_bh(&ppe_lock);
388 if (entry->hash == 0xffff)
389 goto out;
361 390
391 hwe = &ppe->foe_table[entry->hash];
392 memcpy(&foe, hwe, sizeof(foe));
393 if (!mtk_flow_entry_match(entry, &foe)) {
394 entry->hash = 0xffff;
395 goto out;
396 }
397
398 entry->data.ib1 = foe.ib1;
399
400out:
401 spin_unlock_bh(&ppe_lock);
402}
403
404static void
405__mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
406 u16 hash)
407{
408 struct mtk_foe_entry *hwe;
409 u16 timestamp;
410
411 timestamp = mtk_eth_timestamp(ppe->eth);
362 timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP;412 timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP;
363 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;413 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
364 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp);414 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp);
365 415
366 hash = mtk_ppe_hash_entry(entry);
367 hwe = &ppe->foe_table[hash];416 hwe = &ppe->foe_table[hash];
368 if (!mtk_foe_entry_usable(hwe)) {
369 hwe++;
370 hash++;
371
372 if (!mtk_foe_entry_usable(hwe))
373 return -ENOSPC;
374 }
375
376 memcpy(&hwe->data, &entry->data, sizeof(hwe->data));417 memcpy(&hwe->data, &entry->data, sizeof(hwe->data));
377 wmb();418 wmb();
378 hwe->ib1 = entry->ib1;419 hwe->ib1 = entry->ib1;
@@ -380,13 +421,77 @@ int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, · 1 unmodified line
380 dma_wmb();421 dma_wmb();
381 422
382 mtk_ppe_cache_clear(ppe);423 mtk_ppe_cache_clear(ppe);
424}
383 425
384 return hash;426void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
427{
428 spin_lock_bh(&ppe_lock);
429 hlist_del_init(&entry->list);
430 if (entry->hash != 0xffff) {
431 ppe->foe_table[entry->hash].ib1 &= ~MTK_FOE_IB1_STATE;
432 ppe->foe_table[entry->hash].ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE,
433 MTK_FOE_STATE_BIND);
434 dma_wmb();
435 }
436 entry->hash = 0xffff;
437 spin_unlock_bh(&ppe_lock);
438}
439
440int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
441{
442 u32 hash = mtk_ppe_hash_entry(&entry->data);
443
444 entry->hash = 0xffff;
445 spin_lock_bh(&ppe_lock);
446 hlist_add_head(&entry->list, &ppe->foe_flow[hash / 2]);
447 spin_unlock_bh(&ppe_lock);
448
449 return 0;
450}
451
452void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
453{
454 struct hlist_head *head = &ppe->foe_flow[hash / 2];
455 struct mtk_flow_entry *entry;
456 struct mtk_foe_entry *hwe = &ppe->foe_table[hash];
457 bool found = false;
458
459 if (hlist_empty(head))
460 return;
461
462 spin_lock_bh(&ppe_lock);
463 hlist_for_each_entry(entry, head, list) {
464 if (found || !mtk_flow_entry_match(entry, hwe)) {
465 if (entry->hash != 0xffff)
466 entry->hash = 0xffff;
467 continue;
468 }
469
470 entry->hash = hash;
471 __mtk_foe_entry_commit(ppe, &entry->data, hash);
472 found = true;
473 }
474 spin_unlock_bh(&ppe_lock);
475}
476
477int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
478{
479 u16 now = mtk_eth_timestamp(ppe->eth) & MTK_FOE_IB1_BIND_TIMESTAMP;
480 u16 timestamp;
481
482 mtk_flow_entry_update(ppe, entry);
483 timestamp = entry->data.ib1 & MTK_FOE_IB1_BIND_TIMESTAMP;
484
485 if (timestamp > now)
486 return MTK_FOE_IB1_BIND_TIMESTAMP + 1 - timestamp + now;
487 else
488 return now - timestamp;
385}489}
386 490
387struct mtk_ppe *mtk_ppe_init(struct device *dev, void __iomem *base,491struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
388 int version)492 int version)
389{493{
494 struct device *dev = eth->dev;
390 struct mtk_foe_entry *foe;495 struct mtk_foe_entry *foe;
391 struct mtk_ppe *ppe;496 struct mtk_ppe *ppe;
392 497
@@ -398,6 +503,7 @@ struct mtk_ppe *mtk_ppe_init(struct device *dev, void __iomem *base, · 5 unmodified lines
398 * not coherent.503 * not coherent.
399 */504 */
400 ppe->base = base;505 ppe->base = base;
506 ppe->eth = eth;
401 ppe->dev = dev;507 ppe->dev = dev;
402 ppe->version = version;508 ppe->version = version;
403 509
drivers/net/ethernet/mediatek/mtk_ppe.h+32−6
@@ -235,7 +235,17 @@ enum { · 234 unmodified lines
235 MTK_PPE_CPU_REASON_INVALID = 0x1f,235 MTK_PPE_CPU_REASON_INVALID = 0x1f,
236};236};
237 237
238struct mtk_flow_entry {
239 struct rhash_head node;
240 struct hlist_node list;
241 unsigned long cookie;
242 struct mtk_foe_entry data;
243 u16 hash;
244 s8 wed_index;
245};
246
238struct mtk_ppe {247struct mtk_ppe {
248 struct mtk_eth *eth;
239 struct device *dev;249 struct device *dev;
240 void __iomem *base;250 void __iomem *base;
241 int version;251 int version;
@@ -243,18 +253,33 @@ struct mtk_ppe { · 1 unmodified line
243 struct mtk_foe_entry *foe_table;253 struct mtk_foe_entry *foe_table;
244 dma_addr_t foe_phys;254 dma_addr_t foe_phys;
245 255
256 u16 foe_check_time[MTK_PPE_ENTRIES];
257 struct hlist_head foe_flow[MTK_PPE_ENTRIES / 2];
258
246 void *acct_table;259 void *acct_table;
247};260};
248 261
249struct mtk_ppe *mtk_ppe_init(struct device *dev, void __iomem *base, int version);262struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int version);
250int mtk_ppe_start(struct mtk_ppe *ppe);263int mtk_ppe_start(struct mtk_ppe *ppe);
251int mtk_ppe_stop(struct mtk_ppe *ppe);264int mtk_ppe_stop(struct mtk_ppe *ppe);
252 265
266void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash);
267
253static inline void268static inline void
254mtk_foe_entry_clear(struct mtk_ppe *ppe, u16 hash)269mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
255{270{
256 ppe->foe_table[hash].ib1 = 0;271 u16 now, diff;
257 dma_wmb();272
273 if (!ppe)
274 return;
275
276 now = (u16)jiffies;
277 diff = now - ppe->foe_check_time[hash];
278 if (diff < HZ / 10)
279 return;
280
281 ppe->foe_check_time[hash] = now;
282 __mtk_ppe_check_skb(ppe, skb, hash);
258}283}
259 284
260static inline int285static inline int
@@ -282,8 +307,9 @@ int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid); · 21 unmodified lines
282int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid);307int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid);
283int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq,308int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq,
284 int bss, int wcid);309 int bss, int wcid);
285int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,310int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
286 u16 timestamp);311void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
312int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
287int mtk_ppe_debugfs_init(struct mtk_ppe *ppe);313int mtk_ppe_debugfs_init(struct mtk_ppe *ppe);
288 314
289#endif315#endif
drivers/net/ethernet/mediatek/mtk_ppe_offload.c+10−32
@@ -42,13 +42,6 @@ struct mtk_flow_data { · 41 unmodified lines
42 } pppoe;42 } pppoe;
43};43};
44 44
45struct mtk_flow_entry {
46 struct rhash_head node;
47 unsigned long cookie;
48 u16 hash;
49 s8 wed_index;
50};
51
52static const struct rhashtable_params mtk_flow_ht_params = {45static const struct rhashtable_params mtk_flow_ht_params = {
53 .head_offset = offsetof(struct mtk_flow_entry, node),46 .head_offset = offsetof(struct mtk_flow_entry, node),
54 .key_offset = offsetof(struct mtk_flow_entry, cookie),47 .key_offset = offsetof(struct mtk_flow_entry, cookie),
@@ -56,12 +49,6 @@ static const struct rhashtable_params mtk_flow_ht_params = { · 1 unmodified line
56 .automatic_shrinking = true,49 .automatic_shrinking = true,
57};50};
58 51
59static u32
60mtk_eth_timestamp(struct mtk_eth *eth)
61{
62 return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP;
63}
64
65static int52static int
66mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data,53mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data,
67 bool egress)54 bool egress)
@@ -237,10 +224,8 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) · 169 unmodified lines
237 int offload_type = 0;224 int offload_type = 0;
238 int wed_index = -1;225 int wed_index = -1;
239 u16 addr_type = 0;226 u16 addr_type = 0;
240 u32 timestamp;
241 u8 l4proto = 0;227 u8 l4proto = 0;
242 int err = 0;228 int err = 0;
243 int hash;
244 int i;229 int i;
245 230
246 if (rhashtable_lookup(&eth->flow_table, &f->cookie, mtk_flow_ht_params))231 if (rhashtable_lookup(&eth->flow_table, &f->cookie, mtk_flow_ht_params))
@@ -410,23 +395,21 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) · 163 unmodified lines
410 return -ENOMEM;395 return -ENOMEM;
411 396
412 entry->cookie = f->cookie;397 entry->cookie = f->cookie;
413 timestamp = mtk_eth_timestamp(eth);398 memcpy(&entry->data, &foe, sizeof(entry->data));
414 hash = mtk_foe_entry_commit(eth->ppe, &foe, timestamp);399 entry->wed_index = wed_index;
415 if (hash < 0) {400
416 err = hash;401 if (mtk_foe_entry_commit(eth->ppe, entry) < 0)
417 goto free;402 goto free;
418 }
419 403
420 entry->hash = hash;
421 entry->wed_index = wed_index;
422 err = rhashtable_insert_fast(&eth->flow_table, &entry->node,404 err = rhashtable_insert_fast(&eth->flow_table, &entry->node,
423 mtk_flow_ht_params);405 mtk_flow_ht_params);
424 if (err < 0)406 if (err < 0)
425 goto clear_flow;407 goto clear;
426 408
427 return 0;409 return 0;
428clear_flow:410
429 mtk_foe_entry_clear(eth->ppe, hash);411clear:
412 mtk_foe_entry_clear(eth->ppe, entry);
430free:413free:
431 kfree(entry);414 kfree(entry);
432 if (wed_index >= 0)415 if (wed_index >= 0)
@@ -444,7 +427,7 @@ mtk_flow_offload_destroy(struct mtk_eth *eth, struct flow_cls_offload *f) · 11 unmodified lines
444 if (!entry)427 if (!entry)
445 return -ENOENT;428 return -ENOENT;
446 429
447 mtk_foe_entry_clear(eth->ppe, entry->hash);430 mtk_foe_entry_clear(eth->ppe, entry);
448 rhashtable_remove_fast(&eth->flow_table, &entry->node,431 rhashtable_remove_fast(&eth->flow_table, &entry->node,
449 mtk_flow_ht_params);432 mtk_flow_ht_params);
450 if (entry->wed_index >= 0)433 if (entry->wed_index >= 0)
@@ -458,7 +441,6 @@ static int · 7 unmodified lines
458mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)441mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)
459{442{
460 struct mtk_flow_entry *entry;443 struct mtk_flow_entry *entry;
461 int timestamp;
462 u32 idle;444 u32 idle;
463 445
464 entry = rhashtable_lookup(&eth->flow_table, &f->cookie,446 entry = rhashtable_lookup(&eth->flow_table, &f->cookie,
@@ -466,11 +448,7 @@ mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f) · 1 unmodified line
466 if (!entry)448 if (!entry)
467 return -ENOENT;449 return -ENOENT;
468 450
469 timestamp = mtk_foe_entry_timestamp(eth->ppe, entry->hash);451 idle = mtk_foe_entry_idle_time(eth->ppe, entry);
470 if (timestamp < 0)
471 return -ETIMEDOUT;
472
473 idle = mtk_eth_timestamp(eth) - timestamp;
474 f->stats.lastused = jiffies - idle * HZ;452 f->stats.lastused = jiffies - idle * HZ;
475 453
476 return 0;454 return 0;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t