History

net: ethernet: mtk_eth_soc: rework hardware flow table management

The hardware was designed to handle flow detection and creation of flow entries by itself, relying on the software primarily for filling in egress routing information. When there is a hash collision between multiple flows, this allows the hardware to maintain the entry for the most active flow. Additionally, the hardware only keeps offloading active for entries with at least 30 packets per second. With this rework, the code no longer creates a hardware entries directly. Instead, the hardware entry is only created when the PPE reports a matching unbound flow with the minimum target rate. In order to reduce CPU overhead, looking for flows belonging to a hash entry is rate limited to once every 100ms. This rework is also used as preparation for emulating bridge offload by managing L4 offload entries on demand.

Felix Fietkau committed 4 years ago

Commit c4f033d1 parent 1ccc7234 files · +170 −54Browse files

Unified · Split

4 files
  1. drivers/net/ethernet/mediatek/mtk_eth_soc.c +8 −2
  2. drivers/net/ethernet/mediatek/mtk_ppe.c +120 −14
  3. drivers/net/ethernet/mediatek/mtk_ppe.h +32 −6
  4. drivers/net/ethernet/mediatek/mtk_ppe_offload.c +10 −32
drivers/net/ethernet/mediatek/mtk_eth_soc.c+8−2
@@ -21,6 +21,7 @@ · 20 unmodified lines
2121 #include <linux/pinctrl/devinfo.h>
2222 #include <linux/phylink.h>
2323 #include <linux/jhash.h>
24+#include <linux/bitfield.h>
2425 #include <net/dsa.h>
2526
2627 #include "mtk_eth_soc.h"
@@ -1239,7 +1240,7 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, · 1212 unmodified lines
12391240 struct net_device *netdev;
12401241 unsigned int pktlen;
12411242 dma_addr_t dma_addr;
1242− u32 hash;
1243+ u32 hash, reason;
12431244 int mac;
12441245
12451246 ring = mtk_get_rx_ring(eth);
@@ -1315,6 +1316,11 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, · 69 unmodified lines
13151316 skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
13161317 }
13171318
1319+ reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4);
1320+ if (reason == MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
1321+ mtk_ppe_check_skb(eth->ppe, skb,
1322+ trxd.rxd4 & MTK_RXD4_FOE_ENTRY);
1323+
13181324 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
13191325 (trxd.rxd2 & RX_DMA_VTAG))
13201326 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
@@ -3262,7 +3268,7 @@ static int mtk_probe(struct platform_device *pdev) · 1941 unmodified lines
32623268 }
32633269
32643270 if (eth->soc->offload_version) {
3265− eth->ppe = mtk_ppe_init(eth->dev, eth->base + MTK_ETH_PPE_BASE, 2);
3271+ eth->ppe = mtk_ppe_init(eth, eth->base + MTK_ETH_PPE_BASE, 2);
32663272 if (!eth->ppe) {
32673273 err = -ENOMEM;
32683274 goto err_free_dev;
drivers/net/ethernet/mediatek/mtk_ppe.c+120−14
@@ -6,9 +6,12 @@ · 5 unmodified lines
66 #include <linux/iopoll.h>
77 #include <linux/etherdevice.h>
88 #include <linux/platform_device.h>
9+#include "mtk_eth_soc.h"
910 #include "mtk_ppe.h"
1011 #include "mtk_ppe_regs.h"
1112
13+static DEFINE_SPINLOCK(ppe_lock);
14+
1215 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)
1316 {
1417 writel(val, ppe->base + reg);
@@ -41,6 +44,11 @@ static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) · 26 unmodified lines
4144 return ppe_m32(ppe, reg, val, 0);
4245 }
4346
47+static u32 mtk_eth_timestamp(struct mtk_eth *eth)
48+{
49+ return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP;
50+}
51+
4452 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
4553 {
4654 int ret;
@@ -353,26 +361,59 @@ static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry) · 306 unmodified lines
353361 FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1) != MTK_FOE_STATE_BIND;
354362 }
355363
356−int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
357− u16 timestamp)
364+static bool
365+mtk_flow_entry_match(struct mtk_flow_entry *entry, struct mtk_foe_entry *data)
366+{
367+ int type, len;
368+
369+ if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP)
370+ return false;
371+
372+ type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1);
373+ if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE)
374+ len = offsetof(struct mtk_foe_entry, ipv6._rsv);
375+ else
376+ len = offsetof(struct mtk_foe_entry, ipv4.ib2);
377+
378+ return !memcmp(&entry->data.data, &data->data, len - 4);
379+}
380+
381+static void
382+mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
358383 {
359384 struct mtk_foe_entry *hwe;
360− u32 hash;
385+ struct mtk_foe_entry foe;
386+
387+ spin_lock_bh(&ppe_lock);
388+ if (entry->hash == 0xffff)
389+ goto out;
361390
391+ hwe = &ppe->foe_table[entry->hash];
392+ memcpy(&foe, hwe, sizeof(foe));
393+ if (!mtk_flow_entry_match(entry, &foe)) {
394+ entry->hash = 0xffff;
395+ goto out;
396+ }
397+
398+ entry->data.ib1 = foe.ib1;
399+
400+out:
401+ spin_unlock_bh(&ppe_lock);
402+}
403+
404+static void
405+__mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
406+ u16 hash)
407+{
408+ struct mtk_foe_entry *hwe;
409+ u16 timestamp;
410+
411+ timestamp = mtk_eth_timestamp(ppe->eth);
362412 timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP;
363413 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
364414 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp);
365415
366− hash = mtk_ppe_hash_entry(entry);
367416 hwe = &ppe->foe_table[hash];
368− if (!mtk_foe_entry_usable(hwe)) {
369− hwe++;
370− hash++;
371−
372− if (!mtk_foe_entry_usable(hwe))
373− return -ENOSPC;
374− }
375−
376417 memcpy(&hwe->data, &entry->data, sizeof(hwe->data));
377418 wmb();
378419 hwe->ib1 = entry->ib1;
@@ -380,13 +421,77 @@ int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, · 1 unmodified line
380421 dma_wmb();
381422
382423 mtk_ppe_cache_clear(ppe);
424+}
383425
384− return hash;
426+void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
427+{
428+ spin_lock_bh(&ppe_lock);
429+ hlist_del_init(&entry->list);
430+ if (entry->hash != 0xffff) {
431+ ppe->foe_table[entry->hash].ib1 &= ~MTK_FOE_IB1_STATE;
432+ ppe->foe_table[entry->hash].ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE,
433+ MTK_FOE_STATE_BIND);
434+ dma_wmb();
435+ }
436+ entry->hash = 0xffff;
437+ spin_unlock_bh(&ppe_lock);
438+}
439+
440+int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
441+{
442+ u32 hash = mtk_ppe_hash_entry(&entry->data);
443+
444+ entry->hash = 0xffff;
445+ spin_lock_bh(&ppe_lock);
446+ hlist_add_head(&entry->list, &ppe->foe_flow[hash / 2]);
447+ spin_unlock_bh(&ppe_lock);
448+
449+ return 0;
450+}
451+
452+void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
453+{
454+ struct hlist_head *head = &ppe->foe_flow[hash / 2];
455+ struct mtk_flow_entry *entry;
456+ struct mtk_foe_entry *hwe = &ppe->foe_table[hash];
457+ bool found = false;
458+
459+ if (hlist_empty(head))
460+ return;
461+
462+ spin_lock_bh(&ppe_lock);
463+ hlist_for_each_entry(entry, head, list) {
464+ if (found || !mtk_flow_entry_match(entry, hwe)) {
465+ if (entry->hash != 0xffff)
466+ entry->hash = 0xffff;
467+ continue;
468+ }
469+
470+ entry->hash = hash;
471+ __mtk_foe_entry_commit(ppe, &entry->data, hash);
472+ found = true;
473+ }
474+ spin_unlock_bh(&ppe_lock);
475+}
476+
477+int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
478+{
479+ u16 now = mtk_eth_timestamp(ppe->eth) & MTK_FOE_IB1_BIND_TIMESTAMP;
480+ u16 timestamp;
481+
482+ mtk_flow_entry_update(ppe, entry);
483+ timestamp = entry->data.ib1 & MTK_FOE_IB1_BIND_TIMESTAMP;
484+
485+ if (timestamp > now)
486+ return MTK_FOE_IB1_BIND_TIMESTAMP + 1 - timestamp + now;
487+ else
488+ return now - timestamp;
385489 }
386490
387−struct mtk_ppe *mtk_ppe_init(struct device *dev, void __iomem *base,
491+struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
388492 int version)
389493 {
494+ struct device *dev = eth->dev;
390495 struct mtk_foe_entry *foe;
391496 struct mtk_ppe *ppe;
392497
@@ -398,6 +503,7 @@ struct mtk_ppe *mtk_ppe_init(struct device *dev, void __iomem *base, · 5 unmodified lines
398503 * not coherent.
399504 */
400505 ppe->base = base;
506+ ppe->eth = eth;
401507 ppe->dev = dev;
402508 ppe->version = version;
403509
drivers/net/ethernet/mediatek/mtk_ppe.h+32−6
@@ -235,7 +235,17 @@ enum { · 234 unmodified lines
235235 MTK_PPE_CPU_REASON_INVALID = 0x1f,
236236 };
237237
238+struct mtk_flow_entry {
239+ struct rhash_head node;
240+ struct hlist_node list;
241+ unsigned long cookie;
242+ struct mtk_foe_entry data;
243+ u16 hash;
244+ s8 wed_index;
245+};
246+
238247 struct mtk_ppe {
248+ struct mtk_eth *eth;
239249 struct device *dev;
240250 void __iomem *base;
241251 int version;
@@ -243,18 +253,33 @@ struct mtk_ppe { · 1 unmodified line
243253 struct mtk_foe_entry *foe_table;
244254 dma_addr_t foe_phys;
245255
256+ u16 foe_check_time[MTK_PPE_ENTRIES];
257+ struct hlist_head foe_flow[MTK_PPE_ENTRIES / 2];
258+
246259 void *acct_table;
247260 };
248261
249−struct mtk_ppe *mtk_ppe_init(struct device *dev, void __iomem *base, int version);
262+struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int version);
250263 int mtk_ppe_start(struct mtk_ppe *ppe);
251264 int mtk_ppe_stop(struct mtk_ppe *ppe);
252265
266+void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash);
267+
253268 static inline void
254−mtk_foe_entry_clear(struct mtk_ppe *ppe, u16 hash)
269+mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
255270 {
256− ppe->foe_table[hash].ib1 = 0;
257− dma_wmb();
271+ u16 now, diff;
272+
273+ if (!ppe)
274+ return;
275+
276+ now = (u16)jiffies;
277+ diff = now - ppe->foe_check_time[hash];
278+ if (diff < HZ / 10)
279+ return;
280+
281+ ppe->foe_check_time[hash] = now;
282+ __mtk_ppe_check_skb(ppe, skb, hash);
258283 }
259284
260285 static inline int
@@ -282,8 +307,9 @@ int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid); · 21 unmodified lines
282307 int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid);
283308 int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq,
284309 int bss, int wcid);
285−int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
286− u16 timestamp);
310+int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
311+void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
312+int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
287313 int mtk_ppe_debugfs_init(struct mtk_ppe *ppe);
288314
289315 #endif
drivers/net/ethernet/mediatek/mtk_ppe_offload.c+10−32
@@ -42,13 +42,6 @@ struct mtk_flow_data { · 41 unmodified lines
4242 } pppoe;
4343 };
4444
45−struct mtk_flow_entry {
46− struct rhash_head node;
47− unsigned long cookie;
48− u16 hash;
49− s8 wed_index;
50−};
51−
5245 static const struct rhashtable_params mtk_flow_ht_params = {
5346 .head_offset = offsetof(struct mtk_flow_entry, node),
5447 .key_offset = offsetof(struct mtk_flow_entry, cookie),
@@ -56,12 +49,6 @@ static const struct rhashtable_params mtk_flow_ht_params = { · 1 unmodified line
5649 .automatic_shrinking = true,
5750 };
5851
59−static u32
60−mtk_eth_timestamp(struct mtk_eth *eth)
61−{
62− return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP;
63−}
64−
6552 static int
6653 mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data,
6754 bool egress)
@@ -237,10 +224,8 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) · 169 unmodified lines
237224 int offload_type = 0;
238225 int wed_index = -1;
239226 u16 addr_type = 0;
240− u32 timestamp;
241227 u8 l4proto = 0;
242228 int err = 0;
243− int hash;
244229 int i;
245230
246231 if (rhashtable_lookup(&eth->flow_table, &f->cookie, mtk_flow_ht_params))
@@ -410,23 +395,21 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) · 163 unmodified lines
410395 return -ENOMEM;
411396
412397 entry->cookie = f->cookie;
413− timestamp = mtk_eth_timestamp(eth);
414− hash = mtk_foe_entry_commit(eth->ppe, &foe, timestamp);
415− if (hash < 0) {
416− err = hash;
398+ memcpy(&entry->data, &foe, sizeof(entry->data));
399+ entry->wed_index = wed_index;
400+
401+ if (mtk_foe_entry_commit(eth->ppe, entry) < 0)
417402 goto free;
418− }
419403
420− entry->hash = hash;
421− entry->wed_index = wed_index;
422404 err = rhashtable_insert_fast(&eth->flow_table, &entry->node,
423405 mtk_flow_ht_params);
424406 if (err < 0)
425− goto clear_flow;
407+ goto clear;
426408
427409 return 0;
428−clear_flow:
429− mtk_foe_entry_clear(eth->ppe, hash);
410+
411+clear:
412+ mtk_foe_entry_clear(eth->ppe, entry);
430413 free:
431414 kfree(entry);
432415 if (wed_index >= 0)
@@ -444,7 +427,7 @@ mtk_flow_offload_destroy(struct mtk_eth *eth, struct flow_cls_offload *f) · 11 unmodified lines
444427 if (!entry)
445428 return -ENOENT;
446429
447− mtk_foe_entry_clear(eth->ppe, entry->hash);
430+ mtk_foe_entry_clear(eth->ppe, entry);
448431 rhashtable_remove_fast(&eth->flow_table, &entry->node,
449432 mtk_flow_ht_params);
450433 if (entry->wed_index >= 0)
@@ -458,7 +441,6 @@ static int · 7 unmodified lines
458441 mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)
459442 {
460443 struct mtk_flow_entry *entry;
461− int timestamp;
462444 u32 idle;
463445
464446 entry = rhashtable_lookup(&eth->flow_table, &f->cookie,
@@ -466,11 +448,7 @@ mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f) · 1 unmodified line
466448 if (!entry)
467449 return -ENOENT;
468450
469− timestamp = mtk_foe_entry_timestamp(eth->ppe, entry->hash);
470− if (timestamp < 0)
471− return -ETIMEDOUT;
472−
473− idle = mtk_eth_timestamp(eth) - timestamp;
451+ idle = mtk_foe_entry_idle_time(eth->ppe, entry);
474452 f->stats.lastused = jiffies - idle * HZ;
475453
476454 return 0;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t