1 From: Felix Fietkau <nbd@nbd.name>
2 Date: Thu, 10 Dec 2020 12:19:18 +0100
3 Subject: [PATCH] net: ethernet: mtk_eth_soc: add flow offloading support
5 This adds support for offloading IPv4 routed flows, including SNAT/DNAT,
8 Signed-off-by: Felix Fietkau <nbd@nbd.name>
9 Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
11 create mode 100644 drivers/net/ethernet/mediatek/mtk_ppe_offload.c
13 --- a/drivers/net/ethernet/mediatek/Makefile
14 +++ b/drivers/net/ethernet/mediatek/Makefile
18 obj-$(CONFIG_NET_MEDIATEK_SOC) += mtk_eth.o
19 -mtk_eth-y := mtk_eth_soc.o mtk_sgmii.o mtk_eth_path.o mtk_ppe.o mtk_ppe_debugfs.o
20 +mtk_eth-y := mtk_eth_soc.o mtk_sgmii.o mtk_eth_path.o mtk_ppe.o mtk_ppe_debugfs.o mtk_ppe_offload.o
21 obj-$(CONFIG_NET_MEDIATEK_STAR_EMAC) += mtk_star_emac.o
22 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
23 +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
24 @@ -2896,6 +2896,7 @@ static const struct net_device_ops mtk_n
25 #ifdef CONFIG_NET_POLL_CONTROLLER
26 .ndo_poll_controller = mtk_poll_controller,
28 + .ndo_setup_tc = mtk_eth_setup_tc,
31 static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
32 @@ -3161,6 +3162,10 @@ static int mtk_probe(struct platform_dev
33 eth->base + MTK_ETH_PPE_BASE, 2);
37 + err = mtk_eth_offload_init(eth);
42 for (i = 0; i < MTK_MAX_DEVS; i++) {
43 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
44 +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
46 #include <linux/refcount.h>
47 #include <linux/phylink.h>
48 #include <linux/dim.h>
49 +#include <linux/rhashtable.h>
52 #define MTK_QDMA_PAGE_SIZE 2048
54 NETIF_F_HW_VLAN_CTAG_RX | \
55 NETIF_F_SG | NETIF_F_TSO | \
58 + NETIF_F_IPV6_CSUM |\
60 #define MTK_HW_FEATURES_MT7628 (NETIF_F_SG | NETIF_F_RXCSUM)
61 #define NEXT_DESP_IDX(X, Y) (((X) + 1) & ((Y) - 1))
63 @@ -929,6 +931,7 @@ struct mtk_eth {
67 + struct rhashtable flow_table;
70 /* struct mtk_mac - the structure that holds the info about the MACs of the
71 @@ -973,4 +976,9 @@ int mtk_gmac_sgmii_path_setup(struct mtk
72 int mtk_gmac_gephy_path_setup(struct mtk_eth *eth, int mac_id);
73 int mtk_gmac_rgmii_path_setup(struct mtk_eth *eth, int mac_id);
75 +int mtk_eth_offload_init(struct mtk_eth *eth);
76 +int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type,
80 #endif /* MTK_ETH_H */
82 +++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
84 +// SPDX-License-Identifier: GPL-2.0-only
86 + * Copyright (C) 2020 Felix Fietkau <nbd@nbd.name>
89 +#include <linux/if_ether.h>
90 +#include <linux/rhashtable.h>
91 +#include <linux/if_ether.h>
92 +#include <linux/ip.h>
93 +#include <net/flow_offload.h>
94 +#include <net/pkt_cls.h>
96 +#include "mtk_eth_soc.h"
98 +struct mtk_flow_data {
122 +struct mtk_flow_entry {
123 + struct rhash_head node;
124 + unsigned long cookie;
128 +static const struct rhashtable_params mtk_flow_ht_params = {
129 + .head_offset = offsetof(struct mtk_flow_entry, node),
130 + .head_offset = offsetof(struct mtk_flow_entry, cookie),
131 + .key_len = sizeof(unsigned long),
132 + .automatic_shrinking = true,
136 +mtk_eth_timestamp(struct mtk_eth *eth)
138 + return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP;
142 +mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data,
145 + return mtk_foe_entry_set_ipv4_tuple(foe, egress,
146 + data->v4.src_addr, data->src_port,
147 + data->v4.dst_addr, data->dst_port);
151 +mtk_flow_offload_mangle_eth(const struct flow_action_entry *act, void *eth)
153 + void *dest = eth + act->mangle.offset;
154 + const void *src = &act->mangle.val;
156 + if (act->mangle.offset > 8)
159 + if (act->mangle.mask == 0xffff) {
164 + memcpy(dest, src, act->mangle.mask ? 2 : 4);
169 +mtk_flow_mangle_ports(const struct flow_action_entry *act,
170 + struct mtk_flow_data *data)
172 + u32 val = ntohl(act->mangle.val);
174 + switch (act->mangle.offset) {
176 + if (act->mangle.mask == ~htonl(0xffff))
177 + data->dst_port = cpu_to_be16(val);
179 + data->src_port = cpu_to_be16(val >> 16);
182 + data->dst_port = cpu_to_be16(val);
192 +mtk_flow_mangle_ipv4(const struct flow_action_entry *act,
193 + struct mtk_flow_data *data)
197 + switch (act->mangle.offset) {
198 + case offsetof(struct iphdr, saddr):
199 + dest = &data->v4.src_addr;
201 + case offsetof(struct iphdr, daddr):
202 + dest = &data->v4.dst_addr;
208 + memcpy(dest, &act->mangle.val, sizeof(u32));
214 +mtk_flow_get_dsa_port(struct net_device **dev)
216 +#if IS_ENABLED(CONFIG_NET_DSA)
217 + struct dsa_port *dp;
219 + dp = dsa_port_from_netdev(*dev);
226 + if (!dp->cpu_dp->tag_ops)
229 + if (dp->cpu_dp->tag_ops->proto != DSA_TAG_PROTO_MTK)
232 + *dev = dp->cpu_dp->master;
241 +mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe,
242 + struct net_device *dev)
244 + int pse_port, dsa_port;
246 + dsa_port = mtk_flow_get_dsa_port(&dev);
248 + mtk_foe_entry_set_dsa(foe, dsa_port);
250 + if (dev == eth->netdev[0])
252 + else if (dev == eth->netdev[1])
255 + return -EOPNOTSUPP;
257 + mtk_foe_entry_set_pse_port(foe, pse_port);
263 +mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
265 + struct flow_rule *rule = flow_cls_offload_flow_rule(f);
266 + struct flow_action_entry *act;
267 + struct mtk_flow_data data = {};
268 + struct mtk_foe_entry foe;
269 + struct net_device *odev = NULL;
270 + struct mtk_flow_entry *entry;
271 + int offload_type = 0;
279 + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) {
280 + struct flow_match_meta match;
282 + flow_rule_match_meta(rule, &match);
284 + return -EOPNOTSUPP;
287 + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
288 + struct flow_match_control match;
290 + flow_rule_match_control(rule, &match);
291 + addr_type = match.key->addr_type;
293 + return -EOPNOTSUPP;
296 + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
297 + struct flow_match_basic match;
299 + flow_rule_match_basic(rule, &match);
300 + l4proto = match.key->ip_proto;
302 + return -EOPNOTSUPP;
305 + flow_action_for_each(i, act, &rule->action) {
307 + case FLOW_ACTION_MANGLE:
308 + if (act->mangle.htype == FLOW_ACT_MANGLE_HDR_TYPE_ETH)
309 + mtk_flow_offload_mangle_eth(act, &data.eth);
311 + case FLOW_ACTION_REDIRECT:
314 + case FLOW_ACTION_CSUM:
316 + case FLOW_ACTION_VLAN_PUSH:
317 + if (data.vlan.num == 1 ||
318 + act->vlan.proto != htons(ETH_P_8021Q))
319 + return -EOPNOTSUPP;
321 + data.vlan.id = act->vlan.vid;
322 + data.vlan.proto = act->vlan.proto;
325 + case FLOW_ACTION_PPPOE_PUSH:
326 + if (data.pppoe.num == 1)
327 + return -EOPNOTSUPP;
329 + data.pppoe.sid = act->pppoe.sid;
333 + return -EOPNOTSUPP;
337 + switch (addr_type) {
338 + case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
339 + offload_type = MTK_PPE_PKT_TYPE_IPV4_HNAPT;
342 + return -EOPNOTSUPP;
345 + if (!is_valid_ether_addr(data.eth.h_source) ||
346 + !is_valid_ether_addr(data.eth.h_dest))
349 + err = mtk_foe_entry_prepare(&foe, offload_type, l4proto, 0,
355 + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
356 + struct flow_match_ports ports;
358 + flow_rule_match_ports(rule, &ports);
359 + data.src_port = ports.key->src;
360 + data.dst_port = ports.key->dst;
362 + return -EOPNOTSUPP;
365 + if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
366 + struct flow_match_ipv4_addrs addrs;
368 + flow_rule_match_ipv4_addrs(rule, &addrs);
370 + data.v4.src_addr = addrs.key->src;
371 + data.v4.dst_addr = addrs.key->dst;
373 + mtk_flow_set_ipv4_addr(&foe, &data, false);
376 + flow_action_for_each(i, act, &rule->action) {
377 + if (act->id != FLOW_ACTION_MANGLE)
380 + switch (act->mangle.htype) {
381 + case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
382 + case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
383 + err = mtk_flow_mangle_ports(act, &data);
385 + case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
386 + err = mtk_flow_mangle_ipv4(act, &data);
388 + case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
389 + /* handled earlier */
392 + return -EOPNOTSUPP;
399 + if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
400 + err = mtk_flow_set_ipv4_addr(&foe, &data, true);
405 + if (data.vlan.num == 1) {
406 + if (data.vlan.proto != htons(ETH_P_8021Q))
407 + return -EOPNOTSUPP;
409 + mtk_foe_entry_set_vlan(&foe, data.vlan.id);
411 + if (data.pppoe.num == 1)
412 + mtk_foe_entry_set_pppoe(&foe, data.pppoe.sid);
414 + err = mtk_flow_set_output_device(eth, &foe, odev);
418 + entry = kzalloc(sizeof(*entry), GFP_KERNEL);
422 + entry->cookie = f->cookie;
423 + timestamp = mtk_eth_timestamp(eth);
424 + hash = mtk_foe_entry_commit(ð->ppe, &foe, timestamp);
430 + entry->hash = hash;
431 + err = rhashtable_insert_fast(ð->flow_table, &entry->node,
432 + mtk_flow_ht_params);
438 + mtk_foe_entry_clear(ð->ppe, hash);
445 +mtk_flow_offload_destroy(struct mtk_eth *eth, struct flow_cls_offload *f)
447 + struct mtk_flow_entry *entry;
449 + entry = rhashtable_lookup(ð->flow_table, &f->cookie,
450 + mtk_flow_ht_params);
454 + mtk_foe_entry_clear(ð->ppe, entry->hash);
455 + rhashtable_remove_fast(ð->flow_table, &entry->node,
456 + mtk_flow_ht_params);
463 +mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)
465 + struct mtk_flow_entry *entry;
469 + entry = rhashtable_lookup(ð->flow_table, &f->cookie,
470 + mtk_flow_ht_params);
474 + timestamp = mtk_foe_entry_timestamp(ð->ppe, entry->hash);
478 + idle = mtk_eth_timestamp(eth) - timestamp;
479 + f->stats.lastused = jiffies - idle * HZ;
485 +mtk_eth_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
487 + struct flow_cls_offload *cls = type_data;
488 + struct net_device *dev = cb_priv;
489 + struct mtk_mac *mac = netdev_priv(dev);
490 + struct mtk_eth *eth = mac->hw;
492 + if (!tc_can_offload(dev))
493 + return -EOPNOTSUPP;
495 + if (type != TC_SETUP_CLSFLOWER)
496 + return -EOPNOTSUPP;
498 + switch (cls->command) {
499 + case FLOW_CLS_REPLACE:
500 + return mtk_flow_offload_replace(eth, cls);
501 + case FLOW_CLS_DESTROY:
502 + return mtk_flow_offload_destroy(eth, cls);
503 + case FLOW_CLS_STATS:
504 + return mtk_flow_offload_stats(eth, cls);
506 + return -EOPNOTSUPP;
513 +mtk_eth_setup_tc_block(struct net_device *dev, struct flow_block_offload *f)
515 + struct mtk_mac *mac = netdev_priv(dev);
516 + struct mtk_eth *eth = mac->hw;
517 + static LIST_HEAD(block_cb_list);
518 + struct flow_block_cb *block_cb;
519 + flow_setup_cb_t *cb;
521 + if (!eth->ppe.foe_table)
522 + return -EOPNOTSUPP;
524 + if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
525 + return -EOPNOTSUPP;
527 + cb = mtk_eth_setup_tc_block_cb;
528 + f->driver_block_list = &block_cb_list;
530 + switch (f->command) {
531 + case FLOW_BLOCK_BIND:
532 + block_cb = flow_block_cb_lookup(f->block, cb, dev);
534 + flow_block_cb_incref(block_cb);
537 + block_cb = flow_block_cb_alloc(cb, dev, dev, NULL);
538 + if (IS_ERR(block_cb))
539 + return PTR_ERR(block_cb);
541 + flow_block_cb_add(block_cb, f);
542 + list_add_tail(&block_cb->driver_list, &block_cb_list);
544 + case FLOW_BLOCK_UNBIND:
545 + block_cb = flow_block_cb_lookup(f->block, cb, dev);
549 + if (flow_block_cb_decref(block_cb)) {
550 + flow_block_cb_remove(block_cb, f);
551 + list_del(&block_cb->driver_list);
555 + return -EOPNOTSUPP;
559 +int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type,
562 + if (type == TC_SETUP_FT)
563 + return mtk_eth_setup_tc_block(dev, type_data);
565 + return -EOPNOTSUPP;
568 +int mtk_eth_offload_init(struct mtk_eth *eth)
570 + if (!eth->ppe.foe_table)
573 + return rhashtable_init(ð->flow_table, &mtk_flow_ht_params);