target/linux/generic/backport-5.15/706-01-v6.0-net-ethernet-mtk_eth_soc-add-basic-XDP-support.patch

   1 From 7c26c20da5d420cde55618263be4aa2f6de53056 Mon Sep 17 00:00:00 2001
   2 From: Lorenzo Bianconi <lorenzo@kernel.org>
   3 Date: Fri, 22 Jul 2022 09:19:37 +0200
   4 Subject: [PATCH] net: ethernet: mtk_eth_soc: add basic XDP support
   5
   6 Introduce basic XDP support to mtk_eth_soc driver.
   7 Supported XDP verdicts:
   8 - XDP_PASS
   9 - XDP_DROP
  10 - XDP_REDIRECT
  11
  12 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
  13 Signed-off-by: David S. Miller <davem@davemloft.net>
  14 ---
  15  drivers/net/ethernet/mediatek/mtk_eth_soc.c | 162 +++++++++++++++++---
  16  drivers/net/ethernet/mediatek/mtk_eth_soc.h |   2 +
  17  2 files changed, 145 insertions(+), 19 deletions(-)
  18
  19 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
  20 +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
  21 @@ -1389,6 +1389,11 @@ static void mtk_update_rx_cpu_idx(struct
  22         }
  23  }
  24
  25 +static bool mtk_page_pool_enabled(struct mtk_eth *eth)
  26 +{
  27 +       return !eth->hwlro;
  28 +}
  29 +
  30  static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth,
  31                                               struct xdp_rxq_info *xdp_q,
  32                                               int id, int size)
  33 @@ -1451,11 +1456,52 @@ static void mtk_rx_put_buff(struct mtk_r
  34                 skb_free_frag(data);
  35  }
  36
  37 +static u32 mtk_xdp_run(struct mtk_eth *eth, struct mtk_rx_ring *ring,
  38 +                      struct xdp_buff *xdp, struct net_device *dev)
  39 +{
  40 +       struct bpf_prog *prog;
  41 +       u32 act = XDP_PASS;
  42 +
  43 +       rcu_read_lock();
  44 +
  45 +       prog = rcu_dereference(eth->prog);
  46 +       if (!prog)
  47 +               goto out;
  48 +
  49 +       act = bpf_prog_run_xdp(prog, xdp);
  50 +       switch (act) {
  51 +       case XDP_PASS:
  52 +               goto out;
  53 +       case XDP_REDIRECT:
  54 +               if (unlikely(xdp_do_redirect(dev, xdp, prog))) {
  55 +                       act = XDP_DROP;
  56 +                       break;
  57 +               }
  58 +               goto out;
  59 +       default:
  60 +               bpf_warn_invalid_xdp_action(act);
  61 +               fallthrough;
  62 +       case XDP_ABORTED:
  63 +               trace_xdp_exception(dev, prog, act);
  64 +               fallthrough;
  65 +       case XDP_DROP:
  66 +               break;
  67 +       }
  68 +
  69 +       page_pool_put_full_page(ring->page_pool,
  70 +                               virt_to_head_page(xdp->data), true);
  71 +out:
  72 +       rcu_read_unlock();
  73 +
  74 +       return act;
  75 +}
  76 +
  77  static int mtk_poll_rx(struct napi_struct *napi, int budget,
  78                        struct mtk_eth *eth)
  79  {
  80         struct dim_sample dim_sample = {};
  81         struct mtk_rx_ring *ring;
  82 +       bool xdp_flush = false;
  83         int idx;
  84         struct sk_buff *skb;
  85         u8 *data, *new_data;
  86 @@ -1464,9 +1510,9 @@ static int mtk_poll_rx(struct napi_struc
  87
  88         while (done < budget) {
  89                 unsigned int pktlen, *rxdcsum;
  90 -               u32 hash, reason, reserve_len;
  91                 struct net_device *netdev;
  92                 dma_addr_t dma_addr;
  93 +               u32 hash, reason;
  94                 int mac = 0;
  95
  96                 ring = mtk_get_rx_ring(eth);
  97 @@ -1496,8 +1542,14 @@ static int mtk_poll_rx(struct napi_struc
  98                 if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
  99                         goto release_desc;
 100
 101 +               pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
 102 +
 103                 /* alloc new buffer */
 104                 if (ring->page_pool) {
 105 +                       struct page *page = virt_to_head_page(data);
 106 +                       struct xdp_buff xdp;
 107 +                       u32 ret;
 108 +
 109                         new_data = mtk_page_pool_get_buff(ring->page_pool,
 110                                                           &dma_addr,
 111                                                           GFP_ATOMIC);
 112 @@ -1505,6 +1557,34 @@ static int mtk_poll_rx(struct napi_struc
 113                                 netdev->stats.rx_dropped++;
 114                                 goto release_desc;
 115                         }
 116 +
 117 +                       dma_sync_single_for_cpu(eth->dma_dev,
 118 +                               page_pool_get_dma_addr(page) + MTK_PP_HEADROOM,
 119 +                               pktlen, page_pool_get_dma_dir(ring->page_pool));
 120 +
 121 +                       xdp_init_buff(&xdp, PAGE_SIZE, &ring->xdp_q);
 122 +                       xdp_prepare_buff(&xdp, data, MTK_PP_HEADROOM, pktlen,
 123 +                                        false);
 124 +                       xdp_buff_clear_frags_flag(&xdp);
 125 +
 126 +                       ret = mtk_xdp_run(eth, ring, &xdp, netdev);
 127 +                       if (ret == XDP_REDIRECT)
 128 +                               xdp_flush = true;
 129 +
 130 +                       if (ret != XDP_PASS)
 131 +                               goto skip_rx;
 132 +
 133 +                       skb = build_skb(data, PAGE_SIZE);
 134 +                       if (unlikely(!skb)) {
 135 +                               page_pool_put_full_page(ring->page_pool,
 136 +                                                       page, true);
 137 +                               netdev->stats.rx_dropped++;
 138 +                               goto skip_rx;
 139 +                       }
 140 +
 141 +                       skb_reserve(skb, xdp.data - xdp.data_hard_start);
 142 +                       skb_put(skb, xdp.data_end - xdp.data);
 143 +                       skb_mark_for_recycle(skb);
 144                 } else {
 145                         if (ring->frag_size <= PAGE_SIZE)
 146                                 new_data = napi_alloc_frag(ring->frag_size);
 147 @@ -1528,27 +1608,20 @@ static int mtk_poll_rx(struct napi_struc
 148
 149                         dma_unmap_single(eth->dma_dev, trxd.rxd1,
 150                                          ring->buf_size, DMA_FROM_DEVICE);
 151 -               }
 152
 153 -               /* receive data */
 154 -               skb = build_skb(data, ring->frag_size);
 155 -               if (unlikely(!skb)) {
 156 -                       mtk_rx_put_buff(ring, data, true);
 157 -                       netdev->stats.rx_dropped++;
 158 -                       goto skip_rx;
 159 -               }
 160 +                       skb = build_skb(data, ring->frag_size);
 161 +                       if (unlikely(!skb)) {
 162 +                               netdev->stats.rx_dropped++;
 163 +                               skb_free_frag(data);
 164 +                               goto skip_rx;
 165 +                       }
 166
 167 -               if (ring->page_pool) {
 168 -                       reserve_len = MTK_PP_HEADROOM;
 169 -                       skb_mark_for_recycle(skb);
 170 -               } else {
 171 -                       reserve_len = NET_SKB_PAD + NET_IP_ALIGN;
 172 +                       skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
 173 +                       skb_put(skb, pktlen);
 174                 }
 175 -               skb_reserve(skb, reserve_len);
 176
 177 -               pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
 178                 skb->dev = netdev;
 179 -               skb_put(skb, pktlen);
 180 +               bytes += skb->len;
 181
 182                 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
 183                         rxdcsum = &trxd.rxd3;
 184 @@ -1560,7 +1633,6 @@ static int mtk_poll_rx(struct napi_struc
 185                 else
 186                         skb_checksum_none_assert(skb);
 187                 skb->protocol = eth_type_trans(skb, netdev);
 188 -               bytes += pktlen;
 189
 190                 hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY;
 191                 if (hash != MTK_RXD4_FOE_ENTRY) {
 192 @@ -1623,6 +1695,9 @@ rx_done:
 193                           &dim_sample);
 194         net_dim(&eth->rx_dim, dim_sample);
 195
 196 +       if (xdp_flush)
 197 +               xdp_do_flush_map();
 198 +
 199         return done;
 200  }
 201
 202 @@ -1968,7 +2043,7 @@ static int mtk_rx_alloc(struct mtk_eth *
 203         if (!ring->data)
 204                 return -ENOMEM;
 205
 206 -       if (!eth->hwlro) {
 207 +       if (mtk_page_pool_enabled(eth)) {
 208                 struct page_pool *pp;
 209
 210                 pp = mtk_create_page_pool(eth, &ring->xdp_q, ring_no,
 211 @@ -2710,6 +2785,48 @@ static int mtk_stop(struct net_device *d
 212         return 0;
 213  }
 214
 215 +static int mtk_xdp_setup(struct net_device *dev, struct bpf_prog *prog,
 216 +                        struct netlink_ext_ack *extack)
 217 +{
 218 +       struct mtk_mac *mac = netdev_priv(dev);
 219 +       struct mtk_eth *eth = mac->hw;
 220 +       struct bpf_prog *old_prog;
 221 +       bool need_update;
 222 +
 223 +       if (eth->hwlro) {
 224 +               NL_SET_ERR_MSG_MOD(extack, "XDP not supported with HWLRO");
 225 +               return -EOPNOTSUPP;
 226 +       }
 227 +
 228 +       if (dev->mtu > MTK_PP_MAX_BUF_SIZE) {
 229 +               NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP");
 230 +               return -EOPNOTSUPP;
 231 +       }
 232 +
 233 +       need_update = !!eth->prog != !!prog;
 234 +       if (netif_running(dev) && need_update)
 235 +               mtk_stop(dev);
 236 +
 237 +       old_prog = rcu_replace_pointer(eth->prog, prog, lockdep_rtnl_is_held());
 238 +       if (old_prog)
 239 +               bpf_prog_put(old_prog);
 240 +
 241 +       if (netif_running(dev) && need_update)
 242 +               return mtk_open(dev);
 243 +
 244 +       return 0;
 245 +}
 246 +
 247 +static int mtk_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 248 +{
 249 +       switch (xdp->command) {
 250 +       case XDP_SETUP_PROG:
 251 +               return mtk_xdp_setup(dev, xdp->prog, xdp->extack);
 252 +       default:
 253 +               return -EINVAL;
 254 +       }
 255 +}
 256 +
 257  static void ethsys_reset(struct mtk_eth *eth, u32 reset_bits)
 258  {
 259         regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL,
 260 @@ -2988,6 +3105,12 @@ static int mtk_change_mtu(struct net_dev
 261         struct mtk_eth *eth = mac->hw;
 262         u32 mcr_cur, mcr_new;
 263
 264 +       if (rcu_access_pointer(eth->prog) &&
 265 +           length > MTK_PP_MAX_BUF_SIZE) {
 266 +               netdev_err(dev, "Invalid MTU for XDP mode\n");
 267 +               return -EINVAL;
 268 +       }
 269 +
 270         if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
 271                 mcr_cur = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id));
 272                 mcr_new = mcr_cur & ~MAC_MCR_MAX_RX_MASK;
 273 @@ -3314,6 +3437,7 @@ static const struct net_device_ops mtk_n
 274         .ndo_poll_controller    = mtk_poll_controller,
 275  #endif
 276         .ndo_setup_tc           = mtk_eth_setup_tc,
 277 +       .ndo_bpf                = mtk_xdp,
 278  };
 279
 280  static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
 281 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
 282 +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
 283 @@ -1086,6 +1086,8 @@ struct mtk_eth {
 284
 285         struct mtk_ppe                  *ppe;
 286         struct rhashtable               flow_table;
 287 +
 288 +       struct bpf_prog                 __rcu *prog;
 289  };
 290
 291  /* struct mtk_mac -    the structure that holds the info about the MACs of the