target/linux/generic/backport-5.15/706-00-v6.0-net-ethernet-mtk_eth_soc-rely-on-page_pool-for-singl.patch

   1 From 23233e577ef973c2c5d0dd757a0a4605e34ecb57 Mon Sep 17 00:00:00 2001
   2 From: Lorenzo Bianconi <lorenzo@kernel.org>
   3 Date: Fri, 22 Jul 2022 09:19:36 +0200
   4 Subject: [PATCH] net: ethernet: mtk_eth_soc: rely on page_pool for single page
   5  buffers
   6
   7 Rely on page_pool allocator for single page buffers in order to keep
   8 them dma mapped and add skb recycling support.
   9
  10 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
  11 Signed-off-by: David S. Miller <davem@davemloft.net>
  12 ---
  13  drivers/net/ethernet/mediatek/Kconfig       |   1 +
  14  drivers/net/ethernet/mediatek/mtk_eth_soc.c | 185 +++++++++++++++-----
  15  drivers/net/ethernet/mediatek/mtk_eth_soc.h |  10 ++
  16  3 files changed, 156 insertions(+), 40 deletions(-)
  17
  18 --- a/drivers/net/ethernet/mediatek/Kconfig
  19 +++ b/drivers/net/ethernet/mediatek/Kconfig
  20 @@ -16,6 +16,7 @@ config NET_MEDIATEK_SOC
  21         depends on NET_DSA || !NET_DSA
  22         select PHYLINK
  23         select DIMLIB
  24 +       select PAGE_POOL
  25         help
  26           This driver supports the gigabit ethernet MACs in the
  27           MediaTek SoC family.
  28 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
  29 +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
  30 @@ -1389,6 +1389,68 @@ static void mtk_update_rx_cpu_idx(struct
  31         }
  32  }
  33
  34 +static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth,
  35 +                                             struct xdp_rxq_info *xdp_q,
  36 +                                             int id, int size)
  37 +{
  38 +       struct page_pool_params pp_params = {
  39 +               .order = 0,
  40 +               .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
  41 +               .pool_size = size,
  42 +               .nid = NUMA_NO_NODE,
  43 +               .dev = eth->dma_dev,
  44 +               .dma_dir = DMA_FROM_DEVICE,
  45 +               .offset = MTK_PP_HEADROOM,
  46 +               .max_len = MTK_PP_MAX_BUF_SIZE,
  47 +       };
  48 +       struct page_pool *pp;
  49 +       int err;
  50 +
  51 +       pp = page_pool_create(&pp_params);
  52 +       if (IS_ERR(pp))
  53 +               return pp;
  54 +
  55 +       err = xdp_rxq_info_reg(xdp_q, &eth->dummy_dev, id,
  56 +                               eth->rx_napi.napi_id);
  57 +       if (err < 0)
  58 +               goto err_free_pp;
  59 +
  60 +       err = xdp_rxq_info_reg_mem_model(xdp_q, MEM_TYPE_PAGE_POOL, pp);
  61 +       if (err)
  62 +               goto err_unregister_rxq;
  63 +
  64 +       return pp;
  65 +
  66 +err_unregister_rxq:
  67 +       xdp_rxq_info_unreg(xdp_q);
  68 +err_free_pp:
  69 +       page_pool_destroy(pp);
  70 +
  71 +       return ERR_PTR(err);
  72 +}
  73 +
  74 +static void *mtk_page_pool_get_buff(struct page_pool *pp, dma_addr_t *dma_addr,
  75 +                                   gfp_t gfp_mask)
  76 +{
  77 +       struct page *page;
  78 +
  79 +       page = page_pool_alloc_pages(pp, gfp_mask | __GFP_NOWARN);
  80 +       if (!page)
  81 +               return NULL;
  82 +
  83 +       *dma_addr = page_pool_get_dma_addr(page) + MTK_PP_HEADROOM;
  84 +       return page_address(page);
  85 +}
  86 +
  87 +static void mtk_rx_put_buff(struct mtk_rx_ring *ring, void *data, bool napi)
  88 +{
  89 +       if (ring->page_pool)
  90 +               page_pool_put_full_page(ring->page_pool,
  91 +                                       virt_to_head_page(data), napi);
  92 +       else
  93 +               skb_free_frag(data);
  94 +}
  95 +
  96  static int mtk_poll_rx(struct napi_struct *napi, int budget,
  97                        struct mtk_eth *eth)
  98  {
  99 @@ -1402,9 +1464,9 @@ static int mtk_poll_rx(struct napi_struc
 100
 101         while (done < budget) {
 102                 unsigned int pktlen, *rxdcsum;
 103 +               u32 hash, reason, reserve_len;
 104                 struct net_device *netdev;
 105                 dma_addr_t dma_addr;
 106 -               u32 hash, reason;
 107                 int mac = 0;
 108
 109                 ring = mtk_get_rx_ring(eth);
 110 @@ -1435,36 +1497,54 @@ static int mtk_poll_rx(struct napi_struc
 111                         goto release_desc;
 112
 113                 /* alloc new buffer */
 114 -               if (ring->frag_size <= PAGE_SIZE)
 115 -                       new_data = napi_alloc_frag(ring->frag_size);
 116 -               else
 117 -                       new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC);
 118 -               if (unlikely(!new_data)) {
 119 -                       netdev->stats.rx_dropped++;
 120 -                       goto release_desc;
 121 -               }
 122 -               dma_addr = dma_map_single(eth->dma_dev,
 123 -                                         new_data + NET_SKB_PAD +
 124 -                                         eth->ip_align,
 125 -                                         ring->buf_size,
 126 -                                         DMA_FROM_DEVICE);
 127 -               if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr))) {
 128 -                       skb_free_frag(new_data);
 129 -                       netdev->stats.rx_dropped++;
 130 -                       goto release_desc;
 131 -               }
 132 +               if (ring->page_pool) {
 133 +                       new_data = mtk_page_pool_get_buff(ring->page_pool,
 134 +                                                         &dma_addr,
 135 +                                                         GFP_ATOMIC);
 136 +                       if (unlikely(!new_data)) {
 137 +                               netdev->stats.rx_dropped++;
 138 +                               goto release_desc;
 139 +                       }
 140 +               } else {
 141 +                       if (ring->frag_size <= PAGE_SIZE)
 142 +                               new_data = napi_alloc_frag(ring->frag_size);
 143 +                       else
 144 +                               new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC);
 145 +
 146 +                       if (unlikely(!new_data)) {
 147 +                               netdev->stats.rx_dropped++;
 148 +                               goto release_desc;
 149 +                       }
 150
 151 -               dma_unmap_single(eth->dma_dev, trxd.rxd1,
 152 -                                ring->buf_size, DMA_FROM_DEVICE);
 153 +                       dma_addr = dma_map_single(eth->dma_dev,
 154 +                               new_data + NET_SKB_PAD + eth->ip_align,
 155 +                               ring->buf_size, DMA_FROM_DEVICE);
 156 +                       if (unlikely(dma_mapping_error(eth->dma_dev,
 157 +                                                      dma_addr))) {
 158 +                               skb_free_frag(new_data);
 159 +                               netdev->stats.rx_dropped++;
 160 +                               goto release_desc;
 161 +                       }
 162 +
 163 +                       dma_unmap_single(eth->dma_dev, trxd.rxd1,
 164 +                                        ring->buf_size, DMA_FROM_DEVICE);
 165 +               }
 166
 167                 /* receive data */
 168                 skb = build_skb(data, ring->frag_size);
 169                 if (unlikely(!skb)) {
 170 -                       skb_free_frag(data);
 171 +                       mtk_rx_put_buff(ring, data, true);
 172                         netdev->stats.rx_dropped++;
 173                         goto skip_rx;
 174                 }
 175 -               skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
 176 +
 177 +               if (ring->page_pool) {
 178 +                       reserve_len = MTK_PP_HEADROOM;
 179 +                       skb_mark_for_recycle(skb);
 180 +               } else {
 181 +                       reserve_len = NET_SKB_PAD + NET_IP_ALIGN;
 182 +               }
 183 +               skb_reserve(skb, reserve_len);
 184
 185                 pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
 186                 skb->dev = netdev;
 187 @@ -1518,7 +1598,6 @@ static int mtk_poll_rx(struct napi_struc
 188  skip_rx:
 189                 ring->data[idx] = new_data;
 190                 rxd->rxd1 = (unsigned int)dma_addr;
 191 -
 192  release_desc:
 193                 if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
 194                         rxd->rxd2 = RX_DMA_LSO;
 195 @@ -1526,7 +1605,6 @@ release_desc:
 196                         rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size);
 197
 198                 ring->calc_idx = idx;
 199 -
 200                 done++;
 201         }
 202
 203 @@ -1890,13 +1968,15 @@ static int mtk_rx_alloc(struct mtk_eth *
 204         if (!ring->data)
 205                 return -ENOMEM;
 206
 207 -       for (i = 0; i < rx_dma_size; i++) {
 208 -               if (ring->frag_size <= PAGE_SIZE)
 209 -                       ring->data[i] = netdev_alloc_frag(ring->frag_size);
 210 -               else
 211 -                       ring->data[i] = mtk_max_lro_buf_alloc(GFP_KERNEL);
 212 -               if (!ring->data[i])
 213 -                       return -ENOMEM;
 214 +       if (!eth->hwlro) {
 215 +               struct page_pool *pp;
 216 +
 217 +               pp = mtk_create_page_pool(eth, &ring->xdp_q, ring_no,
 218 +                                         rx_dma_size);
 219 +               if (IS_ERR(pp))
 220 +                       return PTR_ERR(pp);
 221 +
 222 +               ring->page_pool = pp;
 223         }
 224
 225         ring->dma = dma_alloc_coherent(eth->dma_dev,
 226 @@ -1907,16 +1987,33 @@ static int mtk_rx_alloc(struct mtk_eth *
 227
 228         for (i = 0; i < rx_dma_size; i++) {
 229                 struct mtk_rx_dma_v2 *rxd;
 230 -
 231 -               dma_addr_t dma_addr = dma_map_single(eth->dma_dev,
 232 -                               ring->data[i] + NET_SKB_PAD + eth->ip_align,
 233 -                               ring->buf_size,
 234 -                               DMA_FROM_DEVICE);
 235 -               if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr)))
 236 -                       return -ENOMEM;
 237 +               dma_addr_t dma_addr;
 238 +               void *data;
 239
 240                 rxd = ring->dma + i * eth->soc->txrx.rxd_size;
 241 +               if (ring->page_pool) {
 242 +                       data = mtk_page_pool_get_buff(ring->page_pool,
 243 +                                                     &dma_addr, GFP_KERNEL);
 244 +                       if (!data)
 245 +                               return -ENOMEM;
 246 +               } else {
 247 +                       if (ring->frag_size <= PAGE_SIZE)
 248 +                               data = netdev_alloc_frag(ring->frag_size);
 249 +                       else
 250 +                               data = mtk_max_lro_buf_alloc(GFP_KERNEL);
 251 +
 252 +                       if (!data)
 253 +                               return -ENOMEM;
 254 +
 255 +                       dma_addr = dma_map_single(eth->dma_dev,
 256 +                               data + NET_SKB_PAD + eth->ip_align,
 257 +                               ring->buf_size, DMA_FROM_DEVICE);
 258 +                       if (unlikely(dma_mapping_error(eth->dma_dev,
 259 +                                                      dma_addr)))
 260 +                               return -ENOMEM;
 261 +               }
 262                 rxd->rxd1 = (unsigned int)dma_addr;
 263 +               ring->data[i] = data;
 264
 265                 if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
 266                         rxd->rxd2 = RX_DMA_LSO;
 267 @@ -1932,6 +2029,7 @@ static int mtk_rx_alloc(struct mtk_eth *
 268                         rxd->rxd8 = 0;
 269                 }
 270         }
 271 +
 272         ring->dma_size = rx_dma_size;
 273         ring->calc_idx_update = false;
 274         ring->calc_idx = rx_dma_size - 1;
 275 @@ -1983,7 +2081,7 @@ static void mtk_rx_clean(struct mtk_eth
 276
 277                         dma_unmap_single(eth->dma_dev, rxd->rxd1,
 278                                          ring->buf_size, DMA_FROM_DEVICE);
 279 -                       skb_free_frag(ring->data[i]);
 280 +                       mtk_rx_put_buff(ring, ring->data[i], false);
 281                 }
 282                 kfree(ring->data);
 283                 ring->data = NULL;
 284 @@ -1995,6 +2093,13 @@ static void mtk_rx_clean(struct mtk_eth
 285                                   ring->dma, ring->phys);
 286                 ring->dma = NULL;
 287         }
 288 +
 289 +       if (ring->page_pool) {
 290 +               if (xdp_rxq_info_is_reg(&ring->xdp_q))
 291 +                       xdp_rxq_info_unreg(&ring->xdp_q);
 292 +               page_pool_destroy(ring->page_pool);
 293 +               ring->page_pool = NULL;
 294 +       }
 295  }
 296
 297  static int mtk_hwlro_rx_init(struct mtk_eth *eth)
 298 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
 299 +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
 300 @@ -18,6 +18,8 @@
 301  #include <linux/rhashtable.h>
 302  #include <linux/dim.h>
 303  #include <linux/bitfield.h>
 304 +#include <net/page_pool.h>
 305 +#include <linux/bpf_trace.h>
 306  #include "mtk_ppe.h"
 307
 308  #define MTK_QDMA_PAGE_SIZE     2048
 309 @@ -49,6 +51,11 @@
 310  #define MTK_HW_FEATURES_MT7628 (NETIF_F_SG | NETIF_F_RXCSUM)
 311  #define NEXT_DESP_IDX(X, Y)    (((X) + 1) & ((Y) - 1))
 312
 313 +#define MTK_PP_HEADROOM                XDP_PACKET_HEADROOM
 314 +#define MTK_PP_PAD             (MTK_PP_HEADROOM + \
 315 +                                SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
 316 +#define MTK_PP_MAX_BUF_SIZE    (PAGE_SIZE - MTK_PP_PAD)
 317 +
 318  #define MTK_QRX_OFFSET         0x10
 319
 320  #define MTK_MAX_RX_RING_NUM    4
 321 @@ -743,6 +750,9 @@ struct mtk_rx_ring {
 322         bool calc_idx_update;
 323         u16 calc_idx;
 324         u32 crx_idx_reg;
 325 +       /* page_pool */
 326 +       struct page_pool *page_pool;
 327 +       struct xdp_rxq_info xdp_q;
 328  };
 329
 330  enum mkt_eth_capabilities {