target/linux/generic/pending-6.1/680-net-add-TCP-fraglist-GRO-support.patch

   1 From: Felix Fietkau <nbd@nbd.name>
   2 Date: Tue, 23 Apr 2024 11:23:03 +0200
   3 Subject: [PATCH] net: add TCP fraglist GRO support
   4
   5 When forwarding TCP after GRO, software segmentation is very expensive,
   6 especially when the checksum needs to be recalculated.
   7 One case where that's currently unavoidable is when routing packets over
   8 PPPoE. Performance improves significantly when using fraglist GRO
   9 implemented in the same way as for UDP.
  10
  11 Here's a measurement of running 2 TCP streams through a MediaTek MT7622
  12 device (2-core Cortex-A53), which runs NAT with flow offload enabled from
  13 one ethernet port to PPPoE on another ethernet port + cake qdisc set to
  14 1Gbps.
  15
  16 rx-gro-list off: 630 Mbit/s, CPU 35% idle
  17 rx-gro-list on:  770 Mbit/s, CPU 40% idle
  18
  19 Signe-off-by: Felix Fietkau <nbd@nbd.name>
  20 ---
  21
  22 --- a/include/net/gro.h
  23 +++ b/include/net/gro.h
  24 @@ -424,6 +424,7 @@ static inline __wsum ip6_gro_compute_pse
  25  }
  26
  27  int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
  28 +int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb);
  29
  30  /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
  31  static inline void gro_normal_list(struct napi_struct *napi)
  32 @@ -446,5 +447,48 @@ static inline void gro_normal_one(struct
  33                 gro_normal_list(napi);
  34  }
  35
  36 +/* This function is the alternative of 'inet_iif' and 'inet_sdif'
  37 + * functions in case we can not rely on fields of IPCB.
  38 + *
  39 + * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized.
  40 + * The caller must hold the RCU read lock.
  41 + */
  42 +static inline void inet_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif)
  43 +{
  44 +       *iif = inet_iif(skb) ?: skb->dev->ifindex;
  45 +       *sdif = 0;
  46 +
  47 +#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
  48 +       if (netif_is_l3_slave(skb->dev)) {
  49 +               struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev);
  50 +
  51 +               *sdif = *iif;
  52 +               *iif = master ? master->ifindex : 0;
  53 +       }
  54 +#endif
  55 +}
  56 +
  57 +/* This function is the alternative of 'inet6_iif' and 'inet6_sdif'
  58 + * functions in case we can not rely on fields of IP6CB.
  59 + *
  60 + * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized.
  61 + * The caller must hold the RCU read lock.
  62 + */
  63 +static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif)
  64 +{
  65 +       /* using skb->dev->ifindex because skb_dst(skb) is not initialized */
  66 +       *iif = skb->dev->ifindex;
  67 +       *sdif = 0;
  68 +
  69 +#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
  70 +       if (netif_is_l3_slave(skb->dev)) {
  71 +               struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev);
  72 +
  73 +               *sdif = *iif;
  74 +               *iif = master ? master->ifindex : 0;
  75 +       }
  76 +#endif
  77 +}
  78 +
  79
  80  #endif /* _NET_IPV6_GRO_H */
  81 --- a/include/net/tcp.h
  82 +++ b/include/net/tcp.h
  83 @@ -2057,7 +2057,10 @@ void tcp_v4_destroy_sock(struct sock *sk
  84
  85  struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
  86                                 netdev_features_t features);
  87 -struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
  88 +struct tcphdr *tcp_gro_pull_header(struct sk_buff *skb);
  89 +struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th);
  90 +struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb,
  91 +                               struct tcphdr *th);
  92  INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff));
  93  INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb));
  94  INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff));
  95 --- a/net/core/gro.c
  96 +++ b/net/core/gro.c
  97 @@ -290,6 +290,33 @@ done:
  98         return 0;
  99  }
 100
 101 +int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
 102 +{
 103 +       if (unlikely(p->len + skb->len >= 65536))
 104 +               return -E2BIG;
 105 +
 106 +       if (NAPI_GRO_CB(p)->last == p)
 107 +               skb_shinfo(p)->frag_list = skb;
 108 +       else
 109 +               NAPI_GRO_CB(p)->last->next = skb;
 110 +
 111 +       skb_pull(skb, skb_gro_offset(skb));
 112 +
 113 +       NAPI_GRO_CB(p)->last = skb;
 114 +       NAPI_GRO_CB(p)->count++;
 115 +       p->data_len += skb->len;
 116 +
 117 +       /* sk ownership - if any - completely transferred to the aggregated packet */
 118 +       skb->destructor = NULL;
 119 +       skb->sk = NULL;
 120 +       p->truesize += skb->truesize;
 121 +       p->len += skb->len;
 122 +
 123 +       NAPI_GRO_CB(skb)->same_flow = 1;
 124 +
 125 +       return 0;
 126 +}
 127 +
 128
 129  static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
 130  {
 131 --- a/net/ipv4/tcp_offload.c
 132 +++ b/net/ipv4/tcp_offload.c
 133 @@ -27,6 +27,70 @@ static void tcp_gso_tstamp(struct sk_buf
 134         }
 135  }
 136
 137 +static void __tcpv4_gso_segment_csum(struct sk_buff *seg,
 138 +                                    __be32 *oldip, __be32 newip,
 139 +                                    __be16 *oldport, __be16 newport)
 140 +{
 141 +       struct tcphdr *th;
 142 +       struct iphdr *iph;
 143 +
 144 +       if (*oldip == newip && *oldport == newport)
 145 +               return;
 146 +
 147 +       th = tcp_hdr(seg);
 148 +       iph = ip_hdr(seg);
 149 +
 150 +       inet_proto_csum_replace4(&th->check, seg, *oldip, newip, true);
 151 +       inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false);
 152 +       *oldport = newport;
 153 +
 154 +       csum_replace4(&iph->check, *oldip, newip);
 155 +       *oldip = newip;
 156 +}
 157 +
 158 +static struct sk_buff *__tcpv4_gso_segment_list_csum(struct sk_buff *segs)
 159 +{
 160 +       const struct tcphdr *th;
 161 +       const struct iphdr *iph;
 162 +       struct sk_buff *seg;
 163 +       struct tcphdr *th2;
 164 +       struct iphdr *iph2;
 165 +
 166 +       seg = segs;
 167 +       th = tcp_hdr(seg);
 168 +       iph = ip_hdr(seg);
 169 +       th2 = tcp_hdr(seg->next);
 170 +       iph2 = ip_hdr(seg->next);
 171 +
 172 +       if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) &&
 173 +           iph->daddr == iph2->daddr && iph->saddr == iph2->saddr)
 174 +               return segs;
 175 +
 176 +       while ((seg = seg->next)) {
 177 +               th2 = tcp_hdr(seg);
 178 +               iph2 = ip_hdr(seg);
 179 +
 180 +               __tcpv4_gso_segment_csum(seg,
 181 +                                        &iph2->saddr, iph->saddr,
 182 +                                        &th2->source, th->source);
 183 +               __tcpv4_gso_segment_csum(seg,
 184 +                                        &iph2->daddr, iph->daddr,
 185 +                                        &th2->dest, th->dest);
 186 +       }
 187 +
 188 +       return segs;
 189 +}
 190 +
 191 +static struct sk_buff *__tcp4_gso_segment_list(struct sk_buff *skb,
 192 +                                             netdev_features_t features)
 193 +{
 194 +       skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
 195 +       if (IS_ERR(skb))
 196 +               return skb;
 197 +
 198 +       return __tcpv4_gso_segment_list_csum(skb);
 199 +}
 200 +
 201  static struct sk_buff *tcp4_gso_segment(struct sk_buff *skb,
 202                                         netdev_features_t features)
 203  {
 204 @@ -36,6 +100,9 @@ static struct sk_buff *tcp4_gso_segment(
 205         if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
 206                 return ERR_PTR(-EINVAL);
 207
 208 +       if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)
 209 +               return __tcp4_gso_segment_list(skb, features);
 210 +
 211         if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
 212                 const struct iphdr *iph = ip_hdr(skb);
 213                 struct tcphdr *th = tcp_hdr(skb);
 214 @@ -177,61 +244,76 @@ out:
 215         return segs;
 216  }
 217
 218 -struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb)
 219 +struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th)
 220  {
 221 -       struct sk_buff *pp = NULL;
 222 +       struct tcphdr *th2;
 223         struct sk_buff *p;
 224 +
 225 +       list_for_each_entry(p, head, list) {
 226 +               if (!NAPI_GRO_CB(p)->same_flow)
 227 +                       continue;
 228 +
 229 +               th2 = tcp_hdr(p);
 230 +               if (*(u32 *)&th->source ^ *(u32 *)&th2->source) {
 231 +                       NAPI_GRO_CB(p)->same_flow = 0;
 232 +                       continue;
 233 +               }
 234 +
 235 +               return p;
 236 +       }
 237 +
 238 +       return NULL;
 239 +}
 240 +
 241 +struct tcphdr *tcp_gro_pull_header(struct sk_buff *skb)
 242 +{
 243 +       unsigned int thlen, hlen, off;
 244         struct tcphdr *th;
 245 -       struct tcphdr *th2;
 246 -       unsigned int len;
 247 -       unsigned int thlen;
 248 -       __be32 flags;
 249 -       unsigned int mss = 1;
 250 -       unsigned int hlen;
 251 -       unsigned int off;
 252 -       int flush = 1;
 253 -       int i;
 254
 255         off = skb_gro_offset(skb);
 256         hlen = off + sizeof(*th);
 257         th = skb_gro_header(skb, hlen, off);
 258         if (unlikely(!th))
 259 -               goto out;
 260 +               return NULL;
 261
 262         thlen = th->doff * 4;
 263         if (thlen < sizeof(*th))
 264 -               goto out;
 265 +               return NULL;
 266
 267         hlen = off + thlen;
 268         if (skb_gro_header_hard(skb, hlen)) {
 269                 th = skb_gro_header_slow(skb, hlen, off);
 270                 if (unlikely(!th))
 271 -                       goto out;
 272 +                       return NULL;
 273         }
 274
 275         skb_gro_pull(skb, thlen);
 276
 277 -       len = skb_gro_len(skb);
 278 -       flags = tcp_flag_word(th);
 279 -
 280 -       list_for_each_entry(p, head, list) {
 281 -               if (!NAPI_GRO_CB(p)->same_flow)
 282 -                       continue;
 283 +       return th;
 284 +}
 285
 286 -               th2 = tcp_hdr(p);
 287 +struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb,
 288 +                               struct tcphdr *th)
 289 +{
 290 +       unsigned int thlen = th->doff * 4;
 291 +       struct sk_buff *pp = NULL;
 292 +       struct sk_buff *p;
 293 +       struct tcphdr *th2;
 294 +       unsigned int len;
 295 +       __be32 flags;
 296 +       unsigned int mss = 1;
 297 +       int flush = 1;
 298 +       int i;
 299
 300 -               if (*(u32 *)&th->source ^ *(u32 *)&th2->source) {
 301 -                       NAPI_GRO_CB(p)->same_flow = 0;
 302 -                       continue;
 303 -               }
 304 +       len = skb_gro_len(skb);
 305 +       flags = tcp_flag_word(th);
 306
 307 -               goto found;
 308 -       }
 309 -       p = NULL;
 310 -       goto out_check_final;
 311 +       p = tcp_gro_lookup(head, th);
 312 +       if (!p)
 313 +               goto out_check_final;
 314
 315 -found:
 316         /* Include the IP ID check below from the inner most IP hdr */
 317 +       th2 = tcp_hdr(p);
 318         flush = NAPI_GRO_CB(p)->flush;
 319         flush |= (__force int)(flags & TCP_FLAG_CWR);
 320         flush |= (__force int)((flags ^ tcp_flag_word(th2)) &
 321 @@ -268,6 +350,19 @@ found:
 322         flush |= p->decrypted ^ skb->decrypted;
 323  #endif
 324
 325 +       if (unlikely(NAPI_GRO_CB(p)->is_flist)) {
 326 +               flush |= (__force int)(flags ^ tcp_flag_word(th2));
 327 +               flush |= skb->ip_summed != p->ip_summed;
 328 +               flush |= skb->csum_level != p->csum_level;
 329 +               flush |= !pskb_may_pull(skb, skb_gro_offset(skb));
 330 +               flush |= NAPI_GRO_CB(p)->count >= 64;
 331 +
 332 +               if (flush || skb_gro_receive_list(p, skb))
 333 +                       mss = 1;
 334 +
 335 +               goto out_check_final;
 336 +       }
 337 +
 338         if (flush || skb_gro_receive(p, skb)) {
 339                 mss = 1;
 340                 goto out_check_final;
 341 @@ -289,7 +384,6 @@ out_check_final:
 342         if (p && (!NAPI_GRO_CB(skb)->same_flow || flush))
 343                 pp = p;
 344
 345 -out:
 346         NAPI_GRO_CB(skb)->flush |= (flush != 0);
 347
 348         return pp;
 349 @@ -315,18 +409,58 @@ int tcp_gro_complete(struct sk_buff *skb
 350  }
 351  EXPORT_SYMBOL(tcp_gro_complete);
 352
 353 +static void tcp4_check_fraglist_gro(struct list_head *head, struct sk_buff *skb,
 354 +                                   struct tcphdr *th)
 355 +{
 356 +       const struct iphdr *iph;
 357 +       struct sk_buff *p;
 358 +       struct sock *sk;
 359 +       struct net *net;
 360 +       int iif, sdif;
 361 +
 362 +       if (!(skb->dev->features & NETIF_F_GRO_FRAGLIST))
 363 +               return;
 364 +
 365 +       p = tcp_gro_lookup(head, th);
 366 +       if (p) {
 367 +               NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
 368 +               return;
 369 +       }
 370 +
 371 +       inet_get_iif_sdif(skb, &iif, &sdif);
 372 +       iph = skb_gro_network_header(skb);
 373 +       net = dev_net(skb->dev);
 374 +       sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 375 +                                      iph->saddr, th->source,
 376 +                                      iph->daddr, ntohs(th->dest),
 377 +                                      iif, sdif);
 378 +       NAPI_GRO_CB(skb)->is_flist = !sk;
 379 +       if (sk)
 380 +               sock_put(sk);
 381 +}
 382 +
 383  INDIRECT_CALLABLE_SCOPE
 384  struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)
 385  {
 386 +       struct tcphdr *th;
 387 +
 388         /* Don't bother verifying checksum if we're going to flush anyway. */
 389         if (!NAPI_GRO_CB(skb)->flush &&
 390             skb_gro_checksum_validate(skb, IPPROTO_TCP,
 391 -                                     inet_gro_compute_pseudo)) {
 392 -               NAPI_GRO_CB(skb)->flush = 1;
 393 -               return NULL;
 394 -       }
 395 +                                     inet_gro_compute_pseudo))
 396 +               goto flush;
 397 +
 398 +       th = tcp_gro_pull_header(skb);
 399 +       if (!th)
 400 +               goto flush;
 401
 402 -       return tcp_gro_receive(head, skb);
 403 +       tcp4_check_fraglist_gro(head, skb, th);
 404 +
 405 +       return tcp_gro_receive(head, skb, th);
 406 +
 407 +flush:
 408 +       NAPI_GRO_CB(skb)->flush = 1;
 409 +       return NULL;
 410  }
 411
 412  INDIRECT_CALLABLE_SCOPE int tcp4_gro_complete(struct sk_buff *skb, int thoff)
 413 @@ -334,6 +468,15 @@ INDIRECT_CALLABLE_SCOPE int tcp4_gro_com
 414         const struct iphdr *iph = ip_hdr(skb);
 415         struct tcphdr *th = tcp_hdr(skb);
 416
 417 +       if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
 418 +               skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV4;
 419 +               skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
 420 +
 421 +               __skb_incr_checksum_unnecessary(skb);
 422 +
 423 +               return 0;
 424 +       }
 425 +
 426         th->check = ~tcp_v4_check(skb->len - thoff, iph->saddr,
 427                                   iph->daddr, 0);
 428         skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4;
 429 --- a/net/ipv4/udp_offload.c
 430 +++ b/net/ipv4/udp_offload.c
 431 @@ -425,33 +425,6 @@ out:
 432         return segs;
 433  }
 434
 435 -static int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
 436 -{
 437 -       if (unlikely(p->len + skb->len >= 65536))
 438 -               return -E2BIG;
 439 -
 440 -       if (NAPI_GRO_CB(p)->last == p)
 441 -               skb_shinfo(p)->frag_list = skb;
 442 -       else
 443 -               NAPI_GRO_CB(p)->last->next = skb;
 444 -
 445 -       skb_pull(skb, skb_gro_offset(skb));
 446 -
 447 -       NAPI_GRO_CB(p)->last = skb;
 448 -       NAPI_GRO_CB(p)->count++;
 449 -       p->data_len += skb->len;
 450 -
 451 -       /* sk ownership - if any - completely transferred to the aggregated packet */
 452 -       skb->destructor = NULL;
 453 -       skb->sk = NULL;
 454 -       p->truesize += skb->truesize;
 455 -       p->len += skb->len;
 456 -
 457 -       NAPI_GRO_CB(skb)->same_flow = 1;
 458 -
 459 -       return 0;
 460 -}
 461 -
 462
 463  #define UDP_GRO_CNT_MAX 64
 464  static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
 465 --- a/net/ipv6/tcpv6_offload.c
 466 +++ b/net/ipv6/tcpv6_offload.c
 467 @@ -7,24 +7,67 @@
 468   */
 469  #include <linux/indirect_call_wrapper.h>
 470  #include <linux/skbuff.h>
 471 +#include <net/inet6_hashtables.h>
 472  #include <net/gro.h>
 473  #include <net/protocol.h>
 474  #include <net/tcp.h>
 475  #include <net/ip6_checksum.h>
 476  #include "ip6_offload.h"
 477
 478 +static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb,
 479 +                                   struct tcphdr *th)
 480 +{
 481 +#if IS_ENABLED(CONFIG_IPV6)
 482 +       const struct ipv6hdr *hdr;
 483 +       struct sk_buff *p;
 484 +       struct sock *sk;
 485 +       struct net *net;
 486 +       int iif, sdif;
 487 +
 488 +       if (!(skb->dev->features & NETIF_F_GRO_FRAGLIST))
 489 +               return;
 490 +
 491 +       p = tcp_gro_lookup(head, th);
 492 +       if (p) {
 493 +               NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
 494 +               return;
 495 +       }
 496 +
 497 +       inet6_get_iif_sdif(skb, &iif, &sdif);
 498 +       hdr = skb_gro_network_header(skb);
 499 +       net = dev_net(skb->dev);
 500 +       sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 501 +                                       &hdr->saddr, th->source,
 502 +                                       &hdr->daddr, ntohs(th->dest),
 503 +                                       iif, sdif);
 504 +       NAPI_GRO_CB(skb)->is_flist = !sk;
 505 +       if (sk)
 506 +               sock_put(sk);
 507 +#endif /* IS_ENABLED(CONFIG_IPV6) */
 508 +}
 509 +
 510  INDIRECT_CALLABLE_SCOPE
 511  struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)
 512  {
 513 +       struct tcphdr *th;
 514 +
 515         /* Don't bother verifying checksum if we're going to flush anyway. */
 516         if (!NAPI_GRO_CB(skb)->flush &&
 517             skb_gro_checksum_validate(skb, IPPROTO_TCP,
 518 -                                     ip6_gro_compute_pseudo)) {
 519 -               NAPI_GRO_CB(skb)->flush = 1;
 520 -               return NULL;
 521 -       }
 522 +                                     ip6_gro_compute_pseudo))
 523 +               goto flush;
 524
 525 -       return tcp_gro_receive(head, skb);
 526 +       th = tcp_gro_pull_header(skb);
 527 +       if (!th)
 528 +               goto flush;
 529 +
 530 +       tcp6_check_fraglist_gro(head, skb, th);
 531 +
 532 +       return tcp_gro_receive(head, skb, th);
 533 +
 534 +flush:
 535 +       NAPI_GRO_CB(skb)->flush = 1;
 536 +       return NULL;
 537  }
 538
 539  INDIRECT_CALLABLE_SCOPE int tcp6_gro_complete(struct sk_buff *skb, int thoff)
 540 @@ -32,6 +75,15 @@ INDIRECT_CALLABLE_SCOPE int tcp6_gro_com
 541         const struct ipv6hdr *iph = ipv6_hdr(skb);
 542         struct tcphdr *th = tcp_hdr(skb);
 543
 544 +       if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
 545 +               skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV6;
 546 +               skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
 547 +
 548 +               __skb_incr_checksum_unnecessary(skb);
 549 +
 550 +               return 0;
 551 +       }
 552 +
 553         th->check = ~tcp_v6_check(skb->len - thoff, &iph->saddr,
 554                                   &iph->daddr, 0);
 555         skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
 556 @@ -39,6 +91,61 @@ INDIRECT_CALLABLE_SCOPE int tcp6_gro_com
 557         return tcp_gro_complete(skb);
 558  }
 559
 560 +static void __tcpv6_gso_segment_csum(struct sk_buff *seg,
 561 +                                    __be16 *oldport, __be16 newport)
 562 +{
 563 +       struct tcphdr *th;
 564 +
 565 +       if (*oldport == newport)
 566 +               return;
 567 +
 568 +       th = tcp_hdr(seg);
 569 +       inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false);
 570 +       *oldport = newport;
 571 +}
 572 +
 573 +static struct sk_buff *__tcpv6_gso_segment_list_csum(struct sk_buff *segs)
 574 +{
 575 +       const struct tcphdr *th;
 576 +       const struct ipv6hdr *iph;
 577 +       struct sk_buff *seg;
 578 +       struct tcphdr *th2;
 579 +       struct ipv6hdr *iph2;
 580 +
 581 +       seg = segs;
 582 +       th = tcp_hdr(seg);
 583 +       iph = ipv6_hdr(seg);
 584 +       th2 = tcp_hdr(seg->next);
 585 +       iph2 = ipv6_hdr(seg->next);
 586 +
 587 +       if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) &&
 588 +           ipv6_addr_equal(&iph->saddr, &iph2->saddr) &&
 589 +           ipv6_addr_equal(&iph->daddr, &iph2->daddr))
 590 +               return segs;
 591 +
 592 +       while ((seg = seg->next)) {
 593 +               th2 = tcp_hdr(seg);
 594 +               iph2 = ipv6_hdr(seg);
 595 +
 596 +               iph2->saddr = iph->saddr;
 597 +               iph2->daddr = iph->daddr;
 598 +               __tcpv6_gso_segment_csum(seg, &th2->source, th->source);
 599 +               __tcpv6_gso_segment_csum(seg, &th2->dest, th->dest);
 600 +       }
 601 +
 602 +       return segs;
 603 +}
 604 +
 605 +static struct sk_buff *__tcp6_gso_segment_list(struct sk_buff *skb,
 606 +                                             netdev_features_t features)
 607 +{
 608 +       skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
 609 +       if (IS_ERR(skb))
 610 +               return skb;
 611 +
 612 +       return __tcpv6_gso_segment_list_csum(skb);
 613 +}
 614 +
 615  static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb,
 616                                         netdev_features_t features)
 617  {
 618 @@ -50,6 +157,9 @@ static struct sk_buff *tcp6_gso_segment(
 619         if (!pskb_may_pull(skb, sizeof(*th)))
 620                 return ERR_PTR(-EINVAL);
 621
 622 +       if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)
 623 +               return __tcp6_gso_segment_list(skb, features);
 624 +
 625         if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
 626                 const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
 627                 struct tcphdr *th = tcp_hdr(skb);