target/linux/generic/backport-6.1/020-v6.3-17-UPSTREAM-mm-multi-gen-LRU-improve-walk_pmd_range.patch

   1 From fc0e3b06e0f19917b7ecad7967a72f61d4743644 Mon Sep 17 00:00:00 2001
   2 From: "T.J. Alumbaugh" <talumbau@google.com>
   3 Date: Wed, 18 Jan 2023 00:18:26 +0000
   4 Subject: [PATCH 17/19] UPSTREAM: mm: multi-gen LRU: improve walk_pmd_range()
   5
   6 Improve readability of walk_pmd_range() and walk_pmd_range_locked().
   7
   8 Link: https://lkml.kernel.org/r/20230118001827.1040870-7-talumbau@google.com
   9 Change-Id: Ia084fbf53fe989673b7804ca8ca520af12d7d52a
  10 Signed-off-by: T.J. Alumbaugh <talumbau@google.com>
  11 Cc: Yu Zhao <yuzhao@google.com>
  12 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  13 (cherry picked from commit b5ff4133617d0eced35b685da0bd0929dd9fabb7)
  14 Bug: 274865848
  15 Signed-off-by: T.J. Mercier <tjmercier@google.com>
  16 ---
  17  mm/vmscan.c | 40 ++++++++++++++++++++--------------------
  18  1 file changed, 20 insertions(+), 20 deletions(-)
  19
  20 diff --git a/mm/vmscan.c b/mm/vmscan.c
  21 index d5d6f8d94f58a..8f496c2e670a9 100644
  22 --- a/mm/vmscan.c
  23 +++ b/mm/vmscan.c
  24 @@ -3980,8 +3980,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
  25  }
  26
  27  #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
  28 -static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
  29 -                                 struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
  30 +static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
  31 +                                 struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
  32  {
  33         int i;
  34         pmd_t *pmd;
  35 @@ -3994,18 +3994,19 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
  36         VM_WARN_ON_ONCE(pud_leaf(*pud));
  37
  38         /* try to batch at most 1+MIN_LRU_BATCH+1 entries */
  39 -       if (*start == -1) {
  40 -               *start = next;
  41 +       if (*first == -1) {
  42 +               *first = addr;
  43 +               bitmap_zero(bitmap, MIN_LRU_BATCH);
  44                 return;
  45         }
  46
  47 -       i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start);
  48 +       i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
  49         if (i && i <= MIN_LRU_BATCH) {
  50                 __set_bit(i - 1, bitmap);
  51                 return;
  52         }
  53
  54 -       pmd = pmd_offset(pud, *start);
  55 +       pmd = pmd_offset(pud, *first);
  56
  57         ptl = pmd_lockptr(args->mm, pmd);
  58         if (!spin_trylock(ptl))
  59 @@ -4016,15 +4017,16 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
  60         do {
  61                 unsigned long pfn;
  62                 struct folio *folio;
  63 -               unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start;
  64 +
  65 +               /* don't round down the first address */
  66 +               addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first;
  67
  68                 pfn = get_pmd_pfn(pmd[i], vma, addr);
  69                 if (pfn == -1)
  70                         goto next;
  71
  72                 if (!pmd_trans_huge(pmd[i])) {
  73 -                       if (arch_has_hw_nonleaf_pmd_young() &&
  74 -                           get_cap(LRU_GEN_NONLEAF_YOUNG))
  75 +                       if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))
  76                                 pmdp_test_and_clear_young(vma, addr, pmd + i);
  77                         goto next;
  78                 }
  79 @@ -4053,12 +4055,11 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
  80         arch_leave_lazy_mmu_mode();
  81         spin_unlock(ptl);
  82  done:
  83 -       *start = -1;
  84 -       bitmap_zero(bitmap, MIN_LRU_BATCH);
  85 +       *first = -1;
  86  }
  87  #else
  88 -static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
  89 -                                 struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
  90 +static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
  91 +                                 struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
  92  {
  93  }
  94  #endif
  95 @@ -4071,9 +4072,9 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
  96         unsigned long next;
  97         unsigned long addr;
  98         struct vm_area_struct *vma;
  99 -       unsigned long pos = -1;
 100 +       unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)];
 101 +       unsigned long first = -1;
 102         struct lru_gen_mm_walk *walk = args->private;
 103 -       unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};
 104
 105         VM_WARN_ON_ONCE(pud_leaf(*pud));
 106
 107 @@ -4115,18 +4116,17 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
 108                         if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
 109                                 continue;
 110
 111 -                       walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
 112 +                       walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
 113                         continue;
 114                 }
 115  #endif
 116                 walk->mm_stats[MM_NONLEAF_TOTAL]++;
 117
 118 -               if (arch_has_hw_nonleaf_pmd_young() &&
 119 -                   get_cap(LRU_GEN_NONLEAF_YOUNG)) {
 120 +               if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) {
 121                         if (!pmd_young(val))
 122                                 continue;
 123
 124 -                       walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
 125 +                       walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
 126                 }
 127
 128                 if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
 129 @@ -4143,7 +4143,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
 130                 update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
 131         }
 132
 133 -       walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos);
 134 +       walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
 135
 136         if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
 137                 goto restart;
 138 --
 139 2.40.1
 140