target/linux/generic/backport-6.1/020-v6.3-10-UPSTREAM-mm-add-vma_has_recency.patch

   1 From 70d216c71ff5c5b17dd1da6294f97b91fb6aba7a Mon Sep 17 00:00:00 2001
   2 From: Yu Zhao <yuzhao@google.com>
   3 Date: Fri, 30 Dec 2022 14:52:51 -0700
   4 Subject: [PATCH 10/19] UPSTREAM: mm: add vma_has_recency()
   5
   6 Add vma_has_recency() to indicate whether a VMA may exhibit temporal
   7 locality that the LRU algorithm relies on.
   8
   9 This function returns false for VMAs marked by VM_SEQ_READ or
  10 VM_RAND_READ.  While the former flag indicates linear access, i.e., a
  11 special case of spatial locality, both flags indicate a lack of temporal
  12 locality, i.e., the reuse of an area within a relatively small duration.
  13
  14 "Recency" is chosen over "locality" to avoid confusion between temporal
  15 and spatial localities.
  16
  17 Before this patch, the active/inactive LRU only ignored the accessed bit
  18 from VMAs marked by VM_SEQ_READ.  After this patch, the active/inactive
  19 LRU and MGLRU share the same logic: they both ignore the accessed bit if
  20 vma_has_recency() returns false.
  21
  22 For the active/inactive LRU, the following fio test showed a [6, 8]%
  23 increase in IOPS when randomly accessing mapped files under memory
  24 pressure.
  25
  26   kb=$(awk '/MemTotal/ { print $2 }' /proc/meminfo)
  27   kb=$((kb - 8*1024*1024))
  28
  29   modprobe brd rd_nr=1 rd_size=$kb
  30   dd if=/dev/zero of=/dev/ram0 bs=1M
  31
  32   mkfs.ext4 /dev/ram0
  33   mount /dev/ram0 /mnt/
  34   swapoff -a
  35
  36   fio --name=test --directory=/mnt/ --ioengine=mmap --numjobs=8 \
  37       --size=8G --rw=randrw --time_based --runtime=10m \
  38       --group_reporting
  39
  40 The discussion that led to this patch is here [1].  Additional test
  41 results are available in that thread.
  42
  43 [1] https://lore.kernel.org/r/Y31s%2FK8T85jh05wH@google.com/
  44
  45 Link: https://lkml.kernel.org/r/20221230215252.2628425-1-yuzhao@google.com
  46 Change-Id: I291dcb795197659e40e46539cd32b857677c34ad
  47 Signed-off-by: Yu Zhao <yuzhao@google.com>
  48 Cc: Alexander Viro <viro@zeniv.linux.org.uk>
  49 Cc: Andrea Righi <andrea.righi@canonical.com>
  50 Cc: Johannes Weiner <hannes@cmpxchg.org>
  51 Cc: Michael Larabel <Michael@MichaelLarabel.com>
  52 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  53 (cherry picked from commit 8788f6781486769d9598dcaedc3fe0eb12fc3e59)
  54 Bug: 274865848
  55 Signed-off-by: T.J. Mercier <tjmercier@google.com>
  56 ---
  57  include/linux/mm_inline.h |  8 ++++++++
  58  mm/memory.c               |  7 +++----
  59  mm/rmap.c                 | 42 +++++++++++++++++----------------------
  60  mm/vmscan.c               |  5 ++++-
  61  4 files changed, 33 insertions(+), 29 deletions(-)
  62
  63 --- a/include/linux/mm_inline.h
  64 +++ b/include/linux/mm_inline.h
  65 @@ -600,4 +600,12 @@ pte_install_uffd_wp_if_needed(struct vm_
  66  #endif
  67  }
  68
  69 +static inline bool vma_has_recency(struct vm_area_struct *vma)
  70 +{
  71 +       if (vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))
  72 +               return false;
  73 +
  74 +       return true;
  75 +}
  76 +
  77  #endif
  78 --- a/mm/memory.c
  79 +++ b/mm/memory.c
  80 @@ -1445,8 +1445,7 @@ again:
  81                                         force_flush = 1;
  82                                         set_page_dirty(page);
  83                                 }
  84 -                               if (pte_young(ptent) &&
  85 -                                   likely(!(vma->vm_flags & VM_SEQ_READ)))
  86 +                               if (pte_young(ptent) && likely(vma_has_recency(vma)))
  87                                         mark_page_accessed(page);
  88                         }
  89                         rss[mm_counter(page)]--;
  90 @@ -5219,8 +5218,8 @@ static inline void mm_account_fault(stru
  91  #ifdef CONFIG_LRU_GEN
  92  static void lru_gen_enter_fault(struct vm_area_struct *vma)
  93  {
  94 -       /* the LRU algorithm doesn't apply to sequential or random reads */
  95 -       current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ));
  96 +       /* the LRU algorithm only applies to accesses with recency */
  97 +       current->in_lru_fault = vma_has_recency(vma);
  98  }
  99
 100  static void lru_gen_exit_fault(void)
 101 --- a/mm/rmap.c
 102 +++ b/mm/rmap.c
 103 @@ -823,25 +823,14 @@ static bool folio_referenced_one(struct
 104                 }
 105
 106                 if (pvmw.pte) {
 107 -                       if (lru_gen_enabled() && pte_young(*pvmw.pte) &&
 108 -                           !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) {
 109 +                       if (lru_gen_enabled() && pte_young(*pvmw.pte)) {
 110                                 lru_gen_look_around(&pvmw);
 111                                 referenced++;
 112                         }
 113
 114                         if (ptep_clear_flush_young_notify(vma, address,
 115 -                                               pvmw.pte)) {
 116 -                               /*
 117 -                                * Don't treat a reference through
 118 -                                * a sequentially read mapping as such.
 119 -                                * If the folio has been used in another mapping,
 120 -                                * we will catch it; if this other mapping is
 121 -                                * already gone, the unmap path will have set
 122 -                                * the referenced flag or activated the folio.
 123 -                                */
 124 -                               if (likely(!(vma->vm_flags & VM_SEQ_READ)))
 125 -                                       referenced++;
 126 -                       }
 127 +                                               pvmw.pte))
 128 +                               referenced++;
 129                 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
 130                         if (pmdp_clear_flush_young_notify(vma, address,
 131                                                 pvmw.pmd))
 132 @@ -875,7 +864,20 @@ static bool invalid_folio_referenced_vma
 133         struct folio_referenced_arg *pra = arg;
 134         struct mem_cgroup *memcg = pra->memcg;
 135
 136 -       if (!mm_match_cgroup(vma->vm_mm, memcg))
 137 +       /*
 138 +        * Ignore references from this mapping if it has no recency. If the
 139 +        * folio has been used in another mapping, we will catch it; if this
 140 +        * other mapping is already gone, the unmap path will have set the
 141 +        * referenced flag or activated the folio in zap_pte_range().
 142 +        */
 143 +       if (!vma_has_recency(vma))
 144 +               return true;
 145 +
 146 +       /*
 147 +        * If we are reclaiming on behalf of a cgroup, skip counting on behalf
 148 +        * of references from different cgroups.
 149 +        */
 150 +       if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
 151                 return true;
 152
 153         return false;
 154 @@ -906,6 +908,7 @@ int folio_referenced(struct folio *folio
 155                 .arg = (void *)&pra,
 156                 .anon_lock = folio_lock_anon_vma_read,
 157                 .try_lock = true,
 158 +               .invalid_vma = invalid_folio_referenced_vma,
 159         };
 160
 161         *vm_flags = 0;
 162 @@ -921,15 +924,6 @@ int folio_referenced(struct folio *folio
 163                         return 1;
 164         }
 165
 166 -       /*
 167 -        * If we are reclaiming on behalf of a cgroup, skip
 168 -        * counting on behalf of references from different
 169 -        * cgroups
 170 -        */
 171 -       if (memcg) {
 172 -               rwc.invalid_vma = invalid_folio_referenced_vma;
 173 -       }
 174 -
 175         rmap_walk(folio, &rwc);
 176         *vm_flags = pra.vm_flags;
 177
 178 --- a/mm/vmscan.c
 179 +++ b/mm/vmscan.c
 180 @@ -3778,7 +3778,10 @@ static int should_skip_vma(unsigned long
 181         if (is_vm_hugetlb_page(vma))
 182                 return true;
 183
 184 -       if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ))
 185 +       if (!vma_has_recency(vma))
 186 +               return true;
 187 +
 188 +       if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL))
 189                 return true;
 190
 191         if (vma == get_gate_vma(vma->vm_mm))