target/linux/generic/backport-6.1/020-v6.4-19-mm-Multi-gen-LRU-remove-wait_event_killable.patch

   1 From 418038c22452df38cde519cc8c662bb15139764a Mon Sep 17 00:00:00 2001
   2 From: Kalesh Singh <kaleshsingh@google.com>
   3 Date: Thu, 13 Apr 2023 14:43:26 -0700
   4 Subject: [PATCH 19/19] mm: Multi-gen LRU: remove wait_event_killable()
   5
   6 Android 14 and later default to MGLRU [1] and field telemetry showed
   7 occasional long tail latency (>100ms) in the reclaim path.
   8
   9 Tracing revealed priority inversion in the reclaim path.  In
  10 try_to_inc_max_seq(), when high priority tasks were blocked on
  11 wait_event_killable(), the preemption of the low priority task to call
  12 wake_up_all() caused those high priority tasks to wait longer than
  13 necessary.  In general, this problem is not different from others of its
  14 kind, e.g., one caused by mutex_lock().  However, it is specific to MGLRU
  15 because it introduced the new wait queue lruvec->mm_state.wait.
  16
  17 The purpose of this new wait queue is to avoid the thundering herd
  18 problem.  If many direct reclaimers rush into try_to_inc_max_seq(), only
  19 one can succeed, i.e., the one to wake up the rest, and the rest who
  20 failed might cause premature OOM kills if they do not wait.  So far there
  21 is no evidence supporting this scenario, based on how often the wait has
  22 been hit.  And this begs the question how useful the wait queue is in
  23 practice.
  24
  25 Based on Minchan's recommendation, which is in line with his commit
  26 6d4675e60135 ("mm: don't be stuck to rmap lock on reclaim path") and the
  27 rest of the MGLRU code which also uses trylock when possible, remove the
  28 wait queue.
  29
  30 [1] https://android-review.googlesource.com/q/I7ed7fbfd6ef9ce10053347528125dd98c39e50bf
  31
  32 Link: https://lkml.kernel.org/r/20230413214326.2147568-1-kaleshsingh@google.com
  33 Fixes: bd74fdaea146 ("mm: multi-gen LRU: support page table walks")
  34 Signed-off-by: Kalesh Singh <kaleshsingh@google.com>
  35 Suggested-by: Minchan Kim <minchan@kernel.org>
  36 Reported-by: Wei Wang <wvw@google.com>
  37 Acked-by: Yu Zhao <yuzhao@google.com>
  38 Cc: Minchan Kim <minchan@kernel.org>
  39 Cc: Jan Alexander Steffens (heftig) <heftig@archlinux.org>
  40 Cc: Oleksandr Natalenko <oleksandr@natalenko.name>
  41 Cc: Suleiman Souhlal <suleiman@google.com>
  42 Cc: Suren Baghdasaryan <surenb@google.com>
  43 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  44 ---
  45  include/linux/mmzone.h |   8 +--
  46  mm/vmscan.c            | 112 +++++++++++++++--------------------------
  47  2 files changed, 42 insertions(+), 78 deletions(-)
  48
  49 --- a/include/linux/mmzone.h
  50 +++ b/include/linux/mmzone.h
  51 @@ -453,18 +453,14 @@ enum {
  52  struct lru_gen_mm_state {
  53         /* set to max_seq after each iteration */
  54         unsigned long seq;
  55 -       /* where the current iteration continues (inclusive) */
  56 +       /* where the current iteration continues after */
  57         struct list_head *head;
  58 -       /* where the last iteration ended (exclusive) */
  59 +       /* where the last iteration ended before */
  60         struct list_head *tail;
  61 -       /* to wait for the last page table walker to finish */
  62 -       struct wait_queue_head wait;
  63         /* Bloom filters flip after each iteration */
  64         unsigned long *filters[NR_BLOOM_FILTERS];
  65         /* the mm stats for debugging */
  66         unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
  67 -       /* the number of concurrent page table walkers */
  68 -       int nr_walkers;
  69  };
  70
  71  struct lru_gen_mm_walk {
  72 --- a/mm/vmscan.c
  73 +++ b/mm/vmscan.c
  74 @@ -3371,18 +3371,13 @@ void lru_gen_del_mm(struct mm_struct *mm
  75                 if (!lruvec)
  76                         continue;
  77
  78 -               /* where the last iteration ended (exclusive) */
  79 +               /* where the current iteration continues after */
  80 +               if (lruvec->mm_state.head == &mm->lru_gen.list)
  81 +                       lruvec->mm_state.head = lruvec->mm_state.head->prev;
  82 +
  83 +               /* where the last iteration ended before */
  84                 if (lruvec->mm_state.tail == &mm->lru_gen.list)
  85                         lruvec->mm_state.tail = lruvec->mm_state.tail->next;
  86 -
  87 -               /* where the current iteration continues (inclusive) */
  88 -               if (lruvec->mm_state.head != &mm->lru_gen.list)
  89 -                       continue;
  90 -
  91 -               lruvec->mm_state.head = lruvec->mm_state.head->next;
  92 -               /* the deletion ends the current iteration */
  93 -               if (lruvec->mm_state.head == &mm_list->fifo)
  94 -                       WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1);
  95         }
  96
  97         list_del_init(&mm->lru_gen.list);
  98 @@ -3478,68 +3473,54 @@ static bool iterate_mm_list(struct lruve
  99                             struct mm_struct **iter)
 100  {
 101         bool first = false;
 102 -       bool last = true;
 103 +       bool last = false;
 104         struct mm_struct *mm = NULL;
 105         struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 106         struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
 107         struct lru_gen_mm_state *mm_state = &lruvec->mm_state;
 108
 109         /*
 110 -        * There are four interesting cases for this page table walker:
 111 -        * 1. It tries to start a new iteration of mm_list with a stale max_seq;
 112 -        *    there is nothing left to do.
 113 -        * 2. It's the first of the current generation, and it needs to reset
 114 -        *    the Bloom filter for the next generation.
 115 -        * 3. It reaches the end of mm_list, and it needs to increment
 116 -        *    mm_state->seq; the iteration is done.
 117 -        * 4. It's the last of the current generation, and it needs to reset the
 118 -        *    mm stats counters for the next generation.
 119 +        * mm_state->seq is incremented after each iteration of mm_list. There
 120 +        * are three interesting cases for this page table walker:
 121 +        * 1. It tries to start a new iteration with a stale max_seq: there is
 122 +        *    nothing left to do.
 123 +        * 2. It started the next iteration: it needs to reset the Bloom filter
 124 +        *    so that a fresh set of PTE tables can be recorded.
 125 +        * 3. It ended the current iteration: it needs to reset the mm stats
 126 +        *    counters and tell its caller to increment max_seq.
 127          */
 128         spin_lock(&mm_list->lock);
 129
 130         VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq);
 131 -       VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq);
 132 -       VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers);
 133
 134 -       if (walk->max_seq <= mm_state->seq) {
 135 -               if (!*iter)
 136 -                       last = false;
 137 +       if (walk->max_seq <= mm_state->seq)
 138                 goto done;
 139 -       }
 140
 141 -       if (!mm_state->nr_walkers) {
 142 -               VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo);
 143 +       if (!mm_state->head)
 144 +               mm_state->head = &mm_list->fifo;
 145
 146 -               mm_state->head = mm_list->fifo.next;
 147 +       if (mm_state->head == &mm_list->fifo)
 148                 first = true;
 149 -       }
 150 -
 151 -       while (!mm && mm_state->head != &mm_list->fifo) {
 152 -               mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list);
 153
 154 +       do {
 155                 mm_state->head = mm_state->head->next;
 156 +               if (mm_state->head == &mm_list->fifo) {
 157 +                       WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
 158 +                       last = true;
 159 +                       break;
 160 +               }
 161
 162                 /* force scan for those added after the last iteration */
 163 -               if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) {
 164 -                       mm_state->tail = mm_state->head;
 165 +               if (!mm_state->tail || mm_state->tail == mm_state->head) {
 166 +                       mm_state->tail = mm_state->head->next;
 167                         walk->force_scan = true;
 168                 }
 169
 170 +               mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list);
 171                 if (should_skip_mm(mm, walk))
 172                         mm = NULL;
 173 -       }
 174 -
 175 -       if (mm_state->head == &mm_list->fifo)
 176 -               WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
 177 +       } while (!mm);
 178  done:
 179 -       if (*iter && !mm)
 180 -               mm_state->nr_walkers--;
 181 -       if (!*iter && mm)
 182 -               mm_state->nr_walkers++;
 183 -
 184 -       if (mm_state->nr_walkers)
 185 -               last = false;
 186 -
 187         if (*iter || last)
 188                 reset_mm_stats(lruvec, walk, last);
 189
 190 @@ -3567,9 +3548,9 @@ static bool iterate_mm_list_nowalk(struc
 191
 192         VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);
 193
 194 -       if (max_seq > mm_state->seq && !mm_state->nr_walkers) {
 195 -               VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo);
 196 -
 197 +       if (max_seq > mm_state->seq) {
 198 +               mm_state->head = NULL;
 199 +               mm_state->tail = NULL;
 200                 WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
 201                 reset_mm_stats(lruvec, NULL, true);
 202                 success = true;
 203 @@ -4172,10 +4153,6 @@ restart:
 204
 205                 walk_pmd_range(&val, addr, next, args);
 206
 207 -               /* a racy check to curtail the waiting time */
 208 -               if (wq_has_sleeper(&walk->lruvec->mm_state.wait))
 209 -                       return 1;
 210 -
 211                 if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
 212                         end = (addr | ~PUD_MASK) + 1;
 213                         goto done;
 214 @@ -4208,8 +4185,14 @@ static void walk_mm(struct lruvec *lruve
 215         walk->next_addr = FIRST_USER_ADDRESS;
 216
 217         do {
 218 +               DEFINE_MAX_SEQ(lruvec);
 219 +
 220                 err = -EBUSY;
 221
 222 +               /* another thread might have called inc_max_seq() */
 223 +               if (walk->max_seq != max_seq)
 224 +                       break;
 225 +
 226                 /* folio_update_gen() requires stable folio_memcg() */
 227                 if (!mem_cgroup_trylock_pages(memcg))
 228                         break;
 229 @@ -4444,25 +4427,12 @@ static bool try_to_inc_max_seq(struct lr
 230                 success = iterate_mm_list(lruvec, walk, &mm);
 231                 if (mm)
 232                         walk_mm(lruvec, mm, walk);
 233 -
 234 -               cond_resched();
 235         } while (mm);
 236  done:
 237 -       if (!success) {
 238 -               if (sc->priority <= DEF_PRIORITY - 2)
 239 -                       wait_event_killable(lruvec->mm_state.wait,
 240 -                                           max_seq < READ_ONCE(lrugen->max_seq));
 241 -               return false;
 242 -       }
 243 +       if (success)
 244 +               inc_max_seq(lruvec, can_swap, force_scan);
 245
 246 -       VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq));
 247 -
 248 -       inc_max_seq(lruvec, can_swap, force_scan);
 249 -       /* either this sees any waiters or they will see updated max_seq */
 250 -       if (wq_has_sleeper(&lruvec->mm_state.wait))
 251 -               wake_up_all(&lruvec->mm_state.wait);
 252 -
 253 -       return true;
 254 +       return success;
 255  }
 256
 257  /******************************************************************************
 258 @@ -6117,7 +6087,6 @@ void lru_gen_init_lruvec(struct lruvec *
 259                 INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
 260
 261         lruvec->mm_state.seq = MIN_NR_GENS;
 262 -       init_waitqueue_head(&lruvec->mm_state.wait);
 263  }
 264
 265  #ifdef CONFIG_MEMCG
 266 @@ -6150,7 +6119,6 @@ void lru_gen_exit_memcg(struct mem_cgrou
 267         for_each_node(nid) {
 268                 struct lruvec *lruvec = get_lruvec(memcg, nid);
 269
 270 -               VM_WARN_ON_ONCE(lruvec->mm_state.nr_walkers);
 271                 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
 272                                            sizeof(lruvec->lrugen.nr_pages)));
 273