target/linux/generic/backport-6.1/020-v6.3-02-UPSTREAM-mm-multi-gen-LRU-rename-lrugen-lists-to-lru.patch

   1 From 656287d55d9cfc72a4bcd4d9bd098570f12ce409 Mon Sep 17 00:00:00 2001
   2 From: Yu Zhao <yuzhao@google.com>
   3 Date: Wed, 21 Dec 2022 21:19:00 -0700
   4 Subject: [PATCH 02/19] UPSTREAM: mm: multi-gen LRU: rename lrugen->lists[] to
   5  lrugen->folios[]
   6
   7 lru_gen_folio will be chained into per-node lists by the coming
   8 lrugen->list.
   9
  10 Link: https://lkml.kernel.org/r/20221222041905.2431096-3-yuzhao@google.com
  11 Signed-off-by: Yu Zhao <yuzhao@google.com>
  12 Cc: Johannes Weiner <hannes@cmpxchg.org>
  13 Cc: Jonathan Corbet <corbet@lwn.net>
  14 Cc: Michael Larabel <Michael@MichaelLarabel.com>
  15 Cc: Michal Hocko <mhocko@kernel.org>
  16 Cc: Mike Rapoport <rppt@kernel.org>
  17 Cc: Roman Gushchin <roman.gushchin@linux.dev>
  18 Cc: Suren Baghdasaryan <surenb@google.com>
  19 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  20 Bug: 274865848
  21 (cherry picked from commit 6df1b2212950aae2b2188c6645ea18e2a9e3fdd5)
  22 Change-Id: I09f53e0fb2cd6b8b3adbb8a80b15dc5efbeae857
  23 Signed-off-by: T.J. Mercier <tjmercier@google.com>
  24 ---
  25  Documentation/mm/multigen_lru.rst |  8 ++++----
  26  include/linux/mm_inline.h         |  4 ++--
  27  include/linux/mmzone.h            |  8 ++++----
  28  mm/vmscan.c                       | 20 ++++++++++----------
  29  4 files changed, 20 insertions(+), 20 deletions(-)
  30
  31 --- a/Documentation/mm/multigen_lru.rst
  32 +++ b/Documentation/mm/multigen_lru.rst
  33 @@ -89,15 +89,15 @@ variables are monotonically increasing.
  34
  35  Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)``
  36  bits in order to fit into the gen counter in ``folio->flags``. Each
  37 -truncated generation number is an index to ``lrugen->lists[]``. The
  38 +truncated generation number is an index to ``lrugen->folios[]``. The
  39  sliding window technique is used to track at least ``MIN_NR_GENS`` and
  40  at most ``MAX_NR_GENS`` generations. The gen counter stores a value
  41  within ``[1, MAX_NR_GENS]`` while a page is on one of
  42 -``lrugen->lists[]``; otherwise it stores zero.
  43 +``lrugen->folios[]``; otherwise it stores zero.
  44
  45  Each generation is divided into multiple tiers. A page accessed ``N``
  46  times through file descriptors is in tier ``order_base_2(N)``. Unlike
  47 -generations, tiers do not have dedicated ``lrugen->lists[]``. In
  48 +generations, tiers do not have dedicated ``lrugen->folios[]``. In
  49  contrast to moving across generations, which requires the LRU lock,
  50  moving across tiers only involves atomic operations on
  51  ``folio->flags`` and therefore has a negligible cost. A feedback loop
  52 @@ -127,7 +127,7 @@ page mapped by this PTE to ``(max_seq%MA
  53  Eviction
  54  --------
  55  The eviction consumes old generations. Given an ``lruvec``, it
  56 -increments ``min_seq`` when ``lrugen->lists[]`` indexed by
  57 +increments ``min_seq`` when ``lrugen->folios[]`` indexed by
  58  ``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to
  59  evict from, it first compares ``min_seq[]`` to select the older type.
  60  If both types are equally old, it selects the one whose first tier has
  61 --- a/include/linux/mm_inline.h
  62 +++ b/include/linux/mm_inline.h
  63 @@ -256,9 +256,9 @@ static inline bool lru_gen_add_folio(str
  64         lru_gen_update_size(lruvec, folio, -1, gen);
  65         /* for folio_rotate_reclaimable() */
  66         if (reclaiming)
  67 -               list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]);
  68 +               list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
  69         else
  70 -               list_add(&folio->lru, &lrugen->lists[gen][type][zone]);
  71 +               list_add(&folio->lru, &lrugen->folios[gen][type][zone]);
  72
  73         return true;
  74  }
  75 --- a/include/linux/mmzone.h
  76 +++ b/include/linux/mmzone.h
  77 @@ -312,7 +312,7 @@ enum lruvec_flags {
  78   * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An
  79   * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the
  80   * corresponding generation. The gen counter in folio->flags stores gen+1 while
  81 - * a page is on one of lrugen->lists[]. Otherwise it stores 0.
  82 + * a page is on one of lrugen->folios[]. Otherwise it stores 0.
  83   *
  84   * A page is added to the youngest generation on faulting. The aging needs to
  85   * check the accessed bit at least twice before handing this page over to the
  86 @@ -324,8 +324,8 @@ enum lruvec_flags {
  87   * rest of generations, if they exist, are considered inactive. See
  88   * lru_gen_is_active().
  89   *
  90 - * PG_active is always cleared while a page is on one of lrugen->lists[] so that
  91 - * the aging needs not to worry about it. And it's set again when a page
  92 + * PG_active is always cleared while a page is on one of lrugen->folios[] so
  93 + * that the aging needs not to worry about it. And it's set again when a page
  94   * considered active is isolated for non-reclaiming purposes, e.g., migration.
  95   * See lru_gen_add_folio() and lru_gen_del_folio().
  96   *
  97 @@ -412,7 +412,7 @@ struct lru_gen_folio {
  98         /* the birth time of each generation in jiffies */
  99         unsigned long timestamps[MAX_NR_GENS];
 100         /* the multi-gen LRU lists, lazily sorted on eviction */
 101 -       struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
 102 +       struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
 103         /* the multi-gen LRU sizes, eventually consistent */
 104         long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
 105         /* the exponential moving average of refaulted */
 106 --- a/mm/vmscan.c
 107 +++ b/mm/vmscan.c
 108 @@ -4258,7 +4258,7 @@ static bool inc_min_seq(struct lruvec *l
 109
 110         /* prevent cold/hot inversion if force_scan is true */
 111         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
 112 -               struct list_head *head = &lrugen->lists[old_gen][type][zone];
 113 +               struct list_head *head = &lrugen->folios[old_gen][type][zone];
 114
 115                 while (!list_empty(head)) {
 116                         struct folio *folio = lru_to_folio(head);
 117 @@ -4269,7 +4269,7 @@ static bool inc_min_seq(struct lruvec *l
 118                         VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
 119
 120                         new_gen = folio_inc_gen(lruvec, folio, false);
 121 -                       list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]);
 122 +                       list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
 123
 124                         if (!--remaining)
 125                                 return false;
 126 @@ -4297,7 +4297,7 @@ static bool try_to_inc_min_seq(struct lr
 127                         gen = lru_gen_from_seq(min_seq[type]);
 128
 129                         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
 130 -                               if (!list_empty(&lrugen->lists[gen][type][zone]))
 131 +                               if (!list_empty(&lrugen->folios[gen][type][zone]))
 132                                         goto next;
 133                         }
 134
 135 @@ -4762,7 +4762,7 @@ static bool sort_folio(struct lruvec *lr
 136
 137         /* promoted */
 138         if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
 139 -               list_move(&folio->lru, &lrugen->lists[gen][type][zone]);
 140 +               list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
 141                 return true;
 142         }
 143
 144 @@ -4771,7 +4771,7 @@ static bool sort_folio(struct lruvec *lr
 145                 int hist = lru_hist_from_seq(lrugen->min_seq[type]);
 146
 147                 gen = folio_inc_gen(lruvec, folio, false);
 148 -               list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]);
 149 +               list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
 150
 151                 WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
 152                            lrugen->protected[hist][type][tier - 1] + delta);
 153 @@ -4783,7 +4783,7 @@ static bool sort_folio(struct lruvec *lr
 154         if (folio_test_locked(folio) || folio_test_writeback(folio) ||
 155             (type == LRU_GEN_FILE && folio_test_dirty(folio))) {
 156                 gen = folio_inc_gen(lruvec, folio, true);
 157 -               list_move(&folio->lru, &lrugen->lists[gen][type][zone]);
 158 +               list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
 159                 return true;
 160         }
 161
 162 @@ -4850,7 +4850,7 @@ static int scan_folios(struct lruvec *lr
 163         for (zone = sc->reclaim_idx; zone >= 0; zone--) {
 164                 LIST_HEAD(moved);
 165                 int skipped = 0;
 166 -               struct list_head *head = &lrugen->lists[gen][type][zone];
 167 +               struct list_head *head = &lrugen->folios[gen][type][zone];
 168
 169                 while (!list_empty(head)) {
 170                         struct folio *folio = lru_to_folio(head);
 171 @@ -5250,7 +5250,7 @@ static bool __maybe_unused state_is_vali
 172                 int gen, type, zone;
 173
 174                 for_each_gen_type_zone(gen, type, zone) {
 175 -                       if (!list_empty(&lrugen->lists[gen][type][zone]))
 176 +                       if (!list_empty(&lrugen->folios[gen][type][zone]))
 177                                 return false;
 178                 }
 179         }
 180 @@ -5295,7 +5295,7 @@ static bool drain_evictable(struct lruve
 181         int remaining = MAX_LRU_BATCH;
 182
 183         for_each_gen_type_zone(gen, type, zone) {
 184 -               struct list_head *head = &lruvec->lrugen.lists[gen][type][zone];
 185 +               struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];
 186
 187                 while (!list_empty(head)) {
 188                         bool success;
 189 @@ -5832,7 +5832,7 @@ void lru_gen_init_lruvec(struct lruvec *
 190                 lrugen->timestamps[i] = jiffies;
 191
 192         for_each_gen_type_zone(gen, type, zone)
 193 -               INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]);
 194 +               INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
 195
 196         lruvec->mm_state.seq = MIN_NR_GENS;
 197         init_waitqueue_head(&lruvec->mm_state.wait);