target/linux/generic/backport-5.15/020-v6.1-14-mm-multi-gen-LRU-retry-pages-written-back-while-isol.patch

   1 From 6f315879ad750391a0b1fab8c9170bc054a5f5d7 Mon Sep 17 00:00:00 2001
   2 From: Yu Zhao <yuzhao@google.com>
   3 Date: Tue, 15 Nov 2022 18:38:07 -0700
   4 Subject: [PATCH 14/29] mm: multi-gen LRU: retry pages written back while
   5  isolated
   6
   7 The page reclaim isolates a batch of pages from the tail of one of the
   8 LRU lists and works on those pages one by one.  For a suitable
   9 swap-backed page, if the swap device is async, it queues that page for
  10 writeback.  After the page reclaim finishes an entire batch, it puts back
  11 the pages it queued for writeback to the head of the original LRU list.
  12
  13 In the meantime, the page writeback flushes the queued pages also by
  14 batches.  Its batching logic is independent from that of the page reclaim.
  15 For each of the pages it writes back, the page writeback calls
  16 rotate_reclaimable_page() which tries to rotate a page to the tail.
  17
  18 rotate_reclaimable_page() only works for a page after the page reclaim
  19 has put it back.  If an async swap device is fast enough, the page
  20 writeback can finish with that page while the page reclaim is still
  21 working on the rest of the batch containing it.  In this case, that page
  22 will remain at the head and the page reclaim will not retry it before
  23 reaching there.
  24
  25 This patch adds a retry to evict_pages().  After evict_pages() has
  26 finished an entire batch and before it puts back pages it cannot free
  27 immediately, it retries those that may have missed the rotation.
  28
  29 Before this patch, ~60% of pages swapped to an Intel Optane missed
  30 rotate_reclaimable_page().  After this patch, ~99% of missed pages were
  31 reclaimed upon retry.
  32
  33 This problem affects relatively slow async swap devices like Samsung 980
  34 Pro much less and does not affect sync swap devices like zram or zswap at
  35 all.
  36
  37 Link: https://lkml.kernel.org/r/20221116013808.3995280-1-yuzhao@google.com
  38 Fixes: ac35a4902374 ("mm: multi-gen LRU: minimal implementation")
  39 Signed-off-by: Yu Zhao <yuzhao@google.com>
  40 Cc: "Yin, Fengwei" <fengwei.yin@intel.com>
  41 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  42 ---
  43  mm/vmscan.c | 48 +++++++++++++++++++++++++++++++++++++-----------
  44  1 file changed, 37 insertions(+), 11 deletions(-)
  45
  46 --- a/mm/vmscan.c
  47 +++ b/mm/vmscan.c
  48 @@ -4723,10 +4723,13 @@ static int evict_pages(struct lruvec *lr
  49         int scanned;
  50         int reclaimed;
  51         LIST_HEAD(list);
  52 +       LIST_HEAD(clean);
  53         struct page *page;
  54 +       struct page *next;
  55         enum vm_event_item item;
  56         struct reclaim_stat stat;
  57         struct lru_gen_mm_walk *walk;
  58 +       bool skip_retry = false;
  59         struct mem_cgroup *memcg = lruvec_memcg(lruvec);
  60         struct pglist_data *pgdat = lruvec_pgdat(lruvec);
  61
  62 @@ -4743,20 +4746,37 @@ static int evict_pages(struct lruvec *lr
  63
  64         if (list_empty(&list))
  65                 return scanned;
  66 -
  67 +retry:
  68         reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false);
  69 +       sc->nr_reclaimed += reclaimed;
  70
  71 -       list_for_each_entry(page, &list, lru) {
  72 -               /* restore LRU_REFS_FLAGS cleared by isolate_page() */
  73 -               if (PageWorkingset(page))
  74 -                       SetPageReferenced(page);
  75 +       list_for_each_entry_safe_reverse(page, next, &list, lru) {
  76 +               if (!page_evictable(page)) {
  77 +                       list_del(&page->lru);
  78 +                       putback_lru_page(page);
  79 +                       continue;
  80 +               }
  81
  82 -               /* don't add rejected pages to the oldest generation */
  83                 if (PageReclaim(page) &&
  84 -                   (PageDirty(page) || PageWriteback(page)))
  85 -                       ClearPageActive(page);
  86 -               else
  87 -                       SetPageActive(page);
  88 +                   (PageDirty(page) || PageWriteback(page))) {
  89 +                       /* restore LRU_REFS_FLAGS cleared by isolate_page() */
  90 +                       if (PageWorkingset(page))
  91 +                               SetPageReferenced(page);
  92 +                       continue;
  93 +               }
  94 +
  95 +               if (skip_retry || PageActive(page) || PageReferenced(page) ||
  96 +                   page_mapped(page) || PageLocked(page) ||
  97 +                   PageDirty(page) || PageWriteback(page)) {
  98 +                       /* don't add rejected pages to the oldest generation */
  99 +                       set_mask_bits(&page->flags, LRU_REFS_MASK | LRU_REFS_FLAGS,
 100 +                                     BIT(PG_active));
 101 +                       continue;
 102 +               }
 103 +
 104 +               /* retry pages that may have missed rotate_reclaimable_page() */
 105 +               list_move(&page->lru, &clean);
 106 +               sc->nr_scanned -= thp_nr_pages(page);
 107         }
 108
 109         spin_lock_irq(&lruvec->lru_lock);
 110 @@ -4778,7 +4798,13 @@ static int evict_pages(struct lruvec *lr
 111         mem_cgroup_uncharge_list(&list);
 112         free_unref_page_list(&list);
 113
 114 -       sc->nr_reclaimed += reclaimed;
 115 +       INIT_LIST_HEAD(&list);
 116 +       list_splice_init(&clean, &list);
 117 +
 118 +       if (!list_empty(&list)) {
 119 +               skip_retry = true;
 120 +               goto retry;
 121 +       }
 122
 123         if (need_swapping && type == LRU_GEN_ANON)
 124                 *need_swapping = true;