target/linux/generic/backport-5.15/020-v6.1-02-mm-x86-add-CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG.patch

   1 From 493de1c4b0f2cd909169401da8c445f6c8a7e29d Mon Sep 17 00:00:00 2001
   2 From: Yu Zhao <yuzhao@google.com>
   3 Date: Sun, 18 Sep 2022 01:59:59 -0600
   4 Subject: [PATCH 02/29] mm: x86: add CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG
   5 MIME-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8
   9 Some architectures support the accessed bit in non-leaf PMD entries, e.g.,
  10 x86 sets the accessed bit in a non-leaf PMD entry when using it as part of
  11 linear address translation [1].  Page table walkers that clear the
  12 accessed bit may use this capability to reduce their search space.
  13
  14 Note that:
  15 1. Although an inline function is preferable, this capability is added
  16    as a configuration option for consistency with the existing macros.
  17 2. Due to the little interest in other varieties, this capability was
  18    only tested on Intel and AMD CPUs.
  19
  20 Thanks to the following developers for their efforts [2][3].
  21   Randy Dunlap <rdunlap@infradead.org>
  22   Stephen Rothwell <sfr@canb.auug.org.au>
  23
  24 [1]: Intel 64 and IA-32 Architectures Software Developer's Manual
  25      Volume 3 (June 2021), section 4.8
  26 [2] https://lore.kernel.org/r/bfdcc7c8-922f-61a9-aa15-7e7250f04af7@infradead.org/
  27 [3] https://lore.kernel.org/r/20220413151513.5a0d7a7e@canb.auug.org.au/
  28
  29 Link: https://lkml.kernel.org/r/20220918080010.2920238-3-yuzhao@google.com
  30 Signed-off-by: Yu Zhao <yuzhao@google.com>
  31 Reviewed-by: Barry Song <baohua@kernel.org>
  32 Acked-by: Brian Geffon <bgeffon@google.com>
  33 Acked-by: Jan Alexander Steffens (heftig) <heftig@archlinux.org>
  34 Acked-by: Oleksandr Natalenko <oleksandr@natalenko.name>
  35 Acked-by: Steven Barrett <steven@liquorix.net>
  36 Acked-by: Suleiman Souhlal <suleiman@google.com>
  37 Tested-by: Daniel Byrne <djbyrne@mtu.edu>
  38 Tested-by: Donald Carr <d@chaos-reins.com>
  39 Tested-by: Holger Hoffstätte <holger@applied-asynchrony.com>
  40 Tested-by: Konstantin Kharlamov <Hi-Angel@yandex.ru>
  41 Tested-by: Shuang Zhai <szhai2@cs.rochester.edu>
  42 Tested-by: Sofia Trinh <sofia.trinh@edi.works>
  43 Tested-by: Vaibhav Jain <vaibhav@linux.ibm.com>
  44 Cc: Andi Kleen <ak@linux.intel.com>
  45 Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
  46 Cc: Catalin Marinas <catalin.marinas@arm.com>
  47 Cc: Dave Hansen <dave.hansen@linux.intel.com>
  48 Cc: Hillf Danton <hdanton@sina.com>
  49 Cc: Jens Axboe <axboe@kernel.dk>
  50 Cc: Johannes Weiner <hannes@cmpxchg.org>
  51 Cc: Jonathan Corbet <corbet@lwn.net>
  52 Cc: Linus Torvalds <torvalds@linux-foundation.org>
  53 Cc: Matthew Wilcox <willy@infradead.org>
  54 Cc: Mel Gorman <mgorman@suse.de>
  55 Cc: Miaohe Lin <linmiaohe@huawei.com>
  56 Cc: Michael Larabel <Michael@MichaelLarabel.com>
  57 Cc: Michal Hocko <mhocko@kernel.org>
  58 Cc: Mike Rapoport <rppt@kernel.org>
  59 Cc: Mike Rapoport <rppt@linux.ibm.com>
  60 Cc: Peter Zijlstra <peterz@infradead.org>
  61 Cc: Qi Zheng <zhengqi.arch@bytedance.com>
  62 Cc: Tejun Heo <tj@kernel.org>
  63 Cc: Vlastimil Babka <vbabka@suse.cz>
  64 Cc: Will Deacon <will@kernel.org>
  65 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  66 ---
  67  arch/Kconfig                   | 8 ++++++++
  68  arch/x86/Kconfig               | 1 +
  69  arch/x86/include/asm/pgtable.h | 3 ++-
  70  arch/x86/mm/pgtable.c          | 5 ++++-
  71  include/linux/pgtable.h        | 4 ++--
  72  5 files changed, 17 insertions(+), 4 deletions(-)
  73
  74 diff --git a/arch/Kconfig b/arch/Kconfig
  75 index 5987363b41c2..62d55b7ccca1 100644
  76 --- a/arch/Kconfig
  77 +++ b/arch/Kconfig
  78 @@ -1295,6 +1295,14 @@ config ARCH_HAS_ELFCORE_COMPAT
  79  config ARCH_HAS_PARANOID_L1D_FLUSH
  80         bool
  81
  82 +config ARCH_HAS_NONLEAF_PMD_YOUNG
  83 +       bool
  84 +       help
  85 +         Architectures that select this option are capable of setting the
  86 +         accessed bit in non-leaf PMD entries when using them as part of linear
  87 +         address translations. Page table walkers that clear the accessed bit
  88 +         may use this capability to reduce their search space.
  89 +
  90  source "kernel/gcov/Kconfig"
  91
  92  source "scripts/gcc-plugins/Kconfig"
  93 diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
  94 index a08ce6360382..38e1d231d52a 100644
  95 --- a/arch/x86/Kconfig
  96 +++ b/arch/x86/Kconfig
  97 @@ -84,6 +84,7 @@ config X86
  98         select ARCH_HAS_PMEM_API                if X86_64
  99         select ARCH_HAS_PTE_DEVMAP              if X86_64
 100         select ARCH_HAS_PTE_SPECIAL
 101 +       select ARCH_HAS_NONLEAF_PMD_YOUNG       if PGTABLE_LEVELS > 2
 102         select ARCH_HAS_UACCESS_FLUSHCACHE      if X86_64
 103         select ARCH_HAS_COPY_MC                 if X86_64
 104         select ARCH_HAS_SET_MEMORY
 105 diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
 106 index 3908780fc408..01a1763123ff 100644
 107 --- a/arch/x86/include/asm/pgtable.h
 108 +++ b/arch/x86/include/asm/pgtable.h
 109 @@ -817,7 +817,8 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 110
 111  static inline int pmd_bad(pmd_t pmd)
 112  {
 113 -       return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
 114 +       return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) !=
 115 +              (_KERNPG_TABLE & ~_PAGE_ACCESSED);
 116  }
 117
 118  static inline unsigned long pages_to_mb(unsigned long npg)
 119 diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
 120 index 3481b35cb4ec..a224193d84bf 100644
 121 --- a/arch/x86/mm/pgtable.c
 122 +++ b/arch/x86/mm/pgtable.c
 123 @@ -550,7 +550,7 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma,
 124         return ret;
 125  }
 126
 127 -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 128 +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
 129  int pmdp_test_and_clear_young(struct vm_area_struct *vma,
 130                               unsigned long addr, pmd_t *pmdp)
 131  {
 132 @@ -562,6 +562,9 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma,
 133
 134         return ret;
 135  }
 136 +#endif
 137 +
 138 +#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 139  int pudp_test_and_clear_young(struct vm_area_struct *vma,
 140                               unsigned long addr, pud_t *pudp)
 141  {
 142 diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
 143 index 2f1188980baf..e6889556e0bf 100644
 144 --- a/include/linux/pgtable.h
 145 +++ b/include/linux/pgtable.h
 146 @@ -212,7 +212,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
 147  #endif
 148
 149  #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
 150 -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 151 +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
 152  static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
 153                                             unsigned long address,
 154                                             pmd_t *pmdp)
 155 @@ -233,7 +233,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
 156         BUILD_BUG();
 157         return 0;
 158  }
 159 -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 160 +#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
 161  #endif
 162
 163  #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
 164 --
 165 2.40.0
 166