target/linux/generic/pending-5.15/020-00-mm-x86-arm64-add-arch_has_hw_pte_young.patch

   1 From a8e6015d9534f39abc08e6804566af059e498a60 Mon Sep 17 00:00:00 2001
   2 From: Yu Zhao <yuzhao@google.com>
   3 Date: Wed, 4 Aug 2021 01:31:34 -0600
   4 Subject: [PATCH 01/10] mm: x86, arm64: add arch_has_hw_pte_young()
   5
   6 Some architectures automatically set the accessed bit in PTEs, e.g.,
   7 x86 and arm64 v8.2. On architectures that do not have this capability,
   8 clearing the accessed bit in a PTE triggers a page fault following the
   9 TLB miss of this PTE.
  10
  11 Being aware of this capability can help make better decisions, i.e.,
  12 whether to limit the size of each batch of PTEs and the burst of
  13 batches when clearing the accessed bit.
  14
  15 Signed-off-by: Yu Zhao <yuzhao@google.com>
  16 Change-Id: Ib49b44fb56df3333a2ff1fcc496fb1980b976e7a
  17 ---
  18  arch/arm64/include/asm/cpufeature.h |  5 +++++
  19  arch/arm64/include/asm/pgtable.h    | 13 ++++++++-----
  20  arch/arm64/kernel/cpufeature.c      | 10 ++++++++++
  21  arch/arm64/tools/cpucaps            |  1 +
  22  arch/x86/include/asm/pgtable.h      |  6 +++---
  23  include/linux/pgtable.h             | 13 +++++++++++++
  24  mm/memory.c                         | 14 +-------------
  25  7 files changed, 41 insertions(+), 21 deletions(-)
  26
  27 --- a/arch/arm64/include/asm/cpufeature.h
  28 +++ b/arch/arm64/include/asm/cpufeature.h
  29 @@ -808,6 +808,11 @@ static inline bool system_supports_tlb_r
  30                 cpus_have_const_cap(ARM64_HAS_TLB_RANGE);
  31  }
  32
  33 +static inline bool system_has_hw_af(void)
  34 +{
  35 +       return IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && cpus_have_const_cap(ARM64_HW_AF);
  36 +}
  37 +
  38  extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
  39
  40  static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange)
  41 --- a/arch/arm64/include/asm/pgtable.h
  42 +++ b/arch/arm64/include/asm/pgtable.h
  43 @@ -999,13 +999,16 @@ static inline void update_mmu_cache(stru
  44   * page after fork() + CoW for pfn mappings. We don't always have a
  45   * hardware-managed access flag on arm64.
  46   */
  47 -static inline bool arch_faults_on_old_pte(void)
  48 +static inline bool arch_has_hw_pte_young(bool local)
  49  {
  50 -       WARN_ON(preemptible());
  51 +       if (local) {
  52 +               WARN_ON(preemptible());
  53 +               return cpu_has_hw_af();
  54 +       }
  55
  56 -       return !cpu_has_hw_af();
  57 +       return system_has_hw_af();
  58  }
  59 -#define arch_faults_on_old_pte         arch_faults_on_old_pte
  60 +#define arch_has_hw_pte_young          arch_has_hw_pte_young
  61
  62  /*
  63   * Experimentally, it's cheap to set the access flag in hardware and we
  64 @@ -1013,7 +1016,7 @@ static inline bool arch_faults_on_old_pt
  65   */
  66  static inline bool arch_wants_old_prefaulted_pte(void)
  67  {
  68 -       return !arch_faults_on_old_pte();
  69 +       return arch_has_hw_pte_young(true);
  70  }
  71  #define arch_wants_old_prefaulted_pte  arch_wants_old_prefaulted_pte
  72
  73 --- a/arch/arm64/kernel/cpufeature.c
  74 +++ b/arch/arm64/kernel/cpufeature.c
  75 @@ -2184,6 +2184,16 @@ static const struct arm64_cpu_capabiliti
  76                 .matches = has_hw_dbm,
  77                 .cpu_enable = cpu_enable_hw_dbm,
  78         },
  79 +       {
  80 +               .desc = "Hardware update of the Access flag",
  81 +               .type = ARM64_CPUCAP_SYSTEM_FEATURE,
  82 +               .capability = ARM64_HW_AF,
  83 +               .sys_reg = SYS_ID_AA64MMFR1_EL1,
  84 +               .sign = FTR_UNSIGNED,
  85 +               .field_pos = ID_AA64MMFR1_HADBS_SHIFT,
  86 +               .min_field_value = 1,
  87 +               .matches = has_cpuid_feature,
  88 +       },
  89  #endif
  90         {
  91                 .desc = "CRC32 instructions",
  92 --- a/arch/arm64/tools/cpucaps
  93 +++ b/arch/arm64/tools/cpucaps
  94 @@ -35,6 +35,7 @@ HAS_STAGE2_FWB
  95  HAS_SYSREG_GIC_CPUIF
  96  HAS_TLB_RANGE
  97  HAS_VIRT_HOST_EXTN
  98 +HW_AF
  99  HW_DBM
 100  KVM_PROTECTED_MODE
 101  MISMATCHED_CACHE_TYPE
 102 --- a/arch/x86/include/asm/pgtable.h
 103 +++ b/arch/x86/include/asm/pgtable.h
 104 @@ -1397,10 +1397,10 @@ static inline bool arch_has_pfn_modify_c
 105         return boot_cpu_has_bug(X86_BUG_L1TF);
 106  }
 107
 108 -#define arch_faults_on_old_pte arch_faults_on_old_pte
 109 -static inline bool arch_faults_on_old_pte(void)
 110 +#define arch_has_hw_pte_young arch_has_hw_pte_young
 111 +static inline bool arch_has_hw_pte_young(bool local)
 112  {
 113 -       return false;
 114 +       return true;
 115  }
 116
 117  #endif /* __ASSEMBLY__ */
 118 --- a/include/linux/pgtable.h
 119 +++ b/include/linux/pgtable.h
 120 @@ -259,6 +259,19 @@ static inline int pmdp_clear_flush_young
 121  #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 122  #endif
 123
 124 +#ifndef arch_has_hw_pte_young
 125 +/*
 126 + * Return whether the accessed bit is supported by the local CPU or all CPUs.
 127 + *
 128 + * Those arches which have hw access flag feature need to implement their own
 129 + * helper. By default, "false" means pagefault will be hit on old pte.
 130 + */
 131 +static inline bool arch_has_hw_pte_young(bool local)
 132 +{
 133 +       return false;
 134 +}
 135 +#endif
 136 +
 137  #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
 138  static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
 139                                        unsigned long address,
 140 --- a/mm/memory.c
 141 +++ b/mm/memory.c
 142 @@ -121,18 +121,6 @@ int randomize_va_space __read_mostly =
 143                                         2;
 144  #endif
 145
 146 -#ifndef arch_faults_on_old_pte
 147 -static inline bool arch_faults_on_old_pte(void)
 148 -{
 149 -       /*
 150 -        * Those arches which don't have hw access flag feature need to
 151 -        * implement their own helper. By default, "true" means pagefault
 152 -        * will be hit on old pte.
 153 -        */
 154 -       return true;
 155 -}
 156 -#endif
 157 -
 158  #ifndef arch_wants_old_prefaulted_pte
 159  static inline bool arch_wants_old_prefaulted_pte(void)
 160  {
 161 @@ -2782,7 +2770,7 @@ static inline bool cow_user_page(struct
 162          * On architectures with software "accessed" bits, we would
 163          * take a double page fault, so mark it accessed here.
 164          */
 165 -       if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) {
 166 +       if (!arch_has_hw_pte_young(true) && !pte_young(vmf->orig_pte)) {
 167                 pte_t entry;
 168
 169                 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);