projects / openwrt / staging / stintel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
kernel: bump 5.15 to 5.15.132
[openwrt/staging/stintel.git] / target / linux / generic / backport-5.15 / 020-v6.1-07-mm-multi-gen-LRU-exploit-locality-in-rmap.patch
1 From e4277535f6d6708bb19b88c4bad155832671d69b Mon Sep 17 00:00:00 2001
2 From: Yu Zhao <yuzhao@google.com>
3 Date: Sun, 18 Sep 2022 02:00:04 -0600
4 Subject: [PATCH 07/29] mm: multi-gen LRU: exploit locality in rmap
5 MIME-Version: 1.0
6 Content-Type: text/plain; charset=UTF-8
7 Content-Transfer-Encoding: 8bit
8
9 Searching the rmap for PTEs mapping each page on an LRU list (to test and
10 clear the accessed bit) can be expensive because pages from different VMAs
11 (PA space) are not cache friendly to the rmap (VA space). For workloads
12 mostly using mapped pages, searching the rmap can incur the highest CPU
13 cost in the reclaim path.
14
15 This patch exploits spatial locality to reduce the trips into the rmap.
16 When shrink_page_list() walks the rmap and finds a young PTE, a new
17 function lru_gen_look_around() scans at most BITS_PER_LONG-1 adjacent
18 PTEs. On finding another young PTE, it clears the accessed bit and
19 updates the gen counter of the page mapped by this PTE to
20 (max_seq%MAX_NR_GENS)+1.
21
22 Server benchmark results:
23 Single workload:
24 fio (buffered I/O): no change
25
26 Single workload:
27 memcached (anon): +[3, 5]%
28 Ops/sec KB/sec
29 patch1-6: 1106168.46 43025.04
30 patch1-7: 1147696.57 44640.29
31
32 Configurations:
33 no change
34
35 Client benchmark results:
36 kswapd profiles:
37 patch1-6
38 39.03% lzo1x_1_do_compress (real work)
39 18.47% page_vma_mapped_walk (overhead)
40 6.74% _raw_spin_unlock_irq
41 3.97% do_raw_spin_lock
42 2.49% ptep_clear_flush
43 2.48% anon_vma_interval_tree_iter_first
44 1.92% page_referenced_one
45 1.88% __zram_bvec_write
46 1.48% memmove
47 1.31% vma_interval_tree_iter_next
48
49 patch1-7
50 48.16% lzo1x_1_do_compress (real work)
51 8.20% page_vma_mapped_walk (overhead)
52 7.06% _raw_spin_unlock_irq
53 2.92% ptep_clear_flush
54 2.53% __zram_bvec_write
55 2.11% do_raw_spin_lock
56 2.02% memmove
57 1.93% lru_gen_look_around
58 1.56% free_unref_page_list
59 1.40% memset
60
61 Configurations:
62 no change
63
64 Link: https://lkml.kernel.org/r/20220918080010.2920238-8-yuzhao@google.com
65 Signed-off-by: Yu Zhao <yuzhao@google.com>
66 Acked-by: Barry Song <baohua@kernel.org>
67 Acked-by: Brian Geffon <bgeffon@google.com>
68 Acked-by: Jan Alexander Steffens (heftig) <heftig@archlinux.org>
69 Acked-by: Oleksandr Natalenko <oleksandr@natalenko.name>
70 Acked-by: Steven Barrett <steven@liquorix.net>
71 Acked-by: Suleiman Souhlal <suleiman@google.com>
72 Tested-by: Daniel Byrne <djbyrne@mtu.edu>
73 Tested-by: Donald Carr <d@chaos-reins.com>
74 Tested-by: Holger Hoffstätte <holger@applied-asynchrony.com>
75 Tested-by: Konstantin Kharlamov <Hi-Angel@yandex.ru>
76 Tested-by: Shuang Zhai <szhai2@cs.rochester.edu>
77 Tested-by: Sofia Trinh <sofia.trinh@edi.works>
78 Tested-by: Vaibhav Jain <vaibhav@linux.ibm.com>
79 Cc: Andi Kleen <ak@linux.intel.com>
80 Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
81 Cc: Catalin Marinas <catalin.marinas@arm.com>
82 Cc: Dave Hansen <dave.hansen@linux.intel.com>
83 Cc: Hillf Danton <hdanton@sina.com>
84 Cc: Jens Axboe <axboe@kernel.dk>
85 Cc: Johannes Weiner <hannes@cmpxchg.org>
86 Cc: Jonathan Corbet <corbet@lwn.net>
87 Cc: Linus Torvalds <torvalds@linux-foundation.org>
88 Cc: Matthew Wilcox <willy@infradead.org>
89 Cc: Mel Gorman <mgorman@suse.de>
90 Cc: Miaohe Lin <linmiaohe@huawei.com>
91 Cc: Michael Larabel <Michael@MichaelLarabel.com>
92 Cc: Michal Hocko <mhocko@kernel.org>
93 Cc: Mike Rapoport <rppt@kernel.org>
94 Cc: Mike Rapoport <rppt@linux.ibm.com>
95 Cc: Peter Zijlstra <peterz@infradead.org>
96 Cc: Qi Zheng <zhengqi.arch@bytedance.com>
97 Cc: Tejun Heo <tj@kernel.org>
98 Cc: Vlastimil Babka <vbabka@suse.cz>
99 Cc: Will Deacon <will@kernel.org>
100 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
101 ---
102 include/linux/memcontrol.h | 31 +++++++
103 include/linux/mmzone.h | 6 ++
104 mm/internal.h | 1 +
105 mm/memcontrol.c | 1 +
106 mm/rmap.c | 7 ++
107 mm/swap.c | 4 +-
108 mm/vmscan.c | 184 +++++++++++++++++++++++++++++++++++++
109 7 files changed, 232 insertions(+), 2 deletions(-)
110
111 --- a/include/linux/memcontrol.h
112 +++ b/include/linux/memcontrol.h
113 @@ -447,6 +447,7 @@ static inline struct obj_cgroup *__page_
114 * - LRU isolation
115 * - lock_page_memcg()
116 * - exclusive reference
117 + * - mem_cgroup_trylock_pages()
118 *
119 * For a kmem page a caller should hold an rcu read lock to protect memcg
120 * associated with a kmem page from being released.
121 @@ -502,6 +503,7 @@ static inline struct mem_cgroup *page_me
122 * - LRU isolation
123 * - lock_page_memcg()
124 * - exclusive reference
125 + * - mem_cgroup_trylock_pages()
126 *
127 * For a kmem page a caller should hold an rcu read lock to protect memcg
128 * associated with a kmem page from being released.
129 @@ -958,6 +960,23 @@ void unlock_page_memcg(struct page *page
130
131 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val);
132
133 +/* try to stablize page_memcg() for all the pages in a memcg */
134 +static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
135 +{
136 + rcu_read_lock();
137 +
138 + if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account))
139 + return true;
140 +
141 + rcu_read_unlock();
142 + return false;
143 +}
144 +
145 +static inline void mem_cgroup_unlock_pages(void)
146 +{
147 + rcu_read_unlock();
148 +}
149 +
150 /* idx can be of type enum memcg_stat_item or node_stat_item */
151 static inline void mod_memcg_state(struct mem_cgroup *memcg,
152 int idx, int val)
153 @@ -1374,6 +1393,18 @@ static inline void unlock_page_memcg(str
154 {
155 }
156
157 +static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
158 +{
159 + /* to match page_memcg_rcu() */
160 + rcu_read_lock();
161 + return true;
162 +}
163 +
164 +static inline void mem_cgroup_unlock_pages(void)
165 +{
166 + rcu_read_unlock();
167 +}
168 +
169 static inline void mem_cgroup_handle_over_high(void)
170 {
171 }
172 --- a/include/linux/mmzone.h
173 +++ b/include/linux/mmzone.h
174 @@ -352,6 +352,7 @@ enum lruvec_flags {
175 #ifndef __GENERATING_BOUNDS_H
176
177 struct lruvec;
178 +struct page_vma_mapped_walk;
179
180 #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
181 #define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
182 @@ -407,6 +408,7 @@ struct lru_gen_struct {
183 };
184
185 void lru_gen_init_lruvec(struct lruvec *lruvec);
186 +void lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
187
188 #ifdef CONFIG_MEMCG
189 void lru_gen_init_memcg(struct mem_cgroup *memcg);
190 @@ -419,6 +421,10 @@ static inline void lru_gen_init_lruvec(s
191 {
192 }
193
194 +static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
195 +{
196 +}
197 +
198 #ifdef CONFIG_MEMCG
199 static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
200 {
201 --- a/mm/internal.h
202 +++ b/mm/internal.h
203 @@ -35,6 +35,7 @@
204 void page_writeback_init(void);
205
206 vm_fault_t do_swap_page(struct vm_fault *vmf);
207 +void activate_page(struct page *page);
208
209 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
210 unsigned long floor, unsigned long ceiling);
211 --- a/mm/memcontrol.c
212 +++ b/mm/memcontrol.c
213 @@ -2798,6 +2798,7 @@ static void commit_charge(struct page *p
214 * - LRU isolation
215 * - lock_page_memcg()
216 * - exclusive reference
217 + * - mem_cgroup_trylock_pages()
218 */
219 page->memcg_data = (unsigned long)memcg;
220 }
221 --- a/mm/rmap.c
222 +++ b/mm/rmap.c
223 @@ -73,6 +73,7 @@
224 #include <linux/page_idle.h>
225 #include <linux/memremap.h>
226 #include <linux/userfaultfd_k.h>
227 +#include <linux/mm_inline.h>
228
229 #include <asm/tlbflush.h>
230
231 @@ -793,6 +794,12 @@ static bool page_referenced_one(struct p
232 }
233
234 if (pvmw.pte) {
235 + if (lru_gen_enabled() && pte_young(*pvmw.pte) &&
236 + !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) {
237 + lru_gen_look_around(&pvmw);
238 + referenced++;
239 + }
240 +
241 if (ptep_clear_flush_young_notify(vma, address,
242 pvmw.pte)) {
243 /*
244 --- a/mm/swap.c
245 +++ b/mm/swap.c
246 @@ -325,7 +325,7 @@ static bool need_activate_page_drain(int
247 return pagevec_count(&per_cpu(lru_pvecs.activate_page, cpu)) != 0;
248 }
249
250 -static void activate_page(struct page *page)
251 +void activate_page(struct page *page)
252 {
253 page = compound_head(page);
254 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
255 @@ -345,7 +345,7 @@ static inline void activate_page_drain(i
256 {
257 }
258
259 -static void activate_page(struct page *page)
260 +void activate_page(struct page *page)
261 {
262 struct lruvec *lruvec;
263
264 --- a/mm/vmscan.c
265 +++ b/mm/vmscan.c
266 @@ -1409,6 +1409,11 @@ retry:
267 if (!sc->may_unmap && page_mapped(page))
268 goto keep_locked;
269
270 + /* page_update_gen() tried to promote this page? */
271 + if (lru_gen_enabled() && !ignore_references &&
272 + page_mapped(page) && PageReferenced(page))
273 + goto keep_locked;
274 +
275 may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
276 (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
277
278 @@ -2990,6 +2995,29 @@ static bool positive_ctrl_err(struct ctr
279 * the aging
280 ******************************************************************************/
281
282 +/* promote pages accessed through page tables */
283 +static int page_update_gen(struct page *page, int gen)
284 +{
285 + unsigned long new_flags, old_flags = READ_ONCE(page->flags);
286 +
287 + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
288 + VM_WARN_ON_ONCE(!rcu_read_lock_held());
289 +
290 + do {
291 + /* lru_gen_del_page() has isolated this page? */
292 + if (!(old_flags & LRU_GEN_MASK)) {
293 + /* for shrink_page_list() */
294 + new_flags = old_flags | BIT(PG_referenced);
295 + continue;
296 + }
297 +
298 + new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS);
299 + new_flags |= (gen + 1UL) << LRU_GEN_PGOFF;
300 + } while (!try_cmpxchg(&page->flags, &old_flags, new_flags));
301 +
302 + return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
303 +}
304 +
305 /* protect pages accessed multiple times through file descriptors */
306 static int page_inc_gen(struct lruvec *lruvec, struct page *page, bool reclaiming)
307 {
308 @@ -3001,6 +3029,11 @@ static int page_inc_gen(struct lruvec *l
309 VM_WARN_ON_ONCE_PAGE(!(old_flags & LRU_GEN_MASK), page);
310
311 do {
312 + new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
313 + /* page_update_gen() has promoted this page? */
314 + if (new_gen >= 0 && new_gen != old_gen)
315 + return new_gen;
316 +
317 new_gen = (old_gen + 1) % MAX_NR_GENS;
318
319 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS);
320 @@ -3015,6 +3048,43 @@ static int page_inc_gen(struct lruvec *l
321 return new_gen;
322 }
323
324 +static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr)
325 +{
326 + unsigned long pfn = pte_pfn(pte);
327 +
328 + VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end);
329 +
330 + if (!pte_present(pte) || is_zero_pfn(pfn))
331 + return -1;
332 +
333 + if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte)))
334 + return -1;
335 +
336 + if (WARN_ON_ONCE(!pfn_valid(pfn)))
337 + return -1;
338 +
339 + return pfn;
340 +}
341 +
342 +static struct page *get_pfn_page(unsigned long pfn, struct mem_cgroup *memcg,
343 + struct pglist_data *pgdat)
344 +{
345 + struct page *page;
346 +
347 + /* try to avoid unnecessary memory loads */
348 + if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
349 + return NULL;
350 +
351 + page = compound_head(pfn_to_page(pfn));
352 + if (page_to_nid(page) != pgdat->node_id)
353 + return NULL;
354 +
355 + if (page_memcg_rcu(page) != memcg)
356 + return NULL;
357 +
358 + return page;
359 +}
360 +
361 static void inc_min_seq(struct lruvec *lruvec, int type)
362 {
363 struct lru_gen_struct *lrugen = &lruvec->lrugen;
364 @@ -3214,6 +3284,114 @@ static void lru_gen_age_node(struct pgli
365 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
366 }
367
368 +/*
369 + * This function exploits spatial locality when shrink_page_list() walks the
370 + * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages.
371 + */
372 +void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
373 +{
374 + int i;
375 + pte_t *pte;
376 + unsigned long start;
377 + unsigned long end;
378 + unsigned long addr;
379 + unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};
380 + struct page *page = pvmw->page;
381 + struct mem_cgroup *memcg = page_memcg(page);
382 + struct pglist_data *pgdat = page_pgdat(page);
383 + struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
384 + DEFINE_MAX_SEQ(lruvec);
385 + int old_gen, new_gen = lru_gen_from_seq(max_seq);
386 +
387 + lockdep_assert_held(pvmw->ptl);
388 + VM_WARN_ON_ONCE_PAGE(PageLRU(page), page);
389 +
390 + if (spin_is_contended(pvmw->ptl))
391 + return;
392 +
393 + start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start);
394 + end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1;
395 +
396 + if (end - start > MIN_LRU_BATCH * PAGE_SIZE) {
397 + if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2)
398 + end = start + MIN_LRU_BATCH * PAGE_SIZE;
399 + else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2)
400 + start = end - MIN_LRU_BATCH * PAGE_SIZE;
401 + else {
402 + start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2;
403 + end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2;
404 + }
405 + }
406 +
407 + pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE;
408 +
409 + rcu_read_lock();
410 + arch_enter_lazy_mmu_mode();
411 +
412 + for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) {
413 + unsigned long pfn;
414 +
415 + pfn = get_pte_pfn(pte[i], pvmw->vma, addr);
416 + if (pfn == -1)
417 + continue;
418 +
419 + if (!pte_young(pte[i]))
420 + continue;
421 +
422 + page = get_pfn_page(pfn, memcg, pgdat);
423 + if (!page)
424 + continue;
425 +
426 + if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i))
427 + VM_WARN_ON_ONCE(true);
428 +
429 + if (pte_dirty(pte[i]) && !PageDirty(page) &&
430 + !(PageAnon(page) && PageSwapBacked(page) &&
431 + !PageSwapCache(page)))
432 + set_page_dirty(page);
433 +
434 + old_gen = page_lru_gen(page);
435 + if (old_gen < 0)
436 + SetPageReferenced(page);
437 + else if (old_gen != new_gen)
438 + __set_bit(i, bitmap);
439 + }
440 +
441 + arch_leave_lazy_mmu_mode();
442 + rcu_read_unlock();
443 +
444 + if (bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) {
445 + for_each_set_bit(i, bitmap, MIN_LRU_BATCH) {
446 + page = pte_page(pte[i]);
447 + activate_page(page);
448 + }
449 + return;
450 + }
451 +
452 + /* page_update_gen() requires stable page_memcg() */
453 + if (!mem_cgroup_trylock_pages(memcg))
454 + return;
455 +
456 + spin_lock_irq(&lruvec->lru_lock);
457 + new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq);
458 +
459 + for_each_set_bit(i, bitmap, MIN_LRU_BATCH) {
460 + page = compound_head(pte_page(pte[i]));
461 + if (page_memcg_rcu(page) != memcg)
462 + continue;
463 +
464 + old_gen = page_update_gen(page, new_gen);
465 + if (old_gen < 0 || old_gen == new_gen)
466 + continue;
467 +
468 + lru_gen_update_size(lruvec, page, old_gen, new_gen);
469 + }
470 +
471 + spin_unlock_irq(&lruvec->lru_lock);
472 +
473 + mem_cgroup_unlock_pages();
474 +}
475 +
476 /******************************************************************************
477 * the eviction
478 ******************************************************************************/
479 @@ -3250,6 +3428,12 @@ static bool sort_page(struct lruvec *lru
480 return true;
481 }
482
483 + /* promoted */
484 + if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
485 + list_move(&page->lru, &lrugen->lists[gen][type][zone]);
486 + return true;
487 + }
488 +
489 /* protected */
490 if (tier > tier_idx) {
491 int hist = lru_hist_from_seq(lrugen->min_seq[type]);
Staging tree of Stijn Tintel