Some backported patches generated with git-format-patch were not
refreshed. Use 'make target/linux/refresh' to align them with OpenWrt's
patch style.
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
19 files changed:
mm/workingset.c | 4 ++--
4 files changed, 24 insertions(+), 24 deletions(-)
mm/workingset.c | 4 ++--
4 files changed, 24 insertions(+), 24 deletions(-)
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index e8ed225d8f7ca..f63968bd7de59 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -178,7 +178,7 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli
+@@ -178,7 +178,7 @@ static inline void lru_gen_update_size(s
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
enum lru_list lru = type * LRU_INACTIVE_FILE;
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
enum lru_list lru = type * LRU_INACTIVE_FILE;
VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS);
VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS);
VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS);
VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS);
-@@ -224,7 +224,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
+@@ -224,7 +224,7 @@ static inline bool lru_gen_add_folio(str
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);
VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 5f74891556f33..bd3e4689f72dc 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -404,7 +404,7 @@ enum {
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -404,7 +404,7 @@ enum {
/* to concurrently iterate lru_gen_mm_list */
struct lru_gen_mm_state mm_state;
#endif
/* to concurrently iterate lru_gen_mm_list */
struct lru_gen_mm_state mm_state;
#endif
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index d18296109aa7e..27142caf284c1 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3190,7 +3190,7 @@ static int get_nr_gens(struct lruvec *lruvec, int type)
+@@ -3190,7 +3190,7 @@ static int get_nr_gens(struct lruvec *lr
static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
{
static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
{
int hist = lru_hist_from_seq(lrugen->min_seq[type]);
pos->refaulted = lrugen->avg_refaulted[type][tier] +
int hist = lru_hist_from_seq(lrugen->min_seq[type]);
pos->refaulted = lrugen->avg_refaulted[type][tier] +
-@@ -3611,7 +3611,7 @@ static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
+@@ -3611,7 +3611,7 @@ static void read_ctrl_pos(struct lruvec
static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover)
{
int hist, tier;
static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover)
{
int hist, tier;
bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1;
unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1;
unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
-@@ -3688,7 +3688,7 @@ static int folio_update_gen(struct folio *folio, int gen)
+@@ -3688,7 +3688,7 @@ static int folio_update_gen(struct folio
static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
int type = folio_is_file_lru(folio);
static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
int type = folio_is_file_lru(folio);
int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
-@@ -3733,7 +3733,7 @@ static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio,
+@@ -3733,7 +3733,7 @@ static void update_batch_size(struct lru
static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk)
{
int gen, type, zone;
static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk)
{
int gen, type, zone;
-@@ -4250,7 +4250,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
+@@ -4250,7 +4250,7 @@ static bool inc_min_seq(struct lruvec *l
{
int zone;
int remaining = MAX_LRU_BATCH;
{
int zone;
int remaining = MAX_LRU_BATCH;
int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
if (type == LRU_GEN_ANON && !can_swap)
int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
if (type == LRU_GEN_ANON && !can_swap)
-@@ -4286,7 +4286,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
+@@ -4286,7 +4286,7 @@ static bool try_to_inc_min_seq(struct lr
{
int gen, type, zone;
bool success = false;
{
int gen, type, zone;
bool success = false;
DEFINE_MIN_SEQ(lruvec);
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
DEFINE_MIN_SEQ(lruvec);
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
-@@ -4307,7 +4307,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
+@@ -4307,7 +4307,7 @@ next:
if (can_swap) {
min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]);
min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);
if (can_swap) {
min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]);
min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);
-@@ -4329,7 +4329,7 @@ static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan)
+@@ -4329,7 +4329,7 @@ static void inc_max_seq(struct lruvec *l
{
int prev, next;
int type, zone;
{
int prev, next;
int type, zone;
spin_lock_irq(&lruvec->lru_lock);
spin_lock_irq(&lruvec->lru_lock);
-@@ -4387,7 +4387,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4387,7 +4387,7 @@ static bool try_to_inc_max_seq(struct lr
bool success;
struct lru_gen_mm_walk *walk;
struct mm_struct *mm = NULL;
bool success;
struct lru_gen_mm_walk *walk;
struct mm_struct *mm = NULL;
VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));
VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));
-@@ -4452,7 +4452,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsig
+@@ -4452,7 +4452,7 @@ static bool should_run_aging(struct lruv
unsigned long old = 0;
unsigned long young = 0;
unsigned long total = 0;
unsigned long old = 0;
unsigned long young = 0;
unsigned long total = 0;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
for (type = !can_swap; type < ANON_AND_FILE; type++) {
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
for (type = !can_swap; type < ANON_AND_FILE; type++) {
-@@ -4737,7 +4737,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
+@@ -4737,7 +4737,7 @@ static bool sort_folio(struct lruvec *lr
int delta = folio_nr_pages(folio);
int refs = folio_lru_refs(folio);
int tier = lru_tier_from_refs(refs);
int delta = folio_nr_pages(folio);
int refs = folio_lru_refs(folio);
int tier = lru_tier_from_refs(refs);
VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
-@@ -4837,7 +4837,7 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
+@@ -4837,7 +4837,7 @@ static int scan_folios(struct lruvec *lr
int scanned = 0;
int isolated = 0;
int remaining = MAX_LRU_BATCH;
int scanned = 0;
int isolated = 0;
int remaining = MAX_LRU_BATCH;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
VM_WARN_ON_ONCE(!list_empty(list));
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
VM_WARN_ON_ONCE(!list_empty(list));
-@@ -5237,7 +5237,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5237,7 +5237,7 @@ done:
static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
{
static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
{
if (lrugen->enabled) {
enum lru_list lru;
if (lrugen->enabled) {
enum lru_list lru;
-@@ -5519,7 +5519,7 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
+@@ -5519,7 +5519,7 @@ static void lru_gen_seq_show_full(struct
int i;
int type, tier;
int hist = lru_hist_from_seq(seq);
int i;
int type, tier;
int hist = lru_hist_from_seq(seq);
for (tier = 0; tier < MAX_NR_TIERS; tier++) {
seq_printf(m, " %10d", tier);
for (tier = 0; tier < MAX_NR_TIERS; tier++) {
seq_printf(m, " %10d", tier);
-@@ -5569,7 +5569,7 @@ static int lru_gen_seq_show(struct seq_file *m, void *v)
+@@ -5569,7 +5569,7 @@ static int lru_gen_seq_show(struct seq_f
unsigned long seq;
bool full = !debugfs_real_fops(m->file)->write;
struct lruvec *lruvec = v;
unsigned long seq;
bool full = !debugfs_real_fops(m->file)->write;
struct lruvec *lruvec = v;
int nid = lruvec_pgdat(lruvec)->node_id;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MAX_SEQ(lruvec);
int nid = lruvec_pgdat(lruvec)->node_id;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MAX_SEQ(lruvec);
-@@ -5823,7 +5823,7 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -5823,7 +5823,7 @@ void lru_gen_init_lruvec(struct lruvec *
{
int i;
int gen, type, zone;
{
int i;
int gen, type, zone;
lrugen->max_seq = MIN_NR_GENS + 1;
lrugen->enabled = lru_gen_enabled();
lrugen->max_seq = MIN_NR_GENS + 1;
lrugen->enabled = lru_gen_enabled();
-diff --git a/mm/workingset.c b/mm/workingset.c
-index ae7e984b23c6b..688aaa73f64e8 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
--- a/mm/workingset.c
+++ b/mm/workingset.c
-@@ -223,7 +223,7 @@ static void *lru_gen_eviction(struct folio *folio)
+@@ -223,7 +223,7 @@ static void *lru_gen_eviction(struct fol
unsigned long token;
unsigned long min_seq;
struct lruvec *lruvec;
unsigned long token;
unsigned long min_seq;
struct lruvec *lruvec;
int type = folio_is_file_lru(folio);
int delta = folio_nr_pages(folio);
int refs = folio_lru_refs(folio);
int type = folio_is_file_lru(folio);
int delta = folio_nr_pages(folio);
int refs = folio_lru_refs(folio);
-@@ -252,7 +252,7 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
+@@ -252,7 +252,7 @@ static void lru_gen_refault(struct folio
unsigned long token;
unsigned long min_seq;
struct lruvec *lruvec;
unsigned long token;
unsigned long min_seq;
struct lruvec *lruvec;
struct mem_cgroup *memcg;
struct pglist_data *pgdat;
int type = folio_is_file_lru(folio);
struct mem_cgroup *memcg;
struct pglist_data *pgdat;
int type = folio_is_file_lru(folio);
mm/vmscan.c | 20 ++++++++++----------
4 files changed, 20 insertions(+), 20 deletions(-)
mm/vmscan.c | 20 ++++++++++----------
4 files changed, 20 insertions(+), 20 deletions(-)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index d7062c6a89464..d8f721f98868a 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
@@ -89,15 +89,15 @@ variables are monotonically increasing.
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
@@ -89,15 +89,15 @@ variables are monotonically increasing.
contrast to moving across generations, which requires the LRU lock,
moving across tiers only involves atomic operations on
``folio->flags`` and therefore has a negligible cost. A feedback loop
contrast to moving across generations, which requires the LRU lock,
moving across tiers only involves atomic operations on
``folio->flags`` and therefore has a negligible cost. A feedback loop
-@@ -127,7 +127,7 @@ page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``.
+@@ -127,7 +127,7 @@ page mapped by this PTE to ``(max_seq%MA
Eviction
--------
The eviction consumes old generations. Given an ``lruvec``, it
Eviction
--------
The eviction consumes old generations. Given an ``lruvec``, it
``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to
evict from, it first compares ``min_seq[]`` to select the older type.
If both types are equally old, it selects the one whose first tier has
``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to
evict from, it first compares ``min_seq[]`` to select the older type.
If both types are equally old, it selects the one whose first tier has
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index f63968bd7de59..da38e3d962e2f 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -256,9 +256,9 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
+@@ -256,9 +256,9 @@ static inline bool lru_gen_add_folio(str
lru_gen_update_size(lruvec, folio, -1, gen);
/* for folio_rotate_reclaimable() */
if (reclaiming)
lru_gen_update_size(lruvec, folio, -1, gen);
/* for folio_rotate_reclaimable() */
if (reclaiming)
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index bd3e4689f72dc..02e4323744715 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -312,7 +312,7 @@ enum lruvec_flags {
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -312,7 +312,7 @@ enum lruvec_flags {
/* the multi-gen LRU sizes, eventually consistent */
long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the exponential moving average of refaulted */
/* the multi-gen LRU sizes, eventually consistent */
long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the exponential moving average of refaulted */
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 27142caf284c1..b02fed912f742 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4258,7 +4258,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
+@@ -4258,7 +4258,7 @@ static bool inc_min_seq(struct lruvec *l
/* prevent cold/hot inversion if force_scan is true */
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
/* prevent cold/hot inversion if force_scan is true */
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
while (!list_empty(head)) {
struct folio *folio = lru_to_folio(head);
while (!list_empty(head)) {
struct folio *folio = lru_to_folio(head);
-@@ -4269,7 +4269,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
+@@ -4269,7 +4269,7 @@ static bool inc_min_seq(struct lruvec *l
VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
new_gen = folio_inc_gen(lruvec, folio, false);
VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
new_gen = folio_inc_gen(lruvec, folio, false);
if (!--remaining)
return false;
if (!--remaining)
return false;
-@@ -4297,7 +4297,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
+@@ -4297,7 +4297,7 @@ static bool try_to_inc_min_seq(struct lr
gen = lru_gen_from_seq(min_seq[type]);
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
gen = lru_gen_from_seq(min_seq[type]);
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
-@@ -4762,7 +4762,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
+@@ -4762,7 +4762,7 @@ static bool sort_folio(struct lruvec *lr
/* promoted */
if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
/* promoted */
if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
-@@ -4771,7 +4771,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
+@@ -4771,7 +4771,7 @@ static bool sort_folio(struct lruvec *lr
int hist = lru_hist_from_seq(lrugen->min_seq[type]);
gen = folio_inc_gen(lruvec, folio, false);
int hist = lru_hist_from_seq(lrugen->min_seq[type]);
gen = folio_inc_gen(lruvec, folio, false);
WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
lrugen->protected[hist][type][tier - 1] + delta);
WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
lrugen->protected[hist][type][tier - 1] + delta);
-@@ -4783,7 +4783,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
+@@ -4783,7 +4783,7 @@ static bool sort_folio(struct lruvec *lr
if (folio_test_locked(folio) || folio_test_writeback(folio) ||
(type == LRU_GEN_FILE && folio_test_dirty(folio))) {
gen = folio_inc_gen(lruvec, folio, true);
if (folio_test_locked(folio) || folio_test_writeback(folio) ||
(type == LRU_GEN_FILE && folio_test_dirty(folio))) {
gen = folio_inc_gen(lruvec, folio, true);
-@@ -4850,7 +4850,7 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
+@@ -4850,7 +4850,7 @@ static int scan_folios(struct lruvec *lr
for (zone = sc->reclaim_idx; zone >= 0; zone--) {
LIST_HEAD(moved);
int skipped = 0;
for (zone = sc->reclaim_idx; zone >= 0; zone--) {
LIST_HEAD(moved);
int skipped = 0;
while (!list_empty(head)) {
struct folio *folio = lru_to_folio(head);
while (!list_empty(head)) {
struct folio *folio = lru_to_folio(head);
-@@ -5250,7 +5250,7 @@ static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
+@@ -5250,7 +5250,7 @@ static bool __maybe_unused state_is_vali
int gen, type, zone;
for_each_gen_type_zone(gen, type, zone) {
int gen, type, zone;
for_each_gen_type_zone(gen, type, zone) {
-@@ -5295,7 +5295,7 @@ static bool drain_evictable(struct lruvec *lruvec)
+@@ -5295,7 +5295,7 @@ static bool drain_evictable(struct lruve
int remaining = MAX_LRU_BATCH;
for_each_gen_type_zone(gen, type, zone) {
int remaining = MAX_LRU_BATCH;
for_each_gen_type_zone(gen, type, zone) {
while (!list_empty(head)) {
bool success;
while (!list_empty(head)) {
bool success;
-@@ -5832,7 +5832,7 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -5832,7 +5832,7 @@ void lru_gen_init_lruvec(struct lruvec *
lrugen->timestamps[i] = jiffies;
for_each_gen_type_zone(gen, type, zone)
lrugen->timestamps[i] = jiffies;
for_each_gen_type_zone(gen, type, zone)
lruvec->mm_state.seq = MIN_NR_GENS;
init_waitqueue_head(&lruvec->mm_state.wait);
lruvec->mm_state.seq = MIN_NR_GENS;
init_waitqueue_head(&lruvec->mm_state.wait);
mm/vmscan.c | 81 +++++++++++++++--------------------------------------
1 file changed, 23 insertions(+), 58 deletions(-)
mm/vmscan.c | 81 +++++++++++++++--------------------------------------
1 file changed, 23 insertions(+), 58 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index b02fed912f742..991961180b320 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -448,6 +448,11 @@ static bool cgroup_reclaim(struct scan_control *sc)
+@@ -448,6 +448,11 @@ static bool cgroup_reclaim(struct scan_c
return sc->target_mem_cgroup;
}
return sc->target_mem_cgroup;
}
/**
* writeback_throttling_sane - is the usual dirty throttling mechanism available?
* @sc: scan_control in question
/**
* writeback_throttling_sane - is the usual dirty throttling mechanism available?
* @sc: scan_control in question
-@@ -498,6 +503,11 @@ static bool cgroup_reclaim(struct scan_control *sc)
+@@ -498,6 +503,11 @@ static bool cgroup_reclaim(struct scan_c
static bool writeback_throttling_sane(struct scan_control *sc)
{
return true;
static bool writeback_throttling_sane(struct scan_control *sc)
{
return true;
-@@ -4993,8 +5003,7 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int sw
+@@ -4993,8 +5003,7 @@ static int isolate_folios(struct lruvec
-@@ -5083,9 +5092,6 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
+@@ -5083,9 +5092,6 @@ retry:
-@@ -5124,67 +5130,26 @@ static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *
+@@ -5124,67 +5130,26 @@ done:
return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0;
}
return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0;
}
-@@ -5208,7 +5173,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5208,7 +5173,7 @@ static void lru_gen_shrink_lruvec(struct
if (!nr_to_scan)
goto done;
if (!nr_to_scan)
goto done;
-@@ -5216,7 +5181,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5216,7 +5181,7 @@ static void lru_gen_shrink_lruvec(struct
if (scanned >= nr_to_scan)
break;
if (scanned >= nr_to_scan)
break;
-@@ -5666,7 +5631,7 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
+@@ -5666,7 +5631,7 @@ static int run_eviction(struct lruvec *l
if (sc->nr_reclaimed >= nr_to_reclaim)
return 0;
if (sc->nr_reclaimed >= nr_to_reclaim)
return 0;
return 0;
cond_resched();
return 0;
cond_resched();
mm/vmscan.c | 126 ++++++++++++++++++++++++----------------------------
1 file changed, 59 insertions(+), 67 deletions(-)
mm/vmscan.c | 126 ++++++++++++++++++++++++----------------------------
1 file changed, 59 insertions(+), 67 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 991961180b320..5a2e83e673232 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -136,7 +136,6 @@ struct scan_control {
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -136,7 +136,6 @@ struct scan_control {
unsigned long last_reclaimed;
#endif
unsigned long last_reclaimed;
#endif
-@@ -4455,7 +4454,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4455,7 +4454,7 @@ done:
struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
{
int gen, type, zone;
struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
{
int gen, type, zone;
-@@ -4464,6 +4463,13 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsig
+@@ -4464,6 +4463,13 @@ static bool should_run_aging(struct lruv
unsigned long total = 0;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
unsigned long total = 0;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
for (type = !can_swap; type < ANON_AND_FILE; type++) {
unsigned long seq;
for (type = !can_swap; type < ANON_AND_FILE; type++) {
unsigned long seq;
-@@ -4492,8 +4498,6 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsig
+@@ -4492,8 +4498,6 @@ static bool should_run_aging(struct lruv
* stalls when the number of generations reaches MIN_NR_GENS. Hence, the
* ideal number of generations is MIN_NR_GENS+1.
*/
* stalls when the number of generations reaches MIN_NR_GENS. Hence, the
* ideal number of generations is MIN_NR_GENS+1.
*/
if (min_seq[!can_swap] + MIN_NR_GENS < max_seq)
return false;
if (min_seq[!can_swap] + MIN_NR_GENS < max_seq)
return false;
-@@ -4512,40 +4516,54 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsig
+@@ -4512,40 +4516,54 @@ static bool should_run_aging(struct lruv
}
/* to protect the working set of the last N jiffies */
}
/* to protect the working set of the last N jiffies */
-@@ -4554,46 +4572,32 @@ static unsigned long lru_gen_min_ttl __read_mostly;
+@@ -4554,46 +4572,32 @@ static unsigned long lru_gen_min_ttl __r
static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
{
struct mem_cgroup *memcg;
static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
{
struct mem_cgroup *memcg;
*/
if (mutex_trylock(&oom_lock)) {
struct oom_control oc = {
*/
if (mutex_trylock(&oom_lock)) {
struct oom_control oc = {
-@@ -5101,33 +5105,27 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
+@@ -5101,33 +5105,27 @@ retry:
* reclaim.
*/
static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
* reclaim.
*/
static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
}
static unsigned long get_nr_to_reclaim(struct scan_control *sc)
}
static unsigned long get_nr_to_reclaim(struct scan_control *sc)
-@@ -5146,9 +5144,7 @@ static unsigned long get_nr_to_reclaim(struct scan_control *sc)
+@@ -5146,9 +5144,7 @@ static unsigned long get_nr_to_reclaim(s
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
{
struct blk_plug plug;
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
{
struct blk_plug plug;
unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
lru_add_drain();
unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
lru_add_drain();
-@@ -5169,13 +5165,13 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5169,13 +5165,13 @@ static void lru_gen_shrink_lruvec(struct
scanned += delta;
if (scanned >= nr_to_scan)
scanned += delta;
if (scanned >= nr_to_scan)
-@@ -5187,10 +5183,6 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5187,10 +5183,6 @@ static void lru_gen_shrink_lruvec(struct
clear_mm_walk();
blk_finish_plug(&plug);
clear_mm_walk();
blk_finish_plug(&plug);
mm/vmscan.c | 124 ++++++++++++++++++++++++++--------------------------
1 file changed, 62 insertions(+), 62 deletions(-)
mm/vmscan.c | 124 ++++++++++++++++++++++++++--------------------------
1 file changed, 62 insertions(+), 62 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 5a2e83e673232..0c47952714b26 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4454,68 +4454,6 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4454,68 +4454,6 @@ done:
static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
{
int gen, type, zone;
static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
{
int gen, type, zone;
-@@ -5099,6 +5037,68 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
+@@ -5099,6 +5037,68 @@ retry:
/*
* For future optimizations:
* 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
/*
* For future optimizations:
* 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
mm/vmscan.c | 374 +++++++++++++++++++++++++++++++++----
6 files changed, 500 insertions(+), 35 deletions(-)
mm/vmscan.c | 374 +++++++++++++++++++++++++++++++++----
6 files changed, 500 insertions(+), 35 deletions(-)
-diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
-index e039763029563..82d28b052a9e5 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
-@@ -790,6 +790,11 @@ static inline void obj_cgroup_put(struct obj_cgroup *objcg)
+@@ -790,6 +790,11 @@ static inline void obj_cgroup_put(struct
percpu_ref_put(&objcg->refcnt);
}
percpu_ref_put(&objcg->refcnt);
}
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
if (memcg)
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
if (memcg)
-@@ -1290,6 +1295,11 @@ static inline void obj_cgroup_put(struct obj_cgroup *objcg)
+@@ -1290,6 +1295,11 @@ static inline void obj_cgroup_put(struct
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
}
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
}
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index da38e3d962e2f..c1fd3922dc5dd 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -122,6 +122,18 @@ static inline bool lru_gen_in_fault(void)
+@@ -122,6 +122,18 @@ static inline bool lru_gen_in_fault(void
return current->in_lru_fault;
}
return current->in_lru_fault;
}
static inline int lru_gen_from_seq(unsigned long seq)
{
return seq % MAX_NR_GENS;
static inline int lru_gen_from_seq(unsigned long seq)
{
return seq % MAX_NR_GENS;
-@@ -297,6 +309,11 @@ static inline bool lru_gen_in_fault(void)
+@@ -297,6 +309,11 @@ static inline bool lru_gen_in_fault(void
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
return false;
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
return false;
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 02e4323744715..66e067a635682 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -7,6 +7,7 @@
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -7,6 +7,7 @@
-@@ -479,12 +497,87 @@ void lru_gen_init_lruvec(struct lruvec *lruvec);
+@@ -479,12 +497,87 @@ void lru_gen_init_lruvec(struct lruvec *
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
#ifdef CONFIG_MEMCG
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
#ifdef CONFIG_MEMCG
static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
{
}
static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
{
}
-@@ -494,6 +587,7 @@ static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -494,6 +587,7 @@ static inline void lru_gen_look_around(s
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
}
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
}
-@@ -501,7 +595,24 @@ static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
+@@ -501,7 +595,24 @@ static inline void lru_gen_init_memcg(st
static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg)
{
}
static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg)
{
}
#endif
CACHELINE_PADDING(_pad2_);
#endif
CACHELINE_PADDING(_pad2_);
-diff --git a/mm/memcontrol.c b/mm/memcontrol.c
-index 3e8f1ad0fe9db..7815d556e38cc 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
-@@ -477,6 +477,16 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid)
+@@ -477,6 +477,16 @@ static void mem_cgroup_update_tree(struc
struct mem_cgroup_per_node *mz;
struct mem_cgroup_tree_per_node *mctz;
struct mem_cgroup_per_node *mz;
struct mem_cgroup_tree_per_node *mctz;
mctz = soft_limit_tree.rb_tree_per_node[nid];
if (!mctz)
return;
mctz = soft_limit_tree.rb_tree_per_node[nid];
if (!mctz)
return;
-@@ -3522,6 +3532,9 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
+@@ -3522,6 +3532,9 @@ unsigned long mem_cgroup_soft_limit_recl
struct mem_cgroup_tree_per_node *mctz;
unsigned long excess;
struct mem_cgroup_tree_per_node *mctz;
unsigned long excess;
-@@ -5382,6 +5395,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
+@@ -5382,6 +5395,7 @@ static int mem_cgroup_css_online(struct
if (unlikely(mem_cgroup_is_root(memcg)))
queue_delayed_work(system_unbound_wq, &stats_flush_dwork,
2UL*HZ);
if (unlikely(mem_cgroup_is_root(memcg)))
queue_delayed_work(system_unbound_wq, &stats_flush_dwork,
2UL*HZ);
return 0;
offline_kmem:
memcg_offline_kmem(memcg);
return 0;
offline_kmem:
memcg_offline_kmem(memcg);
-@@ -5413,6 +5427,7 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
+@@ -5413,6 +5427,7 @@ static void mem_cgroup_css_offline(struc
memcg_offline_kmem(memcg);
reparent_shrinker_deferred(memcg);
wb_memcg_offline(memcg);
memcg_offline_kmem(memcg);
reparent_shrinker_deferred(memcg);
wb_memcg_offline(memcg);
-@@ -5424,6 +5439,7 @@ static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
+@@ -5424,6 +5439,7 @@ static void mem_cgroup_css_released(stru
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
invalidate_reclaim_iterators(memcg);
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
invalidate_reclaim_iterators(memcg);
}
static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
}
static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
-diff --git a/mm/page_alloc.c b/mm/page_alloc.c
-index 69668817fed37..473057b81a9df 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
-@@ -7957,6 +7957,7 @@ static void __init free_area_init_node(int nid)
+@@ -7957,6 +7957,7 @@ static void __init free_area_init_node(i
pgdat_set_deferred_range(pgdat);
free_area_init_core(pgdat);
pgdat_set_deferred_range(pgdat);
free_area_init_core(pgdat);
}
static void __init free_area_init_memoryless_node(int nid)
}
static void __init free_area_init_memoryless_node(int nid)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 0c47952714b26..65eb28448f216 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -54,6 +54,8 @@
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -54,6 +54,8 @@
/* Allocation order */
s8 order;
/* Allocation order */
s8 order;
-@@ -3160,6 +3157,9 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS);
+@@ -3160,6 +3157,9 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_ca
for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
{
struct pglist_data *pgdat = NODE_DATA(nid);
static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
{
struct pglist_data *pgdat = NODE_DATA(nid);
-@@ -4440,8 +4440,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4440,8 +4440,7 @@ done:
if (sc->priority <= DEF_PRIORITY - 2)
wait_event_killable(lruvec->mm_state.wait,
max_seq < READ_ONCE(lrugen->max_seq));
if (sc->priority <= DEF_PRIORITY - 2)
wait_event_killable(lruvec->mm_state.wait,
max_seq < READ_ONCE(lrugen->max_seq));
}
VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq));
}
VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq));
-@@ -4514,8 +4513,6 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -4514,8 +4513,6 @@ static void lru_gen_age_node(struct pgli
VM_WARN_ON_ONCE(!current_is_kswapd());
VM_WARN_ON_ONCE(!current_is_kswapd());
/* check the order to exclude compaction-induced reclaim */
if (!min_ttl || sc->order || sc->priority == DEF_PRIORITY)
return;
/* check the order to exclude compaction-induced reclaim */
if (!min_ttl || sc->order || sc->priority == DEF_PRIORITY)
return;
-@@ -5104,8 +5101,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
+@@ -5104,8 +5101,7 @@ static bool should_run_aging(struct lruv
* 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
* reclaim.
*/
* 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
* reclaim.
*/
{
unsigned long nr_to_scan;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
{
unsigned long nr_to_scan;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
-@@ -5122,10 +5118,8 @@ static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *
+@@ -5122,10 +5118,8 @@ static unsigned long get_nr_to_scan(stru
if (sc->priority == DEF_PRIORITY)
return nr_to_scan;
if (sc->priority == DEF_PRIORITY)
return nr_to_scan;
}
static unsigned long get_nr_to_reclaim(struct scan_control *sc)
}
static unsigned long get_nr_to_reclaim(struct scan_control *sc)
-@@ -5134,29 +5128,18 @@ static unsigned long get_nr_to_reclaim(struct scan_control *sc)
+@@ -5134,29 +5128,18 @@ static unsigned long get_nr_to_reclaim(s
if (!global_reclaim(sc))
return -1;
if (!global_reclaim(sc))
return -1;
if (sc->may_swap)
swappiness = get_swappiness(lruvec, sc);
if (sc->may_swap)
swappiness = get_swappiness(lruvec, sc);
-@@ -5166,7 +5149,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5166,7 +5149,7 @@ static void lru_gen_shrink_lruvec(struct
swappiness = 0;
nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
swappiness = 0;
nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
break;
delta = evict_folios(lruvec, sc, swappiness);
break;
delta = evict_folios(lruvec, sc, swappiness);
-@@ -5183,11 +5166,252 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5183,10 +5166,251 @@ static void lru_gen_shrink_lruvec(struct
+ if (try_to_shrink_lruvec(lruvec, sc))
+ lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
+
+ if (try_to_shrink_lruvec(lruvec, sc))
+ lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
+
- clear_mm_walk();
-
- blk_finish_plug(&plug);
- }
-
++ clear_mm_walk();
++
++ blk_finish_plug(&plug);
++}
++
+#else /* !CONFIG_MEMCG */
+
+static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
+#else /* !CONFIG_MEMCG */
+
+static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
+ if (current_is_kswapd())
+ sc->nr_reclaimed += reclaimed;
+
+ if (current_is_kswapd())
+ sc->nr_reclaimed += reclaimed;
+
-+ clear_mm_walk();
-+
-+ blk_finish_plug(&plug);
+ clear_mm_walk();
+
+ blk_finish_plug(&plug);
+
+ /* kswapd should never fail */
+ pgdat->kswapd_failures = 0;
+
+ /* kswapd should never fail */
+ pgdat->kswapd_failures = 0;
+ WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
+
+ spin_unlock(&pgdat->memcg_lru.lock);
+ WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
+
+ spin_unlock(&pgdat->memcg_lru.lock);
/******************************************************************************
* state change
/******************************************************************************
* state change
- ******************************************************************************/
-@@ -5644,11 +5868,11 @@ static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq,
+@@ -5644,11 +5868,11 @@ static int run_cmd(char cmd, int memcg_i
if (!mem_cgroup_disabled()) {
rcu_read_lock();
if (!mem_cgroup_disabled()) {
rcu_read_lock();
rcu_read_unlock();
if (!memcg)
rcu_read_unlock();
if (!memcg)
-@@ -5796,6 +6020,19 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -5796,6 +6020,19 @@ void lru_gen_init_lruvec(struct lruvec *
void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
INIT_LIST_HEAD(&memcg->mm_list.fifo);
void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
INIT_LIST_HEAD(&memcg->mm_list.fifo);
-@@ -5819,7 +6056,69 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -5819,7 +6056,69 @@ void lru_gen_exit_memcg(struct mem_cgrou
static int __init init_lru_gen(void)
{
static int __init init_lru_gen(void)
{
-@@ -5846,6 +6145,10 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5846,6 +6145,10 @@ static void lru_gen_shrink_lruvec(struct
#endif /* CONFIG_LRU_GEN */
static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
#endif /* CONFIG_LRU_GEN */
static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
-@@ -5859,7 +6162,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
+@@ -5859,7 +6162,7 @@ static void shrink_lruvec(struct lruvec
bool proportional_reclaim;
struct blk_plug plug;
bool proportional_reclaim;
struct blk_plug plug;
lru_gen_shrink_lruvec(lruvec, sc);
return;
}
lru_gen_shrink_lruvec(lruvec, sc);
return;
}
-@@ -6102,6 +6405,11 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
+@@ -6102,6 +6405,11 @@ static void shrink_node(pg_data_t *pgdat
struct lruvec *target_lruvec;
bool reclaimable = false;
struct lruvec *target_lruvec;
bool reclaimable = false;
target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
again:
target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
again:
mm/vmscan.c | 55 +++++++++++++++++++++++++++--------------------------
1 file changed, 28 insertions(+), 27 deletions(-)
mm/vmscan.c | 55 +++++++++++++++++++++++++++--------------------------
1 file changed, 28 insertions(+), 27 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 65eb28448f216..0a0e1250ffc87 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3185,6 +3185,9 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
+@@ -3185,6 +3185,9 @@ static int get_swappiness(struct lruvec
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
if (!can_demote(pgdat->node_id, sc) &&
mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
return 0;
if (!can_demote(pgdat->node_id, sc) &&
mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
return 0;
-@@ -4223,7 +4226,7 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_
+@@ -4223,7 +4226,7 @@ static void walk_mm(struct lruvec *lruve
} while (err == -EAGAIN);
}
} while (err == -EAGAIN);
}
{
struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
{
struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
-@@ -4231,7 +4234,7 @@ static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat)
+@@ -4231,7 +4234,7 @@ static struct lru_gen_mm_walk *set_mm_wa
VM_WARN_ON_ONCE(walk);
walk = &pgdat->mm_walk;
VM_WARN_ON_ONCE(walk);
walk = &pgdat->mm_walk;
VM_WARN_ON_ONCE(current_is_kswapd());
walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
VM_WARN_ON_ONCE(current_is_kswapd());
walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
-@@ -4417,7 +4420,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4417,7 +4420,7 @@ static bool try_to_inc_max_seq(struct lr
if (!walk) {
success = iterate_mm_list_nowalk(lruvec, max_seq);
goto done;
if (!walk) {
success = iterate_mm_list_nowalk(lruvec, max_seq);
goto done;
-@@ -4486,8 +4489,6 @@ static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc
+@@ -4486,8 +4489,6 @@ static bool lruvec_is_reclaimable(struct
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MIN_SEQ(lruvec);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MIN_SEQ(lruvec);
/* see the comment on lru_gen_folio */
gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]);
birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
/* see the comment on lru_gen_folio */
gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]);
birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
-@@ -4743,12 +4744,8 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
+@@ -4743,12 +4744,8 @@ static bool isolate_folio(struct lruvec
(folio_test_dirty(folio) ||
(folio_test_anon(folio) && !folio_test_swapcache(folio))))
return false;
(folio_test_dirty(folio) ||
(folio_test_anon(folio) && !folio_test_swapcache(folio))))
return false;
-@@ -4845,9 +4842,8 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
+@@ -4845,9 +4842,8 @@ static int scan_folios(struct lruvec *lr
__count_vm_events(PGSCAN_ANON + type, isolated);
/*
__count_vm_events(PGSCAN_ANON + type, isolated);
/*
*/
return isolated || !remaining ? scanned : 0;
}
*/
return isolated || !remaining ? scanned : 0;
}
-@@ -5107,8 +5103,7 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool
+@@ -5107,8 +5103,7 @@ static long get_nr_to_scan(struct lruvec
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MAX_SEQ(lruvec);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MAX_SEQ(lruvec);
return 0;
if (!should_run_aging(lruvec, max_seq, sc, can_swap, &nr_to_scan))
return 0;
if (!should_run_aging(lruvec, max_seq, sc, can_swap, &nr_to_scan))
-@@ -5136,17 +5131,14 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
+@@ -5136,17 +5131,14 @@ static bool try_to_shrink_lruvec(struct
long nr_to_scan;
unsigned long scanned = 0;
unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
long nr_to_scan;
unsigned long scanned = 0;
unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
if (nr_to_scan <= 0)
nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
if (nr_to_scan <= 0)
-@@ -5277,12 +5269,13 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5277,12 +5269,13 @@ static void lru_gen_shrink_lruvec(struct
struct blk_plug plug;
VM_WARN_ON_ONCE(global_reclaim(sc));
struct blk_plug plug;
VM_WARN_ON_ONCE(global_reclaim(sc));
if (try_to_shrink_lruvec(lruvec, sc))
lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
if (try_to_shrink_lruvec(lruvec, sc))
lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
-@@ -5338,11 +5331,19 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
+@@ -5338,11 +5331,19 @@ static void lru_gen_shrink_node(struct p
VM_WARN_ON_ONCE(!global_reclaim(sc));
VM_WARN_ON_ONCE(!global_reclaim(sc));
set_initial_priority(pgdat, sc);
set_initial_priority(pgdat, sc);
-@@ -5360,7 +5361,7 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
+@@ -5360,7 +5361,7 @@ static void lru_gen_shrink_node(struct p
clear_mm_walk();
blk_finish_plug(&plug);
clear_mm_walk();
blk_finish_plug(&plug);
/* kswapd should never fail */
pgdat->kswapd_failures = 0;
}
/* kswapd should never fail */
pgdat->kswapd_failures = 0;
}
-@@ -5932,7 +5933,7 @@ static ssize_t lru_gen_seq_write(struct file *file, const char __user *src,
+@@ -5932,7 +5933,7 @@ static ssize_t lru_gen_seq_write(struct
set_task_reclaim_state(current, &sc.reclaim_state);
flags = memalloc_noreclaim_save();
blk_start_plug(&plug);
set_task_reclaim_state(current, &sc.reclaim_state);
flags = memalloc_noreclaim_save();
blk_start_plug(&plug);
err = -ENOMEM;
goto done;
}
err = -ENOMEM;
goto done;
}
mm/vmscan.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
mm/vmscan.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 0a0e1250ffc87..aa9746f2bc80b 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4415,7 +4415,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4415,7 +4415,7 @@ static bool try_to_inc_max_seq(struct lr
* handful of PTEs. Spreading the work out over a period of time usually
* is less efficient, but it avoids bursty page faults.
*/
* handful of PTEs. Spreading the work out over a period of time usually
* is less efficient, but it avoids bursty page faults.
*/
success = iterate_mm_list_nowalk(lruvec, max_seq);
goto done;
}
success = iterate_mm_list_nowalk(lruvec, max_seq);
goto done;
}
mm/vmscan.c | 25 +++++++++++++++----------
1 file changed, 15 insertions(+), 10 deletions(-)
mm/vmscan.c | 25 +++++++++++++++----------
1 file changed, 15 insertions(+), 10 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index aa9746f2bc80b..49da02f841c81 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -5206,18 +5206,20 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
+@@ -5206,18 +5206,20 @@ static int shrink_one(struct lruvec *lru
static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
{
static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
{
gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq));
rcu_read_lock();
gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq));
rcu_read_lock();
-@@ -5241,14 +5243,22 @@ static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -5241,14 +5243,22 @@ restart:
op = shrink_one(lruvec, sc);
op = shrink_one(lruvec, sc);
/* restart if raced with lru_gen_rotate_memcg() */
if (gen != get_nulls_value(pos))
goto restart;
/* restart if raced with lru_gen_rotate_memcg() */
if (gen != get_nulls_value(pos))
goto restart;
-@@ -5257,11 +5267,6 @@ static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -5257,11 +5267,6 @@ restart:
bin = get_memcg_bin(bin + 1);
if (bin != first_bin)
goto restart;
bin = get_memcg_bin(bin + 1);
if (bin != first_bin)
goto restart;
}
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
}
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
mm/vmscan.c | 5 ++++-
4 files changed, 33 insertions(+), 29 deletions(-)
mm/vmscan.c | 5 ++++-
4 files changed, 33 insertions(+), 29 deletions(-)
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index c1fd3922dc5dd..7bb2e5f94734c 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -595,4 +595,12 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
+@@ -595,4 +595,12 @@ pte_install_uffd_wp_if_needed(struct vm_
-diff --git a/mm/memory.c b/mm/memory.c
-index 747b7ea30f890..c2f48f8003c2e 100644
--- a/mm/memory.c
+++ b/mm/memory.c
--- a/mm/memory.c
+++ b/mm/memory.c
-@@ -1435,8 +1435,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
+@@ -1435,8 +1435,7 @@ again:
force_flush = 1;
set_page_dirty(page);
}
force_flush = 1;
set_page_dirty(page);
}
mark_page_accessed(page);
}
rss[mm_counter(page)]--;
mark_page_accessed(page);
}
rss[mm_counter(page)]--;
-@@ -5170,8 +5169,8 @@ static inline void mm_account_fault(struct pt_regs *regs,
+@@ -5170,8 +5169,8 @@ static inline void mm_account_fault(stru
#ifdef CONFIG_LRU_GEN
static void lru_gen_enter_fault(struct vm_area_struct *vma)
{
#ifdef CONFIG_LRU_GEN
static void lru_gen_enter_fault(struct vm_area_struct *vma)
{
}
static void lru_gen_exit_fault(void)
}
static void lru_gen_exit_fault(void)
-diff --git a/mm/rmap.c b/mm/rmap.c
-index 7da2d8d097d9b..825dac3caa1e5 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
--- a/mm/rmap.c
+++ b/mm/rmap.c
-@@ -823,25 +823,14 @@ static bool folio_referenced_one(struct folio *folio,
+@@ -823,25 +823,14 @@ static bool folio_referenced_one(struct
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
if (pmdp_clear_flush_young_notify(vma, address,
pvmw.pmd))
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
if (pmdp_clear_flush_young_notify(vma, address,
pvmw.pmd))
-@@ -875,7 +864,20 @@ static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
+@@ -875,7 +864,20 @@ static bool invalid_folio_referenced_vma
struct folio_referenced_arg *pra = arg;
struct mem_cgroup *memcg = pra->memcg;
struct folio_referenced_arg *pra = arg;
struct mem_cgroup *memcg = pra->memcg;
return true;
return false;
return true;
return false;
-@@ -906,6 +908,7 @@ int folio_referenced(struct folio *folio, int is_locked,
+@@ -906,6 +908,7 @@ int folio_referenced(struct folio *folio
.arg = (void *)&pra,
.anon_lock = folio_lock_anon_vma_read,
.try_lock = true,
.arg = (void *)&pra,
.anon_lock = folio_lock_anon_vma_read,
.try_lock = true,
-@@ -921,15 +924,6 @@ int folio_referenced(struct folio *folio, int is_locked,
+@@ -921,15 +924,6 @@ int folio_referenced(struct folio *folio
rmap_walk(folio, &rwc);
*vm_flags = pra.vm_flags;
rmap_walk(folio, &rwc);
*vm_flags = pra.vm_flags;
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 49da02f841c81..596fed6ae0439 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3778,7 +3778,10 @@ static int should_skip_vma(unsigned long start, unsigned long end, struct mm_wal
+@@ -3778,7 +3778,10 @@ static int should_skip_vma(unsigned long
if (is_vm_hugetlb_page(vma))
return true;
if (is_vm_hugetlb_page(vma))
return true;
return true;
if (vma == get_gate_vma(vma->vm_mm))
return true;
if (vma == get_gate_vma(vma->vm_mm))
mm/fadvise.c | 5 ++++-
3 files changed, 9 insertions(+), 1 deletion(-)
mm/fadvise.c | 5 ++++-
3 files changed, 9 insertions(+), 1 deletion(-)
-diff --git a/include/linux/fs.h b/include/linux/fs.h
-index f14ecbeab2a9d..97f9c41c1a43a 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
-@@ -166,6 +166,8 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
+@@ -166,6 +166,8 @@ typedef int (dio_iodone_t)(struct kiocb
/* File supports DIRECT IO */
#define FMODE_CAN_ODIRECT ((__force fmode_t)0x400000)
/* File supports DIRECT IO */
#define FMODE_CAN_ODIRECT ((__force fmode_t)0x400000)
/* File was opened by fanotify and shouldn't generate fanotify events */
#define FMODE_NONOTIFY ((__force fmode_t)0x4000000)
/* File was opened by fanotify and shouldn't generate fanotify events */
#define FMODE_NONOTIFY ((__force fmode_t)0x4000000)
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index 7bb2e5f94734c..9a8e2049333c0 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -600,6 +600,9 @@ static inline bool vma_has_recency(struct vm_area_struct *vma)
+@@ -600,6 +600,9 @@ static inline bool vma_has_recency(struc
if (vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))
return false;
if (vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))
return false;
-diff --git a/mm/fadvise.c b/mm/fadvise.c
-index c76ee665355a4..2ba24d865bf5f 100644
--- a/mm/fadvise.c
+++ b/mm/fadvise.c
--- a/mm/fadvise.c
+++ b/mm/fadvise.c
-@@ -80,7 +80,7 @@ int generic_fadvise(struct file *file, loff_t offset, loff_t len, int advice)
+@@ -80,7 +80,7 @@ int generic_fadvise(struct file *file, l
case POSIX_FADV_NORMAL:
file->f_ra.ra_pages = bdi->ra_pages;
spin_lock(&file->f_lock);
case POSIX_FADV_NORMAL:
file->f_ra.ra_pages = bdi->ra_pages;
spin_lock(&file->f_lock);
spin_unlock(&file->f_lock);
break;
case POSIX_FADV_RANDOM:
spin_unlock(&file->f_lock);
break;
case POSIX_FADV_RANDOM:
-@@ -107,6 +107,9 @@ int generic_fadvise(struct file *file, loff_t offset, loff_t len, int advice)
+@@ -107,6 +107,9 @@ int generic_fadvise(struct file *file, l
force_page_cache_readahead(mapping, file, start_index, nrpages);
break;
case POSIX_FADV_NOREUSE:
force_page_cache_readahead(mapping, file, start_index, nrpages);
break;
case POSIX_FADV_NOREUSE:
break;
case POSIX_FADV_DONTNEED:
__filemap_fdatawrite_range(mapping, offset, endbyte,
break;
case POSIX_FADV_DONTNEED:
__filemap_fdatawrite_range(mapping, offset, endbyte,
mm/vmscan.c | 4 ++++
2 files changed, 19 insertions(+)
mm/vmscan.c | 4 ++++
2 files changed, 19 insertions(+)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index d8f721f98868a..6e1483e70fdca 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
-@@ -141,6 +141,21 @@ loop has detected outlying refaults from the tier this page is in. To
+@@ -141,6 +141,21 @@ loop has detected outlying refaults from
this end, the feedback loop uses the first tier as the baseline, for
the reason stated earlier.
this end, the feedback loop uses the first tier as the baseline, for
the reason stated earlier.
Summary
-------
The multi-gen LRU can be disassembled into the following parts:
Summary
-------
The multi-gen LRU can be disassembled into the following parts:
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 596fed6ae0439..ab0b8d3b9d88f 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4459,6 +4459,10 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4459,6 +4459,10 @@ done:
static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
{
int gen, type, zone;
static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
{
int gen, type, zone;
mm/vmscan.c | 4 ++++
2 files changed, 18 insertions(+)
mm/vmscan.c | 4 ++++
2 files changed, 18 insertions(+)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index 6e1483e70fdca..bd988a142bc2f 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
-@@ -156,6 +156,20 @@ This time-based approach has the following advantages:
+@@ -156,6 +156,20 @@ This time-based approach has the followi
and memory sizes.
2. It is more reliable because it is directly wired to the OOM killer.
and memory sizes.
2. It is more reliable because it is directly wired to the OOM killer.
Summary
-------
The multi-gen LRU can be disassembled into the following parts:
Summary
-------
The multi-gen LRU can be disassembled into the following parts:
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index ab0b8d3b9d88f..8fa82630240d6 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4553,6 +4553,10 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -4553,6 +4553,10 @@ static void lru_gen_age_node(struct pgli
/*
* This function exploits spatial locality when shrink_folio_list() walks the
* rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If
/*
* This function exploits spatial locality when shrink_folio_list() walks the
* rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If
mm/vmscan.c | 180 +++++++++++++++---------------
2 files changed, 108 insertions(+), 88 deletions(-)
mm/vmscan.c | 180 +++++++++++++++---------------
2 files changed, 108 insertions(+), 88 deletions(-)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index bd988a142bc2f..770b5d539856c 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
-@@ -170,6 +170,22 @@ promotes hot pages. If the scan was done cacheline efficiently, it
+@@ -170,6 +170,22 @@ promotes hot pages. If the scan was done
adds the PMD entry pointing to the PTE table to the Bloom filter. This
forms a feedback loop between the eviction and the aging.
adds the PMD entry pointing to the PTE table to the Bloom filter. This
forms a feedback loop between the eviction and the aging.
Summary
-------
The multi-gen LRU can be disassembled into the following parts:
Summary
-------
The multi-gen LRU can be disassembled into the following parts:
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 8fa82630240d6..74b4f9d660b56 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3208,6 +3208,98 @@ static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
- get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS;
+@@ -3209,6 +3209,98 @@ static bool __maybe_unused seq_is_valid(
-+/******************************************************************************
+ /******************************************************************************
+ * Bloom filters
+ ******************************************************************************/
+
+ * Bloom filters
+ ******************************************************************************/
+
+ WRITE_ONCE(lruvec->mm_state.filters[gen], filter);
+}
+
+ WRITE_ONCE(lruvec->mm_state.filters[gen], filter);
+}
+
- /******************************************************************************
++/******************************************************************************
* mm_struct list
******************************************************************************/
* mm_struct list
******************************************************************************/
-@@ -3333,94 +3425,6 @@ void lru_gen_migrate_mm(struct mm_struct *mm)
+
+@@ -3333,94 +3425,6 @@ void lru_gen_migrate_mm(struct mm_struct
static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last)
{
int i;
static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last)
{
int i;
mm/vmscan.c | 250 +++++++++++++++++-------------
5 files changed, 178 insertions(+), 143 deletions(-)
mm/vmscan.c | 250 +++++++++++++++++-------------
5 files changed, 178 insertions(+), 143 deletions(-)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index 770b5d539856c..5f1f6ecbb79b9 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
-@@ -186,9 +186,40 @@ is false positive, the cost is an additional scan of a range of PTEs,
+@@ -186,9 +186,40 @@ is false positive, the cost is an additi
which may yield hot pages anyway. Parameters of the filter itself can
control the false positive rate in the limit.
which may yield hot pages anyway. Parameters of the filter itself can
control the false positive rate in the limit.
* Generations
* Rmap walks
* Generations
* Rmap walks
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index 9a8e2049333c0..5567f4850243b 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -122,18 +122,6 @@ static inline bool lru_gen_in_fault(void)
+@@ -122,18 +122,6 @@ static inline bool lru_gen_in_fault(void
return current->in_lru_fault;
}
return current->in_lru_fault;
}
static inline int lru_gen_from_seq(unsigned long seq)
{
return seq % MAX_NR_GENS;
static inline int lru_gen_from_seq(unsigned long seq)
{
return seq % MAX_NR_GENS;
-@@ -309,11 +297,6 @@ static inline bool lru_gen_in_fault(void)
+@@ -309,11 +297,6 @@ static inline bool lru_gen_in_fault(void
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
return false;
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
return false;
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 66e067a635682..403c7461e7a70 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -368,15 +368,6 @@ struct page_vma_mapped_walk;
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -368,15 +368,6 @@ struct page_vma_mapped_walk;
#ifdef CONFIG_LRU_GEN
enum {
#ifdef CONFIG_LRU_GEN
enum {
-@@ -557,7 +548,7 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg);
+@@ -557,7 +548,7 @@ void lru_gen_exit_memcg(struct mem_cgrou
void lru_gen_online_memcg(struct mem_cgroup *memcg);
void lru_gen_offline_memcg(struct mem_cgroup *memcg);
void lru_gen_release_memcg(struct mem_cgroup *memcg);
void lru_gen_online_memcg(struct mem_cgroup *memcg);
void lru_gen_offline_memcg(struct mem_cgroup *memcg);
void lru_gen_release_memcg(struct mem_cgroup *memcg);
#else /* !CONFIG_MEMCG */
#else /* !CONFIG_MEMCG */
-@@ -608,7 +599,7 @@ static inline void lru_gen_release_memcg(struct mem_cgroup *memcg)
+@@ -608,7 +599,7 @@ static inline void lru_gen_release_memcg
-diff --git a/mm/memcontrol.c b/mm/memcontrol.c
-index 7815d556e38cc..5397aeb43986d 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
-@@ -478,12 +478,8 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid)
+@@ -478,12 +478,8 @@ static void mem_cgroup_update_tree(struc
struct mem_cgroup_tree_per_node *mctz;
if (lru_gen_enabled()) {
struct mem_cgroup_tree_per_node *mctz;
if (lru_gen_enabled()) {
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 74b4f9d660b56..ccde215c084ca 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4689,6 +4689,148 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
- mem_cgroup_unlock_pages();
+@@ -4690,6 +4690,148 @@ void lru_gen_look_around(struct page_vma
-+/******************************************************************************
+ /******************************************************************************
+ * memcg LRU
+ ******************************************************************************/
+
+ * memcg LRU
+ ******************************************************************************/
+
- /******************************************************************************
++/******************************************************************************
* the eviction
******************************************************************************/
* the eviction
******************************************************************************/
-@@ -5386,53 +5528,6 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
+
+@@ -5386,53 +5528,6 @@ done:
pgdat->kswapd_failures = 0;
}
pgdat->kswapd_failures = 0;
}
/******************************************************************************
* state change
******************************************************************************/
/******************************************************************************
* state change
******************************************************************************/
-@@ -6078,67 +6173,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -6078,67 +6173,6 @@ void lru_gen_exit_memcg(struct mem_cgrou
#endif /* CONFIG_MEMCG */
static int __init init_lru_gen(void)
#endif /* CONFIG_MEMCG */
static int __init init_lru_gen(void)
mm/vmscan.c | 5 +++++
1 file changed, 5 insertions(+)
mm/vmscan.c | 5 +++++
1 file changed, 5 insertions(+)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index ccde215c084ca..d5d6f8d94f58a 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -6160,12 +6160,17 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -6160,12 +6160,17 @@ void lru_gen_exit_memcg(struct mem_cgrou
for (i = 0; i < NR_BLOOM_FILTERS; i++) {
bitmap_free(lruvec->mm_state.filters[i]);
lruvec->mm_state.filters[i] = NULL;
for (i = 0; i < NR_BLOOM_FILTERS; i++) {
bitmap_free(lruvec->mm_state.filters[i]);
lruvec->mm_state.filters[i] = NULL;
mm/vmscan.c | 40 ++++++++++++++++++++--------------------
1 file changed, 20 insertions(+), 20 deletions(-)
mm/vmscan.c | 40 ++++++++++++++++++++--------------------
1 file changed, 20 insertions(+), 20 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index d5d6f8d94f58a..8f496c2e670a9 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3980,8 +3980,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
+@@ -3980,8 +3980,8 @@ restart:
}
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
}
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
-@@ -3994,18 +3994,19 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
+@@ -3994,18 +3994,19 @@ static void walk_pmd_range_locked(pud_t
VM_WARN_ON_ONCE(pud_leaf(*pud));
/* try to batch at most 1+MIN_LRU_BATCH+1 entries */
VM_WARN_ON_ONCE(pud_leaf(*pud));
/* try to batch at most 1+MIN_LRU_BATCH+1 entries */
ptl = pmd_lockptr(args->mm, pmd);
if (!spin_trylock(ptl))
ptl = pmd_lockptr(args->mm, pmd);
if (!spin_trylock(ptl))
-@@ -4016,15 +4017,16 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
+@@ -4016,15 +4017,16 @@ static void walk_pmd_range_locked(pud_t
do {
unsigned long pfn;
struct folio *folio;
do {
unsigned long pfn;
struct folio *folio;
pmdp_test_and_clear_young(vma, addr, pmd + i);
goto next;
}
pmdp_test_and_clear_young(vma, addr, pmd + i);
goto next;
}
-@@ -4053,12 +4055,11 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
+@@ -4053,12 +4055,11 @@ next:
arch_leave_lazy_mmu_mode();
spin_unlock(ptl);
done:
arch_leave_lazy_mmu_mode();
spin_unlock(ptl);
done:
-@@ -4071,9 +4072,9 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+@@ -4071,9 +4072,9 @@ static void walk_pmd_range(pud_t *pud, u
unsigned long next;
unsigned long addr;
struct vm_area_struct *vma;
unsigned long next;
unsigned long addr;
struct vm_area_struct *vma;
VM_WARN_ON_ONCE(pud_leaf(*pud));
VM_WARN_ON_ONCE(pud_leaf(*pud));
-@@ -4115,18 +4116,17 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+@@ -4115,18 +4116,17 @@ restart:
if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
continue;
if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
continue;
}
if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
}
if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
-@@ -4143,7 +4143,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+@@ -4143,7 +4143,7 @@ restart:
update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
}
update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
}
if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
goto restart;
if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
goto restart;
mm/vmscan.c | 73 +++++++++++++++++------------------------------------
1 file changed, 23 insertions(+), 50 deletions(-)
mm/vmscan.c | 73 +++++++++++++++++------------------------------------
1 file changed, 23 insertions(+), 50 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 8f496c2e670a9..f6ce7a1fd78a3 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4571,13 +4571,12 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -4571,13 +4571,12 @@ static void lru_gen_age_node(struct pgli
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
{
int i;
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
{
int i;
struct folio *folio = pfn_folio(pvmw->pfn);
struct mem_cgroup *memcg = folio_memcg(folio);
struct pglist_data *pgdat = folio_pgdat(folio);
struct folio *folio = pfn_folio(pvmw->pfn);
struct mem_cgroup *memcg = folio_memcg(folio);
struct pglist_data *pgdat = folio_pgdat(folio);
-@@ -4594,25 +4593,28 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -4594,25 +4593,28 @@ void lru_gen_look_around(struct page_vma
/* avoid taking the LRU lock under the PTL when possible */
walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL;
/* avoid taking the LRU lock under the PTL when possible */
walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL;
for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) {
unsigned long pfn;
for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) {
unsigned long pfn;
-@@ -4637,56 +4639,27 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -4637,56 +4639,27 @@ void lru_gen_look_around(struct page_vma
!folio_test_swapcache(folio)))
folio_mark_dirty(folio);
!folio_test_swapcache(folio)))
folio_mark_dirty(folio);
}
/******************************************************************************
}
/******************************************************************************
mm/vmscan.c | 112 +++++++++++++++--------------------------
2 files changed, 42 insertions(+), 78 deletions(-)
mm/vmscan.c | 112 +++++++++++++++--------------------------
2 files changed, 42 insertions(+), 78 deletions(-)
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 403c7461e7a70..d62a5accf1be4 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -453,18 +453,14 @@ enum {
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -453,18 +453,14 @@ enum {
};
struct lru_gen_mm_walk {
};
struct lru_gen_mm_walk {
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index f6ce7a1fd78a3..851758303dbf4 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3371,18 +3371,13 @@ void lru_gen_del_mm(struct mm_struct *mm)
+@@ -3371,18 +3371,13 @@ void lru_gen_del_mm(struct mm_struct *mm
}
list_del_init(&mm->lru_gen.list);
}
list_del_init(&mm->lru_gen.list);
-@@ -3478,68 +3473,54 @@ static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk,
+@@ -3478,68 +3473,54 @@ static bool iterate_mm_list(struct lruve
struct mm_struct **iter)
{
bool first = false;
struct mm_struct **iter)
{
bool first = false;
if (*iter || last)
reset_mm_stats(lruvec, walk, last);
if (*iter || last)
reset_mm_stats(lruvec, walk, last);
-@@ -3567,9 +3548,9 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq)
+@@ -3567,9 +3548,9 @@ static bool iterate_mm_list_nowalk(struc
VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);
VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);
WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
reset_mm_stats(lruvec, NULL, true);
success = true;
WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
reset_mm_stats(lruvec, NULL, true);
success = true;
-@@ -4172,10 +4153,6 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
+@@ -4172,10 +4153,6 @@ restart:
walk_pmd_range(&val, addr, next, args);
walk_pmd_range(&val, addr, next, args);
if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
end = (addr | ~PUD_MASK) + 1;
goto done;
if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
end = (addr | ~PUD_MASK) + 1;
goto done;
-@@ -4208,8 +4185,14 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_
+@@ -4208,8 +4185,14 @@ static void walk_mm(struct lruvec *lruve
walk->next_addr = FIRST_USER_ADDRESS;
do {
walk->next_addr = FIRST_USER_ADDRESS;
do {
/* folio_update_gen() requires stable folio_memcg() */
if (!mem_cgroup_trylock_pages(memcg))
break;
/* folio_update_gen() requires stable folio_memcg() */
if (!mem_cgroup_trylock_pages(memcg))
break;
-@@ -4442,25 +4425,12 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4442,25 +4425,12 @@ static bool try_to_inc_max_seq(struct lr
success = iterate_mm_list(lruvec, walk, &mm);
if (mm)
walk_mm(lruvec, mm, walk);
success = iterate_mm_list(lruvec, walk, &mm);
if (mm)
walk_mm(lruvec, mm, walk);
}
/******************************************************************************
}
/******************************************************************************
-@@ -6105,7 +6075,6 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -6105,7 +6075,6 @@ void lru_gen_init_lruvec(struct lruvec *
INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
lruvec->mm_state.seq = MIN_NR_GENS;
INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
lruvec->mm_state.seq = MIN_NR_GENS;
-@@ -6138,7 +6107,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -6138,7 +6107,6 @@ void lru_gen_exit_memcg(struct mem_cgrou
for_each_node(nid) {
struct lruvec *lruvec = get_lruvec(memcg, nid);
for_each_node(nid) {
struct lruvec *lruvec = get_lruvec(memcg, nid);
VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
sizeof(lruvec->lrugen.nr_pages)));
VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
sizeof(lruvec->lrugen.nr_pages)));
projects / openwrt / staging / hauke.git / commitdiff