diff --git a/target/linux/generic/backport-5.4/020-01-UPSTREAM-mm-vmscan.c-use-update_lru_size-in-update_l.patch b/target/linux/generic/backport-5.4/020-01-UPSTREAM-mm-vmscan.c-use-update_lru_size-in-update_l.patch new file mode 100644 index 0000000000..50c0b75422 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-01-UPSTREAM-mm-vmscan.c-use-update_lru_size-in-update_l.patch @@ -0,0 +1,47 @@ +From 6bf249661e5e9b053469c352c25d7a72ad9ac5e1 Mon Sep 17 00:00:00 2001 +From: Wei Yang +Date: Wed, 3 Jun 2020 16:01:12 -0700 +Subject: [PATCH] UPSTREAM: mm/vmscan.c: use update_lru_size() in + update_lru_sizes() + +We already defined the helper update_lru_size(). + +Let's use this to reduce code duplication. + +Signed-off-by: Wei Yang +Signed-off-by: Andrew Morton +Reviewed-by: Baoquan He +Reviewed-by: Andrew Morton +Acked-by: Michal Hocko +Link: http://lkml.kernel.org/r/20200331221550.1011-1-richard.weiyang@gmail.com +Signed-off-by: Linus Torvalds +(cherry picked from commit a892cb6b977ffe209683809e5e9d627656d20aa8) +Signed-off-by: Yu Zhao + +BUG=b:123039911 +TEST=Built + +Change-Id: I2997bda67966cb097475644a65275b2f221b93bf +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940108 +Reviewed-by: Sean Paul +Reviewed-by: Sonny Rao +Tested-by: Yu Zhao +Commit-Queue: Yu Zhao +--- + mm/vmscan.c | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -1655,10 +1655,7 @@ static __always_inline void update_lru_s + if (!nr_zone_taken[zid]) + continue; + +- __update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); +-#ifdef CONFIG_MEMCG +- mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); +-#endif ++ update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); + } + + } diff --git a/target/linux/generic/backport-5.4/020-02-BACKPORT-mm-remove-superfluous-__ClearPageActive.patch b/target/linux/generic/backport-5.4/020-02-BACKPORT-mm-remove-superfluous-__ClearPageActive.patch new file mode 100644 index 0000000000..4e7f3a5a77 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-02-BACKPORT-mm-remove-superfluous-__ClearPageActive.patch @@ -0,0 +1,69 @@ +From 45e277af28e0517cea1581d4f7f0c48d116314a9 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Sat, 18 Jul 2020 01:46:37 -0600 +Subject: [PATCH] BACKPORT: mm: remove superfluous __ClearPageActive() + +To activate a page, mark_page_accessed() always holds a reference on it. +It either gets a new reference when adding a page to +lru_pvecs.activate_page or reuses an existing one it previously got when +it added a page to lru_pvecs.lru_add. So it doesn't call SetPageActive() +on a page that doesn't have any reference left. Therefore, the race is +impossible these days (I didn't brother to dig into its history). + +For other paths, namely reclaim and migration, a reference count is always +held while calling SetPageActive() on a page. + +SetPageSlabPfmemalloc() also uses SetPageActive(), but it's irrelevant to +LRU pages. + +Signed-off-by: Yu Zhao +Signed-off-by: Andrew Morton +Reviewed-by: Yang Shi +Cc: Alexander Duyck +Cc: David Hildenbrand +Cc: Huang Ying +Cc: Hugh Dickins +Cc: Joonsoo Kim +Cc: Mel Gorman +Cc: Michal Hocko +Cc: Nicholas Piggin +Cc: Qian Cai +Link: http://lkml.kernel.org/r/20200818184704.3625199-2-yuzhao@google.com +Signed-off-by: Linus Torvalds +(cherry picked from commit 6f4dd8de4835563de9bae797ce1d7a13465a7a7d) + +BUG=b:123039911 +TEST=Built + +Change-Id: I3e50ae28408b2936b1eb72210b3046eac8485701 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940109 +Tested-by: Yu Zhao +Reviewed-by: Sonny Rao +Commit-Queue: Yu Zhao +--- + mm/memremap.c | 2 -- + mm/swap.c | 2 -- + 2 files changed, 4 deletions(-) + +--- a/mm/memremap.c ++++ b/mm/memremap.c +@@ -419,8 +419,6 @@ void __put_devmap_managed_page(struct pa + * holds a reference on the page. + */ + if (count == 1) { +- /* Clear Active bit in case of parallel mark_page_accessed */ +- __ClearPageActive(page); + __ClearPageWaiters(page); + + mem_cgroup_uncharge(page); +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -829,8 +829,6 @@ void release_pages(struct page **pages, + del_page_from_lru_list(page, lruvec, page_off_lru(page)); + } + +- /* Clear Active bit in case of parallel mark_page_accessed */ +- __ClearPageActive(page); + __ClearPageWaiters(page); + + list_add(&page->lru, &pages_to_free); diff --git a/target/linux/generic/backport-5.4/020-03-BACKPORT-mm-use-self-explanatory-macros-rather-than-.patch b/target/linux/generic/backport-5.4/020-03-BACKPORT-mm-use-self-explanatory-macros-rather-than-.patch new file mode 100644 index 0000000000..9325df0ff9 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-03-BACKPORT-mm-use-self-explanatory-macros-rather-than-.patch @@ -0,0 +1,104 @@ +From f8c45effbd31e3b66802c0c2556933e7f122b180 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Thu, 23 Jul 2020 14:01:45 -0600 +Subject: [PATCH] BACKPORT: mm: use self-explanatory macros rather than + "2" + +Signed-off-by: Yu Zhao +Signed-off-by: Andrew Morton +Cc: Alex Shi +Link: http://lkml.kernel.org/r/20200831175042.3527153-2-yuzhao@google.com +Signed-off-by: Linus Torvalds +(cherry picked from commit ed0173733dd468883198c3136284394320b8fad6) + +BUG=b:123039911 +TEST=Built + +Change-Id: Idb88d5839e920893589bca08121196a5dd719354 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940110 +Tested-by: Yu Zhao +Reviewed-by: Sonny Rao +Commit-Queue: Yu Zhao +--- + include/linux/mmzone.h | 14 ++++++++------ + mm/memcontrol.c | 4 ++-- + mm/vmscan.c | 4 +--- + 3 files changed, 11 insertions(+), 11 deletions(-) + +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -282,17 +282,17 @@ static inline int is_active_lru(enum lru + return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); + } + ++#define ANON_AND_FILE 2 ++ + struct zone_reclaim_stat { + /* + * The pageout code in vmscan.c keeps track of how many of the + * mem/swap backed and file backed pages are referenced. + * The higher the rotated/scanned ratio, the more valuable + * that cache is. +- * +- * The anon LRU stats live in [0], file LRU stats in [1] + */ +- unsigned long recent_rotated[2]; +- unsigned long recent_scanned[2]; ++ unsigned long recent_rotated[ANON_AND_FILE]; ++ unsigned long recent_scanned[ANON_AND_FILE]; + }; + + struct lruvec { +@@ -414,6 +414,8 @@ enum zone_type { + + #ifndef __GENERATING_BOUNDS_H + ++#define ASYNC_AND_SYNC 2 ++ + struct zone { + /* Read-mostly fields */ + +@@ -533,8 +535,8 @@ struct zone { + #if defined CONFIG_COMPACTION || defined CONFIG_CMA + /* pfn where compaction free scanner should start */ + unsigned long compact_cached_free_pfn; +- /* pfn where async and sync compaction migration scanner should start */ +- unsigned long compact_cached_migrate_pfn[2]; ++ /* pfn where compaction migration scanner should start */ ++ unsigned long compact_cached_migrate_pfn[ASYNC_AND_SYNC]; + unsigned long compact_init_migrate_pfn; + unsigned long compact_init_free_pfn; + #endif +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -3980,8 +3980,8 @@ static int memcg_stat_show(struct seq_fi + pg_data_t *pgdat; + struct mem_cgroup_per_node *mz; + struct zone_reclaim_stat *rstat; +- unsigned long recent_rotated[2] = {0, 0}; +- unsigned long recent_scanned[2] = {0, 0}; ++ unsigned long recent_rotated[ANON_AND_FILE] = {}; ++ unsigned long recent_scanned[ANON_AND_FILE] = {}; + + for_each_online_pgdat(pgdat) { + mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id); +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -2307,7 +2307,7 @@ static void get_scan_count(struct lruvec + { + int swappiness = mem_cgroup_swappiness(memcg); + struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; +- u64 fraction[2]; ++ u64 fraction[ANON_AND_FILE]; + u64 denominator = 0; /* gcc */ + struct pglist_data *pgdat = lruvec_pgdat(lruvec); + unsigned long anon_prio, file_prio; +@@ -2418,8 +2418,6 @@ static void get_scan_count(struct lruvec + * Because workloads change over time (and to avoid overflow) + * we keep these statistics as a floating average, which ends + * up weighing recent references more than old ones. +- * +- * anon in [0], file in [1] + */ + + anon = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) + diff --git a/target/linux/generic/backport-5.4/020-04-BACKPORT-mm-vmscan.c-use-add_page_to_lru_list.patch b/target/linux/generic/backport-5.4/020-04-BACKPORT-mm-vmscan.c-use-add_page_to_lru_list.patch new file mode 100644 index 0000000000..e477b1ba7b --- /dev/null +++ b/target/linux/generic/backport-5.4/020-04-BACKPORT-mm-vmscan.c-use-add_page_to_lru_list.patch @@ -0,0 +1,110 @@ +From a36a2ac4005e9c1e58bd5e1ffd2672337d535c51 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Mon, 13 Jul 2020 11:37:30 -0600 +Subject: [PATCH] BACKPORT: mm/vmscan.c: use add_page_to_lru_list() + +Patch series "mm: lru related cleanups", v2. + +The cleanups are intended to reduce the verbosity in lru list operations +and make them less error-prone. A typical example would be how the +patches change __activate_page(): + + static void __activate_page(struct page *page, struct lruvec *lruvec) + { + if (!PageActive(page) && !PageUnevictable(page)) { +- int lru = page_lru_base_type(page); + int nr_pages = thp_nr_pages(page); + +- del_page_from_lru_list(page, lruvec, lru); ++ del_page_from_lru_list(page, lruvec); + SetPageActive(page); +- lru += LRU_ACTIVE; +- add_page_to_lru_list(page, lruvec, lru); ++ add_page_to_lru_list(page, lruvec); + trace_mm_lru_activate(page); + +There are a few more places like __activate_page() and they are +unnecessarily repetitive in terms of figuring out which list a page should +be added onto or deleted from. And with the duplicated code removed, they +are easier to read, IMO. + +Patch 1 to 5 basically cover the above. Patch 6 and 7 make code more +robust by improving bug reporting. Patch 8, 9 and 10 take care of some +dangling helpers left in header files. + +This patch (of 10): + +There is add_page_to_lru_list(), and move_pages_to_lru() should reuse it, +not duplicate it. + +Link: https://lkml.kernel.org/r/20210122220600.906146-1-yuzhao@google.com +Link: https://lore.kernel.org/linux-mm/20201207220949.830352-2-yuzhao@google.com/ +Link: https://lkml.kernel.org/r/20210122220600.906146-2-yuzhao@google.com +Signed-off-by: Yu Zhao +Reviewed-by: Alex Shi +Acked-by: Vlastimil Babka +Reviewed-by: Miaohe Lin +Cc: Hugh Dickins +Cc: Michal Hocko +Cc: Johannes Weiner +Cc: Vladimir Davydov +Cc: Roman Gushchin +Cc: Matthew Wilcox +Signed-off-by: Andrew Morton +Signed-off-by: Linus Torvalds +(cherry picked from commit 42895ea73bcd37c4a79e4c9f681ab8b82243c7f7) + +BUG=b:123039911 +TEST=Built + +Change-Id: I11a7493ee7263b03453c2bc203e9770e12f2577a +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940111 +Tested-by: Yu Zhao +Reviewed-by: Sonny Rao +Commit-Queue: Yu Zhao +--- + mm/vmscan.c | 11 ++++------- + 1 file changed, 4 insertions(+), 7 deletions(-) + +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -1879,7 +1879,7 @@ static unsigned noinline_for_stack move_ + struct list_head *list) + { + struct pglist_data *pgdat = lruvec_pgdat(lruvec); +- int nr_pages, nr_moved = 0; ++ int nr_moved = 0; + LIST_HEAD(pages_to_free); + struct page *page; + enum lru_list lru; +@@ -1887,8 +1887,8 @@ static unsigned noinline_for_stack move_ + while (!list_empty(list)) { + page = lru_to_page(list); + VM_BUG_ON_PAGE(PageLRU(page), page); ++ list_del(&page->lru); + if (unlikely(!page_evictable(page))) { +- list_del(&page->lru); + spin_unlock_irq(&pgdat->lru_lock); + putback_lru_page(page); + spin_lock_irq(&pgdat->lru_lock); +@@ -1898,10 +1898,7 @@ static unsigned noinline_for_stack move_ + + SetPageLRU(page); + lru = page_lru(page); +- +- nr_pages = hpage_nr_pages(page); +- update_lru_size(lruvec, lru, page_zonenum(page), nr_pages); +- list_move(&page->lru, &lruvec->lists[lru]); ++ add_page_to_lru_list(page, lruvec, lru); + + if (put_page_testzero(page)) { + __ClearPageLRU(page); +@@ -1915,7 +1912,7 @@ static unsigned noinline_for_stack move_ + } else + list_add(&page->lru, &pages_to_free); + } else { +- nr_moved += nr_pages; ++ nr_moved += hpage_nr_pages(page); + } + } + diff --git a/target/linux/generic/backport-5.4/020-05-BACKPORT-include-linux-mm_inline.h-shuffle-lru-list-.patch b/target/linux/generic/backport-5.4/020-05-BACKPORT-include-linux-mm_inline.h-shuffle-lru-list-.patch new file mode 100644 index 0000000000..c781a0fd28 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-05-BACKPORT-include-linux-mm_inline.h-shuffle-lru-list-.patch @@ -0,0 +1,93 @@ +From 982961cba9ff379a006cdbbd1339496a9819d519 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Sat, 18 Jul 2020 01:48:19 -0600 +Subject: [PATCH] BACKPORT: include/linux/mm_inline.h: shuffle lru list + addition and deletion functions + +These functions will call page_lru() in the following patches. Move them +below page_lru() to avoid the forward declaration. + +Link: https://lore.kernel.org/linux-mm/20201207220949.830352-3-yuzhao@google.com/ +Link: https://lkml.kernel.org/r/20210122220600.906146-3-yuzhao@google.com +Signed-off-by: Yu Zhao +Acked-by: Vlastimil Babka +Reviewed-by: Miaohe Lin +Cc: Alex Shi +Cc: Hugh Dickins +Cc: Johannes Weiner +Cc: Matthew Wilcox +Cc: Michal Hocko +Cc: Roman Gushchin +Cc: Vladimir Davydov +Signed-off-by: Andrew Morton +Signed-off-by: Linus Torvalds +(cherry picked from commit f90d8191ac864df33b1898bc7edc54eaa24e22bc) + +BUG=b:123039911 +TEST=Built + +Change-Id: I1ab7d53b954c0e6dca45b522929c61b3fe781a5f +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940112 +Tested-by: Yu Zhao +Reviewed-by: Sonny Rao +Commit-Queue: Yu Zhao +--- + include/linux/mm_inline.h | 42 +++++++++++++++++++-------------------- + 1 file changed, 21 insertions(+), 21 deletions(-) + +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -44,27 +44,6 @@ static __always_inline void update_lru_s + #endif + } + +-static __always_inline void add_page_to_lru_list(struct page *page, +- struct lruvec *lruvec, enum lru_list lru) +-{ +- update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page)); +- list_add(&page->lru, &lruvec->lists[lru]); +-} +- +-static __always_inline void add_page_to_lru_list_tail(struct page *page, +- struct lruvec *lruvec, enum lru_list lru) +-{ +- update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page)); +- list_add_tail(&page->lru, &lruvec->lists[lru]); +-} +- +-static __always_inline void del_page_from_lru_list(struct page *page, +- struct lruvec *lruvec, enum lru_list lru) +-{ +- list_del(&page->lru); +- update_lru_size(lruvec, lru, page_zonenum(page), -hpage_nr_pages(page)); +-} +- + /** + * page_lru_base_type - which LRU list type should a page be on? + * @page: the page to test +@@ -124,4 +103,25 @@ static __always_inline enum lru_list pag + } + return lru; + } ++ ++static __always_inline void add_page_to_lru_list(struct page *page, ++ struct lruvec *lruvec, enum lru_list lru) ++{ ++ update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page)); ++ list_add(&page->lru, &lruvec->lists[lru]); ++} ++ ++static __always_inline void add_page_to_lru_list_tail(struct page *page, ++ struct lruvec *lruvec, enum lru_list lru) ++{ ++ update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page)); ++ list_add_tail(&page->lru, &lruvec->lists[lru]); ++} ++ ++static __always_inline void del_page_from_lru_list(struct page *page, ++ struct lruvec *lruvec, enum lru_list lru) ++{ ++ list_del(&page->lru); ++ update_lru_size(lruvec, lru, page_zonenum(page), -hpage_nr_pages(page)); ++} + #endif diff --git a/target/linux/generic/backport-5.4/020-06-BACKPORT-mm-don-t-pass-enum-lru_list-to-lru-list-add.patch b/target/linux/generic/backport-5.4/020-06-BACKPORT-mm-don-t-pass-enum-lru_list-to-lru-list-add.patch new file mode 100644 index 0000000000..26aaacdd30 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-06-BACKPORT-mm-don-t-pass-enum-lru_list-to-lru-list-add.patch @@ -0,0 +1,179 @@ +From aba67dd60f603313f58b8a1faaf1d8da9e932757 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Fri, 2 Oct 2020 01:29:59 -0600 +Subject: [PATCH] BACKPORT: mm: don't pass "enum lru_list" to lru list + addition functions + +The "enum lru_list" parameter to add_page_to_lru_list() and +add_page_to_lru_list_tail() is redundant in the sense that it can +be extracted from the "struct page" parameter by page_lru(). + +A caveat is that we need to make sure PageActive() or +PageUnevictable() is correctly set or cleared before calling +these two functions. And they are indeed. + +Link: https://lore.kernel.org/linux-mm/20201207220949.830352-4-yuzhao@google.com/ +Link: https://lkml.kernel.org/r/20210122220600.906146-4-yuzhao@google.com +Signed-off-by: Yu Zhao +Cc: Alex Shi +Cc: Hugh Dickins +Cc: Johannes Weiner +Cc: Matthew Wilcox +Cc: Michal Hocko +Cc: Roman Gushchin +Cc: Vladimir Davydov +Cc: Vlastimil Babka +Signed-off-by: Andrew Morton +Signed-off-by: Linus Torvalds +(cherry picked from commit 3a9c9788a3149d9745b7eb2eae811e57ef3b127c) + +BUG=b:123039911 +TEST=Built + +Change-Id: Ib58324f3641a83a43d752af5177c40f47a42d8e1 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940113 +Tested-by: Yu Zhao +Reviewed-by: Sonny Rao +Commit-Queue: Yu Zhao +--- + include/linux/mm_inline.h | 8 ++++++-- + mm/memcontrol.c | 2 +- + mm/swap.c | 18 ++++++++---------- + mm/vmscan.c | 6 ++---- + 4 files changed, 17 insertions(+), 17 deletions(-) + +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -105,15 +105,19 @@ static __always_inline enum lru_list pag + } + + static __always_inline void add_page_to_lru_list(struct page *page, +- struct lruvec *lruvec, enum lru_list lru) ++ struct lruvec *lruvec) + { ++ enum lru_list lru = page_lru(page); ++ + update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page)); + list_add(&page->lru, &lruvec->lists[lru]); + } + + static __always_inline void add_page_to_lru_list_tail(struct page *page, +- struct lruvec *lruvec, enum lru_list lru) ++ struct lruvec *lruvec) + { ++ enum lru_list lru = page_lru(page); ++ + update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page)); + list_add_tail(&page->lru, &lruvec->lists[lru]); + } +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -2757,7 +2757,7 @@ static void unlock_page_lru(struct page + lruvec = mem_cgroup_page_lruvec(page, pgdat); + VM_BUG_ON_PAGE(PageLRU(page), page); + SetPageLRU(page); +- add_page_to_lru_list(page, lruvec, page_lru(page)); ++ add_page_to_lru_list(page, lruvec); + } + spin_unlock_irq(&pgdat->lru_lock); + } +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -224,7 +224,7 @@ static void pagevec_move_tail_fn(struct + if (PageLRU(page) && !PageUnevictable(page)) { + del_page_from_lru_list(page, lruvec, page_lru(page)); + ClearPageActive(page); +- add_page_to_lru_list_tail(page, lruvec, page_lru(page)); ++ add_page_to_lru_list_tail(page, lruvec); + (*pgmoved)++; + } + } +@@ -281,8 +281,7 @@ static void __activate_page(struct page + + del_page_from_lru_list(page, lruvec, lru); + SetPageActive(page); +- lru += LRU_ACTIVE; +- add_page_to_lru_list(page, lruvec, lru); ++ add_page_to_lru_list(page, lruvec); + trace_mm_lru_activate(page); + + __count_vm_event(PGACTIVATE); +@@ -523,14 +522,14 @@ static void lru_deactivate_file_fn(struc + * It can make readahead confusing. But race window + * is _really_ small and it's non-critical problem. + */ +- add_page_to_lru_list(page, lruvec, lru); ++ add_page_to_lru_list(page, lruvec); + SetPageReclaim(page); + } else { + /* + * The page's writeback ends up during pagevec + * We moves tha page into tail of inactive. + */ +- add_page_to_lru_list_tail(page, lruvec, lru); ++ add_page_to_lru_list_tail(page, lruvec); + __count_vm_event(PGROTATED); + } + +@@ -549,7 +548,7 @@ static void lru_deactivate_fn(struct pag + del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); + ClearPageActive(page); + ClearPageReferenced(page); +- add_page_to_lru_list(page, lruvec, lru); ++ add_page_to_lru_list(page, lruvec); + + __count_vm_events(PGDEACTIVATE, hpage_nr_pages(page)); + update_page_reclaim_stat(lruvec, file, 0); +@@ -573,7 +572,7 @@ static void lru_lazyfree_fn(struct page + * pages + */ + ClearPageSwapBacked(page); +- add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); ++ add_page_to_lru_list(page, lruvec); + + __count_vm_events(PGLAZYFREE, hpage_nr_pages(page)); + count_memcg_page_event(page, PGLAZYFREE); +@@ -891,8 +890,7 @@ void lru_add_page_tail(struct page *page + * Put page_tail on the list at the correct position + * so they all end up in order. + */ +- add_page_to_lru_list_tail(page_tail, lruvec, +- page_lru(page_tail)); ++ add_page_to_lru_list_tail(page_tail, lruvec); + } + + if (!PageUnevictable(page)) +@@ -951,7 +949,7 @@ static void __pagevec_lru_add_fn(struct + count_vm_event(UNEVICTABLE_PGCULLED); + } + +- add_page_to_lru_list(page, lruvec, lru); ++ add_page_to_lru_list(page, lruvec); + trace_mm_lru_insertion(page, lru); + } + +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -1898,7 +1898,7 @@ static unsigned noinline_for_stack move_ + + SetPageLRU(page); + lru = page_lru(page); +- add_page_to_lru_list(page, lruvec, lru); ++ add_page_to_lru_list(page, lruvec); + + if (put_page_testzero(page)) { + __ClearPageLRU(page); +@@ -4379,12 +4379,10 @@ void check_move_unevictable_pages(struct + continue; + + if (page_evictable(page)) { +- enum lru_list lru = page_lru_base_type(page); +- + VM_BUG_ON_PAGE(PageActive(page), page); + ClearPageUnevictable(page); + del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE); +- add_page_to_lru_list(page, lruvec, lru); ++ add_page_to_lru_list(page, lruvec); + pgrescued++; + } + } diff --git a/target/linux/generic/backport-5.4/020-07-UPSTREAM-mm-swap.c-don-t-pass-enum-lru_list-to-trace.patch b/target/linux/generic/backport-5.4/020-07-UPSTREAM-mm-swap.c-don-t-pass-enum-lru_list-to-trace.patch new file mode 100644 index 0000000000..5ce5a3950e --- /dev/null +++ b/target/linux/generic/backport-5.4/020-07-UPSTREAM-mm-swap.c-don-t-pass-enum-lru_list-to-trace.patch @@ -0,0 +1,102 @@ +From 9aa0ad973ffdb49292d8aa288f81351d3e711df7 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Fri, 2 Oct 2020 01:32:40 -0600 +Subject: [PATCH] UPSTREAM: mm/swap.c: don't pass "enum lru_list" to + trace_mm_lru_insertion() + +The parameter is redundant in the sense that it can be extracted +from the "struct page" parameter by page_lru() correctly. + +Link: https://lore.kernel.org/linux-mm/20201207220949.830352-5-yuzhao@google.com/ +Link: https://lkml.kernel.org/r/20210122220600.906146-5-yuzhao@google.com +Signed-off-by: Yu Zhao +Reviewed-by: Alex Shi +Cc: Hugh Dickins +Cc: Johannes Weiner +Cc: Matthew Wilcox +Cc: Michal Hocko +Cc: Roman Gushchin +Cc: Vladimir Davydov +Cc: Vlastimil Babka +Signed-off-by: Andrew Morton +Signed-off-by: Linus Torvalds +(cherry picked from commit 861404536a3af3c39f1b10959a40def3d8efa2dd) + +BUG=b:123039911 +TEST=Built + +Change-Id: I06661696d32705a0753b45d5886ae87a59953ee7 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940114 +Reviewed-by: Sean Paul +Reviewed-by: Sonny Rao +Tested-by: Yu Zhao +Commit-Queue: Yu Zhao +--- + include/trace/events/pagemap.h | 11 ++++------- + mm/swap.c | 5 +---- + 2 files changed, 5 insertions(+), 11 deletions(-) + +--- a/include/trace/events/pagemap.h ++++ b/include/trace/events/pagemap.h +@@ -27,24 +27,21 @@ + + TRACE_EVENT(mm_lru_insertion, + +- TP_PROTO( +- struct page *page, +- int lru +- ), ++ TP_PROTO(struct page *page), + +- TP_ARGS(page, lru), ++ TP_ARGS(page), + + TP_STRUCT__entry( + __field(struct page *, page ) + __field(unsigned long, pfn ) +- __field(int, lru ) ++ __field(enum lru_list, lru ) + __field(unsigned long, flags ) + ), + + TP_fast_assign( + __entry->page = page; + __entry->pfn = page_to_pfn(page); +- __entry->lru = lru; ++ __entry->lru = page_lru(page); + __entry->flags = trace_pagemap_flags(page); + ), + +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -901,7 +901,6 @@ void lru_add_page_tail(struct page *page + static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, + void *arg) + { +- enum lru_list lru; + int was_unevictable = TestClearPageUnevictable(page); + + VM_BUG_ON_PAGE(PageLRU(page), page); +@@ -936,13 +935,11 @@ static void __pagevec_lru_add_fn(struct + smp_mb(); + + if (page_evictable(page)) { +- lru = page_lru(page); + update_page_reclaim_stat(lruvec, page_is_file_cache(page), + PageActive(page)); + if (was_unevictable) + count_vm_event(UNEVICTABLE_PGRESCUED); + } else { +- lru = LRU_UNEVICTABLE; + ClearPageActive(page); + SetPageUnevictable(page); + if (!was_unevictable) +@@ -950,7 +947,7 @@ static void __pagevec_lru_add_fn(struct + } + + add_page_to_lru_list(page, lruvec); +- trace_mm_lru_insertion(page, lru); ++ trace_mm_lru_insertion(page); + } + + /* diff --git a/target/linux/generic/backport-5.4/020-08-BACKPORT-mm-swap.c-don-t-pass-enum-lru_list-to-del_p.patch b/target/linux/generic/backport-5.4/020-08-BACKPORT-mm-swap.c-don-t-pass-enum-lru_list-to-del_p.patch new file mode 100644 index 0000000000..d435640f86 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-08-BACKPORT-mm-swap.c-don-t-pass-enum-lru_list-to-del_p.patch @@ -0,0 +1,227 @@ +From f9aa82051ae34c6907d10d3cf40d44988ae7f1f3 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Fri, 2 Oct 2020 01:43:39 -0600 +Subject: [PATCH] BACKPORT: mm/swap.c: don't pass "enum lru_list" to + del_page_from_lru_list() + +The parameter is redundant in the sense that it can be potentially +extracted from the "struct page" parameter by page_lru(). We need to +make sure that existing PageActive() or PageUnevictable() remains +until the function returns. A few places don't conform, and simple +reordering fixes them. + +This patch may have left page_off_lru() seemingly odd, and we'll take +care of it in the next patch. + +Link: https://lore.kernel.org/linux-mm/20201207220949.830352-6-yuzhao@google.com/ +Link: https://lkml.kernel.org/r/20210122220600.906146-6-yuzhao@google.com +Signed-off-by: Yu Zhao +Cc: Alex Shi +Cc: Hugh Dickins +Cc: Johannes Weiner +Cc: Matthew Wilcox +Cc: Michal Hocko +Cc: Roman Gushchin +Cc: Vladimir Davydov +Cc: Vlastimil Babka +Signed-off-by: Andrew Morton +Signed-off-by: Linus Torvalds +(cherry picked from commit 46ae6b2cc2a47904a368d238425531ea91f3a2a5) + +BUG=b:123039911 +TEST=Built + +Change-Id: Iaf7a9c8c71da7d41c40f566ef9be8ac33c4e012d +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940115 +Tested-by: Yu Zhao +Reviewed-by: Sonny Rao +Commit-Queue: Yu Zhao +--- + include/linux/mm_inline.h | 5 +++-- + mm/compaction.c | 2 +- + mm/memcontrol.c | 2 +- + mm/mlock.c | 2 +- + mm/swap.c | 24 ++++++++++-------------- + mm/vmscan.c | 9 +++------ + 6 files changed, 19 insertions(+), 25 deletions(-) + +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -123,9 +123,10 @@ static __always_inline void add_page_to_ + } + + static __always_inline void del_page_from_lru_list(struct page *page, +- struct lruvec *lruvec, enum lru_list lru) ++ struct lruvec *lruvec) + { + list_del(&page->lru); +- update_lru_size(lruvec, lru, page_zonenum(page), -hpage_nr_pages(page)); ++ update_lru_size(lruvec, page_lru(page), page_zonenum(page), ++ -hpage_nr_pages(page)); + } + #endif +--- a/mm/compaction.c ++++ b/mm/compaction.c +@@ -984,7 +984,7 @@ isolate_migratepages_block(struct compac + VM_BUG_ON_PAGE(PageCompound(page), page); + + /* Successfully isolated */ +- del_page_from_lru_list(page, lruvec, page_lru(page)); ++ del_page_from_lru_list(page, lruvec); + inc_node_page_state(page, + NR_ISOLATED_ANON + page_is_file_cache(page)); + +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -2741,7 +2741,7 @@ static void lock_page_lru(struct page *p + + lruvec = mem_cgroup_page_lruvec(page, pgdat); + ClearPageLRU(page); +- del_page_from_lru_list(page, lruvec, page_lru(page)); ++ del_page_from_lru_list(page, lruvec); + *isolated = 1; + } else + *isolated = 0; +--- a/mm/mlock.c ++++ b/mm/mlock.c +@@ -115,7 +115,7 @@ static bool __munlock_isolate_lru_page(s + if (getpage) + get_page(page); + ClearPageLRU(page); +- del_page_from_lru_list(page, lruvec, page_lru(page)); ++ del_page_from_lru_list(page, lruvec); + return true; + } + +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -68,7 +68,8 @@ static void __page_cache_release(struct + lruvec = mem_cgroup_page_lruvec(page, pgdat); + VM_BUG_ON_PAGE(!PageLRU(page), page); + __ClearPageLRU(page); +- del_page_from_lru_list(page, lruvec, page_off_lru(page)); ++ del_page_from_lru_list(page, lruvec); ++ page_off_lru(page); + spin_unlock_irqrestore(&pgdat->lru_lock, flags); + } + __ClearPageWaiters(page); +@@ -222,7 +223,7 @@ static void pagevec_move_tail_fn(struct + int *pgmoved = arg; + + if (PageLRU(page) && !PageUnevictable(page)) { +- del_page_from_lru_list(page, lruvec, page_lru(page)); ++ del_page_from_lru_list(page, lruvec); + ClearPageActive(page); + add_page_to_lru_list_tail(page, lruvec); + (*pgmoved)++; +@@ -277,9 +278,8 @@ static void __activate_page(struct page + { + if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { + int file = page_is_file_cache(page); +- int lru = page_lru_base_type(page); + +- del_page_from_lru_list(page, lruvec, lru); ++ del_page_from_lru_list(page, lruvec); + SetPageActive(page); + add_page_to_lru_list(page, lruvec); + trace_mm_lru_activate(page); +@@ -495,7 +495,7 @@ void lru_cache_add_active_or_unevictable + static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, + void *arg) + { +- int lru, file; ++ int file; + bool active; + + if (!PageLRU(page)) +@@ -510,9 +510,8 @@ static void lru_deactivate_file_fn(struc + + active = PageActive(page); + file = page_is_file_cache(page); +- lru = page_lru_base_type(page); + +- del_page_from_lru_list(page, lruvec, lru + active); ++ del_page_from_lru_list(page, lruvec); + ClearPageActive(page); + ClearPageReferenced(page); + +@@ -543,9 +542,8 @@ static void lru_deactivate_fn(struct pag + { + if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { + int file = page_is_file_cache(page); +- int lru = page_lru_base_type(page); + +- del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); ++ del_page_from_lru_list(page, lruvec); + ClearPageActive(page); + ClearPageReferenced(page); + add_page_to_lru_list(page, lruvec); +@@ -560,10 +558,7 @@ static void lru_lazyfree_fn(struct page + { + if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && + !PageSwapCache(page) && !PageUnevictable(page)) { +- bool active = PageActive(page); +- +- del_page_from_lru_list(page, lruvec, +- LRU_INACTIVE_ANON + active); ++ del_page_from_lru_list(page, lruvec); + ClearPageActive(page); + ClearPageReferenced(page); + /* +@@ -825,7 +820,8 @@ void release_pages(struct page **pages, + lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); + VM_BUG_ON_PAGE(!PageLRU(page), page); + __ClearPageLRU(page); +- del_page_from_lru_list(page, lruvec, page_off_lru(page)); ++ del_page_from_lru_list(page, lruvec); ++ page_off_lru(page); + } + + __ClearPageWaiters(page); +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -1807,10 +1807,9 @@ int isolate_lru_page(struct page *page) + spin_lock_irq(&pgdat->lru_lock); + lruvec = mem_cgroup_page_lruvec(page, pgdat); + if (PageLRU(page)) { +- int lru = page_lru(page); + get_page(page); + ClearPageLRU(page); +- del_page_from_lru_list(page, lruvec, lru); ++ del_page_from_lru_list(page, lruvec); + ret = 0; + } + spin_unlock_irq(&pgdat->lru_lock); +@@ -1882,7 +1881,6 @@ static unsigned noinline_for_stack move_ + int nr_moved = 0; + LIST_HEAD(pages_to_free); + struct page *page; +- enum lru_list lru; + + while (!list_empty(list)) { + page = lru_to_page(list); +@@ -1897,13 +1895,12 @@ static unsigned noinline_for_stack move_ + lruvec = mem_cgroup_page_lruvec(page, pgdat); + + SetPageLRU(page); +- lru = page_lru(page); + add_page_to_lru_list(page, lruvec); + + if (put_page_testzero(page)) { + __ClearPageLRU(page); ++ del_page_from_lru_list(page, lruvec); + __ClearPageActive(page); +- del_page_from_lru_list(page, lruvec, lru); + + if (unlikely(PageCompound(page))) { + spin_unlock_irq(&pgdat->lru_lock); +@@ -4380,8 +4377,8 @@ void check_move_unevictable_pages(struct + + if (page_evictable(page)) { + VM_BUG_ON_PAGE(PageActive(page), page); ++ del_page_from_lru_list(page, lruvec); + ClearPageUnevictable(page); +- del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE); + add_page_to_lru_list(page, lruvec); + pgrescued++; + } diff --git a/target/linux/generic/backport-5.4/020-09-BACKPORT-mm-add-__clear_page_lru_flags-to-replace-pa.patch b/target/linux/generic/backport-5.4/020-09-BACKPORT-mm-add-__clear_page_lru_flags-to-replace-pa.patch new file mode 100644 index 0000000000..96c0a37f2a --- /dev/null +++ b/target/linux/generic/backport-5.4/020-09-BACKPORT-mm-add-__clear_page_lru_flags-to-replace-pa.patch @@ -0,0 +1,120 @@ +From 16d3c016eabd1964657d5099ae745b57c9d6b10d Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Fri, 2 Oct 2020 01:54:36 -0600 +Subject: [PATCH] BACKPORT: mm: add __clear_page_lru_flags() to replace + page_off_lru() + +Similar to page_off_lru(), the new function does non-atomic clearing +of PageLRU() in addition to PageActive() and PageUnevictable(), on a +page that has no references left. + +If PageActive() and PageUnevictable() are both set, refuse to clear +either and leave them to bad_page(). This is a behavior change that +is meant to help debug. + +Link: https://lore.kernel.org/linux-mm/20201207220949.830352-7-yuzhao@google.com/ +Link: https://lkml.kernel.org/r/20210122220600.906146-7-yuzhao@google.com +Signed-off-by: Yu Zhao +Cc: Alex Shi +Cc: Hugh Dickins +Cc: Johannes Weiner +Cc: Matthew Wilcox +Cc: Michal Hocko +Cc: Roman Gushchin +Cc: Vladimir Davydov +Cc: Vlastimil Babka +Signed-off-by: Andrew Morton +Signed-off-by: Linus Torvalds +(cherry picked from commit 875601796267214f286d3581fe74f2805d060fe8) + +BUG=b:123039911 +TEST=Built + +Change-Id: I86b973cd52a0ddb0fb1453c5fcd787aa885297e6 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940116 +Tested-by: Yu Zhao +Reviewed-by: Sonny Rao +Commit-Queue: Yu Zhao +--- + include/linux/mm_inline.h | 28 ++++++++++------------------ + mm/swap.c | 6 ++---- + mm/vmscan.c | 3 +-- + 3 files changed, 13 insertions(+), 24 deletions(-) + +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -60,27 +60,19 @@ static inline enum lru_list page_lru_bas + } + + /** +- * page_off_lru - which LRU list was page on? clearing its lru flags. +- * @page: the page to test +- * +- * Returns the LRU list a page was on, as an index into the array of LRU +- * lists; and clears its Unevictable or Active flags, ready for freeing. ++ * __clear_page_lru_flags - clear page lru flags before releasing a page ++ * @page: the page that was on lru and now has a zero reference + */ +-static __always_inline enum lru_list page_off_lru(struct page *page) ++static __always_inline void __clear_page_lru_flags(struct page *page) + { +- enum lru_list lru; ++ __ClearPageLRU(page); + +- if (PageUnevictable(page)) { +- __ClearPageUnevictable(page); +- lru = LRU_UNEVICTABLE; +- } else { +- lru = page_lru_base_type(page); +- if (PageActive(page)) { +- __ClearPageActive(page); +- lru += LRU_ACTIVE; +- } +- } +- return lru; ++ /* this shouldn't happen, so leave the flags to bad_page() */ ++ if (PageActive(page) && PageUnevictable(page)) ++ return; ++ ++ __ClearPageActive(page); ++ __ClearPageUnevictable(page); + } + + /** +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -67,9 +67,8 @@ static void __page_cache_release(struct + spin_lock_irqsave(&pgdat->lru_lock, flags); + lruvec = mem_cgroup_page_lruvec(page, pgdat); + VM_BUG_ON_PAGE(!PageLRU(page), page); +- __ClearPageLRU(page); + del_page_from_lru_list(page, lruvec); +- page_off_lru(page); ++ __clear_page_lru_flags(page); + spin_unlock_irqrestore(&pgdat->lru_lock, flags); + } + __ClearPageWaiters(page); +@@ -819,9 +818,8 @@ void release_pages(struct page **pages, + + lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); + VM_BUG_ON_PAGE(!PageLRU(page), page); +- __ClearPageLRU(page); + del_page_from_lru_list(page, lruvec); +- page_off_lru(page); ++ __clear_page_lru_flags(page); + } + + __ClearPageWaiters(page); +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -1898,9 +1898,8 @@ static unsigned noinline_for_stack move_ + add_page_to_lru_list(page, lruvec); + + if (put_page_testzero(page)) { +- __ClearPageLRU(page); + del_page_from_lru_list(page, lruvec); +- __ClearPageActive(page); ++ __clear_page_lru_flags(page); + + if (unlikely(PageCompound(page))) { + spin_unlock_irq(&pgdat->lru_lock); diff --git a/target/linux/generic/backport-5.4/020-10-UPSTREAM-mm-VM_BUG_ON-lru-page-flags.patch b/target/linux/generic/backport-5.4/020-10-UPSTREAM-mm-VM_BUG_ON-lru-page-flags.patch new file mode 100644 index 0000000000..3bb5d24ad4 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-10-UPSTREAM-mm-VM_BUG_ON-lru-page-flags.patch @@ -0,0 +1,86 @@ +From b135b6b637ee1035d2f69187ed38d8c85ff45ba2 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Fri, 2 Oct 2020 02:08:58 -0600 +Subject: [PATCH] UPSTREAM: mm: VM_BUG_ON lru page flags + +Move scattered VM_BUG_ONs to two essential places that cover all +lru list additions and deletions. + +Link: https://lore.kernel.org/linux-mm/20201207220949.830352-8-yuzhao@google.com/ +Link: https://lkml.kernel.org/r/20210122220600.906146-8-yuzhao@google.com +Signed-off-by: Yu Zhao +Cc: Alex Shi +Cc: Hugh Dickins +Cc: Johannes Weiner +Cc: Matthew Wilcox +Cc: Michal Hocko +Cc: Roman Gushchin +Cc: Vladimir Davydov +Cc: Vlastimil Babka +Signed-off-by: Andrew Morton +Signed-off-by: Linus Torvalds +(cherry picked from commit bc7112719e1e80e4208eef3fc9bd8d2b6c263e7d) + +BUG=b:123039911 +TEST=Built + +Change-Id: I46712058a18b740251a7c1c80b9dcbcc42dac457 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940117 +Reviewed-by: Sean Paul +Reviewed-by: Sonny Rao +Tested-by: Yu Zhao +Commit-Queue: Yu Zhao +--- + include/linux/mm_inline.h | 4 ++++ + mm/swap.c | 2 -- + mm/vmscan.c | 1 - + 3 files changed, 4 insertions(+), 3 deletions(-) + +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -65,6 +65,8 @@ static inline enum lru_list page_lru_bas + */ + static __always_inline void __clear_page_lru_flags(struct page *page) + { ++ VM_BUG_ON_PAGE(!PageLRU(page), page); ++ + __ClearPageLRU(page); + + /* this shouldn't happen, so leave the flags to bad_page() */ +@@ -86,6 +88,8 @@ static __always_inline enum lru_list pag + { + enum lru_list lru; + ++ VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); ++ + if (PageUnevictable(page)) + lru = LRU_UNEVICTABLE; + else { +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -66,7 +66,6 @@ static void __page_cache_release(struct + + spin_lock_irqsave(&pgdat->lru_lock, flags); + lruvec = mem_cgroup_page_lruvec(page, pgdat); +- VM_BUG_ON_PAGE(!PageLRU(page), page); + del_page_from_lru_list(page, lruvec); + __clear_page_lru_flags(page); + spin_unlock_irqrestore(&pgdat->lru_lock, flags); +@@ -817,7 +816,6 @@ void release_pages(struct page **pages, + } + + lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); +- VM_BUG_ON_PAGE(!PageLRU(page), page); + del_page_from_lru_list(page, lruvec); + __clear_page_lru_flags(page); + } +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -4375,7 +4375,6 @@ void check_move_unevictable_pages(struct + continue; + + if (page_evictable(page)) { +- VM_BUG_ON_PAGE(PageActive(page), page); + del_page_from_lru_list(page, lruvec); + ClearPageUnevictable(page); + add_page_to_lru_list(page, lruvec); diff --git a/target/linux/generic/backport-5.4/020-11-BACKPORT-include-linux-mm_inline.h-fold-page_lru_bas.patch b/target/linux/generic/backport-5.4/020-11-BACKPORT-include-linux-mm_inline.h-fold-page_lru_bas.patch new file mode 100644 index 0000000000..be57867b8f --- /dev/null +++ b/target/linux/generic/backport-5.4/020-11-BACKPORT-include-linux-mm_inline.h-fold-page_lru_bas.patch @@ -0,0 +1,78 @@ +From 02323fd32189b0912bab89b9cb7d863e7d11a394 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Fri, 2 Oct 2020 02:11:33 -0600 +Subject: [PATCH] BACKPORT: include/linux/mm_inline.h: fold + page_lru_base_type() into its sole caller + +We've removed all other references to this function. + +Link: https://lore.kernel.org/linux-mm/20201207220949.830352-9-yuzhao@google.com/ +Link: https://lkml.kernel.org/r/20210122220600.906146-9-yuzhao@google.com +Signed-off-by: Yu Zhao +Reviewed-by: Alex Shi +Cc: Hugh Dickins +Cc: Johannes Weiner +Cc: Matthew Wilcox +Cc: Michal Hocko +Cc: Roman Gushchin +Cc: Vladimir Davydov +Cc: Vlastimil Babka +Signed-off-by: Andrew Morton +Signed-off-by: Linus Torvalds +(cherry picked from commit c1770e34f3e7640887d8129fc05d13fe17101301) + +BUG=b:123039911 +TEST=Built + +Change-Id: I26244fd85e60f46f74281add8138a0a9d65a6916 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940118 +Tested-by: Yu Zhao +Reviewed-by: Sonny Rao +Commit-Queue: Yu Zhao +--- + include/linux/mm_inline.h | 27 ++++++--------------------- + 1 file changed, 6 insertions(+), 21 deletions(-) + +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -45,21 +45,6 @@ static __always_inline void update_lru_s + } + + /** +- * page_lru_base_type - which LRU list type should a page be on? +- * @page: the page to test +- * +- * Used for LRU list index arithmetic. +- * +- * Returns the base LRU type - file or anon - @page should be on. +- */ +-static inline enum lru_list page_lru_base_type(struct page *page) +-{ +- if (page_is_file_cache(page)) +- return LRU_INACTIVE_FILE; +- return LRU_INACTIVE_ANON; +-} +- +-/** + * __clear_page_lru_flags - clear page lru flags before releasing a page + * @page: the page that was on lru and now has a zero reference + */ +@@ -91,12 +76,12 @@ static __always_inline enum lru_list pag + VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); + + if (PageUnevictable(page)) +- lru = LRU_UNEVICTABLE; +- else { +- lru = page_lru_base_type(page); +- if (PageActive(page)) +- lru += LRU_ACTIVE; +- } ++ return LRU_UNEVICTABLE; ++ ++ lru = page_is_file_cache(page) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON; ++ if (PageActive(page)) ++ lru += LRU_ACTIVE; ++ + return lru; + } + diff --git a/target/linux/generic/backport-5.4/020-12-UPSTREAM-include-linux-mm_inline.h-fold-__update_lru.patch b/target/linux/generic/backport-5.4/020-12-UPSTREAM-include-linux-mm_inline.h-fold-__update_lru.patch new file mode 100644 index 0000000000..c6f387bc39 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-12-UPSTREAM-include-linux-mm_inline.h-fold-__update_lru.patch @@ -0,0 +1,62 @@ +From a13fb50b6ae0de88dba9057c061b343ef2e41170 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Fri, 2 Oct 2020 02:41:47 -0600 +Subject: [PATCH] UPSTREAM: include/linux/mm_inline.h: fold + __update_lru_size() into its sole caller + +All other references to the function were removed after commit +a892cb6b977f ("mm/vmscan.c: use update_lru_size() in update_lru_sizes()"). + +Link: https://lore.kernel.org/linux-mm/20201207220949.830352-10-yuzhao@google.com/ +Link: https://lkml.kernel.org/r/20210122220600.906146-10-yuzhao@google.com +Signed-off-by: Yu Zhao +Reviewed-by: Alex Shi +Cc: Hugh Dickins +Cc: Johannes Weiner +Cc: Matthew Wilcox +Cc: Michal Hocko +Cc: Roman Gushchin +Cc: Vladimir Davydov +Cc: Vlastimil Babka +Signed-off-by: Andrew Morton +Signed-off-by: Linus Torvalds +(cherry picked from commit 289ccba18af436f2b65ec69b2be1b086ec9f24a4) + +BUG=b:123039911 +TEST=Built + +Change-Id: I81f7635f8d2ddd4f2269a2d56647914817fc9951 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2940119 +Reviewed-by: Sean Paul +Reviewed-by: Sonny Rao +Tested-by: Yu Zhao +Commit-Queue: Yu Zhao +--- + include/linux/mm_inline.h | 9 +-------- + 1 file changed, 1 insertion(+), 8 deletions(-) + +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -23,7 +23,7 @@ static inline int page_is_file_cache(str + return !PageSwapBacked(page); + } + +-static __always_inline void __update_lru_size(struct lruvec *lruvec, ++static __always_inline void update_lru_size(struct lruvec *lruvec, + enum lru_list lru, enum zone_type zid, + int nr_pages) + { +@@ -32,13 +32,6 @@ static __always_inline void __update_lru + __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); + __mod_zone_page_state(&pgdat->node_zones[zid], + NR_ZONE_LRU_BASE + lru, nr_pages); +-} +- +-static __always_inline void update_lru_size(struct lruvec *lruvec, +- enum lru_list lru, enum zone_type zid, +- int nr_pages) +-{ +- __update_lru_size(lruvec, lru, zid, nr_pages); + #ifdef CONFIG_MEMCG + mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); + #endif diff --git a/target/linux/generic/backport-5.4/020-13-BACKPORT-mm-swapcache-support-to-handle-the-shadow-e.patch b/target/linux/generic/backport-5.4/020-13-BACKPORT-mm-swapcache-support-to-handle-the-shadow-e.patch new file mode 100644 index 0000000000..8f409c7e36 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-13-BACKPORT-mm-swapcache-support-to-handle-the-shadow-e.patch @@ -0,0 +1,261 @@ +From 1b950746b18526adb4534e656651850f73a9d1c1 Mon Sep 17 00:00:00 2001 +From: Joonsoo Kim +Date: Tue, 11 Aug 2020 18:30:47 -0700 +Subject: [PATCH] BACKPORT: mm/swapcache: support to handle the shadow entries + +Workingset detection for anonymous page will be implemented in the +following patch and it requires to store the shadow entries into the +swapcache. This patch implements an infrastructure to store the shadow +entry in the swapcache. + +Signed-off-by: Joonsoo Kim +Signed-off-by: Andrew Morton +Acked-by: Johannes Weiner +Cc: Hugh Dickins +Cc: Matthew Wilcox +Cc: Mel Gorman +Cc: Michal Hocko +Cc: Minchan Kim +Cc: Vlastimil Babka +Link: http://lkml.kernel.org/r/1595490560-15117-5-git-send-email-iamjoonsoo.kim@lge.com +Signed-off-by: Linus Torvalds +(cherry picked from commit 3852f6768ede542ed48b9077bedf482c7ecb6327) +Signed-off-by: Yu Zhao + +BUG=b:123039911 +TEST=Built + +Change-Id: I913ad8bfa0bbb744e35a0da4b684cdb5c557f394 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2951281 +Reviewed-by: Yu Zhao +Commit-Queue: Yu Zhao +Tested-by: Yu Zhao +--- + include/linux/swap.h | 17 +++++++++---- + mm/shmem.c | 2 +- + mm/swap_state.c | 57 +++++++++++++++++++++++++++++++++++++++----- + mm/swapfile.c | 2 ++ + mm/vmscan.c | 2 +- + 5 files changed, 68 insertions(+), 12 deletions(-) + +--- a/include/linux/swap.h ++++ b/include/linux/swap.h +@@ -408,10 +408,14 @@ extern struct address_space *swapper_spa + extern unsigned long total_swapcache_pages(void); + extern void show_swap_cache_info(void); + extern int add_to_swap(struct page *page); +-extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t); ++extern int add_to_swap_cache(struct page *page, swp_entry_t entry, ++ gfp_t gfp, void **shadowp); + extern int __add_to_swap_cache(struct page *page, swp_entry_t entry); +-extern void __delete_from_swap_cache(struct page *, swp_entry_t entry); ++extern void __delete_from_swap_cache(struct page *page, ++ swp_entry_t entry, void *shadow); + extern void delete_from_swap_cache(struct page *); ++extern void clear_shadow_from_swap_cache(int type, unsigned long begin, ++ unsigned long end); + extern void free_page_and_swap_cache(struct page *); + extern void free_pages_and_swap_cache(struct page **, int); + extern struct page *lookup_swap_cache(swp_entry_t entry, +@@ -566,13 +570,13 @@ static inline int add_to_swap(struct pag + } + + static inline int add_to_swap_cache(struct page *page, swp_entry_t entry, +- gfp_t gfp_mask) ++ gfp_t gfp_mask, void **shadowp) + { + return -1; + } + + static inline void __delete_from_swap_cache(struct page *page, +- swp_entry_t entry) ++ swp_entry_t entry, void *shadow) + { + } + +@@ -580,6 +584,11 @@ static inline void delete_from_swap_cach + { + } + ++static inline void clear_shadow_from_swap_cache(int type, unsigned long begin, ++ unsigned long end) ++{ ++} ++ + static inline int page_swapcount(struct page *page) + { + return 0; +--- a/mm/shmem.c ++++ b/mm/shmem.c +@@ -1374,7 +1374,7 @@ static int shmem_writepage(struct page * + if (list_empty(&info->swaplist)) + list_add(&info->swaplist, &shmem_swaplist); + +- if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { ++ if (add_to_swap_cache(page, swap, GFP_ATOMIC, NULL) == 0) { + spin_lock_irq(&info->lock); + shmem_recalc_inode(inode); + info->swapped++; +--- a/mm/swap_state.c ++++ b/mm/swap_state.c +@@ -112,12 +112,14 @@ void show_swap_cache_info(void) + * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space, + * but sets SwapCache flag and private instead of mapping and index. + */ +-int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp) ++int add_to_swap_cache(struct page *page, swp_entry_t entry, ++ gfp_t gfp, void **shadowp) + { + struct address_space *address_space = swap_address_space(entry); + pgoff_t idx = swp_offset(entry); + XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page)); + unsigned long i, nr = compound_nr(page); ++ void *old; + + VM_BUG_ON_PAGE(!PageLocked(page), page); + VM_BUG_ON_PAGE(PageSwapCache(page), page); +@@ -127,16 +129,25 @@ int add_to_swap_cache(struct page *page, + SetPageSwapCache(page); + + do { ++ unsigned long nr_shadows = 0; ++ + xas_lock_irq(&xas); + xas_create_range(&xas); + if (xas_error(&xas)) + goto unlock; + for (i = 0; i < nr; i++) { + VM_BUG_ON_PAGE(xas.xa_index != idx + i, page); ++ old = xas_load(&xas); ++ if (xa_is_value(old)) { ++ nr_shadows++; ++ if (shadowp) ++ *shadowp = old; ++ } + set_page_private(page + i, entry.val + i); + xas_store(&xas, page); + xas_next(&xas); + } ++ address_space->nrexceptional -= nr_shadows; + address_space->nrpages += nr; + __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); + ADD_CACHE_INFO(add_total, nr); +@@ -156,7 +167,8 @@ unlock: + * This must be called only on pages that have + * been verified to be in the swap cache. + */ +-void __delete_from_swap_cache(struct page *page, swp_entry_t entry) ++void __delete_from_swap_cache(struct page *page, ++ swp_entry_t entry, void *shadow) + { + struct address_space *address_space = swap_address_space(entry); + int i, nr = hpage_nr_pages(page); +@@ -168,12 +180,14 @@ void __delete_from_swap_cache(struct pag + VM_BUG_ON_PAGE(PageWriteback(page), page); + + for (i = 0; i < nr; i++) { +- void *entry = xas_store(&xas, NULL); ++ void *entry = xas_store(&xas, shadow); + VM_BUG_ON_PAGE(entry != page, entry); + set_page_private(page + i, 0); + xas_next(&xas); + } + ClearPageSwapCache(page); ++ if (shadow) ++ address_space->nrexceptional += nr; + address_space->nrpages -= nr; + __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); + ADD_CACHE_INFO(del_total, nr); +@@ -210,7 +224,7 @@ int add_to_swap(struct page *page) + * Add it to the swap cache. + */ + err = add_to_swap_cache(page, entry, +- __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN); ++ __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN, NULL); + if (err) + /* + * add_to_swap_cache() doesn't return -EEXIST, so we can safely +@@ -248,13 +262,44 @@ void delete_from_swap_cache(struct page + struct address_space *address_space = swap_address_space(entry); + + xa_lock_irq(&address_space->i_pages); +- __delete_from_swap_cache(page, entry); ++ __delete_from_swap_cache(page, entry, NULL); + xa_unlock_irq(&address_space->i_pages); + + put_swap_page(page, entry); + page_ref_sub(page, hpage_nr_pages(page)); + } + ++void clear_shadow_from_swap_cache(int type, unsigned long begin, ++ unsigned long end) ++{ ++ unsigned long curr = begin; ++ void *old; ++ ++ for (;;) { ++ unsigned long nr_shadows = 0; ++ swp_entry_t entry = swp_entry(type, curr); ++ struct address_space *address_space = swap_address_space(entry); ++ XA_STATE(xas, &address_space->i_pages, curr); ++ ++ xa_lock_irq(&address_space->i_pages); ++ xas_for_each(&xas, old, end) { ++ if (!xa_is_value(old)) ++ continue; ++ xas_store(&xas, NULL); ++ nr_shadows++; ++ } ++ address_space->nrexceptional -= nr_shadows; ++ xa_unlock_irq(&address_space->i_pages); ++ ++ /* search the next swapcache until we meet end */ ++ curr >>= SWAP_ADDRESS_SPACE_SHIFT; ++ curr++; ++ curr <<= SWAP_ADDRESS_SPACE_SHIFT; ++ if (curr > end) ++ break; ++ } ++} ++ + /* + * If we are the only user, then try to free up the swap cache. + * +@@ -420,7 +465,7 @@ struct page *__read_swap_cache_async(swp + __SetPageLocked(new_page); + __SetPageSwapBacked(new_page); + err = add_to_swap_cache(new_page, entry, +- gfp_mask & GFP_RECLAIM_MASK); ++ gfp_mask & GFP_RECLAIM_MASK, NULL); + if (likely(!err)) { + /* Initiate read into locked page */ + SetPageWorkingset(new_page); +--- a/mm/swapfile.c ++++ b/mm/swapfile.c +@@ -715,6 +715,7 @@ static void add_to_avail_list(struct swa + static void swap_range_free(struct swap_info_struct *si, unsigned long offset, + unsigned int nr_entries) + { ++ unsigned long begin = offset; + unsigned long end = offset + nr_entries - 1; + void (*swap_slot_free_notify)(struct block_device *, unsigned long); + +@@ -740,6 +741,7 @@ static void swap_range_free(struct swap_ + swap_slot_free_notify(si->bdev, offset); + offset++; + } ++ clear_shadow_from_swap_cache(si->type, begin, end); + } + + static int scan_swap_map_slots(struct swap_info_struct *si, +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -948,7 +948,7 @@ static int __remove_mapping(struct addre + if (PageSwapCache(page)) { + swp_entry_t swap = { .val = page_private(page) }; + mem_cgroup_swapout(page, swap); +- __delete_from_swap_cache(page, swap); ++ __delete_from_swap_cache(page, swap, NULL); + xa_unlock_irqrestore(&mapping->i_pages, flags); + put_swap_page(page, swap); + } else { diff --git a/target/linux/generic/backport-5.4/020-14-BACKPORT-FROMLIST-include-linux-mm.h-do-not-warn-in-.patch b/target/linux/generic/backport-5.4/020-14-BACKPORT-FROMLIST-include-linux-mm.h-do-not-warn-in-.patch new file mode 100644 index 0000000000..88d49ddf1e --- /dev/null +++ b/target/linux/generic/backport-5.4/020-14-BACKPORT-FROMLIST-include-linux-mm.h-do-not-warn-in-.patch @@ -0,0 +1,57 @@ +From 5e63b5c96495b43418b56abc29abf15438fdd2fd Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Tue, 29 Sep 2020 14:54:14 -0600 +Subject: [PATCH] BACKPORT: FROMLIST: include/linux/mm.h: do not warn in + page_memcg_rcu() if !CONFIG_MEMCG + +page_memcg_rcu() warns on !rcu_read_lock_held() regardless of +CONFIG_MEMCG. The following legit code trips the warning when +!CONFIG_MEMCG, since lock_page_memcg() and unlock_page_memcg() are +empty for this config. + + memcg = lock_page_memcg(page1) + (rcu_read_lock() if CONFIG_MEMCG=y) + + do something to page1 + + if (page_memcg_rcu(page2) == memcg) + do something to page2 too as it cannot be migrated away from the + memcg either. + + unlock_page_memcg(page1) + (rcu_read_unlock() if CONFIG_MEMCG=y) + +Locking/unlocking rcu consistently for both configs is rigorous but it +also forces unnecessary locking upon users who have no interest in +CONFIG_MEMCG. + +This patch removes the assertion for !CONFIG_MEMCG, because +page_memcg_rcu() has a few callers and there are no concerns regarding +their correctness at the moment. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432176/) + +BUG=b:123039911 +TEST=Built + +Change-Id: I0d3d8d5cfc69827d2214641d689480cded0557e2 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2951282 +Reviewed-by: Yu Zhao +Commit-Queue: Yu Zhao +Tested-by: Yu Zhao +--- + include/linux/mm.h | 1 - + 1 file changed, 1 deletion(-) + +--- a/include/linux/mm.h ++++ b/include/linux/mm.h +@@ -1327,7 +1327,6 @@ static inline struct mem_cgroup *page_me + } + static inline struct mem_cgroup *page_memcg_rcu(struct page *page) + { +- WARN_ON_ONCE(!rcu_read_lock_held()); + return NULL; + } + #endif diff --git a/target/linux/generic/backport-5.4/020-15-FROMLIST-include-linux-nodemask.h-define-next_memory.patch b/target/linux/generic/backport-5.4/020-15-FROMLIST-include-linux-nodemask.h-define-next_memory.patch new file mode 100644 index 0000000000..0e32ac477e --- /dev/null +++ b/target/linux/generic/backport-5.4/020-15-FROMLIST-include-linux-nodemask.h-define-next_memory.patch @@ -0,0 +1,36 @@ +From ddfe84b424ff3de54e131f78b3581ee60a6954f6 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Mon, 14 Dec 2020 17:51:32 -0700 +Subject: [PATCH] FROMLIST: include/linux/nodemask.h: define next_memory_node() + if !CONFIG_NUMA + +Currently next_memory_node only exists when CONFIG_NUMA=y. This patch +adds the macro for !CONFIG_NUMA. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432177/) + +BUG=b:123039911 +TEST=Built + +Change-Id: Iab6316f6a4cc5a2508b80d1b9a28f2c1504580b1 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2951283 +Reviewed-by: Sean Paul +Reviewed-by: Yu Zhao +Commit-Queue: Yu Zhao +Tested-by: Yu Zhao +--- + include/linux/nodemask.h | 1 + + 1 file changed, 1 insertion(+) + +--- a/include/linux/nodemask.h ++++ b/include/linux/nodemask.h +@@ -484,6 +484,7 @@ static inline int num_node_state(enum no + #define first_online_node 0 + #define first_memory_node 0 + #define next_online_node(nid) (MAX_NUMNODES) ++#define next_memory_node(nid) (MAX_NUMNODES) + #define nr_node_ids 1U + #define nr_online_nodes 1U + diff --git a/target/linux/generic/backport-5.4/020-16-FROMLIST-include-linux-cgroup.h-export-cgroup_mutex.patch b/target/linux/generic/backport-5.4/020-16-FROMLIST-include-linux-cgroup.h-export-cgroup_mutex.patch new file mode 100644 index 0000000000..267771a5d4 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-16-FROMLIST-include-linux-cgroup.h-export-cgroup_mutex.patch @@ -0,0 +1,62 @@ +From d9ded8eaeed518314ef1920a792aaba8ae13d79c Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Tue, 29 Sep 2020 14:14:00 -0600 +Subject: [PATCH] FROMLIST: include/linux/cgroup.h: export cgroup_mutex + +cgroup_mutex is needed to synchronize with memcg creations. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432179/) + +BUG=b:123039911 +TEST=Built + +Change-Id: Idb2aade14d795d4a6b90a65baa3e1747b89ab8cf +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2951284 +Reviewed-by: Sean Paul +Reviewed-by: Yu Zhao +Commit-Queue: Yu Zhao +Tested-by: Yu Zhao +--- + include/linux/cgroup.h | 15 ++++++++++++++- + 1 file changed, 14 insertions(+), 1 deletion(-) + +--- a/include/linux/cgroup.h ++++ b/include/linux/cgroup.h +@@ -426,6 +426,18 @@ static inline void cgroup_put(struct cgr + css_put(&cgrp->self); + } + ++extern struct mutex cgroup_mutex; ++ ++static inline void cgroup_lock(void) ++{ ++ mutex_lock(&cgroup_mutex); ++} ++ ++static inline void cgroup_unlock(void) ++{ ++ mutex_unlock(&cgroup_mutex); ++} ++ + /** + * task_css_set_check - obtain a task's css_set with extra access conditions + * @task: the task to obtain css_set for +@@ -440,7 +452,6 @@ static inline void cgroup_put(struct cgr + * as locks used during the cgroup_subsys::attach() methods. + */ + #ifdef CONFIG_PROVE_RCU +-extern struct mutex cgroup_mutex; + extern spinlock_t css_set_lock; + #define task_css_set_check(task, __c) \ + rcu_dereference_check((task)->cgroups, \ +@@ -703,6 +714,8 @@ struct cgroup; + + static inline void css_get(struct cgroup_subsys_state *css) {} + static inline void css_put(struct cgroup_subsys_state *css) {} ++static inline void cgroup_lock(void) {} ++static inline void cgroup_unlock(void) {} + static inline int cgroup_attach_task_all(struct task_struct *from, + struct task_struct *t) { return 0; } + static inline int cgroupstats_build(struct cgroupstats *stats, diff --git a/target/linux/generic/backport-5.4/020-17-BACKPORT-FROMLIST-mm-x86-support-the-access-bit-on-n.patch b/target/linux/generic/backport-5.4/020-17-BACKPORT-FROMLIST-mm-x86-support-the-access-bit-on-n.patch new file mode 100644 index 0000000000..5a9b566cd7 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-17-BACKPORT-FROMLIST-mm-x86-support-the-access-bit-on-n.patch @@ -0,0 +1,118 @@ +From 9b2a6300b255465ce69c462746da1c401267a5c6 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Sat, 26 Sep 2020 21:17:18 -0600 +Subject: [PATCH] BACKPORT: FROMLIST: mm, x86: support the access bit on + non-leaf PMD entries + +Some architectures support the accessed bit on non-leaf PMD entries +(parents) in addition to leaf PTE entries (children) where pages are +mapped, e.g., x86_64 sets the accessed bit on a parent when using it +as part of linear-address translation [1]. Page table walkers who are +interested in the accessed bit on children can take advantage of this: +they do not need to search the children when the accessed bit is not +set on a parent, given that they have previously cleared the accessed +bit on this parent. + +[1]: Intel 64 and IA-32 Architectures Software Developer's Manual + Volume 3 (October 2019), section 4.8 + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432178/) + +BUG=b:123039911 +TEST=Built + +Change-Id: I1a17be3ae926f721f7b17ea1539e5c39e8c4f9a8 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2951285 +Reviewed-by: Yu Zhao +Commit-Queue: Yu Zhao +Tested-by: Yu Zhao +--- + arch/Kconfig | 9 +++++++++ + arch/x86/Kconfig | 1 + + arch/x86/include/asm/pgtable.h | 2 +- + arch/x86/mm/pgtable.c | 5 ++++- + include/asm-generic/pgtable.h | 4 ++-- + 5 files changed, 17 insertions(+), 4 deletions(-) + +--- a/arch/Kconfig ++++ b/arch/Kconfig +@@ -599,6 +599,15 @@ config HAVE_ARCH_TRANSPARENT_HUGEPAGE + config HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD + bool + ++config HAVE_ARCH_PARENT_PMD_YOUNG ++ bool ++ depends on PGTABLE_LEVELS > 2 ++ help ++ Architectures that select this are able to set the accessed bit on ++ non-leaf PMD entries in addition to leaf PTE entries where pages are ++ mapped. For them, page table walkers that clear the accessed bit may ++ stop at non-leaf PMD entries if they do not see the accessed bit. ++ + config HAVE_ARCH_HUGE_VMAP + bool + +--- a/arch/x86/Kconfig ++++ b/arch/x86/Kconfig +@@ -145,6 +145,7 @@ config X86 + select HAVE_ARCH_TRACEHOOK + select HAVE_ARCH_TRANSPARENT_HUGEPAGE + select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD if X86_64 ++ select HAVE_ARCH_PARENT_PMD_YOUNG if X86_64 + select HAVE_ARCH_VMAP_STACK if X86_64 + select HAVE_ARCH_WITHIN_STACK_FRAMES + select HAVE_ASM_MODVERSIONS +--- a/arch/x86/include/asm/pgtable.h ++++ b/arch/x86/include/asm/pgtable.h +@@ -842,7 +842,7 @@ static inline pte_t *pte_offset_kernel(p + + static inline int pmd_bad(pmd_t pmd) + { +- return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; ++ return ((pmd_flags(pmd) | _PAGE_ACCESSED) & ~_PAGE_USER) != _KERNPG_TABLE; + } + + static inline unsigned long pages_to_mb(unsigned long npg) +--- a/arch/x86/mm/pgtable.c ++++ b/arch/x86/mm/pgtable.c +@@ -543,7 +543,7 @@ int ptep_test_and_clear_young(struct vm_ + return ret; + } + +-#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG) + int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) + { +@@ -555,6 +555,9 @@ int pmdp_test_and_clear_young(struct vm_ + + return ret; + } ++#endif ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE + int pudp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pud_t *pudp) + { +--- a/include/asm-generic/pgtable.h ++++ b/include/asm-generic/pgtable.h +@@ -74,7 +74,7 @@ static inline int ptep_test_and_clear_yo + #endif + + #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +-#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG) + static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +@@ -95,7 +95,7 @@ static inline int pmdp_test_and_clear_yo + BUILD_BUG(); + return 0; + } +-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ ++#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG */ + #endif + + #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH diff --git a/target/linux/generic/backport-5.4/020-18-FROMLIST-mm-workingset.c-refactor-pack_shadow-and-un.patch b/target/linux/generic/backport-5.4/020-18-FROMLIST-mm-workingset.c-refactor-pack_shadow-and-un.patch new file mode 100644 index 0000000000..6a2c85b77e --- /dev/null +++ b/target/linux/generic/backport-5.4/020-18-FROMLIST-mm-workingset.c-refactor-pack_shadow-and-un.patch @@ -0,0 +1,145 @@ +From 42d051841615dd1e9d6806616f6a7ed4a78ba1c5 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Wed, 5 May 2021 12:21:41 -0600 +Subject: [PATCH] FROMLIST: mm/workingset.c: refactor pack_shadow() and + unpack_shadow() + +This patches moves the bucket order and PageWorkingset() out of +pack_shadow() and unpack_shadow(). It has no merits on its own but +makes the upcoming changes to mm/workingset.c less diffy. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432181/) + +BUG=b:123039911 +TEST=Built + +Change-Id: I474378b573d2e1614a9798dca0949d1d6fba3e25 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2951286 +Reviewed-by: Yu Zhao +Commit-Queue: Yu Zhao +Tested-by: Yu Zhao +--- + mm/workingset.c | 55 +++++++++++++++++++++---------------------------- + 1 file changed, 23 insertions(+), 32 deletions(-) + +--- a/mm/workingset.c ++++ b/mm/workingset.c +@@ -167,9 +167,9 @@ + * refault distance will immediately activate the refaulting page. + */ + +-#define EVICTION_SHIFT ((BITS_PER_LONG - BITS_PER_XA_VALUE) + \ +- 1 + NODES_SHIFT + MEM_CGROUP_ID_SHIFT) +-#define EVICTION_MASK (~0UL >> EVICTION_SHIFT) ++#define EVICTION_SHIFT (BITS_PER_XA_VALUE - MEM_CGROUP_ID_SHIFT - NODES_SHIFT) ++#define EVICTION_MASK (BIT(EVICTION_SHIFT) - 1) ++#define WORKINGSET_WIDTH 1 + + /* + * Eviction timestamps need to be able to cover the full range of +@@ -181,36 +181,23 @@ + */ + static unsigned int bucket_order __read_mostly; + +-static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, +- bool workingset) ++static void *pack_shadow(int memcg_id, struct pglist_data *pgdat, unsigned long val) + { +- eviction >>= bucket_order; +- eviction &= EVICTION_MASK; +- eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; +- eviction = (eviction << NODES_SHIFT) | pgdat->node_id; +- eviction = (eviction << 1) | workingset; ++ val = (val << MEM_CGROUP_ID_SHIFT) | memcg_id; ++ val = (val << NODES_SHIFT) | pgdat->node_id; + +- return xa_mk_value(eviction); ++ return xa_mk_value(val); + } + +-static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, +- unsigned long *evictionp, bool *workingsetp) ++static unsigned long unpack_shadow(void *shadow, int *memcg_id, struct pglist_data **pgdat) + { +- unsigned long entry = xa_to_value(shadow); +- int memcgid, nid; +- bool workingset; ++ unsigned long val = xa_to_value(shadow); + +- workingset = entry & 1; +- entry >>= 1; +- nid = entry & ((1UL << NODES_SHIFT) - 1); +- entry >>= NODES_SHIFT; +- memcgid = entry & ((1UL << MEM_CGROUP_ID_SHIFT) - 1); +- entry >>= MEM_CGROUP_ID_SHIFT; +- +- *memcgidp = memcgid; +- *pgdat = NODE_DATA(nid); +- *evictionp = entry << bucket_order; +- *workingsetp = workingset; ++ *pgdat = NODE_DATA(val & (BIT(NODES_SHIFT) - 1)); ++ val >>= NODES_SHIFT; ++ *memcg_id = val & (BIT(MEM_CGROUP_ID_SHIFT) - 1); ++ ++ return val >> MEM_CGROUP_ID_SHIFT; + } + + /** +@@ -235,7 +222,9 @@ void *workingset_eviction(struct page *p + + lruvec = mem_cgroup_lruvec(pgdat, memcg); + eviction = atomic_long_inc_return(&lruvec->inactive_age); +- return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page)); ++ eviction >>= bucket_order; ++ eviction = (eviction << WORKINGSET_WIDTH) | PageWorkingset(page); ++ return pack_shadow(memcgid, pgdat, eviction); + } + + /** +@@ -258,7 +247,7 @@ void workingset_refault(struct page *pag + bool workingset; + int memcgid; + +- unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset); ++ eviction = unpack_shadow(shadow, &memcgid, &pgdat); + + rcu_read_lock(); + /* +@@ -282,6 +271,8 @@ void workingset_refault(struct page *pag + goto out; + lruvec = mem_cgroup_lruvec(pgdat, memcg); + refault = atomic_long_read(&lruvec->inactive_age); ++ workingset = eviction & (BIT(WORKINGSET_WIDTH) - 1); ++ eviction = (eviction >> WORKINGSET_WIDTH) << bucket_order; + active_file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES); + + /* +@@ -300,7 +291,7 @@ void workingset_refault(struct page *pag + * longest time, so the occasional inappropriate activation + * leading to pressure on the active list is not a problem. + */ +- refault_distance = (refault - eviction) & EVICTION_MASK; ++ refault_distance = (refault - eviction) & (EVICTION_MASK >> WORKINGSET_WIDTH); + + inc_lruvec_state(lruvec, WORKINGSET_REFAULT); + +@@ -539,7 +530,7 @@ static int __init workingset_init(void) + unsigned int max_order; + int ret; + +- BUILD_BUG_ON(BITS_PER_LONG < EVICTION_SHIFT); ++ BUILD_BUG_ON(EVICTION_SHIFT < WORKINGSET_WIDTH); + /* + * Calculate the eviction bucket size to cover the longest + * actionable refault distance, which is currently half of +@@ -547,7 +538,7 @@ static int __init workingset_init(void) + * some more pages at runtime, so keep working with up to + * double the initial memory by using totalram_pages as-is. + */ +- timestamp_bits = BITS_PER_LONG - EVICTION_SHIFT; ++ timestamp_bits = EVICTION_SHIFT - WORKINGSET_WIDTH; + max_order = fls_long(totalram_pages() - 1); + if (max_order > timestamp_bits) + bucket_order = max_order - timestamp_bits; diff --git a/target/linux/generic/backport-5.4/020-19-BACKPORT-FROMLIST-mm-multigenerational-lru-groundwor.patch b/target/linux/generic/backport-5.4/020-19-BACKPORT-FROMLIST-mm-multigenerational-lru-groundwor.patch new file mode 100644 index 0000000000..9ea5f8456f --- /dev/null +++ b/target/linux/generic/backport-5.4/020-19-BACKPORT-FROMLIST-mm-multigenerational-lru-groundwor.patch @@ -0,0 +1,913 @@ +From 852d55075c4ec01270f87b9d6978fdada30b5fa7 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Mon, 25 Jan 2021 21:12:33 -0700 +Subject: [PATCH] BACKPORT: FROMLIST: mm: multigenerational lru: + groundwork + +For each lruvec, evictable pages are divided into multiple +generations. The youngest generation number is stored in +lrugen->max_seq for both anon and file types as they are aged on an +equal footing. The oldest generation numbers are stored in +lrugen->min_seq[2] separately for anon and file types as clean file +pages can be evicted regardless of may_swap or may_writepage. These +three variables are monotonically increasing. Generation numbers are +truncated into order_base_2(MAX_NR_GENS+1) bits in order to fit into +page->flags. The sliding window technique is used to prevent truncated +generation numbers from overlapping. Each truncated generation number +is an index to +lrugen->lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]. Evictable +pages are added to the per-zone lists indexed by lrugen->max_seq or +lrugen->min_seq[2] (modulo MAX_NR_GENS), depending on their types. + +Each generation is then divided into multiple tiers. Tiers represent +levels of usage from file descriptors only. Pages accessed N times via +file descriptors belong to tier order_base_2(N). Each generation +contains at most MAX_NR_TIERS tiers, and they require additional +MAX_NR_TIERS-2 bits in page->flags. In contrast to moving across +generations which requires the lru lock for the list operations, +moving across tiers only involves an atomic operation on page->flags +and therefore has a negligible cost. A feedback loop modeled after the +PID controller monitors the refault rates across all tiers and decides +when to activate pages from which tiers in the reclaim path. + +The framework comprises two conceptually independent components: the +aging and the eviction, which can be invoked separately from user +space for the purpose of working set estimation and proactive reclaim. + +The aging produces young generations. Given an lruvec, the aging scans +page tables for referenced pages of this lruvec. Upon finding one, the +aging updates its generation number to max_seq. After each round of +scan, the aging increments max_seq. The aging is due when both of +min_seq[2] reaches max_seq-1, assuming both anon and file types are +reclaimable. + +The eviction consumes old generations. Given an lruvec, the eviction +scans the pages on the per-zone lists indexed by either of min_seq[2]. +It tries to select a type based on the values of min_seq[2] and +swappiness. During a scan, the eviction sorts pages according to their +new generation numbers, if the aging has found them referenced. When +it finds all the per-zone lists of a selected type are empty, the +eviction increments min_seq[2] indexed by this selected type. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432182/) + +BUG=b:123039911 +TEST=Built + +Change-Id: I71de7cd15b8dfa6f9fdd838023474693c4fee0a7 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2987186 +Reviewed-by: Yu Zhao +Tested-by: Yu Zhao +Commit-Queue: Sonny Rao +Commit-Queue: Yu Zhao +--- + fs/fuse/dev.c | 3 +- + include/linux/mm.h | 2 + + include/linux/mm_inline.h | 194 +++++++++++++++++++ + include/linux/mmzone.h | 106 +++++++++++ + include/linux/page-flags-layout.h | 15 +- + include/linux/page-flags.h | 4 +- + kernel/bounds.c | 6 + + mm/huge_memory.c | 3 +- + mm/mm_init.c | 8 +- + mm/mmzone.c | 2 + + mm/swapfile.c | 4 + + mm/vmscan.c | 307 ++++++++++++++++++++++++++++++ + 12 files changed, 645 insertions(+), 9 deletions(-) + +--- a/fs/fuse/dev.c ++++ b/fs/fuse/dev.c +@@ -772,7 +772,8 @@ static int fuse_check_page(struct page * + 1 << PG_active | + 1 << PG_workingset | + 1 << PG_reclaim | +- 1 << PG_waiters))) { ++ 1 << PG_waiters | ++ LRU_GEN_MASK | LRU_USAGE_MASK))) { + pr_warn("trying to steal weird page\n"); + pr_warn(" page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping); + return 1; +--- a/include/linux/mm.h ++++ b/include/linux/mm.h +@@ -924,6 +924,8 @@ vm_fault_t finish_mkwrite_fault(struct v + #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) + #define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) + #define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) ++#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH) ++#define LRU_USAGE_PGOFF (LRU_GEN_PGOFF - LRU_USAGE_WIDTH) + + /* + * Define the bit shifts to access each section. For non-existent +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -78,11 +78,199 @@ static __always_inline enum lru_list pag + return lru; + } + ++#ifdef CONFIG_LRU_GEN ++ ++#ifdef CONFIG_LRU_GEN_ENABLED ++DECLARE_STATIC_KEY_TRUE(lru_gen_static_key); ++ ++static inline bool lru_gen_enabled(void) ++{ ++ return static_branch_likely(&lru_gen_static_key); ++} ++#else ++DECLARE_STATIC_KEY_FALSE(lru_gen_static_key); ++ ++static inline bool lru_gen_enabled(void) ++{ ++ return static_branch_unlikely(&lru_gen_static_key); ++} ++#endif ++ ++/* We track at most MAX_NR_GENS generations using the sliding window technique. */ ++static inline int lru_gen_from_seq(unsigned long seq) ++{ ++ return seq % MAX_NR_GENS; ++} ++ ++/* Return a proper index regardless whether we keep a full history of stats. */ ++static inline int hist_from_seq_or_gen(int seq_or_gen) ++{ ++ return seq_or_gen % NR_STAT_GENS; ++} ++ ++/* The youngest and the second youngest generations are counted as active. */ ++static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) ++{ ++ unsigned long max_seq = READ_ONCE(lruvec->evictable.max_seq); ++ ++ VM_BUG_ON(!max_seq); ++ VM_BUG_ON(gen >= MAX_NR_GENS); ++ ++ return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); ++} ++ ++/* Update the sizes of the multigenerational lru lists. */ ++static inline void lru_gen_update_size(struct page *page, struct lruvec *lruvec, ++ int old_gen, int new_gen) ++{ ++ int type = page_is_file_cache(page); ++ int zone = page_zonenum(page); ++ int delta = hpage_nr_pages(page); ++ enum lru_list lru = type * LRU_FILE; ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock); ++ VM_BUG_ON(old_gen != -1 && old_gen >= MAX_NR_GENS); ++ VM_BUG_ON(new_gen != -1 && new_gen >= MAX_NR_GENS); ++ VM_BUG_ON(old_gen == -1 && new_gen == -1); ++ ++ if (old_gen >= 0) ++ WRITE_ONCE(lrugen->sizes[old_gen][type][zone], ++ lrugen->sizes[old_gen][type][zone] - delta); ++ if (new_gen >= 0) ++ WRITE_ONCE(lrugen->sizes[new_gen][type][zone], ++ lrugen->sizes[new_gen][type][zone] + delta); ++ ++ if (old_gen < 0) { ++ if (lru_gen_is_active(lruvec, new_gen)) ++ lru += LRU_ACTIVE; ++ update_lru_size(lruvec, lru, zone, delta); ++ return; ++ } ++ ++ if (new_gen < 0) { ++ if (lru_gen_is_active(lruvec, old_gen)) ++ lru += LRU_ACTIVE; ++ update_lru_size(lruvec, lru, zone, -delta); ++ return; ++ } ++ ++ if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { ++ update_lru_size(lruvec, lru, zone, -delta); ++ update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta); ++ } ++ ++ VM_BUG_ON(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); ++} ++ ++/* Add a page to one of the multigenerational lru lists. Return true on success. */ ++static inline bool lru_gen_addition(struct page *page, struct lruvec *lruvec, bool front) ++{ ++ int gen; ++ unsigned long old_flags, new_flags; ++ int type = page_is_file_cache(page); ++ int zone = page_zonenum(page); ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ if (PageUnevictable(page) || !lrugen->enabled[type]) ++ return false; ++ /* ++ * If a page is being faulted in, add it to the youngest generation. ++ * try_walk_mm_list() may look at the size of the youngest generation to ++ * determine if the aging is due. ++ * ++ * If a page can't be evicted immediately, i.e., an anon page not in ++ * swap cache, a dirty file page under reclaim, or a page rejected by ++ * evict_pages() due to races, dirty buffer heads, etc., add it to the ++ * second oldest generation. ++ * ++ * If a page could be evicted immediately, i.e., a clean file page, add ++ * it to the oldest generation. ++ */ ++ if (PageActive(page)) ++ gen = lru_gen_from_seq(lrugen->max_seq); ++ else if ((!type && !PageSwapCache(page)) || ++ (PageReclaim(page) && (PageDirty(page) || PageWriteback(page))) || ++ (!PageReferenced(page) && PageWorkingset(page))) ++ gen = lru_gen_from_seq(lrugen->min_seq[type] + 1); ++ else ++ gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ ++ do { ++ old_flags = READ_ONCE(page->flags); ++ VM_BUG_ON_PAGE(old_flags & LRU_GEN_MASK, page); ++ ++ new_flags = (old_flags & ~(LRU_GEN_MASK | BIT(PG_active))) | ++ ((gen + 1UL) << LRU_GEN_PGOFF); ++ /* see the comment in evict_pages() */ ++ if (!(old_flags & BIT(PG_referenced))) ++ new_flags &= ~(LRU_USAGE_MASK | LRU_TIER_FLAGS); ++ } while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags); ++ ++ lru_gen_update_size(page, lruvec, -1, gen); ++ if (front) ++ list_add(&page->lru, &lrugen->lists[gen][type][zone]); ++ else ++ list_add_tail(&page->lru, &lrugen->lists[gen][type][zone]); ++ ++ return true; ++} ++ ++/* Delete a page from one of the multigenerational lru lists. Return true on success. */ ++static inline bool lru_gen_deletion(struct page *page, struct lruvec *lruvec) ++{ ++ int gen; ++ unsigned long old_flags, new_flags; ++ ++ do { ++ old_flags = READ_ONCE(page->flags); ++ if (!(old_flags & LRU_GEN_MASK)) ++ return false; ++ ++ VM_BUG_ON_PAGE(PageActive(page), page); ++ VM_BUG_ON_PAGE(PageUnevictable(page), page); ++ ++ gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++ ++ new_flags = old_flags & ~LRU_GEN_MASK; ++ /* mark page active accordingly */ ++ if (lru_gen_is_active(lruvec, gen)) ++ new_flags |= BIT(PG_active); ++ } while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags); ++ ++ lru_gen_update_size(page, lruvec, gen, -1); ++ list_del(&page->lru); ++ ++ return true; ++} ++ ++#else /* CONFIG_LRU_GEN */ ++ ++static inline bool lru_gen_enabled(void) ++{ ++ return false; ++} ++ ++static inline bool lru_gen_addition(struct page *page, struct lruvec *lruvec, bool front) ++{ ++ return false; ++} ++ ++static inline bool lru_gen_deletion(struct page *page, struct lruvec *lruvec) ++{ ++ return false; ++} ++ ++#endif /* CONFIG_LRU_GEN */ ++ + static __always_inline void add_page_to_lru_list(struct page *page, + struct lruvec *lruvec) + { + enum lru_list lru = page_lru(page); + ++ if (lru_gen_addition(page, lruvec, true)) ++ return; ++ + update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page)); + list_add(&page->lru, &lruvec->lists[lru]); + } +@@ -92,6 +280,9 @@ static __always_inline void add_page_to_ + { + enum lru_list lru = page_lru(page); + ++ if (lru_gen_addition(page, lruvec, false)) ++ return; ++ + update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page)); + list_add_tail(&page->lru, &lruvec->lists[lru]); + } +@@ -99,6 +290,9 @@ static __always_inline void add_page_to_ + static __always_inline void del_page_from_lru_list(struct page *page, + struct lruvec *lruvec) + { ++ if (lru_gen_deletion(page, lruvec)) ++ return; ++ + list_del(&page->lru); + update_lru_size(lruvec, page_lru(page), page_zonenum(page), + -hpage_nr_pages(page)); +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -295,6 +295,108 @@ struct zone_reclaim_stat { + unsigned long recent_scanned[ANON_AND_FILE]; + }; + ++struct lruvec; ++ ++#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) ++#define LRU_USAGE_MASK ((BIT(LRU_USAGE_WIDTH) - 1) << LRU_USAGE_PGOFF) ++ ++#ifdef CONFIG_LRU_GEN ++ ++/* ++ * For each lruvec, evictable pages are divided into multiple generations. The ++ * youngest and the oldest generation numbers, AKA max_seq and min_seq, are ++ * monotonically increasing. The sliding window technique is used to track at ++ * most MAX_NR_GENS and at least MIN_NR_GENS generations. An offset within the ++ * window, AKA gen, indexes an array of per-type and per-zone lists for the ++ * corresponding generation. The counter in page->flags stores gen+1 while a ++ * page is on one of the multigenerational lru lists. Otherwise, it stores 0. ++ */ ++#define MAX_NR_GENS ((unsigned int)CONFIG_NR_LRU_GENS) ++ ++/* ++ * Each generation is then divided into multiple tiers. Tiers represent levels ++ * of usage from file descriptors, i.e., mark_page_accessed(). In contrast to ++ * moving across generations which requires the lru lock, moving across tiers ++ * only involves an atomic operation on page->flags and therefore has a ++ * negligible cost. ++ * ++ * The purposes of tiers are to: ++ * 1) estimate whether pages accessed multiple times via file descriptors are ++ * more active than pages accessed only via page tables by separating the two ++ * access types into upper tiers and the base tier and comparing refault rates ++ * across tiers. ++ * 2) improve buffered io performance by deferring activations of pages ++ * accessed multiple times until the eviction. That is activations happen in ++ * the reclaim path, not the access path. ++ * ++ * Pages accessed N times via file descriptors belong to tier order_base_2(N). ++ * The base tier uses the following page flag: ++ * !PageReferenced() -- readahead pages ++ * PageReferenced() -- single-access pages ++ * All upper tiers use the following page flags: ++ * PageReferenced() && PageWorkingset() -- multi-access pages ++ * in addition to the bits storing N-2 accesses. Therefore, we can support one ++ * upper tier without using additional bits in page->flags. ++ * ++ * Note that ++ * 1) PageWorkingset() is always set for upper tiers because we want to ++ * maintain the existing psi behavior. ++ * 2) !PageReferenced() && PageWorkingset() is not a valid tier. See the ++ * comment in evict_pages(). ++ * ++ * Pages from the base tier are evicted regardless of its refault rate. Pages ++ * from upper tiers will be moved to the next generation, if their refault rates ++ * are higher than that of the base tier. ++ */ ++#define MAX_NR_TIERS ((unsigned int)CONFIG_TIERS_PER_GEN) ++#define LRU_TIER_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) ++#define LRU_USAGE_SHIFT (CONFIG_TIERS_PER_GEN - 1) ++ ++/* Whether to keep historical stats for each generation. */ ++#ifdef CONFIG_LRU_GEN_STATS ++#define NR_STAT_GENS ((unsigned int)CONFIG_NR_LRU_GENS) ++#else ++#define NR_STAT_GENS 1U ++#endif ++ ++struct lrugen { ++ /* the aging increments the max generation number */ ++ unsigned long max_seq; ++ /* the eviction increments the min generation numbers */ ++ unsigned long min_seq[ANON_AND_FILE]; ++ /* the birth time of each generation in jiffies */ ++ unsigned long timestamps[MAX_NR_GENS]; ++ /* the multigenerational lru lists */ ++ struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; ++ /* the sizes of the multigenerational lru lists in pages */ ++ unsigned long sizes[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; ++ /* to determine which type and its tiers to evict */ ++ atomic_long_t evicted[NR_STAT_GENS][ANON_AND_FILE][MAX_NR_TIERS]; ++ atomic_long_t refaulted[NR_STAT_GENS][ANON_AND_FILE][MAX_NR_TIERS]; ++ /* the base tier won't be activated */ ++ unsigned long activated[NR_STAT_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1]; ++ /* arithmetic mean weighted by geometric series 1/2, 1/4, ... */ ++ unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; ++ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; ++ /* whether the multigenerational lru is enabled */ ++ bool enabled[ANON_AND_FILE]; ++}; ++ ++void lru_gen_init_lruvec(struct lruvec *lruvec); ++void lru_gen_set_state(bool enable, bool main, bool swap); ++ ++#else /* CONFIG_LRU_GEN */ ++ ++static inline void lru_gen_init_lruvec(struct lruvec *lruvec) ++{ ++} ++ ++static inline void lru_gen_set_state(bool enable, bool main, bool swap) ++{ ++} ++ ++#endif /* CONFIG_LRU_GEN */ ++ + struct lruvec { + struct list_head lists[NR_LRU_LISTS]; + struct zone_reclaim_stat reclaim_stat; +@@ -302,6 +404,10 @@ struct lruvec { + atomic_long_t inactive_age; + /* Refaults at the time of last reclaim cycle */ + unsigned long refaults; ++#ifdef CONFIG_LRU_GEN ++ /* unevictable pages are on LRU_UNEVICTABLE */ ++ struct lrugen evictable; ++#endif + #ifdef CONFIG_MEMCG + struct pglist_data *pgdat; + #endif +--- a/include/linux/page-flags-layout.h ++++ b/include/linux/page-flags-layout.h +@@ -24,6 +24,14 @@ + #error ZONES_SHIFT -- too many zones configured adjust calculation + #endif + ++#ifdef CONFIG_LRU_GEN ++/* LRU_GEN_WIDTH is generated from order_base_2(CONFIG_NR_LRU_GENS + 1). */ ++#define LRU_USAGE_WIDTH (CONFIG_TIERS_PER_GEN - 2) ++#else ++#define LRU_GEN_WIDTH 0 ++#define LRU_USAGE_WIDTH 0 ++#endif ++ + #ifdef CONFIG_SPARSEMEM + #include + +@@ -56,7 +64,8 @@ + + #define ZONES_WIDTH ZONES_SHIFT + +-#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS ++#if SECTIONS_WIDTH+ZONES_WIDTH+LRU_GEN_WIDTH+LRU_USAGE_WIDTH+NODES_SHIFT \ ++ <= BITS_PER_LONG - NR_PAGEFLAGS + #define NODES_WIDTH NODES_SHIFT + #else + #ifdef CONFIG_SPARSEMEM_VMEMMAP +@@ -83,14 +92,14 @@ + #define KASAN_TAG_WIDTH 0 + #endif + +-#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_CPUPID_SHIFT+KASAN_TAG_WIDTH \ ++#if SECTIONS_WIDTH+ZONES_WIDTH+LRU_GEN_WIDTH+LRU_USAGE_WIDTH+NODES_WIDTH+LAST_CPUPID_SHIFT+KASAN_TAG_WIDTH \ + <= BITS_PER_LONG - NR_PAGEFLAGS + #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT + #else + #define LAST_CPUPID_WIDTH 0 + #endif + +-#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH+LAST_CPUPID_WIDTH+KASAN_TAG_WIDTH \ ++#if SECTIONS_WIDTH+ZONES_WIDTH+LRU_GEN_WIDTH+LRU_USAGE_WIDTH+NODES_WIDTH+LAST_CPUPID_WIDTH+KASAN_TAG_WIDTH \ + > BITS_PER_LONG - NR_PAGEFLAGS + #error "Not enough bits in page flags" + #endif +--- a/include/linux/page-flags.h ++++ b/include/linux/page-flags.h +@@ -827,7 +827,7 @@ static inline void ClearPageSlabPfmemall + 1UL << PG_private | 1UL << PG_private_2 | \ + 1UL << PG_writeback | 1UL << PG_reserved | \ + 1UL << PG_slab | 1UL << PG_active | \ +- 1UL << PG_unevictable | __PG_MLOCKED) ++ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) + + /* + * Flags checked when a page is prepped for return by the page allocator. +@@ -838,7 +838,7 @@ static inline void ClearPageSlabPfmemall + * alloc-free cycle to prevent from reusing the page. + */ + #define PAGE_FLAGS_CHECK_AT_PREP \ +- (((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON) ++ ((((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_USAGE_MASK) + + #define PAGE_FLAGS_PRIVATE \ + (1UL << PG_private | 1UL << PG_private_2) +--- a/kernel/bounds.c ++++ b/kernel/bounds.c +@@ -22,6 +22,12 @@ int main(void) + DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS)); + #endif + DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); ++#ifdef CONFIG_LRU_GEN ++ /* bits needed to represent internal values stored in page->flags */ ++ DEFINE(LRU_GEN_WIDTH, order_base_2(CONFIG_NR_LRU_GENS + 1)); ++ /* bits needed to represent normalized values for external uses */ ++ DEFINE(LRU_GEN_SHIFT, order_base_2(CONFIG_NR_LRU_GENS)); ++#endif + /* End of constants */ + + return 0; +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -2508,7 +2508,8 @@ static void __split_huge_page_tail(struc + (1L << PG_workingset) | + (1L << PG_locked) | + (1L << PG_unevictable) | +- (1L << PG_dirty))); ++ (1L << PG_dirty) | ++ LRU_GEN_MASK | LRU_USAGE_MASK)); + + /* ->mapping in first tail page is compound_mapcount */ + VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, +--- a/mm/mm_init.c ++++ b/mm/mm_init.c +@@ -67,13 +67,17 @@ void __init mminit_verify_pageflags_layo + unsigned long or_mask, add_mask; + + shift = 8 * sizeof(unsigned long); +- width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH - LAST_CPUPID_SHIFT; ++ width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH - LAST_CPUPID_SHIFT ++ - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_USAGE_WIDTH; + mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", +- "Section %d Node %d Zone %d Lastcpupid %d Flags %d\n", ++ "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", + SECTIONS_WIDTH, + NODES_WIDTH, + ZONES_WIDTH, + LAST_CPUPID_WIDTH, ++ KASAN_TAG_WIDTH, ++ LRU_GEN_WIDTH, ++ LRU_USAGE_WIDTH, + NR_PAGEFLAGS); + mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", + "Section %d Node %d Zone %d Lastcpupid %d\n", +--- a/mm/mmzone.c ++++ b/mm/mmzone.c +@@ -80,6 +80,8 @@ void lruvec_init(struct lruvec *lruvec) + + for_each_lru(lru) + INIT_LIST_HEAD(&lruvec->lists[lru]); ++ ++ lru_gen_init_lruvec(lruvec); + } + + #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) +--- a/mm/swapfile.c ++++ b/mm/swapfile.c +@@ -2697,6 +2697,8 @@ SYSCALL_DEFINE1(swapoff, const char __us + err = 0; + atomic_inc(&proc_poll_event); + wake_up_interruptible(&proc_poll_wait); ++ /* stop tracking anon if the multigenerational lru is turned off */ ++ lru_gen_set_state(false, false, true); + + out_dput: + filp_close(victim, NULL); +@@ -3323,6 +3325,8 @@ SYSCALL_DEFINE2(swapon, const char __use + mutex_unlock(&swapon_mutex); + atomic_inc(&proc_poll_event); + wake_up_interruptible(&proc_poll_wait); ++ /* start tracking anon if the multigenerational lru is turned on */ ++ lru_gen_set_state(true, false, true); + + error = 0; + goto out; +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -51,6 +51,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -2557,6 +2558,312 @@ out: + } + } + ++#ifdef CONFIG_LRU_GEN ++ ++/* ++ * After pages are faulted in, the aging must scan them twice before the ++ * eviction can consider them. The first scan clears the accessed bit set during ++ * initial faults. And the second scan makes sure they haven't been used since ++ * the first scan. ++ */ ++#define MIN_NR_GENS 2 ++ ++#define MAX_BATCH_SIZE 8192 ++ ++/****************************************************************************** ++ * shorthand helpers ++ ******************************************************************************/ ++ ++#define DEFINE_MAX_SEQ() \ ++ unsigned long max_seq = READ_ONCE(lruvec->evictable.max_seq) ++ ++#define DEFINE_MIN_SEQ() \ ++ unsigned long min_seq[ANON_AND_FILE] = { \ ++ READ_ONCE(lruvec->evictable.min_seq[0]), \ ++ READ_ONCE(lruvec->evictable.min_seq[1]), \ ++ } ++ ++#define for_each_type_zone(type, zone) \ ++ for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ ++ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) ++ ++#define for_each_gen_type_zone(gen, type, zone) \ ++ for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ ++ for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ ++ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) ++ ++static int page_lru_gen(struct page *page) ++{ ++ return ((page->flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++} ++ ++static int get_lo_wmark(unsigned long max_seq, unsigned long *min_seq, int swappiness) ++{ ++ return max_seq - max(min_seq[!swappiness], min_seq[1]) + 1; ++} ++ ++static int get_hi_wmark(unsigned long max_seq, unsigned long *min_seq, int swappiness) ++{ ++ return max_seq - min(min_seq[!swappiness], min_seq[1]) + 1; ++} ++ ++static int get_nr_gens(struct lruvec *lruvec, int type) ++{ ++ return lruvec->evictable.max_seq - lruvec->evictable.min_seq[type] + 1; ++} ++ ++static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) ++{ ++ lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock); ++ ++ return get_nr_gens(lruvec, 0) >= MIN_NR_GENS && ++ get_nr_gens(lruvec, 0) <= MAX_NR_GENS && ++ get_nr_gens(lruvec, 1) >= MIN_NR_GENS && ++ get_nr_gens(lruvec, 1) <= MAX_NR_GENS; ++} ++ ++/****************************************************************************** ++ * state change ++ ******************************************************************************/ ++ ++#ifdef CONFIG_LRU_GEN_ENABLED ++DEFINE_STATIC_KEY_TRUE(lru_gen_static_key); ++#else ++DEFINE_STATIC_KEY_FALSE(lru_gen_static_key); ++#endif ++ ++static DEFINE_MUTEX(lru_gen_state_mutex); ++static int lru_gen_nr_swapfiles __read_mostly; ++ ++static bool __maybe_unused state_is_valid(struct lruvec *lruvec) ++{ ++ int gen, type, zone; ++ enum lru_list lru; ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ for_each_evictable_lru(lru) { ++ type = is_file_lru(lru); ++ ++ if (lrugen->enabled[type] && !list_empty(&lruvec->lists[lru])) ++ return false; ++ } ++ ++ for_each_gen_type_zone(gen, type, zone) { ++ if (!lrugen->enabled[type] && !list_empty(&lrugen->lists[gen][type][zone])) ++ return false; ++ ++ VM_WARN_ON_ONCE(!lrugen->enabled[type] && lrugen->sizes[gen][type][zone]); ++ } ++ ++ return true; ++} ++ ++static bool fill_lru_gen_lists(struct lruvec *lruvec) ++{ ++ enum lru_list lru; ++ int batch_size = 0; ++ ++ for_each_evictable_lru(lru) { ++ int type = is_file_lru(lru); ++ bool active = is_active_lru(lru); ++ struct list_head *head = &lruvec->lists[lru]; ++ ++ if (!lruvec->evictable.enabled[type]) ++ continue; ++ ++ while (!list_empty(head)) { ++ bool success; ++ struct page *page = lru_to_page(head); ++ ++ VM_BUG_ON_PAGE(PageTail(page), page); ++ VM_BUG_ON_PAGE(PageUnevictable(page), page); ++ VM_BUG_ON_PAGE(PageActive(page) != active, page); ++ VM_BUG_ON_PAGE(page_lru_gen(page) != -1, page); ++ VM_BUG_ON_PAGE(page_is_file_cache(page) != type, page); ++ ++ prefetchw_prev_lru_page(page, head, flags); ++ ++ del_page_from_lru_list(page, lruvec); ++ success = lru_gen_addition(page, lruvec, true); ++ VM_BUG_ON(!success); ++ ++ if (++batch_size == MAX_BATCH_SIZE) ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++static bool drain_lru_gen_lists(struct lruvec *lruvec) ++{ ++ int gen, type, zone; ++ int batch_size = 0; ++ ++ for_each_gen_type_zone(gen, type, zone) { ++ struct list_head *head = &lruvec->evictable.lists[gen][type][zone]; ++ ++ if (lruvec->evictable.enabled[type]) ++ continue; ++ ++ while (!list_empty(head)) { ++ bool success; ++ struct page *page = lru_to_page(head); ++ ++ VM_BUG_ON_PAGE(PageTail(page), page); ++ VM_BUG_ON_PAGE(PageUnevictable(page), page); ++ VM_BUG_ON_PAGE(PageActive(page), page); ++ VM_BUG_ON_PAGE(page_is_file_cache(page) != type, page); ++ VM_BUG_ON_PAGE(page_zonenum(page) != zone, page); ++ ++ prefetchw_prev_lru_page(page, head, flags); ++ ++ success = lru_gen_deletion(page, lruvec); ++ VM_BUG_ON(!success); ++ add_page_to_lru_list(page, lruvec); ++ ++ if (++batch_size == MAX_BATCH_SIZE) ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++/* ++ * For file page tracking, we enable/disable it according to the main switch. ++ * For anon page tracking, we only enabled it when the main switch is on and ++ * there is at least one swapfile; we disable it when there are no swapfiles ++ * regardless of the value of the main switch. Otherwise, we will eventually ++ * reach the max size of the sliding window and have to call inc_min_seq(), ++ * which brings an unnecessary overhead. ++ */ ++void lru_gen_set_state(bool enable, bool main, bool swap) ++{ ++ struct mem_cgroup *memcg; ++ ++ mem_hotplug_begin(); ++ mutex_lock(&lru_gen_state_mutex); ++ cgroup_lock(); ++ ++ main = main && enable != lru_gen_enabled(); ++ swap = swap && !(enable ? lru_gen_nr_swapfiles++ : --lru_gen_nr_swapfiles); ++ swap = swap && lru_gen_enabled(); ++ if (!main && !swap) ++ goto unlock; ++ ++ if (main) { ++ if (enable) ++ static_branch_enable(&lru_gen_static_key); ++ else ++ static_branch_disable(&lru_gen_static_key); ++ } ++ ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ int nid; ++ ++ for_each_node_state(nid, N_MEMORY) { ++ struct pglist_data *pgdat = NODE_DATA(nid); ++ struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg); ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ spin_lock_irq(&pgdat->lru_lock); ++ ++ VM_BUG_ON(!seq_is_valid(lruvec)); ++ VM_BUG_ON(!state_is_valid(lruvec)); ++ ++ WRITE_ONCE(lrugen->enabled[0], lru_gen_enabled() && lru_gen_nr_swapfiles); ++ WRITE_ONCE(lrugen->enabled[1], lru_gen_enabled()); ++ ++ while (!(enable ? fill_lru_gen_lists(lruvec) : ++ drain_lru_gen_lists(lruvec))) { ++ spin_unlock_irq(&pgdat->lru_lock); ++ cond_resched(); ++ spin_lock_irq(&pgdat->lru_lock); ++ } ++ ++ spin_unlock_irq(&pgdat->lru_lock); ++ } ++ ++ cond_resched(); ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); ++unlock: ++ cgroup_unlock(); ++ mutex_unlock(&lru_gen_state_mutex); ++ mem_hotplug_done(); ++} ++ ++static int __meminit __maybe_unused lru_gen_online_mem(struct notifier_block *self, ++ unsigned long action, void *arg) ++{ ++ struct mem_cgroup *memcg; ++ struct memory_notify *mnb = arg; ++ int nid = mnb->status_change_nid; ++ ++ if (action != MEM_GOING_ONLINE || nid == NUMA_NO_NODE) ++ return NOTIFY_DONE; ++ ++ mutex_lock(&lru_gen_state_mutex); ++ cgroup_lock(); ++ ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg); ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ VM_BUG_ON(!seq_is_valid(lruvec)); ++ VM_BUG_ON(!state_is_valid(lruvec)); ++ ++ WRITE_ONCE(lrugen->enabled[0], lru_gen_enabled() && lru_gen_nr_swapfiles); ++ WRITE_ONCE(lrugen->enabled[1], lru_gen_enabled()); ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); ++ ++ cgroup_unlock(); ++ mutex_unlock(&lru_gen_state_mutex); ++ ++ return NOTIFY_DONE; ++} ++ ++/****************************************************************************** ++ * initialization ++ ******************************************************************************/ ++ ++void lru_gen_init_lruvec(struct lruvec *lruvec) ++{ ++ int i; ++ int gen, type, zone; ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ lrugen->max_seq = MIN_NR_GENS + 1; ++ lrugen->enabled[0] = lru_gen_enabled() && lru_gen_nr_swapfiles; ++ lrugen->enabled[1] = lru_gen_enabled(); ++ ++ for (i = 0; i <= MIN_NR_GENS + 1; i++) ++ lrugen->timestamps[i] = jiffies; ++ ++ for_each_gen_type_zone(gen, type, zone) ++ INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); ++} ++ ++static int __init init_lru_gen(void) ++{ ++ BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); ++ BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); ++ ++ if (hotplug_memory_notifier(lru_gen_online_mem, 0)) ++ pr_err("lru_gen: failed to subscribe hotplug notifications\n"); ++ ++ return 0; ++}; ++/* ++ * We want to run as early as possible because debug code may call mm_alloc() ++ * and mmput(). Our only dependency mm_kobj is initialized one stage earlier. ++ */ ++arch_initcall(init_lru_gen); ++ ++#endif /* CONFIG_LRU_GEN */ ++ + /* + * This is a basic per-node page freer. Used by both kswapd and direct reclaim. + */ diff --git a/target/linux/generic/backport-5.4/020-20-BACKPORT-FROMLIST-mm-multigenerational-lru-activatio.patch b/target/linux/generic/backport-5.4/020-20-BACKPORT-FROMLIST-mm-multigenerational-lru-activatio.patch new file mode 100644 index 0000000000..6c0bcbcd39 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-20-BACKPORT-FROMLIST-mm-multigenerational-lru-activatio.patch @@ -0,0 +1,679 @@ +From 50673cccb10ccb11f0fd27106aefe0ae3cc62d26 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Mon, 5 Apr 2021 11:44:28 -0600 +Subject: [PATCH] BACKPORT: FROMLIST: mm: multigenerational lru: + activation + +For pages mapped upon page faults, the accessed bit is set during the +initial faults. We add them to the per-zone lists index by max_seq, +i.e., the youngest generation, so that eviction will not consider them +before the aging has scanned them. Readahead pages allocated in the +page fault path will also be added to the youngest generation, since +it is assumed that they may be needed soon. + +For pages accessed multiple times via file descriptors, instead of +activating them upon the second access, we activate them based on the +refault rates of their tiers. Each generation contains at most +MAX_NR_TIERS tiers, and they require additional MAX_NR_TIERS-2 bits in +page->flags. Pages accessed N times via file descriptors belong to +tier order_base_2(N). Tier 0 is the base tier and it contains pages +read ahead, accessed once via file descriptors and accessed only via +page tables. Pages from the base tier are evicted regardless of the +refault rate. Pages from upper tiers that have higher refault rates +than the base tier will be moved to the next generation. A feedback +loop modeled after the PID controller monitors refault rates across +all tiers and decides when to activate pages from which upper tiers +in the reclaim path. The advantages of this model are: + 1) It has a negligible cost in the buffered IO access path because + activations are done optionally in the reclaim path. + 2) It takes mapped pages into account and avoids overprotecting + pages accessed multiple times via file descriptors. + 3) More tiers offer better protection to pages accessed more than + twice when workloads doing intensive buffered IO are under memory + pressure. + +Finally, we need to make sure deactivation works when the +multigenerational lru is enabled. We cannot use PageActive() because +it is not set on pages from active generations, in order to spare the +aging the trouble of clearing it when active generations become +inactive. So we deactivate pages unconditionally since deactivation is +not a hot code path worth additional optimizations. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432183/) + +BUG=b:123039911 +TEST=Built + +Change-Id: Ibc9c90757fd095cdcc0a49823ada6b55f17ffc06 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2987187 +Reviewed-by: Yu Zhao +Tested-by: Yu Zhao +Commit-Queue: Sonny Rao +Commit-Queue: Yu Zhao +--- + include/linux/memcontrol.h | 20 ------- + include/linux/mm.h | 30 +++++++++++ + include/linux/mm_inline.h | 40 ++++++++++++++ + include/linux/mmzone.h | 11 ++++ + include/linux/sched.h | 2 +- + mm/memcontrol.c | 2 +- + mm/memory.c | 23 ++++++-- + mm/swap.c | 19 ++++++- + mm/swap_state.c | 9 +++- + mm/vmscan.c | 94 ++++++++++++++++++++++++++++++++- + mm/workingset.c | 105 +++++++++++++++++++++++++++++++++++++ + 11 files changed, 323 insertions(+), 32 deletions(-) + +--- a/include/linux/memcontrol.h ++++ b/include/linux/memcontrol.h +@@ -594,18 +594,6 @@ void mem_cgroup_print_oom_context(struct + + void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg); + +-static inline void mem_cgroup_enter_user_fault(void) +-{ +- WARN_ON(current->in_user_fault); +- current->in_user_fault = 1; +-} +- +-static inline void mem_cgroup_exit_user_fault(void) +-{ +- WARN_ON(!current->in_user_fault); +- current->in_user_fault = 0; +-} +- + static inline bool task_in_memcg_oom(struct task_struct *p) + { + return p->memcg_in_oom; +@@ -1063,14 +1051,6 @@ static inline void mem_cgroup_handle_ove + { + } + +-static inline void mem_cgroup_enter_user_fault(void) +-{ +-} +- +-static inline void mem_cgroup_exit_user_fault(void) +-{ +-} +- + static inline bool task_in_memcg_oom(struct task_struct *p) + { + return false; +--- a/include/linux/mm.h ++++ b/include/linux/mm.h +@@ -1513,6 +1513,23 @@ void unmap_mapping_pages(struct address_ + pgoff_t start, pgoff_t nr, bool even_cows); + void unmap_mapping_range(struct address_space *mapping, + loff_t const holebegin, loff_t const holelen, int even_cows); ++ ++static inline void task_enter_user_fault(void) ++{ ++ WARN_ON(current->in_user_fault); ++ current->in_user_fault = 1; ++} ++ ++static inline void task_exit_user_fault(void) ++{ ++ WARN_ON(!current->in_user_fault); ++ current->in_user_fault = 0; ++} ++ ++static inline bool task_in_user_fault(void) ++{ ++ return current->in_user_fault; ++} + #else + static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, + unsigned long address, unsigned int flags) +@@ -1534,6 +1551,19 @@ static inline void unmap_mapping_pages(s + pgoff_t start, pgoff_t nr, bool even_cows) { } + static inline void unmap_mapping_range(struct address_space *mapping, + loff_t const holebegin, loff_t const holelen, int even_cows) { } ++ ++static inline void task_enter_user_fault(void) ++{ ++} ++ ++static inline void task_exit_user_fault(void) ++{ ++} ++ ++static inline bool task_in_user_fault(void) ++{ ++ return false; ++} + #endif + + static inline void unmap_shared_mapping_range(struct address_space *mapping, +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -102,6 +102,12 @@ static inline int lru_gen_from_seq(unsig + return seq % MAX_NR_GENS; + } + ++/* Convert the level of usage to a tier. See the comment on MAX_NR_TIERS. */ ++static inline int lru_tier_from_usage(int usage) ++{ ++ return order_base_2(usage + 1); ++} ++ + /* Return a proper index regardless whether we keep a full history of stats. */ + static inline int hist_from_seq_or_gen(int seq_or_gen) + { +@@ -244,6 +250,36 @@ static inline bool lru_gen_deletion(stru + return true; + } + ++/* Return the level of usage of a page. See the comment on MAX_NR_TIERS. */ ++static inline int page_tier_usage(struct page *page) ++{ ++ unsigned long flags = READ_ONCE(page->flags); ++ ++ return flags & BIT(PG_workingset) ? ++ ((flags & LRU_USAGE_MASK) >> LRU_USAGE_PGOFF) + 1 : 0; ++} ++ ++/* Increment the usage counter after a page is accessed via file descriptors. */ ++static inline void page_inc_usage(struct page *page) ++{ ++ unsigned long usage; ++ unsigned long old_flags, new_flags; ++ ++ do { ++ old_flags = READ_ONCE(page->flags); ++ ++ if (!(old_flags & BIT(PG_workingset))) { ++ new_flags = old_flags | BIT(PG_workingset); ++ continue; ++ } ++ ++ usage = (old_flags & LRU_USAGE_MASK) + BIT(LRU_USAGE_PGOFF); ++ ++ new_flags = (old_flags & ~LRU_USAGE_MASK) | min(usage, LRU_USAGE_MASK); ++ } while (new_flags != old_flags && ++ cmpxchg(&page->flags, old_flags, new_flags) != old_flags); ++} ++ + #else /* CONFIG_LRU_GEN */ + + static inline bool lru_gen_enabled(void) +@@ -261,6 +297,10 @@ static inline bool lru_gen_deletion(stru + return false; + } + ++static inline void page_inc_usage(struct page *page) ++{ ++} ++ + #endif /* CONFIG_LRU_GEN */ + + static __always_inline void add_page_to_lru_list(struct page *page, +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -384,6 +384,8 @@ struct lrugen { + + void lru_gen_init_lruvec(struct lruvec *lruvec); + void lru_gen_set_state(bool enable, bool main, bool swap); ++void *lru_gen_eviction(struct page *page); ++void lru_gen_refault(struct page *page, void *shadow); + + #else /* CONFIG_LRU_GEN */ + +@@ -395,6 +397,15 @@ static inline void lru_gen_set_state(boo + { + } + ++static inline void *lru_gen_eviction(struct page *page) ++{ ++ return NULL; ++} ++ ++static inline void lru_gen_refault(struct page *page, void *shadow) ++{ ++} ++ + #endif /* CONFIG_LRU_GEN */ + + struct lruvec { +--- a/include/linux/sched.h ++++ b/include/linux/sched.h +@@ -768,7 +768,7 @@ struct task_struct { + #ifndef TIF_RESTORE_SIGMASK + unsigned restore_sigmask:1; + #endif +-#ifdef CONFIG_MEMCG ++#ifdef CONFIG_MMU + unsigned in_user_fault:1; + #endif + #ifdef CONFIG_COMPAT_BRK +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -1922,7 +1922,7 @@ static enum oom_status mem_cgroup_oom(st + * victim and then we have to bail out from the charge path. + */ + if (memcg->oom_kill_disable) { +- if (!current->in_user_fault) ++ if (!task_in_user_fault()) + return OOM_SKIPPED; + css_get(&memcg->css); + current->memcg_in_oom = memcg; +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -71,6 +71,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -2887,6 +2888,19 @@ void unmap_mapping_range(struct address_ + } + EXPORT_SYMBOL(unmap_mapping_range); + ++static void lru_gen_swap_refault(struct page *page, swp_entry_t entry) ++{ ++ if (lru_gen_enabled()) { ++ void *item; ++ struct address_space *mapping = swap_address_space(entry); ++ pgoff_t index = swp_offset(entry); ++ ++ item = xa_load(&mapping->i_pages, index); ++ if (xa_is_value(item)) ++ lru_gen_refault(page, item); ++ } ++} ++ + /* + * We enter with non-exclusive mmap_sem (to exclude vma changes, + * but allow concurrent faults), and pte mapped but not yet locked. +@@ -2943,6 +2957,7 @@ vm_fault_t do_swap_page(struct vm_fault + __SetPageLocked(page); + __SetPageSwapBacked(page); + set_page_private(page, entry.val); ++ lru_gen_swap_refault(page, entry); + lru_cache_add_anon(page); + swap_readpage(page, true); + } +@@ -4174,7 +4189,7 @@ vm_fault_t handle_mm_fault(struct vm_are + * space. Kernel faults are handled more gracefully. + */ + if (flags & FAULT_FLAG_USER) +- mem_cgroup_enter_user_fault(); ++ task_enter_user_fault(); + + if (unlikely(is_vm_hugetlb_page(vma))) + ret = hugetlb_fault(vma->vm_mm, vma, address, flags); +@@ -4182,7 +4197,7 @@ vm_fault_t handle_mm_fault(struct vm_are + ret = __handle_mm_fault(vma, address, flags); + + if (flags & FAULT_FLAG_USER) { +- mem_cgroup_exit_user_fault(); ++ task_exit_user_fault(); + /* + * The task may have entered a memcg OOM situation but + * if the allocation error was handled gracefully (no +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -303,6 +303,9 @@ static bool need_activate_page_drain(int + + void activate_page(struct page *page) + { ++ if (lru_gen_enabled()) ++ return; ++ + page = compound_head(page); + if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { + struct pagevec *pvec = &get_cpu_var(activate_page_pvecs); +@@ -323,6 +326,9 @@ void activate_page(struct page *page) + { + pg_data_t *pgdat = page_pgdat(page); + ++ if (lru_gen_enabled()) ++ return; ++ + page = compound_head(page); + spin_lock_irq(&pgdat->lru_lock); + __activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL); +@@ -372,6 +378,10 @@ void mark_page_accessed(struct page *pag + page = compound_head(page); + if (!PageActive(page) && !PageUnevictable(page) && + PageReferenced(page)) { ++ if (lru_gen_enabled()) { ++ page_inc_usage(page); ++ goto done; ++ } + + /* + * If the page is on the LRU, queue it for activation via +@@ -389,6 +399,7 @@ void mark_page_accessed(struct page *pag + } else if (!PageReferenced(page)) { + SetPageReferenced(page); + } ++done: + if (page_is_idle(page)) + clear_page_idle(page); + } +@@ -398,6 +409,10 @@ static void __lru_cache_add(struct page + { + struct pagevec *pvec = &get_cpu_var(lru_add_pvec); + ++ if (lru_gen_enabled() && !PageActive(page) && !PageUnevictable(page) && ++ task_in_user_fault() && !(current->flags & PF_MEMALLOC)) ++ SetPageActive(page); ++ + get_page(page); + if (!pagevec_add(pvec, page) || PageCompound(page)) + __pagevec_lru_add(pvec); +@@ -538,7 +553,7 @@ static void lru_deactivate_file_fn(struc + static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, + void *arg) + { +- if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { ++ if (PageLRU(page) && !PageUnevictable(page) && (PageActive(page) || lru_gen_enabled())) { + int file = page_is_file_cache(page); + + del_page_from_lru_list(page, lruvec); +@@ -646,7 +661,7 @@ void deactivate_file_page(struct page *p + */ + void deactivate_page(struct page *page) + { +- if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { ++ if (PageLRU(page) && !PageUnevictable(page) && (PageActive(page) || lru_gen_enabled())) { + struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs); + + get_page(page); +--- a/mm/swap_state.c ++++ b/mm/swap_state.c +@@ -21,6 +21,7 @@ + #include + #include + #include ++#include + + #include + #include "internal.h" +@@ -409,6 +410,7 @@ struct page *__read_swap_cache_async(swp + struct page *found_page = NULL, *new_page = NULL; + struct swap_info_struct *si; + int err; ++ void *shadow; + *new_page_allocated = false; + + do { +@@ -465,10 +467,13 @@ struct page *__read_swap_cache_async(swp + __SetPageLocked(new_page); + __SetPageSwapBacked(new_page); + err = add_to_swap_cache(new_page, entry, +- gfp_mask & GFP_RECLAIM_MASK, NULL); ++ gfp_mask & GFP_RECLAIM_MASK, &shadow); + if (likely(!err)) { + /* Initiate read into locked page */ +- SetPageWorkingset(new_page); ++ if (!lru_gen_enabled()) ++ SetPageWorkingset(new_page); ++ else if (shadow) ++ lru_gen_refault(new_page, shadow); + lru_cache_add_anon(new_page); + *new_page_allocated = true; + return new_page; +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -907,6 +907,7 @@ static int __remove_mapping(struct addre + { + unsigned long flags; + int refcount; ++ void *shadow = NULL; + + BUG_ON(!PageLocked(page)); + BUG_ON(mapping != page_mapping(page)); +@@ -948,13 +949,15 @@ static int __remove_mapping(struct addre + + if (PageSwapCache(page)) { + swp_entry_t swap = { .val = page_private(page) }; ++ ++ if (lru_gen_enabled()) ++ shadow = lru_gen_eviction(page); + mem_cgroup_swapout(page, swap); +- __delete_from_swap_cache(page, swap, NULL); ++ __delete_from_swap_cache(page, swap, shadow); + xa_unlock_irqrestore(&mapping->i_pages, flags); + put_swap_page(page, swap); + } else { + void (*freepage)(struct page *); +- void *shadow = NULL; + + freepage = mapping->a_ops->freepage; + /* +@@ -2623,6 +2626,93 @@ static bool __maybe_unused seq_is_valid( + } + + /****************************************************************************** ++ * refault feedback loop ++ ******************************************************************************/ ++ ++/* ++ * A feedback loop modeled after the PID controller. Currently supports the ++ * proportional (P) and the integral (I) terms; the derivative (D) term can be ++ * added if necessary. The setpoint (SP) is the desired position; the process ++ * variable (PV) is the measured position. The error is the difference between ++ * the SP and the PV. A positive error results in a positive control output ++ * correction, which, in our case, is to allow eviction. ++ * ++ * The P term is the current refault rate refaulted/(evicted+activated), which ++ * has a weight of 1. The I term is the arithmetic mean of the last N refault ++ * rates, weighted by geometric series 1/2, 1/4, ..., 1/(1<evictable; ++ int hist = hist_from_seq_or_gen(lrugen->min_seq[type]); ++ ++ pos->refaulted = lrugen->avg_refaulted[type][tier] + ++ atomic_long_read(&lrugen->refaulted[hist][type][tier]); ++ pos->total = lrugen->avg_total[type][tier] + ++ atomic_long_read(&lrugen->evicted[hist][type][tier]); ++ if (tier) ++ pos->total += lrugen->activated[hist][type][tier - 1]; ++ pos->gain = gain; ++} ++ ++static void reset_controller_pos(struct lruvec *lruvec, int gen, int type) ++{ ++ int tier; ++ int hist = hist_from_seq_or_gen(gen); ++ struct lrugen *lrugen = &lruvec->evictable; ++ bool carryover = gen == lru_gen_from_seq(lrugen->min_seq[type]); ++ ++ if (!carryover && NR_STAT_GENS == 1) ++ return; ++ ++ for (tier = 0; tier < MAX_NR_TIERS; tier++) { ++ if (carryover) { ++ unsigned long sum; ++ ++ sum = lrugen->avg_refaulted[type][tier] + ++ atomic_long_read(&lrugen->refaulted[hist][type][tier]); ++ WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); ++ ++ sum = lrugen->avg_total[type][tier] + ++ atomic_long_read(&lrugen->evicted[hist][type][tier]); ++ if (tier) ++ sum += lrugen->activated[hist][type][tier - 1]; ++ WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); ++ ++ if (NR_STAT_GENS > 1) ++ continue; ++ } ++ ++ atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); ++ atomic_long_set(&lrugen->evicted[hist][type][tier], 0); ++ if (tier) ++ WRITE_ONCE(lrugen->activated[hist][type][tier - 1], 0); ++ } ++} ++ ++static bool positive_ctrl_err(struct controller_pos *sp, struct controller_pos *pv) ++{ ++ /* ++ * Allow eviction if the PV has a limited number of refaulted pages or a ++ * lower refault rate than the SP. ++ */ ++ return pv->refaulted < SWAP_CLUSTER_MAX || ++ pv->refaulted * max(sp->total, 1UL) * sp->gain <= ++ sp->refaulted * max(pv->total, 1UL) * pv->gain; ++} ++ ++/****************************************************************************** + * state change + ******************************************************************************/ + +--- a/mm/workingset.c ++++ b/mm/workingset.c +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + + /* + * Double CLOCK lists +@@ -200,6 +201,102 @@ static unsigned long unpack_shadow(void + return val >> MEM_CGROUP_ID_SHIFT; + } + ++#ifdef CONFIG_LRU_GEN ++ ++#if LRU_GEN_SHIFT + LRU_USAGE_SHIFT >= EVICTION_SHIFT ++#error "Please try smaller NODES_SHIFT, NR_LRU_GENS and TIERS_PER_GEN configurations" ++#endif ++ ++static void page_set_usage(struct page *page, int usage) ++{ ++ unsigned long old_flags, new_flags; ++ ++ VM_BUG_ON(usage > BIT(LRU_USAGE_WIDTH)); ++ ++ if (!usage) ++ return; ++ ++ do { ++ old_flags = READ_ONCE(page->flags); ++ new_flags = (old_flags & ~LRU_USAGE_MASK) | LRU_TIER_FLAGS | ++ ((usage - 1UL) << LRU_USAGE_PGOFF); ++ } while (new_flags != old_flags && ++ cmpxchg(&page->flags, old_flags, new_flags) != old_flags); ++} ++ ++/* Return a token to be stored in the shadow entry of a page being evicted. */ ++void *lru_gen_eviction(struct page *page) ++{ ++ int hist, tier; ++ unsigned long token; ++ unsigned long min_seq; ++ struct lruvec *lruvec; ++ struct lrugen *lrugen; ++ int type = page_is_file_cache(page); ++ int usage = page_tier_usage(page); ++ struct mem_cgroup *memcg = page_memcg(page); ++ struct pglist_data *pgdat = page_pgdat(page); ++ ++ if (!mem_cgroup_disabled() && !memcg) ++ return NULL; ++ ++ lruvec = mem_cgroup_lruvec(pgdat, memcg); ++ lrugen = &lruvec->evictable; ++ min_seq = READ_ONCE(lrugen->min_seq[type]); ++ token = (min_seq << LRU_USAGE_SHIFT) | usage; ++ ++ hist = hist_from_seq_or_gen(min_seq); ++ tier = lru_tier_from_usage(usage); ++ atomic_long_add(hpage_nr_pages(page), &lrugen->evicted[hist][type][tier]); ++ ++ return pack_shadow(mem_cgroup_id(memcg), pgdat, token); ++} ++ ++/* Account a refaulted page based on the token stored in its shadow entry. */ ++void lru_gen_refault(struct page *page, void *shadow) ++{ ++ int hist, tier, usage; ++ int memcg_id; ++ unsigned long token; ++ unsigned long min_seq; ++ struct lruvec *lruvec; ++ struct lrugen *lrugen; ++ struct pglist_data *pgdat; ++ struct mem_cgroup *memcg; ++ int type = page_is_file_cache(page); ++ ++ token = unpack_shadow(shadow, &memcg_id, &pgdat); ++ if (page_pgdat(page) != pgdat) ++ return; ++ ++ rcu_read_lock(); ++ memcg = mem_cgroup_from_id(memcg_id); ++ if (!mem_cgroup_disabled() && !memcg) ++ goto unlock; ++ ++ usage = token & (BIT(LRU_USAGE_SHIFT) - 1); ++ token >>= LRU_USAGE_SHIFT; ++ ++ lruvec = mem_cgroup_lruvec(pgdat, memcg); ++ lrugen = &lruvec->evictable; ++ min_seq = READ_ONCE(lrugen->min_seq[type]); ++ if (token != (min_seq & (EVICTION_MASK >> LRU_USAGE_SHIFT))) ++ goto unlock; ++ ++ page_set_usage(page, usage); ++ ++ hist = hist_from_seq_or_gen(min_seq); ++ tier = lru_tier_from_usage(usage); ++ atomic_long_add(hpage_nr_pages(page), &lrugen->refaulted[hist][type][tier]); ++ inc_lruvec_state(lruvec, WORKINGSET_REFAULT); ++ if (tier) ++ inc_lruvec_state(lruvec, WORKINGSET_RESTORE); ++unlock: ++ rcu_read_unlock(); ++} ++ ++#endif /* CONFIG_LRU_GEN */ ++ + /** + * workingset_eviction - note the eviction of a page from memory + * @page: the page being evicted +@@ -220,6 +317,9 @@ void *workingset_eviction(struct page *p + VM_BUG_ON_PAGE(page_count(page), page); + VM_BUG_ON_PAGE(!PageLocked(page), page); + ++ if (lru_gen_enabled()) ++ return lru_gen_eviction(page); ++ + lruvec = mem_cgroup_lruvec(pgdat, memcg); + eviction = atomic_long_inc_return(&lruvec->inactive_age); + eviction >>= bucket_order; +@@ -247,6 +347,11 @@ void workingset_refault(struct page *pag + bool workingset; + int memcgid; + ++ if (lru_gen_enabled()) { ++ lru_gen_refault(page, shadow); ++ return; ++ } ++ + eviction = unpack_shadow(shadow, &memcgid, &pgdat); + + rcu_read_lock(); diff --git a/target/linux/generic/backport-5.4/020-21-BACKPORT-FROMLIST-mm-multigenerational-lru-mm_struct.patch b/target/linux/generic/backport-5.4/020-21-BACKPORT-FROMLIST-mm-multigenerational-lru-mm_struct.patch new file mode 100644 index 0000000000..cce26c4b3a --- /dev/null +++ b/target/linux/generic/backport-5.4/020-21-BACKPORT-FROMLIST-mm-multigenerational-lru-mm_struct.patch @@ -0,0 +1,686 @@ +From 7adc80c85d8815b4e2f5b7454c31bedc7f04de34 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Mon, 5 Apr 2021 04:17:41 -0600 +Subject: [PATCH] BACKPORT: FROMLIST: mm: multigenerational lru: + mm_struct list + +In order to scan page tables, we add an infrastructure to maintain +either a system-wide mm_struct list or per-memcg mm_struct lists, and +track whether an mm_struct is being used or has been used since the +last scan. + +Multiple threads can concurrently work on the same mm_struct list, and +each of them will be given a different mm_struct belonging to a +process that has been scheduled since the last scan. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432184/) + +BUG=b:123039911 +TEST=Built + +Change-Id: I25d9eda8c6bdc7c3653b9f210a159d6c247c81e8 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2987188 +Reviewed-by: Yu Zhao +Tested-by: Yu Zhao +Commit-Queue: Sonny Rao +Commit-Queue: Yu Zhao +--- + fs/exec.c | 2 + + include/linux/memcontrol.h | 6 + + include/linux/mm_types.h | 107 ++++++++++++ + kernel/exit.c | 1 + + kernel/fork.c | 10 ++ + kernel/sched/core.c | 2 + + mm/memcontrol.c | 28 ++++ + mm/mmu_context.c | 1 + + mm/vmscan.c | 324 +++++++++++++++++++++++++++++++++++++ + 9 files changed, 481 insertions(+) + +--- a/fs/exec.c ++++ b/fs/exec.c +@@ -1059,6 +1059,7 @@ static int exec_mmap(struct mm_struct *m + active_mm = tsk->active_mm; + tsk->active_mm = mm; + tsk->mm = mm; ++ lru_gen_add_mm(mm); + /* + * This prevents preemption while active_mm is being loaded and + * it and mm are being updated, which could cause problems for +@@ -1069,6 +1070,7 @@ static int exec_mmap(struct mm_struct *m + if (!IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); + activate_mm(active_mm, mm); ++ lru_gen_switch_mm(active_mm, mm); + if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); + tsk->mm->vmacache_seqnum = 0; +--- a/include/linux/memcontrol.h ++++ b/include/linux/memcontrol.h +@@ -182,6 +182,8 @@ struct memcg_padding { + #define MEMCG_PADDING(name) + #endif + ++struct lru_gen_mm_list; ++ + /* + * Remember four most recent foreign writebacks with dirty pages in this + * cgroup. Inode sharing is expected to be uncommon and, even if we miss +@@ -334,6 +336,10 @@ struct mem_cgroup { + struct deferred_split deferred_split_queue; + #endif + ++#ifdef CONFIG_LRU_GEN ++ struct lru_gen_mm_list *mm_list; ++#endif ++ + struct mem_cgroup_per_node *nodeinfo[0]; + /* WARNING: nodeinfo must be the last member here */ + }; +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -14,6 +14,8 @@ + #include + #include + #include ++#include ++#include + + #include + +@@ -524,6 +526,22 @@ struct mm_struct { + atomic_long_t hugetlb_usage; + #endif + struct work_struct async_put_work; ++#ifdef CONFIG_LRU_GEN ++ struct { ++ /* the node of a global or per-memcg mm_struct list */ ++ struct list_head list; ++#ifdef CONFIG_MEMCG ++ /* points to the memcg of the owner task above */ ++ struct mem_cgroup *memcg; ++#endif ++ /* whether this mm_struct has been used since the last walk */ ++ nodemask_t nodes; ++#ifndef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH ++ /* the number of CPUs using this mm_struct */ ++ atomic_t nr_cpus; ++#endif ++ } lrugen; ++#endif + } __randomize_layout; + + /* +@@ -550,6 +568,95 @@ static inline cpumask_t *mm_cpumask(stru + return (struct cpumask *)&mm->cpu_bitmap; + } + ++#ifdef CONFIG_LRU_GEN ++ ++void lru_gen_init_mm(struct mm_struct *mm); ++void lru_gen_add_mm(struct mm_struct *mm); ++void lru_gen_del_mm(struct mm_struct *mm); ++#ifdef CONFIG_MEMCG ++int lru_gen_alloc_mm_list(struct mem_cgroup *memcg); ++void lru_gen_free_mm_list(struct mem_cgroup *memcg); ++void lru_gen_migrate_mm(struct mm_struct *mm); ++#endif ++ ++/* Track the usage of each mm_struct so that we can skip inactive ones. */ ++static inline void lru_gen_switch_mm(struct mm_struct *old, struct mm_struct *new) ++{ ++ /* exclude init_mm, efi_mm, etc. */ ++ if (!core_kernel_data((unsigned long)old)) { ++ VM_BUG_ON(old == &init_mm); ++ ++ nodes_setall(old->lrugen.nodes); ++#ifndef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH ++ atomic_dec(&old->lrugen.nr_cpus); ++ VM_BUG_ON_MM(atomic_read(&old->lrugen.nr_cpus) < 0, old); ++#endif ++ } else ++ VM_BUG_ON_MM(READ_ONCE(old->lrugen.list.prev) || ++ READ_ONCE(old->lrugen.list.next), old); ++ ++ if (!core_kernel_data((unsigned long)new)) { ++ VM_BUG_ON(new == &init_mm); ++ ++#ifndef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH ++ atomic_inc(&new->lrugen.nr_cpus); ++ VM_BUG_ON_MM(atomic_read(&new->lrugen.nr_cpus) < 0, new); ++#endif ++ } else ++ VM_BUG_ON_MM(READ_ONCE(new->lrugen.list.prev) || ++ READ_ONCE(new->lrugen.list.next), new); ++} ++ ++/* Return whether this mm_struct is being used on any CPUs. */ ++static inline bool lru_gen_mm_is_active(struct mm_struct *mm) ++{ ++#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH ++ return !cpumask_empty(mm_cpumask(mm)); ++#else ++ return atomic_read(&mm->lrugen.nr_cpus); ++#endif ++} ++ ++#else /* CONFIG_LRU_GEN */ ++ ++static inline void lru_gen_init_mm(struct mm_struct *mm) ++{ ++} ++ ++static inline void lru_gen_add_mm(struct mm_struct *mm) ++{ ++} ++ ++static inline void lru_gen_del_mm(struct mm_struct *mm) ++{ ++} ++ ++#ifdef CONFIG_MEMCG ++static inline int lru_gen_alloc_mm_list(struct mem_cgroup *memcg) ++{ ++ return 0; ++} ++ ++static inline void lru_gen_free_mm_list(struct mem_cgroup *memcg) ++{ ++} ++ ++static inline void lru_gen_migrate_mm(struct mm_struct *mm) ++{ ++} ++#endif ++ ++static inline void lru_gen_switch_mm(struct mm_struct *old, struct mm_struct *new) ++{ ++} ++ ++static inline bool lru_gen_mm_is_active(struct mm_struct *mm) ++{ ++ return false; ++} ++ ++#endif /* CONFIG_LRU_GEN */ ++ + struct mmu_gather; + extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, + unsigned long start, unsigned long end); +--- a/kernel/exit.c ++++ b/kernel/exit.c +@@ -423,6 +423,7 @@ assign_new_owner: + goto retry; + } + WRITE_ONCE(mm->owner, c); ++ lru_gen_migrate_mm(mm); + task_unlock(c); + put_task_struct(c); + } +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -672,6 +672,7 @@ static void check_mm(struct mm_struct *m + #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS + VM_BUG_ON_MM(mm->pmd_huge_pte, mm); + #endif ++ VM_BUG_ON_MM(lru_gen_mm_is_active(mm), mm); + } + + #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL)) +@@ -1045,6 +1046,7 @@ static struct mm_struct *mm_init(struct + goto fail_nocontext; + + mm->user_ns = get_user_ns(user_ns); ++ lru_gen_init_mm(mm); + return mm; + + fail_nocontext: +@@ -1087,6 +1089,7 @@ static inline void __mmput(struct mm_str + } + if (mm->binfmt) + module_put(mm->binfmt->module); ++ lru_gen_del_mm(mm); + mmdrop(mm); + } + +@@ -2399,6 +2402,13 @@ long _do_fork(struct kernel_clone_args * + get_task_struct(p); + } + ++ if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) { ++ /* lock the task to synchronize with memcg migration */ ++ task_lock(p); ++ lru_gen_add_mm(p->mm); ++ task_unlock(p); ++ } ++ + wake_up_new_task(p); + + /* forking complete and child started to run, tell ptracer */ +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -3468,6 +3468,7 @@ context_switch(struct rq *rq, struct tas + * finish_task_switch()'s mmdrop(). + */ + switch_mm_irqs_off(prev->active_mm, next->mm, next); ++ lru_gen_switch_mm(prev->active_mm, next->mm); + + if (!prev->mm) { // from kernel + /* will mmdrop() in finish_task_switch(). */ +@@ -6284,6 +6285,7 @@ void idle_task_exit(void) + + if (mm != &init_mm) { + switch_mm(mm, &init_mm, current); ++ lru_gen_switch_mm(mm, &init_mm); + finish_arch_post_lock_switch(); + } + +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -5087,6 +5087,7 @@ static void __mem_cgroup_free(struct mem + free_mem_cgroup_per_node_info(memcg, node); + free_percpu(memcg->vmstats_percpu); + free_percpu(memcg->vmstats_local); ++ lru_gen_free_mm_list(memcg); + kfree(memcg); + } + +@@ -5137,6 +5138,9 @@ static struct mem_cgroup *mem_cgroup_all + if (alloc_mem_cgroup_per_node_info(memcg, node)) + goto fail; + ++ if (lru_gen_alloc_mm_list(memcg)) ++ goto fail; ++ + if (memcg_wb_domain_init(memcg, GFP_KERNEL)) + goto fail; + +@@ -6055,6 +6059,29 @@ static void mem_cgroup_move_task(void) + } + #endif + ++#ifdef CONFIG_LRU_GEN ++static void mem_cgroup_attach(struct cgroup_taskset *tset) ++{ ++ struct cgroup_subsys_state *css; ++ struct task_struct *task = NULL; ++ ++ cgroup_taskset_for_each_leader(task, css, tset) ++ ; ++ ++ if (!task) ++ return; ++ ++ task_lock(task); ++ if (task->mm && task->mm->owner == task) ++ lru_gen_migrate_mm(task->mm); ++ task_unlock(task); ++} ++#else ++static void mem_cgroup_attach(struct cgroup_taskset *tset) ++{ ++} ++#endif ++ + /* + * Cgroup retains root cgroups across [un]mount cycles making it necessary + * to verify whether we're attached to the default hierarchy on each mount +@@ -6355,6 +6382,7 @@ struct cgroup_subsys memory_cgrp_subsys + .css_free = mem_cgroup_css_free, + .css_reset = mem_cgroup_css_reset, + .can_attach = mem_cgroup_can_attach, ++ .attach = mem_cgroup_attach, + .cancel_attach = mem_cgroup_cancel_attach, + .post_attach = mem_cgroup_move_task, + .bind = mem_cgroup_bind, +--- a/mm/mmu_context.c ++++ b/mm/mmu_context.c +@@ -34,6 +34,7 @@ void use_mm(struct mm_struct *mm) + } + tsk->mm = mm; + switch_mm_irqs_off(active_mm, mm, tsk); ++ lru_gen_switch_mm(active_mm, mm); + local_irq_enable(); + task_unlock(tsk); + #ifdef finish_arch_post_lock_switch +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -2713,6 +2713,323 @@ static bool positive_ctrl_err(struct con + } + + /****************************************************************************** ++ * mm_struct list ++ ******************************************************************************/ ++ ++enum { ++ MM_SCHED_ACTIVE, /* running processes */ ++ MM_SCHED_INACTIVE, /* sleeping processes */ ++ MM_LOCK_CONTENTION, /* lock contentions */ ++ MM_VMA_INTERVAL, /* VMAs within the range of each PUD/PMD/PTE */ ++ MM_LEAF_OTHER_NODE, /* entries not from the node under reclaim */ ++ MM_LEAF_OTHER_MEMCG, /* entries not from the memcg under reclaim */ ++ MM_LEAF_OLD, /* old entries */ ++ MM_LEAF_YOUNG, /* young entries */ ++ MM_LEAF_DIRTY, /* dirty entries */ ++ MM_LEAF_HOLE, /* non-present entries */ ++ MM_NONLEAF_OLD, /* old non-leaf PMD entries */ ++ MM_NONLEAF_YOUNG, /* young non-leaf PMD entries */ ++ NR_MM_STATS ++}; ++ ++/* mnemonic codes for the stats above */ ++#define MM_STAT_CODES "aicvnmoydhlu" ++ ++struct lru_gen_mm_list { ++ /* the head of a global or per-memcg mm_struct list */ ++ struct list_head head; ++ /* protects the list */ ++ spinlock_t lock; ++ struct { ++ /* set to max_seq after each round of walk */ ++ unsigned long cur_seq; ++ /* the next mm on the list to walk */ ++ struct list_head *iter; ++ /* to wait for the last worker to finish */ ++ struct wait_queue_head wait; ++ /* the number of concurrent workers */ ++ int nr_workers; ++ /* stats for debugging */ ++ unsigned long stats[NR_STAT_GENS][NR_MM_STATS]; ++ } nodes[0]; ++}; ++ ++static struct lru_gen_mm_list *global_mm_list; ++ ++static struct lru_gen_mm_list *alloc_mm_list(void) ++{ ++ int nid; ++ struct lru_gen_mm_list *mm_list; ++ ++ mm_list = kzalloc(struct_size(mm_list, nodes, nr_node_ids), GFP_KERNEL); ++ if (!mm_list) ++ return NULL; ++ ++ INIT_LIST_HEAD(&mm_list->head); ++ spin_lock_init(&mm_list->lock); ++ ++ for_each_node(nid) { ++ mm_list->nodes[nid].cur_seq = MIN_NR_GENS; ++ mm_list->nodes[nid].iter = &mm_list->head; ++ init_waitqueue_head(&mm_list->nodes[nid].wait); ++ } ++ ++ return mm_list; ++} ++ ++static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) ++{ ++#ifdef CONFIG_MEMCG ++ if (!mem_cgroup_disabled()) ++ return memcg ? memcg->mm_list : root_mem_cgroup->mm_list; ++#endif ++ VM_BUG_ON(memcg); ++ ++ return global_mm_list; ++} ++ ++void lru_gen_init_mm(struct mm_struct *mm) ++{ ++ INIT_LIST_HEAD(&mm->lrugen.list); ++#ifdef CONFIG_MEMCG ++ mm->lrugen.memcg = NULL; ++#endif ++#ifndef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH ++ atomic_set(&mm->lrugen.nr_cpus, 0); ++#endif ++ nodes_clear(mm->lrugen.nodes); ++} ++ ++void lru_gen_add_mm(struct mm_struct *mm) ++{ ++ struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); ++ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); ++ ++ VM_BUG_ON_MM(!list_empty(&mm->lrugen.list), mm); ++#ifdef CONFIG_MEMCG ++ VM_BUG_ON_MM(mm->lrugen.memcg, mm); ++ WRITE_ONCE(mm->lrugen.memcg, memcg); ++#endif ++ spin_lock(&mm_list->lock); ++ list_add_tail(&mm->lrugen.list, &mm_list->head); ++ spin_unlock(&mm_list->lock); ++} ++ ++void lru_gen_del_mm(struct mm_struct *mm) ++{ ++ int nid; ++#ifdef CONFIG_MEMCG ++ struct lru_gen_mm_list *mm_list = get_mm_list(mm->lrugen.memcg); ++#else ++ struct lru_gen_mm_list *mm_list = get_mm_list(NULL); ++#endif ++ ++ spin_lock(&mm_list->lock); ++ ++ for_each_node(nid) { ++ if (mm_list->nodes[nid].iter != &mm->lrugen.list) ++ continue; ++ ++ mm_list->nodes[nid].iter = mm_list->nodes[nid].iter->next; ++ if (mm_list->nodes[nid].iter == &mm_list->head) ++ WRITE_ONCE(mm_list->nodes[nid].cur_seq, ++ mm_list->nodes[nid].cur_seq + 1); ++ } ++ ++ list_del_init(&mm->lrugen.list); ++ ++ spin_unlock(&mm_list->lock); ++ ++#ifdef CONFIG_MEMCG ++ mem_cgroup_put(mm->lrugen.memcg); ++ WRITE_ONCE(mm->lrugen.memcg, NULL); ++#endif ++} ++ ++#ifdef CONFIG_MEMCG ++int lru_gen_alloc_mm_list(struct mem_cgroup *memcg) ++{ ++ if (mem_cgroup_disabled()) ++ return 0; ++ ++ memcg->mm_list = alloc_mm_list(); ++ ++ return memcg->mm_list ? 0 : -ENOMEM; ++} ++ ++void lru_gen_free_mm_list(struct mem_cgroup *memcg) ++{ ++ kfree(memcg->mm_list); ++ memcg->mm_list = NULL; ++} ++ ++void lru_gen_migrate_mm(struct mm_struct *mm) ++{ ++ struct mem_cgroup *memcg; ++ ++ lockdep_assert_held(&mm->owner->alloc_lock); ++ ++ if (mem_cgroup_disabled()) ++ return; ++ ++ rcu_read_lock(); ++ memcg = mem_cgroup_from_task(mm->owner); ++ rcu_read_unlock(); ++ if (memcg == mm->lrugen.memcg) ++ return; ++ ++ VM_BUG_ON_MM(!mm->lrugen.memcg, mm); ++ VM_BUG_ON_MM(list_empty(&mm->lrugen.list), mm); ++ ++ lru_gen_del_mm(mm); ++ lru_gen_add_mm(mm); ++} ++ ++static bool mm_has_migrated(struct mm_struct *mm, struct mem_cgroup *memcg) ++{ ++ return READ_ONCE(mm->lrugen.memcg) != memcg; ++} ++#else ++static bool mm_has_migrated(struct mm_struct *mm, struct mem_cgroup *memcg) ++{ ++ return false; ++} ++#endif ++ ++struct mm_walk_args { ++ struct mem_cgroup *memcg; ++ unsigned long max_seq; ++ unsigned long start_pfn; ++ unsigned long end_pfn; ++ unsigned long next_addr; ++ int node_id; ++ int swappiness; ++ int batch_size; ++ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; ++ int mm_stats[NR_MM_STATS]; ++ unsigned long bitmap[0]; ++}; ++ ++static int size_of_mm_walk_args(void) ++{ ++ int size = sizeof(struct mm_walk_args); ++ ++ if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) || ++ IS_ENABLED(CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG)) ++ size += sizeof(unsigned long) * BITS_TO_LONGS(PTRS_PER_PMD); ++ ++ return size; ++} ++ ++static void reset_mm_stats(struct lru_gen_mm_list *mm_list, bool last, ++ struct mm_walk_args *args) ++{ ++ int i; ++ int nid = args->node_id; ++ int hist = hist_from_seq_or_gen(args->max_seq); ++ ++ lockdep_assert_held(&mm_list->lock); ++ ++ for (i = 0; i < NR_MM_STATS; i++) { ++ WRITE_ONCE(mm_list->nodes[nid].stats[hist][i], ++ mm_list->nodes[nid].stats[hist][i] + args->mm_stats[i]); ++ args->mm_stats[i] = 0; ++ } ++ ++ if (!last || NR_STAT_GENS == 1) ++ return; ++ ++ hist = hist_from_seq_or_gen(args->max_seq + 1); ++ for (i = 0; i < NR_MM_STATS; i++) ++ WRITE_ONCE(mm_list->nodes[nid].stats[hist][i], 0); ++} ++ ++static bool should_skip_mm(struct mm_struct *mm, struct mm_walk_args *args) ++{ ++ int type; ++ unsigned long size = 0; ++ ++ if (!lru_gen_mm_is_active(mm) && !node_isset(args->node_id, mm->lrugen.nodes)) ++ return true; ++ ++ if (mm_is_oom_victim(mm)) ++ return true; ++ ++ for (type = !args->swappiness; type < ANON_AND_FILE; type++) { ++ size += type ? get_mm_counter(mm, MM_FILEPAGES) : ++ get_mm_counter(mm, MM_ANONPAGES) + ++ get_mm_counter(mm, MM_SHMEMPAGES); ++ } ++ ++ /* leave the legwork to the rmap if mappings are too sparse */ ++ if (size < max(SWAP_CLUSTER_MAX, mm_pgtables_bytes(mm) / PAGE_SIZE)) ++ return true; ++ ++ return !mmget_not_zero(mm); ++} ++ ++/* To support multiple workers that concurrently walk an mm_struct list. */ ++static bool get_next_mm(struct mm_walk_args *args, struct mm_struct **iter) ++{ ++ bool last = true; ++ struct mm_struct *mm = NULL; ++ int nid = args->node_id; ++ struct lru_gen_mm_list *mm_list = get_mm_list(args->memcg); ++ ++ if (*iter) ++ mmput_async(*iter); ++ else if (args->max_seq <= READ_ONCE(mm_list->nodes[nid].cur_seq)) ++ return false; ++ ++ spin_lock(&mm_list->lock); ++ ++ VM_BUG_ON(args->max_seq > mm_list->nodes[nid].cur_seq + 1); ++ VM_BUG_ON(*iter && args->max_seq < mm_list->nodes[nid].cur_seq); ++ VM_BUG_ON(*iter && !mm_list->nodes[nid].nr_workers); ++ ++ if (args->max_seq <= mm_list->nodes[nid].cur_seq) { ++ last = *iter; ++ goto done; ++ } ++ ++ if (mm_list->nodes[nid].iter == &mm_list->head) { ++ VM_BUG_ON(*iter || mm_list->nodes[nid].nr_workers); ++ mm_list->nodes[nid].iter = mm_list->nodes[nid].iter->next; ++ } ++ ++ while (!mm && mm_list->nodes[nid].iter != &mm_list->head) { ++ mm = list_entry(mm_list->nodes[nid].iter, struct mm_struct, lrugen.list); ++ mm_list->nodes[nid].iter = mm_list->nodes[nid].iter->next; ++ if (should_skip_mm(mm, args)) ++ mm = NULL; ++ ++ args->mm_stats[mm ? MM_SCHED_ACTIVE : MM_SCHED_INACTIVE]++; ++ } ++ ++ if (mm_list->nodes[nid].iter == &mm_list->head) ++ WRITE_ONCE(mm_list->nodes[nid].cur_seq, ++ mm_list->nodes[nid].cur_seq + 1); ++done: ++ if (*iter && !mm) ++ mm_list->nodes[nid].nr_workers--; ++ if (!*iter && mm) ++ mm_list->nodes[nid].nr_workers++; ++ ++ last = last && !mm_list->nodes[nid].nr_workers && ++ mm_list->nodes[nid].iter == &mm_list->head; ++ ++ reset_mm_stats(mm_list, last, args); ++ ++ spin_unlock(&mm_list->lock); ++ ++ *iter = mm; ++ if (mm) ++ node_clear(nid, mm->lrugen.nodes); ++ ++ return last; ++} ++ ++/****************************************************************************** + * state change + ******************************************************************************/ + +@@ -2940,6 +3257,13 @@ static int __init init_lru_gen(void) + { + BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); + BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); ++ BUILD_BUG_ON(sizeof(MM_STAT_CODES) != NR_MM_STATS + 1); ++ ++ if (mem_cgroup_disabled()) { ++ global_mm_list = alloc_mm_list(); ++ if (WARN_ON_ONCE(!global_mm_list)) ++ return -ENOMEM; ++ } + + if (hotplug_memory_notifier(lru_gen_online_mem, 0)) + pr_err("lru_gen: failed to subscribe hotplug notifications\n"); diff --git a/target/linux/generic/backport-5.4/020-22-BACKPORT-FROMLIST-mm-multigenerational-lru-aging.patch b/target/linux/generic/backport-5.4/020-22-BACKPORT-FROMLIST-mm-multigenerational-lru-aging.patch new file mode 100644 index 0000000000..170781896c --- /dev/null +++ b/target/linux/generic/backport-5.4/020-22-BACKPORT-FROMLIST-mm-multigenerational-lru-aging.patch @@ -0,0 +1,952 @@ +From cfa8da4c6e850fcbd7202b4257a5f7e1cb608328 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Mon, 5 Apr 2021 04:35:07 -0600 +Subject: [PATCH] BACKPORT: FROMLIST: mm: multigenerational lru: aging + +The aging produces young generations. Given an lruvec, the aging scans +page tables for referenced pages of this lruvec. Upon finding one, the +aging updates its generation number to max_seq. After each round of +scan, the aging increments max_seq. The aging is due when both of +min_seq[2] reaches max_seq-1, assuming both anon and file types are +reclaimable. + +The aging uses the following optimizations when scanning page tables: + 1) It will not scan page tables from processes that have been + sleeping since the last scan. + 2) It will not scan PTE tables under non-leaf PMD entries that do + not have the accessed bit set, when + CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG=y. + 3) It will not zigzag between the PGD table and the same PMD or PTE + table spanning multiple VMAs. In other words, it finishes all the + VMAs within the range of the same PMD or PTE table before it returns + to the PGD table. This optimizes workloads that have large numbers + of tiny VMAs, especially when CONFIG_PGTABLE_LEVELS=5. + +The aging also takes advantage of the spatial locality: pages mapped +around a referenced PTE may also have been referenced. If the rmap +finds the PTE mapping a page under reclaim referenced, it will call a +new function lru_gen_scan_around() to scan the vicinity of this PTE. +And for each additional PTE found referenced, lru_gen_scan_around() +will update the generation number of the page mapped by this PTE. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432185/) + +BUG=b:123039911 +TEST=Built + +Change-Id: I3ae8abc3100d023cecb3a699d86020ae6fc10a45 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2987189 +Reviewed-by: Yu Zhao +Tested-by: Yu Zhao +Commit-Queue: Sonny Rao +Commit-Queue: Yu Zhao +--- + include/linux/mmzone.h | 6 + + include/linux/pagewalk.h | 2 + + mm/pagewalk.c | 7 +- + mm/rmap.c | 6 + + mm/vmscan.c | 791 +++++++++++++++++++++++++++++++++++++++ + 5 files changed, 811 insertions(+), 1 deletion(-) + +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -296,6 +296,7 @@ struct zone_reclaim_stat { + }; + + struct lruvec; ++struct page_vma_mapped_walk; + + #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) + #define LRU_USAGE_MASK ((BIT(LRU_USAGE_WIDTH) - 1) << LRU_USAGE_PGOFF) +@@ -386,6 +387,7 @@ void lru_gen_init_lruvec(struct lruvec * + void lru_gen_set_state(bool enable, bool main, bool swap); + void *lru_gen_eviction(struct page *page); + void lru_gen_refault(struct page *page, void *shadow); ++void lru_gen_scan_around(struct page_vma_mapped_walk *pvmw); + + #else /* CONFIG_LRU_GEN */ + +@@ -406,6 +408,10 @@ static inline void lru_gen_refault(struc + { + } + ++static inline void lru_gen_scan_around(struct page_vma_mapped_walk *pvmw) ++{ ++} ++ + #endif /* CONFIG_LRU_GEN */ + + struct lruvec { +--- a/include/linux/pagewalk.h ++++ b/include/linux/pagewalk.h +@@ -26,6 +26,8 @@ struct mm_walk; + * right now" and returning 1 means "skip the current vma" + */ + struct mm_walk_ops { ++ int (*p4d_entry)(p4d_t *p4d, unsigned long addr, ++ unsigned long next, struct mm_walk *walk); + int (*pud_entry)(pud_t *pud, unsigned long addr, + unsigned long next, struct mm_walk *walk); + int (*pmd_entry)(pmd_t *pmd, unsigned long addr, +--- a/mm/pagewalk.c ++++ b/mm/pagewalk.c +@@ -135,6 +135,11 @@ static int walk_p4d_range(pgd_t *pgd, un + break; + continue; + } ++ if (ops->p4d_entry) { ++ err = ops->p4d_entry(p4d, addr, next, walk); ++ if (err) ++ break; ++ } + if (ops->pmd_entry || ops->pte_entry) + err = walk_pud_range(p4d, addr, next, walk); + if (err) +@@ -162,7 +167,7 @@ static int walk_pgd_range(unsigned long + break; + continue; + } +- if (ops->pmd_entry || ops->pte_entry) ++ if (ops->p4d_entry || ops->pmd_entry || ops->pte_entry) + err = walk_p4d_range(pgd, addr, next, walk); + if (err) + break; +--- a/mm/rmap.c ++++ b/mm/rmap.c +@@ -66,6 +66,7 @@ + #include + #include + #include ++#include + + #include + +@@ -772,6 +773,11 @@ static bool page_referenced_one(struct p + } + + if (pvmw.pte) { ++ /* the multigenerational lru exploits the spatial locality */ ++ if (lru_gen_enabled() && pte_young(*pvmw.pte)) { ++ lru_gen_scan_around(&pvmw); ++ referenced++; ++ } + if (ptep_clear_flush_young_notify(vma, address, + pvmw.pte)) { + /* +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -52,6 +52,8 @@ + #include + #include + #include ++#include ++#include + + #include + #include +@@ -3030,6 +3032,792 @@ done: + } + + /****************************************************************************** ++ * the aging ++ ******************************************************************************/ ++ ++static void update_batch_size(struct page *page, int old_gen, int new_gen, ++ struct mm_walk_args *args) ++{ ++ int type = page_is_file_cache(page); ++ int zone = page_zonenum(page); ++ int delta = hpage_nr_pages(page); ++ ++ VM_BUG_ON(old_gen >= MAX_NR_GENS); ++ VM_BUG_ON(new_gen >= MAX_NR_GENS); ++ ++ args->batch_size++; ++ ++ args->nr_pages[old_gen][type][zone] -= delta; ++ args->nr_pages[new_gen][type][zone] += delta; ++} ++ ++static void reset_batch_size(struct lruvec *lruvec, struct mm_walk_args *args) ++{ ++ int gen, type, zone; ++ struct lrugen *lrugen = &lruvec->evictable; ++ struct pglist_data *pgdat = lruvec_pgdat(lruvec); ++ ++ if (!args->batch_size) ++ return; ++ ++ args->batch_size = 0; ++ ++ spin_lock_irq(&pgdat->lru_lock); ++ ++ for_each_gen_type_zone(gen, type, zone) { ++ enum lru_list lru = type * LRU_FILE; ++ int total = args->nr_pages[gen][type][zone]; ++ ++ if (!total) ++ continue; ++ ++ args->nr_pages[gen][type][zone] = 0; ++ WRITE_ONCE(lrugen->sizes[gen][type][zone], ++ lrugen->sizes[gen][type][zone] + total); ++ ++ if (lru_gen_is_active(lruvec, gen)) ++ lru += LRU_ACTIVE; ++ update_lru_size(lruvec, lru, zone, total); ++ } ++ ++ spin_unlock_irq(&pgdat->lru_lock); ++} ++ ++static int page_update_gen(struct page *page, int new_gen) ++{ ++ int old_gen; ++ unsigned long old_flags, new_flags; ++ ++ VM_BUG_ON(new_gen >= MAX_NR_GENS); ++ ++ do { ++ old_flags = READ_ONCE(page->flags); ++ ++ old_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++ if (old_gen < 0) { ++ new_flags = old_flags | BIT(PG_referenced); ++ continue; ++ } ++ ++ new_flags = (old_flags & ~(LRU_GEN_MASK | LRU_USAGE_MASK | LRU_TIER_FLAGS)) | ++ ((new_gen + 1UL) << LRU_GEN_PGOFF); ++ } while (new_flags != old_flags && ++ cmpxchg(&page->flags, old_flags, new_flags) != old_flags); ++ ++ return old_gen; ++} ++ ++static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *walk) ++{ ++ struct address_space *mapping; ++ struct vm_area_struct *vma = walk->vma; ++ struct mm_walk_args *args = walk->private; ++ ++ if (is_vm_hugetlb_page(vma) || (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) || ++ !(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) ++ return true; ++ ++ if (vma_is_anonymous(vma)) ++ return !args->swappiness; ++ ++ if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) ++ return true; ++ ++ mapping = vma->vm_file->f_mapping; ++ if (!mapping->a_ops->writepage) ++ return true; ++ ++ return (shmem_mapping(mapping) && !args->swappiness) || mapping_unevictable(mapping); ++} ++ ++/* ++ * Some userspace memory allocators create many single-page VMAs. So instead of ++ * returning back to the PGD table for each of such VMAs, we finish at least an ++ * entire PMD table and therefore avoid many zigzags. This optimizes page table ++ * walks for workloads that have large numbers of tiny VMAs. ++ * ++ * We scan PMD tables in two passes. The first pass reaches to PTE tables and ++ * doesn't take the PMD lock. The second pass clears the accessed bit on PMD ++ * entries and needs to take the PMD lock. The second pass is only done on the ++ * PMD entries that first pass has found the accessed bit is set, namely ++ * 1) leaf entries mapping huge pages from the node under reclaim, and ++ * 2) non-leaf entries whose leaf entries only map pages from the node under ++ * reclaim, when CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG=y. ++ */ ++static bool get_next_vma(struct mm_walk *walk, unsigned long mask, unsigned long size, ++ unsigned long *start, unsigned long *end) ++{ ++ unsigned long next = round_up(*end, size); ++ struct mm_walk_args *args = walk->private; ++ ++ VM_BUG_ON(mask & size); ++ VM_BUG_ON(*start >= *end); ++ VM_BUG_ON((next & mask) != (*start & mask)); ++ ++ while (walk->vma) { ++ if (next >= walk->vma->vm_end) { ++ walk->vma = walk->vma->vm_next; ++ continue; ++ } ++ ++ if ((next & mask) != (walk->vma->vm_start & mask)) ++ return false; ++ ++ if (should_skip_vma(walk->vma->vm_start, walk->vma->vm_end, walk)) { ++ walk->vma = walk->vma->vm_next; ++ continue; ++ } ++ ++ *start = max(next, walk->vma->vm_start); ++ next = (next | ~mask) + 1; ++ /* rounded-up boundaries can wrap to 0 */ ++ *end = next && next < walk->vma->vm_end ? next : walk->vma->vm_end; ++ ++ args->mm_stats[MM_VMA_INTERVAL]++; ++ ++ return true; ++ } ++ ++ return false; ++} ++ ++static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, ++ struct mm_walk *walk) ++{ ++ int i; ++ pte_t *pte; ++ spinlock_t *ptl; ++ unsigned long addr; ++ int remote = 0; ++ struct mm_walk_args *args = walk->private; ++ int old_gen, new_gen = lru_gen_from_seq(args->max_seq); ++ ++ VM_BUG_ON(pmd_trans_huge(*pmd) || pmd_devmap(*pmd)); ++ ++ pte = pte_offset_map_lock(walk->mm, pmd, start & PMD_MASK, &ptl); ++ arch_enter_lazy_mmu_mode(); ++restart: ++ i = (start >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); ++ for (addr = start; addr != end; i++, addr += PAGE_SIZE) { ++ struct page *page; ++ unsigned long pfn = pte_pfn(pte[i]); ++ ++ if (!pte_present(pte[i]) || is_zero_pfn(pfn)) { ++ args->mm_stats[MM_LEAF_HOLE]++; ++ continue; ++ } ++ ++ if (WARN_ON_ONCE(pte_devmap(pte[i]) || pte_special(pte[i]))) ++ continue; ++ ++ if (!pte_young(pte[i])) { ++ args->mm_stats[MM_LEAF_OLD]++; ++ continue; ++ } ++ ++ VM_BUG_ON(!pfn_valid(pfn)); ++ if (pfn < args->start_pfn || pfn >= args->end_pfn) { ++ args->mm_stats[MM_LEAF_OTHER_NODE]++; ++ remote++; ++ continue; ++ } ++ ++ page = compound_head(pfn_to_page(pfn)); ++ if (page_to_nid(page) != args->node_id) { ++ args->mm_stats[MM_LEAF_OTHER_NODE]++; ++ remote++; ++ continue; ++ } ++ ++ if (page_memcg_rcu(page) != args->memcg) { ++ args->mm_stats[MM_LEAF_OTHER_MEMCG]++; ++ continue; ++ } ++ ++ VM_BUG_ON(addr < walk->vma->vm_start || addr >= walk->vma->vm_end); ++ if (!ptep_test_and_clear_young(walk->vma, addr, pte + i)) ++ continue; ++ ++ if (pte_dirty(pte[i]) && !PageDirty(page) && ++ !(PageAnon(page) && PageSwapBacked(page) && !PageSwapCache(page))) { ++ set_page_dirty(page); ++ args->mm_stats[MM_LEAF_DIRTY]++; ++ } ++ ++ old_gen = page_update_gen(page, new_gen); ++ if (old_gen >= 0 && old_gen != new_gen) ++ update_batch_size(page, old_gen, new_gen, args); ++ args->mm_stats[MM_LEAF_YOUNG]++; ++ } ++ ++ if (i < PTRS_PER_PTE && get_next_vma(walk, PMD_MASK, PAGE_SIZE, &start, &end)) ++ goto restart; ++ ++ arch_leave_lazy_mmu_mode(); ++ pte_unmap_unlock(pte, ptl); ++ ++ return IS_ENABLED(CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG) && !remote; ++} ++ ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG) ++static void __walk_pmd_range(pud_t *pud, unsigned long start, ++ struct vm_area_struct *vma, struct mm_walk *walk) ++{ ++ int i; ++ pmd_t *pmd; ++ spinlock_t *ptl; ++ struct mm_walk_args *args = walk->private; ++ int old_gen, new_gen = lru_gen_from_seq(args->max_seq); ++ ++ VM_BUG_ON(pud_trans_huge(*pud) || pud_devmap(*pud)); ++ ++ start &= PUD_MASK; ++ pmd = pmd_offset(pud, start); ++ ptl = pmd_lock(walk->mm, pmd); ++ arch_enter_lazy_mmu_mode(); ++ ++ for_each_set_bit(i, args->bitmap, PTRS_PER_PMD) { ++ struct page *page; ++ unsigned long pfn = pmd_pfn(pmd[i]); ++ unsigned long addr = start + i * PMD_SIZE; ++ ++ if (!pmd_present(pmd[i]) || is_huge_zero_pmd(pmd[i])) { ++ args->mm_stats[MM_LEAF_HOLE]++; ++ continue; ++ } ++ ++ if (WARN_ON_ONCE(pmd_devmap(pmd[i]))) ++ continue; ++ ++ if (!pmd_young(pmd[i])) { ++ args->mm_stats[MM_LEAF_OLD]++; ++ continue; ++ } ++ ++ if (!pmd_trans_huge(pmd[i])) { ++ if (IS_ENABLED(CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG) && ++ pmdp_test_and_clear_young(vma, addr, pmd + i)) ++ args->mm_stats[MM_NONLEAF_YOUNG]++; ++ continue; ++ } ++ ++ VM_BUG_ON(!pfn_valid(pfn)); ++ if (pfn < args->start_pfn || pfn >= args->end_pfn) { ++ args->mm_stats[MM_LEAF_OTHER_NODE]++; ++ continue; ++ } ++ ++ page = pfn_to_page(pfn); ++ VM_BUG_ON_PAGE(PageTail(page), page); ++ if (page_to_nid(page) != args->node_id) { ++ args->mm_stats[MM_LEAF_OTHER_NODE]++; ++ continue; ++ } ++ ++ if (page_memcg_rcu(page) != args->memcg) { ++ args->mm_stats[MM_LEAF_OTHER_MEMCG]++; ++ continue; ++ } ++ ++ VM_BUG_ON(addr < vma->vm_start || addr >= vma->vm_end); ++ if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) ++ continue; ++ ++ if (pmd_dirty(pmd[i]) && !PageDirty(page) && ++ !(PageAnon(page) && PageSwapBacked(page) && !PageSwapCache(page))) { ++ set_page_dirty(page); ++ args->mm_stats[MM_LEAF_DIRTY]++; ++ } ++ ++ old_gen = page_update_gen(page, new_gen); ++ if (old_gen >= 0 && old_gen != new_gen) ++ update_batch_size(page, old_gen, new_gen, args); ++ args->mm_stats[MM_LEAF_YOUNG]++; ++ } ++ ++ arch_leave_lazy_mmu_mode(); ++ spin_unlock(ptl); ++ ++ bitmap_zero(args->bitmap, PTRS_PER_PMD); ++} ++#else ++static void __walk_pmd_range(pud_t *pud, unsigned long start, ++ struct vm_area_struct *vma, struct mm_walk *walk) ++{ ++} ++#endif ++ ++static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, ++ struct mm_walk *walk) ++{ ++ int i; ++ pmd_t *pmd; ++ unsigned long next; ++ unsigned long addr; ++ struct vm_area_struct *vma; ++ int leaf = 0; ++ int nonleaf = 0; ++ struct mm_walk_args *args = walk->private; ++ ++ VM_BUG_ON(pud_trans_huge(*pud) || pud_devmap(*pud)); ++ ++ pmd = pmd_offset(pud, start & PUD_MASK); ++restart: ++ vma = walk->vma; ++ i = (start >> PMD_SHIFT) & (PTRS_PER_PMD - 1); ++ for (addr = start; addr != end; i++, addr = next) { ++ pmd_t val = pmd_read_atomic(pmd + i); ++ ++ /* for pmd_read_atomic() */ ++ barrier(); ++ ++ next = pmd_addr_end(addr, end); ++ ++ if (!pmd_present(val)) { ++ args->mm_stats[MM_LEAF_HOLE]++; ++ continue; ++ } ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ if (pmd_trans_huge(val)) { ++ unsigned long pfn = pmd_pfn(val); ++ ++ if (is_huge_zero_pmd(val)) { ++ args->mm_stats[MM_LEAF_HOLE]++; ++ continue; ++ } ++ ++ if (!pmd_young(val)) { ++ args->mm_stats[MM_LEAF_OLD]++; ++ continue; ++ } ++ ++ if (pfn < args->start_pfn || pfn >= args->end_pfn) { ++ args->mm_stats[MM_LEAF_OTHER_NODE]++; ++ continue; ++ } ++ ++ __set_bit(i, args->bitmap); ++ leaf++; ++ continue; ++ } ++#endif ++ ++#ifdef CONFIG_HAVE_ARCH_PARENT_PMD_YOUNG ++ if (!pmd_young(val)) { ++ args->mm_stats[MM_NONLEAF_OLD]++; ++ continue; ++ } ++#endif ++ if (walk_pte_range(&val, addr, next, walk)) { ++ __set_bit(i, args->bitmap); ++ nonleaf++; ++ } ++ } ++ ++ if (leaf) { ++ __walk_pmd_range(pud, start, vma, walk); ++ leaf = nonleaf = 0; ++ } ++ ++ if (i < PTRS_PER_PMD && get_next_vma(walk, PUD_MASK, PMD_SIZE, &start, &end)) ++ goto restart; ++ ++ if (nonleaf) ++ __walk_pmd_range(pud, start, vma, walk); ++} ++ ++static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, ++ struct mm_walk *walk) ++{ ++ int i; ++ pud_t *pud; ++ unsigned long addr; ++ unsigned long next; ++ struct mm_walk_args *args = walk->private; ++ ++ pud = pud_offset(p4d, start & P4D_MASK); ++restart: ++ i = (start >> PUD_SHIFT) & (PTRS_PER_PUD - 1); ++ for (addr = start; addr != end; i++, addr = next) { ++ pud_t val = READ_ONCE(pud[i]); ++ ++ next = pud_addr_end(addr, end); ++ ++ if (!pud_present(val) || WARN_ON_ONCE(pud_trans_huge(val) || pud_devmap(val))) ++ continue; ++ ++ walk_pmd_range(&val, addr, next, walk); ++ ++ if (args->batch_size >= MAX_BATCH_SIZE) { ++ end = (addr | ~PUD_MASK) + 1; ++ goto done; ++ } ++ } ++ ++ if (i < PTRS_PER_PUD && get_next_vma(walk, P4D_MASK, PUD_SIZE, &start, &end)) ++ goto restart; ++ ++ end = round_up(end, P4D_SIZE); ++done: ++ /* rounded-up boundaries can wrap to 0 */ ++ args->next_addr = end && walk->vma ? max(end, walk->vma->vm_start) : 0; ++ ++ return -EAGAIN; ++} ++ ++static void walk_mm(struct mm_walk_args *args, struct mm_struct *mm) ++{ ++ static const struct mm_walk_ops mm_walk_ops = { ++ .test_walk = should_skip_vma, ++ .p4d_entry = walk_pud_range, ++ }; ++ ++ int err; ++ struct mem_cgroup *memcg = args->memcg; ++ struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(args->node_id), memcg); ++ ++ args->next_addr = FIRST_USER_ADDRESS; ++ ++ do { ++ unsigned long start = args->next_addr; ++ unsigned long end = mm->highest_vm_end; ++ ++ err = -EBUSY; ++ ++ rcu_read_lock(); ++#ifdef CONFIG_MEMCG ++ if (memcg && atomic_read(&memcg->moving_account)) { ++ args->mm_stats[MM_LOCK_CONTENTION]++; ++ goto contended; ++ } ++#endif ++ if (!down_read_trylock(&mm->mmap_sem)) { ++ args->mm_stats[MM_LOCK_CONTENTION]++; ++ goto contended; ++ } ++ ++ err = walk_page_range(mm, start, end, &mm_walk_ops, args); ++ ++ up_read(&mm->mmap_sem); ++ ++ reset_batch_size(lruvec, args); ++contended: ++ rcu_read_unlock(); ++ ++ cond_resched(); ++ } while (err == -EAGAIN && args->next_addr && ++ !mm_is_oom_victim(mm) && !mm_has_migrated(mm, memcg)); ++} ++ ++static void page_inc_gen(struct page *page, struct lruvec *lruvec, bool front) ++{ ++ int old_gen, new_gen; ++ unsigned long old_flags, new_flags; ++ int type = page_is_file_cache(page); ++ int zone = page_zonenum(page); ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ old_gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ ++ do { ++ old_flags = READ_ONCE(page->flags); ++ ++ /* in case the aging has updated old_gen */ ++ new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++ VM_BUG_ON_PAGE(new_gen < 0, page); ++ if (new_gen >= 0 && new_gen != old_gen) ++ goto sort; ++ ++ new_gen = (old_gen + 1) % MAX_NR_GENS; ++ ++ new_flags = (old_flags & ~(LRU_GEN_MASK | LRU_USAGE_MASK | LRU_TIER_FLAGS)) | ++ ((new_gen + 1UL) << LRU_GEN_PGOFF); ++ /* mark the page for reclaim if it's pending writeback */ ++ if (front) ++ new_flags |= BIT(PG_reclaim); ++ } while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags); ++ ++ lru_gen_update_size(page, lruvec, old_gen, new_gen); ++sort: ++ if (front) ++ list_move(&page->lru, &lrugen->lists[new_gen][type][zone]); ++ else ++ list_move_tail(&page->lru, &lrugen->lists[new_gen][type][zone]); ++} ++ ++static bool try_inc_min_seq(struct lruvec *lruvec, int type) ++{ ++ int gen, zone; ++ bool success = false; ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ VM_BUG_ON(!seq_is_valid(lruvec)); ++ ++ while (get_nr_gens(lruvec, type) > MIN_NR_GENS) { ++ gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) { ++ if (!list_empty(&lrugen->lists[gen][type][zone])) ++ return success; ++ } ++ ++ reset_controller_pos(lruvec, gen, type); ++ WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); ++ ++ success = true; ++ } ++ ++ return success; ++} ++ ++static bool inc_min_seq(struct lruvec *lruvec, int type) ++{ ++ int gen, zone; ++ int batch_size = 0; ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ VM_BUG_ON(!seq_is_valid(lruvec)); ++ ++ if (get_nr_gens(lruvec, type) != MAX_NR_GENS) ++ return true; ++ ++ gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) { ++ struct list_head *head = &lrugen->lists[gen][type][zone]; ++ ++ while (!list_empty(head)) { ++ struct page *page = lru_to_page(head); ++ ++ VM_BUG_ON_PAGE(PageTail(page), page); ++ VM_BUG_ON_PAGE(PageUnevictable(page), page); ++ VM_BUG_ON_PAGE(PageActive(page), page); ++ VM_BUG_ON_PAGE(page_is_file_cache(page) != type, page); ++ VM_BUG_ON_PAGE(page_zonenum(page) != zone, page); ++ ++ prefetchw_prev_lru_page(page, head, flags); ++ ++ page_inc_gen(page, lruvec, false); ++ ++ if (++batch_size == MAX_BATCH_SIZE) ++ return false; ++ } ++ ++ VM_BUG_ON(lrugen->sizes[gen][type][zone]); ++ } ++ ++ reset_controller_pos(lruvec, gen, type); ++ WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); ++ ++ return true; ++} ++ ++static void inc_max_seq(struct lruvec *lruvec, unsigned long max_seq) ++{ ++ int gen, type, zone; ++ struct lrugen *lrugen = &lruvec->evictable; ++ struct pglist_data *pgdat = lruvec_pgdat(lruvec); ++ ++ spin_lock_irq(&pgdat->lru_lock); ++ ++ VM_BUG_ON(!seq_is_valid(lruvec)); ++ ++ if (lrugen->max_seq != max_seq) ++ goto unlock; ++ ++ for (type = 0; type < ANON_AND_FILE; type++) { ++ if (try_inc_min_seq(lruvec, type)) ++ continue; ++ ++ while (!inc_min_seq(lruvec, type)) { ++ spin_unlock_irq(&pgdat->lru_lock); ++ cond_resched(); ++ spin_lock_irq(&pgdat->lru_lock); ++ } ++ } ++ ++ gen = lru_gen_from_seq(lrugen->max_seq - 1); ++ for_each_type_zone(type, zone) { ++ enum lru_list lru = type * LRU_FILE; ++ long total = lrugen->sizes[gen][type][zone]; ++ ++ if (!total) ++ continue; ++ ++ WARN_ON_ONCE(total != (int)total); ++ ++ update_lru_size(lruvec, lru, zone, total); ++ update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -total); ++ } ++ ++ gen = lru_gen_from_seq(lrugen->max_seq + 1); ++ for_each_type_zone(type, zone) { ++ VM_BUG_ON(lrugen->sizes[gen][type][zone]); ++ VM_BUG_ON(!list_empty(&lrugen->lists[gen][type][zone])); ++ } ++ ++ for (type = 0; type < ANON_AND_FILE; type++) ++ reset_controller_pos(lruvec, gen, type); ++ ++ WRITE_ONCE(lrugen->timestamps[gen], jiffies); ++ /* make sure all preceding modifications appear first */ ++ smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); ++unlock: ++ spin_unlock_irq(&pgdat->lru_lock); ++} ++ ++/* Main function used by the foreground, the background and the user-triggered aging. */ ++static bool walk_mm_list(struct lruvec *lruvec, unsigned long max_seq, ++ struct scan_control *sc, int swappiness, struct mm_walk_args *args) ++{ ++ bool last; ++ bool alloc = !args; ++ struct mm_struct *mm = NULL; ++ struct lrugen *lrugen = &lruvec->evictable; ++ struct pglist_data *pgdat = lruvec_pgdat(lruvec); ++ int nid = pgdat->node_id; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); ++ ++ VM_BUG_ON(max_seq > READ_ONCE(lrugen->max_seq)); ++ ++ if (alloc) { ++ args = kvzalloc_node(size_of_mm_walk_args(), GFP_KERNEL, nid); ++ if (WARN_ON_ONCE(!args)) ++ return false; ++ } ++ ++ args->memcg = memcg; ++ args->max_seq = max_seq; ++ args->start_pfn = pgdat->node_start_pfn; ++ args->end_pfn = pgdat_end_pfn(pgdat); ++ args->node_id = nid; ++ args->swappiness = swappiness; ++ ++ do { ++ last = get_next_mm(args, &mm); ++ if (mm) { ++ walk_mm(args, mm); ++ } ++ ++ cond_resched(); ++ } while (mm); ++ ++ if (alloc) ++ kvfree(args); ++ ++ if (!last) { ++ /* the foreground aging prefers not to wait */ ++ if (!current_is_kswapd() && sc->priority < DEF_PRIORITY - 2) ++ wait_event_killable(mm_list->nodes[nid].wait, ++ max_seq < READ_ONCE(lrugen->max_seq)); ++ ++ return max_seq < READ_ONCE(lrugen->max_seq); ++ } ++ ++ VM_BUG_ON(max_seq != READ_ONCE(lrugen->max_seq)); ++ ++ inc_max_seq(lruvec, max_seq); ++ /* either we see any waiters or they will see updated max_seq */ ++ if (wq_has_sleeper(&mm_list->nodes[nid].wait)) ++ wake_up_all(&mm_list->nodes[nid].wait); ++ ++ wakeup_flusher_threads(WB_REASON_VMSCAN); ++ ++ return true; ++} ++ ++void lru_gen_scan_around(struct page_vma_mapped_walk *pvmw) ++{ ++ int i; ++ pte_t *pte; ++ int old_gen, new_gen; ++ unsigned long start; ++ unsigned long end; ++ unsigned long addr; ++ struct lruvec *lruvec; ++ struct mem_cgroup *memcg; ++ struct pglist_data *pgdat = page_pgdat(pvmw->page); ++ unsigned long bitmap[BITS_TO_LONGS(SWAP_CLUSTER_MAX * 2)] = {}; ++ ++ lockdep_assert_held(pvmw->ptl); ++ VM_BUG_ON_PAGE(PageTail(pvmw->page), pvmw->page); ++ ++ start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); ++ end = pmd_addr_end(pvmw->address, pvmw->vma->vm_end); ++ ++ if (end - start > SWAP_CLUSTER_MAX * 2 * PAGE_SIZE) { ++ if (pvmw->address - start < SWAP_CLUSTER_MAX * PAGE_SIZE) ++ end = start + SWAP_CLUSTER_MAX * 2 * PAGE_SIZE; ++ else if (end - pvmw->address < SWAP_CLUSTER_MAX * PAGE_SIZE) ++ start = end - SWAP_CLUSTER_MAX * 2 * PAGE_SIZE; ++ else { ++ start = pvmw->address - SWAP_CLUSTER_MAX * PAGE_SIZE; ++ end = pvmw->address + SWAP_CLUSTER_MAX * PAGE_SIZE; ++ } ++ } ++ ++ pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; ++ ++ arch_enter_lazy_mmu_mode(); ++ ++ memcg = lock_page_memcg(pvmw->page); ++ if (WARN_ON_ONCE(!mem_cgroup_disabled() && !memcg)) ++ goto out; ++ ++ spin_lock_irq(&pgdat->lru_lock); ++ ++ lruvec = mem_cgroup_lruvec(pgdat, memcg); ++ new_gen = lru_gen_from_seq(lruvec->evictable.max_seq); ++ ++ for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { ++ struct page *page; ++ unsigned long pfn = pte_pfn(pte[i]); ++ ++ if (!pte_present(pte[i]) || is_zero_pfn(pfn)) ++ continue; ++ ++ if (WARN_ON_ONCE(pte_devmap(pte[i]) || pte_special(pte[i]))) ++ continue; ++ ++ if (!pte_young(pte[i])) ++ continue; ++ ++ VM_BUG_ON(!pfn_valid(pfn)); ++ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) ++ continue; ++ ++ page = compound_head(pfn_to_page(pfn)); ++ if (page_to_nid(page) != pgdat->node_id) ++ continue; ++ ++ if (page_memcg_rcu(page) != memcg) ++ continue; ++ ++ VM_BUG_ON(addr < pvmw->vma->vm_start || addr >= pvmw->vma->vm_end); ++ if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) ++ continue; ++ ++ if (pte_dirty(pte[i]) && !PageDirty(page) && ++ !(PageAnon(page) && PageSwapBacked(page) && !PageSwapCache(page))) ++ __set_bit(i, bitmap); ++ ++ old_gen = page_update_gen(page, new_gen); ++ if (old_gen >= 0 && old_gen != new_gen) ++ lru_gen_update_size(page, lruvec, old_gen, new_gen); ++ } ++ ++ spin_unlock_irq(&pgdat->lru_lock); ++ unlock_page_memcg(pvmw->page); ++out: ++ arch_leave_lazy_mmu_mode(); ++ ++ for_each_set_bit(i, bitmap, SWAP_CLUSTER_MAX * 2) ++ set_page_dirty(pte_page(pte[i])); ++} ++ ++/****************************************************************************** + * state change + ******************************************************************************/ + +@@ -3259,6 +4047,10 @@ static int __init init_lru_gen(void) + BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); + BUILD_BUG_ON(sizeof(MM_STAT_CODES) != NR_MM_STATS + 1); + ++ VM_BUG_ON(PMD_SIZE / PAGE_SIZE != PTRS_PER_PTE); ++ VM_BUG_ON(PUD_SIZE / PMD_SIZE != PTRS_PER_PMD); ++ VM_BUG_ON(P4D_SIZE / PUD_SIZE != PTRS_PER_PUD); ++ + if (mem_cgroup_disabled()) { + global_mm_list = alloc_mm_list(); + if (WARN_ON_ONCE(!global_mm_list)) diff --git a/target/linux/generic/backport-5.4/020-23-BACKPORT-FROMLIST-mm-multigenerational-lru-eviction.patch b/target/linux/generic/backport-5.4/020-23-BACKPORT-FROMLIST-mm-multigenerational-lru-eviction.patch new file mode 100644 index 0000000000..57b3dd64b9 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-23-BACKPORT-FROMLIST-mm-multigenerational-lru-eviction.patch @@ -0,0 +1,662 @@ +From ebd2497c0f0383fd24e536c5de0f73cbe939c5af Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Mon, 5 Apr 2021 04:38:24 -0600 +Subject: [PATCH] BACKPORT: FROMLIST: mm: multigenerational lru: eviction + +The eviction consumes old generations. Given an lruvec, the eviction +scans the pages on the per-zone lists indexed by either of min_seq[2]. +It first tries to select a type based on the values of min_seq[2]. +When anon and file types are both available from the same generation, +it selects the one that has a lower refault rate. + +During a scan, the eviction sorts pages according to their new +generation numbers, if the aging has found them referenced. It also +moves pages from the tiers that have higher refault rates than tier 0 +to the next generation. When it finds all the per-zone lists of a +selected type are empty, the eviction increments min_seq[2] indexed by +this selected type. + +With the aging and the eviction in place, we can build page reclaim in +a straightforward manner: + 1) In order to reduce the latency, direct reclaim only invokes the + aging when both min_seq[2] reaches max_seq-1; otherwise it invokes + the eviction. + 2) In order to avoid the aging in the direct reclaim path, kswapd + does the background aging. It invokes the aging when either of + min_seq[2] reaches max_seq-1; otherwise it invokes the eviction. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432186/) + +BUG=b:123039911 +TEST=Built + +Change-Id: I64c06d8f2cdb83ac7d56c7e1d07f043483956cac +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2987190 +Reviewed-by: Yu Zhao +Tested-by: Yu Zhao +Commit-Queue: Sonny Rao +Commit-Queue: Yu Zhao +--- + include/linux/mmzone.h | 5 + + mm/vmscan.c | 531 +++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 536 insertions(+) + +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -811,6 +811,8 @@ struct deferred_split { + }; + #endif + ++struct mm_walk_args; ++ + /* + * On NUMA machines, each NUMA node would have a pg_data_t to describe + * it's memory layout. On UMA machines there is a single pglist_data which +@@ -900,6 +902,9 @@ typedef struct pglist_data { + + unsigned long flags; + ++#ifdef CONFIG_LRU_GEN ++ struct mm_walk_args *mm_walk_args; ++#endif + ZONE_PADDING(_pad2_) + + /* Per-node vmstats */ +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -1169,6 +1169,11 @@ static unsigned long shrink_page_list(st + if (!sc->may_unmap && page_mapped(page)) + goto keep_locked; + ++ /* in case the page was found accessed by lru_gen_scan_around() */ ++ if (lru_gen_enabled() && !ignore_references && ++ page_mapped(page) && PageReferenced(page)) ++ goto keep_locked; ++ + may_enter_fs = (sc->gfp_mask & __GFP_FS) || + (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO)); + +@@ -3818,6 +3823,482 @@ out: + } + + /****************************************************************************** ++ * the eviction ++ ******************************************************************************/ ++ ++static bool should_skip_page(struct page *page, struct scan_control *sc) ++{ ++ if (!sc->may_unmap && page_mapped(page)) ++ return true; ++ ++ if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && ++ (PageDirty(page) || (PageAnon(page) && !PageSwapCache(page)))) ++ return true; ++ ++ if (!get_page_unless_zero(page)) ++ return true; ++ ++ return false; ++} ++ ++static bool sort_page(struct page *page, struct lruvec *lruvec, int tier_to_isolate) ++{ ++ bool success; ++ int gen = page_lru_gen(page); ++ int type = page_is_file_cache(page); ++ int zone = page_zonenum(page); ++ int tier = lru_tier_from_usage(page_tier_usage(page)); ++ struct lrugen *lrugen = &lruvec->evictable; ++ ++ VM_BUG_ON_PAGE(gen == -1, page); ++ VM_BUG_ON_PAGE(tier_to_isolate < 0, page); ++ ++ /* a lazy-free page that has been written into? */ ++ if (type && PageDirty(page) && PageAnon(page)) { ++ success = lru_gen_deletion(page, lruvec); ++ VM_BUG_ON_PAGE(!success, page); ++ SetPageSwapBacked(page); ++ add_page_to_lru_list_tail(page, lruvec); ++ return true; ++ } ++ ++ /* page_update_gen() has updated the gen #? */ ++ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { ++ list_move(&page->lru, &lrugen->lists[gen][type][zone]); ++ return true; ++ } ++ ++ /* activate this page if its tier has a higher refault rate */ ++ if (tier_to_isolate < tier) { ++ int hist = hist_from_seq_or_gen(gen); ++ ++ page_inc_gen(page, lruvec, false); ++ WRITE_ONCE(lrugen->activated[hist][type][tier - 1], ++ lrugen->activated[hist][type][tier - 1] + hpage_nr_pages(page)); ++ inc_lruvec_state(lruvec, WORKINGSET_ACTIVATE); ++ return true; ++ } ++ ++ /* mark this page for reclaim if it's pending writeback */ ++ if (PageWriteback(page) || (type && PageDirty(page))) { ++ page_inc_gen(page, lruvec, true); ++ return true; ++ } ++ ++ return false; ++} ++ ++static void isolate_page(struct page *page, struct lruvec *lruvec) ++{ ++ bool success; ++ ++ success = lru_gen_deletion(page, lruvec); ++ VM_BUG_ON_PAGE(!success, page); ++ ++ ClearPageLRU(page); ++ ++ if (PageActive(page)) { ++ ClearPageActive(page); ++ /* make sure shrink_page_list() rejects this page */ ++ SetPageReferenced(page); ++ return; ++ } ++ ++ /* make sure shrink_page_list() doesn't try to write this page */ ++ ClearPageReclaim(page); ++ /* make sure shrink_page_list() doesn't reject this page */ ++ ClearPageReferenced(page); ++} ++ ++static int scan_pages(struct lruvec *lruvec, struct scan_control *sc, long *nr_to_scan, ++ int type, int tier, struct list_head *list) ++{ ++ bool success; ++ int gen, zone; ++ enum vm_event_item item; ++ int sorted = 0; ++ int scanned = 0; ++ int isolated = 0; ++ int batch_size = 0; ++ struct lrugen *lrugen = &lruvec->evictable; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ ++ VM_BUG_ON(!list_empty(list)); ++ ++ if (get_nr_gens(lruvec, type) == MIN_NR_GENS) ++ return -ENOENT; ++ ++ gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ ++ for (zone = sc->reclaim_idx; zone >= 0; zone--) { ++ LIST_HEAD(moved); ++ int skipped = 0; ++ struct list_head *head = &lrugen->lists[gen][type][zone]; ++ ++ while (!list_empty(head)) { ++ struct page *page = lru_to_page(head); ++ int delta = hpage_nr_pages(page); ++ ++ VM_BUG_ON_PAGE(PageTail(page), page); ++ VM_BUG_ON_PAGE(PageUnevictable(page), page); ++ VM_BUG_ON_PAGE(PageActive(page), page); ++ VM_BUG_ON_PAGE(page_is_file_cache(page) != type, page); ++ VM_BUG_ON_PAGE(page_zonenum(page) != zone, page); ++ ++ prefetchw_prev_lru_page(page, head, flags); ++ ++ scanned += delta; ++ ++ if (sort_page(page, lruvec, tier)) ++ sorted += delta; ++ else if (should_skip_page(page, sc)) { ++ list_move(&page->lru, &moved); ++ skipped += delta; ++ } else { ++ isolate_page(page, lruvec); ++ list_add(&page->lru, list); ++ isolated += delta; ++ } ++ ++ if (scanned >= *nr_to_scan || isolated >= SWAP_CLUSTER_MAX || ++ ++batch_size == MAX_BATCH_SIZE) ++ break; ++ } ++ ++ list_splice(&moved, head); ++ __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); ++ ++ if (scanned >= *nr_to_scan || isolated >= SWAP_CLUSTER_MAX || ++ batch_size == MAX_BATCH_SIZE) ++ break; ++ } ++ ++ success = try_inc_min_seq(lruvec, type); ++ ++ item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; ++ if (global_reclaim(sc)) { ++ __count_vm_events(item, isolated); ++ __count_vm_events(PGREFILL, sorted); ++ } ++ __count_memcg_events(memcg, item, isolated); ++ __count_memcg_events(memcg, PGREFILL, sorted); ++ ++ *nr_to_scan -= scanned; ++ ++ if (*nr_to_scan <= 0 || success || isolated) ++ return isolated; ++ /* ++ * We may have trouble finding eligible pages due to reclaim_idx, ++ * may_unmap and may_writepage. The following check makes sure we won't ++ * be stuck if we aren't making enough progress. ++ */ ++ return batch_size == MAX_BATCH_SIZE && sorted >= SWAP_CLUSTER_MAX ? 0 : -ENOENT; ++} ++ ++static int get_tier_to_isolate(struct lruvec *lruvec, int type) ++{ ++ int tier; ++ struct controller_pos sp, pv; ++ ++ /* ++ * Ideally we don't want to evict upper tiers that have higher refault ++ * rates. However, we need to leave a margin for the fluctuations in ++ * refault rates. So we use a larger gain factor to make sure upper ++ * tiers are indeed more active. We choose 2 because the lowest upper ++ * tier would have twice of the refault rate of the base tier, according ++ * to their numbers of accesses. ++ */ ++ read_controller_pos(&sp, lruvec, type, 0, 1); ++ for (tier = 1; tier < MAX_NR_TIERS; tier++) { ++ read_controller_pos(&pv, lruvec, type, tier, 2); ++ if (!positive_ctrl_err(&sp, &pv)) ++ break; ++ } ++ ++ return tier - 1; ++} ++ ++static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_to_isolate) ++{ ++ int type, tier; ++ struct controller_pos sp, pv; ++ int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; ++ ++ /* ++ * Compare the refault rates between the base tiers of anon and file to ++ * determine which type to evict. Also need to compare the refault rates ++ * of the upper tiers of the selected type with that of the base tier of ++ * the other type to determine which tier of the selected type to evict. ++ */ ++ read_controller_pos(&sp, lruvec, 0, 0, gain[0]); ++ read_controller_pos(&pv, lruvec, 1, 0, gain[1]); ++ type = positive_ctrl_err(&sp, &pv); ++ ++ read_controller_pos(&sp, lruvec, !type, 0, gain[!type]); ++ for (tier = 1; tier < MAX_NR_TIERS; tier++) { ++ read_controller_pos(&pv, lruvec, type, tier, gain[type]); ++ if (!positive_ctrl_err(&sp, &pv)) ++ break; ++ } ++ ++ *tier_to_isolate = tier - 1; ++ ++ return type; ++} ++ ++static int isolate_pages(struct lruvec *lruvec, struct scan_control *sc, int swappiness, ++ long *nr_to_scan, int *type_to_scan, struct list_head *list) ++{ ++ int i; ++ int type; ++ int isolated; ++ int tier = -1; ++ DEFINE_MAX_SEQ(); ++ DEFINE_MIN_SEQ(); ++ ++ VM_BUG_ON(!seq_is_valid(lruvec)); ++ ++ if (get_hi_wmark(max_seq, min_seq, swappiness) == MIN_NR_GENS) ++ return 0; ++ /* ++ * Try to select a type based on generations and swappiness, and if that ++ * fails, fall back to get_type_to_scan(). When anon and file are both ++ * available from the same generation, swappiness 200 is interpreted as ++ * anon first and swappiness 1 is interpreted as file first. ++ */ ++ type = !swappiness || min_seq[0] > min_seq[1] || ++ (min_seq[0] == min_seq[1] && swappiness != 200 && ++ (swappiness == 1 || get_type_to_scan(lruvec, swappiness, &tier))); ++ ++ if (tier == -1) ++ tier = get_tier_to_isolate(lruvec, type); ++ ++ for (i = !swappiness; i < ANON_AND_FILE; i++) { ++ isolated = scan_pages(lruvec, sc, nr_to_scan, type, tier, list); ++ if (isolated >= 0) ++ break; ++ ++ type = !type; ++ tier = get_tier_to_isolate(lruvec, type); ++ } ++ ++ if (isolated < 0) ++ isolated = *nr_to_scan = 0; ++ ++ *type_to_scan = type; ++ ++ return isolated; ++} ++ ++/* Main function used by the foreground, the background and the user-triggered eviction. */ ++static bool evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swappiness, ++ long *nr_to_scan) ++{ ++ int type; ++ int isolated; ++ int reclaimed; ++ LIST_HEAD(list); ++ struct page *page; ++ enum vm_event_item item; ++ struct reclaim_stat stat; ++ struct pglist_data *pgdat = lruvec_pgdat(lruvec); ++ ++ spin_lock_irq(&pgdat->lru_lock); ++ ++ isolated = isolate_pages(lruvec, sc, swappiness, nr_to_scan, &type, &list); ++ VM_BUG_ON(list_empty(&list) == !!isolated); ++ ++ if (isolated) ++ __mod_node_page_state(pgdat, NR_ISOLATED_ANON + type, isolated); ++ ++ spin_unlock_irq(&pgdat->lru_lock); ++ ++ if (!isolated) ++ goto done; ++ ++ reclaimed = shrink_page_list(&list, pgdat, sc, 0, &stat, false); ++ /* ++ * We need to prevent rejected pages from being added back to the same ++ * lists they were isolated from. Otherwise we may risk looping on them ++ * forever. We use PageActive() or !PageReferenced() && PageWorkingset() ++ * to tell lru_gen_addition() not to add them to the oldest generation. ++ */ ++ list_for_each_entry(page, &list, lru) { ++ if (PageMlocked(page)) ++ continue; ++ ++ if (page_mapped(page) && PageReferenced(page)) ++ SetPageActive(page); ++ else if (!PageActive(page)) ++ SetPageWorkingset(page); ++ ClearPageReferenced(page); ++ } ++ ++ spin_lock_irq(&pgdat->lru_lock); ++ ++ move_pages_to_lru(lruvec, &list); ++ ++ __mod_node_page_state(pgdat, NR_ISOLATED_ANON + type, -isolated); ++ ++ item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; ++ if (global_reclaim(sc)) ++ __count_vm_events(item, reclaimed); ++ __count_memcg_events(lruvec_memcg(lruvec), item, reclaimed); ++ ++ spin_unlock_irq(&pgdat->lru_lock); ++ ++ mem_cgroup_uncharge_list(&list); ++ free_unref_page_list(&list); ++ ++ sc->nr_reclaimed += reclaimed; ++done: ++ return *nr_to_scan > 0 && sc->nr_reclaimed < sc->nr_to_reclaim; ++} ++ ++/****************************************************************************** ++ * page reclaim ++ ******************************************************************************/ ++ ++static int get_swappiness(struct lruvec *lruvec) ++{ ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ int swappiness = mem_cgroup_get_nr_swap_pages(memcg) >= (long)SWAP_CLUSTER_MAX ? ++ mem_cgroup_swappiness(memcg) : 0; ++ ++ VM_BUG_ON(swappiness > 200U); ++ ++ return swappiness; ++} ++ ++static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, int swappiness) ++{ ++ int gen, type, zone; ++ long nr_to_scan = 0; ++ struct lrugen *lrugen = &lruvec->evictable; ++ DEFINE_MAX_SEQ(); ++ DEFINE_MIN_SEQ(); ++ ++ lru_add_drain(); ++ ++ for (type = !swappiness; type < ANON_AND_FILE; type++) { ++ unsigned long seq; ++ ++ for (seq = min_seq[type]; seq <= max_seq; seq++) { ++ gen = lru_gen_from_seq(seq); ++ ++ for (zone = 0; zone <= sc->reclaim_idx; zone++) ++ nr_to_scan += READ_ONCE(lrugen->sizes[gen][type][zone]); ++ } ++ } ++ ++ nr_to_scan = max(nr_to_scan, 0L); ++ nr_to_scan = round_up(nr_to_scan >> sc->priority, SWAP_CLUSTER_MAX); ++ ++ if (get_hi_wmark(max_seq, min_seq, swappiness) > MIN_NR_GENS) ++ return nr_to_scan; ++ ++ /* kswapd uses lru_gen_age_node() */ ++ if (current_is_kswapd()) ++ return 0; ++ ++ return walk_mm_list(lruvec, max_seq, sc, swappiness, NULL) ? nr_to_scan : 0; ++} ++ ++static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) ++{ ++ struct blk_plug plug; ++ long scanned = 0; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ enum mem_cgroup_protection prot = mem_cgroup_protected(sc->target_mem_cgroup, memcg); ++ ++ blk_start_plug(&plug); ++ ++ while (true) { ++ long nr_to_scan; ++ int swappiness = sc->may_swap ? get_swappiness(lruvec) : 0; ++ ++ nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness) - scanned; ++ if (nr_to_scan < (long)SWAP_CLUSTER_MAX) ++ break; ++ ++ scanned += nr_to_scan; ++ ++ if (!evict_pages(lruvec, sc, swappiness, &nr_to_scan)) ++ break; ++ ++ scanned -= nr_to_scan; ++ ++ if (prot == MEMCG_PROT_MIN || (prot == MEMCG_PROT_LOW && !sc->memcg_low_reclaim)) ++ break; ++ ++ cond_resched(); ++ } ++ ++ blk_finish_plug(&plug); ++} ++ ++/****************************************************************************** ++ * the background aging ++ ******************************************************************************/ ++ ++static int lru_gen_spread = MIN_NR_GENS; ++ ++static void try_walk_mm_list(struct lruvec *lruvec, struct scan_control *sc) ++{ ++ int gen, type, zone; ++ long old_and_young[2] = {}; ++ int spread = READ_ONCE(lru_gen_spread); ++ int swappiness = get_swappiness(lruvec); ++ struct lrugen *lrugen = &lruvec->evictable; ++ struct pglist_data *pgdat = lruvec_pgdat(lruvec); ++ DEFINE_MAX_SEQ(); ++ DEFINE_MIN_SEQ(); ++ ++ lru_add_drain(); ++ ++ for (type = !swappiness; type < ANON_AND_FILE; type++) { ++ unsigned long seq; ++ ++ for (seq = min_seq[type]; seq <= max_seq; seq++) { ++ gen = lru_gen_from_seq(seq); ++ ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) ++ old_and_young[seq == max_seq] += ++ READ_ONCE(lrugen->sizes[gen][type][zone]); ++ } ++ } ++ ++ old_and_young[0] = max(old_and_young[0], 0L); ++ old_and_young[1] = max(old_and_young[1], 0L); ++ ++ /* try to spread pages out across spread+1 generations */ ++ if (old_and_young[0] >= old_and_young[1] * spread && ++ get_lo_wmark(max_seq, min_seq, swappiness) > max(spread, MIN_NR_GENS)) ++ return; ++ ++ walk_mm_list(lruvec, max_seq, sc, swappiness, pgdat->mm_walk_args); ++} ++ ++static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) ++{ ++ struct mem_cgroup *memcg; ++ ++ VM_BUG_ON(!current_is_kswapd()); ++ ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg); ++ enum mem_cgroup_protection prot = mem_cgroup_protected(sc->target_mem_cgroup, ++ memcg); ++ ++ if (prot != MEMCG_PROT_MIN && (prot != MEMCG_PROT_LOW || sc->memcg_low_reclaim)) ++ try_walk_mm_list(lruvec, sc); ++ ++ cond_resched(); ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); ++} ++ ++/****************************************************************************** + * state change + ******************************************************************************/ + +@@ -4020,6 +4501,21 @@ static int __meminit __maybe_unused lru_ + return NOTIFY_DONE; + } + ++static void lru_gen_start_kswapd(int nid) ++{ ++ struct pglist_data *pgdat = NODE_DATA(nid); ++ ++ pgdat->mm_walk_args = kvzalloc_node(size_of_mm_walk_args(), GFP_KERNEL, nid); ++ WARN_ON_ONCE(!pgdat->mm_walk_args); ++} ++ ++static void lru_gen_stop_kswapd(int nid) ++{ ++ struct pglist_data *pgdat = NODE_DATA(nid); ++ ++ kvfree(pgdat->mm_walk_args); ++} ++ + /****************************************************************************** + * initialization + ******************************************************************************/ +@@ -4068,6 +4564,24 @@ static int __init init_lru_gen(void) + */ + arch_initcall(init_lru_gen); + ++#else /* CONFIG_LRU_GEN */ ++ ++static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) ++{ ++} ++ ++static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) ++{ ++} ++ ++static void lru_gen_start_kswapd(int nid) ++{ ++} ++ ++static void lru_gen_stop_kswapd(int nid) ++{ ++} ++ + #endif /* CONFIG_LRU_GEN */ + + /* +@@ -4086,6 +4600,12 @@ static void shrink_node_memcg(struct pgl + struct blk_plug plug; + bool scan_adjusted; + ++ if (lru_gen_enabled()) { ++ *lru_pages = 0; ++ lru_gen_shrink_lruvec(lruvec, sc); ++ return; ++ } ++ + get_scan_count(lruvec, memcg, sc, nr, lru_pages); + + /* Record the original scan target for proportional adjustments later */ +@@ -4560,6 +5080,9 @@ static void snapshot_refaults(struct mem + { + struct mem_cgroup *memcg; + ++ if (lru_gen_enabled()) ++ return; ++ + memcg = mem_cgroup_iter(root_memcg, NULL, NULL); + do { + unsigned long refaults; +@@ -4922,6 +5445,11 @@ static void age_active_anon(struct pglis + { + struct mem_cgroup *memcg; + ++ if (lru_gen_enabled()) { ++ lru_gen_age_node(pgdat, sc); ++ return; ++ } ++ + if (!total_swap_pages) + return; + +@@ -5615,6 +6143,8 @@ int kswapd_run(int nid) + if (pgdat->kswapd) + return 0; + ++ lru_gen_start_kswapd(nid); ++ + pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid); + if (IS_ERR(pgdat->kswapd)) { + /* failure at boot is fatal */ +@@ -5637,6 +6167,7 @@ void kswapd_stop(int nid) + if (kswapd) { + kthread_stop(kswapd); + NODE_DATA(nid)->kswapd = NULL; ++ lru_gen_stop_kswapd(nid); + } + } + diff --git a/target/linux/generic/backport-5.4/020-24-BACKPORT-FROMLIST-mm-multigenerational-lru-user-inte.patch b/target/linux/generic/backport-5.4/020-24-BACKPORT-FROMLIST-mm-multigenerational-lru-user-inte.patch new file mode 100644 index 0000000000..76a37f1b51 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-24-BACKPORT-FROMLIST-mm-multigenerational-lru-user-inte.patch @@ -0,0 +1,485 @@ +From b7969d0fb00b3a717c64c9e355bfaa67978c322d Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Mon, 25 Jan 2021 21:38:02 -0700 +Subject: [PATCH] BACKPORT: FROMLIST: mm: multigenerational lru: user + interface + +Add a sysfs file /sys/kernel/mm/lru_gen/enabled to enable and disable +the multigenerational lru at runtime. + +Add a sysfs file /sys/kernel/mm/lru_gen/spread to optionally spread +pages out across more than three generations. More generations make +the background aging more aggressive. + +Add a debugfs file /sys/kernel/debug/lru_gen to monitor the +multigenerational lru and trigger the aging and the eviction. This +file has the following output: + memcg memcg_id memcg_path + node node_id + min_gen birth_time anon_size file_size + ... + max_gen birth_time anon_size file_size + +Given a memcg and a node, "min_gen" is the oldest generation (number) +and "max_gen" is the youngest. Birth time is in milliseconds. The +sizes of anon and file types are in pages. + +This file takes the following input: + + memcg_id node_id gen [swappiness] + - memcg_id node_id gen [swappiness] [nr_to_reclaim] + +The first command line accounts referenced pages to generation +"max_gen" and creates the next generation "max_gen"+1. In this case, +"gen" should be equal to "max_gen". A swap file and a non-zero +"swappiness" are required to scan anon type. If swapping is not +desired, set vm.swappiness to 0. The second command line evicts +generations less than or equal to "gen". In this case, "gen" should be +less than "max_gen"-1 as "max_gen" and "max_gen"-1 are active +generations and therefore protected from the eviction. Use +"nr_to_reclaim" to limit the number of pages to evict. Multiple +command lines are supported, so does concatenation with delimiters "," +and ";". + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432187/) + +BUG=b:123039911 +TEST=Built + +Change-Id: I4448e60029badbe347aa3b624f429b280cc3a3d3 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2987191 +Reviewed-by: Yu Zhao +Tested-by: Yu Zhao +Commit-Queue: Sonny Rao +Commit-Queue: Yu Zhao +--- + mm/vmscan.c | 403 ++++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 403 insertions(+) + +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -54,6 +54,8 @@ + #include + #include + #include ++#include ++#include + + #include + #include +@@ -4517,6 +4519,401 @@ static void lru_gen_stop_kswapd(int nid) + } + + /****************************************************************************** ++ * sysfs interface ++ ******************************************************************************/ ++ ++static ssize_t show_lru_gen_spread(struct kobject *kobj, struct kobj_attribute *attr, ++ char *buf) ++{ ++ return sprintf(buf, "%d\n", READ_ONCE(lru_gen_spread)); ++} ++ ++static ssize_t store_lru_gen_spread(struct kobject *kobj, struct kobj_attribute *attr, ++ const char *buf, size_t len) ++{ ++ int spread; ++ ++ if (kstrtoint(buf, 10, &spread) || spread >= MAX_NR_GENS) ++ return -EINVAL; ++ ++ WRITE_ONCE(lru_gen_spread, spread); ++ ++ return len; ++} ++ ++static struct kobj_attribute lru_gen_spread_attr = __ATTR( ++ spread, 0644, show_lru_gen_spread, store_lru_gen_spread ++); ++ ++static ssize_t show_lru_gen_enabled(struct kobject *kobj, struct kobj_attribute *attr, ++ char *buf) ++{ ++ return snprintf(buf, PAGE_SIZE, "%d\n", lru_gen_enabled()); ++} ++ ++static ssize_t store_lru_gen_enabled(struct kobject *kobj, struct kobj_attribute *attr, ++ const char *buf, size_t len) ++{ ++ int enable; ++ ++ if (kstrtoint(buf, 10, &enable)) ++ return -EINVAL; ++ ++ lru_gen_set_state(enable, true, false); ++ ++ return len; ++} ++ ++static struct kobj_attribute lru_gen_enabled_attr = __ATTR( ++ enabled, 0644, show_lru_gen_enabled, store_lru_gen_enabled ++); ++ ++static struct attribute *lru_gen_attrs[] = { ++ &lru_gen_spread_attr.attr, ++ &lru_gen_enabled_attr.attr, ++ NULL ++}; ++ ++static struct attribute_group lru_gen_attr_group = { ++ .name = "lru_gen", ++ .attrs = lru_gen_attrs, ++}; ++ ++/****************************************************************************** ++ * debugfs interface ++ ******************************************************************************/ ++ ++static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) ++{ ++ struct mem_cgroup *memcg; ++ loff_t nr_to_skip = *pos; ++ ++ m->private = kzalloc(PATH_MAX, GFP_KERNEL); ++ if (!m->private) ++ return ERR_PTR(-ENOMEM); ++ ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ int nid; ++ ++ for_each_node_state(nid, N_MEMORY) { ++ if (!nr_to_skip--) ++ return mem_cgroup_lruvec(NODE_DATA(nid), memcg); ++ } ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); ++ ++ return NULL; ++} ++ ++static void lru_gen_seq_stop(struct seq_file *m, void *v) ++{ ++ if (!IS_ERR_OR_NULL(v)) ++ mem_cgroup_iter_break(NULL, lruvec_memcg(v)); ++ ++ kfree(m->private); ++ m->private = NULL; ++} ++ ++static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) ++{ ++ int nid = lruvec_pgdat(v)->node_id; ++ struct mem_cgroup *memcg = lruvec_memcg(v); ++ ++ ++*pos; ++ ++ nid = next_memory_node(nid); ++ if (nid == MAX_NUMNODES) { ++ memcg = mem_cgroup_iter(NULL, memcg, NULL); ++ if (!memcg) ++ return NULL; ++ ++ nid = first_memory_node; ++ } ++ ++ return mem_cgroup_lruvec(NODE_DATA(nid), memcg); ++} ++ ++static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, ++ unsigned long max_seq, unsigned long *min_seq, ++ unsigned long seq) ++{ ++ int i; ++ int type, tier; ++ int hist = hist_from_seq_or_gen(seq); ++ struct lrugen *lrugen = &lruvec->evictable; ++ int nid = lruvec_pgdat(lruvec)->node_id; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); ++ ++ for (tier = 0; tier < MAX_NR_TIERS; tier++) { ++ seq_printf(m, " %10d", tier); ++ for (type = 0; type < ANON_AND_FILE; type++) { ++ unsigned long n[3] = {}; ++ ++ if (seq == max_seq) { ++ n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); ++ n[1] = READ_ONCE(lrugen->avg_total[type][tier]); ++ ++ seq_printf(m, " %10luR %10luT %10lu ", n[0], n[1], n[2]); ++ } else if (seq == min_seq[type] || NR_STAT_GENS > 1) { ++ n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); ++ n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); ++ if (tier) ++ n[2] = READ_ONCE(lrugen->activated[hist][type][tier - 1]); ++ ++ seq_printf(m, " %10lur %10lue %10lua", n[0], n[1], n[2]); ++ } else ++ seq_puts(m, " 0 0 0 "); ++ } ++ seq_putc(m, '\n'); ++ } ++ ++ seq_puts(m, " "); ++ for (i = 0; i < NR_MM_STATS; i++) { ++ if (seq == max_seq && NR_STAT_GENS == 1) ++ seq_printf(m, " %10lu%c", READ_ONCE(mm_list->nodes[nid].stats[hist][i]), ++ toupper(MM_STAT_CODES[i])); ++ else if (seq != max_seq && NR_STAT_GENS > 1) ++ seq_printf(m, " %10lu%c", READ_ONCE(mm_list->nodes[nid].stats[hist][i]), ++ MM_STAT_CODES[i]); ++ else ++ seq_puts(m, " 0 "); ++ } ++ seq_putc(m, '\n'); ++} ++ ++static int lru_gen_seq_show(struct seq_file *m, void *v) ++{ ++ unsigned long seq; ++ bool full = !debugfs_real_fops(m->file)->write; ++ struct lruvec *lruvec = v; ++ struct lrugen *lrugen = &lruvec->evictable; ++ int nid = lruvec_pgdat(lruvec)->node_id; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ DEFINE_MAX_SEQ(); ++ DEFINE_MIN_SEQ(); ++ ++ if (nid == first_memory_node) { ++#ifdef CONFIG_MEMCG ++ if (memcg) ++ cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); ++#endif ++ seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), (char *)m->private); ++ } ++ ++ seq_printf(m, " node %5d\n", nid); ++ ++ seq = full ? (max_seq < MAX_NR_GENS ? 0 : max_seq - MAX_NR_GENS + 1) : ++ min(min_seq[0], min_seq[1]); ++ ++ for (; seq <= max_seq; seq++) { ++ int gen, type, zone; ++ unsigned int msecs; ++ ++ gen = lru_gen_from_seq(seq); ++ msecs = jiffies_to_msecs(jiffies - READ_ONCE(lrugen->timestamps[gen])); ++ ++ seq_printf(m, " %10lu %10u", seq, msecs); ++ ++ for (type = 0; type < ANON_AND_FILE; type++) { ++ long size = 0; ++ ++ if (seq < min_seq[type]) { ++ seq_puts(m, " -0 "); ++ continue; ++ } ++ ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) ++ size += READ_ONCE(lrugen->sizes[gen][type][zone]); ++ ++ seq_printf(m, " %10lu ", max(size, 0L)); ++ } ++ ++ seq_putc(m, '\n'); ++ ++ if (full) ++ lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); ++ } ++ ++ return 0; ++} ++ ++static const struct seq_operations lru_gen_seq_ops = { ++ .start = lru_gen_seq_start, ++ .stop = lru_gen_seq_stop, ++ .next = lru_gen_seq_next, ++ .show = lru_gen_seq_show, ++}; ++ ++static int advance_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness) ++{ ++ struct scan_control sc = { ++ .target_mem_cgroup = lruvec_memcg(lruvec), ++ }; ++ DEFINE_MAX_SEQ(); ++ ++ if (seq == max_seq) ++ walk_mm_list(lruvec, max_seq, &sc, swappiness, NULL); ++ ++ return seq > max_seq ? -EINVAL : 0; ++} ++ ++static int advance_min_seq(struct lruvec *lruvec, unsigned long seq, int swappiness, ++ unsigned long nr_to_reclaim) ++{ ++ struct blk_plug plug; ++ int err = -EINTR; ++ long nr_to_scan = LONG_MAX; ++ struct scan_control sc = { ++ .nr_to_reclaim = nr_to_reclaim, ++ .target_mem_cgroup = lruvec_memcg(lruvec), ++ .may_writepage = 1, ++ .may_unmap = 1, ++ .may_swap = 1, ++ .reclaim_idx = MAX_NR_ZONES - 1, ++ .gfp_mask = GFP_KERNEL, ++ }; ++ DEFINE_MAX_SEQ(); ++ ++ if (seq >= max_seq - 1) ++ return -EINVAL; ++ ++ blk_start_plug(&plug); ++ ++ while (!signal_pending(current)) { ++ DEFINE_MIN_SEQ(); ++ ++ if (seq < min(min_seq[!swappiness], min_seq[swappiness < 200]) || ++ !evict_pages(lruvec, &sc, swappiness, &nr_to_scan)) { ++ err = 0; ++ break; ++ } ++ ++ cond_resched(); ++ } ++ ++ blk_finish_plug(&plug); ++ ++ return err; ++} ++ ++static int advance_seq(char cmd, int memcg_id, int nid, unsigned long seq, ++ int swappiness, unsigned long nr_to_reclaim) ++{ ++ struct lruvec *lruvec; ++ int err = -EINVAL; ++ struct mem_cgroup *memcg = NULL; ++ ++ if (!mem_cgroup_disabled()) { ++ rcu_read_lock(); ++ memcg = mem_cgroup_from_id(memcg_id); ++#ifdef CONFIG_MEMCG ++ if (memcg && !css_tryget(&memcg->css)) ++ memcg = NULL; ++#endif ++ rcu_read_unlock(); ++ ++ if (!memcg) ++ goto done; ++ } ++ if (memcg_id != mem_cgroup_id(memcg)) ++ goto done; ++ ++ if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) ++ goto done; ++ ++ lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg); ++ ++ if (swappiness == -1) ++ swappiness = get_swappiness(lruvec); ++ else if (swappiness > 200U) ++ goto done; ++ ++ switch (cmd) { ++ case '+': ++ err = advance_max_seq(lruvec, seq, swappiness); ++ break; ++ case '-': ++ err = advance_min_seq(lruvec, seq, swappiness, nr_to_reclaim); ++ break; ++ } ++done: ++ mem_cgroup_put(memcg); ++ ++ return err; ++} ++ ++static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, ++ size_t len, loff_t *pos) ++{ ++ void *buf; ++ char *cur, *next; ++ int err = 0; ++ ++ buf = kvmalloc(len + 1, GFP_USER); ++ if (!buf) ++ return -ENOMEM; ++ ++ if (copy_from_user(buf, src, len)) { ++ kvfree(buf); ++ return -EFAULT; ++ } ++ ++ next = buf; ++ next[len] = '\0'; ++ ++ while ((cur = strsep(&next, ",;\n"))) { ++ int n; ++ int end; ++ char cmd; ++ unsigned int memcg_id; ++ unsigned int nid; ++ unsigned long seq; ++ unsigned int swappiness = -1; ++ unsigned long nr_to_reclaim = -1; ++ ++ cur = skip_spaces(cur); ++ if (!*cur) ++ continue; ++ ++ n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, ++ &seq, &end, &swappiness, &end, &nr_to_reclaim, &end); ++ if (n < 4 || cur[end]) { ++ err = -EINVAL; ++ break; ++ } ++ ++ err = advance_seq(cmd, memcg_id, nid, seq, swappiness, nr_to_reclaim); ++ if (err) ++ break; ++ } ++ ++ kvfree(buf); ++ ++ return err ? : len; ++} ++ ++static int lru_gen_seq_open(struct inode *inode, struct file *file) ++{ ++ return seq_open(file, &lru_gen_seq_ops); ++} ++ ++static const struct file_operations lru_gen_rw_fops = { ++ .open = lru_gen_seq_open, ++ .read = seq_read, ++ .write = lru_gen_seq_write, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; ++ ++static const struct file_operations lru_gen_ro_fops = { ++ .open = lru_gen_seq_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; ++ ++/****************************************************************************** + * initialization + ******************************************************************************/ + +@@ -4556,6 +4953,12 @@ static int __init init_lru_gen(void) + if (hotplug_memory_notifier(lru_gen_online_mem, 0)) + pr_err("lru_gen: failed to subscribe hotplug notifications\n"); + ++ if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) ++ pr_err("lru_gen: failed to create sysfs group\n"); ++ ++ debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); ++ debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); ++ + return 0; + }; + /* diff --git a/target/linux/generic/backport-5.4/020-25-BACKPORT-FROMLIST-mm-multigenerational-lru-Kconfig.patch b/target/linux/generic/backport-5.4/020-25-BACKPORT-FROMLIST-mm-multigenerational-lru-Kconfig.patch new file mode 100644 index 0000000000..503e8eb423 --- /dev/null +++ b/target/linux/generic/backport-5.4/020-25-BACKPORT-FROMLIST-mm-multigenerational-lru-Kconfig.patch @@ -0,0 +1,90 @@ +From 8c987beaae2bdfa3fb35468fe681a9b57f064fc8 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Mon, 25 Jan 2021 21:47:24 -0700 +Subject: [PATCH] BACKPORT: FROMLIST: mm: multigenerational lru: Kconfig + +Add configuration options for the multigenerational lru. + +Signed-off-by: Yu Zhao +Tested-by: Konstantin Kharlamov +(am from https://lore.kernel.org/patchwork/patch/1432188/) + +BUG=b:123039911 +TEST=Built + +Change-Id: Ic74ea07f8fb5f56e6904a1b80c3c286bc2911635 +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2987192 +Reviewed-by: Yu Zhao +Tested-by: Yu Zhao +Commit-Queue: Sonny Rao +Commit-Queue: Yu Zhao +--- + mm/Kconfig | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 57 insertions(+) + +--- a/mm/Kconfig ++++ b/mm/Kconfig +@@ -723,4 +723,63 @@ config ARCH_HAS_PTE_SPECIAL + config ARCH_HAS_HUGEPD + bool + ++# the multigenerational lru { ++config LRU_GEN ++ bool "Multigenerational LRU" ++ depends on MMU ++ # the following options may consume too many bits in page flags ++ depends on !MAXSMP && (64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP) ++ help ++ A high performance LRU implementation to heavily overcommit workloads ++ that are not IO bound. See Documentation/vm/multigen_lru.rst for ++ details. ++ ++ Warning: do not enable this option unless you plan to use it because ++ it introduces a small per-process and per-memcg and per-node memory ++ overhead. ++ ++config LRU_GEN_ENABLED ++ bool "Turn on by default" ++ depends on LRU_GEN ++ help ++ The default value of /sys/kernel/mm/lru_gen/enabled is 0. This option ++ changes it to 1. ++ ++ Warning: the default value is the fast path. See ++ Documentation/static-keys.txt for details. ++ ++config LRU_GEN_STATS ++ bool "Full stats for debugging" ++ depends on LRU_GEN ++ help ++ This option keeps full stats for each generation, which can be read ++ from /sys/kernel/debug/lru_gen_full. ++ ++ Warning: do not enable this option unless you plan to use it because ++ it introduces an additional small per-process and per-memcg and ++ per-node memory overhead. ++ ++config NR_LRU_GENS ++ int "Max number of generations" ++ depends on LRU_GEN ++ range 4 31 ++ default 7 ++ help ++ This will use order_base_2(N+1) spare bits from page flags. ++ ++ Warning: do not use numbers larger than necessary because each ++ generation introduces a small per-node and per-memcg memory overhead. ++ ++config TIERS_PER_GEN ++ int "Number of tiers per generation" ++ depends on LRU_GEN ++ range 2 5 ++ default 4 ++ help ++ This will use N-2 spare bits from page flags. ++ ++ Larger values generally offer better protection to active pages under ++ heavy buffered I/O workloads. ++# } ++ + endmenu diff --git a/target/linux/generic/backport-5.4/020-26-CHROMIUM-mm-multigenerational-lru-add-arch_has_hw_pt.patch b/target/linux/generic/backport-5.4/020-26-CHROMIUM-mm-multigenerational-lru-add-arch_has_hw_pt.patch new file mode 100644 index 0000000000..c381c9a7cf --- /dev/null +++ b/target/linux/generic/backport-5.4/020-26-CHROMIUM-mm-multigenerational-lru-add-arch_has_hw_pt.patch @@ -0,0 +1,155 @@ +From a667dec5b7b10ed855e9e45437372e4e298770d9 Mon Sep 17 00:00:00 2001 +From: Yu Zhao +Date: Sat, 26 Jun 2021 18:30:43 -0600 +Subject: [PATCH] CHROMIUM: mm: multigenerational lru: add + arch_has_hw_pte_young() + +Add arch_has_hw_pte_young() to indicate whether the h/w accessed bit +is supported. For those CPUs that don't, e.g., arm64 v8.0, we skip +walk_mm_list() and solely rely on lru_gen_scan_around(). This still +retains ~80% of the performance improvement. + +BUG=b:123039911 +TEST=Built + +Signed-off-by: Yu Zhao +Change-Id: I32e1380bc5966b4cb3ba5a9796c95af61fa8fc4a +Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2991721 +Reviewed-by: Yu Zhao +Tested-by: Yu Zhao +Commit-Queue: Sonny Rao +Commit-Queue: Yu Zhao +--- + arch/arm64/include/asm/cpucaps.h | 3 ++- + arch/arm64/include/asm/cpufeature.h | 6 ++++++ + arch/arm64/include/asm/pgtable.h | 7 +++++++ + arch/arm64/kernel/cpufeature.c | 10 ++++++++++ + arch/x86/include/asm/pgtable.h | 6 ++++++ + include/asm-generic/pgtable.h | 7 +++++++ + mm/vmscan.c | 8 ++++++++ + 7 files changed, 46 insertions(+), 1 deletion(-) + +--- a/arch/arm64/include/asm/cpucaps.h ++++ b/arch/arm64/include/asm/cpucaps.h +@@ -56,7 +56,8 @@ + #define ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM 46 + #define ARM64_WORKAROUND_1542419 47 + #define ARM64_SPECTRE_BHB 48 ++#define ARM64_HW_AF 49 + +-#define ARM64_NCAPS 49 ++#define ARM64_NCAPS 50 + + #endif /* __ASM_CPUCAPS_H */ +--- a/arch/arm64/include/asm/cpufeature.h ++++ b/arch/arm64/include/asm/cpufeature.h +@@ -643,6 +643,12 @@ static inline bool system_has_prio_mask_ + system_uses_irq_prio_masking(); + } + ++static inline bool system_has_hw_af(void) ++{ ++ return IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && ++ cpus_have_const_cap(ARM64_HW_AF); ++} ++ + #define ARM64_BP_HARDEN_UNKNOWN -1 + #define ARM64_BP_HARDEN_WA_NEEDED 0 + #define ARM64_BP_HARDEN_NOT_REQUIRED 1 +--- a/arch/arm64/include/asm/pgtable.h ++++ b/arch/arm64/include/asm/pgtable.h +@@ -31,6 +31,7 @@ + + #include + #include ++#include + #include + #include + #include +@@ -848,6 +849,12 @@ static inline pmd_t pmdp_establish(struc + + extern int kern_addr_valid(unsigned long addr); + ++static inline bool arch_has_hw_pte_young(void) ++{ ++ return system_has_hw_af(); ++} ++#define arch_has_hw_pte_young arch_has_hw_pte_young ++ + #include + + /* +--- a/arch/arm64/kernel/cpufeature.c ++++ b/arch/arm64/kernel/cpufeature.c +@@ -1475,6 +1475,16 @@ static const struct arm64_cpu_capabiliti + .matches = has_hw_dbm, + .cpu_enable = cpu_enable_hw_dbm, + }, ++ { ++ .desc = "Hardware update of the Access flag", ++ .type = ARM64_CPUCAP_SYSTEM_FEATURE, ++ .capability = ARM64_HW_AF, ++ .sys_reg = SYS_ID_AA64MMFR1_EL1, ++ .sign = FTR_UNSIGNED, ++ .field_pos = ID_AA64MMFR1_HADBS_SHIFT, ++ .min_field_value = 1, ++ .matches = has_cpuid_feature, ++ }, + #endif + { + .desc = "CRC32 instructions", +--- a/arch/x86/include/asm/pgtable.h ++++ b/arch/x86/include/asm/pgtable.h +@@ -1467,6 +1467,12 @@ static inline bool arch_has_pfn_modify_c + return boot_cpu_has_bug(X86_BUG_L1TF); + } + ++static inline bool arch_has_hw_pte_young(void) ++{ ++ return true; ++} ++#define arch_has_hw_pte_young arch_has_hw_pte_young ++ + #include + #endif /* __ASSEMBLY__ */ + +--- a/include/asm-generic/pgtable.h ++++ b/include/asm-generic/pgtable.h +@@ -121,6 +121,13 @@ static inline int pmdp_clear_flush_young + #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + #endif + ++#ifndef arch_has_hw_pte_young ++static inline bool arch_has_hw_pte_young(void) ++{ ++ return false; ++} ++#endif ++ + #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR + static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -4199,6 +4199,11 @@ static long get_nr_to_scan(struct lruvec + if (get_hi_wmark(max_seq, min_seq, swappiness) > MIN_NR_GENS) + return nr_to_scan; + ++ if (!arch_has_hw_pte_young()) { ++ inc_max_seq(lruvec, max_seq); ++ return nr_to_scan; ++ } ++ + /* kswapd uses lru_gen_age_node() */ + if (current_is_kswapd()) + return 0; +@@ -4287,6 +4292,9 @@ static void lru_gen_age_node(struct pgli + + VM_BUG_ON(!current_is_kswapd()); + ++ if (!arch_has_hw_pte_young()) ++ return; ++ + memcg = mem_cgroup_iter(NULL, NULL, NULL); + do { + struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg); diff --git a/target/linux/generic/config-5.4 b/target/linux/generic/config-5.4 index 576649bbb2..3cca0b2741 100644 --- a/target/linux/generic/config-5.4 +++ b/target/linux/generic/config-5.4 @@ -2786,6 +2786,9 @@ CONFIG_LOG_CPU_MAX_BUF_SHIFT=12 # CONFIG_LPC_ICH is not set # CONFIG_LPC_SCH is not set # CONFIG_LP_CONSOLE is not set +CONFIG_LRU_GEN=y +CONFIG_LRU_GEN_ENABLED=y +# CONFIG_LRU_GEN_STATS is not set # CONFIG_LSI_ET1011C_PHY is not set CONFIG_LSM="lockdown,yama,loadpin,safesetid,integrity" CONFIG_LSM_MMAP_MIN_ADDR=65536 @@ -3828,6 +3831,7 @@ CONFIG_NMI_LOG_BUF_SHIFT=13 # CONFIG_NO_HZ is not set # CONFIG_NO_HZ_FULL is not set # CONFIG_NO_HZ_IDLE is not set +CONFIG_NR_LRU_GENS=7 # CONFIG_NS83820 is not set # CONFIG_NTB is not set # CONFIG_NTFS_DEBUG is not set @@ -5657,6 +5661,7 @@ CONFIG_THIN_ARCHIVES=y # CONFIG_THUNDER_NIC_VF is not set # CONFIG_TICK_CPU_ACCOUNTING is not set CONFIG_TICK_ONESHOT=y +CONFIG_TIERS_PER_GEN=4 # CONFIG_TIFM_CORE is not set # CONFIG_TIGON3 is not set # CONFIG_TIMB_DMA is not set