From 8749cfea11f3fffe8f7cad891470a77b36e0185f Mon Sep 17 00:00:00 2001 From: Vladimir Davydov Date: Fri, 15 Jan 2016 16:54:45 -0800 Subject: mm: add page_check_address_transhuge() helper page_referenced_one() and page_idle_clear_pte_refs_one() duplicate the code for looking up pte of a (possibly transhuge) page. Move this code to a new helper function, page_check_address_transhuge(), and make the above mentioned functions use it. This is just a cleanup, no functional changes are intended. Signed-off-by: Vladimir Davydov Reviewed-by: Kirill A. Shutemov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/rmap.h | 19 +++++++++ mm/page_idle.c | 63 ++++------------------------ mm/rmap.c | 115 +++++++++++++++++++++++++++++++-------------------- 3 files changed, 99 insertions(+), 98 deletions(-) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index ebf3750e42b2..77d1ba57d495 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -215,6 +215,25 @@ static inline pte_t *page_check_address(struct page *page, struct mm_struct *mm, return ptep; } +/* + * Used by idle page tracking to check if a page was referenced via page + * tables. + */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +bool page_check_address_transhuge(struct page *page, struct mm_struct *mm, + unsigned long address, pmd_t **pmdp, + pte_t **ptep, spinlock_t **ptlp); +#else +static inline bool page_check_address_transhuge(struct page *page, + struct mm_struct *mm, unsigned long address, + pmd_t **pmdp, pte_t **ptep, spinlock_t **ptlp) +{ + *ptep = page_check_address(page, mm, address, ptlp, 0); + *pmdp = NULL; + return !!*ptep; +} +#endif + /* * Used by swapoff to help locate where page is expected in vma. */ diff --git a/mm/page_idle.c b/mm/page_idle.c index 2c553ba969f8..4ea9c4ef5146 100644 --- a/mm/page_idle.c +++ b/mm/page_idle.c @@ -55,71 +55,26 @@ static int page_idle_clear_pte_refs_one(struct page *page, unsigned long addr, void *arg) { struct mm_struct *mm = vma->vm_mm; - spinlock_t *ptl; - pgd_t *pgd; - pud_t *pud; pmd_t *pmd; pte_t *pte; + spinlock_t *ptl; bool referenced = false; - pgd = pgd_offset(mm, addr); - if (!pgd_present(*pgd)) - return SWAP_AGAIN; - pud = pud_offset(pgd, addr); - if (!pud_present(*pud)) + if (!page_check_address_transhuge(page, mm, addr, &pmd, &pte, &ptl)) return SWAP_AGAIN; - pmd = pmd_offset(pud, addr); - - if (pmd_trans_huge(*pmd)) { - ptl = pmd_lock(mm, pmd); - if (!pmd_present(*pmd)) - goto unlock_pmd; - if (unlikely(!pmd_trans_huge(*pmd))) { - spin_unlock(ptl); - goto map_pte; - } - - if (pmd_page(*pmd) != page) - goto unlock_pmd; + if (pte) { + referenced = ptep_clear_young_notify(vma, addr, pte); + pte_unmap(pte); + } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { referenced = pmdp_clear_young_notify(vma, addr, pmd); - spin_unlock(ptl); - goto found; -unlock_pmd: - spin_unlock(ptl); - return SWAP_AGAIN; } else { - pmd_t pmde = *pmd; - - barrier(); - if (!pmd_present(pmde) || pmd_trans_huge(pmde)) - return SWAP_AGAIN; - - } -map_pte: - pte = pte_offset_map(pmd, addr); - if (!pte_present(*pte)) { - pte_unmap(pte); - return SWAP_AGAIN; + /* unexpected pmd-mapped page? */ + WARN_ON_ONCE(1); } - ptl = pte_lockptr(mm, pmd); - spin_lock(ptl); - - if (!pte_present(*pte)) { - pte_unmap_unlock(pte, ptl); - return SWAP_AGAIN; - } - - /* THP can be referenced by any subpage */ - if (pte_pfn(*pte) - page_to_pfn(page) >= hpage_nr_pages(page)) { - pte_unmap_unlock(pte, ptl); - return SWAP_AGAIN; - } + spin_unlock(ptl); - referenced = ptep_clear_young_notify(vma, addr, pte); - pte_unmap_unlock(pte, ptl); -found: if (referenced) { clear_page_idle(page); /* diff --git a/mm/rmap.c b/mm/rmap.c index 6127c00b2262..cdc2a885a4cd 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -798,48 +798,44 @@ int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) return 1; } -struct page_referenced_arg { - int mapcount; - int referenced; - unsigned long vm_flags; - struct mem_cgroup *memcg; -}; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE /* - * arg: page_referenced_arg will be passed + * Check that @page is mapped at @address into @mm. In contrast to + * page_check_address(), this function can handle transparent huge pages. + * + * On success returns true with pte mapped and locked. For PMD-mapped + * transparent huge pages *@ptep is set to NULL. */ -static int page_referenced_one(struct page *page, struct vm_area_struct *vma, - unsigned long address, void *arg) +bool page_check_address_transhuge(struct page *page, struct mm_struct *mm, + unsigned long address, pmd_t **pmdp, + pte_t **ptep, spinlock_t **ptlp) { - struct mm_struct *mm = vma->vm_mm; - spinlock_t *ptl; - int referenced = 0; - struct page_referenced_arg *pra = arg; pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; + spinlock_t *ptl; if (unlikely(PageHuge(page))) { /* when pud is not present, pte will be NULL */ pte = huge_pte_offset(mm, address); if (!pte) - return SWAP_AGAIN; + return false; ptl = huge_pte_lockptr(page_hstate(page), mm, pte); + pmd = NULL; goto check_pte; } pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) - return SWAP_AGAIN; + return false; pud = pud_offset(pgd, address); if (!pud_present(*pud)) - return SWAP_AGAIN; + return false; pmd = pmd_offset(pud, address); if (pmd_trans_huge(*pmd)) { - int ret = SWAP_AGAIN; - ptl = pmd_lock(mm, pmd); if (!pmd_present(*pmd)) goto unlock_pmd; @@ -851,31 +847,23 @@ static int page_referenced_one(struct page *page, struct vm_area_struct *vma, if (pmd_page(*pmd) != page) goto unlock_pmd; - if (vma->vm_flags & VM_LOCKED) { - pra->vm_flags |= VM_LOCKED; - ret = SWAP_FAIL; /* To break the loop */ - goto unlock_pmd; - } - - if (pmdp_clear_flush_young_notify(vma, address, pmd)) - referenced++; - spin_unlock(ptl); + pte = NULL; goto found; unlock_pmd: spin_unlock(ptl); - return ret; + return false; } else { pmd_t pmde = *pmd; barrier(); if (!pmd_present(pmde) || pmd_trans_huge(pmde)) - return SWAP_AGAIN; + return false; } map_pte: pte = pte_offset_map(pmd, address); if (!pte_present(*pte)) { pte_unmap(pte); - return SWAP_AGAIN; + return false; } ptl = pte_lockptr(mm, pmd); @@ -884,35 +872,74 @@ check_pte: if (!pte_present(*pte)) { pte_unmap_unlock(pte, ptl); - return SWAP_AGAIN; + return false; } /* THP can be referenced by any subpage */ if (pte_pfn(*pte) - page_to_pfn(page) >= hpage_nr_pages(page)) { pte_unmap_unlock(pte, ptl); - return SWAP_AGAIN; + return false; } +found: + *ptep = pte; + *pmdp = pmd; + *ptlp = ptl; + return true; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +struct page_referenced_arg { + int mapcount; + int referenced; + unsigned long vm_flags; + struct mem_cgroup *memcg; +}; +/* + * arg: page_referenced_arg will be passed + */ +static int page_referenced_one(struct page *page, struct vm_area_struct *vma, + unsigned long address, void *arg) +{ + struct mm_struct *mm = vma->vm_mm; + struct page_referenced_arg *pra = arg; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + int referenced = 0; + + if (!page_check_address_transhuge(page, mm, address, &pmd, &pte, &ptl)) + return SWAP_AGAIN; if (vma->vm_flags & VM_LOCKED) { - pte_unmap_unlock(pte, ptl); + if (pte) + pte_unmap(pte); + spin_unlock(ptl); pra->vm_flags |= VM_LOCKED; return SWAP_FAIL; /* To break the loop */ } - if (ptep_clear_flush_young_notify(vma, address, pte)) { - /* - * Don't treat a reference through a sequentially read - * mapping as such. If the page has been used in - * another mapping, we will catch it; if this other - * mapping is already gone, the unmap path will have - * set PG_referenced or activated the page. - */ - if (likely(!(vma->vm_flags & VM_SEQ_READ))) + if (pte) { + if (ptep_clear_flush_young_notify(vma, address, pte)) { + /* + * Don't treat a reference through a sequentially read + * mapping as such. If the page has been used in + * another mapping, we will catch it; if this other + * mapping is already gone, the unmap path will have + * set PG_referenced or activated the page. + */ + if (likely(!(vma->vm_flags & VM_SEQ_READ))) + referenced++; + } + pte_unmap(pte); + } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + if (pmdp_clear_flush_young_notify(vma, address, pmd)) referenced++; + } else { + /* unexpected pmd-mapped page? */ + WARN_ON_ONCE(1); } - pte_unmap_unlock(pte, ptl); + spin_unlock(ptl); -found: if (referenced) clear_page_idle(page); if (test_and_clear_page_young(page)) -- cgit v1.2.3-58-ga151