diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/hmm.c | 2 | ||||
| -rw-r--r-- | mm/huge_memory.c | 4 | ||||
| -rw-r--r-- | mm/madvise.c | 16 | ||||
| -rw-r--r-- | mm/memory-failure.c | 210 |
4 files changed, 186 insertions, 46 deletions
@@ -968,6 +968,8 @@ static void hmm_devmem_free(struct page *page, void *data) { struct hmm_devmem *devmem = data; + page->mapping = NULL; + devmem->ops->free(devmem, page); } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 08b544383d74..c3bc7e9c9a2a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -752,7 +752,7 @@ static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, spin_unlock(ptl); } -int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, +vm_fault_t vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, pfn_t pfn, bool write) { pgprot_t pgprot = vma->vm_page_prot; @@ -812,7 +812,7 @@ static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, spin_unlock(ptl); } -int vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, +vm_fault_t vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, pud_t *pud, pfn_t pfn, bool write) { pgprot_t pgprot = vma->vm_page_prot; diff --git a/mm/madvise.c b/mm/madvise.c index 4d3c922ea1a1..972a9eaa898b 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -631,11 +631,13 @@ static int madvise_inject_error(int behavior, for (; start < end; start += PAGE_SIZE << order) { + unsigned long pfn; int ret; ret = get_user_pages_fast(start, 1, 0, &page); if (ret != 1) return ret; + pfn = page_to_pfn(page); /* * When soft offlining hugepages, after migrating the page @@ -651,17 +653,25 @@ static int madvise_inject_error(int behavior, if (behavior == MADV_SOFT_OFFLINE) { pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n", - page_to_pfn(page), start); + pfn, start); ret = soft_offline_page(page, MF_COUNT_INCREASED); if (ret) return ret; continue; } + pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n", - page_to_pfn(page), start); + pfn, start); - ret = memory_failure(page_to_pfn(page), MF_COUNT_INCREASED); + /* + * Drop the page reference taken by get_user_pages_fast(). In + * the absence of MF_COUNT_INCREASED the memory_failure() + * routine is responsible for pinning the page to prevent it + * from being released back to the page allocator. + */ + put_page(page); + ret = memory_failure(pfn, 0); if (ret) return ret; } diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 192d0bbfc9ea..0cd3de3550f0 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -55,6 +55,7 @@ #include <linux/hugetlb.h> #include <linux/memory_hotplug.h> #include <linux/mm_inline.h> +#include <linux/memremap.h> #include <linux/kfifo.h> #include <linux/ratelimit.h> #include <linux/page-isolation.h> @@ -175,22 +176,51 @@ int hwpoison_filter(struct page *p) EXPORT_SYMBOL_GPL(hwpoison_filter); /* + * Kill all processes that have a poisoned page mapped and then isolate + * the page. + * + * General strategy: + * Find all processes having the page mapped and kill them. + * But we keep a page reference around so that the page is not + * actually freed yet. + * Then stash the page away + * + * There's no convenient way to get back to mapped processes + * from the VMAs. So do a brute-force search over all + * running processes. + * + * Remember that machine checks are not common (or rather + * if they are common you have other problems), so this shouldn't + * be a performance issue. + * + * Also there are some races possible while we get from the + * error detection to actually handle it. + */ + +struct to_kill { + struct list_head nd; + struct task_struct *tsk; + unsigned long addr; + short size_shift; + char addr_valid; +}; + +/* * Send all the processes who have the page mapped a signal. * ``action optional'' if they are not immediately affected by the error * ``action required'' if error happened in current execution context */ -static int kill_proc(struct task_struct *t, unsigned long addr, - unsigned long pfn, struct page *page, int flags) +static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags) { - short addr_lsb; + struct task_struct *t = tk->tsk; + short addr_lsb = tk->size_shift; int ret; pr_err("Memory failure: %#lx: Killing %s:%d due to hardware memory corruption\n", pfn, t->comm, t->pid); - addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT; if ((flags & MF_ACTION_REQUIRED) && t->mm == current->mm) { - ret = force_sig_mceerr(BUS_MCEERR_AR, (void __user *)addr, + ret = force_sig_mceerr(BUS_MCEERR_AR, (void __user *)tk->addr, addr_lsb, current); } else { /* @@ -199,7 +229,7 @@ static int kill_proc(struct task_struct *t, unsigned long addr, * This could cause a loop when the user sets SIGBUS * to SIG_IGN, but hopefully no one will do that? */ - ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)addr, + ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, addr_lsb, t); /* synchronous? */ } if (ret < 0) @@ -235,34 +265,39 @@ void shake_page(struct page *p, int access) } EXPORT_SYMBOL_GPL(shake_page); -/* - * Kill all processes that have a poisoned page mapped and then isolate - * the page. - * - * General strategy: - * Find all processes having the page mapped and kill them. - * But we keep a page reference around so that the page is not - * actually freed yet. - * Then stash the page away - * - * There's no convenient way to get back to mapped processes - * from the VMAs. So do a brute-force search over all - * running processes. - * - * Remember that machine checks are not common (or rather - * if they are common you have other problems), so this shouldn't - * be a performance issue. - * - * Also there are some races possible while we get from the - * error detection to actually handle it. - */ - -struct to_kill { - struct list_head nd; - struct task_struct *tsk; - unsigned long addr; - char addr_valid; -}; +static unsigned long dev_pagemap_mapping_shift(struct page *page, + struct vm_area_struct *vma) +{ + unsigned long address = vma_address(page, vma); + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + pgd = pgd_offset(vma->vm_mm, address); + if (!pgd_present(*pgd)) + return 0; + p4d = p4d_offset(pgd, address); + if (!p4d_present(*p4d)) + return 0; + pud = pud_offset(p4d, address); + if (!pud_present(*pud)) + return 0; + if (pud_devmap(*pud)) + return PUD_SHIFT; + pmd = pmd_offset(pud, address); + if (!pmd_present(*pmd)) + return 0; + if (pmd_devmap(*pmd)) + return PMD_SHIFT; + pte = pte_offset_map(pmd, address); + if (!pte_present(*pte)) + return 0; + if (pte_devmap(*pte)) + return PAGE_SHIFT; + return 0; +} /* * Failure handling: if we can't find or can't kill a process there's @@ -293,6 +328,10 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, } tk->addr = page_address_in_vma(p, vma); tk->addr_valid = 1; + if (is_zone_device_page(p)) + tk->size_shift = dev_pagemap_mapping_shift(p, vma); + else + tk->size_shift = compound_order(compound_head(p)) + PAGE_SHIFT; /* * In theory we don't have to kill when the page was @@ -300,7 +339,7 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, * likely very rare kill anyways just out of paranoia, but use * a SIGKILL because the error is not contained anymore. */ - if (tk->addr == -EFAULT) { + if (tk->addr == -EFAULT || tk->size_shift == 0) { pr_info("Memory failure: Unable to find user space address %lx in %s\n", page_to_pfn(p), tsk->comm); tk->addr_valid = 0; @@ -318,9 +357,8 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, * Also when FAIL is set do a force kill because something went * wrong earlier. */ -static void kill_procs(struct list_head *to_kill, int forcekill, - bool fail, struct page *page, unsigned long pfn, - int flags) +static void kill_procs(struct list_head *to_kill, int forcekill, bool fail, + unsigned long pfn, int flags) { struct to_kill *tk, *next; @@ -343,8 +381,7 @@ static void kill_procs(struct list_head *to_kill, int forcekill, * check for that, but we need to tell the * process anyways. */ - else if (kill_proc(tk->tsk, tk->addr, - pfn, page, flags) < 0) + else if (kill_proc(tk, pfn, flags) < 0) pr_err("Memory failure: %#lx: Cannot send advisory machine check signal to %s:%d\n", pfn, tk->tsk->comm, tk->tsk->pid); } @@ -516,6 +553,7 @@ static const char * const action_page_types[] = { [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page", [MF_MSG_BUDDY] = "free buddy page", [MF_MSG_BUDDY_2ND] = "free buddy page (2nd try)", + [MF_MSG_DAX] = "dax page", [MF_MSG_UNKNOWN] = "unknown page", }; @@ -1013,7 +1051,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, * any accesses to the poisoned memory. */ forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL); - kill_procs(&tokill, forcekill, !unmap_success, p, pfn, flags); + kill_procs(&tokill, forcekill, !unmap_success, pfn, flags); return unmap_success; } @@ -1113,6 +1151,83 @@ out: return res; } +static int memory_failure_dev_pagemap(unsigned long pfn, int flags, + struct dev_pagemap *pgmap) +{ + struct page *page = pfn_to_page(pfn); + const bool unmap_success = true; + unsigned long size = 0; + struct to_kill *tk; + LIST_HEAD(tokill); + int rc = -EBUSY; + loff_t start; + + /* + * Prevent the inode from being freed while we are interrogating + * the address_space, typically this would be handled by + * lock_page(), but dax pages do not use the page lock. This + * also prevents changes to the mapping of this pfn until + * poison signaling is complete. + */ + if (!dax_lock_mapping_entry(page)) + goto out; + + if (hwpoison_filter(page)) { + rc = 0; + goto unlock; + } + + switch (pgmap->type) { + case MEMORY_DEVICE_PRIVATE: + case MEMORY_DEVICE_PUBLIC: + /* + * TODO: Handle HMM pages which may need coordination + * with device-side memory. + */ + goto unlock; + default: + break; + } + + /* + * Use this flag as an indication that the dax page has been + * remapped UC to prevent speculative consumption of poison. + */ + SetPageHWPoison(page); + + /* + * Unlike System-RAM there is no possibility to swap in a + * different physical page at a given virtual address, so all + * userspace consumption of ZONE_DEVICE memory necessitates + * SIGBUS (i.e. MF_MUST_KILL) + */ + flags |= MF_ACTION_REQUIRED | MF_MUST_KILL; + collect_procs(page, &tokill, flags & MF_ACTION_REQUIRED); + + list_for_each_entry(tk, &tokill, nd) + if (tk->size_shift) + size = max(size, 1UL << tk->size_shift); + if (size) { + /* + * Unmap the largest mapping to avoid breaking up + * device-dax mappings which are constant size. The + * actual size of the mapping being torn down is + * communicated in siginfo, see kill_proc() + */ + start = (page->index << PAGE_SHIFT) & ~(size - 1); + unmap_mapping_range(page->mapping, start, start + size, 0); + } + kill_procs(&tokill, flags & MF_MUST_KILL, !unmap_success, pfn, flags); + rc = 0; +unlock: + dax_unlock_mapping_entry(page); +out: + /* drop pgmap ref acquired in caller */ + put_dev_pagemap(pgmap); + action_result(pfn, MF_MSG_DAX, rc ? MF_FAILED : MF_RECOVERED); + return rc; +} + /** * memory_failure - Handle memory failure of a page. * @pfn: Page Number of the corrupted page @@ -1135,6 +1250,7 @@ int memory_failure(unsigned long pfn, int flags) struct page *p; struct page *hpage; struct page *orig_head; + struct dev_pagemap *pgmap; int res; unsigned long page_flags; @@ -1147,6 +1263,10 @@ int memory_failure(unsigned long pfn, int flags) return -ENXIO; } + pgmap = get_dev_pagemap(pfn, NULL); + if (pgmap) + return memory_failure_dev_pagemap(pfn, flags, pgmap); + p = pfn_to_page(pfn); if (PageHuge(p)) return memory_failure_hugetlb(pfn, flags); @@ -1777,6 +1897,14 @@ int soft_offline_page(struct page *page, int flags) int ret; unsigned long pfn = page_to_pfn(page); + if (is_zone_device_page(page)) { + pr_debug_ratelimited("soft_offline: %#lx page is device page\n", + pfn); + if (flags & MF_COUNT_INCREASED) + put_page(page); + return -EIO; + } + if (PageHWPoison(page)) { pr_info("soft offline: %#lx page already poisoned\n", pfn); if (flags & MF_COUNT_INCREASED) |