mm: add tracepoint for scanning pages (7d2eba05) · Commits · 戴 / test

include/trace/events/huge_memory.h

0 → 100644

+136 −0

Original line number	Diff line number	Diff line
		#undef TRACE_SYSTEM
		#define TRACE_SYSTEM huge_memory

		#if !defined(__HUGE_MEMORY_H) \|\| defined(TRACE_HEADER_MULTI_READ)
		#define __HUGE_MEMORY_H

		#include <linux/tracepoint.h>

		#include <trace/events/gfpflags.h>

		#define SCAN_STATUS \
		EM( SCAN_FAIL, "failed") \
		EM( SCAN_SUCCEED, "succeeded") \
		EM( SCAN_PMD_NULL, "pmd_null") \
		EM( SCAN_EXCEED_NONE_PTE, "exceed_none_pte") \
		EM( SCAN_PTE_NON_PRESENT, "pte_non_present") \
		EM( SCAN_PAGE_RO, "no_writable_page") \
		EM( SCAN_NO_REFERENCED_PAGE, "no_referenced_page") \
		EM( SCAN_PAGE_NULL, "page_null") \
		EM( SCAN_SCAN_ABORT, "scan_aborted") \
		EM( SCAN_PAGE_COUNT, "not_suitable_page_count") \
		EM( SCAN_PAGE_LRU, "page_not_in_lru") \
		EM( SCAN_PAGE_LOCK, "page_locked") \
		EM( SCAN_PAGE_ANON, "page_not_anon") \
		EM( SCAN_ANY_PROCESS, "no_process_for_page") \
		EM( SCAN_VMA_NULL, "vma_null") \
		EM( SCAN_VMA_CHECK, "vma_check_failed") \
		EM( SCAN_ADDRESS_RANGE, "not_suitable_address_range") \
		EM( SCAN_SWAP_CACHE_PAGE, "page_swap_cache") \
		EM( SCAN_DEL_PAGE_LRU, "could_not_delete_page_from_lru")\
		EM( SCAN_ALLOC_HUGE_PAGE_FAIL, "alloc_huge_page_failed") \
		EMe( SCAN_CGROUP_CHARGE_FAIL, "ccgroup_charge_failed")

		#undef EM
		#undef EMe
		#define EM(a, b) TRACE_DEFINE_ENUM(a);
		#define EMe(a, b) TRACE_DEFINE_ENUM(a);

		SCAN_STATUS

		#undef EM
		#undef EMe
		#define EM(a, b) {a, b},
		#define EMe(a, b) {a, b}

		TRACE_EVENT(mm_khugepaged_scan_pmd,

		TP_PROTO(struct mm_struct *mm, unsigned long pfn, bool writable,
		bool referenced, int none_or_zero, int status),

		TP_ARGS(mm, pfn, writable, referenced, none_or_zero, status),

		TP_STRUCT__entry(
		__field(struct mm_struct *, mm)
		__field(unsigned long, pfn)
		__field(bool, writable)
		__field(bool, referenced)
		__field(int, none_or_zero)
		__field(int, status)
		),

		TP_fast_assign(
		__entry->mm = mm;
		__entry->pfn = pfn;
		__entry->writable = writable;
		__entry->referenced = referenced;
		__entry->none_or_zero = none_or_zero;
		__entry->status = status;
		),

		TP_printk("mm=%p, scan_pfn=0x%lx, writable=%d, referenced=%d, none_or_zero=%d, status=%s",
		__entry->mm,
		__entry->pfn,
		__entry->writable,
		__entry->referenced,
		__entry->none_or_zero,
		__print_symbolic(__entry->status, SCAN_STATUS))
		);

		TRACE_EVENT(mm_collapse_huge_page,

		TP_PROTO(struct mm_struct *mm, int isolated, int status),

		TP_ARGS(mm, isolated, status),

		TP_STRUCT__entry(
		__field(struct mm_struct *, mm)
		__field(int, isolated)
		__field(int, status)
		),

		TP_fast_assign(
		__entry->mm = mm;
		__entry->isolated = isolated;
		__entry->status = status;
		),

		TP_printk("mm=%p, isolated=%d, status=%s",
		__entry->mm,
		__entry->isolated,
		__print_symbolic(__entry->status, SCAN_STATUS))
		);

		TRACE_EVENT(mm_collapse_huge_page_isolate,

		TP_PROTO(unsigned long pfn, int none_or_zero,
		bool referenced, bool writable, int status),

		TP_ARGS(pfn, none_or_zero, referenced, writable, status),

		TP_STRUCT__entry(
		__field(unsigned long, pfn)
		__field(int, none_or_zero)
		__field(bool, referenced)
		__field(bool, writable)
		__field(int, status)
		),

		TP_fast_assign(
		__entry->pfn = pfn;
		__entry->none_or_zero = none_or_zero;
		__entry->referenced = referenced;
		__entry->writable = writable;
		__entry->status = status;
		),

		TP_printk("scan_pfn=0x%lx, none_or_zero=%d, referenced=%d, writable=%d, status=%s",
		__entry->pfn,
		__entry->none_or_zero,
		__entry->referenced,
		__entry->writable,
		__print_symbolic(__entry->status, SCAN_STATUS))
		);

		#endif /* __HUGE_MEMORY_H */
		#include <trace/define_trace.h>

mm/huge_memory.c

+134 −32

Original line number	Diff line number	Diff line
		@@ -31,6 +31,33 @@
		#include <asm/pgalloc.h>
		#include "internal.h"

		enum scan_result {
		SCAN_FAIL,
		SCAN_SUCCEED,
		SCAN_PMD_NULL,
		SCAN_EXCEED_NONE_PTE,
		SCAN_PTE_NON_PRESENT,
		SCAN_PAGE_RO,
		SCAN_NO_REFERENCED_PAGE,
		SCAN_PAGE_NULL,
		SCAN_SCAN_ABORT,
		SCAN_PAGE_COUNT,
		SCAN_PAGE_LRU,
		SCAN_PAGE_LOCK,
		SCAN_PAGE_ANON,
		SCAN_ANY_PROCESS,
		SCAN_VMA_NULL,
		SCAN_VMA_CHECK,
		SCAN_ADDRESS_RANGE,
		SCAN_SWAP_CACHE_PAGE,
		SCAN_DEL_PAGE_LRU,
		SCAN_ALLOC_HUGE_PAGE_FAIL,
		SCAN_CGROUP_CHARGE_FAIL
		};

		#define CREATE_TRACE_POINTS
		#include <trace/events/huge_memory.h>

		/*
		* By default transparent hugepage support is disabled in order that avoid
		* to risk increase the memory footprint of applications without a guaranteed
		@@ -2198,26 +2225,33 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
		unsigned long address,
		pte_t *pte)
		{
		struct page *page;
		struct page *page = NULL;
		pte_t *_pte;
		int none_or_zero = 0;
		int none_or_zero = 0, result = 0;
		bool referenced = false, writable = false;

		for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
		_pte++, address += PAGE_SIZE) {
		pte_t pteval = *_pte;
		if (pte_none(pteval) \|\| (pte_present(pteval) &&
		is_zero_pfn(pte_pfn(pteval)))) {
		if (!userfaultfd_armed(vma) &&
		++none_or_zero <= khugepaged_max_ptes_none)
		++none_or_zero <= khugepaged_max_ptes_none) {
		continue;
		else
		} else {
		result = SCAN_EXCEED_NONE_PTE;
		goto out;
		}
		if (!pte_present(pteval))
		}
		if (!pte_present(pteval)) {
		result = SCAN_PTE_NON_PRESENT;
		goto out;
		}
		page = vm_normal_page(vma, address, pteval);
		if (unlikely(!page))
		if (unlikely(!page)) {
		result = SCAN_PAGE_NULL;
		goto out;
		}

		VM_BUG_ON_PAGE(PageCompound(page), page);
		VM_BUG_ON_PAGE(!PageAnon(page), page);
		@@ -2229,8 +2263,10 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
		* is needed to serialize against split_huge_page
		* when invoked from the VM.
		*/
		if (!trylock_page(page))
		if (!trylock_page(page)) {
		result = SCAN_PAGE_LOCK;
		goto out;
		}

		/*
		* cannot use mapcount: can't collapse if there's a gup pin.
		@@ -2239,6 +2275,7 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
		*/
		if (page_count(page) != 1 + !!PageSwapCache(page)) {
		unlock_page(page);
		result = SCAN_PAGE_COUNT;
		goto out;
		}
		if (pte_write(pteval)) {
		@@ -2246,6 +2283,7 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
		} else {
		if (PageSwapCache(page) && !reuse_swap_page(page)) {
		unlock_page(page);
		result = SCAN_SWAP_CACHE_PAGE;
		goto out;
		}
		/*
		@@ -2260,6 +2298,7 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
		*/
		if (isolate_lru_page(page)) {
		unlock_page(page);
		result = SCAN_DEL_PAGE_LRU;
		goto out;
		}
		/* 0 stands for page_is_file_cache(page) == false */
		@@ -2273,10 +2312,21 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
		mmu_notifier_test_young(vma->vm_mm, address))
		referenced = true;
		}
		if (likely(referenced && writable))
		if (likely(writable)) {
		if (likely(referenced)) {
		result = SCAN_SUCCEED;
		trace_mm_collapse_huge_page_isolate(page_to_pfn(page), none_or_zero,
		referenced, writable, result);
		return 1;
		}
		} else {
		result = SCAN_PAGE_RO;
		}

		out:
		release_pte_pages(pte, _pte);
		trace_mm_collapse_huge_page_isolate(page_to_pfn(page), none_or_zero,
		referenced, writable, result);
		return 0;
		}

		@@ -2513,7 +2563,7 @@ static void collapse_huge_page(struct mm_struct *mm,
		pgtable_t pgtable;
		struct page *new_page;
		spinlock_t pmd_ptl, pte_ptl;
		int isolated;
		int isolated, result = 0;
		unsigned long hstart, hend;
		struct mem_cgroup *memcg;
		unsigned long mmun_start; /* For mmu_notifiers */
		@@ -2528,12 +2578,15 @@ static void collapse_huge_page(struct mm_struct *mm,

		/* release the mmap_sem read lock. */
		new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node);
		if (!new_page)
		return;
		if (!new_page) {
		result = SCAN_ALLOC_HUGE_PAGE_FAIL;
		goto out_nolock;
		}

		if (unlikely(mem_cgroup_try_charge(new_page, mm,
		gfp, &memcg)))
		return;
		if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg))) {
		result = SCAN_CGROUP_CHARGE_FAIL;
		goto out_nolock;
		}

		/*
		* Prevent all access to pagetables with the exception of
		@@ -2541,21 +2594,31 @@ static void collapse_huge_page(struct mm_struct *mm,
		* handled by the anon_vma lock + PG_lock.
		*/
		down_write(&mm->mmap_sem);
		if (unlikely(khugepaged_test_exit(mm)))
		if (unlikely(khugepaged_test_exit(mm))) {
		result = SCAN_ANY_PROCESS;
		goto out;
		}

		vma = find_vma(mm, address);
		if (!vma)
		if (!vma) {
		result = SCAN_VMA_NULL;
		goto out;
		}
		hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
		hend = vma->vm_end & HPAGE_PMD_MASK;
		if (address < hstart \|\| address + HPAGE_PMD_SIZE > hend)
		if (address < hstart \|\| address + HPAGE_PMD_SIZE > hend) {
		result = SCAN_ADDRESS_RANGE;
		goto out;
		if (!hugepage_vma_check(vma))
		}
		if (!hugepage_vma_check(vma)) {
		result = SCAN_VMA_CHECK;
		goto out;
		}
		pmd = mm_find_pmd(mm, address);
		if (!pmd)
		if (!pmd) {
		result = SCAN_PMD_NULL;
		goto out;
		}

		anon_vma_lock_write(vma->anon_vma);

		@@ -2592,6 +2655,7 @@ static void collapse_huge_page(struct mm_struct *mm,
		pmd_populate(mm, pmd, pmd_pgtable(_pmd));
		spin_unlock(pmd_ptl);
		anon_vma_unlock_write(vma->anon_vma);
		result = SCAN_FAIL;
		goto out;
		}

		@@ -2629,10 +2693,15 @@ static void collapse_huge_page(struct mm_struct *mm,
		*hpage = NULL;

		khugepaged_pages_collapsed++;
		result = SCAN_SUCCEED;
		out_up_write:
		up_write(&mm->mmap_sem);
		trace_mm_collapse_huge_page(mm, isolated, result);
		return;

		out_nolock:
		trace_mm_collapse_huge_page(mm, isolated, result);
		return;
		out:
		mem_cgroup_cancel_charge(new_page, memcg);
		goto out_up_write;
		@@ -2645,8 +2714,8 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
		{
		pmd_t *pmd;
		pte_t pte, _pte;
		int ret = 0, none_or_zero = 0;
		struct page *page;
		int ret = 0, none_or_zero = 0, result = 0;
		struct page *page = NULL;
		unsigned long _address;
		spinlock_t *ptl;
		int node = NUMA_NO_NODE;
		@@ -2655,8 +2724,10 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
		VM_BUG_ON(address & ~HPAGE_PMD_MASK);

		pmd = mm_find_pmd(mm, address);
		if (!pmd)
		if (!pmd) {
		result = SCAN_PMD_NULL;
		goto out;
		}

		memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
		pte = pte_offset_map_lock(mm, pmd, address, &ptl);
		@@ -2665,19 +2736,25 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
		pte_t pteval = *_pte;
		if (pte_none(pteval) \|\| is_zero_pfn(pte_pfn(pteval))) {
		if (!userfaultfd_armed(vma) &&
		++none_or_zero <= khugepaged_max_ptes_none)
		++none_or_zero <= khugepaged_max_ptes_none) {
		continue;
		else
		} else {
		result = SCAN_EXCEED_NONE_PTE;
		goto out_unmap;
		}
		if (!pte_present(pteval))
		}
		if (!pte_present(pteval)) {
		result = SCAN_PTE_NON_PRESENT;
		goto out_unmap;
		}
		if (pte_write(pteval))
		writable = true;

		page = vm_normal_page(vma, _address, pteval);
		if (unlikely(!page))
		if (unlikely(!page)) {
		result = SCAN_PAGE_NULL;
		goto out_unmap;
		}
		/*
		* Record which node the original page is from and save this
		* information to khugepaged_node_load[].
		@@ -2685,26 +2762,49 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
		* hit record.
		*/
		node = page_to_nid(page);
		if (khugepaged_scan_abort(node))
		if (khugepaged_scan_abort(node)) {
		result = SCAN_SCAN_ABORT;
		goto out_unmap;
		}
		khugepaged_node_load[node]++;
		VM_BUG_ON_PAGE(PageCompound(page), page);
		if (!PageLRU(page) \|\| PageLocked(page) \|\| !PageAnon(page))
		if (!PageLRU(page)) {
		result = SCAN_SCAN_ABORT;
		goto out_unmap;
		}
		if (PageLocked(page)) {
		result = SCAN_PAGE_LOCK;
		goto out_unmap;
		}
		if (!PageAnon(page)) {
		result = SCAN_PAGE_ANON;
		goto out_unmap;
		}

		/*
		* cannot use mapcount: can't collapse if there's a gup pin.
		* The page must only be referenced by the scanned process
		* and page swap cache.
		*/
		if (page_count(page) != 1 + !!PageSwapCache(page))
		if (page_count(page) != 1 + !!PageSwapCache(page)) {
		result = SCAN_PAGE_COUNT;
		goto out_unmap;
		}
		if (pte_young(pteval) \|\|
		page_is_young(page) \|\| PageReferenced(page) \|\|
		mmu_notifier_test_young(vma->vm_mm, address))
		referenced = true;
		}
		if (referenced && writable)
		if (writable) {
		if (referenced) {
		result = SCAN_SUCCEED;
		ret = 1;
		} else {
		result = SCAN_NO_REFERENCED_PAGE;
		}
		} else {
		result = SCAN_PAGE_RO;
		}
		out_unmap:
		pte_unmap_unlock(pte, ptl);
		if (ret) {
		@@ -2713,6 +2813,8 @@ out_unmap:
		collapse_huge_page(mm, address, hpage, vma, node);
		}
		out:
		trace_mm_khugepaged_scan_pmd(mm, page_to_pfn(page), writable, referenced,
		none_or_zero, result);
		return ret;
		}

Admin message