[PATCH] swsusp: Use memory bitmaps during resume (940864dd) · Commits · 戴 / test

include/linux/suspend.h

+0 −9

Original line number	Diff line number	Diff line
		@@ -16,15 +16,6 @@ struct pbe {
		struct pbe *next;
		};

		#define for_each_pbe(pbe, pblist) \
		for (pbe = pblist ; pbe ; pbe = pbe->next)

		#define PBES_PER_PAGE (PAGE_SIZE/sizeof(struct pbe))
		#define PB_PAGE_SKIP (PBES_PER_PAGE-1)

		#define for_each_pb_page(pbe, pblist) \
		for (pbe = pblist ; pbe ; pbe = (pbe+PB_PAGE_SKIP)->next)

		/* mm/page_alloc.c */
		extern void drain_local_pages(void);
		extern void mark_free_pages(struct zone *zone);

kernel/power/power.h

+1 −10

Original line number	Diff line number	Diff line
		@@ -81,16 +81,6 @@ struct snapshot_handle {
		unsigned int prev; /* number of the block of PAGE_SIZE bytes that
		* was the current one previously
		*/
		struct pbe pbe; / PBE that corresponds to 'buffer' */
		struct pbe last_pbe; / When the image is restored (eg. read
		* from disk) we can store some image
		* data directly in the page frames
		* in which they were before suspend.
		* In such a case the PBEs that
		* correspond to them will be unused.
		* This is the last PBE, so far, that
		* does not correspond to such data.
		*/
		void buffer; / address of the block to read from
		* or write to
		*/
		@@ -113,6 +103,7 @@ extern unsigned int snapshot_additional_pages(struct zone *zone);
		extern int snapshot_read_next(struct snapshot_handle *handle, size_t count);
		extern int snapshot_write_next(struct snapshot_handle *handle, size_t count);
		extern int snapshot_image_loaded(struct snapshot_handle *handle);
		extern void snapshot_free_unused_memory(struct snapshot_handle *handle);

		#define SNAPSHOT_IOC_MAGIC '3'
		#define SNAPSHOT_FREEZE _IO(SNAPSHOT_IOC_MAGIC, 1)

kernel/power/snapshot.c

+182 −234

Original line number	Diff line number	Diff line
		@@ -39,7 +39,7 @@ struct pbe *restore_pblist;

		static unsigned int nr_copy_pages;
		static unsigned int nr_meta_pages;
		static unsigned long *buffer;
		static void *buffer;

		#ifdef CONFIG_HIGHMEM
		unsigned int count_highmem_pages(void)
		@@ -172,7 +172,7 @@ static inline int restore_highmem(void) {return 0;}
		#define PG_UNSAFE_CLEAR 1
		#define PG_UNSAFE_KEEP 0

		static unsigned int unsafe_pages;
		static unsigned int allocated_unsafe_pages;

		static void *alloc_image_page(gfp_t gfp_mask, int safe_needed)
		{
		@@ -183,7 +183,7 @@ static void *alloc_image_page(gfp_t gfp_mask, int safe_needed)
		while (res && PageNosaveFree(virt_to_page(res))) {
		/* The page is unsafe, mark it for swsusp_free() */
		SetPageNosave(virt_to_page(res));
		unsafe_pages++;
		allocated_unsafe_pages++;
		res = (void *)get_zeroed_page(gfp_mask);
		}
		if (res) {
		@@ -772,101 +772,10 @@ copy_data_pages(struct memory_bitmap copy_bm, struct memory_bitmap orig_bm)
		}

		/**
		* free_pagedir - free pages allocated with alloc_pagedir()
		*/

		static void free_pagedir(struct pbe *pblist, int clear_nosave_free)
		{
		struct pbe *pbe;

		while (pblist) {
		pbe = (pblist + PB_PAGE_SKIP)->next;
		free_image_page(pblist, clear_nosave_free);
		pblist = pbe;
		}
		}

		/**
		* fill_pb_page - Create a list of PBEs on a given memory page
		*/

		static inline void fill_pb_page(struct pbe *pbpage, unsigned int n)
		{
		struct pbe *p;

		p = pbpage;
		pbpage += n - 1;
		do
		p->next = p + 1;
		while (++p < pbpage);
		}

		/**
		* create_pbe_list - Create a list of PBEs on top of a given chain
		* of memory pages allocated with alloc_pagedir()
		*
		* This function assumes that pages allocated by alloc_image_page() will
		* always be zeroed.
		*/

		static inline void create_pbe_list(struct pbe *pblist, unsigned int nr_pages)
		{
		struct pbe *pbpage;
		unsigned int num = PBES_PER_PAGE;

		for_each_pb_page (pbpage, pblist) {
		if (num >= nr_pages)
		break;

		fill_pb_page(pbpage, PBES_PER_PAGE);
		num += PBES_PER_PAGE;
		}
		if (pbpage) {
		num -= PBES_PER_PAGE;
		fill_pb_page(pbpage, nr_pages - num);
		}
		}

		/**
		* alloc_pagedir - Allocate the page directory.
		*
		* First, determine exactly how many pages we need and
		* allocate them.
		*
		* We arrange the pages in a chain: each page is an array of PBES_PER_PAGE
		* struct pbe elements (pbes) and the last element in the page points
		* to the next page.
		* swsusp_free - free pages allocated for the suspend.
		*
		* On each page we set up a list of struct_pbe elements.
		*/

		static struct pbe *alloc_pagedir(unsigned int nr_pages, gfp_t gfp_mask,
		int safe_needed)
		{
		unsigned int num;
		struct pbe pblist, pbe;

		if (!nr_pages)
		return NULL;

		pblist = alloc_image_page(gfp_mask, safe_needed);
		pbe = pblist;
		for (num = PBES_PER_PAGE; num < nr_pages; num += PBES_PER_PAGE) {
		if (!pbe) {
		free_pagedir(pblist, PG_UNSAFE_CLEAR);
		return NULL;
		}
		pbe += PB_PAGE_SKIP;
		pbe->next = alloc_image_page(gfp_mask, safe_needed);
		pbe = pbe->next;
		}
		create_pbe_list(pblist, nr_pages);
		return pblist;
		}

		/**
		* Free pages we allocated for suspend. Suspend pages are alocated
		* before atomic copy, so we need to free them after resume.
		* Suspend pages are alocated before the atomic copy is made, so we
		* need to release them after the resume.
		*/

		void swsusp_free(void)
		@@ -904,14 +813,18 @@ void swsusp_free(void)
		static int enough_free_mem(unsigned int nr_pages)
		{
		struct zone *zone;
		unsigned int n = 0;
		unsigned int free = 0, meta = 0;

		for_each_zone (zone)
		if (!is_highmem(zone))
		n += zone->free_pages;
		pr_debug("swsusp: available memory: %u pages\n", n);
		return n > (nr_pages + PAGES_FOR_IO +
		(nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE);
		if (!is_highmem(zone)) {
		free += zone->free_pages;
		meta += snapshot_additional_pages(zone);
		}

		pr_debug("swsusp: pages needed: %u + %u + %u, available pages: %u\n",
		nr_pages, PAGES_FOR_IO, meta, free);

		return free > nr_pages + PAGES_FOR_IO + meta;
		}

		static int
		@@ -961,11 +874,6 @@ asmlinkage int swsusp_save(void)
		nr_pages = count_data_pages();
		printk("swsusp: Need to copy %u pages\n", nr_pages);

		pr_debug("swsusp: pages needed: %u + %lu + %u, free: %u\n",
		nr_pages,
		(nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE,
		PAGES_FOR_IO, nr_free_pages());

		if (!enough_free_mem(nr_pages)) {
		printk(KERN_ERR "swsusp: Not enough free memory\n");
		return -ENOMEM;
		@@ -1007,22 +915,19 @@ static void init_header(struct swsusp_info *info)
		}

		/**
		* pack_addresses - the addresses corresponding to pfns found in the
		* bitmap @bm are stored in the array @buf[] (1 page)
		* pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
		* are stored in the array @buf[] (1 page at a time)
		*/

		static inline void
		pack_addresses(unsigned long buf, struct memory_bitmap bm)
		pack_pfns(unsigned long buf, struct memory_bitmap bm)
		{
		int j;

		for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
		unsigned long pfn = memory_bm_next_pfn(bm);

		if (unlikely(pfn == BM_END_OF_MAP))
		buf[j] = memory_bm_next_pfn(bm);
		if (unlikely(buf[j] == BM_END_OF_MAP))
		break;

		buf[j] = (unsigned long)page_address(pfn_to_page(pfn));
		}
		}

		@@ -1068,7 +973,7 @@ int snapshot_read_next(struct snapshot_handle *handle, size_t count)
		if (handle->prev < handle->cur) {
		if (handle->cur <= nr_meta_pages) {
		memset(buffer, 0, PAGE_SIZE);
		pack_addresses(buffer, &orig_bm);
		pack_pfns(buffer, &orig_bm);
		} else {
		unsigned long pfn = memory_bm_next_pfn(&copy_bm);

		@@ -1094,14 +999,10 @@ int snapshot_read_next(struct snapshot_handle *handle, size_t count)
		* had been used before suspend
		*/

		static int mark_unsafe_pages(struct pbe *pblist)
		static int mark_unsafe_pages(struct memory_bitmap *bm)
		{
		struct zone *zone;
		unsigned long pfn, max_zone_pfn;
		struct pbe *p;

		if (!pblist) /* a sanity check */
		return -EINVAL;

		/* Clear page flags */
		for_each_zone (zone) {
		@@ -1111,30 +1012,37 @@ static int mark_unsafe_pages(struct pbe *pblist)
		ClearPageNosaveFree(pfn_to_page(pfn));
		}

		/* Mark orig addresses */
		for_each_pbe (p, pblist) {
		if (virt_addr_valid(p->orig_address))
		SetPageNosaveFree(virt_to_page(p->orig_address));
		/* Mark pages that correspond to the "original" pfns as "unsafe" */
		memory_bm_position_reset(bm);
		do {
		pfn = memory_bm_next_pfn(bm);
		if (likely(pfn != BM_END_OF_MAP)) {
		if (likely(pfn_valid(pfn)))
		SetPageNosaveFree(pfn_to_page(pfn));
		else
		return -EFAULT;
		}
		} while (pfn != BM_END_OF_MAP);

		unsafe_pages = 0;
		allocated_unsafe_pages = 0;

		return 0;
		}

		static void copy_page_backup_list(struct pbe dst, struct pbe src)
		static void
		duplicate_memory_bitmap(struct memory_bitmap dst, struct memory_bitmap src)
		{
		/* We assume both lists contain the same number of elements */
		while (src) {
		dst->orig_address = src->orig_address;
		dst = dst->next;
		src = src->next;
		unsigned long pfn;

		memory_bm_position_reset(src);
		pfn = memory_bm_next_pfn(src);
		while (pfn != BM_END_OF_MAP) {
		memory_bm_set_bit(dst, pfn);
		pfn = memory_bm_next_pfn(src);
		}
		}

		static int check_header(struct swsusp_info *info)
		static inline int check_header(struct swsusp_info *info)
		{
		char *reason = NULL;

		@@ -1161,19 +1069,14 @@ static int check_header(struct swsusp_info *info)
		* load header - check the image header and copy data from it
		*/

		static int load_header(struct snapshot_handle *handle,
		struct swsusp_info *info)
		static int
		load_header(struct swsusp_info *info)
		{
		int error;
		struct pbe *pblist;

		restore_pblist = NULL;
		error = check_header(info);
		if (!error) {
		pblist = alloc_pagedir(info->image_pages, GFP_ATOMIC, PG_ANY);
		if (!pblist)
		return -ENOMEM;
		restore_pblist = pblist;
		handle->pbe = pblist;
		nr_copy_pages = info->image_pages;
		nr_meta_pages = info->pages - info->image_pages - 1;
		}
		@@ -1181,108 +1084,137 @@ static int load_header(struct snapshot_handle *handle,
		}

		/**
		* unpack_orig_addresses - copy the elements of @buf[] (1 page) to
		* the PBEs in the list starting at @pbe
		* unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
		* the corresponding bit in the memory bitmap @bm
		*/

		static inline struct pbe unpack_orig_addresses(unsigned long buf,
		struct pbe *pbe)
		static inline void
		unpack_orig_pfns(unsigned long buf, struct memory_bitmap bm)
		{
		int j;

		for (j = 0; j < PAGE_SIZE / sizeof(long) && pbe; j++) {
		pbe->orig_address = buf[j];
		pbe = pbe->next;
		for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
		if (unlikely(buf[j] == BM_END_OF_MAP))
		break;

		memory_bm_set_bit(bm, buf[j]);
		}
		return pbe;
		}

		/**
		* prepare_image - use metadata contained in the PBE list
		* pointed to by restore_pblist to mark the pages that will
		* be overwritten in the process of restoring the system
		* memory state from the image ("unsafe" pages) and allocate
		* memory for the image
		* prepare_image - use the memory bitmap @bm to mark the pages that will
		* be overwritten in the process of restoring the system memory state
		* from the suspend image ("unsafe" pages) and allocate memory for the
		* image.
		*
		* The idea is to allocate the PBE list first and then
		* allocate as many pages as it's needed for the image data,
		* but not to assign these pages to the PBEs initially.
		* Instead, we just mark them as allocated and create a list
		* of "safe" which will be used later
		* The idea is to allocate a new memory bitmap first and then allocate
		* as many pages as needed for the image data, but not to assign these
		* pages to specific tasks initially. Instead, we just mark them as
		* allocated and create a list of "safe" pages that will be used later.
		*/

		static struct linked_page *safe_pages;
		#define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))

		static struct linked_page *safe_pages_list;

		static int prepare_image(struct snapshot_handle *handle)
		static int
		prepare_image(struct memory_bitmap new_bm, struct memory_bitmap bm)
		{
		int error = 0;
		unsigned int nr_pages = nr_copy_pages;
		struct pbe p, pblist = NULL;
		unsigned int nr_pages;
		struct linked_page sp_list, lp;
		int error;

		p = restore_pblist;
		error = mark_unsafe_pages(p);
		if (!error) {
		pblist = alloc_pagedir(nr_pages, GFP_ATOMIC, PG_SAFE);
		if (pblist)
		copy_page_backup_list(pblist, p);
		free_pagedir(p, PG_UNSAFE_KEEP);
		if (!pblist)
		error = mark_unsafe_pages(bm);
		if (error)
		goto Free;

		error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
		if (error)
		goto Free;

		duplicate_memory_bitmap(new_bm, bm);
		memory_bm_free(bm, PG_UNSAFE_KEEP);
		/* Reserve some safe pages for potential later use.
		*
		* NOTE: This way we make sure there will be enough safe pages for the
		* chain_alloc() in get_buffer(). It is a bit wasteful, but
		* nr_copy_pages cannot be greater than 50% of the memory anyway.
		*/
		sp_list = NULL;
		/* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
		nr_pages = nr_copy_pages - allocated_unsafe_pages;
		nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
		while (nr_pages > 0) {
		lp = alloc_image_page(GFP_ATOMIC, PG_SAFE);
		if (!lp) {
		error = -ENOMEM;
		goto Free;
		}
		safe_pages = NULL;
		if (!error && nr_pages > unsafe_pages) {
		nr_pages -= unsafe_pages;
		while (nr_pages--) {
		struct linked_page *ptr;

		ptr = (void *)get_zeroed_page(GFP_ATOMIC);
		if (!ptr) {
		lp->next = sp_list;
		sp_list = lp;
		nr_pages--;
		}
		/* Preallocate memory for the image */
		safe_pages_list = NULL;
		nr_pages = nr_copy_pages - allocated_unsafe_pages;
		while (nr_pages > 0) {
		lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
		if (!lp) {
		error = -ENOMEM;
		break;
		goto Free;
		}
		if (!PageNosaveFree(virt_to_page(ptr))) {
		if (!PageNosaveFree(virt_to_page(lp))) {
		/* The page is "safe", add it to the list */
		ptr->next = safe_pages;
		safe_pages = ptr;
		lp->next = safe_pages_list;
		safe_pages_list = lp;
		}
		/* Mark the page as allocated */
		SetPageNosave(virt_to_page(ptr));
		SetPageNosaveFree(virt_to_page(ptr));
		SetPageNosave(virt_to_page(lp));
		SetPageNosaveFree(virt_to_page(lp));
		nr_pages--;
		}
		/* Free the reserved safe pages so that chain_alloc() can use them */
		while (sp_list) {
		lp = sp_list->next;
		free_image_page(sp_list, PG_UNSAFE_CLEAR);
		sp_list = lp;
		}
		if (!error) {
		restore_pblist = pblist;
		} else {
		handle->pbe = NULL;
		return 0;

		Free:
		swsusp_free();
		}
		return error;
		}

		static void get_buffer(struct snapshot_handle handle)
		/**
		* get_buffer - compute the address that snapshot_write_next() should
		* set for its caller to write to.
		*/

		static void get_buffer(struct memory_bitmap bm, struct chain_allocator *ca)
		{
		struct pbe pbe = handle->pbe, last = handle->last_pbe;
		struct page *page = virt_to_page(pbe->orig_address);
		struct pbe *pbe;
		struct page *page = pfn_to_page(memory_bm_next_pfn(bm));

		if (PageNosave(page) && PageNosaveFree(page)) {
		/*
		* We have allocated the "original" page frame and we can
		* use it directly to store the read page
		if (PageNosave(page) && PageNosaveFree(page))
		/* We have allocated the "original" page frame and we can
		* use it directly to store the loaded page.
		*/
		pbe->address = 0;
		if (last && last->next)
		last->next = NULL;
		return (void *)pbe->orig_address;
		}
		/*
		* The "original" page frame has not been allocated and we have to
		* use a "safe" page frame to store the read page
		return page_address(page);

		/* The "original" page frame has not been allocated and we have to
		* use a "safe" page frame to store the loaded page.
		*/
		pbe->address = (unsigned long)safe_pages;
		safe_pages = safe_pages->next;
		if (last)
		last->next = pbe;
		handle->last_pbe = pbe;
		pbe = chain_alloc(ca, sizeof(struct pbe));
		if (!pbe) {
		swsusp_free();
		return NULL;
		}
		pbe->orig_address = (unsigned long)page_address(page);
		pbe->address = (unsigned long)safe_pages_list;
		safe_pages_list = safe_pages_list->next;
		pbe->next = restore_pblist;
		restore_pblist = pbe;
		return (void *)pbe->address;
		}

		@@ -1310,10 +1242,13 @@ static void get_buffer(struct snapshot_handle handle)

		int snapshot_write_next(struct snapshot_handle *handle, size_t count)
		{
		static struct chain_allocator ca;
		int error = 0;

		/* Check if we have already loaded the entire image */
		if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages)
		return 0;

		if (!buffer) {
		/* This makes the buffer be freed by swsusp_free() */
		buffer = alloc_image_page(GFP_ATOMIC, PG_ANY);
		@@ -1324,26 +1259,32 @@ int snapshot_write_next(struct snapshot_handle *handle, size_t count)
		handle->buffer = buffer;
		handle->sync_read = 1;
		if (handle->prev < handle->cur) {
		if (!handle->prev) {
		error = load_header(handle,
		(struct swsusp_info *)buffer);
		if (handle->prev == 0) {
		error = load_header(buffer);
		if (error)
		return error;

		error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
		if (error)
		return error;

		} else if (handle->prev <= nr_meta_pages) {
		handle->pbe = unpack_orig_addresses(buffer,
		handle->pbe);
		if (!handle->pbe) {
		error = prepare_image(handle);
		unpack_orig_pfns(buffer, &copy_bm);
		if (handle->prev == nr_meta_pages) {
		error = prepare_image(&orig_bm, &copy_bm);
		if (error)
		return error;
		handle->pbe = restore_pblist;
		handle->last_pbe = NULL;
		handle->buffer = get_buffer(handle);

		chain_init(&ca, GFP_ATOMIC, PG_SAFE);
		memory_bm_position_reset(&orig_bm);
		restore_pblist = NULL;
		handle->buffer = get_buffer(&orig_bm, &ca);
		handle->sync_read = 0;
		if (!handle->buffer)
		return -ENOMEM;
		}
		} else {
		handle->pbe = handle->pbe->next;
		handle->buffer = get_buffer(handle);
		handle->buffer = get_buffer(&orig_bm, &ca);
		handle->sync_read = 0;
		}
		handle->prev = handle->cur;
		@@ -1362,6 +1303,13 @@ int snapshot_write_next(struct snapshot_handle *handle, size_t count)

		int snapshot_image_loaded(struct snapshot_handle *handle)
		{
		return !(!handle->pbe \|\| handle->pbe->next \|\| !nr_copy_pages \|\|
		return !(!nr_copy_pages \|\|
		handle->cur <= nr_meta_pages + nr_copy_pages);
		}

		void snapshot_free_unused_memory(struct snapshot_handle *handle)
		{
		/* Free only if we have loaded the image entirely */
		if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages)
		memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
		}

kernel/power/swap.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -331,8 +331,7 @@ static int enough_swap(unsigned int nr_pages)
		unsigned int free_swap = count_swap_pages(root_swap, 1);

		pr_debug("swsusp: free swap pages: %u\n", free_swap);
		return free_swap > (nr_pages + PAGES_FOR_IO +
		(nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE);
		return free_swap > nr_pages + PAGES_FOR_IO;
		}

		/**
		@@ -547,6 +546,7 @@ static int load_image(struct swap_map_handle *handle,
		error = err2;
		if (!error) {
		printk("\b\b\b\bdone\n");
		snapshot_free_unused_memory(snapshot);
		if (!snapshot_image_loaded(snapshot))
		error = -ENODATA;
		}

kernel/power/user.c

+1 −0

Original line number	Diff line number	Diff line
		@@ -193,6 +193,7 @@ static int snapshot_ioctl(struct inode inode, struct file filp,
		error = -EPERM;
		break;
		}
		snapshot_free_unused_memory(&data->handle);
		down(&pm_sem);
		pm_prepare_console();
		error = device_suspend(PMSG_FREEZE);

Admin message