mm, swap: fix race between swapoff and some swap operations (eb085574) · Commits · 戴 / test

include/linux/swap.h

+10 −3

Original line number	Diff line number	Diff line
		@@ -175,8 +175,9 @@ enum {
		SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */
		SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */
		SWP_SYNCHRONOUS_IO = (1 << 12), /* synchronous IO is efficient */
		SWP_VALID = (1 << 13), /* swap is valid to be operated on? */
		/* add others here before... */
		SWP_SCANNING = (1 << 13), /* refcount in scan_swap_map */
		SWP_SCANNING = (1 << 14), /* refcount in scan_swap_map */
		};

		#define SWAP_CLUSTER_MAX 32UL
		@@ -460,7 +461,7 @@ extern unsigned int count_swap_pages(int, int);
		extern sector_t map_swap_page(struct page , struct block_device *);
		extern sector_t swapdev_block(int, pgoff_t);
		extern int page_swapcount(struct page *);
		extern int __swap_count(struct swap_info_struct *si, swp_entry_t entry);
		extern int __swap_count(swp_entry_t entry);
		extern int __swp_swapcount(swp_entry_t entry);
		extern int swp_swapcount(swp_entry_t entry);
		extern struct swap_info_struct page_swap_info(struct page );
		@@ -470,6 +471,12 @@ extern int try_to_free_swap(struct page *);
		struct backing_dev_info;
		extern int init_swap_address_space(unsigned int type, unsigned long nr_pages);
		extern void exit_swap_address_space(unsigned int type);
		extern struct swap_info_struct *get_swap_device(swp_entry_t entry);

		static inline void put_swap_device(struct swap_info_struct *si)
		{
		rcu_read_unlock();
		}

		#else /* CONFIG_SWAP */

		@@ -576,7 +583,7 @@ static inline int page_swapcount(struct page *page)
		return 0;
		}

		static inline int __swap_count(struct swap_info_struct *si, swp_entry_t entry)
		static inline int __swap_count(swp_entry_t entry)
		{
		return 0;
		}

mm/memory.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -2805,7 +2805,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
		struct swap_info_struct *si = swp_swap_info(entry);

		if (si->flags & SWP_SYNCHRONOUS_IO &&
		__swap_count(si, entry) == 1) {
		__swap_count(entry) == 1) {
		/* skip swapcache */
		page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
		vmf->address);

mm/swap_state.c

+13 −3

Original line number	Diff line number	Diff line
		@@ -310,8 +310,13 @@ struct page lookup_swap_cache(swp_entry_t entry, struct vm_area_struct vma,
		unsigned long addr)
		{
		struct page *page;
		struct swap_info_struct *si;

		si = get_swap_device(entry);
		if (!si)
		return NULL;
		page = find_get_page(swap_address_space(entry), swp_offset(entry));
		put_swap_device(si);

		INC_CACHE_INFO(find_total);
		if (page) {
		@@ -354,8 +359,8 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
		struct vm_area_struct *vma, unsigned long addr,
		bool *new_page_allocated)
		{
		struct page found_page, new_page = NULL;
		struct address_space *swapper_space = swap_address_space(entry);
		struct page found_page = NULL, new_page = NULL;
		struct swap_info_struct *si;
		int err;
		*new_page_allocated = false;

		@@ -365,7 +370,12 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
		* called after lookup_swap_cache() failed, re-calling
		* that would confuse statistics.
		*/
		found_page = find_get_page(swapper_space, swp_offset(entry));
		si = get_swap_device(entry);
		if (!si)
		break;
		found_page = find_get_page(swap_address_space(entry),
		swp_offset(entry));
		put_swap_device(si);
		if (found_page)
		break;

mm/swapfile.c

+122 −32

Original line number	Diff line number	Diff line
		@@ -1079,12 +1079,11 @@ fail:
		static struct swap_info_struct *__swap_info_get(swp_entry_t entry)
		{
		struct swap_info_struct *p;
		unsigned long offset, type;
		unsigned long offset;

		if (!entry.val)
		goto out;
		type = swp_type(entry);
		p = swap_type_to_swap_info(type);
		p = swp_swap_info(entry);
		if (!p)
		goto bad_nofile;
		if (!(p->flags & SWP_USED))
		@@ -1187,6 +1186,69 @@ static unsigned char __swap_entry_free_locked(struct swap_info_struct *p,
		return usage;
		}

		/*
		* Check whether swap entry is valid in the swap device. If so,
		* return pointer to swap_info_struct, and keep the swap entry valid
		* via preventing the swap device from being swapoff, until
		* put_swap_device() is called. Otherwise return NULL.
		*
		* The entirety of the RCU read critical section must come before the
		* return from or after the call to synchronize_rcu() in
		* enable_swap_info() or swapoff(). So if "si->flags & SWP_VALID" is
		* true, the si->map, si->cluster_info, etc. must be valid in the
		* critical section.
		*
		* Notice that swapoff or swapoff+swapon can still happen before the
		* rcu_read_lock() in get_swap_device() or after the rcu_read_unlock()
		* in put_swap_device() if there isn't any other way to prevent
		* swapoff, such as page lock, page table lock, etc. The caller must
		* be prepared for that. For example, the following situation is
		* possible.
		*
		* CPU1 CPU2
		* do_swap_page()
		* ... swapoff+swapon
		* __read_swap_cache_async()
		* swapcache_prepare()
		* __swap_duplicate()
		* // check swap_map
		* // verify PTE not changed
		*
		* In __swap_duplicate(), the swap_map need to be checked before
		* changing partly because the specified swap entry may be for another
		* swap device which has been swapoff. And in do_swap_page(), after
		* the page is read from the swap device, the PTE is verified not
		* changed with the page table locked to check whether the swap device
		* has been swapoff or swapoff+swapon.
		*/
		struct swap_info_struct *get_swap_device(swp_entry_t entry)
		{
		struct swap_info_struct *si;
		unsigned long offset;

		if (!entry.val)
		goto out;
		si = swp_swap_info(entry);
		if (!si)
		goto bad_nofile;

		rcu_read_lock();
		if (!(si->flags & SWP_VALID))
		goto unlock_out;
		offset = swp_offset(entry);
		if (offset >= si->max)
		goto unlock_out;

		return si;
		bad_nofile:
		pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
		out:
		return NULL;
		unlock_out:
		rcu_read_unlock();
		return NULL;
		}

		static unsigned char __swap_entry_free(struct swap_info_struct *p,
		swp_entry_t entry, unsigned char usage)
		{
		@@ -1358,11 +1420,18 @@ int page_swapcount(struct page *page)
		return count;
		}

		int __swap_count(struct swap_info_struct *si, swp_entry_t entry)
		int __swap_count(swp_entry_t entry)
		{
		struct swap_info_struct *si;
		pgoff_t offset = swp_offset(entry);
		int count = 0;

		return swap_count(si->swap_map[offset]);
		si = get_swap_device(entry);
		if (si) {
		count = swap_count(si->swap_map[offset]);
		put_swap_device(si);
		}
		return count;
		}

		static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
		@@ -1387,9 +1456,11 @@ int __swp_swapcount(swp_entry_t entry)
		int count = 0;
		struct swap_info_struct *si;

		si = __swap_info_get(entry);
		if (si)
		si = get_swap_device(entry);
		if (si) {
		count = swap_swapcount(si, entry);
		put_swap_device(si);
		}
		return count;
		}

		@@ -2335,7 +2406,7 @@ static int swap_node(struct swap_info_struct *p)
		return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE;
		}

		static void _enable_swap_info(struct swap_info_struct *p, int prio,
		static void setup_swap_info(struct swap_info_struct *p, int prio,
		unsigned char *swap_map,
		struct swap_cluster_info *cluster_info)
		{
		@@ -2362,7 +2433,11 @@ static void _enable_swap_info(struct swap_info_struct *p, int prio,
		}
		p->swap_map = swap_map;
		p->cluster_info = cluster_info;
		p->flags \|= SWP_WRITEOK;
		}

		static void _enable_swap_info(struct swap_info_struct *p)
		{
		p->flags \|= SWP_WRITEOK \| SWP_VALID;
		atomic_long_add(p->pages, &nr_swap_pages);
		total_swap_pages += p->pages;

		@@ -2389,7 +2464,17 @@ static void enable_swap_info(struct swap_info_struct *p, int prio,
		frontswap_init(p->type, frontswap_map);
		spin_lock(&swap_lock);
		spin_lock(&p->lock);
		_enable_swap_info(p, prio, swap_map, cluster_info);
		setup_swap_info(p, prio, swap_map, cluster_info);
		spin_unlock(&p->lock);
		spin_unlock(&swap_lock);
		/*
		* Guarantee swap_map, cluster_info, etc. fields are valid
		* between get/put_swap_device() if SWP_VALID bit is set
		*/
		synchronize_rcu();
		spin_lock(&swap_lock);
		spin_lock(&p->lock);
		_enable_swap_info(p);
		spin_unlock(&p->lock);
		spin_unlock(&swap_lock);
		}
		@@ -2398,7 +2483,8 @@ static void reinsert_swap_info(struct swap_info_struct *p)
		{
		spin_lock(&swap_lock);
		spin_lock(&p->lock);
		_enable_swap_info(p, p->prio, p->swap_map, p->cluster_info);
		setup_swap_info(p, p->prio, p->swap_map, p->cluster_info);
		_enable_swap_info(p);
		spin_unlock(&p->lock);
		spin_unlock(&swap_lock);
		}
		@@ -2501,6 +2587,17 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)

		reenable_swap_slots_cache_unlock();

		spin_lock(&swap_lock);
		spin_lock(&p->lock);
		p->flags &= ~SWP_VALID; /* mark swap device as invalid */
		spin_unlock(&p->lock);
		spin_unlock(&swap_lock);
		/*
		* wait for swap operations protected by get/put_swap_device()
		* to complete
		*/
		synchronize_rcu();

		flush_work(&p->discard_work);

		destroy_swap_extents(p);
		@@ -3265,17 +3362,11 @@ static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
		unsigned char has_cache;
		int err = -EINVAL;

		if (non_swap_entry(entry))
		goto out;

		p = swp_swap_info(entry);
		p = get_swap_device(entry);
		if (!p)
		goto bad_file;

		offset = swp_offset(entry);
		if (unlikely(offset >= p->max))
		goto out;

		offset = swp_offset(entry);
		ci = lock_cluster_or_swap_info(p, offset);

		count = p->swap_map[offset];
		@@ -3321,11 +3412,9 @@ static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
		unlock_out:
		unlock_cluster_or_swap_info(p, ci);
		out:
		if (p)
		put_swap_device(p);
		return err;

		bad_file:
		pr_err("swap_dup: %s%08lx\n", Bad_file, entry.val);
		goto out;
		}

		/*
		@@ -3417,6 +3506,7 @@ int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
		struct page *list_page;
		pgoff_t offset;
		unsigned char count;
		int ret = 0;

		/*
		* When debugging, it's easier to use __GFP_ZERO here; but it's better
		@@ -3424,15 +3514,15 @@ int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
		*/
		page = alloc_page(gfp_mask \| __GFP_HIGHMEM);

		si = swap_info_get(entry);
		si = get_swap_device(entry);
		if (!si) {
		/*
		* An acceptable race has occurred since the failing
		* __swap_duplicate(): the swap entry has been freed,
		* perhaps even the whole swap_map cleared for swapoff.
		* __swap_duplicate(): the swap device may be swapoff
		*/
		goto outer;
		}
		spin_lock(&si->lock);

		offset = swp_offset(entry);

		@@ -3450,9 +3540,8 @@ int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
		}

		if (!page) {
		unlock_cluster(ci);
		spin_unlock(&si->lock);
		return -ENOMEM;
		ret = -ENOMEM;
		goto out;
		}

		/*
		@@ -3504,10 +3593,11 @@ out_unlock_cont:
		out:
		unlock_cluster(ci);
		spin_unlock(&si->lock);
		put_swap_device(si);
		outer:
		if (page)
		__free_page(page);
		return 0;
		return ret;
		}

		/*

Admin message