Commit c01d5b30 authored by Hugh Dickins's avatar Hugh Dickins Committed by Linus Torvalds
Browse files

shmem: get_unmapped_area align huge page 

Provide a shmem_get_unmapped_area method in file_operations, called at
mmap time to decide the mapping address.  It could be conditional on
CONFIG_TRANSPARENT_HUGEPAGE, but save #ifdefs in other places by making
it unconditional.

shmem_get_unmapped_area() first calls the usual mm->get_unmapped_area
(which we treat as a black box, highly dependent on architecture and
config and executable layout).  Lots of conditions, and in most cases it
just goes with the address that chose; but when our huge stars are
rightly aligned, yet that did not provide a suitable address, go back to
ask for a larger arena, within which to align the mapping suitably.

There have to be some direct calls to shmem_get_unmapped_area(), not via
the file_operations: because of the way shmem_zero_setup() is called to
create a shmem object late in the mmap sequence, when MAP_SHARED is
requested with MAP_ANONYMOUS or /dev/zero.  Though this only matters
when /proc/sys/vm/shmem_huge has been set.

Link: http://lkml.kernel.org/r/1466021202-61880-29-git-send-email-kirill.shutemov@linux.intel.com


Signed-off-by: default avatarHugh Dickins <hughd@google.com>
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>

Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 5a6e75f8

drivers/char/mem.c

+24 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ 
#include <linux/device.h>

#include <linux/highmem.h>

#include <linux/backing-dev.h>

#include <linux/shmem_fs.h>

#include <linux/splice.h>

#include <linux/pfn.h>

#include <linux/export.h>
@@ -657,6 +658,28 @@ static int mmap_zero(struct file *file, struct vm_area_struct *vma) 
	return 0;

}



static unsigned long get_unmapped_area_zero(struct file *file,

				unsigned long addr, unsigned long len,

				unsigned long pgoff, unsigned long flags)

{

#ifdef CONFIG_MMU

	if (flags & MAP_SHARED) {

		/*

		 * mmap_zero() will call shmem_zero_setup() to create a file,

		 * so use shmem's get_unmapped_area in case it can be huge;

		 * and pass NULL for file as in mmap.c's get_unmapped_area(),

		 * so as not to confuse shmem with our handle on "/dev/zero".

		 */

		return shmem_get_unmapped_area(NULL, addr, len, pgoff, flags);

	}



	/* Otherwise flags & MAP_PRIVATE: with no shmem object beneath it */

	return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);

#else

	return -ENOSYS;

#endif

}



static ssize_t write_full(struct file *file, const char __user *buf,

			  size_t count, loff_t *ppos)

{
@@ -764,6 +787,7 @@ static const struct file_operations zero_fops = { 
	.read_iter	= read_iter_zero,

	.write_iter	= write_iter_zero,

	.mmap		= mmap_zero,

	.get_unmapped_area = get_unmapped_area_zero,

#ifndef CONFIG_MMU

	.mmap_capabilities = zero_mmap_capabilities,

#endif

include/linux/shmem_fs.h

+2 −0
Original line number Diff line number Diff line
@@ -50,6 +50,8 @@ extern struct file *shmem_file_setup(const char *name, 
extern struct file *shmem_kernel_file_setup(const char *name, loff_t size,

					    unsigned long flags);

extern int shmem_zero_setup(struct vm_area_struct *);

extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr,

		unsigned long len, unsigned long pgoff, unsigned long flags);

extern int shmem_lock(struct file *file, int lock, struct user_struct *user);

extern bool shmem_mapping(struct address_space *mapping);

extern void shmem_unlock_mapping(struct address_space *mapping);

ipc/shm.c

+4 −2
Original line number Diff line number Diff line
@@ -476,13 +476,15 @@ static const struct file_operations shm_file_operations = { 
	.mmap		= shm_mmap,

	.fsync		= shm_fsync,

	.release	= shm_release,

#ifndef CONFIG_MMU

	.get_unmapped_area	= shm_get_unmapped_area,

#endif

	.llseek		= noop_llseek,

	.fallocate	= shm_fallocate,

};



/*

 * shm_file_operations_huge is now identical to shm_file_operations,

 * but we keep it distinct for the sake of is_file_shm_hugepages().

 */

static const struct file_operations shm_file_operations_huge = {

	.mmap		= shm_mmap,

	.fsync		= shm_fsync,

mm/mmap.c

+14 −2
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include <linux/personality.h>

#include <linux/security.h>

#include <linux/hugetlb.h>

#include <linux/shmem_fs.h>

#include <linux/profile.h>

#include <linux/export.h>

#include <linux/mount.h>
@@ -1897,8 +1898,19 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 
		return -ENOMEM;



	get_area = current->mm->get_unmapped_area;

	if (file && file->f_op->get_unmapped_area)

	if (file) {

		if (file->f_op->get_unmapped_area)

			get_area = file->f_op->get_unmapped_area;

	} else if (flags & MAP_SHARED) {

		/*

		 * mmap_region() will call shmem_zero_setup() to create a file,

		 * so use shmem's get_unmapped_area in case it can be huge.

		 * do_mmap_pgoff() will clear pgoff, so match alignment.

		 */

		pgoff = 0;

		get_area = shmem_get_unmapped_area;

	}



	addr = get_area(file, addr, len, pgoff, flags);

	if (IS_ERR_VALUE(addr))

		return addr;

mm/shmem.c

+98 −0
Original line number Diff line number Diff line
@@ -1513,6 +1513,94 @@ static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 
	return ret;

}



unsigned long shmem_get_unmapped_area(struct file *file,

				      unsigned long uaddr, unsigned long len,

				      unsigned long pgoff, unsigned long flags)

{

	unsigned long (*get_area)(struct file *,

		unsigned long, unsigned long, unsigned long, unsigned long);

	unsigned long addr;

	unsigned long offset;

	unsigned long inflated_len;

	unsigned long inflated_addr;

	unsigned long inflated_offset;



	if (len > TASK_SIZE)

		return -ENOMEM;



	get_area = current->mm->get_unmapped_area;

	addr = get_area(file, uaddr, len, pgoff, flags);



	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))

		return addr;

	if (IS_ERR_VALUE(addr))

		return addr;

	if (addr & ~PAGE_MASK)

		return addr;

	if (addr > TASK_SIZE - len)

		return addr;



	if (shmem_huge == SHMEM_HUGE_DENY)

		return addr;

	if (len < HPAGE_PMD_SIZE)

		return addr;

	if (flags & MAP_FIXED)

		return addr;

	/*

	 * Our priority is to support MAP_SHARED mapped hugely;

	 * and support MAP_PRIVATE mapped hugely too, until it is COWed.

	 * But if caller specified an address hint, respect that as before.

	 */

	if (uaddr)

		return addr;



	if (shmem_huge != SHMEM_HUGE_FORCE) {

		struct super_block *sb;



		if (file) {

			VM_BUG_ON(file->f_op != &shmem_file_operations);

			sb = file_inode(file)->i_sb;

		} else {

			/*

			 * Called directly from mm/mmap.c, or drivers/char/mem.c

			 * for "/dev/zero", to create a shared anonymous object.

			 */

			if (IS_ERR(shm_mnt))

				return addr;

			sb = shm_mnt->mnt_sb;

		}

		if (SHMEM_SB(sb)->huge != SHMEM_HUGE_NEVER)

			return addr;

	}



	offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);

	if (offset && offset + len < 2 * HPAGE_PMD_SIZE)

		return addr;

	if ((addr & (HPAGE_PMD_SIZE-1)) == offset)

		return addr;



	inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;

	if (inflated_len > TASK_SIZE)

		return addr;

	if (inflated_len < len)

		return addr;



	inflated_addr = get_area(NULL, 0, inflated_len, 0, flags);

	if (IS_ERR_VALUE(inflated_addr))

		return addr;

	if (inflated_addr & ~PAGE_MASK)

		return addr;



	inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);

	inflated_addr += offset - inflated_offset;

	if (inflated_offset > offset)

		inflated_addr += HPAGE_PMD_SIZE;



	if (inflated_addr > TASK_SIZE - len)

		return addr;

	return inflated_addr;

}



#ifdef CONFIG_NUMA

static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)

{
@@ -3261,6 +3349,7 @@ static const struct address_space_operations shmem_aops = { 


static const struct file_operations shmem_file_operations = {

	.mmap		= shmem_mmap,

	.get_unmapped_area = shmem_get_unmapped_area,

#ifdef CONFIG_TMPFS

	.llseek		= shmem_file_llseek,

	.read_iter	= shmem_file_read_iter,
@@ -3496,6 +3585,15 @@ void shmem_unlock_mapping(struct address_space *mapping) 
{

}



#ifdef CONFIG_MMU

unsigned long shmem_get_unmapped_area(struct file *file,

				      unsigned long addr, unsigned long len,

				      unsigned long pgoff, unsigned long flags)

{

	return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);

}

#endif



void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)

{

	truncate_inode_pages_range(inode->i_mapping, lstart, lend);

CRA Git | Maintained and supported by SUSTech CRA and CCSE