Commit 34c0fd54 authored by Dan Williams's avatar Dan Williams Committed by Linus Torvalds
Browse files

mm, dax, pmem: introduce pfn_t 



For the purpose of communicating the optional presence of a 'struct
page' for the pfn returned from ->direct_access(), introduce a type that
encapsulates a page-frame-number plus flags.  These flags contain the
historical "page_link" encoding for a scatterlist entry, but can also
denote "device memory".  Where "device memory" is a set of pfns that are
not part of the kernel's linear mapping by default, but are accessed via
the same memory controller as ram.

The motivation for this new type is large capacity persistent memory
that needs struct page entries in the 'memmap' to support 3rd party DMA
(i.e.  O_DIRECT I/O with a persistent memory source/target).  However,
we also need it in support of maintaining a list of mapped inodes which
need to be unmapped at driver teardown or freeze_bdev() time.

Signed-off-by: default avatarDan Williams <dan.j.williams@intel.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Dave Hansen <dave@sr71.net>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent ba049e93

arch/powerpc/sysdev/axonram.c

+4 −5
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ 
#include <linux/types.h>

#include <linux/of_device.h>

#include <linux/of_platform.h>

#include <linux/pfn_t.h>



#include <asm/page.h>

#include <asm/prom.h>
@@ -142,15 +143,13 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio) 
 */

static long

axon_ram_direct_access(struct block_device *device, sector_t sector,

		       void __pmem **kaddr, unsigned long *pfn)

		       void __pmem **kaddr, pfn_t *pfn)

{

	struct axon_ram_bank *bank = device->bd_disk->private_data;

	loff_t offset = (loff_t)sector << AXON_RAM_SECTOR_SHIFT;

	void *addr = (void *)(bank->ph_addr + offset);



	*kaddr = (void __pmem *)addr;

	*pfn = virt_to_phys(addr) >> PAGE_SHIFT;



	*kaddr = (void __pmem __force *) bank->io_addr + offset;

	*pfn = phys_to_pfn_t(bank->ph_addr + offset, PFN_DEV);

	return bank->size - offset;

}

drivers/block/brd.c

+5 −2
Original line number Diff line number Diff line
@@ -19,6 +19,9 @@ 
#include <linux/radix-tree.h>

#include <linux/fs.h>

#include <linux/slab.h>

#ifdef CONFIG_BLK_DEV_RAM_DAX

#include <linux/pfn_t.h>

#endif



#include <asm/uaccess.h>
@@ -378,7 +381,7 @@ static int brd_rw_page(struct block_device *bdev, sector_t sector, 


#ifdef CONFIG_BLK_DEV_RAM_DAX

static long brd_direct_access(struct block_device *bdev, sector_t sector,

			void __pmem **kaddr, unsigned long *pfn)

			void __pmem **kaddr, pfn_t *pfn)

{

	struct brd_device *brd = bdev->bd_disk->private_data;

	struct page *page;
@@ -389,7 +392,7 @@ static long brd_direct_access(struct block_device *bdev, sector_t sector, 
	if (!page)

		return -ENOSPC;

	*kaddr = (void __pmem *)page_address(page);

	*pfn = page_to_pfn(page);

	*pfn = page_to_pfn_t(page);



	return PAGE_SIZE;

}

drivers/nvdimm/pmem.c

+9 −4
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include <linux/moduleparam.h>

#include <linux/badblocks.h>

#include <linux/vmalloc.h>

#include <linux/pfn_t.h>

#include <linux/slab.h>

#include <linux/pmem.h>

#include <linux/nd.h>
@@ -40,6 +41,7 @@ struct pmem_device { 
	phys_addr_t		phys_addr;

	/* when non-zero this device is hosting a 'pfn' instance */

	phys_addr_t		data_offset;

	unsigned long		pfn_flags;

	void __pmem		*virt_addr;

	size_t			size;

	struct badblocks	bb;
@@ -135,13 +137,13 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector, 
}



static long pmem_direct_access(struct block_device *bdev, sector_t sector,

		      void __pmem **kaddr, unsigned long *pfn)

		      void __pmem **kaddr, pfn_t *pfn)

{

	struct pmem_device *pmem = bdev->bd_disk->private_data;

	resource_size_t offset = sector * 512 + pmem->data_offset;



	*kaddr = pmem->virt_addr + offset;

	*pfn = (pmem->phys_addr + offset) >> PAGE_SHIFT;

	*pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);



	return pmem->size - offset;

}
@@ -174,9 +176,11 @@ static struct pmem_device *pmem_alloc(struct device *dev, 
		return ERR_PTR(-EBUSY);

	}



	if (pmem_should_map_pages(dev))

	pmem->pfn_flags = PFN_DEV;

	if (pmem_should_map_pages(dev)) {

		pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, res);

	else

		pmem->pfn_flags |= PFN_MAP;

	} else

		pmem->virt_addr = (void __pmem *) devm_memremap(dev,

				pmem->phys_addr, pmem->size,

				ARCH_MEMREMAP_PMEM);
@@ -384,6 +388,7 @@ static int nvdimm_namespace_attach_pfn(struct nd_namespace_common *ndns) 
	pmem = dev_get_drvdata(dev);

	devm_memunmap(dev, (void __force *) pmem->virt_addr);

	pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, &nsio->res);

	pmem->pfn_flags |= PFN_MAP;

	if (IS_ERR(pmem->virt_addr)) {

		rc = PTR_ERR(pmem->virt_addr);

		goto err;

drivers/s390/block/dcssblk.c

+5 −6
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
#include <linux/completion.h>

#include <linux/interrupt.h>

#include <linux/platform_device.h>

#include <linux/pfn_t.h>

#include <asm/extmem.h>

#include <asm/io.h>
@@ -30,7 +31,7 @@ static void dcssblk_release(struct gendisk *disk, fmode_t mode); 
static blk_qc_t dcssblk_make_request(struct request_queue *q,

						struct bio *bio);

static long dcssblk_direct_access(struct block_device *bdev, sector_t secnum,

			 void __pmem **kaddr, unsigned long *pfn);

			 void __pmem **kaddr, pfn_t *pfn);



static char dcssblk_segments[DCSSBLK_PARM_LEN] = "\0";
@@ -883,20 +884,18 @@ fail: 


static long

dcssblk_direct_access (struct block_device *bdev, sector_t secnum,

			void __pmem **kaddr, unsigned long *pfn)

			void __pmem **kaddr, pfn_t *pfn)

{

	struct dcssblk_dev_info *dev_info;

	unsigned long offset, dev_sz;

	void *addr;



	dev_info = bdev->bd_disk->private_data;

	if (!dev_info)

		return -ENODEV;

	dev_sz = dev_info->end - dev_info->start;

	offset = secnum * 512;

	addr = (void *) (dev_info->start + offset);

	*pfn = virt_to_phys(addr) >> PAGE_SHIFT;

	*kaddr = (void __pmem *) addr;

	*kaddr = (void __pmem *) (dev_info->start + offset);

	*pfn = __pfn_to_pfn_t(PFN_DOWN(dev_info->start + offset), PFN_DEV);



	return dev_sz - offset;

}

fs/dax.c

+7 −4
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ 
#include <linux/sched.h>

#include <linux/uio.h>

#include <linux/vmstat.h>

#include <linux/pfn_t.h>

#include <linux/sizes.h>



static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax)
@@ -362,7 +363,7 @@ static int dax_insert_mapping(struct inode *inode, struct buffer_head *bh, 
	}

	dax_unmap_atomic(bdev, &dax);



	error = vm_insert_mixed(vma, vaddr, dax.pfn);

	error = vm_insert_mixed(vma, vaddr, pfn_t_to_pfn(dax.pfn));



 out:

	i_mmap_unlock_read(mapping);
@@ -667,7 +668,8 @@ int __dax_pmd_fault(struct vm_area_struct *vma, unsigned long address, 
			result = VM_FAULT_SIGBUS;

			goto out;

		}

		if ((length < PMD_SIZE) || (dax.pfn & PG_PMD_COLOUR)) {

		if (length < PMD_SIZE

				|| (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR)) {

			dax_unmap_atomic(bdev, &dax);

			goto fallback;

		}
@@ -676,7 +678,7 @@ int __dax_pmd_fault(struct vm_area_struct *vma, unsigned long address, 
		 * TODO: teach vmf_insert_pfn_pmd() to support

		 * 'pte_special' for pmds

		 */

		if (pfn_valid(dax.pfn)) {

		if (pfn_t_has_page(dax.pfn)) {

			dax_unmap_atomic(bdev, &dax);

			goto fallback;

		}
@@ -690,7 +692,8 @@ int __dax_pmd_fault(struct vm_area_struct *vma, unsigned long address, 
		}

		dax_unmap_atomic(bdev, &dax);



		result |= vmf_insert_pfn_pmd(vma, address, pmd, dax.pfn, write);

		result |= vmf_insert_pfn_pmd(vma, address, pmd,

				pfn_t_to_pfn(dax.pfn), write);

	}



 out:

CRA Git | Maintained and supported by SUSTech CRA and CCSE