misc: mic: SCIF remote memory map/unmap interface

/*

 * Intel MIC Platform Software Stack (MPSS)

 *

 * Copyright(c) 2015 Intel Corporation.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License, version 2, as

 * published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful, but

 * WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

 * General Public License for more details.

 *

 * Intel SCIF driver.

 *

 */

#include "scif_main.h"

/*

 * struct scif_vma_info - Information about a remote memory mapping

 *			  created via scif_mmap(..)

 * @vma: VM area struct

 * @list: link to list of active vmas

 */

struct scif_vma_info {

	struct vm_area_struct *vma;

	struct list_head list;

};

void scif_recv_munmap(struct scif_dev *scifdev, struct scifmsg *msg)

{

	struct scif_rma_req req;

	struct scif_window *window = NULL;

	struct scif_window *recv_window =

		(struct scif_window *)msg->payload[0];

	struct scif_endpt *ep;

	ep = (struct scif_endpt *)recv_window->ep;

	req.out_window = &window;

	req.offset = recv_window->offset;

	req.prot = recv_window->prot;

	req.nr_bytes = recv_window->nr_pages << PAGE_SHIFT;

	req.type = SCIF_WINDOW_FULL;

	req.head = &ep->rma_info.reg_list;

	msg->payload[0] = ep->remote_ep;

	mutex_lock(&ep->rma_info.rma_lock);

	/* Does a valid window exist? */

	if (scif_query_window(&req)) {

		dev_err(&scifdev->sdev->dev,

			"%s %d -ENXIO\n", __func__, __LINE__);

		msg->uop = SCIF_UNREGISTER_ACK;

		goto error;

	}

	scif_put_window(window, window->nr_pages);

	if (!window->ref_count) {

		atomic_inc(&ep->rma_info.tw_refcount);

		ep->rma_info.async_list_del = 1;

		list_del_init(&window->list);

		scif_free_window_offset(ep, window, window->offset);

	}

error:

	mutex_unlock(&ep->rma_info.rma_lock);

	if (window && !window->ref_count)

		scif_queue_for_cleanup(window, &scif_info.rma);

}

/*

 * Remove valid remote memory mappings created via scif_mmap(..) from the

 * process address space since the remote node is lost

 */

static void __scif_zap_mmaps(struct scif_endpt *ep)

{

	struct list_head *item;

	struct scif_vma_info *info;

	struct vm_area_struct *vma;

	unsigned long size;

	spin_lock(&ep->lock);

	list_for_each(item, &ep->rma_info.vma_list) {

		info = list_entry(item, struct scif_vma_info, list);

		vma = info->vma;

		size = vma->vm_end - vma->vm_start;

		zap_vma_ptes(vma, vma->vm_start, size);

		dev_dbg(scif_info.mdev.this_device,

			"%s ep %p zap vma %p size 0x%lx\n",

			__func__, ep, info->vma, size);

	}

	spin_unlock(&ep->lock);

}

/*

 * Traverse the list of endpoints for a particular remote node and

 * zap valid remote memory mappings since the remote node is lost

 */

static void _scif_zap_mmaps(int node, struct list_head *head)

{

	struct scif_endpt *ep;

	struct list_head *item;

	mutex_lock(&scif_info.connlock);

	list_for_each(item, head) {

		ep = list_entry(item, struct scif_endpt, list);

		if (ep->remote_dev->node == node)

			__scif_zap_mmaps(ep);

	}

	mutex_unlock(&scif_info.connlock);

}

/*

 * Wrapper for removing remote memory mappings for a particular node. This API

 * is called by peer nodes as part of handling a lost node.

 */

void scif_zap_mmaps(int node)

{

	_scif_zap_mmaps(node, &scif_info.connected);

	_scif_zap_mmaps(node, &scif_info.disconnected);

}

/*

 * This API is only called while handling a lost node:

 * a) Remote node is dead.

 * b) Remote memory mappings have been zapped

 * So we can traverse the remote_reg_list without any locks. Since

 * the window has not yet been unregistered we can drop the ref count

 * and queue it to the cleanup thread.

 */

static void __scif_cleanup_rma_for_zombies(struct scif_endpt *ep)

{

	struct list_head *pos, *tmp;

	struct scif_window *window;

	list_for_each_safe(pos, tmp, &ep->rma_info.remote_reg_list) {

		window = list_entry(pos, struct scif_window, list);

		if (window->ref_count)

			scif_put_window(window, window->nr_pages);

		else

			dev_err(scif_info.mdev.this_device,

				"%s %d unexpected\n",

				__func__, __LINE__);

		if (!window->ref_count) {

			atomic_inc(&ep->rma_info.tw_refcount);

			list_del_init(&window->list);

			scif_queue_for_cleanup(window, &scif_info.rma);

		}

	}

}

/* Cleanup remote registration lists for zombie endpoints */

void scif_cleanup_rma_for_zombies(int node)

{

	struct scif_endpt *ep;

	struct list_head *item;

	mutex_lock(&scif_info.eplock);

	list_for_each(item, &scif_info.zombie) {

		ep = list_entry(item, struct scif_endpt, list);

		if (ep->remote_dev && ep->remote_dev->node == node)

			__scif_cleanup_rma_for_zombies(ep);

	}

	mutex_unlock(&scif_info.eplock);

	flush_work(&scif_info.misc_work);

}

/* Insert the VMA into the per endpoint VMA list */

static int scif_insert_vma(struct scif_endpt *ep, struct vm_area_struct *vma)

{

	struct scif_vma_info *info;

	int err = 0;

	info = kzalloc(sizeof(*info), GFP_KERNEL);

	if (!info) {

		err = -ENOMEM;

		goto done;

	}

	info->vma = vma;

	spin_lock(&ep->lock);

	list_add_tail(&info->list, &ep->rma_info.vma_list);

	spin_unlock(&ep->lock);

done:

	return err;

}

/* Delete the VMA from the per endpoint VMA list */

static void scif_delete_vma(struct scif_endpt *ep, struct vm_area_struct *vma)

{

	struct list_head *item;

	struct scif_vma_info *info;

	spin_lock(&ep->lock);

	list_for_each(item, &ep->rma_info.vma_list) {

		info = list_entry(item, struct scif_vma_info, list);

		if (info->vma == vma) {

			list_del(&info->list);

			kfree(info);

			break;

		}

	}

	spin_unlock(&ep->lock);

}

static phys_addr_t scif_get_phys(phys_addr_t phys, struct scif_endpt *ep)

{

	struct scif_dev *scifdev = (struct scif_dev *)ep->remote_dev;

	struct scif_hw_dev *sdev = scifdev->sdev;

	phys_addr_t out_phys, apt_base = 0;

	/*

	 * If the DMA address is card relative then we need to add the

	 * aperture base for mmap to work correctly

	 */

	if (!scifdev_self(scifdev) && sdev->aper && sdev->card_rel_da)

		apt_base = sdev->aper->pa;

	out_phys = apt_base + phys;

	return out_phys;

}

int scif_get_pages(scif_epd_t epd, off_t offset, size_t len,

		   struct scif_range **pages)

{

	struct scif_endpt *ep = (struct scif_endpt *)epd;

	struct scif_rma_req req;

	struct scif_window *window = NULL;

	int nr_pages, err, i;

	dev_dbg(scif_info.mdev.this_device,

		"SCIFAPI get_pinned_pages: ep %p offset 0x%lx len 0x%lx\n",

		ep, offset, len);

	err = scif_verify_epd(ep);

	if (err)

		return err;

	if (!len || (offset < 0) ||

	    (offset + len < offset) ||

	    (ALIGN(offset, PAGE_SIZE) != offset) ||

	    (ALIGN(len, PAGE_SIZE) != len))

		return -EINVAL;

	nr_pages = len >> PAGE_SHIFT;

	req.out_window = &window;

	req.offset = offset;

	req.prot = 0;

	req.nr_bytes = len;

	req.type = SCIF_WINDOW_SINGLE;

	req.head = &ep->rma_info.remote_reg_list;

	mutex_lock(&ep->rma_info.rma_lock);

	/* Does a valid window exist? */

	err = scif_query_window(&req);

	if (err) {

		dev_err(&ep->remote_dev->sdev->dev,

			"%s %d err %d\n", __func__, __LINE__, err);

		goto error;

	}

	/* Allocate scif_range */

	*pages = kzalloc(sizeof(**pages), GFP_KERNEL);

	if (!*pages) {

		err = -ENOMEM;

		goto error;

	}

	/* Allocate phys addr array */

	(*pages)->phys_addr = scif_zalloc(nr_pages * sizeof(dma_addr_t));

	if (!((*pages)->phys_addr)) {

		err = -ENOMEM;

		goto error;

	}

	if (scif_is_mgmt_node() && !scifdev_self(ep->remote_dev)) {

		/* Allocate virtual address array */

		((*pages)->va = scif_zalloc(nr_pages * sizeof(void *)));

		if (!(*pages)->va) {

			err = -ENOMEM;

			goto error;

		}

	}

	/* Populate the values */

	(*pages)->cookie = window;

	(*pages)->nr_pages = nr_pages;

	(*pages)->prot_flags = window->prot;

	for (i = 0; i < nr_pages; i++) {

		(*pages)->phys_addr[i] =

			__scif_off_to_dma_addr(window, offset +

					       (i * PAGE_SIZE));

		(*pages)->phys_addr[i] = scif_get_phys((*pages)->phys_addr[i],

							ep);

		if (scif_is_mgmt_node() && !scifdev_self(ep->remote_dev))

			(*pages)->va[i] =

				ep->remote_dev->sdev->aper->va +

				(*pages)->phys_addr[i] -

				ep->remote_dev->sdev->aper->pa;

	}

	scif_get_window(window, nr_pages);

error:

	mutex_unlock(&ep->rma_info.rma_lock);

	if (err) {

		if (*pages) {

			scif_free((*pages)->phys_addr,

				  nr_pages * sizeof(dma_addr_t));

			scif_free((*pages)->va,

				  nr_pages * sizeof(void *));

			kfree(*pages);

			*pages = NULL;

		}

		dev_err(&ep->remote_dev->sdev->dev,

			"%s %d err %d\n", __func__, __LINE__, err);

	}

	return err;

}

EXPORT_SYMBOL_GPL(scif_get_pages);

int scif_put_pages(struct scif_range *pages)

{

	struct scif_endpt *ep;

	struct scif_window *window;

	struct scifmsg msg;

	if (!pages || !pages->cookie)

		return -EINVAL;

	window = pages->cookie;

	if (!window || window->magic != SCIFEP_MAGIC)

		return -EINVAL;

	ep = (struct scif_endpt *)window->ep;

	/*

	 * If the state is SCIFEP_CONNECTED or SCIFEP_DISCONNECTED then the

	 * callee should be allowed to release references to the pages,

	 * else the endpoint was not connected in the first place,

	 * hence the ENOTCONN.

	 */

	if (ep->state != SCIFEP_CONNECTED && ep->state != SCIFEP_DISCONNECTED)

		return -ENOTCONN;

	mutex_lock(&ep->rma_info.rma_lock);

	scif_put_window(window, pages->nr_pages);

	/* Initiate window destruction if ref count is zero */

	if (!window->ref_count) {

		list_del(&window->list);

		mutex_unlock(&ep->rma_info.rma_lock);

		scif_drain_dma_intr(ep->remote_dev->sdev,

				    ep->rma_info.dma_chan);

		/* Inform the peer about this window being destroyed. */

		msg.uop = SCIF_MUNMAP;

		msg.src = ep->port;

		msg.payload[0] = window->peer_window;

		/* No error handling for notification messages */

		scif_nodeqp_send(ep->remote_dev, &msg);

		/* Destroy this window from the peer's registered AS */

		scif_destroy_remote_window(window);

	} else {

		mutex_unlock(&ep->rma_info.rma_lock);

	}

	scif_free(pages->phys_addr, pages->nr_pages * sizeof(dma_addr_t));

	scif_free(pages->va, pages->nr_pages * sizeof(void *));

	kfree(pages);

	return 0;

}

EXPORT_SYMBOL_GPL(scif_put_pages);

/*

 * scif_rma_list_mmap:

 *

 * Traverse the remote registration list starting from start_window:

 * 1) Create VtoP mappings via remap_pfn_range(..)

 * 2) Once step 1) and 2) complete successfully then traverse the range of

 *    windows again and bump the reference count.

 * RMA lock must be held.

 */

static int scif_rma_list_mmap(struct scif_window *start_window, s64 offset,

			      int nr_pages, struct vm_area_struct *vma)

{

	s64 end_offset, loop_offset = offset;

	struct scif_window *window = start_window;

	int loop_nr_pages, nr_pages_left = nr_pages;

	struct scif_endpt *ep = (struct scif_endpt *)start_window->ep;

	struct list_head *head = &ep->rma_info.remote_reg_list;

	int i, err = 0;

	dma_addr_t phys_addr;

	struct scif_window_iter src_win_iter;

	size_t contig_bytes = 0;

	might_sleep();

	list_for_each_entry_from(window, head, list) {

		end_offset = window->offset +

			(window->nr_pages << PAGE_SHIFT);

		loop_nr_pages = min_t(int,

				      (end_offset - loop_offset) >> PAGE_SHIFT,

				      nr_pages_left);

		scif_init_window_iter(window, &src_win_iter);

		for (i = 0; i < loop_nr_pages; i++) {

			phys_addr = scif_off_to_dma_addr(window, loop_offset,

							 &contig_bytes,

							 &src_win_iter);

			phys_addr = scif_get_phys(phys_addr, ep);

			err = remap_pfn_range(vma,

					      vma->vm_start +

					      loop_offset - offset,

					      phys_addr >> PAGE_SHIFT,

					      PAGE_SIZE,

					      vma->vm_page_prot);

			if (err)

				goto error;

			loop_offset += PAGE_SIZE;

		}

		nr_pages_left -= loop_nr_pages;

		if (!nr_pages_left)

			break;

	}

	/*

	 * No more failures expected. Bump up the ref count for all

	 * the windows. Another traversal from start_window required

	 * for handling errors encountered across windows during

	 * remap_pfn_range(..).

	 */

	loop_offset = offset;

	nr_pages_left = nr_pages;

	window = start_window;

	head = &ep->rma_info.remote_reg_list;

	list_for_each_entry_from(window, head, list) {

		end_offset = window->offset +

			(window->nr_pages << PAGE_SHIFT);

		loop_nr_pages = min_t(int,

				      (end_offset - loop_offset) >> PAGE_SHIFT,

				      nr_pages_left);

		scif_get_window(window, loop_nr_pages);

		nr_pages_left -= loop_nr_pages;

		loop_offset += (loop_nr_pages << PAGE_SHIFT);

		if (!nr_pages_left)

			break;

	}

error:

	if (err)

		dev_err(scif_info.mdev.this_device,

			"%s %d err %d\n", __func__, __LINE__, err);

	return err;

}

/*

 * scif_rma_list_munmap:

 *

 * Traverse the remote registration list starting from window:

 * 1) Decrement ref count.

 * 2) If the ref count drops to zero then send a SCIF_MUNMAP message to peer.

 * RMA lock must be held.

 */

static void scif_rma_list_munmap(struct scif_window *start_window,

				 s64 offset, int nr_pages)

{

	struct scifmsg msg;

	s64 loop_offset = offset, end_offset;

	int loop_nr_pages, nr_pages_left = nr_pages;

	struct scif_endpt *ep = (struct scif_endpt *)start_window->ep;

	struct list_head *head = &ep->rma_info.remote_reg_list;

	struct scif_window *window = start_window, *_window;

	msg.uop = SCIF_MUNMAP;

	msg.src = ep->port;

	loop_offset = offset;

	nr_pages_left = nr_pages;

	list_for_each_entry_safe_from(window, _window, head, list) {

		end_offset = window->offset +

			(window->nr_pages << PAGE_SHIFT);

		loop_nr_pages = min_t(int,

				      (end_offset - loop_offset) >> PAGE_SHIFT,

				      nr_pages_left);

		scif_put_window(window, loop_nr_pages);

		if (!window->ref_count) {

			struct scif_dev *rdev = ep->remote_dev;

			scif_drain_dma_intr(rdev->sdev,

					    ep->rma_info.dma_chan);

			/* Inform the peer about this munmap */

			msg.payload[0] = window->peer_window;

			/* No error handling for Notification messages. */

			scif_nodeqp_send(ep->remote_dev, &msg);

			list_del(&window->list);

			/* Destroy this window from the peer's registered AS */

			scif_destroy_remote_window(window);

		}

		nr_pages_left -= loop_nr_pages;

		loop_offset += (loop_nr_pages << PAGE_SHIFT);

		if (!nr_pages_left)

			break;

	}

}

/*

 * The private data field of each VMA used to mmap a remote window

 * points to an instance of struct vma_pvt

 */

struct vma_pvt {

	struct scif_endpt *ep;	/* End point for remote window */

	s64 offset;		/* offset within remote window */

	bool valid_offset;	/* offset is valid only if the original

				 * mmap request was for a single page

				 * else the offset within the vma is

				 * the correct offset

				 */

	struct kref ref;

};

static void vma_pvt_release(struct kref *ref)

{

	struct vma_pvt *vmapvt = container_of(ref, struct vma_pvt, ref);

	kfree(vmapvt);

}

/**

 * scif_vma_open - VMA open driver callback

 * @vma: VMM memory area.

 * The open method is called by the kernel to allow the subsystem implementing

 * the VMA to initialize the area. This method is invoked any time a new

 * reference to the VMA is made (when a process forks, for example).

 * The one exception happens when the VMA is first created by mmap;

 * in this case, the driver's mmap method is called instead.

 * This function is also invoked when an existing VMA is split by the kernel

 * due to a call to munmap on a subset of the VMA resulting in two VMAs.

 * The kernel invokes this function only on one of the two VMAs.

 */

static void scif_vma_open(struct vm_area_struct *vma)

{

	struct vma_pvt *vmapvt = vma->vm_private_data;

	dev_dbg(scif_info.mdev.this_device,

		"SCIFAPI vma open: vma_start 0x%lx vma_end 0x%lx\n",

		vma->vm_start, vma->vm_end);

	scif_insert_vma(vmapvt->ep, vma);

	kref_get(&vmapvt->ref);

}

/**

 * scif_munmap - VMA close driver callback.

 * @vma: VMM memory area.

 * When an area is destroyed, the kernel calls its close operation.

 * Note that there's no usage count associated with VMA's; the area

 * is opened and closed exactly once by each process that uses it.

 */

static void scif_munmap(struct vm_area_struct *vma)

{

	struct scif_endpt *ep;

	struct vma_pvt *vmapvt = vma->vm_private_data;

	int nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;

	s64 offset;

	struct scif_rma_req req;

	struct scif_window *window = NULL;

	int err;

	might_sleep();

	dev_dbg(scif_info.mdev.this_device,

		"SCIFAPI munmap: vma_start 0x%lx vma_end 0x%lx\n",

		vma->vm_start, vma->vm_end);

	ep = vmapvt->ep;

	offset = vmapvt->valid_offset ? vmapvt->offset :

		(vma->vm_pgoff) << PAGE_SHIFT;

	dev_dbg(scif_info.mdev.this_device,

		"SCIFAPI munmap: ep %p nr_pages 0x%x offset 0x%llx\n",

		ep, nr_pages, offset);

	req.out_window = &window;

	req.offset = offset;

	req.nr_bytes = vma->vm_end - vma->vm_start;

	req.prot = vma->vm_flags & (VM_READ | VM_WRITE);

	req.type = SCIF_WINDOW_PARTIAL;

	req.head = &ep->rma_info.remote_reg_list;

	mutex_lock(&ep->rma_info.rma_lock);

	err = scif_query_window(&req);

	if (err)

		dev_err(scif_info.mdev.this_device,

			"%s %d err %d\n", __func__, __LINE__, err);

	else

		scif_rma_list_munmap(window, offset, nr_pages);

	mutex_unlock(&ep->rma_info.rma_lock);

	/*

	 * The kernel probably zeroes these out but we still want

	 * to clean up our own mess just in case.

	 */

	vma->vm_ops = NULL;

	vma->vm_private_data = NULL;

	kref_put(&vmapvt->ref, vma_pvt_release);

	scif_delete_vma(ep, vma);

}

static const struct vm_operations_struct scif_vm_ops = {

	.open = scif_vma_open,

	.close = scif_munmap,

};

/**

 * scif_mmap - Map pages in virtual address space to a remote window.

 * @vma: VMM memory area.

 * @epd: endpoint descriptor

 *

 * Return: Upon successful completion, scif_mmap() returns zero

 * else an apt error is returned as documented in scif.h

 */

int scif_mmap(struct vm_area_struct *vma, scif_epd_t epd)

{

	struct scif_rma_req req;

	struct scif_window *window = NULL;

	struct scif_endpt *ep = (struct scif_endpt *)epd;

	s64 start_offset = vma->vm_pgoff << PAGE_SHIFT;

	int nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;

	int err;

	struct vma_pvt *vmapvt;

	dev_dbg(scif_info.mdev.this_device,

		"SCIFAPI mmap: ep %p start_offset 0x%llx nr_pages 0x%x\n",

		ep, start_offset, nr_pages);

	err = scif_verify_epd(ep);

	if (err)

		return err;

	might_sleep();

	err = scif_insert_vma(ep, vma);

	if (err)

		return err;

	vmapvt = kzalloc(sizeof(*vmapvt), GFP_KERNEL);

	if (!vmapvt) {

		scif_delete_vma(ep, vma);

		return -ENOMEM;

	}

	vmapvt->ep = ep;

	kref_init(&vmapvt->ref);

	req.out_window = &window;

	req.offset = start_offset;

	req.nr_bytes = vma->vm_end - vma->vm_start;

	req.prot = vma->vm_flags & (VM_READ | VM_WRITE);

	req.type = SCIF_WINDOW_PARTIAL;

	req.head = &ep->rma_info.remote_reg_list;

	mutex_lock(&ep->rma_info.rma_lock);

	/* Does a valid window exist? */

	err = scif_query_window(&req);

	if (err) {

		dev_err(&ep->remote_dev->sdev->dev,

			"%s %d err %d\n", __func__, __LINE__, err);

		goto error_unlock;

	}

	/* Default prot for loopback */

	if (!scifdev_self(ep->remote_dev))

		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);

	/*

	 * VM_DONTCOPY - Do not copy this vma on fork

	 * VM_DONTEXPAND - Cannot expand with mremap()

	 * VM_RESERVED - Count as reserved_vm like IO

	 * VM_PFNMAP - Page-ranges managed without "struct page"

	 * VM_IO - Memory mapped I/O or similar

	 *

	 * We do not want to copy this VMA automatically on a fork(),

	 * expand this VMA due to mremap() or swap out these pages since

	 * the VMA is actually backed by physical pages in the remote

	 * node's physical memory and not via a struct page.

	 */

	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;

	if (!scifdev_self(ep->remote_dev))

		vma->vm_flags |= VM_IO | VM_PFNMAP;

	/* Map this range of windows */