userfaultfd: non-cooperative: add ability to report non-PF events from uffd descriptor (9cd75c3c) · Commits · 戴 / test

fs/userfaultfd.c

+96 −2

Original line number	Original line	Diff line number	Diff line
	@@ -12,6 +12,7 @@
	* mm/ksm.c (mm hashing).		* mm/ksm.c (mm hashing).
	*/		*/

			#include <linux/list.h>
	#include <linux/hashtable.h>		#include <linux/hashtable.h>
	#include <linux/sched.h>		#include <linux/sched.h>
	#include <linux/mm.h>		#include <linux/mm.h>
	@@ -45,12 +46,16 @@ struct userfaultfd_ctx {
	wait_queue_head_t fault_wqh;		wait_queue_head_t fault_wqh;
	/* waitqueue head for the pseudo fd to wakeup poll/read */		/* waitqueue head for the pseudo fd to wakeup poll/read */
	wait_queue_head_t fd_wqh;		wait_queue_head_t fd_wqh;
			/* waitqueue head for events */
			wait_queue_head_t event_wqh;
	/* a refile sequence protected by fault_pending_wqh lock */		/* a refile sequence protected by fault_pending_wqh lock */
	struct seqcount refile_seq;		struct seqcount refile_seq;
	/* pseudo fd refcounting */		/* pseudo fd refcounting */
	atomic_t refcount;		atomic_t refcount;
	/* userfaultfd syscall flags */		/* userfaultfd syscall flags */
	unsigned int flags;		unsigned int flags;
			/* features requested from the userspace */
			unsigned int features;
	/* state machine */		/* state machine */
	enum userfaultfd_state state;		enum userfaultfd_state state;
	/* released */		/* released */
	@@ -142,6 +147,8 @@ static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
	VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));		VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
	VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));		VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
	VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));		VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
			VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
			VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
	VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));		VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
	VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));		VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
	mmdrop(ctx->mm);		mmdrop(ctx->mm);
	@@ -458,6 +465,59 @@ out:
	return ret;		return ret;
	}		}

			static int __maybe_unused userfaultfd_event_wait_completion(
			struct userfaultfd_ctx *ctx,
			struct userfaultfd_wait_queue *ewq)
			{
			int ret = 0;

			ewq->ctx = ctx;
			init_waitqueue_entry(&ewq->wq, current);

			spin_lock(&ctx->event_wqh.lock);
			/*
			* After the __add_wait_queue the uwq is visible to userland
			* through poll/read().
			*/
			__add_wait_queue(&ctx->event_wqh, &ewq->wq);
			for (;;) {
			set_current_state(TASK_KILLABLE);
			if (ewq->msg.event == 0)
			break;
			if (ACCESS_ONCE(ctx->released) \|\|
			fatal_signal_pending(current)) {
			ret = -1;
			__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
			break;
			}

			spin_unlock(&ctx->event_wqh.lock);

			wake_up_poll(&ctx->fd_wqh, POLLIN);
			schedule();

			spin_lock(&ctx->event_wqh.lock);
			}
			__set_current_state(TASK_RUNNING);
			spin_unlock(&ctx->event_wqh.lock);

			/*
			* ctx may go away after this if the userfault pseudo fd is
			* already released.
			*/

			userfaultfd_ctx_put(ctx);
			return ret;
			}

			static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
			struct userfaultfd_wait_queue *ewq)
			{
			ewq->msg.event = 0;
			wake_up_locked(&ctx->event_wqh);
			__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
			}

	static int userfaultfd_release(struct inode inode, struct file file)		static int userfaultfd_release(struct inode inode, struct file file)
	{		{
	struct userfaultfd_ctx *ctx = file->private_data;		struct userfaultfd_ctx *ctx = file->private_data;
	@@ -546,6 +606,12 @@ static inline struct userfaultfd_wait_queue *find_userfault(
	return find_userfault_in(&ctx->fault_pending_wqh);		return find_userfault_in(&ctx->fault_pending_wqh);
	}		}

			static inline struct userfaultfd_wait_queue *find_userfault_evt(
			struct userfaultfd_ctx *ctx)
			{
			return find_userfault_in(&ctx->event_wqh);
			}

	static unsigned int userfaultfd_poll(struct file file, poll_table wait)		static unsigned int userfaultfd_poll(struct file file, poll_table wait)
	{		{
	struct userfaultfd_ctx *ctx = file->private_data;		struct userfaultfd_ctx *ctx = file->private_data;
	@@ -577,6 +643,9 @@ static unsigned int userfaultfd_poll(struct file file, poll_table wait)
	smp_mb();		smp_mb();
	if (waitqueue_active(&ctx->fault_pending_wqh))		if (waitqueue_active(&ctx->fault_pending_wqh))
	ret = POLLIN;		ret = POLLIN;
			else if (waitqueue_active(&ctx->event_wqh))
			ret = POLLIN;

	return ret;		return ret;
	default:		default:
	WARN_ON_ONCE(1);		WARN_ON_ONCE(1);
	@@ -641,6 +710,19 @@ static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
	break;		break;
	}		}
	spin_unlock(&ctx->fault_pending_wqh.lock);		spin_unlock(&ctx->fault_pending_wqh.lock);

			spin_lock(&ctx->event_wqh.lock);
			uwq = find_userfault_evt(ctx);
			if (uwq) {
			*msg = uwq->msg;

			userfaultfd_event_complete(ctx, uwq);
			spin_unlock(&ctx->event_wqh.lock);
			ret = 0;
			break;
			}
			spin_unlock(&ctx->event_wqh.lock);

	if (signal_pending(current)) {		if (signal_pending(current)) {
	ret = -ERESTARTSYS;		ret = -ERESTARTSYS;
	break;		break;
	@@ -1184,6 +1266,14 @@ out:
	return ret;		return ret;
	}		}

			static inline unsigned int uffd_ctx_features(__u64 user_features)
			{
			/*
			* For the current set of features the bits just coincide
			*/
			return (unsigned int)user_features;
			}

	/*		/*
	* userland asks for a certain API version and we return which bits		* userland asks for a certain API version and we return which bits
	* and ioctl commands are implemented in this kernel for such API		* and ioctl commands are implemented in this kernel for such API
	@@ -1202,19 +1292,21 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx,
	ret = -EFAULT;		ret = -EFAULT;
	if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))		if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
	goto out;		goto out;
	if (uffdio_api.api != UFFD_API \|\| uffdio_api.features) {		if (uffdio_api.api != UFFD_API \|\|
			(uffdio_api.features & ~UFFD_API_FEATURES)) {
	memset(&uffdio_api, 0, sizeof(uffdio_api));		memset(&uffdio_api, 0, sizeof(uffdio_api));
	if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))		if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
	goto out;		goto out;
	ret = -EINVAL;		ret = -EINVAL;
	goto out;		goto out;
	}		}
	uffdio_api.features = UFFD_API_FEATURES;		uffdio_api.features &= UFFD_API_FEATURES;
	uffdio_api.ioctls = UFFD_API_IOCTLS;		uffdio_api.ioctls = UFFD_API_IOCTLS;
	ret = -EFAULT;		ret = -EFAULT;
	if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))		if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
	goto out;		goto out;
	ctx->state = UFFD_STATE_RUNNING;		ctx->state = UFFD_STATE_RUNNING;
			ctx->features = uffd_ctx_features(uffdio_api.features);
	ret = 0;		ret = 0;
	out:		out:
	return ret;		return ret;
	@@ -1301,6 +1393,7 @@ static void init_once_userfaultfd_ctx(void *mem)

	init_waitqueue_head(&ctx->fault_pending_wqh);		init_waitqueue_head(&ctx->fault_pending_wqh);
	init_waitqueue_head(&ctx->fault_wqh);		init_waitqueue_head(&ctx->fault_wqh);
			init_waitqueue_head(&ctx->event_wqh);
	init_waitqueue_head(&ctx->fd_wqh);		init_waitqueue_head(&ctx->fd_wqh);
	seqcount_init(&ctx->refile_seq);		seqcount_init(&ctx->refile_seq);
	}		}
	@@ -1341,6 +1434,7 @@ static struct file *userfaultfd_file_create(int flags)

	atomic_set(&ctx->refcount, 1);		atomic_set(&ctx->refcount, 1);
	ctx->flags = flags;		ctx->flags = flags;
			ctx->features = 0;
	ctx->state = UFFD_STATE_WAIT_API;		ctx->state = UFFD_STATE_WAIT_API;
	ctx->released = false;		ctx->released = false;
	ctx->mm = current->mm;		ctx->mm = current->mm;

Admin message