Commit 79305ec6 authored by Dave Airlie's avatar Dave Airlie
Browse files

Merge tag 'amdkfd-fixes-2015-01-06' of git://people.freedesktop.org/~gabbayo/linux into drm-fixes 

- Complete overhaul to the main IOCTL function, kfd_ioctl(), according to
  drm_ioctl() example. This includes changing the IOCTL definitions, so it
  breaks compatibility with previous versions of the userspace. However,
  because the kernel was not officialy released yet, and this the first
  kernel that includes amdkfd, I assume I can still do that at this stage.

- A couple of bug fixes for the non-HWS path (used for bring-ups and
  debugging purposes only).

* tag 'amdkfd-fixes-2015-01-06' of git://people.freedesktop.org/~gabbayo/linux:
  drm/amdkfd: rewrite kfd_ioctl() according to drm_ioctl()
  drm/amdkfd: reformat IOCTL definitions to drm-style
  drm/amdkfd: Do copy_to/from_user in general kfd_ioctl()
  drm/amdkfd: unmap VMID<-->PASID when relesing VMID (non-HWS)
  drm/radeon: Assign VMID to PASID for IH in non-HWS mode
  drm/radeon: do not leave queue acquired if timeout happens in kgd_hqd_destroy()
  drm/amdkfd: Load mqd to hqd in non-HWS mode
  drm/amd: Fixing typos in kfd<->kgd interface
parents eaee8ec4 76baee6c

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c

+168 −147
Original line number Diff line number Diff line
@@ -126,17 +126,14 @@ static int kfd_open(struct inode *inode, struct file *filep) 
	return 0;

}



static long kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,

					void __user *arg)

static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,

					void *data)

{

	struct kfd_ioctl_get_version_args args;

	struct kfd_ioctl_get_version_args *args = data;

	int err = 0;



	args.major_version = KFD_IOCTL_MAJOR_VERSION;

	args.minor_version = KFD_IOCTL_MINOR_VERSION;



	if (copy_to_user(arg, &args, sizeof(args)))

		err = -EFAULT;

	args->major_version = KFD_IOCTL_MAJOR_VERSION;

	args->minor_version = KFD_IOCTL_MINOR_VERSION;



	return err;

}
@@ -220,10 +217,10 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties, 
	return 0;

}



static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,

					void __user *arg)

static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,

					void *data)

{

	struct kfd_ioctl_create_queue_args args;

	struct kfd_ioctl_create_queue_args *args = data;

	struct kfd_dev *dev;

	int err = 0;

	unsigned int queue_id;
@@ -232,16 +229,13 @@ static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 


	memset(&q_properties, 0, sizeof(struct queue_properties));



	if (copy_from_user(&args, arg, sizeof(args)))

		return -EFAULT;



	pr_debug("kfd: creating queue ioctl\n");



	err = set_queue_properties_from_user(&q_properties, &args);

	err = set_queue_properties_from_user(&q_properties, args);

	if (err)

		return err;



	dev = kfd_device_by_id(args.gpu_id);

	dev = kfd_device_by_id(args->gpu_id);

	if (dev == NULL)

		return -EINVAL;
@@ -249,7 +243,7 @@ static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 


	pdd = kfd_bind_process_to_device(dev, p);

	if (IS_ERR(pdd)) {

		err = PTR_ERR(pdd);

		err = -ESRCH;

		goto err_bind_process;

	}
@@ -262,33 +256,26 @@ static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 
	if (err != 0)

		goto err_create_queue;



	args.queue_id = queue_id;

	args->queue_id = queue_id;



	/* Return gpu_id as doorbell offset for mmap usage */

	args.doorbell_offset = args.gpu_id << PAGE_SHIFT;



	if (copy_to_user(arg, &args, sizeof(args))) {

		err = -EFAULT;

		goto err_copy_args_out;

	}

	args->doorbell_offset = args->gpu_id << PAGE_SHIFT;



	mutex_unlock(&p->mutex);



	pr_debug("kfd: queue id %d was created successfully\n", args.queue_id);

	pr_debug("kfd: queue id %d was created successfully\n", args->queue_id);



	pr_debug("ring buffer address == 0x%016llX\n",

			args.ring_base_address);

			args->ring_base_address);



	pr_debug("read ptr address    == 0x%016llX\n",

			args.read_pointer_address);

			args->read_pointer_address);



	pr_debug("write ptr address   == 0x%016llX\n",

			args.write_pointer_address);

			args->write_pointer_address);



	return 0;



err_copy_args_out:

	pqm_destroy_queue(&p->pqm, queue_id);

err_create_queue:

err_bind_process:

	mutex_unlock(&p->mutex);
@@ -296,99 +283,90 @@ err_bind_process: 
}



static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,

					void __user *arg)

					void *data)

{

	int retval;

	struct kfd_ioctl_destroy_queue_args args;



	if (copy_from_user(&args, arg, sizeof(args)))

		return -EFAULT;

	struct kfd_ioctl_destroy_queue_args *args = data;



	pr_debug("kfd: destroying queue id %d for PASID %d\n",

				args.queue_id,

				args->queue_id,

				p->pasid);



	mutex_lock(&p->mutex);



	retval = pqm_destroy_queue(&p->pqm, args.queue_id);

	retval = pqm_destroy_queue(&p->pqm, args->queue_id);



	mutex_unlock(&p->mutex);

	return retval;

}



static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,

					void __user *arg)

					void *data)

{

	int retval;

	struct kfd_ioctl_update_queue_args args;

	struct kfd_ioctl_update_queue_args *args = data;

	struct queue_properties properties;



	if (copy_from_user(&args, arg, sizeof(args)))

		return -EFAULT;



	if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {

	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {

		pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");

		return -EINVAL;

	}



	if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) {

	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {

		pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");

		return -EINVAL;

	}



	if ((args.ring_base_address) &&

	if ((args->ring_base_address) &&

		(!access_ok(VERIFY_WRITE,

			(const void __user *) args.ring_base_address,

			(const void __user *) args->ring_base_address,

			sizeof(uint64_t)))) {

		pr_err("kfd: can't access ring base address\n");

		return -EFAULT;

	}



	if (!is_power_of_2(args.ring_size) && (args.ring_size != 0)) {

	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {

		pr_err("kfd: ring size must be a power of 2 or 0\n");

		return -EINVAL;

	}



	properties.queue_address = args.ring_base_address;

	properties.queue_size = args.ring_size;

	properties.queue_percent = args.queue_percentage;

	properties.priority = args.queue_priority;

	properties.queue_address = args->ring_base_address;

	properties.queue_size = args->ring_size;

	properties.queue_percent = args->queue_percentage;

	properties.priority = args->queue_priority;



	pr_debug("kfd: updating queue id %d for PASID %d\n",

			args.queue_id, p->pasid);

			args->queue_id, p->pasid);



	mutex_lock(&p->mutex);



	retval = pqm_update_queue(&p->pqm, args.queue_id, &properties);

	retval = pqm_update_queue(&p->pqm, args->queue_id, &properties);



	mutex_unlock(&p->mutex);



	return retval;

}



static long kfd_ioctl_set_memory_policy(struct file *filep,

				struct kfd_process *p, void __user *arg)

static int kfd_ioctl_set_memory_policy(struct file *filep,

					struct kfd_process *p, void *data)

{

	struct kfd_ioctl_set_memory_policy_args args;

	struct kfd_ioctl_set_memory_policy_args *args = data;

	struct kfd_dev *dev;

	int err = 0;

	struct kfd_process_device *pdd;

	enum cache_policy default_policy, alternate_policy;



	if (copy_from_user(&args, arg, sizeof(args)))

		return -EFAULT;



	if (args.default_policy != KFD_IOC_CACHE_POLICY_COHERENT

	    && args.default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {

	if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT

	    && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {

		return -EINVAL;

	}



	if (args.alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT

	    && args.alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {

	if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT

	    && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {

		return -EINVAL;

	}



	dev = kfd_device_by_id(args.gpu_id);

	dev = kfd_device_by_id(args->gpu_id);

	if (dev == NULL)

		return -EINVAL;
@@ -396,23 +374,23 @@ static long kfd_ioctl_set_memory_policy(struct file *filep, 


	pdd = kfd_bind_process_to_device(dev, p);

	if (IS_ERR(pdd)) {

		err = PTR_ERR(pdd);

		err = -ESRCH;

		goto out;

	}



	default_policy = (args.default_policy == KFD_IOC_CACHE_POLICY_COHERENT)

	default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)

			 ? cache_policy_coherent : cache_policy_noncoherent;



	alternate_policy =

		(args.alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)

		(args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)

		   ? cache_policy_coherent : cache_policy_noncoherent;



	if (!dev->dqm->set_cache_memory_policy(dev->dqm,

				&pdd->qpd,

				default_policy,

				alternate_policy,

				(void __user *)args.alternate_aperture_base,

				args.alternate_aperture_size))

				(void __user *)args->alternate_aperture_base,

				args->alternate_aperture_size))

		err = -EINVAL;



out:
@@ -421,53 +399,44 @@ out: 
	return err;

}



static long kfd_ioctl_get_clock_counters(struct file *filep,

				struct kfd_process *p, void __user *arg)

static int kfd_ioctl_get_clock_counters(struct file *filep,

				struct kfd_process *p, void *data)

{

	struct kfd_ioctl_get_clock_counters_args args;

	struct kfd_ioctl_get_clock_counters_args *args = data;

	struct kfd_dev *dev;

	struct timespec time;



	if (copy_from_user(&args, arg, sizeof(args)))

		return -EFAULT;



	dev = kfd_device_by_id(args.gpu_id);

	dev = kfd_device_by_id(args->gpu_id);

	if (dev == NULL)

		return -EINVAL;



	/* Reading GPU clock counter from KGD */

	args.gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd);

	args->gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd);



	/* No access to rdtsc. Using raw monotonic time */

	getrawmonotonic(&time);

	args.cpu_clock_counter = (uint64_t)timespec_to_ns(&time);

	args->cpu_clock_counter = (uint64_t)timespec_to_ns(&time);



	get_monotonic_boottime(&time);

	args.system_clock_counter = (uint64_t)timespec_to_ns(&time);

	args->system_clock_counter = (uint64_t)timespec_to_ns(&time);



	/* Since the counter is in nano-seconds we use 1GHz frequency */

	args.system_clock_freq = 1000000000;



	if (copy_to_user(arg, &args, sizeof(args)))

		return -EFAULT;

	args->system_clock_freq = 1000000000;



	return 0;

}





static int kfd_ioctl_get_process_apertures(struct file *filp,

				struct kfd_process *p, void __user *arg)

				struct kfd_process *p, void *data)

{

	struct kfd_ioctl_get_process_apertures_args args;

	struct kfd_ioctl_get_process_apertures_args *args = data;

	struct kfd_process_device_apertures *pAperture;

	struct kfd_process_device *pdd;



	dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid);



	if (copy_from_user(&args, arg, sizeof(args)))

		return -EFAULT;



	args.num_of_nodes = 0;

	args->num_of_nodes = 0;



	mutex_lock(&p->mutex);
@@ -476,7 +445,8 @@ static int kfd_ioctl_get_process_apertures(struct file *filp, 
		/* Run over all pdd of the process */

		pdd = kfd_get_first_process_device_data(p);

		do {

			pAperture = &args.process_apertures[args.num_of_nodes];

			pAperture =

				&args->process_apertures[args->num_of_nodes];

			pAperture->gpu_id = pdd->dev->id;

			pAperture->lds_base = pdd->lds_base;

			pAperture->lds_limit = pdd->lds_limit;
@@ -486,7 +456,7 @@ static int kfd_ioctl_get_process_apertures(struct file *filp, 
			pAperture->scratch_limit = pdd->scratch_limit;



			dev_dbg(kfd_device,

				"node id %u\n", args.num_of_nodes);

				"node id %u\n", args->num_of_nodes);

			dev_dbg(kfd_device,

				"gpu id %u\n", pdd->dev->id);

			dev_dbg(kfd_device,
@@ -502,80 +472,131 @@ static int kfd_ioctl_get_process_apertures(struct file *filp, 
			dev_dbg(kfd_device,

				"scratch_limit %llX\n", pdd->scratch_limit);



			args.num_of_nodes++;

			args->num_of_nodes++;

		} while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL &&

				(args.num_of_nodes < NUM_OF_SUPPORTED_GPUS));

				(args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));

	}



	mutex_unlock(&p->mutex);



	if (copy_to_user(arg, &args, sizeof(args)))

		return -EFAULT;



	return 0;

}



#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \

	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl}



/** Ioctl table */

static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {

	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,

			kfd_ioctl_get_version, 0),



	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,

			kfd_ioctl_create_queue, 0),



	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,

			kfd_ioctl_destroy_queue, 0),



	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,

			kfd_ioctl_set_memory_policy, 0),



	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,

			kfd_ioctl_get_clock_counters, 0),



	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,

			kfd_ioctl_get_process_apertures, 0),



	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,

			kfd_ioctl_update_queue, 0),

};



#define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)



static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)

{

	struct kfd_process *process;

	long err = -EINVAL;

	amdkfd_ioctl_t *func;

	const struct amdkfd_ioctl_desc *ioctl = NULL;

	unsigned int nr = _IOC_NR(cmd);

	char stack_kdata[128];

	char *kdata = NULL;

	unsigned int usize, asize;

	int retcode = -EINVAL;



	dev_dbg(kfd_device,

		"ioctl cmd 0x%x (#%d), arg 0x%lx\n",

		cmd, _IOC_NR(cmd), arg);

	if (nr >= AMDKFD_CORE_IOCTL_COUNT)

		goto err_i1;



	if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {

		u32 amdkfd_size;



		ioctl = &amdkfd_ioctls[nr];



		amdkfd_size = _IOC_SIZE(ioctl->cmd);

		usize = asize = _IOC_SIZE(cmd);

		if (amdkfd_size > asize)

			asize = amdkfd_size;



		cmd = ioctl->cmd;

	} else

		goto err_i1;



	dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg);



	process = kfd_get_process(current);

	if (IS_ERR(process))

		return PTR_ERR(process);

	if (IS_ERR(process)) {

		dev_dbg(kfd_device, "no process\n");

		goto err_i1;

	}



	switch (cmd) {

	case KFD_IOC_GET_VERSION:

		err = kfd_ioctl_get_version(filep, process, (void __user *)arg);

		break;

	case KFD_IOC_CREATE_QUEUE:

		err = kfd_ioctl_create_queue(filep, process,

						(void __user *)arg);

		break;



	case KFD_IOC_DESTROY_QUEUE:

		err = kfd_ioctl_destroy_queue(filep, process,

						(void __user *)arg);

		break;



	case KFD_IOC_SET_MEMORY_POLICY:

		err = kfd_ioctl_set_memory_policy(filep, process,

						(void __user *)arg);

		break;



	case KFD_IOC_GET_CLOCK_COUNTERS:

		err = kfd_ioctl_get_clock_counters(filep, process,

						(void __user *)arg);

		break;



	case KFD_IOC_GET_PROCESS_APERTURES:

		err = kfd_ioctl_get_process_apertures(filep, process,

						(void __user *)arg);

		break;



	case KFD_IOC_UPDATE_QUEUE:

		err = kfd_ioctl_update_queue(filep, process,

						(void __user *)arg);

		break;



	default:

		dev_err(kfd_device,

			"unknown ioctl cmd 0x%x, arg 0x%lx)\n",

			cmd, arg);

		err = -EINVAL;

		break;

	/* Do not trust userspace, use our own definition */

	func = ioctl->func;



	if (unlikely(!func)) {

		dev_dbg(kfd_device, "no function\n");

		retcode = -EINVAL;

		goto err_i1;

	}



	if (err < 0)

		dev_err(kfd_device,

			"ioctl error %ld for ioctl cmd 0x%x (#%d)\n",

			err, cmd, _IOC_NR(cmd));

	if (cmd & (IOC_IN | IOC_OUT)) {

		if (asize <= sizeof(stack_kdata)) {

			kdata = stack_kdata;

		} else {

			kdata = kmalloc(asize, GFP_KERNEL);

			if (!kdata) {

				retcode = -ENOMEM;

				goto err_i1;

			}

		}

		if (asize > usize)

			memset(kdata + usize, 0, asize - usize);

	}



	return err;

	if (cmd & IOC_IN) {

		if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {

			retcode = -EFAULT;

			goto err_i1;

		}

	} else if (cmd & IOC_OUT) {

		memset(kdata, 0, usize);

	}



	retcode = func(filep, process, kdata);



	if (cmd & IOC_OUT)

		if (copy_to_user((void __user *)arg, kdata, usize) != 0)

			retcode = -EFAULT;



err_i1:

	if (!ioctl)

		dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",

			  task_pid_nr(current), cmd, nr);



	if (kdata != stack_kdata)

		kfree(kdata);



	if (retcode)

		dev_dbg(kfd_device, "ret = %d\n", retcode);



	return retcode;

}



static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c

+15 −0
Original line number Diff line number Diff line
@@ -161,6 +161,9 @@ static void deallocate_vmid(struct device_queue_manager *dqm, 
{

	int bit = qpd->vmid - KFD_VMID_START_OFFSET;



	/* Release the vmid mapping */

	set_pasid_vmid_mapping(dqm, 0, qpd->vmid);



	set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);

	qpd->vmid = 0;

	q->properties.vmid = 0;
@@ -272,6 +275,18 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 
		return retval;

	}



	pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",

			q->pipe,

			q->queue);



	retval = mqd->load_mqd(mqd, q->mqd, q->pipe,

			q->queue, q->properties.write_ptr);

	if (retval != 0) {

		deallocate_hqd(dqm, q);

		mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);

		return retval;

	}



	return 0;

}

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c

+1 −1
Original line number Diff line number Diff line
@@ -184,7 +184,7 @@ static bool is_occupied(struct mqd_manager *mm, void *mqd, 
			uint32_t queue_id)

{



	return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address,

	return kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address,

					pipe_id, queue_id);



}

drivers/gpu/drm/amd/amdkfd/kfd_priv.h

+18 −0
Original line number Diff line number Diff line
@@ -463,6 +463,24 @@ struct kfd_process { 
	bool is_32bit_user_mode;

};



/**

 * Ioctl function type.

 *

 * \param filep pointer to file structure.

 * \param p amdkfd process pointer.

 * \param data pointer to arg that was copied from user.

 */

typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,

				void *data);



struct amdkfd_ioctl_desc {

	unsigned int cmd;

	int flags;

	amdkfd_ioctl_t *func;

	unsigned int cmd_drv;

	const char *name;

};



void kfd_process_create_wq(void);

void kfd_process_destroy_wq(void);

struct kfd_process *kfd_create_process(const struct task_struct *);

drivers/gpu/drm/amd/include/kgd_kfd_interface.h

+1 −1
Original line number Diff line number Diff line
@@ -183,7 +183,7 @@ struct kfd2kgd_calls { 
	int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,

			uint32_t queue_id, uint32_t __user *wptr);



	bool (*hqd_is_occupies)(struct kgd_dev *kgd, uint64_t queue_address,

	bool (*hqd_is_occupied)(struct kgd_dev *kgd, uint64_t queue_address,

				uint32_t pipe_id, uint32_t queue_id);



	int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type,

CRA Git | Maintained and supported by SUSTech CRA and CCSE