Commit eff6f4a0 authored by Oded Gabbay's avatar Oded Gabbay Committed by Greg Kroah-Hartman
Browse files

habanalabs: add command submission module 



This patch adds the main flow for the user to submit work to the device.

Each work is described by a command submission object (CS). The CS contains
3 arrays of command buffers: One for execution, and two for context-switch
(store and restore).

For each CB, the user specifies on which queue to put that CB. In case of
an internal queue, the entry doesn't contain a pointer to the CB but the
address in the on-chip memory that the CB resides at.

The driver parses some of the CBs to enforce security restrictions.

The user receives a sequence number that represents the CS object. The user
can then query the driver regarding the status of the CS, using that
sequence number.

In case the CS doesn't finish before the timeout expires, the driver will
perform a soft-reset of the device.

Reviewed-by: default avatarMike Rapoport <rppt@linux.ibm.com>
Signed-off-by: default avatarOded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent f8c8c7d5

drivers/misc/habanalabs/Makefile

+2 −1
Original line number Diff line number Diff line
@@ -5,7 +5,8 @@ 
obj-m	:= habanalabs.o



habanalabs-y := habanalabs_drv.o device.o context.o asid.o habanalabs_ioctl.o \

		command_buffer.o hw_queue.o irq.o sysfs.o hwmon.o

		command_buffer.o hw_queue.o irq.o sysfs.o hwmon.o memory.o \

		command_submission.o



include $(src)/goya/Makefile

habanalabs-y += $(HL_GOYA_FILES)

drivers/misc/habanalabs/command_submission.c

0 → 100644
+766 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0



/*

 * Copyright 2016-2019 HabanaLabs, Ltd.

 * All Rights Reserved.

 */



#include <uapi/misc/habanalabs.h>

#include "habanalabs.h"



#include <linux/uaccess.h>

#include <linux/slab.h>



static void job_wq_completion(struct work_struct *work);

static long _hl_cs_wait_ioctl(struct hl_device *hdev,

		struct hl_ctx *ctx, u64 timeout_us, u64 seq);

static void cs_do_release(struct kref *ref);



static const char *hl_fence_get_driver_name(struct dma_fence *fence)

{

	return "HabanaLabs";

}



static const char *hl_fence_get_timeline_name(struct dma_fence *fence)

{

	struct hl_dma_fence *hl_fence =

		container_of(fence, struct hl_dma_fence, base_fence);



	return dev_name(hl_fence->hdev->dev);

}



static bool hl_fence_enable_signaling(struct dma_fence *fence)

{

	return true;

}



static void hl_fence_release(struct dma_fence *fence)

{

	struct hl_dma_fence *hl_fence =

		container_of(fence, struct hl_dma_fence, base_fence);



	kfree_rcu(hl_fence, base_fence.rcu);

}



static const struct dma_fence_ops hl_fence_ops = {

	.get_driver_name = hl_fence_get_driver_name,

	.get_timeline_name = hl_fence_get_timeline_name,

	.enable_signaling = hl_fence_enable_signaling,

	.wait = dma_fence_default_wait,

	.release = hl_fence_release

};



static void cs_get(struct hl_cs *cs)

{

	kref_get(&cs->refcount);

}



static int cs_get_unless_zero(struct hl_cs *cs)

{

	return kref_get_unless_zero(&cs->refcount);

}



static void cs_put(struct hl_cs *cs)

{

	kref_put(&cs->refcount, cs_do_release);

}



/*

 * cs_parser - parse the user command submission

 *

 * @hpriv	: pointer to the private data of the fd

 * @job        : pointer to the job that holds the command submission info

 *

 * The function parses the command submission of the user. It calls the

 * ASIC specific parser, which returns a list of memory blocks to send

 * to the device as different command buffers

 *

 */

static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job)

{

	struct hl_device *hdev = hpriv->hdev;

	struct hl_cs_parser parser;

	int rc;



	parser.ctx_id = job->cs->ctx->asid;

	parser.cs_sequence = job->cs->sequence;

	parser.job_id = job->id;



	parser.hw_queue_id = job->hw_queue_id;

	parser.job_userptr_list = &job->userptr_list;

	parser.patched_cb = NULL;

	parser.user_cb = job->user_cb;

	parser.user_cb_size = job->user_cb_size;

	parser.ext_queue = job->ext_queue;

	job->patched_cb = NULL;

	parser.use_virt_addr = hdev->mmu_enable;



	rc = hdev->asic_funcs->cs_parser(hdev, &parser);

	if (job->ext_queue) {

		if (!rc) {

			job->patched_cb = parser.patched_cb;

			job->job_cb_size = parser.patched_cb_size;



			spin_lock(&job->patched_cb->lock);

			job->patched_cb->cs_cnt++;

			spin_unlock(&job->patched_cb->lock);

		}



		/*

		 * Whether the parsing worked or not, we don't need the

		 * original CB anymore because it was already parsed and

		 * won't be accessed again for this CS

		 */

		spin_lock(&job->user_cb->lock);

		job->user_cb->cs_cnt--;

		spin_unlock(&job->user_cb->lock);

		hl_cb_put(job->user_cb);

		job->user_cb = NULL;

	}



	return rc;

}



static void free_job(struct hl_device *hdev, struct hl_cs_job *job)

{

	struct hl_cs *cs = job->cs;



	if (job->ext_queue) {

		hl_userptr_delete_list(hdev, &job->userptr_list);



		/*

		 * We might arrive here from rollback and patched CB wasn't

		 * created, so we need to check it's not NULL

		 */

		if (job->patched_cb) {

			spin_lock(&job->patched_cb->lock);

			job->patched_cb->cs_cnt--;

			spin_unlock(&job->patched_cb->lock);



			hl_cb_put(job->patched_cb);

		}

	}



	/*

	 * This is the only place where there can be multiple threads

	 * modifying the list at the same time

	 */

	spin_lock(&cs->job_lock);

	list_del(&job->cs_node);

	spin_unlock(&cs->job_lock);



	if (job->ext_queue)

		cs_put(cs);



	kfree(job);

}



static void cs_do_release(struct kref *ref)

{

	struct hl_cs *cs = container_of(ref, struct hl_cs,

						refcount);

	struct hl_device *hdev = cs->ctx->hdev;

	struct hl_cs_job *job, *tmp;



	cs->completed = true;



	/*

	 * Although if we reached here it means that all external jobs have

	 * finished, because each one of them took refcnt to CS, we still

	 * need to go over the internal jobs and free them. Otherwise, we

	 * will have leaked memory and what's worse, the CS object (and

	 * potentially the CTX object) could be released, while the JOB

	 * still holds a pointer to them (but no reference).

	 */

	list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)

		free_job(hdev, job);



	/* We also need to update CI for internal queues */

	if (cs->submitted) {

		hl_int_hw_queue_update_ci(cs);



		spin_lock(&hdev->hw_queues_mirror_lock);

		/* remove CS from hw_queues mirror list */

		list_del_init(&cs->mirror_node);

		spin_unlock(&hdev->hw_queues_mirror_lock);



		/*

		 * Don't cancel TDR in case this CS was timedout because we

		 * might be running from the TDR context

		 */

		if ((!cs->timedout) &&

			(hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT)) {

			struct hl_cs *next;



			if (cs->tdr_active)

				cancel_delayed_work_sync(&cs->work_tdr);



			spin_lock(&hdev->hw_queues_mirror_lock);



			/* queue TDR for next CS */

			next = list_first_entry_or_null(

					&hdev->hw_queues_mirror_list,

					struct hl_cs, mirror_node);



			if ((next) && (!next->tdr_active)) {

				next->tdr_active = true;

				schedule_delayed_work(&next->work_tdr,

							hdev->timeout_jiffies);

			}



			spin_unlock(&hdev->hw_queues_mirror_lock);

		}

	}



	hl_ctx_put(cs->ctx);



	if (cs->timedout)

		dma_fence_set_error(cs->fence, -ETIMEDOUT);

	else if (cs->aborted)

		dma_fence_set_error(cs->fence, -EIO);



	dma_fence_signal(cs->fence);

	dma_fence_put(cs->fence);



	kfree(cs);

}



static void cs_timedout(struct work_struct *work)

{

	struct hl_device *hdev;

	int ctx_asid, rc;

	struct hl_cs *cs = container_of(work, struct hl_cs,

						 work_tdr.work);

	rc = cs_get_unless_zero(cs);

	if (!rc)

		return;



	if ((!cs->submitted) || (cs->completed)) {

		cs_put(cs);

		return;

	}



	/* Mark the CS is timed out so we won't try to cancel its TDR */

	cs->timedout = true;



	hdev = cs->ctx->hdev;

	ctx_asid = cs->ctx->asid;



	/* TODO: add information about last signaled seq and last emitted seq */

	dev_err(hdev->dev, "CS %d.%llu got stuck!\n", ctx_asid, cs->sequence);



	cs_put(cs);



	if (hdev->reset_on_lockup)

		hl_device_reset(hdev, false, false);

}



static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,

			struct hl_cs **cs_new)

{

	struct hl_dma_fence *fence;

	struct dma_fence *other = NULL;

	struct hl_cs *cs;

	int rc;



	cs = kzalloc(sizeof(*cs), GFP_ATOMIC);

	if (!cs)

		return -ENOMEM;



	cs->ctx = ctx;

	cs->submitted = false;

	cs->completed = false;

	INIT_LIST_HEAD(&cs->job_list);

	INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout);

	kref_init(&cs->refcount);

	spin_lock_init(&cs->job_lock);



	fence = kmalloc(sizeof(*fence), GFP_ATOMIC);

	if (!fence) {

		rc = -ENOMEM;

		goto free_cs;

	}



	fence->hdev = hdev;

	spin_lock_init(&fence->lock);

	cs->fence = &fence->base_fence;



	spin_lock(&ctx->cs_lock);



	fence->cs_seq = ctx->cs_sequence;

	other = ctx->cs_pending[fence->cs_seq & (HL_MAX_PENDING_CS - 1)];

	if ((other) && (!dma_fence_is_signaled(other))) {

		spin_unlock(&ctx->cs_lock);

		rc = -EAGAIN;

		goto free_fence;

	}



	dma_fence_init(&fence->base_fence, &hl_fence_ops, &fence->lock,

			ctx->asid, ctx->cs_sequence);



	cs->sequence = fence->cs_seq;



	ctx->cs_pending[fence->cs_seq & (HL_MAX_PENDING_CS - 1)] =

							&fence->base_fence;

	ctx->cs_sequence++;



	dma_fence_get(&fence->base_fence);



	dma_fence_put(other);



	spin_unlock(&ctx->cs_lock);



	*cs_new = cs;



	return 0;



free_fence:

	kfree(fence);

free_cs:

	kfree(cs);

	return rc;

}



static void cs_rollback(struct hl_device *hdev, struct hl_cs *cs)

{

	struct hl_cs_job *job, *tmp;



	list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)

		free_job(hdev, job);

}



void hl_cs_rollback_all(struct hl_device *hdev)

{

	struct hl_cs *cs, *tmp;



	/* flush all completions */

	flush_workqueue(hdev->cq_wq);



	/* Make sure we don't have leftovers in the H/W queues mirror list */

	list_for_each_entry_safe(cs, tmp, &hdev->hw_queues_mirror_list,

				mirror_node) {

		cs_get(cs);

		cs->aborted = true;

		dev_warn_ratelimited(hdev->dev, "Killing CS %d.%llu\n",

					cs->ctx->asid, cs->sequence);

		cs_rollback(hdev, cs);

		cs_put(cs);

	}

}



static void job_wq_completion(struct work_struct *work)

{

	struct hl_cs_job *job = container_of(work, struct hl_cs_job,

						finish_work);

	struct hl_cs *cs = job->cs;

	struct hl_device *hdev = cs->ctx->hdev;



	/* job is no longer needed */

	free_job(hdev, job);

}



static struct hl_cb *validate_queue_index(struct hl_device *hdev,

					struct hl_cb_mgr *cb_mgr,

					struct hl_cs_chunk *chunk,

					bool *ext_queue)

{

	struct asic_fixed_properties *asic = &hdev->asic_prop;

	struct hw_queue_properties *hw_queue_prop;

	u32 cb_handle;

	struct hl_cb *cb;



	/* Assume external queue */

	*ext_queue = true;



	hw_queue_prop = &asic->hw_queues_props[chunk->queue_index];



	if ((chunk->queue_index >= HL_MAX_QUEUES) ||

			(hw_queue_prop->type == QUEUE_TYPE_NA)) {

		dev_err(hdev->dev, "Queue index %d is invalid\n",

			chunk->queue_index);

		return NULL;

	}



	if (hw_queue_prop->kmd_only) {

		dev_err(hdev->dev, "Queue index %d is restricted for KMD\n",

			chunk->queue_index);

		return NULL;

	} else if (hw_queue_prop->type == QUEUE_TYPE_INT) {

		*ext_queue = false;

		return (struct hl_cb *) (uintptr_t) chunk->cb_handle;

	}



	/* Retrieve CB object */

	cb_handle = (u32) (chunk->cb_handle >> PAGE_SHIFT);



	cb = hl_cb_get(hdev, cb_mgr, cb_handle);

	if (!cb) {

		dev_err(hdev->dev, "CB handle 0x%x invalid\n", cb_handle);

		return NULL;

	}



	if ((chunk->cb_size < 8) || (chunk->cb_size > cb->size)) {

		dev_err(hdev->dev, "CB size %u invalid\n", chunk->cb_size);

		goto release_cb;

	}



	spin_lock(&cb->lock);

	cb->cs_cnt++;

	spin_unlock(&cb->lock);



	return cb;



release_cb:

	hl_cb_put(cb);

	return NULL;

}



struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev, bool ext_queue)

{

	struct hl_cs_job *job;



	job = kzalloc(sizeof(*job), GFP_ATOMIC);

	if (!job)

		return NULL;



	job->ext_queue = ext_queue;



	if (job->ext_queue) {

		INIT_LIST_HEAD(&job->userptr_list);

		INIT_WORK(&job->finish_work, job_wq_completion);

	}



	return job;

}



static int _hl_cs_ioctl(struct hl_fpriv *hpriv, void __user *chunks,

			u32 num_chunks, u64 *cs_seq)

{

	struct hl_device *hdev = hpriv->hdev;

	struct hl_cs_chunk *cs_chunk_array;

	struct hl_cs_job *job;

	struct hl_cs *cs;

	struct hl_cb *cb;

	bool ext_queue_present = false;

	u32 size_to_copy;

	int rc, i, parse_cnt;



	*cs_seq = ULLONG_MAX;



	if (num_chunks > HL_MAX_JOBS_PER_CS) {

		dev_err(hdev->dev,

			"Number of chunks can NOT be larger than %d\n",

			HL_MAX_JOBS_PER_CS);

		rc = -EINVAL;

		goto out;

	}



	cs_chunk_array = kmalloc_array(num_chunks, sizeof(*cs_chunk_array),

					GFP_ATOMIC);

	if (!cs_chunk_array) {

		rc = -ENOMEM;

		goto out;

	}



	size_to_copy = num_chunks * sizeof(struct hl_cs_chunk);

	if (copy_from_user(cs_chunk_array, chunks, size_to_copy)) {

		dev_err(hdev->dev, "Failed to copy cs chunk array from user\n");

		rc = -EFAULT;

		goto free_cs_chunk_array;

	}



	/* increment refcnt for context */

	hl_ctx_get(hdev, hpriv->ctx);



	rc = allocate_cs(hdev, hpriv->ctx, &cs);

	if (rc) {

		hl_ctx_put(hpriv->ctx);

		goto free_cs_chunk_array;

	}



	*cs_seq = cs->sequence;



	/* Validate ALL the CS chunks before submitting the CS */

	for (i = 0, parse_cnt = 0 ; i < num_chunks ; i++, parse_cnt++) {

		struct hl_cs_chunk *chunk = &cs_chunk_array[i];

		bool ext_queue;



		cb = validate_queue_index(hdev, &hpriv->cb_mgr, chunk,

					&ext_queue);

		if (ext_queue) {

			ext_queue_present = true;

			if (!cb) {

				rc = -EINVAL;

				goto free_cs_object;

			}

		}



		job = hl_cs_allocate_job(hdev, ext_queue);

		if (!job) {

			dev_err(hdev->dev, "Failed to allocate a new job\n");

			rc = -ENOMEM;

			if (ext_queue)

				goto release_cb;

			else

				goto free_cs_object;

		}



		job->id = i + 1;

		job->cs = cs;

		job->user_cb = cb;

		job->user_cb_size = chunk->cb_size;

		if (job->ext_queue)

			job->job_cb_size = cb->size;

		else

			job->job_cb_size = chunk->cb_size;

		job->hw_queue_id = chunk->queue_index;



		cs->jobs_in_queue_cnt[job->hw_queue_id]++;



		list_add_tail(&job->cs_node, &cs->job_list);



		/*

		 * Increment CS reference. When CS reference is 0, CS is

		 * done and can be signaled to user and free all its resources

		 * Only increment for JOB on external queues, because only

		 * for those JOBs we get completion

		 */

		if (job->ext_queue)

			cs_get(cs);



		rc = cs_parser(hpriv, job);

		if (rc) {

			dev_err(hdev->dev,

				"Failed to parse JOB %d.%llu.%d, err %d, rejecting the CS\n",

				cs->ctx->asid, cs->sequence, job->id, rc);

			goto free_cs_object;

		}

	}



	if (!ext_queue_present) {

		dev_err(hdev->dev,

			"Reject CS %d.%llu because no external queues jobs\n",

			cs->ctx->asid, cs->sequence);

		rc = -EINVAL;

		goto free_cs_object;

	}



	rc = hl_hw_queue_schedule_cs(cs);

	if (rc) {

		dev_err(hdev->dev,

			"Failed to submit CS %d.%llu to H/W queues, error %d\n",

			cs->ctx->asid, cs->sequence, rc);

		goto free_cs_object;

	}



	rc = HL_CS_STATUS_SUCCESS;

	goto put_cs;



release_cb:

	spin_lock(&cb->lock);

	cb->cs_cnt--;

	spin_unlock(&cb->lock);

	hl_cb_put(cb);

free_cs_object:

	cs_rollback(hdev, cs);

	*cs_seq = ULLONG_MAX;

	/* The path below is both for good and erroneous exits */

put_cs:

	/* We finished with the CS in this function, so put the ref */

	cs_put(cs);

free_cs_chunk_array:

	kfree(cs_chunk_array);

out:

	return rc;

}



int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)

{

	struct hl_device *hdev = hpriv->hdev;

	union hl_cs_args *args = data;

	struct hl_ctx *ctx = hpriv->ctx;

	void __user *chunks;

	u32 num_chunks;

	u64 cs_seq = ULONG_MAX;

	int rc, do_restore;

	bool need_soft_reset = false;



	if (hl_device_disabled_or_in_reset(hdev)) {

		dev_warn(hdev->dev,

			"Device is %s. Can't submit new CS\n",

			atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");

		rc = -EBUSY;

		goto out;

	}



	do_restore = atomic_cmpxchg(&ctx->thread_restore_token, 1, 0);



	if (do_restore || (args->in.cs_flags & HL_CS_FLAGS_FORCE_RESTORE)) {

		long ret;



		chunks = (void __user *)(uintptr_t)args->in.chunks_restore;

		num_chunks = args->in.num_chunks_restore;



		mutex_lock(&hpriv->restore_phase_mutex);



		if (do_restore) {

			rc = hdev->asic_funcs->context_switch(hdev, ctx->asid);

			if (rc) {

				dev_err_ratelimited(hdev->dev,

					"Failed to switch to context %d, rejecting CS! %d\n",

					ctx->asid, rc);

				/*

				 * If we timedout, we need to soft-reset because

				 * QMAN is probably stuck. However, we can't

				 * call to reset here directly because of

				 * deadlock, so need to do it at the very end

				 * of this function

				 */

				if (rc == -ETIMEDOUT)

					need_soft_reset = true;

				mutex_unlock(&hpriv->restore_phase_mutex);

				goto out;

			}

		}



		hdev->asic_funcs->restore_phase_topology(hdev);



		if (num_chunks == 0) {

			dev_dbg(hdev->dev,

			"Need to run restore phase but restore CS is empty\n");

			rc = 0;

		} else {

			rc = _hl_cs_ioctl(hpriv, chunks, num_chunks,

						&cs_seq);

		}



		mutex_unlock(&hpriv->restore_phase_mutex);



		if (rc) {

			dev_err(hdev->dev,

				"Failed to submit restore CS for context %d (%d)\n",

				ctx->asid, rc);

			goto out;

		}



		/* Need to wait for restore completion before execution phase */

		if (num_chunks > 0) {

			ret = _hl_cs_wait_ioctl(hdev, ctx,

					jiffies_to_usecs(hdev->timeout_jiffies),

					cs_seq);

			if (ret <= 0) {

				dev_err(hdev->dev,

					"Restore CS for context %d failed to complete %ld\n",

					ctx->asid, ret);

				rc = -ENOEXEC;

				goto out;

			}

		}



		ctx->thread_restore_wait_token = 1;

	} else if (!ctx->thread_restore_wait_token) {

		u32 tmp;



		rc = hl_poll_timeout_memory(hdev,

			(u64) (uintptr_t) &ctx->thread_restore_wait_token,

			jiffies_to_usecs(hdev->timeout_jiffies),

			&tmp);



		if (rc || !tmp) {

			dev_err(hdev->dev,

				"restore phase hasn't finished in time\n");

			rc = -ETIMEDOUT;

			goto out;

		}

	}



	chunks = (void __user *)(uintptr_t)args->in.chunks_execute;

	num_chunks = args->in.num_chunks_execute;



	if (num_chunks == 0) {

		dev_err(hdev->dev,

			"Got execute CS with 0 chunks, context %d\n",

			ctx->asid);

		rc = -EINVAL;

		goto out;

	}



	rc = _hl_cs_ioctl(hpriv, chunks, num_chunks, &cs_seq);



out:

	if (rc != -EAGAIN) {

		memset(args, 0, sizeof(*args));

		args->out.status = rc;

		args->out.seq = cs_seq;

	}



	if ((rc == -ETIMEDOUT) && (need_soft_reset))

		hl_device_reset(hdev, false, false);



	return rc;

}



static long _hl_cs_wait_ioctl(struct hl_device *hdev,

		struct hl_ctx *ctx, u64 timeout_us, u64 seq)

{

	struct dma_fence *fence;

	unsigned long timeout;

	long rc;



	if (timeout_us == MAX_SCHEDULE_TIMEOUT)

		timeout = timeout_us;

	else

		timeout = usecs_to_jiffies(timeout_us);



	hl_ctx_get(hdev, ctx);



	fence = hl_ctx_get_fence(ctx, seq);

	if (IS_ERR(fence)) {

		rc = PTR_ERR(fence);

	} else if (fence) {

		rc = dma_fence_wait_timeout(fence, true, timeout);

		if (fence->error == -ETIMEDOUT)

			rc = -ETIMEDOUT;

		else if (fence->error == -EIO)

			rc = -EIO;

		dma_fence_put(fence);

	} else

		rc = 1;



	hl_ctx_put(ctx);



	return rc;

}



int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)

{

	struct hl_device *hdev = hpriv->hdev;

	union hl_wait_cs_args *args = data;

	u64 seq = args->in.seq;

	long rc;



	rc = _hl_cs_wait_ioctl(hdev, hpriv->ctx, args->in.timeout_us, seq);



	memset(args, 0, sizeof(*args));



	if (rc < 0) {

		dev_err(hdev->dev, "Error %ld on waiting for CS handle %llu\n",

			rc, seq);

		if (rc == -ERESTARTSYS) {

			args->out.status = HL_WAIT_CS_STATUS_INTERRUPTED;

			rc = -EINTR;

		} else if (rc == -ETIMEDOUT) {

			args->out.status = HL_WAIT_CS_STATUS_TIMEDOUT;

		} else if (rc == -EIO) {

			args->out.status = HL_WAIT_CS_STATUS_ABORTED;

		}

		return rc;

	}



	if (rc == 0)

		args->out.status = HL_WAIT_CS_STATUS_BUSY;

	else

		args->out.status = HL_WAIT_CS_STATUS_COMPLETED;



	return 0;

}

drivers/misc/habanalabs/context.c

+48 −4
Original line number Diff line number Diff line
@@ -12,6 +12,18 @@ 
static void hl_ctx_fini(struct hl_ctx *ctx)

{

	struct hl_device *hdev = ctx->hdev;

	int i;



	/*

	 * If we arrived here, there are no jobs waiting for this context

	 * on its queues so we can safely remove it.

	 * This is because for each CS, we increment the ref count and for

	 * every CS that was finished we decrement it and we won't arrive

	 * to this function unless the ref count is 0

	 */



	for (i = 0 ; i < HL_MAX_PENDING_CS ; i++)

		dma_fence_put(ctx->cs_pending[i]);



	if (ctx->asid != HL_KERNEL_ASID_ID)

		hl_asid_free(hdev, ctx->asid);
@@ -23,8 +35,6 @@ void hl_ctx_do_release(struct kref *ref) 


	ctx = container_of(ref, struct hl_ctx, refcount);



	dev_dbg(ctx->hdev->dev, "Now really releasing context %d\n", ctx->asid);



	hl_ctx_fini(ctx);



	if (ctx->hpriv)
@@ -90,6 +100,11 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) 


	kref_init(&ctx->refcount);



	ctx->cs_sequence = 1;

	spin_lock_init(&ctx->cs_lock);

	atomic_set(&ctx->thread_restore_token, 1);

	ctx->thread_restore_wait_token = 0;



	if (is_kernel_ctx) {

		ctx->asid = HL_KERNEL_ASID_ID; /* KMD gets ASID 0 */

	} else {
@@ -100,8 +115,6 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) 
		}

	}



	dev_dbg(hdev->dev, "Created context with ASID %u\n", ctx->asid);



	return 0;

}
@@ -115,6 +128,37 @@ int hl_ctx_put(struct hl_ctx *ctx) 
	return kref_put(&ctx->refcount, hl_ctx_do_release);

}



struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)

{

	struct hl_device *hdev = ctx->hdev;

	struct dma_fence *fence;



	spin_lock(&ctx->cs_lock);



	if (seq >= ctx->cs_sequence) {

		dev_notice(hdev->dev,

			"Can't wait on seq %llu because current CS is at seq %llu\n",

			seq, ctx->cs_sequence);

		spin_unlock(&ctx->cs_lock);

		return ERR_PTR(-EINVAL);

	}





	if (seq + HL_MAX_PENDING_CS < ctx->cs_sequence) {

		dev_dbg(hdev->dev,

			"Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",

			seq, ctx->cs_sequence);

		spin_unlock(&ctx->cs_lock);

		return NULL;

	}



	fence = dma_fence_get(

			ctx->cs_pending[seq & (HL_MAX_PENDING_CS - 1)]);

	spin_unlock(&ctx->cs_lock);



	return fence;

}



/*

 * hl_ctx_mgr_init - initialize the context manager

 *

drivers/misc/habanalabs/device.c

+16 −0
Original line number Diff line number Diff line
@@ -30,6 +30,8 @@ static void hpriv_release(struct kref *ref) 


	put_pid(hpriv->taskpid);



	mutex_destroy(&hpriv->restore_phase_mutex);



	kfree(hpriv);



	/* Now the FD is really closed */
@@ -208,6 +210,8 @@ static int device_early_init(struct hl_device *hdev) 


	mutex_init(&hdev->fd_open_cnt_lock);

	mutex_init(&hdev->send_cpu_message_lock);

	INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);

	spin_lock_init(&hdev->hw_queues_mirror_lock);

	atomic_set(&hdev->in_reset, 0);

	atomic_set(&hdev->fd_open_cnt, 0);
@@ -593,6 +597,9 @@ again: 
	 */

	hdev->asic_funcs->halt_engines(hdev, hard_reset);



	/* Go over all the queues, release all CS and their jobs */

	hl_cs_rollback_all(hdev);



	if (hard_reset) {

		/* Release kernel context */

		if (hl_ctx_put(hdev->kernel_ctx) != 1) {
@@ -616,6 +623,12 @@ again: 
	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)

		hl_cq_reset(hdev, &hdev->completion_queue[i]);



	/* Make sure the setup phase for the user context will run again */

	if (hdev->user_ctx) {

		atomic_set(&hdev->user_ctx->thread_restore_token, 1);

		hdev->user_ctx->thread_restore_wait_token = 0;

	}



	/* Finished tear-down, starting to re-initialize */



	if (hard_reset) {
@@ -952,6 +965,9 @@ void hl_device_fini(struct hl_device *hdev) 
	 */

	hdev->asic_funcs->halt_engines(hdev, true);



	/* Go over all the queues, release all CS and their jobs */

	hl_cs_rollback_all(hdev);



	hl_cb_pool_fini(hdev);



	/* Release kernel context */

drivers/misc/habanalabs/goya/goya.c

+1126 −0

File changed.

Preview size limit exceeded, changes collapsed.

CRA Git | Maintained and supported by SUSTech CRA and CCSE