Commit 9096d6e5 authored by Christian König's avatar Christian König Committed by Alex Deucher
Browse files

drm/amdgpu: implement gmc_v9_0_emit_flush_gpu_tlb 



Unify tlb flushing for gmc v9.

Signed-off-by: default avatarChristian König <christian.koenig@amd.com>
Acked-by: default avatarChunming Zhou <david1.zhou@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent 5518625d

drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h

+0 −1
Original line number Diff line number Diff line
@@ -69,7 +69,6 @@ struct amdgpu_gmc_funcs { 
	/* get the pde for a given mc addr */

	void (*get_vm_pde)(struct amdgpu_device *adev, int level,

			   u64 *dst, u64 *flags);

	uint32_t (*get_invalidate_req)(unsigned int vmid);

};



struct amdgpu_gmc {

drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

+7 −22
Original line number Diff line number Diff line
@@ -3688,31 +3688,16 @@ static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 
					uint64_t pd_addr)

{

	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];

	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);

	uint32_t req = ring->adev->gmc.gmc_funcs->get_invalidate_req(vmid);

	uint64_t flags = AMDGPU_PTE_VALID;

	unsigned eng = ring->vm_inv_eng;



	amdgpu_gmc_get_vm_pde(ring->adev, -1, &pd_addr, &flags);

	pd_addr |= flags;



	gfx_v9_0_write_data_to_reg(ring, usepfp, true,

				   hub->ctx0_ptb_addr_lo32 + (2 * vmid),

				   lower_32_bits(pd_addr));



	gfx_v9_0_write_data_to_reg(ring, usepfp, true,

				   hub->ctx0_ptb_addr_hi32 + (2 * vmid),

				   upper_32_bits(pd_addr));



	gfx_v9_0_write_data_to_reg(ring, usepfp, true,

				   hub->vm_inv_eng0_req + eng, req);

	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);



	/* wait for the invalidate to complete */

	gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, hub->vm_inv_eng0_ack +

			      eng, 0, 1 << vmid, 1 << vmid, 0x20);

	gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, hub->vm_inv_eng0_ack + eng,

			      0, 1 << vmid, 1 << vmid, 0x20);



	/* compute doesn't have PFP */

	if (usepfp) {

	if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {

		/* sync PFP to ME, otherwise we might get invalid PFP reads */

		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));

		amdgpu_ring_write(ring, 0x0);
@@ -4312,7 +4297,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 
	.emit_frame_size = /* totally 242 maximum if 16 IBs */

		5 +  /* COND_EXEC */

		7 +  /* PIPELINE_SYNC */

		24 + /* VM_FLUSH */

		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */

		8 +  /* FENCE for VM_FLUSH */

		20 + /* GDS switch */

		4 + /* double SWITCH_BUFFER,
@@ -4361,7 +4346,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 
		7 + /* gfx_v9_0_ring_emit_hdp_flush */

		5 + /* gfx_v9_0_ring_emit_hdp_invalidate */

		7 + /* gfx_v9_0_ring_emit_pipeline_sync */

		24 + /* gfx_v9_0_ring_emit_vm_flush */

		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* gfx_v9_0_ring_emit_vm_flush */

		8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */

	.emit_ib_size =	4, /* gfx_v9_0_ring_emit_ib_compute */

	.emit_ib = gfx_v9_0_ring_emit_ib_compute,
@@ -4393,7 +4378,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 
		7 + /* gfx_v9_0_ring_emit_hdp_flush */

		5 + /* gfx_v9_0_ring_emit_hdp_invalidate */

		7 + /* gfx_v9_0_ring_emit_pipeline_sync */

		24 + /* gfx_v9_0_ring_emit_vm_flush */

		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* gfx_v9_0_ring_emit_vm_flush */

		8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */

	.emit_ib_size =	4, /* gfx_v9_0_ring_emit_ib_compute */

	.emit_ib = gfx_v9_0_ring_emit_ib_compute,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c

+24 −1
Original line number Diff line number Diff line
@@ -366,6 +366,29 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, 
	spin_unlock(&adev->gmc.invalidate_lock);

}



static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,

					    unsigned vmid, unsigned pasid,

					    uint64_t pd_addr)

{

	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];

	uint32_t req = gmc_v9_0_get_invalidate_req(vmid);

	uint64_t flags = AMDGPU_PTE_VALID;

	unsigned eng = ring->vm_inv_eng;



	amdgpu_gmc_get_vm_pde(ring->adev, -1, &pd_addr, &flags);

	pd_addr |= flags;



	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid),

			      lower_32_bits(pd_addr));



	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid),

			      upper_32_bits(pd_addr));



	amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_req + eng, req);



	return pd_addr;

}



/**

 * gmc_v9_0_set_pte_pde - update the page tables using MMIO

 *
@@ -491,8 +514,8 @@ static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 


static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {

	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,

	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,

	.set_pte_pde = gmc_v9_0_set_pte_pde,

	.get_invalidate_req = gmc_v9_0_get_invalidate_req,

	.get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags,

	.get_vm_pde = gmc_v9_0_get_vm_pde

};

drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c

+2 −21
Original line number Diff line number Diff line
@@ -1137,28 +1137,9 @@ static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 
					 uint64_t pd_addr)

{

	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];

	uint32_t req = ring->adev->gmc.gmc_funcs->get_invalidate_req(vmid);

	uint64_t flags = AMDGPU_PTE_VALID;

	unsigned eng = ring->vm_inv_eng;



	amdgpu_gmc_get_vm_pde(ring->adev, -1, &pd_addr, &flags);

	pd_addr |= flags;



	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |

			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));

	amdgpu_ring_write(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2);

	amdgpu_ring_write(ring, lower_32_bits(pd_addr));



	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |

			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));

	amdgpu_ring_write(ring, hub->ctx0_ptb_addr_hi32 + vmid * 2);

	amdgpu_ring_write(ring, upper_32_bits(pd_addr));



	/* flush TLB */

	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |

			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));

	amdgpu_ring_write(ring, hub->vm_inv_eng0_req + eng);

	amdgpu_ring_write(ring, req);

	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);



	/* wait for flush */

	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
@@ -1604,7 +1585,7 @@ static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs = { 
		6 + /* sdma_v4_0_ring_emit_hdp_flush */

		3 + /* sdma_v4_0_ring_emit_hdp_invalidate */

		6 + /* sdma_v4_0_ring_emit_pipeline_sync */

		18 + /* sdma_v4_0_ring_emit_vm_flush */

		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + 6 + /* sdma_v4_0_ring_emit_vm_flush */

		10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */

	.emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */

	.emit_ib = sdma_v4_0_ring_emit_ib,

drivers/gpu/drm/amd/amdgpu/soc15.h

+2 −0
Original line number Diff line number Diff line
@@ -27,6 +27,8 @@ 
#include "nbio_v6_1.h"

#include "nbio_v7_0.h"



#define SOC15_FLUSH_GPU_TLB_NUM_WREG	3



extern const struct amd_ip_funcs soc15_common_ip_funcs;



struct soc15_reg_golden {

CRA Git | Maintained and supported by SUSTech CRA and CCSE