Commit b2cff0db authored by Chris Wilson's avatar Chris Wilson Committed by Daniel Vetter
Browse files

drm/i915: Reduce duplicated forcewake logic 



Introduce a structure to track the individual forcewake domains and use
that to eliminate duplicate logic.

v2: - Rebase on latest dinq (Mika)
    - for_each_fw_domain macro (Mika)
    - Handle reset atomically, keeping the timer running (Mika)
    - for_each_fw_domain parameter ordering (Chris)
    - defer timer on new register access (Mika)

v3: - Fix forcewake_reset/get race by waiting pending timers

v4: - cond_resched and verbose warning on timer deletion (Chris)
    - need to run pending timers manually on reset

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> (v1)
Signed-off-by: default avatarMika Kuoppala <mika.kuoppala@intel.com>
Acked-by: Deepak S <deepak.s@linux.intel.com> (v2)
Signed-off-by: default avatarDaniel Vetter <daniel.vetter@ffwll.ch>
parent 51f67885

drivers/gpu/drm/i915/i915_debugfs.c

+26 −39
Original line number Diff line number Diff line
@@ -1288,14 +1288,36 @@ static int ironlake_drpc_info(struct seq_file *m) 
	return 0;

}



static int vlv_drpc_info(struct seq_file *m)

static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)

{

	struct drm_info_node *node = m->private;

	struct drm_device *dev = node->minor->dev;

	struct drm_i915_private *dev_priv = dev->dev_private;

	struct intel_uncore_forcewake_domain *fw_domain;

	const char *domain_names[] = {

		"render",

		"blitter",

		"media",

	};

	int i;



	spin_lock_irq(&dev_priv->uncore.lock);

	for_each_fw_domain(fw_domain, dev_priv, i) {

		seq_printf(m, "%s.wake_count = %u\n",

			   domain_names[i],

			   fw_domain->wake_count);

	}

	spin_unlock_irq(&dev_priv->uncore.lock);



	return 0;

}



static int vlv_drpc_info(struct seq_file *m)

{

	struct drm_info_node *node = m->private;

	struct drm_device *dev = node->minor->dev;

	struct drm_i915_private *dev_priv = dev->dev_private;

	u32 rpmodectl1, rcctl1, pw_status;

	unsigned fw_rendercount = 0, fw_mediacount = 0;



	intel_runtime_pm_get(dev_priv);
@@ -1327,22 +1349,11 @@ static int vlv_drpc_info(struct seq_file *m) 
	seq_printf(m, "Media RC6 residency since boot: %u\n",

		   I915_READ(VLV_GT_MEDIA_RC6));



	spin_lock_irq(&dev_priv->uncore.lock);

	fw_rendercount = dev_priv->uncore.fw_rendercount;

	fw_mediacount = dev_priv->uncore.fw_mediacount;

	spin_unlock_irq(&dev_priv->uncore.lock);



	seq_printf(m, "Forcewake Render Count = %u\n", fw_rendercount);

	seq_printf(m, "Forcewake Media Count = %u\n", fw_mediacount);





	return 0;

	return i915_gen6_forcewake_count_info(m, NULL);

}





static int gen6_drpc_info(struct seq_file *m)

{



	struct drm_info_node *node = m->private;

	struct drm_device *dev = node->minor->dev;

	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1356,7 +1367,7 @@ static int gen6_drpc_info(struct seq_file *m) 
	intel_runtime_pm_get(dev_priv);



	spin_lock_irq(&dev_priv->uncore.lock);

	forcewake_count = dev_priv->uncore.forcewake_count;

	forcewake_count = dev_priv->uncore.fw_domain[FW_DOMAIN_ID_RENDER].wake_count;

	spin_unlock_irq(&dev_priv->uncore.lock);



	if (forcewake_count) {
@@ -1984,30 +1995,6 @@ static int i915_execlists(struct seq_file *m, void *data) 
	return 0;

}



static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)

{

	struct drm_info_node *node = m->private;

	struct drm_device *dev = node->minor->dev;

	struct drm_i915_private *dev_priv = dev->dev_private;

	unsigned forcewake_count = 0, fw_rendercount = 0, fw_mediacount = 0;



	spin_lock_irq(&dev_priv->uncore.lock);

	if (IS_VALLEYVIEW(dev)) {

		fw_rendercount = dev_priv->uncore.fw_rendercount;

		fw_mediacount = dev_priv->uncore.fw_mediacount;

	} else

		forcewake_count = dev_priv->uncore.forcewake_count;

	spin_unlock_irq(&dev_priv->uncore.lock);



	if (IS_VALLEYVIEW(dev)) {

		seq_printf(m, "fw_rendercount = %u\n", fw_rendercount);

		seq_printf(m, "fw_mediacount = %u\n", fw_mediacount);

	} else

		seq_printf(m, "forcewake count = %u\n", forcewake_count);



	return 0;

}



static const char *swizzle_string(unsigned swizzle)

{

	switch (swizzle) {

drivers/gpu/drm/i915/i915_drv.h

+37 −17
Original line number Diff line number Diff line
@@ -596,20 +596,45 @@ struct intel_uncore_funcs { 
				uint64_t val, bool trace);

};



enum {

	FW_DOMAIN_ID_RENDER = 0,

	FW_DOMAIN_ID_BLITTER,

	FW_DOMAIN_ID_MEDIA,



	FW_DOMAIN_ID_COUNT

};



struct intel_uncore {

	spinlock_t lock; /** lock is also taken in irq contexts. */



	struct intel_uncore_funcs funcs;



	unsigned fifo_count;

	unsigned forcewake_count;



	unsigned fw_rendercount;

	unsigned fw_mediacount;

	unsigned fw_blittercount;

	unsigned fw_domains;



	struct timer_list force_wake_timer;

};

	struct intel_uncore_forcewake_domain {

		struct drm_i915_private *i915;

		int id;

		unsigned wake_count;

		struct timer_list timer;

	} fw_domain[FW_DOMAIN_ID_COUNT];

#define FORCEWAKE_RENDER	(1 << FW_DOMAIN_ID_RENDER)

#define FORCEWAKE_BLITTER	(1 << FW_DOMAIN_ID_BLITTER)

#define FORCEWAKE_MEDIA		(1 << FW_DOMAIN_ID_MEDIA)

#define FORCEWAKE_ALL		(FORCEWAKE_RENDER | \

				 FORCEWAKE_BLITTER | \

				 FORCEWAKE_MEDIA)

};



/* Iterate over initialised fw domains */

#define for_each_fw_domain_mask(domain__, mask__, dev_priv__, i__) \

	for ((i__) = 0, (domain__) = &(dev_priv__)->uncore.fw_domain[0]; \

	     (i__) < FW_DOMAIN_ID_COUNT; \

	     (i__)++, (domain__) = &(dev_priv__)->uncore.fw_domain[i__]) \

		if (((mask__) & (dev_priv__)->uncore.fw_domains) & (1 << (i__)))



#define for_each_fw_domain(domain__, dev_priv__, i__) \

	for_each_fw_domain_mask(domain__, FORCEWAKE_ALL, dev_priv__, i__)



#define DEV_INFO_FOR_EACH_FLAG(func, sep) \

	func(is_mobile) sep \
@@ -3167,8 +3192,10 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, 
 * must be set to prevent GT core from power down and stale values being

 * returned.

 */

void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine);

void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine);

void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,

			    unsigned fw_domains);

void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,

			    unsigned fw_domains);

void assert_force_wake_inactive(struct drm_i915_private *dev_priv);



int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val);
@@ -3200,13 +3227,6 @@ void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); 
int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val);

int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val);



#define FORCEWAKE_RENDER	(1 << 0)

#define FORCEWAKE_MEDIA		(1 << 1)

#define FORCEWAKE_BLITTER	(1 << 2)

#define FORCEWAKE_ALL		(FORCEWAKE_RENDER | FORCEWAKE_MEDIA | \

					FORCEWAKE_BLITTER)





#define I915_READ8(reg)		dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true)

#define I915_WRITE8(reg, val)	dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true)

drivers/gpu/drm/i915/intel_uncore.c

+183 −262
Original line number Diff line number Diff line
@@ -229,7 +229,6 @@ static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, 
static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,

				 int fw_engine)

{



	/* Check for Render Engine */

	if (FORCEWAKE_RENDER & fw_engine)

		__raw_i915_write32(dev_priv, FORCEWAKE_VLV,
@@ -247,37 +246,6 @@ static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, 
		gen6_gt_check_fifodbg(dev_priv);

}



static void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)

{

	if (fw_engine & FORCEWAKE_RENDER &&

	    dev_priv->uncore.fw_rendercount++ != 0)

		fw_engine &= ~FORCEWAKE_RENDER;

	if (fw_engine & FORCEWAKE_MEDIA &&

	    dev_priv->uncore.fw_mediacount++ != 0)

		fw_engine &= ~FORCEWAKE_MEDIA;



	if (fw_engine)

		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine);

}



static void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)

{

	if (fw_engine & FORCEWAKE_RENDER) {

		WARN_ON(!dev_priv->uncore.fw_rendercount);

		if (--dev_priv->uncore.fw_rendercount != 0)

			fw_engine &= ~FORCEWAKE_RENDER;

	}



	if (fw_engine & FORCEWAKE_MEDIA) {

		WARN_ON(!dev_priv->uncore.fw_mediacount);

		if (--dev_priv->uncore.fw_mediacount != 0)

			fw_engine &= ~FORCEWAKE_MEDIA;

	}



	if (fw_engine)

		dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine);

}



static void __gen9_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)

{

	__raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9,
@@ -367,80 +335,72 @@ __gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) 
				_MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));

}



static void

gen9_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)

static void gen6_force_wake_timer(unsigned long arg)

{

	if (FORCEWAKE_RENDER & fw_engine) {

		if (dev_priv->uncore.fw_rendercount++ == 0)

			dev_priv->uncore.funcs.force_wake_get(dev_priv,

							FORCEWAKE_RENDER);

	}

	struct intel_uncore_forcewake_domain *domain = (void *)arg;

	unsigned long irqflags;



	if (FORCEWAKE_MEDIA & fw_engine) {

		if (dev_priv->uncore.fw_mediacount++ == 0)

			dev_priv->uncore.funcs.force_wake_get(dev_priv,

							FORCEWAKE_MEDIA);

	}

	assert_device_not_suspended(domain->i915);



	if (FORCEWAKE_BLITTER & fw_engine) {

		if (dev_priv->uncore.fw_blittercount++ == 0)

			dev_priv->uncore.funcs.force_wake_get(dev_priv,

							FORCEWAKE_BLITTER);

	}

	spin_lock_irqsave(&domain->i915->uncore.lock, irqflags);

	if (WARN_ON(domain->wake_count == 0))

		domain->wake_count++;



	if (--domain->wake_count == 0)

		domain->i915->uncore.funcs.force_wake_put(domain->i915,

							  1 << domain->id);



	spin_unlock_irqrestore(&domain->i915->uncore.lock, irqflags);

}



static void

gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)

void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)

{

	if (FORCEWAKE_RENDER & fw_engine) {

		WARN_ON(dev_priv->uncore.fw_rendercount == 0);

		if (--dev_priv->uncore.fw_rendercount == 0)

			dev_priv->uncore.funcs.force_wake_put(dev_priv,

							FORCEWAKE_RENDER);

	}

	struct drm_i915_private *dev_priv = dev->dev_private;

	unsigned long irqflags, fw = 0;

	struct intel_uncore_forcewake_domain *domain;

	int id, active_domains, retry_count = 100;



	if (FORCEWAKE_MEDIA & fw_engine) {

		WARN_ON(dev_priv->uncore.fw_mediacount == 0);

		if (--dev_priv->uncore.fw_mediacount == 0)

			dev_priv->uncore.funcs.force_wake_put(dev_priv,

							FORCEWAKE_MEDIA);

	}

	/* Hold uncore.lock across reset to prevent any register access

	 * with forcewake not set correctly. Wait until all pending

	 * timers are run before holding.

	 */

	while (1) {

		active_domains = 0;



	if (FORCEWAKE_BLITTER & fw_engine) {

		WARN_ON(dev_priv->uncore.fw_blittercount == 0);

		if (--dev_priv->uncore.fw_blittercount == 0)

			dev_priv->uncore.funcs.force_wake_put(dev_priv,

							FORCEWAKE_BLITTER);

	}

		for_each_fw_domain(domain, dev_priv, id) {

			if (del_timer_sync(&domain->timer) == 0)

				continue;



			gen6_force_wake_timer((unsigned long)domain);

		}



static void gen6_force_wake_timer(unsigned long arg)

{

	struct drm_i915_private *dev_priv = (void *)arg;

	unsigned long irqflags;

		spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);



	assert_device_not_suspended(dev_priv);

		for_each_fw_domain(domain, dev_priv, id) {

			if (timer_pending(&domain->timer))

				active_domains |= (1 << id);

		}



	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);

	WARN_ON(!dev_priv->uncore.forcewake_count);

		if (active_domains == 0)

			break;



		if (--retry_count == 0) {

			DRM_ERROR("Timed out waiting for forcewake timers to finish\n");

			break;

		}



	if (--dev_priv->uncore.forcewake_count == 0)

		dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);

		spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

		cond_resched();

	}



void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)

{

	struct drm_i915_private *dev_priv = dev->dev_private;

	unsigned long irqflags;

	WARN_ON(active_domains);



	if (del_timer_sync(&dev_priv->uncore.force_wake_timer))

		gen6_force_wake_timer((unsigned long)dev_priv);

	for_each_fw_domain(domain, dev_priv, id)

		if (domain->wake_count)

			fw |= 1 << id;



	/* Hold uncore.lock across reset to prevent any register access

	 * with forcewake not set correctly

	 */

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);

	if (fw)

		dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);



	if (IS_VALLEYVIEW(dev))

		vlv_force_wake_reset(dev_priv);
@@ -454,28 +414,6 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) 
		__gen9_gt_force_wake_mt_reset(dev_priv);



	if (restore) { /* If reset with a user forcewake, try to restore */

		unsigned fw = 0;



		if (IS_VALLEYVIEW(dev)) {

			if (dev_priv->uncore.fw_rendercount)

				fw |= FORCEWAKE_RENDER;



			if (dev_priv->uncore.fw_mediacount)

				fw |= FORCEWAKE_MEDIA;

		} else if (IS_GEN9(dev)) {

			if (dev_priv->uncore.fw_rendercount)

				fw |= FORCEWAKE_RENDER;



			if (dev_priv->uncore.fw_mediacount)

				fw |= FORCEWAKE_MEDIA;



			if (dev_priv->uncore.fw_blittercount)

				fw |= FORCEWAKE_BLITTER;

		} else {

			if (dev_priv->uncore.forcewake_count)

				fw = FORCEWAKE_ALL;

		}



		if (fw)

			dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
@@ -485,6 +423,9 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) 
				GT_FIFO_FREE_ENTRIES_MASK;

	}



	if (!restore)

		assert_force_wake_inactive(dev_priv);



	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

}
@@ -533,53 +474,59 @@ void intel_uncore_sanitize(struct drm_device *dev) 
 * be called at the beginning of the sequence followed by a call to

 * gen6_gt_force_wake_put() at the end of the sequence.

 */

void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)

void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,

			    unsigned fw_domains)

{

	unsigned long irqflags;

	struct intel_uncore_forcewake_domain *domain;

	int id;



	if (!dev_priv->uncore.funcs.force_wake_get)

		return;



	WARN_ON(dev_priv->pm.suspended);



	fw_domains &= dev_priv->uncore.fw_domains;



	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);



	if (IS_GEN9(dev_priv->dev)) {

		gen9_force_wake_get(dev_priv, fw_engine);

	} else if (IS_VALLEYVIEW(dev_priv->dev)) {

		vlv_force_wake_get(dev_priv, fw_engine);

	} else {

		if (dev_priv->uncore.forcewake_count++ == 0)

			dev_priv->uncore.funcs.force_wake_get(dev_priv,

							      FORCEWAKE_ALL);

	for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) {

		if (domain->wake_count++)

			fw_domains &= ~(1 << id);

	}



	if (fw_domains)

		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);



	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

}



/*

 * see gen6_gt_force_wake_get()

 */

void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)

void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,

			    unsigned fw_domains)

{

	unsigned long irqflags;

	struct intel_uncore_forcewake_domain *domain;

	int id;



	if (!dev_priv->uncore.funcs.force_wake_put)

		return;



	fw_domains &= dev_priv->uncore.fw_domains;



	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);



	if (IS_GEN9(dev_priv->dev)) {

		gen9_force_wake_put(dev_priv, fw_engine);

	} else if (IS_VALLEYVIEW(dev_priv->dev)) {

		vlv_force_wake_put(dev_priv, fw_engine);

	} else {

		WARN_ON(!dev_priv->uncore.forcewake_count);

		if (--dev_priv->uncore.forcewake_count == 0) {

			dev_priv->uncore.forcewake_count++;

			mod_timer_pinned(&dev_priv->uncore.force_wake_timer,

					 jiffies + 1);

		}

	for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) {

		if (WARN_ON(domain->wake_count == 0))

			continue;



		if (--domain->wake_count)

			continue;



		domain->wake_count++;

		mod_timer_pinned(&domain->timer, jiffies + 1);

	}



	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
@@ -587,10 +534,14 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) 


void assert_force_wake_inactive(struct drm_i915_private *dev_priv)

{

	struct intel_uncore_forcewake_domain *domain;

	int i;



	if (!dev_priv->uncore.funcs.force_wake_get)

		return;



	WARN_ON(dev_priv->uncore.forcewake_count > 0);

	for_each_fw_domain(domain, dev_priv, i)

		WARN_ON(domain->wake_count);

}



/* We give fast paths for the really cool registers */
@@ -753,19 +704,37 @@ __gen2_read(64) 
	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \

	return val



static inline void __force_wake_get(struct drm_i915_private *dev_priv,

				    unsigned fw_domains)

{

	struct intel_uncore_forcewake_domain *domain;

	int i;



	if (WARN_ON(!fw_domains))

		return;



	/* Ideally GCC would be constant-fold and eliminate this loop */

	for_each_fw_domain_mask(domain, fw_domains, dev_priv, i) {

		if (domain->wake_count) {

			fw_domains &= ~(1 << i);

			continue;

		}



		domain->wake_count++;

		mod_timer_pinned(&domain->timer, jiffies + 1);

	}



	if (fw_domains)

		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);

}



#define __gen6_read(x) \

static u##x \

gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \

	GEN6_READ_HEADER(x); \

	hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \

	if (dev_priv->uncore.forcewake_count == 0 && \

	    NEEDS_FORCE_WAKE((dev_priv), (reg))) { \

		dev_priv->uncore.funcs.force_wake_get(dev_priv, \

						      FORCEWAKE_ALL); \

		dev_priv->uncore.forcewake_count++; \

		mod_timer_pinned(&dev_priv->uncore.force_wake_timer, \

				 jiffies + 1); \

	} \

	if (NEEDS_FORCE_WAKE((dev_priv), (reg))) \

		__force_wake_get(dev_priv, FORCEWAKE_RENDER); \

	val = __raw_i915_read##x(dev_priv, reg); \

	hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \

	GEN6_READ_FOOTER; \
@@ -774,45 +743,27 @@ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 
#define __vlv_read(x) \

static u##x \

vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \

	unsigned fwengine = 0; \

	GEN6_READ_HEADER(x); \

	if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \

		if (dev_priv->uncore.fw_rendercount == 0) \

			fwengine = FORCEWAKE_RENDER; \

	} else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \

		if (dev_priv->uncore.fw_mediacount == 0) \

			fwengine = FORCEWAKE_MEDIA; \

	}  \

	if (fwengine) \

		dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \

	if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) \

		__force_wake_get(dev_priv, FORCEWAKE_RENDER); \

	else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) \

		__force_wake_get(dev_priv, FORCEWAKE_MEDIA); \

	val = __raw_i915_read##x(dev_priv, reg); \

	if (fwengine) \

		dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \

	GEN6_READ_FOOTER; \

}



#define __chv_read(x) \

static u##x \

chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \

	unsigned fwengine = 0; \

	GEN6_READ_HEADER(x); \

	if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \

		if (dev_priv->uncore.fw_rendercount == 0) \

			fwengine = FORCEWAKE_RENDER; \

	} else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \

		if (dev_priv->uncore.fw_mediacount == 0) \

			fwengine = FORCEWAKE_MEDIA; \

	} else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \

		if (dev_priv->uncore.fw_rendercount == 0) \

			fwengine |= FORCEWAKE_RENDER; \

		if (dev_priv->uncore.fw_mediacount == 0) \

			fwengine |= FORCEWAKE_MEDIA; \

	} \

	if (fwengine) \

		dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \

	if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \

		__force_wake_get(dev_priv, FORCEWAKE_RENDER); \

	else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \

		__force_wake_get(dev_priv, FORCEWAKE_MEDIA); \

	else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \

		__force_wake_get(dev_priv, \

				 FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \

	val = __raw_i915_read##x(dev_priv, reg); \

	if (fwengine) \

		dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \

	GEN6_READ_FOOTER; \

}
@@ -822,32 +773,21 @@ chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 
#define __gen9_read(x) \

static u##x \

gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \

	unsigned fw_engine; \

	GEN6_READ_HEADER(x); \

	if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg))) { \

		val = __raw_i915_read##x(dev_priv, reg); \

	} else { \

		unsigned fwengine = 0; \

		if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \

			if (dev_priv->uncore.fw_rendercount == 0) \

				fwengine = FORCEWAKE_RENDER; \

		} else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \

			if (dev_priv->uncore.fw_mediacount == 0) \

				fwengine = FORCEWAKE_MEDIA; \

		} else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \

			if (dev_priv->uncore.fw_rendercount == 0) \

				fwengine |= FORCEWAKE_RENDER; \

			if (dev_priv->uncore.fw_mediacount == 0) \

				fwengine |= FORCEWAKE_MEDIA; \

		} else { \

			if (dev_priv->uncore.fw_blittercount == 0) \

				fwengine = FORCEWAKE_BLITTER; \

		} \

		if (fwengine) \

			dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \

	if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)))	\

		fw_engine = 0; \

	else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg))	\

		fw_engine = FORCEWAKE_RENDER; \

	else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \

		fw_engine = FORCEWAKE_MEDIA; \

	else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \

		fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \

	else \

		fw_engine = FORCEWAKE_BLITTER; \

	if (fw_engine) \

		__force_wake_get(dev_priv, fw_engine); \

	val = __raw_i915_read##x(dev_priv, reg); \

		if (fwengine) \

			dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \

	} \

	GEN6_READ_FOOTER; \

}
@@ -981,17 +921,9 @@ static void \ 
gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \

	GEN6_WRITE_HEADER; \

	hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \

	if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \

		if (dev_priv->uncore.forcewake_count == 0) \

			dev_priv->uncore.funcs.force_wake_get(dev_priv,	\

							      FORCEWAKE_ALL); \

		__raw_i915_write##x(dev_priv, reg, val); \

		if (dev_priv->uncore.forcewake_count == 0) \

			dev_priv->uncore.funcs.force_wake_put(dev_priv, \

							      FORCEWAKE_ALL); \

	} else { \

	if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) \

		__force_wake_get(dev_priv, FORCEWAKE_RENDER); \

	__raw_i915_write##x(dev_priv, reg, val); \

	} \

	hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \

	hsw_unclaimed_reg_detect(dev_priv); \

	GEN6_WRITE_FOOTER; \
@@ -1000,28 +932,17 @@ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace 
#define __chv_write(x) \

static void \

chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \

	unsigned fwengine = 0; \

	bool shadowed = is_gen8_shadowed(dev_priv, reg); \

	GEN6_WRITE_HEADER; \

	if (!shadowed) { \

		if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \

			if (dev_priv->uncore.fw_rendercount == 0) \

				fwengine = FORCEWAKE_RENDER; \

		} else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \

			if (dev_priv->uncore.fw_mediacount == 0) \

				fwengine = FORCEWAKE_MEDIA; \

		} else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \

			if (dev_priv->uncore.fw_rendercount == 0) \

				fwengine |= FORCEWAKE_RENDER; \

			if (dev_priv->uncore.fw_mediacount == 0) \

				fwengine |= FORCEWAKE_MEDIA; \

		if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \

			__force_wake_get(dev_priv, FORCEWAKE_RENDER); \

		else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \

			__force_wake_get(dev_priv, FORCEWAKE_MEDIA); \

		else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \

			__force_wake_get(dev_priv, FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \

	} \

	} \

	if (fwengine) \

		dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \

	__raw_i915_write##x(dev_priv, reg, val); \

	if (fwengine) \

		dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \

	GEN6_WRITE_FOOTER; \

}
@@ -1052,35 +973,22 @@ static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, u32 reg) 
static void \

gen9_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, \

		bool trace) { \

	unsigned fw_engine; \

	GEN6_WRITE_HEADER; \

	if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) ||	\

			is_gen9_shadowed(dev_priv, reg)) { \

		__raw_i915_write##x(dev_priv, reg, val); \

	} else { \

		unsigned fwengine = 0; \

		if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \

			if (dev_priv->uncore.fw_rendercount == 0) \

				fwengine = FORCEWAKE_RENDER; \

		} else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \

			if (dev_priv->uncore.fw_mediacount == 0) \

				fwengine = FORCEWAKE_MEDIA; \

		} else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \

			if (dev_priv->uncore.fw_rendercount == 0) \

				fwengine |= FORCEWAKE_RENDER; \

			if (dev_priv->uncore.fw_mediacount == 0) \

				fwengine |= FORCEWAKE_MEDIA; \

		} else { \

			if (dev_priv->uncore.fw_blittercount == 0) \

				fwengine = FORCEWAKE_BLITTER; \

		} \

		if (fwengine) \

			dev_priv->uncore.funcs.force_wake_get(dev_priv, \

					fwengine); \

	    is_gen9_shadowed(dev_priv, reg)) \

		fw_engine = 0; \

	else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \

		fw_engine = FORCEWAKE_RENDER; \

	else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \

		fw_engine = FORCEWAKE_MEDIA; \

	else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \

		fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \

	else \

		fw_engine = FORCEWAKE_BLITTER; \

	if (fw_engine) \

		__force_wake_get(dev_priv, fw_engine); \

	__raw_i915_write##x(dev_priv, reg, val); \

		if (fwengine) \

			dev_priv->uncore.funcs.force_wake_put(dev_priv, \

					fwengine); \

	} \

	GEN6_WRITE_FOOTER; \

}
@@ -1132,21 +1040,24 @@ do { \ 
void intel_uncore_init(struct drm_device *dev)

{

	struct drm_i915_private *dev_priv = dev->dev_private;



	setup_timer(&dev_priv->uncore.force_wake_timer,

		    gen6_force_wake_timer, (unsigned long)dev_priv);

	struct intel_uncore_forcewake_domain *domain;

	int i;



	__intel_uncore_early_sanitize(dev, false);



	if (IS_GEN9(dev)) {

		dev_priv->uncore.funcs.force_wake_get = __gen9_force_wake_get;

		dev_priv->uncore.funcs.force_wake_put = __gen9_force_wake_put;

		dev_priv->uncore.fw_domains = FORCEWAKE_RENDER |

			FORCEWAKE_BLITTER | FORCEWAKE_MEDIA;

	} else if (IS_VALLEYVIEW(dev)) {

		dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;

		dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;

		dev_priv->uncore.fw_domains = FORCEWAKE_RENDER | FORCEWAKE_MEDIA;

	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {

		dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;

		dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;

		dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;

	} else if (IS_IVYBRIDGE(dev)) {

		u32 ecobus;
@@ -1178,11 +1089,21 @@ void intel_uncore_init(struct drm_device *dev) 
			dev_priv->uncore.funcs.force_wake_put =

				__gen6_gt_force_wake_put;

		}

		dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;

	} else if (IS_GEN6(dev)) {

		dev_priv->uncore.funcs.force_wake_get =

			__gen6_gt_force_wake_get;

		dev_priv->uncore.funcs.force_wake_put =

			__gen6_gt_force_wake_put;

		dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;

	}



	for_each_fw_domain(domain, dev_priv, i) {

		domain->i915 = dev_priv;

		domain->id = i;



		setup_timer(&domain->timer, gen6_force_wake_timer,

			    (unsigned long)domain);

	}



	switch (INTEL_INFO(dev)->gen) {

CRA Git | Maintained and supported by SUSTech CRA and CCSE