Commit 9d5c5a52 authored by Paul Greenwalt's avatar Paul Greenwalt Committed by Jeff Kirsher
Browse files

ice: update malicious driver detection event handling 



Update the PF VFs MDD event message to rate limit once per second and
report the total number Rx|Tx event count. Add support to print pending
MDD events that occur during the rate limit. The use of net_ratelimit did
not allow for per VF Rx|Tx granularity.

Additional PF MDD log messages are guarded by netif_msg_[rx|tx]_err().

Since VF RX MDD events disable the queue, add ethtool private flag
mdd-auto-reset-vf to configure VF reset to re-enable the queue.

Disable anti-spoof detection interrupt to prevent spurious events
during a function reset.

To avoid race condition do not make PF MDD register reads conditional
on global MDD result.

Signed-off-by: default avatarPaul Greenwalt <paul.greenwalt@intel.com>
Signed-off-by: default avatarTony Nguyen <anthony.l.nguyen@intel.com>
Tested-by: default avatarAndrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: default avatarJeff Kirsher <jeffrey.t.kirsher@intel.com>
parent a29a912d

drivers/net/ethernet/intel/ice/ice.h

+4 −0
Original line number Diff line number Diff line
@@ -212,6 +212,7 @@ enum ice_state { 
	__ICE_SERVICE_SCHED,

	__ICE_SERVICE_DIS,

	__ICE_OICR_INTR_DIS,		/* Global OICR interrupt disabled */

	__ICE_MDD_VF_PRINT_PENDING,	/* set when MDD event handle */

	__ICE_STATE_NBITS		/* must be last */

};
@@ -340,6 +341,7 @@ enum ice_pf_flags { 
	ICE_FLAG_FW_LLDP_AGENT,

	ICE_FLAG_ETHTOOL_CTXT,		/* set when ethtool holds RTNL lock */

	ICE_FLAG_LEGACY_RX,

	ICE_FLAG_MDD_AUTO_RESET_VF,

	ICE_PF_FLAGS_NBITS		/* must be last */

};
@@ -363,6 +365,8 @@ struct ice_pf { 
	u16 num_vfs_supported;		/* num VFs supported for this PF */

	u16 num_vf_qps;			/* num queue pairs per VF */

	u16 num_vf_msix;		/* num vectors per VF */

	/* used to ratelimit the MDD event logging */

	unsigned long last_printed_mdd_jiffies;

	DECLARE_BITMAP(state, __ICE_STATE_NBITS);

	DECLARE_BITMAP(flags, ICE_PF_FLAGS_NBITS);

	unsigned long *avail_txqs;	/* bitmap to track PF Tx queue usage */

drivers/net/ethernet/intel/ice/ice_ethtool.c

+1 −0
Original line number Diff line number Diff line
@@ -157,6 +157,7 @@ struct ice_priv_flag { 
static const struct ice_priv_flag ice_gstrings_priv_flags[] = {

	ICE_PRIV_FLAG("link-down-on-close", ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA),

	ICE_PRIV_FLAG("fw-lldp-agent", ICE_FLAG_FW_LLDP_AGENT),

	ICE_PRIV_FLAG("mdd-auto-reset-vf", ICE_FLAG_MDD_AUTO_RESET_VF),

	ICE_PRIV_FLAG("legacy-rx", ICE_FLAG_LEGACY_RX),

};

drivers/net/ethernet/intel/ice/ice_hw_autogen.h

+2 −0
Original line number Diff line number Diff line
@@ -217,6 +217,8 @@ 
#define VPLAN_TX_QBASE_VFNUMQ_M			ICE_M(0xFF, 16)

#define VPLAN_TXQ_MAPENA(_VF)			(0x00073800 + ((_VF) * 4))

#define VPLAN_TXQ_MAPENA_TX_ENA_M		BIT(0)

#define GL_MDCK_TX_TDPU				0x00049348

#define GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M BIT(1)

#define GL_MDET_RX				0x00294C00

#define GL_MDET_RX_QNUM_S			0

#define GL_MDET_RX_QNUM_M			ICE_M(0x7FFF, 0)

drivers/net/ethernet/intel/ice/ice_main.c

+70 −56
Original line number Diff line number Diff line
@@ -1187,20 +1187,28 @@ static void ice_service_timer(struct timer_list *t) 
 * ice_handle_mdd_event - handle malicious driver detect event

 * @pf: pointer to the PF structure

 *

 * Called from service task. OICR interrupt handler indicates MDD event

 * Called from service task. OICR interrupt handler indicates MDD event.

 * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log

 * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events

 * disable the queue, the PF can be configured to reset the VF using ethtool

 * private flag mdd-auto-reset-vf.

 */

static void ice_handle_mdd_event(struct ice_pf *pf)

{

	struct device *dev = ice_pf_to_dev(pf);

	struct ice_hw *hw = &pf->hw;

	bool mdd_detected = false;

	u32 reg;

	int i;



	if (!test_and_clear_bit(__ICE_MDD_EVENT_PENDING, pf->state))

	if (!test_and_clear_bit(__ICE_MDD_EVENT_PENDING, pf->state)) {

		/* Since the VF MDD event logging is rate limited, check if

		 * there are pending MDD events.

		 */

		ice_print_vfs_mdd_events(pf);

		return;

	}



	/* find what triggered the MDD event */

	/* find what triggered an MDD event */

	reg = rd32(hw, GL_MDET_TX_PQM);

	if (reg & GL_MDET_TX_PQM_VALID_M) {

		u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >>
@@ -1216,7 +1224,6 @@ static void ice_handle_mdd_event(struct ice_pf *pf) 
			dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",

				 event, queue, pf_num, vf_num);

		wr32(hw, GL_MDET_TX_PQM, 0xffffffff);

		mdd_detected = true;

	}



	reg = rd32(hw, GL_MDET_TX_TCLAN);
@@ -1234,7 +1241,6 @@ static void ice_handle_mdd_event(struct ice_pf *pf) 
			dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",

				 event, queue, pf_num, vf_num);

		wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff);

		mdd_detected = true;

	}



	reg = rd32(hw, GL_MDET_RX);
@@ -1252,85 +1258,85 @@ static void ice_handle_mdd_event(struct ice_pf *pf) 
			dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n",

				 event, queue, pf_num, vf_num);

		wr32(hw, GL_MDET_RX, 0xffffffff);

		mdd_detected = true;

	}



	if (mdd_detected) {

		bool pf_mdd_detected = false;



	/* check to see if this PF caused an MDD event */

	reg = rd32(hw, PF_MDET_TX_PQM);

	if (reg & PF_MDET_TX_PQM_VALID_M) {

		wr32(hw, PF_MDET_TX_PQM, 0xFFFF);

			dev_info(dev, "TX driver issue detected, PF reset issued\n");

			pf_mdd_detected = true;

		if (netif_msg_tx_err(pf))

			dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n");

	}



	reg = rd32(hw, PF_MDET_TX_TCLAN);

	if (reg & PF_MDET_TX_TCLAN_VALID_M) {

		wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF);

			dev_info(dev, "TX driver issue detected, PF reset issued\n");

			pf_mdd_detected = true;

		if (netif_msg_tx_err(pf))

			dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n");

	}



	reg = rd32(hw, PF_MDET_RX);

	if (reg & PF_MDET_RX_VALID_M) {

		wr32(hw, PF_MDET_RX, 0xFFFF);

			dev_info(dev, "RX driver issue detected, PF reset issued\n");

			pf_mdd_detected = true;

		}

		/* Queue belongs to the PF initiate a reset */

		if (pf_mdd_detected) {

			set_bit(__ICE_NEEDS_RESTART, pf->state);

			ice_service_task_schedule(pf);

		}

		if (netif_msg_rx_err(pf))

			dev_info(dev, "Malicious Driver Detection event RX detected on PF\n");

	}



	/* check to see if one of the VFs caused the MDD */

	/* Check to see if one of the VFs caused an MDD event, and then

	 * increment counters and set print pending

	 */

	ice_for_each_vf(pf, i) {

		struct ice_vf *vf = &pf->vf[i];



		bool vf_mdd_detected = false;



		reg = rd32(hw, VP_MDET_TX_PQM(i));

		if (reg & VP_MDET_TX_PQM_VALID_M) {

			wr32(hw, VP_MDET_TX_PQM(i), 0xFFFF);

			vf_mdd_detected = true;

			dev_info(dev, "TX driver issue detected on VF %d\n",

			vf->mdd_tx_events.count++;

			set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state);

			if (netif_msg_tx_err(pf))

				dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n",

					 i);

		}



		reg = rd32(hw, VP_MDET_TX_TCLAN(i));

		if (reg & VP_MDET_TX_TCLAN_VALID_M) {

			wr32(hw, VP_MDET_TX_TCLAN(i), 0xFFFF);

			vf_mdd_detected = true;

			dev_info(dev, "TX driver issue detected on VF %d\n",

			vf->mdd_tx_events.count++;

			set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state);

			if (netif_msg_tx_err(pf))

				dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n",

					 i);

		}



		reg = rd32(hw, VP_MDET_TX_TDPU(i));

		if (reg & VP_MDET_TX_TDPU_VALID_M) {

			wr32(hw, VP_MDET_TX_TDPU(i), 0xFFFF);

			vf_mdd_detected = true;

			dev_info(dev, "TX driver issue detected on VF %d\n",

			vf->mdd_tx_events.count++;

			set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state);

			if (netif_msg_tx_err(pf))

				dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n",

					 i);

		}



		reg = rd32(hw, VP_MDET_RX(i));

		if (reg & VP_MDET_RX_VALID_M) {

			wr32(hw, VP_MDET_RX(i), 0xFFFF);

			vf_mdd_detected = true;

			dev_info(dev, "RX driver issue detected on VF %d\n",

			vf->mdd_rx_events.count++;

			set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state);

			if (netif_msg_rx_err(pf))

				dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n",

					 i);

		}



		if (vf_mdd_detected) {

			vf->num_mdd_events++;

			if (vf->num_mdd_events &&

			    vf->num_mdd_events <= ICE_MDD_EVENTS_THRESHOLD)

				dev_info(dev, "VF %d has had %llu MDD events since last boot, Admin might need to reload AVF driver with this number of events\n",

					 i, vf->num_mdd_events);

			/* Since the queue is disabled on VF Rx MDD events, the

			 * PF can be configured to reset the VF through ethtool

			 * private flag mdd-auto-reset-vf.

			 */

			if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags))

				ice_reset_vf(&pf->vf[i], false);

		}

	}



	ice_print_vfs_mdd_events(pf);

}



/**
@@ -1995,6 +2001,14 @@ static void ice_ena_misc_vector(struct ice_pf *pf) 
	struct ice_hw *hw = &pf->hw;

	u32 val;



	/* Disable anti-spoof detection interrupt to prevent spurious event

	 * interrupts during a function reset. Anti-spoof functionally is

	 * still supported.

	 */

	val = rd32(hw, GL_MDCK_TX_TDPU);

	val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M;

	wr32(hw, GL_MDCK_TX_TDPU, val);



	/* clear things first */

	wr32(hw, PFINT_OICR_ENA, 0);	/* disable all */

	rd32(hw, PFINT_OICR);		/* read to clear */

drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c

+55 −1
Original line number Diff line number Diff line
@@ -171,6 +171,11 @@ static void ice_free_vf_res(struct ice_vf *vf) 
	}



	last_vector_idx = vf->first_vector_idx + pf->num_vf_msix - 1;



	/* clear VF MDD event information */

	memset(&vf->mdd_tx_events, 0, sizeof(vf->mdd_tx_events));

	memset(&vf->mdd_rx_events, 0, sizeof(vf->mdd_rx_events));



	/* Disable interrupts so that VF starts in a known state */

	for (i = vf->first_vector_idx; i <= last_vector_idx; i++) {

		wr32(&pf->hw, GLINT_DYN_CTL(i), GLINT_DYN_CTL_CLEARPBA_M);
@@ -1175,7 +1180,7 @@ static bool ice_is_vf_disabled(struct ice_vf *vf) 
 *

 * Returns true if the VF is reset, false otherwise.

 */

static bool ice_reset_vf(struct ice_vf *vf, bool is_vflr)

bool ice_reset_vf(struct ice_vf *vf, bool is_vflr)

{

	struct ice_pf *pf = vf->pf;

	struct ice_vsi *vsi;
@@ -3529,3 +3534,52 @@ int ice_get_vf_stats(struct net_device *netdev, int vf_id, 


	return 0;

}



/**

 * ice_print_vfs_mdd_event - print VFs malicious driver detect event

 * @pf: pointer to the PF structure

 *

 * Called from ice_handle_mdd_event to rate limit and print VFs MDD events.

 */

void ice_print_vfs_mdd_events(struct ice_pf *pf)

{

	struct device *dev = ice_pf_to_dev(pf);

	struct ice_hw *hw = &pf->hw;

	int i;



	/* check that there are pending MDD events to print */

	if (!test_and_clear_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state))

		return;



	/* VF MDD event logs are rate limited to one second intervals */

	if (time_is_after_jiffies(pf->last_printed_mdd_jiffies + HZ * 1))

		return;



	pf->last_printed_mdd_jiffies = jiffies;



	ice_for_each_vf(pf, i) {

		struct ice_vf *vf = &pf->vf[i];



		/* only print Rx MDD event message if there are new events */

		if (vf->mdd_rx_events.count != vf->mdd_rx_events.last_printed) {

			vf->mdd_rx_events.last_printed =

							vf->mdd_rx_events.count;



			dev_info(dev, "%d Rx Malicious Driver Detection events detected on PF %d VF %d MAC %pM. mdd-auto-reset-vfs=%s\n",

				 vf->mdd_rx_events.count, hw->pf_id, i,

				 vf->dflt_lan_addr.addr,

				 test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)

					  ? "on" : "off");

		}



		/* only print Tx MDD event message if there are new events */

		if (vf->mdd_tx_events.count != vf->mdd_tx_events.last_printed) {

			vf->mdd_tx_events.last_printed =

							vf->mdd_tx_events.count;



			dev_info(dev, "%d Tx Malicious Driver Detection events detected on PF %d VF %d MAC %pM.\n",

				 vf->mdd_tx_events.count, hw->pf_id, i,

				 vf->dflt_lan_addr.addr);

		}

	}

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE