Merge branch 'Devlink-health-reporting-and-recovery-system'

The health mechanism is targeted for Real Time Alerting, in order to know when

something bad had happened to a PCI device

- Provide alert debug information

- Self healing

- If problem needs vendor support, provide a way to gather all needed debugging

  information.

The main idea is to unify and centralize driver health reports in the

generic devlink instance and allow the user to set different

attributes of the health reporting and recovery procedures.

The devlink health reporter:

Device driver creates a "health reporter" per each error/health type.

Error/Health type can be a known/generic (eg pci error, fw error, rx/tx error)

or unknown (driver specific).

For each registered health reporter a driver can issue error/health reports

asynchronously. All health reports handling is done by devlink.

Device driver can provide specific callbacks for each "health reporter", e.g.

 - Recovery procedures

 - Diagnostics and object dump procedures

 - OOB initial parameters

Different parts of the driver can register different types of health reporters

with different handlers.

Once an error is reported, devlink health will do the following actions:

  * A log is being send to the kernel trace events buffer

  * Health status and statistics are being updated for the reporter instance

  * Object dump is being taken and saved at the reporter instance (as long as

    there is no other dump which is already stored)

  * Auto recovery attempt is being done. Depends on:

    - Auto-recovery configuration

    - Grace period vs. time passed since last recover

The user interface:

User can access/change each reporter's parameters and driver specific callbacks

via devlink, e.g per error type (per health reporter)

 - Configure reporter's generic parameters (like: disable/enable auto recovery)

 - Invoke recovery procedure

 - Run diagnostics

 - Object dump

The devlink health interface (via netlink):

DEVLINK_CMD_HEALTH_REPORTER_GET

  Retrieves status and configuration info per DEV and reporter.

DEVLINK_CMD_HEALTH_REPORTER_SET

  Allows reporter-related configuration setting.

DEVLINK_CMD_HEALTH_REPORTER_RECOVER

  Triggers a reporter's recovery procedure.

DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE

  Retrieves diagnostics data from a reporter on a device.

DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET

  Retrieves the last stored dump. Devlink health

  saves a single dump. If an dump is not already stored by the devlink

  for this reporter, devlink generates a new dump.

  dump output is defined by the reporter.

DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR

  Clears the last saved dump file for the specified reporter.

                                               netlink

                                      +--------------------------+

                                      |                          |

                                      |            +             |

                                      |            |             |

                                      +--------------------------+

                                                   |request for ops

                                                   |(diagnose,

 mlx5_core                             devlink     |recover,

                                                   |dump)

+--------+                            +--------------------------+

|        |                            |    reporter|             |

|        |                            |  +---------v----------+  |

|        |   ops execution            |  |                    |  |

|     <----------------------------------+                    |  |

|        |                            |  |                    |  |

|        |                            |  + ^------------------+  |

|        |                            |    | request for ops     |

|        |                            |    | (recover, dump)     |

|        |                            |    |                     |

|        |                            |  +-+------------------+  |

|        |     health report          |  | health handler     |  |

|        +------------------------------->                    |  |

|        |                            |  +--------------------+  |

|        |     health reporter create |                          |

|        +---------------------------->                          |

+--------+                            +--------------------------+

#

mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \

		en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \

		en_selftest.o en/port.o en/monitor_stats.o

		en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o

#

# Netdev extra

	struct mlx5e_channel      *channel;

	int                        txq_ix;

	u32                        rate_limit;

	struct mlx5e_txqsq_recover {

	struct work_struct         recover_work;

		u64                        last_recover;

	} recover;

} ____cacheline_aligned_in_smp;

struct mlx5e_dma_info {

	u8	hfunc;

};

struct mlx5e_modify_sq_param {

	int curr_state;

	int next_state;

	int rl_update;

	int rl_index;

};

struct mlx5e_priv {

	/* priv data path fields - start */

	struct mlx5e_txqsq *txq2sq[MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC];

#ifdef CONFIG_MLX5_EN_TLS

	struct mlx5e_tls          *tls;

#endif

	struct devlink_health_reporter *tx_reporter;

};

struct mlx5e_profile {

void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,

			       struct mlx5e_params *params);

int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,

		    struct mlx5e_modify_sq_param *p);

void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq);

void mlx5e_tx_disable_queue(struct netdev_queue *txq);

static inline bool mlx5e_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev)

{

	return (MLX5_CAP_ETH(mdev, tunnel_stateless_gre) &&

/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */

/* Copyright (c) 2018 Mellanox Technologies. */

#ifndef __MLX5E_EN_REPORTER_H

#define __MLX5E_EN_REPORTER_H

#include <linux/mlx5/driver.h>

#include "en.h"

int mlx5e_tx_reporter_create(struct mlx5e_priv *priv);

void mlx5e_tx_reporter_destroy(struct mlx5e_priv *priv);

void mlx5e_tx_reporter_err_cqe(struct mlx5e_txqsq *sq);

void mlx5e_tx_reporter_timeout(struct mlx5e_txqsq *sq);

#endif

/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */

/* Copyright (c) 2018 Mellanox Technologies. */

#include <net/devlink.h>

#include "reporter.h"

#include "lib/eq.h"

#define MLX5E_TX_REPORTER_PER_SQ_MAX_LEN 256

struct mlx5e_tx_err_ctx {

	int (*recover)(struct mlx5e_txqsq *sq);

	struct mlx5e_txqsq *sq;

};

static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq)

{

	unsigned long exp_time = jiffies + msecs_to_jiffies(2000);

	while (time_before(jiffies, exp_time)) {

		if (sq->cc == sq->pc)

			return 0;

		msleep(20);

	}

	netdev_err(sq->channel->netdev,

		   "Wait for SQ 0x%x flush timeout (sq cc = 0x%x, sq pc = 0x%x)\n",

		   sq->sqn, sq->cc, sq->pc);

	return -ETIMEDOUT;

}

static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq)

{

	WARN_ONCE(sq->cc != sq->pc,

		  "SQ 0x%x: cc (0x%x) != pc (0x%x)\n",

		  sq->sqn, sq->cc, sq->pc);

	sq->cc = 0;

	sq->dma_fifo_cc = 0;

	sq->pc = 0;

}

static int mlx5e_sq_to_ready(struct mlx5e_txqsq *sq, int curr_state)

{

	struct mlx5_core_dev *mdev = sq->channel->mdev;

	struct net_device *dev = sq->channel->netdev;

	struct mlx5e_modify_sq_param msp = {0};

	int err;

	msp.curr_state = curr_state;

	msp.next_state = MLX5_SQC_STATE_RST;

	err = mlx5e_modify_sq(mdev, sq->sqn, &msp);

	if (err) {

		netdev_err(dev, "Failed to move sq 0x%x to reset\n", sq->sqn);

		return err;

	}

	memset(&msp, 0, sizeof(msp));

	msp.curr_state = MLX5_SQC_STATE_RST;

	msp.next_state = MLX5_SQC_STATE_RDY;

	err = mlx5e_modify_sq(mdev, sq->sqn, &msp);

	if (err) {

		netdev_err(dev, "Failed to move sq 0x%x to ready\n", sq->sqn);

		return err;

	}

	return 0;

}

static int mlx5e_tx_reporter_err_cqe_recover(struct mlx5e_txqsq *sq)

{

	struct mlx5_core_dev *mdev = sq->channel->mdev;

	struct net_device *dev = sq->channel->netdev;

	u8 state;

	int err;

	if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state))

		return 0;

	err = mlx5_core_query_sq_state(mdev, sq->sqn, &state);

	if (err) {

		netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n",

			   sq->sqn, err);

		return err;

	}

	if (state != MLX5_RQC_STATE_ERR) {

		netdev_err(dev, "SQ 0x%x not in ERROR state\n", sq->sqn);

		return -EINVAL;

	}

	mlx5e_tx_disable_queue(sq->txq);

	err = mlx5e_wait_for_sq_flush(sq);

	if (err)

		return err;

	/* At this point, no new packets will arrive from the stack as TXQ is

	 * marked with QUEUE_STATE_DRV_XOFF. In addition, NAPI cleared all

	 * pending WQEs.  SQ can safely reset the SQ.

	 */

	err = mlx5e_sq_to_ready(sq, state);

	if (err)

		return err;

	mlx5e_reset_txqsq_cc_pc(sq);

	sq->stats->recover++;

	mlx5e_activate_txqsq(sq);

	return 0;

}

void mlx5e_tx_reporter_err_cqe(struct mlx5e_txqsq *sq)

{

	char err_str[MLX5E_TX_REPORTER_PER_SQ_MAX_LEN];

	struct mlx5e_tx_err_ctx err_ctx = {0};

	err_ctx.sq       = sq;

	err_ctx.recover  = mlx5e_tx_reporter_err_cqe_recover;

	sprintf(err_str, "ERR CQE on SQ: 0x%x", sq->sqn);

	devlink_health_report(sq->channel->priv->tx_reporter, err_str,

			      &err_ctx);

}

static int mlx5e_tx_reporter_timeout_recover(struct mlx5e_txqsq *sq)

{

	struct mlx5_eq_comp *eq = sq->cq.mcq.eq;

	u32 eqe_count;

	netdev_err(sq->channel->netdev, "EQ 0x%x: Cons = 0x%x, irqn = 0x%x\n",

		   eq->core.eqn, eq->core.cons_index, eq->core.irqn);

	eqe_count = mlx5_eq_poll_irq_disabled(eq);

	if (!eqe_count) {

		clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);

		return 1;

	}

	netdev_err(sq->channel->netdev, "Recover %d eqes on EQ 0x%x\n",

		   eqe_count, eq->core.eqn);

	sq->channel->stats->eq_rearm++;

	return 0;

}

void mlx5e_tx_reporter_timeout(struct mlx5e_txqsq *sq)

{

	struct mlx5e_tx_err_ctx err_ctx;

	char err_str[MLX5E_TX_REPORTER_PER_SQ_MAX_LEN];

	err_ctx.sq       = sq;

	err_ctx.recover  = mlx5e_tx_reporter_timeout_recover;

	sprintf(err_str,

		"TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u\n",

		sq->channel->ix, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc,

		jiffies_to_usecs(jiffies - sq->txq->trans_start));

	devlink_health_report(sq->channel->priv->tx_reporter, err_str,

			      &err_ctx);

}

/* state lock cannot be grabbed within this function.

 * It can cause a dead lock or a read-after-free.

 */

int mlx5e_tx_reporter_recover_from_ctx(struct mlx5e_tx_err_ctx *err_ctx)

{

	return err_ctx->recover(err_ctx->sq);

}

static int mlx5e_tx_reporter_recover_all(struct mlx5e_priv *priv)

{

	int err;

	mutex_lock(&priv->state_lock);

	mlx5e_close_locked(priv->netdev);

	err = mlx5e_open_locked(priv->netdev);

	mutex_unlock(&priv->state_lock);

	return err;

}

static int mlx5e_tx_reporter_recover(struct devlink_health_reporter *reporter,

				     void *context)

{

	struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter);

	struct mlx5e_tx_err_ctx *err_ctx = context;

	return err_ctx ? mlx5e_tx_reporter_recover_from_ctx(err_ctx) :

			 mlx5e_tx_reporter_recover_all(priv);

}

static int

mlx5e_tx_reporter_build_diagnose_output(struct devlink_health_buffer *buffer,

					u32 sqn, u8 state, u8 stopped)

{

	int err, i;

	int nest = 0;

	char name[20];

	err = devlink_health_buffer_nest_start(buffer,

					       DEVLINK_ATTR_HEALTH_BUFFER_OBJECT);

	if (err)

		goto buffer_error;

	nest++;

	err = devlink_health_buffer_nest_start(buffer,

					       DEVLINK_ATTR_HEALTH_BUFFER_OBJECT_PAIR);

	if (err)

		goto buffer_error;

	nest++;

	sprintf(name, "SQ 0x%x", sqn);

	err = devlink_health_buffer_put_object_name(buffer, name);

	if (err)

		goto buffer_error;

	err = devlink_health_buffer_nest_start(buffer,

					       DEVLINK_ATTR_HEALTH_BUFFER_OBJECT_VALUE);

	if (err)

		goto buffer_error;

	nest++;

	err = devlink_health_buffer_nest_start(buffer,

					       DEVLINK_ATTR_HEALTH_BUFFER_OBJECT);

	if (err)

		goto buffer_error;

	nest++;

	err = devlink_health_buffer_nest_start(buffer,

					       DEVLINK_ATTR_HEALTH_BUFFER_OBJECT_PAIR);

	if (err)

		goto buffer_error;

	nest++;

	err = devlink_health_buffer_put_object_name(buffer, "HW state");

	if (err)

		goto buffer_error;

	err = devlink_health_buffer_nest_start(buffer,

					       DEVLINK_ATTR_HEALTH_BUFFER_OBJECT_VALUE);

	if (err)

		goto buffer_error;

	nest++;

	err = devlink_health_buffer_put_value_u8(buffer, state);

	if (err)

		goto buffer_error;

	devlink_health_buffer_nest_end(buffer); /* DEVLINK_ATTR_HEALTH_BUFFER_OBJECT_VALUE */

	nest--;

	devlink_health_buffer_nest_end(buffer); /* DEVLINK_ATTR_HEALTH_BUFFER_OBJECT_PAIR */

	nest--;

	err = devlink_health_buffer_nest_start(buffer,

					       DEVLINK_ATTR_HEALTH_BUFFER_OBJECT_PAIR);

	if (err)

		goto buffer_error;

	nest++;

	err = devlink_health_buffer_put_object_name(buffer, "stopped");

	if (err)

		goto buffer_error;

	err = devlink_health_buffer_nest_start(buffer,

					       DEVLINK_ATTR_HEALTH_BUFFER_OBJECT_VALUE);

	if (err)

		goto buffer_error;

	nest++;

	err = devlink_health_buffer_put_value_u8(buffer, stopped);

	if (err)

		goto buffer_error;

	for (i = 0; i < nest; i++)

		devlink_health_buffer_nest_end(buffer);

	return 0;

buffer_error:

	for (i = 0; i < nest; i++)

		devlink_health_buffer_nest_cancel(buffer);

	return err;

}

static int mlx5e_tx_reporter_diagnose(struct devlink_health_reporter *reporter,

				      struct devlink_health_buffer **buffers_array,

				      unsigned int buffer_size,

				      unsigned int num_buffers)

{

	struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter);

	unsigned int buff = 0;

	int i = 0, err = 0;

	if (buffer_size < MLX5E_TX_REPORTER_PER_SQ_MAX_LEN)

		return -ENOMEM;

	mutex_lock(&priv->state_lock);

	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {

		mutex_unlock(&priv->state_lock);

		return 0;

	}

	while (i < priv->channels.num * priv->channels.params.num_tc) {

		struct mlx5e_txqsq *sq = priv->txq2sq[i];

		u8 state;

		err = mlx5_core_query_sq_state(priv->mdev, sq->sqn, &state);

		if (err)

			break;

		err = mlx5e_tx_reporter_build_diagnose_output(buffers_array[buff],

							      sq->sqn, state,

							      netif_xmit_stopped(sq->txq));

		if (err) {

			if (++buff == num_buffers)

				break;

		} else {

			i++;

		}

	}

	mutex_unlock(&priv->state_lock);

	return err;

}

static const struct devlink_health_reporter_ops mlx5_tx_reporter_ops = {

		.name = "TX",

		.recover = mlx5e_tx_reporter_recover,

		.diagnose_size = MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC *

				 MLX5E_TX_REPORTER_PER_SQ_MAX_LEN,

		.diagnose = mlx5e_tx_reporter_diagnose,

		.dump_size = 0,

		.dump = NULL,

};

#define MLX5_REPORTER_TX_GRACEFUL_PERIOD 500

int mlx5e_tx_reporter_create(struct mlx5e_priv *priv)

{

	struct mlx5_core_dev *mdev = priv->mdev;

	struct devlink *devlink = priv_to_devlink(mdev);

	priv->tx_reporter =

		devlink_health_reporter_create(devlink, &mlx5_tx_reporter_ops,

					       MLX5_REPORTER_TX_GRACEFUL_PERIOD,

					       true, priv);

	return PTR_ERR_OR_ZERO(priv->tx_reporter);

}

void mlx5e_tx_reporter_destroy(struct mlx5e_priv *priv)

{

	devlink_health_reporter_destroy(priv->tx_reporter);

}