Commit a25717d2 authored by Jakub Kicinski's avatar Jakub Kicinski Committed by Daniel Borkmann
Browse files

xdp: support simultaneous driver and hw XDP attachment 



Split the query of HW-attached program from the software one.
Introduce new .ndo_bpf command to query HW-attached program.
This will allow drivers to install different programs in HW
and SW at the same time.  Netlink can now also carry multiple
programs on dump (in which case mode will be set to
XDP_ATTACHED_MULTI and user has to check per-attachment point
attributes, IFLA_XDP_PROG_ID will not be present).  We reuse
IFLA_XDP_PROG_ID skb space for second mode, so rtnl_xdp_size()
doesn't need to be updated.

Note that the installation side is still not there, since all
drivers currently reject installing more than one program at
the time.

Signed-off-by: default avatarJakub Kicinski <jakub.kicinski@netronome.com>
Reviewed-by: default avatarQuentin Monnet <quentin.monnet@netronome.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent 05296620

drivers/net/ethernet/netronome/nfp/nfp_net_common.c

+6 −0
Original line number Diff line number Diff line
@@ -3453,6 +3453,12 @@ static int nfp_net_xdp(struct net_device *netdev, struct netdev_bpf *xdp) 
	case XDP_SETUP_PROG_HW:

		return nfp_net_xdp_setup(nn, xdp);

	case XDP_QUERY_PROG:

		if (nn->dp.bpf_offload_xdp)

			return 0;

		return xdp_attachment_query(&nn->xdp, xdp);

	case XDP_QUERY_PROG_HW:

		if (!nn->dp.bpf_offload_xdp)

			return 0;

		return xdp_attachment_query(&nn->xdp, xdp);

	default:

		return nfp_app_bpf(nn->app, nn, xdp);

drivers/net/netdevsim/bpf.c

+6 −0
Original line number Diff line number Diff line
@@ -561,6 +561,12 @@ int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf) 
		nsim_bpf_destroy_prog(bpf->offload.prog);

		return 0;

	case XDP_QUERY_PROG:

		if (ns->xdp_prog_mode != XDP_ATTACHED_DRV)

			return 0;

		return xdp_attachment_query(&ns->xdp, bpf);

	case XDP_QUERY_PROG_HW:

		if (ns->xdp_prog_mode != XDP_ATTACHED_HW)

			return 0;

		return xdp_attachment_query(&ns->xdp, bpf);

	case XDP_SETUP_PROG:

		err = nsim_setup_prog_checks(ns, bpf);

include/linux/netdevice.h

+4 −3
Original line number Diff line number Diff line
@@ -820,6 +820,7 @@ enum bpf_netdev_command { 
	XDP_SETUP_PROG,

	XDP_SETUP_PROG_HW,

	XDP_QUERY_PROG,

	XDP_QUERY_PROG_HW,

	/* BPF program for offload callbacks, invoked at program load time. */

	BPF_OFFLOAD_VERIFIER_PREP,

	BPF_OFFLOAD_TRANSLATE,
@@ -843,7 +844,7 @@ struct netdev_bpf { 
			struct bpf_prog *prog;

			struct netlink_ext_ack *extack;

		};

		/* XDP_QUERY_PROG */

		/* XDP_QUERY_PROG, XDP_QUERY_PROG_HW */

		struct {

			u32 prog_id;

			/* flags with which program was installed */
@@ -3533,8 +3534,8 @@ struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, 
typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);

int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,

		      int fd, u32 flags);

void __dev_xdp_query(struct net_device *dev, bpf_op_t xdp_op,

		     struct netdev_bpf *xdp);

u32 __dev_xdp_query(struct net_device *dev, bpf_op_t xdp_op,

		    enum bpf_netdev_command cmd);



int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);

int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);

include/uapi/linux/if_link.h

+1 −0
Original line number Diff line number Diff line
@@ -920,6 +920,7 @@ enum { 
	XDP_ATTACHED_DRV,

	XDP_ATTACHED_SKB,

	XDP_ATTACHED_HW,

	XDP_ATTACHED_MULTI,

};



enum {

net/core/dev.c

+27 −18
Original line number Diff line number Diff line
@@ -7582,21 +7582,19 @@ int dev_change_proto_down(struct net_device *dev, bool proto_down) 
}

EXPORT_SYMBOL(dev_change_proto_down);



void __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op,

		     struct netdev_bpf *xdp)

u32 __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op,

		    enum bpf_netdev_command cmd)

{

	memset(xdp, 0, sizeof(*xdp));

	xdp->command = XDP_QUERY_PROG;

	struct netdev_bpf xdp;



	/* Query must always succeed. */

	WARN_ON(bpf_op(dev, xdp) < 0);

}

	if (!bpf_op)

		return 0;



static bool __dev_xdp_attached(struct net_device *dev, bpf_op_t bpf_op)

{

	struct netdev_bpf xdp;

	memset(&xdp, 0, sizeof(xdp));

	xdp.command = cmd;



	__dev_xdp_query(dev, bpf_op, &xdp);

	/* Query must always succeed. */

	WARN_ON(bpf_op(dev, &xdp) < 0 && cmd == XDP_QUERY_PROG);



	return xdp.prog_id;

}
@@ -7632,12 +7630,19 @@ static void dev_xdp_uninstall(struct net_device *dev) 
	if (!ndo_bpf)

		return;



	__dev_xdp_query(dev, ndo_bpf, &xdp);

	if (!xdp.prog_id)

		return;

	memset(&xdp, 0, sizeof(xdp));

	xdp.command = XDP_QUERY_PROG;

	WARN_ON(ndo_bpf(dev, &xdp));

	if (xdp.prog_id)

		WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,

					NULL));



	/* Program removal should always succeed */

	WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags, NULL));

	/* Remove HW offload */

	memset(&xdp, 0, sizeof(xdp));

	xdp.command = XDP_QUERY_PROG_HW;

	if (!ndo_bpf(dev, &xdp) && xdp.prog_id)

		WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,

					NULL));

}



/**
@@ -7653,12 +7658,15 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 
		      int fd, u32 flags)

{

	const struct net_device_ops *ops = dev->netdev_ops;

	enum bpf_netdev_command query;

	struct bpf_prog *prog = NULL;

	bpf_op_t bpf_op, bpf_chk;

	int err;



	ASSERT_RTNL();



	query = flags & XDP_FLAGS_HW_MODE ? XDP_QUERY_PROG_HW : XDP_QUERY_PROG;



	bpf_op = bpf_chk = ops->ndo_bpf;

	if (!bpf_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE)))

		return -EOPNOTSUPP;
@@ -7668,10 +7676,11 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 
		bpf_chk = generic_xdp_install;



	if (fd >= 0) {

		if (bpf_chk && __dev_xdp_attached(dev, bpf_chk))

		if (__dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG) ||

		    __dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG_HW))

			return -EEXIST;

		if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&

		    __dev_xdp_attached(dev, bpf_op))

		    __dev_xdp_query(dev, bpf_op, query))

			return -EBUSY;



		prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,

CRA Git | Maintained and supported by SUSTech CRA and CCSE