Commit da927466 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'libbpf-field-existence' 



Andrii Nakryiko says:

====================
This patch set generalizes libbpf's CO-RE relocation support. In addition to
existing field's byte offset relocation, libbpf now supports field existence
relocations, which are emitted by Clang when using
__builtin_preserve_field_info(<field>, BPF_FIELD_EXISTS). A convenience
bpf_core_field_exists() macro is added to bpf_core_read.h BPF-side header,
along the bpf_field_info_kind enum containing currently supported types of
field information libbpf supports. This list will grow as libbpf gains support
for other relo kinds.

This patch set upgrades the format of .BTF.ext's relocation record to match
latest Clang's format (12 -> 16 bytes). This is not a breaking change, as the
previous format hasn't been released yet as part of official Clang version
release.

v1->v2:
- unify bpf_field_info_kind enum and naming changes (Alexei);
- added bpf_core_field_exists() to bpf_core_read.h.
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 14f2cf60 c7566a69

tools/lib/bpf/bpf_core_read.h

+23 −1
Original line number Diff line number Diff line
@@ -2,6 +2,28 @@ 
#ifndef __BPF_CORE_READ_H__

#define __BPF_CORE_READ_H__



/*

 * enum bpf_field_info_kind is passed as a second argument into

 * __builtin_preserve_field_info() built-in to get a specific aspect of

 * a field, captured as a first argument. __builtin_preserve_field_info(field,

 * info_kind) returns __u32 integer and produces BTF field relocation, which

 * is understood and processed by libbpf during BPF object loading. See

 * selftests/bpf for examples.

 */

enum bpf_field_info_kind {

	BPF_FIELD_BYTE_OFFSET = 0,	/* field byte offset */

	BPF_FIELD_EXISTS = 2,		/* field existence in target kernel */

};



/*

 * Convenience macro to check that field actually exists in target kernel's.

 * Returns:

 *    1, if matching field is present in target kernel;

 *    0, if no matching field found.

 */

#define bpf_core_field_exists(field)					    \

	__builtin_preserve_field_info(field, BPF_FIELD_EXISTS)



/*

 * bpf_core_read() abstracts away bpf_probe_read() call and captures offset

 * relocation for source address using __builtin_preserve_access_index()
@@ -12,7 +34,7 @@ 
 * a relocation, which records BTF type ID describing root struct/union and an

 * accessor string which describes exact embedded field that was used to take

 * an address. See detailed description of this relocation format and

 * semantics in comments to struct bpf_offset_reloc in libbpf_internal.h.

 * semantics in comments to struct bpf_field_reloc in libbpf_internal.h.

 *

 * This relocation allows libbpf to adjust BPF instruction to use correct

 * actual field offset, based on target kernel BTF type that matches original

tools/lib/bpf/btf.c

+8 −8
Original line number Diff line number Diff line
@@ -888,14 +888,14 @@ static int btf_ext_setup_line_info(struct btf_ext *btf_ext) 
	return btf_ext_setup_info(btf_ext, &param);

}



static int btf_ext_setup_offset_reloc(struct btf_ext *btf_ext)

static int btf_ext_setup_field_reloc(struct btf_ext *btf_ext)

{

	struct btf_ext_sec_setup_param param = {

		.off = btf_ext->hdr->offset_reloc_off,

		.len = btf_ext->hdr->offset_reloc_len,

		.min_rec_size = sizeof(struct bpf_offset_reloc),

		.ext_info = &btf_ext->offset_reloc_info,

		.desc = "offset_reloc",

		.off = btf_ext->hdr->field_reloc_off,

		.len = btf_ext->hdr->field_reloc_len,

		.min_rec_size = sizeof(struct bpf_field_reloc),

		.ext_info = &btf_ext->field_reloc_info,

		.desc = "field_reloc",

	};



	return btf_ext_setup_info(btf_ext, &param);
@@ -975,9 +975,9 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size) 
		goto done;



	if (btf_ext->hdr->hdr_len <

	    offsetofend(struct btf_ext_header, offset_reloc_len))

	    offsetofend(struct btf_ext_header, field_reloc_len))

		goto done;

	err = btf_ext_setup_offset_reloc(btf_ext);

	err = btf_ext_setup_field_reloc(btf_ext);

	if (err)

		goto done;

tools/lib/bpf/btf.h

+2 −2
Original line number Diff line number Diff line
@@ -60,8 +60,8 @@ struct btf_ext_header { 
	__u32	line_info_len;



	/* optional part of .BTF.ext header */

	__u32	offset_reloc_off;

	__u32	offset_reloc_len;

	__u32	field_reloc_off;

	__u32	field_reloc_len;

};



LIBBPF_API void btf__free(struct btf *btf);

tools/lib/bpf/libbpf.c

+105 −64
Original line number Diff line number Diff line
@@ -249,6 +249,7 @@ struct bpf_object { 


	bool loaded;

	bool has_pseudo_calls;

	bool relaxed_core_relocs;



	/*

	 * Information when doing elf related work. Only valid if fd
@@ -1322,9 +1323,9 @@ static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) 
	return 0;

}



static int bpf_object__init_maps(struct bpf_object *obj, int flags)

static int bpf_object__init_maps(struct bpf_object *obj, bool relaxed_maps)

{

	bool strict = !(flags & MAPS_RELAX_COMPAT);

	bool strict = !relaxed_maps;

	int err;



	err = bpf_object__init_user_maps(obj, strict);
@@ -1521,7 +1522,7 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 
	return 0;

}



static int bpf_object__elf_collect(struct bpf_object *obj, int flags)

static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps)

{

	Elf *elf = obj->efile.elf;

	GElf_Ehdr *ep = &obj->efile.ehdr;
@@ -1652,7 +1653,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 
	}

	err = bpf_object__init_btf(obj, btf_data, btf_ext_data);

	if (!err)

		err = bpf_object__init_maps(obj, flags);

		err = bpf_object__init_maps(obj, relaxed_maps);

	if (!err)

		err = bpf_object__sanitize_and_load_btf(obj);

	if (!err)
@@ -2326,7 +2327,7 @@ static bool str_is_empty(const char *s) 
}



/*

 * Turn bpf_offset_reloc into a low- and high-level spec representation,

 * Turn bpf_field_reloc into a low- and high-level spec representation,

 * validating correctness along the way, as well as calculating resulting

 * field offset (in bytes), specified by accessor string. Low-level spec

 * captures every single level of nestedness, including traversing anonymous
@@ -2771,26 +2772,54 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, 


/*

 * Patch relocatable BPF instruction.

 * Expected insn->imm value is provided for validation, as well as the new

 * relocated value.

 *

 * Patched value is determined by relocation kind and target specification.

 * For field existence relocation target spec will be NULL if field is not

 * found.

 * Expected insn->imm value is determined using relocation kind and local

 * spec, and is checked before patching instruction. If actual insn->imm value

 * is wrong, bail out with error.

 *

 * Currently three kinds of BPF instructions are supported:

 * 1. rX = <imm> (assignment with immediate operand);

 * 2. rX += <imm> (arithmetic operations with immediate operand);

 * 3. *(rX) = <imm> (indirect memory assignment with immediate operand).

 *

 * If actual insn->imm value is wrong, bail out.

 */

static int bpf_core_reloc_insn(struct bpf_program *prog, int insn_off,

			       __u32 orig_off, __u32 new_off)

static int bpf_core_reloc_insn(struct bpf_program *prog,

			       const struct bpf_field_reloc *relo,

			       const struct bpf_core_spec *local_spec,

			       const struct bpf_core_spec *targ_spec)

{

	__u32 orig_val, new_val;

	struct bpf_insn *insn;

	int insn_idx;

	__u8 class;



	if (insn_off % sizeof(struct bpf_insn))

	if (relo->insn_off % sizeof(struct bpf_insn))

		return -EINVAL;

	insn_idx = insn_off / sizeof(struct bpf_insn);

	insn_idx = relo->insn_off / sizeof(struct bpf_insn);



	switch (relo->kind) {

	case BPF_FIELD_BYTE_OFFSET:

		orig_val = local_spec->offset;

		if (targ_spec) {

			new_val = targ_spec->offset;

		} else {

			pr_warning("prog '%s': patching insn #%d w/ failed reloc, imm %d -> %d\n",

				   bpf_program__title(prog, false), insn_idx,

				   orig_val, -1);

			new_val = (__u32)-1;

		}

		break;

	case BPF_FIELD_EXISTS:

		orig_val = 1; /* can't generate EXISTS relo w/o local field */

		new_val = targ_spec ? 1 : 0;

		break;

	default:

		pr_warning("prog '%s': unknown relo %d at insn #%d'\n",

			   bpf_program__title(prog, false),

			   relo->kind, insn_idx);

		return -EINVAL;

	}



	insn = &prog->insns[insn_idx];

	class = BPF_CLASS(insn->code);
@@ -2798,12 +2827,12 @@ static int bpf_core_reloc_insn(struct bpf_program *prog, int insn_off, 
	if (class == BPF_ALU || class == BPF_ALU64) {

		if (BPF_SRC(insn->code) != BPF_K)

			return -EINVAL;

		if (insn->imm != orig_off)

		if (insn->imm != orig_val)

			return -EINVAL;

		insn->imm = new_off;

		insn->imm = new_val;

		pr_debug("prog '%s': patched insn #%d (ALU/ALU64) imm %d -> %d\n",

			 bpf_program__title(prog, false),

			 insn_idx, orig_off, new_off);

			 insn_idx, orig_val, new_val);

	} else {

		pr_warning("prog '%s': trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n",

			   bpf_program__title(prog, false),
@@ -2811,6 +2840,7 @@ static int bpf_core_reloc_insn(struct bpf_program *prog, int insn_off, 
			   insn->off, insn->imm);

		return -EINVAL;

	}



	return 0;

}
@@ -2977,7 +3007,7 @@ static void *u32_as_hash_key(__u32 x) 
 *    types should be compatible (see bpf_core_fields_are_compat for details).

 * 3. It is supported and expected that there might be multiple flavors

 *    matching the spec. As long as all the specs resolve to the same set of

 *    offsets across all candidates, there is not error. If there is any

 *    offsets across all candidates, there is no error. If there is any

 *    ambiguity, CO-RE relocation will fail. This is necessary to accomodate

 *    imprefection of BTF deduplication, which can cause slight duplication of

 *    the same BTF type, if some directly or indirectly referenced (by
@@ -2992,12 +3022,12 @@ static void *u32_as_hash_key(__u32 x) 
 *    CPU-wise compared to prebuilding a map from all local type names to

 *    a list of candidate type names. It's also sped up by caching resolved

 *    list of matching candidates per each local "root" type ID, that has at

 *    least one bpf_offset_reloc associated with it. This list is shared

 *    least one bpf_field_reloc associated with it. This list is shared

 *    between multiple relocations for the same type ID and is updated as some

 *    of the candidates are pruned due to structural incompatibility.

 */

static int bpf_core_reloc_offset(struct bpf_program *prog,

				 const struct bpf_offset_reloc *relo,

static int bpf_core_reloc_field(struct bpf_program *prog,

				 const struct bpf_field_reloc *relo,

				 int relo_idx,

				 const struct btf *local_btf,

				 const struct btf *targ_btf,
@@ -3087,15 +3117,26 @@ static int bpf_core_reloc_offset(struct bpf_program *prog, 
		cand_ids->data[j++] = cand_spec.spec[0].type_id;

	}



	/*

	 * For BPF_FIELD_EXISTS relo or when relaxed CO-RE reloc mode is

	 * requested, it's expected that we might not find any candidates.

	 * In this case, if field wasn't found in any candidate, the list of

	 * candidates shouldn't change at all, we'll just handle relocating

	 * appropriately, depending on relo's kind.

	 */

	if (j > 0)

		cand_ids->len = j;

	if (cand_ids->len == 0) {



	if (j == 0 && !prog->obj->relaxed_core_relocs &&

	    relo->kind != BPF_FIELD_EXISTS) {

		pr_warning("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n",

			   prog_name, relo_idx, local_id, local_name, spec_str);

		return -ESRCH;

	}



	err = bpf_core_reloc_insn(prog, relo->insn_off,

				  local_spec.offset, targ_spec.offset);

	/* bpf_core_reloc_insn should know how to handle missing targ_spec */

	err = bpf_core_reloc_insn(prog, relo, &local_spec,

				  j ? &targ_spec : NULL);

	if (err) {

		pr_warning("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",

			   prog_name, relo_idx, relo->insn_off, err);
@@ -3106,10 +3147,10 @@ static int bpf_core_reloc_offset(struct bpf_program *prog, 
}



static int

bpf_core_reloc_offsets(struct bpf_object *obj, const char *targ_btf_path)

bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path)

{

	const struct btf_ext_info_sec *sec;

	const struct bpf_offset_reloc *rec;

	const struct bpf_field_reloc *rec;

	const struct btf_ext_info *seg;

	struct hashmap_entry *entry;

	struct hashmap *cand_cache = NULL;
@@ -3134,7 +3175,7 @@ bpf_core_reloc_offsets(struct bpf_object *obj, const char *targ_btf_path) 
		goto out;

	}



	seg = &obj->btf_ext->offset_reloc_info;

	seg = &obj->btf_ext->field_reloc_info;

	for_each_btf_ext_sec(seg, sec) {

		sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);

		if (str_is_empty(sec_name)) {
@@ -3153,7 +3194,7 @@ bpf_core_reloc_offsets(struct bpf_object *obj, const char *targ_btf_path) 
			 sec_name, sec->num_info);



		for_each_btf_ext_rec(seg, sec, i, rec) {

			err = bpf_core_reloc_offset(prog, rec, i, obj->btf,

			err = bpf_core_reloc_field(prog, rec, i, obj->btf,

						   targ_btf, cand_cache);

			if (err) {

				pr_warning("prog '%s': relo #%d: failed to relocate: %d\n",
@@ -3179,8 +3220,8 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 
{

	int err = 0;



	if (obj->btf_ext->offset_reloc_info.len)

		err = bpf_core_reloc_offsets(obj, targ_btf_path);

	if (obj->btf_ext->field_reloc_info.len)

		err = bpf_core_reloc_fields(obj, targ_btf_path);



	return err;

}
@@ -3554,24 +3595,46 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) 


static struct bpf_object *

__bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,

		   const char *obj_name, int flags)

		   struct bpf_object_open_opts *opts)

{

	struct bpf_object *obj;

	const char *obj_name;

	char tmp_name[64];

	bool relaxed_maps;

	int err;



	if (elf_version(EV_CURRENT) == EV_NONE) {

		pr_warning("failed to init libelf for %s\n", path);

		pr_warning("failed to init libelf for %s\n",

			   path ? : "(mem buf)");

		return ERR_PTR(-LIBBPF_ERRNO__LIBELF);

	}



	if (!OPTS_VALID(opts, bpf_object_open_opts))

		return ERR_PTR(-EINVAL);



	obj_name = OPTS_GET(opts, object_name, path);

	if (obj_buf) {

		if (!obj_name) {

			snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",

				 (unsigned long)obj_buf,

				 (unsigned long)obj_buf_sz);

			obj_name = tmp_name;

		}

		path = obj_name;

		pr_debug("loading object '%s' from buffer\n", obj_name);

	}



	obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);

	if (IS_ERR(obj))

		return obj;



	obj->relaxed_core_relocs = OPTS_GET(opts, relaxed_core_relocs, false);

	relaxed_maps = OPTS_GET(opts, relaxed_maps, false);



	CHECK_ERR(bpf_object__elf_init(obj), err, out);

	CHECK_ERR(bpf_object__check_endianness(obj), err, out);

	CHECK_ERR(bpf_object__probe_caps(obj), err, out);

	CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out);

	CHECK_ERR(bpf_object__elf_collect(obj, relaxed_maps), err, out);

	CHECK_ERR(bpf_object__collect_reloc(obj), err, out);



	bpf_object__elf_finish(obj);
@@ -3584,13 +3647,16 @@ out: 
static struct bpf_object *

__bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)

{

	LIBBPF_OPTS(bpf_object_open_opts, opts,

		.relaxed_maps = flags & MAPS_RELAX_COMPAT,

	);



	/* param validation */

	if (!attr->file)

		return NULL;



	pr_debug("loading %s\n", attr->file);



	return __bpf_object__open(attr->file, NULL, 0, NULL, flags);

	return __bpf_object__open(attr->file, NULL, 0, &opts);

}



struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
@@ -3611,47 +3677,22 @@ struct bpf_object *bpf_object__open(const char *path) 
struct bpf_object *

bpf_object__open_file(const char *path, struct bpf_object_open_opts *opts)

{

	const char *obj_name;

	bool relaxed_maps;



	if (!OPTS_VALID(opts, bpf_object_open_opts))

		return ERR_PTR(-EINVAL);

	if (!path)

		return ERR_PTR(-EINVAL);



	pr_debug("loading %s\n", path);



	obj_name = OPTS_GET(opts, object_name, path);

	relaxed_maps = OPTS_GET(opts, relaxed_maps, false);

	return __bpf_object__open(path, NULL, 0, obj_name,

				  relaxed_maps ? MAPS_RELAX_COMPAT : 0);

	return __bpf_object__open(path, NULL, 0, opts);

}



struct bpf_object *

bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,

		     struct bpf_object_open_opts *opts)

{

	char tmp_name[64];

	const char *obj_name;

	bool relaxed_maps;



	if (!OPTS_VALID(opts, bpf_object_open_opts))

		return ERR_PTR(-EINVAL);

	if (!obj_buf || obj_buf_sz == 0)

		return ERR_PTR(-EINVAL);



	obj_name = OPTS_GET(opts, object_name, NULL);

	if (!obj_name) {

		snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",

			 (unsigned long)obj_buf,

			 (unsigned long)obj_buf_sz);

		obj_name = tmp_name;

	}

	pr_debug("loading object '%s' from buffer\n", obj_name);



	relaxed_maps = OPTS_GET(opts, relaxed_maps, false);

	return __bpf_object__open(obj_name, obj_buf, obj_buf_sz, obj_name,

				  relaxed_maps ? MAPS_RELAX_COMPAT : 0);

	return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);

}



struct bpf_object *

tools/lib/bpf/libbpf.h

+3 −1
Original line number Diff line number Diff line
@@ -96,8 +96,10 @@ struct bpf_object_open_opts { 
	const char *object_name;

	/* parse map definitions non-strictly, allowing extra attributes/data */

	bool relaxed_maps;

	/* process CO-RE relocations non-strictly, allowing them to fail */

	bool relaxed_core_relocs;

};

#define bpf_object_open_opts__last_field relaxed_maps

#define bpf_object_open_opts__last_field relaxed_core_relocs



LIBBPF_API struct bpf_object *bpf_object__open(const char *path);

LIBBPF_API struct bpf_object *

CRA Git | Maintained and supported by SUSTech CRA and CCSE