Commit 1f607504 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'libbpf-global-vars' 

Andrii Nakryiko says:

====================
This patch set salvages all the non-extern-specific changes out of blocked
externs patch set ([0]). In addition to small clean ups, it also refactors
libbpf's handling of relocations and allows support for global (non-static)
variables.

  [0] https://patchwork.ozlabs.org/project/netdev/list/?series=143358&state=*


====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents a8fdaad5 393cdfbe

tools/lib/bpf/libbpf.c

+158 −134
Original line number Diff line number Diff line
@@ -276,8 +276,8 @@ struct bpf_object { 
		struct {

			GElf_Shdr shdr;

			Elf_Data *data;

		} *reloc;

		int nr_reloc;

		} *reloc_sects;

		int nr_reloc_sects;

		int maps_shndx;

		int btf_maps_shndx;

		int text_shndx;
@@ -575,8 +575,8 @@ static void bpf_object__elf_finish(struct bpf_object *obj) 
	obj->efile.rodata = NULL;

	obj->efile.bss = NULL;



	zfree(&obj->efile.reloc);

	obj->efile.nr_reloc = 0;

	zfree(&obj->efile.reloc_sects);

	obj->efile.nr_reloc_sects = 0;

	zclose(obj->efile.fd);

	obj->efile.obj_buf = NULL;

	obj->efile.obj_buf_sz = 0;
@@ -965,8 +965,7 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 
		 obj->path, nr_maps, data->d_size);



	if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {

		pr_warn("unable to determine map definition size "

			"section %s, %d maps in %zd bytes\n",

		pr_warn("unable to determine map definition size section %s, %d maps in %zd bytes\n",

			obj->path, nr_maps, data->d_size);

		return -EINVAL;

	}
@@ -1030,12 +1029,11 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 
			 * incompatible.

			 */

			char *b;



			for (b = ((char *)def) + sizeof(struct bpf_map_def);

			     b < ((char *)def) + map_def_sz; b++) {

				if (*b != 0) {

					pr_warn("maps section in %s: \"%s\" "

						"has unrecognized, non-zero "

						"options\n",

					pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",

						obj->path, map_name);

					if (strict)

						return -EINVAL;
@@ -1073,7 +1071,8 @@ skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 
 */

static bool get_map_field_int(const char *map_name, const struct btf *btf,

			      const struct btf_type *def,

			      const struct btf_member *m, __u32 *res) {

			      const struct btf_member *m, __u32 *res)

{

	const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);

	const char *name = btf__name_by_offset(btf, m->name_off);

	const struct btf_array *arr_info;
@@ -1387,7 +1386,8 @@ static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 
	for (i = 0; i < vlen; i++) {

		err = bpf_object__init_user_btf_map(obj, sec, i,

						    obj->efile.btf_maps_shndx,

						    data, strict, pin_root_path);

						    data, strict,

						    pin_root_path);

		if (err)

			return err;

	}
@@ -1673,12 +1673,14 @@ static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps, 
				if (strcmp(name, ".text") == 0)

					obj->efile.text_shndx = idx;

				err = bpf_object__add_program(obj, data->d_buf,

							      data->d_size, name, idx);

							      data->d_size,

							      name, idx);

				if (err) {

					char errmsg[STRERR_BUFSIZE];

					char *cp = libbpf_strerror_r(-err, errmsg,

								     sizeof(errmsg));

					char *cp;



					cp = libbpf_strerror_r(-err, errmsg,

							       sizeof(errmsg));

					pr_warn("failed to alloc program %s (%s): %s",

						name, obj->path, cp);

					return err;
@@ -1693,8 +1695,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps, 
				pr_debug("skip section(%d) %s\n", idx, name);

			}

		} else if (sh.sh_type == SHT_REL) {

			int nr_reloc = obj->efile.nr_reloc;

			void *reloc = obj->efile.reloc;

			int nr_sects = obj->efile.nr_reloc_sects;

			void *sects = obj->efile.reloc_sects;

			int sec = sh.sh_info; /* points to other section */



			/* Only do relo for section with exec instructions */
@@ -1704,18 +1706,18 @@ static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps, 
				continue;

			}



			reloc = reallocarray(reloc, nr_reloc + 1,

					     sizeof(*obj->efile.reloc));

			if (!reloc) {

				pr_warn("realloc failed\n");

			sects = reallocarray(sects, nr_sects + 1,

					     sizeof(*obj->efile.reloc_sects));

			if (!sects) {

				pr_warn("reloc_sects realloc failed\n");

				return -ENOMEM;

			}



			obj->efile.reloc = reloc;

			obj->efile.nr_reloc++;

			obj->efile.reloc_sects = sects;

			obj->efile.nr_reloc_sects++;



			obj->efile.reloc[nr_reloc].shdr = sh;

			obj->efile.reloc[nr_reloc].data = data;

			obj->efile.reloc_sects[nr_sects].shdr = sh;

			obj->efile.reloc_sects[nr_sects].data = data;

		} else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) {

			obj->efile.bss = data;

			obj->efile.bss_shndx = idx;
@@ -1780,14 +1782,6 @@ static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 
	       shndx == obj->efile.btf_maps_shndx;

}



static bool bpf_object__relo_in_known_section(const struct bpf_object *obj,

					      int shndx)

{

	return shndx == obj->efile.text_shndx ||

	       bpf_object__shndx_is_maps(obj, shndx) ||

	       bpf_object__shndx_is_data(obj, shndx);

}



static enum libbpf_map_type

bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)

{
@@ -1801,14 +1795,119 @@ bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 
		return LIBBPF_MAP_UNSPEC;

}



static int bpf_program__record_reloc(struct bpf_program *prog,

				     struct reloc_desc *reloc_desc,

				     __u32 insn_idx, const char *name,

				     const GElf_Sym *sym, const GElf_Rel *rel)

{

	struct bpf_insn *insn = &prog->insns[insn_idx];

	size_t map_idx, nr_maps = prog->obj->nr_maps;

	struct bpf_object *obj = prog->obj;

	__u32 shdr_idx = sym->st_shndx;

	enum libbpf_map_type type;

	struct bpf_map *map;



	/* sub-program call relocation */

	if (insn->code == (BPF_JMP | BPF_CALL)) {

		if (insn->src_reg != BPF_PSEUDO_CALL) {

			pr_warn("incorrect bpf_call opcode\n");

			return -LIBBPF_ERRNO__RELOC;

		}

		/* text_shndx can be 0, if no default "main" program exists */

		if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {

			pr_warn("bad call relo against section %u\n", shdr_idx);

			return -LIBBPF_ERRNO__RELOC;

		}

		if (sym->st_value % 8) {

			pr_warn("bad call relo offset: %lu\n", sym->st_value);

			return -LIBBPF_ERRNO__RELOC;

		}

		reloc_desc->type = RELO_CALL;

		reloc_desc->insn_idx = insn_idx;

		reloc_desc->text_off = sym->st_value / 8;

		obj->has_pseudo_calls = true;

		return 0;

	}



	if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {

		pr_warn("invalid relo for insns[%d].code 0x%x\n",

			insn_idx, insn->code);

		return -LIBBPF_ERRNO__RELOC;

	}

	if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {

		pr_warn("invalid relo for \'%s\' in special section 0x%x; forgot to initialize global var?..\n",

			name, shdr_idx);

		return -LIBBPF_ERRNO__RELOC;

	}



	type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);



	/* generic map reference relocation */

	if (type == LIBBPF_MAP_UNSPEC) {

		if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {

			pr_warn("bad map relo against section %u\n",

				shdr_idx);

			return -LIBBPF_ERRNO__RELOC;

		}

		for (map_idx = 0; map_idx < nr_maps; map_idx++) {

			map = &obj->maps[map_idx];

			if (map->libbpf_type != type ||

			    map->sec_idx != sym->st_shndx ||

			    map->sec_offset != sym->st_value)

				continue;

			pr_debug("found map %zd (%s, sec %d, off %zu) for insn %u\n",

				 map_idx, map->name, map->sec_idx,

				 map->sec_offset, insn_idx);

			break;

		}

		if (map_idx >= nr_maps) {

			pr_warn("map relo failed to find map for sec %u, off %llu\n",

				shdr_idx, (__u64)sym->st_value);

			return -LIBBPF_ERRNO__RELOC;

		}

		reloc_desc->type = RELO_LD64;

		reloc_desc->insn_idx = insn_idx;

		reloc_desc->map_idx = map_idx;

		return 0;

	}



	/* global data map relocation */

	if (!bpf_object__shndx_is_data(obj, shdr_idx)) {

		pr_warn("bad data relo against section %u\n", shdr_idx);

		return -LIBBPF_ERRNO__RELOC;

	}

	if (!obj->caps.global_data) {

		pr_warn("relocation: kernel does not support global \'%s\' variable access in insns[%d]\n",

			name, insn_idx);

		return -LIBBPF_ERRNO__RELOC;

	}

	for (map_idx = 0; map_idx < nr_maps; map_idx++) {

		map = &obj->maps[map_idx];

		if (map->libbpf_type != type)

			continue;

		pr_debug("found data map %zd (%s, sec %d, off %zu) for insn %u\n",

			 map_idx, map->name, map->sec_idx, map->sec_offset,

			 insn_idx);

		break;

	}

	if (map_idx >= nr_maps) {

		pr_warn("data relo failed to find map for sec %u\n",

			shdr_idx);

		return -LIBBPF_ERRNO__RELOC;

	}



	reloc_desc->type = RELO_DATA;

	reloc_desc->insn_idx = insn_idx;

	reloc_desc->map_idx = map_idx;

	return 0;

}



static int

bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,

			   Elf_Data *data, struct bpf_object *obj)

{

	Elf_Data *symbols = obj->efile.symbols;

	struct bpf_map *maps = obj->maps;

	size_t nr_maps = obj->nr_maps;

	int i, nrels;

	int err, i, nrels;



	pr_debug("collecting relocating info for: '%s'\n", prog->section_name);

	nrels = shdr->sh_size / shdr->sh_entsize;
@@ -1821,12 +1920,8 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 
	prog->nr_reloc = nrels;



	for (i = 0; i < nrels; i++) {

		struct bpf_insn *insns = prog->insns;

		enum libbpf_map_type type;

		unsigned int insn_idx;

		unsigned int shdr_idx;

		const char *name;

		size_t map_idx;

		__u32 insn_idx;

		GElf_Sym sym;

		GElf_Rel rel;
@@ -1834,101 +1929,28 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 
			pr_warn("relocation: failed to get %d reloc\n", i);

			return -LIBBPF_ERRNO__FORMAT;

		}



		if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {

			pr_warn("relocation: symbol %"PRIx64" not found\n",

				GELF_R_SYM(rel.r_info));

			return -LIBBPF_ERRNO__FORMAT;

		}

		if (rel.r_offset % sizeof(struct bpf_insn))

			return -LIBBPF_ERRNO__FORMAT;



		insn_idx = rel.r_offset / sizeof(struct bpf_insn);

		name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,

				  sym.st_name) ? : "<?>";



		pr_debug("relo for %lld value %lld name %d (\'%s\')\n",

			 (long long) (rel.r_info >> 32),

			 (long long) sym.st_value, sym.st_name, name);



		shdr_idx = sym.st_shndx;

		insn_idx = rel.r_offset / sizeof(struct bpf_insn);

		pr_debug("relocation: insn_idx=%u, shdr_idx=%u\n",

			 insn_idx, shdr_idx);



		if (shdr_idx >= SHN_LORESERVE) {

			pr_warn("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n",

				name, shdr_idx, insn_idx,

				insns[insn_idx].code);

			return -LIBBPF_ERRNO__RELOC;

		}

		if (!bpf_object__relo_in_known_section(obj, shdr_idx)) {

			pr_warn("Program '%s' contains unrecognized relo data pointing to section %u\n",

				prog->section_name, shdr_idx);

			return -LIBBPF_ERRNO__RELOC;

		}



		if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {

			if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {

				pr_warn("incorrect bpf_call opcode\n");

				return -LIBBPF_ERRNO__RELOC;

			}

			if (sym.st_value % 8) {

				pr_warn("bad call relo offset: %lu\n", sym.st_value);

				return -LIBBPF_ERRNO__RELOC;

			}

			prog->reloc_desc[i].type = RELO_CALL;

			prog->reloc_desc[i].insn_idx = insn_idx;

			prog->reloc_desc[i].text_off = sym.st_value / 8;

			obj->has_pseudo_calls = true;

			continue;

		}



		if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {

			pr_warn("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",

				insn_idx, insns[insn_idx].code);

			return -LIBBPF_ERRNO__RELOC;

		}



		if (bpf_object__shndx_is_maps(obj, shdr_idx) ||

		    bpf_object__shndx_is_data(obj, shdr_idx)) {

			type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);

			if (type != LIBBPF_MAP_UNSPEC) {

				if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) {

					pr_warn("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n",

						name, insn_idx, insns[insn_idx].code);

					return -LIBBPF_ERRNO__RELOC;

				}

				if (!obj->caps.global_data) {

					pr_warn("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n",

						name, insn_idx);

					return -LIBBPF_ERRNO__RELOC;

				}

			}



			for (map_idx = 0; map_idx < nr_maps; map_idx++) {

				if (maps[map_idx].libbpf_type != type)

					continue;

				if (type != LIBBPF_MAP_UNSPEC ||

				    (maps[map_idx].sec_idx == sym.st_shndx &&

				     maps[map_idx].sec_offset == sym.st_value)) {

					pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n",

						 map_idx, maps[map_idx].name,

						 maps[map_idx].sec_idx,

						 maps[map_idx].sec_offset,

		pr_debug("relo for shdr %u, symb %llu, value %llu, type %d, bind %d, name %d (\'%s\'), insn %u\n",

			 (__u32)sym.st_shndx, (__u64)GELF_R_SYM(rel.r_info),

			 (__u64)sym.st_value, GELF_ST_TYPE(sym.st_info),

			 GELF_ST_BIND(sym.st_info), sym.st_name, name,

			 insn_idx);

					break;

				}

			}



			if (map_idx >= nr_maps) {

				pr_warn("bpf relocation: map_idx %d larger than %d\n",

					(int)map_idx, (int)nr_maps - 1);

				return -LIBBPF_ERRNO__RELOC;

			}



			prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ?

						   RELO_DATA : RELO_LD64;

			prog->reloc_desc[i].insn_idx = insn_idx;

			prog->reloc_desc[i].map_idx = map_idx;

		}

		err = bpf_program__record_reloc(prog, &prog->reloc_desc[i],

						insn_idx, name, &sym, &rel);

		if (err)

			return err;

	}

	return 0;

}
@@ -2120,7 +2142,7 @@ bpf_object__probe_global_data(struct bpf_object *obj) 


static int bpf_object__probe_btf_func(struct bpf_object *obj)

{

	const char strs[] = "\0int\0x\0a";

	static const char strs[] = "\0int\0x\0a";

	/* void x(int a) {} */

	__u32 types[] = {

		/* int */
@@ -2146,7 +2168,7 @@ static int bpf_object__probe_btf_func(struct bpf_object *obj) 


static int bpf_object__probe_btf_datasec(struct bpf_object *obj)

{

	const char strs[] = "\0x\0.data";

	static const char strs[] = "\0x\0.data";

	/* static int a; */

	__u32 types[] = {

		/* int */
@@ -3671,9 +3693,9 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) 
		return -LIBBPF_ERRNO__INTERNAL;

	}



	for (i = 0; i < obj->efile.nr_reloc; i++) {

		GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;

		Elf_Data *data = obj->efile.reloc[i].data;

	for (i = 0; i < obj->efile.nr_reloc_sects; i++) {

		GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;

		Elf_Data *data = obj->efile.reloc_sects[i].data;

		int idx = shdr->sh_info;

		struct bpf_program *prog;
@@ -5087,7 +5109,7 @@ int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 
		*expected_attach_type = section_names[i].expected_attach_type;

		return 0;

	}

	pr_warn("failed to guess program type based on ELF section name '%s'\n", name);

	pr_warn("failed to guess program type from ELF section '%s'\n", name);

	type_names = libbpf_get_type_names(false);

	if (type_names != NULL) {

		pr_info("supported section(type) names are:%s\n", type_names);
@@ -6313,7 +6335,8 @@ static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 


};



static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset)

static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,

					   int offset)

{

	__u32 *array = (__u32 *)info;
@@ -6322,7 +6345,8 @@ static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offse 
	return -(int)offset;

}



static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset)

static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,

					   int offset)

{

	__u64 *array = (__u64 *)info;

tools/testing/selftests/bpf/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -161,7 +161,7 @@ $(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h 
define CLANG_BPF_BUILD_RULE

	($(CLANG) $3 -O2 -target bpf -emit-llvm				\

		-c $1 -o - || echo "BPF obj compilation failed") | 	\

	$(LLC) -march=bpf -mcpu=probe $4 -filetype=obj -o $2

	$(LLC) -mattr=dwarfris -march=bpf -mcpu=probe $4 -filetype=obj -o $2

endef

# Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32

define CLANG_NOALU32_BPF_BUILD_RULE

tools/testing/selftests/bpf/progs/test_core_reloc_arrays.c

+2 −2
Original line number Diff line number Diff line
@@ -8,10 +8,10 @@ 


char _license[] SEC("license") = "GPL";



static volatile struct data {

struct {

	char in[256];

	char out[256];

} data;

} data = {};



struct core_reloc_arrays_output {

	int a2;

tools/testing/selftests/bpf/progs/test_core_reloc_bitfields_direct.c

+2 −2
Original line number Diff line number Diff line
@@ -8,10 +8,10 @@ 


char _license[] SEC("license") = "GPL";



static volatile struct data {

struct {

	char in[256];

	char out[256];

} data;

} data = {};



struct core_reloc_bitfields {

	/* unsigned bitfields */

tools/testing/selftests/bpf/progs/test_core_reloc_bitfields_probed.c

+2 −2
Original line number Diff line number Diff line
@@ -8,10 +8,10 @@ 


char _license[] SEC("license") = "GPL";



static volatile struct data {

struct {

	char in[256];

	char out[256];

} data;

} data = {};



struct core_reloc_bitfields {

	/* unsigned bitfields */

CRA Git | Maintained and supported by SUSTech CRA and CCSE