Commit ac53a0d3 authored by Daniel Borkmann's avatar Daniel Borkmann
Browse files

Merge branch 'bpf-llvm-reg-alloc-patterns' 

Alexei Starovoitov says:

====================
Make two verifier improvements:

- The llvm register allocator may use two different registers representing
  the same virtual register. Teach the verifier to recognize that.
- Track bounded scalar spill/fill.

The profiler[123] test in patch 3 will fail to load without patches 1 and 2.
The profiler[23] test may fail to load on older llvm due to speculative
code motion nd instruction combining optimizations that are fixed in
https://reviews.llvm.org/D85570



v1 -> v2:
  - fixed 32-bit mov issue spotted by John.
  - allowed r2=r1; r3=r2; sequence as suggested by John.
  - added comments, acks, more tests.
====================

Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parents eca43ee6 54fada41

kernel/bpf/verifier.c

+65 −1
Original line number Diff line number Diff line
@@ -2227,6 +2227,20 @@ static bool register_is_const(struct bpf_reg_state *reg) 
	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);

}



static bool __is_scalar_unbounded(struct bpf_reg_state *reg)

{

	return tnum_is_unknown(reg->var_off) &&

	       reg->smin_value == S64_MIN && reg->smax_value == S64_MAX &&

	       reg->umin_value == 0 && reg->umax_value == U64_MAX &&

	       reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX &&

	       reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX;

}



static bool register_is_bounded(struct bpf_reg_state *reg)

{

	return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg);

}



static bool __is_pointer_value(bool allow_ptr_leaks,

			       const struct bpf_reg_state *reg)

{
@@ -2278,7 +2292,7 @@ static int check_stack_write(struct bpf_verifier_env *env, 
	if (value_regno >= 0)

		reg = &cur->regs[value_regno];



	if (reg && size == BPF_REG_SIZE && register_is_const(reg) &&

	if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) &&

	    !register_is_null(reg) && env->bpf_capable) {

		if (dst_reg != BPF_REG_FP) {

			/* The backtracking logic can only recognize explicit
@@ -6436,6 +6450,11 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 
	src_reg = NULL;

	if (dst_reg->type != SCALAR_VALUE)

		ptr_reg = dst_reg;

	else

		/* Make sure ID is cleared otherwise dst_reg min/max could be

		 * incorrectly propagated into other registers by find_equal_scalars()

		 */

		dst_reg->id = 0;

	if (BPF_SRC(insn->code) == BPF_X) {

		src_reg = &regs[insn->src_reg];

		if (src_reg->type != SCALAR_VALUE) {
@@ -6569,6 +6588,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 
				/* case: R1 = R2

				 * copy register state to dest reg

				 */

				if (src_reg->type == SCALAR_VALUE && !src_reg->id)

					/* Assign src and dst registers the same ID

					 * that will be used by find_equal_scalars()

					 * to propagate min/max range.

					 */

					src_reg->id = ++env->id_gen;

				*dst_reg = *src_reg;

				dst_reg->live |= REG_LIVE_WRITTEN;

				dst_reg->subreg_def = DEF_NOT_SUBREG;
@@ -6581,6 +6606,11 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 
					return -EACCES;

				} else if (src_reg->type == SCALAR_VALUE) {

					*dst_reg = *src_reg;

					/* Make sure ID is cleared otherwise

					 * dst_reg min/max could be incorrectly

					 * propagated into src_reg by find_equal_scalars()

					 */

					dst_reg->id = 0;

					dst_reg->live |= REG_LIVE_WRITTEN;

					dst_reg->subreg_def = env->insn_idx + 1;

				} else {
@@ -7369,6 +7399,30 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn, 
	return true;

}



static void find_equal_scalars(struct bpf_verifier_state *vstate,

			       struct bpf_reg_state *known_reg)

{

	struct bpf_func_state *state;

	struct bpf_reg_state *reg;

	int i, j;



	for (i = 0; i <= vstate->curframe; i++) {

		state = vstate->frame[i];

		for (j = 0; j < MAX_BPF_REG; j++) {

			reg = &state->regs[j];

			if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)

				*reg = *known_reg;

		}



		bpf_for_each_spilled_reg(j, state, reg) {

			if (!reg)

				continue;

			if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)

				*reg = *known_reg;

		}

	}

}



static int check_cond_jmp_op(struct bpf_verifier_env *env,

			     struct bpf_insn *insn, int *insn_idx)

{
@@ -7497,6 +7551,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, 
				reg_combine_min_max(&other_branch_regs[insn->src_reg],

						    &other_branch_regs[insn->dst_reg],

						    src_reg, dst_reg, opcode);

			if (src_reg->id) {

				find_equal_scalars(this_branch, src_reg);

				find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);

			}



		}

	} else if (dst_reg->type == SCALAR_VALUE) {

		reg_set_min_max(&other_branch_regs[insn->dst_reg],
@@ -7504,6 +7563,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, 
					opcode, is_jmp32);

	}



	if (dst_reg->type == SCALAR_VALUE && dst_reg->id) {

		find_equal_scalars(this_branch, dst_reg);

		find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);

	}



	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().

	 * NOTE: these optimizations below are related with pointer comparison

	 *       which will never be JMP32.

tools/testing/selftests/bpf/README.rst

+38 −0
Original line number Diff line number Diff line
@@ -7,6 +7,44 @@ General instructions on running selftests can be found in 
Additional information about selftest failures are

documented here.



profiler[23] test failures with clang/llvm <12.0.0

==================================================



With clang/llvm <12.0.0, the profiler[23] test may fail.

The symptom looks like



.. code-block:: c



  // r9 is a pointer to map_value

  // r7 is a scalar

  17:       bf 96 00 00 00 00 00 00 r6 = r9

  18:       0f 76 00 00 00 00 00 00 r6 += r7

  math between map_value pointer and register with unbounded min value is not allowed



  // the instructions below will not be seen in the verifier log

  19:       a5 07 01 00 01 01 00 00 if r7 < 257 goto +1

  20:       bf 96 00 00 00 00 00 00 r6 = r9

  // r6 is used here



The verifier will reject such code with above error.

At insn 18 the r7 is indeed unbounded. The later insn 19 checks the bounds and

the insn 20 undoes map_value addition. It is currently impossible for the

verifier to understand such speculative pointer arithmetic.

Hence

    https://reviews.llvm.org/D85570

addresses it on the compiler side. It was committed on llvm 12.



The corresponding C code

.. code-block:: c



  for (int i = 0; i < MAX_CGROUPS_PATH_DEPTH; i++) {

          filepart_length = bpf_probe_read_str(payload, ...);

          if (filepart_length <= MAX_PATH) {

                  barrier_var(filepart_length); // workaround

                  payload += filepart_length;

          }

  }



bpf_iter test failures with clang/llvm 10.0.0

=============================================

tools/testing/selftests/bpf/prog_tests/align.c

+8 −8
Original line number Diff line number Diff line
@@ -195,13 +195,13 @@ static struct bpf_align_test tests[] = { 
		.prog_type = BPF_PROG_TYPE_SCHED_CLS,

		.matches = {

			{7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},

			{8, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},

			{8, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},

			{9, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},

			{10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},

			{10, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},

			{11, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},

			{12, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},

			{12, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},

			{13, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},

			{14, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},

			{14, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},

			{15, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},

			{16, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},

		},
@@ -518,7 +518,7 @@ static struct bpf_align_test tests[] = { 
			 * the total offset is 4-byte aligned and meets the

			 * load's requirements.

			 */

			{20, "R5=pkt(id=1,off=0,r=4,umin_value=2,umax_value=1034,var_off=(0x2; 0x7fc)"},

			{20, "R5=pkt(id=2,off=0,r=4,umin_value=2,umax_value=1034,var_off=(0x2; 0x7fc)"},



		},

	},
@@ -561,18 +561,18 @@ static struct bpf_align_test tests[] = { 
			/* Adding 14 makes R6 be (4n+2) */

			{11, "R6_w=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"},

			/* Subtracting from packet pointer overflows ubounds */

			{13, "R5_w=pkt(id=1,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c)"},

			{13, "R5_w=pkt(id=2,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c)"},

			/* New unknown value in R7 is (4n), >= 76 */

			{15, "R7_w=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"},

			/* Adding it to packet pointer gives nice bounds again */

			{16, "R5_w=pkt(id=2,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},

			{16, "R5_w=pkt(id=3,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},

			/* At the time the word size load is performed from R5,

			 * its total fixed offset is NET_IP_ALIGN + reg->off (0)

			 * which is 2.  Then the variable offset is (4n+2), so

			 * the total offset is 4-byte aligned and meets the

			 * load's requirements.

			 */

			{20, "R5=pkt(id=2,off=0,r=4,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},

			{20, "R5=pkt(id=3,off=0,r=4,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},

		},

	},

};

tools/testing/selftests/bpf/prog_tests/test_profiler.c

0 → 100644
+72 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/* Copyright (c) 2020 Facebook */

#include <test_progs.h>

#include "progs/profiler.h"

#include "profiler1.skel.h"

#include "profiler2.skel.h"

#include "profiler3.skel.h"



static int sanity_run(struct bpf_program *prog)

{

	struct bpf_prog_test_run_attr test_attr = {};

	__u64 args[] = {1, 2, 3};

	__u32 duration = 0;

	int err, prog_fd;



	prog_fd = bpf_program__fd(prog);

	test_attr.prog_fd = prog_fd;

	test_attr.ctx_in = args;

	test_attr.ctx_size_in = sizeof(args);

	err = bpf_prog_test_run_xattr(&test_attr);

	if (CHECK(err || test_attr.retval, "test_run",

		  "err %d errno %d retval %d duration %d\n",

		  err, errno, test_attr.retval, duration))

		return -1;

	return 0;

}



void test_test_profiler(void)

{

	struct profiler1 *profiler1_skel = NULL;

	struct profiler2 *profiler2_skel = NULL;

	struct profiler3 *profiler3_skel = NULL;

	__u32 duration = 0;

	int err;



	profiler1_skel = profiler1__open_and_load();

	if (CHECK(!profiler1_skel, "profiler1_skel_load", "profiler1 skeleton failed\n"))

		goto cleanup;



	err = profiler1__attach(profiler1_skel);

	if (CHECK(err, "profiler1_attach", "profiler1 attach failed: %d\n", err))

		goto cleanup;



	if (sanity_run(profiler1_skel->progs.raw_tracepoint__sched_process_exec))

		goto cleanup;



	profiler2_skel = profiler2__open_and_load();

	if (CHECK(!profiler2_skel, "profiler2_skel_load", "profiler2 skeleton failed\n"))

		goto cleanup;



	err = profiler2__attach(profiler2_skel);

	if (CHECK(err, "profiler2_attach", "profiler2 attach failed: %d\n", err))

		goto cleanup;



	if (sanity_run(profiler2_skel->progs.raw_tracepoint__sched_process_exec))

		goto cleanup;



	profiler3_skel = profiler3__open_and_load();

	if (CHECK(!profiler3_skel, "profiler3_skel_load", "profiler3 skeleton failed\n"))

		goto cleanup;



	err = profiler3__attach(profiler3_skel);

	if (CHECK(err, "profiler3_attach", "profiler3 attach failed: %d\n", err))

		goto cleanup;



	if (sanity_run(profiler3_skel->progs.raw_tracepoint__sched_process_exec))

		goto cleanup;

cleanup:

	profiler1__destroy(profiler1_skel);

	profiler2__destroy(profiler2_skel);

	profiler3__destroy(profiler3_skel);

}

tools/testing/selftests/bpf/progs/profiler.h

0 → 100644
+177 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/* Copyright (c) 2020 Facebook */

#pragma once



#define TASK_COMM_LEN 16

#define MAX_ANCESTORS 4

#define MAX_PATH 256

#define KILL_TARGET_LEN 64

#define CTL_MAXNAME 10

#define MAX_ARGS_LEN 4096

#define MAX_FILENAME_LEN 512

#define MAX_ENVIRON_LEN 8192

#define MAX_PATH_DEPTH 32

#define MAX_FILEPATH_LENGTH (MAX_PATH_DEPTH * MAX_PATH)

#define MAX_CGROUPS_PATH_DEPTH 8



#define MAX_METADATA_PAYLOAD_LEN TASK_COMM_LEN



#define MAX_CGROUP_PAYLOAD_LEN \

	(MAX_PATH * 2 + (MAX_PATH * MAX_CGROUPS_PATH_DEPTH))



#define MAX_CAP_PAYLOAD_LEN (MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN)



#define MAX_SYSCTL_PAYLOAD_LEN \

	(MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + CTL_MAXNAME + MAX_PATH)



#define MAX_KILL_PAYLOAD_LEN \

	(MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + TASK_COMM_LEN + \

	 KILL_TARGET_LEN)



#define MAX_EXEC_PAYLOAD_LEN \

	(MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + MAX_FILENAME_LEN + \

	 MAX_ARGS_LEN + MAX_ENVIRON_LEN)



#define MAX_FILEMOD_PAYLOAD_LEN \

	(MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + MAX_FILEPATH_LENGTH + \

	 MAX_FILEPATH_LENGTH)



enum data_type {

	INVALID_EVENT,

	EXEC_EVENT,

	FORK_EVENT,

	KILL_EVENT,

	SYSCTL_EVENT,

	FILEMOD_EVENT,

	MAX_DATA_TYPE_EVENT

};



enum filemod_type {

	FMOD_OPEN,

	FMOD_LINK,

	FMOD_SYMLINK,

};



struct ancestors_data_t {

	pid_t ancestor_pids[MAX_ANCESTORS];

	uint32_t ancestor_exec_ids[MAX_ANCESTORS];

	uint64_t ancestor_start_times[MAX_ANCESTORS];

	uint32_t num_ancestors;

};



struct var_metadata_t {

	enum data_type type;

	pid_t pid;

	uint32_t exec_id;

	uid_t uid;

	gid_t gid;

	uint64_t start_time;

	uint32_t cpu_id;

	uint64_t bpf_stats_num_perf_events;

	uint64_t bpf_stats_start_ktime_ns;

	uint8_t comm_length;

};



struct cgroup_data_t {

	ino_t cgroup_root_inode;

	ino_t cgroup_proc_inode;

	uint64_t cgroup_root_mtime;

	uint64_t cgroup_proc_mtime;

	uint16_t cgroup_root_length;

	uint16_t cgroup_proc_length;

	uint16_t cgroup_full_length;

	int cgroup_full_path_root_pos;

};



struct var_sysctl_data_t {

	struct var_metadata_t meta;

	struct cgroup_data_t cgroup_data;

	struct ancestors_data_t ancestors_info;

	uint8_t sysctl_val_length;

	uint16_t sysctl_path_length;

	char payload[MAX_SYSCTL_PAYLOAD_LEN];

};



struct var_kill_data_t {

	struct var_metadata_t meta;

	struct cgroup_data_t cgroup_data;

	struct ancestors_data_t ancestors_info;

	pid_t kill_target_pid;

	int kill_sig;

	uint32_t kill_count;

	uint64_t last_kill_time;

	uint8_t kill_target_name_length;

	uint8_t kill_target_cgroup_proc_length;

	char payload[MAX_KILL_PAYLOAD_LEN];

	size_t payload_length;

};



struct var_exec_data_t {

	struct var_metadata_t meta;

	struct cgroup_data_t cgroup_data;

	pid_t parent_pid;

	uint32_t parent_exec_id;

	uid_t parent_uid;

	uint64_t parent_start_time;

	uint16_t bin_path_length;

	uint16_t cmdline_length;

	uint16_t environment_length;

	char payload[MAX_EXEC_PAYLOAD_LEN];

};



struct var_fork_data_t {

	struct var_metadata_t meta;

	pid_t parent_pid;

	uint32_t parent_exec_id;

	uint64_t parent_start_time;

	char payload[MAX_METADATA_PAYLOAD_LEN];

};



struct var_filemod_data_t {

	struct var_metadata_t meta;

	struct cgroup_data_t cgroup_data;

	enum filemod_type fmod_type;

	unsigned int dst_flags;

	uint32_t src_device_id;

	uint32_t dst_device_id;

	ino_t src_inode;

	ino_t dst_inode;

	uint16_t src_filepath_length;

	uint16_t dst_filepath_length;

	char payload[MAX_FILEMOD_PAYLOAD_LEN];

};



struct profiler_config_struct {

	bool fetch_cgroups_from_bpf;

	ino_t cgroup_fs_inode;

	ino_t cgroup_login_session_inode;

	uint64_t kill_signals_mask;

	ino_t inode_filter;

	uint32_t stale_info_secs;

	bool use_variable_buffers;

	bool read_environ_from_exec;

	bool enable_cgroup_v1_resolver;

};



struct bpf_func_stats_data {

	uint64_t time_elapsed_ns;

	uint64_t num_executions;

	uint64_t num_perf_events;

};



struct bpf_func_stats_ctx {

	uint64_t start_time_ns;

	struct bpf_func_stats_data* bpf_func_stats_data_val;

};



enum bpf_function_id {

	profiler_bpf_proc_sys_write,

	profiler_bpf_sched_process_exec,

	profiler_bpf_sched_process_exit,

	profiler_bpf_sys_enter_kill,

	profiler_bpf_do_filp_open_ret,

	profiler_bpf_sched_process_fork,

	profiler_bpf_vfs_link,

	profiler_bpf_vfs_symlink,

	profiler_bpf_max_function_id

};

CRA Git | Maintained and supported by SUSTech CRA and CCSE