rcu/tree: Make debug_objects logic independent of rcu_head (446044eb) · Commits · 戴 / test

kernel/rcu/tree.c

+13 −16

Original line number	Diff line number	Diff line
		@@ -2970,13 +2970,11 @@ EXPORT_SYMBOL_GPL(call_rcu);
		* @nr_records: Number of active pointers in the array
		* @records: Array of the kfree_rcu() pointers
		* @next: Next bulk object in the block chain
		* @head_free_debug: For debug, when CONFIG_DEBUG_OBJECTS_RCU_HEAD is set
		*/
		struct kfree_rcu_bulk_data {
		unsigned long nr_records;
		void *records[KFREE_BULK_MAX_ENTR];
		struct kfree_rcu_bulk_data *next;
		struct rcu_head *head_free_debug;
		};

		/**
		@@ -3026,11 +3024,13 @@ struct kfree_rcu_cpu {
		static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc);

		static __always_inline void
		debug_rcu_head_unqueue_bulk(struct rcu_head *head)
		debug_rcu_bhead_unqueue(struct kfree_rcu_bulk_data *bhead)
		{
		#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
		for (; head; head = head->next)
		debug_rcu_head_unqueue(head);
		int i;

		for (i = 0; i < bhead->nr_records; i++)
		debug_rcu_head_unqueue((struct rcu_head *)(bhead->records[i]));
		#endif
		}

		@@ -3060,7 +3060,7 @@ static void kfree_rcu_work(struct work_struct *work)
		for (; bhead; bhead = bnext) {
		bnext = bhead->next;

		debug_rcu_head_unqueue_bulk(bhead->head_free_debug);
		debug_rcu_bhead_unqueue(bhead);

		rcu_lock_acquire(&rcu_callback_map);
		trace_rcu_invoke_kfree_bulk_callback(rcu_state.name,
		@@ -3082,14 +3082,15 @@ static void kfree_rcu_work(struct work_struct *work)
		*/
		for (; head; head = next) {
		unsigned long offset = (unsigned long)head->func;
		void ptr = (void )head - offset;

		next = head->next;
		debug_rcu_head_unqueue(head);
		debug_rcu_head_unqueue((struct rcu_head *)ptr);
		rcu_lock_acquire(&rcu_callback_map);
		trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset);

		if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset)))
		kfree((void *)head - offset);
		kfree(ptr);

		rcu_lock_release(&rcu_callback_map);
		cond_resched_tasks_rcu_qs();
		@@ -3228,18 +3229,11 @@ kfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp,
		/* Initialize the new block. */
		bnode->nr_records = 0;
		bnode->next = krcp->bhead;
		bnode->head_free_debug = NULL;

		/* Attach it to the head. */
		krcp->bhead = bnode;
		}

		#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
		head->func = func;
		head->next = krcp->bhead->head_free_debug;
		krcp->bhead->head_free_debug = head;
		#endif

		/* Finally insert. */
		krcp->bhead->records[krcp->bhead->nr_records++] =
		(void *) head - (unsigned long) func;
		@@ -3263,14 +3257,17 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
		{
		unsigned long flags;
		struct kfree_rcu_cpu *krcp;
		void *ptr;

		local_irq_save(flags); // For safely calling this_cpu_ptr().
		krcp = this_cpu_ptr(&krc);
		if (krcp->initialized)
		raw_spin_lock(&krcp->lock);

		ptr = (void *)head - (unsigned long)func;

		// Queue the object but don't yet schedule the batch.
		if (debug_rcu_head_queue(head)) {
		if (debug_rcu_head_queue(ptr)) {
		// Probable double kfree_rcu(), just leak.
		WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
		__func__, head);

Admin message