Commit ad3fee79 authored by Lars Ellenberg's avatar Lars Ellenberg Committed by Philipp Reisner
Browse files

drbd: improve throttling decisions of background resynchronisation 



Background resynchronisation does some "side-stepping", or throttles
itself, if it detects application IO activity, and the current resync
rate estimate is above the configured "cmin-rate".

What was not detected: if there is no application IO,
because it blocks on activity log transactions.

Introduce a new atomic_t ap_actlog_cnt, tracking such blocked requests,
and count non-zero as application IO activity.
This counter is exposed at proc_details level 2 and above.

Also make sure to release the currently locked resync extent
if we side-step due to such voluntary throttling.

Signed-off-by: default avatarPhilipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: default avatarLars Ellenberg <lars.ellenberg@linbit.com>
parent 7753a4c1

drivers/block/drbd/drbd_actlog.c

+26 −3
Original line number Diff line number Diff line
@@ -991,6 +991,15 @@ int drbd_try_rs_begin_io(struct drbd_device *device, sector_t sector) 
	struct lc_element *e;

	struct bm_extent *bm_ext;

	int i;

	bool throttle = drbd_rs_should_slow_down(device, sector, true);



	/* If we need to throttle, a half-locked (only marked BME_NO_WRITES,

	 * not yet BME_LOCKED) extent needs to be kicked out explicitly if we

	 * need to throttle. There is at most one such half-locked extent,

	 * which is remembered in resync_wenr. */



	if (throttle && device->resync_wenr != enr)

		return -EAGAIN;



	spin_lock_irq(&device->al_lock);

	if (device->resync_wenr != LC_FREE && device->resync_wenr != enr) {
@@ -1014,8 +1023,10 @@ int drbd_try_rs_begin_io(struct drbd_device *device, sector_t sector) 
			D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));

			clear_bit(BME_NO_WRITES, &bm_ext->flags);

			device->resync_wenr = LC_FREE;

			if (lc_put(device->resync, &bm_ext->lce) == 0)

			if (lc_put(device->resync, &bm_ext->lce) == 0) {

				bm_ext->flags = 0;

				device->resync_locked--;

			}

			wake_up(&device->al_wait);

		} else {

			drbd_alert(device, "LOGIC BUG\n");
@@ -1077,8 +1088,20 @@ proceed: 
	return 0;



try_again:

	if (bm_ext)

	if (bm_ext) {

		if (throttle) {

			D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));

			D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));

			clear_bit(BME_NO_WRITES, &bm_ext->flags);

			device->resync_wenr = LC_FREE;

			if (lc_put(device->resync, &bm_ext->lce) == 0) {

				bm_ext->flags = 0;

				device->resync_locked--;

			}

			wake_up(&device->al_wait);

		} else

			device->resync_wenr = enr;

	}

	spin_unlock_irq(&device->al_lock);

	return -EAGAIN;

}

drivers/block/drbd/drbd_int.h

+3 −1
Original line number Diff line number Diff line
@@ -797,6 +797,7 @@ struct drbd_device { 
	unsigned int al_writ_cnt;

	unsigned int bm_writ_cnt;

	atomic_t ap_bio_cnt;	 /* Requests we need to complete */

	atomic_t ap_actlog_cnt;  /* Requests waiting for activity log */

	atomic_t ap_pending_cnt; /* AP data packets on the wire, ack expected */

	atomic_t rs_pending_cnt; /* RS request/data packets on the wire */

	atomic_t unacked_cnt;	 /* Need to send replies for */
@@ -1454,7 +1455,8 @@ extern void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req); 
extern int drbd_receiver(struct drbd_thread *thi);

extern int drbd_asender(struct drbd_thread *thi);

extern bool drbd_rs_c_min_rate_throttle(struct drbd_device *device);

extern bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector);

extern bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector,

		bool throttle_if_app_is_waiting);

extern int drbd_submit_peer_request(struct drbd_device *,

				    struct drbd_peer_request *, const unsigned,

				    const int);

drivers/block/drbd/drbd_main.c

+1 −0
Original line number Diff line number Diff line
@@ -1909,6 +1909,7 @@ void drbd_init_set_defaults(struct drbd_device *device) 
	drbd_set_defaults(device);



	atomic_set(&device->ap_bio_cnt, 0);

	atomic_set(&device->ap_actlog_cnt, 0);

	atomic_set(&device->ap_pending_cnt, 0);

	atomic_set(&device->rs_pending_cnt, 0);

	atomic_set(&device->unacked_cnt, 0);

drivers/block/drbd/drbd_proc.c

+3 −0
Original line number Diff line number Diff line
@@ -335,6 +335,9 @@ static int drbd_seq_show(struct seq_file *seq, void *v) 
			lc_seq_printf_stats(seq, device->act_log);

			put_ldev(device);

		}



		if (proc_details >= 2)

			seq_printf(seq, "\tblocked on activity log: %d\n", atomic_read(&device->ap_actlog_cnt));

	}

	rcu_read_unlock();

drivers/block/drbd/drbd_receiver.c

+12 −7
Original line number Diff line number Diff line
@@ -2417,13 +2417,14 @@ out_interrupted: 
 * The current sync rate used here uses only the most recent two step marks,

 * to have a short time average so we can react faster.

 */

bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector)

bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector,

		bool throttle_if_app_is_waiting)

{

	struct lc_element *tmp;

	bool throttle = true;

	bool throttle = drbd_rs_c_min_rate_throttle(device);



	if (!drbd_rs_c_min_rate_throttle(device))

		return false;

	if (!throttle || throttle_if_app_is_waiting)

		return throttle;



	spin_lock_irq(&device->al_lock);

	tmp = lc_find(device->resync, BM_SECT_TO_EXT(sector));
@@ -2431,7 +2432,8 @@ bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector) 
		struct bm_extent *bm_ext = lc_entry(tmp, struct bm_extent, lce);

		if (test_bit(BME_PRIORITY, &bm_ext->flags))

			throttle = false;

		/* Do not slow down if app IO is already waiting for this extent */

		/* Do not slow down if app IO is already waiting for this extent,

		 * and our progress is necessary for application IO to complete. */

	}

	spin_unlock_irq(&device->al_lock);
@@ -2456,7 +2458,9 @@ bool drbd_rs_c_min_rate_throttle(struct drbd_device *device) 
	curr_events = (int)part_stat_read(&disk->part0, sectors[0]) +

		      (int)part_stat_read(&disk->part0, sectors[1]) -

			atomic_read(&device->rs_sect_ev);

	if (!device->rs_last_events || curr_events - device->rs_last_events > 64) {



	if (atomic_read(&device->ap_actlog_cnt)

	    || !device->rs_last_events || curr_events - device->rs_last_events > 64) {

		unsigned long rs_left;

		int i;
@@ -2646,7 +2650,8 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet 
	 * we would also throttle its application reads.

	 * In that case, throttling is done on the SyncTarget only.

	 */

	if (device->state.peer != R_PRIMARY && drbd_rs_should_slow_down(device, sector))

	if (device->state.peer != R_PRIMARY

	&& drbd_rs_should_slow_down(device, sector, false))

		schedule_timeout_uninterruptible(HZ/10);

	if (drbd_rs_begin_io(device, sector))

		goto out_free_e;

CRA Git | Maintained and supported by SUSTech CRA and CCSE