Commit 5a22db89 authored by Lars Ellenberg's avatar Lars Ellenberg Committed by Philipp Reisner
Browse files

drbd: serialize sending of resync uuid with pending w_send_oos 



To improve the latency of IO requests during bitmap exchange,
we recently allowed writes while waiting for the bitmap, sending "set
out-of-sync" information packets for any newly dirtied bits.

We have to make sure that the new resync-uuid does not overtake
these "set oos" packets. Once the resync-uuid is received, the
sync target starts the resync process, and expects the bitmap to
only be cleared, not re-set.

If we use this protocol extension, we queue the generation and sending
of the resync-uuid on the worker, which naturally serializes with all
previously queued packets.

Signed-off-by: default avatarPhilipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: default avatarLars Ellenberg <lars.ellenberg@linbit.com>
parent f735e363

drivers/block/drbd/drbd_int.h

+1 −1
Original line number Diff line number Diff line
@@ -1204,7 +1204,7 @@ extern int drbd_send(struct drbd_conf *mdev, struct socket *sock, 
extern int drbd_send_protocol(struct drbd_conf *mdev);

extern int drbd_send_uuids(struct drbd_conf *mdev);

extern int drbd_send_uuids_skip_initial_sync(struct drbd_conf *mdev);

extern int drbd_send_sync_uuid(struct drbd_conf *mdev, u64 val);

extern int drbd_gen_and_send_sync_uuid(struct drbd_conf *mdev);

extern int drbd_send_sizes(struct drbd_conf *mdev, int trigger_reply, enum dds_flags flags);

extern int _drbd_send_state(struct drbd_conf *mdev);

extern int drbd_send_state(struct drbd_conf *mdev);

drivers/block/drbd/drbd_main.c

+19 −3
Original line number Diff line number Diff line
@@ -1387,6 +1387,17 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os, 
		spin_unlock_irq(&mdev->req_lock);

	}



	/* Became sync source.  With protocol >= 96, we still need to send out

	 * the sync uuid now. Need to do that before any drbd_send_state, or

	 * the other side may go "paused sync" before receiving the sync uuids,

	 * which is unexpected. */

	if ((os.conn != C_SYNC_SOURCE && os.conn != C_PAUSED_SYNC_S) &&

	    (ns.conn == C_SYNC_SOURCE || ns.conn == C_PAUSED_SYNC_S) &&

	    mdev->agreed_pro_version >= 96 && get_ldev(mdev)) {

		drbd_gen_and_send_sync_uuid(mdev);

		put_ldev(mdev);

	}



	/* Do not change the order of the if above and the two below... */

	if (os.pdsk == D_DISKLESS && ns.pdsk > D_DISKLESS) {      /* attach on the peer */

		drbd_send_uuids(mdev);
@@ -1980,12 +1991,17 @@ int drbd_send_uuids_skip_initial_sync(struct drbd_conf *mdev) 
	return _drbd_send_uuids(mdev, 8);

}





int drbd_send_sync_uuid(struct drbd_conf *mdev, u64 val)

int drbd_gen_and_send_sync_uuid(struct drbd_conf *mdev)

{

	struct p_rs_uuid p;

	u64 uuid;



	D_ASSERT(mdev->state.disk == D_UP_TO_DATE);



	p.uuid = cpu_to_be64(val);

	get_random_bytes(&uuid, sizeof(u64));

	drbd_uuid_set(mdev, UI_BITMAP, uuid);

	drbd_md_sync(mdev);

	p.uuid = cpu_to_be64(uuid);



	return drbd_send_cmd(mdev, USE_DATA_SOCKET, P_SYNC_UUID,

			     (struct p_header80 *)&p, sizeof(p));

drivers/block/drbd/drbd_worker.c

+14 −13
Original line number Diff line number Diff line
@@ -1516,18 +1516,6 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side) 
		return;

	}



	if (side == C_SYNC_TARGET) {

		mdev->bm_resync_fo = 0;

	} else /* side == C_SYNC_SOURCE */ {

		u64 uuid;



		get_random_bytes(&uuid, sizeof(u64));

		drbd_uuid_set(mdev, UI_BITMAP, uuid);

		drbd_send_sync_uuid(mdev, uuid);



		D_ASSERT(mdev->state.disk == D_UP_TO_DATE);

	}



	write_lock_irq(&global_state_lock);

	ns = mdev->state;
@@ -1565,7 +1553,19 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side) 
		_drbd_pause_after(mdev);

	}

	write_unlock_irq(&global_state_lock);

	put_ldev(mdev);



	if (side == C_SYNC_TARGET)

		mdev->bm_resync_fo = 0;



	/* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid

	 * with w_send_oos, or the sync target will get confused as to

	 * how much bits to resync.  We cannot do that always, because for an

	 * empty resync and protocol < 95, we need to do it here, as we call

	 * drbd_resync_finished from here in that case.

	 * We drbd_gen_and_send_sync_uuid here for protocol < 96,

	 * and from after_state_ch otherwise. */

	if (side == C_SYNC_SOURCE && mdev->agreed_pro_version < 96)

		drbd_gen_and_send_sync_uuid(mdev);



	if (r == SS_SUCCESS) {

		dev_info(DEV, "Began resync as %s (will sync %lu KB [%lu bits set]).\n",
@@ -1601,6 +1601,7 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side) 


		drbd_md_sync(mdev);

	}

	put_ldev(mdev);

	drbd_state_unlock(mdev);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE