Commit 82dcabad authored by Ilya Dryomov's avatar Ilya Dryomov
Browse files

libceph: revamp subs code, switch to SUBSCRIBE2 protocol 



It is currently hard-coded in the mon_client that mdsmap and monmap
subs are continuous, while osdmap sub is always "onetime".  To better
handle full clusters/pools in the osd_client, we need to be able to
issue continuous osdmap subs.  Revamp subs code to allow us to specify
for each sub whether it should be continuous or not.

Although not strictly required for the above, switch to SUBSCRIBE2
protocol while at it, eliminating the ambiguity between a request for
"every map since X" and a request for "just the latest" when we don't
have a map yet (i.e. have epoch 0).  SUBSCRIBE2 feature bit is now
required - it's been supported since pre-argonaut (2010).

Move "got mdsmap" call to the end of ceph_mdsc_handle_map() - calling
in before we validate the epoch and successfully install the new map
can mess up mon_client sub state.

Signed-off-by: default avatarIlya Dryomov <idryomov@gmail.com>
parent 0f9af169

fs/ceph/mds_client.c

+2 −1
Original line number Diff line number Diff line
@@ -3764,7 +3764,6 @@ void ceph_mdsc_handle_map(struct ceph_mds_client *mdsc, struct ceph_msg *msg) 
	dout("handle_map epoch %u len %d\n", epoch, (int)maplen);



	/* do we need it? */

	ceph_monc_got_mdsmap(&mdsc->fsc->client->monc, epoch);

	mutex_lock(&mdsc->mutex);

	if (mdsc->mdsmap && epoch <= mdsc->mdsmap->m_epoch) {

		dout("handle_map epoch %u <= our %u\n",
@@ -3791,6 +3790,8 @@ void ceph_mdsc_handle_map(struct ceph_mds_client *mdsc, struct ceph_msg *msg) 
	mdsc->fsc->sb->s_maxbytes = mdsc->mdsmap->m_max_file_size;



	__wake_requests(mdsc, &mdsc->waiting_for_map);

	ceph_monc_got_map(&mdsc->fsc->client->monc, CEPH_SUB_MDSMAP,

			  mdsc->mdsmap->m_epoch);



	mutex_unlock(&mdsc->mutex);

	schedule_delayed(mdsc);

fs/ceph/super.c

+1 −1
Original line number Diff line number Diff line
@@ -530,7 +530,7 @@ static struct ceph_fs_client *create_fs_client(struct ceph_mount_options *fsopt, 
		goto fail;

	}

	fsc->client->extra_mon_dispatch = extra_mon_dispatch;

	fsc->client->monc.want_mdsmap = 1;

	ceph_monc_want_map(&fsc->client->monc, CEPH_SUB_MDSMAP, 0, true);



	fsc->mount_options = fsopt;

include/linux/ceph/ceph_features.h

+2 −0
Original line number Diff line number Diff line
@@ -105,6 +105,7 @@ static inline u64 ceph_sanitize_features(u64 features) 
 */

#define CEPH_FEATURES_SUPPORTED_DEFAULT		\

	(CEPH_FEATURE_NOSRCADDR |		\

	 CEPH_FEATURE_SUBSCRIBE2 |		\

	 CEPH_FEATURE_RECONNECT_SEQ |		\

	 CEPH_FEATURE_PGID64 |			\

	 CEPH_FEATURE_PGPOOL3 |			\
@@ -127,6 +128,7 @@ static inline u64 ceph_sanitize_features(u64 features) 


#define CEPH_FEATURES_REQUIRED_DEFAULT   \

	(CEPH_FEATURE_NOSRCADDR |	 \

	 CEPH_FEATURE_SUBSCRIBE2 |	 \

	 CEPH_FEATURE_RECONNECT_SEQ |	 \

	 CEPH_FEATURE_PGID64 |		 \

	 CEPH_FEATURE_PGPOOL3 |		 \

include/linux/ceph/ceph_fs.h

+2 −2
Original line number Diff line number Diff line
@@ -198,8 +198,8 @@ struct ceph_client_mount { 
#define CEPH_SUBSCRIBE_ONETIME    1  /* i want only 1 update after have */



struct ceph_mon_subscribe_item {

	__le64 have_version;    __le64 have;

	__u8 onetime;

	__le64 start;

	__u8 flags;

} __attribute__ ((packed));



struct ceph_mon_subscribe_ack {

include/linux/ceph/mon_client.h

+20 −8
Original line number Diff line number Diff line
@@ -68,7 +68,8 @@ struct ceph_mon_client { 


	bool hunting;

	int cur_mon;                       /* last monitor i contacted */

	unsigned long sub_sent, sub_renew_after;

	unsigned long sub_renew_after;

	unsigned long sub_renew_sent;

	struct ceph_connection con;



	/* pending generic requests */
@@ -76,10 +77,12 @@ struct ceph_mon_client { 
	int num_generic_requests;

	u64 last_tid;



	/* mds/osd map */

	int want_mdsmap;

	int want_next_osdmap; /* 1 = want, 2 = want+asked */

	u32 have_osdmap, have_mdsmap;

	/* subs, indexed with CEPH_SUB_* */

	struct {

		struct ceph_mon_subscribe_item item;

		bool want;

		u32 have; /* epoch */

	} subs[3];



#ifdef CONFIG_DEBUG_FS

	struct dentry *debugfs_file;
@@ -93,14 +96,23 @@ extern int ceph_monmap_contains(struct ceph_monmap *m, 
extern int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl);

extern void ceph_monc_stop(struct ceph_mon_client *monc);



enum {

	CEPH_SUB_MDSMAP = 0,

	CEPH_SUB_MONMAP,

	CEPH_SUB_OSDMAP,

};



extern const char *ceph_sub_str[];



/*

 * The model here is to indicate that we need a new map of at least

 * epoch @want, and also call in when we receive a map.  We will

 * epoch @epoch, and also call in when we receive a map.  We will

 * periodically rerequest the map from the monitor cluster until we

 * get what we want.

 */

extern int ceph_monc_got_mdsmap(struct ceph_mon_client *monc, u32 have);

extern int ceph_monc_got_osdmap(struct ceph_mon_client *monc, u32 have);

bool ceph_monc_want_map(struct ceph_mon_client *monc, int sub, u32 epoch,

			bool continuous);

void ceph_monc_got_map(struct ceph_mon_client *monc, int sub, u32 epoch);



extern void ceph_monc_request_next_osdmap(struct ceph_mon_client *monc);

extern int ceph_monc_wait_osdmap(struct ceph_mon_client *monc, u32 epoch,

CRA Git | Maintained and supported by SUSTech CRA and CCSE