Commit e0c2a9aa authored by David Teigland's avatar David Teigland Committed by Steven Whitehouse
Browse files

GFS2: dlm based recovery coordination 



This new method of managing recovery is an alternative to
the previous approach of using the userland gfs_controld.

- use dlm slot numbers to assign journal id's
- use dlm recovery callbacks to initiate journal recovery
- use a dlm lock to determine the first node to mount fs
- use a dlm lock to track journals that need recovery

Signed-off-by: default avatarDavid Teigland <teigland@redhat.com>
Signed-off-by: default avatarSteven Whitehouse <swhiteho@redhat.com>
parent e343a895

fs/gfs2/glock.c

+1 −1
Original line number Diff line number Diff line
@@ -1353,7 +1353,7 @@ void gfs2_glock_complete(struct gfs2_glock *gl, int ret) 
	spin_lock(&gl->gl_spin);

	gl->gl_reply = ret;



	if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_flags))) {

	if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {

		if (gfs2_should_freeze(gl)) {

			set_bit(GLF_FROZEN, &gl->gl_flags);

			spin_unlock(&gl->gl_spin);

fs/gfs2/glock.h

+5 −2
Original line number Diff line number Diff line
@@ -121,7 +121,10 @@ enum { 


struct lm_lockops {

	const char *lm_proto_name;

	int (*lm_mount) (struct gfs2_sbd *sdp, const char *fsname);

	int (*lm_mount) (struct gfs2_sbd *sdp, const char *table);

	void (*lm_first_done) (struct gfs2_sbd *sdp);

	void (*lm_recovery_result) (struct gfs2_sbd *sdp, unsigned int jid,

				    unsigned int result);

	void (*lm_unmount) (struct gfs2_sbd *sdp);

	void (*lm_withdraw) (struct gfs2_sbd *sdp);

	void (*lm_put_lock) (struct gfs2_glock *gl);

fs/gfs2/incore.h

+54 −4
Original line number Diff line number Diff line
@@ -139,8 +139,45 @@ struct gfs2_bufdata { 
#define GDLM_STRNAME_BYTES	25

#define GDLM_LVB_SIZE		32



/*

 * ls_recover_flags:

 *

 * DFL_BLOCK_LOCKS: dlm is in recovery and will grant locks that had been

 * held by failed nodes whose journals need recovery.  Those locks should

 * only be used for journal recovery until the journal recovery is done.

 * This is set by the dlm recover_prep callback and cleared by the

 * gfs2_control thread when journal recovery is complete.  To avoid

 * races between recover_prep setting and gfs2_control clearing, recover_spin

 * is held while changing this bit and reading/writing recover_block

 * and recover_start.

 *

 * DFL_NO_DLM_OPS: dlm lockspace ops/callbacks are not being used.

 *

 * DFL_FIRST_MOUNT: this node is the first to mount this fs and is doing

 * recovery of all journals before allowing other nodes to mount the fs.

 * This is cleared when FIRST_MOUNT_DONE is set.

 *

 * DFL_FIRST_MOUNT_DONE: this node was the first mounter, and has finished

 * recovery of all journals, and now allows other nodes to mount the fs.

 *

 * DFL_MOUNT_DONE: gdlm_mount has completed successfully and cleared

 * BLOCK_LOCKS for the first time.  The gfs2_control thread should now

 * control clearing BLOCK_LOCKS for further recoveries.

 *

 * DFL_UNMOUNT: gdlm_unmount sets to keep sdp off gfs2_control_wq.

 *

 * DFL_DLM_RECOVERY: set while dlm is in recovery, between recover_prep()

 * and recover_done(), i.e. set while recover_block == recover_start.

 */



enum {

	DFL_BLOCK_LOCKS		= 0,

	DFL_NO_DLM_OPS		= 1,

	DFL_FIRST_MOUNT		= 2,

	DFL_FIRST_MOUNT_DONE	= 3,

	DFL_MOUNT_DONE		= 4,

	DFL_UNMOUNT		= 5,

	DFL_DLM_RECOVERY	= 6,

};



struct lm_lockname {
@@ -499,14 +536,26 @@ struct gfs2_sb_host { 
struct lm_lockstruct {

	int ls_jid;

	unsigned int ls_first;

	unsigned int ls_first_done;

	unsigned int ls_nodir;

	const struct lm_lockops *ls_ops;

	unsigned long ls_flags;

	dlm_lockspace_t *ls_dlm;



	int ls_recover_jid_done;

	int ls_recover_jid_status;

	int ls_recover_jid_done;   /* These two are deprecated, */

	int ls_recover_jid_status; /* used previously by gfs_controld */



	struct dlm_lksb ls_mounted_lksb; /* mounted_lock */

	struct dlm_lksb ls_control_lksb; /* control_lock */

	char ls_control_lvb[GDLM_LVB_SIZE]; /* control_lock lvb */

	struct completion ls_sync_wait; /* {control,mounted}_{lock,unlock} */



	spinlock_t ls_recover_spin; /* protects following fields */

	unsigned long ls_recover_flags; /* DFL_ */

	uint32_t ls_recover_mount; /* gen in first recover_done cb */

	uint32_t ls_recover_start; /* gen in last recover_done cb */

	uint32_t ls_recover_block; /* copy recover_start in last recover_prep */

	uint32_t ls_recover_size; /* size of recover_submit, recover_result */

	uint32_t *ls_recover_submit; /* gen in last recover_slot cb per jid */

	uint32_t *ls_recover_result; /* result of last jid recovery */

};



struct gfs2_sbd {
@@ -544,6 +593,7 @@ struct gfs2_sbd { 
	wait_queue_head_t sd_glock_wait;

	atomic_t sd_glock_disposal;

	struct completion sd_locking_init;

	struct delayed_work sd_control_work;



	/* Inode Stuff */

fs/gfs2/lock_dlm.c

+983 −10

File changed.

Preview size limit exceeded, changes collapsed.

fs/gfs2/main.c

+10 −0
Original line number Diff line number Diff line
@@ -28,6 +28,8 @@ 
#include "recovery.h"

#include "dir.h"



struct workqueue_struct *gfs2_control_wq;



static struct shrinker qd_shrinker = {

	.shrink = gfs2_shrink_qd_memory,

	.seeks = DEFAULT_SEEKS,
@@ -146,12 +148,19 @@ static int __init init_gfs2_fs(void) 
	if (!gfs_recovery_wq)

		goto fail_wq;



	gfs2_control_wq = alloc_workqueue("gfs2_control",

			       WQ_NON_REENTRANT | WQ_UNBOUND | WQ_FREEZABLE, 0);

	if (!gfs2_control_wq)

		goto fail_control;



	gfs2_register_debugfs();



	printk("GFS2 installed\n");



	return 0;



fail_control:

	destroy_workqueue(gfs_recovery_wq);

fail_wq:

	unregister_filesystem(&gfs2meta_fs_type);

fail_unregister:
@@ -195,6 +204,7 @@ static void __exit exit_gfs2_fs(void) 
	unregister_filesystem(&gfs2_fs_type);

	unregister_filesystem(&gfs2meta_fs_type);

	destroy_workqueue(gfs_recovery_wq);

	destroy_workqueue(gfs2_control_wq);



	rcu_barrier();

CRA Git | Maintained and supported by SUSTech CRA and CCSE