Commit 2e60d768 authored by Benjamin Marzinski's avatar Benjamin Marzinski Committed by Steven Whitehouse
Browse files

GFS2: update freeze code to use freeze/thaw_super on all nodes 



The current gfs2 freezing code is considerably more complicated than it
should be because it doesn't use the vfs freezing code on any node except
the one that begins the freeze.  This is because it needs to acquire a
cluster glock before calling the vfs code to prevent a deadlock, and
without the new freeze_super and thaw_super hooks, that was impossible. To
deal with the issue, gfs2 had to do some hacky locking tricks to make sure
that a frozen node couldn't be holding on a lock it needed to do the
unfreeze ioctl.

This patch makes use of the new hooks to simply the gfs2 locking code. Now,
all the nodes in the cluster freeze and thaw in exactly the same way. Every
node in the cluster caches the freeze glock in the shared state.  The new
freeze_super hook allows the freezing node to grab this freeze glock in
the exclusive state without first calling the vfs freeze_super function.
All the nodes in the cluster see this lock change, and call the vfs
freeze_super function. The vfs locking code guarantees that the nodes can't
get stuck holding the glocks necessary to unfreeze the system.  To
unfreeze, the freezing node uses the new thaw_super hook to drop the freeze
glock. Again, all the nodes notice this, reacquire the glock in shared mode
and call the vfs thaw_super function.

Signed-off-by: default avatarBenjamin Marzinski <bmarzins@redhat.com>
Signed-off-by: default avatarSteven Whitehouse <swhiteho@redhat.com>
parent 48b6bca6

fs/gfs2/glops.c

+12 −14
Original line number Diff line number Diff line
@@ -28,6 +28,8 @@ 
#include "trans.h"

#include "dir.h"



struct workqueue_struct *gfs2_freeze_wq;



static void gfs2_ail_error(struct gfs2_glock *gl, const struct buffer_head *bh)

{

	fs_err(gl->gl_sbd, "AIL buffer %p: blocknr %llu state 0x%08lx mapping %p page state 0x%lx\n",
@@ -94,11 +96,8 @@ static void gfs2_ail_empty_gl(struct gfs2_glock *gl) 
         * on the stack */

	tr.tr_reserved = 1 + gfs2_struct2blk(sdp, tr.tr_revokes, sizeof(u64));

	tr.tr_ip = _RET_IP_;

	sb_start_intwrite(sdp->sd_vfs);

	if (gfs2_log_reserve(sdp, tr.tr_reserved) < 0) {

		sb_end_intwrite(sdp->sd_vfs);

	if (gfs2_log_reserve(sdp, tr.tr_reserved) < 0)

		return;

	}

	WARN_ON_ONCE(current->journal_info);

	current->journal_info = &tr;
@@ -469,20 +468,19 @@ static void inode_go_dump(struct seq_file *seq, const struct gfs2_glock *gl) 


static void freeze_go_sync(struct gfs2_glock *gl)

{

	int error = 0;

	struct gfs2_sbd *sdp = gl->gl_sbd;

	DEFINE_WAIT(wait);



	if (gl->gl_state == LM_ST_SHARED &&

	    test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) {

		atomic_set(&sdp->sd_log_freeze, 1);

		wake_up(&sdp->sd_logd_waitq);

		do {

			prepare_to_wait(&sdp->sd_log_frozen_wait, &wait,

					TASK_UNINTERRUPTIBLE);

			if (atomic_read(&sdp->sd_log_freeze))

				io_schedule();

		} while(atomic_read(&sdp->sd_log_freeze));

		finish_wait(&sdp->sd_log_frozen_wait, &wait);

		atomic_set(&sdp->sd_freeze_state, SFS_STARTING_FREEZE);

		error = freeze_super(sdp->sd_vfs);

		if (error) {

			printk(KERN_INFO "GFS2: couldn't freeze filesystem: %d\n", error);

			gfs2_assert_withdraw(sdp, 0);

		}

		queue_work(gfs2_freeze_wq, &sdp->sd_freeze_work);

		gfs2_log_flush(sdp, NULL, FREEZE_FLUSH);

	}

}

fs/gfs2/glops.h

+2 −0
Original line number Diff line number Diff line
@@ -12,6 +12,8 @@ 


#include "incore.h"



extern struct workqueue_struct *gfs2_freeze_wq;



extern const struct gfs2_glock_operations gfs2_meta_glops;

extern const struct gfs2_glock_operations gfs2_inode_glops;

extern const struct gfs2_glock_operations gfs2_rgrp_glops;

fs/gfs2/incore.h

+12 −6
Original line number Diff line number Diff line
@@ -588,6 +588,12 @@ enum { 
	SDF_SKIP_DLM_UNLOCK	= 8,

};



enum gfs2_freeze_state {

	SFS_UNFROZEN		= 0,

	SFS_STARTING_FREEZE	= 1,

	SFS_FROZEN		= 2,

};



#define GFS2_FSNAME_LEN		256



struct gfs2_inum_host {
@@ -685,6 +691,7 @@ struct gfs2_sbd { 
	struct gfs2_holder sd_live_gh;

	struct gfs2_glock *sd_rename_gl;

	struct gfs2_glock *sd_freeze_gl;

	struct work_struct sd_freeze_work;

	wait_queue_head_t sd_glock_wait;

	atomic_t sd_glock_disposal;

	struct completion sd_locking_init;
@@ -789,6 +796,9 @@ struct gfs2_sbd { 
	wait_queue_head_t sd_log_flush_wait;

	int sd_log_error;



	atomic_t sd_reserving_log;

	wait_queue_head_t sd_reserving_log_wait;



	unsigned int sd_log_flush_head;

	u64 sd_log_flush_wrapped;
@@ -798,12 +808,8 @@ struct gfs2_sbd { 


	/* For quiescing the filesystem */

	struct gfs2_holder sd_freeze_gh;

	struct gfs2_holder sd_freeze_root_gh;

	struct gfs2_holder sd_thaw_gh;

	atomic_t sd_log_freeze;

	atomic_t sd_frozen_root;

	wait_queue_head_t sd_frozen_root_wait;

	wait_queue_head_t sd_log_frozen_wait;

	atomic_t sd_freeze_state;

	struct mutex sd_freeze_mutex;



	char sd_fsname[GFS2_FSNAME_LEN];

	char sd_table_name[GFS2_FSNAME_LEN];

fs/gfs2/inode.c

+10 −30
Original line number Diff line number Diff line
@@ -1618,19 +1618,12 @@ int gfs2_permission(struct inode *inode, int mask) 
{

	struct gfs2_inode *ip;

	struct gfs2_holder i_gh;

	struct gfs2_sbd *sdp = GFS2_SB(inode);

	int error;

	int unlock = 0;

	int frozen_root = 0;





	ip = GFS2_I(inode);

	if (gfs2_glock_is_locked_by_me(ip->i_gl) == NULL) {

		if (unlikely(gfs2_glock_is_held_excl(sdp->sd_freeze_gl) &&

			     inode == sdp->sd_root_dir->d_inode &&

			     atomic_inc_not_zero(&sdp->sd_frozen_root)))

			frozen_root = 1;

		else {

		if (mask & MAY_NOT_BLOCK)

			return -ECHILD;

		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh);
@@ -1638,7 +1631,6 @@ int gfs2_permission(struct inode *inode, int mask) 
			return error;

		unlock = 1;

	}

	}



	if ((mask & MAY_WRITE) && IS_IMMUTABLE(inode))

		error = -EACCES;
@@ -1646,8 +1638,6 @@ int gfs2_permission(struct inode *inode, int mask) 
		error = generic_permission(inode, mask);

	if (unlock)

		gfs2_glock_dq_uninit(&i_gh);

	else if (frozen_root && atomic_dec_and_test(&sdp->sd_frozen_root))

		wake_up(&sdp->sd_frozen_root_wait);



	return error;

}
@@ -1820,29 +1810,19 @@ static int gfs2_getattr(struct vfsmount *mnt, struct dentry *dentry, 
	struct inode *inode = dentry->d_inode;

	struct gfs2_inode *ip = GFS2_I(inode);

	struct gfs2_holder gh;

	struct gfs2_sbd *sdp = GFS2_SB(inode);

	int error;

	int unlock = 0;

	int frozen_root = 0;



	if (gfs2_glock_is_locked_by_me(ip->i_gl) == NULL) {

		if (unlikely(gfs2_glock_is_held_excl(sdp->sd_freeze_gl) &&

			     inode == sdp->sd_root_dir->d_inode &&

			     atomic_inc_not_zero(&sdp->sd_frozen_root)))

			frozen_root = 1;

		else {

		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &gh);

		if (error)

			return error;

		unlock = 1;

	}

	}



	generic_fillattr(inode, stat);

	if (unlock)

		gfs2_glock_dq_uninit(&gh);

	else if (frozen_root && atomic_dec_and_test(&sdp->sd_frozen_root))

		wake_up(&sdp->sd_frozen_root_wait);



	return 0;

}

fs/gfs2/log.c

+21 −21
Original line number Diff line number Diff line
@@ -339,6 +339,7 @@ void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks) 


int gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks)

{

	int ret = 0;

	unsigned reserved_blks = 7 * (4096 / sdp->sd_vfs->s_blocksize);

	unsigned wanted = blks + reserved_blks;

	DEFINE_WAIT(wait);
@@ -362,9 +363,13 @@ int gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks) 
		} while(free_blocks <= wanted);

		finish_wait(&sdp->sd_log_waitq, &wait);

	}

	atomic_inc(&sdp->sd_reserving_log);

	if (atomic_cmpxchg(&sdp->sd_log_blks_free, free_blocks,

				free_blocks - blks) != free_blocks)

				free_blocks - blks) != free_blocks) {

		if (atomic_dec_and_test(&sdp->sd_reserving_log))

			wake_up(&sdp->sd_reserving_log_wait);

		goto retry;

	}

	trace_gfs2_log_blocks(sdp, -blks);



	/*
@@ -377,9 +382,11 @@ int gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks) 
	down_read(&sdp->sd_log_flush_lock);

	if (unlikely(!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags))) {

		gfs2_log_release(sdp, blks);

		return -EROFS;

		ret = -EROFS;

	}

	return 0;

	if (atomic_dec_and_test(&sdp->sd_reserving_log))

		wake_up(&sdp->sd_reserving_log_wait);

	return ret;

}



/**
@@ -652,9 +659,12 @@ static void log_write_header(struct gfs2_sbd *sdp, u32 flags) 
	u32 hash;

	int rw = WRITE_FLUSH_FUA | REQ_META;

	struct page *page = mempool_alloc(gfs2_page_pool, GFP_NOIO);

	enum gfs2_freeze_state state = atomic_read(&sdp->sd_freeze_state);

	lh = page_address(page);

	clear_page(lh);



	gfs2_assert_withdraw(sdp, (state != SFS_FROZEN));



	tail = current_tail(sdp);



	lh->lh_header.mh_magic = cpu_to_be32(GFS2_MAGIC);
@@ -695,6 +705,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, 
		    enum gfs2_flush_type type)

{

	struct gfs2_trans *tr;

	enum gfs2_freeze_state state = atomic_read(&sdp->sd_freeze_state);



	down_write(&sdp->sd_log_flush_lock);
@@ -713,8 +724,12 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, 
		INIT_LIST_HEAD(&tr->tr_ail1_list);

		INIT_LIST_HEAD(&tr->tr_ail2_list);

		tr->tr_first = sdp->sd_log_flush_head;

		if (unlikely (state == SFS_FROZEN))

			gfs2_assert_withdraw(sdp, !tr->tr_num_buf_new && !tr->tr_num_databuf_new);

	}



	if (unlikely(state == SFS_FROZEN))

		gfs2_assert_withdraw(sdp, !sdp->sd_log_num_revoke);

	gfs2_assert_withdraw(sdp,

			sdp->sd_log_num_revoke == sdp->sd_log_commited_revoke);
@@ -745,8 +760,6 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, 
	spin_unlock(&sdp->sd_ail_lock);

	gfs2_log_unlock(sdp);



	if (atomic_read(&sdp->sd_log_freeze))

		type = FREEZE_FLUSH;

	if (type != NORMAL_FLUSH) {

		if (!sdp->sd_log_idle) {

			for (;;) {
@@ -763,21 +776,8 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, 
		}

		if (type == SHUTDOWN_FLUSH || type == FREEZE_FLUSH)

			gfs2_log_shutdown(sdp);

		if (type == FREEZE_FLUSH) {

			int error;



			atomic_set(&sdp->sd_log_freeze, 0);

			wake_up(&sdp->sd_log_frozen_wait);

			error = gfs2_glock_nq_init(sdp->sd_freeze_gl,

						   LM_ST_SHARED, 0,

						   &sdp->sd_thaw_gh);

			if (error) {

				printk(KERN_INFO "GFS2: couln't get freeze lock : %d\n", error);

				gfs2_assert_withdraw(sdp, 0);

			}

			else

				gfs2_glock_dq_uninit(&sdp->sd_thaw_gh);

		}

		if (type == FREEZE_FLUSH)

			atomic_set(&sdp->sd_freeze_state, SFS_FROZEN);

	}



	trace_gfs2_log_flush(sdp, 0);
@@ -888,7 +888,7 @@ void gfs2_log_shutdown(struct gfs2_sbd *sdp) 


static inline int gfs2_jrnl_flush_reqd(struct gfs2_sbd *sdp)

{

	return (atomic_read(&sdp->sd_log_pinned) >= atomic_read(&sdp->sd_log_thresh1) || atomic_read(&sdp->sd_log_freeze));

	return (atomic_read(&sdp->sd_log_pinned) >= atomic_read(&sdp->sd_log_thresh1));

}



static inline int gfs2_ail_flush_reqd(struct gfs2_sbd *sdp)