Commit e56dc9e2 authored by J. Bruce Fields's avatar J. Bruce Fields
Browse files

nfsd: remove fault injection code 



It was an interesting idea but nobody seems to be using it, it's buggy
at this point, and nfs4state.c is already complicated enough without it.
The new nfsd/clients/ code provides some of the same functionality, and
could probably do more if desired.

This feature has been deprecated since 9d60d931 ("Deprecate nfsd
fault injection").

Signed-off-by: default avatarJ. Bruce Fields <bfields@redhat.com>
parent 856deb86

Documentation/admin-guide/nfs/fault_injection.rst

deleted100644 → 0
+0 −70
Original line number Diff line number Diff line 
===================

NFS Fault Injection

===================



Fault injection is a method for forcing errors that may not normally occur, or

may be difficult to reproduce.  Forcing these errors in a controlled environment

can help the developer find and fix bugs before their code is shipped in a

production system.  Injecting an error on the Linux NFS server will allow us to

observe how the client reacts and if it manages to recover its state correctly.



NFSD_FAULT_INJECTION must be selected when configuring the kernel to use this

feature.





Using Fault Injection

=====================

On the client, mount the fault injection server through NFS v4.0+ and do some

work over NFS (open files, take locks, ...).



On the server, mount the debugfs filesystem to <debug_dir> and ls

<debug_dir>/nfsd.  This will show a list of files that will be used for

injecting faults on the NFS server.  As root, write a number n to the file

corresponding to the action you want the server to take.  The server will then

process the first n items it finds.  So if you want to forget 5 locks, echo '5'

to <debug_dir>/nfsd/forget_locks.  A value of 0 will tell the server to forget

all corresponding items.  A log message will be created containing the number

of items forgotten (check dmesg).



Go back to work on the client and check if the client recovered from the error

correctly.





Available Faults

================

forget_clients:

     The NFS server keeps a list of clients that have placed a mount call.  If

     this list is cleared, the server will have no knowledge of who the client

     is, forcing the client to reauthenticate with the server.



forget_openowners:

     The NFS server keeps a list of what files are currently opened and who

     they were opened by.  Clearing this list will force the client to reopen

     its files.



forget_locks:

     The NFS server keeps a list of what files are currently locked in the VFS.

     Clearing this list will force the client to reclaim its locks (files are

     unlocked through the VFS as they are cleared from this list).



forget_delegations:

     A delegation is used to assure the client that a file, or part of a file,

     has not changed since the delegation was awarded.  Clearing this list will

     force the client to reacquire its delegation before accessing the file

     again.



recall_delegations:

     Delegations can be recalled by the server when another client attempts to

     access a file.  This test will notify the client that its delegation has

     been revoked, forcing the client to reacquire the delegation before using

     the file again.





tools/nfs/inject_faults.sh script

=================================

This script has been created to ease the fault injection process.  This script

will detect the mounted debugfs directory and write to the files located there

based on the arguments passed by the user.  For example, running

`inject_faults.sh forget_locks 1` as root will instruct the server to forget

one lock.  Running `inject_faults forget_locks` will instruct the server to

forgetall locks.

Documentation/admin-guide/nfs/index.rst

+0 −1
Original line number Diff line number Diff line
@@ -12,4 +12,3 @@ NFS 
    nfs-idmapper

    pnfs-block-server

    pnfs-scsi-server
 
    fault_injection

fs/nfsd/Kconfig

+0 −10
Original line number Diff line number Diff line
@@ -156,13 +156,3 @@ config NFSD_V4_SECURITY_LABEL 


	If you do not wish to enable fine-grained security labels SELinux or

	Smack policies on NFSv4 files, say N.
 


config NFSD_FAULT_INJECTION

	bool "NFS server manual fault injection"

	depends on NFSD_V4 && DEBUG_KERNEL && DEBUG_FS && BROKEN

	help

	  This option enables support for manually injecting faults

	  into the NFS server.  This is intended to be used for

	  testing error recovery on the NFS client.



	  If unsure, say N.

fs/nfsd/Makefile

+0 −1
Original line number Diff line number Diff line
@@ -13,7 +13,6 @@ nfsd-y += trace.o 
nfsd-y 			+= nfssvc.o nfsctl.o nfsproc.o nfsfh.o vfs.o \

			   export.o auth.o lockd.o nfscache.o nfsxdr.o \

			   stats.o filecache.o

nfsd-$(CONFIG_NFSD_FAULT_INJECTION) += fault_inject.o

nfsd-$(CONFIG_NFSD_V2_ACL) += nfs2acl.o

nfsd-$(CONFIG_NFSD_V3)	+= nfs3proc.o nfs3xdr.o

nfsd-$(CONFIG_NFSD_V3_ACL) += nfs3acl.o

fs/nfsd/nfs4state.c

+0 −593
Original line number Diff line number Diff line
@@ -7253,599 +7253,6 @@ nfs4_check_open_reclaim(clientid_t *clid, 
	return nfs_ok;

}



#ifdef CONFIG_NFSD_FAULT_INJECTION

static inline void

put_client(struct nfs4_client *clp)

{

	atomic_dec(&clp->cl_rpc_users);

}



static struct nfs4_client *

nfsd_find_client(struct sockaddr_storage *addr, size_t addr_size)

{

	struct nfs4_client *clp;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

					  nfsd_net_id);



	if (!nfsd_netns_ready(nn))

		return NULL;



	list_for_each_entry(clp, &nn->client_lru, cl_lru) {

		if (memcmp(&clp->cl_addr, addr, addr_size) == 0)

			return clp;

	}

	return NULL;

}



u64

nfsd_inject_print_clients(void)

{

	struct nfs4_client *clp;

	u64 count = 0;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

					  nfsd_net_id);

	char buf[INET6_ADDRSTRLEN];



	if (!nfsd_netns_ready(nn))

		return 0;



	spin_lock(&nn->client_lock);

	list_for_each_entry(clp, &nn->client_lru, cl_lru) {

		rpc_ntop((struct sockaddr *)&clp->cl_addr, buf, sizeof(buf));

		pr_info("NFS Client: %s\n", buf);

		++count;

	}

	spin_unlock(&nn->client_lock);



	return count;

}



u64

nfsd_inject_forget_client(struct sockaddr_storage *addr, size_t addr_size)

{

	u64 count = 0;

	struct nfs4_client *clp;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

					  nfsd_net_id);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	clp = nfsd_find_client(addr, addr_size);

	if (clp) {

		if (mark_client_expired_locked(clp) == nfs_ok)

			++count;

		else

			clp = NULL;

	}

	spin_unlock(&nn->client_lock);



	if (clp)

		expire_client(clp);



	return count;

}



u64

nfsd_inject_forget_clients(u64 max)

{

	u64 count = 0;

	struct nfs4_client *clp, *next;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	LIST_HEAD(reaplist);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) {

		if (mark_client_expired_locked(clp) == nfs_ok) {

			list_add(&clp->cl_lru, &reaplist);

			if (max != 0 && ++count >= max)

				break;

		}

	}

	spin_unlock(&nn->client_lock);



	list_for_each_entry_safe(clp, next, &reaplist, cl_lru)

		expire_client(clp);



	return count;

}



static void nfsd_print_count(struct nfs4_client *clp, unsigned int count,

			     const char *type)

{

	char buf[INET6_ADDRSTRLEN];

	rpc_ntop((struct sockaddr *)&clp->cl_addr, buf, sizeof(buf));

	printk(KERN_INFO "NFS Client: %s has %u %s\n", buf, count, type);

}



static void

nfsd_inject_add_lock_to_list(struct nfs4_ol_stateid *lst,

			     struct list_head *collect)

{

	struct nfs4_client *clp = lst->st_stid.sc_client;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

					  nfsd_net_id);



	if (!collect)

		return;



	lockdep_assert_held(&nn->client_lock);

	atomic_inc(&clp->cl_rpc_users);

	list_add(&lst->st_locks, collect);

}



static u64 nfsd_foreach_client_lock(struct nfs4_client *clp, u64 max,

				    struct list_head *collect,

				    bool (*func)(struct nfs4_ol_stateid *))

{

	struct nfs4_openowner *oop;

	struct nfs4_ol_stateid *stp, *st_next;

	struct nfs4_ol_stateid *lst, *lst_next;

	u64 count = 0;



	spin_lock(&clp->cl_lock);

	list_for_each_entry(oop, &clp->cl_openowners, oo_perclient) {

		list_for_each_entry_safe(stp, st_next,

				&oop->oo_owner.so_stateids, st_perstateowner) {

			list_for_each_entry_safe(lst, lst_next,

					&stp->st_locks, st_locks) {

				if (func) {

					if (func(lst))

						nfsd_inject_add_lock_to_list(lst,

									collect);

				}

				++count;

				/*

				 * Despite the fact that these functions deal

				 * with 64-bit integers for "count", we must

				 * ensure that it doesn't blow up the

				 * clp->cl_rpc_users. Throw a warning if we

				 * start to approach INT_MAX here.

				 */

				WARN_ON_ONCE(count == (INT_MAX / 2));

				if (count == max)

					goto out;

			}

		}

	}

out:

	spin_unlock(&clp->cl_lock);



	return count;

}



static u64

nfsd_collect_client_locks(struct nfs4_client *clp, struct list_head *collect,

			  u64 max)

{

	return nfsd_foreach_client_lock(clp, max, collect, unhash_lock_stateid);

}



static u64

nfsd_print_client_locks(struct nfs4_client *clp)

{

	u64 count = nfsd_foreach_client_lock(clp, 0, NULL, NULL);

	nfsd_print_count(clp, count, "locked files");

	return count;

}



u64

nfsd_inject_print_locks(void)

{

	struct nfs4_client *clp;

	u64 count = 0;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);



	if (!nfsd_netns_ready(nn))

		return 0;



	spin_lock(&nn->client_lock);

	list_for_each_entry(clp, &nn->client_lru, cl_lru)

		count += nfsd_print_client_locks(clp);

	spin_unlock(&nn->client_lock);



	return count;

}



static void

nfsd_reap_locks(struct list_head *reaplist)

{

	struct nfs4_client *clp;

	struct nfs4_ol_stateid *stp, *next;



	list_for_each_entry_safe(stp, next, reaplist, st_locks) {

		list_del_init(&stp->st_locks);

		clp = stp->st_stid.sc_client;

		nfs4_put_stid(&stp->st_stid);

		put_client(clp);

	}

}



u64

nfsd_inject_forget_client_locks(struct sockaddr_storage *addr, size_t addr_size)

{

	unsigned int count = 0;

	struct nfs4_client *clp;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	LIST_HEAD(reaplist);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	clp = nfsd_find_client(addr, addr_size);

	if (clp)

		count = nfsd_collect_client_locks(clp, &reaplist, 0);

	spin_unlock(&nn->client_lock);

	nfsd_reap_locks(&reaplist);

	return count;

}



u64

nfsd_inject_forget_locks(u64 max)

{

	u64 count = 0;

	struct nfs4_client *clp;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	LIST_HEAD(reaplist);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	list_for_each_entry(clp, &nn->client_lru, cl_lru) {

		count += nfsd_collect_client_locks(clp, &reaplist, max - count);

		if (max != 0 && count >= max)

			break;

	}

	spin_unlock(&nn->client_lock);

	nfsd_reap_locks(&reaplist);

	return count;

}



static u64

nfsd_foreach_client_openowner(struct nfs4_client *clp, u64 max,

			      struct list_head *collect,

			      void (*func)(struct nfs4_openowner *))

{

	struct nfs4_openowner *oop, *next;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	u64 count = 0;



	lockdep_assert_held(&nn->client_lock);



	spin_lock(&clp->cl_lock);

	list_for_each_entry_safe(oop, next, &clp->cl_openowners, oo_perclient) {

		if (func) {

			func(oop);

			if (collect) {

				atomic_inc(&clp->cl_rpc_users);

				list_add(&oop->oo_perclient, collect);

			}

		}

		++count;

		/*

		 * Despite the fact that these functions deal with

		 * 64-bit integers for "count", we must ensure that

		 * it doesn't blow up the clp->cl_rpc_users. Throw a

		 * warning if we start to approach INT_MAX here.

		 */

		WARN_ON_ONCE(count == (INT_MAX / 2));

		if (count == max)

			break;

	}

	spin_unlock(&clp->cl_lock);



	return count;

}



static u64

nfsd_print_client_openowners(struct nfs4_client *clp)

{

	u64 count = nfsd_foreach_client_openowner(clp, 0, NULL, NULL);



	nfsd_print_count(clp, count, "openowners");

	return count;

}



static u64

nfsd_collect_client_openowners(struct nfs4_client *clp,

			       struct list_head *collect, u64 max)

{

	return nfsd_foreach_client_openowner(clp, max, collect,

						unhash_openowner_locked);

}



u64

nfsd_inject_print_openowners(void)

{

	struct nfs4_client *clp;

	u64 count = 0;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);



	if (!nfsd_netns_ready(nn))

		return 0;



	spin_lock(&nn->client_lock);

	list_for_each_entry(clp, &nn->client_lru, cl_lru)

		count += nfsd_print_client_openowners(clp);

	spin_unlock(&nn->client_lock);



	return count;

}



static void

nfsd_reap_openowners(struct list_head *reaplist)

{

	struct nfs4_client *clp;

	struct nfs4_openowner *oop, *next;



	list_for_each_entry_safe(oop, next, reaplist, oo_perclient) {

		list_del_init(&oop->oo_perclient);

		clp = oop->oo_owner.so_client;

		release_openowner(oop);

		put_client(clp);

	}

}



u64

nfsd_inject_forget_client_openowners(struct sockaddr_storage *addr,

				     size_t addr_size)

{

	unsigned int count = 0;

	struct nfs4_client *clp;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	LIST_HEAD(reaplist);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	clp = nfsd_find_client(addr, addr_size);

	if (clp)

		count = nfsd_collect_client_openowners(clp, &reaplist, 0);

	spin_unlock(&nn->client_lock);

	nfsd_reap_openowners(&reaplist);

	return count;

}



u64

nfsd_inject_forget_openowners(u64 max)

{

	u64 count = 0;

	struct nfs4_client *clp;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	LIST_HEAD(reaplist);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	list_for_each_entry(clp, &nn->client_lru, cl_lru) {

		count += nfsd_collect_client_openowners(clp, &reaplist,

							max - count);

		if (max != 0 && count >= max)

			break;

	}

	spin_unlock(&nn->client_lock);

	nfsd_reap_openowners(&reaplist);

	return count;

}



static u64 nfsd_find_all_delegations(struct nfs4_client *clp, u64 max,

				     struct list_head *victims)

{

	struct nfs4_delegation *dp, *next;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	u64 count = 0;



	lockdep_assert_held(&nn->client_lock);



	spin_lock(&state_lock);

	list_for_each_entry_safe(dp, next, &clp->cl_delegations, dl_perclnt) {

		if (victims) {

			/*

			 * It's not safe to mess with delegations that have a

			 * non-zero dl_time. They might have already been broken

			 * and could be processed by the laundromat outside of

			 * the state_lock. Just leave them be.

			 */

			if (dp->dl_time != 0)

				continue;



			atomic_inc(&clp->cl_rpc_users);

			WARN_ON(!unhash_delegation_locked(dp));

			list_add(&dp->dl_recall_lru, victims);

		}

		++count;

		/*

		 * Despite the fact that these functions deal with

		 * 64-bit integers for "count", we must ensure that

		 * it doesn't blow up the clp->cl_rpc_users. Throw a

		 * warning if we start to approach INT_MAX here.

		 */

		WARN_ON_ONCE(count == (INT_MAX / 2));

		if (count == max)

			break;

	}

	spin_unlock(&state_lock);

	return count;

}



static u64

nfsd_print_client_delegations(struct nfs4_client *clp)

{

	u64 count = nfsd_find_all_delegations(clp, 0, NULL);



	nfsd_print_count(clp, count, "delegations");

	return count;

}



u64

nfsd_inject_print_delegations(void)

{

	struct nfs4_client *clp;

	u64 count = 0;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);



	if (!nfsd_netns_ready(nn))

		return 0;



	spin_lock(&nn->client_lock);

	list_for_each_entry(clp, &nn->client_lru, cl_lru)

		count += nfsd_print_client_delegations(clp);

	spin_unlock(&nn->client_lock);



	return count;

}



static void

nfsd_forget_delegations(struct list_head *reaplist)

{

	struct nfs4_client *clp;

	struct nfs4_delegation *dp, *next;



	list_for_each_entry_safe(dp, next, reaplist, dl_recall_lru) {

		list_del_init(&dp->dl_recall_lru);

		clp = dp->dl_stid.sc_client;

		revoke_delegation(dp);

		put_client(clp);

	}

}



u64

nfsd_inject_forget_client_delegations(struct sockaddr_storage *addr,

				      size_t addr_size)

{

	u64 count = 0;

	struct nfs4_client *clp;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	LIST_HEAD(reaplist);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	clp = nfsd_find_client(addr, addr_size);

	if (clp)

		count = nfsd_find_all_delegations(clp, 0, &reaplist);

	spin_unlock(&nn->client_lock);



	nfsd_forget_delegations(&reaplist);

	return count;

}



u64

nfsd_inject_forget_delegations(u64 max)

{

	u64 count = 0;

	struct nfs4_client *clp;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	LIST_HEAD(reaplist);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	list_for_each_entry(clp, &nn->client_lru, cl_lru) {

		count += nfsd_find_all_delegations(clp, max - count, &reaplist);

		if (max != 0 && count >= max)

			break;

	}

	spin_unlock(&nn->client_lock);

	nfsd_forget_delegations(&reaplist);

	return count;

}



static void

nfsd_recall_delegations(struct list_head *reaplist)

{

	struct nfs4_client *clp;

	struct nfs4_delegation *dp, *next;



	list_for_each_entry_safe(dp, next, reaplist, dl_recall_lru) {

		list_del_init(&dp->dl_recall_lru);

		clp = dp->dl_stid.sc_client;



		trace_nfsd_deleg_recall(&dp->dl_stid.sc_stateid);



		/*

		 * We skipped all entries that had a zero dl_time before,

		 * so we can now reset the dl_time back to 0. If a delegation

		 * break comes in now, then it won't make any difference since

		 * we're recalling it either way.

		 */

		spin_lock(&state_lock);

		dp->dl_time = 0;

		spin_unlock(&state_lock);

		nfsd_break_one_deleg(dp);

		put_client(clp);

	}

}



u64

nfsd_inject_recall_client_delegations(struct sockaddr_storage *addr,

				      size_t addr_size)

{

	u64 count = 0;

	struct nfs4_client *clp;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	LIST_HEAD(reaplist);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	clp = nfsd_find_client(addr, addr_size);

	if (clp)

		count = nfsd_find_all_delegations(clp, 0, &reaplist);

	spin_unlock(&nn->client_lock);



	nfsd_recall_delegations(&reaplist);

	return count;

}



u64

nfsd_inject_recall_delegations(u64 max)

{

	u64 count = 0;

	struct nfs4_client *clp, *next;

	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,

						nfsd_net_id);

	LIST_HEAD(reaplist);



	if (!nfsd_netns_ready(nn))

		return count;



	spin_lock(&nn->client_lock);

	list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) {

		count += nfsd_find_all_delegations(clp, max - count, &reaplist);

		if (max != 0 && ++count >= max)

			break;

	}

	spin_unlock(&nn->client_lock);

	nfsd_recall_delegations(&reaplist);

	return count;

}

#endif /* CONFIG_NFSD_FAULT_INJECTION */



/*

 * Since the lifetime of a delegation isn't limited to that of an open, a

 * client may quite reasonably hang on to a delegation as long as it has