Commit 5721d4e5 authored by Nathan Huckleberry's avatar Nathan Huckleberry Committed by Mike Snitzer
Browse files

dm verity: Add optional "try_verify_in_tasklet" feature 



Using tasklets for disk verification can reduce IO latency. When there
are accelerated hash instructions it is often better to compute the
hash immediately using a tasklet rather than deferring verification to
a work-queue. This reduces time spent waiting to schedule work-queue
jobs, but requires spending slightly more time in interrupt context.

If the dm-bufio cache does not have the required hashes we fallback to
the work-queue implementation. FEC is only possible using work-queue
because code to support the FEC feature may sleep.

The following shows a speed comparison of random reads on a dm-verity
device. The dm-verity device uses a 1G ramdisk for data and a 1G
ramdisk for hashes. One test was run using tasklets and one test was
run using the existing work-queue solution. Both tests were run when
the dm-bufio cache was hot. The tasklet implementation performs
significantly better since there is no time spent waiting for
work-queue jobs to be scheduled.

   READ: bw=181MiB/s (190MB/s), 181MiB/s-181MiB/s (190MB/s-190MB/s),
   io=512MiB (537MB), run=2827-2827msec
   READ: bw=23.6MiB/s (24.8MB/s), 23.6MiB/s-23.6MiB/s (24.8MB/s-24.8MB/s),
   io=512MiB (537MB), run=21688-21688msec

Signed-off-by: default avatarNathan Huckleberry <nhuck@google.com>
Signed-off-by: default avatarMike Snitzer <snitzer@kernel.org>
parent b32d4582

drivers/md/dm-verity-target.c

+86 −17
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ 
#define DM_VERITY_OPT_PANIC		"panic_on_corruption"

#define DM_VERITY_OPT_IGN_ZEROES	"ignore_zero_blocks"

#define DM_VERITY_OPT_AT_MOST_ONCE	"check_at_most_once"

#define DM_VERITY_OPT_TASKLET_VERIFY	"try_verify_in_tasklet"



#define DM_VERITY_OPTS_MAX		(3 + DM_VERITY_OPTS_FEC + \

					 DM_VERITY_ROOT_HASH_VERIFICATION_OPTS)
@@ -220,7 +221,7 @@ static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, 
	struct mapped_device *md = dm_table_get_md(v->ti->table);



	/* Corruption should be visible in device status in all modes */

	v->hash_failed = 1;

	v->hash_failed = true;



	if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)

		goto out;
@@ -286,7 +287,19 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, 


	verity_hash_at_level(v, block, level, &hash_block, &offset);



	if (io->in_tasklet) {

		data = dm_bufio_get(v->bufio, hash_block, &buf);

		if (data == NULL) {

			/*

			 * In tasklet and the hash was not in the bufio cache.

			 * Return early and resume execution from a work-queue

			 * to read the hash from disk.

			 */

			return -EAGAIN;

		}

	} else

		data = dm_bufio_read(v->bufio, hash_block, &buf);



	if (IS_ERR(data))

		return PTR_ERR(data);
@@ -307,6 +320,14 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, 
		if (likely(memcmp(verity_io_real_digest(v, io), want_digest,

				  v->digest_size) == 0))

			aux->hash_verified = 1;

		else if (io->in_tasklet) {

			/*

			 * Error handling code (FEC included) cannot be run in a

			 * tasklet since it may sleep, so fallback to work-queue.

			 */

			r = -EAGAIN;

			goto release_ret_r;

		}

		else if (verity_fec_decode(v, io,

					   DM_VERITY_BLOCK_TYPE_METADATA,

					   hash_block, data, NULL) == 0)
@@ -474,9 +495,14 @@ static int verity_verify_io(struct dm_verity_io *io) 
	bool is_zero;

	struct dm_verity *v = io->v;

	struct bvec_iter start;

	unsigned b;

	/*

	 * Copy the iterator in case we need to restart verification in a

	 * work-queue.

	 */

	struct bvec_iter iter_copy = io->iter;

	struct crypto_wait wait;

	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);

	unsigned int b;



	for (b = 0; b < io->n_blocks; b++) {

		int r;
@@ -485,7 +511,7 @@ static int verity_verify_io(struct dm_verity_io *io) 


		if (v->validated_blocks &&

		    likely(test_bit(cur_block, v->validated_blocks))) {

			verity_bv_skip_block(v, io, &io->iter);

			verity_bv_skip_block(v, io, &iter_copy);

			continue;

		}
@@ -500,7 +526,7 @@ static int verity_verify_io(struct dm_verity_io *io) 
			 * If we expect a zero block, don't validate, just

			 * return zeros.

			 */

			r = verity_for_bv_block(v, io, &io->iter,

			r = verity_for_bv_block(v, io, &iter_copy,

						verity_bv_zero);

			if (unlikely(r < 0))

				return r;
@@ -512,8 +538,8 @@ static int verity_verify_io(struct dm_verity_io *io) 
		if (unlikely(r < 0))

			return r;



		start = io->iter;

		r = verity_for_io_block(v, io, &io->iter, &wait);

		start = iter_copy;

		r = verity_for_io_block(v, io, &iter_copy, &wait);

		if (unlikely(r < 0))

			return r;
@@ -527,6 +553,12 @@ static int verity_verify_io(struct dm_verity_io *io) 
			if (v->validated_blocks)

				set_bit(cur_block, v->validated_blocks);

			continue;

		} else if (io->in_tasklet) {

			/*

			 * Error handling code (FEC included) cannot be run in a

			 * tasklet since it may sleep, so fallback to work-queue.

			 */

			return -EAGAIN;

		} else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA,

					     cur_block, NULL, &start) == 0) {

			continue;
@@ -566,6 +598,7 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) 
	bio->bi_end_io = io->orig_bi_end_io;

	bio->bi_status = status;



	if (!io->in_tasklet)

		verity_fec_finish_io(io);



	bio_endio(bio);
@@ -575,9 +608,29 @@ static void verity_work(struct work_struct *w) 
{

	struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);



	io->in_tasklet = false;



	verity_fec_init_io(io);

	verity_finish_io(io, errno_to_blk_status(verity_verify_io(io)));

}



static void verity_tasklet(unsigned long data)

{

	struct dm_verity_io *io = (struct dm_verity_io *)data;

	int err;



	io->in_tasklet = true;

	err = verity_verify_io(io);

	if (err == -EAGAIN) {

		/* fallback to retrying with work-queue */

		INIT_WORK(&io->work, verity_work);

		queue_work(io->v->verify_wq, &io->work);

		return;

	}



	verity_finish_io(io, errno_to_blk_status(err));

}



static void verity_end_io(struct bio *bio)

{

	struct dm_verity_io *io = bio->bi_private;
@@ -588,9 +641,14 @@ static void verity_end_io(struct bio *bio) 
		return;

	}



	if (io->v->use_tasklet) {

		tasklet_init(&io->tasklet, verity_tasklet, (unsigned long)io);

		tasklet_schedule(&io->tasklet);

	} else {

		INIT_WORK(&io->work, verity_work);

		queue_work(io->v->verify_wq, &io->work);

	}

}



/*

 * Prefetch buffers for the specified io.
@@ -700,8 +758,6 @@ static int verity_map(struct dm_target *ti, struct bio *bio) 
	bio->bi_private = io;

	io->iter = bio->bi_iter;



	verity_fec_init_io(io);



	verity_submit_prefetch(v, io);



	submit_bio_noacct(bio);
@@ -751,6 +807,8 @@ static void verity_status(struct dm_target *ti, status_type_t type, 
			args++;

		if (v->validated_blocks)

			args++;

		if (v->use_tasklet)

			args++;

		if (v->signature_key_desc)

			args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;

		if (!args)
@@ -776,6 +834,8 @@ static void verity_status(struct dm_target *ti, status_type_t type, 
			DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);

		if (v->validated_blocks)

			DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);

		if (v->use_tasklet)

			DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);

		sz = verity_fec_status_table(v, sz, result, maxlen);

		if (v->signature_key_desc)

			DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY
@@ -1011,11 +1071,16 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, 
				return r;

			continue;



		} else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {

			v->use_tasklet = true;

			continue;



		} else if (verity_is_fec_opt_arg(arg_name)) {

			r = verity_fec_parse_opt_args(as, v, &argc, arg_name);

			if (r)

				return r;

			continue;



		} else if (verity_verify_is_sig_opt_arg(arg_name)) {

			r = verity_verify_sig_parse_opt_args(as, v,

							     verify_args,
@@ -1023,7 +1088,6 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, 
			if (r)

				return r;

			continue;



		}



		ti->error = "Unrecognized verity feature request";
@@ -1155,7 +1219,11 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) 
		goto bad;

	}



	v->tfm = crypto_alloc_ahash(v->alg_name, 0, 0);

	/*

	 * FIXME: CRYPTO_ALG_ASYNC should be conditional on v->use_tasklet

	 * but verity_parse_opt_args() happens below and has data dep on tfm.

	 */

	v->tfm = crypto_alloc_ahash(v->alg_name, 0, CRYPTO_ALG_ASYNC);

	if (IS_ERR(v->tfm)) {

		ti->error = "Cannot initialize hash function";

		r = PTR_ERR(v->tfm);
@@ -1265,7 +1333,8 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) 


	v->bufio = dm_bufio_client_create(v->hash_dev->bdev,

		1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),

		dm_bufio_alloc_callback, NULL, 0);

		dm_bufio_alloc_callback, NULL,

		v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0);

	if (IS_ERR(v->bufio)) {

		ti->error = "Cannot initialize dm-bufio";

		r = PTR_ERR(v->bufio);
@@ -1312,7 +1381,7 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) 
static struct target_type verity_target = {

	.name		= "verity",

	.features	= DM_TARGET_IMMUTABLE,

	.version	= {1, 8, 0},

	.version	= {1, 9, 0},

	.module		= THIS_MODULE,

	.ctr		= verity_ctr,

	.dtr		= verity_dtr,

drivers/md/dm-verity.h

+5 −1
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 


#include <linux/dm-bufio.h>

#include <linux/device-mapper.h>

#include <linux/interrupt.h>

#include <crypto/hash.h>



#define DM_VERITY_MAX_LEVELS		63
@@ -51,9 +52,10 @@ struct dm_verity { 
	unsigned char hash_per_block_bits;	/* log2(hashes in hash block) */

	unsigned char levels;	/* the number of tree levels */

	unsigned char version;

	bool hash_failed:1;	/* set if hash of any block failed */

	bool use_tasklet:1;	/* try to verify in tasklet before work-queue */

	unsigned digest_size;	/* digest size for the current hash algorithm */

	unsigned int ahash_reqsize;/* the size of temporary space for crypto */

	int hash_failed;	/* set to 1 if hash of any block failed */

	enum verity_mode mode;	/* mode for handling verification errors */

	unsigned corrupted_errs;/* Number of errors for corrupted blocks */
@@ -76,10 +78,12 @@ struct dm_verity_io { 


	sector_t block;

	unsigned n_blocks;

	bool in_tasklet;



	struct bvec_iter iter;



	struct work_struct work;

	struct tasklet_struct tasklet;



	/*

	 * Three variably-size fields follow this struct: