Merge remote-tracking branch 'kwolf/for-anthony' into staging (503cb22e) · Commits · SUMMER2020 / students / proj-2021291

block-migration.c

+4 −3

Original line number	Diff line number	Diff line
		@@ -23,7 +23,8 @@
		#include "sysemu/blockdev.h"
		#include <assert.h>

		#define BLOCK_SIZE (BDRV_SECTORS_PER_DIRTY_CHUNK << BDRV_SECTOR_BITS)
		#define BLOCK_SIZE (1 << 20)
		#define BDRV_SECTORS_PER_DIRTY_CHUNK (BLOCK_SIZE >> BDRV_SECTOR_BITS)

		#define BLK_MIG_FLAG_DEVICE_BLOCK 0x01
		#define BLK_MIG_FLAG_EOS 0x02
		@@ -254,7 +255,7 @@ static void set_dirty_tracking(int enable)
		BlkMigDevState *bmds;

		QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
		bdrv_set_dirty_tracking(bmds->bs, enable);
		bdrv_set_dirty_tracking(bmds->bs, enable ? BLOCK_SIZE : 0);
		}
		}

		@@ -478,7 +479,7 @@ static int64_t get_remaining_dirty(void)
		dirty += bdrv_get_dirty_count(bmds->bs);
		}

		return dirty * BLOCK_SIZE;
		return dirty << BDRV_SECTOR_BITS;
		}

		static void blk_mig_cleanup(void)

block.c

+31 −93

Original line number	Diff line number	Diff line
		@@ -1286,7 +1286,6 @@ static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
		bs_dest->iostatus = bs_src->iostatus;

		/* dirty bitmap */
		bs_dest->dirty_count = bs_src->dirty_count;
		bs_dest->dirty_bitmap = bs_src->dirty_bitmap;

		/* job */
		@@ -1674,7 +1673,7 @@ static void tracked_request_begin(BdrvTrackedRequest *req,
		/**
		* Round a region to cluster boundaries
		*/
		static void round_to_clusters(BlockDriverState *bs,
		void bdrv_round_to_clusters(BlockDriverState *bs,
		int64_t sector_num, int nb_sectors,
		int64_t *cluster_sector_num,
		int *cluster_nb_sectors)
		@@ -1719,7 +1718,7 @@ static void coroutine_fn wait_for_overlapping_requests(BlockDriverState *bs,
		* CoR read and write operations are atomic and guest writes cannot
		* interleave between them.
		*/
		round_to_clusters(bs, sector_num, nb_sectors,
		bdrv_round_to_clusters(bs, sector_num, nb_sectors,
		&cluster_sector_num, &cluster_nb_sectors);

		do {
		@@ -2035,36 +2034,6 @@ int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
		return ret;
		}

		#define BITS_PER_LONG (sizeof(unsigned long) * 8)

		static void set_dirty_bitmap(BlockDriverState *bs, int64_t sector_num,
		int nb_sectors, int dirty)
		{
		int64_t start, end;
		unsigned long val, idx, bit;

		start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
		end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;

		for (; start <= end; start++) {
		idx = start / BITS_PER_LONG;
		bit = start % BITS_PER_LONG;
		val = bs->dirty_bitmap[idx];
		if (dirty) {
		if (!(val & (1UL << bit))) {
		bs->dirty_count++;
		val \|= 1UL << bit;
		}
		} else {
		if (val & (1UL << bit)) {
		bs->dirty_count--;
		val &= ~(1UL << bit);
		}
		}
		bs->dirty_bitmap[idx] = val;
		}
		}

		/* Return < 0 if error. Important errors are:
		-EIO generic I/O error (may happen for all errors)
		-ENOMEDIUM No media inserted.
		@@ -2216,7 +2185,7 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
		/* Cover entire cluster so no additional backing file I/O is required when
		* allocating cluster in the image file.
		*/
		round_to_clusters(bs, sector_num, nb_sectors,
		bdrv_round_to_clusters(bs, sector_num, nb_sectors,
		&cluster_sector_num, &cluster_nb_sectors);

		trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
		@@ -2863,8 +2832,9 @@ BlockInfo bdrv_query_info(BlockDriverState bs)
		if (bs->dirty_bitmap) {
		info->has_dirty = true;
		info->dirty = g_malloc0(sizeof(*info->dirty));
		info->dirty->count = bdrv_get_dirty_count(bs) *
		BDRV_SECTORS_PER_DIRTY_CHUNK * BDRV_SECTOR_SIZE;
		info->dirty->count = bdrv_get_dirty_count(bs) * BDRV_SECTOR_SIZE;
		info->dirty->granularity =
		((int64_t) BDRV_SECTOR_SIZE << hbitmap_granularity(bs->dirty_bitmap));
		}

		if (bs->drv) {
		@@ -4173,7 +4143,7 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
		}

		if (bs->dirty_bitmap) {
		set_dirty_bitmap(bs, sector_num, nb_sectors, 0);
		bdrv_reset_dirty(bs, sector_num, nb_sectors);
		}

		if (bs->drv->bdrv_co_discard) {
		@@ -4331,22 +4301,20 @@ bool bdrv_qiov_is_aligned(BlockDriverState bs, QEMUIOVector qiov)
		return true;
		}

		void bdrv_set_dirty_tracking(BlockDriverState *bs, int enable)
		void bdrv_set_dirty_tracking(BlockDriverState *bs, int granularity)
		{
		int64_t bitmap_size;

		bs->dirty_count = 0;
		if (enable) {
		if (!bs->dirty_bitmap) {
		bitmap_size = (bdrv_getlength(bs) >> BDRV_SECTOR_BITS) +
		BDRV_SECTORS_PER_DIRTY_CHUNK * BITS_PER_LONG - 1;
		bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * BITS_PER_LONG;
		assert((granularity & (granularity - 1)) == 0);

		bs->dirty_bitmap = g_new0(unsigned long, bitmap_size);
		}
		if (granularity) {
		granularity >>= BDRV_SECTOR_BITS;
		assert(!bs->dirty_bitmap);
		bitmap_size = (bdrv_getlength(bs) >> BDRV_SECTOR_BITS);
		bs->dirty_bitmap = hbitmap_alloc(bitmap_size, ffs(granularity) - 1);
		} else {
		if (bs->dirty_bitmap) {
		g_free(bs->dirty_bitmap);
		hbitmap_free(bs->dirty_bitmap);
		bs->dirty_bitmap = NULL;
		}
		}
		@@ -4354,67 +4322,37 @@ void bdrv_set_dirty_tracking(BlockDriverState *bs, int enable)

		int bdrv_get_dirty(BlockDriverState *bs, int64_t sector)
		{
		int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;

		if (bs->dirty_bitmap &&
		(sector << BDRV_SECTOR_BITS) < bdrv_getlength(bs)) {
		return !!(bs->dirty_bitmap[chunk / BITS_PER_LONG] &
		(1UL << (chunk % BITS_PER_LONG)));
		if (bs->dirty_bitmap) {
		return hbitmap_get(bs->dirty_bitmap, sector);
		} else {
		return 0;
		}
		}

		int64_t bdrv_get_next_dirty(BlockDriverState *bs, int64_t sector)
		void bdrv_dirty_iter_init(BlockDriverState bs, HBitmapIter hbi)
		{
		int64_t chunk;
		int bit, elem;

		/* Avoid an infinite loop. */
		assert(bs->dirty_count > 0);

		sector = (sector \| (BDRV_SECTORS_PER_DIRTY_CHUNK - 1)) + 1;
		chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;

		QEMU_BUILD_BUG_ON(sizeof(bs->dirty_bitmap[0]) * 8 != BITS_PER_LONG);
		elem = chunk / BITS_PER_LONG;
		bit = chunk % BITS_PER_LONG;
		for (;;) {
		if (sector >= bs->total_sectors) {
		sector = 0;
		bit = elem = 0;
		}
		if (bit == 0 && bs->dirty_bitmap[elem] == 0) {
		sector += BDRV_SECTORS_PER_DIRTY_CHUNK * BITS_PER_LONG;
		elem++;
		} else {
		if (bs->dirty_bitmap[elem] & (1UL << bit)) {
		return sector;
		}
		sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
		if (++bit == BITS_PER_LONG) {
		bit = 0;
		elem++;
		}
		}
		}
		hbitmap_iter_init(hbi, bs->dirty_bitmap, 0);
		}

		void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
		int nr_sectors)
		{
		set_dirty_bitmap(bs, cur_sector, nr_sectors, 1);
		hbitmap_set(bs->dirty_bitmap, cur_sector, nr_sectors);
		}

		void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector,
		int nr_sectors)
		{
		set_dirty_bitmap(bs, cur_sector, nr_sectors, 0);
		hbitmap_reset(bs->dirty_bitmap, cur_sector, nr_sectors);
		}

		int64_t bdrv_get_dirty_count(BlockDriverState *bs)
		{
		return bs->dirty_count;
		if (bs->dirty_bitmap) {
		return hbitmap_count(bs->dirty_bitmap);
		} else {
		return 0;
		}
		}

		void bdrv_set_in_use(BlockDriverState *bs, int in_use)

block/bochs.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -126,7 +126,7 @@ static int bochs_open(BlockDriverState *bs, int flags)
		strcmp(bochs.subtype, GROWING_TYPE) \|\|
		((le32_to_cpu(bochs.version) != HEADER_VERSION) &&
		(le32_to_cpu(bochs.version) != HEADER_V1))) {
		goto fail;
		return -EMEDIUMTYPE;
		}

		if (le32_to_cpu(bochs.version) == HEADER_V1) {

block/cow.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -73,7 +73,7 @@ static int cow_open(BlockDriverState *bs, int flags)
		}

		if (be32_to_cpu(cow_header.magic) != COW_MAGIC) {
		ret = -EINVAL;
		ret = -EMEDIUMTYPE;
		goto fail;
		}

block/mirror.c

+322 −60

Original line number	Diff line number	Diff line
		@@ -15,17 +15,17 @@
		#include "block/blockjob.h"
		#include "block/block_int.h"
		#include "qemu/ratelimit.h"

		enum {
		/*
		* Size of data buffer for populating the image file. This should be large
		* enough to process multiple clusters in a single call, so that populating
		* contiguous regions of the image is efficient.
		*/
		BLOCK_SIZE = 512 * BDRV_SECTORS_PER_DIRTY_CHUNK, /* in bytes */
		};
		#include "qemu/bitmap.h"

		#define SLICE_TIME 100000000ULL /* ns */
		#define MAX_IN_FLIGHT 16

		/* The mirroring buffer is a list of granularity-sized chunks.
		* Free chunks are organized in a list.
		*/
		typedef struct MirrorBuffer {
		QSIMPLEQ_ENTRY(MirrorBuffer) next;
		} MirrorBuffer;

		typedef struct MirrorBlockJob {
		BlockJob common;
		@@ -36,9 +36,26 @@ typedef struct MirrorBlockJob {
		bool synced;
		bool should_complete;
		int64_t sector_num;
		int64_t granularity;
		size_t buf_size;
		unsigned long *cow_bitmap;
		HBitmapIter hbi;
		uint8_t *buf;
		QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
		int buf_free_count;

		unsigned long *in_flight_bitmap;
		int in_flight;
		int ret;
		} MirrorBlockJob;

		typedef struct MirrorOp {
		MirrorBlockJob *s;
		QEMUIOVector qiov;
		int64_t sector_num;
		int nb_sectors;
		} MirrorOp;

		static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
		int error)
		{
		@@ -52,51 +69,234 @@ static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
		}
		}

		static int coroutine_fn mirror_iteration(MirrorBlockJob *s,
		BlockErrorAction *p_action)
		static void mirror_iteration_done(MirrorOp *op, int ret)
		{
		MirrorBlockJob *s = op->s;
		struct iovec *iov;
		int64_t chunk_num;
		int i, nb_chunks, sectors_per_chunk;

		trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret);

		s->in_flight--;
		iov = op->qiov.iov;
		for (i = 0; i < op->qiov.niov; i++) {
		MirrorBuffer buf = (MirrorBuffer ) iov[i].iov_base;
		QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
		s->buf_free_count++;
		}

		sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
		chunk_num = op->sector_num / sectors_per_chunk;
		nb_chunks = op->nb_sectors / sectors_per_chunk;
		bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
		if (s->cow_bitmap && ret >= 0) {
		bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
		}

		g_slice_free(MirrorOp, op);
		qemu_coroutine_enter(s->common.co, NULL);
		}

		static void mirror_write_complete(void *opaque, int ret)
		{
		MirrorOp *op = opaque;
		MirrorBlockJob *s = op->s;
		if (ret < 0) {
		BlockDriverState *source = s->common.bs;
		BlockDriverState *target = s->target;
		QEMUIOVector qiov;
		int ret, nb_sectors;
		int64_t end;
		struct iovec iov;
		BlockErrorAction action;

		bdrv_set_dirty(source, op->sector_num, op->nb_sectors);
		action = mirror_error_action(s, false, -ret);
		if (action == BDRV_ACTION_REPORT && s->ret >= 0) {
		s->ret = ret;
		}
		}
		mirror_iteration_done(op, ret);
		}

		static void mirror_read_complete(void *opaque, int ret)
		{
		MirrorOp *op = opaque;
		MirrorBlockJob *s = op->s;
		if (ret < 0) {
		BlockDriverState *source = s->common.bs;
		BlockErrorAction action;

		bdrv_set_dirty(source, op->sector_num, op->nb_sectors);
		action = mirror_error_action(s, true, -ret);
		if (action == BDRV_ACTION_REPORT && s->ret >= 0) {
		s->ret = ret;
		}

		mirror_iteration_done(op, ret);
		return;
		}
		bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors,
		mirror_write_complete, op);
		}

		static void coroutine_fn mirror_iteration(MirrorBlockJob *s)
		{
		BlockDriverState *source = s->common.bs;
		int nb_sectors, sectors_per_chunk, nb_chunks;
		int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector;
		MirrorOp *op;

		s->sector_num = hbitmap_iter_next(&s->hbi);
		if (s->sector_num < 0) {
		bdrv_dirty_iter_init(source, &s->hbi);
		s->sector_num = hbitmap_iter_next(&s->hbi);
		trace_mirror_restart_iter(s, bdrv_get_dirty_count(source));
		assert(s->sector_num >= 0);
		}

		hbitmap_next_sector = s->sector_num;
		sector_num = s->sector_num;
		sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
		end = s->common.len >> BDRV_SECTOR_BITS;
		s->sector_num = bdrv_get_next_dirty(source, s->sector_num);
		nb_sectors = MIN(BDRV_SECTORS_PER_DIRTY_CHUNK, end - s->sector_num);
		bdrv_reset_dirty(source, s->sector_num, nb_sectors);

		/* Extend the QEMUIOVector to include all adjacent blocks that will
		* be copied in this operation.
		*
		* We have to do this if we have no backing file yet in the destination,
		* and the cluster size is very large. Then we need to do COW ourselves.
		* The first time a cluster is copied, copy it entirely. Note that,
		* because both the granularity and the cluster size are powers of two,
		* the number of sectors to copy cannot exceed one cluster.
		*
		* We also want to extend the QEMUIOVector to include more adjacent
		* dirty blocks if possible, to limit the number of I/O operations and
		* run efficiently even with a small granularity.
		*/
		nb_chunks = 0;
		nb_sectors = 0;
		next_sector = sector_num;
		next_chunk = sector_num / sectors_per_chunk;

		/* Wait for I/O to this cluster (from a previous iteration) to be done. */
		while (test_bit(next_chunk, s->in_flight_bitmap)) {
		trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
		qemu_coroutine_yield();
		}

		do {
		int added_sectors, added_chunks;

		if (!bdrv_get_dirty(source, next_sector) \|\|
		test_bit(next_chunk, s->in_flight_bitmap)) {
		assert(nb_sectors > 0);
		break;
		}

		added_sectors = sectors_per_chunk;
		if (s->cow_bitmap && !test_bit(next_chunk, s->cow_bitmap)) {
		bdrv_round_to_clusters(s->target,
		next_sector, added_sectors,
		&next_sector, &added_sectors);

		/* On the first iteration, the rounding may make us copy
		* sectors before the first dirty one.
		*/
		if (next_sector < sector_num) {
		assert(nb_sectors == 0);
		sector_num = next_sector;
		next_chunk = next_sector / sectors_per_chunk;
		}
		}

		added_sectors = MIN(added_sectors, end - (sector_num + nb_sectors));
		added_chunks = (added_sectors + sectors_per_chunk - 1) / sectors_per_chunk;

		/* When doing COW, it may happen that there is not enough space for
		* a full cluster. Wait if that is the case.
		*/
		while (nb_chunks == 0 && s->buf_free_count < added_chunks) {
		trace_mirror_yield_buf_busy(s, nb_chunks, s->in_flight);
		qemu_coroutine_yield();
		}
		if (s->buf_free_count < nb_chunks + added_chunks) {
		trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight);
		break;
		}

		/* We have enough free space to copy these sectors. */
		bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks);

		nb_sectors += added_sectors;
		nb_chunks += added_chunks;
		next_sector += added_sectors;
		next_chunk += added_chunks;
		} while (next_sector < end);

		/* Allocate a MirrorOp that is used as an AIO callback. */
		op = g_slice_new(MirrorOp);
		op->s = s;
		op->sector_num = sector_num;
		op->nb_sectors = nb_sectors;

		/* Now make a QEMUIOVector taking enough granularity-sized chunks
		* from s->buf_free.
		*/
		qemu_iovec_init(&op->qiov, nb_chunks);
		next_sector = sector_num;
		while (nb_chunks-- > 0) {
		MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
		QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
		s->buf_free_count--;
		qemu_iovec_add(&op->qiov, buf, s->granularity);

		/* Advance the HBitmapIter in parallel, so that we do not examine
		* the same sector twice.
		*/
		if (next_sector > hbitmap_next_sector && bdrv_get_dirty(source, next_sector)) {
		hbitmap_next_sector = hbitmap_iter_next(&s->hbi);
		}

		next_sector += sectors_per_chunk;
		}

		bdrv_reset_dirty(source, sector_num, nb_sectors);

		/* Copy the dirty cluster. */
		iov.iov_base = s->buf;
		iov.iov_len = nb_sectors * 512;
		qemu_iovec_init_external(&qiov, &iov, 1);
		s->in_flight++;
		trace_mirror_one_iteration(s, sector_num, nb_sectors);
		bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors,
		mirror_read_complete, op);
		}

		trace_mirror_one_iteration(s, s->sector_num, nb_sectors);
		ret = bdrv_co_readv(source, s->sector_num, nb_sectors, &qiov);
		if (ret < 0) {
		*p_action = mirror_error_action(s, true, -ret);
		goto fail;
		static void mirror_free_init(MirrorBlockJob *s)
		{
		int granularity = s->granularity;
		size_t buf_size = s->buf_size;
		uint8_t *buf = s->buf;

		assert(s->buf_free_count == 0);
		QSIMPLEQ_INIT(&s->buf_free);
		while (buf_size != 0) {
		MirrorBuffer cur = (MirrorBuffer )buf;
		QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
		s->buf_free_count++;
		buf_size -= granularity;
		buf += granularity;
		}
		ret = bdrv_co_writev(target, s->sector_num, nb_sectors, &qiov);
		if (ret < 0) {
		*p_action = mirror_error_action(s, false, -ret);
		s->synced = false;
		goto fail;
		}
		return 0;

		fail:
		/* Try again later. */
		bdrv_set_dirty(source, s->sector_num, nb_sectors);
		return ret;
		static void mirror_drain(MirrorBlockJob *s)
		{
		while (s->in_flight > 0) {
		qemu_coroutine_yield();
		}
		}

		static void coroutine_fn mirror_run(void *opaque)
		{
		MirrorBlockJob *s = opaque;
		BlockDriverState *bs = s->common.bs;
		int64_t sector_num, end;
		int64_t sector_num, end, sectors_per_chunk, length;
		uint64_t last_pause_ns;
		BlockDriverInfo bdi;
		char backing_filename[1024];
		int ret = 0;
		int n;

		@@ -105,20 +305,39 @@ static void coroutine_fn mirror_run(void *opaque)
		}

		s->common.len = bdrv_getlength(bs);
		if (s->common.len < 0) {
		if (s->common.len <= 0) {
		block_job_completed(&s->common, s->common.len);
		return;
		}

		length = (bdrv_getlength(bs) + s->granularity - 1) / s->granularity;
		s->in_flight_bitmap = bitmap_new(length);

		/* If we have no backing file yet in the destination, we cannot let
		* the destination do COW. Instead, we copy sectors around the
		* dirty data if needed. We need a bitmap to do that.
		*/
		bdrv_get_backing_filename(s->target, backing_filename,
		sizeof(backing_filename));
		if (backing_filename[0] && !s->target->backing_hd) {
		bdrv_get_info(s->target, &bdi);
		if (s->granularity < bdi.cluster_size) {
		s->buf_size = MAX(s->buf_size, bdi.cluster_size);
		s->cow_bitmap = bitmap_new(length);
		}
		}

		end = s->common.len >> BDRV_SECTOR_BITS;
		s->buf = qemu_blockalign(bs, BLOCK_SIZE);
		s->buf = qemu_blockalign(bs, s->buf_size);
		sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
		mirror_free_init(s);

		if (s->mode != MIRROR_SYNC_MODE_NONE) {
		/* First part, loop on the sectors and initialize the dirty bitmap. */
		BlockDriverState *base;
		base = s->mode == MIRROR_SYNC_MODE_FULL ? NULL : bs->backing_hd;
		for (sector_num = 0; sector_num < end; ) {
		int64_t next = (sector_num \| (BDRV_SECTORS_PER_DIRTY_CHUNK - 1)) + 1;
		int64_t next = (sector_num \| (sectors_per_chunk - 1)) + 1;
		ret = bdrv_co_is_allocated_above(bs, base,
		sector_num, next - sector_num, &n);

		@@ -136,24 +355,40 @@ static void coroutine_fn mirror_run(void *opaque)
		}
		}

		s->sector_num = -1;
		bdrv_dirty_iter_init(bs, &s->hbi);
		last_pause_ns = qemu_get_clock_ns(rt_clock);
		for (;;) {
		uint64_t delay_ns;
		int64_t cnt;
		bool should_complete;

		cnt = bdrv_get_dirty_count(bs);
		if (cnt != 0) {
		BlockErrorAction action = BDRV_ACTION_REPORT;
		ret = mirror_iteration(s, &action);
		if (ret < 0 && action == BDRV_ACTION_REPORT) {
		if (s->ret < 0) {
		ret = s->ret;
		goto immediate_exit;
		}

		cnt = bdrv_get_dirty_count(bs);

		/* Note that even when no rate limit is applied we need to yield
		* periodically with no pending I/O so that qemu_aio_flush() returns.
		* We do so every SLICE_TIME nanoseconds, or when there is an error,
		* or when the source is clean, whichever comes first.
		*/
		if (qemu_get_clock_ns(rt_clock) - last_pause_ns < SLICE_TIME &&
		s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
		if (s->in_flight == MAX_IN_FLIGHT \|\| s->buf_free_count == 0 \|\|
		(cnt == 0 && s->in_flight > 0)) {
		trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt);
		qemu_coroutine_yield();
		continue;
		} else if (cnt != 0) {
		mirror_iteration(s);
		continue;
		}
		}

		should_complete = false;
		if (cnt == 0) {
		if (s->in_flight == 0 && cnt == 0) {
		trace_mirror_before_flush(s);
		ret = bdrv_flush(s->target);
		if (ret < 0) {
		@@ -196,23 +431,20 @@ static void coroutine_fn mirror_run(void *opaque)
		trace_mirror_before_sleep(s, cnt, s->synced);
		if (!s->synced) {
		/* Publish progress */
		s->common.offset = end * BDRV_SECTOR_SIZE - cnt * BLOCK_SIZE;
		s->common.offset = (end - cnt) * BDRV_SECTOR_SIZE;

		if (s->common.speed) {
		delay_ns = ratelimit_calculate_delay(&s->limit, BDRV_SECTORS_PER_DIRTY_CHUNK);
		delay_ns = ratelimit_calculate_delay(&s->limit, sectors_per_chunk);
		} else {
		delay_ns = 0;
		}

		/* Note that even when no rate limit is applied we need to yield
		* with no pending I/O here so that bdrv_drain_all() returns.
		*/
		block_job_sleep_ns(&s->common, rt_clock, delay_ns);
		if (block_job_is_cancelled(&s->common)) {
		break;
		}
		} else if (!should_complete) {
		delay_ns = (cnt == 0 ? SLICE_TIME : 0);
		delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0);
		block_job_sleep_ns(&s->common, rt_clock, delay_ns);
		} else if (cnt == 0) {
		/* The two disks are in sync. Exit and report successful
		@@ -222,11 +454,24 @@ static void coroutine_fn mirror_run(void *opaque)
		s->common.cancelled = false;
		break;
		}
		last_pause_ns = qemu_get_clock_ns(rt_clock);
		}

		immediate_exit:
		if (s->in_flight > 0) {
		/* We get here only if something went wrong. Either the job failed,
		* or it was cancelled prematurely so that we do not guarantee that
		* the target is a copy of the source.
		*/
		assert(ret < 0 \|\| (!s->synced && block_job_is_cancelled(&s->common)));
		mirror_drain(s);
		}

		assert(s->in_flight == 0);
		qemu_vfree(s->buf);
		bdrv_set_dirty_tracking(bs, false);
		g_free(s->cow_bitmap);
		g_free(s->in_flight_bitmap);
		bdrv_set_dirty_tracking(bs, 0);
		bdrv_iostatus_disable(s->target);
		if (s->should_complete && ret == 0) {
		if (bdrv_get_flags(s->target) != bdrv_get_flags(s->common.bs)) {
		@@ -288,14 +533,28 @@ static BlockJobType mirror_job_type = {
		};

		void mirror_start(BlockDriverState bs, BlockDriverState target,
		int64_t speed, MirrorSyncMode mode,
		BlockdevOnError on_source_error,
		int64_t speed, int64_t granularity, int64_t buf_size,
		MirrorSyncMode mode, BlockdevOnError on_source_error,
		BlockdevOnError on_target_error,
		BlockDriverCompletionFunc *cb,
		void opaque, Error *errp)
		{
		MirrorBlockJob *s;

		if (granularity == 0) {
		/* Choose the default granularity based on the target file's cluster
		* size, clamped between 4k and 64k. */
		BlockDriverInfo bdi;
		if (bdrv_get_info(target, &bdi) >= 0 && bdi.cluster_size != 0) {
		granularity = MAX(4096, bdi.cluster_size);
		granularity = MIN(65536, granularity);
		} else {
		granularity = 65536;
		}
		}

		assert ((granularity & (granularity - 1)) == 0);

		if ((on_source_error == BLOCKDEV_ON_ERROR_STOP \|\|
		on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
		!bdrv_iostatus_is_enabled(bs)) {
		@@ -312,7 +571,10 @@ void mirror_start(BlockDriverState bs, BlockDriverState target,
		s->on_target_error = on_target_error;
		s->target = target;
		s->mode = mode;
		bdrv_set_dirty_tracking(bs, true);
		s->granularity = granularity;
		s->buf_size = MAX(buf_size, granularity);

		bdrv_set_dirty_tracking(bs, granularity);
		bdrv_set_enable_write_cache(s->target, true);
		bdrv_set_on_error(s->target, on_target_error, on_target_error);
		bdrv_iostatus_enable(s->target);