Merge remote-tracking branch 'remotes/maxreitz/tags/pull-block-2020-05-13' into staging (d8f9d57d) · Commits · SUMMER2020 / students / proj-2021291

block/block-copy.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -591,13 +591,13 @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
		}
		if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
		block_copy_task_end(task, 0);
		g_free(task);
		progress_set_remaining(s->progress,
		bdrv_get_dirty_count(s->copy_bitmap) +
		s->in_flight_bytes);
		trace_block_copy_skip_range(s, task->offset, task->bytes);
		offset = task_end(task);
		bytes = end - offset;
		g_free(task);
		continue;
		}
		task->zeroes = ret & BDRV_BLOCK_ZERO;

block/qcow2-threads.c

+229 −11

Original line number	Diff line number	Diff line
		@@ -28,6 +28,11 @@
		#define ZLIB_CONST
		#include <zlib.h>

		#ifdef CONFIG_ZSTD
		#include <zstd.h>
		#include <zstd_errors.h>
		#endif

		#include "qcow2.h"
		#include "block/thread-pool.h"
		#include "crypto.h"
		@@ -74,7 +79,9 @@ typedef struct Qcow2CompressData {
		} Qcow2CompressData;

		/*
		* qcow2_compress()
		* qcow2_zlib_compress()
		*
		* Compress @src_size bytes of data using zlib compression method
		*
		* @dest - destination buffer, @dest_size bytes
		* @src - source buffer, @src_size bytes
		@@ -83,7 +90,7 @@ typedef struct Qcow2CompressData {
		* -ENOMEM destination buffer is not enough to store compressed data
		* -EIO on any other error
		*/
		static ssize_t qcow2_compress(void *dest, size_t dest_size,
		static ssize_t qcow2_zlib_compress(void *dest, size_t dest_size,
		const void *src, size_t src_size)
		{
		ssize_t ret;
		@@ -119,10 +126,10 @@ static ssize_t qcow2_compress(void *dest, size_t dest_size,
		}

		/*
		* qcow2_decompress()
		* qcow2_zlib_decompress()
		*
		* Decompress some data (not more than @src_size bytes) to produce exactly
		* @dest_size bytes.
		* @dest_size bytes using zlib compression method
		*
		* @dest - destination buffer, @dest_size bytes
		* @src - source buffer, @src_size bytes
		@@ -130,7 +137,7 @@ static ssize_t qcow2_compress(void *dest, size_t dest_size,
		* Returns: 0 on success
		* -EIO on fail
		*/
		static ssize_t qcow2_decompress(void *dest, size_t dest_size,
		static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size,
		const void *src, size_t src_size)
		{
		int ret;
		@@ -164,6 +171,160 @@ static ssize_t qcow2_decompress(void *dest, size_t dest_size,
		return ret;
		}

		#ifdef CONFIG_ZSTD

		/*
		* qcow2_zstd_compress()
		*
		* Compress @src_size bytes of data using zstd compression method
		*
		* @dest - destination buffer, @dest_size bytes
		* @src - source buffer, @src_size bytes
		*
		* Returns: compressed size on success
		* -ENOMEM destination buffer is not enough to store compressed data
		* -EIO on any other error
		*/
		static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
		const void *src, size_t src_size)
		{
		ssize_t ret;
		size_t zstd_ret;
		ZSTD_outBuffer output = {
		.dst = dest,
		.size = dest_size,
		.pos = 0
		};
		ZSTD_inBuffer input = {
		.src = src,
		.size = src_size,
		.pos = 0
		};
		ZSTD_CCtx *cctx = ZSTD_createCCtx();

		if (!cctx) {
		return -EIO;
		}
		/*
		* Use the zstd streamed interface for symmetry with decompression,
		* where streaming is essential since we don't record the exact
		* compressed size.
		*
		* ZSTD_compressStream2() tries to compress everything it could
		* with a single call. Although, ZSTD docs says that:
		* "You must continue calling ZSTD_compressStream2() with ZSTD_e_end
		* until it returns 0, at which point you are free to start a new frame",
		* in out tests we saw the only case when it returned with >0 -
		* when the output buffer was too small. In that case,
		* ZSTD_compressStream2() expects a bigger buffer on the next call.
		* We can't provide a bigger buffer because we are limited with dest_size
		* which we pass to the ZSTD_compressStream2() at once.
		* So, we don't need any loops and just abort the compression when we
		* don't get 0 result on the first call.
		*/
		zstd_ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);

		if (zstd_ret) {
		if (zstd_ret > output.size - output.pos) {
		ret = -ENOMEM;
		} else {
		ret = -EIO;
		}
		goto out;
		}

		/* make sure that zstd didn't overflow the dest buffer */
		assert(output.pos <= dest_size);
		ret = output.pos;
		out:
		ZSTD_freeCCtx(cctx);
		return ret;
		}

		/*
		* qcow2_zstd_decompress()
		*
		* Decompress some data (not more than @src_size bytes) to produce exactly
		* @dest_size bytes using zstd compression method
		*
		* @dest - destination buffer, @dest_size bytes
		* @src - source buffer, @src_size bytes
		*
		* Returns: 0 on success
		* -EIO on any error
		*/
		static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size,
		const void *src, size_t src_size)
		{
		size_t zstd_ret = 0;
		ssize_t ret = 0;
		ZSTD_outBuffer output = {
		.dst = dest,
		.size = dest_size,
		.pos = 0
		};
		ZSTD_inBuffer input = {
		.src = src,
		.size = src_size,
		.pos = 0
		};
		ZSTD_DCtx *dctx = ZSTD_createDCtx();

		if (!dctx) {
		return -EIO;
		}

		/*
		* The compressed stream from the input buffer may consist of more
		* than one zstd frame. So we iterate until we get a fully
		* uncompressed cluster.
		* From zstd docs related to ZSTD_decompressStream:
		* "return : 0 when a frame is completely decoded and fully flushed"
		* We suppose that this means: each time ZSTD_decompressStream reads
		* only ONE full frame and returns 0 if and only if that frame
		* is completely decoded and flushed. Only after returning 0,
		* ZSTD_decompressStream reads another ONE full frame.
		*/
		while (output.pos < output.size) {
		size_t last_in_pos = input.pos;
		size_t last_out_pos = output.pos;
		zstd_ret = ZSTD_decompressStream(dctx, &output, &input);

		if (ZSTD_isError(zstd_ret)) {
		ret = -EIO;
		break;
		}

		/*
		* The ZSTD manual is vague about what to do if it reads
		* the buffer partially, and we don't want to get stuck
		* in an infinite loop where ZSTD_decompressStream
		* returns > 0 waiting for another input chunk. So, we add
		* a check which ensures that the loop makes some progress
		* on each step.
		*/
		if (last_in_pos >= input.pos &&
		last_out_pos >= output.pos) {
		ret = -EIO;
		break;
		}
		}
		/*
		* Make sure that we have the frame fully flushed here
		* if not, we somehow managed to get uncompressed cluster
		* greater then the cluster size, possibly because of its
		* damage.
		*/
		if (zstd_ret > 0) {
		ret = -EIO;
		}

		ZSTD_freeDCtx(dctx);
		assert(ret == 0 \|\| ret == -EIO);
		return ret;
		}
		#endif

		static int qcow2_compress_pool_func(void *opaque)
		{
		Qcow2CompressData *data = opaque;
		@@ -191,20 +352,77 @@ qcow2_co_do_compress(BlockDriverState bs, void dest, size_t dest_size,
		return arg.ret;
		}

		/*
		* qcow2_co_compress()
		*
		* Compress @src_size bytes of data using the compression
		* method defined by the image compression type
		*
		* @dest - destination buffer, @dest_size bytes
		* @src - source buffer, @src_size bytes
		*
		* Returns: compressed size on success
		* a negative error code on failure
		*/
		ssize_t coroutine_fn
		qcow2_co_compress(BlockDriverState bs, void dest, size_t dest_size,
		const void *src, size_t src_size)
		{
		return qcow2_co_do_compress(bs, dest, dest_size, src, src_size,
		qcow2_compress);
		BDRVQcow2State *s = bs->opaque;
		Qcow2CompressFunc fn;

		switch (s->compression_type) {
		case QCOW2_COMPRESSION_TYPE_ZLIB:
		fn = qcow2_zlib_compress;
		break;

		#ifdef CONFIG_ZSTD
		case QCOW2_COMPRESSION_TYPE_ZSTD:
		fn = qcow2_zstd_compress;
		break;
		#endif
		default:
		abort();
		}

		return qcow2_co_do_compress(bs, dest, dest_size, src, src_size, fn);
		}

		/*
		* qcow2_co_decompress()
		*
		* Decompress some data (not more than @src_size bytes) to produce exactly
		* @dest_size bytes using the compression method defined by the image
		* compression type
		*
		* @dest - destination buffer, @dest_size bytes
		* @src - source buffer, @src_size bytes
		*
		* Returns: 0 on success
		* a negative error code on failure
		*/
		ssize_t coroutine_fn
		qcow2_co_decompress(BlockDriverState bs, void dest, size_t dest_size,
		const void *src, size_t src_size)
		{
		return qcow2_co_do_compress(bs, dest, dest_size, src, src_size,
		qcow2_decompress);
		BDRVQcow2State *s = bs->opaque;
		Qcow2CompressFunc fn;

		switch (s->compression_type) {
		case QCOW2_COMPRESSION_TYPE_ZLIB:
		fn = qcow2_zlib_decompress;
		break;

		#ifdef CONFIG_ZSTD
		case QCOW2_COMPRESSION_TYPE_ZSTD:
		fn = qcow2_zstd_decompress;
		break;
		#endif
		default:
		abort();
		}

		return qcow2_co_do_compress(bs, dest, dest_size, src, src_size, fn);
		}

block/qcow2.c

+120 −0

Original line number	Diff line number	Diff line
		@@ -1242,6 +1242,42 @@ static int qcow2_update_options(BlockDriverState bs, QDict options,
		return ret;
		}

		static int validate_compression_type(BDRVQcow2State s, Error *errp)
		{
		switch (s->compression_type) {
		case QCOW2_COMPRESSION_TYPE_ZLIB:
		#ifdef CONFIG_ZSTD
		case QCOW2_COMPRESSION_TYPE_ZSTD:
		#endif
		break;

		default:
		error_setg(errp, "qcow2: unknown compression type: %u",
		s->compression_type);
		return -ENOTSUP;
		}

		/*
		* if the compression type differs from QCOW2_COMPRESSION_TYPE_ZLIB
		* the incompatible feature flag must be set
		*/
		if (s->compression_type == QCOW2_COMPRESSION_TYPE_ZLIB) {
		if (s->incompatible_features & QCOW2_INCOMPAT_COMPRESSION) {
		error_setg(errp, "qcow2: Compression type incompatible feature "
		"bit must not be set");
		return -EINVAL;
		}
		} else {
		if (!(s->incompatible_features & QCOW2_INCOMPAT_COMPRESSION)) {
		error_setg(errp, "qcow2: Compression type incompatible feature "
		"bit must be set");
		return -EINVAL;
		}
		}

		return 0;
		}

		/* Called with s->lock held. */
		static int coroutine_fn qcow2_do_open(BlockDriverState bs, QDict options,
		int flags, Error **errp)
		@@ -1357,6 +1393,23 @@ static int coroutine_fn qcow2_do_open(BlockDriverState bs, QDict options,
		s->compatible_features = header.compatible_features;
		s->autoclear_features = header.autoclear_features;

		/*
		* Handle compression type
		* Older qcow2 images don't contain the compression type header.
		* Distinguish them by the header length and use
		* the only valid (default) compression type in that case
		*/
		if (header.header_length > offsetof(QCowHeader, compression_type)) {
		s->compression_type = header.compression_type;
		} else {
		s->compression_type = QCOW2_COMPRESSION_TYPE_ZLIB;
		}

		ret = validate_compression_type(s, errp);
		if (ret) {
		goto fail;
		}

		if (s->incompatible_features & ~QCOW2_INCOMPAT_MASK) {
		void *feature_table = NULL;
		qcow2_read_extensions(bs, header.header_length, ext_end,
		@@ -2728,6 +2781,11 @@ int qcow2_update_header(BlockDriverState *bs)
		total_size = bs->total_sectors * BDRV_SECTOR_SIZE;
		refcount_table_clusters = s->refcount_table_size >> (s->cluster_bits - 3);

		ret = validate_compression_type(s, NULL);
		if (ret) {
		goto fail;
		}

		*header = (QCowHeader) {
		/* Version 2 fields */
		.magic = cpu_to_be32(QCOW_MAGIC),
		@@ -2750,6 +2808,7 @@ int qcow2_update_header(BlockDriverState *bs)
		.autoclear_features = cpu_to_be64(s->autoclear_features),
		.refcount_order = cpu_to_be32(s->refcount_order),
		.header_length = cpu_to_be32(header_length),
		.compression_type = s->compression_type,
		};

		/* For older versions, write a shorter header */
		@@ -2849,6 +2908,11 @@ int qcow2_update_header(BlockDriverState *bs)
		.bit = QCOW2_INCOMPAT_DATA_FILE_BITNR,
		.name = "external data file",
		},
		{
		.type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
		.bit = QCOW2_INCOMPAT_COMPRESSION_BITNR,
		.name = "compression type",
		},
		{
		.type = QCOW2_FEAT_TYPE_COMPATIBLE,
		.bit = QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR,
		@@ -3287,6 +3351,7 @@ qcow2_co_create(BlockdevCreateOptions create_options, Error *errp)
		uint64_t* refcount_table;
		Error *local_err = NULL;
		int ret;
		uint8_t compression_type = QCOW2_COMPRESSION_TYPE_ZLIB;

		assert(create_options->driver == BLOCKDEV_DRIVER_QCOW2);
		qcow2_opts = &create_options->u.qcow2;
		@@ -3404,6 +3469,31 @@ qcow2_co_create(BlockdevCreateOptions create_options, Error *errp)
		}
		}

		if (qcow2_opts->has_compression_type &&
		qcow2_opts->compression_type != QCOW2_COMPRESSION_TYPE_ZLIB) {

		ret = -EINVAL;

		if (version < 3) {
		error_setg(errp, "Non-zlib compression type is only supported with "
		"compatibility level 1.1 and above (use version=v3 or "
		"greater)");
		goto out;
		}

		switch (qcow2_opts->compression_type) {
		#ifdef CONFIG_ZSTD
		case QCOW2_COMPRESSION_TYPE_ZSTD:
		break;
		#endif
		default:
		error_setg(errp, "Unknown compression type");
		goto out;
		}

		compression_type = qcow2_opts->compression_type;
		}

		/* Create BlockBackend to write to the image */
		blk = blk_new_with_bs(bs, BLK_PERM_WRITE \| BLK_PERM_RESIZE, BLK_PERM_ALL,
		errp);
		@@ -3426,6 +3516,8 @@ qcow2_co_create(BlockdevCreateOptions create_options, Error *errp)
		.refcount_table_offset = cpu_to_be64(cluster_size),
		.refcount_table_clusters = cpu_to_be32(1),
		.refcount_order = cpu_to_be32(refcount_order),
		/* don't deal with endianness since compression_type is 1 byte long */
		.compression_type = compression_type,
		.header_length = cpu_to_be32(sizeof(*header)),
		};

		@@ -3444,6 +3536,10 @@ qcow2_co_create(BlockdevCreateOptions create_options, Error *errp)
		header->autoclear_features \|=
		cpu_to_be64(QCOW2_AUTOCLEAR_DATA_FILE_RAW);
		}
		if (compression_type != QCOW2_COMPRESSION_TYPE_ZLIB) {
		header->incompatible_features \|=
		cpu_to_be64(QCOW2_INCOMPAT_COMPRESSION);
		}

		ret = blk_pwrite(blk, 0, header, cluster_size, 0);
		g_free(header);
		@@ -3629,6 +3725,7 @@ static int coroutine_fn qcow2_co_create_opts(BlockDriver *drv,
		{ BLOCK_OPT_ENCRYPT, BLOCK_OPT_ENCRYPT_FORMAT },
		{ BLOCK_OPT_COMPAT_LEVEL, "version" },
		{ BLOCK_OPT_DATA_FILE_RAW, "data-file-raw" },
		{ BLOCK_OPT_COMPRESSION_TYPE, "compression-type" },
		{ NULL, NULL },
		};

		@@ -4925,6 +5022,7 @@ static ImageInfoSpecific qcow2_get_specific_info(BlockDriverState bs,
		.data_file = g_strdup(s->image_data_file),
		.has_data_file_raw = has_data_file(bs),
		.data_file_raw = data_file_is_raw(bs),
		.compression_type = s->compression_type,
		};
		} else {
		/* if this assertion fails, this probably means a new version was
		@@ -5330,6 +5428,22 @@ static int qcow2_amend_options(BlockDriverState bs, QemuOpts opts,
		"images");
		return -EINVAL;
		}
		} else if (!strcmp(desc->name, BLOCK_OPT_COMPRESSION_TYPE)) {
		const char *ct_name =
		qemu_opt_get(opts, BLOCK_OPT_COMPRESSION_TYPE);
		int compression_type =
		qapi_enum_parse(&Qcow2CompressionType_lookup, ct_name, -1,
		NULL);
		if (compression_type == -1) {
		error_setg(errp, "Unknown compression type: %s", ct_name);
		return -ENOTSUP;
		}

		if (compression_type != s->compression_type) {
		error_setg(errp, "Changing the compression type "
		"is not supported");
		return -ENOTSUP;
		}
		} else {
		/* if this point is reached, this probably means a new option was
		* added without having it covered here */
		@@ -5596,6 +5710,12 @@ static QemuOptsList qcow2_create_opts = {
		.help = "Width of a reference count entry in bits",
		.def_value_str = "16"
		},
		{
		.name = BLOCK_OPT_COMPRESSION_TYPE,
		.type = QEMU_OPT_STRING,
		.help = "Compression method used for image cluster compression",
		.def_value_str = "zlib"
		},
		{ /* end of list */ }
		}
		};

block/qcow2.h

+19 −1

Original line number	Diff line number	Diff line
		@@ -146,8 +146,16 @@ typedef struct QCowHeader {

		uint32_t refcount_order;
		uint32_t header_length;

		/* Additional fields */
		uint8_t compression_type;

		/* header must be a multiple of 8 */
		uint8_t padding[7];
		} QEMU_PACKED QCowHeader;

		QEMU_BUILD_BUG_ON(!QEMU_IS_ALIGNED(sizeof(QCowHeader), 8));

		typedef struct QEMU_PACKED QCowSnapshotHeader {
		/* header is 8 byte aligned */
		uint64_t l1_table_offset;
		@@ -216,13 +224,16 @@ enum {
		QCOW2_INCOMPAT_DIRTY_BITNR = 0,
		QCOW2_INCOMPAT_CORRUPT_BITNR = 1,
		QCOW2_INCOMPAT_DATA_FILE_BITNR = 2,
		QCOW2_INCOMPAT_COMPRESSION_BITNR = 3,
		QCOW2_INCOMPAT_DIRTY = 1 << QCOW2_INCOMPAT_DIRTY_BITNR,
		QCOW2_INCOMPAT_CORRUPT = 1 << QCOW2_INCOMPAT_CORRUPT_BITNR,
		QCOW2_INCOMPAT_DATA_FILE = 1 << QCOW2_INCOMPAT_DATA_FILE_BITNR,
		QCOW2_INCOMPAT_COMPRESSION = 1 << QCOW2_INCOMPAT_COMPRESSION_BITNR,

		QCOW2_INCOMPAT_MASK = QCOW2_INCOMPAT_DIRTY
		\| QCOW2_INCOMPAT_CORRUPT
		\| QCOW2_INCOMPAT_DATA_FILE,
		\| QCOW2_INCOMPAT_DATA_FILE
		\| QCOW2_INCOMPAT_COMPRESSION,
		};

		/* Compatible feature bits */
		@@ -366,6 +377,13 @@ typedef struct BDRVQcow2State {

		bool metadata_preallocation_checked;
		bool metadata_preallocation;
		/*
		* Compression type used for the image. Default: 0 - ZLIB
		* The image compression type is set on image creation.
		* For now, the only way to change the compression type
		* is to convert the image with the desired compression type set.
		*/
		Qcow2CompressionType compression_type;
		} BDRVQcow2State;

		typedef struct Qcow2COWRegion {

configure

+1 −1

Original line number	Diff line number	Diff line
		@@ -1861,7 +1861,7 @@ disabled with --disable-FEATURE, default is enabled if available:
		lzfse support of lzfse compression library
		(for reading lzfse-compressed dmg images)
		zstd support for zstd compression library
		(for migration compression)
		(for migration compression and qcow2 cluster compression)
		seccomp seccomp support
		coroutine-pool coroutine freelist (better performance)
		glusterfs GlusterFS backend