Commit 6b422e5f authored by Peter Maydell's avatar Peter Maydell
Browse files

Merge remote-tracking branch 'remotes/maxreitz/tags/pull-block-2019-09-03' into staging 



Block patches:
- qemu-io now accepts a file to read a write pattern from
- Ensure that raw files have their first block allocated so we can probe
  the O_DIRECT alignment if necessary
- Various fixes

# gpg: Signature made Tue 03 Sep 2019 13:58:57 BST
# gpg:                using RSA key 91BEB60A30DB3E8857D11829F407DB0061D5CF40
# gpg:                issuer "mreitz@redhat.com"
# gpg: Good signature from "Max Reitz <mreitz@redhat.com>" [full]
# Primary key fingerprint: 91BE B60A 30DB 3E88 57D1  1829 F407 DB00 61D5 CF40

* remotes/maxreitz/tags/pull-block-2019-09-03:
  iotests: Unify cache mode quoting
  tests/check-block: Skip iotests when sanitizers are enabled
  iotests: Check for enabled drivers before testing them
  iotests: Add -display none to the qemu options
  file-posix: fix request_alignment typo
  iotests: Disable 126 for flat vmdk subformats
  iotests: Disable 110 for vmdk.twoGbMaxExtentSparse
  iotests: Disable broken streamOptimized tests
  vmdk: Reject invalid compressed writes
  iotests: Keep testing broken relative extent paths
  vmdk: Use bdrv_dirname() for relative extent paths
  iotests: Fix _filter_img_create()
  iotests: Test allocate_first_block() with O_DIRECT
  block: posix: Always allocate the first block
  block: fix permission update in bdrv_replace_node
  qemu-io: add pattern file for write command

Signed-off-by: default avatarPeter Maydell <peter.maydell@linaro.org>
parents d3714799 755c5fe7

block.c

+2 −3
Original line number Diff line number Diff line
@@ -4165,7 +4165,6 @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, 
{

    BdrvChild *c, *next;

    GSList *list = NULL, *p;

    uint64_t old_perm, old_shared;

    uint64_t perm = 0, shared = BLK_PERM_ALL;

    int ret;
@@ -4211,8 +4210,8 @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, 
        bdrv_unref(from);

    }



    bdrv_get_cumulative_perm(to, &old_perm, &old_shared);

    bdrv_set_perm(to, old_perm | perm, old_shared | shared);

    bdrv_get_cumulative_perm(to, &perm, &shared);

    bdrv_set_perm(to, perm, shared);



out:

    g_slist_free(list);

block/file-posix.c

+52 −1
Original line number Diff line number Diff line
@@ -380,7 +380,7 @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp) 
        for (i = 0; i < ARRAY_SIZE(alignments); i++) {

            align = alignments[i];

            if (raw_is_io_aligned(fd, buf + align, max_align)) {

                /* Fallback to request_aligment. */

                /* Fallback to request_alignment. */

                s->buf_align = (align != 1) ? align : bs->bl.request_alignment;

                break;

            }
@@ -1749,6 +1749,43 @@ static int handle_aiocb_discard(void *opaque) 
    return ret;

}



/*

 * Help alignment probing by allocating the first block.

 *

 * When reading with direct I/O from unallocated area on Gluster backed by XFS,

 * reading succeeds regardless of request length. In this case we fallback to

 * safe alignment which is not optimal. Allocating the first block avoids this

 * fallback.

 *

 * fd may be opened with O_DIRECT, but we don't know the buffer alignment or

 * request alignment, so we use safe values.

 *

 * Returns: 0 on success, -errno on failure. Since this is an optimization,

 * caller may ignore failures.

 */

static int allocate_first_block(int fd, size_t max_size)

{

    size_t write_size = (max_size < MAX_BLOCKSIZE)

        ? BDRV_SECTOR_SIZE

        : MAX_BLOCKSIZE;

    size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize());

    void *buf;

    ssize_t n;

    int ret;



    buf = qemu_memalign(max_align, write_size);

    memset(buf, 0, write_size);



    do {

        n = pwrite(fd, buf, write_size, 0);

    } while (n == -1 && errno == EINTR);



    ret = (n == -1) ? -errno : 0;



    qemu_vfree(buf);

    return ret;

}



static int handle_aiocb_truncate(void *opaque)

{

    RawPosixAIOData *aiocb = opaque;
@@ -1788,6 +1825,17 @@ static int handle_aiocb_truncate(void *opaque) 
                /* posix_fallocate() doesn't set errno. */

                error_setg_errno(errp, -result,

                                 "Could not preallocate new data");

            } else if (current_length == 0) {

                /*

                 * posix_fallocate() uses fallocate() if the filesystem

                 * supports it, or fallback to manually writing zeroes. If

                 * fallocate() was used, unaligned reads from the fallocated

                 * area in raw_probe_alignment() will succeed, hence we need to

                 * allocate the first block.

                 *

                 * Optimize future alignment probing; ignore failures.

                 */

                allocate_first_block(fd, offset);

            }

        } else {

            result = 0;
@@ -1849,6 +1897,9 @@ static int handle_aiocb_truncate(void *opaque) 
        if (ftruncate(fd, offset) != 0) {

            result = -errno;

            error_setg_errno(errp, -result, "Could not resize file");

        } else if (current_length == 0 && offset > current_length) {

            /* Optimize future alignment probing; ignore failures. */

            allocate_first_block(fd, offset);

        }

        return result;

    default:

block/vmdk.c

+44 −20
Original line number Diff line number Diff line
@@ -1076,8 +1076,7 @@ static const char *next_line(const char *s) 
}



static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,

                              const char *desc_file_path, QDict *options,

                              Error **errp)

                              QDict *options, Error **errp)

{

    int ret;

    int matches;
@@ -1087,6 +1086,7 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, 
    const char *p, *np;

    int64_t sectors = 0;

    int64_t flat_offset;

    char *desc_file_dir = NULL;

    char *extent_path;

    BdrvChild *extent_file;

    BDRVVmdkState *s = bs->opaque;
@@ -1130,16 +1130,23 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, 
            continue;

        }



        if (!path_is_absolute(fname) && !path_has_protocol(fname) &&

            !desc_file_path[0])

        {

        if (path_is_absolute(fname)) {

            extent_path = g_strdup(fname);

        } else {

            if (!desc_file_dir) {

                desc_file_dir = bdrv_dirname(bs->file->bs, errp);

                if (!desc_file_dir) {

                    bdrv_refresh_filename(bs->file->bs);

            error_setg(errp, "Cannot use relative extent paths with VMDK "

                       "descriptor file '%s'", bs->file->bs->filename);

            return -EINVAL;

                    error_prepend(errp, "Cannot use relative paths with VMDK "

                                  "descriptor file '%s': ",

                                  bs->file->bs->filename);

                    ret = -EINVAL;

                    goto out;

                }

            }



        extent_path = path_combine(desc_file_path, fname);

            extent_path = g_strconcat(desc_file_dir, fname, NULL);

        }



        ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents);

        assert(ret < 32);
@@ -1149,7 +1156,8 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, 
        g_free(extent_path);

        if (local_err) {

            error_propagate(errp, local_err);

            return -EINVAL;

            ret = -EINVAL;

            goto out;

        }



        /* save to extents array */
@@ -1160,7 +1168,7 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, 
                            0, 0, 0, 0, 0, &extent, errp);

            if (ret < 0) {

                bdrv_unref_child(bs, extent_file);

                return ret;

                goto out;

            }

            extent->flat_start_offset = flat_offset << 9;

        } else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) {
@@ -1175,24 +1183,27 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, 
            g_free(buf);

            if (ret) {

                bdrv_unref_child(bs, extent_file);

                return ret;

                goto out;

            }

            extent = &s->extents[s->num_extents - 1];

        } else if (!strcmp(type, "SESPARSE")) {

            ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp);

            if (ret) {

                bdrv_unref_child(bs, extent_file);

                return ret;

                goto out;

            }

            extent = &s->extents[s->num_extents - 1];

        } else {

            error_setg(errp, "Unsupported extent type '%s'", type);

            bdrv_unref_child(bs, extent_file);

            return -ENOTSUP;

            ret = -ENOTSUP;

            goto out;

        }

        extent->type = g_strdup(type);

    }

    return 0;



    ret = 0;

    goto out;



invalid:

    np = next_line(p);
@@ -1201,7 +1212,11 @@ invalid: 
        np--;

    }

    error_setg(errp, "Invalid extent line: %.*s", (int)(np - p), p);

    return -EINVAL;

    ret = -EINVAL;



out:

    g_free(desc_file_dir);

    return ret;

}



static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
@@ -1228,8 +1243,7 @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf, 
    }

    s->create_type = g_strdup(ct);

    s->desc_offset = 0;

    ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options,

                             errp);

    ret = vmdk_parse_extents(buf, bs, options, errp);

exit:

    return ret;

}
@@ -1720,6 +1734,16 @@ static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset, 
    if (extent->compressed) {

        void *compressed_data;



        /* Only whole clusters */

        if (offset_in_cluster ||

            n_bytes > (extent->cluster_sectors * SECTOR_SIZE) ||

            (n_bytes < (extent->cluster_sectors * SECTOR_SIZE) &&

             offset + n_bytes != extent->end_sector * SECTOR_SIZE))

        {

            ret = -EINVAL;

            goto out;

        }



        if (!extent->has_marker) {

            ret = -EINVAL;

            goto out;

qemu-io-cmds.c

+93 −6
Original line number Diff line number Diff line
@@ -350,6 +350,79 @@ static void qemu_io_free(void *p) 
    qemu_vfree(p);

}



/*

 * qemu_io_alloc_from_file()

 *

 * Allocates the buffer and populates it with the content of the given file

 * up to @len bytes. If the file length is less than @len, then the buffer

 * is populated with the file content cyclically.

 *

 * @blk - the block backend where the buffer content is going to be written to

 * @len - the buffer length

 * @file_name - the file to read the content from

 *

 * Returns: the buffer pointer on success

 *          NULL on error

 */

static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len,

                                     const char *file_name)

{

    char *buf, *buf_origin;

    FILE *f = fopen(file_name, "r");

    int pattern_len;



    if (!f) {

        perror(file_name);

        return NULL;

    }



    if (qemuio_misalign) {

        len += MISALIGN_OFFSET;

    }



    buf_origin = buf = blk_blockalign(blk, len);



    if (qemuio_misalign) {

        buf_origin += MISALIGN_OFFSET;

        buf += MISALIGN_OFFSET;

        len -= MISALIGN_OFFSET;

    }



    pattern_len = fread(buf_origin, 1, len, f);



    if (ferror(f)) {

        perror(file_name);

        goto error;

    }



    if (pattern_len == 0) {

        fprintf(stderr, "%s: file is empty\n", file_name);

        goto error;

    }



    fclose(f);



    if (len > pattern_len) {

        len -= pattern_len;

        buf += pattern_len;



        while (len > 0) {

            size_t len_to_copy = MIN(pattern_len, len);



            memcpy(buf, buf_origin, len_to_copy);



            len -= len_to_copy;

            buf += len_to_copy;

        }

    }



    return buf_origin;



error:

    qemu_io_free(buf_origin);

    return NULL;

}



static void dump_buffer(const void *buffer, int64_t offset, int64_t len)

{

    uint64_t i;
@@ -948,6 +1021,7 @@ static void write_help(void) 
" -n, -- with -z, don't allow slow fallback\n"

" -p, -- ignored for backwards compatibility\n"

" -P, -- use different pattern to fill file\n"

" -s, -- use a pattern file to fill the write buffer\n"

" -C, -- report statistics in a machine parsable format\n"

" -q, -- quiet mode, do not show I/O statistics\n"

" -u, -- with -z, allow unmapping\n"
@@ -964,7 +1038,7 @@ static const cmdinfo_t write_cmd = { 
    .perm       = BLK_PERM_WRITE,

    .argmin     = 2,

    .argmax     = -1,

    .args       = "[-bcCfnquz] [-P pattern] off len",

    .args       = "[-bcCfnquz] [-P pattern | -s source_file] off len",

    .oneline    = "writes a number of bytes at a specified offset",

    .help       = write_help,

};
@@ -973,7 +1047,7 @@ static int write_f(BlockBackend *blk, int argc, char **argv) 
{

    struct timespec t1, t2;

    bool Cflag = false, qflag = false, bflag = false;

    bool Pflag = false, zflag = false, cflag = false;

    bool Pflag = false, zflag = false, cflag = false, sflag = false;

    int flags = 0;

    int c, cnt, ret;

    char *buf = NULL;
@@ -982,8 +1056,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv) 
    /* Some compilers get confused and warn if this is not initialized.  */

    int64_t total = 0;

    int pattern = 0xcd;

    const char *file_name = NULL;



    while ((c = getopt(argc, argv, "bcCfnpP:quz")) != -1) {

    while ((c = getopt(argc, argv, "bcCfnpP:qs:uz")) != -1) {

        switch (c) {

        case 'b':

            bflag = true;
@@ -1013,6 +1088,10 @@ static int write_f(BlockBackend *blk, int argc, char **argv) 
        case 'q':

            qflag = true;

            break;

        case 's':

            sflag = true;

            file_name = optarg;

            break;

        case 'u':

            flags |= BDRV_REQ_MAY_UNMAP;

            break;
@@ -1050,8 +1129,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv) 
        return -EINVAL;

    }



    if (zflag && Pflag) {

        printf("-z and -P cannot be specified at the same time\n");

    if (zflag + Pflag + sflag > 1) {

        printf("Only one of -z, -P, and -s "

               "can be specified at the same time\n");

        return -EINVAL;

    }
@@ -1087,8 +1167,15 @@ static int write_f(BlockBackend *blk, int argc, char **argv) 
    }



    if (!zflag) {

        if (sflag) {

            buf = qemu_io_alloc_from_file(blk, count, file_name);

            if (!buf) {

                return -EINVAL;

            }

        } else {

            buf = qemu_io_alloc(blk, count, pattern);

        }

    }



    clock_gettime(CLOCK_MONOTONIC, &t1);

    if (bflag) {

tests/check-block.sh

+5 −0
Original line number Diff line number Diff line
@@ -21,6 +21,11 @@ if grep -q "TARGET_GPROF=y" *-softmmu/config-target.mak 2>/dev/null ; then 
    exit 0

fi



if grep -q "CFLAGS.*-fsanitize" config-host.mak 2>/dev/null ; then

    echo "Sanitizers are enabled ==> Not running the qemu-iotests."

    exit 0

fi



if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then

    echo "No qemu-system binary available ==> Not running the qemu-iotests."

    exit 0