Commit 0d2ed603 authored by Peter Maydell's avatar Peter Maydell
Browse files

Merge remote-tracking branch 'remotes/kevin/tags/for-upstream' into staging 



Block layer core and image format patches

# gpg: Signature made Fri May 22 16:21:03 2015 BST using RSA key ID C88F2FD6
# gpg: Good signature from "Kevin Wolf <kwolf@redhat.com>"

* remotes/kevin/tags/for-upstream: (22 commits)
  MAINTAINERS: Split "Block QAPI, monitor, command line" off core
  MAINTAINERS: Add header files to Block Layer Core section
  tests: add test case for encrypted qcow2 read/write
  qemu-io: prompt for encryption keys when required
  util: allow \n to terminate password input
  util: move read_password method out of qemu-img into osdep/oslib
  qcow2/qcow: protect against uninitialized encryption key
  qemu-iotests: Make debugging python tests easier
  qemu-iotests: qemu-img info on afl VMDK image with a huge capacity
  block: Detect multiplication overflow in bdrv_getlength
  qemu-io: Use getopt() correctly
  qcow2: style fixes in qcow2-cache.c
  qcow2: make qcow2_cache_put() a void function
  qcow2: use a hash to look for entries in the L2 cache
  qcow2: remove qcow2_cache_find_entry_to_replace()
  qcow2: use an LRU algorithm to replace entries from the L2 cache
  qcow2: simplify qcow2_cache_put() and qcow2_cache_entry_mark_dirty()
  qcow2: use one single memory block for the L2/refcount cache tables
  vmdk: Fix overflow if l1_size is 0x20000000
  vmdk: Fix next_cluster_sector for compressed write
  ...

Signed-off-by: default avatarPeter Maydell <peter.maydell@linaro.org>
parents bb2fa17f 4120201d

MAINTAINERS

+9 −0
Original line number Diff line number Diff line
@@ -785,6 +785,7 @@ S: Supported 
F: block*

F: block/

F: hw/block/

F: include/block/

F: qemu-img*

F: qemu-io*

F: tests/qemu-iotests/
@@ -812,6 +813,14 @@ F: block/stream.h 
F: block/mirror.c

T: git git://github.com/codyprime/qemu-kvm-jtc.git block



Block QAPI, monitor, command line

M: Markus Armbruster <armbru@redhat.com>

S: Supported

F: blockdev.c

F: block/qapi.c

F: qapi/block*.json

T: git git://repo.or.cz/qemu/armbru.git block-next



Character Devices

M: Paolo Bonzini <pbonzini@redhat.com>

S: Maintained

block.c

+1 −0
Original line number Diff line number Diff line
@@ -2341,6 +2341,7 @@ int64_t bdrv_getlength(BlockDriverState *bs) 
{

    int64_t ret = bdrv_nb_sectors(bs);



    ret = ret > INT64_MAX / BDRV_SECTOR_SIZE ? -EFBIG : ret;

    return ret < 0 ? ret : ret * BDRV_SECTOR_SIZE;

}

block/qcow.c

+7 −3
Original line number Diff line number Diff line
@@ -269,6 +269,7 @@ static int qcow_set_key(BlockDriverState *bs, const char *key) 
    for(i = 0;i < len;i++) {

        keybuf[i] = key[i];

    }

    assert(bs->encrypted);

    s->crypt_method = s->crypt_method_header;



    if (AES_set_encrypt_key(keybuf, 128, &s->aes_encrypt_key) != 0)
@@ -411,9 +412,10 @@ static uint64_t get_cluster_offset(BlockDriverState *bs, 
                bdrv_truncate(bs->file, cluster_offset + s->cluster_size);

                /* if encrypted, we must initialize the cluster

                   content which won't be written */

                if (s->crypt_method &&

                if (bs->encrypted &&

                    (n_end - n_start) < s->cluster_sectors) {

                    uint64_t start_sect;

                    assert(s->crypt_method);

                    start_sect = (offset & ~(s->cluster_size - 1)) >> 9;

                    memset(s->cluster_data + 512, 0x00, 512);

                    for(i = 0; i < s->cluster_sectors; i++) {
@@ -590,7 +592,8 @@ static coroutine_fn int qcow_co_readv(BlockDriverState *bs, int64_t sector_num, 
            if (ret < 0) {

                break;

            }

            if (s->crypt_method) {

            if (bs->encrypted) {

                assert(s->crypt_method);

                encrypt_sectors(s, sector_num, buf, buf,

                                n, 0,

                                &s->aes_decrypt_key);
@@ -661,7 +664,8 @@ static coroutine_fn int qcow_co_writev(BlockDriverState *bs, int64_t sector_num, 
            ret = -EIO;

            break;

        }

        if (s->crypt_method) {

        if (bs->encrypted) {

            assert(s->crypt_method);

            if (!cluster_data) {

                cluster_data = g_malloc0(s->cluster_size);

            }

block/qcow2-cache.c

+75 −96
Original line number Diff line number Diff line
@@ -28,10 +28,9 @@ 
#include "trace.h"



typedef struct Qcow2CachedTable {

    void*   table;

    int64_t  offset;

    bool     dirty;

    int     cache_hits;

    uint64_t lru_counter;

    int      ref;

} Qcow2CachedTable;
@@ -40,39 +39,46 @@ struct Qcow2Cache { 
    struct Qcow2Cache      *depends;

    int                     size;

    bool                    depends_on_flush;

    void                   *table_array;

    uint64_t                lru_counter;

};



static inline void *qcow2_cache_get_table_addr(BlockDriverState *bs,

                    Qcow2Cache *c, int table)

{

    BDRVQcowState *s = bs->opaque;

    return (uint8_t *) c->table_array + (size_t) table * s->cluster_size;

}



static inline int qcow2_cache_get_table_idx(BlockDriverState *bs,

                  Qcow2Cache *c, void *table)

{

    BDRVQcowState *s = bs->opaque;

    ptrdiff_t table_offset = (uint8_t *) table - (uint8_t *) c->table_array;

    int idx = table_offset / s->cluster_size;

    assert(idx >= 0 && idx < c->size && table_offset % s->cluster_size == 0);

    return idx;

}



Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables)

{

    BDRVQcowState *s = bs->opaque;

    Qcow2Cache *c;

    int i;



    c = g_new0(Qcow2Cache, 1);

    c->size = num_tables;

    c->entries = g_try_new0(Qcow2CachedTable, num_tables);

    if (!c->entries) {

        goto fail;

    }

    c->table_array = qemu_try_blockalign(bs->file,

                                         (size_t) num_tables * s->cluster_size);



    for (i = 0; i < c->size; i++) {

        c->entries[i].table = qemu_try_blockalign(bs->file, s->cluster_size);

        if (c->entries[i].table == NULL) {

            goto fail;

        }

    if (!c->entries || !c->table_array) {

        qemu_vfree(c->table_array);

        g_free(c->entries);

        g_free(c);

        c = NULL;

    }



    return c;



fail:

    if (c->entries) {

        for (i = 0; i < c->size; i++) {

            qemu_vfree(c->entries[i].table);

        }

    }

    g_free(c->entries);

    g_free(c);

    return NULL;

}



int qcow2_cache_destroy(BlockDriverState *bs, Qcow2Cache *c)
@@ -81,9 +87,9 @@ int qcow2_cache_destroy(BlockDriverState* bs, Qcow2Cache *c) 


    for (i = 0; i < c->size; i++) {

        assert(c->entries[i].ref == 0);

        qemu_vfree(c->entries[i].table);

    }



    qemu_vfree(c->table_array);

    g_free(c->entries);

    g_free(c);
@@ -151,8 +157,8 @@ static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i) 
        BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE);

    }



    ret = bdrv_pwrite(bs->file, c->entries[i].offset, c->entries[i].table,

        s->cluster_size);

    ret = bdrv_pwrite(bs->file, c->entries[i].offset,

                      qcow2_cache_get_table_addr(bs, c, i), s->cluster_size);

    if (ret < 0) {

        return ret;

    }
@@ -228,42 +234,12 @@ int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c) 
    for (i = 0; i < c->size; i++) {

        assert(c->entries[i].ref == 0);

        c->entries[i].offset = 0;

        c->entries[i].cache_hits = 0;

        c->entries[i].lru_counter = 0;

    }



    return 0;

}



static int qcow2_cache_find_entry_to_replace(Qcow2Cache *c)

{

    int i;

    int min_count = INT_MAX;

    int min_index = -1;

    c->lru_counter = 0;





    for (i = 0; i < c->size; i++) {

        if (c->entries[i].ref) {

            continue;

        }



        if (c->entries[i].cache_hits < min_count) {

            min_index = i;

            min_count = c->entries[i].cache_hits;

        }



        /* Give newer hits priority */

        /* TODO Check how to optimize the replacement strategy */

        if (c->entries[i].cache_hits > 1) {

            c->entries[i].cache_hits /= 2;

        }

    }



    if (min_index == -1) {

        /* This can't happen in current synchronous code, but leave the check

         * here as a reminder for whoever starts using AIO with the cache */

        abort();

    }

    return min_index;

    return 0;

}



static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c,
@@ -272,19 +248,37 @@ static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c, 
    BDRVQcowState *s = bs->opaque;

    int i;

    int ret;

    int lookup_index;

    uint64_t min_lru_counter = UINT64_MAX;

    int min_lru_index = -1;



    trace_qcow2_cache_get(qemu_coroutine_self(), c == s->l2_table_cache,

                          offset, read_from_disk);



    /* Check if the table is already cached */

    for (i = 0; i < c->size; i++) {

        if (c->entries[i].offset == offset) {

    i = lookup_index = (offset / s->cluster_size * 4) % c->size;

    do {

        const Qcow2CachedTable *t = &c->entries[i];

        if (t->offset == offset) {

            goto found;

        }

        if (t->ref == 0 && t->lru_counter < min_lru_counter) {

            min_lru_counter = t->lru_counter;

            min_lru_index = i;

        }

        if (++i == c->size) {

            i = 0;

        }

    } while (i != lookup_index);



    /* If not, write a table back and replace it */

    i = qcow2_cache_find_entry_to_replace(c);

    if (min_lru_index == -1) {

        /* This can't happen in current synchronous code, but leave the check

         * here as a reminder for whoever starts using AIO with the cache */

        abort();

    }



    /* Cache miss: write a table back and replace it */

    i = min_lru_index;

    trace_qcow2_cache_get_replace_entry(qemu_coroutine_self(),

                                        c == s->l2_table_cache, i);

    if (i < 0) {
@@ -304,22 +298,19 @@ static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c, 
            BLKDBG_EVENT(bs->file, BLKDBG_L2_LOAD);

        }



        ret = bdrv_pread(bs->file, offset, c->entries[i].table, s->cluster_size);

        ret = bdrv_pread(bs->file, offset, qcow2_cache_get_table_addr(bs, c, i),

                         s->cluster_size);

        if (ret < 0) {

            return ret;

        }

    }



    /* Give the table some hits for the start so that it won't be replaced

     * immediately. The number 32 is completely arbitrary. */

    c->entries[i].cache_hits = 32;

    c->entries[i].offset = offset;



    /* And return the right table */

found:

    c->entries[i].cache_hits++;

    c->entries[i].ref++;

    *table = c->entries[i].table;

    *table = qcow2_cache_get_table_addr(bs, c, i);



    trace_qcow2_cache_get_done(qemu_coroutine_self(),

                               c == s->l2_table_cache, i);
@@ -339,36 +330,24 @@ int qcow2_cache_get_empty(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset, 
    return qcow2_cache_do_get(bs, c, offset, table, false);

}



int qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table)

void qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table)

{

    int i;

    int i = qcow2_cache_get_table_idx(bs, c, *table);



    for (i = 0; i < c->size; i++) {

        if (c->entries[i].table == *table) {

            goto found;

        }

    }

    return -ENOENT;



found:

    c->entries[i].ref--;

    *table = NULL;



    assert(c->entries[i].ref >= 0);

    return 0;

    if (c->entries[i].ref == 0) {

        c->entries[i].lru_counter = ++c->lru_counter;

    }



void qcow2_cache_entry_mark_dirty(Qcow2Cache *c, void *table)

{

    int i;



    for (i = 0; i < c->size; i++) {

        if (c->entries[i].table == table) {

            goto found;

        }

    assert(c->entries[i].ref >= 0);

}

    abort();



found:

void qcow2_cache_entry_mark_dirty(BlockDriverState *bs, Qcow2Cache *c,

     void *table)

{

    int i = qcow2_cache_get_table_idx(bs, c, table);

    assert(c->entries[i].offset != 0);

    c->entries[i].dirty = true;

}

block/qcow2-cluster.c

+18 −47
Original line number Diff line number Diff line
@@ -253,17 +253,14 @@ static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table) 


        memcpy(l2_table, old_table, s->cluster_size);



        ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &old_table);

        if (ret < 0) {

            goto fail;

        }

        qcow2_cache_put(bs, s->l2_table_cache, (void **) &old_table);

    }



    /* write the l2 table to the file */

    BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_WRITE);



    trace_qcow2_l2_allocate_write_l2(bs, l1_index);

    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);

    qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);

    ret = qcow2_cache_flush(bs, s->l2_table_cache);

    if (ret < 0) {

        goto fail;
@@ -403,7 +400,8 @@ static int coroutine_fn copy_sectors(BlockDriverState *bs, 
        goto out;

    }



    if (s->crypt_method) {

    if (bs->encrypted) {

        assert(s->crypt_method);

        qcow2_encrypt_sectors(s, start_sect + n_start,

                        iov.iov_base, iov.iov_base, n, 1,

                        &s->aes_encrypt_key);
@@ -692,12 +690,9 @@ uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs, 
    /* compressed clusters never have the copied flag */



    BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE_COMPRESSED);

    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);

    qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);

    l2_table[l2_index] = cpu_to_be64(cluster_offset);

    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);

    if (ret < 0) {

        return 0;

    }

    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);



    return cluster_offset;

}
@@ -771,7 +766,7 @@ int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m) 
    if (ret < 0) {

        goto err;

    }

    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);

    qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);



    assert(l2_index + m->nb_clusters <= s->l2_size);

    for (i = 0; i < m->nb_clusters; i++) {
@@ -789,10 +784,7 @@ int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m) 
     }





    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);

    if (ret < 0) {

        goto err;

    }

    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);



    /*

     * If this was a COW, we need to decrease the refcount of the old cluster.
@@ -944,7 +936,7 @@ static int handle_copied(BlockDriverState *bs, uint64_t guest_offset, 
    uint64_t *l2_table;

    unsigned int nb_clusters;

    unsigned int keep_clusters;

    int ret, pret;

    int ret;



    trace_qcow2_handle_copied(qemu_coroutine_self(), guest_offset, *host_offset,

                              *bytes);
@@ -1011,10 +1003,7 @@ static int handle_copied(BlockDriverState *bs, uint64_t guest_offset, 


    /* Cleanup */

out:

    pret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);

    if (pret < 0) {

        return pret;

    }

    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);



    /* Only return a host offset if we actually made progress. Otherwise we

     * would make requirements for handle_alloc() that it can't fulfill */
@@ -1139,10 +1128,7 @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset, 
     * wrong with our code. */

    assert(nb_clusters > 0);



    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);

    if (ret < 0) {

        return ret;

    }

    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);



    /* Allocate, if necessary at a given offset in the image file */

    alloc_cluster_offset = start_of_cluster(s, *host_offset);
@@ -1470,7 +1456,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset, 
        }



        /* First remove L2 entries */

        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);

        qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);

        if (!full_discard && s->qcow_version >= 3) {

            l2_table[l2_index + i] = cpu_to_be64(QCOW_OFLAG_ZERO);

        } else {
@@ -1481,10 +1467,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset, 
        qcow2_free_any_clusters(bs, old_l2_entry, 1, type);

    }



    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);

    if (ret < 0) {

        return ret;

    }

    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);



    return nb_clusters;

}
@@ -1558,7 +1541,7 @@ static int zero_single_l2(BlockDriverState *bs, uint64_t offset, 
        old_offset = be64_to_cpu(l2_table[l2_index + i]);



        /* Update L2 entries */

        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);

        qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);

        if (old_offset & QCOW_OFLAG_COMPRESSED) {

            l2_table[l2_index + i] = cpu_to_be64(QCOW_OFLAG_ZERO);

            qcow2_free_any_clusters(bs, old_offset, 1, QCOW2_DISCARD_REQUEST);
@@ -1567,10 +1550,7 @@ static int zero_single_l2(BlockDriverState *bs, uint64_t offset, 
        }

    }



    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);

    if (ret < 0) {

        return ret;

    }

    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);



    return nb_clusters;

}
@@ -1760,14 +1740,10 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table, 


        if (is_active_l1) {

            if (l2_dirty) {

                qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);

                qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);

                qcow2_cache_depends_on_flush(s->l2_table_cache);

            }

            ret = qcow2_cache_put(bs, s->l2_table_cache, (void **)&l2_table);

            if (ret < 0) {

                l2_table = NULL;

                goto fail;

            }

            qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);

        } else {

            if (l2_dirty) {

                ret = qcow2_pre_write_overlap_check(bs,
@@ -1798,12 +1774,7 @@ fail: 
        if (!is_active_l1) {

            qemu_vfree(l2_table);

        } else {

            if (ret < 0) {

            qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);

            } else {

                ret = qcow2_cache_put(bs, s->l2_table_cache,

                        (void **)&l2_table);

            }

        }

    }

    return ret;