Commit befbaf51 authored by Peter Maydell's avatar Peter Maydell
Browse files

Merge remote-tracking branch 'remotes/kevin/tags/for-upstream' into staging 



Mirror block job fixes for 2.6.0-rc3

# gpg: Signature made Wed 20 Apr 2016 15:56:43 BST using RSA key ID C88F2FD6
# gpg: Good signature from "Kevin Wolf <kwolf@redhat.com>"

* remotes/kevin/tags/for-upstream:
  iotests: Test case for drive-mirror with unaligned image size
  iotests: Add iotests.image_size
  mirror: Don't extend the last sub-chunk
  block/mirror: Refresh stale bitmap iterator cache
  block/mirror: Revive dead yielding code

Signed-off-by: default avatarPeter Maydell <peter.maydell@linaro.org>
parents fa59dd95 8ca92f3c

block/mirror.c

+31 −14
Original line number Diff line number Diff line
@@ -108,7 +108,7 @@ static void mirror_iteration_done(MirrorOp *op, int ret) 


    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;

    chunk_num = op->sector_num / sectors_per_chunk;

    nb_chunks = op->nb_sectors / sectors_per_chunk;

    nb_chunks = DIV_ROUND_UP(op->nb_sectors, sectors_per_chunk);

    bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);

    if (ret >= 0) {

        if (s->cow_bitmap) {
@@ -161,6 +161,14 @@ static void mirror_read_complete(void *opaque, int ret) 
                    mirror_write_complete, op);

}



static inline void mirror_clip_sectors(MirrorBlockJob *s,

                                       int64_t sector_num,

                                       int *nb_sectors)

{

    *nb_sectors = MIN(*nb_sectors,

                      s->bdev_length / BDRV_SECTOR_SIZE - sector_num);

}



/* Round sector_num and/or nb_sectors to target cluster if COW is needed, and

 * return the offset of the adjusted tail sector against original. */

static int mirror_cow_align(MirrorBlockJob *s,
@@ -189,6 +197,9 @@ static int mirror_cow_align(MirrorBlockJob *s, 
                                               s->target_cluster_sectors);

        }

    }

    /* Clipping may result in align_nb_sectors unaligned to chunk boundary, but

     * that doesn't matter because it's already the end of source image. */

    mirror_clip_sectors(s, align_sector_num, &align_nb_sectors);



    ret = align_sector_num + align_nb_sectors - (*sector_num + *nb_sectors);

    *sector_num = align_sector_num;
@@ -231,9 +242,8 @@ static int mirror_do_read(MirrorBlockJob *s, int64_t sector_num, 
    /* The sector range must meet granularity because:

     * 1) Caller passes in aligned values;

     * 2) mirror_cow_align is used only when target cluster is larger. */

    assert(!(nb_sectors % sectors_per_chunk));

    assert(!(sector_num % sectors_per_chunk));

    nb_chunks = nb_sectors / sectors_per_chunk;

    nb_chunks = DIV_ROUND_UP(nb_sectors, sectors_per_chunk);



    while (s->buf_free_count < nb_chunks) {

        trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
@@ -298,7 +308,7 @@ static void mirror_do_zero_or_discard(MirrorBlockJob *s, 
static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)

{

    BlockDriverState *source = s->common.bs;

    int64_t sector_num;

    int64_t sector_num, first_chunk;

    uint64_t delay_ns = 0;

    /* At least the first dirty chunk is mirrored in one iteration. */

    int nb_chunks = 1;
@@ -313,6 +323,12 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) 
        assert(sector_num >= 0);

    }



    first_chunk = sector_num / sectors_per_chunk;

    while (test_bit(first_chunk, s->in_flight_bitmap)) {

        trace_mirror_yield_in_flight(s, first_chunk, s->in_flight);

        mirror_wait_for_io(s);

    }



    /* Find the number of consective dirty chunks following the first dirty

     * one, and wait for in flight requests in them. */

    while (nb_chunks * sectors_per_chunk < (s->buf_size >> BDRV_SECTOR_BITS)) {
@@ -324,18 +340,18 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) 
            break;

        }

        if (test_bit(next_chunk, s->in_flight_bitmap)) {

            if (nb_chunks > 0) {

            break;

        }

            trace_mirror_yield_in_flight(s, next_sector, s->in_flight);

            mirror_wait_for_io(s);

            /* Now retry.  */

        } else {



        hbitmap_next = hbitmap_iter_next(&s->hbi);

        if (hbitmap_next > next_sector || hbitmap_next < 0) {

            /* The bitmap iterator's cache is stale, refresh it */

            bdrv_set_dirty_iter(&s->hbi, next_sector);

            hbitmap_next = hbitmap_iter_next(&s->hbi);

        }

        assert(hbitmap_next == next_sector);

        nb_chunks++;

    }

    }



    /* Clear dirty bits before querying the block status, because

     * calling bdrv_get_block_status_above could yield - if some blocks are
@@ -378,6 +394,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) 
            }

        }



        mirror_clip_sectors(s, sector_num, &io_sectors);

        switch (mirror_method) {

        case MIRROR_METHOD_COPY:

            io_sectors = mirror_do_read(s, sector_num, io_sectors);
@@ -393,7 +410,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) 
        }

        assert(io_sectors);

        sector_num += io_sectors;

        nb_chunks -= io_sectors / sectors_per_chunk;

        nb_chunks -= DIV_ROUND_UP(io_sectors, sectors_per_chunk);

        delay_ns += ratelimit_calculate_delay(&s->limit, io_sectors);

    }

    return delay_ns;

tests/qemu-iotests/109.out

+22 −22
Original line number Diff line number Diff line
@@ -10,14 +10,14 @@ Automatically detecting the format is dangerous for raw images, write operations 
Specify the 'raw' format explicitly to remove the restrictions.

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 65536, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 1024, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"return": []}

read 65536/65536 bytes at offset 0

64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

{"return": {}}

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 65536, "offset": 65536, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 1024, "offset": 1024, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 1024, "offset": 1024, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

Warning: Image size mismatch!

Images are identical.
@@ -31,14 +31,14 @@ Automatically detecting the format is dangerous for raw images, write operations 
Specify the 'raw' format explicitly to remove the restrictions.

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 262144, "offset": 65536, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 197120, "offset": 512, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"return": []}

read 65536/65536 bytes at offset 0

64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

{"return": {}}

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 262144, "offset": 262144, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 262144, "offset": 262144, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 197120, "offset": 197120, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 197120, "offset": 197120, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

Warning: Image size mismatch!

Images are identical.
@@ -73,14 +73,14 @@ Automatically detecting the format is dangerous for raw images, write operations 
Specify the 'raw' format explicitly to remove the restrictions.

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 65536, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 1024, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"return": []}

read 65536/65536 bytes at offset 0

64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

{"return": {}}

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 65536, "offset": 65536, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 1024, "offset": 1024, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 1024, "offset": 1024, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

Warning: Image size mismatch!

Images are identical.
@@ -115,14 +115,14 @@ Automatically detecting the format is dangerous for raw images, write operations 
Specify the 'raw' format explicitly to remove the restrictions.

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 65536, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 2560, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"return": []}

read 65536/65536 bytes at offset 0

64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

{"return": {}}

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 65536, "offset": 65536, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 2560, "offset": 2560, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 2560, "offset": 2560, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

Warning: Image size mismatch!

Images are identical.
@@ -135,14 +135,14 @@ Automatically detecting the format is dangerous for raw images, write operations 
Specify the 'raw' format explicitly to remove the restrictions.

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 65536, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 2560, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"return": []}

read 65536/65536 bytes at offset 0

64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

{"return": {}}

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 65536, "offset": 65536, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 2560, "offset": 2560, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 2560, "offset": 2560, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

Image resized.

Warning: Image size mismatch!

Images are identical.
@@ -198,14 +198,14 @@ Automatically detecting the format is dangerous for raw images, write operations 
Specify the 'raw' format explicitly to remove the restrictions.

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 65536, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 2048, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}

{"return": []}

read 65536/65536 bytes at offset 0

64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

{"return": {}}

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 65536, "offset": 65536, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 2048, "offset": 2048, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 2048, "offset": 2048, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

Image resized.

Warning: Image size mismatch!

Images are identical.
@@ -218,14 +218,14 @@ WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed 
Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.

Specify the 'raw' format explicitly to remove the restrictions.

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 65536, "offset": 65536, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 512, "offset": 512, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 512, "offset": 512, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

Warning: Image size mismatch!

Images are identical.

{"return": {}}

{"return": {}}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 65536, "offset": 65536, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 512, "offset": 512, "speed": 0, "type": "mirror"}}

{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 512, "offset": 512, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}

Warning: Image size mismatch!

Images are identical.

*** done

tests/qemu-iotests/152

0 → 100644
+62 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python

#

# Tests for drive-mirror with source size unaligned to granularity

#

# Copyright (C) 2016 Red Hat, Inc.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

#



import os

import iotests

from iotests import qemu_img



test_img = os.path.join(iotests.test_dir, 'test.img')

target_img = os.path.join(iotests.test_dir, 'target.img')



class TestUnaligned(iotests.QMPTestCase):

    def setUp(self):

        qemu_img('create', '-f', iotests.imgfmt, test_img, '512')

        self.vm = iotests.VM().add_drive(test_img)

        self.vm.launch()



    def tearDown(self):

        self.vm.shutdown()

        os.remove(test_img)

        try:

            os.remove(target_img)

        except OSError:

            pass



    def test_unaligned(self):

        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',

                             granularity=65536, target=target_img)

        self.complete_and_wait()

        self.vm.shutdown()

        self.assertEqual(iotests.image_size(test_img), iotests.image_size(target_img),

                         "Target size doesn't match source when granularity when unaligend")



    def test_unaligned_with_update(self):

        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',

                             granularity=65536, target=target_img)

        self.wait_ready()

        self.vm.hmp_qemu_io('drive0', 'write 0 512')

        self.complete_and_wait(wait_ready=False)

        self.vm.shutdown()

        self.assertEqual(iotests.image_size(test_img), iotests.image_size(target_img),

                         "Target size doesn't match source when granularity when unaligend")





if __name__ == '__main__':

    iotests.main(supported_fmts=['raw', 'qcow2'])

tests/qemu-iotests/152.out

0 → 100644
+5 −0
Original line number Diff line number Diff line 
..

----------------------------------------------------------------------

Ran 2 tests



OK

tests/qemu-iotests/group

+1 −0
Original line number Diff line number Diff line
@@ -152,3 +152,4 @@ 
148 rw auto quick

149 rw auto sudo

150 rw auto quick

152 rw auto quick