Commit 55cc1b59 authored by Lidong Chen's avatar Lidong Chen Committed by Juan Quintela
Browse files

migration: create a dedicated connection for rdma return path 



If start a RDMA migration with postcopy enabled, the source qemu
establish a dedicated connection for return path.

Signed-off-by: default avatarLidong Chen <lidongchen@tencent.com>
Reviewed-by: default avatarDr. David Alan Gilbert <dgilbert@redhat.com>
Reviewed-by: default avatarJuan Quintela <quintela@redhat.com>
Signed-off-by: default avatarJuan Quintela <quintela@redhat.com>
parent ccb7e1b5

migration/rdma.c

+91 −3
Original line number Diff line number Diff line
@@ -387,6 +387,10 @@ typedef struct RDMAContext { 
    uint64_t unregistrations[RDMA_SIGNALED_SEND_MAX];



    GHashTable *blockmap;



    /* the RDMAContext for return path */

    struct RDMAContext *return_path;

    bool is_return_path;

} RDMAContext;



#define TYPE_QIO_CHANNEL_RDMA "qio-channel-rdma"
@@ -2323,10 +2327,22 @@ static void qemu_rdma_cleanup(RDMAContext *rdma) 
        rdma_destroy_id(rdma->cm_id);

        rdma->cm_id = NULL;

    }



    /* the destination side, listen_id and channel is shared */

    if (rdma->listen_id) {

        if (!rdma->is_return_path) {

            rdma_destroy_id(rdma->listen_id);

        }

        rdma->listen_id = NULL;



        if (rdma->channel) {

            if (!rdma->is_return_path) {

                rdma_destroy_event_channel(rdma->channel);

            }

            rdma->channel = NULL;

        }

    }



    if (rdma->channel) {

        rdma_destroy_event_channel(rdma->channel);

        rdma->channel = NULL;
@@ -2555,6 +2571,25 @@ err_dest_init_create_listen_id: 


}



static void qemu_rdma_return_path_dest_init(RDMAContext *rdma_return_path,

                                            RDMAContext *rdma)

{

    int idx;



    for (idx = 0; idx < RDMA_WRID_MAX; idx++) {

        rdma_return_path->wr_data[idx].control_len = 0;

        rdma_return_path->wr_data[idx].control_curr = NULL;

    }



    /*the CM channel and CM id is shared*/

    rdma_return_path->channel = rdma->channel;

    rdma_return_path->listen_id = rdma->listen_id;



    rdma->return_path = rdma_return_path;

    rdma_return_path->return_path = rdma;

    rdma_return_path->is_return_path = true;

}



static void *qemu_rdma_data_init(const char *host_port, Error **errp)

{

    RDMAContext *rdma = NULL;
@@ -3012,6 +3047,8 @@ err: 
    return ret;

}



static void rdma_accept_incoming_migration(void *opaque);



static int qemu_rdma_accept(RDMAContext *rdma)

{

    RDMACapabilities cap;
@@ -3106,7 +3143,14 @@ static int qemu_rdma_accept(RDMAContext *rdma) 
        }

    }



    /* Accept the second connection request for return path */

    if (migrate_postcopy() && !rdma->is_return_path) {

        qemu_set_fd_handler(rdma->channel->fd, rdma_accept_incoming_migration,

                            NULL,

                            (void *)(intptr_t)rdma->return_path);

    } else {

        qemu_set_fd_handler(rdma->channel->fd, NULL, NULL, NULL);

    }



    ret = rdma_accept(rdma->cm_id, &conn_param);

    if (ret) {
@@ -3691,6 +3735,10 @@ static void rdma_accept_incoming_migration(void *opaque) 


    trace_qemu_rdma_accept_incoming_migration_accepted();



    if (rdma->is_return_path) {

        return;

    }



    f = qemu_fopen_rdma(rdma, "rb");

    if (f == NULL) {

        ERROR(errp, "could not qemu_fopen_rdma!");
@@ -3705,7 +3753,7 @@ static void rdma_accept_incoming_migration(void *opaque) 
void rdma_start_incoming_migration(const char *host_port, Error **errp)

{

    int ret;

    RDMAContext *rdma;

    RDMAContext *rdma, *rdma_return_path;

    Error *local_err = NULL;



    trace_rdma_start_incoming_migration();
@@ -3732,12 +3780,24 @@ void rdma_start_incoming_migration(const char *host_port, Error **errp) 


    trace_rdma_start_incoming_migration_after_rdma_listen();



    /* initialize the RDMAContext for return path */

    if (migrate_postcopy()) {

        rdma_return_path = qemu_rdma_data_init(host_port, &local_err);



        if (rdma_return_path == NULL) {

            goto err;

        }



        qemu_rdma_return_path_dest_init(rdma_return_path, rdma);

    }



    qemu_set_fd_handler(rdma->channel->fd, rdma_accept_incoming_migration,

                        NULL, (void *)(intptr_t)rdma);

    return;

err:

    error_propagate(errp, local_err);

    g_free(rdma);

    g_free(rdma_return_path);

}



void rdma_start_outgoing_migration(void *opaque,
@@ -3745,6 +3805,7 @@ void rdma_start_outgoing_migration(void *opaque, 
{

    MigrationState *s = opaque;

    RDMAContext *rdma = qemu_rdma_data_init(host_port, errp);

    RDMAContext *rdma_return_path = NULL;

    int ret = 0;



    if (rdma == NULL) {
@@ -3765,6 +3826,32 @@ void rdma_start_outgoing_migration(void *opaque, 
        goto err;

    }



    /* RDMA postcopy need a seprate queue pair for return path */

    if (migrate_postcopy()) {

        rdma_return_path = qemu_rdma_data_init(host_port, errp);



        if (rdma_return_path == NULL) {

            goto err;

        }



        ret = qemu_rdma_source_init(rdma_return_path,

            s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL], errp);



        if (ret) {

            goto err;

        }



        ret = qemu_rdma_connect(rdma_return_path, errp);



        if (ret) {

            goto err;

        }



        rdma->return_path = rdma_return_path;

        rdma_return_path->return_path = rdma;

        rdma_return_path->is_return_path = true;

    }



    trace_rdma_start_outgoing_migration_after_rdma_connect();



    s->to_dst_file = qemu_fopen_rdma(rdma, "wb");
@@ -3772,4 +3859,5 @@ void rdma_start_outgoing_migration(void *opaque, 
    return;

err:

    g_free(rdma);

    g_free(rdma_return_path);

}