Commit 603476c2 authored by Daniel P. Berrangé's avatar Daniel P. Berrangé Committed by Markus Armbruster
Browse files

qdict: implement a qdict_crumple method for un-flattening a dict



The qdict_flatten() method will take a dict whose elements are
further nested dicts/lists and flatten them by concatenating
keys.

The qdict_crumple() method aims to do the reverse, taking a flat
qdict, and turning it into a set of nested dicts/lists. It will
apply nesting based on the key name, with a '.' indicating a
new level in the hierarchy. If the keys in the nested structure
are all numeric, it will create a list, otherwise it will create
a dict.

If the keys are a mixture of numeric and non-numeric, or the
numeric keys are not in strictly ascending order, an error will
be reported.

As an example, a flat dict containing

 {
   'foo.0.bar': 'one',
   'foo.0.wizz': '1',
   'foo.1.bar': 'two',
   'foo.1.wizz': '2'
 }

will get turned into a dict with one element 'foo' whose
value is a list. The list elements will each in turn be
dicts.

 {
   'foo': [
     { 'bar': 'one', 'wizz': '1' },
     { 'bar': 'two', 'wizz': '2' }
   ],
 }

If the key is intended to contain a literal '.', then it must
be escaped as '..'. ie a flat dict

  {
     'foo..bar': 'wizz',
     'bar.foo..bar': 'eek',
     'bar.hello': 'world'
  }

Will end up as

  {
     'foo.bar': 'wizz',
     'bar': {
        'foo.bar': 'eek',
        'hello': 'world'
     }
  }

The intent of this function is that it allows a set of QemuOpts
to be turned into a nested data structure that mirrors the nesting
used when the same object is defined over QMP.

Reviewed-by: default avatarEric Blake <eblake@redhat.com>
Reviewed-by: default avatarKevin Wolf <kwolf@redhat.com>
Reviewed-by: default avatarMarc-André Lureau <marcandre.lureau@redhat.com>
Signed-off-by: default avatarDaniel P. Berrange <berrange@redhat.com>
Message-Id: <1475246744-29302-3-git-send-email-berrange@redhat.com>
Reviewed-by: default avatarMarkus Armbruster <armbru@redhat.com>
[Parameter recursive dropped along with its tests; whitespace style
touched up]
Signed-off-by: default avatarMarkus Armbruster <armbru@redhat.com>
parent b1d2e5f1
Loading
Loading
Loading
Loading
+1 −0
Original line number Diff line number Diff line
@@ -73,6 +73,7 @@ void qdict_flatten(QDict *qdict);
void qdict_extract_subqdict(QDict *src, QDict **dst, const char *start);
void qdict_array_split(QDict *src, QList **dst);
int qdict_array_entries(QDict *src, const char *subqdict);
QObject *qdict_crumple(const QDict *src, Error **errp);

void qdict_join(QDict *dest, QDict *src, bool overwrite);

+277 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@
#include "qapi/qmp/qbool.h"
#include "qapi/qmp/qstring.h"
#include "qapi/qmp/qobject.h"
#include "qapi/error.h"
#include "qemu/queue.h"
#include "qemu-common.h"
#include "qemu/cutils.h"
@@ -683,6 +684,282 @@ void qdict_array_split(QDict *src, QList **dst)
    }
}

/**
 * qdict_split_flat_key:
 * @key: the key string to split
 * @prefix: non-NULL pointer to hold extracted prefix
 * @suffix: non-NULL pointer to remaining suffix
 *
 * Given a flattened key such as 'foo.0.bar', split it into two parts
 * at the first '.' separator. Allows double dot ('..') to escape the
 * normal separator.
 *
 * e.g.
 *    'foo.0.bar' -> prefix='foo' and suffix='0.bar'
 *    'foo..0.bar' -> prefix='foo.0' and suffix='bar'
 *
 * The '..' sequence will be unescaped in the returned 'prefix'
 * string. The 'suffix' string will be left in escaped format, so it
 * can be fed back into the qdict_split_flat_key() key as the input
 * later.
 *
 * The caller is responsible for freeing the string returned in @prefix
 * using g_free().
 */
static void qdict_split_flat_key(const char *key, char **prefix,
                                 const char **suffix)
{
    const char *separator;
    size_t i, j;

    /* Find first '.' separator, but if there is a pair '..'
     * that acts as an escape, so skip over '..' */
    separator = NULL;
    do {
        if (separator) {
            separator += 2;
        } else {
            separator = key;
        }
        separator = strchr(separator, '.');
    } while (separator && separator[1] == '.');

    if (separator) {
        *prefix = g_strndup(key, separator - key);
        *suffix = separator + 1;
    } else {
        *prefix = g_strdup(key);
        *suffix = NULL;
    }

    /* Unescape the '..' sequence into '.' */
    for (i = 0, j = 0; (*prefix)[i] != '\0'; i++, j++) {
        if ((*prefix)[i] == '.') {
            assert((*prefix)[i + 1] == '.');
            i++;
        }
        (*prefix)[j] = (*prefix)[i];
    }
    (*prefix)[j] = '\0';
}

/**
 * qdict_is_list:
 * @maybe_list: dict to check if keys represent list elements.
 *
 * Determine whether all keys in @maybe_list are valid list elements.
 * If @maybe_list is non-zero in length and all the keys look like
 * valid list indexes, this will return 1. If @maybe_list is zero
 * length or all keys are non-numeric then it will return 0 to indicate
 * it is a normal qdict. If there is a mix of numeric and non-numeric
 * keys, or the list indexes are non-contiguous, an error is reported.
 *
 * Returns: 1 if a valid list, 0 if a dict, -1 on error
 */
static int qdict_is_list(QDict *maybe_list, Error **errp)
{
    const QDictEntry *ent;
    ssize_t len = 0;
    ssize_t max = -1;
    int is_list = -1;
    int64_t val;

    for (ent = qdict_first(maybe_list); ent != NULL;
         ent = qdict_next(maybe_list, ent)) {

        if (qemu_strtoll(ent->key, NULL, 10, &val) == 0) {
            if (is_list == -1) {
                is_list = 1;
            } else if (!is_list) {
                error_setg(errp,
                           "Cannot mix list and non-list keys");
                return -1;
            }
            len++;
            if (val > max) {
                max = val;
            }
        } else {
            if (is_list == -1) {
                is_list = 0;
            } else if (is_list) {
                error_setg(errp,
                           "Cannot mix list and non-list keys");
                return -1;
            }
        }
    }

    if (is_list == -1) {
        assert(!qdict_size(maybe_list));
        is_list = 0;
    }

    /* NB this isn't a perfect check - e.g. it won't catch
     * a list containing '1', '+1', '01', '3', but that
     * does not matter - we've still proved that the
     * input is a list. It is up the caller to do a
     * stricter check if desired */
    if (len != (max + 1)) {
        error_setg(errp, "List indices are not contiguous, "
                   "saw %zd elements but %zd largest index",
                   len, max);
        return -1;
    }

    return is_list;
}

/**
 * qdict_crumple:
 * @src: the original flat dictionary (only scalar values) to crumple
 *
 * Takes a flat dictionary whose keys use '.' separator to indicate
 * nesting, and values are scalars, and crumples it into a nested
 * structure.
 *
 * To include a literal '.' in a key name, it must be escaped as '..'
 *
 * For example, an input of:
 *
 * { 'foo.0.bar': 'one', 'foo.0.wizz': '1',
 *   'foo.1.bar': 'two', 'foo.1.wizz': '2' }
 *
 * will result in an output of:
 *
 * {
 *   'foo': [
 *      { 'bar': 'one', 'wizz': '1' },
 *      { 'bar': 'two', 'wizz': '2' }
 *   ],
 * }
 *
 * The following scenarios in the input dict will result in an
 * error being returned:
 *
 *  - Any values in @src are non-scalar types
 *  - If keys in @src imply that a particular level is both a
 *    list and a dict. e.g., "foo.0.bar" and "foo.eek.bar".
 *  - If keys in @src imply that a particular level is a list,
 *    but the indices are non-contiguous. e.g. "foo.0.bar" and
 *    "foo.2.bar" without any "foo.1.bar" present.
 *  - If keys in @src represent list indexes, but are not in
 *    the "%zu" format. e.g. "foo.+0.bar"
 *
 * Returns: either a QDict or QList for the nested data structure, or NULL
 * on error
 */
QObject *qdict_crumple(const QDict *src, Error **errp)
{
    const QDictEntry *ent;
    QDict *two_level, *multi_level = NULL;
    QObject *dst = NULL, *child;
    size_t i;
    char *prefix = NULL;
    const char *suffix = NULL;
    int is_list;

    two_level = qdict_new();

    /* Step 1: split our totally flat dict into a two level dict */
    for (ent = qdict_first(src); ent != NULL; ent = qdict_next(src, ent)) {
        if (qobject_type(ent->value) == QTYPE_QDICT ||
            qobject_type(ent->value) == QTYPE_QLIST) {
            error_setg(errp, "Value %s is not a scalar",
                       ent->key);
            goto error;
        }

        qdict_split_flat_key(ent->key, &prefix, &suffix);

        child = qdict_get(two_level, prefix);
        if (suffix) {
            if (child) {
                if (qobject_type(child) != QTYPE_QDICT) {
                    error_setg(errp, "Key %s prefix is already set as a scalar",
                               prefix);
                    goto error;
                }
            } else {
                child = QOBJECT(qdict_new());
                qdict_put_obj(two_level, prefix, child);
            }
            qobject_incref(ent->value);
            qdict_put_obj(qobject_to_qdict(child), suffix, ent->value);
        } else {
            if (child) {
                error_setg(errp, "Key %s prefix is already set as a dict",
                           prefix);
                goto error;
            }
            qobject_incref(ent->value);
            qdict_put_obj(two_level, prefix, ent->value);
        }

        g_free(prefix);
        prefix = NULL;
    }

    /* Step 2: optionally process the two level dict recursively
     * into a multi-level dict */
    multi_level = qdict_new();
    for (ent = qdict_first(two_level); ent != NULL;
         ent = qdict_next(two_level, ent)) {

        if (qobject_type(ent->value) == QTYPE_QDICT) {
            child = qdict_crumple(qobject_to_qdict(ent->value), errp);
            if (!child) {
                goto error;
            }

            qdict_put_obj(multi_level, ent->key, child);
        } else {
            qobject_incref(ent->value);
            qdict_put_obj(multi_level, ent->key, ent->value);
        }
    }
    QDECREF(two_level);
    two_level = NULL;

    /* Step 3: detect if we need to turn our dict into list */
    is_list = qdict_is_list(multi_level, errp);
    if (is_list < 0) {
        goto error;
    }

    if (is_list) {
        dst = QOBJECT(qlist_new());

        for (i = 0; i < qdict_size(multi_level); i++) {
            char *key = g_strdup_printf("%zu", i);

            child = qdict_get(multi_level, key);
            g_free(key);

            if (!child) {
                error_setg(errp, "Missing list index %zu", i);
                goto error;
            }

            qobject_incref(child);
            qlist_append_obj(qobject_to_qlist(dst), child);
        }
        QDECREF(multi_level);
        multi_level = NULL;
    } else {
        dst = QOBJECT(multi_level);
    }

    return dst;

 error:
    g_free(prefix);
    QDECREF(multi_level);
    QDECREF(two_level);
    qobject_decref(dst);
    return NULL;
}

/**
 * qdict_array_entries(): Returns the number of direct array entries if the
 * sub-QDict of src specified by the prefix in subqdict (or src itself for
+150 −8
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@
#include "qapi/qmp/qint.h"
#include "qapi/qmp/qdict.h"
#include "qapi/qmp/qstring.h"
#include "qapi/error.h"
#include "qemu-common.h"

/*
@@ -412,7 +413,6 @@ static void qdict_array_split_test(void)

    QDECREF(test_dict);


    /*
     * Test the split of
     *
@@ -518,7 +518,6 @@ static void qdict_join_test(void)
    dict1 = qdict_new();
    dict2 = qdict_new();


    /* Test everything once without overwrite and once with */
    do
    {
@@ -528,7 +527,6 @@ static void qdict_join_test(void)
        g_assert(qdict_size(dict1) == 0);
        g_assert(qdict_size(dict2) == 0);


        /* First iteration: Test movement */
        /* Second iteration: Test empty source and non-empty destination */
        qdict_put(dict2, "foo", qint_from_int(42));
@@ -542,7 +540,6 @@ static void qdict_join_test(void)
            g_assert(qdict_get_int(dict1, "foo") == 42);
        }


        /* Test non-empty source and destination without conflict */
        qdict_put(dict2, "bar", qint_from_int(23));

@@ -554,7 +551,6 @@ static void qdict_join_test(void)
        g_assert(qdict_get_int(dict1, "foo") == 42);
        g_assert(qdict_get_int(dict1, "bar") == 23);


        /* Test conflict */
        qdict_put(dict2, "foo", qint_from_int(84));

@@ -570,7 +566,6 @@ static void qdict_join_test(void)
            g_assert(qdict_get_int(dict2, "foo") == 84);
        }


        /* Check the references */
        g_assert(qdict_get(dict1, "foo")->refcnt == 1);
        g_assert(qdict_get(dict1, "bar")->refcnt == 1);
@@ -579,7 +574,6 @@ static void qdict_join_test(void)
            g_assert(qdict_get(dict2, "foo")->refcnt == 1);
        }


        /* Clean up */
        qdict_del(dict1, "foo");
        qdict_del(dict1, "bar");
@@ -590,11 +584,152 @@ static void qdict_join_test(void)
    }
    while (overwrite ^= true);


    QDECREF(dict1);
    QDECREF(dict2);
}

static void qdict_crumple_test_recursive(void)
{
    QDict *src, *dst, *rule, *vnc, *acl, *listen;
    QObject *child, *res;
    QList *rules;

    src = qdict_new();
    qdict_put(src, "vnc.listen.addr", qstring_from_str("127.0.0.1"));
    qdict_put(src, "vnc.listen.port", qstring_from_str("5901"));
    qdict_put(src, "vnc.acl.rules.0.match", qstring_from_str("fred"));
    qdict_put(src, "vnc.acl.rules.0.policy", qstring_from_str("allow"));
    qdict_put(src, "vnc.acl.rules.1.match", qstring_from_str("bob"));
    qdict_put(src, "vnc.acl.rules.1.policy", qstring_from_str("deny"));
    qdict_put(src, "vnc.acl.default", qstring_from_str("deny"));
    qdict_put(src, "vnc.acl..name", qstring_from_str("acl0"));
    qdict_put(src, "vnc.acl.rule..name", qstring_from_str("acl0"));

    res = qdict_crumple(src, &error_abort);

    g_assert_cmpint(qobject_type(res), ==, QTYPE_QDICT);

    dst = qobject_to_qdict(res);

    g_assert_cmpint(qdict_size(dst), ==, 1);

    child = qdict_get(dst, "vnc");
    g_assert_cmpint(qobject_type(child), ==, QTYPE_QDICT);
    vnc = qobject_to_qdict(child);

    child = qdict_get(vnc, "listen");
    g_assert_cmpint(qobject_type(child), ==, QTYPE_QDICT);
    listen = qobject_to_qdict(child);
    g_assert_cmpstr("127.0.0.1", ==, qdict_get_str(listen, "addr"));
    g_assert_cmpstr("5901", ==, qdict_get_str(listen, "port"));

    child = qdict_get(vnc, "acl");
    g_assert_cmpint(qobject_type(child), ==, QTYPE_QDICT);
    acl = qobject_to_qdict(child);

    child = qdict_get(acl, "rules");
    g_assert_cmpint(qobject_type(child), ==, QTYPE_QLIST);
    rules = qobject_to_qlist(child);
    g_assert_cmpint(qlist_size(rules), ==, 2);

    rule = qobject_to_qdict(qlist_pop(rules));
    g_assert_cmpint(qdict_size(rule), ==, 2);
    g_assert_cmpstr("fred", ==, qdict_get_str(rule, "match"));
    g_assert_cmpstr("allow", ==, qdict_get_str(rule, "policy"));
    QDECREF(rule);

    rule = qobject_to_qdict(qlist_pop(rules));
    g_assert_cmpint(qdict_size(rule), ==, 2);
    g_assert_cmpstr("bob", ==, qdict_get_str(rule, "match"));
    g_assert_cmpstr("deny", ==, qdict_get_str(rule, "policy"));
    QDECREF(rule);

    /* With recursive crumpling, we should see all names unescaped */
    g_assert_cmpstr("acl0", ==, qdict_get_str(vnc, "acl.name"));
    child = qdict_get(vnc, "acl");
    g_assert_cmpint(qobject_type(child), ==, QTYPE_QDICT);
    acl = qdict_get_qdict(vnc, "acl");
    g_assert_cmpstr("acl0", ==, qdict_get_str(acl, "rule.name"));

    QDECREF(src);
    QDECREF(dst);
}

static void qdict_crumple_test_empty(void)
{
    QDict *src, *dst;

    src = qdict_new();

    dst = (QDict *)qdict_crumple(src, &error_abort);

    g_assert_cmpint(qdict_size(dst), ==, 0);

    QDECREF(src);
    QDECREF(dst);
}

static void qdict_crumple_test_bad_inputs(void)
{
    QDict *src;
    Error *error = NULL;

    src = qdict_new();
    /* rule.0 can't be both a string and a dict */
    qdict_put(src, "rule.0", qstring_from_str("fred"));
    qdict_put(src, "rule.0.policy", qstring_from_str("allow"));

    g_assert(qdict_crumple(src, &error) == NULL);
    g_assert(error != NULL);
    error_free(error);
    error = NULL;
    QDECREF(src);

    src = qdict_new();
    /* rule can't be both a list and a dict */
    qdict_put(src, "rule.0", qstring_from_str("fred"));
    qdict_put(src, "rule.a", qstring_from_str("allow"));

    g_assert(qdict_crumple(src, &error) == NULL);
    g_assert(error != NULL);
    error_free(error);
    error = NULL;
    QDECREF(src);

    src = qdict_new();
    /* The input should be flat, ie no dicts or lists */
    qdict_put(src, "rule.a", qdict_new());
    qdict_put(src, "rule.b", qstring_from_str("allow"));

    g_assert(qdict_crumple(src, &error) == NULL);
    g_assert(error != NULL);
    error_free(error);
    error = NULL;
    QDECREF(src);

    src = qdict_new();
    /* List indexes must not have gaps */
    qdict_put(src, "rule.0", qstring_from_str("deny"));
    qdict_put(src, "rule.3", qstring_from_str("allow"));

    g_assert(qdict_crumple(src, &error) == NULL);
    g_assert(error != NULL);
    error_free(error);
    error = NULL;
    QDECREF(src);

    src = qdict_new();
    /* List indexes must be in %zu format */
    qdict_put(src, "rule.0", qstring_from_str("deny"));
    qdict_put(src, "rule.+1", qstring_from_str("allow"));

    g_assert(qdict_crumple(src, &error) == NULL);
    g_assert(error != NULL);
    error_free(error);
    error = NULL;
    QDECREF(src);
}

/*
 * Errors test-cases
 */
@@ -742,6 +877,13 @@ int main(int argc, char **argv)
    g_test_add_func("/errors/put_exists", qdict_put_exists_test);
    g_test_add_func("/errors/get_not_exists", qdict_get_not_exists_test);

    g_test_add_func("/public/crumple/recursive",
                    qdict_crumple_test_recursive);
    g_test_add_func("/public/crumple/empty",
                    qdict_crumple_test_empty);
    g_test_add_func("/public/crumple/bad_inputs",
                    qdict_crumple_test_bad_inputs);

    /* The Big one */
    if (g_test_slow()) {
        g_test_add_func("/stress/test", qdict_stress_test);