Commit fb0bc835 authored by Markus Armbruster's avatar Markus Armbruster Committed by Eric Blake
Browse files

qapi-gen: New common driver for code and doc generators 



Whenever qapi-schema.json changes, we run six programs eleven times to
update eleven files.  Similar for qga/qapi-schema.json.  This is
silly.  Replace the six programs by a single program that spits out
all eleven files.

The programs become modules in new Python package qapi, along with the
helper library.  This requires moving them to scripts/qapi/.  While
moving them, consistently drop executable mode bits.

Signed-off-by: default avatarMarkus Armbruster <armbru@redhat.com>
Reviewed-by: default avatarMarc-André Lureau <marcandre.lureau@redhat.com>
Message-Id: <20180211093607.27351-9-armbru@redhat.com>
Reviewed-by: default avatarEric Blake <eblake@redhat.com>
Reviewed-by: default avatarMichael Roth <mdroth@linux.vnet.ibm.com>
[eblake: move change to one-line 'blurb' earlier in series, mention mode
bit change as intentional, update qapi-code-gen.txt to match actual
generated events.c file]
Signed-off-by: default avatarEric Blake <eblake@redhat.com>
parent 26df4e7f

.gitignore

+2 −0
Original line number Diff line number Diff line
@@ -28,9 +28,11 @@ 
/linux-headers/asm

/qga/qapi-generated

/qapi-generated

/qapi-gen-timestamp

/qapi-types.[ch]

/qapi-visit.[ch]

/qapi-event.[ch]

/qapi-doc.texi

/qmp-commands.h

/qmp-introspect.[ch]

/qmp-marshal.c

Makefile

+38 −50
Original line number Diff line number Diff line
@@ -94,6 +94,7 @@ GENERATED_FILES += qmp-commands.h qapi-types.h qapi-visit.h qapi-event.h 
GENERATED_FILES += qmp-marshal.c qapi-types.c qapi-visit.c qapi-event.c

GENERATED_FILES += qmp-introspect.h

GENERATED_FILES += qmp-introspect.c

GENERATED_FILES += qapi-doc.texi



GENERATED_FILES += trace/generated-tcg-tracers.h
@@ -482,25 +483,26 @@ qemu-ga$(EXESUF): QEMU_CFLAGS += -I qga/qapi-generated 
qemu-keymap$(EXESUF): LIBS += $(XKBCOMMON_LIBS)

qemu-keymap$(EXESUF): QEMU_CFLAGS += $(XKBCOMMON_CFLAGS)



gen-out-type = $(subst .,-,$(suffix $@))



qapi-py = $(SRC_PATH)/scripts/qapi.py $(SRC_PATH)/scripts/ordereddict.py



qga/qapi-generated/qga-qapi-types.c qga/qapi-generated/qga-qapi-types.h :\

$(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-types.py \

		$(gen-out-type) -o qga/qapi-generated -p "qga-" $<, \

		"GEN","$@")

qga/qapi-generated/qga-qapi-visit.c qga/qapi-generated/qga-qapi-visit.h :\

$(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-visit.py \

		$(gen-out-type) -o qga/qapi-generated -p "qga-" $<, \

		"GEN","$@")

qga/qapi-generated/qga-qmp-commands.h qga/qapi-generated/qga-qmp-marshal.c :\

$(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-commands.py \

		$(gen-out-type) -o qga/qapi-generated -p "qga-" $<, \

		"GEN","$@")

qapi-py = $(SRC_PATH)/scripts/qapi/commands.py \

$(SRC_PATH)/scripts/qapi/events.py \

$(SRC_PATH)/scripts/qapi/introspect.py \

$(SRC_PATH)/scripts/qapi/types.py \

$(SRC_PATH)/scripts/qapi/visit.py \

$(SRC_PATH)/scripts/qapi/common.py \

$(SRC_PATH)/scripts/qapi/doc.py \

$(SRC_PATH)/scripts/ordereddict.py \

$(SRC_PATH)/scripts/qapi-gen.py



qga/qapi-generated/qga-qapi-types.c qga/qapi-generated/qga-qapi-types.h \

qga/qapi-generated/qga-qapi-visit.c qga/qapi-generated/qga-qapi-visit.h \

qga/qapi-generated/qga-qmp-commands.h qga/qapi-generated/qga-qmp-marshal.c \

qga/qapi-generated/qga-qapi-doc.texi: \

qga/qapi-generated/qapi-gen-timestamp ;

qga/qapi-generated/qapi-gen-timestamp: $(SRC_PATH)/qga/qapi-schema.json $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-gen.py \

		-o qga/qapi-generated -p "qga-" $<, \

		"GEN","$(@:%-timestamp=%)")

	@>$@



qapi-modules = $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/qapi/common.json \

               $(SRC_PATH)/qapi/block.json $(SRC_PATH)/qapi/block-core.json \
@@ -517,31 +519,18 @@ qapi-modules = $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/qapi/common.json \ 
               $(SRC_PATH)/qapi/transaction.json \

               $(SRC_PATH)/qapi/ui.json



qapi-types.c qapi-types.h :\

$(qapi-modules) $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-types.py \

		$(gen-out-type) -o "." -b $<, \

		"GEN","$@")

qapi-visit.c qapi-visit.h :\

$(qapi-modules) $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-visit.py \

		$(gen-out-type) -o "." -b $<, \

		"GEN","$@")

qapi-event.c qapi-event.h :\

$(qapi-modules) $(SRC_PATH)/scripts/qapi-event.py $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-event.py \

		$(gen-out-type) -o "." $<, \

		"GEN","$@")

qmp-commands.h qmp-marshal.c :\

$(qapi-modules) $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-commands.py \

		$(gen-out-type) -o "." $<, \

		"GEN","$@")

qmp-introspect.h qmp-introspect.c :\

$(qapi-modules) $(SRC_PATH)/scripts/qapi-introspect.py $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-introspect.py \

		$(gen-out-type) -o "." $<, \

		"GEN","$@")

qapi-types.c qapi-types.h \

qapi-visit.c qapi-visit.h \

qmp-commands.h qmp-marshal.c \

qapi-event.c qapi-event.h \

qmp-introspect.h qmp-introspect.c \

qapi-doc.texi: \

qapi-gen-timestamp ;

qapi-gen-timestamp: $(qapi-modules) $(qapi-py)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-gen.py \

		-o "." -b $<, \

		"GEN","$(@:%-timestamp=%)")

	@>$@



QGALIB_GEN=$(addprefix qga/qapi-generated/, qga-qapi-types.h qga-qapi-visit.h qga-qmp-commands.h)

$(qga-obj-y): $(QGALIB_GEN)
@@ -601,6 +590,7 @@ clean: 
	rm -f trace/generated-tracers-dtrace.dtrace*

	rm -f trace/generated-tracers-dtrace.h*

	rm -f $(foreach f,$(GENERATED_FILES),$(f) $(f)-timestamp)

	rm -f qapi-gen-timestamp

	rm -rf qapi-generated

	rm -rf qga/qapi-generated

	for d in $(ALL_SUBDIRS); do \
@@ -809,13 +799,11 @@ qemu-monitor-info.texi: $(SRC_PATH)/hmp-commands-info.hx $(SRC_PATH)/scripts/hxt 
qemu-img-cmds.texi: $(SRC_PATH)/qemu-img-cmds.hx $(SRC_PATH)/scripts/hxtool

	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"GEN","$@")



docs/interop/qemu-qmp-qapi.texi docs/interop/qemu-ga-qapi.texi: $(SRC_PATH)/scripts/qapi2texi.py $(qapi-py)



docs/interop/qemu-qmp-qapi.texi: $(qapi-modules)

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi2texi.py $< > $@,"GEN","$@")

docs/interop/qemu-qmp-qapi.texi: qapi-doc.texi

	@cp -p $< $@



docs/interop/qemu-ga-qapi.texi: $(SRC_PATH)/qga/qapi-schema.json

	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi2texi.py $< > $@,"GEN","$@")

docs/interop/qemu-ga-qapi.texi: qga/qapi-generated/qga-qapi-doc.texi

	@cp -p $< $@



qemu.1: qemu-doc.texi qemu-options.texi qemu-monitor.texi qemu-monitor-info.texi

qemu.1: qemu-option-trace.texi

docs/devel/qapi-code-gen.txt

+48 −54
Original line number Diff line number Diff line
@@ -899,12 +899,13 @@ the names of built-in types. Clients should examine member 


== Code generation ==



Schemas are fed into five scripts to generate all the code/files that,

paired with the core QAPI libraries, comprise everything required to

take JSON commands read in by a Client JSON Protocol server, unmarshal

the arguments into the underlying C types, call into the corresponding

C function, map the response back to a Client JSON Protocol response

to be returned to the user, and introspect the commands.

The QAPI code generator qapi-gen.py generates code and documentation

from the schema.  Together with the core QAPI libraries, this code

provides everything required to take JSON commands read in by a Client

JSON Protocol server, unmarshal the arguments into the underlying C

types, call into the corresponding C function, map the response back

to a Client JSON Protocol response to be returned to the user, and

introspect the commands.



As an example, we'll use the following schema, which describes a

single complex user-defined type, along with command which takes a
@@ -922,18 +923,23 @@ qmp_my_command(); everything else is produced by the generator. 


    { 'event': 'MY_EVENT' }



We run qapi-gen.py like this:



    $ python scripts/qapi-gen.py --output-dir="qapi-generated" \

    --prefix="example-" example-schema.json



For a more thorough look at generated code, the testsuite includes

tests/qapi-schema/qapi-schema-tests.json that covers more examples of

what the generator will accept, and compiles the resulting C code as

part of 'make check-unit'.



=== scripts/qapi-types.py ===

=== Code generated for QAPI types ===



Used to generate the C types defined by a schema, along with

supporting code. The following files are created:

The following files are created:



$(prefix)qapi-types.h - C types corresponding to types defined in

                        the schema you pass in

                        the schema



$(prefix)qapi-types.c - Cleanup functions for the above C types



The $(prefix) is an optional parameter used as a namespace to keep the
@@ -943,8 +949,6 @@ created code. 


Example:



    $ python scripts/qapi-types.py --output-dir="qapi-generated" \

    --prefix="example-" example-schema.json

    $ cat qapi-generated/example-qapi-types.h

[Uninteresting stuff omitted...]
@@ -1008,28 +1012,26 @@ Example: 
        visit_free(v);

    }



=== scripts/qapi-visit.py ===

=== Code generated for visiting QAPI types ===



Used to generate the visitor functions used to walk through and

convert between a native QAPI C data structure and some other format

(such as QObject); the generated functions are named visit_type_FOO()

and visit_type_FOO_members().

These are the visitor functions used to walk through and convert

between a native QAPI C data structure and some other format (such as

QObject); the generated functions are named visit_type_FOO() and

visit_type_FOO_members().



The following files are generated:



$(prefix)qapi-visit.c: visitor function for a particular C type, used

$(prefix)qapi-visit.c: Visitor function for a particular C type, used

                       to automagically convert QObjects into the

                       corresponding C type and vice-versa, as well

                       as for deallocating memory for an existing C

                       type



$(prefix)qapi-visit.h: declarations for previously mentioned visitor

$(prefix)qapi-visit.h: Declarations for previously mentioned visitor

                       functions



Example:



    $ python scripts/qapi-visit.py --output-dir="qapi-generated"

    --prefix="example-" example-schema.json

    $ cat qapi-generated/example-qapi-visit.h

[Uninteresting stuff omitted...]
@@ -1137,30 +1139,22 @@ Example: 
        error_propagate(errp, err);

    }



=== scripts/qapi-commands.py ===

=== Code generated for commands ===



These are the marshaling/dispatch functions for the commands defined

in the schema.  The generated code provides qmp_marshal_COMMAND(), and

declares qmp_COMMAND() that the user must implement.



Used to generate the marshaling/dispatch functions for the commands

defined in the schema. The generated code implements

qmp_marshal_COMMAND() (registered automatically), and declares

qmp_COMMAND() that the user must implement.  The following files are

generated:

The following files are generated:



$(prefix)qmp-marshal.c: command marshal/dispatch functions for each

                        QMP command defined in the schema. Functions

                        generated by qapi-visit.py are used to

                        convert QObjects received from the wire into

                        function parameters, and uses the same

                        visitor functions to convert native C return

                        values to QObjects from transmission back

                        over the wire.

$(prefix)qmp-marshal.c: Command marshal/dispatch functions for each

                        QMP command defined in the schema



$(prefix)qmp-commands.h: Function prototypes for the QMP commands

                         specified in the schema.

                         specified in the schema



Example:



    $ python scripts/qapi-commands.py --output-dir="qapi-generated"

    --prefix="example-" example-schema.json

    $ cat qapi-generated/example-qmp-commands.h

[Uninteresting stuff omitted...]
@@ -1242,20 +1236,20 @@ Example: 
                             qmp_marshal_my_command, QCO_NO_OPTIONS);

    }



=== scripts/qapi-event.py ===

=== Code generated for events ===



Used to generate the event-related C code defined by a schema, with

implementations for qapi_event_send_FOO(). The following files are

created:

This is the code related to events defined in the schema, providing

qapi_event_send_EVENT().



The following files are created:



$(prefix)qapi-event.h - Function prototypes for each event type, plus an

                        enumeration of all event names



$(prefix)qapi-event.c - Implementation of functions to send an event



Example:



    $ python scripts/qapi-event.py --output-dir="qapi-generated"

    --prefix="example-" example-schema.json

    $ cat qapi-generated/example-qapi-event.h

[Uninteresting stuff omitted...]
@@ -1301,24 +1295,24 @@ Example: 
        QDECREF(qmp);

    }



    const char *const example_QAPIEvent_lookup[] = {

    const QEnumLookup example_QAPIEvent_lookup = {

        .array = (const char *const[]) {

            [EXAMPLE_QAPI_EVENT_MY_EVENT] = "MY_EVENT",

        [EXAMPLE_QAPI_EVENT__MAX] = NULL,

        },

        .size = EXAMPLE_QAPI_EVENT__MAX

    };



=== scripts/qapi-introspect.py ===

=== Code generated for introspection ===



Used to generate the introspection C code for a schema. The following

files are created:

The following files are created:



$(prefix)qmp-introspect.c - Defines a string holding a JSON

                            description of the schema.

$(prefix)qmp-introspect.h - Declares the above string.

                            description of the schema



$(prefix)qmp-introspect.h - Declares the above string



Example:



    $ python scripts/qapi-introspect.py --output-dir="qapi-generated"

    --prefix="example-" example-schema.json

    $ cat qapi-generated/example-qmp-introspect.h

[Uninteresting stuff omitted...]

monitor.c

+1 −1
Original line number Diff line number Diff line
@@ -951,7 +951,7 @@ EventInfoList *qmp_query_events(Error **errp) 
 * visit_type_SchemaInfoList() into a SchemaInfoList, then marshal it

 * to QObject with generated output marshallers, every time.  Instead,

 * we do it in test-qobject-input-visitor.c, just to make sure

 * qapi-introspect.py's output actually conforms to the schema.

 * qapi-gen.py's output actually conforms to the schema.

 */

static void qmp_query_qmp_schema(QDict *qdict, QObject **ret_data,

                                 Error **errp)

qapi-schema.json

+1 −1
Original line number Diff line number Diff line
@@ -72,7 +72,7 @@ 
        'q_obj_CpuInfo-base'    # CPU, visible through query-cpu

    ] } }



# Documentation generated with qapi2texi.py is in source order, with

# Documentation generated with qapi-gen.py is in source order, with

# included sub-schemas inserted at the first include directive

# (subsequent include directives have no effect).  To get a sane and

# stable order, it's best to include each sub-schema just once, or