Commit 0a7052bb authored by Peter Maydell's avatar Peter Maydell
Browse files

Merge remote-tracking branch 'remotes/stsquad/tags/pull-docker-fixes-for-3.0-240718-1' into staging 



docker fixes & tcg test tweak

  - graceful handling of testing under cross-compile
  - fixes for debootstrap handling
  - more helpful errors (binfmt_misc/EXECUTABLE missing)
  - drop runcom TCG test

# gpg: Signature made Tue 24 Jul 2018 11:48:32 BST
# gpg:                using RSA key FBD0DB095A9E2A44
# gpg: Good signature from "Alex Bennée (Master Work Key) <alex.bennee@linaro.org>"
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8  DF35 FBD0 DB09 5A9E 2A44

* remotes/stsquad/tags/pull-docker-fixes-for-3.0-240718-1:
  tests/tcg: remove runcom test
  docker: perform basic binfmt_misc validation in docker.py
  docker: ignore distro versioning of debootstrap
  docker: add commentary to debian-bootstrap.docker
  docker: Update debootstrap script after Debian migration from Alioth to Salsa
  docker: report hint when docker.py check fails
  docker: drop QEMU_TARGET check, fallback in EXECUTABLE not set
  docker: add expansion for docker-test-FOO to Makefile.include
  docker: add test-unit runner
  docker: Makefile.include don't include partial images
  docker: gracefully skip check_qemu
  docker: move make check into check_qemu helper
  docker: split configure_qemu from build_qemu
  docker: fail more gracefully on docker.py check
  docker: par down QEMU_CONFIGURE_OPTS in debian-tricore-cross
  docker: base debian-tricore on qemu:debian9
  tests/.gitignore: don't ignore docker tests

Signed-off-by: default avatarPeter Maydell <peter.maydell@linaro.org>
parents aabbca41 1a3bdc61

tests/.gitignore

+1 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ qht-bench 
rcutorture

test-*

!test-*.c

!docker/test-*

test-qapi-commands.[ch]

test-qapi-events.[ch]

test-qapi-types.[ch]

tests/docker/Makefile.include

+14 −9
Original line number Diff line number Diff line
@@ -6,7 +6,7 @@ DOCKER_SUFFIX := .docker 
DOCKER_FILES_DIR := $(SRC_PATH)/tests/docker/dockerfiles

DOCKER_DEPRECATED_IMAGES := debian

# we don't run tests on intermediate images (used as base by another image)

DOCKER_INTERMEDIATE_IMAGES := debian8 debian9 debian8-mxe debian-ports debian-sid

DOCKER_PARTIAL_IMAGES := debian debian8 debian9 debian8-mxe debian-ports debian-sid debian-bootstrap

DOCKER_IMAGES := $(filter-out $(DOCKER_DEPRECATED_IMAGES),$(sort $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.docker)))))

DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES))

# Use a global constant ccache directory to speed up repetitive builds
@@ -58,13 +58,11 @@ docker-image-%: $(DOCKER_FILES_DIR)/%.docker 
docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker

	$(if $(EXECUTABLE),,\

		$(error EXECUTABLE not set, debootstrap of debian-$* would fail))

	$(if $(wildcard $(EXECUTABLE)),,\

		$(error Please build $(EXECUTABLE) first))

	$(if $(DEB_ARCH),,\

		$(error DEB_ARCH not set, debootstrap of debian-$* would fail))

	$(if $(DEB_TYPE),,\

		$(error DEB_TYPE not set, debootstrap of debian-$* would fail))

	$(if $(filter $(QEMU_TARGET),$(TARGET_DIRS)),				\

	$(if $(wildcard $(EXECUTABLE)),						\

		$(call quiet-command,						\

			DEB_ARCH=$(DEB_ARCH)					\

			DEB_TYPE=$(DEB_TYPE)					\
@@ -75,7 +73,8 @@ docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker 
			$(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)), \

			"BUILD","binfmt debian-$* (debootstrapped)"),		\

		$(call quiet-command,						\

			$(DOCKER_SCRIPT) check --quiet qemu:debian-$* $<,	\

			$(DOCKER_SCRIPT) check --quiet qemu:debian-$* $< || 	\

			{ echo "You will need to build $(EXECUTABLE)"; exit 1;},\

			"CHECK", "debian-$* exists"))



endif
@@ -121,6 +120,11 @@ docker-image-travis: NOUSER=1 
# Specialist build images, sometimes very limited tools

docker-image-tricore-cross: docker-image-debian9



# These images may be good enough for building tests but not for test builds

DOCKER_PARTIAL_IMAGES += debian-alpha-cross debian-hppa-cross debian-m68k-cross debian-sh4-cross

DOCKER_PARTIAL_IMAGES += debian-sparc64-cross debian-mips64-cross debian-riscv64-cross

DOCKER_PARTIAL_IMAGES += debian-tricore-cross debian-powerpc-cross fedora-i386-cross



# Rules for building linux-user powered images

#

# These are slower than using native cross compiler setups but can
@@ -131,19 +135,19 @@ docker-image-tricore-cross: docker-image-debian9 
# broken so we need a qemu-linux-user for this target

docker-binfmt-image-debian-powerpc-user: DEB_ARCH = powerpc

docker-binfmt-image-debian-powerpc-user: DEB_TYPE = jessie

docker-binfmt-image-debian-powerpc-user: QEMU_TARGET = ppc-linux-user

docker-binfmt-image-debian-powerpc-user: EXECUTABLE = ${BUILD_DIR}/ppc-linux-user/qemu-ppc

docker-image-debian-powerpc-user-cross: docker-binfmt-image-debian-powerpc-user

DOCKER_USER_IMAGES += debian-powerpc-user



# Expand all the pre-requistes for each docker image and test combination

$(foreach i,$(filter-out $(DOCKER_INTERMEDIATE_IMAGES),$(DOCKER_IMAGES) $(DOCKER_DEPRECATED_IMAGES)), \

$(foreach i,$(filter-out $(DOCKER_PARTIAL_IMAGES),$(DOCKER_IMAGES) $(DOCKER_DEPRECATED_IMAGES)), \

	$(foreach t,$(DOCKER_TESTS) $(DOCKER_TOOLS), \

		$(eval .PHONY: docker-$t@$i) \

		$(eval docker-$t@$i: docker-image-$i docker-run-$t@$i) \

	) \

	$(foreach t,$(DOCKER_TESTS), \

		$(eval docker-test: docker-$t@$i) \

		$(eval docker-all-tests: docker-$t@$i) \

		$(eval docker-$t: docker-$t@$i) \

	) \

)
@@ -153,7 +157,8 @@ docker: 
	@echo 'Available targets:'

	@echo

	@echo '    docker:              Print this help.'

	@echo '    docker-test:         Run all image/test combinations.'

	@echo '    docker-all-tests:    Run all image/test combinations.'

	@echo '    docker-TEST:         Run TEST on all image combinations.'

	@echo '    docker-clean:        Kill and remove residual docker testing containers.'

	@echo '    docker-TEST@IMAGE:   Run "TEST" in container "IMAGE".'

	@echo '                         Note: "TEST" is one of the listed test name,'

tests/docker/common.rc

+23 −1
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ requires() 
    done

}



build_qemu()

configure_qemu()

{

    config_opts="--enable-werror \

                 ${TARGET_LIST:+--target-list=${TARGET_LIST}} \
@@ -32,9 +32,31 @@ build_qemu() 
    echo $config_opts

    $QEMU_SRC/configure $config_opts || \

        { cat config.log && test_fail "Failed to run 'configure'"; }

}



build_qemu()

{

    configure_qemu $@

    make $MAKEFLAGS

}



check_qemu()

{

    # default to make check unless the caller specifies

    if test -z "$@"; then

        INVOCATION="check"

    else

        INVOCATION="$@"

    fi



    if command -v gtester > /dev/null 2>&1 && \

           gtester --version > /dev/null 2>&1; then

        make $MAKEFLAGS $INVOCATION

    else

        echo "No working gtester, skipping make $INVOCATION"

    fi

}



test_fail()

{

    echo "$@"

tests/docker/docker.py

+36 −1
Original line number Diff line number Diff line
@@ -112,6 +112,31 @@ def _copy_binary_with_libs(src, dest_dir): 
            so_path = os.path.dirname(l)

            _copy_with_mkdir(l , dest_dir, so_path)





def _check_binfmt_misc(executable):

    """Check binfmt_misc has entry for executable in the right place.



    The details of setting up binfmt_misc are outside the scope of

    this script but we should at least fail early with a useful

    message if it won't work."""



    binary = os.path.basename(executable)

    binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)



    if not os.path.exists(binfmt_entry):

        print ("No binfmt_misc entry for %s" % (binary))

        return False



    with open(binfmt_entry) as x: entry = x.read()



    qpath = "/usr/bin/%s" % (binary)

    if not re.search("interpreter %s\n" % (qpath), entry):

        print ("binfmt_misc for %s does not point to %s" % (binary, qpath))

        return False



    return True





def _read_qemu_dockerfile(img_name):

    # special case for Debian linux-user images

    if img_name.startswith("debian") and img_name.endswith("user"):
@@ -315,6 +340,11 @@ class BuildCommand(SubCommand): 
            # Create a docker context directory for the build

            docker_dir = tempfile.mkdtemp(prefix="docker_build")



            # Validate binfmt_misc will work

            if args.include_executable:

                if not _check_binfmt_misc(args.include_executable):

                    return 1



            # Is there a .pre file to run in the build context?

            docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"

            if os.path.exists(docker_pre):
@@ -479,7 +509,12 @@ class CheckCommand(SubCommand): 
    def run(self, args, argv):

        tag = args.tag



        try:

            dkr = Docker()

        except:

            print("Docker not set up")

            return 1



        info = dkr.inspect_tag(tag)

        if info is None:

            print("Image does not exist")

tests/docker/dockerfiles/debian-bootstrap.docker

+1 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ FROM scratch 
ADD . /



# Patch all mounts as docker already has stuff set up

# (this is not needed for later debootstraps but is harmless atm)

RUN sed -i 's/in_target mount/echo not for docker in_target mount/g' /debootstrap/functions



# Run stage 2