Commit e6f1306b authored by Alex Bennée's avatar Alex Bennée
Browse files

tests/docker: add --registry support to tooling 



This allows us to point the tools towards a registry from which they
can grab pre-built layers instead of doing everything from scratch
each time. To enable this we need to be using the DOCKER_BUILDKIT
engine.

Signed-off-by: default avatarAlex Bennée <alex.bennee@linaro.org>
Message-Id: <20200701135652.1366-25-alex.bennee@linaro.org>
parent 8a8a50a9

tests/docker/Makefile.include

+5 −1
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ DOCKER_IMAGES := $(sort $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.doc 
DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES))

# Use a global constant ccache directory to speed up repetitive builds

DOCKER_CCACHE_DIR := $$HOME/.cache/qemu-docker-ccache

DOCKER_REGISTRY := $(if $(REGISTRY),$(REGISTRY),registry.gitlab.com/qemu-project/qemu)



DOCKER_TESTS := $(notdir $(shell \

	find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f))
@@ -56,7 +57,9 @@ else 
docker-image-%: $(DOCKER_FILES_DIR)/%.docker

	$(call quiet-command,\

		$(DOCKER_SCRIPT) build -t qemu/$* -f $< \

		$(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \

		$(if $V,,--quiet) \

		$(if $(NOCACHE),--no-cache, \

			$(if $(DOCKER_REGISTRY),--registry $(DOCKER_REGISTRY))) \

		$(if $(NOUSER),,--add-current-user) \

		$(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES))\

		$(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\
@@ -213,6 +216,7 @@ endif 
	@echo '                         Include extra files in image.'

	@echo '    ENGINE=auto/docker/podman'

	@echo '                         Specify which container engine to run.'

	@echo '    REGISTRY=url         Cache builds from registry (default:$(DOCKER_REGISTRY))'



# This rule if for directly running against an arbitrary docker target.

# It is called by the expanded docker targets (e.g. make

tests/docker/docker.py

+38 −6
Original line number Diff line number Diff line
@@ -221,6 +221,13 @@ class Docker(object): 
    """ Running Docker commands """

    def __init__(self):

        self._command = _guess_engine_command()



        if "docker" in self._command and "TRAVIS" not in os.environ:

            os.environ["DOCKER_BUILDKIT"] = "1"

            self._buildkit = True

        else:

            self._buildkit = False



        self._instance = None

        atexit.register(self._kill_instances)

        signal.signal(signal.SIGTERM, self._kill_instances)
@@ -289,10 +296,24 @@ class Docker(object): 
        return labels.get("com.qemu.dockerfile-checksum", "")



    def build_image(self, tag, docker_dir, dockerfile,

                    quiet=True, user=False, argv=None, extra_files_cksum=[]):

                    quiet=True, user=False, argv=None, registry=None,

                    extra_files_cksum=[]):

        if argv is None:

            argv = []



        # pre-calculate the docker checksum before any

        # substitutions we make for caching

        checksum = _text_checksum(_dockerfile_preprocess(dockerfile))



        if registry is not None:

            dockerfile = dockerfile.replace("FROM qemu/",

                                            "FROM %s/qemu/" %

                                            (registry))

            # see if we can fetch a cache copy, may fail...

            pull_args = ["pull", "%s/%s" % (registry, tag)]

            self._do(pull_args, quiet=quiet)





        tmp_df = tempfile.NamedTemporaryFile(mode="w+t",

                                             encoding='utf-8',

                                             dir=docker_dir, suffix=".docker")
@@ -306,15 +327,23 @@ class Docker(object): 
                         (uname, uid, uname))



        tmp_df.write("\n")

        tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %

                     _text_checksum(_dockerfile_preprocess(dockerfile)))

        tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % (checksum))

        for f, c in extra_files_cksum:

            tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))



        tmp_df.flush()



        self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv +

                       [docker_dir],

        build_args = ["build", "-t", tag, "-f", tmp_df.name]

        if self._buildkit:

            build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"]



        if registry is not None:

            cache = "%s/%s" % (registry, tag)

            build_args += ["--cache-from", cache]

        build_args += argv

        build_args += [docker_dir]



        self._do_check(build_args,

                       quiet=quiet)



    def update_image(self, tag, tarball, quiet=True):
@@ -403,6 +432,8 @@ class BuildCommand(SubCommand): 
        parser.add_argument("--add-current-user", "-u", dest="user",

                            action="store_true",

                            help="Add the current user to image's passwd")

        parser.add_argument("--registry", "-r",

                            help="cache from docker registry")

        parser.add_argument("-t", dest="tag",

                            help="Image Tag")

        parser.add_argument("-f", dest="dockerfile",
@@ -458,7 +489,8 @@ class BuildCommand(SubCommand): 
                     for k, v in os.environ.items()

                     if k.lower() in FILTERED_ENV_NAMES]

            dkr.build_image(tag, docker_dir, dockerfile,

                            quiet=args.quiet, user=args.user, argv=argv,

                            quiet=args.quiet, user=args.user,

                            argv=argv, registry=args.registry,

                            extra_files_cksum=cksum)



            rmtree(docker_dir)