From 767b6bd22bc94287e88367dc4ec5c7f9a765c603 Mon Sep 17 00:00:00 2001 From: Alex Bennée Date: Wed, 1 Jul 2020 14:56:29 +0100 Subject: tests/docker: change tag naming scheme of our images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We've been misusing the tag naming scheme for some time by overloading the post : section with the image type. Really it should be saved for the revision of that particular build. Move the details to the other side so we have: qemu/image-name with the implied :latest version added by the tooling. Suggested-by: Daniel P. Berrangé Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20200701135652.1366-18-alex.bennee@linaro.org> --- tests/docker/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/docker/docker.py') diff --git a/tests/docker/docker.py b/tests/docker/docker.py index e630aae108..cc6f76caa6 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -204,7 +204,7 @@ def _dockerfile_preprocess(df): for l in df.splitlines(): if len(l.strip()) == 0 or l.startswith("#"): continue - from_pref = "FROM qemu:" + from_pref = "FROM qemu/" if l.startswith(from_pref): # TODO: Alternatively we could replace this line with "FROM $ID" # where $ID is the image's hex id obtained with -- cgit 1.4.1 From e6f1306b1003e2ce4e63c8e5dbe362d27fb8766c Mon Sep 17 00:00:00 2001 From: Alex Bennée Date: Wed, 1 Jul 2020 14:56:36 +0100 Subject: tests/docker: add --registry support to tooling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to point the tools towards a registry from which they can grab pre-built layers instead of doing everything from scratch each time. To enable this we need to be using the DOCKER_BUILDKIT engine. Signed-off-by: Alex Bennée Message-Id: <20200701135652.1366-25-alex.bennee@linaro.org> --- tests/docker/Makefile.include | 6 +++++- tests/docker/docker.py | 44 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 43 insertions(+), 7 deletions(-) (limited to 'tests/docker/docker.py') diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index e23b4af20e..a26177abc2 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -13,6 +13,7 @@ DOCKER_IMAGES := $(sort $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.doc DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES)) # Use a global constant ccache directory to speed up repetitive builds DOCKER_CCACHE_DIR := $$HOME/.cache/qemu-docker-ccache +DOCKER_REGISTRY := $(if $(REGISTRY),$(REGISTRY),registry.gitlab.com/qemu-project/qemu) DOCKER_TESTS := $(notdir $(shell \ find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f)) @@ -56,7 +57,9 @@ else docker-image-%: $(DOCKER_FILES_DIR)/%.docker $(call quiet-command,\ $(DOCKER_SCRIPT) build -t qemu/$* -f $< \ - $(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \ + $(if $V,,--quiet) \ + $(if $(NOCACHE),--no-cache, \ + $(if $(DOCKER_REGISTRY),--registry $(DOCKER_REGISTRY))) \ $(if $(NOUSER),,--add-current-user) \ $(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES))\ $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\ @@ -213,6 +216,7 @@ endif @echo ' Include extra files in image.' @echo ' ENGINE=auto/docker/podman' @echo ' Specify which container engine to run.' + @echo ' REGISTRY=url Cache builds from registry (default:$(DOCKER_REGISTRY))' # This rule if for directly running against an arbitrary docker target. # It is called by the expanded docker targets (e.g. make diff --git a/tests/docker/docker.py b/tests/docker/docker.py index cc6f76caa6..9684f07bde 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -221,6 +221,13 @@ class Docker(object): """ Running Docker commands """ def __init__(self): self._command = _guess_engine_command() + + if "docker" in self._command and "TRAVIS" not in os.environ: + os.environ["DOCKER_BUILDKIT"] = "1" + self._buildkit = True + else: + self._buildkit = False + self._instance = None atexit.register(self._kill_instances) signal.signal(signal.SIGTERM, self._kill_instances) @@ -289,10 +296,24 @@ class Docker(object): return labels.get("com.qemu.dockerfile-checksum", "") def build_image(self, tag, docker_dir, dockerfile, - quiet=True, user=False, argv=None, extra_files_cksum=[]): + quiet=True, user=False, argv=None, registry=None, + extra_files_cksum=[]): if argv is None: argv = [] + # pre-calculate the docker checksum before any + # substitutions we make for caching + checksum = _text_checksum(_dockerfile_preprocess(dockerfile)) + + if registry is not None: + dockerfile = dockerfile.replace("FROM qemu/", + "FROM %s/qemu/" % + (registry)) + # see if we can fetch a cache copy, may fail... + pull_args = ["pull", "%s/%s" % (registry, tag)] + self._do(pull_args, quiet=quiet) + + tmp_df = tempfile.NamedTemporaryFile(mode="w+t", encoding='utf-8', dir=docker_dir, suffix=".docker") @@ -306,15 +327,23 @@ class Docker(object): (uname, uid, uname)) tmp_df.write("\n") - tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % - _text_checksum(_dockerfile_preprocess(dockerfile))) + tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % (checksum)) for f, c in extra_files_cksum: tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c)) tmp_df.flush() - self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + - [docker_dir], + build_args = ["build", "-t", tag, "-f", tmp_df.name] + if self._buildkit: + build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"] + + if registry is not None: + cache = "%s/%s" % (registry, tag) + build_args += ["--cache-from", cache] + build_args += argv + build_args += [docker_dir] + + self._do_check(build_args, quiet=quiet) def update_image(self, tag, tarball, quiet=True): @@ -403,6 +432,8 @@ class BuildCommand(SubCommand): parser.add_argument("--add-current-user", "-u", dest="user", action="store_true", help="Add the current user to image's passwd") + parser.add_argument("--registry", "-r", + help="cache from docker registry") parser.add_argument("-t", dest="tag", help="Image Tag") parser.add_argument("-f", dest="dockerfile", @@ -458,7 +489,8 @@ class BuildCommand(SubCommand): for k, v in os.environ.items() if k.lower() in FILTERED_ENV_NAMES] dkr.build_image(tag, docker_dir, dockerfile, - quiet=args.quiet, user=args.user, argv=argv, + quiet=args.quiet, user=args.user, + argv=argv, registry=args.registry, extra_files_cksum=cksum) rmtree(docker_dir) -- cgit 1.4.1 From 85c5e69923ebd27aa3bc796d322b4c5f304de9e0 Mon Sep 17 00:00:00 2001 From: Alex Bennée Date: Thu, 9 Jul 2020 15:13:25 +0100 Subject: tests/docker: fall back more gracefully when pull fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I only spotted this in the small window between my testing with my registry while waiting for the gitlab PR to go in. As we pre-pull the registry image we know if that fails there isn't any point attempting to use the cache. Fall back to the way we used to do it at that point. Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20200709141327.14631-12-alex.bennee@linaro.org> --- tests/docker/docker.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'tests/docker/docker.py') diff --git a/tests/docker/docker.py b/tests/docker/docker.py index 9684f07bde..2d67bbd15a 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -306,13 +306,14 @@ class Docker(object): checksum = _text_checksum(_dockerfile_preprocess(dockerfile)) if registry is not None: - dockerfile = dockerfile.replace("FROM qemu/", - "FROM %s/qemu/" % - (registry)) # see if we can fetch a cache copy, may fail... pull_args = ["pull", "%s/%s" % (registry, tag)] - self._do(pull_args, quiet=quiet) - + if self._do(pull_args, quiet=quiet) == 0: + dockerfile = dockerfile.replace("FROM qemu/", + "FROM %s/qemu/" % + (registry)) + else: + registry = None tmp_df = tempfile.NamedTemporaryFile(mode="w+t", encoding='utf-8', -- cgit 1.4.1