diff options
Diffstat (limited to 'tests/docker')
| -rw-r--r-- | tests/docker/Makefile.include | 47 | ||||
| -rwxr-xr-x | tests/docker/docker.py | 112 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-alpha-cross.docker | 12 | ||||
| -rwxr-xr-x | tests/docker/dockerfiles/debian-apt-fake.sh | 46 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-hppa-cross.docker | 12 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-m68k-cross.docker | 12 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-mips64-cross.docker | 12 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-powerpc-cross.docker | 39 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-riscv64-cross.docker | 12 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-sh4-cross.docker | 12 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-sid.docker | 32 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-sparc64-cross.docker | 12 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian8.docker | 3 |
13 files changed, 267 insertions, 96 deletions
diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 74fd51c22c..91d9665517 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -15,6 +15,8 @@ DOCKER_TESTS := $(notdir $(shell \ DOCKER_TOOLS := travis +DOCKER_SCRIPT=$(SRC_PATH)/tests/docker/docker.py + TESTS ?= % IMAGES ?= % @@ -32,20 +34,27 @@ docker-qemu-src: $(DOCKER_SRC_COPY) docker-image: ${DOCKER_TARGETS} -# General rule for building docker images +# General rule for building docker images. If we are a sub-make +# invoked with SKIP_DOCKER_BUILD we still check the image is upto date +# though +ifdef SKIP_DOCKER_BUILD +docker-image-%: $(DOCKER_FILES_DIR)/%.docker + $(call quiet-command, \ + $(DOCKER_SCRIPT) check --quiet qemu:$* $<, \ + "CHECK", "$*") +else docker-image-%: $(DOCKER_FILES_DIR)/%.docker @if test "$@" = docker-image-debian-bootstrap -a -z "$(EXECUTABLE)"; then \ echo WARNING: EXECUTABLE is not set, debootstrap may fail. 2>&1 ; \ fi $(call quiet-command,\ - $(SRC_PATH)/tests/docker/docker.py build qemu:$* $< \ + $(DOCKER_SCRIPT) build qemu:$* $< \ $(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \ $(if $(NOUSER),,--add-current-user) \ $(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES))\ $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\ "BUILD","$*") - -docker-image-debian-powerpc-cross: EXTRA_FILES:=$(SRC_PATH)/tests/docker/dockerfiles/debian-apt-fake.sh +endif # Enforce dependencies for composite images docker-image-debian: docker-image-debian9 @@ -55,12 +64,34 @@ docker-image-debian-armel-cross: docker-image-debian9 docker-image-debian-armhf-cross: docker-image-debian9 docker-image-debian-arm64-cross: docker-image-debian9 docker-image-debian-mips-cross: docker-image-debian9 +docker-image-debian-mipsel-cross: docker-image-debian9 docker-image-debian-mips64el-cross: docker-image-debian9 -docker-image-debian-powerpc-cross: docker-image-debian8 docker-image-debian-ppc64el-cross: docker-image-debian9 docker-image-debian-s390x-cross: docker-image-debian9 docker-image-debian-win32-cross: docker-image-debian8-mxe docker-image-debian-win64-cross: docker-image-debian8-mxe + +# Debian SID images - we are tracking a rolling distro so we want to +# force a re-build of the base image if we ever need to build one of +# its children. +ifndef SKIP_DOCKER_BUILD +ifeq ($(HAVE_USER_DOCKER),y) +SID_AGE=$(shell $(DOCKER_SCRIPT) check --checktype=age --olderthan=180 --quiet qemu:debian-sid) +ifeq ($(SID_AGE),) +else +docker-image-debian-sid: NOCACHE=1 +endif +endif +endif + +docker-image-debian-alpha-cross: docker-image-debian-sid +docker-image-debian-hppa-cross: docker-image-debian-sid +docker-image-debian-m68k-cross: docker-image-debian-sid +docker-image-debian-sh4-cross: docker-image-debian-sid +docker-image-debian-sparc64-cross: docker-image-debian-sid +docker-image-debian-mips64-cross: docker-image-debian-sid +docker-image-debian-riscv64-cross: docker-image-debian-sid +docker-image-debian-powerpc-cross: docker-image-debian-sid docker-image-travis: NOUSER=1 # Specialist build images, sometimes very limited tools @@ -133,11 +164,11 @@ docker-run: docker-qemu-src fi $(if $(EXECUTABLE), \ $(call quiet-command, \ - $(SRC_PATH)/tests/docker/docker.py update \ + $(DOCKER_SCRIPT) update \ $(IMAGE) $(EXECUTABLE), \ " COPYING $(EXECUTABLE) to $(IMAGE)")) $(call quiet-command, \ - $(SRC_PATH)/tests/docker/docker.py run \ + $(DOCKER_SCRIPT) run \ $(if $(NOUSER),,-u $(shell id -u)) \ --security-opt seccomp=unconfined \ $(if $V,,--rm) \ @@ -167,4 +198,4 @@ docker-run-%: @$(MAKE) docker-run TEST=$(CMD) IMAGE=qemu:$(IMAGE) docker-clean: - $(call quiet-command, $(SRC_PATH)/tests/docker/docker.py clean) + $(call quiet-command, $(DOCKER_SCRIPT) clean) diff --git a/tests/docker/docker.py b/tests/docker/docker.py index 306e14cf69..b279836154 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -26,9 +26,13 @@ import tempfile import re import signal from tarfile import TarFile, TarInfo -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO from shutil import copy, rmtree from pwd import getpwuid +from datetime import datetime,timedelta FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy'] @@ -49,7 +53,9 @@ def _guess_docker_command(): commands = [["docker"], ["sudo", "-n", "docker"]] for cmd in commands: try: - if subprocess.call(cmd + ["images"], + # docker version will return the client details in stdout + # but still report a status of 1 if it can't contact the daemon + if subprocess.call(cmd + ["version"], stdout=DEVNULL, stderr=DEVNULL) == 0: return cmd except OSError: @@ -179,8 +185,17 @@ class Docker(object): stderr=subprocess.STDOUT, **kwargs) + def inspect_tag(self, tag): + try: + return self._output(["inspect", tag]) + except subprocess.CalledProcessError: + return None + + def get_image_creation_time(self, info): + return json.loads(info)[0]["Created"] + def get_image_dockerfile_checksum(self, tag): - resp = self._output(["inspect", tag]) + resp = self.inspect_tag(tag) labels = json.loads(resp)[0]["Config"].get("Labels", {}) return labels.get("com.qemu.dockerfile-checksum", "") @@ -201,8 +216,10 @@ class Docker(object): tmp_df.write("\n") tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % - _text_checksum("\n".join([dockerfile] + - extra_files_cksum))) + _text_checksum(_dockerfile_preprocess(dockerfile))) + for f, c in extra_files_cksum: + tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c)) + tmp_df.flush() self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \ @@ -317,7 +334,7 @@ class BuildCommand(SubCommand): _copy_binary_with_libs(args.include_executable, docker_dir) for filename in args.extra_files or []: _copy_with_mkdir(filename, docker_dir) - cksum += [_file_checksum(filename)] + cksum += [(filename, _file_checksum(filename))] argv += ["--build-arg=" + k.lower() + "=" + v for k, v in os.environ.iteritems() @@ -409,6 +426,89 @@ class ProbeCommand(SubCommand): return +class CcCommand(SubCommand): + """Compile sources with cc in images""" + name = "cc" + + def args(self, parser): + parser.add_argument("--image", "-i", required=True, + help="The docker image in which to run cc") + parser.add_argument("--cc", default="cc", + help="The compiler executable to call") + parser.add_argument("--user", + help="The user-id to run under") + parser.add_argument("--source-path", "-s", nargs="*", dest="paths", + help="""Extra paths to (ro) mount into container for + reading sources""") + + def run(self, args, argv): + if argv and argv[0] == "--": + argv = argv[1:] + cwd = os.getcwd() + cmd = ["--rm", "-w", cwd, + "-v", "%s:%s:rw" % (cwd, cwd)] + if args.paths: + for p in args.paths: + cmd += ["-v", "%s:%s:ro,z" % (p, p)] + if args.user: + cmd += ["-u", args.user] + cmd += [args.image, args.cc] + cmd += argv + return Docker().command("run", cmd, args.quiet) + + +class CheckCommand(SubCommand): + """Check if we need to re-build a docker image out of a dockerfile. + Arguments: <tag> <dockerfile>""" + name = "check" + + def args(self, parser): + parser.add_argument("tag", + help="Image Tag") + parser.add_argument("dockerfile", default=None, + help="Dockerfile name", nargs='?') + parser.add_argument("--checktype", choices=["checksum", "age"], + default="checksum", help="check type") + parser.add_argument("--olderthan", default=60, type=int, + help="number of minutes") + + def run(self, args, argv): + tag = args.tag + + dkr = Docker() + info = dkr.inspect_tag(tag) + if info is None: + print("Image does not exist") + return 1 + + if args.checktype == "checksum": + if not args.dockerfile: + print("Need a dockerfile for tag:%s" % (tag)) + return 1 + + dockerfile = open(args.dockerfile, "rb").read() + + if dkr.image_matches_dockerfile(tag, dockerfile): + if not args.quiet: + print("Image is up to date") + return 0 + else: + print("Image needs updating") + return 1 + elif args.checktype == "age": + timestr = dkr.get_image_creation_time(info).split(".")[0] + created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S") + past = datetime.now() - timedelta(minutes=args.olderthan) + if created < past: + print ("Image created @ %s more than %d minutes old" % + (timestr, args.olderthan)) + return 1 + else: + if not args.quiet: + print ("Image less than %d minutes old" % (args.olderthan)) + return 0 + + def main(): parser = argparse.ArgumentParser(description="A Docker helper", usage="%s <subcommand> ..." % os.path.basename(sys.argv[0])) diff --git a/tests/docker/dockerfiles/debian-alpha-cross.docker b/tests/docker/dockerfiles/debian-alpha-cross.docker new file mode 100644 index 0000000000..29a25d0dfd --- /dev/null +++ b/tests/docker/dockerfiles/debian-alpha-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian sid base image which +# contains cross compilers for Debian "ports" targets. +# +FROM qemu:debian-sid + +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + gcc-alpha-linux-gnu \ + libc6.1-dev-alpha-cross || { echo "Failed to build - see debian-sid.docker notes"; exit 1; } diff --git a/tests/docker/dockerfiles/debian-apt-fake.sh b/tests/docker/dockerfiles/debian-apt-fake.sh deleted file mode 100755 index 2ec0fdf47a..0000000000 --- a/tests/docker/dockerfiles/debian-apt-fake.sh +++ /dev/null @@ -1,46 +0,0 @@ -#! /bin/sh -# -# Generate fake debian package to resolve unimportant unmet dependencies held -# by upstream multiarch broken packages. -# -# Copyright (c) 2017 Philippe Mathieu-Daudé <f4bug@amsat.org> -# -# This work is licensed under the terms of the GNU GPL, version 2 -# or (at your option) any later version. See the COPYING file in -# the top-level directory. - -test $1 = "install" && shift 1 - -fake_install() -{ - echo "Generating fake $2 $1 $3 ..." - (cd /var/cache/apt/archives - (cat << 'EOF' -Section: misc -Priority: optional -Standards-Version: 3.9.2 - -Package: NAME -Version: VERSION -Maintainer: qemu-devel@nongnu.org -Architecture: any -Multi-Arch: same -Description: fake NAME -EOF - ) | sed s/NAME/$2/g | sed s/VERSION/$3/g > $2.control - equivs-build -a $1 $2.control 1>/dev/null 2>/dev/null - dpkg -i --force-overwrite $2_$3_$1.deb - ) -} - -try_install() -{ - name=$(echo $1|sed "s/\(.*\):\(.*\)=\(.*\)/\1/") - arch=$(echo $1|sed "s/\(.*\):\(.*\)=\(.*\)/\2/") - vers=$(echo $1|sed "s/\(.*\):\(.*\)=\(.*\)/\3/") - apt-get install -q -yy $1 || fake_install $arch $name $vers -} - -for package in $*; do - try_install $package -done diff --git a/tests/docker/dockerfiles/debian-hppa-cross.docker b/tests/docker/dockerfiles/debian-hppa-cross.docker new file mode 100644 index 0000000000..ad443defac --- /dev/null +++ b/tests/docker/dockerfiles/debian-hppa-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian sid base image which +# contains cross compilers for Debian "ports" targets. +# +FROM qemu:debian-sid + +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + gcc-hppa-linux-gnu \ + libc6-dev-hppa-cross diff --git a/tests/docker/dockerfiles/debian-m68k-cross.docker b/tests/docker/dockerfiles/debian-m68k-cross.docker new file mode 100644 index 0000000000..21ba3b0132 --- /dev/null +++ b/tests/docker/dockerfiles/debian-m68k-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian sid base image which +# contains cross compilers for Debian "ports" targets. +# +FROM qemu:debian-sid + +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + gcc-m68k-linux-gnu \ + libc6-dev-m68k-cross diff --git a/tests/docker/dockerfiles/debian-mips64-cross.docker b/tests/docker/dockerfiles/debian-mips64-cross.docker new file mode 100644 index 0000000000..ed1ce0e919 --- /dev/null +++ b/tests/docker/dockerfiles/debian-mips64-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian sid base image which +# contains cross compilers for Debian "ports" targets. +# +FROM qemu:debian-sid + +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + gcc-mips64-linux-gnuabi64 \ + libc6-dev-mips64-cross diff --git a/tests/docker/dockerfiles/debian-powerpc-cross.docker b/tests/docker/dockerfiles/debian-powerpc-cross.docker index a5dd46b4ac..5e62ca0df1 100644 --- a/tests/docker/dockerfiles/debian-powerpc-cross.docker +++ b/tests/docker/dockerfiles/debian-powerpc-cross.docker @@ -1,40 +1,13 @@ # # Docker powerpc cross-compiler target # -# This docker target builds on the debian Jessie base image. +# This docker target builds on the debian sid base image which +# contains cross compilers for Debian "ports" targets. The original +# Jessie based no longer builds. # -FROM qemu:debian8 -MAINTAINER Philippe Mathieu-Daudé <f4bug@amsat.org> +FROM qemu:debian-sid -# Add the foreign architecture we want and install dependencies -RUN dpkg --add-architecture powerpc -RUN apt-get update RUN DEBIAN_FRONTEND=noninteractive eatmydata \ apt-get install -y --no-install-recommends \ - crossbuild-essential-powerpc - -# <kludge> to fix "following packages have unmet dependencies" ... -ADD debian-apt-fake.sh /usr/local/bin/apt-fake -RUN apt-get install -y --no-install-recommends \ - equivs \ - pkg-config -RUN apt-fake install \ - pkg-config:powerpc=0.28-1.1-fake && \ - ln -s pkg-config /usr/bin/powerpc-linux-gnu-pkg-config -ENV PKG_CONFIG_PATH /usr/lib/powerpc-linux-gnu/pkgconfig -# </kludge> - -# Specify the cross prefix for this image (see tests/docker/common.rc) -ENV QEMU_CONFIGURE_OPTS --cross-prefix=powerpc-linux-gnu- - -RUN DEBIAN_FRONTEND=noninteractive eatmydata \ - apt-get build-dep -yy -a powerpc qemu -RUN DEBIAN_FRONTEND=noninteractive \ - apt-get install -y --no-install-recommends \ - glusterfs-common:powerpc \ - libbz2-dev:powerpc \ - liblzo2-dev:powerpc \ - libncursesw5-dev:powerpc \ - libnfs-dev:powerpc \ - librdmacm-dev:powerpc \ - libsnappy-dev:powerpc + gcc-powerpc-linux-gnu \ + libc6-dev-powerpc-cross || { echo "Failed to build - see debian-sid.docker notes"; exit 1; } diff --git a/tests/docker/dockerfiles/debian-riscv64-cross.docker b/tests/docker/dockerfiles/debian-riscv64-cross.docker new file mode 100644 index 0000000000..2b2e64cee6 --- /dev/null +++ b/tests/docker/dockerfiles/debian-riscv64-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian sid base image which +# contains cross compilers for Debian "ports" targets. +# +FROM qemu:debian-sid + +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + gcc-riscv64-linux-gnu \ + libc6-dev-riscv64-cross diff --git a/tests/docker/dockerfiles/debian-sh4-cross.docker b/tests/docker/dockerfiles/debian-sh4-cross.docker new file mode 100644 index 0000000000..88a2423094 --- /dev/null +++ b/tests/docker/dockerfiles/debian-sh4-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian sid base image which +# contains cross compilers for Debian "ports" targets. +# +FROM qemu:debian-sid + +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + gcc-sh4-linux-gnu \ + libc6-dev-sh4-cross diff --git a/tests/docker/dockerfiles/debian-sid.docker b/tests/docker/dockerfiles/debian-sid.docker new file mode 100644 index 0000000000..9a3d168705 --- /dev/null +++ b/tests/docker/dockerfiles/debian-sid.docker @@ -0,0 +1,32 @@ +# +# Debian Sid Base +# +# A number of our guests exist as ports only. We can either use the +# ports repo or get everything from Sid. However Sid is a rolling +# distro which may be broken at any particular time. If you are +# unlucky and try and build your images while gcc is in the process of +# being uploaded this can fail. Your only recourse is to try again in +# a few hours when the repos have re-synced. Once built however you +# won't be affected by repo changes unless the docker recipies are +# updated and trigger a re-build. +# + +FROM debian:sid-slim + +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list + +# Install common build utilities +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + bison \ + build-essential \ + ca-certificates \ + flex \ + git \ + pkg-config \ + psmisc \ + python \ + texinfo || { echo "Failed to build - see debian-sid.docker notes"; exit 1; } diff --git a/tests/docker/dockerfiles/debian-sparc64-cross.docker b/tests/docker/dockerfiles/debian-sparc64-cross.docker new file mode 100644 index 0000000000..1e2c809274 --- /dev/null +++ b/tests/docker/dockerfiles/debian-sparc64-cross.docker @@ -0,0 +1,12 @@ +# +# Docker cross-compiler target +# +# This docker target builds on the debian sid base image which +# contains cross compilers for Debian "ports" targets. +# +FROM qemu:debian-sid + +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + gcc-sparc64-linux-gnu \ + libc6-dev-sparc64-cross diff --git a/tests/docker/dockerfiles/debian8.docker b/tests/docker/dockerfiles/debian8.docker index 1bcf2e3d2f..52945631cd 100644 --- a/tests/docker/dockerfiles/debian8.docker +++ b/tests/docker/dockerfiles/debian8.docker @@ -32,6 +32,3 @@ RUN DEBIAN_FRONTEND=noninteractive eatmydata \ pkg-config \ python-minimal -# Setup Emdebian [emdebian-archive-keyring] -RUN echo "deb http://emdebian.org/tools/debian/ jessie main" > /etc/apt/sources.list.d/emdebian.list && \ - curl http://emdebian.org/tools/debian/emdebian-toolchain-archive.key | apt-key add - |