Merge remote-tracking branch 'remotes/stsquad/tags/pull-tcg-testing-revivial-210618-2' into staging

Add check-tcg machinary

This restores the ability to run TCG smoke tests by using our docker
infrastructure to support cross building simple tests. It represents
the first step to making better cross-architecture testing available
straight from the source tree ;-)

v2
  - fix quoting of target_compiler
  - make docker.py Py3 safe
  - tweak .travis.yml recipe
  - don't probe docker when HAVE_USER_DOCKER not set

# gpg: Signature made Thu 21 Jun 2018 07:23:45 BST
# gpg:                using RSA key FBD0DB095A9E2A44
# gpg: Good signature from "Alex Bennée (Master Work Key) <alex.bennee@linaro.org>"
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8  DF35 FBD0 DB09 5A9E 2A44

* remotes/stsquad/tags/pull-tcg-testing-revivial-210618-2: (57 commits)
  .travis.yml: add check-tcg test
  tests/docker/Makefile.include: only force SID to NOCACHE if old
  docker: docker.py adding age check command
  tests/Makefile: call sub-makes with SKIP_DOCKER_BUILD=1
  docker: docker.py add check sub-command
  docker: docker.py don't conflate checksums for extra_files
  docker: docker.py use "version" to probe usage
  tests: add top-level make dependency for docker builds
  tests/tcg/i386: extend timeout for runcom test
  tests/tcg: override runners for broken tests
  tests/tcg: add run, diff, and skip helper macros
  tests/Makefile.include: add [build|clean|check]-tcg targets
  Makefile.target: add (clean-/build-)guest-tests targets
  tests/tcg/Makefile: update to be called from Makefile.target
  tests/tcg: enable building for PowerPC
  docker: move debian-powerpc-cross to sid based build
  tests/tcg: enable building for RISCV64
  tests/tcg: enable building for mips64
  tests/tcg: enable building for sparc64
  tests/tcg: enable building for sh4
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

1 files changed, 106 insertions, 6 deletions

diff --git a/tests/docker/docker.py b/tests/docker/docker.py
index 306e14cf69..b279836154 100755
--- a/tests/docker/docker.py
+++ b/tests/docker/docker.py
@@ -26,9 +26,13 @@ import tempfile
 import re
 import signal
 from tarfile import TarFile, TarInfo
-from StringIO import StringIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 from shutil import copy, rmtree
 from pwd import getpwuid
+from datetime import datetime,timedelta
 
 
 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
@@ -49,7 +53,9 @@ def _guess_docker_command():
     commands = [["docker"], ["sudo", "-n", "docker"]]
     for cmd in commands:
         try:
-            if subprocess.call(cmd + ["images"],
+            # docker version will return the client details in stdout
+            # but still report a status of 1 if it can't contact the daemon
+            if subprocess.call(cmd + ["version"],
                                stdout=DEVNULL, stderr=DEVNULL) == 0:
                 return cmd
         except OSError:
@@ -179,8 +185,17 @@ class Docker(object):
                                        stderr=subprocess.STDOUT,
                                        **kwargs)
 
+    def inspect_tag(self, tag):
+        try:
+            return self._output(["inspect", tag])
+        except subprocess.CalledProcessError:
+            return None
+
+    def get_image_creation_time(self, info):
+        return json.loads(info)[0]["Created"]
+
     def get_image_dockerfile_checksum(self, tag):
-        resp = self._output(["inspect", tag])
+        resp = self.inspect_tag(tag)
         labels = json.loads(resp)[0]["Config"].get("Labels", {})
         return labels.get("com.qemu.dockerfile-checksum", "")
 
@@ -201,8 +216,10 @@ class Docker(object):
 
         tmp_df.write("\n")
         tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
-                     _text_checksum("\n".join([dockerfile] +
-                                    extra_files_cksum)))
+                     _text_checksum(_dockerfile_preprocess(dockerfile)))
+        for f, c in extra_files_cksum:
+            tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
+
         tmp_df.flush()
 
         self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \
@@ -317,7 +334,7 @@ class BuildCommand(SubCommand):
                 _copy_binary_with_libs(args.include_executable, docker_dir)
             for filename in args.extra_files or []:
                 _copy_with_mkdir(filename, docker_dir)
-                cksum += [_file_checksum(filename)]
+                cksum += [(filename, _file_checksum(filename))]
 
             argv += ["--build-arg=" + k.lower() + "=" + v
                         for k, v in os.environ.iteritems()
@@ -409,6 +426,89 @@ class ProbeCommand(SubCommand):
         return
 
 
+class CcCommand(SubCommand):
+    """Compile sources with cc in images"""
+    name = "cc"
+
+    def args(self, parser):
+        parser.add_argument("--image", "-i", required=True,
+                            help="The docker image in which to run cc")
+        parser.add_argument("--cc", default="cc",
+                            help="The compiler executable to call")
+        parser.add_argument("--user",
+                            help="The user-id to run under")
+        parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
+                            help="""Extra paths to (ro) mount into container for
+                            reading sources""")
+
+    def run(self, args, argv):
+        if argv and argv[0] == "--":
+            argv = argv[1:]
+        cwd = os.getcwd()
+        cmd = ["--rm", "-w", cwd,
+               "-v", "%s:%s:rw" % (cwd, cwd)]
+        if args.paths:
+            for p in args.paths:
+                cmd += ["-v", "%s:%s:ro,z" % (p, p)]
+        if args.user:
+            cmd += ["-u", args.user]
+        cmd += [args.image, args.cc]
+        cmd += argv
+        return Docker().command("run", cmd, args.quiet)
+
+
+class CheckCommand(SubCommand):
+    """Check if we need to re-build a docker image out of a dockerfile.
+    Arguments: <tag> <dockerfile>"""
+    name = "check"
+
+    def args(self, parser):
+        parser.add_argument("tag",
+                            help="Image Tag")
+        parser.add_argument("dockerfile", default=None,
+                            help="Dockerfile name", nargs='?')
+        parser.add_argument("--checktype", choices=["checksum", "age"],
+                            default="checksum", help="check type")
+        parser.add_argument("--olderthan", default=60, type=int,
+                            help="number of minutes")
+
+    def run(self, args, argv):
+        tag = args.tag
+
+        dkr = Docker()
+        info = dkr.inspect_tag(tag)
+        if info is None:
+            print("Image does not exist")
+            return 1
+
+        if args.checktype == "checksum":
+            if not args.dockerfile:
+                print("Need a dockerfile for tag:%s" % (tag))
+                return 1
+
+            dockerfile = open(args.dockerfile, "rb").read()
+
+            if dkr.image_matches_dockerfile(tag, dockerfile):
+                if not args.quiet:
+                    print("Image is up to date")
+                return 0
+            else:
+                print("Image needs updating")
+                return 1
+        elif args.checktype == "age":
+            timestr = dkr.get_image_creation_time(info).split(".")[0]
+            created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
+            past = datetime.now() - timedelta(minutes=args.olderthan)
+            if created < past:
+                print ("Image created @ %s more than %d minutes old" %
+                       (timestr, args.olderthan))
+                return 1
+            else:
+                if not args.quiet:
+                    print ("Image less than %d minutes old" % (args.olderthan))
+                return 0
+
+
 def main():
     parser = argparse.ArgumentParser(description="A Docker helper",
             usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))