Merge remote-tracking branch 'remotes/stsquad/tags/pull-testing-next-110219-1' into staging

Testing updates:

  - .travis.yml tweaks and optimisations
  - .cirrus.yml enabled for FreeBSD CI
  - docker.py clean-ups for binfmt_misc
  - more control of vm-test builds

# gpg: Signature made Mon 11 Feb 2019 13:03:14 GMT
# gpg:                using RSA key F715F7CD46F94435F4F588658E520D61289519AE
# gpg: Good signature from "Alex Bennée (Master Work Key) <alex.bennee@linaro.org>" [full]
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8  DF35 FBD0 DB09 5A9E 2A44
#      Subkey fingerprint: F715 F7CD 46F9 4435 F4F5  8865 8E52 0D61 2895 19AE

* remotes/stsquad/tags/pull-testing-next-110219-1:
  tests/vm: Be verbose while extracting compressed images
  docs/devel/testing: Add -a option to usermod command on docker setup
  scripts/qemu.py: allow arches use KVM for their 32bit cousins
  tests/vm: expose BUILD_TARGET, TARGET_LIST and EXTRA_CONFIGURE_OPTS
  tests/vm: add --build-target option
  tests/vm: call make check directly for netbsd/freebsd/ubuntu.i386
  tests/vm: move images to $HOME/.cache/qemu-vm/images
  tests: PEP8 cleanup of docker.py, mostly white space
  tests: docker.py be even smarter with persistent binfmt_misc
  tests: make docker.py check for persistent configs
  tests: make docker.py update use configured binfmt path
  docker: add debian-buster-arm64-cross
  archive-source.sh: Clone the submodules locally
  MAINTAINERS: Add an entry for scripts/archive-source.sh
  .travis.yml: fold --disable-tcg into alternate coroutine builds
  .travis.yml: separate tools and docs into another entry
  .travis.yml: stop requesting libffi & gettext from homebrew
  .cirrus.yml: basic compile and test for FreeBSD

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

7 files changed, 30 insertions, 16 deletions

diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include
index a98fb3027f..992d823f6b 100644
--- a/tests/vm/Makefile.include
+++ b/tests/vm/Makefile.include
@@ -3,7 +3,8 @@
 .PHONY: vm-build-all vm-clean-all
 
 IMAGES := ubuntu.i386 freebsd netbsd openbsd centos
-IMAGE_FILES := $(patsubst %, tests/vm/%.img, $(IMAGES))
+IMAGES_DIR := $(HOME)/.cache/qemu-vm/images
+IMAGE_FILES := $(patsubst %, $(IMAGES_DIR)/%.img, $(IMAGES))
 
 .PRECIOUS: $(IMAGE_FILES)
 
@@ -18,15 +19,21 @@ vm-test:
 	@echo ""
 	@echo "  vm-build-all                    - Build QEMU in all VMs"
 	@echo "  vm-clean-all                    - Clean up VM images"
+	@echo
+	@echo "Special variables:"
+	@echo "    BUILD_TARGET=foo		 - override the build target"
+	@echo "    TARGET_LIST=a,b,c    	 - Override target list in builds."
+	@echo '    EXTRA_CONFIGURE_OPTS="..."'
 
 vm-build-all: $(addprefix vm-build-, $(IMAGES))
 
 vm-clean-all:
 	rm -f $(IMAGE_FILES)
 
-tests/vm/%.img: $(SRC_PATH)/tests/vm/% \
-		$(SRC_PATH)/tests/vm/basevm.py \
-		$(SRC_PATH)/tests/vm/Makefile.include
+$(IMAGES_DIR)/%.img:	$(SRC_PATH)/tests/vm/% \
+			$(SRC_PATH)/tests/vm/basevm.py \
+			$(SRC_PATH)/tests/vm/Makefile.include
+	@mkdir -p $(IMAGES_DIR)
 	$(call quiet-command, \
 		$< \
 		$(if $(V)$(DEBUG), --debug) \
@@ -37,7 +44,7 @@ tests/vm/%.img: $(SRC_PATH)/tests/vm/% \
 
 
 # Build in VM $(IMAGE)
-vm-build-%: tests/vm/%.img
+vm-build-%: $(IMAGES_DIR)/%.img
 	$(call quiet-command, \
 		$(SRC_PATH)/tests/vm/$* \
 		$(if $(V)$(DEBUG), --debug) \
@@ -45,6 +52,9 @@ vm-build-%: tests/vm/%.img
 		$(if $(J),--jobs $(J)) \
 		$(if $(V),--verbose) \
 		--image "$<" \
-		--build-qemu $(SRC_PATH), \
+		$(if $(BUILD_TARGET),--build-target $(BUILD_TARGET)) \
+		--build-qemu $(SRC_PATH) -- \
+		$(if $(TARGET_LIST),--target-list=$(TARGET_LIST)) \
+		$(if $(EXTRA_CONFIGURE_OPTS),$(EXTRA_CONFIGURE_OPTS)), \
 		"  VM-BUILD $*")
 
diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py
index 5caf77d6b8..bdca6cb2fc 100755
--- a/tests/vm/basevm.py
+++ b/tests/vm/basevm.py
@@ -228,6 +228,8 @@ def parse_args(vmcls):
                       help="build image")
     parser.add_option("--build-qemu",
                       help="build QEMU from source in guest")
+    parser.add_option("--build-target",
+                      help="QEMU build target", default="check")
     parser.add_option("--interactive", "-I", action="store_true",
                       help="Interactively run command")
     parser.add_option("--snapshot", "-s", action="store_true",
@@ -255,6 +257,7 @@ def main(vmcls):
             cmd = [vm.BUILD_SCRIPT.format(
                    configure_opts = " ".join(argv),
                    jobs=args.jobs,
+                   target=args.build_target,
                    verbose = "V=1" if args.verbose else "")]
         else:
             cmd = argv
diff --git a/tests/vm/centos b/tests/vm/centos
index daa2dbca03..ba133ea429 100755
--- a/tests/vm/centos
+++ b/tests/vm/centos
@@ -65,8 +65,9 @@ class CentosVM(basevm.BaseVM):
     def build_image(self, img):
         cimg = self._download_with_cache("https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1802.qcow2.xz")
         img_tmp = img + ".tmp"
+        sys.stderr.write("Extracting the image...\n")
         subprocess.check_call(["cp", "-f", cimg, img_tmp + ".xz"])
-        subprocess.check_call(["xz", "-df", img_tmp + ".xz"])
+        subprocess.check_call(["xz", "-dvf", img_tmp + ".xz"])
         subprocess.check_call(["qemu-img", "resize", img_tmp, "50G"])
         self.boot(img_tmp, extra_args = ["-cdrom", self._gen_cloud_init_iso()])
         self.wait_ssh()
diff --git a/tests/vm/freebsd b/tests/vm/freebsd
index 19a3729172..b0066017a6 100755
--- a/tests/vm/freebsd
+++ b/tests/vm/freebsd
@@ -25,8 +25,7 @@ class FreeBSDVM(basevm.BaseVM):
         cd $(mktemp -d /var/tmp/qemu-test.XXXXXX);
         tar -xf /dev/vtbd1;
         ./configure {configure_opts};
-        gmake --output-sync -j{jobs} {verbose};
-        gmake --output-sync -j{jobs} check {verbose};
+        gmake --output-sync -j{jobs} {target} {verbose};
     """
 
     def build_image(self, img):
@@ -34,8 +33,9 @@ class FreeBSDVM(basevm.BaseVM):
                 sha256sum='adcb771549b37bc63826c501f05121a206ed3d9f55f49145908f7e1432d65891')
         img_tmp_xz = img + ".tmp.xz"
         img_tmp = img + ".tmp"
+        sys.stderr.write("Extracting the image...\n")
         subprocess.check_call(["cp", "-f", cimg, img_tmp_xz])
-        subprocess.check_call(["xz", "-df", img_tmp_xz])
+        subprocess.check_call(["xz", "-dvf", img_tmp_xz])
         if os.path.exists(img):
             os.remove(img)
         os.rename(img_tmp, img)
diff --git a/tests/vm/netbsd b/tests/vm/netbsd
index fac6a7ce51..4c6624ea5e 100755
--- a/tests/vm/netbsd
+++ b/tests/vm/netbsd
@@ -25,8 +25,7 @@ class NetBSDVM(basevm.BaseVM):
         cd $(mktemp -d /var/tmp/qemu-test.XXXXXX);
         tar -xf /dev/rld1a;
         ./configure --python=python2.7 {configure_opts};
-        gmake --output-sync -j{jobs} {verbose};
-        gmake --output-sync -j{jobs} check {verbose};
+        gmake --output-sync -j{jobs} {target} {verbose};
     """
 
     def build_image(self, img):
@@ -34,8 +33,9 @@ class NetBSDVM(basevm.BaseVM):
                                          sha256sum='b633d565b0eac3d02015cd0c81440bd8a7a8df8512615ac1ee05d318be015732')
         img_tmp_xz = img + ".tmp.xz"
         img_tmp = img + ".tmp"
+        sys.stderr.write("Extracting the image...\n")
         subprocess.check_call(["cp", "-f", cimg, img_tmp_xz])
-        subprocess.check_call(["xz", "-df", img_tmp_xz])
+        subprocess.check_call(["xz", "-dvf", img_tmp_xz])
         if os.path.exists(img):
             os.remove(img)
         os.rename(img_tmp, img)
diff --git a/tests/vm/openbsd b/tests/vm/openbsd
index cfe0572c59..2105c01a26 100755
--- a/tests/vm/openbsd
+++ b/tests/vm/openbsd
@@ -35,8 +35,9 @@ class OpenBSDVM(basevm.BaseVM):
                 sha256sum='8c6cedc483e602cfee5e04f0406c64eb99138495e8ca580bc0293bcf0640c1bf')
         img_tmp_xz = img + ".tmp.xz"
         img_tmp = img + ".tmp"
+        sys.stderr.write("Extracting the image...\n")
         subprocess.check_call(["cp", "-f", cimg, img_tmp_xz])
-        subprocess.check_call(["xz", "-df", img_tmp_xz])
+        subprocess.check_call(["xz", "-dvf", img_tmp_xz])
         if os.path.exists(img):
             os.remove(img)
         os.rename(img_tmp, img)
diff --git a/tests/vm/ubuntu.i386 b/tests/vm/ubuntu.i386
index 1b7e1ab8f0..a22d137e76 100755
--- a/tests/vm/ubuntu.i386
+++ b/tests/vm/ubuntu.i386
@@ -26,8 +26,7 @@ class UbuntuX86VM(basevm.BaseVM):
         sudo chmod a+r /dev/vdb;
         tar -xf /dev/vdb;
         ./configure {configure_opts};
-        make --output-sync -j{jobs};
-        make --output-sync check -j{jobs} {verbose};
+        make --output-sync {target} -j{jobs} {verbose};
     """
 
     def _gen_cloud_init_iso(self):