Merge remote-tracking branch 'remotes/stsquad/tags/pull-fixes-for-rc2-270720-1' into staging

Various fixes for rc2:

  - get shippable working again
  - semihosting bug fixes
  - tweak tb-size handling for low memory machines
  - i386 compound literal float fix
  - linux-user MAP_FIXED->MAP_NOREPLACE on fallback
  - docker binfmt_misc fixes
  - linux-user nanosleep fix
  - tests/vm drain console fixes

# gpg: Signature made Mon 27 Jul 2020 09:45:31 BST
# gpg:                using RSA key 6685AE99E75167BCAFC8DF35FBD0DB095A9E2A44
# gpg: Good signature from "Alex Bennée (Master Work Key) <alex.bennee@linaro.org>" [full]
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8  DF35 FBD0 DB09 5A9E 2A44

* remotes/stsquad/tags/pull-fixes-for-rc2-270720-1:
  tests/vm: add shutdown timeout in basevm.py
  python/qemu: Change ConsoleSocket to optionally drain socket.
  python/qemu: Cleanup changes to ConsoleSocket
  linux-user, ppc: fix clock_nanosleep() for linux-user-ppc
  linux-user: fix clock_nanosleep()
  tests/docker: add support for DEB_KEYRING
  tests/docker: fix binfmt_misc image building
  tests/docker: fix update command due to python3 str/bytes distinction
  linux-user: don't use MAP_FIXED in pgd_find_hole_fallback
  target/i386: floatx80: avoid compound literals in static initializers
  accel/tcg: better handle memory constrained systems
  util/oslib-win32: add qemu_get_host_physmem implementation
  util: add qemu_get_host_physmem utility function
  semihosting: don't send the trailing '\0'
  semihosting: defer connect_chardevs a little more to use serialx
  shippable: add one more qemu to registry url

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

4 files changed, 25 insertions, 12 deletions

diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include
index a104e9df28..9119dff97d 100644
--- a/tests/docker/Makefile.include
+++ b/tests/docker/Makefile.include
@@ -78,7 +78,7 @@ docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker
 			DEB_ARCH=$(DEB_ARCH)					\
 			DEB_TYPE=$(DEB_TYPE) 					\
 			$(if $(DEB_URL),DEB_URL=$(DEB_URL),)			\
-			$(DOCKER_SCRIPT) build qemu/debian-$* $< 		\
+			$(DOCKER_SCRIPT) build -t qemu/debian-$* -f $< 		\
 			$(if $V,,--quiet) $(if $(NOCACHE),--no-cache) 		\
 			$(if $(NOUSER),,--add-current-user) 			\
 			$(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES))	\
diff --git a/tests/docker/docker.py b/tests/docker/docker.py
index c9f20d8d09..356d7618f1 100755
--- a/tests/docker/docker.py
+++ b/tests/docker/docker.py
@@ -24,7 +24,7 @@ import tempfile
 import re
 import signal
 from tarfile import TarFile, TarInfo
-from io import StringIO
+from io import StringIO, BytesIO
 from shutil import copy, rmtree
 from pwd import getpwuid
 from datetime import datetime, timedelta
@@ -541,13 +541,14 @@ class UpdateCommand(SubCommand):
 
         # Create a Docker buildfile
         df = StringIO()
-        df.write("FROM %s\n" % args.tag)
-        df.write("ADD . /\n")
-        df.seek(0)
+        df.write(u"FROM %s\n" % args.tag)
+        df.write(u"ADD . /\n")
+
+        df_bytes = BytesIO(bytes(df.getvalue(), "UTF-8"))
 
         df_tar = TarInfo(name="Dockerfile")
-        df_tar.size = len(df.buf)
-        tmp_tar.addfile(df_tar, fileobj=df)
+        df_tar.size = df_bytes.getbuffer().nbytes
+        tmp_tar.addfile(df_tar, fileobj=df_bytes)
 
         tmp_tar.close()
 
diff --git a/tests/docker/dockerfiles/debian-bootstrap.pre b/tests/docker/dockerfiles/debian-bootstrap.pre
index c164778c30..35c85f7db8 100755
--- a/tests/docker/dockerfiles/debian-bootstrap.pre
+++ b/tests/docker/dockerfiles/debian-bootstrap.pre
@@ -80,6 +80,13 @@ else
 fi
 
 #
+# Add optional args
+#
+if [ -n "${DEB_KEYRING}" ]; then
+    DEBOOTSTRAP="${DEBOOTSTRAP} --keyring=${DEB_KEYRING}"
+fi
+
+#
 # Finally check to see if any qemu's are installed
 #
 BINFMT_DIR=/proc/sys/fs/binfmt_misc
diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py
index 7acb48b876..3fac20e929 100644
--- a/tests/vm/basevm.py
+++ b/tests/vm/basevm.py
@@ -80,6 +80,8 @@ class BaseVM(object):
     arch = "#arch"
     # command to halt the guest, can be overridden by subclasses
     poweroff = "poweroff"
+    # Time to wait for shutdown to finish.
+    shutdown_timeout_default = 30
     # enable IPv6 networking
     ipv6 = True
     # This is the timeout on the wait for console bytes.
@@ -87,7 +89,7 @@ class BaseVM(object):
     # Scale up some timeouts under TCG.
     # 4 is arbitrary, but greater than 2,
     # since we found we need to wait more than twice as long.
-    tcg_ssh_timeout_multiplier = 4
+    tcg_timeout_multiplier = 4
     def __init__(self, args, config=None):
         self._guest = None
         self._genisoimage = args.genisoimage
@@ -141,9 +143,12 @@ class BaseVM(object):
         if args.jobs and args.jobs > 1:
             self._args += ["-smp", "%d" % args.jobs]
         if kvm_available(self.arch):
+            self._shutdown_timeout = self.shutdown_timeout_default
             self._args += ["-enable-kvm"]
         else:
             logging.info("KVM not available, not using -enable-kvm")
+            self._shutdown_timeout = \
+                self.shutdown_timeout_default * self.tcg_timeout_multiplier
         self._data_args = []
 
         if self._config['qemu_args'] != None:
@@ -423,7 +428,7 @@ class BaseVM(object):
     def wait_ssh(self, wait_root=False, seconds=300, cmd="exit 0"):
         # Allow more time for VM to boot under TCG.
         if not kvm_available(self.arch):
-            seconds *= self.tcg_ssh_timeout_multiplier
+            seconds *= self.tcg_timeout_multiplier
         starttime = datetime.datetime.now()
         endtime = starttime + datetime.timedelta(seconds=seconds)
         cmd_success = False
@@ -441,14 +446,14 @@ class BaseVM(object):
             raise Exception("Timeout while waiting for guest ssh")
 
     def shutdown(self):
-        self._guest.shutdown()
+        self._guest.shutdown(timeout=self._shutdown_timeout)
 
     def wait(self):
-        self._guest.wait()
+        self._guest.wait(timeout=self._shutdown_timeout)
 
     def graceful_shutdown(self):
         self.ssh_root(self.poweroff)
-        self._guest.wait()
+        self._guest.wait(timeout=self._shutdown_timeout)
 
     def qmp(self, *args, **kwargs):
         return self._guest.qmp(*args, **kwargs)