Merge remote-tracking branch 'remotes/cleber/tags/python-next-pull-request' into staging

Python queue, 2019-02-22

Python:
* introduce "python" directory with module namespace
* log QEMU launch command line on qemu.QEMUMachine

Acceptance Tests:
* initrd 4GiB+ test
* migration test
* multi vm support in test class
* bump Avocado version and drop ":avocado: enable"

# gpg: Signature made Fri 22 Feb 2019 19:37:07 GMT
# gpg:                using RSA key 657E8D33A5F209F3
# gpg: Good signature from "Cleber Rosa <crosa@redhat.com>" [marginal]
# gpg: WARNING: This key is not certified with sufficiently trusted signatures!
# gpg:          It is not certain that the signature belongs to the owner.
# Primary key fingerprint: 7ABB 96EB 8B46 B94D 5E0F  E9BB 657E 8D33 A5F2 09F3

* remotes/cleber/tags/python-next-pull-request:
  Acceptance tests: expect boot to extract 2GiB+ initrd with linux-v4.16
  Acceptance tests: use linux-3.6 and set vm memory to 4GiB
  tests.acceptance: adds simple migration test
  tests.acceptance: adds multi vm capability for acceptance tests
  scripts/qemu.py: log QEMU launch command line
  Introduce a Python module structure
  Acceptance tests: drop usage of ":avocado: enable"

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

23 files changed, 193 insertions, 40 deletions

diff --git a/configure b/configure
index 47bf617fcc..5921d08cb3 100755
--- a/configure
+++ b/configure
@@ -7674,6 +7674,7 @@ LINKS="$LINKS pc-bios/qemu-icon.bmp"
 LINKS="$LINKS .gdbinit scripts" # scripts needed by relative path in .gdbinit
 LINKS="$LINKS tests/acceptance tests/data"
 LINKS="$LINKS tests/qemu-iotests/check"
+LINKS="$LINKS python"
 for bios_file in \
     $source_path/pc-bios/*.bin \
     $source_path/pc-bios/*.lid \
diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst
index 135743a2bf..60f897d915 100644
--- a/docs/devel/testing.rst
+++ b/docs/devel/testing.rst
@@ -600,7 +600,6 @@ the ``avocado_qemu.Test`` class.  Here's a simple usage example:
 
   class Version(Test):
       """
-      :avocado: enable
       :avocado: tags=quick
       """
       def test_qmp_human_info_version(self):
@@ -634,7 +633,46 @@ instance, available at ``self.vm``.  Because many tests will tweak the
 QEMU command line, launching the QEMUMachine (by using ``self.vm.launch()``)
 is left to the test writer.
 
-At test "tear down", ``avocado_qemu.Test`` handles the QEMUMachine
+The base test class has also support for tests with more than one
+QEMUMachine. The way to get machines is through the ``self.get_vm()``
+method which will return a QEMUMachine instance. The ``self.get_vm()``
+method accepts arguments that will be passed to the QEMUMachine creation
+and also an optional `name` attribute so you can identify a specific
+machine and get it more than once through the tests methods. A simple
+and hypothetical example follows:
+
+.. code::
+
+  from avocado_qemu import Test
+
+
+  class MultipleMachines(Test):
+      """
+      :avocado: enable
+      """
+      def test_multiple_machines(self):
+          first_machine = self.get_vm()
+          second_machine = self.get_vm()
+          self.get_vm(name='third_machine').launch()
+
+          first_machine.launch()
+          second_machine.launch()
+
+          first_res = first_machine.command(
+              'human-monitor-command',
+              command_line='info version')
+
+          second_res = second_machine.command(
+              'human-monitor-command',
+              command_line='info version')
+
+          third_res = self.get_vm(name='third_machine').command(
+              'human-monitor-command',
+              command_line='info version')
+
+          self.assertEquals(first_res, second_res, third_res)
+
+At test "tear down", ``avocado_qemu.Test`` handles all the QEMUMachines
 shutdown.
 
 QEMUMachine
diff --git a/scripts/qemu.py b/python/qemu/__init__.py
index f7269eefbb..fd144c0006 100644
--- a/scripts/qemu.py
+++ b/python/qemu/__init__.py
@@ -16,12 +16,13 @@ import errno
 import logging
 import os
 import subprocess
-import qmp.qmp
 import re
 import shutil
 import socket
 import tempfile
 
+from . import qmp
+
 
 LOG = logging.getLogger(__name__)
 
@@ -66,7 +67,7 @@ class QEMUMachineAddDeviceError(QEMUMachineError):
     failures reported by the QEMU binary itself.
     """
 
-class MonitorResponseError(qmp.qmp.QMPError):
+class MonitorResponseError(qmp.QMPError):
     """
     Represents erroneous QMP monitor reply
     """
@@ -266,8 +267,8 @@ class QEMUMachine(object):
         self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log")
         self._qemu_log_file = open(self._qemu_log_path, 'wb')
 
-        self._qmp = qmp.qmp.QEMUMonitorProtocol(self._vm_monitor,
-                                                server=True)
+        self._qmp = qmp.QEMUMonitorProtocol(self._vm_monitor,
+                                            server=True)
 
     def _post_launch(self):
         self._qmp.accept()
@@ -319,6 +320,7 @@ class QEMUMachine(object):
         self._pre_launch()
         self._qemu_full_args = (self._wrapper + [self._binary] +
                                 self._base_args() + self._args)
+        LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args))
         self._popen = subprocess.Popen(self._qemu_full_args,
                                        stdin=devnull,
                                        stdout=self._qemu_log_file,
@@ -383,7 +385,7 @@ class QEMUMachine(object):
         """
         reply = self.qmp(cmd, conv_keys, **args)
         if reply is None:
-            raise qmp.qmp.QMPError("Monitor is closed")
+            raise qmp.QMPError("Monitor is closed")
         if "error" in reply:
             raise MonitorResponseError(reply)
         return reply["return"]
diff --git a/scripts/qmp/qmp.py b/python/qemu/qmp.py
index 5c8cf6a056..5c8cf6a056 100644
--- a/scripts/qmp/qmp.py
+++ b/python/qemu/qmp.py
diff --git a/scripts/qtest.py b/python/qemu/qtest.py
index afac3fe900..eb45824dd0 100644
--- a/scripts/qtest.py
+++ b/python/qemu/qtest.py
@@ -13,7 +13,8 @@
 
 import socket
 import os
-import qemu
+
+from . import QEMUMachine
 
 
 class QEMUQtestProtocol(object):
@@ -79,7 +80,7 @@ class QEMUQtestProtocol(object):
         self._sock.settimeout(timeout)
 
 
-class QEMUQtestMachine(qemu.QEMUMachine):
+class QEMUQtestMachine(QEMUMachine):
     '''A QEMU VM'''
 
     def __init__(self, binary, args=None, name=None, test_dir="/var/tmp",
diff --git a/scripts/device-crash-test b/scripts/device-crash-test
index 2a13fa4f84..a6748910ad 100755
--- a/scripts/device-crash-test
+++ b/scripts/device-crash-test
@@ -25,6 +25,7 @@ check for crashes and unexpected errors.
 """
 from __future__ import print_function
 
+import os
 import sys
 import glob
 import logging
@@ -34,6 +35,7 @@ import random
 import argparse
 from itertools import chain
 
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python'))
 from qemu import QEMUMachine
 
 logger = logging.getLogger('device-crash-test')
diff --git a/scripts/qmp/__init__.py b/scripts/qmp/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/scripts/qmp/__init__.py
+++ /dev/null
diff --git a/scripts/qmp/qemu-ga-client b/scripts/qmp/qemu-ga-client
index e8cb7646a0..30cf8a9a0d 100755
--- a/scripts/qmp/qemu-ga-client
+++ b/scripts/qmp/qemu-ga-client
@@ -37,10 +37,13 @@
 #
 
 from __future__ import print_function
+import os
+import sys
 import base64
 import random
 
-import qmp
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu import qmp
 
 
 class QemuGuestAgent(qmp.QEMUMonitorProtocol):
diff --git a/scripts/qmp/qmp-shell b/scripts/qmp/qmp-shell
index 770140772d..9fec46e2ed 100755
--- a/scripts/qmp/qmp-shell
+++ b/scripts/qmp/qmp-shell
@@ -66,7 +66,6 @@
 # sent to QEMU, which is useful for debugging and documentation generation.
 
 from __future__ import print_function
-import qmp
 import json
 import ast
 import readline
@@ -76,6 +75,9 @@ import errno
 import atexit
 import shlex
 
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu import qmp
+
 class QMPCompleter(list):
     def complete(self, text, state):
         for cmd in self:
diff --git a/scripts/render_block_graph.py b/scripts/render_block_graph.py
index ed7e581b4f..3e9d282a49 100755
--- a/scripts/render_block_graph.py
+++ b/scripts/render_block_graph.py
@@ -23,6 +23,8 @@ import sys
 import subprocess
 import json
 from graphviz import Digraph
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python'))
 from qemu import MonitorResponseError
 
 
diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py
index 1e54fd5932..a66ec72daa 100644
--- a/tests/acceptance/avocado_qemu/__init__.py
+++ b/tests/acceptance/avocado_qemu/__init__.py
@@ -10,12 +10,12 @@
 
 import os
 import sys
+import uuid
 
 import avocado
 
-SRC_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
-SRC_ROOT_DIR = os.path.abspath(os.path.dirname(SRC_ROOT_DIR))
-sys.path.append(os.path.join(SRC_ROOT_DIR, 'scripts'))
+SRC_ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..')
+sys.path.append(os.path.join(SRC_ROOT_DIR, 'python'))
 
 from qemu import QEMUMachine
 
@@ -42,13 +42,29 @@ def pick_default_qemu_bin():
 
 class Test(avocado.Test):
     def setUp(self):
-        self.vm = None
+        self._vms = {}
         self.qemu_bin = self.params.get('qemu_bin',
                                         default=pick_default_qemu_bin())
         if self.qemu_bin is None:
             self.cancel("No QEMU binary defined or found in the source tree")
-        self.vm = QEMUMachine(self.qemu_bin)
+
+    def _new_vm(self, *args):
+        vm = QEMUMachine(self.qemu_bin)
+        if args:
+            vm.add_args(*args)
+        return vm
+
+    @property
+    def vm(self):
+        return self.get_vm(name='default')
+
+    def get_vm(self, *args, name=None):
+        if not name:
+            name = str(uuid.uuid4())
+        if self._vms.get(name) is None:
+            self._vms[name] = self._new_vm(*args)
+        return self._vms[name]
 
     def tearDown(self):
-        if self.vm is not None:
-            self.vm.shutdown()
+        for vm in self._vms.values():
+            vm.shutdown()
diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py
index 98324f7591..beeb1e59e8 100644
--- a/tests/acceptance/boot_linux_console.py
+++ b/tests/acceptance/boot_linux_console.py
@@ -18,7 +18,6 @@ class BootLinuxConsole(Test):
     Boots a x86_64 Linux kernel and checks that the console is operational
     and the kernel command line is properly passed from QEMU to the kernel
 
-    :avocado: enable
     :avocado: tags=x86_64
     """
 
diff --git a/tests/acceptance/linux_initrd.py b/tests/acceptance/linux_initrd.py
index 737355c2ef..fbdb48e43f 100644
--- a/tests/acceptance/linux_initrd.py
+++ b/tests/acceptance/linux_initrd.py
@@ -8,6 +8,7 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
+import logging
 import tempfile
 from avocado.utils.process import run
 
@@ -18,20 +19,21 @@ class LinuxInitrd(Test):
     """
     Checks QEMU evaluates correctly the initrd file passed as -initrd option.
 
-    :avocado: enable
     :avocado: tags=x86_64
     """
 
-    timeout = 60
+    timeout = 300
 
-    def test_with_2gib_file_should_exit_error_msg(self):
+    def test_with_2gib_file_should_exit_error_msg_with_linux_v3_6(self):
         """
         Pretends to boot QEMU with an initrd file with size of 2GiB
         and expect it exits with error message.
+        Fedora-18 shipped with linux-3.6 which have not supported xloadflags
+        cannot support more than 2GiB initrd.
         """
-        kernel_url = ('https://mirrors.kernel.org/fedora/releases/28/'
-                      'Everything/x86_64/os/images/pxeboot/vmlinuz')
-        kernel_hash = '238e083e114c48200f80d889f7e32eeb2793e02a'
+        kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora/li'
+                      'nux/releases/18/Fedora/x86_64/os/images/pxeboot/vmlinuz')
+        kernel_hash = '41464f68efe42b9991250bed86c7081d2ccdbb21'
         kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
         max_size = 2 * (1024 ** 3) - 1
 
@@ -39,10 +41,44 @@ class LinuxInitrd(Test):
             initrd.seek(max_size)
             initrd.write(b'\0')
             initrd.flush()
-            cmd = "%s -kernel %s -initrd %s" % (self.qemu_bin, kernel_path,
-                                                initrd.name)
+            cmd = "%s -kernel %s -initrd %s -m 4096" % (
+                  self.qemu_bin, kernel_path, initrd.name)
             res = run(cmd, ignore_status=True)
             self.assertEqual(res.exit_status, 1)
             expected_msg = r'.*initrd is too large.*max: \d+, need %s.*' % (
                 max_size + 1)
             self.assertRegex(res.stderr_text, expected_msg)
+
+    def test_with_2gib_file_should_work_with_linux_v4_16(self):
+        """
+        QEMU has supported up to 4 GiB initrd for recent kernel
+        Expect guest can reach 'Unpacking initramfs...'
+        """
+        kernel_url = ('https://mirrors.kernel.org/fedora/releases/28/'
+                      'Everything/x86_64/os/images/pxeboot/vmlinuz')
+        kernel_hash = '238e083e114c48200f80d889f7e32eeb2793e02a'
+        kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
+        max_size = 2 * (1024 ** 3) + 1
+
+        with tempfile.NamedTemporaryFile() as initrd:
+            initrd.seek(max_size)
+            initrd.write(b'\0')
+            initrd.flush()
+
+            self.vm.set_machine('pc')
+            self.vm.set_console()
+            kernel_command_line = 'console=ttyS0'
+            self.vm.add_args('-kernel', kernel_path,
+                             '-append', kernel_command_line,
+                             '-initrd', initrd.name,
+                             '-m', '5120')
+            self.vm.launch()
+            console = self.vm.console_socket.makefile()
+            console_logger = logging.getLogger('console')
+            while True:
+                msg = console.readline()
+                console_logger.debug(msg.strip())
+                if 'Unpacking initramfs...' in msg:
+                    break
+                if 'Kernel panic - not syncing' in msg:
+                    self.fail("Kernel panic reached")
diff --git a/tests/acceptance/migration.py b/tests/acceptance/migration.py
new file mode 100644
index 0000000000..6115cf6c24
--- /dev/null
+++ b/tests/acceptance/migration.py
@@ -0,0 +1,53 @@
+# Migration test
+#
+# Copyright (c) 2019 Red Hat, Inc.
+#
+# Authors:
+#  Cleber Rosa <crosa@redhat.com>
+#  Caio Carrara <ccarrara@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+
+
+from avocado_qemu import Test
+
+from avocado.utils import network
+from avocado.utils import wait
+
+
+class Migration(Test):
+    """
+    :avocado: enable
+    """
+
+    timeout = 10
+
+    @staticmethod
+    def migration_finished(vm):
+        return vm.command('query-migrate')['status'] in ('completed', 'failed')
+
+    def _get_free_port(self):
+        port = network.find_free_port()
+        if port is None:
+            self.cancel('Failed to find a free port')
+        return port
+
+
+    def test_migration_with_tcp_localhost(self):
+        source_vm = self.get_vm()
+        dest_uri = 'tcp:localhost:%u' % self._get_free_port()
+        dest_vm = self.get_vm('-incoming', dest_uri)
+        dest_vm.launch()
+        source_vm.launch()
+        source_vm.qmp('migrate', uri=dest_uri)
+        wait.wait_for(
+            self.migration_finished,
+            timeout=self.timeout,
+            step=0.1,
+            args=(source_vm,)
+        )
+        self.assertEqual(dest_vm.command('query-migrate')['status'], 'completed')
+        self.assertEqual(source_vm.command('query-migrate')['status'], 'completed')
+        self.assertEqual(dest_vm.command('query-status')['status'], 'running')
+        self.assertEqual(source_vm.command('query-status')['status'], 'postmigrate')
diff --git a/tests/acceptance/version.py b/tests/acceptance/version.py
index 13b0a7440d..67c2192c93 100644
--- a/tests/acceptance/version.py
+++ b/tests/acceptance/version.py
@@ -14,7 +14,6 @@ from avocado_qemu import Test
 
 class Version(Test):
     """
-    :avocado: enable
     :avocado: tags=quick
     """
     def test_qmp_human_info_version(self):
diff --git a/tests/acceptance/virtio_version.py b/tests/acceptance/virtio_version.py
index ce990250d8..37fc01ea18 100644
--- a/tests/acceptance/virtio_version.py
+++ b/tests/acceptance/virtio_version.py
@@ -11,7 +11,7 @@ Check compatibility of virtio device types
 import sys
 import os
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 from qemu import QEMUMachine
 from avocado_qemu import Test
 
@@ -61,7 +61,6 @@ class VirtioVersionCheck(Test):
     same device tree created by `disable-modern` and
     `disable-legacy`.
 
-    :avocado: enable
     :avocado: tags=x86_64
     """
 
diff --git a/tests/acceptance/vnc.py b/tests/acceptance/vnc.py
index b1ef9d71b1..064ceabcc1 100644
--- a/tests/acceptance/vnc.py
+++ b/tests/acceptance/vnc.py
@@ -13,7 +13,6 @@ from avocado_qemu import Test
 
 class Vnc(Test):
     """
-    :avocado: enable
     :avocado: tags=vnc,quick
     """
     def test_no_vnc(self):
diff --git a/tests/migration/guestperf/engine.py b/tests/migration/guestperf/engine.py
index 398e3f2706..0e304660b8 100644
--- a/tests/migration/guestperf/engine.py
+++ b/tests/migration/guestperf/engine.py
@@ -24,13 +24,14 @@ import re
 import sys
 import time
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts'))
-import qemu
-import qmp.qmp
 from guestperf.progress import Progress, ProgressStats
 from guestperf.report import Report
 from guestperf.timings import TimingRecord, Timings
 
+sys.path.append(os.path.join(os.path.dirname(__file__),
+                             '..', '..', '..', 'python'))
+import qemu
+
 
 class Engine(object):
 
diff --git a/tests/qemu-iotests/235 b/tests/qemu-iotests/235
index d6edd97ab4..75c203b30c 100755
--- a/tests/qemu-iotests/235
+++ b/tests/qemu-iotests/235
@@ -23,7 +23,7 @@ import os
 import iotests
 from iotests import qemu_img_create, qemu_io, file_path, log
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts'))
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 
 from qemu import QEMUMachine
 
diff --git a/tests/qemu-iotests/238 b/tests/qemu-iotests/238
index f81ee1112f..688abc9acb 100755
--- a/tests/qemu-iotests/238
+++ b/tests/qemu-iotests/238
@@ -23,7 +23,7 @@ import os
 import iotests
 from iotests import log
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts'))
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 
 from qemu import QEMUMachine
 
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 4910fb2005..3d15571688 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -32,8 +32,8 @@ import atexit
 import io
 from collections import OrderedDict
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts'))
-import qtest
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu import qtest
 
 
 # This will not work if arguments contain spaces but is necessary if we
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 64c6e27a94..002ded6a22 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,4 +1,4 @@
 # Add Python module requirements, one per line, to be installed
 # in the tests/venv Python virtual environment. For more info,
 # refer to: https://pip.pypa.io/en/stable/user_guide/#id1
-avocado-framework==65.0
+avocado-framework==68.0
diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py
index bdca6cb2fc..0556bdcf9e 100755
--- a/tests/vm/basevm.py
+++ b/tests/vm/basevm.py
@@ -17,7 +17,7 @@ import sys
 import logging
 import time
 import datetime
-sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 from qemu import QEMUMachine, kvm_available
 import subprocess
 import hashlib