Merge remote-tracking branch 'remotes/maxreitz/tags/pull-block-2020-07-06' into staging

Block patches for 5.1:
- LUKS keyslot amendment
  (+ patches to make the iotests pass on non-Linux systems, and to keep
     the tests passing for qcow v1, and to skip LUKS tests (including
     qcow2 LUKS) when the built qemu does not support it)
- Refactoring in the block layer: Drop the basically unnecessary
  unallocated_blocks_are_zero field from BlockDriverInfo
- Fix qcow2 preallocation when the image size is not a multiple of the
  cluster size
- Fix in block-copy code

# gpg: Signature made Mon 06 Jul 2020 11:02:53 BST
# gpg:                using RSA key 91BEB60A30DB3E8857D11829F407DB0061D5CF40
# gpg:                issuer "mreitz@redhat.com"
# gpg: Good signature from "Max Reitz <mreitz@redhat.com>" [full]
# Primary key fingerprint: 91BE B60A 30DB 3E88 57D1  1829 F407 DB00 61D5 CF40

* remotes/maxreitz/tags/pull-block-2020-07-06: (31 commits)
  qed: Simplify backing reads
  block: drop unallocated_blocks_are_zero
  block/vhdx: drop unallocated_blocks_are_zero
  block/file-posix: drop unallocated_blocks_are_zero
  block/iscsi: drop unallocated_blocks_are_zero
  block/crypto: drop unallocated_blocks_are_zero
  block/vpc: return ZERO block-status when appropriate
  block/vdi: return ZERO block-status when appropriate
  block: inline bdrv_unallocated_blocks_are_zero()
  qemu-img: convert: don't use unallocated_blocks_are_zero
  iotests: add tests for blockdev-amend
  block/qcow2: implement blockdev-amend
  block/crypto: implement blockdev-amend
  block/core: add generic infrastructure for x-blockdev-amend qmp command
  iotests: qemu-img tests for luks key management
  block/qcow2: extend qemu-img amend interface with crypto options
  block/crypto: implement the encryption key management
  block/crypto: rename two functions
  block/amend: refactor qcow2 amend options
  block/amend: separate amend and create options for qemu-img
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

1 files changed, 64 insertions, 20 deletions

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index ef739dd1e3..f1e0733dda 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -29,7 +29,7 @@ import struct
 import subprocess
 import sys
 from typing import (Any, Callable, Dict, Iterable,
-                    List, Optional, Sequence, TypeVar)
+                    List, Optional, Sequence, Tuple, TypeVar)
 import unittest
 
 # pylint: disable=import-error, wrong-import-position
@@ -90,15 +90,24 @@ luks_default_secret_object = 'secret,id=keysec0,data=' + \
 luks_default_key_secret_opt = 'key-secret=keysec0'
 
 
-def qemu_img(*args):
-    '''Run qemu-img and return the exit code'''
-    devnull = open('/dev/null', 'r+')
-    exitcode = subprocess.call(qemu_img_args + list(args),
-                               stdin=devnull, stdout=devnull)
-    if exitcode < 0:
+def qemu_img_pipe_and_status(*args: str) -> Tuple[str, int]:
+    """
+    Run qemu-img and return both its output and its exit code
+    """
+    subp = subprocess.Popen(qemu_img_args + list(args),
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT,
+                            universal_newlines=True)
+    output = subp.communicate()[0]
+    if subp.returncode < 0:
         sys.stderr.write('qemu-img received signal %i: %s\n'
-                         % (-exitcode, ' '.join(qemu_img_args + list(args))))
-    return exitcode
+                         % (-subp.returncode,
+                            ' '.join(qemu_img_args + list(args))))
+    return (output, subp.returncode)
+
+def qemu_img(*args: str) -> int:
+    '''Run qemu-img and return the exit code'''
+    return qemu_img_pipe_and_status(*args)[1]
 
 def ordered_qmp(qmsg, conv_keys=True):
     # Dictionaries are not ordered prior to 3.6, therefore:
@@ -140,18 +149,9 @@ def qemu_img_verbose(*args):
                          % (-exitcode, ' '.join(qemu_img_args + list(args))))
     return exitcode
 
-def qemu_img_pipe(*args):
+def qemu_img_pipe(*args: str) -> str:
     '''Run qemu-img and return its output'''
-    subp = subprocess.Popen(qemu_img_args + list(args),
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.STDOUT,
-                            universal_newlines=True)
-    output = subp.communicate()[0]
-    if subp.returncode < 0:
-        sys.stderr.write('qemu-img received signal %i: %s\n'
-                         % (-subp.returncode,
-                            ' '.join(qemu_img_args + list(args))))
-    return output
+    return qemu_img_pipe_and_status(*args)[0]
 
 def qemu_img_log(*args):
     result = qemu_img_pipe(*args)
@@ -1010,12 +1010,17 @@ def _verify_image_format(supported_fmts: Sequence[str] = (),
         # similar to
         #   _supported_fmt generic
         # for bash tests
+        if imgfmt == 'luks':
+            verify_working_luks()
         return
 
     not_sup = supported_fmts and (imgfmt not in supported_fmts)
     if not_sup or (imgfmt in unsupported_fmts):
         notrun('not suitable for this image format: %s' % imgfmt)
 
+    if imgfmt == 'luks':
+        verify_working_luks()
+
 def _verify_protocol(supported: Sequence[str] = (),
                      unsupported: Sequence[str] = ()) -> None:
     assert not (supported and unsupported)
@@ -1052,6 +1057,45 @@ def verify_quorum():
     if not supports_quorum():
         notrun('quorum support missing')
 
+def has_working_luks() -> Tuple[bool, str]:
+    """
+    Check whether our LUKS driver can actually create images
+    (this extends to LUKS encryption for qcow2).
+
+    If not, return the reason why.
+    """
+
+    img_file = f'{test_dir}/luks-test.luks'
+    (output, status) = \
+        qemu_img_pipe_and_status('create', '-f', 'luks',
+                                 '--object', luks_default_secret_object,
+                                 '-o', luks_default_key_secret_opt,
+                                 '-o', 'iter-time=10',
+                                 img_file, '1G')
+    try:
+        os.remove(img_file)
+    except OSError:
+        pass
+
+    if status != 0:
+        reason = output
+        for line in output.splitlines():
+            if img_file + ':' in line:
+                reason = line.split(img_file + ':', 1)[1].strip()
+                break
+
+        return (False, reason)
+    else:
+        return (True, '')
+
+def verify_working_luks():
+    """
+    Skip test suite if LUKS does not work
+    """
+    (working, reason) = has_working_luks()
+    if not working:
+        notrun(reason)
+
 def qemu_pipe(*args):
     """
     Run qemu with an option to print something and exit (e.g. a help option).