diff options
Diffstat (limited to '')
38 files changed, 743 insertions, 416 deletions
diff --git a/scripts/archive-source.sh b/scripts/archive-source.sh index 23e042dacd..4899630491 100755 --- a/scripts/archive-source.sh +++ b/scripts/archive-source.sh @@ -26,8 +26,7 @@ sub_file="${sub_tdir}/submodule.tar" # independent of what the developer currently has initialized # in their checkout, because the build environment is completely # different to the host OS. -submodules="dtc meson ui/keycodemapdb" -submodules="$submodules tests/fp/berkeley-softfloat-3 tests/fp/berkeley-testfloat-3" +subprojects="dtc keycodemapdb libvfio-user berkeley-softfloat-3 berkeley-testfloat-3" sub_deinit="" function cleanup() { @@ -51,23 +50,11 @@ function tree_ish() { git archive --format tar "$(tree_ish)" > "$tar_file" test $? -ne 0 && error "failed to archive qemu" -for sm in $submodules; do - status="$(git submodule status "$sm")" - smhash="${status#[ +-]}" - smhash="${smhash%% *}" - case "$status" in - -*) - sub_deinit="$sub_deinit $sm" - git submodule update --init "$sm" - test $? -ne 0 && error "failed to update submodule $sm" - ;; - +*) - echo "WARNING: submodule $sm is out of sync" - ;; - esac - (cd $sm; git archive --format tar --prefix "$sm/" $(tree_ish)) > "$sub_file" - test $? -ne 0 && error "failed to archive submodule $sm ($smhash)" - tar --concatenate --file "$tar_file" "$sub_file" - test $? -ne 0 && error "failed append submodule $sm to $tar_file" + +for sp in $subprojects; do + meson subprojects download $sp + test $? -ne 0 && error "failed to download subproject $sp" + tar --append --file "$tar_file" --exclude=.git subprojects/$sp + test $? -ne 0 && error "failed to append subproject $sp to $tar_file" done exit 0 diff --git a/scripts/block-coroutine-wrapper.py b/scripts/block-coroutine-wrapper.py index 60e9b3107c..d4a183db61 100644 --- a/scripts/block-coroutine-wrapper.py +++ b/scripts/block-coroutine-wrapper.py @@ -88,16 +88,7 @@ class FuncDecl: raise ValueError(f"no_co function can't be rdlock: {self.name}") self.target_name = f'{subsystem}_{subname}' - t = self.args[0].type - if t == 'BlockDriverState *': - ctx = 'bdrv_get_aio_context(bs)' - elif t == 'BdrvChild *': - ctx = 'bdrv_get_aio_context(child->bs)' - elif t == 'BlockBackend *': - ctx = 'blk_get_aio_context(blk)' - else: - ctx = 'qemu_get_aio_context()' - self.ctx = ctx + self.ctx = self.gen_ctx() self.get_result = 's->ret = ' self.ret = 'return s.ret;' @@ -109,6 +100,17 @@ class FuncDecl: self.co_ret = '' self.return_field = '' + def gen_ctx(self, prefix: str = '') -> str: + t = self.args[0].type + if t == 'BlockDriverState *': + return f'bdrv_get_aio_context({prefix}bs)' + elif t == 'BdrvChild *': + return f'bdrv_get_aio_context({prefix}child->bs)' + elif t == 'BlockBackend *': + return f'blk_get_aio_context({prefix}blk)' + else: + return 'qemu_get_aio_context()' + def gen_list(self, format: str) -> str: return ', '.join(format.format_map(arg.__dict__) for arg in self.args) @@ -262,8 +264,11 @@ typedef struct {struct_name} {{ static void {name}_bh(void *opaque) {{ {struct_name} *s = opaque; + AioContext *ctx = {func.gen_ctx('s->')}; + aio_context_acquire(ctx); {func.get_result}{name}({ func.gen_list('s->{name}') }); + aio_context_release(ctx); aio_co_wake(s->co); }} diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index d768171dcf..eeaec436eb 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -2865,6 +2865,14 @@ sub process { if ($line =~ /\bsignal\s*\(/ && !($line =~ /SIG_(?:IGN|DFL)/)) { ERROR("use sigaction to establish signal handlers; signal is not portable\n" . $herecurr); } +# recommend qemu_bh_new_guarded instead of qemu_bh_new + if ($realfile =~ /.*\/hw\/.*/ && $line =~ /\bqemu_bh_new\s*\(/) { + ERROR("use qemu_bh_new_guarded() instead of qemu_bh_new() to avoid reentrancy problems\n" . $herecurr); + } +# recommend aio_bh_new_guarded instead of aio_bh_new + if ($realfile =~ /.*\/hw\/.*/ && $line =~ /\baio_bh_new\s*\(/) { + ERROR("use aio_bh_new_guarded() instead of aio_bh_new() to avoid reentrancy problems\n" . $herecurr); + } # check for module_init(), use category-specific init macros explicitly please if ($line =~ /^module_init\s*\(/) { ERROR("please use block_init(), type_init() etc. instead of module_init()\n" . $herecurr); diff --git a/scripts/ci/gitlab-kubernetes-runners/values.yaml b/scripts/ci/gitlab-kubernetes-runners/values.yaml new file mode 100644 index 0000000000..204a96a842 --- /dev/null +++ b/scripts/ci/gitlab-kubernetes-runners/values.yaml @@ -0,0 +1,30 @@ +gitlabUrl: "https://gitlab.com/" +runnerRegistrationToken: "" +rbac: + create: true +concurrent: 200 +runners: + privileged: true + config: | + [[runners]] + limit = 100 + environment = [ + "DOCKER_HOST=tcp://docker:2376", + "DOCKER_TLS_CERTDIR=/certs", + "DOCKER_TLS_VERIFY=1", + "DOCKER_CERT_PATH=/certs/client" + ] + [runners.kubernetes] + poll_timeout = 1200 + image = "ubuntu:20.04" + cpu_request = "0.5" + service_cpu_request = "0.5" + helper_cpu_request = "0.25" + cpu_request_overwrite_max_allowed = "7" + memory_request_overwrite_max_allowed = "30Gi" + [[runners.kubernetes.volumes.empty_dir]] + name = "docker-certs" + mount_path = "/certs/client" + medium = "Memory" + [runners.kubernetes.node_selector] + agentpool = "jobs" diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure index 6e8983f39c..d02b09a4b9 100755 --- a/scripts/ci/org.centos/stream/8/x86_64/configure +++ b/scripts/ci/org.centos/stream/8/x86_64/configure @@ -29,14 +29,11 @@ --extra-cflags="-O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection" \ --with-suffix="qemu-kvm" \ --firmwarepath=/usr/share/qemu-firmware \ ---with-git=meson \ ---with-git-submodules=update \ --target-list="x86_64-softmmu" \ --block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \ --audio-drv-list="" \ --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \ --with-coroutine=ucontext \ ---with-git=git \ --tls-priority=@QEMU,SYSTEM \ --disable-attr \ --disable-auth-pam \ diff --git a/scripts/ci/org.centos/stream/8/x86_64/test-avocado b/scripts/ci/org.centos/stream/8/x86_64/test-avocado index d2c0e5fb4c..e0443fc8ae 100755 --- a/scripts/ci/org.centos/stream/8/x86_64/test-avocado +++ b/scripts/ci/org.centos/stream/8/x86_64/test-avocado @@ -30,7 +30,8 @@ make get-vm-images tests/avocado/cpu_queries.py:QueryCPUModelExpansion.test \ tests/avocado/empty_cpu_model.py:EmptyCPUModel.test \ tests/avocado/hotplug_cpu.py:HotPlugCPU.test \ - tests/avocado/igb.py:IGB.test \ + tests/avocado/netdev-ethtool.py:NetDevEthtool.test_igb \ + tests/avocado/netdev-ethtool.py:NetDevEthtool.test_igb_nomsi \ tests/avocado/info_usernet.py:InfoUsernet.test_hostfwd \ tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu \ tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu_pt \ diff --git a/scripts/ci/setup/build-environment.yml b/scripts/ci/setup/build-environment.yml index 78b1021cd4..f344d1a850 100644 --- a/scripts/ci/setup/build-environment.yml +++ b/scripts/ci/setup/build-environment.yml @@ -24,7 +24,6 @@ when: - ansible_facts['distribution'] == 'Ubuntu' - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '20.04' - name: Update apt cache / upgrade packages via apt apt: @@ -33,87 +32,131 @@ when: - ansible_facts['distribution'] == 'Ubuntu' - - name: Install basic packages to build QEMU on Ubuntu 20.04 + # lcitool variables -f json ubuntu-2204 qemu | jq -r '.pkgs[]' | xargs -n 1 echo "-" + - name: Install basic packages to build QEMU on Ubuntu 22.04 package: name: + - bash + - bc + - bison + - bsdextrautils + - bzip2 + - ca-certificates - ccache + - clang + - dbus + - debianutils + - diffutils + - exuberant-ctags + - findutils + - flex + - g++ - gcc + - gcovr + - genisoimage - gettext - git - - glusterfs-common + - hostname - libaio-dev + - libasan5 + - libasound2-dev - libattr1-dev + - libbpf-dev - libbrlapi-dev - libbz2-dev + - libc6-dev - libcacard-dev - - libcapstone-dev - libcap-ng-dev + - libcapstone-dev + - libcmocka-dev - libcurl4-gnutls-dev + - libdaxctl-dev - libdrm-dev - libepoxy-dev - libfdt-dev + - libffi-dev - libgbm-dev + - libgcrypt20-dev + - libglib2.0-dev + - libglusterfs-dev + - libgnutls28-dev - libgtk-3-dev + - libibumad-dev - libibverbs-dev - libiscsi-dev - libjemalloc-dev - libjpeg-turbo8-dev + - libjson-c-dev + - liblttng-ust-dev - liblzo2-dev - - libncurses5-dev - libncursesw5-dev - libnfs-dev - - libnss3-dev - libnuma-dev + - libpam0g-dev + - libpcre2-dev - libpixman-1-dev - - librados-dev + - libpmem-dev + - libpng-dev + - libpulse-dev - librbd-dev - librdmacm-dev - libsasl2-dev - libsdl2-dev + - libsdl2-image-dev - libseccomp-dev + - libslirp-dev - libsnappy-dev - libspice-protocol-dev + - libspice-server-dev - libssh-dev + - libsystemd-dev + - libtasn1-6-dev + - libubsan1 + - libudev-dev + - liburing-dev - libusb-1.0-0-dev - libusbredirhost-dev - libvdeplug-dev + - libvirglrenderer-dev - libvte-2.91-dev + - libxen-dev + - libxml2-dev - libzstd-dev + - llvm + - locales - make - - python3-yaml + - meson + - multipath-tools + - ncat + - nettle-dev + - ninja-build + - openssh-client + - pkgconf + - python3 + - python3-numpy + - python3-opencv + - python3-pillow + - python3-pip - python3-sphinx - python3-sphinx-rtd-theme - - ninja-build + - python3-venv + - python3-yaml + - rpm2cpio + - sed - sparse + - systemtap-sdt-dev + - tar + - tesseract-ocr + - tesseract-ocr-eng + - texinfo - xfslibs-dev + - zlib1g-dev state: present when: - ansible_facts['distribution'] == 'Ubuntu' + - ansible_facts['distribution_version'] == '22.04' - - name: Install packages to build QEMU on Ubuntu 20.04 on non-s390x - package: - name: - - libspice-server-dev - - libxen-dev - state: present - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' or ansible_facts['architecture'] == 'x86_64' - - - name: Install basic packages to build QEMU on Ubuntu 20.04 - package: - name: - # Originally from tests/docker/dockerfiles/ubuntu2004.docker - - clang-10 - - genisoimage - - liblttng-ust-dev - - libslirp-dev - - netcat-openbsd - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['distribution_version'] == '20.04' - - - name: Install armhf cross-compile packages to build QEMU on AArch64 Ubuntu 20.04 + - name: Install armhf cross-compile packages to build QEMU on AArch64 Ubuntu 22.04 package: name: - binutils-arm-linux-gnueabihf @@ -128,7 +171,7 @@ - zlib1g-dev:armhf when: - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['distribution_version'] == '20.04' + - ansible_facts['distribution_version'] == '22.04' - ansible_facts['architecture'] == 'aarch64' - name: Enable EPEL repo on EL8 diff --git a/scripts/ci/setup/gitlab-runner.yml b/scripts/ci/setup/gitlab-runner.yml index 1a1b270ff2..7bdafab511 100644 --- a/scripts/ci/setup/gitlab-runner.yml +++ b/scripts/ci/setup/gitlab-runner.yml @@ -26,6 +26,7 @@ user: user: gitlab-runner group: gitlab-runner + groups: kvm comment: GitLab Runner home: /home/gitlab-runner shell: /bin/bash diff --git a/scripts/coverity-scan/COMPONENTS.md b/scripts/coverity-scan/COMPONENTS.md index 7c48e0f1d2..883da95aff 100644 --- a/scripts/coverity-scan/COMPONENTS.md +++ b/scripts/coverity-scan/COMPONENTS.md @@ -24,6 +24,9 @@ hppa i386 ~ (/qemu)?((/include)?/hw/i386/.*|/target/i386/.*|/hw/intc/[^/]*apic[^/]*\.c) +loongarch + ~ (/qemu)?((/include)?/hw/(loongarch/.*|.*/loongarch.*)|/target/loongarch/.*) + m68k ~ (/qemu)?((/include)?/hw/m68k/.*|/target/m68k/.*|(/include)?/hw(/.*)?/mcf.*|(/include)?/hw/nubus/.*) @@ -36,11 +39,14 @@ mips nios2 ~ (/qemu)?((/include)?/hw/nios2/.*|/target/nios2/.*) +openrisc + ~ (/qemu)?((/include)?/hw/openrisc/.*|/target/openrisc/.*) + ppc ~ (/qemu)?((/include)?/hw/ppc/.*|/target/ppc/.*|/hw/pci-host/(uninorth.*|dec.*|prep.*|ppc.*)|/hw/misc/macio/.*|(/include)?/hw/.*/(xics|openpic|spapr).*) riscv - ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*) + ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*|/hw/.*/(riscv_|ibex_|sifive_).*) rx ~ (/qemu)?((/include)?/hw/rx/.*|/target/rx/.*) @@ -54,12 +60,12 @@ sh4 sparc ~ (/qemu)?((/include)?/hw/sparc(64)?.*|/target/sparc/.*|/hw/.*/grlib.*|/hw/display/cg3.c) -tilegx - ~ (/qemu)?(/target/tilegx/.*) - tricore ~ (/qemu)?((/include)?/hw/tricore/.*|/target/tricore/.*) +xtensa + ~ (/qemu)?((/include)?/hw/xtensa/.*|/target/xtensa/.*) + 9pfs ~ (/qemu)?(/hw/9pfs/.*|/fsdev/.*) @@ -73,7 +79,7 @@ char ~ (/qemu)?(/qemu-char\.c|/include/sysemu/char\.h|(/include)?/hw/char/.*) crypto - ~ (/qemu)?((/include)?/crypto/.*|/hw/.*/crypto.*) + ~ (/qemu)?((/include)?/crypto/.*|/hw/.*/.*crypto.*|(/include/sysemu|/backends)/cryptodev.*) disas ~ (/qemu)?((/include)?/disas.*) @@ -100,7 +106,7 @@ net ~ (/qemu)?((/include)?(/hw)?/(net|rdma)/.*) pci - ~ (/qemu)?(/hw/pci.*|/include/hw/pci.*) + ~ (/qemu)?(/include)?/hw/(cxl/|pci).* qemu-ga ~ (/qemu)?(/qga/.*) @@ -108,9 +114,6 @@ qemu-ga scsi ~ (/qemu)?(/scsi/.*|/hw/scsi/.*|/include/hw/scsi/.*) -tcg - ~ (/qemu)?(/accel/tcg/.*|/replay/.*|/(.*/)?softmmu.*) - trace ~ (/qemu)?(/.*trace.*\.[ch]) @@ -126,9 +129,27 @@ user util ~ (/qemu)?(/util/.*|/include/qemu/.*) +vfio + ~ (/qemu)?(/include)?/hw/vfio/.* + +virtio + ~ (/qemu)?(/include)?/hw/virtio/.* + xen ~ (/qemu)?(.*/xen.*) +hvf + ~ (/qemu)?(.*/hvf.*) + +kvm + ~ (/qemu)?(.*/kvm.*) + +tcg + ~ (/qemu)?(/accel/tcg|/replay|/tcg)/.* + +sysemu + ~ (/qemu)?(/softmmu/.*|/accel/.*) + (headers) ~ (/qemu)?(/include/.*) @@ -137,9 +158,3 @@ testlibs tests ~ (/qemu)?(/tests/.*) - -loongarch - ~ (/qemu)?((/include)?/hw/(loongarch/.*|.*/loongarch.*)|/target/loongarch/.*) - -riscv - ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*|/hw/.*/(riscv_|ibex_|sifive_).*) diff --git a/scripts/decodetree.py b/scripts/decodetree.py index a03dc6b5e3..a8a6cb69cd 100644 --- a/scripts/decodetree.py +++ b/scripts/decodetree.py @@ -35,12 +35,14 @@ arguments = {} formats = {} allpatterns = [] anyextern = False +testforerror = False translate_prefix = 'trans' translate_scope = 'static ' input_file = '' output_file = None output_fd = None +output_null = False insntype = 'uint32_t' decode_function = 'decode' @@ -53,6 +55,80 @@ re_fld_ident = '%[a-zA-Z0-9_]*' re_fmt_ident = '@[a-zA-Z0-9_]*' re_pat_ident = '[a-zA-Z0-9_]*' +# Local implementation of a topological sort. We use the same API that +# the Python graphlib does, so that when QEMU moves forward to a +# baseline of Python 3.9 or newer this code can all be dropped and +# replaced with: +# from graphlib import TopologicalSorter, CycleError +# +# https://docs.python.org/3.9/library/graphlib.html#graphlib.TopologicalSorter +# +# We only implement the parts of TopologicalSorter we care about: +# ts = TopologicalSorter(graph=None) +# create the sorter. graph is a dictionary whose keys are +# nodes and whose values are lists of the predecessors of that node. +# (That is, if graph contains "A" -> ["B", "C"] then we must output +# B and C before A.) +# ts.static_order() +# returns a list of all the nodes in sorted order, or raises CycleError +# CycleError +# exception raised if there are cycles in the graph. The second +# element in the args attribute is a list of nodes which form a +# cycle; the first and last element are the same, eg [a, b, c, a] +# (Our implementation doesn't give the order correctly.) +# +# For our purposes we can assume that the data set is always small +# (typically 10 nodes or less, actual links in the graph very rare), +# so we don't need to worry about efficiency of implementation. +# +# The core of this implementation is from +# https://code.activestate.com/recipes/578272-topological-sort/ +# (but updated to Python 3), and is under the MIT license. + +class CycleError(ValueError): + """Subclass of ValueError raised if cycles exist in the graph""" + pass + +class TopologicalSorter: + """Topologically sort a graph""" + def __init__(self, graph=None): + self.graph = graph + + def static_order(self): + # We do the sort right here, unlike the stdlib version + from functools import reduce + data = {} + r = [] + + if not self.graph: + return [] + + # This code wants the values in the dict to be specifically sets + for k, v in self.graph.items(): + data[k] = set(v) + + # Find all items that don't depend on anything. + extra_items_in_deps = (reduce(set.union, data.values()) + - set(data.keys())) + # Add empty dependencies where needed + data.update({item:{} for item in extra_items_in_deps}) + while True: + ordered = set(item for item, dep in data.items() if not dep) + if not ordered: + break + r.extend(ordered) + data = {item: (dep - ordered) + for item, dep in data.items() + if item not in ordered} + if data: + # This doesn't give as nice results as the stdlib, which + # gives you the cycle by listing the nodes in order. Here + # we only know the nodes in the cycle but not their order. + raise CycleError(f'nodes are in a cycle', list(data.keys())) + + return r +# end TopologicalSorter + def error_with_file(file, lineno, *args): """Print an error message from file:line and args and exit.""" global output_file @@ -71,7 +147,7 @@ def error_with_file(file, lineno, *args): if output_file and output_fd: output_fd.close() os.remove(output_file) - exit(1) + exit(0 if testforerror else 1) # end error_with_file @@ -205,11 +281,14 @@ class Field: s = '' return str(self.pos) + ':' + s + str(self.len) - def str_extract(self): + def str_extract(self, lvalue_formatter): global bitop_width s = 's' if self.sign else '' return f'{s}extract{bitop_width}(insn, {self.pos}, {self.len})' + def referenced_fields(self): + return [] + def __eq__(self, other): return self.sign == other.sign and self.mask == other.mask @@ -228,12 +307,12 @@ class MultiField: def __str__(self): return str(self.subs) - def str_extract(self): + def str_extract(self, lvalue_formatter): global bitop_width ret = '0' pos = 0 for f in reversed(self.subs): - ext = f.str_extract() + ext = f.str_extract(lvalue_formatter) if pos == 0: ret = ext else: @@ -241,6 +320,12 @@ class MultiField: pos += f.len return ret + def referenced_fields(self): + l = [] + for f in self.subs: + l.extend(f.referenced_fields()) + return l + def __ne__(self, other): if len(self.subs) != len(other.subs): return True @@ -264,9 +349,12 @@ class ConstField: def __str__(self): return str(self.value) - def str_extract(self): + def str_extract(self, lvalue_formatter): return str(self.value) + def referenced_fields(self): + return [] + def __cmp__(self, other): return self.value - other.value # end ConstField @@ -283,8 +371,12 @@ class FunctionField: def __str__(self): return self.func + '(' + str(self.base) + ')' - def str_extract(self): - return self.func + '(ctx, ' + self.base.str_extract() + ')' + def str_extract(self, lvalue_formatter): + return (self.func + '(ctx, ' + + self.base.str_extract(lvalue_formatter) + ')') + + def referenced_fields(self): + return self.base.referenced_fields() def __eq__(self, other): return self.func == other.func and self.base == other.base @@ -304,9 +396,12 @@ class ParameterField: def __str__(self): return self.func - def str_extract(self): + def str_extract(self, lvalue_formatter): return self.func + '(ctx)' + def referenced_fields(self): + return [] + def __eq__(self, other): return self.func == other.func @@ -314,6 +409,32 @@ class ParameterField: return not self.__eq__(other) # end ParameterField +class NamedField: + """Class representing a field already named in the pattern""" + def __init__(self, name, sign, len): + self.mask = 0 + self.sign = sign + self.len = len + self.name = name + + def __str__(self): + return self.name + + def str_extract(self, lvalue_formatter): + global bitop_width + s = 's' if self.sign else '' + lvalue = lvalue_formatter(self.name) + return f'{s}extract{bitop_width}({lvalue}, 0, {self.len})' + + def referenced_fields(self): + return [self.name] + + def __eq__(self, other): + return self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) +# end NamedField class Arguments: """Class representing the extracted fields of a format""" @@ -337,7 +458,6 @@ class Arguments: output('} ', self.struct_name(), ';\n\n') # end Arguments - class General: """Common code between instruction formats and instruction patterns""" def __init__(self, name, lineno, base, fixb, fixm, udfm, fldm, flds, w): @@ -351,12 +471,59 @@ class General: self.fieldmask = fldm self.fields = flds self.width = w + self.dangling = None def __str__(self): return self.name + ' ' + str_match_bits(self.fixedbits, self.fixedmask) def str1(self, i): return str_indent(i) + self.__str__() + + def dangling_references(self): + # Return a list of all named references which aren't satisfied + # directly by this format/pattern. This will be either: + # * a format referring to a field which is specified by the + # pattern(s) using it + # * a pattern referring to a field which is specified by the + # format it uses + # * a user error (referring to a field that doesn't exist at all) + if self.dangling is None: + # Compute this once and cache the answer + dangling = [] + for n, f in self.fields.items(): + for r in f.referenced_fields(): + if r not in self.fields: + dangling.append(r) + self.dangling = dangling + return self.dangling + + def output_fields(self, indent, lvalue_formatter): + # We use a topological sort to ensure that any use of NamedField + # comes after the initialization of the field it is referencing. + graph = {} + for n, f in self.fields.items(): + refs = f.referenced_fields() + graph[n] = refs + + try: + ts = TopologicalSorter(graph) + for n in ts.static_order(): + # We only want to emit assignments for the keys + # in our fields list, not for anything that ends up + # in the tsort graph only because it was referenced as + # a NamedField. + try: + f = self.fields[n] + output(indent, lvalue_formatter(n), ' = ', + f.str_extract(lvalue_formatter), ';\n') + except KeyError: + pass + except CycleError as e: + # The second element of args is a list of nodes which form + # a cycle (there might be others too, but only one is reported). + # Pretty-print it to tell the user. + cycle = ' => '.join(e.args[1]) + error(self.lineno, 'field definitions form a cycle: ' + cycle) # end General @@ -370,8 +537,7 @@ class Format(General): def output_extract(self): output('static void ', self.extract_name(), '(DisasContext *ctx, ', self.base.struct_name(), ' *a, ', insntype, ' insn)\n{\n') - for n, f in self.fields.items(): - output(' a->', n, ' = ', f.str_extract(), ';\n') + self.output_fields(str_indent(4), lambda n: 'a->' + n) output('}\n\n') # end Format @@ -392,11 +558,36 @@ class Pattern(General): ind = str_indent(i) arg = self.base.base.name output(ind, '/* ', self.file, ':', str(self.lineno), ' */\n') + # We might have named references in the format that refer to fields + # in the pattern, or named references in the pattern that refer + # to fields in the format. This affects whether we extract the fields + # for the format before or after the ones for the pattern. + # For simplicity we don't allow cross references in both directions. + # This is also where we catch the syntax error of referring to + # a nonexistent field. + fmt_refs = self.base.dangling_references() + for r in fmt_refs: + if r not in self.fields: + error(self.lineno, f'format refers to undefined field {r}') + pat_refs = self.dangling_references() + for r in pat_refs: + if r not in self.base.fields: + error(self.lineno, f'pattern refers to undefined field {r}') + if pat_refs and fmt_refs: + error(self.lineno, ('pattern that uses fields defined in format ' + 'cannot use format that uses fields defined ' + 'in pattern')) + if fmt_refs: + # pattern fields first + self.output_fields(ind, lambda n: 'u.f_' + arg + '.' + n) + assert not extracted, "dangling fmt refs but it was already extracted" if not extracted: output(ind, self.base.extract_name(), '(ctx, &u.f_', arg, ', insn);\n') - for n, f in self.fields.items(): - output(ind, 'u.f_', arg, '.', n, ' = ', f.str_extract(), ';\n') + if not fmt_refs: + # pattern fields last + self.output_fields(ind, lambda n: 'u.f_' + arg + '.' + n) + output(ind, 'if (', translate_prefix, '_', self.name, '(ctx, &u.f_', arg, ')) return true;\n') @@ -473,7 +664,7 @@ class MultiPattern(General): def prop_format(self): for p in self.pats: - p.build_tree() + p.prop_format() def prop_width(self): width = None @@ -505,6 +696,12 @@ class IncMultiPattern(MultiPattern): output(ind, '}\n') else: p.output_code(i, extracted, p.fixedbits, p.fixedmask) + + def build_tree(self): + if not self.pats: + error_with_file(self.file, self.lineno, 'empty pattern group') + super().build_tree() + #end IncMultiPattern @@ -536,8 +733,10 @@ class Tree: ind = str_indent(i) # If we identified all nodes below have the same format, - # extract the fields now. - if not extracted and self.base: + # extract the fields now. But don't do it if the format relies + # on named fields from the insn pattern, as those won't have + # been initialised at this point. + if not extracted and self.base and not self.base.dangling_references(): output(ind, self.base.extract_name(), '(ctx, &u.f_', self.base.base.name, ', insn);\n') extracted = True @@ -623,7 +822,7 @@ class ExcMultiPattern(MultiPattern): return t def build_tree(self): - super().prop_format() + super().build_tree() self.tree = self.__build_tree(self.pats, self.fixedbits, self.fixedmask) @@ -659,6 +858,7 @@ def parse_field(lineno, name, toks): """Parse one instruction field from TOKS at LINENO""" global fields global insnwidth + global re_C_ident # A "simple" field will have only one entry; # a "multifield" will have several. @@ -673,6 +873,25 @@ def parse_field(lineno, name, toks): func = func[1] continue + if re.fullmatch(re_C_ident + ':s[0-9]+', t): + # Signed named field + subtoks = t.split(':') + n = subtoks[0] + le = int(subtoks[1]) + f = NamedField(n, True, le) + subs.append(f) + width += le + continue + if re.fullmatch(re_C_ident + ':[0-9]+', t): + # Unsigned named field + subtoks = t.split(':') + n = subtoks[0] + le = int(subtoks[1]) + f = NamedField(n, False, le) + subs.append(f) + width += le + continue + if re.fullmatch('[0-9]+:s[0-9]+', t): # Signed field extract subtoks = t.split(':s') @@ -1278,6 +1497,7 @@ def main(): global translate_prefix global output_fd global output_file + global output_null global input_file global insnwidth global insntype @@ -1286,11 +1506,13 @@ def main(): global bitop_width global variablewidth global anyextern + global testforerror decode_scope = 'static ' long_opts = ['decode=', 'translate=', 'output=', 'insnwidth=', - 'static-decode=', 'varinsnwidth='] + 'static-decode=', 'varinsnwidth=', 'test-for-error', + 'output-null'] try: (opts, args) = getopt.gnu_getopt(sys.argv[1:], 'o:vw:', long_opts) except getopt.GetoptError as err: @@ -1319,6 +1541,10 @@ def main(): bitop_width = 64 elif insnwidth != 32: error(0, 'cannot handle insns of width', insnwidth) + elif o == '--test-for-error': + testforerror = True + elif o == '--output-null': + output_null = True else: assert False, 'unhandled option' @@ -1348,7 +1574,9 @@ def main(): stree = build_size_tree(toppat.pats, 8, 0, 0) prop_size(stree) - if output_file: + if output_null: + output_fd = open(os.devnull, 'wt', encoding='utf-8', errors="ignore") + elif output_file: output_fd = open(output_file, 'wt', encoding='utf-8') else: output_fd = io.TextIOWrapper(sys.stdout.buffer, @@ -1417,6 +1645,7 @@ def main(): if output_file: output_fd.close() + exit(1 if testforerror else 0) # end main diff --git a/scripts/device-crash-test b/scripts/device-crash-test index 73bcb98693..b74d887331 100755 --- a/scripts/device-crash-test +++ b/scripts/device-crash-test @@ -397,7 +397,7 @@ def binariesToTest(args, testcase): def accelsToTest(args, testcase): - if getBinaryInfo(args, testcase['binary']).kvm_available: + if getBinaryInfo(args, testcase['binary']).kvm_available and not args.tcg_only: yield 'kvm' yield 'tcg' @@ -510,6 +510,8 @@ def main(): help="Full mode: test cases that are expected to fail") parser.add_argument('--strict', action='store_true', dest='strict', help="Treat all warnings as fatal") + parser.add_argument('--tcg-only', action='store_true', dest='tcg_only', + help="Only test with TCG accelerator") parser.add_argument('qemu', nargs='*', metavar='QEMU', help='QEMU binary to run') args = parser.parse_args() diff --git a/scripts/git-submodule.sh b/scripts/git-submodule.sh index 7be41f5948..11fad2137c 100755 --- a/scripts/git-submodule.sh +++ b/scripts/git-submodule.sh @@ -9,10 +9,10 @@ command=$1 shift maybe_modules="$@" -# if --with-git-submodules=ignore, do nothing +# if not running in a git checkout, do nothing test "$command" = "ignore" && exit 0 -test -z "$GIT" && GIT=git +test -z "$GIT" && GIT=$(command -v git) cd "$(dirname "$0")/.." @@ -21,19 +21,14 @@ update_error() { echo echo "Unable to automatically checkout GIT submodules '$modules'." echo "If you require use of an alternative GIT binary (for example to" - echo "enable use of a transparent proxy), then please specify it by" - echo "running configure by with the '--with-git' argument. e.g." + echo "enable use of a transparent proxy), please disable automatic" + echo "GIT submodule checkout with:" echo - echo " $ ./configure --with-git='tsocks git'" - echo - echo "Alternatively you may disable automatic GIT submodule checkout" - echo "with:" - echo - echo " $ ./configure --with-git-submodules=validate" + echo " $ ./configure --disable-download" echo echo "and then manually update submodules prior to running make, with:" echo - echo " $ scripts/git-submodule.sh update $modules" + echo " $ GIT='tsocks git' scripts/git-submodule.sh update $modules" echo exit 1 } @@ -44,19 +39,30 @@ validate_error() { echo "configured for validate only. Please run" echo " scripts/git-submodule.sh update $maybe_modules" echo "from the source directory or call configure with" - echo " --with-git-submodules=update" - echo "To disable GIT submodules validation, use" - echo " --with-git-submodules=ignore" + echo " --enable-download" fi exit 1 } +check_updated() { + local CURSTATUS OLDSTATUS + CURSTATUS=$($GIT submodule status $module) + OLDSTATUS=$(grep $module $substat) + test "$CURSTATUS" = "$OLDSTATUS" +} + if test -n "$maybe_modules" && ! test -e ".git" then echo "$0: unexpectedly called with submodules but no git checkout exists" exit 1 fi +if test -n "$maybe_modules" && test -z "$GIT" +then + echo "$0: unexpectedly called with submodules but git binary not found" + exit 1 +fi + modules="" for m in $maybe_modules do @@ -71,33 +77,34 @@ done case "$command" in status|validate) - if test -z "$maybe_modules" - then - test -s ${substat} && validate_error "$command" || exit 0 - fi - test -f "$substat" || validate_error "$command" + test -z "$maybe_modules" && exit 0 for module in $modules; do - CURSTATUS=$($GIT submodule status $module) - OLDSTATUS=$(cat $substat | grep $module) - if test "$CURSTATUS" != "$OLDSTATUS"; then - validate_error "$command" - fi + check_updated $module || validate_error "$command" done exit 0 ;; update) - if test -z "$maybe_modules" - then - test -e $substat || touch $substat - exit 0 - fi + test -e $substat || touch $substat + test -z "$maybe_modules" && exit 0 $GIT submodule update --init $modules 1>/dev/null test $? -ne 0 && update_error "failed to update modules" + for module in $modules; do + check_updated $module || echo Updated "$module" + done - $GIT submodule status $modules > "${substat}" - test $? -ne 0 && update_error "failed to save git submodule status" >&2 + (while read -r; do + for module in $modules; do + case $REPLY in + *" $module "*) continue 2 ;; + esac + done + printf '%s\n' "$REPLY" + done + $GIT submodule status $modules + test $? -ne 0 && update_error "failed to save git submodule status" >&2) < $substat > $substat.new + mv -f $substat.new $substat ;; esac diff --git a/scripts/make-config-poison.sh b/scripts/make-config-poison.sh index 1892854261..2b36907e23 100755 --- a/scripts/make-config-poison.sh +++ b/scripts/make-config-poison.sh @@ -4,11 +4,12 @@ if test $# = 0; then exit 0 fi -# Create list of config switches that should be poisoned in common code... -# but filter out CONFIG_TCG and CONFIG_USER_ONLY which are special. +# Create list of config switches that should be poisoned in common code, +# but filter out several which are handled manually. exec sed -n \ -e' /CONFIG_TCG/d' \ -e '/CONFIG_USER_ONLY/d' \ + -e '/CONFIG_SOFTMMU/d' \ -e '/^#define / {' \ -e 's///' \ -e 's/ .*//' \ diff --git a/scripts/make-release b/scripts/make-release index 44a9d86a04..c5db87b3f9 100755 --- a/scripts/make-release +++ b/scripts/make-release @@ -16,6 +16,9 @@ if [ $# -ne 2 ]; then exit 0 fi +# Only include wraps that are invoked with subproject() +SUBPROJECTS="dtc libvfio-user keycodemapdb berkeley-softfloat-3 berkeley-testfloat-3" + src="$1" version="$2" destination=qemu-${version} @@ -26,6 +29,8 @@ git clone --single-branch -b "v${version}" -c advice.detachedHead=false \ pushd ${destination} git submodule update --init --single-branch +meson subprojects download $SUBPROJECTS + (cd roms/seabios && git describe --tags --long --dirty > .version) (cd roms/skiboot && ./make_version.sh > .version) # Fetch edk2 submodule's submodules, since it won't have access to them via diff --git a/scripts/meson-buildoptions.py b/scripts/meson-buildoptions.py index a04dcc70a5..8d2e526132 100755..100644 --- a/scripts/meson-buildoptions.py +++ b/scripts/meson-buildoptions.py @@ -35,6 +35,8 @@ SKIP_OPTIONS = { OPTION_NAMES = { "b_coverage": "gcov", "b_lto": "lto", + "coroutine_backend": "with-coroutine", + "debug": "debug-info", "malloc": "enable-malloc", "pkgversion": "with-pkgversion", "qemu_firmwarepath": "firmwarepath", @@ -46,6 +48,7 @@ BUILTIN_OPTIONS = { "b_coverage", "b_lto", "datadir", + "debug", "includedir", "libdir", "libexecdir", diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 009fab1515..5714fd93d9 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -1,7 +1,8 @@ # This file is generated by meson-buildoptions.py, do not edit! meson_options_help() { printf "%s\n" ' --audio-drv-list=CHOICES Set audio driver list [default] (choices: alsa/co' - printf "%s\n" ' reaudio/default/dsound/jack/oss/pa/sdl/sndio)' + printf "%s\n" ' reaudio/default/dsound/jack/oss/pa/pipewire/sdl/s' + printf "%s\n" ' ndio)' printf "%s\n" ' --block-drv-ro-whitelist=VALUE' printf "%s\n" ' set block driver read-only whitelist (by default' printf "%s\n" ' affects only QEMU, not tools like qemu-img)' @@ -10,10 +11,12 @@ meson_options_help() { printf "%s\n" ' affects only QEMU, not tools like qemu-img)' printf "%s\n" ' --datadir=VALUE Data file directory [share]' printf "%s\n" ' --disable-coroutine-pool coroutine freelist (better performance)' + printf "%s\n" ' --disable-debug-info Enable debug symbols and other information' printf "%s\n" ' --disable-hexagon-idef-parser' printf "%s\n" ' use idef-parser to automatically generate TCG' printf "%s\n" ' code for the Hexagon frontend' printf "%s\n" ' --disable-install-blobs install provided firmware blobs' + printf "%s\n" ' --disable-qom-cast-debug cast debugging support' printf "%s\n" ' --docdir=VALUE Base directory for documentation installation' printf "%s\n" ' (can be empty) [share/doc]' printf "%s\n" ' --enable-block-drv-whitelist-in-tools' @@ -21,6 +24,8 @@ meson_options_help() { printf "%s\n" ' QEMU' printf "%s\n" ' --enable-cfi Control-Flow Integrity (CFI)' printf "%s\n" ' --enable-cfi-debug Verbose errors in case of CFI violation' + printf "%s\n" ' --enable-debug-graph-lock' + printf "%s\n" ' graph lock debugging support' printf "%s\n" ' --enable-debug-mutex mutex debugging support' printf "%s\n" ' --enable-debug-stack-usage' printf "%s\n" ' measure coroutine stack usage' @@ -35,14 +40,17 @@ meson_options_help() { printf "%s\n" ' --enable-module-upgrades try to load modules from alternate paths for' printf "%s\n" ' upgrades' printf "%s\n" ' --enable-profiler profiler support' - printf "%s\n" ' --enable-qom-cast-debug cast debugging support' printf "%s\n" ' --enable-rng-none dummy RNG, avoid using /dev/(u)random and' printf "%s\n" ' getrandom()' + printf "%s\n" ' --enable-safe-stack SafeStack Stack Smash Protection (requires' + printf "%s\n" ' clang/llvm and coroutine backend ucontext)' + printf "%s\n" ' --enable-sanitizers enable default sanitizers' printf "%s\n" ' --enable-strip Strip targets on install' printf "%s\n" ' --enable-tcg-interpreter TCG with bytecode interpreter (slow)' printf "%s\n" ' --enable-trace-backends=CHOICES' printf "%s\n" ' Set available tracing backends [log] (choices:' printf "%s\n" ' dtrace/ftrace/log/nop/simple/syslog/ust)' + printf "%s\n" ' --enable-tsan enable thread sanitizer' printf "%s\n" ' --firmwarepath=VALUES search PATH for firmware files [share/qemu-' printf "%s\n" ' firmware]' printf "%s\n" ' --iasl=VALUE Path to ACPI disassembler' @@ -54,11 +62,11 @@ meson_options_help() { printf "%s\n" ' --localedir=VALUE Locale data directory [share/locale]' printf "%s\n" ' --localstatedir=VALUE Localstate data directory [/var/local]' printf "%s\n" ' --mandir=VALUE Manual page directory [share/man]' - printf "%s\n" ' --sphinx-build=VALUE Use specified sphinx-build for building document' - printf "%s\n" ' [sphinx-build]' printf "%s\n" ' --sysconfdir=VALUE Sysconf data directory [etc]' printf "%s\n" ' --tls-priority=VALUE Default TLS protocol/cipher priority string' printf "%s\n" ' [NORMAL]' + printf "%s\n" ' --with-coroutine=CHOICE coroutine backend to use (choices:' + printf "%s\n" ' auto/sigaltstack/ucontext/windows)' printf "%s\n" ' --with-pkgversion=VALUE use specified string as sub-version of the' printf "%s\n" ' package' printf "%s\n" ' --with-trace-file=VALUE Trace file prefix for simple backend [trace]' @@ -83,6 +91,7 @@ meson_options_help() { printf "%s\n" ' capstone Whether and how to find the capstone library' printf "%s\n" ' cloop cloop image format support' printf "%s\n" ' cocoa Cocoa user interface (macOS only)' + printf "%s\n" ' colo-proxy colo-proxy support' printf "%s\n" ' coreaudio CoreAudio sound support' printf "%s\n" ' crypto-afalg Linux AF_ALG crypto backend driver' printf "%s\n" ' curl CURL block device driver' @@ -126,6 +135,7 @@ meson_options_help() { printf "%s\n" ' lzo lzo compression support' printf "%s\n" ' malloc-trim enable libc malloc_trim() for memory optimization' printf "%s\n" ' membarrier membarrier system call (for Linux 4.14+ or Windows' + printf "%s\n" ' modules modules support (non Windows)' printf "%s\n" ' mpath Multipath persistent reservation passthrough' printf "%s\n" ' multiprocess Out of process device emulation support' printf "%s\n" ' netmap netmap network backend support' @@ -136,6 +146,7 @@ meson_options_help() { printf "%s\n" ' oss OSS sound support' printf "%s\n" ' pa PulseAudio sound support' printf "%s\n" ' parallels parallels image format support' + printf "%s\n" ' pipewire Pipewire sound support' printf "%s\n" ' png PNG support with libpng' printf "%s\n" ' pvrdma Enable PVRDMA support' printf "%s\n" ' qcow1 qcow1 image format support' @@ -156,6 +167,7 @@ meson_options_help() { printf "%s\n" ' sparse sparse checker' printf "%s\n" ' spice Spice server support' printf "%s\n" ' spice-protocol Spice protocol support' + printf "%s\n" ' stack-protector compiler-provided stack protection' printf "%s\n" ' tcg TCG support' printf "%s\n" ' tools build support utilities that come with QEMU' printf "%s\n" ' tpm TPM support' @@ -167,6 +179,7 @@ meson_options_help() { printf "%s\n" ' VDUSE block export support' printf "%s\n" ' vfio-user-server' printf "%s\n" ' vfio-user server support' + printf "%s\n" ' vhdx vhdx image format support' printf "%s\n" ' vhost-crypto vhost-user crypto backend support' printf "%s\n" ' vhost-kernel vhost kernel backend support' printf "%s\n" ' vhost-net vhost-net kernel acceleration support' @@ -176,10 +189,14 @@ meson_options_help() { printf "%s\n" ' vhost-vdpa vhost-vdpa kernel backend support' printf "%s\n" ' virglrenderer virgl rendering support' printf "%s\n" ' virtfs virtio-9p support' + printf "%s\n" ' virtfs-proxy-helper' + printf "%s\n" ' virtio-9p proxy helper support' + printf "%s\n" ' vmdk vmdk image format support' printf "%s\n" ' vmnet vmnet.framework network backend support' printf "%s\n" ' vnc VNC server' printf "%s\n" ' vnc-jpeg JPEG lossy compression for VNC server' printf "%s\n" ' vnc-sasl SASL authentication for VNC server' + printf "%s\n" ' vpc vpc image format support' printf "%s\n" ' vte vte support for the gtk UI' printf "%s\n" ' vvfat vvfat image format support' printf "%s\n" ' whpx WHPX acceleration support' @@ -236,8 +253,11 @@ _meson_option_parse() { --disable-cloop) printf "%s" -Dcloop=disabled ;; --enable-cocoa) printf "%s" -Dcocoa=enabled ;; --disable-cocoa) printf "%s" -Dcocoa=disabled ;; + --enable-colo-proxy) printf "%s" -Dcolo_proxy=enabled ;; + --disable-colo-proxy) printf "%s" -Dcolo_proxy=disabled ;; --enable-coreaudio) printf "%s" -Dcoreaudio=enabled ;; --disable-coreaudio) printf "%s" -Dcoreaudio=disabled ;; + --with-coroutine=*) quote_sh "-Dcoroutine_backend=$2" ;; --enable-coroutine-pool) printf "%s" -Dcoroutine_pool=true ;; --disable-coroutine-pool) printf "%s" -Dcoroutine_pool=false ;; --enable-crypto-afalg) printf "%s" -Dcrypto_afalg=enabled ;; @@ -249,6 +269,10 @@ _meson_option_parse() { --datadir=*) quote_sh "-Ddatadir=$2" ;; --enable-dbus-display) printf "%s" -Ddbus_display=enabled ;; --disable-dbus-display) printf "%s" -Ddbus_display=disabled ;; + --enable-debug-info) printf "%s" -Ddebug=true ;; + --disable-debug-info) printf "%s" -Ddebug=false ;; + --enable-debug-graph-lock) printf "%s" -Ddebug_graph_lock=true ;; + --disable-debug-graph-lock) printf "%s" -Ddebug_graph_lock=false ;; --enable-debug-mutex) printf "%s" -Ddebug_mutex=true ;; --disable-debug-mutex) printf "%s" -Ddebug_mutex=false ;; --enable-debug-stack-usage) printf "%s" -Ddebug_stack_usage=true ;; @@ -350,6 +374,8 @@ _meson_option_parse() { --disable-membarrier) printf "%s" -Dmembarrier=disabled ;; --enable-module-upgrades) printf "%s" -Dmodule_upgrades=true ;; --disable-module-upgrades) printf "%s" -Dmodule_upgrades=false ;; + --enable-modules) printf "%s" -Dmodules=enabled ;; + --disable-modules) printf "%s" -Dmodules=disabled ;; --enable-mpath) printf "%s" -Dmpath=enabled ;; --disable-mpath) printf "%s" -Dmpath=disabled ;; --enable-multiprocess) printf "%s" -Dmultiprocess=enabled ;; @@ -370,6 +396,8 @@ _meson_option_parse() { --disable-pa) printf "%s" -Dpa=disabled ;; --enable-parallels) printf "%s" -Dparallels=enabled ;; --disable-parallels) printf "%s" -Dparallels=disabled ;; + --enable-pipewire) printf "%s" -Dpipewire=enabled ;; + --disable-pipewire) printf "%s" -Dpipewire=disabled ;; --with-pkgversion=*) quote_sh "-Dpkgversion=$2" ;; --enable-png) printf "%s" -Dpng=enabled ;; --disable-png) printf "%s" -Dpng=disabled ;; @@ -394,6 +422,10 @@ _meson_option_parse() { --disable-replication) printf "%s" -Dreplication=disabled ;; --enable-rng-none) printf "%s" -Drng_none=true ;; --disable-rng-none) printf "%s" -Drng_none=false ;; + --enable-safe-stack) printf "%s" -Dsafe_stack=true ;; + --disable-safe-stack) printf "%s" -Dsafe_stack=false ;; + --enable-sanitizers) printf "%s" -Dsanitizers=true ;; + --disable-sanitizers) printf "%s" -Dsanitizers=false ;; --enable-sdl) printf "%s" -Dsdl=enabled ;; --disable-sdl) printf "%s" -Dsdl=disabled ;; --enable-sdl-image) printf "%s" -Dsdl_image=enabled ;; @@ -414,11 +446,12 @@ _meson_option_parse() { --disable-sndio) printf "%s" -Dsndio=disabled ;; --enable-sparse) printf "%s" -Dsparse=enabled ;; --disable-sparse) printf "%s" -Dsparse=disabled ;; - --sphinx-build=*) quote_sh "-Dsphinx_build=$2" ;; --enable-spice) printf "%s" -Dspice=enabled ;; --disable-spice) printf "%s" -Dspice=disabled ;; --enable-spice-protocol) printf "%s" -Dspice_protocol=enabled ;; --disable-spice-protocol) printf "%s" -Dspice_protocol=disabled ;; + --enable-stack-protector) printf "%s" -Dstack_protector=enabled ;; + --disable-stack-protector) printf "%s" -Dstack_protector=disabled ;; --enable-strip) printf "%s" -Dstrip=true ;; --disable-strip) printf "%s" -Dstrip=false ;; --sysconfdir=*) quote_sh "-Dsysconfdir=$2" ;; @@ -433,6 +466,8 @@ _meson_option_parse() { --disable-tpm) printf "%s" -Dtpm=disabled ;; --enable-trace-backends=*) quote_sh "-Dtrace_backends=$2" ;; --with-trace-file=*) quote_sh "-Dtrace_file=$2" ;; + --enable-tsan) printf "%s" -Dtsan=true ;; + --disable-tsan) printf "%s" -Dtsan=false ;; --enable-u2f) printf "%s" -Du2f=enabled ;; --disable-u2f) printf "%s" -Du2f=disabled ;; --enable-usb-redir) printf "%s" -Dusb_redir=enabled ;; @@ -445,6 +480,8 @@ _meson_option_parse() { --disable-vduse-blk-export) printf "%s" -Dvduse_blk_export=disabled ;; --enable-vfio-user-server) printf "%s" -Dvfio_user_server=enabled ;; --disable-vfio-user-server) printf "%s" -Dvfio_user_server=disabled ;; + --enable-vhdx) printf "%s" -Dvhdx=enabled ;; + --disable-vhdx) printf "%s" -Dvhdx=disabled ;; --enable-vhost-crypto) printf "%s" -Dvhost_crypto=enabled ;; --disable-vhost-crypto) printf "%s" -Dvhost_crypto=disabled ;; --enable-vhost-kernel) printf "%s" -Dvhost_kernel=enabled ;; @@ -461,6 +498,10 @@ _meson_option_parse() { --disable-virglrenderer) printf "%s" -Dvirglrenderer=disabled ;; --enable-virtfs) printf "%s" -Dvirtfs=enabled ;; --disable-virtfs) printf "%s" -Dvirtfs=disabled ;; + --enable-virtfs-proxy-helper) printf "%s" -Dvirtfs_proxy_helper=enabled ;; + --disable-virtfs-proxy-helper) printf "%s" -Dvirtfs_proxy_helper=disabled ;; + --enable-vmdk) printf "%s" -Dvmdk=enabled ;; + --disable-vmdk) printf "%s" -Dvmdk=disabled ;; --enable-vmnet) printf "%s" -Dvmnet=enabled ;; --disable-vmnet) printf "%s" -Dvmnet=disabled ;; --enable-vnc) printf "%s" -Dvnc=enabled ;; @@ -469,6 +510,8 @@ _meson_option_parse() { --disable-vnc-jpeg) printf "%s" -Dvnc_jpeg=disabled ;; --enable-vnc-sasl) printf "%s" -Dvnc_sasl=enabled ;; --disable-vnc-sasl) printf "%s" -Dvnc_sasl=disabled ;; + --enable-vpc) printf "%s" -Dvpc=enabled ;; + --disable-vpc) printf "%s" -Dvpc=disabled ;; --enable-vte) printf "%s" -Dvte=enabled ;; --disable-vte) printf "%s" -Dvte=disabled ;; --enable-vvfat) printf "%s" -Dvvfat=enabled ;; diff --git a/scripts/modinfo-collect.py b/scripts/modinfo-collect.py index 4e7584df66..4e7584df66 100755..100644 --- a/scripts/modinfo-collect.py +++ b/scripts/modinfo-collect.py diff --git a/scripts/modinfo-generate.py b/scripts/modinfo-generate.py index b1538fcced..b1538fcced 100755..100644 --- a/scripts/modinfo-generate.py +++ b/scripts/modinfo-generate.py diff --git a/scripts/probe-gdb-support.py b/scripts/probe-gdb-support.py index 5755255966..5755255966 100755..100644 --- a/scripts/probe-gdb-support.py +++ b/scripts/probe-gdb-support.py diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py index a079378d1b..d1fdf4182c 100644 --- a/scripts/qapi/commands.py +++ b/scripts/qapi/commands.py @@ -66,6 +66,7 @@ def gen_call(name: str, elif arg_type: assert not arg_type.variants for memb in arg_type.members: + assert not memb.ifcond.is_present() if memb.need_has(): argstr += 'arg.has_%s, ' % c_name(memb.name) argstr += 'arg.%s, ' % c_name(memb.name) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index ca01ea6f4a..cae0a08359 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -333,62 +333,56 @@ def normalize_members(members: object) -> None: members[key] = {'type': arg} -def check_type(value: Optional[object], - info: QAPISourceInfo, - source: str, - allow_array: bool = False, - allow_dict: Union[bool, str] = False) -> None: +def check_type_name(value: Optional[object], + info: QAPISourceInfo, source: str) -> None: + if value is not None and not isinstance(value, str): + raise QAPISemError(info, "%s should be a type name" % source) + + +def check_type_name_or_array(value: Optional[object], + info: QAPISourceInfo, source: str) -> None: + if value is None or isinstance(value, str): + return + + if not isinstance(value, list): + raise QAPISemError(info, + "%s should be a type name or array" % source) + + if len(value) != 1 or not isinstance(value[0], str): + raise QAPISemError(info, + "%s: array type must contain single type name" % + source) + + +def check_type_implicit(value: Optional[object], + info: QAPISourceInfo, source: str, + parent_name: Optional[str]) -> None: """ - Normalize and validate the QAPI type of ``value``. + Normalize and validate an optional implicit struct type. - Python types of ``str`` or ``None`` are always allowed. + Accept ``None`` or a ``dict`` defining an implicit struct type. + The latter is normalized in place. :param value: The value to check. :param info: QAPI schema source file information. :param source: Error string describing this ``value``. - :param allow_array: - Allow a ``List[str]`` of length 1, which indicates an array of - the type named by the list element. - :param allow_dict: - Allow a dict. Its members can be struct type members or union - branches. When the value of ``allow_dict`` is in pragma - ``member-name-exceptions``, the dict's keys may violate the - member naming rules. The dict members are normalized in place. + :param parent_name: + When the value of ``parent_name`` is in pragma + ``member-name-exceptions``, an implicit struct type may + violate the member naming rules. :raise QAPISemError: When ``value`` fails validation. - :return: None, ``value`` is normalized in-place as needed. + :return: None """ if value is None: return - # Type name - if isinstance(value, str): - return - - # Array type - if isinstance(value, list): - if not allow_array: - raise QAPISemError(info, "%s cannot be an array" % source) - if len(value) != 1 or not isinstance(value[0], str): - raise QAPISemError(info, - "%s: array type must contain single type name" % - source) - return - - # Anonymous type - - if not allow_dict: - raise QAPISemError(info, "%s should be a type name" % source) - if not isinstance(value, dict): raise QAPISemError(info, "%s should be an object or type name" % source) - permissive = False - if isinstance(allow_dict, str): - permissive = allow_dict in info.pragma.member_name_exceptions + permissive = parent_name in info.pragma.member_name_exceptions - # value is a dictionary, check that each member is okay for (key, arg) in value.items(): key_source = "%s member '%s'" % (source, key) if key.startswith('*'): @@ -401,7 +395,16 @@ def check_type(value: Optional[object], check_keys(arg, info, key_source, ['type'], ['if', 'features']) check_if(arg, info, key_source) check_features(arg.get('features'), info) - check_type(arg['type'], info, key_source, allow_array=True) + check_type_name_or_array(arg['type'], info, key_source) + + +def check_type_name_or_implicit(value: Optional[object], + info: QAPISourceInfo, source: str, + parent_name: Optional[str]) -> None: + if value is None or isinstance(value, str): + return + + check_type_implicit(value, info, source, parent_name) def check_features(features: Optional[object], @@ -489,8 +492,8 @@ def check_struct(expr: QAPIExpression) -> None: name = cast(str, expr['struct']) # Checked in check_exprs members = expr['data'] - check_type(members, expr.info, "'data'", allow_dict=name) - check_type(expr.get('base'), expr.info, "'base'") + check_type_implicit(members, expr.info, "'data'", name) + check_type_name(expr.get('base'), expr.info, "'base'") def check_union(expr: QAPIExpression) -> None: @@ -508,7 +511,7 @@ def check_union(expr: QAPIExpression) -> None: members = expr['data'] info = expr.info - check_type(base, info, "'base'", allow_dict=name) + check_type_name_or_implicit(base, info, "'base'", name) check_name_is_str(discriminator, info, "'discriminator'") if not isinstance(members, dict): @@ -518,7 +521,7 @@ def check_union(expr: QAPIExpression) -> None: source = "'data' member '%s'" % key check_keys(value, info, source, ['type'], ['if']) check_if(value, info, source) - check_type(value['type'], info, source, allow_array=not base) + check_type_name(value['type'], info, source) def check_alternate(expr: QAPIExpression) -> None: @@ -544,7 +547,7 @@ def check_alternate(expr: QAPIExpression) -> None: check_name_lower(key, info, source) check_keys(value, info, source, ['type'], ['if']) check_if(value, info, source) - check_type(value['type'], info, source, allow_array=True) + check_type_name_or_array(value['type'], info, source) def check_command(expr: QAPIExpression) -> None: @@ -560,10 +563,13 @@ def check_command(expr: QAPIExpression) -> None: rets = expr.get('returns') boxed = expr.get('boxed', False) - if boxed and args is None: - raise QAPISemError(expr.info, "'boxed': true requires 'data'") - check_type(args, expr.info, "'data'", allow_dict=not boxed) - check_type(rets, expr.info, "'returns'", allow_array=True) + if boxed: + if args is None: + raise QAPISemError(expr.info, "'boxed': true requires 'data'") + check_type_name(args, expr.info, "'data'") + else: + check_type_name_or_implicit(args, expr.info, "'data'", None) + check_type_name_or_array(rets, expr.info, "'returns'") def check_event(expr: QAPIExpression) -> None: @@ -578,9 +584,12 @@ def check_event(expr: QAPIExpression) -> None: args = expr.get('data') boxed = expr.get('boxed', False) - if boxed and args is None: - raise QAPISemError(expr.info, "'boxed': true requires 'data'") - check_type(args, expr.info, "'data'", allow_dict=not boxed) + if boxed: + if args is None: + raise QAPISemError(expr.info, "'boxed': true requires 'data'") + check_type_name(args, expr.info, "'data'") + else: + check_type_name_or_implicit(args, expr.info, "'data'", None) def check_exprs(exprs: List[QAPIExpression]) -> List[QAPIExpression]: diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py index b5a8d03e8e..70bc576a10 100644 --- a/scripts/qapi/gen.py +++ b/scripts/qapi/gen.py @@ -13,6 +13,7 @@ from contextlib import contextmanager import os +import sys import re from typing import ( Dict, @@ -119,6 +120,7 @@ def build_params(arg_type: Optional[QAPISchemaObjectType], elif arg_type: assert not arg_type.variants for memb in arg_type.members: + assert not memb.ifcond.is_present() ret += sep sep = ', ' if memb.need_has(): @@ -161,7 +163,7 @@ class QAPIGenC(QAPIGenCCode): def _top(self) -> str: return mcgen(''' -/* AUTOMATICALLY GENERATED, DO NOT MODIFY */ +/* AUTOMATICALLY GENERATED by %(tool)s DO NOT MODIFY */ /* %(blurb)s @@ -173,6 +175,7 @@ class QAPIGenC(QAPIGenCCode): */ ''', + tool=os.path.basename(sys.argv[0]), blurb=self._blurb, copyright=self._copyright) def _bottom(self) -> str: @@ -194,7 +197,10 @@ class QAPIGenH(QAPIGenC): class QAPIGenTrace(QAPIGen): def _top(self) -> str: - return super()._top() + '# AUTOMATICALLY GENERATED, DO NOT MODIFY\n\n' + return (super()._top() + + '# AUTOMATICALLY GENERATED by ' + + os.path.basename(sys.argv[0]) + + ', DO NOT MODIFY\n\n') @contextmanager diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py index fc216a53d3..316736b6a2 100644 --- a/scripts/qapi/main.py +++ b/scripts/qapi/main.py @@ -98,6 +98,6 @@ def main() -> int: builtins=args.builtins, gen_tracing=not args.suppress_tracing) except QAPIError as err: - print(f"{sys.argv[0]}: {str(err)}", file=sys.stderr) + print(err, file=sys.stderr) return 1 return 0 diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini index 6625356429..3463307ddc 100644 --- a/scripts/qapi/mypy.ini +++ b/scripts/qapi/mypy.ini @@ -1,7 +1,7 @@ [mypy] strict = True disallow_untyped_calls = False -python_version = 3.6 +python_version = 3.7 [mypy-qapi.schema] disallow_untyped_defs = False diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index 878f90b458..22e7bcc4b1 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -346,7 +346,7 @@ class QAPISchemaParser: elif not self.tok.isspace(): # Show up to next structural, whitespace or quote # character - match = must_match('[^[\\]{}:,\\s\'"]+', + match = must_match('[^[\\]{}:,\\s\']+', self.src[self.cursor-1:]) raise QAPIParseError(self, "stray '%s'" % match.group(0)) @@ -468,34 +468,39 @@ class QAPIDoc: class Section: # pylint: disable=too-few-public-methods def __init__(self, parser: QAPISchemaParser, - name: Optional[str] = None, indent: int = 0): - + name: Optional[str] = None): # parser, for error messages about indentation self._parser = parser # optional section name (argument/member or section name) self.name = name + # section text without section name self.text = '' - # the expected indent level of the text of this section - self._indent = indent + # indentation to strip (None means indeterminate) + self._indent = None if self.name else 0 def append(self, line: str) -> None: - # Strip leading spaces corresponding to the expected indent level - # Blank lines are always OK. + line = line.rstrip() + if line: indent = must_match(r'\s*', line).end() - if indent < self._indent: + if self._indent is None: + # indeterminate indentation + if self.text != '': + # non-blank, non-first line determines indentation + self._indent = indent + elif indent < self._indent: raise QAPIParseError( self._parser, "unexpected de-indent (expected at least %d spaces)" % self._indent) line = line[self._indent:] - self.text += line.rstrip() + '\n' + self.text += line + '\n' class ArgSection(Section): def __init__(self, parser: QAPISchemaParser, - name: str, indent: int = 0): - super().__init__(parser, name, indent) + name: str): + super().__init__(parser, name) self.member: Optional['QAPISchemaMember'] = None def connect(self, member: 'QAPISchemaMember') -> None: @@ -558,12 +563,12 @@ class QAPIDoc: self._switch_section(QAPIDoc.NullSection(self._parser)) @staticmethod - def _is_section_tag(name: str) -> bool: - return name in ('Returns:', 'Since:', - # those are often singular or plural - 'Note:', 'Notes:', - 'Example:', 'Examples:', - 'TODO:') + def _match_at_name_colon(string: str): + return re.match(r'@([^:]*): *', string) + + @staticmethod + def _match_section_tag(string: str): + return re.match(r'(Returns|Since|Notes?|Examples?|TODO): *', string) def _append_body_line(self, line: str) -> None: """ @@ -579,7 +584,6 @@ class QAPIDoc: Else, append the line to the current section. """ - name = line.split(' ', 1)[0] # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't # recognized, and get silently treated as ordinary text if not self.symbol and not self.body.text and line.startswith('@'): @@ -593,12 +597,12 @@ class QAPIDoc: self._parser, "name required after '@'") elif self.symbol: # This is a definition documentation block - if name.startswith('@') and name.endswith(':'): + if self._match_at_name_colon(line): self._append_line = self._append_args_line self._append_args_line(line) elif line == 'Features:': self._append_line = self._append_features_line - elif self._is_section_tag(name): + elif self._match_section_tag(line): self._append_line = self._append_various_line self._append_various_line(line) else: @@ -619,25 +623,11 @@ class QAPIDoc: Else, append the line to the current section. """ - name = line.split(' ', 1)[0] - - if name.startswith('@') and name.endswith(':'): - # If line is "@arg: first line of description", find - # the index of 'f', which is the indent we expect for any - # following lines. We then remove the leading "@arg:" - # from line and replace it with spaces so that 'f' has the - # same index as it did in the original line and can be - # handled the same way we will handle following lines. - indent = must_match(r'@\S*:\s*', line).end() - line = line[indent:] - if not line: - # Line was just the "@arg:" header; following lines - # are not indented - indent = 0 - else: - line = ' ' * indent + line - self._start_args_section(name[1:-1], indent) - elif self._is_section_tag(name): + match = self._match_at_name_colon(line) + if match: + line = line[match.end():] + self._start_args_section(match.group(1)) + elif self._match_section_tag(line): self._append_line = self._append_various_line self._append_various_line(line) return @@ -654,25 +644,11 @@ class QAPIDoc: self._append_freeform(line) def _append_features_line(self, line: str) -> None: - name = line.split(' ', 1)[0] - - if name.startswith('@') and name.endswith(':'): - # If line is "@arg: first line of description", find - # the index of 'f', which is the indent we expect for any - # following lines. We then remove the leading "@arg:" - # from line and replace it with spaces so that 'f' has the - # same index as it did in the original line and can be - # handled the same way we will handle following lines. - indent = must_match(r'@\S*:\s*', line).end() - line = line[indent:] - if not line: - # Line was just the "@arg:" header; following lines - # are not indented - indent = 0 - else: - line = ' ' * indent + line - self._start_features_section(name[1:-1], indent) - elif self._is_section_tag(name): + match = self._match_at_name_colon(line) + if match: + line = line[match.end():] + self._start_features_section(match.group(1)) + elif self._match_section_tag(line): self._append_line = self._append_various_line self._append_various_line(line) return @@ -696,36 +672,22 @@ class QAPIDoc: Else, append the line to the current section. """ - name = line.split(' ', 1)[0] - - if name.startswith('@') and name.endswith(':'): + match = self._match_at_name_colon(line) + if match: raise QAPIParseError(self._parser, - "'%s' can't follow '%s' section" - % (name, self.sections[0].name)) - if self._is_section_tag(name): - # If line is "Section: first line of description", find - # the index of 'f', which is the indent we expect for any - # following lines. We then remove the leading "Section:" - # from line and replace it with spaces so that 'f' has the - # same index as it did in the original line and can be - # handled the same way we will handle following lines. - indent = must_match(r'\S*:\s*', line).end() - line = line[indent:] - if not line: - # Line was just the "Section:" header; following lines - # are not indented - indent = 0 - else: - line = ' ' * indent + line - self._start_section(name[:-1], indent) + "description of '@%s:' follows a section" + % match.group(1)) + match = self._match_section_tag(line) + if match: + line = line[match.end():] + self._start_section(match.group(1)) self._append_freeform(line) def _start_symbol_section( self, symbols_dict: Dict[str, 'QAPIDoc.ArgSection'], - name: str, - indent: int) -> None: + name: str) -> None: # FIXME invalid names other than the empty string aren't flagged if not name: raise QAPIParseError(self._parser, "invalid parameter name") @@ -733,27 +695,26 @@ class QAPIDoc: raise QAPIParseError(self._parser, "'%s' parameter name duplicated" % name) assert not self.sections - new_section = QAPIDoc.ArgSection(self._parser, name, indent) + new_section = QAPIDoc.ArgSection(self._parser, name) self._switch_section(new_section) symbols_dict[name] = new_section - def _start_args_section(self, name: str, indent: int) -> None: - self._start_symbol_section(self.args, name, indent) + def _start_args_section(self, name: str) -> None: + self._start_symbol_section(self.args, name) - def _start_features_section(self, name: str, indent: int) -> None: - self._start_symbol_section(self.features, name, indent) + def _start_features_section(self, name: str) -> None: + self._start_symbol_section(self.features, name) - def _start_section(self, name: Optional[str] = None, - indent: int = 0) -> None: + def _start_section(self, name: Optional[str] = None) -> None: if name in ('Returns', 'Since') and self.has_section(name): raise QAPIParseError(self._parser, "duplicated '%s' section" % name) - new_section = QAPIDoc.Section(self._parser, name, indent) + new_section = QAPIDoc.Section(self._parser, name) self._switch_section(new_section) self.sections.append(new_section) def _switch_section(self, new_section: 'QAPIDoc.Section') -> None: - text = self._section.text = self._section.text.strip() + text = self._section.text = self._section.text.strip('\n') # Only the 'body' section is allowed to have an empty body. # All other sections, including anonymous ones, must have text. diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index 207e4d71f3..231ebf61ba 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -259,7 +259,7 @@ class QAPISchemaType(QAPISchemaEntity): return not self.c_type().endswith(POINTER_SUFFIX) def check(self, schema): - QAPISchemaEntity.check(self, schema) + super().check(schema) for feat in self.features: if feat.is_special(): raise QAPISemError( @@ -465,9 +465,10 @@ class QAPISchemaObjectType(QAPISchemaType): # on behalf of info, which is not necessarily self.info def check_clash(self, info, seen): assert self._checked - assert not self.variants # not implemented for m in self.members: m.check_clash(info, seen) + if self.variants: + self.variants.check_clash(info, seen) def connect_doc(self, doc=None): super().connect_doc(doc) @@ -486,6 +487,10 @@ class QAPISchemaObjectType(QAPISchemaType): assert self.members is not None return not self.members and not self.variants + def has_conditional_members(self): + assert self.members is not None + return any(m.ifcond.is_present() for m in self.members) + def c_name(self): assert self.name != 'q_empty' return super().c_name() @@ -652,8 +657,7 @@ class QAPISchemaVariants: self.info, "branch '%s' is not a value of %s" % (v.name, self.tag_member.type.describe())) - if (not isinstance(v.type, QAPISchemaObjectType) - or v.type.variants): + if not isinstance(v.type, QAPISchemaObjectType): raise QAPISemError( self.info, "%s cannot use %s" @@ -697,6 +701,7 @@ class QAPISchemaMember: def describe(self, info): role = self.role + meta = 'type' defined_in = self.defined_in assert defined_in @@ -708,13 +713,17 @@ class QAPISchemaMember: # Implicit type created for a command's dict 'data' assert role == 'member' role = 'parameter' + meta = 'command' + defined_in = defined_in[:-4] elif defined_in.endswith('-base'): # Implicit type created for a union's dict 'base' role = 'base ' + role + defined_in = defined_in[:-5] else: assert False - elif defined_in != info.defn_name: - return "%s '%s' of type '%s'" % (role, self.name, defined_in) + + if defined_in != info.defn_name: + return "%s '%s' of %s '%s'" % (role, self.name, meta, defined_in) return "%s '%s'" % (role, self.name) @@ -817,6 +826,11 @@ class QAPISchemaCommand(QAPISchemaEntity): self.info, "command's 'data' can take %s only with 'boxed': true" % self.arg_type.describe()) + self.arg_type.check(schema) + if self.arg_type.has_conditional_members() and not self.boxed: + raise QAPISemError( + self.info, + "conditional command arguments require 'boxed': true") if self._ret_type_name: self.ret_type = schema.resolve_type( self._ret_type_name, self.info, "command's 'returns'") @@ -872,6 +886,11 @@ class QAPISchemaEvent(QAPISchemaEntity): self.info, "event's 'data' can take %s only with 'boxed': true" % self.arg_type.describe()) + self.arg_type.check(schema) + if self.arg_type.has_conditional_members() and not self.boxed: + raise QAPISemError( + self.info, + "conditional event arguments require 'boxed': true") def connect_doc(self, doc=None): super().connect_doc(doc) diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py index 26a584ee4c..c56ea4d724 100644 --- a/scripts/qapi/visit.py +++ b/scripts/qapi/visit.py @@ -74,11 +74,13 @@ bool visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp) sep = '' for memb in members: if memb.optional and not memb.need_has(): + ret += memb.ifcond.gen_if() ret += mcgen(''' bool has_%(c_name)s = !!obj->%(c_name)s; ''', c_name=c_name(memb.name)) sep = '\n' + ret += memb.ifcond.gen_endif() ret += sep if base: diff --git a/scripts/qom-cast-macro-clean-cocci-gen.py b/scripts/qom-cast-macro-clean-cocci-gen.py new file mode 100644 index 0000000000..2fa8438a14 --- /dev/null +++ b/scripts/qom-cast-macro-clean-cocci-gen.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Generate a Coccinelle semantic patch to remove pointless QOM cast. +# +# Usage: +# +# $ qom-cast-macro-clean-cocci-gen.py $(git ls-files) > qom_pointless_cast.cocci +# $ spatch \ +# --macro-file scripts/cocci-macro-file.h \ +# --sp-file qom_pointless_cast.cocci \ +# --keep-comments \ +# --use-gitgrep \ +# --in-place \ +# --dir . +# +# SPDX-FileContributor: Philippe Mathieu-Daudé <philmd@linaro.org> +# SPDX-FileCopyrightText: 2023 Linaro Ltd. +# SPDX-License-Identifier: GPL-2.0-or-later + +import re +import sys + +assert len(sys.argv) > 0 + +def print_cocci_rule(qom_typedef, qom_cast_macro): + print(f'''@@ +typedef {qom_typedef}; +{qom_typedef} *obj; +@@ +- {qom_cast_macro}(obj) ++ obj +''') + +patterns = [ + r'DECLARE_INSTANCE_CHECKER\((\w+),\W*(\w+),\W*TYPE_\w+\)', + r'DECLARE_OBJ_CHECKERS\((\w+),\W*\w+,\W*(\w+),\W*TYPE_\w+\)', + r'OBJECT_DECLARE_TYPE\((\w+),\W*\w+,\W*(\w+)\)', + r'OBJECT_DECLARE_SIMPLE_TYPE\((\w+),\W*(\w+)\)', + r'INTERFACE_CHECK\((\w+),\W*\(\w+\),\W*TYPE_(\w+)\)', +] + +for fn in sys.argv[1:]: + try: + content = open(fn, 'rt').read() + except: + continue + for pattern in patterns: + for match in re.findall(pattern, content): + print_cocci_rule(match[0], match[1]) diff --git a/scripts/symlink-install-tree.py b/scripts/symlink-install-tree.py index 67cb86dd52..8ed97e3c94 100644 --- a/scripts/symlink-install-tree.py +++ b/scripts/symlink-install-tree.py @@ -28,5 +28,8 @@ for source, dest in json.loads(out).items(): os.symlink(source, bundle_dest) except BaseException as e: if not isinstance(e, OSError) or e.errno != errno.EEXIST: + if os.name == 'nt': + print('Please enable Developer Mode to support soft link ' + 'without Administrator permission') print(f'error making symbolic link {dest}', file=sys.stderr) raise e diff --git a/scripts/test-driver.py b/scripts/test-driver.py deleted file mode 100644 index eef74b29a8..0000000000 --- a/scripts/test-driver.py +++ /dev/null @@ -1,35 +0,0 @@ -#! /usr/bin/env python3 - -# Wrapper for tests that hides the output if they succeed. -# Used by "make check" -# -# Copyright (C) 2020 Red Hat, Inc. -# -# Author: Paolo Bonzini <pbonzini@redhat.com> - -import subprocess -import sys -import os -import argparse - -parser = argparse.ArgumentParser(description='Test driver for QEMU') -parser.add_argument('-C', metavar='DIR', dest='dir', default='.', - help='change to DIR before doing anything else') -parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', - help='be more verbose') -parser.add_argument('test_args', nargs=argparse.REMAINDER) - -args = parser.parse_args() -os.chdir(args.dir) - -test_args = args.test_args -if test_args[0] == '--': - test_args = test_args[1:] - -if args.verbose: - result = subprocess.run(test_args, stdout=None, stderr=None) -else: - result = subprocess.run(test_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if result.returncode: - sys.stdout.buffer.write(result.stdout) -sys.exit(result.returncode) diff --git a/scripts/tracetool/backend/ftrace.py b/scripts/tracetool/backend/ftrace.py index 5fa30ccc08..baed2ae61c 100644 --- a/scripts/tracetool/backend/ftrace.py +++ b/scripts/tracetool/backend/ftrace.py @@ -12,6 +12,8 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" +import os.path + from tracetool import out @@ -45,7 +47,7 @@ def generate_h(event, group): args=event.args, event_id="TRACE_" + event.name.upper(), event_lineno=event.lineno, - event_filename=event.filename, + event_filename=os.path.relpath(event.filename), fmt=event.fmt.rstrip("\n"), argnames=argnames) diff --git a/scripts/tracetool/backend/log.py b/scripts/tracetool/backend/log.py index 17ba1cd90e..de27b7e62e 100644 --- a/scripts/tracetool/backend/log.py +++ b/scripts/tracetool/backend/log.py @@ -12,6 +12,8 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" +import os.path + from tracetool import out @@ -53,7 +55,7 @@ def generate_h(event, group): ' }', cond=cond, event_lineno=event.lineno, - event_filename=event.filename, + event_filename=os.path.relpath(event.filename), name=event.name, fmt=event.fmt.rstrip("\n"), argnames=argnames) diff --git a/scripts/tracetool/backend/syslog.py b/scripts/tracetool/backend/syslog.py index 5a3a00fe31..012970f6cc 100644 --- a/scripts/tracetool/backend/syslog.py +++ b/scripts/tracetool/backend/syslog.py @@ -12,6 +12,8 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" +import os.path + from tracetool import out @@ -41,7 +43,7 @@ def generate_h(event, group): ' }', cond=cond, event_lineno=event.lineno, - event_filename=event.filename, + event_filename=os.path.relpath(event.filename), name=event.name, fmt=event.fmt.rstrip("\n"), argnames=argnames) diff --git a/scripts/tracetool/format/c.py b/scripts/tracetool/format/c.py index c390c1844a..69edf0d588 100644 --- a/scripts/tracetool/format/c.py +++ b/scripts/tracetool/format/c.py @@ -32,19 +32,13 @@ def generate(events, backend, group): out('uint16_t %s;' % e.api(e.QEMU_DSTATE)) for e in events: - if "vcpu" in e.properties: - vcpu_id = 0 - else: - vcpu_id = "TRACE_VCPU_EVENT_NONE" out('TraceEvent %(event)s = {', ' .id = 0,', - ' .vcpu_id = %(vcpu_id)s,', ' .name = \"%(name)s\",', ' .sstate = %(sstate)s,', ' .dstate = &%(dstate)s ', '};', event = e.api(e.QEMU_EVENT), - vcpu_id = vcpu_id, name = e.name, sstate = "TRACE_%s_ENABLED" % e.name.upper(), dstate = e.api(e.QEMU_DSTATE)) diff --git a/scripts/tracetool/format/h.py b/scripts/tracetool/format/h.py index e94f0be7da..ea126b07ea 100644 --- a/scripts/tracetool/format/h.py +++ b/scripts/tracetool/format/h.py @@ -16,10 +16,7 @@ from tracetool import out def generate(events, backend, group): - if group == "root": - header = "trace/control-vcpu.h" - else: - header = "trace/control.h" + header = "trace/control.h" out('/* This file is autogenerated by tracetool, do not edit. */', '', @@ -74,16 +71,7 @@ def generate(events, backend, group): out('}') - # tracer wrapper with checks (per-vCPU tracing) - if "vcpu" in e.properties: - trace_cpu = next(iter(e.args))[1] - cond = "trace_event_get_vcpu_state(%(cpu)s,"\ - " TRACE_%(id)s)"\ - % dict( - cpu=trace_cpu, - id=e.name.upper()) - else: - cond = "true" + cond = "true" out('', 'static inline void %(api)s(%(args)s)', diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh index b1ad99cba8..35a64bb501 100755 --- a/scripts/update-linux-headers.sh +++ b/scripts/update-linux-headers.sh @@ -160,8 +160,8 @@ done rm -rf "$output/linux-headers/linux" mkdir -p "$output/linux-headers/linux" -for header in kvm.h vfio.h vfio_ccw.h vfio_zdev.h vhost.h \ - psci.h psp-sev.h userfaultfd.h mman.h vduse.h; do +for header in const.h stddef.h kvm.h vfio.h vfio_ccw.h vfio_zdev.h vhost.h \ + psci.h psp-sev.h userfaultfd.h memfd.h mman.h nvme_ioctl.h vduse.h; do cp "$tmpdir/include/linux/$header" "$output/linux-headers/linux" done diff --git a/scripts/vmstate-static-checker.py b/scripts/vmstate-static-checker.py index dfeee8231a..9c0e6b81f2 100755 --- a/scripts/vmstate-static-checker.py +++ b/scripts/vmstate-static-checker.py @@ -134,6 +134,11 @@ def exists_in_substruct(fields, item): return check_fields_match(fields["Description"]["name"], substruct_fields[0]["field"], item) +def size_total(entry): + size = entry["size"] + if "num" not in entry: + return size + return size * entry["num"] def check_fields(src_fields, dest_fields, desc, sec): # This function checks for all the fields in a section. If some @@ -249,17 +254,19 @@ def check_fields(src_fields, dest_fields, desc, sec): continue if s_item["field"] == "unused" or d_item["field"] == "unused": - if s_item["size"] == d_item["size"]: + s_size = size_total(s_item) + d_size = size_total(d_item) + if s_size == d_size: continue if d_item["field"] == "unused": advance_dest = False - unused_count = d_item["size"] - s_item["size"] + unused_count = d_size - s_size; continue if s_item["field"] == "unused": advance_src = False - unused_count = s_item["size"] - d_item["size"] + unused_count = s_size - d_size continue print("Section \"" + sec + "\",", end=' ') diff --git a/scripts/xen-detect.c b/scripts/xen-detect.c index 85e8206490..db049e605c 100644 --- a/scripts/xen-detect.c +++ b/scripts/xen-detect.c @@ -138,66 +138,6 @@ return 0; } -#elif CONFIG_XEN_CTRL_INTERFACE_VERSION == 40600 - #include <xenctrl.h> - #include <xenstore.h> - #include <stdint.h> - #include <xen/hvm/hvm_info_table.h> - #if !defined(HVM_MAX_VCPUS) - # error HVM_MAX_VCPUS not defined - #endif - int main(void) { - xc_interface *xc; - xs_daemon_open(); - xc = xc_interface_open(0, 0, 0); - xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0); - xc_gnttab_open(NULL, 0); - xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0); - xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000); - xc_hvm_create_ioreq_server(xc, 0, HVM_IOREQSRV_BUFIOREQ_ATOMIC, NULL); - xc_reserved_device_memory_map(xc, 0, 0, 0, 0, NULL, 0); - return 0; - } - -#elif CONFIG_XEN_CTRL_INTERFACE_VERSION == 40500 - #include <xenctrl.h> - #include <xenstore.h> - #include <stdint.h> - #include <xen/hvm/hvm_info_table.h> - #if !defined(HVM_MAX_VCPUS) - # error HVM_MAX_VCPUS not defined - #endif - int main(void) { - xc_interface *xc; - xs_daemon_open(); - xc = xc_interface_open(0, 0, 0); - xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0); - xc_gnttab_open(NULL, 0); - xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0); - xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000); - xc_hvm_create_ioreq_server(xc, 0, 0, NULL); - return 0; - } - -#elif CONFIG_XEN_CTRL_INTERFACE_VERSION == 40200 - #include <xenctrl.h> - #include <xenstore.h> - #include <stdint.h> - #include <xen/hvm/hvm_info_table.h> - #if !defined(HVM_MAX_VCPUS) - # error HVM_MAX_VCPUS not defined - #endif - int main(void) { - xc_interface *xc; - xs_daemon_open(); - xc = xc_interface_open(0, 0, 0); - xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0); - xc_gnttab_open(NULL, 0); - xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0); - xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000); - return 0; - } - #else #error invalid CONFIG_XEN_CTRL_INTERFACE_VERSION #endif |