From 9c6692db550f739a84fe4b677428df09d9fafdc3 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 10 May 2023 23:54:27 -0400 Subject: tests: Use configure-provided pyvenv for tests This patch changes how the avocado tests are provided, ever so slightly. Instead of creating a new testing venv, use the configure-provided 'pyvenv' instead, and install optional packages into that. Signed-off-by: John Snow Message-Id: <20230511035435.734312-20-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- docs/devel/acpi-bits.rst | 6 +++--- docs/devel/testing.rst | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'docs/devel') diff --git a/docs/devel/acpi-bits.rst b/docs/devel/acpi-bits.rst index 22e2580200..9677b0098f 100644 --- a/docs/devel/acpi-bits.rst +++ b/docs/devel/acpi-bits.rst @@ -61,19 +61,19 @@ Under ``tests/avocado/`` as the root we have: :: $ make check-venv (needed only the first time to create the venv) - $ ./tests/venv/bin/avocado run -t acpi tests/avocado + $ ./pyvenv/bin/avocado run -t acpi tests/avocado The above will run all acpi avocado tests including this one. In order to run the individual tests, perform the following: :: - $ ./tests/venv/bin/avocado run tests/avocado/acpi-bits.py --tap - + $ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py --tap - The above will produce output in tap format. You can omit "--tap -" in the end and it will produce output like the following: :: - $ ./tests/venv/bin/avocado run tests/avocado/acpi-bits.py + $ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py Fetching asset from tests/avocado/acpi-bits.py:AcpiBitsTest.test_acpi_smbios_bits JOB ID : eab225724da7b64c012c65705dc2fa14ab1defef JOB LOG : /home/anisinha/avocado/job-results/job-2022-10-10T17.58-eab2257/job.log diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst index 203facb417..2cafec4178 100644 --- a/docs/devel/testing.rst +++ b/docs/devel/testing.rst @@ -888,9 +888,9 @@ You can run the avocado tests simply by executing: make check-avocado -This involves the automatic creation of Python virtual environment -within the build tree (at ``tests/venv``) which will have all the -right dependencies, and will save tests results also within the +This involves the automatic installation, from PyPI, of all the +necessary avocado-framework dependencies into the QEMU venv within the +build tree (at ``./pyvenv``). Test results are also saved within the build tree (at ``tests/results``). Note: the build environment must be using a Python 3 stack, and have @@ -947,7 +947,7 @@ may be invoked by running: .. code:: - tests/venv/bin/avocado run $OPTION1 $OPTION2 tests/avocado/ + pyvenv/bin/avocado run $OPTION1 $OPTION2 tests/avocado/ Note that if ``make check-avocado`` was not executed before, it is possible to create the Python virtual environment with the dependencies @@ -962,20 +962,20 @@ a test file. To run tests from a single file within the build tree, use: .. code:: - tests/venv/bin/avocado run tests/avocado/$TESTFILE + pyvenv/bin/avocado run tests/avocado/$TESTFILE To run a single test within a test file, use: .. code:: - tests/venv/bin/avocado run tests/avocado/$TESTFILE:$TESTCLASS.$TESTNAME + pyvenv/bin/avocado run tests/avocado/$TESTFILE:$TESTCLASS.$TESTNAME Valid test names are visible in the output from any previous execution of Avocado or ``make check-avocado``, and can also be queried using: .. code:: - tests/venv/bin/avocado list tests/avocado + pyvenv/bin/avocado list tests/avocado Manual Installation ~~~~~~~~~~~~~~~~~~~ -- cgit 1.4.1 From a0cbd2e8496aa8a80241fca3b247a54709c5ed26 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 14 Jul 2022 14:33:49 +0200 Subject: meson: use prefer_static option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The option is new in Meson 0.63 and removes the need to pass "static: true" to all dependency and find_library invocation. Actually cleaning up the invocations is left for a separate patch. Reviewed-by: Marc-André Lureau Signed-off-by: Paolo Bonzini --- configure | 4 +--- docs/devel/build-system.rst | 3 +-- meson.build | 13 +++++-------- qga/meson.build | 2 +- 4 files changed, 8 insertions(+), 14 deletions(-) (limited to 'docs/devel') diff --git a/configure b/configure index 590c7717bc..8bca8acb2a 100755 --- a/configure +++ b/configure @@ -2409,9 +2409,6 @@ fi if test "$solaris" = "yes" ; then echo "CONFIG_SOLARIS=y" >> $config_host_mak fi -if test "$static" = "yes" ; then - echo "CONFIG_STATIC=y" >> $config_host_mak -fi echo "SRC_PATH=$source_path" >> $config_host_mak echo "TARGET_DIRS=$target_list" >> $config_host_mak if test "$modules" = "yes"; then @@ -2654,6 +2651,7 @@ if test "$skip_meson" = no; then # Built-in options test "$bindir" != "bin" && meson_option_add "-Dbindir=$bindir" test "$default_feature" = no && meson_option_add -Dauto_features=disabled + test "$static" = yes && meson_option_add -Dprefer_static=true test "$pie" = no && meson_option_add -Db_pie=false test "$werror" = yes && meson_option_add -Dwerror=true diff --git a/docs/devel/build-system.rst b/docs/devel/build-system.rst index 1894721743..9db18aff15 100644 --- a/docs/devel/build-system.rst +++ b/docs/devel/build-system.rst @@ -311,8 +311,7 @@ dependency will be used:: sdl_image = not_found if not get_option('sdl_image').auto() or have_system sdl_image = dependency('SDL2_image', required: get_option('sdl_image'), - method: 'pkg-config', - static: enable_static) + method: 'pkg-config') endif This avoids warnings on static builds of user-mode emulators, for example. diff --git a/meson.build b/meson.build index d85637edf1..7634f8b990 100644 --- a/meson.build +++ b/meson.build @@ -18,10 +18,7 @@ targetos = host_machine.system() sh = find_program('sh') config_host = keyval.load(meson.current_build_dir() / 'config-host.mak') enable_modules = 'CONFIG_MODULES' in config_host -enable_static = 'CONFIG_STATIC' in config_host - -# Allow both shared and static libraries unless --enable-static -static_kwargs = enable_static ? {'static': true} : {} +static_kwargs = {} cc = meson.get_compiler('c') all_languages = ['c'] @@ -193,7 +190,7 @@ qemu_cflags = config_host['QEMU_CFLAGS'].split() qemu_objcflags = config_host['QEMU_OBJCFLAGS'].split() qemu_ldflags = config_host['QEMU_LDFLAGS'].split() -if enable_static +if get_option('prefer_static') qemu_ldflags += get_option('b_pie') ? '-static-pie' : '-static' endif @@ -841,7 +838,7 @@ if targetos == 'linux' and have_tools and get_option('mpath').allowed() kwargs: static_kwargs) if libmpathpersist.found() mpathlibs += libmpathpersist - if enable_static + if get_option('prefer_static') mpathlibs += cc.find_library('devmapper', required: get_option('mpath'), kwargs: static_kwargs) @@ -1225,7 +1222,7 @@ if not gnutls_crypto.found() # Debian has removed -lgpg-error from libgcrypt-config # as it "spreads unnecessary dependencies" which in # turn breaks static builds... - if gcrypt.found() and enable_static + if gcrypt.found() and get_option('prefer_static') gcrypt = declare_dependency(dependencies: [ gcrypt, cc.find_library('gpg-error', required: true, kwargs: static_kwargs)]) @@ -1657,7 +1654,7 @@ endif # libdw libdw = not_found if not get_option('libdw').auto() or \ - (not enable_static and (have_system or have_user)) + (not get_option('prefer_static') and (have_system or have_user)) libdw = dependency('libdw', method: 'pkg-config', kwargs: static_kwargs, diff --git a/qga/meson.build b/qga/meson.build index ad17dc7dca..622b5f94a2 100644 --- a/qga/meson.build +++ b/qga/meson.build @@ -22,7 +22,7 @@ have_qga_vss = get_option('qga_vss') \ Then run configure with: --extra-cxxflags="-isystem /path/to/vss/inc/win2003"''') \ .require(midl.found() or widl.found(), error_message: 'VSS support requires midl or widl') \ - .require(not enable_static, + .require(not get_option('prefer_static'), error_message: 'VSS support requires dynamic linking with GLib') \ .allowed() -- cgit 1.4.1 From 877c5567050eff2c63267d76f0c0c3c38cebe048 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 12 Oct 2022 13:03:18 +0200 Subject: configure: remove pkg-config functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All uses of pkg-config have been moved to Meson. Reviewed-by: Marc-André Lureau Signed-off-by: Paolo Bonzini --- configure | 21 ++++----------------- docs/devel/build-system.rst | 4 ---- 2 files changed, 4 insertions(+), 21 deletions(-) (limited to 'docs/devel') diff --git a/configure b/configure index b7cae08c43..115cc325db 100755 --- a/configure +++ b/configure @@ -389,11 +389,7 @@ strip="${STRIP-${cross_prefix}strip}" widl="${WIDL-${cross_prefix}widl}" windres="${WINDRES-${cross_prefix}windres}" windmc="${WINDMC-${cross_prefix}windmc}" -pkg_config_exe="${PKG_CONFIG-${cross_prefix}pkg-config}" -query_pkg_config() { - "${pkg_config_exe}" ${QEMU_PKG_CONFIG_FLAGS} "$@" -} -pkg_config=query_pkg_config +pkg_config="${PKG_CONFIG-${cross_prefix}pkg-config}" sdl2_config="${SDL2_CONFIG-${cross_prefix}sdl2-config}" # default flags for all hosts @@ -791,9 +787,7 @@ for opt do ;; --without-default-features) # processed above ;; - --static) - static="yes" - QEMU_PKG_CONFIG_FLAGS="--static $QEMU_PKG_CONFIG_FLAGS" + --static) static="yes" ;; --bindir=*) bindir="$optarg" ;; @@ -1501,13 +1495,6 @@ EOF fi fi -########################################## -# pkg-config probe - -if ! has "$pkg_config_exe"; then - error_exit "pkg-config binary '$pkg_config_exe' not found" -fi - ########################################## # fdt probe @@ -2364,7 +2351,7 @@ echo "PYTHON=$python" >> $config_host_mak echo "GENISOIMAGE=$genisoimage" >> $config_host_mak echo "MESON=$meson" >> $config_host_mak echo "NINJA=$ninja" >> $config_host_mak -echo "PKG_CONFIG=${pkg_config_exe}" >> $config_host_mak +echo "PKG_CONFIG=${pkg_config}" >> $config_host_mak echo "CC=$cc" >> $config_host_mak echo "QEMU_CFLAGS=$QEMU_CFLAGS" >> $config_host_mak echo "QEMU_OBJCFLAGS=$QEMU_OBJCFLAGS" >> $config_host_mak @@ -2514,7 +2501,7 @@ if test "$skip_meson" = no; then test -n "$objcc" && echo "objc = [$(meson_quote $objcc $CPU_CFLAGS)]" >> $cross echo "ar = [$(meson_quote $ar)]" >> $cross echo "nm = [$(meson_quote $nm)]" >> $cross - echo "pkgconfig = [$(meson_quote $pkg_config_exe)]" >> $cross + echo "pkgconfig = [$(meson_quote $pkg_config)]" >> $cross echo "ranlib = [$(meson_quote $ranlib)]" >> $cross if has $sdl2_config; then echo "sdl2-config = [$(meson_quote $sdl2_config)]" >> $cross diff --git a/docs/devel/build-system.rst b/docs/devel/build-system.rst index 9db18aff15..66cfe7b8bd 100644 --- a/docs/devel/build-system.rst +++ b/docs/devel/build-system.rst @@ -103,10 +103,6 @@ developers in checking for system features: Print $MESSAGE to stderr, followed by $MORE... and then exit from the configure script with non-zero status -``query_pkg_config $ARGS...`` - Run pkg-config passing it $ARGS. If QEMU is doing a static build, - then --static will be automatically added to $ARGS - Stage 2: Meson ============== -- cgit 1.4.1 From 34f983d86fe40ffe5975369c1cf5e6a61688383a Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 9 Jan 2023 15:31:51 +0100 Subject: build: move sanitizer tests to meson Signed-off-by: Paolo Bonzini --- configure | 147 ----------------------------------------- docs/devel/build-system.rst | 4 -- meson.build | 56 +++++++++++++++- meson_options.txt | 4 ++ scripts/meson-buildoptions.sh | 6 ++ tests/qemu-iotests/meson.build | 2 +- tests/unit/meson.build | 2 +- 7 files changed, 66 insertions(+), 155 deletions(-) (limited to 'docs/devel') diff --git a/configure b/configure index 1125670039..c3ea6ef112 100755 --- a/configure +++ b/configure @@ -275,9 +275,6 @@ EXTRA_OBJCFLAGS="" EXTRA_LDFLAGS="" debug_tcg="no" -sanitizers="no" -tsan="no" -fortify_source="yes" docs="auto" EXESUF="" prefix="/usr/local" @@ -416,14 +413,6 @@ EOF compile_object } -check_include() { -cat > $TMPC < -int main(void) { return 0; } -EOF - compile_object -} - write_c_skeleton() { cat > $TMPC < /dev/null 2>&1 ; then - fortify_source="no"; - elif test -n "$cxx" && has $cxx && - echo | $cxx -dM -E - | grep __clang__ >/dev/null 2>&1 ; then - fortify_source="no"; - else - fortify_source="yes" - fi -fi - -########################################## -# checks for sanitizers - -have_asan=no -have_ubsan=no -have_asan_iface_h=no -have_asan_iface_fiber=no - -if test "$sanitizers" = "yes" ; then - write_c_skeleton - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=address" ""; then - have_asan=yes - fi - - # we could use a simple skeleton for flags checks, but this also - # detect the static linking issue of ubsan, see also: - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84285 - cat > $TMPC << EOF -int main(int argc, char **argv) -{ - return argc + 1; -} -EOF - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=undefined" ""; then - have_ubsan=yes - fi - - if check_include "sanitizer/asan_interface.h" ; then - have_asan_iface_h=yes - fi - - cat > $TMPC << EOF -#include -int main(void) { - __sanitizer_start_switch_fiber(0, 0, 0); - return 0; -} -EOF - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=address" "" ; then - have_asan_iface_fiber=yes - fi -fi - -# Thread sanitizer is, for now, much noisier than the other sanitizers; -# keep it separate until that is not the case. -if test "$tsan" = "yes" && test "$sanitizers" = "yes"; then - error_exit "TSAN is not supported with other sanitiziers." -fi -have_tsan=no -have_tsan_iface_fiber=no -if test "$tsan" = "yes" ; then - write_c_skeleton - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then - have_tsan=yes - fi - cat > $TMPC << EOF -#include -int main(void) { - __tsan_create_fiber(0); - return 0; -} -EOF - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then - have_tsan_iface_fiber=yes - fi -fi - ########################################## # functions to probe cross compilers @@ -2139,42 +2036,6 @@ case "$vfio_user_server" in ;; esac -########################################## -# End of CC checks -# After here, no more $cc or $ld runs - -write_c_skeleton - -if test "$fortify_source" = "yes" ; then - QEMU_CFLAGS="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $QEMU_CFLAGS" -fi - -if test "$have_asan" = "yes"; then - QEMU_CFLAGS="-fsanitize=address $QEMU_CFLAGS" - QEMU_LDFLAGS="-fsanitize=address $QEMU_LDFLAGS" - if test "$have_asan_iface_h" = "no" ; then - echo "ASAN build enabled, but ASAN header missing." \ - "Without code annotation, the report may be inferior." - elif test "$have_asan_iface_fiber" = "no" ; then - echo "ASAN build enabled, but ASAN header is too old." \ - "Without code annotation, the report may be inferior." - fi -fi -if test "$have_tsan" = "yes" ; then - if test "$have_tsan_iface_fiber" = "yes" ; then - QEMU_CFLAGS="-fsanitize=thread $QEMU_CFLAGS" - QEMU_LDFLAGS="-fsanitize=thread $QEMU_LDFLAGS" - else - error_exit "Cannot enable TSAN due to missing fiber annotation interface." - fi -elif test "$tsan" = "yes" ; then - error_exit "Cannot enable TSAN due to missing sanitize thread interface." -fi -if test "$have_ubsan" = "yes"; then - QEMU_CFLAGS="-fsanitize=undefined $QEMU_CFLAGS" - QEMU_LDFLAGS="-fsanitize=undefined $QEMU_LDFLAGS" -fi - ####################################### # cross-compiled firmware targets @@ -2299,14 +2160,6 @@ fi echo "CONFIG_COROUTINE_BACKEND=$coroutine" >> $config_host_mak -if test "$have_asan_iface_fiber" = "yes" ; then - echo "CONFIG_ASAN_IFACE_FIBER=y" >> $config_host_mak -fi - -if test "$have_tsan" = "yes" && test "$have_tsan_iface_fiber" = "yes" ; then - echo "CONFIG_TSAN=y" >> $config_host_mak -fi - if test "$plugins" = "yes" ; then echo "CONFIG_PLUGIN=y" >> $config_host_mak fi diff --git a/docs/devel/build-system.rst b/docs/devel/build-system.rst index 66cfe7b8bd..4a733fc0a7 100644 --- a/docs/devel/build-system.rst +++ b/docs/devel/build-system.rst @@ -91,10 +91,6 @@ developers in checking for system features: ``check_define $NAME`` Determine if the macro $NAME is defined by the system C compiler -``check_include $NAME`` - Determine if the include $NAME file is available to the system C - compiler. The replacement in Meson is ``cc.has_header()``. - ``write_c_skeleton`` Write a minimal C program main() function to the temporary file indicated by $TMPC diff --git a/meson.build b/meson.build index 2596f336a6..b50b0161ba 100644 --- a/meson.build +++ b/meson.build @@ -211,6 +211,35 @@ if get_option('prefer_static') qemu_ldflags += get_option('b_pie') ? '-static-pie' : '-static' endif +if get_option('sanitizers') + if cc.has_argument('-fsanitize=address') + qemu_cflags = ['-fsanitize=address'] + qemu_cflags + qemu_ldflags = ['-fsanitize=address'] + qemu_ldflags + endif + + # Detect static linking issue with ubsan - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84285 + if cc.links('int main(int argc, char **argv) { return argc + 1; }', + args: [qemu_ldflags, '-fsanitize=undefined']) + qemu_cflags = ['-fsanitize=undefined'] + qemu_cflags + qemu_ldflags = ['-fsanitize=undefined'] + qemu_ldflags + endif +endif + +# Thread sanitizer is, for now, much noisier than the other sanitizers; +# keep it separate until that is not the case. +if get_option('tsan') + if get_option('sanitizers') + error('TSAN is not supported with other sanitizers') + endif + if not cc.has_function('__tsan_create_fiber', + args: '-fsanitize=thread', + prefix: '#include ') + error('Cannot enable TSAN due to missing fiber annotation interface') + endif + qemu_cflags = ['-fsanitize=thread'] + qemu_cflags + qemu_ldflags = ['-fsanitize=thread'] + qemu_ldflags +endif + # Detect support for PT_GNU_RELRO + DT_BIND_NOW. # The combination is known as "full relro", because .got.plt is read-only too. qemu_ldflags += cc.get_supported_link_arguments('-Wl,-z,relro', '-Wl,-z,now') @@ -221,7 +250,7 @@ if targetos == 'windows' endif # Exclude --warn-common with TSan to suppress warnings from the TSan libraries. -if targetos != 'sunos' and not config_host.has_key('CONFIG_TSAN') +if targetos != 'sunos' and not get_option('tsan') qemu_ldflags += cc.get_supported_link_arguments('-Wl,--warn-common') endif @@ -284,6 +313,16 @@ if 'cpp' in all_languages endif endif +# clang does not support glibc + FORTIFY_SOURCE (is it still true?) +if get_option('optimization') != '0' and targetos == 'linux' + if cc.get_id() == 'gcc' + qemu_cflags += ['-U_FORTIFY_SOURCE', '-D_FORTIFY_SOURCE=2'] + endif + if 'cpp' in all_languages and cxx.get_id() == 'gcc' + qemu_cxxflags += ['-U_FORTIFY_SOURCE', '-D_FORTIFY_SOURCE=2'] + endif +endif + add_project_arguments(qemu_cflags, native: false, language: 'c') add_project_arguments(qemu_cxxflags, native: false, language: 'cpp') add_project_arguments(qemu_objcflags, native: false, language: 'objc') @@ -1920,6 +1959,7 @@ if seccomp.found() endif config_host_data.set('CONFIG_SNAPPY', snappy.found()) config_host_data.set('CONFIG_TPM', have_tpm) +config_host_data.set('CONFIG_TSAN', get_option('tsan')) config_host_data.set('CONFIG_USB_LIBUSB', libusb.found()) config_host_data.set('CONFIG_VDE', vde.found()) config_host_data.set('CONFIG_VHOST_NET', have_vhost_net) @@ -2051,6 +2091,18 @@ if rdma.found() prefix: '#include ')) endif +have_asan_fiber = false +if get_option('sanitizers') and \ + not cc.has_function('__sanitizer_start_switch_fiber', + args: '-fsanitize=address', + prefix: '#include ') + warning('Missing ASAN due to missing fiber annotation interface') + warning('Without code annotation, the report may be inferior.') +else + have_asan_fiber = true +endif +config_host_data.set('CONFIG_ASAN_IFACE_FIBER', have_asan_fiber) + # has_header_symbol config_host_data.set('CONFIG_BLKZONED', cc.has_header_symbol('linux/blkzoned.h', 'BLKOPENZONE')) @@ -3898,7 +3950,7 @@ else endif summary_info += {'gprof': gprof_info} summary_info += {'gcov': get_option('b_coverage')} -summary_info += {'thread sanitizer': config_host.has_key('CONFIG_TSAN')} +summary_info += {'thread sanitizer': get_option('tsan')} summary_info += {'CFI support': get_option('cfi')} if get_option('cfi') summary_info += {'CFI debug support': get_option('cfi_debug')} diff --git a/meson_options.txt b/meson_options.txt index 2e78cc251c..7f25cd3413 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -82,6 +82,10 @@ option('tcg', type: 'feature', value: 'enabled', description: 'TCG support') option('tcg_interpreter', type: 'boolean', value: false, description: 'TCG with bytecode interpreter (slow)') +option('sanitizers', type: 'boolean', value: false, + description: 'enable default sanitizers') +option('tsan', type: 'boolean', value: false, + description: 'enable thread sanitizer') option('cfi', type: 'boolean', value: false, description: 'Control-Flow Integrity (CFI)') option('cfi_debug', type: 'boolean', value: false, diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 0d06b1a0f1..6c992f1ab4 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -41,11 +41,13 @@ meson_options_help() { printf "%s\n" ' --enable-profiler profiler support' printf "%s\n" ' --enable-rng-none dummy RNG, avoid using /dev/(u)random and' printf "%s\n" ' getrandom()' + printf "%s\n" ' --enable-sanitizers enable default sanitizers' printf "%s\n" ' --enable-strip Strip targets on install' printf "%s\n" ' --enable-tcg-interpreter TCG with bytecode interpreter (slow)' printf "%s\n" ' --enable-trace-backends=CHOICES' printf "%s\n" ' Set available tracing backends [log] (choices:' printf "%s\n" ' dtrace/ftrace/log/nop/simple/syslog/ust)' + printf "%s\n" ' --enable-tsan enable thread sanitizer' printf "%s\n" ' --firmwarepath=VALUES search PATH for firmware files [share/qemu-' printf "%s\n" ' firmware]' printf "%s\n" ' --iasl=VALUE Path to ACPI disassembler' @@ -408,6 +410,8 @@ _meson_option_parse() { --disable-replication) printf "%s" -Dreplication=disabled ;; --enable-rng-none) printf "%s" -Drng_none=true ;; --disable-rng-none) printf "%s" -Drng_none=false ;; + --enable-sanitizers) printf "%s" -Dsanitizers=true ;; + --disable-sanitizers) printf "%s" -Dsanitizers=false ;; --enable-sdl) printf "%s" -Dsdl=enabled ;; --disable-sdl) printf "%s" -Dsdl=disabled ;; --enable-sdl-image) printf "%s" -Dsdl_image=enabled ;; @@ -446,6 +450,8 @@ _meson_option_parse() { --disable-tpm) printf "%s" -Dtpm=disabled ;; --enable-trace-backends=*) quote_sh "-Dtrace_backends=$2" ;; --with-trace-file=*) quote_sh "-Dtrace_file=$2" ;; + --enable-tsan) printf "%s" -Dtsan=true ;; + --disable-tsan) printf "%s" -Dtsan=false ;; --enable-u2f) printf "%s" -Du2f=enabled ;; --disable-u2f) printf "%s" -Du2f=disabled ;; --enable-usb-redir) printf "%s" -Dusb_redir=enabled ;; diff --git a/tests/qemu-iotests/meson.build b/tests/qemu-iotests/meson.build index 9735071a29..44761e1e4d 100644 --- a/tests/qemu-iotests/meson.build +++ b/tests/qemu-iotests/meson.build @@ -2,7 +2,7 @@ if not have_tools or targetos == 'windows' or get_option('gprof') subdir_done() endif -foreach cflag: config_host['QEMU_CFLAGS'].split() +foreach cflag: qemu_ldflags if cflag.startswith('-fsanitize') and \ not cflag.contains('safe-stack') and not cflag.contains('cfi-icall') message('Sanitizers are enabled ==> Disabled the qemu-iotests.') diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 3bc78d8660..48ae66011b 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -147,7 +147,7 @@ if have_system # Some tests: test-char, test-qdev-global-props, and test-qga, # are not runnable under TSan due to a known issue. # https://github.com/google/sanitizers/issues/1116 - if 'CONFIG_TSAN' not in config_host + if not get_option('tsan') if 'CONFIG_POSIX' in config_host tests += { 'test-char': ['socket-helpers.c', qom, io, chardev] -- cgit 1.4.1 From fe3ab4eb2de46076cbafcbc86b22e71ad24894c6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 18 Oct 2022 09:56:43 +0200 Subject: docs/devel: update build system docs configure is only doing compiler and host setup now, so adjust the relevant documentation. It is also possible to build emulators with ninja directly if one is so inclined, so mention that as well. The Python virtual environment set up is a new major task of configure as well. Mention it in the list of produced files, while leaving it for a future patch to document how it works and how ``mkvenv ensure`` is used. Signed-off-by: Paolo Bonzini --- docs/devel/build-system.rst | 283 +++++++++++++++++++++++++++----------------- 1 file changed, 175 insertions(+), 108 deletions(-) (limited to 'docs/devel') diff --git a/docs/devel/build-system.rst b/docs/devel/build-system.rst index 4a733fc0a7..551c5a5ac0 100644 --- a/docs/devel/build-system.rst +++ b/docs/devel/build-system.rst @@ -4,30 +4,14 @@ The QEMU build system architecture This document aims to help developers understand the architecture of the QEMU build system. As with projects using GNU autotools, the QEMU build -system has two stages, first the developer runs the "configure" script +system has two stages; first the developer runs the "configure" script to determine the local build environment characteristics, then they run -"make" to build the project. There is about where the similarities with +"make" to build the project. This is about where the similarities with GNU autotools end, so try to forget what you know about them. +The two general ways to perform a build are as follows: -Stage 1: configure -================== - -The QEMU configure script is written directly in shell, and should be -compatible with any POSIX shell, hence it uses #!/bin/sh. An important -implication of this is that it is important to avoid using bash-isms on -development platforms where bash is the primary host. - -In contrast to autoconf scripts, QEMU's configure is expected to be -silent while it is checking for features. It will only display output -when an error occurs, or to show the final feature enablement summary -on completion. - -Because QEMU uses the Meson build system under the hood, only VPATH -builds are supported. There are two general ways to invoke configure & -perform a build: - - - VPATH, build artifacts outside of QEMU source tree entirely:: + - build artifacts outside of QEMU source tree entirely:: cd ../ mkdir build @@ -35,80 +19,122 @@ perform a build: ../qemu/configure make - - VPATH, build artifacts in a subdir of QEMU source tree:: + - build artifacts in a subdir of QEMU source tree:: mkdir build cd build ../configure make -The configure script automatically recognizes -command line options for which a same-named Meson option exists; -dashes in the command line are replaced with underscores. +Most of the actual build process uses Meson under the hood, therefore +build artifacts cannot be placed in the source tree itself. + + +Stage 1: configure +================== + +The configure script has five tasks: + + - detect the host architecture + + - list the targets for which to build emulators; the list of + targets also affects which firmware binaries and tests to build -Many checks on the compilation environment are still found in configure -rather than ``meson.build``, but new checks should be added directly to -``meson.build``. + - find the compilers (native and cross) used to build executables, + firmware and tests. The results are written as either Makefile + fragments (``config-host.mak``) or a Meson machine file + (``config-meson.cross``) -Patches are also welcome to move existing checks from the configure -phase to ``meson.build``. When doing so, ensure that ``meson.build`` does -not use anymore the keys that you have removed from ``config-host.mak``. -Typically these will be replaced in ``meson.build`` by boolean variables, -``get_option('optname')`` invocations, or ``dep.found()`` expressions. -In general, the remaining checks have little or no interdependencies, -so they can be moved one by one. + - create a virtual environment in which all Python code runs during + the build, and possibly install packages into it from PyPI + + - invoke Meson in the virtual environment, to perform the actual + configuration step for the emulator build + +The configure script automatically recognizes command line options for +which a same-named Meson option exists; dashes in the command line are +replaced with underscores. + +Almost all QEMU developers that need to modify the build system will +only be concerned with Meson, and therefore can skip the rest of this +section. + + +Modifying ``configure`` +----------------------- + +``configure`` is a shell script; it uses ``#!/bin/sh`` and therefore +should be compatible with any POSIX shell. It is important to avoid +using bash-isms to avoid breaking development platforms where bash is +the primary host. + +The configure script provides a variety of functions to help writing +portable shell code and providing consistent behavior across architectures +and operating systems: + +``error_exit $MESSAGE $MORE...`` + Print $MESSAGE to stderr, followed by $MORE... and then exit from the + configure script with non-zero status. + +``has $COMMAND`` + Determine if $COMMAND exists in the current environment, either as a + shell builtin, or executable binary, returning 0 on success. The + replacement in Meson is ``find_program()``. -Helper functions ----------------- +``probe_target_compiler $TARGET`` + Detect a cross compiler and cross tools for the QEMU target $TARGET (e.g., + ``$CPU-softmmu``, ``$CPU-linux-user``, ``$CPU-bsd-user``). If a working + compiler is present, return success and set variables ``$target_cc``, + ``$target_ar``, etc. to non-empty values. -The configure script provides a variety of helper functions to assist -developers in checking for system features: +``write_target_makefile`` + Write a Makefile fragment to stdout, exposing the result of the most + ``probe_target_compiler`` call as the usual Make variables (``CC``, + ``AR``, ``LD``, etc.). -``do_cc $ARGS...`` - Attempt to run the system C compiler passing it $ARGS... -``do_cxx $ARGS...`` - Attempt to run the system C++ compiler passing it $ARGS... +Configure does not generally perform tests for compiler options beyond +basic checks to detect the host platform and ensure the compiler is +functioning. These are performed using a few more helper functions: ``compile_object $CFLAGS`` Attempt to compile a test program with the system C compiler using $CFLAGS. The test program must have been previously written to a file - called $TMPC. The replacement in Meson is the compiler object ``cc``, - which has methods such as ``cc.compiles()``, - ``cc.check_header()``, ``cc.has_function()``. + called $TMPC. ``compile_prog $CFLAGS $LDFLAGS`` Attempt to compile a test program with the system C compiler using $CFLAGS and link it with the system linker using $LDFLAGS. The test program must have been previously written to a file called $TMPC. - The replacement in Meson is ``cc.find_library()`` and ``cc.links()``. - -``has $COMMAND`` - Determine if $COMMAND exists in the current environment, either as a - shell builtin, or executable binary, returning 0 on success. The - replacement in Meson is ``find_program()``. ``check_define $NAME`` - Determine if the macro $NAME is defined by the system C compiler + Determine if the macro $NAME is defined by the system C compiler. + +``do_compiler $CC $ARGS...`` + Attempt to run the C compiler $CC, passing it $ARGS... This function + does not use flags passed via options such as ``--extra-cflags``, and + therefore can be used to check for cross compilers. However, most + such checks are done at ``make`` time instead (see for example the + ``cc-option`` macro in ``pc-bios/option-rom/Makefile``). ``write_c_skeleton`` Write a minimal C program main() function to the temporary file - indicated by $TMPC + indicated by $TMPC. -``error_exit $MESSAGE $MORE...`` - Print $MESSAGE to stderr, followed by $MORE... and then exit from the - configure script with non-zero status +Python virtual environments and the QEMU build system +----------------------------------------------------- + +TBD Stage 2: Meson ============== -The Meson build system is currently used to describe the build -process for: +The Meson build system describes the build and install process for: 1) executables, which include: - - Tools - ``qemu-img``, ``qemu-nbd``, ``qga`` (guest agent), etc + - Tools - ``qemu-img``, ``qemu-nbd``, ``qemu-ga`` (guest agent), etc - System emulators - ``qemu-system-$ARCH`` @@ -118,7 +144,8 @@ process for: 2) documentation -3) ROMs, which can be either installed as binary blobs or compiled +3) ROMs, whether provided as binary blobs in the QEMU distributions + or cross compiled under the direction of the configure script 4) other data files, such as icons or desktop files @@ -273,8 +300,7 @@ system/userspace emulation target Adding checks ------------- -New checks should be added to Meson. Compiler checks can be as simple as -the following:: +Compiler checks can be as simple as the following:: config_host_data.set('HAVE_BTRFS_H', cc.has_header('linux/btrfs.h')) @@ -351,22 +377,30 @@ script, which may point to something other than the first python3 binary on the path. -Stage 3: makefiles -================== +Stage 3: Make +============= -The use of GNU make is required with the QEMU build system. +The next step in building QEMU is to invoke make. GNU Make is required +to build QEMU, and may be installed as ``gmake`` on some hosts. -The output of Meson is a build.ninja file, which is used with the Ninja -build system. QEMU uses a different approach, where Makefile rules are -synthesized from the build.ninja file. The main Makefile includes these -rules and wraps them so that e.g. submodules are built before QEMU. -The resulting build system is largely non-recursive in nature, in -contrast to common practices seen with automake. +The output of Meson is a ``build.ninja`` file, which is used with the +Ninja build tool. However, QEMU's build comprises other components than +just the emulators (namely firmware and the tests in ``tests/tcg``) which +need different cross compilers. The QEMU Makefile wraps both Ninja and +the smaller build systems for firmware and tests; it also takes care of +running ``configure`` again when the script changes. Apart from invoking +these sub-Makefiles, the resulting build is largely non-recursive. -Tests are also ran by the Makefile with the traditional ``make check`` -phony target, while benchmarks are run with ``make bench``. Meson test -suites such as ``unit`` can be ran with ``make check-unit`` too. It is also -possible to run tests defined in meson.build with ``meson test``. +Tests, whether defined in ``meson.build`` or not, are also ran by the +Makefile with the traditional ``make check`` phony target, while benchmarks +are run with ``make bench``. Meson test suites such as ``unit`` can be ran +with ``make check-unit``, and ``make check-tcg`` builds and runs "non-Meson" +tests for all targets. + +If desired, it is also possible to use ``ninja`` and ``meson test``, +respectively to build emulators and run tests defined in meson.build. +The main difference is that ``make`` needs the ``-jN`` flag in order to +enable parallel builds or tests. Useful make targets ------------------- @@ -378,6 +412,7 @@ Useful make targets Print the value of the variable VAR. Useful for debugging the build system. + Important files for the build system ==================================== @@ -391,8 +426,7 @@ number of dynamically created files listed later. ``Makefile`` The main entry point used when invoking make to build all the components of QEMU. The default 'all' target will naturally result in the build of - every component. Makefile takes care of recursively building submodules - directly via a non-recursive set of rules. + every component. ``*/meson.build`` The meson.build file in the root directory is the main entry point for the @@ -401,59 +435,92 @@ number of dynamically created files listed later. other meson.build files spread throughout the QEMU source tree. ``tests/Makefile.include`` - Rules for external test harnesses. These include the TCG tests, - ``qemu-iotests`` and the Avocado-based integration tests. + Rules for external test harnesses. These include the TCG tests + and the Avocado-based integration tests. ``tests/docker/Makefile.include`` - Rules for Docker tests. Like tests/Makefile, this file is included - directly by the top level Makefile, anything defined in this file will - influence the entire build system. + Rules for Docker tests. Like ``tests/Makefile.include``, this file is + included directly by the top level Makefile, anything defined in this + file will influence the entire build system. ``tests/vm/Makefile.include`` - Rules for VM-based tests. Like tests/Makefile, this file is included - directly by the top level Makefile, anything defined in this file will - influence the entire build system. + Rules for VM-based tests. Like ``tests/Makefile.include``, this file is + included directly by the top level Makefile, anything defined in this + file will influence the entire build system. Dynamically created files ------------------------- -The following files are generated dynamically by configure in order to -control the behaviour of the statically defined makefiles. This avoids -the need for QEMU makefiles to go through any pre-processing as seen -with autotools, where Makefile.am generates Makefile.in which generates -Makefile. +The following files are generated at run-time in order to control the +behaviour of the Makefiles. This avoids the need for QEMU makefiles to +go through any pre-processing as seen with autotools, where configure +generates ``Makefile`` from ``Makefile.in``. Built by configure: ``config-host.mak`` When configure has determined the characteristics of the build host it - will write a long list of variables to config-host.mak file. This - provides the various install directories, compiler / linker flags and a + will write them to this file for use in ``Makefile`` and to a smaller + extent ``meson.build``. These include the paths to various tools and a variety of ``CONFIG_*`` variables related to optionally enabled features. - This is imported by the top level Makefile and meson.build in order to - tailor the build output. - config-host.mak is also used as a dependency checking mechanism. If make + ``config-host.mak`` is also used as a dependency checking mechanism. If make sees that the modification timestamp on configure is newer than that on - config-host.mak, then configure will be re-run. + ``config-host.mak``, then configure will be re-run. + + The variables defined here apply to all QEMU + build outputs. + +``config-meson.cross`` + + A Meson "cross file" (or native file) used to communicate the paths to + the toolchain and other configuration options. + +``config.status`` - The variables defined here are those which are applicable to all QEMU - build outputs. Variables which are potentially different for each - emulator target are defined by the next file... + A small shell script that will invoke configure again with the same + environment variables that were set during the first run. It's used to + rerun configure after changes to the source code, but it can also be + inspected manually to check the contents of the environment. +``Makefile.prereqs`` + + A set of Makefile dependencies that order the build and execution of + firmware and tests after the container images and emulators that they + need. + +``pc-bios/*/config.mak``, ``tests/tcg/config-host.mak``, ``tests/tcg/*/config-target.mak`` + + Configuration variables used to build the firmware and TCG tests, + including paths to cross compilation toolchains. + +``pyvenv`` + + A Python virtual environment that is used for all Python code running + during the build. Using a virtual environment ensures that even code + that is run via ``sphinx-build``, ``meson`` etc. uses the same interpreter + and packages. Built by Meson: +``config-host.h`` + Used by C code to determine the properties of the build environment + and the set of enabled features for the entire build. + ``${TARGET-NAME}-config-devices.mak`` - TARGET-NAME is again the name of a system or userspace emulator. The - config-devices.mak file is automatically generated by make using the - scripts/make_device_config.sh program, feeding it the - default-configs/$TARGET-NAME file as input. - -``config-host.h``, ``$TARGET_NAME-config-target.h``, ``$TARGET_NAME-config-devices.h`` - These files are used by source code to determine what features are - enabled. They are generated from the contents of the corresponding - ``*.mak`` files using Meson's ``configure_file()`` function. + TARGET-NAME is the name of a system emulator. The file is + generated by Meson using files under ``configs/devices`` as input. + +``${TARGET-NAME}-config-target.mak`` + TARGET-NAME is the name of a system or usermode emulator. The file is + generated by Meson using files under ``configs/targets`` as input. + +``$TARGET_NAME-config-target.h``, ``$TARGET_NAME-config-devices.h`` + Used by C code to determine the properties and enabled + features for each target. enabled. They are generated from + the contents of the corresponding ``*.mak`` files using Meson's + ``configure_file()`` function; each target can include them using + the ``CONFIG_TARGET`` and ``CONFIG_DEVICES`` macro respectively. ``build.ninja`` The build rules. -- cgit 1.4.1