diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/git-submodule.sh | 73 | ||||
| -rw-r--r-- | scripts/meson-buildoptions.sh | 3 | ||||
| -rw-r--r-- | scripts/meson.build | 2 | ||||
| -rw-r--r-- | scripts/xml-preprocess-test.py | 136 | ||||
| -rwxr-xr-x | scripts/xml-preprocess.py | 293 |
5 files changed, 472 insertions, 35 deletions
diff --git a/scripts/git-submodule.sh b/scripts/git-submodule.sh index 11fad2137c..335f7f5fdf 100755 --- a/scripts/git-submodule.sh +++ b/scripts/git-submodule.sh @@ -9,13 +9,22 @@ command=$1 shift maybe_modules="$@" -# if not running in a git checkout, do nothing -test "$command" = "ignore" && exit 0 - +test -z "$maybe_modules" && exit 0 test -z "$GIT" && GIT=$(command -v git) cd "$(dirname "$0")/.." +no_git_error= +if ! test -e ".git"; then + no_git_error='no git checkout exists' +elif test -z "$GIT"; then + no_git_error='git binary not found' +fi + +is_git() { + test -z "$no_git_error" +} + update_error() { echo "$0: $*" echo @@ -34,7 +43,7 @@ update_error() { } validate_error() { - if test "$1" = "validate"; then + if is_git && test "$1" = "validate"; then echo "GIT submodules checkout is out of date, and submodules" echo "configured for validate only. Please run" echo " scripts/git-submodule.sh update $maybe_modules" @@ -51,42 +60,42 @@ check_updated() { test "$CURSTATUS" = "$OLDSTATUS" } -if test -n "$maybe_modules" && ! test -e ".git" -then - echo "$0: unexpectedly called with submodules but no git checkout exists" - exit 1 -fi - -if test -n "$maybe_modules" && test -z "$GIT" -then - echo "$0: unexpectedly called with submodules but git binary not found" - exit 1 +if is_git; then + test -e $substat || touch $substat + modules="" + for m in $maybe_modules + do + $GIT submodule status $m 1> /dev/null 2>&1 + if test $? = 0 + then + modules="$modules $m" + grep $m $substat > /dev/null 2>&1 || $GIT submodule status $module >> $substat + else + echo "warn: ignoring non-existent submodule $m" + fi + done +else + modules=$maybe_modules fi -modules="" -for m in $maybe_modules -do - $GIT submodule status $m 1> /dev/null 2>&1 - if test $? = 0 - then - modules="$modules $m" - else - echo "warn: ignoring non-existent submodule $m" - fi -done - case "$command" in status|validate) - test -f "$substat" || validate_error "$command" - test -z "$maybe_modules" && exit 0 for module in $modules; do - check_updated $module || validate_error "$command" + if is_git; then + check_updated $module || validate_error "$command" + elif ! (set xyz "$module"/* && test -e "$2"); then + # The directory does not exist or it contains no files + echo "$0: sources not available for $module and $no_git_error" + validate_error "$command" + fi done - exit 0 ;; + update) - test -e $substat || touch $substat - test -z "$maybe_modules" && exit 0 + is_git || { + echo "$0: unexpectedly called with submodules but $no_git_error" + exit 1 + } $GIT submodule update --init $modules 1>/dev/null test $? -ne 0 && update_error "failed to update modules" diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 5714fd93d9..7dd5709ef4 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -39,7 +39,6 @@ meson_options_help() { printf "%s\n" ' jemalloc/system/tcmalloc)' printf "%s\n" ' --enable-module-upgrades try to load modules from alternate paths for' printf "%s\n" ' upgrades' - printf "%s\n" ' --enable-profiler profiler support' printf "%s\n" ' --enable-rng-none dummy RNG, avoid using /dev/(u)random and' printf "%s\n" ' getrandom()' printf "%s\n" ' --enable-safe-stack SafeStack Stack Smash Protection (requires' @@ -401,8 +400,6 @@ _meson_option_parse() { --with-pkgversion=*) quote_sh "-Dpkgversion=$2" ;; --enable-png) printf "%s" -Dpng=enabled ;; --disable-png) printf "%s" -Dpng=disabled ;; - --enable-profiler) printf "%s" -Dprofiler=true ;; - --disable-profiler) printf "%s" -Dprofiler=false ;; --enable-pvrdma) printf "%s" -Dpvrdma=enabled ;; --disable-pvrdma) printf "%s" -Dpvrdma=disabled ;; --enable-qcow1) printf "%s" -Dqcow1=enabled ;; diff --git a/scripts/meson.build b/scripts/meson.build index 1c89e10a76..532277f5a2 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -1,3 +1,5 @@ if stap.found() install_data('qemu-trace-stap', install_dir: get_option('bindir')) endif + +test('xml-preprocess', files('xml-preprocess-test.py'), suite: ['unit']) diff --git a/scripts/xml-preprocess-test.py b/scripts/xml-preprocess-test.py new file mode 100644 index 0000000000..dd92579969 --- /dev/null +++ b/scripts/xml-preprocess-test.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2023 Red Hat, Inc. +# +# SPDX-License-Identifier: MIT +"""Unit tests for xml-preprocess""" + +import contextlib +import importlib +import os +import platform +import subprocess +import tempfile +import unittest +from io import StringIO + +xmlpp = importlib.import_module("xml-preprocess") + + +class TestXmlPreprocess(unittest.TestCase): + """Tests for xml-preprocess.Preprocessor""" + + def test_preprocess_xml(self): + with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: + temp_file.write("<root></root>") + temp_file_name = temp_file.name + result = xmlpp.preprocess_xml(temp_file_name) + self.assertEqual(result, "<root></root>") + os.remove(temp_file_name) + + def test_save_xml(self): + with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: + temp_file_name = temp_file.name + xmlpp.save_xml("<root></root>", temp_file_name) + self.assertTrue(os.path.isfile(temp_file_name)) + os.remove(temp_file_name) + + def test_include(self): + with tempfile.NamedTemporaryFile(mode="w", delete=False) as inc_file: + inc_file.write("<included>Content from included file</included>") + inc_file_name = inc_file.name + xml_str = f"<?include {inc_file_name} ?>" + expected = "<included>Content from included file</included>" + xpp = xmlpp.Preprocessor() + result = xpp.preprocess(xml_str) + self.assertEqual(result, expected) + os.remove(inc_file_name) + self.assertRaises(FileNotFoundError, xpp.preprocess, xml_str) + + def test_envvar(self): + os.environ["TEST_ENV_VAR"] = "TestValue" + xml_str = "<root>$(env.TEST_ENV_VAR)</root>" + expected = "<root>TestValue</root>" + xpp = xmlpp.Preprocessor() + result = xpp.preprocess(xml_str) + self.assertEqual(result, expected) + self.assertRaises(KeyError, xpp.preprocess, "$(env.UNKNOWN)") + + def test_sys_var(self): + xml_str = "<root>$(sys.ARCH)</root>" + expected = f"<root>{platform.architecture()[0]}</root>" + xpp = xmlpp.Preprocessor() + result = xpp.preprocess(xml_str) + self.assertEqual(result, expected) + self.assertRaises(KeyError, xpp.preprocess, "$(sys.UNKNOWN)") + + def test_cus_var(self): + xml_str = "<root>$(var.USER)</root>" + expected = "<root></root>" + xpp = xmlpp.Preprocessor() + result = xpp.preprocess(xml_str) + self.assertEqual(result, expected) + xml_str = "<?define USER=FOO?><root>$(var.USER)</root>" + expected = "<root>FOO</root>" + xpp = xmlpp.Preprocessor() + result = xpp.preprocess(xml_str) + self.assertEqual(result, expected) + + def test_error_warning(self): + xml_str = "<root><?warning \"test warn\"?></root>" + expected = "<root></root>" + xpp = xmlpp.Preprocessor() + out = StringIO() + with contextlib.redirect_stdout(out): + result = xpp.preprocess(xml_str) + self.assertEqual(result, expected) + self.assertEqual(out.getvalue(), "[Warning]: test warn\n") + self.assertRaises(RuntimeError, xpp.preprocess, "<?error \"test\"?>") + + def test_cmd(self): + xpp = xmlpp.Preprocessor() + result = xpp.preprocess('<root><?cmd "echo hello world"?></root>') + self.assertEqual(result, "<root>hello world</root>") + self.assertRaises( + subprocess.CalledProcessError, + xpp.preprocess, '<?cmd "test-unknown-cmd"?>' + ) + + def test_foreach(self): + xpp = xmlpp.Preprocessor() + result = xpp.preprocess( + '<root><?foreach x in a;b;c?>$(var.x)<?endforeach?></root>' + ) + self.assertEqual(result, "<root>abc</root>") + + def test_if_elseif(self): + xpp = xmlpp.Preprocessor() + result = xpp.preprocess('<root><?if True?>ok<?endif?></root>') + self.assertEqual(result, "<root>ok</root>") + result = xpp.preprocess('<root><?if False?>ok<?endif?></root>') + self.assertEqual(result, "<root></root>") + result = xpp.preprocess('<root><?if True?>ok<?else?>ko<?endif?></root>') + self.assertEqual(result, "<root>ok</root>") + result = xpp.preprocess('<root><?if False?>ok<?else?>ko<?endif?></root>') + self.assertEqual(result, "<root>ko</root>") + result = xpp.preprocess( + '<root><?if False?>ok<?elseif True?>ok2<?else?>ko<?endif?></root>' + ) + self.assertEqual(result, "<root>ok2</root>") + result = xpp.preprocess( + '<root><?if False?>ok<?elseif False?>ok<?else?>ko<?endif?></root>' + ) + self.assertEqual(result, "<root>ko</root>") + + def test_ifdef(self): + xpp = xmlpp.Preprocessor() + result = xpp.preprocess('<root><?ifdef USER?>ok<?else?>ko<?endif?></root>') + self.assertEqual(result, "<root>ko</root>") + result = xpp.preprocess( + '<?define USER=FOO?><root><?ifdef USER?>ok<?else?>ko<?endif?></root>' + ) + self.assertEqual(result, "<root>ok</root>") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/xml-preprocess.py b/scripts/xml-preprocess.py new file mode 100755 index 0000000000..57f1d28912 --- /dev/null +++ b/scripts/xml-preprocess.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2019 Tony Su +# Copyright (c) 2023 Red Hat, Inc. +# +# SPDX-License-Identifier: MIT +# +# Adapted from https://github.com/peitaosu/XML-Preprocessor +# +"""This is a XML Preprocessor which can be used to process your XML file before +you use it, to process conditional statements, variables, iteration +statements, error/warning, execute command, etc. + +## XML Schema + +### Include Files +``` +<?include path/to/file ?> +``` + +### Variables +``` +$(env.EnvironmentVariable) + +$(sys.SystemVariable) + +$(var.CustomVariable) +``` + +### Conditional Statements +``` +<?if ?> + +<?ifdef ?> + +<?ifndef ?> + +<?else?> + +<?elseif ?> + +<?endif?> +``` + +### Iteration Statements +``` +<?foreach VARNAME in 1;2;3?> + $(var.VARNAME) +<?endforeach?> +``` + +### Errors and Warnings +``` +<?error "This is error message!" ?> + +<?warning "This is warning message!" ?> +``` + +### Commands +``` +<? cmd "echo hello world" ?> +``` +""" + +import os +import platform +import re +import subprocess +import sys +from typing import Optional +from xml.dom import minidom + + +class Preprocessor(): + """This class holds the XML preprocessing state""" + + def __init__(self): + self.sys_vars = { + "ARCH": platform.architecture()[0], + "SOURCE": os.path.abspath(__file__), + "CURRENT": os.getcwd(), + } + self.cus_vars = {} + + def _pp_include(self, xml_str: str) -> str: + include_regex = r"(<\?include([\w\s\\/.:_-]+)\s*\?>)" + matches = re.findall(include_regex, xml_str) + for group_inc, group_xml in matches: + inc_file_path = group_xml.strip() + with open(inc_file_path, "r", encoding="utf-8") as inc_file: + inc_file_content = inc_file.read() + xml_str = xml_str.replace(group_inc, inc_file_content) + return xml_str + + def _pp_env_var(self, xml_str: str) -> str: + envvar_regex = r"(\$\(env\.(\w+)\))" + matches = re.findall(envvar_regex, xml_str) + for group_env, group_var in matches: + xml_str = xml_str.replace(group_env, os.environ[group_var]) + return xml_str + + def _pp_sys_var(self, xml_str: str) -> str: + sysvar_regex = r"(\$\(sys\.(\w+)\))" + matches = re.findall(sysvar_regex, xml_str) + for group_sys, group_var in matches: + xml_str = xml_str.replace(group_sys, self.sys_vars[group_var]) + return xml_str + + def _pp_cus_var(self, xml_str: str) -> str: + define_regex = r"(<\?define\s*(\w+)\s*=\s*([\w\s\"]+)\s*\?>)" + matches = re.findall(define_regex, xml_str) + for group_def, group_name, group_var in matches: + group_name = group_name.strip() + group_var = group_var.strip().strip("\"") + self.cus_vars[group_name] = group_var + xml_str = xml_str.replace(group_def, "") + cusvar_regex = r"(\$\(var\.(\w+)\))" + matches = re.findall(cusvar_regex, xml_str) + for group_cus, group_var in matches: + xml_str = xml_str.replace( + group_cus, + self.cus_vars.get(group_var, "") + ) + return xml_str + + def _pp_foreach(self, xml_str: str) -> str: + foreach_regex = r"(<\?foreach\s+(\w+)\s+in\s+([\w;]+)\s*\?>(.*)<\?endforeach\?>)" + matches = re.findall(foreach_regex, xml_str) + for group_for, group_name, group_vars, group_text in matches: + group_texts = "" + for var in group_vars.split(";"): + self.cus_vars[group_name] = var + group_texts += self._pp_cus_var(group_text) + xml_str = xml_str.replace(group_for, group_texts) + return xml_str + + def _pp_error_warning(self, xml_str: str) -> str: + error_regex = r"<\?error\s*\"([^\"]+)\"\s*\?>" + matches = re.findall(error_regex, xml_str) + for group_var in matches: + raise RuntimeError("[Error]: " + group_var) + warning_regex = r"(<\?warning\s*\"([^\"]+)\"\s*\?>)" + matches = re.findall(warning_regex, xml_str) + for group_wrn, group_var in matches: + print("[Warning]: " + group_var) + xml_str = xml_str.replace(group_wrn, "") + return xml_str + + def _pp_if_eval(self, xml_str: str) -> str: + ifelif_regex = ( + r"(<\?(if|elseif)\s*([^\"\s=<>!]+)\s*([!=<>]+)\s*\"*([^\"=<>!]+)\"*\s*\?>)" + ) + matches = re.findall(ifelif_regex, xml_str) + for ifelif, tag, left, operator, right in matches: + if "<" in operator or ">" in operator: + result = eval(f"{left} {operator} {right}") + else: + result = eval(f'"{left}" {operator} "{right}"') + xml_str = xml_str.replace(ifelif, f"<?{tag} {result}?>") + return xml_str + + def _pp_ifdef_ifndef(self, xml_str: str) -> str: + ifndef_regex = r"(<\?(ifdef|ifndef)\s*([\w]+)\s*\?>)" + matches = re.findall(ifndef_regex, xml_str) + for group_ifndef, group_tag, group_var in matches: + if group_tag == "ifdef": + result = group_var in self.cus_vars + else: + result = group_var not in self.cus_vars + xml_str = xml_str.replace(group_ifndef, f"<?if {result}?>") + return xml_str + + def _pp_if_elseif(self, xml_str: str) -> str: + if_elif_else_regex = ( + r"(<\?if\s(True|False)\?>" + r"(.*?)" + r"<\?elseif\s(True|False)\?>" + r"(.*?)" + r"<\?else\?>" + r"(.*?)" + r"<\?endif\?>)" + ) + if_else_regex = ( + r"(<\?if\s(True|False)\?>" + r"(.*?)" + r"<\?else\?>" + r"(.*?)" + r"<\?endif\?>)" + ) + if_regex = r"(<\?if\s(True|False)\?>(.*?)<\?endif\?>)" + matches = re.findall(if_elif_else_regex, xml_str, re.DOTALL) + for (group_full, group_if, group_if_elif, group_elif, + group_elif_else, group_else) in matches: + result = "" + if group_if == "True": + result = group_if_elif + elif group_elif == "True": + result = group_elif_else + else: + result = group_else + xml_str = xml_str.replace(group_full, result) + matches = re.findall(if_else_regex, xml_str, re.DOTALL) + for group_full, group_if, group_if_else, group_else in matches: + result = "" + if group_if == "True": + result = group_if_else + else: + result = group_else + xml_str = xml_str.replace(group_full, result) + matches = re.findall(if_regex, xml_str, re.DOTALL) + for group_full, group_if, group_text in matches: + result = "" + if group_if == "True": + result = group_text + xml_str = xml_str.replace(group_full, result) + return xml_str + + def _pp_command(self, xml_str: str) -> str: + cmd_regex = r"(<\?cmd\s*\"([^\"]+)\"\s*\?>)" + matches = re.findall(cmd_regex, xml_str) + for group_cmd, group_exec in matches: + output = subprocess.check_output( + group_exec, shell=True, + text=True, stderr=subprocess.STDOUT + ) + xml_str = xml_str.replace(group_cmd, output) + return xml_str + + def _pp_blanks(self, xml_str: str) -> str: + right_blank_regex = r">[\n\s\t\r]*" + left_blank_regex = r"[\n\s\t\r]*<" + xml_str = re.sub(right_blank_regex, ">", xml_str) + xml_str = re.sub(left_blank_regex, "<", xml_str) + return xml_str + + def preprocess(self, xml_str: str) -> str: + fns = [ + self._pp_blanks, + self._pp_include, + self._pp_foreach, + self._pp_env_var, + self._pp_sys_var, + self._pp_cus_var, + self._pp_if_eval, + self._pp_ifdef_ifndef, + self._pp_if_elseif, + self._pp_command, + self._pp_error_warning, + ] + + while True: + changed = False + for func in fns: + out_xml = func(xml_str) + if not changed and out_xml != xml_str: + changed = True + xml_str = out_xml + if not changed: + break + + return xml_str + + +def preprocess_xml(path: str) -> str: + with open(path, "r", encoding="utf-8") as original_file: + input_xml = original_file.read() + + proc = Preprocessor() + return proc.preprocess(input_xml) + + +def save_xml(xml_str: str, path: Optional[str]): + xml = minidom.parseString(xml_str) + with open(path, "w", encoding="utf-8") if path else sys.stdout as output_file: + output_file.write(xml.toprettyxml()) + + +def main(): + if len(sys.argv) < 2: + print("Usage: xml-preprocessor input.xml [output.xml]") + sys.exit(1) + + output_file = None + if len(sys.argv) == 3: + output_file = sys.argv[2] + + input_file = sys.argv[1] + output_xml = preprocess_xml(input_file) + save_xml(output_xml, output_file) + + +if __name__ == "__main__": + main() |