diff options
27 files changed, 420 insertions, 135 deletions
diff --git a/README.md b/README.md index 270dd3f7..4ce234cf 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Reverse engineering framework in Python -**Table of Contents** +**Table of Contents** - [What is Miasm?](#user-content-what-is-miasm) - [Basic examples](#user-content-basic-examples) @@ -126,7 +126,7 @@ Working with IR, for instance by getting side effects: ... print 'read: ', [str(x) for x in o_r] ... print 'written:', [str(x) for x in o_w] ... print -... +... read: ['R8', 'R0'] written: ['R2'] @@ -244,8 +244,8 @@ RAX 0000000000000000 RBX 0000000000000000 RCX 0000000000000004 RDX 0000000000000 RSI 0000000000000000 RDI 0000000000000000 RSP 000000000123FFF8 RBP 0000000000000000 zf 0000000000000000 nf 0000000000000000 of 0000000000000000 cf 0000000000000000 RIP 0000000040000013 -40000015 RET ->>> +40000015 RET +>>> ``` @@ -270,7 +270,7 @@ Initializing the IR pool: >>> ira = machine.ira() >>> for b in blocs: ... ira.add_bloc(b) -... +... ``` Initializing the engine with default symbolic values: @@ -424,7 +424,7 @@ How does it work? Miasm embeds its own disassembler, intermediate language and instruction semantic. It is written in Python. -To emulate code, it uses LibTCC, LLVM or Python to JIT the intermediate +To emulate code, it uses LibTCC, LLVM, GCC or Python to JIT the intermediate representation. It can emulate shellcodes and all or parts of binaries. Python callbacks can be executed to interact with the execution, for instance to emulate library functions effects. @@ -447,12 +447,15 @@ Software requirements Miasm uses: -* LibTCC [tinycc (ONLY version 0.9.26)](http://repo.or.cz/w/tinycc.git), to JIT code for emulation mode -* or LLVM v3.2 with python-llvm, see below * python-pyparsing * python-dev * elfesteem from [Elfesteem](https://github.com/serpilliere/elfesteem.git) +To enable code JIT, one of the following module is mandatory: +* GCC +* LLVM v3.2 with python-llvm, see below +* LibTCC [tinycc (ONLY version 0.9.26)](http://repo.or.cz/w/tinycc.git) + Configuration ------------- @@ -464,7 +467,8 @@ python setup.py build sudo python setup.py install ``` -* To use the jitter, TCC or LLVM is recommended +To use the jitter, GCC, TCC or LLVM is recommended +* GCC (any version) * LibTCC needs to be configured with the `--disable-static` option * remove `libtcc-dev` from the system to avoid conflicts * clone [TinyCC](http://repo.or.cz/tinycc.git): `git clone http://repo.or.cz/tinycc.git` @@ -494,7 +498,7 @@ Windows & IDA Most of Miasm's IDA plugins use a subset of Miasm functionnality. A quick way to have them working is to add: * `elfesteem` directory and `pyparsing.py` to `C:\...\IDA\python\` or `pip install pyparsing elfesteem` -* `miasm2/miasm2` directory to `C:\...\IDA\python\` +* `miasm2/miasm2` directory to `C:\...\IDA\python\` All features excepting JITter related ones will be available. For a more complete installation, please refer to above paragraphs. diff --git a/miasm2/jitter/Jitgcc.c b/miasm2/jitter/Jitgcc.c new file mode 100644 index 00000000..3e7225cb --- /dev/null +++ b/miasm2/jitter/Jitgcc.c @@ -0,0 +1,87 @@ +#include <Python.h> +#include <inttypes.h> +#include <stdint.h> + +typedef struct { + uint8_t is_local; + uint64_t address; +} block_id; + +typedef int (*jitted_func)(block_id*, PyObject*); + + +PyObject* gcc_exec_bloc(PyObject* self, PyObject* args) +{ + jitted_func func; + PyObject* jitcpu; + PyObject* func_py; + PyObject* lbl2ptr; + PyObject* breakpoints; + PyObject* retaddr = NULL; + int status; + block_id BlockDst; + + if (!PyArg_ParseTuple(args, "OOOO", &retaddr, &jitcpu, &lbl2ptr, &breakpoints)) + return NULL; + + /* The loop will decref retaddr always once */ + Py_INCREF(retaddr); + + for (;;) { + // Init + BlockDst.is_local = 0; + BlockDst.address = 0; + + // Get the expected jitted function address + func_py = PyDict_GetItem(lbl2ptr, retaddr); + if (func_py) + func = (jitted_func) PyInt_AsLong((PyObject*) func_py); + else { + if (BlockDst.is_local == 1) { + fprintf(stderr, "return on local label!\n"); + exit(1); + } + // retaddr is not jitted yet + return retaddr; + } + + // Execute it + status = func(&BlockDst, jitcpu); + Py_DECREF(retaddr); + retaddr = PyLong_FromUnsignedLongLong(BlockDst.address); + + // Check exception + if (status) + return retaddr; + + // Check breakpoint + if (PyDict_Contains(breakpoints, retaddr)) + return retaddr; + } +} + + + +static PyObject *GccError; + + +static PyMethodDef GccMethods[] = { + {"gcc_exec_bloc", gcc_exec_bloc, METH_VARARGS, + "gcc exec bloc"}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initJitgcc(void) +{ + PyObject *m; + + m = Py_InitModule("Jitgcc", GccMethods); + if (m == NULL) + return; + + GccError = PyErr_NewException("gcc.error", NULL, NULL); + Py_INCREF(GccError); + PyModule_AddObject(m, "error", GccError); +} + diff --git a/miasm2/jitter/jitcore_gcc.py b/miasm2/jitter/jitcore_gcc.py new file mode 100644 index 00000000..cad64df3 --- /dev/null +++ b/miasm2/jitter/jitcore_gcc.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +import os +import tempfile +import ctypes +from distutils.sysconfig import get_python_inc +from subprocess import check_call +from hashlib import md5 + +from miasm2.ir.ir2C import irblocs2C +from miasm2.jitter import jitcore, Jitgcc +from miasm2.core.utils import keydefaultdict + + +def gen_core(arch, attrib): + lib_dir = os.path.dirname(os.path.realpath(__file__)) + + txt = "" + txt += '#include "%s/queue.h"\n' % lib_dir + txt += '#include "%s/vm_mngr.h"\n' % lib_dir + txt += '#include "%s/vm_mngr_py.h"\n' % lib_dir + txt += '#include "%s/JitCore.h"\n' % lib_dir + txt += '#include "%s/arch/JitCore_%s.h"\n' % (lib_dir, arch.name) + txt += r''' +#define RAISE(errtype, msg) {PyObject* p; p = PyErr_Format( errtype, msg ); return p;} +''' + return txt + + +def gen_C_source(ir_arch, func_code): + c_source = "" + c_source += "\n".join(func_code) + + c_source = gen_core(ir_arch.arch, ir_arch.attrib) + c_source + c_source = "#include <Python.h>\n" + c_source + + return c_source + + +class myresolver(object): + + def __init__(self, offset): + self.offset = offset + + def ret(self): + return "return PyLong_FromUnsignedLongLong(0x%X);" % self.offset + + +class resolver(object): + + def __init__(self): + self.resolvers = keydefaultdict(myresolver) + + def get_resolver(self, offset): + return self.resolvers[offset] + + +class JitCore_Gcc(jitcore.JitCore): + + "JiT management, using GCC as backend" + + def __init__(self, ir_arch, bs=None): + self.jitted_block_delete_cb = self.deleteCB + super(JitCore_Gcc, self).__init__(ir_arch, bs) + self.resolver = resolver() + self.gcc_states = {} + self.ir_arch = ir_arch + self.tempdir = os.path.join(tempfile.gettempdir(), "miasm_gcc_cache") + try: + os.mkdir(self.tempdir, 0755) + except OSError: + pass + if not os.access(self.tempdir, os.R_OK | os.W_OK): + raise RuntimeError( + 'Cannot access gcc cache directory %s ' % self.tempdir) + self.exec_wrapper = Jitgcc.gcc_exec_bloc + self.libs = None + self.include_files = None + + def deleteCB(self, offset): + pass + + def load(self): + lib_dir = os.path.dirname(os.path.realpath(__file__)) + libs = [os.path.join(lib_dir, 'VmMngr.so'), + os.path.join(lib_dir, + 'arch/JitCore_%s.so' % (self.ir_arch.arch.name))] + + include_files = [os.path.dirname(__file__), + get_python_inc()] + self.include_files = include_files + self.libs = libs + + def jit_gcc_compil(self, f_name, func_code): + func_hash = md5(func_code).hexdigest() + fname_out = os.path.join(self.tempdir, "%s.so" % func_hash) + if not os.access(fname_out, os.R_OK | os.X_OK): + # Create unique C file + fdesc, fname_in = tempfile.mkstemp(suffix=".c") + os.write(fdesc, func_code) + os.close(fdesc) + + # Create unique SO file + _, fname_tmp = tempfile.mkstemp(suffix=".so") + + inc_dir = ["-I%s" % inc for inc in self.include_files] + libs = ["%s" % lib for lib in self.libs] + args = ["gcc"] + ["-O3"] + [ + "-shared", "-fPIC", fname_in, '-o', fname_tmp] + inc_dir + libs + check_call(args) + # Move temporary file to final file + os.rename(fname_tmp, fname_out) + + lib = ctypes.cdll.LoadLibrary(fname_out) + func = getattr(lib, f_name) + addr = ctypes.cast(func, ctypes.c_void_p).value + return None, addr + + def jitirblocs(self, label, irblocs): + f_name = "bloc_%s" % label.name + f_declaration = 'int %s(block_id * BlockDst, JitCpu* jitcpu)' % f_name + out = irblocs2C(self.ir_arch, self.resolver, label, irblocs, + gen_exception_code=True, + log_mn=self.log_mn, + log_regs=self.log_regs) + out = [f_declaration + '{'] + out + ['}\n'] + c_code = out + + func_code = gen_C_source(self.ir_arch, c_code) + + # open('tmp_%.4d.c'%self.jitcount, "w").write(func_code) + self.jitcount += 1 + gcc_state, mcode = self.jit_gcc_compil(f_name, func_code) + self.lbl2jitbloc[label.offset] = mcode + self.gcc_states[label.offset] = gcc_state diff --git a/miasm2/jitter/jitload.py b/miasm2/jitter/jitload.py index 05a8575e..cc92b0cf 100644 --- a/miasm2/jitter/jitload.py +++ b/miasm2/jitter/jitload.py @@ -20,22 +20,6 @@ log_func = logging.getLogger('jit function call') log_func.addHandler(hnd) log_func.setLevel(logging.CRITICAL) - -try: - from miasm2.jitter.jitcore_tcc import JitCore_Tcc -except ImportError: - log.error('cannot import jit tcc') - -try: - from miasm2.jitter.jitcore_llvm import JitCore_LLVM -except ImportError: - log.error('cannot import jit llvm') - -try: - from miasm2.jitter.jitcore_python import JitCore_Python -except ImportError: - log.error('cannot import jit python') - try: from miasm2.jitter import VmMngr except ImportError: @@ -189,18 +173,22 @@ class jitter: self.arch = ir_arch.arch self.attrib = ir_arch.attrib arch_name = ir_arch.arch.name # (ir_arch.arch.name, ir_arch.attrib) - if arch_name == "x86": - from miasm2.jitter.arch import JitCore_x86 as jcore - elif arch_name == "arm": - from miasm2.jitter.arch import JitCore_arm as jcore - elif arch_name == "aarch64": - from miasm2.jitter.arch import JitCore_aarch64 as jcore - elif arch_name == "msp430": - from miasm2.jitter.arch import JitCore_msp430 as jcore - elif arch_name == "mips32": - from miasm2.jitter.arch import JitCore_mips32 as jcore - else: - raise ValueError("unsupported jit arch!") + + try: + if arch_name == "x86": + from miasm2.jitter.arch import JitCore_x86 as jcore + elif arch_name == "arm": + from miasm2.jitter.arch import JitCore_arm as jcore + elif arch_name == "aarch64": + from miasm2.jitter.arch import JitCore_aarch64 as jcore + elif arch_name == "msp430": + from miasm2.jitter.arch import JitCore_msp430 as jcore + elif arch_name == "mips32": + from miasm2.jitter.arch import JitCore_mips32 as jcore + else: + raise ValueError("unknown jit arch: %s" % arch_name) + except ImportError: + raise RuntimeError('Unsupported jit arch: %s' % arch_name) self.vm = VmMngr.Vm() self.cpu = jcore.JitCpu() @@ -211,14 +199,21 @@ class jitter: self.symbexec = EmulatedSymbExec(self.cpu, self.ir_arch, {}) self.symbexec.reset_regs() - if jit_type == "tcc": - self.jit = JitCore_Tcc(self.ir_arch, self.bs) - elif jit_type == "llvm": - self.jit = JitCore_LLVM(self.ir_arch, self.bs) - elif jit_type == "python": - self.jit = JitCore_Python(self.ir_arch, self.bs) - else: - raise Exception("Unkown JiT Backend") + try: + if jit_type == "tcc": + from miasm2.jitter.jitcore_tcc import JitCore_Tcc as JitCore + elif jit_type == "llvm": + from miasm2.jitter.jitcore_llvm import JitCore_LLVM as JitCore + elif jit_type == "python": + from miasm2.jitter.jitcore_python import JitCore_Python as JitCore + elif jit_type == "gcc": + from miasm2.jitter.jitcore_gcc import JitCore_Gcc as JitCore + else: + raise ValueError("Unknown jitter %s" % jit_type) + except ImportError: + raise RuntimeError('Unsupported jitter: %s' % jit_type) + + self.jit = JitCore(self.ir_arch, self.bs) self.cpu.init_regs() self.vm.init_memory_page_pool() diff --git a/setup.py b/setup.py index 3b504542..03999108 100755 --- a/setup.py +++ b/setup.py @@ -48,6 +48,8 @@ def buil_all(): ["miasm2/jitter/JitCore.c", "miasm2/jitter/vm_mngr.c", "miasm2/jitter/arch/JitCore_mips32.c"]), + Extension("miasm2.jitter.Jitgcc", + ["miasm2/jitter/Jitgcc.c"]), Extension("miasm2.jitter.Jitllvm", ["miasm2/jitter/Jitllvm.c"]), ] @@ -78,6 +80,8 @@ def buil_all(): "miasm2/jitter/arch/JitCore_mips32.c"]), Extension("miasm2.jitter.Jitllvm", ["miasm2/jitter/Jitllvm.c"]), + Extension("miasm2.jitter.Jitgcc", + ["miasm2/jitter/Jitgcc.c"]), Extension("miasm2.jitter.Jittcc", ["miasm2/jitter/Jittcc.c"], libraries=["tcc"]) diff --git a/test/arch/aarch64/unit/asm_test.py b/test/arch/aarch64/unit/asm_test.py index 9e0d5ea8..a9e6cfc6 100644 --- a/test/arch/aarch64/unit/asm_test.py +++ b/test/arch/aarch64/unit/asm_test.py @@ -21,8 +21,8 @@ if filename and os.path.isfile(filename): reg_and_id = dict(mn_aarch64.regs.all_regs_ids_byname) class Asm_Test(object): - def __init__(self): - self.myjit = Machine("aarch64l").jitter() + def __init__(self, jitter): + self.myjit = Machine("aarch64l").jitter(jitter) self.myjit.init_stack() self.myjit.jit.log_regs = False diff --git a/test/arch/aarch64/unit/mn_ubfm.py b/test/arch/aarch64/unit/mn_ubfm.py index 938f13cf..6f1857fb 100644 --- a/test/arch/aarch64/unit/mn_ubfm.py +++ b/test/arch/aarch64/unit/mn_ubfm.py @@ -1,4 +1,6 @@ #! /usr/bin/env python +import sys + from asm_test import Asm_Test from pdb import pm @@ -27,4 +29,4 @@ main: if __name__ == "__main__": - [test()() for test in [Test_UBFM1, Test_UBFM2 ]] + [test(*sys.argv[1:])() for test in [Test_UBFM1, Test_UBFM2 ]] diff --git a/test/arch/mips32/unit/asm_test.py b/test/arch/mips32/unit/asm_test.py index a00d0842..7272c15e 100644 --- a/test/arch/mips32/unit/asm_test.py +++ b/test/arch/mips32/unit/asm_test.py @@ -23,8 +23,8 @@ reg_and_id = dict(mn_mips32.regs.all_regs_ids_byname) class Asm_Test(object): - def __init__(self): - self.myjit = Machine("mips32l").jitter() + def __init__(self, jitter): + self.myjit = Machine("mips32l").jitter(jitter) self.myjit.init_stack() self.myjit.jit.log_regs = False diff --git a/test/arch/mips32/unit/mn_bcc.py b/test/arch/mips32/unit/mn_bcc.py index 729f3679..4818c171 100644 --- a/test/arch/mips32/unit/mn_bcc.py +++ b/test/arch/mips32/unit/mn_bcc.py @@ -1,4 +1,6 @@ #! /usr/bin/env python +import sys + from asm_test import Asm_Test @@ -29,4 +31,4 @@ SKIP: if __name__ == "__main__": - [test()() for test in [Test_BCC]] + [test(*sys.argv[1:])() for test in [Test_BCC]] diff --git a/test/arch/x86/unit/asm_test.py b/test/arch/x86/unit/asm_test.py index 118a57b4..4e1d03b8 100644 --- a/test/arch/x86/unit/asm_test.py +++ b/test/arch/x86/unit/asm_test.py @@ -23,8 +23,8 @@ reg_and_id = dict(mn_x86.regs.all_regs_ids_byname) class Asm_Test(object): run_addr = 0x0 - def __init__(self): - self.myjit = Machine(self.arch_name).jitter() + def __init__(self, jitter_engine): + self.myjit = Machine(self.arch_name).jitter(jitter_engine) self.myjit.init_stack() self.myjit.jit.log_regs = False @@ -84,8 +84,8 @@ class Asm_Test_16(Asm_Test): arch_attrib = 16 ret_addr = 0x1337 - def __init__(self): - self.myjit = Machine(self.arch_name).jitter() + def __init__(self, jitter_engine): + self.myjit = Machine(self.arch_name).jitter(jitter_engine) self.myjit.stack_base = 0x1000 self.myjit.stack_size = 0x1000 self.myjit.init_stack() diff --git a/test/arch/x86/unit/mn_daa.py b/test/arch/x86/unit/mn_daa.py index 7aadf582..d07bf849 100644 --- a/test/arch/x86/unit/mn_daa.py +++ b/test/arch/x86/unit/mn_daa.py @@ -1,4 +1,6 @@ #! /usr/bin/env python +import sys + from asm_test import Asm_Test_32 @@ -73,4 +75,4 @@ array_al_end: if __name__ == "__main__": - [test()() for test in [Test_DAA]] + [test(*sys.argv[1:])() for test in [Test_DAA]] diff --git a/test/arch/x86/unit/mn_das.py b/test/arch/x86/unit/mn_das.py index 0828cafe..2d8102d9 100644 --- a/test/arch/x86/unit/mn_das.py +++ b/test/arch/x86/unit/mn_das.py @@ -1,4 +1,6 @@ #! /usr/bin/env python +import sys + from asm_test import Asm_Test_32 @@ -103,4 +105,4 @@ array_al_end: if __name__ == "__main__": - [test()() for test in [Test_DAS]] + [test(*sys.argv[1:])() for test in [Test_DAS]] diff --git a/test/arch/x86/unit/mn_float.py b/test/arch/x86/unit/mn_float.py index 81eb518b..d704de73 100644 --- a/test/arch/x86/unit/mn_float.py +++ b/test/arch/x86/unit/mn_float.py @@ -1,4 +1,6 @@ #! /usr/bin/env python +import sys + from asm_test import Asm_Test_32 @@ -19,4 +21,4 @@ class Test_FADD(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_FADD]] + [test(*sys.argv[1:])() for test in [Test_FADD]] diff --git a/test/arch/x86/unit/mn_int.py b/test/arch/x86/unit/mn_int.py index 0f4a5717..9d964220 100644 --- a/test/arch/x86/unit/mn_int.py +++ b/test/arch/x86/unit/mn_int.py @@ -1,4 +1,6 @@ #! /usr/bin/env python +import sys + from miasm2.jitter.csts import EXCEPT_INT_XX from asm_test import Asm_Test_32 @@ -15,8 +17,8 @@ class Test_INT(Asm_Test_32): jitter.cpu.set_exception(0) return True - def __init__(self): - super(Test_INT, self).__init__() + def __init__(self, jitter): + super(Test_INT, self).__init__(jitter) self.int_num = 0 self.myjit.add_exception_handler(EXCEPT_INT_XX, self.set_int_num) @@ -28,4 +30,4 @@ class Test_INT(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_INT]] + [test(*sys.argv[1:])() for test in [Test_INT]] diff --git a/test/arch/x86/unit/mn_pcmpeq.py b/test/arch/x86/unit/mn_pcmpeq.py index 06815e76..5d0a59c6 100644 --- a/test/arch/x86/unit/mn_pcmpeq.py +++ b/test/arch/x86/unit/mn_pcmpeq.py @@ -1,7 +1,8 @@ #! /usr/bin/env python -from asm_test import Asm_Test_32 import sys +from asm_test import Asm_Test_32 + class Test_PCMPEQB(Asm_Test_32): TXT = ''' main: @@ -61,4 +62,4 @@ class Test_PCMPEQD(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_PCMPEQB, Test_PCMPEQW, Test_PCMPEQD]] + [test(*sys.argv[1:])() for test in [Test_PCMPEQB, Test_PCMPEQW, Test_PCMPEQD]] diff --git a/test/arch/x86/unit/mn_pextr.py b/test/arch/x86/unit/mn_pextr.py index 0469eed7..696f077b 100644 --- a/test/arch/x86/unit/mn_pextr.py +++ b/test/arch/x86/unit/mn_pextr.py @@ -1,7 +1,8 @@ #! /usr/bin/env python -from asm_test import Asm_Test_32 import sys +from asm_test import Asm_Test_32 + class Test_PEXTRB(Asm_Test_32): TXT = ''' main: @@ -22,4 +23,4 @@ class Test_PEXTRB(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_PEXTRB]] + [test(*sys.argv[1:])() for test in [Test_PEXTRB]] diff --git a/test/arch/x86/unit/mn_pinsr.py b/test/arch/x86/unit/mn_pinsr.py index a10cd286..74120e5c 100644 --- a/test/arch/x86/unit/mn_pinsr.py +++ b/test/arch/x86/unit/mn_pinsr.py @@ -1,7 +1,8 @@ #! /usr/bin/env python -from asm_test import Asm_Test_32 import sys +from asm_test import Asm_Test_32 + class Test_PINSRB(Asm_Test_32): TXT = ''' main: @@ -22,4 +23,4 @@ class Test_PINSRB(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_PINSRB]] + [test(*sys.argv[1:])() for test in [Test_PINSRB]] diff --git a/test/arch/x86/unit/mn_pmaxu.py b/test/arch/x86/unit/mn_pmaxu.py index 50cbff94..527b966f 100644 --- a/test/arch/x86/unit/mn_pmaxu.py +++ b/test/arch/x86/unit/mn_pmaxu.py @@ -1,7 +1,8 @@ #! /usr/bin/env python -from asm_test import Asm_Test_32 import sys +from asm_test import Asm_Test_32 + class Test_PMAXU(Asm_Test_32): TXT = ''' main: @@ -22,4 +23,4 @@ class Test_PMAXU(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_PMAXU]] + [test(*sys.argv[1:])() for test in [Test_PMAXU]] diff --git a/test/arch/x86/unit/mn_pminu.py b/test/arch/x86/unit/mn_pminu.py index 27c9ad1e..d667df4e 100644 --- a/test/arch/x86/unit/mn_pminu.py +++ b/test/arch/x86/unit/mn_pminu.py @@ -1,7 +1,8 @@ #! /usr/bin/env python -from asm_test import Asm_Test_32 import sys +from asm_test import Asm_Test_32 + class Test_PMINU(Asm_Test_32): TXT = ''' main: @@ -22,4 +23,4 @@ class Test_PMINU(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_PMINU]] + [test(*sys.argv[1:])() for test in [Test_PMINU]] diff --git a/test/arch/x86/unit/mn_pmovmskb.py b/test/arch/x86/unit/mn_pmovmskb.py index 796e977c..b7e7b897 100644 --- a/test/arch/x86/unit/mn_pmovmskb.py +++ b/test/arch/x86/unit/mn_pmovmskb.py @@ -1,7 +1,8 @@ #! /usr/bin/env python -from asm_test import Asm_Test_32 import sys +from asm_test import Asm_Test_32 + class Test_PMOVMSKB(Asm_Test_32): TXT = ''' main: @@ -23,4 +24,4 @@ class Test_PMOVMSKB(Asm_Test_32): assert self.myjit.cpu.EAX == 0x00000015 if __name__ == "__main__": - [test()() for test in [Test_PMOVMSKB,]] + [test(*sys.argv[1:])() for test in [Test_PMOVMSKB,]] diff --git a/test/arch/x86/unit/mn_pshufb.py b/test/arch/x86/unit/mn_pshufb.py index 594b0870..5f4c7370 100644 --- a/test/arch/x86/unit/mn_pshufb.py +++ b/test/arch/x86/unit/mn_pshufb.py @@ -1,7 +1,8 @@ #! /usr/bin/env python -from asm_test import Asm_Test_32 import sys +from asm_test import Asm_Test_32 + class Test_PSHUFB(Asm_Test_32): TXT = ''' main: @@ -22,4 +23,4 @@ class Test_PSHUFB(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_PSHUFB]] + [test(*sys.argv[1:])() for test in [Test_PSHUFB]] diff --git a/test/arch/x86/unit/mn_psrl_psll.py b/test/arch/x86/unit/mn_psrl_psll.py index 79125612..44126b96 100644 --- a/test/arch/x86/unit/mn_psrl_psll.py +++ b/test/arch/x86/unit/mn_psrl_psll.py @@ -1,7 +1,8 @@ #! /usr/bin/env python -from asm_test import Asm_Test_32 import sys +from asm_test import Asm_Test_32 + class Test_PSRL(Asm_Test_32): TXT = ''' main: @@ -52,4 +53,4 @@ class Test_PSLL(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_PSRL, Test_PSLL]] + [test(*sys.argv[1:])() for test in [Test_PSRL, Test_PSLL]] diff --git a/test/arch/x86/unit/mn_punpck.py b/test/arch/x86/unit/mn_punpck.py index 8b655aa0..f6a4772e 100644 --- a/test/arch/x86/unit/mn_punpck.py +++ b/test/arch/x86/unit/mn_punpck.py @@ -1,7 +1,8 @@ #! /usr/bin/env python -from asm_test import Asm_Test_32 import sys +from asm_test import Asm_Test_32 + class Test_PUNPCKHBW(Asm_Test_32): TXT = ''' main: @@ -120,5 +121,5 @@ class Test_PUNPCKLDQ(Asm_Test_32): assert self.myjit.cpu.MM1 == 0xEEFF020155667788 if __name__ == "__main__": - [test()() for test in [Test_PUNPCKHBW, Test_PUNPCKHWD, Test_PUNPCKHDQ, - Test_PUNPCKLBW, Test_PUNPCKLWD, Test_PUNPCKLDQ,]] + [test(*sys.argv[1:])() for test in [Test_PUNPCKHBW, Test_PUNPCKHWD, Test_PUNPCKHDQ, + Test_PUNPCKLBW, Test_PUNPCKLWD, Test_PUNPCKLDQ,]] diff --git a/test/arch/x86/unit/mn_pushpop.py b/test/arch/x86/unit/mn_pushpop.py index d230a088..1a3f5517 100644 --- a/test/arch/x86/unit/mn_pushpop.py +++ b/test/arch/x86/unit/mn_pushpop.py @@ -1,4 +1,6 @@ #! /usr/bin/env python +import sys + from asm_test import Asm_Test_16, Asm_Test_32 from miasm2.core.utils import pck16, pck32 @@ -119,7 +121,7 @@ class Test_PUSHAD_16(Asm_Test_16): if __name__ == "__main__": - [test()() for test in [Test_PUSHA_16, Test_PUSHA_32, - Test_PUSHAD_16, Test_PUSHAD_32 - ] - ] + [test(*sys.argv[1:])() for test in [Test_PUSHA_16, Test_PUSHA_32, + Test_PUSHAD_16, Test_PUSHAD_32 + ] + ] diff --git a/test/arch/x86/unit/mn_stack.py b/test/arch/x86/unit/mn_stack.py index 6ae26d67..3bce3979 100644 --- a/test/arch/x86/unit/mn_stack.py +++ b/test/arch/x86/unit/mn_stack.py @@ -1,4 +1,6 @@ #! /usr/bin/env python +import sys + from asm_test import Asm_Test_32 @@ -57,4 +59,4 @@ BAD: if __name__ == "__main__": - [test()() for test in [Test_PUSHPOP]] + [test(*sys.argv[1:])() for test in [Test_PUSHPOP]] diff --git a/test/arch/x86/unit/mn_strings.py b/test/arch/x86/unit/mn_strings.py index f8055665..5e7269f7 100644 --- a/test/arch/x86/unit/mn_strings.py +++ b/test/arch/x86/unit/mn_strings.py @@ -1,4 +1,6 @@ #! /usr/bin/env python +import sys + from asm_test import Asm_Test_32 class Test_SCAS(Asm_Test_32): @@ -45,4 +47,4 @@ class Test_MOVS(Asm_Test_32): if __name__ == "__main__": - [test()() for test in [Test_SCAS, Test_MOVS]] + [test(*sys.argv[1:])() for test in [Test_SCAS, Test_MOVS]] diff --git a/test/test_all.py b/test/test_all.py index 53e8d513..35f081de 100644 --- a/test/test_all.py +++ b/test/test_all.py @@ -13,6 +13,7 @@ TAGS = {"regression": "REGRESSION", # Regression tests "example": "EXAMPLE", # Examples "long": "LONG", # Very time consumming tests "llvm": "LLVM", # LLVM dependency is required + "tcc": "TCC", # TCC dependency is required "z3": "Z3", # Z3 dependecy is needed "qemu": "QEMU", # QEMU tests (several tests) } @@ -34,14 +35,25 @@ class RegressionTest(Test): def get_sample(cls, sample_name): "Return the relative path of @sample_name" return os.path.join(cls.sample_dir, sample_name) - - ## Architecture testset += RegressionTest(["x86/arch.py"], base_dir="arch", products=["x86_speed_reg_test.bin", "regression_test16_ia32.bin", "regression_test32_ia32.bin", "regression_test64_ia32.bin"]) + + + +### ArchUnit regression tests +class ArchUnitTest(RegressionTest): + """Test against arch unit regression tests""" + + jitter_engines = ["tcc", "llvm", "gcc"] + + def __init__(self, script, jitter ,*args, **kwargs): + super(ArchUnitTest, self).__init__([script, jitter], *args, **kwargs) + + for script in ["x86/sem.py", "x86/unit/mn_strings.py", "x86/unit/mn_float.py", @@ -69,7 +81,9 @@ for script in ["x86/sem.py", "mips32/arch.py", "mips32/unit/mn_bcc.py", ]: - testset += RegressionTest([script], base_dir="arch") + for jitter in ArchUnitTest.jitter_engines: + tags = [TAGS[jitter]] if jitter in TAGS else [] + testset += ArchUnitTest(script, jitter, base_dir="arch", tags=tags) ### QEMU regression tests class QEMUTest(RegressionTest): @@ -79,6 +93,7 @@ class QEMUTest(RegressionTest): SCRIPT_NAME = "testqemu.py" SAMPLE_NAME = "test-i386" EXPECTED_PATH = "expected" + jitter_engines = ["tcc", "llvm", "python", "gcc"] def __init__(self, name, jitter, *args, **kwargs): super(QEMUTest, self).__init__([self.SCRIPT_NAME], *args, **kwargs) @@ -95,52 +110,53 @@ class QEMUTest(RegressionTest): # Test name -> supported jitter engines -QEMU_TESTS = { +QEMU_TESTS = [ # Operations - "btr": ("tcc", "python"), - "bts": ("tcc", "python"), - "bt": ("tcc", "python"), - "shrd": ("tcc", "python"), - "shld": ("tcc", "python"), - "rcl": ("tcc", "python"), - "rcr": ("tcc", "python"), - "ror": ("tcc", "python"), - "rol": ("tcc", "python"), - "sar": ("tcc", "python"), - "shr": ("tcc", "python"), - "shl": ("tcc", "python"), - "not": ("tcc", "python"), - "neg": ("tcc", "python"), - "dec": ("tcc", "python"), - "inc": ("tcc", "python"), - "sbb": ("tcc", "python"), - "adc": ("tcc", "python"), - "cmp": ("tcc", "python"), - "or": ("tcc", "python"), - "and": ("tcc", "python"), - "xor": ("tcc", "python"), - "sub": ("tcc", "python"), - "add": ("tcc", "python"), + "btr", + "bts", + "bt", + "shrd", + "shld", + "rcl", + "rcr", + "ror", + "rol", + "sar", + "shr", + "shl", + "not", + "neg", + "dec", + "inc", + "sbb", + "adc", + "cmp", + "or", + "and", + "xor", + "sub", + "add", # Specifics - "bsx": ("tcc", "python"), - "mul": ("tcc", "python"), - "jcc": ("tcc", "python"), - "loop": ("tcc", "python"), - "lea": ("tcc", "python"), - "self_modifying_code": ("tcc", "python"), - "conv": ("tcc", "python"), - "bcd": ("tcc", "python"), - "xchg": ("tcc", "python"), - "string": ("tcc", "python"), - "misc": ("tcc", "python"), + "bsx", + "mul", + "jcc", + "loop", + "lea", + "self_modifying_code", + "conv", + "bcd", + "xchg", + "string", + "misc", # Unsupported # "floats", "segs", "code16", "exceptions", "single_step" -} +] -for test_name, jitters in QEMU_TESTS.iteritems(): - for jitter_engine in jitters: - testset += QEMUTest(test_name, jitter_engine) +for test_name in QEMU_TESTS: + for jitter in QEMUTest.jitter_engines: + tags = [TAGS[jitter]] if jitter in TAGS else [] + testset += QEMUTest(test_name, jitter, tags=tags) ## Semantic @@ -183,7 +199,7 @@ class SemanticTestExec(RegressionTest): input_filename, "-a", hex(address)] self.products = [] - + self.tags.append(TAGS["tcc"]) test_x86_64_mul_div = SemanticTestAsm("x86_64", "PE", ["mul_div"]) @@ -232,7 +248,7 @@ testset += RegressionTest(["smt2.py"], base_dir="ir/translators", ## OS_DEP for script in ["win_api_x86_32.py", ]: - testset += RegressionTest([script], base_dir="os_dep") + testset += RegressionTest([script], base_dir="os_dep", tags=[TAGS['tcc']]) ## Analysis testset += RegressionTest(["depgraph.py"], base_dir="analysis", @@ -305,7 +321,7 @@ for i, test_args in enumerate(test_args): ## Jitter for script in ["jitload.py", ]: - testset += RegressionTest([script], base_dir="jitter") + testset += RegressionTest([script], base_dir="jitter", tags=[TAGS["tcc"]]) # Examples @@ -535,13 +551,14 @@ class ExampleJitter(Example): - script path begins with "jitter/" """ example_dir = "jitter" - jitter_engines = ["tcc", "llvm", "python"] + jitter_engines = ["tcc", "llvm", "python", "gcc"] for jitter in ExampleJitter.jitter_engines: # Take 5 min on a Core i5 tags = {"python": [TAGS["long"]], "llvm": [TAGS["llvm"]], + "tcc": [TAGS["tcc"]], } testset += ExampleJitter(["unpack_upx.py", Example.get_sample("box_upx.exe")] + @@ -567,7 +584,7 @@ for script, dep in [(["x86_32.py", Example.get_sample("x86_32_sc.bin")], []), [test_box[name]]) for name in test_box_names]: for jitter in ExampleJitter.jitter_engines: - tags = [TAGS["llvm"]] if jitter == "llvm" else [] + tags = [TAGS[jitter]] if jitter in TAGS else [] testset += ExampleJitter(script + ["--jitter", jitter], depends=dep, tags=tags) @@ -650,6 +667,13 @@ By default, no tag is omitted." % ", ".join(TAGS.keys()), default="") except ImportError: llvm = False + # Handle tcc modularity + tcc = True + try: + from miasm2.jitter import Jittcc + except ImportError: + tcc = False + # TODO XXX: fix llvm jitter (deactivated for the moment) llvm = False @@ -661,6 +685,14 @@ By default, no tag is omitted." % ", ".join(TAGS.keys()), default="") if TAGS["llvm"] not in exclude_tags: exclude_tags.append(TAGS["llvm"]) + if tcc is False: + print "%(red)s[TCC]%(end)s Python" % cosmetics.colors + \ + "'libtcc' module is required for tcc tests" + + # Remove tcc tests + if TAGS["tcc"] not in exclude_tags: + exclude_tags.append(TAGS["tcc"]) + # Handle Z3 dependency try: import z3 |