diff options
| -rw-r--r-- | .travis.yml | 6 | ||||
| -rw-r--r-- | miasm2/analysis/sandbox.py | 2 | ||||
| -rw-r--r-- | miasm2/jitter/Jittcc.c | 319 | ||||
| -rw-r--r-- | miasm2/jitter/jitcore_tcc.py | 104 | ||||
| -rw-r--r-- | miasm2/jitter/jitload.py | 5 | ||||
| -rwxr-xr-x | setup.py | 88 | ||||
| -rw-r--r-- | test/arch/x86/unit/mn_cdq.py | 2 | ||||
| -rwxr-xr-x | test/test_all.py | 27 |
8 files changed, 10 insertions, 543 deletions
diff --git a/.travis.yml b/.travis.yml index 2a28f290..49d7eef2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,12 +22,6 @@ before_script: - "python /usr/lib/python2.7/dist-packages/virtualenv.py virtualenv;" - "cd virtualenv;" - "source bin/activate;" -# install tcc -- "git clone http://repo.or.cz/tinycc.git tinycc && cd tinycc && git checkout d5e22108a0dc48899e44a158f91d5b3215eb7fe6" -- "./configure --disable-static --tccdir=./ --libdir=$(pwd) --includedir=$(pwd)" -- "make && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd);cd ..;" -- "cp tinycc/libtcc.h include" -- "cp tinycc/libtcc.so.1.0 tinycc/libtcc.so" # install llvmlite - "pip install enum34" - "pip install llvmlite" diff --git a/miasm2/analysis/sandbox.py b/miasm2/analysis/sandbox.py index f29d1776..e77b1669 100644 --- a/miasm2/analysis/sandbox.py +++ b/miasm2/analysis/sandbox.py @@ -86,7 +86,7 @@ class Sandbox(object): parser.add_argument('-g', "--gdbserver", type=int, help="Listen on port @port") parser.add_argument("-j", "--jitter", - help="Jitter engine. Possible values are: gcc (default), tcc, llvm, python", + help="Jitter engine. Possible values are: gcc (default), llvm, python", default="gcc") parser.add_argument( '-q', "--quiet-function-calls", action="store_true", diff --git a/miasm2/jitter/Jittcc.c b/miasm2/jitter/Jittcc.c deleted file mode 100644 index 61a6cda4..00000000 --- a/miasm2/jitter/Jittcc.c +++ /dev/null @@ -1,319 +0,0 @@ -/* -** Copyright (C) 2011 EADS France, Fabrice Desclaux <fabrice.desclaux@eads.net> -** -** This program is free software; you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation; either version 2 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License along -** with this program; if not, write to the Free Software Foundation, Inc., -** 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ -#include <Python.h> - -#include <inttypes.h> -#include <libtcc.h> - -#include <stdint.h> - - - -int include_array_count = 0; -char **include_array = NULL; - - -int lib_array_count = 0; -char **lib_array = NULL; - -//char *libcodenat_path = NULL; - - -TCCState * tcc_init_state(void) -{ - int i; - TCCState *tcc_state = NULL; - tcc_state = tcc_new(); - if (!tcc_state) { - fprintf(stderr, "Impossible de creer un contexte TCC\n"); - exit(EXIT_FAILURE); - } - tcc_set_output_type(tcc_state, TCC_OUTPUT_MEMORY); - - //tcc_add_file(tcc_state, libcodenat_path); - for (i=0;i<lib_array_count; i++){ - tcc_add_file(tcc_state, lib_array[i]); - } - - for (i=0;i<include_array_count; i++){ - tcc_add_include_path(tcc_state, include_array[i]); - } - return tcc_state; -} - - -PyObject* tcc_end(PyObject* self, PyObject* args) -{ - unsigned long long tmp = 0; - - if (!PyArg_ParseTuple(args, "K", &tmp)) - return NULL; - - tcc_delete((TCCState *) (intptr_t) tmp); - - Py_INCREF(Py_None); - return Py_None; -} - -PyObject* tcc_set_emul_lib_path(PyObject* self, PyObject* args) -{ - char* include_arg; - char* lib_arg; - - char* str1, * str2, * init_ptr; - if (!PyArg_ParseTuple(args, "ss", - &include_arg, - &lib_arg)) - return NULL; - - init_ptr = str2 = strdup(include_arg); - while (str2){ - str1 = strsep(&str2, ";"); - if (str1){ - include_array_count ++; - include_array = realloc(include_array, - include_array_count * sizeof(char*)); - if (include_array == NULL) { - fprintf(stderr, "cannot realloc char* include_array\n"); - exit(EXIT_FAILURE); - } - include_array[include_array_count-1] = strdup(str1); - // fprintf(stderr, "adding include file: %s\n", str1); - } - } - if (init_ptr != NULL) - free(init_ptr); - - init_ptr = str2 = strdup(lib_arg); - while (str2){ - str1 = strsep(&str2, ";"); - if (str1){ - lib_array_count ++; - lib_array = realloc(lib_array, - lib_array_count * sizeof(char*)); - if (lib_array == NULL) { - fprintf(stderr, "cannot realloc char* lib_array\n"); - exit(EXIT_FAILURE); - } - lib_array[lib_array_count-1] = strdup(str1); - // fprintf(stderr, "adding lib file: %s\n", str1); - } - } - if (init_ptr != NULL) - free(init_ptr); - - - /* - libcodenat_path = (char*)malloc(strlen(libcodenat_path_arg)+1); - strcpy(libcodenat_path, libcodenat_path_arg); - */ - Py_INCREF(Py_None); - - - return Py_None; -} - - -typedef struct { - uint8_t is_local; - uint64_t address; -} block_id; - -typedef int (*jitted_func)(block_id*, PyObject*); - - -PyObject* tcc_exec_bloc(PyObject* self, PyObject* args) -{ - jitted_func func; - PyObject* jitcpu; - PyObject* func_py; - PyObject* lbl2ptr; - PyObject* breakpoints; - PyObject* retaddr = NULL; - int status; - block_id BlockDst; - uint64_t max_exec_per_call = 0; - uint64_t cpt; - int do_cpt; - - if (!PyArg_ParseTuple(args, "OOOO|K", - &retaddr, &jitcpu, &lbl2ptr, &breakpoints, - &max_exec_per_call)) - return NULL; - - /* The loop will decref retaddr always once */ - Py_INCREF(retaddr); - - if (max_exec_per_call == 0) { - do_cpt = 0; - cpt = 1; - } else { - do_cpt = 1; - cpt = max_exec_per_call; - } - - - for (;;) { - if (cpt == 0) - return retaddr; - if (do_cpt) - cpt --; - // Init - BlockDst.is_local = 0; - BlockDst.address = 0; - - // Get the expected jitted function address - func_py = PyDict_GetItem(lbl2ptr, retaddr); - if (func_py) - func = (jitted_func) PyLong_AsVoidPtr((PyObject*) func_py); - else { - if (BlockDst.is_local == 1) { - fprintf(stderr, "return on local label!\n"); - exit(EXIT_FAILURE); - } - // retaddr is not jitted yet - return retaddr; - } - - // Execute it - status = func(&BlockDst, jitcpu); - Py_DECREF(retaddr); - retaddr = PyLong_FromUnsignedLongLong(BlockDst.address); - - // Check exception - if (status) - return retaddr; - - // Check breakpoint - if (PyDict_Contains(breakpoints, retaddr)) - return retaddr; - } -} - -PyObject* tcc_compil(PyObject* self, PyObject* args) -{ - char* func_name; - char* func_code; - int (*entry)(void); - TCCState *tcc_state = NULL; - PyObject* ret; - - tcc_state = tcc_init_state(); - - if (!PyArg_ParseTuple(args, "ss", &func_name, &func_code)) - return NULL; - - if (tcc_compile_string(tcc_state, func_code) != 0) { - fprintf(stderr, "Error compiling !\n"); - fprintf(stderr, "%s\n", func_code); - exit(EXIT_FAILURE); - } - /* XXX configure tinycc install with --disable-static */ - if (tcc_relocate(tcc_state, TCC_RELOCATE_AUTO) < 0) { - fprintf(stderr, "TCC relocate error\n"); - exit(EXIT_FAILURE); - } - entry = tcc_get_symbol(tcc_state, func_name); - if (!entry){ - fprintf(stderr, "Error getting symbol %s!\n", func_name); - fprintf(stderr, "%s\n", func_name); - exit(EXIT_FAILURE); - } - - ret = PyTuple_New(2); - if (ret == NULL) { - fprintf(stderr, "Error alloc %s!\n", func_name); - fprintf(stderr, "%s\n", func_name); - exit(EXIT_FAILURE); - } - - PyTuple_SetItem(ret, 0, PyLong_FromUnsignedLongLong((intptr_t) tcc_state)); - PyTuple_SetItem(ret, 1, PyLong_FromUnsignedLongLong((intptr_t) entry)); - - return ret; - -} - - - -PyObject* tcc_loop_exec(PyObject* self, PyObject* args) -{ - //PyObject* (*func)(void*, void*); - uint64_t* vm; - uint64_t* cpu; - PyObject* ret; - PyObject* func; - PyObject* pArgs; - - - if (!PyArg_ParseTuple(args, "OKK", &func, &cpu, &vm)) - return NULL; - - while (1) { - if (!PyCallable_Check (func)) { - fprintf(stderr, "function not callable!\n"); - exit(EXIT_FAILURE); - } - - pArgs = PyTuple_New(2); - PyTuple_SetItem(pArgs, 0, PyLong_FromUnsignedLongLong((intptr_t)cpu)); - PyTuple_SetItem(pArgs, 1, PyLong_FromUnsignedLongLong((intptr_t)vm)); - ret = PyObject_CallObject(func, pArgs); - Py_DECREF(2); - - if (ret == Py_None) { - Py_INCREF(Py_None); - return Py_None; - } - func = ret; - } - - return ret; -} - - - -static PyObject *TccError; - - -static PyMethodDef TccMethods[] = { - {"tcc_set_emul_lib_path", tcc_set_emul_lib_path, METH_VARARGS, - "init tcc path"}, - {"tcc_exec_bloc", tcc_exec_bloc, METH_VARARGS, - "tcc exec bloc"}, - {"tcc_compil", tcc_compil, METH_VARARGS, - "tcc compil"}, - {"tcc_end", tcc_end, METH_VARARGS, - "tcc end"}, - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - -PyMODINIT_FUNC -initJittcc(void) -{ - PyObject *m; - - m = Py_InitModule("Jittcc", TccMethods); - if (m == NULL) - return; - - TccError = PyErr_NewException("tcc.error", NULL, NULL); - Py_INCREF(TccError); - PyModule_AddObject(m, "error", TccError); -} - diff --git a/miasm2/jitter/jitcore_tcc.py b/miasm2/jitter/jitcore_tcc.py deleted file mode 100644 index 28288400..00000000 --- a/miasm2/jitter/jitcore_tcc.py +++ /dev/null @@ -1,104 +0,0 @@ -#-*- coding:utf-8 -*- -import os -import tempfile -from subprocess import Popen, PIPE - -from miasm2.jitter import Jittcc -from miasm2.jitter.jitcore_cc_base import JitCore_Cc_Base, gen_core - - -class JitCore_Tcc(JitCore_Cc_Base): - - "JiT management, using LibTCC as backend" - - def __init__(self, ir_arch, bs=None): - super(JitCore_Tcc, self).__init__(ir_arch, bs) - self.exec_wrapper = Jittcc.tcc_exec_bloc - - def deleteCB(self, offset): - "Free the TCCState corresponding to @offset" - if offset in self.states: - Jittcc.tcc_end(self.states[offset]) - del self.states[offset] - - def load(self): - super(JitCore_Tcc, self).load() - libs = ';'.join(self.libs) - jittcc_path = Jittcc.__file__ - include_dir = os.path.dirname(jittcc_path) - include_dir += ";" + os.path.join(include_dir, "arch") - - # XXX HACK - # As debian/ubuntu have moved some include files using arch directory, - # TCC doesn't know them, so we get the info from CC - # For example /usr/include/x86_64-linux-gnu which contains limits.h - p = Popen(["cc", "-Wp,-v", "-E", "-"], - stdout=PIPE, stderr=PIPE, stdin=PIPE) - p.stdin.close() - include_files = p.stderr.read().split('\n') - include_files = [x[1:] - for x in include_files if x.startswith(' /usr/include')] - include_files += self.include_files - include_files = ";".join(include_files) - Jittcc.tcc_set_emul_lib_path(include_files, libs) - - def __del__(self): - for tcc_state in self.states.values(): - Jittcc.tcc_end(tcc_state) - - def jit_tcc_compil(self, func_name, func_code): - return Jittcc.tcc_compil(func_name, func_code) - - def compil_code(self, block, func_code): - """ - Compil the C code of @func_code from @block - @block: original asm_block - @func_code: C code of the block - """ - label = block.label - self.jitcount += 1 - tcc_state, mcode = self.jit_tcc_compil(self.label2fname(label), func_code) - self.lbl2jitbloc[label.offset] = mcode - self.states[label.offset] = tcc_state - - def add_bloc(self, block): - """Add a bloc to JiT and JiT it. - @block: block to jit - """ - block_hash = self.hash_block(block) - fname_out = os.path.join(self.tempdir, "%s.c" % block_hash) - - if os.access(fname_out, os.R_OK): - func_code = open(fname_out, "rb").read() - else: - func_code = self.gen_c_code(block.label, block) - - # Create unique C file - fdesc, fname_tmp = tempfile.mkstemp(suffix=".c") - os.write(fdesc, func_code) - os.close(fdesc) - os.rename(fname_tmp, fname_out) - - self.compil_code(block, func_code) - - @staticmethod - def gen_C_source(ir_arch, func_code): - c_source = "" - c_source += "\n".join(func_code) - - c_source = gen_core(ir_arch.arch, ir_arch.attrib) + c_source - - c_source = """ - #ifdef __x86_64__ - #ifndef __LP64__ - /* - for ubuntu ?!? XXX TODO - /!\ force 64 bit system using 64 bits libc - change this to __ILP32__ to do so. - */ - #define __LP64__ - #endif - #endif - """ + "#include <Python.h>\n" + c_source - - return c_source diff --git a/miasm2/jitter/jitload.py b/miasm2/jitter/jitload.py index b2e16c1d..db486b4f 100644 --- a/miasm2/jitter/jitload.py +++ b/miasm2/jitter/jitload.py @@ -171,7 +171,6 @@ class jitter(object): @ir_arch: ir instance for this architecture @jit_type: JiT backend to use. Available options are: - "gcc" - - "tcc" - "llvm" - "python" """ @@ -210,9 +209,7 @@ class jitter(object): self.symbexec.reset_regs() try: - if jit_type == "tcc": - from miasm2.jitter.jitcore_tcc import JitCore_Tcc as JitCore - elif jit_type == "llvm": + if jit_type == "llvm": from miasm2.jitter.jitcore_llvm import JitCore_LLVM as JitCore elif jit_type == "python": from miasm2.jitter.jitcore_python import JitCore_Python as JitCore diff --git a/setup.py b/setup.py index ff4e98d8..0a1acc00 100755 --- a/setup.py +++ b/setup.py @@ -25,48 +25,6 @@ def buil_all(): 'miasm2/jitter/arch', 'miasm2/jitter/loader', ] - ext_modules_no_tcc = [ - Extension("miasm2.jitter.VmMngr", - ["miasm2/jitter/vm_mngr.c", - "miasm2/jitter/vm_mngr_py.c"]), - Extension("miasm2.jitter.arch.JitCore_x86", - ["miasm2/jitter/JitCore.c", - "miasm2/jitter/vm_mngr.c", - "miasm2/jitter/op_semantics.c", - "miasm2/jitter/arch/JitCore_x86.c"]), - Extension("miasm2.jitter.arch.JitCore_arm", - ["miasm2/jitter/JitCore.c", - "miasm2/jitter/vm_mngr.c", - "miasm2/jitter/op_semantics.c", - "miasm2/jitter/arch/JitCore_arm.c"]), - Extension("miasm2.jitter.arch.JitCore_aarch64", - ["miasm2/jitter/JitCore.c", - "miasm2/jitter/vm_mngr.c", - "miasm2/jitter/op_semantics.c", - "miasm2/jitter/arch/JitCore_aarch64.c"]), - Extension("miasm2.jitter.arch.JitCore_msp430", - ["miasm2/jitter/JitCore.c", - "miasm2/jitter/vm_mngr.c", - "miasm2/jitter/op_semantics.c", - "miasm2/jitter/arch/JitCore_msp430.c"]), - Extension("miasm2.jitter.arch.JitCore_mips32", - ["miasm2/jitter/JitCore.c", - "miasm2/jitter/vm_mngr.c", - "miasm2/jitter/op_semantics.c", - "miasm2/jitter/arch/JitCore_mips32.c"]), - Extension("miasm2.jitter.arch.JitCore_ppc32", - ["miasm2/jitter/JitCore.c", - "miasm2/jitter/vm_mngr.c", - "miasm2/jitter/op_semantics.c", - "miasm2/jitter/arch/JitCore_ppc32.c"], - depends=["miasm2/jitter/arch/JitCore_ppc32.h", - "miasm2/jitter/arch/JitCore_ppc32_regs.h"]), - Extension("miasm2.jitter.Jitgcc", - ["miasm2/jitter/Jitgcc.c"]), - Extension("miasm2.jitter.Jitllvm", - ["miasm2/jitter/Jitllvm.c"]), - ] - ext_modules_all = [ Extension("miasm2.jitter.VmMngr", ["miasm2/jitter/vm_mngr.c", @@ -107,15 +65,12 @@ def buil_all(): ["miasm2/jitter/Jitllvm.c"]), Extension("miasm2.jitter.Jitgcc", ["miasm2/jitter/Jitgcc.c"]), - Extension("miasm2.jitter.Jittcc", - ["miasm2/jitter/Jittcc.c"], - libraries=["tcc"]) ] print 'building' build_ok = False for name, ext_modules in [('all', ext_modules_all), - ('notcc', ext_modules_no_tcc)]: + ]: print 'build with', repr(name) try: s = setup( @@ -141,14 +96,6 @@ def buil_all(): if not build_ok: raise ValueError('Unable to build Miasm!') print 'build', name - if name == 'notcc': - print - print "*"*80 - print "Warning: TCC is not properly installed," - print "Miasm will be installed without TCC Jitter" - print "Etheir install TCC or use LLVM jitter" - print "*"*80 - print # we copy libraries from build dir to current miasm directory build_base = None if 'build' in s.command_options: @@ -160,35 +107,4 @@ def buil_all(): build_base = os.path.join('build','lib' + plat_specifier) print build_base -def buil_no_tcc(): - setup( - name = 'Miasm', - version = '2.0', - packages=['miasm2', 'miasm2/tools', - 'miasm2/expression', 'miasm2/graph', 'miasm2/arch', - 'miasm2/core', 'miasm2/tools/emul_lib' ], - package_data = {'miasm2':['tools/emul_lib/*.h']}, - # data_files = [('toto', ['miasm2/tools/emul_lib/queue.h'])], - # Metadata - author = 'Fabrice Desclaux', - author_email = 'serpilliere@droid-corp.org', - description = 'Machine code manipulation library', - license = 'GPLv2', - # keywords = '', - # url = '', - ) - - -def try_build(): - buil_all() - """ - try: - buil_all() - return - except: - print "WARNING cannot build with libtcc!, trying without it" - print "Miasm will not be able to emulate code" - buil_no_tcc() - """ - -try_build() +buil_all() diff --git a/test/arch/x86/unit/mn_cdq.py b/test/arch/x86/unit/mn_cdq.py index f4e4d6e7..b6abc781 100644 --- a/test/arch/x86/unit/mn_cdq.py +++ b/test/arch/x86/unit/mn_cdq.py @@ -433,7 +433,7 @@ if __name__ == "__main__": Test_CDQE_64, Test_CDQE_64_signed, ] - if sys.argv[1] not in ["gcc", "tcc"]: + if sys.argv[1] not in ["gcc"]: # TODO XXX CQO use 128 bit not supported in gcc yet! tests += [ Test_CQO_64, diff --git a/test/test_all.py b/test/test_all.py index fa27b787..52873f4b 100755 --- a/test/test_all.py +++ b/test/test_all.py @@ -15,7 +15,6 @@ TAGS = {"regression": "REGRESSION", # Regression tests "example": "EXAMPLE", # Examples "long": "LONG", # Very time consumming tests "llvm": "LLVM", # LLVM dependency is required - "tcc": "TCC", # TCC dependency is required "gcc": "GCC", # GCC based tests "z3": "Z3", # Z3 dependecy is needed "qemu": "QEMU", # QEMU tests (several tests) @@ -52,7 +51,7 @@ testset += RegressionTest(["x86/arch.py"], base_dir="arch", class ArchUnitTest(RegressionTest): """Test against arch unit regression tests""" - jitter_engines = ["tcc", "llvm", "gcc", "python"] + jitter_engines = ["llvm", "gcc", "python"] def __init__(self, script, jitter ,*args, **kwargs): super(ArchUnitTest, self).__init__([script, jitter], *args, **kwargs) @@ -60,7 +59,7 @@ class ArchUnitTest(RegressionTest): # script -> blacklisted jitter blacklist = { "x86/unit/mn_float.py": ["python", "llvm"], - "x86/unit/mn_div.py": ["tcc", "gcc"], + "x86/unit/mn_div.py": ["gcc"], } for script in ["x86/sem.py", "x86/unit/mn_strings.py", @@ -107,7 +106,7 @@ class QEMUTest(RegressionTest): SCRIPT_NAME = "testqemu.py" SAMPLE_NAME = "test-i386" EXPECTED_PATH = "expected" - jitter_engines = ["tcc", "llvm", "python", "gcc"] + jitter_engines = ["llvm", "python", "gcc"] def __init__(self, name, jitter, *args, **kwargs): super(QEMUTest, self).__init__([self.SCRIPT_NAME], *args, **kwargs) @@ -655,7 +654,7 @@ class ExampleJitter(Example): - script path begins with "jitter/" """ example_dir = "jitter" - jitter_engines = ["tcc", "llvm", "python", "gcc"] + jitter_engines = ["llvm", "python", "gcc"] class ExampleJitterNoPython(ExampleJitter): @@ -663,14 +662,13 @@ class ExampleJitterNoPython(ExampleJitter): - script path begins with "jitter/" Run jitting script without python support """ - jitter_engines = ["tcc", "llvm", "gcc"] + jitter_engines = ["llvm", "gcc"] for jitter in ExampleJitter.jitter_engines: # Take 5 min on a Core i5 tags = {"python": [TAGS["long"]], "llvm": [TAGS["llvm"]], - "tcc": [TAGS["tcc"]], "gcc": [TAGS["gcc"]], } testset += ExampleJitter(["unpack_upx.py", @@ -804,13 +802,6 @@ By default, all tag are considered." % ", ".join(TAGS.keys()), default="") except ImportError: llvm = False - # Handle tcc modularity - tcc = True - try: - from miasm2.jitter import Jittcc - except ImportError: - tcc = False - if llvm is False: print "%(red)s[LLVM]%(end)s Python" % cosmetics.colors + \ "'llvmlite' module is required for llvm tests" @@ -819,14 +810,6 @@ By default, all tag are considered." % ", ".join(TAGS.keys()), default="") if TAGS["llvm"] not in exclude_tags: exclude_tags.append(TAGS["llvm"]) - if tcc is False: - print "%(red)s[TCC]%(end)s Python" % cosmetics.colors + \ - "'libtcc' module is required for tcc tests" - - # Remove tcc tests - if TAGS["tcc"] not in exclude_tags: - exclude_tags.append(TAGS["tcc"]) - # Handle Z3 dependency try: import z3 |