Merge pull request #367 from serpilliere/jitter_cache_block

Jitter cache block

7 files changed, 157 insertions, 72 deletions

diff --git a/miasm2/ir/ir2C.py b/miasm2/ir/ir2C.py
index d888f586..ebc61e27 100644
--- a/miasm2/ir/ir2C.py
+++ b/miasm2/ir/ir2C.py
@@ -28,12 +28,12 @@ for size in [8, 16, 32, 64]:
 def init_arch_C(arch):
     arch.id2Cid = {}
     for x in arch.regs.all_regs_ids + prefetch_id:
-        arch.id2Cid[x] = m2_expr.ExprId('((vm_cpu_t*)jitcpu->cpu)->' + str(x), x.size)
+        arch.id2Cid[x] = m2_expr.ExprId('mycpu->' + str(x), x.size)
 
     arch.id2newCid = {}
 
     for x in arch.regs.all_regs_ids + prefetch_id:
-        arch.id2newCid[x] = m2_expr.ExprId('((vm_cpu_t*)jitcpu->cpu)->%s_new' % x, x.size)
+        arch.id2newCid[x] = m2_expr.ExprId('mycpu->%s_new' % x, x.size)
 
 
 def patch_c_id(arch, e):
@@ -407,6 +407,8 @@ def irblocs2C(ir_arch, resolvers, label, irblocs,
             l.index -= lbl_index_min
 
     out.append("void* local_labels[] = {%s};"%(', '.join(["&&%s"%l.name for l in lbls_local])))
+    out.append("vm_cpu_t* mycpu = (vm_cpu_t*)jitcpu->cpu;")
+
 
     out.append("goto %s;" % label.name)
     bloc_labels = [x.label for x in irblocs]
diff --git a/miasm2/jitter/JitCore.c b/miasm2/jitter/JitCore.c
index ac8a0624..a6d29a72 100644
--- a/miasm2/jitter/JitCore.c
+++ b/miasm2/jitter/JitCore.c
@@ -54,16 +54,6 @@ PyObject * JitCpu_set_jitter(JitCpu *self, PyObject *value, void *closure)
 	return 0;
 }
 
-
-void Resolve_dst(block_id* b, uint64_t addr, uint64_t is_local)
-{
-	b->address = addr;
-	b->is_local = is_local;
-}
-
-
-
-
 uint8_t __attribute__((weak)) MEM_LOOKUP_08(JitCpu* jitcpu, uint64_t addr)
 {
 	return vm_MEM_LOOKUP_08(&((VmMngr*)jitcpu->pyvm)->vm_mngr, addr);
diff --git a/miasm2/jitter/JitCore.h b/miasm2/jitter/JitCore.h
index c8400701..bae5a417 100644
--- a/miasm2/jitter/JitCore.h
+++ b/miasm2/jitter/JitCore.h
@@ -113,6 +113,8 @@ PyObject * JitCpu_get_jitter(JitCpu *self, void *closure);
 PyObject * JitCpu_set_jitter(JitCpu *self, PyObject *value, void *closure);
 void Resolve_dst(block_id* BlockDst, uint64_t addr, uint64_t is_local);
 
+#define Resolve_dst(b, arg_addr, arg_is_local) do {(b)->address = (arg_addr); (b)->is_local = (arg_is_local);} while(0)
+
 
 
 uint8_t MEM_LOOKUP_08(JitCpu* jitcpu, uint64_t addr);
diff --git a/miasm2/jitter/jitcore_gcc.py b/miasm2/jitter/jitcore_gcc.py
index c703c661..db0b7880 100644
--- a/miasm2/jitter/jitcore_gcc.py
+++ b/miasm2/jitter/jitcore_gcc.py
@@ -92,10 +92,53 @@ class JitCore_Gcc(jitcore.JitCore):
         self.include_files = include_files
         self.libs = libs
 
-    def jit_gcc_compil(self, f_name, func_code):
-        func_hash = md5(func_code).hexdigest()
-        fname_out = os.path.join(self.tempdir, "%s.so" % func_hash)
+    def label2fname(self, label):
+        """
+        Generate function name from @label
+        @label: asm_label instance
+        """
+        return "block_%s" % label.name
+
+    def load_code(self, label, fname_so):
+        f_name = self.label2fname(label)
+        lib = ctypes.cdll.LoadLibrary(fname_so)
+        func = getattr(lib, f_name)
+        addr = ctypes.cast(func, ctypes.c_void_p).value
+        self.lbl2jitbloc[label.offset] = addr
+        self.gcc_states[label.offset] = None
+
+    def gen_c_code(self, label, irblocks):
+        """
+        Return the C code corresponding to the @irblocks
+        @label: asm_label of the block to jit
+        @irblocks: list of irblocks
+        """
+        f_name = self.label2fname(label)
+        f_declaration = 'int %s(block_id * BlockDst, JitCpu* jitcpu)' % f_name
+        out = irblocs2C(self.ir_arch, self.resolver, label, irblocks,
+                        gen_exception_code=True,
+                        log_mn=self.log_mn,
+                        log_regs=self.log_regs)
+        out = [f_declaration + '{'] + out + ['}\n']
+        c_code = out
+
+        return gen_C_source(self.ir_arch, c_code)
+
+    def add_bloc(self, block):
+        """Add a bloc to JiT and JiT it.
+        @block: block to jit
+        """
+        block_raw = "".join(line.b for line in block.lines)
+        block_hash = md5("%X_%s_%s_%s" % (block.label.offset,
+                                          self.log_mn,
+                                          self.log_regs,
+                                          block_raw)).hexdigest()
+        fname_out = os.path.join(self.tempdir, "%s.so" % block_hash)
+
         if not os.access(fname_out, os.R_OK | os.X_OK):
+            irblocks = self.ir_arch.add_bloc(block, gen_pc_updt=True)
+            func_code = self.gen_c_code(block.label, irblocks)
+
             # Create unique C file
             fdesc, fname_in = tempfile.mkstemp(suffix=".c")
             os.write(fdesc, func_code)
@@ -113,25 +156,5 @@ class JitCore_Gcc(jitcore.JitCore):
             # Move temporary file to final file
             os.rename(fname_tmp, fname_out)
             os.remove(fname_in)
-        lib = ctypes.cdll.LoadLibrary(fname_out)
-        func = getattr(lib, f_name)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        return None, addr
-
-    def jitirblocs(self, label, irblocs):
-        f_name = "bloc_%s" % label.name
-        f_declaration = 'int %s(block_id * BlockDst, JitCpu* jitcpu)' % f_name
-        out = irblocs2C(self.ir_arch, self.resolver, label, irblocs,
-                        gen_exception_code=True,
-                        log_mn=self.log_mn,
-                        log_regs=self.log_regs)
-        out = [f_declaration + '{'] + out + ['}\n']
-        c_code = out
-
-        func_code = gen_C_source(self.ir_arch, c_code)
 
-        # open('tmp_%.4d.c'%self.jitcount, "w").write(func_code)
-        self.jitcount += 1
-        gcc_state, mcode = self.jit_gcc_compil(f_name, func_code)
-        self.lbl2jitbloc[label.offset] = mcode
-        self.gcc_states[label.offset] = gcc_state
+        self.load_code(block.label, fname_out)
diff --git a/miasm2/jitter/jitcore_tcc.py b/miasm2/jitter/jitcore_tcc.py
index 304a5bca..151fab7d 100644
--- a/miasm2/jitter/jitcore_tcc.py
+++ b/miasm2/jitter/jitcore_tcc.py
@@ -4,6 +4,8 @@
 import os
 from distutils.sysconfig import get_python_inc
 from subprocess import Popen, PIPE
+from hashlib import md5
+import tempfile
 
 from miasm2.ir.ir2C import irblocs2C
 from miasm2.jitter import jitcore, Jittcc
@@ -100,6 +102,12 @@ class JitCore_Tcc(jitcore.JitCore):
         self.tcc_states = {}
         self.ir_arch = ir_arch
 
+        self.tempdir = os.path.join(tempfile.gettempdir(), "miasm_gcc_cache")
+        try:
+            os.mkdir(self.tempdir, 0755)
+        except OSError:
+            pass
+
     def deleteCB(self, offset):
         "Free the TCCState corresponding to @offset"
         if offset in self.tcc_states:
@@ -111,7 +119,8 @@ class JitCore_Tcc(jitcore.JitCore):
         lib_dir = os.path.dirname(os.path.realpath(__file__))
         libs = []
         libs.append(os.path.join(lib_dir, 'VmMngr.so'))
-        libs.append(os.path.join(lib_dir, 'arch/JitCore_%s.so' % (self.ir_arch.arch.name)))
+        libs.append(
+            os.path.join(lib_dir, 'arch/JitCore_%s.so' % (self.ir_arch.arch.name)))
         libs = ';'.join(libs)
         jittcc_path = Jittcc.__file__
         include_dir = os.path.dirname(jittcc_path)
@@ -127,7 +136,7 @@ class JitCore_Tcc(jitcore.JitCore):
         p.stdin.close()
         include_files = p.stderr.read().split('\n')
         include_files = [x[1:]
-            for x in include_files if x.startswith(' /usr/include')]
+                         for x in include_files if x.startswith(' /usr/include')]
         include_files += [include_dir, get_python_inc()]
         include_files = ";".join(include_files)
         Jittcc.tcc_set_emul_lib_path(include_files, libs)
@@ -136,23 +145,63 @@ class JitCore_Tcc(jitcore.JitCore):
         for tcc_state in self.tcc_states.values():
             Jittcc.tcc_end(tcc_state)
 
-    def jitirblocs(self, label, irblocs):
-        f_name = "bloc_%s" % label.name
+    def label2fname(self, label):
+        """
+        Generate function name from @label
+        @label: asm_label instance
+        """
+        return "block_%s" % label.name
+
+    def compil_code(self, block, func_code):
+        """
+        Compil the C code of @func_code from @block
+        @block: original asm_block
+        @func_code: C code of the block
+        """
+        label = block.label
+        self.jitcount += 1
+        tcc_state, mcode = jit_tcc_compil(self.label2fname(label), func_code)
+        self.lbl2jitbloc[label.offset] = mcode
+        self.tcc_states[label.offset] = tcc_state
+
+    def gen_c_code(self, label, irblocks):
+        """
+        Return the C code corresponding to the @irblocks
+        @label: asm_label of the block to jit
+        @irblocks: list of irblocks
+        """
+        f_name = self.label2fname(label)
         f_declaration = 'int %s(block_id * BlockDst, JitCpu* jitcpu)' % f_name
-        out = irblocs2C(self.ir_arch, self.resolver, label, irblocs,
+        out = irblocs2C(self.ir_arch, self.resolver, label, irblocks,
                         gen_exception_code=True,
                         log_mn=self.log_mn,
                         log_regs=self.log_regs)
         out = [f_declaration + '{'] + out + ['}\n']
         c_code = out
 
-        func_code = gen_C_source(self.ir_arch, c_code)
-
-        # open('tmp_%.4d.c'%self.jitcount, "w").write(func_code)
-        self.jitcount += 1
-        tcc_state, mcode = jit_tcc_compil(f_name, func_code)
-        jcode = jit_tcc_code(mcode)
-        self.lbl2jitbloc[label.offset] = mcode
-        self.tcc_states[label.offset] = tcc_state
-        self.addr2obj[label.offset] = jcode
-        self.addr2objref[label.offset] = objref(jcode)
+        return gen_C_source(self.ir_arch, c_code)
+
+    def add_bloc(self, block):
+        """Add a bloc to JiT and JiT it.
+        @block: block to jit
+        """
+        block_raw = "".join(line.b for line in block.lines)
+        block_hash = md5("%X_%s_%s_%s" % (block.label.offset,
+                                          self.log_mn,
+                                          self.log_regs,
+                                          block_raw)).hexdigest()
+        fname_out = os.path.join(self.tempdir, "%s.c" % block_hash)
+        if os.access(fname_out, os.R_OK):
+            func_code = open(fname_out).read()
+        else:
+            irblocks = self.ir_arch.add_bloc(block, gen_pc_updt=True)
+            block.irblocs = irblocks
+            func_code = self.gen_c_code(block.label, irblocks)
+
+            # Create unique C file
+            fdesc, fname_tmp = tempfile.mkstemp(suffix=".c")
+            os.write(fdesc, func_code)
+            os.close(fdesc)
+            os.rename(fname_tmp, fname_out)
+
+        self.compil_code(block, func_code)
diff --git a/miasm2/jitter/vm_mngr.c b/miasm2/jitter/vm_mngr.c
index 1bb58a17..f5c83b8b 100644
--- a/miasm2/jitter/vm_mngr.c
+++ b/miasm2/jitter/vm_mngr.c
@@ -41,6 +41,41 @@
 #define MAX(a,b)  (((a)>(b))?(a):(b))
 
 
+const uint8_t parity_table[256] = {
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
+    0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
+};
+
 //#define DEBUG_MIASM_AUTOMOD_CODE
 
 
@@ -527,26 +562,6 @@ int is_mapped(vm_mngr_t* vm_mngr, uint64_t addr, uint64_t size)
        return 1;
 }
 
-
-
-unsigned int parity(unsigned int a)
-{
-#if defined(__builtin_parity)
-	return __builtin_parity(a);
-#else
-    unsigned int tmp, cpt;
-
-    tmp = a&0xFF;
-    cpt = 1;
-    while (tmp!=0){
-	    cpt^=tmp&1;
-	    tmp>>=1;
-    }
-    return cpt;
-#endif
-}
-
-
 int shift_right_arith(unsigned int size, int a, unsigned int b)
 {
     int i32_a;
diff --git a/miasm2/jitter/vm_mngr.h b/miasm2/jitter/vm_mngr.h
index eb972392..76b1c0dd 100644
--- a/miasm2/jitter/vm_mngr.h
+++ b/miasm2/jitter/vm_mngr.h
@@ -174,8 +174,12 @@ uint64_t MEM_LOOKUP_64_PASSTHROUGH(uint64_t addr);
 int vm_read_mem(vm_mngr_t* vm_mngr, uint64_t addr, char** buffer_ptr, uint64_t size);
 int vm_write_mem(vm_mngr_t* vm_mngr, uint64_t addr, char *buffer, uint64_t size);
 
+#define CC_P 1
+
+extern const uint8_t parity_table[256];
+
+#define parity(a) (parity_table[(a) & 0xFF])
 
-unsigned int parity(unsigned int a);
 unsigned int my_imul08(unsigned int a, unsigned int b);
 
 int is_mapped(vm_mngr_t* vm_mngr, uint64_t addr, uint64_t size);