add prefetch mem in emulation

6 files changed, 236 insertions, 77 deletions

diff --git a/example/unpack_upx.py b/example/unpack_upx.py
index 765ff4d8..17960ea6 100644
--- a/example/unpack_upx.py
+++ b/example/unpack_upx.py
@@ -39,7 +39,6 @@ else:
     # binary

     all_bloc = asmbloc.dis_bloc_all(x86_mn, in_str, decomp_func, job_done, symbol_pool, bloc_wd=1)

     b = all_bloc[0]

-    

 print b

 

 

@@ -79,7 +78,6 @@ print 'final label'
 print hex(end_label)

 

 

-    

 base_imp = 0

 offset_imp = 0

 libbase_ad = 0x77700000

@@ -95,9 +93,7 @@ def myloadlibexa():
     print repr(libname)

 

     ad = runtime_dll.lib_get_add_base(libname)

-        

     regs = vm_get_gpreg()

-    

     if not base_imp:

         base_imp = regs["edi"]

     if not offset_imp:

@@ -109,18 +105,13 @@ def myloadlibexa():
     vm_set_gpreg(regs)

 

 

-    

-

-

-

-

 def mygetproc():

     global runtime_dll

     ret_ad = vm_pop_uint32_t()

     libbase = vm_pop_uint32_t()

     fname = vm_pop_uint32_t()

     print 'getproc', hex(fname), hex(libbase), hex(ret_ad)

-    

+

     regs = vm_get_gpreg()

     dst_ad = regs['ebx']

     print 'ebx', hex(dst_ad)

@@ -135,8 +126,6 @@ def mygetproc():
 

     ad = runtime_dll.lib_get_add_func(libbase, fname, dst_ad)

 

-    

-    

     regs['eip'] = ret_ad

     regs['eax'] = ad

     vm_set_gpreg(regs)

@@ -163,7 +152,6 @@ if 'kernel32_VirtualProtect' in dll_dyn_funcs:
 

 

 

-    

 dump_memory_page_pool_py()

 

 

@@ -184,7 +172,7 @@ code_blocs_mem_range = []
 def my_run():

     global cpt, my_eip, known_blocs, code_blocs_mem_range

     trace_on = {'log_mn':False, 'log_regs':False}

-    

+

     print 'start'

     while True:

         cpt+=1

@@ -196,11 +184,9 @@ def my_run():
                 sdata = vm_get_str(e.rva2virt(s.addr), s.rawsize)

                 e.virt[e.rva2virt(s.addr)] = sdata

             in_str = bin_stream(e.virt)

-    

-            open('uu.bin', 'wb').write(str(e))

+            #open('uu.bin', 'wb').write(str(e))

             g = asmbloc.bloc2graph([x.b for x in known_blocs.values()], lines = False)

-            open("graph.txt" , "w").write(g)

-    

+            #open("graph.txt" , "w").write(g)

             break

         if my_eip in dyn_func:

             dyn_func[my_eip]()

@@ -236,7 +222,7 @@ for r, v in regs.items():
 

 oo = vm_get_str(decomp_buf_ad_out, decomp_func-decomp_buf_ad_out)

 

-open('uu', 'w').write("A"*0x1000 + oo)

+#open('uu', 'w').write("A"*0x1000 + oo)

 print repr(oo[:0x10])

 print repr(oo[-0x10:])

 

@@ -261,7 +247,7 @@ new_dll = []
 offset_imp = offset_imp - decomp_buf_ad_out - struct.unpack('I', e.virt[ad_tmp:ad_tmp+4])[0]

 print "read ofset imp", hex(offset_imp)

 

-#XXXXX 

+#XXXXX

 ad_base = decomp_buf_ad_out

 

 print repr(e.SHList)

@@ -280,6 +266,7 @@ e.DirRes = pe.DirRes(e)
 print repr(e.DirImport.impdesc)

 new_dll = runtime_dll.gen_new_lib(e)

 print new_dll

+e.DirImport.impdesc = []

 e.DirImport.add_dlldesc(new_dll)

 s_myimp = e.SHList.add_section(name = "myimp", rawsize = len(e.DirImport))

 print repr(e.SHList)

diff --git a/miasm/arch/ia32_sem.py b/miasm/arch/ia32_sem.py
index 1bd9f90e..9e6025e7 100644
--- a/miasm/arch/ia32_sem.py
+++ b/miasm/arch/ia32_sem.py
@@ -227,7 +227,7 @@ float_c2 = ExprId(reg_float_c2)
 float_c3 = ExprId(reg_float_c3)
 float_stack_ptr = ExprId(reg_float_stack_ptr)
 float_control = ExprId(reg_float_control)
-                          
+
 float_st0 = ExprId(reg_float_st0, 64)
 float_st1 = ExprId(reg_float_st1, 64)
 float_st2 = ExprId(reg_float_st2, 64)
@@ -279,15 +279,15 @@ all_registers = [
     eip ,
     esi ,
     edi ,
-    dr0, 
-    dr1, 
-    dr2, 
-    dr3, 
-    dr4, 
-    dr5, 
-    dr6, 
-    dr7, 
-    
+    dr0,
+    dr1,
+    dr2,
+    dr3,
+    dr4,
+    dr5,
+    dr6,
+    dr7,
+
     eflag,
     tmp1,
     zf ,
@@ -307,24 +307,24 @@ all_registers = [
     vif,
     vip,
     i_d,
-    
+
     es ,
     cs ,
     ss ,
     ds ,
     fs ,
     gs ,
-    
+
     tsc1 ,
     tsc2 ,
-    
+
     float_c0 ,
     float_c1 ,
     float_c2 ,
     float_c3 ,
     float_stack_ptr ,
     float_control ,
-    
+
     float_st0 ,
     float_st1 ,
     float_st2 ,
@@ -361,6 +361,8 @@ OF(A+B) = ((A XOR D) AND NOT (A XOR B)) < 0
 OF(A-B) = ((A XOR D) AND (A XOR B)) < 0
 """
 
+
+# XXX TODO make default check against 0 or not 0 (same eq as in C)
 def get_op_msb(a):
     cast_int = tab_uintsize[a.get_size()]
     return ExprOp('==', ExprInt(cast_int(1)), ExprOp('>>', a, ExprInt(cast_int(a.get_size()-1))))
@@ -418,7 +420,6 @@ def update_flag_add_of(cast_int, a, b, c):
     return ExprAff(of, get_op_msb(((a ^ c) & ExprOp('!', (a ^ b)))))
 
 
-
 #checked: ok for sbb add because of b & c before +cf
 def update_flag_sub_cf(cast_int, a, b, c):
     return ExprAff(cf, get_op_msb((a ^ b) ^ c) ^ get_op_msb((a ^ c) & (a ^ b)))
@@ -428,15 +429,12 @@ def update_flag_sub_of(cast_int, a, b, c):
     return ExprAff(of, get_op_msb(((a ^ c) & (a ^ b))))
 
 
-    
-
-
 #z = x+y (+cf?)
 def update_flag_add(x, y, z):
     cast_int = tab_uintsize[z.get_size()]
     e = []
     e.append(update_flag_add_cf(cast_int, x, y, z))
-    e.append(update_flag_add_of(cast_int, x, y, z))    
+    e.append(update_flag_add_of(cast_int, x, y, z))
     return e
 
 #z = x-y (+cf?)
diff --git a/miasm/expression/expression.py b/miasm/expression/expression.py
index 8fa9eb27..c9aba983 100644
--- a/miasm/expression/expression.py
+++ b/miasm/expression/expression.py
@@ -327,11 +327,10 @@ class ExprCond(Expr):
         return self.cond == a.cond and self.src1 == a.src1 and self.src2 == a.src2
     def __hash__(self):
         return hash(self.cond)^hash(self.src1)^hash(self.src2)
-        
+
     def toC(self):
         return "(%s?%s:%s)"%(self.cond.toC(), self.src1.toC(), self.src2.toC())
-        
-    
+
 class ExprMem(Expr):
     def __init__(self, arg, size = 32):
         if not isinstance(arg, Expr): raise 'arg must be expr'
@@ -348,10 +347,11 @@ class ExprMem(Expr):
     def get_size(self):
         return self.size
     def reload_expr(self, g = {}):
+        if self in g:
+            return g[self]
         arg = self.arg
         if isinstance(arg, Expr):
             arg = self.arg.reload_expr(g)
-        
 
         return ExprMem(arg, self.size )
     def __contains__(self, e):
@@ -372,9 +372,8 @@ class ExprMem(Expr):
 
     def toC(self):
         return "MEM_LOOKUP_%.2d(%s)"%(self.size, self.arg.toC())
-        
 
-    
+
 class ExprOp(Expr):
     def __init__(self, op, *args):
         self.op, self.args = op, args
diff --git a/miasm/tools/emul_lib/libcodenat.h b/miasm/tools/emul_lib/libcodenat.h
index d4047f16..9ae1408c 100644
--- a/miasm/tools/emul_lib/libcodenat.h
+++ b/miasm/tools/emul_lib/libcodenat.h
@@ -152,7 +152,6 @@ typedef struct {
 	unsigned int float_c2_new;
 	unsigned int float_c3_new;
 
-
 	unsigned int float_stack_ptr;
 	unsigned int float_stack_ptr_new;
 
@@ -172,6 +171,43 @@ typedef struct {
 	unsigned int cr3;
 	unsigned int cr3_new;
 
+	uint8_t pfmem08_0;
+	uint8_t pfmem08_1;
+	uint8_t pfmem08_2;
+	uint8_t pfmem08_3;
+	uint8_t pfmem08_4;
+	uint8_t pfmem08_5;
+	uint8_t pfmem08_6;
+	uint8_t pfmem08_7;
+
+	uint16_t pfmem16_0;
+	uint16_t pfmem16_1;
+	uint16_t pfmem16_2;
+	uint16_t pfmem16_3;
+	uint16_t pfmem16_4;
+	uint16_t pfmem16_5;
+	uint16_t pfmem16_6;
+	uint16_t pfmem16_7;
+
+	uint32_t pfmem32_0;
+	uint32_t pfmem32_1;
+	uint32_t pfmem32_2;
+	uint32_t pfmem32_3;
+	uint32_t pfmem32_4;
+	uint32_t pfmem32_5;
+	uint32_t pfmem32_6;
+	uint32_t pfmem32_7;
+
+	uint64_t pfmem64_0;
+	uint64_t pfmem64_1;
+	uint64_t pfmem64_2;
+	uint64_t pfmem64_3;
+	uint64_t pfmem64_4;
+	uint64_t pfmem64_5;
+	uint64_t pfmem64_6;
+	uint64_t pfmem64_7;
+
+
 
 }vm_cpu_t;
 
diff --git a/miasm/tools/to_c_helper.py b/miasm/tools/to_c_helper.py
index 7c45f50d..aa2ba091 100644
--- a/miasm/tools/to_c_helper.py
+++ b/miasm/tools/to_c_helper.py
@@ -38,6 +38,43 @@ def id2new(i):
 mask_int = 0xffffffffffffffff
 
 
+pfmem08_0 = ExprId("pfmem08_0", 8)
+pfmem08_1 = ExprId("pfmem08_1", 8)
+pfmem08_2 = ExprId("pfmem08_2", 8)
+pfmem08_3 = ExprId("pfmem08_3", 8)
+pfmem08_4 = ExprId("pfmem08_4", 8)
+pfmem08_5 = ExprId("pfmem08_5", 8)
+pfmem08_6 = ExprId("pfmem08_6", 8)
+pfmem08_7 = ExprId("pfmem08_7", 8)
+
+pfmem16_0 = ExprId("pfmem16_0", 16)
+pfmem16_1 = ExprId("pfmem16_1", 16)
+pfmem16_2 = ExprId("pfmem16_2", 16)
+pfmem16_3 = ExprId("pfmem16_3", 16)
+pfmem16_4 = ExprId("pfmem16_4", 16)
+pfmem16_5 = ExprId("pfmem16_5", 16)
+pfmem16_6 = ExprId("pfmem16_6", 16)
+pfmem16_7 = ExprId("pfmem16_7", 16)
+
+pfmem32_0 = ExprId("pfmem32_0", 32)
+pfmem32_1 = ExprId("pfmem32_1", 32)
+pfmem32_2 = ExprId("pfmem32_2", 32)
+pfmem32_3 = ExprId("pfmem32_3", 32)
+pfmem32_4 = ExprId("pfmem32_4", 32)
+pfmem32_5 = ExprId("pfmem32_5", 32)
+pfmem32_6 = ExprId("pfmem32_6", 32)
+pfmem32_7 = ExprId("pfmem32_7", 32)
+
+pfmem64_0 = ExprId("pfmem64_0", 64)
+pfmem64_1 = ExprId("pfmem64_1", 64)
+pfmem64_2 = ExprId("pfmem64_2", 64)
+pfmem64_3 = ExprId("pfmem64_3", 64)
+pfmem64_4 = ExprId("pfmem64_4", 64)
+pfmem64_5 = ExprId("pfmem64_5", 64)
+pfmem64_6 = ExprId("pfmem64_6", 64)
+pfmem64_7 = ExprId("pfmem64_7", 64)
+
+
 my_C_id = [
     eax,
     ebx,
@@ -93,7 +130,7 @@ my_C_id = [
     #i_d_new,
     #my_tick,
     float_control,
-    cond,
+    #cond,
     ds,
     #vm_exception_flags,
     #vm_exception_flags_new,
@@ -119,6 +156,42 @@ my_C_id = [
     cr3,
 
     float_stack_ptr,
+    pfmem08_0,
+    pfmem08_1,
+    pfmem08_2,
+    pfmem08_3,
+    pfmem08_4,
+    pfmem08_5,
+    pfmem08_6,
+    pfmem08_7,
+
+    pfmem16_0,
+    pfmem16_1,
+    pfmem16_2,
+    pfmem16_3,
+    pfmem16_4,
+    pfmem16_5,
+    pfmem16_6,
+    pfmem16_7,
+
+    pfmem32_0,
+    pfmem32_1,
+    pfmem32_2,
+    pfmem32_3,
+    pfmem32_4,
+    pfmem32_5,
+    pfmem32_6,
+    pfmem32_7,
+
+    pfmem64_0,
+    pfmem64_1,
+    pfmem64_2,
+    pfmem64_3,
+    pfmem64_4,
+    pfmem64_5,
+    pfmem64_6,
+    pfmem64_7,
+
     ]
 
 float_id_e = [
@@ -134,7 +207,7 @@ float_id_e = [
 
 id2Cid = {}
 for x in my_C_id:
-    id2Cid[x] = ExprId('vmcpu.'+str(x))
+    id2Cid[x] = ExprId('vmcpu.'+str(x), x.get_size())
 
 def patch_c_id(e):
     return e.reload_expr(id2Cid)
@@ -142,18 +215,24 @@ def patch_c_id(e):
 
 code_deal_exception_at_instr = r"""
 if (vmcpu.vm_exception_flags > EXCEPT_NUM_UDPT_EIP) {
-    %s = 0x%X; 
-    return vmcpu.eip; 
+    %s = 0x%X;
+    return vmcpu.eip;
 }
 """
 code_deal_exception_post_instr = r"""
 if (vmcpu.vm_exception_flags) {
-    %s = (vmcpu.vm_exception_flags > EXCEPT_NUM_UDPT_EIP) ?  0x%X : 0x%X; 
-    return vmcpu.eip; 
+    %s = (vmcpu.vm_exception_flags > EXCEPT_NUM_UDPT_EIP) ?  0x%X : 0x%X;
+    return vmcpu.eip;
 }
 """
 
-    
+
+tab_uintsize ={8:uint8,
+               16:uint16,
+               32:uint32,
+               64:uint64
+               }
+
 def Exp2C(exprs, l = None, addr2label = None, gen_exception_code = False):
     my_size_mask = {1:1, 8:0xFF, 16:0xFFFF, 32:0xFFFFFFFF,  64:0xFFFFFFFFFFFFFFFFL,
                     2: 3}
@@ -165,6 +244,16 @@ def Exp2C(exprs, l = None, addr2label = None, gen_exception_code = False):
     #print [str(x) for x in exprs]
 
     dst_dict = {}
+    src_mem = {}
+
+    prefect_mem_pool = {8: [pfmem08_0 ,pfmem08_1, pfmem08_2, pfmem08_3,
+                            pfmem08_4, pfmem08_5, pfmem08_6, pfmem08_7],
+                        16: [pfmem16_0 ,pfmem16_1, pfmem16_2, pfmem16_3,
+                            pfmem16_4, pfmem16_5, pfmem16_6, pfmem16_7],
+                        32: [pfmem32_0 ,pfmem32_1, pfmem32_2, pfmem32_3,
+                            pfmem32_4, pfmem32_5, pfmem32_6, pfmem32_7],
+                        64: [pfmem64_0 ,pfmem64_1, pfmem64_2, pfmem64_3,
+                            pfmem64_4, pfmem64_5, pfmem64_6, pfmem64_7],}
 
     new_expr = []
 
@@ -180,7 +269,14 @@ def Exp2C(exprs, l = None, addr2label = None, gen_exception_code = False):
             dst_dict[e.dst].append(e)
         else:
             new_expr.append(e)
-            
+        # search mem lookup for generate mem read prefetch
+        rs = e.src.get_r(mem_read=True)
+        for r in rs:
+            if (not isinstance(r, ExprMem)) or r in src_mem:
+                continue
+            pfmem = prefect_mem_pool[r.get_size()].pop(0)
+            src_mem[r] = pfmem
+
     for dst, exs in dst_dict.items():
         if len(exs) ==1:
             new_expr += exs
@@ -197,19 +293,27 @@ def Exp2C(exprs, l = None, addr2label = None, gen_exception_code = False):
             #print known_intervals
             missing_i = get_missing_interval(known_intervals)
             #print missing_i
-    
             rest = [ExprSliceTo(ExprSlice(dst, *r), *r) for r in missing_i]
             final_dst = ExprCompose(e_colision+ rest)
-            
             new_expr.append(ExprAff(dst, final_dst))
-        
     out_mem = []
-    
+
+    # first, generate mem prefetch
+    mem_k = src_mem.keys()
+    mem_k.sort()
+    for k in mem_k:
+        str_src = patch_c_id(k).toC()
+        str_dst = patch_c_id(src_mem[k]).toC()
+        out.append('%s = %s;'%(str_dst, str_src))
+    src_w_len = {}
+    for k, v in src_mem.items():
+        cast_int = tab_uintsize[k.get_size()]
+        src_w_len[k] = v
     for e in new_expr:
-            
         if True:#e.dst != eip:
-            
             src, dst = e.src, e.dst
+            # reload src using prefetch
+            src = src.reload_expr(src_w_len)
             str_src = patch_c_id(src).toC()
             str_dst = patch_c_id(dst).toC()
             if isinstance(dst, ExprId):
@@ -224,8 +328,7 @@ def Exp2C(exprs, l = None, addr2label = None, gen_exception_code = False):
             elif isinstance(dst, ExprMem):
                 str_dst = str_dst.replace('MEM_LOOKUP', 'MEM_WRITE')
                 out_mem.append('%s, %s);'%(str_dst[:-1], str_src))
-                
-            
+
         if e.dst == eip :
             eip_is_dst = True
             if isinstance(e.src, ExprCond):
@@ -259,7 +362,6 @@ def Exp2C(exprs, l = None, addr2label = None, gen_exception_code = False):
 
     for i in id_to_update:
         out.append('%s = %s;'%(patch_c_id(i), id2new(patch_c_id(i))))
-                            
 
 
 
@@ -926,7 +1028,6 @@ def updt_bloc_emul(known_blocs, in_str, my_eip, symbol_pool, code_blocs_mem_rang
     reset_code_bloc_pool_py()
     for a, b in  code_blocs_mem_range:
             vm_add_code_bloc(a, b)
-    
 #'''
 
 def updt_pe_from_emul(e):
@@ -934,10 +1035,6 @@ def updt_pe_from_emul(e):
         sdata = vm_get_str(e.rva2virt(s.addr), s.rawsize)
         e.virt[e.rva2virt(s.addr)] = sdata
     return bin_stream(e.virt)
-    
-    return bin_stream_vm()
-
-
 
 def updt_automod_code(known_blocs):
     w_ad, w_size = vm_get_last_write_ad(), vm_get_last_write_size()
@@ -951,7 +1048,6 @@ def updt_automod_code(known_blocs):
         vm_add_code_bloc(a, b)
     vm_reset_exception()
 
-
     return known_blocs, code_addr
 
 
diff --git a/miasm/tools/win_api.py b/miasm/tools/win_api.py
index b9077ed6..e3a609b7 100644
--- a/miasm/tools/win_api.py
+++ b/miasm/tools/win_api.py
@@ -838,15 +838,20 @@ def kernel32_GetModuleFileName(funcname, set_str):
     if hmodule in [0]:
         p = module_path[:]
     else:
-        raise ValueError('unknown module h')
+        print ValueError('unknown module h', hex(hmodule))
+        p = None
 
 
-    if nsize < len(p):
+    if p == None:
+        l = 0
+    elif nsize < len(p):
         p = p[:nsize]
-    l = len(p)
+        l = len(p)
+    else:
+        l = len(p)
 
-    print repr(p)
-    vm_set_mem(lpfilename, set_str(p))
+        print repr(p)
+        vm_set_mem(lpfilename, set_str(p))
 
     regs = vm_get_gpreg()
     regs['eip'] = ret_ad
@@ -946,8 +951,7 @@ def kernel32_GetProcAddress():
     if fname < 0x10000:
         fname = fname
     else:
-        fname = vm_get_str(fname, 0x100)
-        fname = fname[:fname.find('\x00')]
+        fname = get_str_ansi(fname, 0x100)
     print repr(fname)
 
     ad = runtime_dll.lib_get_add_func(libbase, fname)
@@ -998,6 +1002,16 @@ def kernel32_GetModuleHandleA():
     regs['eax'] = eax
     vm_set_gpreg(regs)
 
+def kernel32_VirtualLock():
+    ret_ad = vm_pop_uint32_t()
+    lpaddress = vm_pop_uint32_t()
+    dwsize = vm_pop_uint32_t()
+    print whoami(), hex(ret_ad), hex(lpaddress), hex(dwsize)
+    regs = vm_get_gpreg()
+    regs['eip'] = ret_ad
+    regs['eax'] = 1
+    vm_set_gpreg(regs)
+
 
 def kernel32_GetSystemInfo():
     ret_ad = vm_pop_uint32_t()
@@ -2561,13 +2575,27 @@ def kernel32_TlsSetValue():
 
     print whoami(), hex(tlsindex), hex(tlsvalue)
 
-
     tls_values[tlsindex] = tlsvalue
     regs = vm_get_gpreg()
     regs['eip'] = ret_ad
     regs['eax'] = 1
     vm_set_gpreg(regs)
 
+def kernel32_TlsGetValue():
+    global tls_index
+    ret_ad = vm_pop_uint32_t()
+    tlsindex = vm_pop_uint32_t()
+
+    print whoami(), hex(tlsindex)
+
+    if not tlsindex in tls_values:
+        raise ValueError("unknown tls val", repr(tlsindex))
+    regs = vm_get_gpreg()
+    regs['eip'] = ret_ad
+    regs['eax'] = tls_values[tlsindex]
+    vm_set_gpreg(regs)
+
+
 def user32_GetKeyboardType():
     ret_ad = vm_pop_uint32_t()
     typeflag = vm_pop_uint32_t()
@@ -2912,3 +2940,18 @@ def kernel32_VirtualQuery():
     regs['eip'] = ret_ad
     regs['eax'] = dwl
     vm_set_gpreg(regs)
+
+def kernel32_GetProcessAffinityMask():
+    ret_ad = vm_pop_uint32_t()
+    hprocess = vm_pop_uint32_t()
+    procaffmask = vm_pop_uint32_t()
+    systemaffmask = vm_pop_uint32_t()
+
+    print whoami(), hex(ret_ad), hex(hprocess), hex(procaffmask), hex(systemaffmask)
+    vm_set_mem(procaffmask, pdw(1))
+    vm_set_mem(systemaffmask, pdw(1))
+
+    regs = vm_get_gpreg()
+    regs['eip'] = ret_ad
+    regs['eax'] = 1
+    vm_set_gpreg(regs)