diff options
| -rw-r--r-- | miasm2/core/utils.py | 73 | ||||
| -rw-r--r-- | miasm2/jitter/jitcore.py | 11 | ||||
| -rw-r--r-- | miasm2/jitter/jitcore_tcc.py | 14 | ||||
| -rw-r--r-- | test/core/utils.py | 41 | ||||
| -rw-r--r-- | test/test_all.py | 1 |
5 files changed, 133 insertions, 7 deletions
diff --git a/miasm2/core/utils.py b/miasm2/core/utils.py index b2ddff2b..782c21d6 100644 --- a/miasm2/core/utils.py +++ b/miasm2/core/utils.py @@ -1,5 +1,6 @@ import struct import inspect +import UserDict upck8 = lambda x: struct.unpack('B', x)[0] upck16 = lambda x: struct.unpack('H', x)[0] @@ -48,3 +49,75 @@ class keydefaultdict(collections.defaultdict): def whoami(): return inspect.stack()[2][3] + + +class BoundedDict(UserDict.DictMixin): + """Limited in size dictionnary. + + To reduce combinatory cost, once an upper limit @max_size is reached, + @max_size - @min_size elements are suppressed. + The targeted elements are the less accessed. + + One can define a callback called when an element is removed + """ + + def __init__(self, max_size, min_size=None, initialdata=None, + delete_cb=None): + """Create a BoundedDict + @max_size: maximum size of the dictionnary + @min_size: (optional) number of most used element to keep when resizing + @initialdata: (optional) dict instance with initial data + @delete_cb: (optional) callback called when an element is removed + """ + self._data = initialdata.copy() if initialdata else {} + self._min_size = min_size if min_size else max_size / 3 + self._max_size = max_size + self._size = len(self._data) + self._counter = collections.Counter(self._data.keys()) + self._delete_cb = delete_cb + + def __setitem__(self, asked_key, value): + if asked_key not in self._data: + # Update internal size and use's counter + self._size += 1 + + # Bound can only be reached on a new element + if (self._size >= self._max_size): + most_commons = [key for key, _ in self._counter.most_common()] + + # Handle callback + if self._delete_cb is not None: + for key in most_commons[self._min_size - 1:]: + self._delete_cb(key) + + # Keep only the most @_min_size used + self._data = {key:self._data[key] + for key in most_commons[:self._min_size - 1]} + self._size = self._min_size + + # Reset use's counter + self._counter = collections.Counter(self._data.keys()) + + self._data[asked_key] = value + self._counter.update([asked_key]) + + def keys(self): + "Return the list of dict's keys" + return self._data.keys() + + def __getitem__(self, key): + self._counter.update([key]) + return self._data[key] + + def __delitem__(self, key): + if self._delete_cb is not None: + self._delete_cb(key) + del self._data[key] + self._size -= 1 + del self._counter[key] + + def __del__(self): + """Ensure the callback is called when last reference is lost""" + if self._delete_cb: + for key in self._data: + self._delete_cb(key) diff --git a/miasm2/jitter/jitcore.py b/miasm2/jitter/jitcore.py index f1e34870..16f201c9 100644 --- a/miasm2/jitter/jitcore.py +++ b/miasm2/jitter/jitcore.py @@ -17,6 +17,7 @@ # from miasm2.core import asmbloc from miasm2.core.interval import interval +from miasm2.core.utils import BoundedDict from miasm2.jitter.csts import * @@ -24,6 +25,9 @@ class JitCore(object): "JiT management. This is an abstract class" + jitted_block_delete_cb = None + jitted_block_max_size = 10000 + def __init__(self, ir_arch, bs=None): """Initialise a JitCore instance. @ir_arch: ir instance for current architecture @@ -33,7 +37,8 @@ class JitCore(object): self.ir_arch = ir_arch self.bs = bs self.known_blocs = {} - self.lbl2jitbloc = {} + self.lbl2jitbloc = BoundedDict(self.jitted_block_max_size, + delete_cb=self.jitted_block_delete_cb) self.lbl2bloc = {} self.log_mn = False self.log_regs = False @@ -67,7 +72,7 @@ class JitCore(object): def load(self, arch, attrib): "Initialise the Jitter according to arch and attrib" - raise Exception("DO NOT instanciate JitCore") + raise NotImplementedError("Abstract class") def get_bloc_min_max(self, cur_bloc): "Update cur_bloc to set min/max address" @@ -91,7 +96,7 @@ class JitCore(object): @irblocs: a gorup of irblocs """ - raise Exception("DO NOT instanciate JitCore") + raise NotImplementedError("Abstract class") def add_bloc(self, b): """Add a bloc to JiT and JiT it. diff --git a/miasm2/jitter/jitcore_tcc.py b/miasm2/jitter/jitcore_tcc.py index 46b46184..7ea77b15 100644 --- a/miasm2/jitter/jitcore_tcc.py +++ b/miasm2/jitter/jitcore_tcc.py @@ -92,12 +92,19 @@ class JitCore_Tcc(jitcore.JitCore): "JiT management, using LibTCC as backend" def __init__(self, ir_arch, bs=None): + self.jitted_block_delete_cb = self.deleteCB super(JitCore_Tcc, self).__init__(ir_arch, bs) self.resolver = resolver() self.exec_wrapper = Jittcc.tcc_exec_bloc - self.tcc_states =[] + self.tcc_states = {} self.ir_arch = ir_arch + def deleteCB(self, offset): + "Free the TCCState corresponding to @offset" + if offset in self.tcc_states: + Jittcc.tcc_end(self.tcc_states[offset]) + del self.tcc_states[offset] + def load(self): # os.path.join(os.path.dirname(os.path.realpath(__file__)), "jitter") lib_dir = os.path.dirname(os.path.realpath(__file__)) @@ -120,12 +127,11 @@ class JitCore_Tcc(jitcore.JitCore): include_files = [x[1:] for x in include_files if x.startswith(' /usr/include')] include_files += [include_dir, get_python_inc()] - include_files = ";".join(include_files) Jittcc.tcc_set_emul_lib_path(include_files, libs) def __del__(self): - for tcc_state in self.tcc_states: + for tcc_state in self.tcc_states.values(): Jittcc.tcc_end(tcc_state) def jitirblocs(self, label, irblocs): @@ -143,8 +149,8 @@ class JitCore_Tcc(jitcore.JitCore): # open('tmp_%.4d.c'%self.jitcount, "w").write(func_code) self.jitcount += 1 tcc_state, mcode = jit_tcc_compil(f_name, func_code) - self.tcc_states.append(tcc_state) jcode = jit_tcc_code(mcode) self.lbl2jitbloc[label.offset] = mcode + self.tcc_states[label.offset] = tcc_state self.addr2obj[label.offset] = jcode self.addr2objref[label.offset] = objref(jcode) diff --git a/test/core/utils.py b/test/core/utils.py new file mode 100644 index 00000000..bf14df68 --- /dev/null +++ b/test/core/utils.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +import unittest + + +class TestUtils(unittest.TestCase): + + def test_boundedDict(self): + from miasm2.core.utils import BoundedDict + + # Use a callback + def logger(key): + print "DELETE", key + + # Create a 5/2 dictionnary + bd = BoundedDict(5, 2, initialdata={"element": "value"}, + delete_cb=logger) + bd["element2"] = "value2" + assert("element" in bd) + assert("element2" in bd) + self.assertEqual(bd["element"], "value") + self.assertEqual(bd["element2"], "value2") + + # Increase 'element2' use + _ = bd["element2"] + + for i in xrange(6): + bd[i] = i + print "Insert %d -> %s" % (i, bd) + + assert(len(bd) == 2) + + assert("element2" in bd) + self.assertEqual(bd["element2"], "value2") + + +if __name__ == '__main__': + testsuite = unittest.TestLoader().loadTestsFromTestCase(TestUtils) + report = unittest.TextTestRunner(verbosity=2).run(testsuite) + exit(len(report.errors + report.failures)) diff --git a/test/test_all.py b/test/test_all.py index 0ecc677f..8d759053 100644 --- a/test/test_all.py +++ b/test/test_all.py @@ -100,6 +100,7 @@ testset += SemanticTestExec("x86_32", "PE", 0x401000, ["bsr_bsf"], for script in ["interval.py", "graph.py", "parse_asm.py", + "utils.py", ]: testset += RegressionTest([script], base_dir="core") ## Expression |