[DYNAREC] Reworked dynarec memory allocator, and a bit of protection tracking too (might help #455)

5 files changed, 227 insertions, 97 deletions

diff --git a/src/custommem.c b/src/custommem.c
index ec11207d..e11881d1 100644
--- a/src/custommem.c
+++ b/src/custommem.c
@@ -35,7 +35,6 @@
 KHASH_MAP_INIT_INT64(dynablocks, dynablock_t*)
 static mmaplist_t          *mmaplist = NULL;
 static size_t              mmapsize = 0;
-static size_t              mmapcap = 0;
 static kh_dynablocks_t     *dblist_oversized;      // store the list of oversized dynablocks (normal sized are inside mmaplist)
 static uintptr_t***        box64_jmptbl3[1<<JMPTABL_SHIFT];
 static uintptr_t**         box64_jmptbldefault2[1<<JMPTABL_SHIFT];
@@ -411,68 +410,96 @@ void customFree(void* p)
 }
 
 #ifdef DYNAREC
-typedef struct mmaplist_s {
+typedef struct mmapchunk_s {
     void*               block;
     size_t              maxfree;
     size_t              size;
     kh_dynablocks_t*    dblist;
     uint8_t*            helper;
     void*               first;  // first free block, to speed up things
-    int                 locked; // don't try to add stuff on locked block
+    int                 lock;   // don't try to add stuff on locked block
+} mmapchunk_t;
+#define NCHUNK          64
+typedef struct mmaplist_s {
+    mmapchunk_t         chunks[NCHUNK];
+    mmaplist_t*         next;
 } mmaplist_t;
 
+mmapchunk_t* addChunk(size_t mmapsize) {
+    if(!mmaplist)
+        mmaplist = (mmaplist_t*)box_calloc(1, sizeof(mmaplist_t));
+    mmaplist_t* head = mmaplist;
+    size_t i = mmapsize;
+    while(i) {
+        if(i>=NCHUNK) {
+            i-=NCHUNK;
+            if(!head->next) {
+                head->next = (mmaplist_t*)box_calloc(1, sizeof(mmaplist_t));
+            }
+            head=head->next;
+        } else
+            return &head->chunks[i];
+    }
+}
+
 uintptr_t FindFreeDynarecMap(dynablock_t* db, size_t size)
 {
     // look for free space
     void* sub = NULL;
-    for(size_t i=0; i<mmapsize; ++i) {
-        if(mmaplist[i].maxfree>=size+sizeof(blockmark_t) && !mmaplist[i].locked) {
-            mmaplist[i].locked = 1;
-            size_t rsize = 0;
-            sub = getFirstBlock(mmaplist[i].block, size, &rsize, mmaplist[i].first);
-            if(sub) {
-                uintptr_t ret = (uintptr_t)allocBlock(mmaplist[i].block, sub, size, &mmaplist[i].first);
-                if(rsize==mmaplist[i].maxfree) {
-                    mmaplist[i].maxfree = getMaxFreeBlock(mmaplist[i].block, mmaplist[i].size, mmaplist[i].first);
-                }
-                kh_dynablocks_t *blocks = mmaplist[i].dblist;
-                if(!blocks) {
-                    blocks = mmaplist[i].dblist = kh_init(dynablocks);
-                    kh_resize(dynablocks, blocks, 64);
+    mmaplist_t* head = mmaplist;
+    int i = mmapsize;
+    while(head) {
+        const int n = (i>NCHUNK)?NCHUNK:i;
+        i-=n;
+        for(int i=0; i<n; ++i) {
+            mmapchunk_t* chunk = &head->chunks[i];
+            if(chunk->maxfree>=size+sizeof(blockmark_t) && !native_lock_incif0(&chunk->lock)) {
+                size_t rsize = 0;
+                sub = getFirstBlock(chunk->block, size, &rsize, chunk->first);
+                if(sub) {
+                    uintptr_t ret = (uintptr_t)allocBlock(chunk->block, sub, size, &chunk->first);
+                    if(rsize==chunk->maxfree) {
+                        chunk->maxfree = getMaxFreeBlock(chunk->block, chunk->size, chunk->first);
+                    }
+                    kh_dynablocks_t *blocks = chunk->dblist;
+                    if(!blocks) {
+                        blocks = chunk->dblist = kh_init(dynablocks);
+                        kh_resize(dynablocks, blocks, 64);
+                    }
+                    khint_t k;
+                    int r;
+                    k = kh_put(dynablocks, blocks, (uintptr_t)ret, &r);
+                    kh_value(blocks, k) = db;
+                    int size255=(size<256)?size:255;
+                    for(size_t j=0; j<size255; ++j)
+                        chunk->helper[(uintptr_t)ret-(uintptr_t)chunk->block+j] = j;
+                    if(size!=size255)
+                        memset(&chunk->helper[(uintptr_t)ret-(uintptr_t)chunk->block+256], -1, size-255);
+                    native_lock_decifnot0(&chunk->lock);
+                    return ret;
+                } else {
+                    printf_log(LOG_INFO, "BOX64: Warning, sub not found, corrupted mmaplist[%zu] info?\n", i);
+                    native_lock_decifnot0(&chunk->lock);
+                    if(box64_log >= LOG_DEBUG)
+                        printBlock(chunk->block, chunk->first);
                 }
-                khint_t k;
-                int r;
-                k = kh_put(dynablocks, blocks, (uintptr_t)ret, &r);
-                kh_value(blocks, k) = db;
-                int size255=(size<256)?size:255;
-                for(size_t j=0; j<size255; ++j)
-                    mmaplist[i].helper[(uintptr_t)ret-(uintptr_t)mmaplist[i].block+j] = j;
-                if(size!=size255)
-                    memset(&mmaplist[i].helper[(uintptr_t)ret-(uintptr_t)mmaplist[i].block+256], -1, size-255);
-                mmaplist[i].locked = 0;
-                return ret;
-            } else {
-                printf_log(LOG_INFO, "BOX64: Warning, sub not found, corrupted mmaplist[%zu] info?\n", i);
-                if(box64_log >= LOG_DEBUG)
-                    printBlock(mmaplist[i].block, mmaplist[i].first);
             }
         }
+        head = head->next;
     }
     return 0;
 }
 
 uintptr_t AddNewDynarecMap(dynablock_t* db, size_t size)
 {
-    size_t i = mmapsize++;
-    dynarec_log(LOG_DEBUG, "Ask for DynaRec Block Alloc #%zu/%zu\n", mmapsize, mmapcap);
-    if(mmapsize>mmapcap) {
-        mmapcap += 32;
-        mmaplist = (mmaplist_t*)box_realloc(mmaplist, mmapcap*sizeof(mmaplist_t));
-    }
+    dynarec_log(LOG_DEBUG, "Ask for DynaRec Block Alloc #%zu\n", mmapsize);
+    mmapchunk_t* chunk = addChunk(mmapsize++);
+    native_lock_incif0(&chunk->lock);
     #ifndef USE_MMAP
     void *p = NULL;
     if(!(p=box_memalign(box64_pagesize, MMAPSIZE))) {
-        dynarec_log(LOG_INFO, "Cannot create memory map of %d byte for dynarec block #%zu\n", MMAPSIZE, i);
+        dynarec_log(LOG_INFO, "Cannot create memory map of %d byte for dynarec block #%zu\n", MMAPSIZE, mmapsize-1);
+        native_lock_store(&chunk->lock, 0);
         --mmapsize;
         return 0;
     }
@@ -480,18 +507,18 @@ uintptr_t AddNewDynarecMap(dynablock_t* db, size_t size)
     #else
     void* p = mmap(NULL, MMAPSIZE, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
     if(p==(void*)-1) {
-        dynarec_log(LOG_INFO, "Cannot create memory map of %d byte for dynarec block #%zu\n", MMAPSIZE, i);
+        dynarec_log(LOG_INFO, "Cannot create memory map of %d byte for dynarec block #%zu\n", MMAPSIZE, mmapsize-1);
+        native_lock_store(&chunk->lock, 0);
         --mmapsize;
         return 0;
     }
     #endif
     setProtection((uintptr_t)p, MMAPSIZE, PROT_READ | PROT_WRITE | PROT_EXEC);
 
-    mmaplist[i].locked = 1;
-    mmaplist[i].block = p;
-    mmaplist[i].size = MMAPSIZE;
-    mmaplist[i].helper = (uint8_t*)box_calloc(1, MMAPSIZE);
-    mmaplist[i].first = p;
+    chunk->block = p;
+    chunk->size = MMAPSIZE;
+    chunk->helper = (uint8_t*)box_calloc(1, MMAPSIZE);
+    chunk->first = p;
     // setup marks
     blockmark_t* m = (blockmark_t*)p;
     m->prev.x32 = 0;
@@ -502,44 +529,57 @@ uintptr_t AddNewDynarecMap(dynablock_t* db, size_t size)
     n->prev.fill = 0;
     n->prev.size = m->next.size;
     // alloc 1st block
-    uintptr_t sub  = (uintptr_t)allocBlock(mmaplist[i].block, p, size, &mmaplist[i].first);
-    mmaplist[i].maxfree = getMaxFreeBlock(mmaplist[i].block, mmaplist[i].size, mmaplist[i].first);
-    kh_dynablocks_t *blocks = mmaplist[i].dblist = kh_init(dynablocks);
+    uintptr_t sub  = (uintptr_t)allocBlock(chunk->block, p, size, &chunk->first);
+    chunk->maxfree = getMaxFreeBlock(chunk->block, chunk->size, chunk->first);
+    kh_dynablocks_t *blocks = chunk->dblist = kh_init(dynablocks);
     kh_resize(dynablocks, blocks, 64);
     khint_t k;
     int ret;
     k = kh_put(dynablocks, blocks, (uintptr_t)sub, &ret);
     kh_value(blocks, k) = db;
     for(size_t j=0; j<size; ++j)
-        mmaplist[i].helper[(uintptr_t)sub-(uintptr_t)mmaplist[i].block + j] = (j<256)?j:255;
-    mmaplist[i].locked = 0;
+        chunk->helper[(uintptr_t)sub-(uintptr_t)chunk->block + j] = (j<256)?j:255;
+    native_lock_decifnot0(&chunk->lock);
     return sub;
 }
 
 void ActuallyFreeDynarecMap(dynablock_t* db, uintptr_t addr, size_t size)
 {
-    (void)db;
-    if(!addr || !size)
-        return;
-    for(size_t i=0; i<mmapsize; ++i) {
-        if ((addr>(uintptr_t)mmaplist[i].block) 
-         && (addr<((uintptr_t)mmaplist[i].block+mmaplist[i].size))) {
-            void* sub = (void*)(addr-sizeof(blockmark_t));
-            size_t newfree = freeBlock(mmaplist[i].block, sub, &mmaplist[i].first);
-            if(mmaplist[i].maxfree < newfree) mmaplist[i].maxfree = newfree;
-            kh_dynablocks_t *blocks = mmaplist[i].dblist;
-            if(blocks) {
-                khint_t k = kh_get(dynablocks, blocks, (uintptr_t)sub);
-                if(k!=kh_end(blocks))
-                    kh_del(dynablocks, blocks, k);
-                memset(&mmaplist[i].helper[(uintptr_t)sub-(uintptr_t)mmaplist[i].block], 0, size);
-            }
-            if(mmaplist[i].locked) {
-                printf_log(LOG_INFO, "BOX64: Warning, Free a chunk in a locked mmaplist[%zu]\n", i);
-                ++mmaplist[i].locked;
+    mmaplist_t* head = mmaplist;
+    int i = mmapsize;
+    while(head) {
+        const int n = (i>NCHUNK)?NCHUNK:i;
+        i-=n;
+        for(int i=0; i<n; ++i) {
+            mmapchunk_t* chunk = &head->chunks[i];
+            if ((addr>(uintptr_t)(chunk->block)) 
+            && (addr<((uintptr_t)(chunk->block)+chunk->size))) {
+                int loopedwait = 256;
+                while (native_lock_incif0(&chunk->lock) && loopedwait) {
+                    sched_yield();
+                    --loopedwait;
+                }
+                if(!loopedwait) {
+                    printf_log(LOG_INFO, "BOX64: Warning, Free a chunk in a locked mmaplist[%d]\n", i);
+                    //arm_lock_incb(&chunk->lock);
+                    if(cycle_log)
+                        print_cycle_log(LOG_INFO);
+                }
+                void* sub = (void*)(addr-sizeof(blockmark_t));
+                size_t newfree = freeBlock(chunk->block, sub, &chunk->first);
+                if(chunk->maxfree < newfree) chunk->maxfree = newfree;
+                kh_dynablocks_t *blocks = chunk->dblist;
+                if(blocks) {
+                    khint_t k = kh_get(dynablocks, blocks, (uintptr_t)sub);
+                    if(k!=kh_end(blocks))
+                        kh_del(dynablocks, blocks, k);
+                    memset(&chunk->helper[(uintptr_t)sub-(uintptr_t)chunk->block], 0, size);
+                }
+                native_lock_decifnot0(&chunk->lock);
+                return;
             }
-            return;
         }
+        head = head->next;
     }
     if(mmapsize)
         dynarec_log(LOG_NONE, "Warning, block %p (size %zu) not found in mmaplist for Free\n", (void*)addr, size);
@@ -548,20 +588,28 @@ void ActuallyFreeDynarecMap(dynablock_t* db, uintptr_t addr, size_t size)
 dynablock_t* FindDynablockFromNativeAddress(void* addr)
 {
     // look in actual list
-    for(size_t i=0; i<mmapsize; ++i) {
-        if ((uintptr_t)addr>=(uintptr_t)mmaplist[i].block 
-        && ((uintptr_t)addr<(uintptr_t)mmaplist[i].block+mmaplist[i].size)) {
-            if(!mmaplist[i].helper)
-                return FindDynablockDynablocklist(addr, mmaplist[i].dblist);
-            else {
-                uintptr_t p = (uintptr_t)addr - (uintptr_t)mmaplist[i].block;
-                while(mmaplist[i].helper[p]) p -= mmaplist[i].helper[p];
-                khint_t k = kh_get(dynablocks, mmaplist[i].dblist, (uintptr_t)mmaplist[i].block + p);
-                if(k!=kh_end(mmaplist[i].dblist))
-                    return kh_value(mmaplist[i].dblist, k);
-                return NULL;
+   mmaplist_t* head = mmaplist;
+    int i = mmapsize;
+    while(head) {
+        const int n = (i>NCHUNK)?NCHUNK:i;
+        i-=n;
+        for(int i=0; i<n; ++i) {
+            mmapchunk_t* chunk = &head->chunks[i];
+            if ((uintptr_t)addr>=(uintptr_t)chunk->block 
+            && ((uintptr_t)addr<(uintptr_t)chunk->block+chunk->size)) {
+                if(!chunk->helper)
+                    return FindDynablockDynablocklist(addr, chunk->dblist);
+                else {
+                    uintptr_t p = (uintptr_t)addr - (uintptr_t)chunk->block;
+                    while(chunk->helper[p]) p -= chunk->helper[p];
+                    khint_t k = kh_get(dynablocks, chunk->dblist, (uintptr_t)chunk->block + p);
+                    if(k!=kh_end(chunk->dblist))
+                        return kh_value(chunk->dblist, k);
+                    return NULL;
+                }
             }
         }
+        head = head->next;
     }
     // look in oversized
     return FindDynablockDynablocklist(addr, dblist_oversized);
@@ -979,6 +1027,8 @@ void printMapMem()
 
 void addMapMem(uintptr_t begin, uintptr_t end)
 {
+    if(!mapmem)
+        return;
     begin &=~0xfff;
     end = (end&~0xfff)+0xfff; // full page
     // sanitize values
@@ -1016,6 +1066,8 @@ void addMapMem(uintptr_t begin, uintptr_t end)
 }
 void removeMapMem(uintptr_t begin, uintptr_t end)
 {
+    if(!mapmem)
+        return;
     begin &=~0xfff;
     end = (end&~0xfff)+0xfff; // full page
     // sanitize values
@@ -1316,8 +1368,10 @@ uint32_t getProtection(uintptr_t addr)
 {
     if(addr>=(1LL<<48))
         return 0;
+    pthread_mutex_lock(&mutex_prot);
     const uintptr_t idx = (addr>>MEMPROT_SHIFT);
     uint32_t ret = memprot[idx>>16].prot[idx&0xffff];
+    pthread_mutex_unlock(&mutex_prot);
     return ret;
 }
 
@@ -1462,28 +1516,34 @@ void fini_custommem_helper(box64context_t *ctx)
 #ifdef DYNAREC
     if(box64_dynarec) {
         dynarec_log(LOG_DEBUG, "Free global Dynarecblocks\n");
-        for (size_t i=0; i<mmapsize; ++i) {
-            if(mmaplist[i].block)
-                #ifdef USE_MMAP
-                munmap(mmaplist[i].block, mmaplist[i].size);
-                #else
-                box_free(mmaplist[i].block);
-                #endif
-            if(mmaplist[i].dblist) {
-                kh_destroy(dynablocks, mmaplist[i].dblist);
-                mmaplist[i].dblist = NULL;
-            }
-            if(mmaplist[i].helper) {
-                box_free(mmaplist[i].helper);
-                mmaplist[i].helper = NULL;
+        mmaplist_t* head = mmaplist;
+        mmaplist = NULL;
+        while(head) {
+            for (int i=0; i<NCHUNK; ++i) {
+                if(head->chunks[i].block)
+                    #ifdef USE_MMAP
+                    munmap(head->chunks[i].block, head->chunks[i].size);
+                    #else
+                    box_free(head->chunks[i].block);
+                    #endif
+                if(head->chunks[i].dblist) {
+                    kh_destroy(dynablocks, head->chunks[i].dblist);
+                    head->chunks[i].dblist = NULL;
+                }
+                if(head->chunks[i].helper) {
+                    box_free(head->chunks[i].helper);
+                    head->chunks[i].helper = NULL;
+                }
             }
+            mmaplist_t *old = head;
+            head = head->next;
+            free(old);
         }
         if(dblist_oversized) {
             kh_destroy(dynablocks, dblist_oversized);
             dblist_oversized = NULL;
         }
         mmapsize = 0;
-        mmapcap = 0;
 
         box_free(mmaplist);
         for (int i3=0; i3<(1<<DYNAMAP_SHIFT); ++i3)
diff --git a/src/dynarec/arm64/arm64_lock.S b/src/dynarec/arm64/arm64_lock.S
index f1210d17..209a666c 100755
--- a/src/dynarec/arm64/arm64_lock.S
+++ b/src/dynarec/arm64/arm64_lock.S
@@ -21,6 +21,9 @@
 .global arm64_lock_storeifref
 .global arm64_lock_decifnot0b
 .global arm64_lock_storeb
+.global arm64_lock_incif0
+.global arm64_lock_decifnot0
+.global arm64_lock_store
 
 arm64_lock_read_b:
     dmb     ish
@@ -161,3 +164,37 @@ arm64_lock_storeb:
     strb    w1, [x0]
     dmb     ish
     ret
+
+arm64_lock_decifnot0:
+    dmb     ish
+arm64_lock_decifnot0_0:
+    ldaxr   w1, [x0]
+    cmp     w1, #0
+    bne     arm64_lock_decifnot0_exit
+    sub     w3, w1, #1
+    stlxr   w2, w3, [x0]
+    cbnz    w2, arm64_lock_decifnot0_0
+arm64_lock_decifnot0_exit:
+    dmb     ish
+    mov     w0, w1
+    ret
+
+arm64_lock_incif0:
+    dmb     ish
+arm64_lock_incif0_0:
+    ldaxr   w1, [x0]
+    cmp     w1, #0
+    beq     arm64_lock_incif0_exit
+    add     w3, w1, #1
+    stlxr   w2, w3, [x0]
+    cbnz    w2, arm64_lock_incif0_0
+arm64_lock_incif0_exit:
+    dmb     ish
+    mov     w0, w1
+    ret
+
+arm64_lock_store:
+    dmb     ish
+    str     w1, [x0]
+    dmb     ish
+    ret
diff --git a/src/dynarec/arm64/arm64_lock.h b/src/dynarec/arm64/arm64_lock.h
index 2b82319d..8f99321b 100755
--- a/src/dynarec/arm64/arm64_lock.h
+++ b/src/dynarec/arm64/arm64_lock.h
@@ -45,4 +45,13 @@ extern void arm64_lock_decifnot0b(void*p);
 // atomic store (with memory barrier)
 extern void arm64_lock_storeb(void*p, uint8_t b);
 
+// increment atomicaly the int at [p] only if it was 0. Return the old value of [p]
+extern int arm64_lock_incif0(void*p);
+
+// decrement atomicaly the int at [p] (but only if p not 0)
+extern int arm64_lock_decifnot0(void*p);
+
+// atomic store (with memory barrier)
+extern void arm64_lock_store(void*p, uint32_t v);
+
 #endif  //__ARM64_LOCK__H__
\ No newline at end of file
diff --git a/src/dynarec/native_lock.h b/src/dynarec/native_lock.h
index 8be8fe67..09917f37 100755
--- a/src/dynarec/native_lock.h
+++ b/src/dynarec/native_lock.h
@@ -20,6 +20,9 @@
 #define native_lock_storeifnull(A, B)       arm64_lock_storeifnull(A, B)

 #define native_lock_decifnot0b(A)           arm64_lock_decifnot0b(A)

 #define native_lock_storeb(A, B)            arm64_lock_storeb(A, B)

+#define native_lock_incif0(A)               arm64_lock_incif0(A)

+#define native_lock_decifnot0(A)            arm64_lock_decifnot0(A)

+#define native_lock_store(A, B)             arm64_lock_store(A, B)

 

 #else

 #error Unsupported architecture

diff --git a/src/libtools/signals.c b/src/libtools/signals.c
index 657cdad3..394c67db 100755
--- a/src/libtools/signals.c
+++ b/src/libtools/signals.c
@@ -910,6 +910,27 @@ dynarec_log(/*LOG_DEBUG*/LOG_INFO, "Repeated SIGSEGV with Access error on %p for
             glitch_addr = NULL;
             glitch_prot = 0;
         }
+        if(addr && pc && ((prot&(PROT_READ|PROT_WRITE))==(PROT_READ|PROT_WRITE))) {
+            static void* glitch2_pc = NULL;
+            static void* glitch2_addr = NULL;
+            static int glitch2_prot = 0;
+            if((glitch2_pc!=pc || glitch2_addr!=addr || glitch2_prot!=prot)) {
+                dynarec_log(LOG_INFO, "Is that a multi process glitch too?\n");
+                //printf_log(LOG_INFO, "Is that a multi process glitch too?\n");
+                glitch2_pc = pc;
+                glitch2_addr = addr;
+                glitch2_prot = prot;
+                sched_yield();  // give time to the other process
+                refreshProtection((uintptr_t)addr);
+                relockMutex(Locks);
+                sched_yield();  // give time to the other process
+                pthread_mutex_unlock(&mutex_dynarec_prot);
+                return; // try again
+            }
+            glitch2_pc = NULL;
+            glitch2_addr = NULL;
+            glitch2_prot = 0;
+        }
         pthread_mutex_unlock(&mutex_dynarec_prot);
     }
     if(!db_searched)