[DYNAREC] Optimized HotPage handling, and made it lockless

4 files changed, 47 insertions, 23 deletions

diff --git a/src/custommem.c b/src/custommem.c
index 3aae1b90..b6cfa043 100644
--- a/src/custommem.c
+++ b/src/custommem.c
@@ -1174,15 +1174,14 @@ int IsInHotPage(uintptr_t addr) {
     if(addr<=(1LL<<48))
         return 0;
     int idx = (addr>>MEMPROT_SHIFT)>>16;
-    if(!memprot[idx].hot)
+    uint8_t *block = memprot[idx].hot;
+    if(!block)
         return 0;
     int base = (addr>>MEMPROT_SHIFT)&0xffff;
-    if(!memprot[idx].hot[base])
+    if(!block[base])
         return 0;
     // decrement hot
-    pthread_mutex_lock(&mutex_prot);
-    memprot[idx].hot[base]-=1;
-    pthread_mutex_unlock(&mutex_prot);
+    native_lock_decifnot0b(&block[base]);
     return 1;
 }
 
@@ -1196,39 +1195,36 @@ int AreaInHotPage(uintptr_t start, uintptr_t end_) {
         //dynarec_log(LOG_DEBUG, "00\n");
         return 0;
     }
+    int ret = 0;
     for (uintptr_t i=idx; i<=end; ++i) {
         uint8_t *block = memprot[i>>16].hot;
-        uint32_t hot = block?block[i&0xffff]:0;
-        if(hot) {
-            // decrement hot
-            pthread_mutex_lock(&mutex_prot);
-            block[i&0xffff]-=1;
-            pthread_mutex_unlock(&mutex_prot);
-            //dynarec_log(LOG_DEBUG, "1\n");
-            return 1;
+        if(block) {
+            uint32_t hot = block[i&0xffff];
+            if(hot) {
+                // decrement hot
+                native_lock_decifnot0b(&block[i&0xffff]);
+                //dynarec_log(LOG_DEBUG, "1\n");
+                ret = 1;
+            }
+        } else {
+            i+=0xffff;
         }
     }
     //dynarec_log(LOG_DEBUG, "0\n");
-    return 0;
+    return ret;
 
 }
 
 void AddHotPage(uintptr_t addr) {
     int idx = (addr>>MEMPROT_SHIFT)>>16;
     int base = (addr>>MEMPROT_SHIFT)&0xffff;
-    pthread_mutex_lock(&mutex_prot);
     if(!memprot[idx].hot) {
             uint8_t* newblock = box_calloc(1<<16, sizeof(uint8_t));
-#if 0 //def ARM64   //disabled for now, not usefull with the mutex
-            if (native_lock_storeifref(&memprot[i], newblock, memprot_default) != newblock) {
+            if (native_lock_storeifnull(&memprot[idx].hot, newblock) != newblock) {
                 box_free(newblock);
             }
-#else
-            memprot[idx].hot = newblock;
-#endif
     }
-    memprot[idx].hot[base] = HOTPAGE_STEP;
-    pthread_mutex_unlock(&mutex_prot);
+    native_lock_storeb(&memprot[idx].hot[base], HOTPAGE_STEP);
 }
 #endif
 
diff --git a/src/dynarec/arm64/arm64_lock.S b/src/dynarec/arm64/arm64_lock.S
index 92132bad..f1210d17 100755
--- a/src/dynarec/arm64/arm64_lock.S
+++ b/src/dynarec/arm64/arm64_lock.S
@@ -19,7 +19,8 @@
 .global arm64_lock_xchg_d
 .global arm64_lock_storeifnull
 .global arm64_lock_storeifref
-
+.global arm64_lock_decifnot0b
+.global arm64_lock_storeb
 
 arm64_lock_read_b:
     dmb     ish
@@ -141,3 +142,22 @@ arm64_lock_storeifref_exit:
     dmb     ish
     mov     x0, x3
     ret
+
+arm64_lock_decifnot0b:
+    dmb     ish
+arm64_lock_decifnot0b_0:
+    ldaxrb  w1, [x0]
+    cmp     w1, #0
+    beq     arm64_lock_decifnot0b_exit
+    sub     w1, w1, #1
+    stlxrb  w2, w1, [x0]
+    cbnz    w2, arm64_lock_decifnot0b_0
+arm64_lock_decifnot0b_exit:
+    dmb     ish
+    ret
+
+arm64_lock_storeb:
+    dmb     ish
+    strb    w1, [x0]
+    dmb     ish
+    ret
diff --git a/src/dynarec/arm64/arm64_lock.h b/src/dynarec/arm64/arm64_lock.h
index 76a1551c..2b82319d 100755
--- a/src/dynarec/arm64/arm64_lock.h
+++ b/src/dynarec/arm64/arm64_lock.h
@@ -39,4 +39,10 @@ extern void* arm64_lock_storeifnull(void*p, void* val);
 // Atomicaly store value to [p] only if [p] is ref. Return new [p] value (so val or old)
 extern void* arm64_lock_storeifref(void*p, void* val, void* ref);
 
+// decrement atomicaly the byte at [p] (but only if p not 0)
+extern void arm64_lock_decifnot0b(void*p);
+
+// atomic store (with memory barrier)
+extern void arm64_lock_storeb(void*p, uint8_t b);
+
 #endif  //__ARM64_LOCK__H__
\ No newline at end of file
diff --git a/src/dynarec/native_lock.h b/src/dynarec/native_lock.h
index e2070901..8be8fe67 100755
--- a/src/dynarec/native_lock.h
+++ b/src/dynarec/native_lock.h
@@ -18,6 +18,8 @@
 #define native_lock_xchg_d(A, B)            arm64_lock_xchg_d(A, B)

 #define native_lock_storeifref(A, B, C)     arm64_lock_storeifref(A, B, C)

 #define native_lock_storeifnull(A, B)       arm64_lock_storeifnull(A, B)

+#define native_lock_decifnot0b(A)           arm64_lock_decifnot0b(A)

+#define native_lock_storeb(A, B)            arm64_lock_storeb(A, B)

 

 #else

 #error Unsupported architecture