[BOX32] More work on box32 memory management and threads

5 files changed, 62 insertions, 27 deletions

diff --git a/src/custommem.c b/src/custommem.c
index 38caaa03..c0f5c028 100644
--- a/src/custommem.c
+++ b/src/custommem.c
@@ -647,6 +647,8 @@ void* map128_customMalloc(size_t size, int is32bits)
         printf_log(LOG_INFO, "Custommem: Failed to alloc 32bits: allocation %p-%p for 128byte MAP Alloc p_blocks[%d]\n", p, p+allocsize, i);
         #endif
         p_blocks[i].maxfree = allocsize - mapsize;
+        p_blocks[i].is32bits = 0;
+        errno = ENOMEM;
         return NULL;
     }
     #ifdef TRACE_MEMSTAT
@@ -741,6 +743,8 @@ void* map64_customMalloc(size_t size, int is32bits)
 
     if (is32bits && p > (void*)0xffffffffLL) {
         p_blocks[i].maxfree = allocsize - mapsize;
+        p_blocks[i].is32bits = 0;
+        errno = ENOMEM;
         return NULL;
     }
 
@@ -864,6 +868,8 @@ void* internal_customMalloc(size_t size, int is32bits)
         printf_log(LOG_INFO, "Custommem: Failed to alloc 32bits: allocation %p-%p for LIST Alloc p_blocks[%d]\n", p, p+allocsize, i);
         #endif
         p_blocks[i].maxfree = allocsize - sizeof(blockmark_t)*2;
+        p_blocks[i].is32bits = 0;
+        errno = ENOMEM;
         return NULL;
     }
     #ifdef TRACE_MEMSTAT
@@ -1151,6 +1157,7 @@ void* internal_customMemAligned(size_t align, size_t size, int is32bits)
         #endif
         p_blocks[i].is32bits = 0;
         p_blocks[i].maxfree = allocsize - sizeof(blockmark_t)*2;
+        errno = ENOMEM;
         return NULL;
     }
     #ifdef TRACE_MEMSTAT
diff --git a/src/libtools/threads.c b/src/libtools/threads.c
index 71222eca..6fd7ffe6 100644
--- a/src/libtools/threads.c
+++ b/src/libtools/threads.c
@@ -30,6 +30,7 @@
 #endif
 #ifdef BOX32
 #include "box32.h"
+#include "threads32.h"
 #endif
 
 //void _pthread_cleanup_push_defer(void* buffer, void* routine, void* arg);	// declare hidden functions
@@ -125,6 +126,9 @@ int GetStackSize(uintptr_t attr, void** stack, size_t* stacksize)
 }
 
 void my_longjmp(x64emu_t* emu, /*struct __jmp_buf_tag __env[1]*/void *p, int32_t __val);
+#ifdef BOX32
+void my32_longjmp(x64emu_t* emu, /*struct __jmp_buf_tag __env[1]*/void *p, int32_t __val);
+#endif
 
 static pthread_key_t thread_key;
 
@@ -149,7 +153,13 @@ static void emuthread_cancel(void* p)
 	// check cancels threads
 	for(int i=et->cancel_size-1; i>=0; --i) {
 		et->emu->flags.quitonlongjmp = 0;
-		my_longjmp(et->emu, ((x64_unwind_buff_t*)et->cancels[i])->__cancel_jmp_buf, 1);
+		et->emu->quit = 0;
+		#ifdef BOX32
+		if(et->is32bits)
+			my32_longjmp(et->emu, ((i386_unwind_buff_t*)et->cancels[i])->__cancel_jmp_buf, 1);
+		else
+		#endif
+			my_longjmp(et->emu, ((x64_unwind_buff_t*)et->cancels[i])->__cancel_jmp_buf, 1);
 		DynaRun(et->emu);	// will return after a __pthread_unwind_next()
 	}
 	#ifdef BOX32
diff --git a/src/libtools/threads32.c b/src/libtools/threads32.c
index 2c36a532..ce5a4579 100755
--- a/src/libtools/threads32.c
+++ b/src/libtools/threads32.c
@@ -26,6 +26,7 @@
 #include "emu/x64run_private.h"
 #include "x64trace.h"
 #include "bridge.h"
+#include "threads32.h"
 #ifdef DYNAREC
 #include "dynablock.h"
 #endif
@@ -53,32 +54,6 @@ typedef struct threadstack_s {
 	size_t 	stacksize;
 } threadstack_t;
 
-// longjmp / setjmp
-typedef struct jump_buff_i386_s {
- uint32_t save_ebx;
- uint32_t save_esi;
- uint32_t save_edi;
- uint32_t save_ebp;
- uint32_t save_esp;
- uint32_t save_eip;
-} jump_buff_i386_t;
-
-// sigset_t should have the same size on 32bits and 64bits machine (64bits)
-typedef struct __attribute__((packed, aligned(4))) __jmp_buf_tag_s {
-    jump_buff_i386_t __jmpbuf;
-    int              __mask_was_saved;
-    sigset_t         __saved_mask;
-} __jmp_buf_tag_t;
-
-typedef struct i386_unwind_buff_s {
-	struct {
-		jump_buff_i386_t	__cancel_jmp_buf;	
-		int					__mask_was_saved;
-	} __cancel_jmp_buf[1];
-	ptr_t __pad[2];
-	ptr_t __pad3;
-} __attribute__((packed, aligned(4))) i386_unwind_buff_t;
-
 // those are define in thread.c
 emuthread_t* thread_get_et();
 void thread_set_et(emuthread_t* et);
@@ -104,11 +79,14 @@ static void emuthread_cancel(void* p)
 	if(!et)
 		return;
 	// check cancels threads
+	uintptr_t rax = et->emu->regs[_AX].q[0];
 	for(int i=et->cancel_size-1; i>=0; --i) {
 		et->emu->flags.quitonlongjmp = 0;
+		et->emu->quit = 0;
 		my32_longjmp(et->emu, ((i386_unwind_buff_t*)et->cancels[i])->__cancel_jmp_buf, 1);
 		DynaRun(et->emu);	// will return after a __pthread_unwind_next()
 	}
+	et->emu->regs[_AX].q[0] = rax;
 	box_free(et->cancels);
 	et->cancels=NULL;
 	et->cancel_size = et->cancel_cap = 0;
@@ -128,6 +106,7 @@ static void* pthread_routine(void* p)
 	// call the function
 	emuthread_t *et = (emuthread_t*)p;
 	thread_set_et(et);
+	et->is32bits = 1;
 	et->emu->type = EMUTYPE_MAIN;
 	et->self = (uintptr_t)pthread_self();
 	et->hself = to_hash(et->self);
@@ -309,6 +288,7 @@ EXPORT void my32___pthread_unregister_cancel(x64emu_t* emu, i386_unwind_buff_t*
 			if(i!=et->cancel_size-1)
 				memmove(et->cancels+i, et->cancels+i+1, sizeof(i386_unwind_buff_t*)*(et->cancel_size-i-1));
 			et->cancel_size--;
+			return;
 		}
 	}
 }
diff --git a/src/libtools/threads32.h b/src/libtools/threads32.h
index 00a527d3..9bdce018 100644
--- a/src/libtools/threads32.h
+++ b/src/libtools/threads32.h
@@ -12,4 +12,30 @@ typedef struct fake_pthread_mutext_s {
 } fake_phtread_mutex_t;
 #define KIND_SIGN	0xbad001
 
+// longjmp / setjmp
+typedef struct jump_buff_i386_s {
+ uint32_t save_ebx;
+ uint32_t save_esi;
+ uint32_t save_edi;
+ uint32_t save_ebp;
+ uint32_t save_esp;
+ uint32_t save_eip;
+} jump_buff_i386_t;
+
+// sigset_t should have the same size on 32bits and 64bits machine (64bits)
+typedef struct __attribute__((packed, aligned(4))) __jmp_buf_tag_i386_s {
+    jump_buff_i386_t __jmpbuf;
+    int              __mask_was_saved;
+    sigset_t         __saved_mask;
+} __jmp_buf_tag_i386_t;
+
+typedef struct i386_unwind_buff_s {
+	struct {
+		jump_buff_i386_t	__cancel_jmp_buf;	
+		int					__mask_was_saved;
+	} __cancel_jmp_buf[1];
+	ptr_t __pad[2];
+	ptr_t __pad3;
+} __attribute__((packed, aligned(4))) i386_unwind_buff_t;
+
 #endif //__THREADS_32_H_
\ No newline at end of file
diff --git a/src/os/os_linux.c b/src/os/os_linux.c
index d1a940b5..5c1ccac4 100644
--- a/src/os/os_linux.c
+++ b/src/os/os_linux.c
@@ -10,6 +10,8 @@
 #include <stdarg.h>
 #include <stdlib.h>
 #include <errno.h>
+#include <sys/resource.h>
+#include <malloc.h>
 
 #include "os.h"
 #include "signals.h"
@@ -133,6 +135,16 @@ const char* GetNativeName(void* p)
 void PersonalityAddrLimit32Bit(void)
 {
     personality(ADDR_LIMIT_32BIT);
+    /*struct rlimit l;
+    if(getrlimit(RLIMIT_DATA, &l)<0) return;  // failed
+    if(l.rlim_max>3*1024*1024*1024LL) {
+        l.rlim_cur = 3*1024*1024*1024LL;
+        setrlimit(RLIMIT_DATA, &l);
+    }*/
+    // setting 32bits malloc options
+    mallopt(M_ARENA_TEST, 2);
+    mallopt(M_ARENA_MAX, 2);
+    mallopt(M_MMAP_THRESHOLD, 128*1024);
 }
 
 int IsAddrElfOrFileMapped(uintptr_t addr)