Reworked exit, unloading libs and running Fini as it should (plus a workaround for nvidia driver not unloading)

8 files changed, 155 insertions, 32 deletions

diff --git a/src/custommem.c b/src/custommem.c
index a87ac1e9..72c7f2fb 100644
--- a/src/custommem.c
+++ b/src/custommem.c
@@ -1262,7 +1262,7 @@ void refreshProtection(uintptr_t addr)
     uintptr_t bend;
     if (rb_get_end(memprot, addr, &prot, &bend)) {
         int ret = mprotect((void*)(addr&~(box64_pagesize-1)), box64_pagesize, prot&~PROT_CUSTOM);
-printf_log(LOG_INFO, "refreshProtection(%p): %p/0x%x (ret=%d/%s)\n", (void*)addr, (void*)(addr&~(box64_pagesize-1)), prot, ret, ret?strerror(errno):"ok");
+        dynarec_log(LOG_DEBUG, "refreshProtection(%p): %p/0x%x (ret=%d/%s)\n", (void*)addr, (void*)(addr&~(box64_pagesize-1)), prot, ret, ret?strerror(errno):"ok");
     }
     mutex_unlock(&mutex_prot);
 }
diff --git a/src/emu/x64int3.c b/src/emu/x64int3.c
index c78cdd0a..10bd1c95 100644
--- a/src/emu/x64int3.c
+++ b/src/emu/x64int3.c
@@ -48,7 +48,9 @@ x64emu_t* x64emu_fork(x64emu_t* emu, int forktype)
         iFpppp_t forkpty = (iFpppp_t)emu->forkpty_info->f;
         v = forkpty(emu->forkpty_info->amaster, emu->forkpty_info->name, emu->forkpty_info->termp, emu->forkpty_info->winp);
         emu->forkpty_info = NULL;
-    } else
+    } else if(forktype==3)
+        v = vfork();
+    else
         v = fork();
     /*if(type == EMUTYPE_MAIN)
         thread_set_emu(emu);*/
@@ -59,10 +61,6 @@ x64emu_t* x64emu_fork(x64emu_t* emu, int forktype)
         for (int i=0; i<my_context->atfork_sz; --i)
             if(my_context->atforks[i].parent)
                 EmuCall(emu, my_context->atforks[i].parent);
-        if(forktype==3) {
-            // vfork, the parent wait the end or execve of the son
-            waitpid(v, NULL, WEXITED);
-        }
 
     } else if(v==0) {
         ResetSegmentsCache(emu);
diff --git a/src/librarian/librarian.c b/src/librarian/librarian.c
index 9fe8a472..52e4def3 100644
--- a/src/librarian/librarian.c
+++ b/src/librarian/librarian.c
@@ -37,13 +37,12 @@ void FreeLibrarian(lib_t **maplib, x64emu_t *emu)
 
     library_t* owner = (*maplib)->owner;
     (*maplib)->owner = NULL;    // to avoid recursive free...
-
-    /*if((*maplib)->ownlibs && (*maplib)->libsz) {
+    if((*maplib)->ownlibs && (*maplib)->libsz) {
         for(int i=0; i<(*maplib)->libsz; ++i) {
             printf_dump(LOG_DEBUG, "Unloading %s\n", (*maplib)->libraries[i]->name);
             DecRefCount(&(*maplib)->libraries[i], emu);
         }
-    }*/
+    }
     box_free((*maplib)->libraries);
     (*maplib)->libraries = NULL;
 
diff --git a/src/librarian/library.c b/src/librarian/library.c
index e03264c5..eb34ccbb 100644
--- a/src/librarian/library.c
+++ b/src/librarian/library.c
@@ -98,11 +98,15 @@ void WrappedLib_CommonInit(library_t *lib) {
     lib->w.mydatamap = kh_init(datamap);
 }
 
+static void initDummyLib(library_t *lib);
+
 void EmuLib_Fini(library_t* lib)
 {
 }
 void WrappedLib_FinishFini(library_t* lib)
 {
+    if(lib->type!=LIB_WRAPPED)
+        return;
     if(lib->w.lib)
         dlclose(lib->w.lib);
     lib->w.lib = NULL;
@@ -115,6 +119,9 @@ void WrappedLib_FinishFini(library_t* lib)
     }
     FreeBridge(&lib->w.bridge);
 }
+void DummyLib_Fini(library_t* lib)
+{
+}
 
 int WrappedLib_GetWeak(library_t* lib, const char* name, uintptr_t *offs, uintptr_t *sz, size_t asked_size, int* weak, int version, const char* vername, int local, const char* defver)
 {
@@ -148,6 +155,11 @@ int EmuLib_GetWeak(library_t* lib, const char* name, uintptr_t *offs, uintptr_t
     }
     return 0;
 }
+int DummyLib_GetWeak(library_t* lib, const char* name, uintptr_t *offs, uintptr_t *sz, size_t asked_size, int* weak, int version, const char* vername, int local, const char* defver)
+{
+    return 0;
+}
+
 int WrappedLib_GetGlobal(library_t* lib, const char* name, uintptr_t *offs, uintptr_t *sz, size_t asked_size, int* weak, int version, const char* vername, int local, const char* defver)
 {
     // ignoring asked size on wrapped libs
@@ -178,6 +190,10 @@ int EmuLib_GetGlobal(library_t* lib, const char* name, uintptr_t *offs, uintptr_
     }
     return 0;
 }
+int DummyLib_GetGlobal(library_t* lib, const char* name, uintptr_t *offs, uintptr_t *sz, size_t asked_size, int* weak, int version, const char* vername, int local, const char* defver)
+{
+    return 0;
+}
 int EmuLib_GetLocal(library_t* lib, const char* name, uintptr_t *offs, uintptr_t *sz, size_t asked_size, int* weak, int version, const char* vername, int local, const char* defver)
 {
     // ignoring asked size on wrapped libs
@@ -197,6 +213,11 @@ int WrappedLib_GetLocal(library_t* lib, const char* name, uintptr_t *offs, uintp
     (void)lib; (void)name; (void)offs; (void)sz; (void)version; (void)vername; (void)local;
     return 0;
 }
+int DummyLib_GetLocal(library_t* lib, const char* name, uintptr_t *offs, uintptr_t *sz, size_t asked_size, int* weak, int version, const char* vername, int local, const char* defver)
+{
+    (void)lib; (void)name; (void)offs; (void)sz; (void)version; (void)vername; (void)local;
+    return 0;
+}
 
 static void initWrappedLib(library_t *lib, box64context_t* context) {
     int nb = sizeof(wrappedlibs) / sizeof(wrappedlib_t);
@@ -346,6 +367,18 @@ static void initEmulatedLib(const char* path, library_t *lib, box64context_t* co
         }
 }
 
+static void initDummyLib(library_t *lib)
+{
+    memset(lib, 0, sizeof(library_t));
+    lib->fini = DummyLib_Fini;
+    lib->getglobal = DummyLib_GetGlobal;
+    lib->getweak = DummyLib_GetWeak;
+    lib->getlocal = DummyLib_GetLocal;
+    lib->name = NULL;
+    lib->path = NULL;
+    lib->type = LIB_UNNKNOW;
+}
+
 static const char* essential_libs[] = {
     "libc.so.6", "libpthread.so.0", "librt.so.1", "libGL.so.1", "libGL.so", "libX11.so.6", 
     "libasound.so.2", "libdl.so.2", "libm.so.6", "libbsd.so.0", "libutil.so.1", "libresolv.so.2", "libresolv.so", 
@@ -364,10 +397,35 @@ static int isEssentialLib(const char* name) {
     return 0;
 }
 
+#define NLIBS       16
+typedef struct lib_brick_s {
+    library_t           libs[NLIBS];
+    struct lib_brick_s* next;
+} lib_brick_t;
+
+static lib_brick_t* lib_head = NULL;
+static lib_brick_t* cur_brick = NULL;
+static size_t cur_lib = 0;
+static size_t lib_cap = 0;
+
 library_t *NewLibrary(const char* path, box64context_t* context, elfheader_t* verneeded)
 {
     printf_dump(LOG_DEBUG, "Trying to load \"%s\"\n", path);
-    library_t *lib = (library_t*)box_calloc(1, sizeof(library_t));
+    //library_t *lib = (library_t*)box_calloc(1, sizeof(library_t));
+    if(cur_lib==lib_cap) {
+        lib_brick_t *new_brick = calloc(1, sizeof(lib_brick_t));
+        if(!lib_head)
+            cur_brick = lib_head = new_brick;
+        else {
+            cur_brick->next = new_brick;
+            cur_brick = new_brick;
+        }
+        lib_cap += NLIBS;
+    }
+    library_t *lib = &cur_brick->libs[cur_lib%NLIBS];
+    ++cur_lib;
+    memset(lib, 0, sizeof(library_t));
+
     lib->path = box_realpath(path, NULL);
     if(!lib->path)
         lib->path = box_strdup(path);
@@ -382,7 +440,8 @@ library_t *NewLibrary(const char* path, box64context_t* context, elfheader_t* ve
         if(strstr(lib->name, "libpulse.so")==lib->name || strstr(lib->name, "libpulse-simple.so")==lib->name) {
             box_free(lib->name);
             box_free(lib->path);
-            box_free(lib);
+            initDummyLib(lib);
+            //box_free(lib);
             return NULL;
         }
     }
@@ -390,7 +449,8 @@ library_t *NewLibrary(const char* path, box64context_t* context, elfheader_t* ve
         if(strstr(lib->name, "libvulkan.so")==lib->name) {
             box_free(lib->name);
             box_free(lib->path);
-            box_free(lib);
+            initDummyLib(lib);
+            //box_free(lib);
             return NULL;
         }
     }
@@ -419,7 +479,8 @@ library_t *NewLibrary(const char* path, box64context_t* context, elfheader_t* ve
     {
         box_free(lib->name);
         box_free(lib->path);
-        box_free(lib);
+        initDummyLib(lib);
+        //box_free(lib);
         return NULL;
     }
     lib->gbridgemap = kh_init(bridgemap);
@@ -496,7 +557,7 @@ void Free1Library(library_t **the_lib, x64emu_t* emu)
 
     library_t* lib = *the_lib;
 
-    printf_dump(LOG_DEBUG, "Free1Library %s\n", lib->name);
+    printf_dump(LOG_DEBUG, "Free1Library %s\n", lib->name?:"???");
     // remove lib from maplib/local_maplib...
     if(my_context) {
         MapLibRemoveLib(my_context->maplib, lib);
@@ -559,10 +620,11 @@ void Free1Library(library_t **the_lib, x64emu_t* emu)
             kh_destroy(symbol2map, lib->w.symbol2map);
     }
 
-    // remove frim the dlopen collection
+    // remove from the dlopen collection
     if(lib->dlopen)
         RemoveDlopen(the_lib, lib->dlopen);
-    box_free(lib);
+    initDummyLib(lib);
+    //box_free(lib);
     if(*the_lib == lib)
         *the_lib = NULL;
 }
@@ -610,7 +672,7 @@ int IsSameLib(library_t* lib, const char* path)
 }
 int GetLibWeakSymbolStartEnd(library_t* lib, const char* name, uintptr_t* start, uintptr_t* end, size_t size, int* weak, int version, const char* vername, int local, const char* defver)
 {
-    if(!lib || !name[0])
+    if(!name[0] || !lib || (lib->type==LIB_UNNKNOW))
         return 0;
     khint_t k;
     kh_bridgemap_t *map = local?lib->lbridgemap:((*weak)?lib->wbridgemap:lib->gbridgemap);
@@ -637,7 +699,7 @@ int GetLibWeakSymbolStartEnd(library_t* lib, const char* name, uintptr_t* start,
 }
 int GetLibGlobalSymbolStartEnd(library_t* lib, const char* name, uintptr_t* start, uintptr_t* end, size_t size, int* weak, int version, const char* vername, int local, const char* defver)
 {
-    if(!name[0] || !lib)
+    if(!name[0] || !lib || (lib->type==LIB_UNNKNOW))
         return 0;
     khint_t k;
     kh_bridgemap_t *map = local?lib->lbridgemap:((*weak)?lib->wbridgemap:lib->gbridgemap);
@@ -664,7 +726,7 @@ int GetLibGlobalSymbolStartEnd(library_t* lib, const char* name, uintptr_t* star
 }
 int GetLibLocalSymbolStartEnd(library_t* lib, const char* name, uintptr_t* start, uintptr_t* end, size_t size, int* weak, int version, const char* vername, int local, const char* defver)
 {
-    if(!name[0])
+    if(!name[0] || !lib || (lib->type==LIB_UNNKNOW))
         return 0;
     khint_t k;
     // check first if already in the map
@@ -1111,23 +1173,28 @@ void setNeededLibs(library_t* lib, int n, ...)
     va_end (va);
 }
 
+#define LIB_MAXCNT 255
 void IncRefCount(library_t* lib, x64emu_t* emu)
 {
     if(!lib || lib->type==LIB_UNNKNOW)
         return;
     switch (lib->type) {
         case LIB_WRAPPED:
+            if(lib->w.refcnt==LIB_MAXCNT)
+                return;
             ++lib->w.refcnt;
-            /*if(lib->w.needed)
+            if(lib->w.needed)
                 for(int i=0; i<lib->w.needed->size; ++i) {
                     IncRefCount(lib->w.needed->libs[i], emu);
-                }*/
+                }
             break;
         case LIB_EMULATED:
+            if(lib->e.elf->refcnt==LIB_MAXCNT)
+                return;
             ++lib->e.elf->refcnt;
-            /*if(lib->e.elf->needed)
+            if(lib->e.elf->needed)
                 for(int i=0; i<lib->e.elf->needed->size; ++i)   // some libs may not be loaded yet
-                    IncRefCount(lib->e.elf->needed->libs[i], emu);*/
+                    IncRefCount(lib->e.elf->needed->libs[i], emu);
     }
 }
 
@@ -1144,6 +1211,8 @@ int DecRefCount(library_t** lib, x64emu_t* emu)
     int freed = 0;
     switch ((*lib)->type) {
         case LIB_WRAPPED:
+            if((*lib)->w.refcnt==LIB_MAXCNT)
+                return ret;
             needed = (*lib)->w.needed;
             ret=--(*lib)->w.refcnt;
             if(!ret) {
@@ -1153,6 +1222,8 @@ int DecRefCount(library_t** lib, x64emu_t* emu)
             }
             break;
         case LIB_EMULATED:
+            if((*lib)->e.elf->refcnt==LIB_MAXCNT)
+                return ret;
             needed = (*lib)->e.elf->needed;
             ret=--(*lib)->e.elf->refcnt;
             if(!ret) {
@@ -1164,9 +1235,9 @@ int DecRefCount(library_t** lib, x64emu_t* emu)
             }
             break;
     }
-    /*if(needed)
+    if(needed)
         for(int i=0; i<needed->size; ++i)
-            DecRefCount(&needed->libs[i], emu);*/
+            DecRefCount(&needed->libs[i], emu);
     if(freed)
         free_neededlib(needed);
     return ret;
diff --git a/src/libtools/signals.c b/src/libtools/signals.c
index a7253de5..1c654983 100644
--- a/src/libtools/signals.c
+++ b/src/libtools/signals.c
@@ -1221,6 +1221,7 @@ static pthread_mutex_t mutex_dynarec_prot = PTHREAD_ERRORCHECK_MUTEX_INITIALIZER
 #endif
 
 extern int box64_quit;
+extern int box64_exit_code;
 
 void my_box64signalhandler(int32_t sig, siginfo_t* info, void * ucntx)
 {
@@ -1237,7 +1238,7 @@ void my_box64signalhandler(int32_t sig, siginfo_t* info, void * ucntx)
         longjmp(SIG_JMPBUF, 1);
     if((sig==SIGSEGV || sig==SIGBUS) && box64_quit) {
         printf_log(LOG_INFO, "Sigfault/Segbus while quitting, exiting silently\n");
-        exit(0);    // Hack, segfault while quiting, exit silently
+        _exit(box64_exit_code);    // Hack, segfault while quiting, exit silently
     }
     ucontext_t *p = (ucontext_t *)ucntx;
     void* addr = (void*)info->si_addr;  // address that triggered the issue
diff --git a/src/main.c b/src/main.c
index 5ec7f910..a17e3eb8 100644
--- a/src/main.c
+++ b/src/main.c
@@ -39,6 +39,7 @@
 
 box64context_t *my_context = NULL;
 int box64_quit = 0;
+int box64_exit_code = 0;
 int box64_log = LOG_INFO; //LOG_NONE;
 int box64_dump = 0;
 int box64_nobanner = 0;
@@ -240,6 +241,7 @@ void my_child_fork()
         if(!ftrace_name)
             fclose(ftrace);
         openFTrace(NULL);
+        printf_log(/*LOG_DEBUG*/LOG_INFO, "Forked child of %s\n", GetLastApplyName());
     }
 }
 
@@ -1352,6 +1354,8 @@ void endBox64()
     RunElfFini(my_context->elfs[0], emu);
     FreeLibrarian(&my_context->local_maplib, emu);    // unload all libs
     FreeLibrarian(&my_context->maplib, emu);    // unload all libs
+    void closeAllDLOpenned();
+    closeAllDLOpenned();    // close residual dlopenned libs
     #if 0
     // waiting for all thread except this one to finish
     int this_thread = GetTID();
@@ -1378,8 +1382,8 @@ void endBox64()
                 if(tid!=this_thread) {
                     if(attempt>4000) {
                         printf_log(LOG_INFO, "Stop waiting for remaining thread %04d\n", tid);
-                        // enough wait, kill all thread!
-                        syscall(__NR_tgkill, pid, tid, SIGABRT);
+                        // enough wait, exit
+                        _exit(box64_exit_code);
                     } else {
                         running = 1;
                         ++attempt;
diff --git a/src/wrapped/wrappedlibc.c b/src/wrapped/wrappedlibc.c
index 347a38e6..583f9b2e 100644
--- a/src/wrapped/wrappedlibc.c
+++ b/src/wrapped/wrappedlibc.c
@@ -522,7 +522,7 @@ pid_t EXPORT my_fork(x64emu_t* emu)
 {
     #if 1
     emu->quit = 1;
-    emu->fork = 3;  // use regular fork...
+    emu->fork = 1;  // use regular fork...
     return 0;
     #else
     // execute atforks prepare functions, in reverse order
@@ -3372,6 +3372,17 @@ EXPORT int my_register_printf_type(x64emu_t* emu, void* f)
 }
 
 extern int box64_quit;
+extern int box64_exit_code;
+void endBox64();
+#if !defined(ANDROID)
+static void* timed_exit_thread(void*)
+{
+    // this is a workaround for some NVidia drivers on ARM64 that may freeze at exit
+    // waiting on a pthread_cond_destroy
+    usleep(500000); // wait 1/2 a second
+    _exit(box64_exit_code); // force exit, something is wrong
+}
+#endif
 EXPORT void my_exit(x64emu_t* emu, int code)
 {
     if(emu->flags.quitonexit) {
@@ -3381,7 +3392,12 @@ EXPORT void my_exit(x64emu_t* emu, int code)
         return;
     }
     emu->quit = 1;
-    box64_quit = 1;
+    box64_exit_code = code;
+    endBox64();
+#if !defined(ANDROID)
+    pthread_t exit_thread;
+    pthread_create(&exit_thread, NULL, timed_exit_thread, NULL);
+#endif
     exit(code);
 }
 
@@ -3472,6 +3488,7 @@ EXPORT char my___libc_single_threaded = 0;
         setNeededLibs(lib, NEEDED_LIBS);
 
 #define CUSTOM_FINI \
-    freeMy();
+    freeMy();       \
+    return;     // do not unload...
 
 #include "wrappedlib_init.h"
diff --git a/src/wrapped/wrappedlibdl.c b/src/wrapped/wrappedlibdl.c
index 7ccf0e1c..11278c1c 100644
--- a/src/wrapped/wrappedlibdl.c
+++ b/src/wrapped/wrappedlibdl.c
@@ -176,8 +176,16 @@ void* my_dlopen(x64emu_t* emu, void *filename, int flag)
             return NULL;
         }
         dlopened = (lib==NULL);
-        // Then open the lib
+        // cleanup the old deferredInit state
+        int old_deferredInit = my_context->deferredInit;
         my_context->deferredInit = 1;
+        elfheader_t** old_deferredInitList = my_context->deferredInitList;
+        my_context->deferredInitList = NULL;
+        int old_deferredInitSz = my_context->deferredInitSz;
+        int old_deferredInitCap = my_context->deferredInitCap;
+        my_context->deferredInitSz = my_context->deferredInitCap = 0;
+
+        // Then open the lib
         int bindnow = (!box64_musl && (flag&0x2))?1:0;
         needed_libs_t *tmp = new_neededlib(1);
         tmp->names[0] = rfilename;
@@ -187,11 +195,21 @@ void* my_dlopen(x64emu_t* emu, void *filename, int flag)
                 dl->last_error = box_calloc(1, 129);
             snprintf(dl->last_error, 129, "Cannot dlopen(\"%s\"/%p, %X)\n", rfilename, filename, flag);
             RemoveNeededLib(NULL, is_local, tmp, my_context, emu);
+            if(my_context->deferredInitList)
+                box_free(my_context->deferredInitList);
+            my_context->deferredInit = old_deferredInit;
+            my_context->deferredInitList = old_deferredInitList;
+            my_context->deferredInitSz = old_deferredInitSz;
+            my_context->deferredInitCap = old_deferredInitCap;
             return NULL;
         }
         free_neededlib(tmp);
         lib = GetLibInternal(rfilename);
         RunDeferredElfInit(emu);
+        my_context->deferredInit = old_deferredInit;
+        my_context->deferredInitList = old_deferredInitList;
+        my_context->deferredInitSz = old_deferredInitSz;
+        my_context->deferredInitCap = old_deferredInitCap;
     } else {
         // check if already dlopenned...
         for (size_t i=MIN_NLIB; i<dl->lib_sz; ++i) {
@@ -585,6 +603,21 @@ EXPORT int my__dl_find_object(x64emu_t* emu, void* addr, my_dl_find_object_t* re
     return -1;
 }
 
+void closeAllDLOpenned()
+{
+    dlprivate_t *dl = my_context->dlprivate;
+    if(dl) {
+        x64emu_t* emu = thread_get_emu();
+        for(size_t i=0; i<dl->lib_sz; ++i)
+            while(dl->dllibs[i].count) {
+                printf_log(LOG_DEBUG, "  closing %s\n", dl->dllibs[i].lib->name);
+                my_dlclose(emu, (void*)(i+1));
+            }
+    }
+}
+
+#define CUSTOM_FINI \
+    closeAllDLOpenned();
 
 // define all standard library functions
 #include "wrappedlib_init.h"