Added back some isnan testing to integer conversion (converting a nan, infinite or overflow value to int is UB, and so should be avoided for max cross platform compatibility)

4 files changed, 59 insertions, 46 deletions

diff --git a/src/emu/x64run0f.c b/src/emu/x64run0f.c
index 7c30a086..8cd5b312 100644
--- a/src/emu/x64run0f.c
+++ b/src/emu/x64run0f.c
@@ -212,12 +212,12 @@ uintptr_t Run0F(x64emu_t *emu, rex_t rex, uintptr_t addr, int *step)
             GETEX(0);

             GETGM;

             tmp64s = EX->f[1];

-            if (tmp64s==(int32_t)tmp64s)

+            if (tmp64s==(int32_t)tmp64s && !isnanf(EX->f[1]))

                 GM->sd[1] = (int32_t)tmp64s;

             else

                 GM->sd[1] = INT32_MIN;

             tmp64s = EX->f[0];

-            if (tmp64s==(int32_t)tmp64s)

+            if (tmp64s==(int32_t)tmp64s && !isnanf(EX->f[0]))

                 GM->sd[0] = (int32_t)tmp64s;

             else

                 GM->sd[0] = INT32_MIN;

@@ -228,20 +228,23 @@ uintptr_t Run0F(x64emu_t *emu, rex_t rex, uintptr_t addr, int *step)
             GETEX(0);

             GETGM;

             for(int i=1; i>=0; --i) {

-                switch(emu->mxcsr.f.MXCSR_RC) {

-                    case ROUND_Nearest:

-                        tmp64s = nearbyintf(EX->f[i]);

-                        break;

-                    case ROUND_Down:

-                        tmp64s = floorf(EX->f[i]);

-                        break;

-                    case ROUND_Up:

-                        tmp64s = ceilf(EX->f[i]);

-                        break;

-                    case ROUND_Chop:

-                        tmp64s = EX->f[i];

-                        break;

-                }

+                if(isnanf(EX->f[i]))

+                    tmp64s = INT32_MIN;

+                else

+                    switch(emu->mxcsr.f.MXCSR_RC) {

+                        case ROUND_Nearest:

+                            tmp64s = nearbyintf(EX->f[i]);

+                            break;

+                        case ROUND_Down:

+                            tmp64s = floorf(EX->f[i]);

+                            break;

+                        case ROUND_Up:

+                            tmp64s = ceilf(EX->f[i]);

+                            break;

+                        case ROUND_Chop:

+                            tmp64s = EX->f[i];

+                            break;

+                    }

                 if (tmp64s==(int32_t)tmp64s)

                     GM->sd[i] = (int32_t)tmp64s;

                 else

diff --git a/src/emu/x64run660f.c b/src/emu/x64run660f.c
index 506da029..47691ecb 100644
--- a/src/emu/x64run660f.c
+++ b/src/emu/x64run660f.c
@@ -205,12 +205,12 @@ uintptr_t Run660F(x64emu_t *emu, rex_t rex, uintptr_t addr)
         GETEX(0);

         GETGM;

         tmp64s = EX->d[0];

-        if (tmp64s==(int32_t)tmp64s)

+        if (tmp64s==(int32_t)tmp64s && !isnan(EX->d[0]))

             GX->sd[0] = (int32_t)tmp64s;

         else

             GX->sd[0] = INT32_MIN;

         tmp64s = EX->d[1];

-        if (tmp64s==(int32_t)tmp64s)

+        if (tmp64s==(int32_t)tmp64s && !isnan(EX->d[1]))

             GX->sd[1] = (int32_t)tmp64s;

         else

             GX->sd[1] = INT32_MIN;

@@ -238,7 +238,7 @@ uintptr_t Run660F(x64emu_t *emu, rex_t rex, uintptr_t addr)
                 break;

         }

         for(int i=0; i<2; ++i)

-            if (tmp64s==(int32_t)i64[i])

+            if (i64[i]==(int32_t)i64[i] && !isnan(EX->d[i]))

                 GM->sd[i] = (int32_t)i64[i];

             else

                 GM->sd[i] = INT32_MIN;

@@ -1084,20 +1084,23 @@ uintptr_t Run660F(x64emu_t *emu, rex_t rex, uintptr_t addr)
         GETEX(0);

         GETGX;

         for(int i=0; i<4; ++i) {

-            switch(emu->mxcsr.f.MXCSR_RC) {

-                case ROUND_Nearest:

-                    tmp64s = nearbyintf(EX->f[i]);

-                    break;

-                case ROUND_Down:

-                    tmp64s = floorf(EX->f[i]);

-                    break;

-                case ROUND_Up:

-                    tmp64s = ceilf(EX->f[i]);

-                    break;

-                case ROUND_Chop:

-                    tmp64s = EX->f[i];

-                    break;

-            }

+            if(isnanf(EX->f[i]))

+                tmp64s = INT32_MIN;

+            else

+                switch(emu->mxcsr.f.MXCSR_RC) {

+                    case ROUND_Nearest:

+                        tmp64s = nearbyintf(EX->f[i]);

+                        break;

+                    case ROUND_Down:

+                        tmp64s = floorf(EX->f[i]);

+                        break;

+                    case ROUND_Up:

+                        tmp64s = ceilf(EX->f[i]);

+                        break;

+                    case ROUND_Chop:

+                        tmp64s = EX->f[i];

+                        break;

+                }

             if (tmp64s==(int32_t)tmp64s) {

                 GX->sd[i] = (int32_t)tmp64s;

             } else {

diff --git a/src/emu/x64runf20f.c b/src/emu/x64runf20f.c
index eb94958d..10da2f09 100644
--- a/src/emu/x64runf20f.c
+++ b/src/emu/x64runf20f.c
@@ -338,12 +338,12 @@ uintptr_t RunF20F(x64emu_t *emu, rex_t rex, uintptr_t addr, int *step)
                 tmp64s1 = EX->d[1];

                 break;

         }

-        if (tmp64s0==(int32_t)tmp64s0) {

+        if (tmp64s0==(int32_t)tmp64s0 && !isnan(EX->d[0])) {

             GX->sd[0] = (int32_t)tmp64s0;

         } else {

             GX->sd[0] = INT32_MIN;

         }

-        if (tmp64s1==(int32_t)tmp64s1) {

+        if (tmp64s1==(int32_t)tmp64s1 && !isnan(EX->d[1])) {

             GX->sd[1] = (int32_t)tmp64s1;

         } else {

             GX->sd[1] = INT32_MIN;

diff --git a/src/emu/x64runf30f.c b/src/emu/x64runf30f.c
index e2830c4a..d5b70b20 100644
--- a/src/emu/x64runf30f.c
+++ b/src/emu/x64runf30f.c
@@ -97,12 +97,12 @@ uintptr_t RunF30F(x64emu_t *emu, rex_t rex, uintptr_t addr)
         nextop = F8;

         GETEX(0);

         GETGD;

-        if (rex.w)

-            if(isnanf(EX->f[0]) || isinff(EX->f[0]) || EX->f[0]>0x7fffffffffffffffLL)

+        if (rex.w) {

+            if(isnanf(EX->f[0]) || isinff(EX->f[0]) || EX->f[0]>(float)0x7fffffffffffffffLL)

                 GD->q[0] = 0x8000000000000000LL;

             else

                 GD->sq[0] = EX->f[0];

-        else {

+        } else {

             if(isnanf(EX->f[0]) || isinff(EX->f[0]) || EX->f[0]>0x7fffffff)

                 GD->dword[0] = 0x80000000;

             else

@@ -115,7 +115,7 @@ uintptr_t RunF30F(x64emu_t *emu, rex_t rex, uintptr_t addr)
         GETEX(0);

         GETGD;

         if(rex.w) {

-            if(isnanf(EX->f[0]) || isinff(EX->f[0]) || EX->f[0]>0x7fffffffffffffffLL)

+            if(isnanf(EX->f[0]) || isinff(EX->f[0]) || EX->f[0]>(float)0x7fffffffffffffffLL)

                 GD->q[0] = 0x8000000000000000LL;

             else

                 switch(emu->mxcsr.f.MXCSR_RC) {

@@ -133,23 +133,27 @@ uintptr_t RunF30F(x64emu_t *emu, rex_t rex, uintptr_t addr)
                         break;

                 }

         } else {

-            if(isnanf(EX->f[0]) || isinff(EX->f[0]) || EX->f[0]>0x7fffffff)

-                GD->dword[0] = 0x80000000;

+            if(isnanf(EX->f[0]))

+                tmp64s = INT32_MIN;

             else

                 switch(emu->mxcsr.f.MXCSR_RC) {

                     case ROUND_Nearest:

-                        GD->sdword[0] = nearbyintf(EX->f[0]);

+                        tmp64s = nearbyintf(EX->f[0]);

                         break;

                     case ROUND_Down:

-                        GD->sdword[0] = floorf(EX->f[0]);

+                        tmp64s = floorf(EX->f[0]);

                         break;

                     case ROUND_Up:

-                        GD->sdword[0] = ceilf(EX->f[0]);

+                        tmp64s = ceilf(EX->f[0]);

                         break;

                     case ROUND_Chop:

-                        GD->sdword[0] = EX->f[0];

+                        tmp64s = EX->f[0];

                         break;

                 }

+            if (tmp64s==(int32_t)tmp64s)

+                GD->sdword[0] = (int32_t)tmp64s;

+            else

+                GD->sdword[0] = INT32_MIN;

             GD->dword[1] = 0;

         }

         break;

@@ -196,7 +200,10 @@ uintptr_t RunF30F(x64emu_t *emu, rex_t rex, uintptr_t addr)
         GETEX(0);

         GETGX;

         for(int i=0; i<4; ++i) {

-            tmp64s = EX->f[i];

+            if(isnanf(EX->f[i]))

+                tmp64s = INT32_MIN;

+            else

+                tmp64s = EX->f[i];

             if (tmp64s==(int32_t)tmp64s) {

                 GX->sd[i] = (int32_t)tmp64s;

             } else {