From b436bd460a10fefedbfc7ec8faa2634d1308e788 Mon Sep 17 00:00:00 2001
From: Joel Dice <joel.dice@gmail.com>
Date: Wed, 3 Apr 2013 16:08:40 -0600
Subject: [PATCH] fix NaN handling in floating point comparisons

---
 src/codegen/target/x86/encode.cpp |  3 +++
 src/compile.cpp                   | 28 ++++++++++++++++++++++++----
 src/interpret.cpp                 | 28 ++++++++++++++++++++++++----
 test/Floats.java                  | 18 ++++++++++++++++++
 4 files changed, 69 insertions(+), 8 deletions(-)

diff --git a/src/codegen/target/x86/encode.cpp b/src/codegen/target/x86/encode.cpp
index f6c97dcac0..e2f1ee287e 100644
--- a/src/codegen/target/x86/encode.cpp
+++ b/src/codegen/target/x86/encode.cpp
@@ -243,6 +243,9 @@ void branch(Context* c, lir::TernaryOperation op, lir::Constant* target) {
 void branchFloat(Context* c, lir::TernaryOperation op, lir::Constant* target) {
   switch (op) {
   case lir::JumpIfFloatEqual:
+    // jp past the je so we don't jump to the target if unordered:
+    c->code.append(0x7a);
+    c->code.append(6);
     conditional(c, 0x84, target);
     break;
 
diff --git a/src/compile.cpp b/src/compile.cpp
index 01d3f5c5f2..fd5ebc1f74 100644
--- a/src/compile.cpp
+++ b/src/compile.cpp
@@ -2467,13 +2467,27 @@ getJClassFromReference(MyThread* t, object pair)
        referenceName(t, pairSecond(t, pair)))));
 }
 
+bool
+isNaN(double v)
+{
+  return fpclassify(v) == FP_NAN;
+}
+
+bool
+isNaN(float v)
+{
+  return fpclassify(v) == FP_NAN;
+}
+
 int64_t
 compareDoublesG(uint64_t bi, uint64_t ai)
 {
   double a = bitsToDouble(ai);
   double b = bitsToDouble(bi);
   
-  if (a < b) {
+  if (isNaN(a) or isNaN(b)) {
+    return 1;
+  } else if (a < b) {
     return -1;
   } else if (a > b) {
     return 1;
@@ -2490,7 +2504,9 @@ compareDoublesL(uint64_t bi, uint64_t ai)
   double a = bitsToDouble(ai);
   double b = bitsToDouble(bi);
   
-  if (a < b) {
+  if (isNaN(a) or isNaN(b)) {
+    return -1;
+  } else if (a < b) {
     return -1;
   } else if (a > b) {
     return 1;
@@ -2507,7 +2523,9 @@ compareFloatsG(uint32_t bi, uint32_t ai)
   float a = bitsToFloat(ai);
   float b = bitsToFloat(bi);
   
-  if (a < b) {
+  if (isNaN(a) or isNaN(b)) {
+    return 1;
+  } if (a < b) {
     return -1;
   } else if (a > b) {
     return 1;
@@ -2524,7 +2542,9 @@ compareFloatsL(uint32_t bi, uint32_t ai)
   float a = bitsToFloat(ai);
   float b = bitsToFloat(bi);
   
-  if (a < b) {
+  if (isNaN(a) or isNaN(b)) {
+    return -1;
+  } if (a < b) {
     return -1;
   } else if (a > b) {
     return 1;
diff --git a/src/interpret.cpp b/src/interpret.cpp
index cc68a4b79f..d222d4c1b5 100644
--- a/src/interpret.cpp
+++ b/src/interpret.cpp
@@ -680,6 +680,18 @@ store(Thread* t, unsigned index)
          BytesPerWord * 2);
 }
 
+bool
+isNaN(double v)
+{
+  return fpclassify(v) == FP_NAN;
+}
+
+bool
+isNaN(float v)
+{
+  return fpclassify(v) == FP_NAN;
+}
+
 uint64_t
 findExceptionHandler(Thread* t, object method, unsigned ip)
 {
@@ -1157,7 +1169,9 @@ interpret3(Thread* t, const int base)
     double b = popDouble(t);
     double a = popDouble(t);
     
-    if (a < b) {
+    if (isNaN(a) or isNaN(b)) {
+      pushInt(t, 1);
+    } if (a < b) {
       pushInt(t, static_cast<unsigned>(-1));
     } else if (a > b) {
       pushInt(t, 1);
@@ -1172,7 +1186,9 @@ interpret3(Thread* t, const int base)
     double b = popDouble(t);
     double a = popDouble(t);
     
-    if (a < b) {
+    if (isNaN(a) or isNaN(b)) {
+      pushInt(t, static_cast<unsigned>(-1));
+    } if (a < b) {
       pushInt(t, static_cast<unsigned>(-1));
     } else if (a > b) {
       pushInt(t, 1);
@@ -1370,7 +1386,9 @@ interpret3(Thread* t, const int base)
     float b = popFloat(t);
     float a = popFloat(t);
     
-    if (a < b) {
+    if (isNaN(a) or isNaN(b)) {
+      pushInt(t, 1);
+    } if (a < b) {
       pushInt(t, static_cast<unsigned>(-1));
     } else if (a > b) {
       pushInt(t, 1);
@@ -1385,7 +1403,9 @@ interpret3(Thread* t, const int base)
     float b = popFloat(t);
     float a = popFloat(t);
     
-    if (a < b) {
+    if (isNaN(a) or isNaN(b)) {
+      pushInt(t, static_cast<unsigned>(-1));
+    } if (a < b) {
       pushInt(t, static_cast<unsigned>(-1));
     } else if (a > b) {
       pushInt(t, 1);
diff --git a/test/Floats.java b/test/Floats.java
index b8b80cbd75..eca59b1bc9 100644
--- a/test/Floats.java
+++ b/test/Floats.java
@@ -329,5 +329,23 @@ public class Floats {
 
     { float v = Float.POSITIVE_INFINITY;
       expect(Long.MAX_VALUE == (long) v); }
+
+    expect(Double.NaN != Double.NaN);
+    expect(! (Double.NaN == Double.NaN));
+
+    { double d = Double.NaN;
+      expect(Double.NaN != d);
+      expect(! (Double.NaN == d));
+      expect(d != d);
+      expect(! (d == d)); }
+
+    expect(Float.NaN != Float.NaN);
+    expect(! (Float.NaN == Float.NaN));
+
+    { float d = Float.NaN;
+      expect(Float.NaN != d);
+      expect(! (Float.NaN == d));
+      expect(d != d);
+      expect(! (d == d)); }
   }
 }