x86_64: Fix compares with NaNs.

Comparisons with unordered doubles was broken, NaNs always
compare unequal (and unordered) to everything, including
to itself.

tests/tcctest.c

@@ -87,6 +87,7 @@ void builtin_test(void);
 void weak_test(void);
 void global_data_test(void);
 void cmp_comparison_test(void);
+void math_cmp_test(void);
 
 int fib(int n);
 void num(int n);
@@ -594,6 +595,7 @@ int main(int argc, char **argv)
     weak_test();
     global_data_test();
     cmp_comparison_test();
+    math_cmp_test();
     return 0; 
 }
 
@@ -2592,3 +2594,67 @@ void cmp_comparison_test(void)
   compare_comparisons (&s);
   return 0;
 }
+
+int fcompare (double a, double b, int code)
+{
+  switch (code) {
+    case 0: return a == b;
+    case 1: return a != b;
+    case 2: return a < b;
+    case 3: return a >= b;
+    case 4: return a > b;
+    case 5: return a <= b;
+  }
+}
+
+void math_cmp_test(void)
+{
+  double nan = 0.0/0.0;
+  double one = 1.0;
+  double two = 2.0;
+  int comp = 0;
+#define bug(a,b,op,iop,part) printf("Test broken: %s %s %s %s %d\n", #a, #b, #op, #iop, part)
+
+  /* This asserts that "a op b" is _not_ true, but "a iop b" is true.
+     And it does this in various ways so that all code generation paths
+     are checked (generating inverted tests, or non-inverted tests, or
+     producing a 0/1 value without jumps (that's done in the fcompare
+     function).  */
+#define FCMP(a,b,op,iop,code) \
+  if (fcompare (a,b,code))    \
+    bug (a,b,op,iop,1); \
+  if (a op b) \
+    bug (a,b,op,iop,2); \
+  if (a iop b) \
+    ; \
+  else \
+    bug (a,b,op,iop,3); \
+  if ((a op b) || comp) \
+    bug (a,b,op,iop,4); \
+  if ((a iop b) || comp) \
+    ; \
+  else \
+    bug (a,b,op,iop,5);
+
+  /* Equality tests.  */
+  FCMP(nan, nan, ==, !=, 0);
+  FCMP(one, two, ==, !=, 0);
+  FCMP(one, one, !=, ==, 1);
+  /* Non-equality is a bit special.  */
+  if (!fcompare (nan, nan, 1))
+    bug (nan, nan, !=, ==, 6);
+
+  /* Relational tests on numbers.  */
+  FCMP(two, one, <, >=, 2);
+  FCMP(one, two, >=, <, 3);
+  FCMP(one, two, >, <=, 4);
+  FCMP(two, one, <=, >, 5);
+
+  /* Relational tests on NaNs.  Note that the inverse op here is
+     always !=, there's no operator in C that is equivalent to !(a < b),
+     when NaNs are involved, same for the other relational ops.  */
+  FCMP(nan, nan, <, !=, 2);
+  FCMP(nan, nan, >=, !=, 3);
+  FCMP(nan, nan, >, !=, 4);
+  FCMP(nan, nan, <=, !=, 5);
+}

x86_64-gen.c

@@ -429,7 +429,18 @@ void load(int r, SValue *sv)
             gen_modrm(r, VT_LOCAL, sv->sym, fc);
         } else if (v == VT_CMP) {
             orex(0,r,0,0);
-            oad(0xb8 + REG_VALUE(r), 0); /* mov $0, r */
+	    if ((fc & ~0x100) != TOK_NE)
+              oad(0xb8 + REG_VALUE(r), 0); /* mov $0, r */
+	    else
+              oad(0xb8 + REG_VALUE(r), 1); /* mov $1, r */
+	    if (fc & 0x100)
+	      {
+	        /* This was a float compare.  If the parity bit is
+		   set the result was unordered, meaning false for everything
+		   except TOK_NE, and true for TOK_NE.  */
+		fc &= ~0x100;
+		o(0x037a + (REX_BASE(r) << 8));
+	      }
             orex(0,r,0, 0x0f); /* setxx %br */
             o(fc);
             o(0xc0 + REG_VALUE(r));
@@ -1161,6 +1172,24 @@ int gtst(int inv, int t)
     v = vtop->r & VT_VALMASK;
     if (v == VT_CMP) {
         /* fast case : can jump directly since flags are set */
+	if (vtop->c.i & 0x100)
+	  {
+	    /* This was a float compare.  If the parity flag is set
+	       the result was unordered.  For anything except != this
+	       means false and we don't jump (anding both conditions).
+	       For != this means true (oring both).
+	       Take care about inverting the test.  We need to jump
+	       to our target if the result was unordered and test wasn't NE,
+	       otherwise if unordered we don't want to jump.  */
+	    vtop->c.i &= ~0x100;
+	    if (!inv == (vtop->c.i != TOK_NE))
+	      o(0x067a);  /* jp +6 */
+	    else
+	      {
+	        g(0x0f);
+		t = psym(0x8a, t); /* jp t */
+	      }
+	  }
         g(0x0f);
         t = psym((vtop->c.i - 16) ^ inv, t);
     } else if (v == VT_JMP || v == VT_JMPI) {
@@ -1469,7 +1498,7 @@ void gen_opf(int op)
 
             vtop--;
             vtop->r = VT_CMP;
-            vtop->c.i = op;
+            vtop->c.i = op | 0x100;
         } else {
             /* no memory reference possible for long double operations */
             if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {