fix race condition leading to deadlock on exit

There is a delay between when we tell the OS to start a thread and
when it actually starts, and during that time a thread might
mistakenly think it was the last to exit, try to shut down the VM, and
then block in joinAll when it finds it wasn't the last one after all.

The solution is to increment Machine::liveCount and add the new thread
to the process tree before starting it -- all while holding
Machine::stateLock for atomicity.  This helps guarantee that when
liveCount is one, we can be sure there's really only one thread
running or staged to run.

src/machine.cpp

@@ -2258,7 +2258,7 @@ Thread::Thread(Machine* m, object javaThread, Thread* parent):
   vtable(&(m->jniEnvVTable)),
   m(m),
   parent(parent),
-  peer((parent ? parent->child : 0)),
+  peer(0),
   child(0),
   waitNext(0),
   state(NoState),
@@ -2331,9 +2331,6 @@ Thread::init()
     javaThread = m->classpath->makeThread(this, 0);
 
     threadPeer(this, javaThread) = reinterpret_cast<jlong>(this);
-  } else {
-    peer = parent->child;
-    parent->child = this;
   }
 
   expect(this, m->system->success(m->system->make(&lock)));
@@ -2538,6 +2535,7 @@ enter(Thread* t, Thread::State s)
         -- t->m->daemonCount;
       }
     }
+
     t->state = s;
 
     t->m->stateLock->notifyAll(t->systemThread);
@@ -3739,7 +3737,10 @@ collect(Thread* t, Heap::CollectionType type)
 
     m->finalizeThread = m->processor->makeThread(m, javaThread, m->rootThread);
     
+    addThread(t, m->finalizeThread);
+
     if (not startThread(t, m->finalizeThread)) {
+      removeThread(t, m->finalizeThread);
       m->finalizeThread = 0;
     }
   }

src/machine.h

@@ -1753,14 +1753,45 @@ startThread(Thread* t, Thread* p)
   return t->m->system->success(t->m->system->start(&(p->runnable)));
 }
 
+inline void
+addThread(Thread* t, Thread* p)
+{
+  ACQUIRE_RAW(t, t->m->stateLock);
+
+  assert(t, p->state == Thread::NoState);
+
+  p->state = Thread::IdleState;
+  ++ t->m->liveCount;
+
+  p->peer = p->parent->child;
+  p->parent->child = p;
+}
+
+inline void
+removeThread(Thread* t, Thread* p)
+{
+  ACQUIRE_RAW(t, t->m->stateLock);
+
+  assert(t, p->state == Thread::IdleState);
+
+  -- t->m->liveCount;
+
+  t->m->stateLock->notifyAll(t->systemThread);
+
+  p->parent->child = p->peer;
+}
+
 inline Thread*
 startThread(Thread* t, object javaThread)
 {
   Thread* p = t->m->processor->makeThread(t->m, javaThread, t);
 
+  addThread(t, p);
+
   if (startThread(t, p)) {
     return p;
   } else {
+    removeThread(t, p);
     return 0;
   }
 }
@@ -1791,6 +1822,8 @@ attachThread(Machine* m, bool daemon)
   Thread* t = m->processor->makeThread(m, 0, m->rootThread);
   m->system->attach(&(t->runnable));
 
+  addThread(t, t);
+
   enter(t, Thread::ActiveState);
 
   t->javaThread = m->classpath->makeThread(t, m->rootThread);