support multiple sequential VM instances with bootimage build

Until now, the bootimage build hasn't supported using the Java
invocation API to create a VM, destroy it, and create another in the
same process.  Ideally, we would be able to create multiple VMs
simultaneously without any interference between them.  In fact, Avian
is designed to support this for the most part, but there are a few
places we use global, mutable state which prevent this from working.
Most notably, the bootimage is modified in-place at runtime, so the
best we can do without extensive changes is to clean up the bootimage
when the VM is destroyed so it's ready for later instances.  Hence
this commit.

Ultimately, we can move towards a fully reentrant VM by making the
bootimage immutable, but this will require some care to avoid
performance regressions.  Another challenge is our Posix signal
handlers, which currently rely on a global handle to the VM, since you
can't, to my knowledge, pass a context pointer when registering a
signal handler.  Thread local variables won't necessarily help, since
a thread might attatch to more than one VM at a time.

src/compile.cpp

@@ -2011,7 +2011,7 @@ findExceptionHandler(Thread* t, object method, void* ip)
         if (key >= start and key < end) {
           object catchType = arrayBody(t, table, i + 1);
 
-          if (catchType == 0 or instanceOf(t, catchType, t->exception)) {
+          if (exceptionMatch(t, catchType, t->exception)) {
             return compiled + intArrayBody(t, index, (i * 3) + 2);
           }
         }
@@ -8249,7 +8249,7 @@ class SignalHandler: public System::SignalHandler {
           t->exception = vm::root(t, root);
         }
 
-        //printTrace(t, t->exception);
+        // printTrace(t, t->exception);
 
         object continuation;
         findUnwindTarget(t, ip, frame, stack, &continuation);
@@ -9237,6 +9237,52 @@ fixupHeap(MyThread* t UNUSED, uintptr_t* map, unsigned size, uintptr_t* heap)
   }
 }
 
+void
+resetClassRuntimeState(Thread* t, object c, uintptr_t* heap, unsigned heapSize)
+{
+  classRuntimeDataIndex(t, c) = 0;
+
+  if (classArrayElementSize(t, c) == 0) {
+    object staticTable = classStaticTable(t, c);
+    if (staticTable) {
+      for (unsigned i = 0; i < singletonCount(t, staticTable); ++i) {
+        if (singletonIsObject(t, staticTable, i)
+            and (reinterpret_cast<uintptr_t*>
+                 (singletonObject(t, staticTable, i)) < heap or
+                 reinterpret_cast<uintptr_t*>
+                 (singletonObject(t, staticTable, i)) > heap + heapSize))
+        {
+          singletonObject(t, staticTable, i) = 0;
+        }
+      }
+    }
+  }
+
+  if (classMethodTable(t, c)) {
+    for (unsigned i = 0; i < arrayLength(t, classMethodTable(t, c)); ++i) {
+      object m = arrayBody(t, classMethodTable(t, c), i);
+
+      methodNativeID(t, m) = 0;
+      methodRuntimeDataIndex(t, m) = 0;
+
+      if (methodVmFlags(t, m) & ClassInitFlag) {
+        classVmFlags(t, c) |= NeedInitFlag;
+        classVmFlags(t, c) &= ~InitErrorFlag;
+      }
+    }
+  }
+
+  t->m->processor->initVtable(t, c);
+}
+
+void
+resetRuntimeState(Thread* t, object map, uintptr_t* heap, unsigned heapSize)
+{
+  for (HashMapIterator it(t, map); it.hasMore();) {
+    resetClassRuntimeState(t, tripleSecond(t, it.next()), heap, heapSize);
+  }
+}
+
 void
 fixupMethods(Thread* t, object map, BootImage* image UNUSED, uint8_t* code)
 {
@@ -9339,7 +9385,11 @@ boot(MyThread* t, BootImage* image, uint8_t* code)
   // fprintf(stderr, "code from %p to %p\n",
   //         code, code + image->codeSize);
  
-  fixupHeap(t, heapMap, heapMapSizeInWords, heap);
+  static bool fixed = false;
+
+  if (not fixed) {
+    fixupHeap(t, heapMap, heapMapSizeInWords, heap);
+  }
   
   t->m->heap->setImmortalHeap(heap, image->heapSize / BytesPerWord);
 
@@ -9384,11 +9434,30 @@ boot(MyThread* t, BootImage* image, uint8_t* code)
     
   findThunks(t, image, code);
 
-  fixupVirtualThunks(t, code);
+  if (fixed) {
+    resetRuntimeState
+      (t, classLoaderMap(t, root(t, Machine::BootLoader)), heap,
+       image->heapSize);
 
-  fixupMethods
+    resetRuntimeState
-    (t, classLoaderMap(t, root(t, Machine::BootLoader)), image, code);
+      (t, classLoaderMap(t, root(t, Machine::AppLoader)), heap,
-  fixupMethods(t, classLoaderMap(t, root(t, Machine::AppLoader)), image, code);
+       image->heapSize);
+
+    for (unsigned i = 0; i < arrayLength(t, t->m->types); ++i) {
+      resetClassRuntimeState
+        (t, type(t, static_cast<Machine::Type>(i)), heap, image->heapSize);
+    }
+  } else {
+    fixupVirtualThunks(t, code);
+
+    fixupMethods
+      (t, classLoaderMap(t, root(t, Machine::BootLoader)), image, code);
+
+    fixupMethods
+      (t, classLoaderMap(t, root(t, Machine::AppLoader)), image, code);
+  }
+
+  fixed = true;
 
   setRoot(t, Machine::BootstrapClassMap, makeHashMap(t, 0, 0));
 }

src/heap.cpp

@@ -848,13 +848,20 @@ bitset(Context* c UNUSED, void* o)
 void
 free(Context* c, Fixie** fixies)
 {
-  for (Fixie** p = fixies; *p;) {
+  for (Fixie* p = *fixies; p;) {
-    Fixie* f = *p;
+    Fixie* f = p;
+    p = f->next;
 
     if (f->immortal()) {
-      p = &(f->next);
+      if (DebugFixies) {
+        fprintf(stderr, "reset immortal fixie %p\n", f);
+      }
+      memset(f->mask(), 0, Fixie::maskSize(f->size, f->hasMask));
+      f->next = 0;
+      f->handle = 0;
+      f->marked = false;
+      f->dirty = false;
     } else {
-      *p = f->next;
       if (DebugFixies) {
         fprintf(stderr, "free fixie %p\n", f);
       }

src/interpret.cpp

@@ -698,7 +698,7 @@ findExceptionHandler(Thread* t, object method, unsigned ip)
           }
         }
 
-        if (catchType == 0 or instanceOf(t, catchType, t->exception)) {
+        if (exceptionMatch(t, catchType, t->exception)) {
           return eh;
         }
       }

src/jnienv.cpp

@@ -3024,6 +3024,8 @@ boot(Thread* t, uintptr_t*)
   setRoot(t, Machine::OutOfMemoryError,
           makeThrowable(t, Machine::OutOfMemoryErrorType));
 
+  setRoot(t, Machine::Shutdown, makeThrowable(t, Machine::ThrowableType));
+
   setRoot(t, Machine::FinalizerThread, t->m->classpath->makeThread(t, t));
 
   threadDaemon(t, root(t, Machine::FinalizerThread)) = true;

src/machine.cpp

@@ -138,35 +138,37 @@ dispose(Thread* t, Thread* o, bool remove)
 }
 
 void
-joinAll(Thread* m, Thread* o)
+visitAll(Thread* m, Thread* o, void (*visit)(Thread*, Thread*))
 {
   for (Thread* p = o->child; p;) {
     Thread* child = p;
     p = p->peer;
-    joinAll(m, child);
+    visitAll(m, child, visit);
   }
 
-  join(m, o);
+  visit(m, o);
 }
 
 void
-disposeAll(Thread* m, Thread* o)
+disposeNoRemove(Thread* m, Thread* o)
 {
-  for (Thread* p = o->child; p;) {
-    Thread* child = p;
-    p = p->peer;
-    disposeAll(m, child);
-  }
-
   dispose(m, o, false);
 }
 
+void
+interruptDaemon(Thread* m, Thread* o)
+{
+  if (o->flags & Thread::DaemonFlag) {
+    interrupt(m, o);
+  }
+}
+
 void
 turnOffTheLights(Thread* t)
 {
   expect(t, t->m->liveCount == 1);
 
-  joinAll(t, t->m->rootThread);
+  visitAll(t, t->m->rootThread, join);
 
   enter(t, Thread::ExitState);
 
@@ -215,7 +217,7 @@ turnOffTheLights(Thread* t)
 
   Machine* m = t->m;
 
-  disposeAll(t, t->m->rootThread);
+  visitAll(t, t->m->rootThread, disposeNoRemove);
 
   System* s = m->system;
   Heap* h = m->heap;
@@ -2443,6 +2445,7 @@ Machine::Machine(System* system, Heap* heap, Finder* bootFinder,
   collecting(false),
   triedBuiltinOnLoad(false),
   dumpedHeapOnOOM(false),
+  alive(true),
   heapPoolIndex(0)
 {
   heap->setClient(heapClient);
@@ -2694,6 +2697,17 @@ shutDown(Thread* t)
       }
     }
   }
+
+  // interrupt daemon threads and tell them to die
+
+  // todo: be more aggressive about killing daemon threads, e.g. at
+  // any GC point, not just at waits/sleeps
+  { ACQUIRE(t, t->m->stateLock);
+
+    t->m->alive = false;
+
+    visitAll(t, t->m->rootThread, interruptDaemon);
+  }
 }
 
 void
@@ -3247,9 +3261,7 @@ classInitializer(Thread* t, object class_)
     {
       object o = arrayBody(t, classMethodTable(t, class_), i);
 
-      if (vm::strcmp(reinterpret_cast<const int8_t*>("<clinit>"),
+      if (methodVmFlags(t, o) & ClassInitFlag) {
-                     &byteArrayBody(t, methodName(t, o), 0)) == 0)
-      {
         return o;
       }               
     }

src/machine.h

@@ -157,8 +157,7 @@ const unsigned ContinuationFlag = 1 << 11;
 
 // method vmFlags:
 const unsigned ClassInitFlag = 1 << 0;
-const unsigned CompiledFlag = 1 << 1;
+const unsigned ConstructorFlag = 1 << 1;
-const unsigned ConstructorFlag = 1 << 2;
 
 #ifndef JNI_VERSION_1_6
 #define JNI_VERSION_1_6 0x00010006
@@ -1266,6 +1265,7 @@ class Machine {
     ArithmeticException,
     ArrayIndexOutOfBoundsException,
     OutOfMemoryError,
+    Shutdown,
     VirtualFileFinders,
     VirtualFiles
   };
@@ -1322,6 +1322,7 @@ class Machine {
   bool collecting;
   bool triedBuiltinOnLoad;
   bool dumpedHeapOnOOM;
+  bool alive;
   JavaVMVTable javaVMVTable;
   JNIEnvVTable jniEnvVTable;
   uintptr_t* heapPool[ThreadHeapPoolSize];
@@ -1357,6 +1358,9 @@ run(Thread* t, uint64_t (*function)(Thread*, uintptr_t*),
 void
 checkDaemon(Thread* t);
 
+object&
+root(Thread* t, Machine::Root root);
+
 extern "C" uint64_t
 vmRun(uint64_t (*function)(Thread*, uintptr_t*), uintptr_t* arguments,
       void* checkpoint);
@@ -1507,7 +1511,7 @@ class Thread {
 
       vm::run(t, runThread, 0);
 
-      if (t->exception) {
+      if (t->exception and t->exception != root(t, Machine::Shutdown)) {
         printTrace(t, t->exception);
       }
 
@@ -3182,7 +3186,11 @@ wait(Thread* t, object o, int64_t milliseconds)
     bool interrupted = monitorWait(t, m, milliseconds);
 
     if (interrupted) {
-      throwNew(t, Machine::InterruptedExceptionType);
+      if (t->m->alive or (t->flags & Thread::DaemonFlag) == 0) {
+        throwNew(t, Machine::InterruptedExceptionType);
+      } else {
+        throw_(t, root(t, Machine::Shutdown));
+      }
     }
   } else {
     throwNew(t, Machine::IllegalMonitorStateExceptionType);
@@ -3256,6 +3264,14 @@ getAndClearInterrupted(Thread* t, Thread* target)
   }
 }
 
+inline bool
+exceptionMatch(Thread* t, object type, object exception)
+{
+  return type == 0
+    or (exception != root(t, Machine::Shutdown)
+        and instanceOf(t, type, t->exception));
+}
+
 object
 intern(Thread* t, object s);