early sketch of continuation support

src/assembler.h

@@ -258,7 +258,7 @@ class Assembler {
     virtual int thread() = 0;
     virtual int returnLow() = 0;
     virtual int returnHigh() = 0;
-    virtual int virtualCallClass() = 0;
+    virtual int virtualCallTarget() = 0;
     virtual int virtualCallIndex() = 0;
 
     virtual bool condensedAddressing() = 0;

src/compile-x86.S

@@ -47,23 +47,70 @@ vmInvoke:
 
    // copy arguments into place
    movq   $0,%r9
-   jmp    LOCAL(test)
+   jmp    LOCAL(vmInvoke_argumentTest)
 
-LOCAL(loop):
+LOCAL(vmInvoke_argumentLoop):
    movq   (%rdx,%r9,1),%r8
    movq   %r8,(%rsp,%r9,1)
    addq   $8,%r9
 
-LOCAL(test):
+LOCAL(vmInvoke_argumentTest):
    cmpq   %rcx,%r9
-   jb     LOCAL(loop)
+   jb     LOCAL(vmInvoke_argumentLoop)
    
    // call function
    call   *%rsi
    
+LOCAL(vmInvoke_returnAddress):
    // restore stack pointer
    movq   %rbp,%rsp
+
+   // call the next continuation frame, if any
+   movq   THREAD_CONTINUATION(%rbx),%rcx
+   cmpq   $0,%rcx
+   je     LOCAL(vmInvoke_exit)
+
+   movq   CONTINUATION_SIZE(%rcx),%rsi
+	 subq   %rsi,%rsp
    
+   movq   CONTINUATION_BODY(%rcx),%rdi
+   
+   movq   $0,%r9
+   jmp    LOCAL(vmInvoke_continuationTest)
+
+LOCAL(vmInvoke_continuationLoop):
+   movq   (%rdi,%r9,1),%r8
+   movq   %r8,(%rsp,%r9,1)
+   addq   $8,%r9
+   
+LOCAL(vmInvoke_continuationTest):
+   cmpq   %rsi,%r9
+   jb     LOCAL(vmInvoke_continuationLoop)
+   
+   movq   CONTINUATION_RETURN_ADDRESS_OFFSET(%rcx),%rdi
+   movq   LOCAL(vmInvoke_returnAddress),(%rsp,%rdi,1)
+   
+   movq   CONTINUATION_FRAME_POINTER_OFFSET(%rcx),%rdi
+   movq   %rbp,(%rsp,%rdi,1)
+   subq   %rdi,%rbp
+   
+   movq   CONTINUATION_NEXT(%rcx),%rdi
+   movq   %rdi,THREAD_CONTINUATION(%rbx)
+
+   // call the continuation unless we're handling an exception
+   movq   THREAD_EXCEPTION(%rbx),%rsi
+   cmpq   $0,%rsi
+   je     *CONTINUATION_ADDRESS(%rcx)
+
+   // we're handling an exception - call the exception handler instead
+   movq   $0,THREAD_EXCEPTION(%rbx)
+   movq   THREAD_EXCEPTION_STACK(%rbx),%rsp
+   movq   THREAD_EXCEPTION_OFFSET(%rbx),%rdi
+   movq   %rsi,(%rsp,%rdi,1)
+   
+   jmp    *THREAD_EXCEPTION_HANDLER(%rbx)
+
+LOCAL(vmInvoke_exit):
    // restore callee-saved registers
    movq   %rsp,%r9
    subq   $48,%r9
@@ -77,7 +124,24 @@ LOCAL(test):
    // return
    popq   %rbp
    ret
-
+   
+.globl vmCallWithContinuation
+vmCallWithContinuation:
+   // %rdi: thread
+   // %rsi: address
+   // %rdx: targetObject
+   // %rcx: continuation
+   // %r8 : base
+   // %r9 : stack
+   
+   movq   %rdi,%rdx
+   movq   %r8,%rbp
+   movq   %r9,%rsp
+   movq   LOCAL(vmInvoke_returnAddress),(%rsp)
+   movq   %rcx,8(%rsp)
+   movq   %rdx,16(%rsp)
+   jmp    *%rsi
+   
 #elif defined __i386__
 
 #  if defined __APPLE__ || defined __MINGW32__ || defined __CYGWIN32__

src/compile.cpp

@@ -27,7 +27,7 @@ vmCall();
 
 namespace {
 
-const bool DebugCompile = true;
+const bool DebugCompile = false;
 const bool DebugNatives = false;
 const bool DebugCallTable = false;
 const bool DebugMethodTree = false;
@@ -49,6 +49,7 @@ class MyThread: public Thread {
       t(t),
       base(t->base),
       stack(t->stack),
+      continuation(t->continuation),
       nativeMethod(0),
       targetMethod(0),
       next(t->trace)
@@ -61,13 +62,14 @@ class MyThread: public Thread {
     ~CallTrace() {
       t->stack = stack;
       t->base = base;
+      t->continuation = continuation;
       t->trace = next;
     }
 
     MyThread* t;
-    void* ip;
     void* base;
     void* stack;
+    object continuation;
     object nativeMethod;
     object targetMethod;
     CallTrace* next;
@@ -78,8 +80,12 @@ class MyThread: public Thread {
     ip(0),
     base(0),
     stack(0),
+    continuation(0),
+    exceptionStack(0),
+    exceptionOffset(0),
+    exceptionHandler(0),
     tailAddress(0),
-    virtualCallClass(0),
+    virtualCallTarget(0),
     virtualCallIndex(0),
     trace(0),
     reference(0),
@@ -91,8 +97,12 @@ class MyThread: public Thread {
   void* ip;
   void* base;
   void* stack;
+  object continuation;
+  void* exceptionStack;
+  uintptr_t exceptionOffset;
+  void* exceptionHandler;
   void* tailAddress;
-  void* virtualCallClass;
+  void* virtualCallTarget;
   uintptr_t virtualCallIndex;
   CallTrace* trace;
   Reference* reference;
@@ -110,7 +120,8 @@ parameterOffset(MyThread* t, object method)
 object
 resolveThisPointer(MyThread* t, void* stack, object method)
 {
-  return reinterpret_cast<object*>(stack)[parameterOffset(t, method)];
+  return reinterpret_cast<object*>(stack)
+    [t->arch->frameFooterSize() + t->arch->frameReturnAddressSize()];
 }
 
 object
@@ -213,6 +224,7 @@ class MyStackWalker: public Processor::StackWalker {
 
     virtual void visit(Heap::Visitor* v) {
       v->visit(&(walker->method_));
+      v->visit(&(walker->continuation));
     }
 
     MyStackWalker* walker;
@@ -226,6 +238,7 @@ class MyStackWalker: public Processor::StackWalker {
     stack(t->stack),
     trace(t->trace),
     method_(0),
+    continuation(t->continuation),
     protector(this)
   { }
 
@@ -272,7 +285,10 @@ class MyStackWalker: public Processor::StackWalker {
           method_ = methodForIp(t, ip_);
           if (method_) {
             state = Method;
+          } else if (continuation) {
+            state = Continuation;
           } else if (trace) {
+            continuation = trace->continuation;
             stack = trace->stack;
             base = trace->base;
             ip_ = t->arch->frameIp(stack);
@@ -290,6 +306,7 @@ class MyStackWalker: public Processor::StackWalker {
         }
         break;
 
+      case Continuation:
       case Method:
       case NativeMethod:
         return true;
@@ -305,6 +322,10 @@ class MyStackWalker: public Processor::StackWalker {
     
   void next() {
     switch (state) {
+    case Continuation:
+      continuation = continuationNext(t, continuation);
+      break;
+
     case Method:
       t->arch->nextFrame(&stack, &base);
       ip_ = t->arch->frameIp(stack);
@@ -329,6 +350,10 @@ class MyStackWalker: public Processor::StackWalker {
 
   virtual int ip() {
     switch (state) {
+    case Continuation:
+      return continuationAddress(t, continuation)
+        - methodCompiled(t, continuationMethod(t, continuation));
+
     case Method:
       return reinterpret_cast<intptr_t>(ip_) - methodCompiled(t, method_);
         
@@ -358,6 +383,7 @@ class MyStackWalker: public Processor::StackWalker {
   void* stack;
   MyThread::CallTrace* trace;
   object method_;
+  object continuation;
   MyProtector protector;
 };
 
@@ -403,21 +429,30 @@ localOffset(MyThread* t, int v, object method)
   return offset;
 }
 
+int
+localOffsetFromStack(MyThread* t, int index, object method)
+{
+  return localOffset(t, index, method)
+    + (t->arch->frameReturnAddressSize() * BytesPerWord);
+}
+
 object*
 localObject(MyThread* t, void* stack, object method, unsigned index)
 {
   return reinterpret_cast<object*>
-    (static_cast<uint8_t*>(stack)
+    (static_cast<uint8_t*>(stack) + localOffsetFromStack(t, index, method));
-     + localOffset(t, index, method)
+}
-     + (t->arch->frameReturnAddressSize() * BytesPerWord));
+
+int
+stackOffsetFromFrame(MyThread* t, object method)
+{
+  return alignedFrameSize(t, method) + t->arch->frameHeaderSize();
 }
 
 void*
 stackForFrame(MyThread* t, void* frame, object method)
 {
-  return static_cast<void**>(frame)
+  return static_cast<void**>(frame) - stackOffsetFromFrame(t, method);
-    - alignedFrameSize(t, method)
-    - t->arch->frameHeaderSize();
 }
 
 class PoolElement: public Promise {
@@ -684,6 +719,41 @@ class Context {
   MyProtector protector;
 };
 
+unsigned
+translateLocalIndex(Context* context, unsigned footprint, unsigned index)
+{
+  unsigned parameterFootprint = methodParameterFootprint
+    (context->thread, context->method);
+
+  if (index < parameterFootprint) {
+    return parameterFootprint - index - footprint;
+  } else {
+    return index;
+  }
+}
+
+void
+initLocal(Context* context, unsigned footprint, unsigned index)
+{
+  context->compiler->initLocal
+    (footprint, translateLocalIndex(context, footprint, index));
+}
+
+Compiler::Operand*
+loadLocal(Context* context, unsigned footprint, unsigned index)
+{
+  return context->compiler->loadLocal
+    (footprint, translateLocalIndex(context, footprint, index));
+}
+
+void
+storeLocal(Context* context, unsigned footprint, Compiler::Operand* value,
+           unsigned index)
+{
+  context->compiler->storeLocal
+    (footprint, value, translateLocalIndex(context, footprint, index));
+}
+
 class Frame {
  public:
   enum StackType {
@@ -1055,36 +1125,36 @@ class Frame {
 
   void loadInt(unsigned index) {
     assert(t, index < localSize());
-    pushInt(c->loadLocal(1, index));
+    pushInt(loadLocal(context, 1, index));
   }
 
   void loadLong(unsigned index) {
     assert(t, index < static_cast<unsigned>(localSize() - 1));
-    pushLong(c->loadLocal(2, index));
+    pushLong(loadLocal(context, 2, index));
   }
 
   void loadObject(unsigned index) {
     assert(t, index < localSize());
-    pushObject(c->loadLocal(1, index));
+    pushObject(loadLocal(context, 1, index));
   }
 
   void storeInt(unsigned index) {
-    c->storeLocal(1, popInt(), index);
+    storeLocal(context, 1, popInt(), index);
     storedInt(index);
   }
 
   void storeLong(unsigned index) {
-    c->storeLocal(2, popLong(), index);
+    storeLocal(context, 2, popLong(), index);
     storedLong(index);
   }
 
   void storeObject(unsigned index) {
-    c->storeLocal(1, popObject(), index);
+    storeLocal(context, 1, popObject(), index);
     storedObject(index);
   }
 
   void storeObjectOrAddress(unsigned index) {
-    c->storeLocal(1, popQuiet(1), index);
+    storeLocal(context, 1, popQuiet(1), index);
 
     assert(t, sp >= 1);
     assert(t, sp - 1 >= localSize());
@@ -1261,22 +1331,25 @@ insertCallNode(MyThread* t, object node);
 void*
 findExceptionHandler(Thread* t, object method, void* ip)
 {
-  object table = codeExceptionHandlerTable(t, methodCode(t, method));
+  if (t->exception) {
-  if (table) {
+    object table = codeExceptionHandlerTable(t, methodCode(t, method));
-    object index = arrayBody(t, table, 0);
+    if (table) {
+      object index = arrayBody(t, table, 0);
       
-    uint8_t* compiled = reinterpret_cast<uint8_t*>(methodCompiled(t, method));
+      uint8_t* compiled = reinterpret_cast<uint8_t*>
+        (methodCompiled(t, method));
 
-    for (unsigned i = 0; i < arrayLength(t, table) - 1; ++i) {
+      for (unsigned i = 0; i < arrayLength(t, table) - 1; ++i) {
-      unsigned start = intArrayBody(t, index, i * 3);
+        unsigned start = intArrayBody(t, index, i * 3);
-      unsigned end = intArrayBody(t, index, (i * 3) + 1);
+        unsigned end = intArrayBody(t, index, (i * 3) + 1);
-      unsigned key = difference(ip, compiled) - 1;
+        unsigned key = difference(ip, compiled) - 1;
 
-      if (key >= start and key < end) {
+        if (key >= start and key < end) {
-        object catchType = arrayBody(t, table, i + 1);
+          object catchType = arrayBody(t, table, i + 1);
 
-        if (catchType == 0 or instanceOf(t, catchType, t->exception)) {
+          if (catchType == 0 or instanceOf(t, catchType, t->exception)) {
-          return compiled + intArrayBody(t, index, (i * 3) + 2);
+            return compiled + intArrayBody(t, index, (i * 3) + 2);
+          }
         }
       }
     }
@@ -1285,6 +1358,23 @@ findExceptionHandler(Thread* t, object method, void* ip)
   return 0;
 }
 
+void
+releaseLock(MyThread* t, object method)
+{
+  if (methodFlags(t, method) & ACC_SYNCHRONIZED) {
+    object lock;
+    if (methodFlags(t, method) & ACC_STATIC) {
+      lock = methodClass(t, method);
+    } else {
+      lock = *localObject
+        (t, stackForFrame(t, stack, method), method,
+         savedTargetIndex(t, method));
+    }
+    
+    release(t, lock);
+  }
+}
+
 void
 findUnwindTarget(MyThread* t, void** targetIp, void** targetBase,
                  void** targetStack)
@@ -1323,19 +1413,98 @@ findUnwindTarget(MyThread* t, void** targetIp, void** targetBase,
         t->arch->nextFrame(&stack, &base);
         ip = t->arch->frameIp(stack);
 
-        if (methodFlags(t, method) & ACC_SYNCHRONIZED) {
+        releaseLock(t, method);
-          object lock;
-          if (methodFlags(t, method) & ACC_STATIC) {
-            lock = methodClass(t, method);
-          } else {
-            lock = *localObject
-              (t, stackForFrame(t, stack, method), method,
-               savedTargetIndex(t, method));
-          }
-    
-          release(t, lock);
-        }
       }
+    } else {
+      *targetIp = ip;
+      *targetBase = base;
+      *targetStack = static_cast<void**>(stack)
+        + t->arch->frameReturnAddressSize();
+
+      while (t->continuation) {
+        void* handler = findExceptionHandler
+          (t, continuationMethod(t, t->continuation),
+           continuationAddress(t, t->continuation));
+
+        if (handler) {
+          t->exceptionHandler = handler;
+
+          t->exceptionStack = stackForFrame
+            (t, *targetStack, continuationMethod(t, t->continuation));
+
+          t->exceptionOffset = localOffset(t, localSize(t, method), method);
+          break;
+        } else {
+          releaseLock(t, continuationMethod(t, t->continuation));
+        }
+
+        t->continuation = continuationNext(t, t->continuation)
+      }
+    }
+  }
+}
+
+object
+makeCurrentContinuation(MyThread* t, void** targetIp, void** targetBase,
+                        void** targetStack)
+{
+  void* ip = t->ip;
+  void* base = t->base;
+  void* stack = t->stack;
+  if (ip == 0) {
+    ip = t->arch->frameIp(stack);
+  }
+
+  object target = t->trace->targetMethod;
+  PROTECT(t, target);
+
+  object first = 0;
+  PROTECT(t, first);
+
+  object last = 0;
+  PROTECT(t, last);
+
+  *targetIp = 0;
+  while (*targetIp == 0) {
+    object method = methodForIp(t, ip);
+    if (method) {
+      PROTECT(t, method);
+
+      void** top = static_cast<void**>(stack) - t->arch->frameHeader();;
+      unsigned argumentFootprint
+        = t->arch->argumentFootprint(methodParameterFootprint(t, target));
+      unsigned alignment = t->arch->stackAlignmentInWords();
+      if (argumentFootprint > alignment) {
+        top += argumentFootprint - alignment;
+      }
+
+      t->arch->nextFrame(&stack, &base);
+
+      void** bottom = static_cast<void**>(stack);
+      unsigned frameSize = bottom - top;
+      unsigned totalSize = frameSize
+        + t->arch->frameHeaderSize()
+        + t->arch->frameFooterSize()
+        + t->arch->argumentFootprint(methodParameterFootprint(t, method));
+
+      object c = makeContinuation
+        (t, 0, method, ip,
+         ((frameSize + t->arch->returnAddressOffset()) * BytesPerWord),
+         ((frameSize + t->arch->framePointerOffset()) * BytesPerWord),
+         totalSize);
+
+      memcpy(&continuationBody(t, c, 0), top, totalSize * BytesPerWord);
+
+      if (last) {
+        set(t, last, ContinuationNext, c);
+      } else {
+        first = c;
+      }
+      last = c;
+
+      ip = t->arch->frameIp(stack);
+
+      target = method;
     } else {
       *targetIp = ip;
       *targetBase = base;
@@ -1343,6 +1512,11 @@ findUnwindTarget(MyThread* t, void** targetIp, void** targetBase,
         + t->arch->frameReturnAddressSize();
     }
   }
+
+  expect(t, last);
+  set(t, last, ContinuationNext, t->continuation);
+
+  return first;
 }
 
 void NO_RETURN
@@ -2044,7 +2218,7 @@ handleMonitorEvent(MyThread* t, Frame* frame, intptr_t function)
     if (methodFlags(t, method) & ACC_STATIC) {
       lock = frame->append(methodClass(t, method));
     } else {
-      lock = c->loadLocal(1, savedTargetIndex(t, method));
+      lock = loadLocal(frame->context, 1, savedTargetIndex(t, method));
     }
     
     c->call(c->constant(function),
@@ -2063,11 +2237,9 @@ handleEntrance(MyThread* t, Frame* frame)
   if ((methodFlags(t, method) & (ACC_SYNCHRONIZED | ACC_STATIC))
       == ACC_SYNCHRONIZED)
   {
-    Compiler* c = frame->c;
-
     // save 'this' pointer in case it is overwritten.
     unsigned index = savedTargetIndex(t, method);
-    c->storeLocal(1, c->loadLocal(1, 0), index);
+    storeLocal(frame->context, 1, loadLocal(frame->context, 1, 0), index);
     frame->set(index, Frame::Object);
   }
 
@@ -3080,7 +3252,8 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
       int8_t count = codeBody(t, code, ip++);
 
       c->storeLocal
-        (1, c->add(4, c->constant(count), c->loadLocal(1, index)), index);
+        (1, c->add(4, c->constant(count), loadLocal(context, 1, index)),
+         index);
     } break;
 
     case iload:
@@ -3217,21 +3390,16 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
 
       bool tailCall = isTailCall(t, code, ip, context->method, target);
 
-      Compiler::Operand* classOperand = c->and_
-        (BytesPerWord, c->constant(PointerMask),
-         c->memory(instance, 0, 0, 1));
-
-      c->freezeRegister(t->arch->virtualCallClass(), classOperand);
-
       Compiler::Operand* result = c->stackCall
-        (c->memory(classOperand, offset, 0, 1),
+        (c->memory
+         (c->and_
+          (BytesPerWord, c->constant(PointerMask),
+           c->memory(instance, 0, 0, 1)), offset, 0, 1),
          tailCall ? Compiler::TailJump : 0,
          frame->trace(0, 0),
          rSize,
          parameterFootprint);
 
-      c->thawRegister(t->arch->virtualCallClass());
-
       frame->pop(parameterFootprint);
 
       if (rSize) {
@@ -3338,7 +3506,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
         // wasn't already a live value there, which is something we
         // should verify once we have complete data flow information
         // (todo).
-        c->storeLocal(1, c->constant(0), index);
+        storeLocal(context, 1, c->constant(0), index);
         frame->storedObject(index);
       }
 
@@ -3840,7 +4008,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
     } break;
 
     case ret:
-      c->jmp(c->loadLocal(1, codeBody(t, code, ip)));
+      c->jmp(loadLocal(context, 1, codeBody(t, code, ip)));
       c->endSubroutine(frame->subroutine);
       return;
 
@@ -3940,7 +4108,8 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
         uint16_t count = codeReadInt16(t, code, ip);
 
         c->storeLocal
-          (1, c->add(4, c->constant(count), c->loadLocal(1, index)), index);
+          (1, c->add(4, c->constant(count), loadLocal(context, 1, index)),
+           index);
       } break;
 
       case iload: {
@@ -3960,7 +4129,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
       } break;
 
       case ret:
-        c->jmp(c->loadLocal(1, codeReadInt16(t, code, ip)));
+        c->jmp(loadLocal(context, 1, codeReadInt16(t, code, ip)));
         c->endSubroutine(frame->subroutine);
         return;
 
@@ -4434,7 +4603,7 @@ compile(MyThread* t, Allocator* allocator, Context* context)
 
   unsigned index = 0;
   if ((methodFlags(t, context->method) & ACC_STATIC) == 0) {
-    c->initLocal(1, index);
+    initLocal(context, 1, index);
     frame.set(index++, Frame::Object);    
   }
 
@@ -4446,19 +4615,19 @@ compile(MyThread* t, Allocator* allocator, Context* context)
     switch (*it.next()) {
     case 'L':
     case '[':
-      c->initLocal(1, index);
+      initLocal(context, 1, index);
       frame.set(index++, Frame::Object);
       break;
       
     case 'J':
     case 'D':
-      c->initLocal(2, index);
+      initLocal(context, 2, index);
       frame.set(index++, Frame::Long);
       frame.set(index++, Frame::Long);
       break;
 
     default:
-      c->initLocal(1, index);
+      initLocal(context, 1, index);
       frame.set(index++, Frame::Integer);
       break;
     }
@@ -4636,8 +4805,8 @@ compileVirtualMethod2(MyThread* t, object class_, unsigned index)
 uint64_t
 compileVirtualMethod(MyThread* t)
 {
-  object class_ = static_cast<object>(t->virtualCallClass);
+  object class_ = objectClass(t, static_cast<object>(t->virtualCallTarget));
-  t->virtualCallClass = 0;
+  t->virtualCallTarget = 0;
 
   unsigned index = t->virtualCallIndex;
   t->virtualCallIndex = 0;
@@ -4651,31 +4820,21 @@ compileVirtualMethod(MyThread* t)
   }
 }
 
+inline uint64_t
+invokeNativeFast(MyThread* t, object method)
+{
+  return reinterpret_cast<FastNativeFunction>(methodCompiled(t, method))
+    (t, method,
+     static_cast<uintptr_t*>(t->stack)
+     + t->arch->frameFooterSize()
+     + t->arch->frameReturnAddressSize());
+}
+
 uint64_t
-invokeNative2(MyThread* t, object method)
+invokeNativeSlow(MyThread* t, object method)
 {
   PROTECT(t, method);
 
-  assert(t, methodFlags(t, method) & ACC_NATIVE);
-
-  initClass(t, methodClass(t, method));
-  if (UNLIKELY(t->exception)) return 0;
-
-  if (methodCompiled(t, method) == defaultThunk(t)) {
-    void* function = resolveNativeMethod(t, method);
-    if (UNLIKELY(function == 0)) {
-      object message = makeString
-        (t, "%s.%s%s",
-         &byteArrayBody(t, className(t, methodClass(t, method)), 0),
-         &byteArrayBody(t, methodName(t, method), 0),
-         &byteArrayBody(t, methodSpec(t, method), 0));
-      t->exception = makeUnsatisfiedLinkError(t, message);
-      return 0;
-    }
-
-    methodCompiled(t, method) = reinterpret_cast<uintptr_t>(function);
-  }
-
   object class_ = methodClass(t, method);
   PROTECT(t, class_);
 
@@ -4694,12 +4853,13 @@ invokeNative2(MyThread* t, object method)
   types[typeOffset++] = POINTER_TYPE;
 
   uintptr_t* sp = static_cast<uintptr_t*>(t->stack)
-    + parameterOffset(t, method);
+    + t->arch->frameFooterSize()
+    + t->arch->frameReturnAddressSize();
 
   if (methodFlags(t, method) & ACC_STATIC) {
     args[argOffset++] = reinterpret_cast<uintptr_t>(&class_);
   } else {
-    args[argOffset++] = reinterpret_cast<uintptr_t>(sp--);
+    args[argOffset++] = reinterpret_cast<uintptr_t>(sp++);
   }
   types[typeOffset++] = POINTER_TYPE;
 
@@ -4716,14 +4876,14 @@ invokeNative2(MyThread* t, object method)
     case INT16_TYPE:
     case INT32_TYPE:
     case FLOAT_TYPE:
-      args[argOffset++] = *(sp--);
+      args[argOffset++] = *(sp++);
       break;
 
     case INT64_TYPE:
     case DOUBLE_TYPE: {
-      memcpy(args + argOffset, sp - 1, 8);
+      memcpy(args + argOffset, sp, 8);
       argOffset += (8 / BytesPerWord);
-      sp -= 2;
+      sp += 2;
     } break;
 
     case POINTER_TYPE: {
@@ -4732,7 +4892,7 @@ invokeNative2(MyThread* t, object method)
       } else {
         args[argOffset++] = 0;
       }
-      -- sp;
+      ++ sp;
     } break;
 
     default: abort(t);
@@ -4842,6 +5002,16 @@ invokeNative2(MyThread* t, object method)
 
   return result;
 }
+  
+uint64_t
+invokeNative2(MyThread* t, object method)
+{
+  if (methodVmFlags(t, method) & FastNative) {
+    return invokeNativeFast(t, method);
+  } else {
+    return invokeNativeSlow(t, method);
+  }
+}
 
 uint64_t
 invokeNative(MyThread* t)
@@ -4870,7 +5040,11 @@ invokeNative(MyThread* t)
   t->trace->targetMethod = t->trace->nativeMethod;
 
   if (LIKELY(t->exception == 0)) {
-    result = invokeNative2(t, t->trace->nativeMethod);
+    resolveNative(t, t->trace->nativeMethod);
+
+    if (LIKELY(t->exception == 0)) {
+      result = invokeNative2(t, t->trace->nativeMethod);
+    }
   }
 
   t->trace->targetMethod = 0;
@@ -5019,6 +5193,61 @@ visitStack(MyThread* t, Heap::Visitor* v)
   }
 }
 
+void
+walkContinuationBody(MyThread* t, Heap::Walker* w, object c, int start)
+{
+  const int ContinuationReferenceCount = 3;
+
+  int bodyStart = max(0, start - ContinuationReferenceCount);
+
+  object method = t->m->heap->follow(continuationMethod(t, c));
+  unsigned count = frameMapSizeInBits(t, method);
+
+  if (count) {
+    object map = codePool(t, methodCode(t, method));
+    int index = frameMapIndex
+      (t, method, difference
+       (continuationAddress(t, c),
+        reinterpret_cast<void*>(methodAddress(t, method))));
+
+    for (unsigned i = bodyStart; i < count; ++i) {
+      int j = index + i;
+      if ((intArrayBody(t, map, j / 32)
+           & (static_cast<int32_t>(1) << (j % 32))))
+      {
+        if (not w->visit
+            ((continuationFramePointerOffset(t, c) / BytesPerWord)
+             - t->arch->framePointerOffset()
+             - stackOffsetFromFrame(t, method)
+             + localOffsetFromStack(c, i, method)))
+        {
+          return;
+        }        
+      }
+    }
+  }
+}
+
+void
+callWithContinuation(MyThread* t, object method, object this_,
+                     object continuation, void* base, void* stack)
+{
+  t->trace->targetMethod = 0;
+
+  if (methodFlags(t, method) & ACC_NATIVE) {
+    t->trace->nativeMethod = method;
+  } else {
+    t->trace->nativeMethod = 0;
+  }
+  
+  vmCallWithContinuation
+    (t, methodAddress(t, method), this_, continuation, base,
+     static_cast<void**>(stack)
+     - t->arch->argumentFootprint(methodParameterFootprint(t, method))
+     - t->arch->frameFooterSize()
+     - t->arch->returnAddressSize());
+}
+
 class ArgumentList {
  public:
   ArgumentList(Thread* t, uintptr_t* array, unsigned size, bool* objectMask,
@@ -5028,7 +5257,7 @@ class ArgumentList {
     array(array),
     objectMask(objectMask),
     size(size),
-    position(size),
+    position(0),
     protector(this)
   {
     if (this_) {
@@ -5065,7 +5294,7 @@ class ArgumentList {
     array(array),
     objectMask(objectMask),
     size(size),
-    position(size),
+    position(0),
     protector(this)
   {
     if (this_) {
@@ -5095,35 +5324,30 @@ class ArgumentList {
   }
 
   void addObject(object v) {
-    assert(t, position);
+    assert(t, position < size);
 
-    -- position;
     array[position] = reinterpret_cast<uintptr_t>(v);
     objectMask[position] = true;
+    ++ position;
   }
 
   void addInt(uintptr_t v) {
-    assert(t, position);
+    assert(t, position < size);
 
-    -- position;
     array[position] = v;
     objectMask[position] = false;
+    ++ position;
   }
 
   void addLong(uint64_t v) {
-    assert(t, position >= 2);
+    assert(t, position < size - 1);
 
-    position -= 2;
+    memcpy(array + position, &v, 8);
-
-    if (BytesPerWord == 8) {
-      memcpy(array + position, &v, 8);
-    } else {
-      array[position] = v;
-      array[position + 1] = v >> 32;
-    }
 
     objectMask[position] = false;
     objectMask[position + 1] = false;
+
+    position += 2;
   }
 
   MyThread* t;
@@ -5378,10 +5602,13 @@ class MyProcessor: public Processor {
     }
 
     for (MyThread::CallTrace* trace = t->trace; trace; trace = trace->next) {
+      v->visit(&(trace->continuation));
       v->visit(&(trace->nativeMethod));
       v->visit(&(trace->targetMethod));
     }
 
+    v->visit(&(t->continuation));
+
     for (Reference* r = t->reference; r; r = r->next) {
       v->visit(&(r->target));
     }
@@ -5677,6 +5904,53 @@ class MyProcessor: public Processor {
     expect(t, t->m->system->success
            (t->m->system->handleSegFault(&segFaultHandler)));
   }
+
+  virtual void callWithCurrentContinuation(Thread* t, object method,
+                                           object this_)
+  {
+    object coninuation;
+    void* base;
+    void* stack;
+
+    { PROTECT(t, method);
+      PROTECT(t, this_);
+
+      compile(static_cast<MyThread*>(t), 
+              ::codeAllocator(static_cast<MyThread*>(t)), 0, method);
+
+      if (LIKELY(t->exception == 0)) {
+        void* ip;
+        continuation = makeCurrentContinuation
+          (static_cast<MyThread*>(t), &ip, &base, &stack);
+      }
+    }
+
+    if (LIKELY(t->exception == 0)) {
+      callWithContinuation(t, method, this_, continuation, base, stack);
+    }
+  }
+
+  virtual void callContinuation(Thread* t, object continuation, object result)
+  {
+    assert(t, t->exception == 0);
+
+    void* ip;
+    void* base;
+    void* stack;
+    findUnwindTarget(t, &ip, &base, &stack);
+
+    t->trace->nativeMethod = 0;
+    t->trace->targetMethod = 0;
+
+    t->continuation = continuation;
+    vmJump(ip, base, stack, t, result, 0);
+  }
+
+  virtual void walkContinuationBody(Thread* t, Heap::Walker* w, object o,
+                                    unsigned start)
+  {
+    ::walkContinuationBody(static_cast<MyThread*>(t), w, o, start);
+  }
   
   System* s;
   Allocator* allocator;
@@ -6124,15 +6398,24 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p,
 
   { Assembler* a = defaultVirtualContext.context.assembler;
 
-    Assembler::Register class_(t->arch->virtualCallClass());
+    Assembler::Register class_(t->arch->virtualCallTarget());
-    Assembler::Memory virtualCallClass
+    Assembler::Memory virtualCallTargetSrc
-      (t->arch->thread(), difference(&(t->virtualCallClass), t));
+      (t->arch->stack(),
+       t->arch->frameFooterSize() + t->arch->returnAddressSize());
+
+    a->apply(Move, BytesPerWord, MemoryOperand, &virtualCallTargetSrc,
+             BytesPerWord, RegisterOperand, &class_);
+
+    Assembler::Memory virtualCallTargetDst
+      (t->arch->thread(), difference(&(t->virtualCallTarget), t));
+
     a->apply(Move, BytesPerWord, RegisterOperand, &class_,
-             BytesPerWord, MemoryOperand, &virtualCallClass);
+             BytesPerWord, MemoryOperand, &virtualCallTargetDst);
 
     Assembler::Register index(t->arch->virtualCallIndex());
     Assembler::Memory virtualCallIndex
       (t->arch->thread(), difference(&(t->virtualCallIndex), t));
+
     a->apply(Move, BytesPerWord, RegisterOperand, &index,
              BytesPerWord, MemoryOperand, &virtualCallIndex);
     

src/compiler.cpp

@@ -2327,51 +2327,41 @@ class CallEvent: public Event {
     stackArgumentFootprint(stackArgumentFootprint)
   {
     uint32_t registerMask = ~0;
-    Stack* s = argumentStack;
-    unsigned index = 0;
-    unsigned frameIndex;
-    int returnAddressIndex = -1;
-    int framePointerIndex = -1;
 
-    if (flags & Compiler::TailJump) {
+    if (argumentCount) {
-      assert(c, argumentCount == 0);
+      assert(c, (flags & Compiler::TailJump) == 0);
+      assert(c, stackArgumentFootprint == 0);
 
-      int base = frameBase(c);
+      Stack* s = argumentStack;
-      returnAddressIndex = base + c->arch->returnAddressOffset();
+      unsigned frameIndex = 0;
-      framePointerIndex = base + c->arch->framePointerOffset();
+      unsigned index = 0;
 
-      frameIndex = totalFrameSize(c) - c->arch->argumentFootprint
+      while (true) {
-        (stackArgumentFootprint);
+        Read* target;
-    } else {
+        if (index < c->arch->argumentRegisterCount()) {
-      frameIndex = 0;
+          int number = c->arch->argumentRegister(index);
-
-      if (argumentCount) {
-        while (true) {
-          Read* target;
-          if (index < c->arch->argumentRegisterCount()) {
-            int number = c->arch->argumentRegister(index);
         
-            if (DebugReads) {
+          if (DebugReads) {
-              fprintf(stderr, "reg %d arg read %p\n", number, s->value);
+            fprintf(stderr, "reg %d arg read %p\n", number, s->value);
-            }
-
-            target = fixedRegisterRead(c, number);
-            registerMask &= ~(1 << number);
-          } else {
-            if (DebugReads) {
-              fprintf(stderr, "stack %d arg read %p\n", frameIndex, s->value);
-            }
-
-            target = read(c, SiteMask(1 << MemoryOperand, 0, frameIndex));
-            ++ frameIndex;
           }
-          addRead(c, this, s->value, target);
 
-          if ((++ index) < argumentCount) {
+          target = fixedRegisterRead(c, number);
-            s = s->next;
+          registerMask &= ~(1 << number);
-          } else {
+        } else {
-            break;
+          if (DebugReads) {
+            fprintf(stderr, "stack %d arg read %p\n", frameIndex, s->value);
           }
+
+          target = read(c, SiteMask(1 << MemoryOperand, 0, frameIndex));
+          ++ frameIndex;
+        }
+
+        addRead(c, this, s->value, target);
+
+        if ((++ index) < argumentCount) {
+          s = s->next;
+        } else {
+          break;
         }
       }
     }
@@ -2394,41 +2384,53 @@ class CallEvent: public Event {
                (typeMask, registerMask & planRegisterMask, AnyFrameIndex)));
     }
 
-    int footprint = stackArgumentFootprint;
+    Stack* stack = stackBefore;
-    for (Stack* s = stackBefore; s; s = s->next) {
+
-      if (s->value) {
+    if (stackArgumentFootprint) {
-        if (footprint > 0) {
+      int footprint = stackArgumentFootprint;
+      int returnAddressIndex;
+      int framePointerIndex;
+      int frameOffset;
+
+      if (flags & Compiler::TailJump) {
+        assert(c, argumentCount == 0);
+
+        int base = frameBase(c);
+        returnAddressIndex = base + c->arch->returnAddressOffset();
+        framePointerIndex = base + c->arch->framePointerOffset();
+
+        frameOffset = totalFrameSize(c)
+          - c->arch->argumentFootprint(stackArgumentFootprint) - 1;
+      } else {
+        returnAddressIndex = -1;
+        framePointerIndex = -1;
+        frameOffset = -1;
+      }
+
+      while (footprint > 0) {
+        if (stack->value) {
+          int frameIndex = footprint + frameOffset;
+
           if (DebugReads) {
             fprintf(stderr, "stack arg read %p at %d of %d\n",
-                    s->value, frameIndex, totalFrameSize(c));
+                    stack->value, frameIndex, totalFrameSize(c));
           }
 
           if (static_cast<int>(frameIndex) == returnAddressIndex) {
-            returnAddressSurrogate = s->value;
+            returnAddressSurrogate = stack->value;
-            addRead(c, this, s->value, anyRegisterRead(c));
+            addRead(c, this, stack->value, anyRegisterRead(c));
           } else if (static_cast<int>(frameIndex) == framePointerIndex) {
             framePointerSurrogate = s->value;
-            addRead(c, this, s->value, anyRegisterRead(c));
+            addRead(c, this, stack->value, anyRegisterRead(c));
           } else {
-            addRead(c, this, s->value, read
+            addRead(c, this, stack->value, read
                     (c, SiteMask(1 << MemoryOperand, 0, frameIndex)));
           }
-        } else if ((flags & Compiler::TailJump) == 0) {
-          unsigned logicalIndex = ::frameIndex
-            (c, s->index + c->localFootprint);
-
-          if (DebugReads) {
-            fprintf(stderr, "stack save read %p at %d of %d\n",
-                    s->value, logicalIndex, totalFrameSize(c));
-          }
-
-          addRead(c, this, s->value, read
-                  (c, SiteMask(1 << MemoryOperand, 0, logicalIndex)));
         }
-      }
 
-      -- footprint;
+        stack = stack->next;
-      ++ frameIndex;
+        -- footprint;
+      }
     }
 
     if ((flags & Compiler::TailJump) == 0) {
@@ -2445,6 +2447,23 @@ class CallEvent: public Event {
 
       assert(c, static_cast<int>(popIndex) >= 0);
 
+      while (stack) {
+        if (stack->value) {
+          unsigned logicalIndex = ::frameIndex
+            (c, stack->index + c->localFootprint);
+
+          if (DebugReads) {
+            fprintf(stderr, "stack save read %p at %d of %d\n",
+                    stack->value, logicalIndex, totalFrameSize(c));
+          }
+
+          addRead(c, this, stack->value, read
+                  (c, SiteMask(1 << MemoryOperand, 0, logicalIndex)));
+        }
+
+        stack = stack->next;
+      }
+
       saveLocals(c, this);
     }
   }
@@ -3289,7 +3308,7 @@ push(Context* c, unsigned footprint, Value* v)
   
   Value* low = v;
   
-  if (bigEndian) {
+  if (not bigEndian) {
     v = pushWord(c, v);
   }
 
@@ -3308,7 +3327,7 @@ push(Context* c, unsigned footprint, Value* v)
     high = 0;
   }
   
-  if (not bigEndian) {
+  if (bigEndian) {
     v = pushWord(c, v);
   }
 
@@ -3341,7 +3360,7 @@ pop(Context* c, unsigned footprint)
 
   bool bigEndian = c->arch->bigEndian();
 
-  if (not bigEndian) {
+  if (bigEndian) {
     s = c->stack;
   }
 
@@ -3352,11 +3371,11 @@ pop(Context* c, unsigned footprint)
     Stack* low;
     Stack* high;
     if (bigEndian) {
-      high = c->stack;
-      low = high->next;
-    } else {
       low = c->stack;
       high = low->next;
+    } else {
+      high = c->stack;
+      low = high->next;
     }
 
     assert(c, low->value->high == high->value
@@ -3366,7 +3385,7 @@ pop(Context* c, unsigned footprint)
     popWord(c);
   }
 
-  if (bigEndian) {
+  if (not bigEndian) {
     s = c->stack;
   }
 
@@ -3997,61 +4016,6 @@ appendSaveLocals(Context* c)
          SaveLocalsEvent(c));
 }
 
-class FreezeRegisterEvent: public Event {
- public:
-  FreezeRegisterEvent(Context* c, int number, Value* value):
-    Event(c), number(number), value(value)
-  {
-    addRead(c, this, value, fixedRegisterRead(c, number));
-  }
-
-  virtual const char* name() {
-    return "FreezeRegisterEvent";
-  }
-
-  virtual void compile(Context* c) {
-    c->registerResources[number].freeze(c, value);
-
-    for (Read* r = reads; r; r = r->eventNext) {
-      popRead(c, this, r->value);
-    }
-  }
-
-  int number;
-  Value* value;
-};
-
-void
-appendFreezeRegister(Context* c, int number, Value* value)
-{
-  append(c, new (c->zone->allocate(sizeof(FreezeRegisterEvent)))
-         FreezeRegisterEvent(c, number, value));
-}
-
-class ThawRegisterEvent: public Event {
- public:
-  ThawRegisterEvent(Context* c, int number):
-    Event(c), number(number)
-  { }
-
-  virtual const char* name() {
-    return "ThawRegisterEvent";
-  }
-
-  virtual void compile(Context* c) {
-    c->registerResources[number].thaw(c, 0);
-  }
-
-  int number;
-};
-
-void
-appendThawRegister(Context* c, int number)
-{
-  append(c, new (c->zone->allocate(sizeof(ThawRegisterEvent)))
-         ThawRegisterEvent(c, number));
-}
-
 class DummyEvent: public Event {
  public:
   DummyEvent(Context* c):
@@ -5133,14 +5097,6 @@ class MyCompiler: public Compiler {
     return value(&c, s, s);
   }
 
-  virtual void freezeRegister(int number, Operand* value) {
-    appendFreezeRegister(&c, number, static_cast<Value*>(value));
-  }
-
-  virtual void thawRegister(int number) {
-    appendThawRegister(&c, number);
-  }
-
   Promise* machineIp() {
     return codePromise(&c, c.logicalCode[c.logicalIp]->lastEvent);
   }
@@ -5230,18 +5186,18 @@ class MyCompiler: public Compiler {
       Stack* low;
       Stack* high;
       if (bigEndian) {
-        high = s;
-        low = s->next;
-      } else {
         low = s;
         high = s->next;
+      } else {
+        high = s;
+        low = s->next;
       }
 
       assert(&c, low->value->high == high->value
              and ((BytesPerWord == 8) xor (low->value->high != 0)));
 #endif // not NDEBUG
 
-      if (bigEndian) {
+      if (not bigEndian) {
         s = s->next;
       }
     }

src/compiler.h

@@ -67,9 +67,6 @@ class Compiler {
 
   virtual Operand* register_(int number) = 0;
 
-  virtual void freezeRegister(int number, Operand* value) = 0;
-  virtual void thawRegister(int number) = 0;
-
   virtual void push(unsigned footprint) = 0;
   virtual void push(unsigned footprint, Operand* value) = 0;
   virtual void save(unsigned footprint, Operand* value) = 0;

src/interpret.cpp

@@ -460,7 +460,7 @@ makeNativeMethodData(Thread* t, object method, void* function)
   return data;
 }
 
-inline object
+inline void
 resolveNativeMethodData(Thread* t, object method)
 {
   if (methodCode(t, method) == 0) {
@@ -479,9 +479,7 @@ resolveNativeMethodData(Thread* t, object method)
       t->exception = makeUnsatisfiedLinkError(t, message);
       return 0;
     }
-  } else {
+  } 
-    return methodCode(t, method);
-  }  
 }
 
 inline void
@@ -498,16 +496,73 @@ checkStack(Thread* t, object method)
   }
 }
 
+void
+pushResult(Thread* t, unsigned returnCode, uint64_t result)
+{
+  switch (returnCode) {
+  case ByteField:
+  case BooleanField:
+    if (DebugRun) {
+      fprintf(stderr, "result: %d\n", static_cast<int8_t>(result));
+    }
+    pushInt(t, static_cast<int8_t>(result));
+    break;
+
+  case CharField:
+    if (DebugRun) {
+      fprintf(stderr, "result: %d\n", static_cast<uint16_t>(result));
+    }
+    pushInt(t, static_cast<uint16_t>(result));
+    break;
+
+  case ShortField:
+    if (DebugRun) {
+      fprintf(stderr, "result: %d\n", static_cast<int16_t>(result));
+    }
+    pushInt(t, static_cast<int16_t>(result));
+    break;
+
+  case FloatField:
+  case IntField:
+    if (DebugRun) {
+      fprintf(stderr, "result: %d\n", static_cast<int32_t>(result));
+    }
+    pushInt(t, result);
+    break;
+
+  case LongField:
+  case DoubleField:
+    if (DebugRun) {
+      fprintf(stderr, "result: %"LLD"\n", result);
+    }
+    pushLong(t, result);
+    break;
+
+  case ObjectField:
+    if (DebugRun) {
+      fprintf(stderr, "result: %p at %p\n",
+              static_cast<uintptr_t>(result) == 0 ? 0 :
+              *reinterpret_cast<object*>(static_cast<uintptr_t>(result)),
+              reinterpret_cast<object*>(static_cast<uintptr_t>(result)));
+    }
+    pushObject(t, static_cast<uintptr_t>(result) == 0 ? 0 :
+               *reinterpret_cast<object*>(static_cast<uintptr_t>(result)));
+    break;
+
+  case VoidField:
+    break;
+
+  default:
+    abort(t);
+  }
+}
+
 unsigned
-invokeNative(Thread* t, object method)
+invokeNativeSlow(Thread* t, object method)
 {
   PROTECT(t, method);
 
-  object data = resolveNativeMethodData(t, method);
+  object data = methodCode(t, method);
-  if (UNLIKELY(t->exception)) {
-    return VoidField;
-  }
-
   PROTECT(t, data);
 
   pushFrame(t, method);
@@ -609,66 +664,46 @@ invokeNative(Thread* t, object method)
     return VoidField;
   }
 
-  switch (returnCode) {
+  pushResult(t, returnCode, result);
-  case ByteField:
-  case BooleanField:
-    if (DebugRun) {
-      fprintf(stderr, "result: %d\n", static_cast<int8_t>(result));
-    }
-    pushInt(t, static_cast<int8_t>(result));
-    break;
-
-  case CharField:
-    if (DebugRun) {
-      fprintf(stderr, "result: %d\n", static_cast<uint16_t>(result));
-    }
-    pushInt(t, static_cast<uint16_t>(result));
-    break;
-
-  case ShortField:
-    if (DebugRun) {
-      fprintf(stderr, "result: %d\n", static_cast<int16_t>(result));
-    }
-    pushInt(t, static_cast<int16_t>(result));
-    break;
-
-  case FloatField:
-  case IntField:
-    if (DebugRun) {
-      fprintf(stderr, "result: %d\n", static_cast<int32_t>(result));
-    }
-    pushInt(t, result);
-    break;
-
-  case LongField:
-  case DoubleField:
-    if (DebugRun) {
-      fprintf(stderr, "result: %"LLD"\n", result);
-    }
-    pushLong(t, result);
-    break;
-
-  case ObjectField:
-    if (DebugRun) {
-      fprintf(stderr, "result: %p at %p\n",
-              static_cast<uintptr_t>(result) == 0 ? 0 :
-              *reinterpret_cast<object*>(static_cast<uintptr_t>(result)),
-              reinterpret_cast<object*>(static_cast<uintptr_t>(result)));
-    }
-    pushObject(t, static_cast<uintptr_t>(result) == 0 ? 0 :
-               *reinterpret_cast<object*>(static_cast<uintptr_t>(result)));
-    break;
-
-  case VoidField:
-    break;
-
-  default:
-    abort(t);
-  }
 
   return returnCode;
 }
 
+unsigned
+invokeNative(MyThread* t, object method)
+{
+  PROTECT(t, method);
+
+  resolveNativeMethodData(t, method);
+
+  if (UNLIKELY(t->exception)) {
+    return VoidField;
+  }
+
+  if (methodVmFlags(t, method) & FastNative) {
+    pushFrame(t, method);
+
+    uint64_t result = reinterpret_cast<FastNativeFunction>
+      (methodCompiled(t, method))
+      (t, method,
+       static_cast<uintptr_t*>(t->stack)
+       + t->arch->frameFooterSize()
+       + t->arch->frameReturnAddressSize());
+
+    popFrame(t);
+
+    if (UNLIKELY(t->exception)) {
+      return VoidField;
+    }
+
+    pushResult(t, methodReturnCode(t, method), result);
+
+    return methodReturnCode(t, method);
+  } else {
+    return invokeNativeSlow(t, method);
+  }
+}
+
 bool
 classInit2(Thread* t, object class_, unsigned ipOffset)
 {

src/machine.cpp

@@ -1544,9 +1544,12 @@ boot(Thread* t)
 
   m->unsafe = false;
 
-  classVmFlags(t, arrayBody(t, m->types, Machine::SingletonType))
+  classFlags(t, arrayBody(t, m->types, Machine::SingletonType))
     |= SingletonFlag;
 
+  classFlags(t, arrayBody(t, m->types, Machine::ContinuationType))
+    |= ContinuationFlag;
+
   classVmFlags(t, arrayBody(t, m->types, Machine::JreferenceType))
     |= ReferenceFlag;
   classVmFlags(t, arrayBody(t, m->types, Machine::WeakReferenceType))
@@ -2754,6 +2757,8 @@ collect(Thread* t, Heap::CollectionType type)
 
   Machine* m = t->m;
 
+  m->continuationClass = arrayBody(t, t->m->types, Machine::ContinuationType);
+
   m->unsafe = true;
   m->heap->collect(type, footprint(m->rootThread));
   m->unsafe = false;
@@ -2800,7 +2805,7 @@ walk(Thread* t, Heap::Walker* w, object o, unsigned start)
            intArrayLength(t, objectMask) * 4);
 
     ::walk(t, w, mask, fixedSize, arrayElementSize, arrayLength, start);
-  } else if (classVmFlags(t, class_) & SingletonFlag) {
+  } else if (classFlags(t, class_) & SingletonFlag) {
     unsigned length = singletonLength(t, o);
     if (length) {
       ::walk(t, w, singletonMask(t, o),
@@ -2811,6 +2816,10 @@ walk(Thread* t, Heap::Walker* w, object o, unsigned start)
   } else if (start == 0) {
     w->visit(0);
   }
+
+  if (classFlags(t, class_) & ContinuationFlag) {
+    t->m->processor->walkContinuationBody(t, w, o, start);
+  }
 }
 
 int
@@ -2843,13 +2852,13 @@ visitRoots(Machine* m, Heap::Visitor* v)
   v->visit(&(m->types));
   v->visit(&(m->jniMethodTable));
 
-  for (Reference* r = m->jniReferences; r; r = r->next) {
-    v->visit(&(r->target));
-  }
-
   for (Thread* t = m->rootThread; t; t = t->peer) {
     ::visitRoots(t, v);
   }
+
+  for (Reference* r = m->jniReferences; r; r = r->next) {
+    v->visit(&(r->target));
+  }
 }
 
 void

src/machine.h

@@ -74,20 +74,26 @@ enum StackTag {
 const int NativeLine = -1;
 const int UnknownLine = -2;
 
-// class flags:
+// class flags (note that we must be careful not to overlap the
+// standard ACC_* flags):
+const unsigned SingletonFlag = 1 << 14;
+const unsigned ContinuationFlag = 1 << 15;
+
+// class vmFlags:
 const unsigned ReferenceFlag = 1 << 0;
 const unsigned WeakReferenceFlag = 1 << 1;
 const unsigned NeedInitFlag = 1 << 2;
 const unsigned InitFlag = 1 << 3;
 const unsigned PrimitiveFlag = 1 << 4;
-const unsigned SingletonFlag = 1 << 5;
+const unsigned BootstrapFlag = 1 << 5;
-const unsigned BootstrapFlag = 1 << 6;
+const unsigned HasFinalMemberFlag = 1 << 6;
-const unsigned HasFinalMemberFlag = 1 << 7;
 
-// method flags:
+// method vmFlags:
 const unsigned ClassInitFlag = 1 << 0;
 const unsigned CompiledFlag = 1 << 1;
 const unsigned ConstructorFlag = 1 << 2;
+const unsigned NativeResolved = 1 << 3;
+const unsigned FastNative = 1 << 4;
 
 typedef Machine JavaVM;
 typedef Thread JNIEnv;
@@ -1320,6 +1326,8 @@ class Thread {
 #endif // VM_STRESS
 };
 
+typedef uint64_t (*FastNativeFunction)(Thread*, object, uintptr_t*) = 0;
+
 inline object
 objectClass(Thread*, object o)
 {

src/process.cpp

@@ -67,7 +67,7 @@ mangle(int8_t c, char* dst)
 unsigned
 jniNameLength(Thread* t, object method, bool decorate)
 {
-  unsigned size = 5;
+  unsigned size = 0;
 
   object className = ::className(t, methodClass(t, method));
   for (unsigned i = 0; i < byteArrayLength(t, className) - 1; ++i) {
@@ -96,10 +96,11 @@ jniNameLength(Thread* t, object method, bool decorate)
 }
 
 void
-makeJNIName(Thread* t, char* name, object method, bool decorate)
+makeJNIName(Thread* t, const char* prefix, unsigned prefixLength, char* name,
+            object method, bool decorate)
 {
-  memcpy(name, "Java_", 5);
+  memcpy(name, prefix, prefixLength);
-  name += 5;
+  name += prefixLength;
 
   object className = ::className(t, methodClass(t, method));
   for (unsigned i = 0; i < byteArrayLength(t, className) - 1; ++i) {
@@ -146,22 +147,19 @@ resolveNativeMethod(Thread* t, const char* undecorated, const char* decorated)
   return 0;
 }
 
-} // namespace
-
-namespace vm {
-
 void*
-resolveNativeMethod2(Thread* t, object method)
+resolveNativeMethod(Thread* t, object method, const char* prefix,
+                    unsigned prefixLength)
 {
-  unsigned undecoratedSize = jniNameLength(t, method, false);
+  unsigned undecoratedSize = prefixLength + jniNameLength(t, method, false);
   char undecorated[undecoratedSize + 1 + 6]; // extra 6 is for code below
-  makeJNIName(t, undecorated + 1, method, false);
+  makeJNIName(t, prefix, prefixLength, undecorated + 1, method, false);
 
-  unsigned decoratedSize = jniNameLength(t, method, true);
+  unsigned decoratedSize = prefixLength + jniNameLength(t, method, true);
   char decorated[decoratedSize + 1 + 6]; // extra 6 is for code below
-  makeJNIName(t, decorated + 1, method, true);
+  makeJNIName(t, prefix, prefixLength, decorated + 1, method, true);
 
-  void* p = ::resolveNativeMethod(t, undecorated + 1, decorated + 1);
+  void* p = resolveNativeMethod(t, undecorated + 1, decorated + 1);
   if (p) {
     return p;
   }
@@ -181,13 +179,13 @@ resolveNativeMethod2(Thread* t, object method)
   snprintf(decorated + decoratedSize + 1, 5, "@%d",
            footprint * BytesPerWord);
 
-  p = ::resolveNativeMethod(t, undecorated, decorated);
+  p = resolveNativeMethod(t, undecorated, decorated);
   if (p) {
     return p;
   }
 
   // one more try without the leading underscore
-  p = ::resolveNativeMethod(t, undecorated + 1, decorated + 1);
+  p = resolveNativeMethod(t, undecorated + 1, decorated + 1);
   if (p) {
     return p;
   }
@@ -196,6 +194,28 @@ resolveNativeMethod2(Thread* t, object method)
   return 0;
 }
 
+} // namespace
+
+namespace vm {
+
+void*
+resolveNativeMethod2(Thread* t, object method)
+{
+  void* p = ::resolveNativeMethod(t, method, "Java_", 5);
+  if (p) {
+    methodVmFlags(t, method) |= NativeResolved;
+    return p;
+  }
+
+  p = ::resolveNativeMethod(t, method, "Avian_", 6);
+  if (p) {
+    methodVmFlags(t, method) |= NativeResolved & FastNative;
+    return p;
+  }
+
+  return 0;
+}
+
 int
 findLineNumber(Thread* t, object method, unsigned ip)
 {

src/process.h

@@ -128,13 +128,31 @@ isSpecialMethod(Thread* t, object method, object class_)
 void*
 resolveNativeMethod2(Thread* t, object method);
 
-inline void*
+inline void
-resolveNativeMethod(Thread* t, object method)
+resolveNativeMethod(MyThread* t, object method)
 {
-  if (methodCode(t, method)) {
+  PROTECT(t, method);
-    return pointerValue(t, methodCode(t, method));
+
-  } else {
+  assert(t, methodFlags(t, method) & ACC_NATIVE);
-    return resolveNativeMethod2(t, method);
+
+  initClass(t, methodClass(t, method));
+  if (UNLIKELY(t->exception)) return 0;
+
+  unsigned flags = methodVmFlags(t, method);
+  uintptr_t address = methodCompiled(t, method);
+  if ((flags & NativeResolved) == 0 or address == defaultThunk(t)) {
+    void* function = resolveNativeMethod2(t, method);
+    if (UNLIKELY(function == 0)) {
+      object message = makeString
+        (t, "%s.%s%s",
+         &byteArrayBody(t, className(t, methodClass(t, method)), 0),
+         &byteArrayBody(t, methodName(t, method), 0),
+         &byteArrayBody(t, methodSpec(t, method), 0));
+      t->exception = makeUnsatisfiedLinkError(t, message);
+      return;
+    }
+
+    methodCompiled(t, method) = reinterpret_cast<uintptr_t>(function);
   }
 }
 

src/processor.h

@@ -133,6 +133,16 @@ class Processor {
   virtual void
   boot(Thread* t, BootImage* image) = 0;
 
+  virtual void
+  callWithCurrentContinuation(Thread* t, object method, object this_) = 0;
+
+  virtual void
+  callContinuation(Thread* t, object continuation, object result) = 0;
+
+  virtual void
+  walkContiuationBody(Thread* t, Heap::Walker* w, object o, unsigned start)
+  = 0;
+
   object
   invoke(Thread* t, object method, object this_, ...)
   {

src/types.def

@@ -103,6 +103,14 @@
 (type array
   (noassert array object body))
 
+(type continuation
+  (object next)
+  (object method)
+  (void* address)
+  (uintptr_t returnAddressOffset)
+  (uintptr_t framePointerOffset)
+  (array uintptr_t body))
+
 (type string java/lang/String)
 
 (type thread java/lang/Thread)

src/x86.cpp

@@ -2034,7 +2034,7 @@ class MyArchitecture: public Assembler::Architecture {
     return (BytesPerWord == 4 ? rdx : NoRegister);
   }
 
-  virtual int virtualCallClass() {
+  virtual int virtualCallTarget() {
     return rax;
   }