check for stack overflow on entry to all non-leaf methods

We now check for stack overflow in the JIT build as well as the
interpreted build, throwing a StackOverflowError if the limit
(currently hard-coded to 64KB, but should be easy to make
configurable) is exceeded.

src/bootimage.cpp

@@ -30,7 +30,7 @@ const unsigned HeapCapacity = 768 * 1024 * 1024;
 // One of the advantages of a bootimage-based build is that reduces
 // the overhead of major GCs at runtime since we can avoid scanning
 // the pre-built heap image entirely.  However, this only works if we
-// can ensure that no part of the heap image (with an exception noted
+// can ensure that no part of the heap image (with exceptions noted
 // below) ever points to runtime-allocated objects.  Therefore (most)
 // references in the heap image are considered immutable, and any
 // attempt to update them at runtime will cause the process to abort.

src/bootimage.h

@@ -45,6 +45,7 @@ class BootImage {
     Thunk defaultVirtual;
     Thunk native;
     Thunk aioob;
+    Thunk stackOverflow;
     Thunk table;
   };
 
@@ -75,6 +76,7 @@ class BootImage {
   unsigned compileVirtualMethodCall;
   unsigned invokeNativeCall;
   unsigned throwArrayIndexOutOfBoundsCall;
+  unsigned throwStackOverflowCall;
 
 #define THUNK(s) unsigned s##Call;
 #include "thunks.cpp"

src/compile.cpp

@@ -225,7 +225,8 @@ class MyThread: public Thread {
          ? parent->arch
          : makeArchitecture(m->system, useNativeFeatures)),
     transition(0),
-    traceContext(0)
+    traceContext(0),
+    stackLimit(0)
   {
     arch->acquire();
   }
@@ -245,6 +246,7 @@ class MyThread: public Thread {
   Assembler::Architecture* arch;
   Context* transition;
   TraceContext* traceContext;
+  uintptr_t stackLimit;
 };
 
 void
@@ -1118,6 +1120,7 @@ class Context {
     objectPoolCount(0),
     traceLogCount(0),
     dirtyRoots(false),
+    leaf(true),
     eventLog(t->m->system, t->m->heap, 1024),
     protector(this)
   { }
@@ -1139,6 +1142,7 @@ class Context {
     objectPoolCount(0),
     traceLogCount(0),
     dirtyRoots(false),
+    leaf(true),
     eventLog(t->m->system, t->m->heap, 0),
     protector(this)
   { }
@@ -1164,6 +1168,7 @@ class Context {
   unsigned objectPoolCount;
   unsigned traceLogCount;
   bool dirtyRoots;
+  bool leaf;
   Vector eventLog;
   MyProtector protector;
 };
@@ -2099,6 +2104,9 @@ bootNativeThunk(MyThread* t);
 uintptr_t
 aioobThunk(MyThread* t);
 
+uintptr_t
+stackOverflowThunk(MyThread* t);
+
 uintptr_t
 virtualThunk(MyThread* t, unsigned index);
 
@@ -2611,6 +2619,14 @@ throwArrayIndexOutOfBounds(MyThread* t)
   unwind(t);
 }
 
+void NO_RETURN
+throwStackOverflow(MyThread* t)
+{
+  t->exception = makeThrowable(t, Machine::StackOverflowErrorType); 
+
+  unwind(t);
+}
+
 void NO_RETURN
 throw_(MyThread* t, object o)
 {
@@ -4121,6 +4137,8 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
     } break;
 
     case invokeinterface: {
+      context->leaf = false;
+
       uint16_t index = codeReadInt16(t, code, ip);
       ip += 2;
 
@@ -4162,6 +4180,8 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
     } break;
 
     case invokespecial: {
+      context->leaf = false;
+
       uint16_t index = codeReadInt16(t, code, ip);
       object target = resolveMethod(t, context->method, index - 1);
       if (UNLIKELY(t->exception)) return;
@@ -4179,6 +4199,8 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
     } break;
 
     case invokestatic: {
+      context->leaf = false;
+
       uint16_t index = codeReadInt16(t, code, ip);
 
       object target = resolveMethod(t, context->method, index - 1);
@@ -4193,6 +4215,8 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
     } break;
 
     case invokevirtual: {
+      context->leaf = false;
+
       uint16_t index = codeReadInt16(t, code, ip);
 
       object target = resolveMethod(t, context->method, index - 1);
@@ -5745,7 +5769,9 @@ finish(MyThread* t, Allocator* allocator, Context* context)
   // parallelism (the downside being that it may end up being a waste
   // of cycles if another thread compiles the same method in parallel,
   // which might be mitigated by fine-grained, per-method locking):
-  unsigned codeSize = c->compile();
+  unsigned codeSize = c->compile
+    (context->leaf ? 0 : stackOverflowThunk(t),
+     difference(&(t->stackLimit), t));
 
   uintptr_t* code = static_cast<uintptr_t*>
     (allocator->allocate(pad(codeSize) + pad(c->poolSize()) + BytesPerWord));
@@ -7092,6 +7118,15 @@ invoke(Thread* thread, object method, ArgumentList* arguments)
 {
   MyThread* t = static_cast<MyThread*>(thread);
 
+  uintptr_t stackLimit = t->stackLimit;
+  uintptr_t stackPosition = reinterpret_cast<uintptr_t>(&t);
+  if (stackLimit == 0) {
+    t->stackLimit = stackPosition - StackSizeInBytes;
+  } else if (stackPosition < stackLimit) {
+    t->exception = makeThrowable(t, Machine::StackOverflowErrorType);
+    return 0;
+  }
+
   unsigned returnCode = methodReturnCode(t, method);
   unsigned returnType = fieldType(t, returnCode);
 
@@ -7111,6 +7146,8 @@ invoke(Thread* thread, object method, ArgumentList* arguments)
        returnType);
   }
 
+  t->stackLimit = stackLimit;
+
   if (t->exception) { 
     if (UNLIKELY(t->flags & Thread::UseBackupHeapFlag)) {
       collect(t, Heap::MinorCollection);
@@ -7243,6 +7280,7 @@ class MyProcessor: public Processor {
     Thunk defaultVirtual;
     Thunk native;
     Thunk aioob;
+    Thunk stackOverflow;
     Thunk table;
   };
 
@@ -7801,7 +7839,7 @@ isThunkUnsafeStack(MyProcessor::Thunk* thunk, void* ip)
 bool
 isThunkUnsafeStack(MyProcessor::ThunkCollection* thunks, void* ip)
 {
-  const unsigned NamedThunkCount = 4;
+  const unsigned NamedThunkCount = 5;
 
   MyProcessor::Thunk table[NamedThunkCount + ThunkCount];
 
@@ -7809,6 +7847,7 @@ isThunkUnsafeStack(MyProcessor::ThunkCollection* thunks, void* ip)
   table[1] = thunks->defaultVirtual;
   table[2] = thunks->native;
   table[3] = thunks->aioob;
+  table[4] = thunks->stackOverflow;
     
   for (unsigned i = 0; i < ThunkCount; ++i) {
     new (table + NamedThunkCount + i) MyProcessor::Thunk
@@ -8137,6 +8176,8 @@ fixupThunks(MyThread* t, BootImage* image, uint8_t* code)
     = thunkToThunk(image->thunks.defaultVirtual, code);
   p->bootThunks.native = thunkToThunk(image->thunks.native, code);
   p->bootThunks.aioob = thunkToThunk(image->thunks.aioob, code);
+  p->bootThunks.stackOverflow
+    = thunkToThunk(image->thunks.stackOverflow, code);
   p->bootThunks.table = thunkToThunk(image->thunks.table, code);
 
   updateCall(t, LongCall, code + image->compileMethodCall,
@@ -8151,6 +8192,9 @@ fixupThunks(MyThread* t, BootImage* image, uint8_t* code)
   updateCall(t, LongCall, code + image->throwArrayIndexOutOfBoundsCall,
              voidPointer(throwArrayIndexOutOfBounds));
 
+  updateCall(t, LongCall, code + image->throwStackOverflowCall,
+             voidPointer(throwStackOverflow));
+
 #define THUNK(s)                                                        \
   updateCall(t, LongJump, code + image->s##Call, voidPointer(s));
 
@@ -8391,6 +8435,23 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p)
     p->thunks.aioob.length = a->endBlock(false)->resolve(0, 0);
   }
 
+  ThunkContext stackOverflowContext(t, &zone);
+
+  { Assembler* a = stackOverflowContext.context.assembler;
+      
+    a->saveFrame(difference(&(t->stack), t), difference(&(t->base), t));
+
+    p->thunks.stackOverflow.frameSavedOffset = a->length();
+
+    Assembler::Register thread(t->arch->thread());
+    a->pushFrame(1, BytesPerWord, RegisterOperand, &thread);
+
+    Assembler::Constant proc(&(stackOverflowContext.promise));
+    a->apply(LongCall, BytesPerWord, ConstantOperand, &proc);
+
+    p->thunks.stackOverflow.length = a->endBlock(false)->resolve(0, 0);
+  }
+
   ThunkContext tableContext(t, &zone);
 
   { Assembler* a = tableContext.context.assembler;
@@ -8463,6 +8524,21 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p)
     }
   }
 
+  p->thunks.stackOverflow.start = finish
+    (t, allocator, stackOverflowContext.context.assembler, "stackOverflow",
+     p->thunks.stackOverflow.length);
+
+  { void* call;
+    stackOverflowContext.promise.listener->resolve
+      (reinterpret_cast<intptr_t>(voidPointer(throwStackOverflow)),
+       &call);
+
+    if (image) {
+      image->throwStackOverflowCall
+        = static_cast<uint8_t*>(call) - imageBase;
+    }
+  }
+
   p->thunks.table.start = static_cast<uint8_t*>
     (allocator->allocate(p->thunks.table.length * ThunkCount));
 
@@ -8472,6 +8548,8 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p)
       = thunkToThunk(p->thunks.defaultVirtual, imageBase);
     image->thunks.native = thunkToThunk(p->thunks.native, imageBase);
     image->thunks.aioob = thunkToThunk(p->thunks.aioob, imageBase);
+    image->thunks.stackOverflow
+      = thunkToThunk(p->thunks.stackOverflow, imageBase);
     image->thunks.table = thunkToThunk(p->thunks.table, imageBase);
   }
 
@@ -8539,6 +8617,12 @@ aioobThunk(MyThread* t)
   return reinterpret_cast<uintptr_t>(processor(t)->thunks.aioob.start);
 }
 
+uintptr_t
+stackOverflowThunk(MyThread* t)
+{
+  return reinterpret_cast<uintptr_t>(processor(t)->thunks.stackOverflow.start);
+}
+
 bool
 unresolved(MyThread* t, uintptr_t methodAddress)
 {

src/compiler.cpp

@@ -4919,8 +4919,7 @@ class BoundsCheckEvent: public Event {
                         lengthOffset, NoRegister, 1);
       length.acquired = true;
 
-      CodePromise* nextPromise = codePromise
-        (c, static_cast<Promise*>(0));
+      CodePromise* nextPromise = codePromise(c, static_cast<Promise*>(0));
 
       freezeSource(c, BytesPerWord, index);
 
@@ -5649,7 +5648,7 @@ block(Context* c, Event* head)
 }
 
 unsigned
-compile(Context* c)
+compile(Context* c, uintptr_t stackOverflowHandler, unsigned stackLimitOffset)
 {
   if (c->logicalCode[c->logicalIp]->lastEvent == 0) {
     appendDummy(c);
@@ -5660,6 +5659,15 @@ compile(Context* c)
   Block* firstBlock = block(c, c->firstEvent);
   Block* block = firstBlock;
 
+  if (stackOverflowHandler) {
+    Assembler::Register stack(c->arch->stack());
+    Assembler::Memory stackLimit(c->arch->thread(), stackLimitOffset);
+    Assembler::Constant handler(resolved(c, stackOverflowHandler));
+    a->apply(JumpIfGreaterOrEqual, BytesPerWord, RegisterOperand, &stack,
+             BytesPerWord, MemoryOperand, &stackLimit,
+             BytesPerWord, ConstantOperand, &handler);
+  }
+
   a->allocateFrame(c->alignedFrameSize);
 
   for (Event* e = c->firstEvent; e; e = e->next) {
@@ -6406,8 +6414,8 @@ class MyCompiler: public Compiler {
   virtual void checkBounds(Operand* object, unsigned lengthOffset,
                            Operand* index, intptr_t handler)
   {
-    appendBoundsCheck(&c, static_cast<Value*>(object),
-                      lengthOffset, static_cast<Value*>(index), handler);
+    appendBoundsCheck(&c, static_cast<Value*>(object), lengthOffset,
+                      static_cast<Value*>(index), handler);
   }
 
   virtual void store(unsigned srcSize, Operand* src, unsigned dstSize,
@@ -6827,8 +6835,11 @@ class MyCompiler: public Compiler {
     appendBarrier(&c, StoreLoadBarrier);
   }
 
-  virtual unsigned compile() {
-    return c.machineCodeSize = local::compile(&c);
+  virtual unsigned compile(uintptr_t stackOverflowHandler,
+                           unsigned stackLimitOffset)
+  {
+    return c.machineCodeSize = local::compile
+      (&c, stackOverflowHandler, stackLimitOffset);
   }
 
   virtual unsigned poolSize() {

src/compiler.h

@@ -188,7 +188,8 @@ class Compiler {
   virtual void storeStoreBarrier() = 0;
   virtual void storeLoadBarrier() = 0;
 
-  virtual unsigned compile() = 0;
+  virtual unsigned compile(uintptr_t stackOverflowHandler,
+                           unsigned stackLimitOffset) = 0;
   virtual unsigned poolSize() = 0;
   virtual void writeTo(uint8_t* dst) = 0;
 

src/interpret.cpp

@@ -30,9 +30,6 @@ class ClassInitList;
 
 class Thread: public vm::Thread {
  public:
-  static const unsigned StackSizeInBytes = 64 * 1024;
-  static const unsigned StackSizeInWords = StackSizeInBytes / BytesPerWord;
-
   Thread(Machine* m, object javaThread, vm::Thread* parent):
     vm::Thread(m, javaThread, parent),
     ip(0),
@@ -78,7 +75,7 @@ pushObject(Thread* t, object o)
     fprintf(stderr, "push object %p at %d\n", o, t->sp);
   }
 
-  assert(t, t->sp + 1 < Thread::StackSizeInWords / 2);
+  assert(t, t->sp + 1 < StackSizeInWords / 2);
   t->stack[(t->sp * 2)    ] = ObjectTag;
   t->stack[(t->sp * 2) + 1] = reinterpret_cast<uintptr_t>(o);
   ++ t->sp;
@@ -91,7 +88,7 @@ pushInt(Thread* t, uint32_t v)
     fprintf(stderr, "push int %d at %d\n", v, t->sp);
   }
 
-  assert(t, t->sp + 1 < Thread::StackSizeInWords / 2);
+  assert(t, t->sp + 1 < StackSizeInWords / 2);
   t->stack[(t->sp * 2)    ] = IntTag;
   t->stack[(t->sp * 2) + 1] = v;
   ++ t->sp;
@@ -184,7 +181,7 @@ peekObject(Thread* t, unsigned index)
             index);
   }
 
-  assert(t, index < Thread::StackSizeInWords / 2);
+  assert(t, index < StackSizeInWords / 2);
   assert(t, t->stack[index * 2] == ObjectTag);
   return *reinterpret_cast<object*>(t->stack + (index * 2) + 1);
 }
@@ -198,7 +195,7 @@ peekInt(Thread* t, unsigned index)
             index);
   }
 
-  assert(t, index < Thread::StackSizeInWords / 2);
+  assert(t, index < StackSizeInWords / 2);
   assert(t, t->stack[index * 2] == IntTag);
   return t->stack[(index * 2) + 1];
 }
@@ -254,7 +251,7 @@ inline object*
 pushReference(Thread* t, object o)
 {
   if (o) {
-    expect(t, t->sp + 1 < Thread::StackSizeInWords / 2);
+    expect(t, t->sp + 1 < StackSizeInWords / 2);
     pushObject(t, o);
     return reinterpret_cast<object*>(t->stack + ((t->sp - 1) * 2) + 1);
   } else {
@@ -436,7 +433,7 @@ checkStack(Thread* t, object method)
                + codeMaxLocals(t, methodCode(t, method))
                + FrameFootprint
                + codeMaxStack(t, methodCode(t, method))
-               > Thread::StackSizeInWords / 2))
+               > StackSizeInWords / 2))
   {
     t->exception = makeThrowable(t, Machine::StackOverflowErrorType);
   }
@@ -3136,7 +3133,7 @@ class MyProcessor: public Processor {
     assert(t, ((methodFlags(t, method) & ACC_STATIC) == 0) xor (this_ == 0));
 
     if (UNLIKELY(t->sp + methodParameterFootprint(t, method) + 1
-                 > Thread::StackSizeInWords / 2))
+                 > StackSizeInWords / 2))
     {
       t->exception = makeThrowable(t, Machine::StackOverflowErrorType);
       return 0;
@@ -3161,7 +3158,7 @@ class MyProcessor: public Processor {
     assert(t, ((methodFlags(t, method) & ACC_STATIC) == 0) xor (this_ == 0));
 
     if (UNLIKELY(t->sp + methodParameterFootprint(t, method) + 1
-                 > Thread::StackSizeInWords / 2))
+                 > StackSizeInWords / 2))
     {
       t->exception = makeThrowable(t, Machine::StackOverflowErrorType);
       return 0;
@@ -3185,7 +3182,7 @@ class MyProcessor: public Processor {
            or t->state == Thread::ExclusiveState);
 
     if (UNLIKELY(t->sp + parameterFootprint(vmt, methodSpec, false)
-                 > Thread::StackSizeInWords / 2))
+                 > StackSizeInWords / 2))
     {
       t->exception = makeThrowable(t, Machine::StackOverflowErrorType);
       return 0;

src/machine.h

@@ -56,6 +56,9 @@ const unsigned ThreadBackupHeapSizeInBytes = 2 * 1024;
 const unsigned ThreadBackupHeapSizeInWords
 = ThreadBackupHeapSizeInBytes / BytesPerWord;
 
+const unsigned StackSizeInBytes = 64 * 1024;
+const unsigned StackSizeInWords = StackSizeInBytes / BytesPerWord;
+
 const unsigned ThreadHeapPoolSize = 64;
 
 const unsigned FixedFootprintThresholdInBytes

test/StackOverflow.java

@@ -0,0 +1,29 @@
+public class StackOverflow {
+  private static void test1() {
+    test1();
+  }
+
+  private static void test2() {
+    test3();
+  }
+
+  private static void test3() {
+    test2();
+  }
+
+  public static void main(String[] args) {
+    try {
+      test1();
+      throw new RuntimeException();
+    } catch (StackOverflowError e) {
+      e.printStackTrace();
+    }
+
+    try {
+      test2();
+      throw new RuntimeException();
+    } catch (StackOverflowError e) {
+      e.printStackTrace();
+    }
+  }
+}