fix StackOverflowError stack walking in tails=true builds

The various Architecture::nextFrame implementations were not walking
the stack correctly when a StackOverflowError was thrown.  The
throwStackOverflow thunk is called before the frame of the most
recently called method has been fully created, and because tails=true
builds use a different calling convention, we need to treat this
situation carefully when building a stack trace or unwinding.
Otherwise, we will skip past all the java frames to the next native
frame, which is what was happening.

include/avian/codegen/architecture.h

@@ -88,7 +88,7 @@ virtual unsigned alignFrameSize(unsigned sizeInWords) = 0;
 
 virtual void nextFrame(void* start, unsigned size, unsigned footprint,
                        void* link, bool mostRecent,
-                       unsigned targetParameterFootprint, void** ip,
+                       int targetParameterFootprint, void** ip,
                        void** stack) = 0;
 virtual void* frameIp(void* stack) = 0;
 virtual unsigned frameHeaderSize() = 0;
@@ -138,4 +138,4 @@ virtual void release() = 0;
 } // namespace codegen
 } // namespace avian
 
-#endif // AVIAN_CODEGEN_ARCHITECTURE_H
+#endif // AVIAN_CODEGEN_ARCHITECTURE_H

src/codegen/target/arm/assembler.cpp

@@ -83,7 +83,7 @@ argumentFootprint(unsigned footprint)
 void
 nextFrame(ArchitectureContext* con, uint32_t* start, unsigned size UNUSED,
           unsigned footprint, void* link, bool,
-          unsigned targetParameterFootprint UNUSED, void** ip, void** stack)
+          int targetParameterFootprint UNUSED, void** ip, void** stack)
 {
   assert(con, *ip >= start);
   assert(con, *ip <= start + (size / TargetBytesPerWord));
@@ -113,13 +113,13 @@ nextFrame(ArchitectureContext* con, uint32_t* start, unsigned size UNUSED,
     return;
   }
 
-  if (TailCalls) {
+  if (TailCalls and targetParameterFootprint >= 0) {
     if (argumentFootprint(targetParameterFootprint) > StackAlignmentInWords) {
       offset += argumentFootprint(targetParameterFootprint)
         - StackAlignmentInWords;
     }
 
-    // check for post-non-tail-call stack adjustment of the form "add
+    // check for post-non-tail-call stack adjustment of the form "sub
     // sp, sp, #offset":
     if ((*instruction >> 12) == 0xe24dd) {
       unsigned value = *instruction & 0xff;
@@ -292,7 +292,7 @@ class MyArchitecture: public Architecture {
 
   virtual void nextFrame(void* start, unsigned size, unsigned footprint,
                          void* link, bool mostRecent,
-                         unsigned targetParameterFootprint, void** ip,
+                         int targetParameterFootprint, void** ip,
                          void** stack)
   {
     arm::nextFrame(&con, static_cast<uint32_t*>(start), size, footprint, link,

src/codegen/target/powerpc/assembler.cpp

@@ -191,7 +191,7 @@ argumentFootprint(unsigned footprint)
 void
 nextFrame(ArchitectureContext* c UNUSED, int32_t* start, unsigned size,
           unsigned footprint, void* link, bool,
-          unsigned targetParameterFootprint, void** ip, void** stack)
+          int targetParameterFootprint, void** ip, void** stack)
 {
   assert(c, *ip >= start);
   assert(c, *ip <= start + (size / BytesPerWord));
@@ -214,7 +214,7 @@ nextFrame(ArchitectureContext* c UNUSED, int32_t* start, unsigned size,
 
   unsigned offset = footprint;
 
-  if (TailCalls) {
+  if (TailCalls and targetParameterFootprint >= 0) {
     if (argumentFootprint(targetParameterFootprint) > StackAlignmentInWords) {
       offset += argumentFootprint(targetParameterFootprint)
         - StackAlignmentInWords;
@@ -391,7 +391,7 @@ class MyArchitecture: public Architecture {
 
   virtual void nextFrame(void* start, unsigned size, unsigned footprint,
                          void* link, bool mostRecent,
-                         unsigned targetParameterFootprint, void** ip,
+                         int targetParameterFootprint, void** ip,
                          void** stack)
   {
     powerpc::nextFrame(&c, static_cast<int32_t*>(start), size, footprint, link,

src/codegen/target/x86/assembler.cpp

@@ -72,7 +72,7 @@ read4(uint8_t* p)
 void
 nextFrame(ArchitectureContext* c UNUSED, uint8_t* start, unsigned size UNUSED,
           unsigned footprint, void*, bool mostRecent,
-          unsigned targetParameterFootprint, void** ip, void** stack)
+          int targetParameterFootprint, void** ip, void** stack)
 {
   assert(c, *ip >= start);
   assert(c, *ip <= start + size);
@@ -114,13 +114,13 @@ nextFrame(ArchitectureContext* c UNUSED, uint8_t* start, unsigned size UNUSED,
 
   unsigned offset = footprint + FrameHeaderSize - (mostRecent ? 1 : 0);
 
-  if (TailCalls) {
+  if (TailCalls and targetParameterFootprint >= 0) {
     if (argumentFootprint(targetParameterFootprint) > StackAlignmentInWords) {
       offset += argumentFootprint(targetParameterFootprint)
         - StackAlignmentInWords;
     }
 
-    // check for post-non-tail-call stack adjustment of the form "add
+    // check for post-non-tail-call stack adjustment of the form "sub
     // $offset,%rsp":
     if (TargetBytesPerWord == 4) {
       if ((*instruction == 0x83 or *instruction == 0x81)
@@ -141,6 +141,14 @@ nextFrame(ArchitectureContext* c UNUSED, uint8_t* start, unsigned size UNUSED,
 
     // todo: check for and handle tail calls
   }
+  
+  if (UseFramePointer and not mostRecent) {
+    assert(c, static_cast<void***>(*stack)[-1] + 1
+           == static_cast<void**>(*stack) + offset);
+
+    assert(c, static_cast<void***>(*stack)[-1][1]
+           == static_cast<void**>(*stack)[offset]);
+  }
 
   *ip = static_cast<void**>(*stack)[offset];
   *stack = static_cast<void**>(*stack) + offset;
@@ -369,7 +377,7 @@ class MyArchitecture: public Architecture {
 
   virtual void nextFrame(void* start, unsigned size, unsigned footprint,
                          void* link, bool mostRecent,
-                         unsigned targetParameterFootprint, void** ip,
+                         int targetParameterFootprint, void** ip,
                          void** stack)
   {
     x86::nextFrame(&c, static_cast<uint8_t*>(start), size, footprint,

test/StackOverflow.java

@@ -1,5 +1,17 @@
 public class StackOverflow {
+  private static int add(int[] numbers, int offset, int length) {
+    if (length == 0) {
+      return 0;
+    } else {
+      return numbers[offset] + add(numbers, offset + 1, length - 1);
+    }
+  }
+  private static int add(int ... numbers) {
+    return add(numbers, 0, numbers.length);
+  }
+
   private static int test1() {
+    add(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
     return test1() + 1;
   }