fix incorrect stack unwinding for asynchronous stack traces

Our Thread.getStackTrace implementation is tricky because it might be
invoked on a thread executing arbitrary native or Java code, and there
are numerous edge cases to consider.  Unsurprisingly, there were a few
lingering, non-fatal bugs revealed by Valgrind recently, one involving
the brief interval just before and after returning from invokeNative,
and the other involving an off-by-one error in x86.cpp's nextFrame
implementation.  This commit fixes both.

src/compile-arm.S

@@ -23,12 +23,12 @@
 #endif
 
 #define THREAD_STACK 2148
-#define THREAD_SCRATCH 2152
-#define THREAD_CONTINUATION 2156
+#define THREAD_SCRATCH 2156
+#define THREAD_CONTINUATION 2160
 #define THREAD_EXCEPTION 44
-#define THREAD_EXCEPTION_STACK_ADJUSTMENT 2160
-#define THREAD_EXCEPTION_OFFSET 2164
-#define THREAD_EXCEPTION_HANDLER 2168
+#define THREAD_EXCEPTION_STACK_ADJUSTMENT 2164
+#define THREAD_EXCEPTION_OFFSET 2168
+#define THREAD_EXCEPTION_HANDLER 2172
 
 #define CONTINUATION_NEXT 4
 #define CONTINUATION_ADDRESS 16

src/compile-powerpc.S

@@ -30,11 +30,11 @@
 #define ARGUMENT_BASE BYTES_PER_WORD * LINKAGE_AREA
       
 #define THREAD_STACK 2148
-#define THREAD_CONTINUATION 2156
+#define THREAD_CONTINUATION 2160
 #define THREAD_EXCEPTION 44
-#define THREAD_EXCEPTION_STACK_ADJUSTMENT 2160
-#define THREAD_EXCEPTION_OFFSET 2164
-#define THREAD_EXCEPTION_HANDLER 2168
+#define THREAD_EXCEPTION_STACK_ADJUSTMENT 2164
+#define THREAD_EXCEPTION_OFFSET 2168
+#define THREAD_EXCEPTION_HANDLER 2172
 
 #define CONTINUATION_NEXT 4
 #define CONTINUATION_ADDRESS 16

src/compile-x86.S

@@ -24,7 +24,7 @@
 #ifdef __x86_64__
 
 #define THREAD_STACK 2224
-#define THREAD_SCRATCH 2232
+#define THREAD_SCRATCH 2240
 
 #ifdef AVIAN_USE_FRAME_POINTER
 #  define ALIGNMENT_ADJUSTMENT 0
@@ -300,7 +300,7 @@ LOCAL(vmJumpAndInvoke_argumentTest):
 #elif defined __i386__
 
 #define THREAD_STACK 2148
-#define THREAD_SCRATCH 2152
+#define THREAD_SCRATCH 2156
 
 #ifdef AVIAN_USE_FRAME_POINTER
 #  define ALIGNMENT_ADJUSTMENT 0

src/compile.cpp

@@ -235,6 +235,7 @@ class MyThread: public Thread {
     Thread(m, javaThread, parent),
     ip(0),
     stack(0),
+    newStack(0),
     scratch(0),
     continuation(0),
     exceptionStackAdjustment(0),
@@ -261,6 +262,7 @@ class MyThread: public Thread {
 
   void* ip;
   void* stack;
+  void* newStack;
   void* scratch;
   object continuation;
   uintptr_t exceptionStackAdjustment;
@@ -7620,11 +7622,11 @@ invokeNative(MyThread* t)
 
   stack += t->arch->frameReturnAddressSize();
 
-  transition(t, getIp(t), stack, t->continuation, t->trace);
-
   t->trace->targetMethod = 0;
   t->trace->nativeMethod = 0;
 
+  t->newStack = stack;
+
   return result;
 }
 
@@ -9903,7 +9905,7 @@ compileThunks(MyThread* t, FixedAllocator* allocator)
     compileCall(t, &context, invokeNativeIndex);
   
     a->popFrameAndUpdateStackAndReturn
-      (t->arch->alignFrameSize(1), TargetThreadStack);
+      (t->arch->alignFrameSize(1), TargetThreadNewStack);
 
     p->thunks.native.length = a->endBlock(false)->resolve(0, 0);
 

src/continuations-x86.S

@@ -10,11 +10,11 @@
 
 #ifdef __x86_64__
 
-#define THREAD_CONTINUATION 2240
+#define THREAD_CONTINUATION 2248
 #define THREAD_EXCEPTION 80
-#define THREAD_EXCEPTION_STACK_ADJUSTMENT 2248
-#define THREAD_EXCEPTION_OFFSET 2256
-#define THREAD_EXCEPTION_HANDLER 2264
+#define THREAD_EXCEPTION_STACK_ADJUSTMENT 2256
+#define THREAD_EXCEPTION_OFFSET 2264
+#define THREAD_EXCEPTION_HANDLER 2272
 
 #define CONTINUATION_NEXT 8
 #define CONTINUATION_ADDRESS 32
@@ -91,11 +91,11 @@ LOCAL(vmInvoke_exit):
 
 #elif defined __i386__
    
-#define THREAD_CONTINUATION 2156
+#define THREAD_CONTINUATION 2160
 #define THREAD_EXCEPTION 44
-#define THREAD_EXCEPTION_STACK_ADJUSTMENT 2160
-#define THREAD_EXCEPTION_OFFSET 2164
-#define THREAD_EXCEPTION_HANDLER 2168
+#define THREAD_EXCEPTION_STACK_ADJUSTMENT 2164
+#define THREAD_EXCEPTION_OFFSET 2168
+#define THREAD_EXCEPTION_HANDLER 2172
 
 #define CONTINUATION_NEXT 4
 #define CONTINUATION_ADDRESS 16

src/target.h

@@ -94,13 +94,14 @@ const unsigned TargetBytesPerWord = 8;
 
 const unsigned TargetThreadIp = 2216;
 const unsigned TargetThreadStack = 2224;
-const unsigned TargetThreadTailAddress = 2272;
-const unsigned TargetThreadVirtualCallTarget = 2280;
-const unsigned TargetThreadVirtualCallIndex = 2288;
-const unsigned TargetThreadHeapImage = 2296;
-const unsigned TargetThreadCodeImage = 2304;
-const unsigned TargetThreadThunkTable = 2312;
-const unsigned TargetThreadStackLimit = 2360;
+const unsigned TargetThreadNewStack = 2232;
+const unsigned TargetThreadTailAddress = 2280;
+const unsigned TargetThreadVirtualCallTarget = 2288;
+const unsigned TargetThreadVirtualCallIndex = 2296;
+const unsigned TargetThreadHeapImage = 2304;
+const unsigned TargetThreadCodeImage = 2312;
+const unsigned TargetThreadThunkTable = 2320;
+const unsigned TargetThreadStackLimit = 2368;
 
 const unsigned TargetClassFixedSize = 12;
 const unsigned TargetClassArrayElementSize = 14;
@@ -124,13 +125,14 @@ const unsigned TargetBytesPerWord = 4;
 
 const unsigned TargetThreadIp = 2144;
 const unsigned TargetThreadStack = 2148;
-const unsigned TargetThreadTailAddress = 2172;
-const unsigned TargetThreadVirtualCallTarget = 2176;
-const unsigned TargetThreadVirtualCallIndex = 2180;
-const unsigned TargetThreadHeapImage = 2184;
-const unsigned TargetThreadCodeImage = 2188;
-const unsigned TargetThreadThunkTable = 2192;
-const unsigned TargetThreadStackLimit = 2216;
+const unsigned TargetThreadNewStack = 2152;
+const unsigned TargetThreadTailAddress = 2176;
+const unsigned TargetThreadVirtualCallTarget = 2180;
+const unsigned TargetThreadVirtualCallIndex = 2184;
+const unsigned TargetThreadHeapImage = 2188;
+const unsigned TargetThreadCodeImage = 2192;
+const unsigned TargetThreadThunkTable = 2198;
+const unsigned TargetThreadStackLimit = 2220;
 
 const unsigned TargetClassFixedSize = 8;
 const unsigned TargetClassArrayElementSize = 10;

src/x86.cpp

@@ -2524,10 +2524,10 @@ nextFrame(ArchitectureContext* c UNUSED, uint8_t* start, unsigned size UNUSED,
   // skip stack overflow check, if present:
   if (TargetBytesPerWord == 4) {
     if (*start == 0x39) {
-      start += 11;
+      start += 12;
     }
   } else if (*start == 0x48 and start[1] == 0x39) {
-    start += 12;
+    start += 13;
   }
 
   if (instruction <= start) {