snapshot

src/assembler.h

@@ -161,6 +161,8 @@ class Assembler {
   virtual unsigned argumentRegisterCount() = 0;
   virtual int argumentRegister(unsigned index) = 0;
 
+  virtual unsigned stackAlignment() = 0;
+
   virtual void plan(UnaryOperation op, unsigned size, uint8_t* typeMask,
                     uint64_t* registerMask, bool* thunk) = 0;
 

src/common.h

@@ -60,12 +60,6 @@
 #  define SO_SUFFIX ".so"
 #endif
 
-#if (defined __APPLE__) && (defined __i386__)
-#  define FORCE_ALIGN __attribute__((force_align_arg_pointer))
-#else
-#  define FORCE_ALIGN
-#endif
-
 #define NO_RETURN __attribute__((noreturn))
 
 #define LIKELY(v) __builtin_expect((v) != 0, true)

src/compile.cpp

@@ -386,6 +386,8 @@ class TraceElement: public TraceHandler {
 };
 
 enum Event {
+  PushContextEvent,
+  PopContextEvent,
   PushEvent,
   PopEvent,
   IpEvent,
@@ -407,15 +409,16 @@ localSize(MyThread* t, object method)
 }
 
 unsigned
-frameSize(MyThread* t, object method)
+alignedFrameSize(MyThread* t, object method)
 {
-  return localSize(t, method) + codeMaxStack(t, methodCode(t, method));
+  return Assembler::alignFrameSize
+    (localSize(t, method) + codeMaxStack(t, methodCode(t, method)));
 }
 
 unsigned
 frameMapSizeInWords(MyThread* t, object method)
 {
-  return ceiling(frameSize(t, method), BitsPerWord) * BytesPerWord;
+  return ceiling(alignedFrameSize(t, method), BitsPerWord) * BytesPerWord;
 }
 
 uint16_t*
@@ -591,7 +594,7 @@ class Frame {
            (t, methodCode(t, context->method)));
 
     if (level > 1) {
-      context->eventLog.append(PushEvent);
+      context->eventLog.append(PushContextEvent);
     }
   }
 
@@ -604,7 +607,7 @@ class Frame {
       }
 
       if (level > 1) {
-        context->eventLog.append(PopEvent);      
+        context->eventLog.append(PopContextEvent);      
       }
     }
   }
@@ -830,54 +833,82 @@ class Frame {
     this->ip = ip;
   }
 
+  void pushQuiet(unsigned size, Compiler::Operand* o) {
+    if (size == 8 and BytesPerWord == 8) {
+      c->push(8);
+
+      context->eventLog.append(PushEvent);
+      context->eventLog.appendAddress(c->top());
+    }
+
+    c->push(size, o);
+    
+    context->eventLog.append(PushEvent);
+    context->eventLog.appendAddress(c->top());
+  }
+
+  Compiler::Operand* popQuiet(unsigned size) {
+    context->eventLog.append(PopEvent);
+    context->eventLog.appendAddress(c->top());
+
+    Compiler::Operand* r = c->pop(8);
+
+    if (size == 8 and BytesPerWord == 8) {
+      context->eventLog.append(PopEvent);
+      context->eventLog.appendAddress(c->top());
+
+      c->pop(8);
+    }
+
+    return r;
+  }
+
   void pushInt(Compiler::Operand* o) {
-    c->push(4, o);
+    pushQuiet(4, o);
     pushedInt();
   }
 
   void pushAddress(Compiler::Operand* o) {
-    c->push(BytesPerWord, o);
+    pushQuiet(BytesPerWord, o);
     pushedInt();
   }
 
   void pushObject(Compiler::Operand* o) {
-    c->push(BytesPerWord, o);
+    pushQuiet(BytesPerWord, o);
     pushedObject();
   }
 
   void pushObject() {
-    c->pushed(1);
+    c->pushed();
+
+    context->eventLog.append(PushEvent);
+    context->eventLog.appendAddress(c->top());
+
     pushedObject();
   }
 
-  void pushLongQuiet(Compiler::Operand* o) {
-    if (BytesPerWord == 8) {
-      c->push(8);
-    }
-    c->push(8, o);
-  }
-
   void pushLong(Compiler::Operand* o) {
-    pushLongQuiet(o);
+    pushQuiet(8, o);
     pushedLong();
   }
 
   void pop(unsigned count) {
     popped(count);
-    c->popped(count);
+
+    for (unsigned i = count; i;) {
+      Compiler::StackElement* s = c->top();
+
+      context->eventLog.append(PopEvent);
+      context->eventLog.appendAddress(s);
+
+      c->popped();
+      i -= c->size(s);
+    }
   }
 
   Compiler::Operand* popInt() {
     poppedInt();
-    return c->pop(4);
+    return popQuiet(4);
-  }
-
-  Compiler::Operand* popLongQuiet() {
-    Compiler::Operand* r = c->pop(8);
-    if (BytesPerWord == 8) {
-      c->pop(8);
-    }
-    return r;
   }
 
   Compiler::Operand* peekLong(unsigned index) {
@@ -886,12 +917,12 @@ class Frame {
 
   Compiler::Operand* popLong() {
     poppedLong();
-    return popLongQuiet();
+    return popQuiet(8);
   }
 
   Compiler::Operand* popObject() {
     poppedObject();
-    return c->pop(BytesPerWord);
+    return popQuiet(BytesPerWord);
   }
 
   void loadInt(unsigned index) {
@@ -925,7 +956,7 @@ class Frame {
   }
 
   void storeObjectOrAddress(unsigned index) {
-    c->storeLocal(BytesPerWord, c->pop(BytesPerWord), index);
+    c->storeLocal(BytesPerWord, popQuiet(BytesPerWord), index);
 
     assert(t, sp >= 1);
     assert(t, sp - 1 >= localSize());
@@ -939,39 +970,39 @@ class Frame {
   }
 
   void dup() {
-    c->push(BytesPerWord, c->peek(BytesPerWord, 0));
+    pushQuiet(BytesPerWord, c->peek(BytesPerWord, 0));
 
     dupped();
   }
 
   void dupX1() {
-    Compiler::Operand* s0 = c->pop(BytesPerWord);
+    Compiler::Operand* s0 = popQuiet(BytesPerWord);
-    Compiler::Operand* s1 = c->pop(BytesPerWord);
+    Compiler::Operand* s1 = popQuiet(BytesPerWord);
 
-    c->push(BytesPerWord, s0);
+    pushQuiet(BytesPerWord, s0);
-    c->push(BytesPerWord, s1);
+    pushQuiet(BytesPerWord, s1);
-    c->push(BytesPerWord, s0);
+    pushQuiet(BytesPerWord, s0);
 
     duppedX1();
   }
 
   void dupX2() {
-    Compiler::Operand* s0 = c->pop(BytesPerWord);
+    Compiler::Operand* s0 = popQuiet(BytesPerWord);
 
     if (get(sp - 2) == Long) {
-      Compiler::Operand* s1 = popLongQuiet();
+      Compiler::Operand* s1 = popQuiet(8);
 
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
-      pushLongQuiet(s1);
+      pushQuiet(8, s1);
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
     } else {
-      Compiler::Operand* s1 = c->pop(BytesPerWord);
+      Compiler::Operand* s1 = popQuiet(BytesPerWord);
-      Compiler::Operand* s2 = c->pop(BytesPerWord);
+      Compiler::Operand* s2 = popQuiet(BytesPerWord);
 
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
-      c->push(BytesPerWord, s2);
+      pushQuiet(BytesPerWord, s2);
-      c->push(BytesPerWord, s1);
+      pushQuiet(BytesPerWord, s1);
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
     }
 
     duppedX2();
@@ -979,15 +1010,15 @@ class Frame {
 
   void dup2() {
     if (get(sp - 1) == Long) {
-      pushLongQuiet(peekLong(0));
+      pushQuiet(8, peekLong(0));
     } else {
-      Compiler::Operand* s0 = c->pop(BytesPerWord);
+      Compiler::Operand* s0 = popQuiet(BytesPerWord);
-      Compiler::Operand* s1 = c->pop(BytesPerWord);
+      Compiler::Operand* s1 = popQuiet(BytesPerWord);
 
-      c->push(BytesPerWord, s1);
+      pushQuiet(BytesPerWord, s1);
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
-      c->push(BytesPerWord, s1);
+      pushQuiet(BytesPerWord, s1);
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
     }
 
     dupped2();
@@ -995,22 +1026,22 @@ class Frame {
 
   void dup2X1() {
     if (get(sp - 1) == Long) {
-      Compiler::Operand* s0 = popLongQuiet();
+      Compiler::Operand* s0 = popQuiet(8);
-      Compiler::Operand* s1 = c->pop(BytesPerWord);
+      Compiler::Operand* s1 = popQuiet(BytesPerWord);
 
-      pushLongQuiet(s0);
+      pushQuiet(8, s0);
-      c->push(BytesPerWord, s1);
+      pushQuiet(BytesPerWord, s1);
-      pushLongQuiet(s0);
+      pushQuiet(8, s0);
     } else {
-      Compiler::Operand* s0 = c->pop(BytesPerWord);
+      Compiler::Operand* s0 = popQuiet(BytesPerWord);
-      Compiler::Operand* s1 = c->pop(BytesPerWord);
+      Compiler::Operand* s1 = popQuiet(BytesPerWord);
-      Compiler::Operand* s2 = c->pop(BytesPerWord);
+      Compiler::Operand* s2 = popQuiet(BytesPerWord);
 
-      c->push(BytesPerWord, s1);
+      pushQuiet(BytesPerWord, s1);
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
-      c->push(BytesPerWord, s2);
+      pushQuiet(BytesPerWord, s2);
-      c->push(BytesPerWord, s1);
+      pushQuiet(BytesPerWord, s1);
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
     }
 
     dupped2X1();
@@ -1018,46 +1049,46 @@ class Frame {
 
   void dup2X2() {
     if (get(sp - 1) == Long) {
-      Compiler::Operand* s0 = popLongQuiet();
+      Compiler::Operand* s0 = popQuiet(8);
 
       if (get(sp - 3) == Long) {
-        Compiler::Operand* s1 = popLongQuiet();
+        Compiler::Operand* s1 = popQuiet(8);
 
-        pushLongQuiet(s0);
+        pushQuiet(8, s0);
-        pushLongQuiet(s1);
+        pushQuiet(8, s1);
-        pushLongQuiet(s0);
+        pushQuiet(8, s0);
       } else {
-        Compiler::Operand* s1 = c->pop(BytesPerWord);
+        Compiler::Operand* s1 = popQuiet(BytesPerWord);
-        Compiler::Operand* s2 = c->pop(BytesPerWord);
+        Compiler::Operand* s2 = popQuiet(BytesPerWord);
 
-        pushLongQuiet(s0);
+        pushQuiet(8, s0);
-        c->push(BytesPerWord, s2);
+        pushQuiet(BytesPerWord, s2);
-        c->push(BytesPerWord, s1);
+        pushQuiet(BytesPerWord, s1);
-        pushLongQuiet(s0);
+        pushQuiet(8, s0);
       }
     } else {
-      Compiler::Operand* s0 = c->pop(BytesPerWord);
+      Compiler::Operand* s0 = popQuiet(BytesPerWord);
-      Compiler::Operand* s1 = c->pop(BytesPerWord);
+      Compiler::Operand* s1 = popQuiet(BytesPerWord);
-      Compiler::Operand* s2 = c->pop(BytesPerWord);
+      Compiler::Operand* s2 = popQuiet(BytesPerWord);
-      Compiler::Operand* s3 = c->pop(BytesPerWord);
+      Compiler::Operand* s3 = popQuiet(BytesPerWord);
 
-      c->push(BytesPerWord, s1);
+      pushQuiet(BytesPerWord, s1);
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
-      c->push(BytesPerWord, s3);
+      pushQuiet(BytesPerWord, s3);
-      c->push(BytesPerWord, s2);
+      pushQuiet(BytesPerWord, s2);
-      c->push(BytesPerWord, s1);
+      pushQuiet(BytesPerWord, s1);
-      c->push(BytesPerWord, s0);
+      pushQuiet(BytesPerWord, s0);
     }
 
     dupped2X2();
   }
 
   void swap() {
-    Compiler::Operand* s0 = c->pop(BytesPerWord);
+    Compiler::Operand* s0 = popQuiet(BytesPerWord);
-    Compiler::Operand* s1 = c->pop(BytesPerWord);
+    Compiler::Operand* s1 = popQuiet(BytesPerWord);
 
-    c->push(BytesPerWord, s0);
+    pushQuiet(BytesPerWord, s0);
-    c->push(BytesPerWord, s1);
+    pushQuiet(BytesPerWord, s1);
 
     swapped();
   }
@@ -1192,14 +1223,14 @@ unwind(MyThread* t)
   vmJump(ip, base, stack, t);
 }
 
-void FORCE_ALIGN
+void
 tryInitClass(MyThread* t, object class_)
 {
   initClass(t, class_);
   if (UNLIKELY(t->exception)) unwind(t);
 }
 
-void* FORCE_ALIGN
+void*
 findInterfaceMethodFromInstance(MyThread* t, object method, object instance)
 {
   if (instance) {
@@ -1212,7 +1243,7 @@ findInterfaceMethodFromInstance(MyThread* t, object method, object instance)
   }
 }
 
-intptr_t FORCE_ALIGN
+intptr_t
 compareDoublesG(uint64_t bi, uint64_t ai)
 {
   double a = bitsToDouble(ai);
@@ -1229,7 +1260,7 @@ compareDoublesG(uint64_t bi, uint64_t ai)
   }
 }
 
-intptr_t FORCE_ALIGN
+intptr_t
 compareDoublesL(uint64_t bi, uint64_t ai)
 {
   double a = bitsToDouble(ai);
@@ -1246,7 +1277,7 @@ compareDoublesL(uint64_t bi, uint64_t ai)
   }
 }
 
-intptr_t FORCE_ALIGN
+intptr_t
 compareFloatsG(uint32_t bi, uint32_t ai)
 {
   float a = bitsToFloat(ai);
@@ -1263,7 +1294,7 @@ compareFloatsG(uint32_t bi, uint32_t ai)
   }
 }
 
-intptr_t FORCE_ALIGN
+intptr_t
 compareFloatsL(uint32_t bi, uint32_t ai)
 {
   float a = bitsToFloat(ai);
@@ -1280,151 +1311,151 @@ compareFloatsL(uint32_t bi, uint32_t ai)
   }
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 addDouble(uint64_t b, uint64_t a)
 {
   return doubleToBits(bitsToDouble(a) + bitsToDouble(b));
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 subtractDouble(uint64_t b, uint64_t a)
 {
   return doubleToBits(bitsToDouble(a) - bitsToDouble(b));
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 multiplyDouble(uint64_t b, uint64_t a)
 {
   return doubleToBits(bitsToDouble(a) * bitsToDouble(b));
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 divideDouble(uint64_t b, uint64_t a)
 {
   return doubleToBits(bitsToDouble(a) / bitsToDouble(b));
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 moduloDouble(uint64_t b, uint64_t a)
 {
   return doubleToBits(fmod(bitsToDouble(a), bitsToDouble(b)));
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 negateDouble(uint64_t a)
 {
   return doubleToBits(- bitsToDouble(a));
 }
 
-uint32_t FORCE_ALIGN
+uint32_t
 doubleToFloat(int64_t a)
 {
   return floatToBits(static_cast<float>(bitsToDouble(a)));
 }
 
-int32_t FORCE_ALIGN
+int32_t
 doubleToInt(int64_t a)
 {
   return static_cast<int32_t>(bitsToDouble(a));
 }
 
-int64_t FORCE_ALIGN
+int64_t
 doubleToLong(int64_t a)
 {
   return static_cast<int64_t>(bitsToDouble(a));
 }
 
-uint32_t FORCE_ALIGN
+uint32_t
 addFloat(uint32_t b, uint32_t a)
 {
   return floatToBits(bitsToFloat(a) + bitsToFloat(b));
 }
 
-uint32_t FORCE_ALIGN
+uint32_t
 subtractFloat(uint32_t b, uint32_t a)
 {
   return floatToBits(bitsToFloat(a) - bitsToFloat(b));
 }
 
-uint32_t FORCE_ALIGN
+uint32_t
 multiplyFloat(uint32_t b, uint32_t a)
 {
   return floatToBits(bitsToFloat(a) * bitsToFloat(b));
 }
 
-uint32_t FORCE_ALIGN
+uint32_t
 divideFloat(uint32_t b, uint32_t a)
 {
   return floatToBits(bitsToFloat(a) / bitsToFloat(b));
 }
 
-uint32_t FORCE_ALIGN
+uint32_t
 moduloFloat(uint32_t b, uint32_t a)
 {
   return floatToBits(fmod(bitsToFloat(a), bitsToFloat(b)));
 }
 
-uint32_t FORCE_ALIGN
+uint32_t
 negateFloat(uint32_t a)
 {
   return floatToBits(- bitsToFloat(a));
 }
 
-int64_t FORCE_ALIGN
+int64_t
 divideLong(int64_t b, int64_t a)
 {
   return a / b;
 }
 
-int64_t FORCE_ALIGN
+int64_t
 moduloLong(int64_t b, int64_t a)
 {
   return a % b;
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 floatToDouble(int32_t a)
 {
   return doubleToBits(static_cast<double>(bitsToFloat(a)));
 }
 
-int32_t FORCE_ALIGN
+int32_t
 floatToInt(int32_t a)
 {
   return static_cast<int32_t>(bitsToFloat(a));
 }
 
-int64_t FORCE_ALIGN
+int64_t
 floatToLong(int32_t a)
 {
   return static_cast<int64_t>(bitsToFloat(a));
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 intToDouble(int32_t a)
 {
   return doubleToBits(static_cast<double>(a));
 }
 
-uint32_t FORCE_ALIGN
+uint32_t
 intToFloat(int32_t a)
 {
   return floatToBits(static_cast<float>(a));
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 longToDouble(int64_t a)
 {
   return doubleToBits(static_cast<double>(a));
 }
 
-uint32_t FORCE_ALIGN
+uint32_t
 longToFloat(int64_t a)
 {
   return floatToBits(static_cast<float>(a));
 }
 
-object FORCE_ALIGN
+object
 makeBlankObjectArray(MyThread* t, object class_, int32_t length)
 {
   if (length >= 0) {
@@ -1436,7 +1467,7 @@ makeBlankObjectArray(MyThread* t, object class_, int32_t length)
   }
 }
 
-object FORCE_ALIGN
+object
 makeBlankArray(MyThread* t, object (*constructor)(Thread*, uintptr_t, bool),
                int32_t length)
 {
@@ -1449,7 +1480,7 @@ makeBlankArray(MyThread* t, object (*constructor)(Thread*, uintptr_t, bool),
   }
 }
 
-uintptr_t FORCE_ALIGN
+uintptr_t
 lookUpAddress(int32_t key, uintptr_t* start, int32_t count,
               uintptr_t default_)
 {
@@ -1472,7 +1503,7 @@ lookUpAddress(int32_t key, uintptr_t* start, int32_t count,
   return default_;
 }
 
-void FORCE_ALIGN
+void
 setMaybeNull(MyThread* t, object o, unsigned offset, object value)
 {
   if (LIKELY(o)) {
@@ -1483,7 +1514,7 @@ setMaybeNull(MyThread* t, object o, unsigned offset, object value)
   }
 }
 
-void FORCE_ALIGN
+void
 acquireMonitorForObject(MyThread* t, object o)
 {
   if (LIKELY(o)) {
@@ -1494,7 +1525,7 @@ acquireMonitorForObject(MyThread* t, object o)
   }
 }
 
-void FORCE_ALIGN
+void
 releaseMonitorForObject(MyThread* t, object o)
 {
   if (LIKELY(o)) {
@@ -1530,7 +1561,7 @@ makeMultidimensionalArray2(MyThread* t, object class_, uintptr_t* stack,
   return array;
 }
 
-object FORCE_ALIGN
+object
 makeMultidimensionalArray(MyThread* t, object class_, int32_t dimensions,
                           uintptr_t* stack)
 {
@@ -1563,7 +1594,7 @@ traceSize(Thread* t)
     + (counter.count * FixedSizeOfTraceElement);
 }
 
-void NO_RETURN FORCE_ALIGN
+void NO_RETURN
 throwArrayIndexOutOfBounds(MyThread* t)
 {
   ensure(t, FixedSizeOfArrayIndexOutOfBoundsException + traceSize(t));
@@ -1575,7 +1606,7 @@ throwArrayIndexOutOfBounds(MyThread* t)
   unwind(t);
 }
 
-void NO_RETURN FORCE_ALIGN
+void NO_RETURN
 throw_(MyThread* t, object o)
 {
   if (LIKELY(o)) {
@@ -1586,7 +1617,7 @@ throw_(MyThread* t, object o)
   unwind(t);
 }
 
-void FORCE_ALIGN
+void
 checkCast(MyThread* t, object class_, object o)
 {
   if (UNLIKELY(o and not isAssignableFrom(t, class_, objectClass(t, o)))) {
@@ -1599,7 +1630,7 @@ checkCast(MyThread* t, object class_, object o)
   }
 }
 
-void FORCE_ALIGN
+void
 gcIfNecessary(MyThread* t)
 {
   if (UNLIKELY(t->backupHeap)) {
@@ -1687,32 +1718,32 @@ compileDirectInvoke(MyThread* t, Frame* frame, object target)
 
   if (not emptyMethod(t, target)) {
     if (methodFlags(t, target) & ACC_NATIVE) {
-      result = c->call
+      result = c->stackCall
         (c->constant
          (reinterpret_cast<intptr_t>
           (&singletonBody(t, nativeThunk(t), 0))),
          0,
          frame->trace(target, false),
          rSize,
-         0);
+         methodParameterFootprint(t, target));
     } else if (methodCompiled(t, target) == defaultThunk(t)) {
-      result = c->call
+      result = c->stackCall
         (c->constant
          (reinterpret_cast<intptr_t>
           (&singletonBody(t, defaultThunk(t), 0))),
          Compiler::Aligned,
          frame->trace(target, false),
          rSize,
-         0);
+         methodParameterFootprint(t, target));
     } else {
-      result = c->call
+      result = c->stackCall
         (c->constant
          (reinterpret_cast<intptr_t>
           (&singletonBody(t, methodCompiled(t, target), 0))),
          0,
          frame->trace(0, false),
          rSize,
-         0);
+         methodParameterFootprint(t, target));
     }
   }
 
@@ -2629,7 +2660,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
 
       unsigned rSize = resultSize(t, methodReturnCode(t, target));
 
-      Compiler::Operand* result = c->call
+      Compiler::Operand* result = c->stackCall
         (c->call
          (c->constant
           (getThunk(t, findInterfaceMethodFromInstanceThunk)),
@@ -2641,7 +2672,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
          0,
          frame->trace(target, true),
          rSize,
-         0);
+         parameterFootprint);
 
       frame->pop(parameterFootprint);
 
@@ -2688,7 +2719,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
 
       unsigned rSize = resultSize(t, methodReturnCode(t, target));
 
-      Compiler::Operand* result = c->call
+      Compiler::Operand* result = c->stackCall
         (c->memory
          (c->and_
           (BytesPerWord, c->constant(PointerMask),
@@ -2696,7 +2727,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip,
          0,
          frame->trace(target, true),
          rSize,
-         0);
+         parameterFootprint);
 
       frame->pop(parameterFootprint);
 
@@ -3486,7 +3517,7 @@ printSet(uintptr_t m)
 
 unsigned
 calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots,
-                   unsigned eventIndex)
+                   unsigned stackPadding, unsigned eventIndex)
 {
   // for each instruction with more than one predecessor, and for each
   // stack position, determine if there exists a path to that
@@ -3494,6 +3525,9 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots,
   // stack position (i.e. it is uninitialized or contains primitive
   // data).
 
+  Compiler* c = context->compiler;
+
+  unsigned localSize = ::localSize(t, context->method);
   unsigned mapSize = frameMapSizeInWords(t, context->method);
 
   uintptr_t roots[mapSize];
@@ -3517,13 +3551,30 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots,
   while (eventIndex < length) {
     Event e = static_cast<Event>(context->eventLog.get(eventIndex++));
     switch (e) {
-    case PushEvent: {
+    case PushContextEvent: {
-      eventIndex = calculateFrameMaps(t, context, roots, eventIndex);
+      eventIndex = calculateFrameMaps
+        (t, context, roots, stackPadding, eventIndex);
     } break;
 
-    case PopEvent:
+    case PopContextEvent:
       return eventIndex;
 
+    case PushEvent: {
+      Compiler::StackElement* s;
+      context->eventLog.get(eventIndex, &s, BytesPerWord);
+      stackPadding += c->padding(s);
+
+      eventIndex += BytesPerWord;
+    } break;
+
+    case PopEvent: {
+      Compiler::StackElement* s;
+      context->eventLog.get(eventIndex, &s, BytesPerWord);
+      stackPadding -= c->padding(s);
+
+      eventIndex += BytesPerWord;
+    } break;
+
     case IpEvent: {
       ip = context->eventLog.get2(eventIndex);
       eventIndex += 2;
@@ -3541,7 +3592,7 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots,
           uintptr_t newRoots = tableRoots[wi] & roots[wi];
 
           if ((eventIndex == length
-               or context->eventLog.get(eventIndex) == PopEvent)
+               or context->eventLog.get(eventIndex) == PopContextEvent)
               and newRoots != tableRoots[wi])
           {
             if (DebugFrameMaps) {
@@ -3569,6 +3620,10 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots,
       unsigned i = context->eventLog.get2(eventIndex);
       eventIndex += 2;
 
+      if (i > localSize) {
+        i += stackPadding;
+      }
+
       markBit(roots, i);
     } break;
 
@@ -3576,6 +3631,10 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots,
       unsigned i = context->eventLog.get2(eventIndex);
       eventIndex += 2;
 
+      if (i > localSize) {
+        i += stackPadding;
+      }
+
       clearBit(roots, i);
     } break;
 
@@ -3628,7 +3687,7 @@ compareMethodBounds(Thread* t, object a, object b)
 unsigned
 frameObjectMapSize(MyThread* t, object method, object map)
 {
-  int size = frameSize(t, method);
+  int size = alignedFrameSize(t, method);
   return ceiling(intArrayLength(t, map) * size, 32 + size);
 }
 
@@ -3719,7 +3778,7 @@ finish(MyThread* t, Context* context)
     qsort(elements, context->traceLogCount, sizeof(TraceElement*),
           compareTraceElementPointers);
 
-    unsigned size = frameSize(t, context->method);
+    unsigned size = alignedFrameSize(t, context->method);
     object map = makeIntArray
       (t, context->traceLogCount
        + ceiling(context->traceLogCount * size, 32),
@@ -3835,7 +3894,7 @@ compile(MyThread* t, Context* context)
   if (UNLIKELY(t->exception)) return 0;
 
   context->dirtyRoots = false;
-  unsigned eventIndex = calculateFrameMaps(t, context, 0, 0);
+  unsigned eventIndex = calculateFrameMaps(t, context, 0, 0, 0);
 
   object eht = codeExceptionHandlerTable(t, methodCode(t, context->method));
   if (eht) {
@@ -3881,7 +3940,7 @@ compile(MyThread* t, Context* context)
           compile(t, &frame2, exceptionHandlerIp(eh), true);
           if (UNLIKELY(t->exception)) return 0;
 
-          eventIndex = calculateFrameMaps(t, context, 0, eventIndex);
+          eventIndex = calculateFrameMaps(t, context, 0, 0, eventIndex);
         }
       }
 
@@ -3891,7 +3950,7 @@ compile(MyThread* t, Context* context)
 
   while (context->dirtyRoots) {
     context->dirtyRoots = false;
-    calculateFrameMaps(t, context, 0, 0);
+    calculateFrameMaps(t, context, 0, 0, 0);
   }
 
   return finish(t, context);
@@ -3935,7 +3994,7 @@ compileMethod2(MyThread* t)
   }
 }
 
-void* FORCE_ALIGN
+void*
 compileMethod(MyThread* t)
 {
   void* r = compileMethod2(t);
@@ -4129,7 +4188,7 @@ invokeNative2(MyThread* t, object method)
   return result;
 }
 
-uint64_t FORCE_ALIGN
+uint64_t
 invokeNative(MyThread* t)
 {
   if (t->trace->nativeMethod == 0) {
@@ -4170,7 +4229,7 @@ frameMapIndex(MyThread* t, object method, int32_t offset)
     int32_t v = intArrayBody(t, map, middle);
       
     if (offset == v) {
-      return (indexSize * 32) + (frameSize(t, method) * middle);
+      return (indexSize * 32) + (alignedFrameSize(t, method) * middle);
     } else if (offset < v) {
       top = middle;
     } else {
@@ -4193,7 +4252,7 @@ visitStackAndLocals(MyThread* t, Heap::Visitor* v, void* base, object method,
 
     count = parameterFootprint + height - argumentFootprint;
   } else {
-    count = frameSize(t, method);
+    count = alignedFrameSize(t, method);
   }
       
   if (count) {

src/compiler.cpp

@@ -69,57 +69,37 @@ class Site {
   Site* next;
 };
 
-class Stack {
+class Stack: public StackElement {
  public:
-  Stack(Value* value, unsigned size, unsigned index, Stack* next):
+  Stack(unsigned index, Value* value, Stack* next):
-    value(value), size(size), index(index), next(next), pushEvent(0),
+    index(index), value(value), next(next)
-    pushSite(0), pushed(false)
   { }
 
-  Value* value;
-  unsigned size;
   unsigned index;
+  Value* value;
   Stack* next;
-  PushEvent* pushEvent;
-  Site* pushSite;
-  bool pushed;
 };
 
 class State {
  public:
-  State(State* next, Stack* stack):
+  State(State* next, Stack* stack, Value** locals):
     stack(stack),
+    locals(locals),
     next(next)
   { }
 
   Stack* stack;
+  Value** locals;
   State* next;
 };
 
-class Local {
- public:
-  Local(unsigned size, unsigned index, Value* value, Site* site, Local* old,
-        Local* next):
-    size(size), index(index), reuse(true), value(value), site(site), old(old),
-    next(next)
-  { }
-
-  unsigned size;
-  unsigned index;
-  bool reuse;
-  Value* value;
-  Site* site;
-  Local* old;
-  Local* next;
-};
-
 class LogicalInstruction {
  public:
   Event* firstEvent;
   Event* lastEvent;
   LogicalInstruction* immediatePredecessor;
   Stack* stack;
-  Local* locals;
+  Value** locals;
   unsigned machineOffset;
   bool stackSaved;
 };
@@ -162,18 +142,22 @@ class Junction {
 
 class Read {
  public:
-  Read(unsigned size, Value* value, Site* target, Read* next, Event* event,
+  Read():
-       Read* eventNext):
+    next(0), event(0), eventNext(0)
-    size(size), value(value), target(target), next(next), event(event),
-    eventNext(eventNext)
   { }
   
-  unsigned size;
-  Value* value;
-  Site* target;
   Read* next;
   Event* event;
   Read* eventNext;
+
+  virtual Site* pickSite(Context* c, Value* v) = 0;
+
+  virtual Site* allocateSite(Context* c) = 0;
+
+  virtual void intersect(uint8_t* typeMask, uint64_t* registerMask,
+                         int* frameIndex) = 0;
+  
+  virtual bool valid() = 0;
 };
 
 class Value: public Compiler::Operand {
@@ -189,6 +173,11 @@ class Value: public Compiler::Operand {
   Site* target;
 };
 
+enum Pass {
+  ScanPass,
+  CompilePass
+};
+
 class Context {
  public:
   Context(System* system, Assembler* assembler, Zone* zone,
@@ -212,10 +201,10 @@ class Context {
     nextSequence(0),
     junctions(0),
     machineCode(0),
-    locals(0),
-    localTable(0),
     stackReset(false),
-    constantCompare(CompareNone)
+    constantCompare(CompareNone),
+    pass(ScanPass),
+    stackPadding(0)
   {
     for (unsigned i = 0; i < assembler->registerCount(); ++i) {
       registers[i] = new (zone->allocate(sizeof(Register))) Register(i);
@@ -243,10 +232,10 @@ class Context {
   unsigned nextSequence;
   Junction* junctions;
   uint8_t* machineCode;
-  Local* locals;
-  Local** localTable;
   bool stackReset;
   ConstantCompare constantCompare;
+  Pass pass;
+  unsigned stackPadding;
 };
 
 class PoolPromise: public Promise {
@@ -368,11 +357,9 @@ class Event {
 
   virtual void compile(Context* c) = 0;
 
-  virtual bool skipMove(unsigned) { return false; }
-
   Event* next;
   Stack* stack;
-  Local* locals;
+  Value** locals;
   CodePromise* promises;
   Read* reads;
   unsigned readCount;
@@ -452,13 +439,25 @@ clearSites(Context* c, Value* v)
   v->sites = 0;
 }
 
+bool
+valid(Read* r)
+{
+  return r and r->valid();
+}
+
+bool
+live(Value* v)
+{
+  return valid(v->reads);
+}
+
 void
 nextRead(Context* c, Value* v)
 {
 //   fprintf(stderr, "pop read %p from %p; next: %p\n", v->reads, v, v->reads->next);
 
   v->reads = v->reads->next;
-  if (v->reads == 0) {
+  if (not live(v)) {
     clearSites(c, v);
   }
 }
@@ -709,7 +708,7 @@ decrement(Context* c UNUSED, Register* r)
 class MemorySite: public Site {
  public:
   MemorySite(int base, int offset, int index, unsigned scale):
-    base(0), index(0), value(base, offset, index, scale)
+    base(0), index(0), offsetPromise(0), value(base, offset, index, scale)
   { }
 
   void sync(Context* c UNUSED) {
@@ -772,40 +771,15 @@ memorySite(Context* c, int base, int offset = 0, int index = NoRegister,
     MemorySite(base, offset, index, scale);
 }
 
-bool
-matchRegister(Context* c UNUSED, Site* s, uint64_t mask)
-{
-  assert(c, s->type(c) == RegisterOperand);
-
-  RegisterSite* r = static_cast<RegisterSite*>(s);
-  if (r->low) {
-    r->sync(c);
-    return ((static_cast<uint64_t>(1) << r->register_.low) & mask)
-      and (r->register_.high == NoRegister
-           or ((static_cast<uint64_t>(1) << (r->register_.high + 32)) & mask));
-  } else {
-    return false;
-  }
-}
-
-bool
-match(Context* c, Site* s, uint8_t typeMask, uint64_t registerMask)
-{
-  OperandType t = s->type(c);
-  return ((1 << t) & typeMask)
-    and (t != RegisterOperand or matchRegister(c, s, registerMask));
-}
-
 Site*
-targetOrNull(Context* c, Read* r)
+targetOrNull(Context* c, Value* v, Read* r)
 {
-  Value* v = r->value;
   if (v->target) {
     return v->target;
-  } else if (r->target) {
-    return r->target->readTarget(c, r);
   } else {
-    return 0;
+    Site* s = r->pickSite(c, v);
+    if (s) return s;
+    return r->allocateSite(c);
   }
 }
 
@@ -814,49 +788,22 @@ targetOrNull(Context* c, Value* v)
 {
   if (v->target) {
     return v->target;
-  } else if (v->reads and v->reads->target) {
+  } else if (live(v)) {
-    return v->reads->target->readTarget(c, v->reads);
+    Site* s = r->pickSite(c, v);
+    if (s) return s;
+    return r->allocateSite(c);
   }
   return 0;
 }
 
-class AbstractSite: public Site {
+Site*
- public:
+pickSite(Context* c, Value* value, uint8_t typeMask, uint64_t registerMask,
-  virtual unsigned copyCost(Context* c, Site*) {
+         int frameIndex)
-    abort(c);
+{
-  }
-
-  virtual void copyTo(Context* c, unsigned, Site*) {
-    abort(c);
-  }
-
-  virtual OperandType type(Context* c) {
-    abort(c);
-  }
-
-  virtual Assembler::Operand* asAssemblerOperand(Context* c) {
-    abort(c);
-  }
-};
-
-class VirtualSite: public AbstractSite {
- public:
-  VirtualSite(Value* value, uint8_t typeMask, uint64_t registerMask):
-    value(value), registerMask(registerMask), typeMask(typeMask)
-  { }
-
-  virtual Site* readTarget(Context* c, Read* r) {
-    if (value) {
-      Site* s = targetOrNull(c, value);
-      if (s and match(c, s, typeMask, registerMask)) {
-        return s;
-      }
-    }
-
   Site* site = 0;
   unsigned copyCost = 0xFFFFFFFF;
-    for (Site* s = r->value->sites; s; s = s->next) {
+  for (Site* s = value->sites; s; s = s->next) {
-      if (match(c, s, typeMask, registerMask)) {
+    if (s->match(c, typeMask, registerMask, frameIndex)) {
       unsigned v = s->copyCost(c, 0);
       if (v < copyCost) {
         site = s;
@@ -864,27 +811,129 @@ class VirtualSite: public AbstractSite {
       }
     }
   }
-
-    if (site) {
   return site;
-    } else {
+}
-      assert(c, typeMask & (1 << RegisterOperand));
+
+Site*
+allocateSite(Context* c, uint8_t typeMask, uint64_t registerMask,
+             int frameIndex)
+{
+  if ((typeMask & (1 << RegisterOperand)) and registerMask) {
     return freeRegisterSite(c, registerMask);
+  } else if (frameIndex >= 0) {
+    return frameSite(c, frameIndex);
+  } else {
+    abort(c);
+  }
+}
+
+class TargetRead: public Read {
+ public:
+  TargetRead(Value* target, uint8_t typeMask, uint64_t registerMask,
+             int frameIndex):
+    target(target), registerMask(registerMask), typeMask(typeMask),
+    frameIndex(frameIndex)
+  { }
+
+  virtual Site* pickSite(Context* c, Value* value) {
+    return ::pickSite(c, value, typeMask, registerMask, frameIndex);
+  }
+
+  virtual Site* allocateSite(Context* c) {
+    if (target) {
+      Site* s = targetOrNull(c, target);
+      if (s and s->match(c, typeMask, registerMask, frameIndex)) {
+        return s;
+      }
+    }
+    return ::allocateSite(c, typeMask, registerMask, frameIndex);
+  }
+
+  virtual void intersect(uint8_t* typeMask, uint64_t* registerMask,
+                         int* frameIndex)
+  {
+    *typeMask &= this->typeMask;
+    *registerMask &= this->registerMask;
+
+    if (*frameIndex == AnyFrameIndex) {
+      *frameIndex = this->frameIndex;
+    } else if (this->frameIndex != AnyFrameIndex
+               and *frameIndex != this->frameIndex)
+    {
+      *frameIndex = NoFrameIndex;
     }
   }
 
-  Value* value;
+  virtual bool valid() {
+    return true;
+  }
+
+  Value* target;
   uint64_t registerMask;
   uint8_t typeMask;
+  int frameIndex;
 };
 
-VirtualSite*
+TargetRead*
-virtualSite(Context* c, Value* v = 0,
+targetRead(Context* c, Value* target = 0,
            uint8_t typeMask = ~static_cast<uint8_t>(0),
-            uint64_t registerMask = ~static_cast<uint64_t>(0))
+           uint64_t registerMask = ~static_cast<uint64_t>(0),
+           int frameIndex = NoFrameIndex)
 {
-  return new (c->zone->allocate(sizeof(VirtualSite)))
+  return new (c->zone->allocate(sizeof(TargetRead)))
-    VirtualSite(v, typeMask, registerMask);
+    TargetRead(target, typeMask, registerMask);
+}
+
+class MultiRead: public Read {
+ public:
+  MultiRead():
+    reads(0)
+  { }
+
+  virtual Site* pickSite(Context* c) {
+    uint8_t typeMask = ~static_cast<uint8_t>(0);
+    uint64_t registerMask = ~static_cast<uint64_t>(0);
+    int frameIndex = AnyFrameIndex;
+    intersect(&typeMask, &registerMask, &frameIndex);
+
+    return ::pickSite(c, value, typeMask, registerMask, frameIndex);
+  }
+
+  virtual Site* allocateSite(Context* c) {
+    uint8_t typeMask = ~static_cast<uint8_t>(0);
+    uint64_t registerMask = ~static_cast<uint64_t>(0);
+    int frameIndex = AnyFrameIndex;
+    intersect(&typeMask, &registerMask, &frameIndex);
+
+    return ::allocateSite(c, typeMask, registerMask, frameIndex);
+  }
+
+  virtual void intersect(uint8_t* typeMask, uint64_t* registerMask,
+                         int* frameIndex)
+  {
+    for (Cell* cell = reads; cell; cell = cell->next) {
+      Read* r = static_cast<Read*>(cell->value);
+      r->intersect(&typeMask, &registerMask, &frameIndex);
+    }
+  }
+
+  virtual bool valid() {
+    for (Cell* cell = reads; cell; cell = cell->next) {
+      Read* r = static_cast<Read*>(cell->value);
+      if (r->valid()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  Cell* reads;
+};
+
+MultiRead*
+multiRead(Context* c)
+{
+  return new (c->zone->allocate(sizeof(MultiRead))) MultiRead;
 }
 
 VirtualSite*
@@ -928,72 +977,8 @@ pick(Context* c, Site* sites, Site* target = 0, unsigned* cost = 0)
   return site;
 }
 
-unsigned
-stackOffset(Context* c)
-{
-  return c->localFootprint - c->parameterFootprint;
-}
-
-Site*
-pushSite(Context* c, unsigned index)
-{
-  return memorySite
-    (c, c->assembler->base(),
-     - (stackOffset(c) + index + 1) * BytesPerWord, NoRegister, 1);
-}
-
-void
-pushNow(Context* c, Stack* start, unsigned count)
-{
-  Stack* segment[count];
-  unsigned index = count;
-  for (Stack* s = start; s and index; s = s->next) {
-    segment[--index] = s;
-  }
-
-  for (unsigned i = 0; i < count; ++i) {
-    Stack* s = segment[i];
-    assert(c, not s->pushed);
-
-    if (s->value and s->value->sites) {
-      Site* source = pick(c, s->value->sites);
-
-      removeMemorySites(c, s->value);
-
-      s->pushSite = pushSite(c, s->index);
-      addSite(c, 0, s->size * BytesPerWord, s->value, s->pushSite);
-
-      apply(c, Push, s->size * BytesPerWord, source);
-    } else {
-      Assembler::Register stack(c->assembler->stack());
-      Assembler::Constant offset(resolved(c, s->size * BytesPerWord));
-      c->assembler->apply
-        (Subtract, BytesPerWord, ConstantOperand, &offset,
-         RegisterOperand, &stack);
-    }
-
-    if (DebugStack) {
-      fprintf(stderr, "pushed %p value: %p sites: %p\n",
-              s, s->value, s->value->sites);
-    }
-
-    s->pushed = true;
-  }
-}
-
-void
-pushNow(Context* c, Stack* start)
-{
-  unsigned count = 0;
-  for (Stack* s = start; s and (not s->pushed); s = s->next) {
-    ++ count;
-  }
-
-  pushNow(c, start, count);
-}
-
 bool
-trySteal(Context* c, Register* r, Stack* stack)
+trySteal(Context* c, Register* r, Stack* stack, Value** locals)
 {
   assert(c, r->refCount == 0);
 
@@ -1233,11 +1218,8 @@ apply(Context* c, BinaryOperation op, unsigned size, Site* a, Site* b)
 }
 
 void
-insertRead(Context* c, Event* event, int sequence, Value* v,
+insertRead(Context* c, Event* event, int sequence, Value* v, Read* r)
-           unsigned size, Site* target)
 {
-  Read* r = new (c->zone->allocate(sizeof(Read)))
-    Read(size, v, target, 0, event, event->reads);
   event->reads = r;
   ++ event->readCount;
 
@@ -1266,9 +1248,9 @@ insertRead(Context* c, Event* event, int sequence, Value* v,
 }
 
 void
-addRead(Context* c, Value* v, unsigned size, Site* target)
+addRead(Context* c, Value* v, Read* r)
 {
-  insertRead(c, c->logicalCode[c->logicalIp].lastEvent, -1, v, size, target);
+  insertRead(c, c->logicalCode[c->logicalIp].lastEvent, -1, v, r);
 }
 
 Site*
@@ -1282,7 +1264,7 @@ class PushEvent: public Event {
     assert(c, s->pushEvent == 0);
 
     s->pushEvent = this;
-    addRead(c, s->value, s->size * BytesPerWord, pushSite(c, this));
+    addRead(c, s->value, s->geometry->size * BytesPerWord, pushSite(c, this));
   }
 
   virtual void compile(Context* c) {
@@ -1305,6 +1287,52 @@ class PushEvent: public Event {
   bool active;
 };
 
+void
+appendPush(Context* c, Stack* s)
+{
+  if (DebugAppend) {
+    fprintf(stderr, "appendPush\n");
+  }
+
+  new (c->zone->allocate(sizeof(PushEvent))) PushEvent(c, s);
+}
+
+void
+appendPush(Context* c)
+{
+  appendPush(c, c->state->stack);
+}
+
+class PushedEvent: public Event {
+ public:
+  PushedEvent(Context* c, Stack* s):
+    Event(c), s(s)
+  { }
+
+  virtual void compile(Context* c) {
+    if (DebugCompile) {
+      fprintf(stderr, "PushedEvent.compile\n");
+    }
+
+    assert(c, c->stackPadding == 0);
+    assert(c, s->geometry->padding == 0);
+
+    s->pushSite = s->value->sites = pushSite(&c, s);
+  }
+
+  Stack* s;
+};
+
+void
+appendPushed(Context* c, Stack* s)
+{
+  if (DebugAppend) {
+    fprintf(stderr, "appendPushed\n");
+  }
+
+  new (c->zone->allocate(sizeof(PushedEvent))) PushedEvent(c, s);
+}
+
 void
 push(Context* c, unsigned size, Value* v);
 
@@ -1332,7 +1360,7 @@ cleanStack(Context* c, Stack* stack, Local* locals, Read* reads)
 
   for (Stack* s = stack; s; s = s->next) {
     if (s->pushSite) {
-      addSite(c, 0, s->size * BytesPerWord, s->value, s->pushSite);
+      addSite(c, 0, s->geometry->size * BytesPerWord, s->value, s->pushSite);
     }
   }
 
@@ -1367,14 +1395,15 @@ class CallEvent: public Event {
  public:
   CallEvent(Context* c, Value* address, unsigned flags,
             TraceHandler* traceHandler, Value* result, unsigned resultSize,
-            Stack* argumentStack, unsigned argumentCount):
+            Stack* argumentStack, unsigned argumentCount, unsigned padding):
     Event(c),
     address(address),
     traceHandler(traceHandler),
     result(result),
     flags(flags),
     resultSize(resultSize),
-    argumentFootprint(0)
+    argumentFootprint(0),
+    paddding(padding)
   {
     uint32_t mask = ~0;
     Stack* s = argumentStack;
@@ -1388,10 +1417,10 @@ class CallEvent: public Event {
       } else {
         target = 0;
         s->pushEvent->active = true;
-        argumentFootprint += s->size;
+        argumentFootprint += s->geometry->size;
       }
-      addRead(c, s->value, s->size * BytesPerWord, target);
+      addRead(c, s->value, s->geometry->size * BytesPerWord, target);
-      index += s->size;
+      index += s->geometry->size;
       s = s->next;
     }
 
@@ -1400,7 +1429,7 @@ class CallEvent: public Event {
 
     for (Stack* s = stack; s; s = s->next) {
       s->pushEvent->active = true;
-      addRead(c, s->value, s->size * BytesPerWord, virtualSite
+      addRead(c, s->value, s->geometry->size * BytesPerWord, virtualSite
               (c, 0, ~0, (static_cast<uint64_t>(mask) << 32) | mask));
     }
 
@@ -1414,6 +1443,8 @@ class CallEvent: public Event {
 
     pushNow(c, stack);
 
+    pad(c, padding);
+    
     UnaryOperation type = ((flags & Compiler::Aligned) ? AlignedCall : Call);
     apply(c, type, BytesPerWord, address->source);
 
@@ -1423,7 +1454,7 @@ class CallEvent: public Event {
 
     cleanStack(c, stack, locals, reads);
 
-    if (resultSize and result->reads) {
+    if (resultSize and live(result)) {
       addSite(c, 0, resultSize, result, registerSite
               (c, c->assembler->returnLow(),
                resultSize > BytesPerWord ?
@@ -1441,12 +1472,13 @@ class CallEvent: public Event {
   unsigned flags;
   unsigned resultSize;
   unsigned argumentFootprint;
+  unsigned padding;
 };
 
 void
 appendCall(Context* c, Value* address, unsigned flags,
            TraceHandler* traceHandler, Value* result, unsigned resultSize,
-           Stack* argumentStack, unsigned argumentCount)
+           Stack* argumentStack, unsigned argumentCount, unsigned padding)
 {
   if (DebugAppend) {
     fprintf(stderr, "appendCall\n");
@@ -1454,7 +1486,7 @@ appendCall(Context* c, Value* address, unsigned flags,
 
   new (c->zone->allocate(sizeof(CallEvent)))
     CallEvent(c, address, flags, traceHandler, result,
-              resultSize, argumentStack, argumentCount);
+              resultSize, argumentStack, argumentCount, unsigned padding);
 }
 
 class ReturnEvent: public Event {
@@ -1515,25 +1547,14 @@ class MoveEvent: public Event {
       fprintf(stderr, "MoveEvent.compile\n");
     }
 
-    bool isLoad = src->reads->next == 0;
+    bool isLoad = not valid(src->reads->next);
-    bool isStore = dst->reads == 0;
+    bool isStore = not valid(dst->reads);
 
-    Site* target;
+    Site* target = targetOrRegister(c, dst);
-    unsigned cost;
+    unsigned cost = src->source->copyCost(c, target);
-    if (type == Move
-        and dst->reads
-        and next == dst->reads->event
-        and dst->reads->event->skipMove(size))
-    {
-      target = src->source;
-      cost = 0;
-    } else {
-      target = targetOrRegister(c, dst);
-      cost = src->source->copyCost(c, target);
     if (cost == 0 and (isLoad or isStore)) {
       target = src->source;
     }
-    }
 
     assert(c, isLoad or isStore or target != src->source);
 
@@ -1695,7 +1716,7 @@ preserve(Context* c, Stack* stack, unsigned size, Value* v, Site* s,
 void
 maybePreserve(Context* c, Stack* stack, unsigned size, Value* v, Site* s)
 {
-  if (v->reads->next and v->sites->next == 0) {
+  if (valid(v->reads->next) and v->sites->next == 0) {
     preserve(c, stack, size, v, s, v->reads->next);
   }
 }
@@ -1741,7 +1762,7 @@ class CombineEvent: public Event {
     nextRead(c, second);
 
     removeSite(c, second, second->source);
-    if (result->reads) {
+    if (live(result)) {
       addSite(c, 0, size, result, second->source);
     }
   }
@@ -1823,7 +1844,7 @@ class TranslateEvent: public Event {
     nextRead(c, value);
 
     removeSite(c, value, value->source);
-    if (result->reads) {
+    if (live(result)) {
       addSite(c, 0, size, result, value->source);
     }
   }
@@ -1926,11 +1947,17 @@ appendMemory(Context* c, Value* base, int displacement, Value* index,
     MemoryEvent(c, base, displacement, index, scale, result);
 }
 
+Stack*
+stack(Context* c, Value* value, StackGeometry* geometry, Stack* next)
+{
+  return new (c->zone->allocate(sizeof(Stack))) Stack(value, geometry, next);
+}
+
 Stack*
 stack(Context* c, Value* value, unsigned size, unsigned index, Stack* next)
 {
-  return new (c->zone->allocate(sizeof(Stack)))
+  return stack(c, value, new (c->zone->allocate(sizeof(StackGeometry)))
-    Stack(value, size, index, next);
+               StackGeometry(size, index), next);
 }
 
 void
@@ -1939,8 +1966,8 @@ resetStack(Context* c)
   unsigned i = 0;
   Stack* p = 0;
   for (Stack* s = c->state->stack; s; s = s->next) {
-    Stack* n = stack(c, value(c), s->size, s->index, 0);
+    Stack* n = stack(c, value(c), s->geometry, 0);
-    n->value->sites = n->pushSite = pushSite(c, s->index);
+    n->value->sites = n->pushSite = pushSite(c, s);
     n->pushed = true;
 
     if (p) {
@@ -1950,7 +1977,7 @@ resetStack(Context* c)
     }
     p = n;
 
-    i += s->size;
+    i += s->geometry->size;
   }
 
   resetLocals(c);
@@ -1970,7 +1997,7 @@ popNow(Context* c, Stack* stack, unsigned count, bool ignore)
       s->pushed = false;
 
       Value* v = s->value;
-      if (v->reads and v->sites == 0 and (not ignore)) {
+      if (live(v) and v->sites == 0 and (not ignore)) {
         ::ignore(c, ignored);
 
         Site* target = targetOrRegister(c, v);
@@ -1980,15 +2007,15 @@ popNow(Context* c, Stack* stack, unsigned count, bool ignore)
                   s, s->value, target);
         }
 
-        addSite(c, stack, s->size * BytesPerWord, v, target);
+        addSite(c, stack, s->geometry->size * BytesPerWord, v, target);
 
-        apply(c, Pop, BytesPerWord * s->size, target);
+        apply(c, Pop, BytesPerWord * s->geometry->size, target);
       } else {
         if (DebugStack) {
           fprintf(stderr, "ignore %p value: %p\n", s, v);
         }
           
-        ignored += s->size;
+        ignored += s->geometry->size;
       }
     } else {
       if (DebugStack) {
@@ -1996,7 +2023,9 @@ popNow(Context* c, Stack* stack, unsigned count, bool ignore)
       }
     }
 
-    i -= s->size;
+    c->stackPadding -= s->geometry->padding;
+    ignored += s->geometry->padding;
+    i -= s->geometry->size;
     s = s->next;
   }
 
@@ -2010,7 +2039,7 @@ class StackSyncEvent: public Event {
   {
     for (Stack* s = stack; s; s = s->next) {
       if (s->pushEvent) s->pushEvent->active = true;
-      addRead(c, s->value, s->size * BytesPerWord, 0);
+      addRead(c, s->value, s->geometry->size * BytesPerWord, 0);
     } 
   }
 
@@ -2019,7 +2048,8 @@ class StackSyncEvent: public Event {
   {
     for (Stack* s = stack; s; s = s->next) {
       if (s->pushEvent) s->pushEvent->active = true;
-      insertRead(c, this, sequence, s->value, s->size * BytesPerWord, 0);
+      insertRead
+        (c, this, sequence, s->value, s->geometry->size * BytesPerWord, 0);
     }
   }
 
@@ -2157,22 +2187,6 @@ pushSite(Context* c, PushEvent* e)
   return new (c->zone->allocate(sizeof(PushSite))) PushSite(e);
 }
 
-void
-appendPush(Context* c, Stack* s)
-{
-  if (DebugAppend) {
-    fprintf(stderr, "appendPush\n");
-  }
-
-  new (c->zone->allocate(sizeof(PushEvent))) PushEvent(c, s);
-}
-
-void
-appendPush(Context* c)
-{
-  appendPush(c, c->state->stack);
-}
-
 class PopEvent: public Event {
  public:
   PopEvent(Context* c, unsigned count, bool ignore):
@@ -2215,7 +2229,7 @@ class ClobberLocalEvent: public Event {
     for (Local* l = local; l; l = l->old) {
       Value* v = l->value;
       Site* s = l->site;
-      if (v->reads
+      if (live(v)
           and v->sites->next == 0
           and v->sites == s)
       {
@@ -2258,7 +2272,7 @@ class LocalEvent: public Event {
     Site* sites = 0;
     if (local->old) {
       Value* v = local->old->value;
-      if (local->old->reuse and v->reads->next == 0) {
+      if (local->old->reuse and valid(v->reads->next) == 0) {
         sites = v->sites;
       }
 
@@ -2266,7 +2280,7 @@ class LocalEvent: public Event {
     }
 
     Value* v = local->value;
-    if (v->reads) {
+    if (live(v)) {
       for (Site* s = sites; s;) {
         Site* t = s->next;
         if (s->type(c) != MemoryOperand) {
@@ -2397,6 +2411,8 @@ compile(Context* c)
 {
   Assembler* a = c->assembler;
 
+  c->pass = CompilePass;
+
   Assembler::Register base(a->base());
   Assembler::Register stack(a->stack());
   a->apply(Push, BytesPerWord, RegisterOperand, &base);
@@ -2424,7 +2440,8 @@ compile(Context* c)
           for (Stack* s = e->stack; s; s = s->next) {
             if (s->value->sites) {
               assert(c, s->value->sites->next == 0);
-              s->value->sites->acquire(c, 0, s->size * BytesPerWord, s->value);
+              s->value->sites->acquire
+                (c, 0, s->geometry->size * BytesPerWord, s->value);
             }
           }
         }
@@ -2513,7 +2530,9 @@ popState(Context* c)
 Stack*
 stack(Context* c, Value* value, unsigned size, Stack* next)
 {
-  return stack(c, value, size, (next ? next->index + next->size : 0), next);
+  return stack
+    (c, value, size, (next ? next->geometry->index + next->geometry->size : 0),
+     next);
 }
 
 void
@@ -2545,7 +2564,7 @@ pop(Context* c, unsigned size UNUSED)
   Stack* s = c->state->stack;
   assert(c, ceiling(size, BytesPerWord) == s->size);
 
-  appendPop(c, s->size, false);
+  appendPop(c, s->geometry->size, false);
 
   c->state->stack = s->next;
   return s->value;
@@ -2778,33 +2797,38 @@ class MyCompiler: public Compiler {
     return ::pop(&c, size);
   }
 
-  virtual void pushed(unsigned count) {
+  virtual void pushed() {
-    for (unsigned i = 0; i < count; ++i) {
     Value* v = value(&c);
     c.state->stack = ::stack(&c, v, 1, c.state->stack);
     c.state->stack->pushed = true;
-      c.state->stack->pushSite
+    appendPushed(&c, c.state->stack);
-        = v->sites = pushSite(&c, c.state->stack->index);
-    }
   }
 
-  virtual void popped(unsigned count) {
+  virtual void popped() {
-    appendPop(&c, count, true);
+    appendPop(&c, c.state->stack->size, true);
 
-    for (unsigned i = count; i;) {
+    c.state->stack = c.state->stack->next;
-      Stack* s = c.state->stack;
-      c.state->stack = s->next;
-      i -= s->size;
   }
+
+  virtual StackElement* top() {
+    return c.state->stack;
+  }
+
+  virtual unsigned size(StackElement* e) {
+    return static_cast<Stack*>(e)->geometry->size;
+  }
+
+  virtual unsigned padding(StackElement* e) {
+    return static_cast<Stack*>(e)->geometry->padding;
   }
 
   virtual Operand* peek(unsigned size UNUSED, unsigned index) {
     Stack* s = c.state->stack;
     for (unsigned i = index; i > 0;) {
-      i -= s->size;
+      i -= s->geometry->size;
       s = s->next;
     }
-    assert(&c, s->size == ceiling(size, BytesPerWord));
+    assert(&c, s->geometry->size == ceiling(size, BytesPerWord));
     return s->value;
   }
 
@@ -2845,18 +2869,56 @@ class MyCompiler: public Compiler {
     }
 
     Stack* oldStack = c.state->stack;
+    Stack* bottomArgument = 0;
 
     for (int i = index - 1; i >= 0; --i) {
       ::push(&c, argumentSizes[i], arguments[i]);
+      if (i == index - 1) {
+        bottomArgument = c.state->stack;
+      }
     }
-
     Stack* argumentStack = c.state->stack;
     c.state->stack = oldStack;
 
+    unsigned padding = c->assembler->stackPadding
+      (c.state->stack->geometry->index + c.state->stack->geometry->size);
+
+    if (bottomArgument) {
+      bottomArgument->geometry->padding = padding;
+    }
+
     Value* result = value(&c);
-    appendCall(&c, static_cast<Value*>(address), flags,
+    appendCall(&c, static_cast<Value*>(address), flags, traceHandler, result,
-               traceHandler, result, resultSize, argumentStack,
+               resultSize, argumentStack, index, argumentCount ? 0 : padding);
-               index);
+
+    return result;
+  }
+
+  virtual Operand* stackCall(Operand* address,
+                             unsigned flags,
+                             TraceHandler* traceHandler,
+                             unsigned resultSize,
+                             unsigned argumentFootprint)
+  {
+    unsigned padding = c->assembler->stackPadding
+      (c.state->stack->geometry->index + c.state->stack->geometry->size);
+
+    int footprint = argumentFootprint;
+    for (Stack* s = c.state->stack; s; s = s->next) {
+      footprint -= s->geometry->size;
+      if (footprint == 0) {
+        s->geometry->padding = padding;
+      }
+
+      if (s->pushEvent == 0) {
+        appendPush(&c, s);
+      }
+      s->pushEvent->active = true;
+    }
+
+    Value* result = value(&c);
+    appendCall(&c, static_cast<Value*>(address), flags, traceHandler, result,
+               resultSize, c.state->stack, 0, argumentFootprint ? 0 : padding);
 
     return result;
   }

src/compiler.h

@@ -31,6 +31,7 @@ class Compiler {
   static const unsigned NoReturn = 1 << 1;
 
   class Operand { };
+  class StackElement { };
 
   virtual ~Compiler() { }
 
@@ -68,8 +69,11 @@ class Compiler {
   virtual void push(unsigned size) = 0;
   virtual void push(unsigned size, Operand* value) = 0;
   virtual Operand* pop(unsigned size) = 0;
-  virtual void pushed(unsigned count) = 0;
+  virtual void pushed() = 0;
-  virtual void popped(unsigned count) = 0;
+  virtual void popped() = 0;
+  virtual StackElement* top() = 0;
+  virtual unsigned size(StackElement*) = 0;
+  virtual unsigned padding(StackElement*) = 0;
   virtual Operand* peek(unsigned size, unsigned index) = 0;
 
   virtual Operand* call(Operand* address,
@@ -79,6 +83,12 @@ class Compiler {
                         unsigned argumentCount,
                         ...) = 0;
 
+  virtual Operand* stackCall(Operand* address,
+                             unsigned flags,
+                             TraceHandler* traceHandler,
+                             unsigned resultSize,
+                             unsigned argumentFootprint) = 0;
+
   virtual void return_(unsigned size, Operand* value) = 0;
 
   virtual void storeLocal(unsigned size, Operand* src, unsigned index) = 0;

src/machine.cpp

@@ -2221,7 +2221,7 @@ isAssignableFrom(Thread* t, object a, object b)
   return false;
 }
 
-bool FORCE_ALIGN
+bool
 instanceOf(Thread* t, object class_, object o)
 {
   if (o == 0) {

src/machine.h

@@ -1507,7 +1507,7 @@ mark(Thread* t, object o, unsigned offset)
   }
 }
 
-inline void FORCE_ALIGN
+inline void
 set(Thread* t, object target, unsigned offset, object value)
 {
   cast<object>(target, offset) = value;
@@ -1694,7 +1694,7 @@ makeExceptionInInitializerError(Thread* t, object cause)
   return makeExceptionInInitializerError(t, 0, trace, cause);
 }
 
-inline object FORCE_ALIGN
+inline object
 makeNew(Thread* t, object class_)
 {
   assert(t, t->state == Thread::ActiveState);
@@ -1709,7 +1709,7 @@ makeNew(Thread* t, object class_)
   return instance;
 }
 
-inline object FORCE_ALIGN
+inline object
 makeNewWeakReference(Thread* t, object class_)
 {
   assert(t, t->state == Thread::ActiveState);

src/x86.cpp

@@ -2030,6 +2030,15 @@ class MyAssembler: public Assembler {
     }
   }
 
+  virtual unsigned stackPadding(unsigned footprint) {
+#if (defined __APPLE__) || (defined __x86_64__)
+    const unsigned alignment = 16 / BytesPerWord;
+    return (ceiling(footprint, alignment) * alignment) - footprint;
+#else
+    return 0;
+#endif
+  }
+
   virtual void plan(UnaryOperation op, unsigned size, uint8_t* typeMask,
                     uint64_t* registerMask, bool* thunk)
   {