diff --git a/src/assembler.h b/src/assembler.h index ca7926965b..ef3f9a7690 100644 --- a/src/assembler.h +++ b/src/assembler.h @@ -161,6 +161,8 @@ class Assembler { virtual unsigned argumentRegisterCount() = 0; virtual int argumentRegister(unsigned index) = 0; + virtual unsigned stackAlignment() = 0; + virtual void plan(UnaryOperation op, unsigned size, uint8_t* typeMask, uint64_t* registerMask, bool* thunk) = 0; diff --git a/src/common.h b/src/common.h index 2a2e548a07..27302dd92f 100644 --- a/src/common.h +++ b/src/common.h @@ -60,12 +60,6 @@ # define SO_SUFFIX ".so" #endif -#if (defined __APPLE__) && (defined __i386__) -# define FORCE_ALIGN __attribute__((force_align_arg_pointer)) -#else -# define FORCE_ALIGN -#endif - #define NO_RETURN __attribute__((noreturn)) #define LIKELY(v) __builtin_expect((v) != 0, true) diff --git a/src/compile.cpp b/src/compile.cpp index f8970cd4cd..217e105c98 100644 --- a/src/compile.cpp +++ b/src/compile.cpp @@ -386,6 +386,8 @@ class TraceElement: public TraceHandler { }; enum Event { + PushContextEvent, + PopContextEvent, PushEvent, PopEvent, IpEvent, @@ -407,15 +409,16 @@ localSize(MyThread* t, object method) } unsigned -frameSize(MyThread* t, object method) +alignedFrameSize(MyThread* t, object method) { - return localSize(t, method) + codeMaxStack(t, methodCode(t, method)); + return Assembler::alignFrameSize + (localSize(t, method) + codeMaxStack(t, methodCode(t, method))); } unsigned frameMapSizeInWords(MyThread* t, object method) { - return ceiling(frameSize(t, method), BitsPerWord) * BytesPerWord; + return ceiling(alignedFrameSize(t, method), BitsPerWord) * BytesPerWord; } uint16_t* @@ -591,7 +594,7 @@ class Frame { (t, methodCode(t, context->method))); if (level > 1) { - context->eventLog.append(PushEvent); + context->eventLog.append(PushContextEvent); } } @@ -604,7 +607,7 @@ class Frame { } if (level > 1) { - context->eventLog.append(PopEvent); + context->eventLog.append(PopContextEvent); } } } @@ -830,54 +833,82 @@ class Frame { this->ip = ip; } + void pushQuiet(unsigned size, Compiler::Operand* o) { + if (size == 8 and BytesPerWord == 8) { + c->push(8); + + context->eventLog.append(PushEvent); + context->eventLog.appendAddress(c->top()); + } + + c->push(size, o); + + context->eventLog.append(PushEvent); + context->eventLog.appendAddress(c->top()); + } + + Compiler::Operand* popQuiet(unsigned size) { + context->eventLog.append(PopEvent); + context->eventLog.appendAddress(c->top()); + + Compiler::Operand* r = c->pop(8); + + if (size == 8 and BytesPerWord == 8) { + context->eventLog.append(PopEvent); + context->eventLog.appendAddress(c->top()); + + c->pop(8); + } + + return r; + } + void pushInt(Compiler::Operand* o) { - c->push(4, o); + pushQuiet(4, o); pushedInt(); } void pushAddress(Compiler::Operand* o) { - c->push(BytesPerWord, o); + pushQuiet(BytesPerWord, o); pushedInt(); } void pushObject(Compiler::Operand* o) { - c->push(BytesPerWord, o); + pushQuiet(BytesPerWord, o); pushedObject(); } void pushObject() { - c->pushed(1); + c->pushed(); + + context->eventLog.append(PushEvent); + context->eventLog.appendAddress(c->top()); + pushedObject(); } - void pushLongQuiet(Compiler::Operand* o) { - if (BytesPerWord == 8) { - c->push(8); - } - c->push(8, o); - } - void pushLong(Compiler::Operand* o) { - pushLongQuiet(o); + pushQuiet(8, o); pushedLong(); } void pop(unsigned count) { popped(count); - c->popped(count); + + for (unsigned i = count; i;) { + Compiler::StackElement* s = c->top(); + + context->eventLog.append(PopEvent); + context->eventLog.appendAddress(s); + + c->popped(); + i -= c->size(s); + } } Compiler::Operand* popInt() { poppedInt(); - return c->pop(4); - } - - Compiler::Operand* popLongQuiet() { - Compiler::Operand* r = c->pop(8); - if (BytesPerWord == 8) { - c->pop(8); - } - return r; + return popQuiet(4); } Compiler::Operand* peekLong(unsigned index) { @@ -886,12 +917,12 @@ class Frame { Compiler::Operand* popLong() { poppedLong(); - return popLongQuiet(); + return popQuiet(8); } Compiler::Operand* popObject() { poppedObject(); - return c->pop(BytesPerWord); + return popQuiet(BytesPerWord); } void loadInt(unsigned index) { @@ -925,7 +956,7 @@ class Frame { } void storeObjectOrAddress(unsigned index) { - c->storeLocal(BytesPerWord, c->pop(BytesPerWord), index); + c->storeLocal(BytesPerWord, popQuiet(BytesPerWord), index); assert(t, sp >= 1); assert(t, sp - 1 >= localSize()); @@ -939,39 +970,39 @@ class Frame { } void dup() { - c->push(BytesPerWord, c->peek(BytesPerWord, 0)); + pushQuiet(BytesPerWord, c->peek(BytesPerWord, 0)); dupped(); } void dupX1() { - Compiler::Operand* s0 = c->pop(BytesPerWord); - Compiler::Operand* s1 = c->pop(BytesPerWord); + Compiler::Operand* s0 = popQuiet(BytesPerWord); + Compiler::Operand* s1 = popQuiet(BytesPerWord); - c->push(BytesPerWord, s0); - c->push(BytesPerWord, s1); - c->push(BytesPerWord, s0); + pushQuiet(BytesPerWord, s0); + pushQuiet(BytesPerWord, s1); + pushQuiet(BytesPerWord, s0); duppedX1(); } void dupX2() { - Compiler::Operand* s0 = c->pop(BytesPerWord); + Compiler::Operand* s0 = popQuiet(BytesPerWord); if (get(sp - 2) == Long) { - Compiler::Operand* s1 = popLongQuiet(); + Compiler::Operand* s1 = popQuiet(8); - c->push(BytesPerWord, s0); - pushLongQuiet(s1); - c->push(BytesPerWord, s0); + pushQuiet(BytesPerWord, s0); + pushQuiet(8, s1); + pushQuiet(BytesPerWord, s0); } else { - Compiler::Operand* s1 = c->pop(BytesPerWord); - Compiler::Operand* s2 = c->pop(BytesPerWord); + Compiler::Operand* s1 = popQuiet(BytesPerWord); + Compiler::Operand* s2 = popQuiet(BytesPerWord); - c->push(BytesPerWord, s0); - c->push(BytesPerWord, s2); - c->push(BytesPerWord, s1); - c->push(BytesPerWord, s0); + pushQuiet(BytesPerWord, s0); + pushQuiet(BytesPerWord, s2); + pushQuiet(BytesPerWord, s1); + pushQuiet(BytesPerWord, s0); } duppedX2(); @@ -979,15 +1010,15 @@ class Frame { void dup2() { if (get(sp - 1) == Long) { - pushLongQuiet(peekLong(0)); + pushQuiet(8, peekLong(0)); } else { - Compiler::Operand* s0 = c->pop(BytesPerWord); - Compiler::Operand* s1 = c->pop(BytesPerWord); + Compiler::Operand* s0 = popQuiet(BytesPerWord); + Compiler::Operand* s1 = popQuiet(BytesPerWord); - c->push(BytesPerWord, s1); - c->push(BytesPerWord, s0); - c->push(BytesPerWord, s1); - c->push(BytesPerWord, s0); + pushQuiet(BytesPerWord, s1); + pushQuiet(BytesPerWord, s0); + pushQuiet(BytesPerWord, s1); + pushQuiet(BytesPerWord, s0); } dupped2(); @@ -995,22 +1026,22 @@ class Frame { void dup2X1() { if (get(sp - 1) == Long) { - Compiler::Operand* s0 = popLongQuiet(); - Compiler::Operand* s1 = c->pop(BytesPerWord); + Compiler::Operand* s0 = popQuiet(8); + Compiler::Operand* s1 = popQuiet(BytesPerWord); - pushLongQuiet(s0); - c->push(BytesPerWord, s1); - pushLongQuiet(s0); + pushQuiet(8, s0); + pushQuiet(BytesPerWord, s1); + pushQuiet(8, s0); } else { - Compiler::Operand* s0 = c->pop(BytesPerWord); - Compiler::Operand* s1 = c->pop(BytesPerWord); - Compiler::Operand* s2 = c->pop(BytesPerWord); + Compiler::Operand* s0 = popQuiet(BytesPerWord); + Compiler::Operand* s1 = popQuiet(BytesPerWord); + Compiler::Operand* s2 = popQuiet(BytesPerWord); - c->push(BytesPerWord, s1); - c->push(BytesPerWord, s0); - c->push(BytesPerWord, s2); - c->push(BytesPerWord, s1); - c->push(BytesPerWord, s0); + pushQuiet(BytesPerWord, s1); + pushQuiet(BytesPerWord, s0); + pushQuiet(BytesPerWord, s2); + pushQuiet(BytesPerWord, s1); + pushQuiet(BytesPerWord, s0); } dupped2X1(); @@ -1018,46 +1049,46 @@ class Frame { void dup2X2() { if (get(sp - 1) == Long) { - Compiler::Operand* s0 = popLongQuiet(); + Compiler::Operand* s0 = popQuiet(8); if (get(sp - 3) == Long) { - Compiler::Operand* s1 = popLongQuiet(); + Compiler::Operand* s1 = popQuiet(8); - pushLongQuiet(s0); - pushLongQuiet(s1); - pushLongQuiet(s0); + pushQuiet(8, s0); + pushQuiet(8, s1); + pushQuiet(8, s0); } else { - Compiler::Operand* s1 = c->pop(BytesPerWord); - Compiler::Operand* s2 = c->pop(BytesPerWord); + Compiler::Operand* s1 = popQuiet(BytesPerWord); + Compiler::Operand* s2 = popQuiet(BytesPerWord); - pushLongQuiet(s0); - c->push(BytesPerWord, s2); - c->push(BytesPerWord, s1); - pushLongQuiet(s0); + pushQuiet(8, s0); + pushQuiet(BytesPerWord, s2); + pushQuiet(BytesPerWord, s1); + pushQuiet(8, s0); } } else { - Compiler::Operand* s0 = c->pop(BytesPerWord); - Compiler::Operand* s1 = c->pop(BytesPerWord); - Compiler::Operand* s2 = c->pop(BytesPerWord); - Compiler::Operand* s3 = c->pop(BytesPerWord); + Compiler::Operand* s0 = popQuiet(BytesPerWord); + Compiler::Operand* s1 = popQuiet(BytesPerWord); + Compiler::Operand* s2 = popQuiet(BytesPerWord); + Compiler::Operand* s3 = popQuiet(BytesPerWord); - c->push(BytesPerWord, s1); - c->push(BytesPerWord, s0); - c->push(BytesPerWord, s3); - c->push(BytesPerWord, s2); - c->push(BytesPerWord, s1); - c->push(BytesPerWord, s0); + pushQuiet(BytesPerWord, s1); + pushQuiet(BytesPerWord, s0); + pushQuiet(BytesPerWord, s3); + pushQuiet(BytesPerWord, s2); + pushQuiet(BytesPerWord, s1); + pushQuiet(BytesPerWord, s0); } dupped2X2(); } void swap() { - Compiler::Operand* s0 = c->pop(BytesPerWord); - Compiler::Operand* s1 = c->pop(BytesPerWord); + Compiler::Operand* s0 = popQuiet(BytesPerWord); + Compiler::Operand* s1 = popQuiet(BytesPerWord); - c->push(BytesPerWord, s0); - c->push(BytesPerWord, s1); + pushQuiet(BytesPerWord, s0); + pushQuiet(BytesPerWord, s1); swapped(); } @@ -1192,14 +1223,14 @@ unwind(MyThread* t) vmJump(ip, base, stack, t); } -void FORCE_ALIGN +void tryInitClass(MyThread* t, object class_) { initClass(t, class_); if (UNLIKELY(t->exception)) unwind(t); } -void* FORCE_ALIGN +void* findInterfaceMethodFromInstance(MyThread* t, object method, object instance) { if (instance) { @@ -1212,7 +1243,7 @@ findInterfaceMethodFromInstance(MyThread* t, object method, object instance) } } -intptr_t FORCE_ALIGN +intptr_t compareDoublesG(uint64_t bi, uint64_t ai) { double a = bitsToDouble(ai); @@ -1229,7 +1260,7 @@ compareDoublesG(uint64_t bi, uint64_t ai) } } -intptr_t FORCE_ALIGN +intptr_t compareDoublesL(uint64_t bi, uint64_t ai) { double a = bitsToDouble(ai); @@ -1246,7 +1277,7 @@ compareDoublesL(uint64_t bi, uint64_t ai) } } -intptr_t FORCE_ALIGN +intptr_t compareFloatsG(uint32_t bi, uint32_t ai) { float a = bitsToFloat(ai); @@ -1263,7 +1294,7 @@ compareFloatsG(uint32_t bi, uint32_t ai) } } -intptr_t FORCE_ALIGN +intptr_t compareFloatsL(uint32_t bi, uint32_t ai) { float a = bitsToFloat(ai); @@ -1280,151 +1311,151 @@ compareFloatsL(uint32_t bi, uint32_t ai) } } -uint64_t FORCE_ALIGN +uint64_t addDouble(uint64_t b, uint64_t a) { return doubleToBits(bitsToDouble(a) + bitsToDouble(b)); } -uint64_t FORCE_ALIGN +uint64_t subtractDouble(uint64_t b, uint64_t a) { return doubleToBits(bitsToDouble(a) - bitsToDouble(b)); } -uint64_t FORCE_ALIGN +uint64_t multiplyDouble(uint64_t b, uint64_t a) { return doubleToBits(bitsToDouble(a) * bitsToDouble(b)); } -uint64_t FORCE_ALIGN +uint64_t divideDouble(uint64_t b, uint64_t a) { return doubleToBits(bitsToDouble(a) / bitsToDouble(b)); } -uint64_t FORCE_ALIGN +uint64_t moduloDouble(uint64_t b, uint64_t a) { return doubleToBits(fmod(bitsToDouble(a), bitsToDouble(b))); } -uint64_t FORCE_ALIGN +uint64_t negateDouble(uint64_t a) { return doubleToBits(- bitsToDouble(a)); } -uint32_t FORCE_ALIGN +uint32_t doubleToFloat(int64_t a) { return floatToBits(static_cast(bitsToDouble(a))); } -int32_t FORCE_ALIGN +int32_t doubleToInt(int64_t a) { return static_cast(bitsToDouble(a)); } -int64_t FORCE_ALIGN +int64_t doubleToLong(int64_t a) { return static_cast(bitsToDouble(a)); } -uint32_t FORCE_ALIGN +uint32_t addFloat(uint32_t b, uint32_t a) { return floatToBits(bitsToFloat(a) + bitsToFloat(b)); } -uint32_t FORCE_ALIGN +uint32_t subtractFloat(uint32_t b, uint32_t a) { return floatToBits(bitsToFloat(a) - bitsToFloat(b)); } -uint32_t FORCE_ALIGN +uint32_t multiplyFloat(uint32_t b, uint32_t a) { return floatToBits(bitsToFloat(a) * bitsToFloat(b)); } -uint32_t FORCE_ALIGN +uint32_t divideFloat(uint32_t b, uint32_t a) { return floatToBits(bitsToFloat(a) / bitsToFloat(b)); } -uint32_t FORCE_ALIGN +uint32_t moduloFloat(uint32_t b, uint32_t a) { return floatToBits(fmod(bitsToFloat(a), bitsToFloat(b))); } -uint32_t FORCE_ALIGN +uint32_t negateFloat(uint32_t a) { return floatToBits(- bitsToFloat(a)); } -int64_t FORCE_ALIGN +int64_t divideLong(int64_t b, int64_t a) { return a / b; } -int64_t FORCE_ALIGN +int64_t moduloLong(int64_t b, int64_t a) { return a % b; } -uint64_t FORCE_ALIGN +uint64_t floatToDouble(int32_t a) { return doubleToBits(static_cast(bitsToFloat(a))); } -int32_t FORCE_ALIGN +int32_t floatToInt(int32_t a) { return static_cast(bitsToFloat(a)); } -int64_t FORCE_ALIGN +int64_t floatToLong(int32_t a) { return static_cast(bitsToFloat(a)); } -uint64_t FORCE_ALIGN +uint64_t intToDouble(int32_t a) { return doubleToBits(static_cast(a)); } -uint32_t FORCE_ALIGN +uint32_t intToFloat(int32_t a) { return floatToBits(static_cast(a)); } -uint64_t FORCE_ALIGN +uint64_t longToDouble(int64_t a) { return doubleToBits(static_cast(a)); } -uint32_t FORCE_ALIGN +uint32_t longToFloat(int64_t a) { return floatToBits(static_cast(a)); } -object FORCE_ALIGN +object makeBlankObjectArray(MyThread* t, object class_, int32_t length) { if (length >= 0) { @@ -1436,7 +1467,7 @@ makeBlankObjectArray(MyThread* t, object class_, int32_t length) } } -object FORCE_ALIGN +object makeBlankArray(MyThread* t, object (*constructor)(Thread*, uintptr_t, bool), int32_t length) { @@ -1449,7 +1480,7 @@ makeBlankArray(MyThread* t, object (*constructor)(Thread*, uintptr_t, bool), } } -uintptr_t FORCE_ALIGN +uintptr_t lookUpAddress(int32_t key, uintptr_t* start, int32_t count, uintptr_t default_) { @@ -1472,7 +1503,7 @@ lookUpAddress(int32_t key, uintptr_t* start, int32_t count, return default_; } -void FORCE_ALIGN +void setMaybeNull(MyThread* t, object o, unsigned offset, object value) { if (LIKELY(o)) { @@ -1483,7 +1514,7 @@ setMaybeNull(MyThread* t, object o, unsigned offset, object value) } } -void FORCE_ALIGN +void acquireMonitorForObject(MyThread* t, object o) { if (LIKELY(o)) { @@ -1494,7 +1525,7 @@ acquireMonitorForObject(MyThread* t, object o) } } -void FORCE_ALIGN +void releaseMonitorForObject(MyThread* t, object o) { if (LIKELY(o)) { @@ -1530,7 +1561,7 @@ makeMultidimensionalArray2(MyThread* t, object class_, uintptr_t* stack, return array; } -object FORCE_ALIGN +object makeMultidimensionalArray(MyThread* t, object class_, int32_t dimensions, uintptr_t* stack) { @@ -1563,7 +1594,7 @@ traceSize(Thread* t) + (counter.count * FixedSizeOfTraceElement); } -void NO_RETURN FORCE_ALIGN +void NO_RETURN throwArrayIndexOutOfBounds(MyThread* t) { ensure(t, FixedSizeOfArrayIndexOutOfBoundsException + traceSize(t)); @@ -1575,7 +1606,7 @@ throwArrayIndexOutOfBounds(MyThread* t) unwind(t); } -void NO_RETURN FORCE_ALIGN +void NO_RETURN throw_(MyThread* t, object o) { if (LIKELY(o)) { @@ -1586,7 +1617,7 @@ throw_(MyThread* t, object o) unwind(t); } -void FORCE_ALIGN +void checkCast(MyThread* t, object class_, object o) { if (UNLIKELY(o and not isAssignableFrom(t, class_, objectClass(t, o)))) { @@ -1599,7 +1630,7 @@ checkCast(MyThread* t, object class_, object o) } } -void FORCE_ALIGN +void gcIfNecessary(MyThread* t) { if (UNLIKELY(t->backupHeap)) { @@ -1687,32 +1718,32 @@ compileDirectInvoke(MyThread* t, Frame* frame, object target) if (not emptyMethod(t, target)) { if (methodFlags(t, target) & ACC_NATIVE) { - result = c->call + result = c->stackCall (c->constant (reinterpret_cast (&singletonBody(t, nativeThunk(t), 0))), 0, frame->trace(target, false), rSize, - 0); + methodParameterFootprint(t, target)); } else if (methodCompiled(t, target) == defaultThunk(t)) { - result = c->call + result = c->stackCall (c->constant (reinterpret_cast (&singletonBody(t, defaultThunk(t), 0))), Compiler::Aligned, frame->trace(target, false), rSize, - 0); + methodParameterFootprint(t, target)); } else { - result = c->call + result = c->stackCall (c->constant (reinterpret_cast (&singletonBody(t, methodCompiled(t, target), 0))), 0, frame->trace(0, false), rSize, - 0); + methodParameterFootprint(t, target)); } } @@ -2629,7 +2660,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, unsigned rSize = resultSize(t, methodReturnCode(t, target)); - Compiler::Operand* result = c->call + Compiler::Operand* result = c->stackCall (c->call (c->constant (getThunk(t, findInterfaceMethodFromInstanceThunk)), @@ -2641,7 +2672,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, 0, frame->trace(target, true), rSize, - 0); + parameterFootprint); frame->pop(parameterFootprint); @@ -2688,7 +2719,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, unsigned rSize = resultSize(t, methodReturnCode(t, target)); - Compiler::Operand* result = c->call + Compiler::Operand* result = c->stackCall (c->memory (c->and_ (BytesPerWord, c->constant(PointerMask), @@ -2696,7 +2727,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, 0, frame->trace(target, true), rSize, - 0); + parameterFootprint); frame->pop(parameterFootprint); @@ -3486,7 +3517,7 @@ printSet(uintptr_t m) unsigned calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots, - unsigned eventIndex) + unsigned stackPadding, unsigned eventIndex) { // for each instruction with more than one predecessor, and for each // stack position, determine if there exists a path to that @@ -3494,6 +3525,9 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots, // stack position (i.e. it is uninitialized or contains primitive // data). + Compiler* c = context->compiler; + + unsigned localSize = ::localSize(t, context->method); unsigned mapSize = frameMapSizeInWords(t, context->method); uintptr_t roots[mapSize]; @@ -3517,13 +3551,30 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots, while (eventIndex < length) { Event e = static_cast(context->eventLog.get(eventIndex++)); switch (e) { - case PushEvent: { - eventIndex = calculateFrameMaps(t, context, roots, eventIndex); + case PushContextEvent: { + eventIndex = calculateFrameMaps + (t, context, roots, stackPadding, eventIndex); } break; - case PopEvent: + case PopContextEvent: return eventIndex; + case PushEvent: { + Compiler::StackElement* s; + context->eventLog.get(eventIndex, &s, BytesPerWord); + stackPadding += c->padding(s); + + eventIndex += BytesPerWord; + } break; + + case PopEvent: { + Compiler::StackElement* s; + context->eventLog.get(eventIndex, &s, BytesPerWord); + stackPadding -= c->padding(s); + + eventIndex += BytesPerWord; + } break; + case IpEvent: { ip = context->eventLog.get2(eventIndex); eventIndex += 2; @@ -3541,7 +3592,7 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots, uintptr_t newRoots = tableRoots[wi] & roots[wi]; if ((eventIndex == length - or context->eventLog.get(eventIndex) == PopEvent) + or context->eventLog.get(eventIndex) == PopContextEvent) and newRoots != tableRoots[wi]) { if (DebugFrameMaps) { @@ -3569,6 +3620,10 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots, unsigned i = context->eventLog.get2(eventIndex); eventIndex += 2; + if (i > localSize) { + i += stackPadding; + } + markBit(roots, i); } break; @@ -3576,6 +3631,10 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots, unsigned i = context->eventLog.get2(eventIndex); eventIndex += 2; + if (i > localSize) { + i += stackPadding; + } + clearBit(roots, i); } break; @@ -3628,7 +3687,7 @@ compareMethodBounds(Thread* t, object a, object b) unsigned frameObjectMapSize(MyThread* t, object method, object map) { - int size = frameSize(t, method); + int size = alignedFrameSize(t, method); return ceiling(intArrayLength(t, map) * size, 32 + size); } @@ -3719,7 +3778,7 @@ finish(MyThread* t, Context* context) qsort(elements, context->traceLogCount, sizeof(TraceElement*), compareTraceElementPointers); - unsigned size = frameSize(t, context->method); + unsigned size = alignedFrameSize(t, context->method); object map = makeIntArray (t, context->traceLogCount + ceiling(context->traceLogCount * size, 32), @@ -3835,7 +3894,7 @@ compile(MyThread* t, Context* context) if (UNLIKELY(t->exception)) return 0; context->dirtyRoots = false; - unsigned eventIndex = calculateFrameMaps(t, context, 0, 0); + unsigned eventIndex = calculateFrameMaps(t, context, 0, 0, 0); object eht = codeExceptionHandlerTable(t, methodCode(t, context->method)); if (eht) { @@ -3881,7 +3940,7 @@ compile(MyThread* t, Context* context) compile(t, &frame2, exceptionHandlerIp(eh), true); if (UNLIKELY(t->exception)) return 0; - eventIndex = calculateFrameMaps(t, context, 0, eventIndex); + eventIndex = calculateFrameMaps(t, context, 0, 0, eventIndex); } } @@ -3891,7 +3950,7 @@ compile(MyThread* t, Context* context) while (context->dirtyRoots) { context->dirtyRoots = false; - calculateFrameMaps(t, context, 0, 0); + calculateFrameMaps(t, context, 0, 0, 0); } return finish(t, context); @@ -3935,7 +3994,7 @@ compileMethod2(MyThread* t) } } -void* FORCE_ALIGN +void* compileMethod(MyThread* t) { void* r = compileMethod2(t); @@ -4129,7 +4188,7 @@ invokeNative2(MyThread* t, object method) return result; } -uint64_t FORCE_ALIGN +uint64_t invokeNative(MyThread* t) { if (t->trace->nativeMethod == 0) { @@ -4170,7 +4229,7 @@ frameMapIndex(MyThread* t, object method, int32_t offset) int32_t v = intArrayBody(t, map, middle); if (offset == v) { - return (indexSize * 32) + (frameSize(t, method) * middle); + return (indexSize * 32) + (alignedFrameSize(t, method) * middle); } else if (offset < v) { top = middle; } else { @@ -4193,7 +4252,7 @@ visitStackAndLocals(MyThread* t, Heap::Visitor* v, void* base, object method, count = parameterFootprint + height - argumentFootprint; } else { - count = frameSize(t, method); + count = alignedFrameSize(t, method); } if (count) { diff --git a/src/compiler.cpp b/src/compiler.cpp index bffbf37b0c..53c7682ba3 100644 --- a/src/compiler.cpp +++ b/src/compiler.cpp @@ -69,57 +69,37 @@ class Site { Site* next; }; -class Stack { +class Stack: public StackElement { public: - Stack(Value* value, unsigned size, unsigned index, Stack* next): - value(value), size(size), index(index), next(next), pushEvent(0), - pushSite(0), pushed(false) + Stack(unsigned index, Value* value, Stack* next): + index(index), value(value), next(next) { } - Value* value; - unsigned size; unsigned index; + Value* value; Stack* next; - PushEvent* pushEvent; - Site* pushSite; - bool pushed; }; class State { public: - State(State* next, Stack* stack): + State(State* next, Stack* stack, Value** locals): stack(stack), + locals(locals), next(next) { } Stack* stack; + Value** locals; State* next; }; -class Local { - public: - Local(unsigned size, unsigned index, Value* value, Site* site, Local* old, - Local* next): - size(size), index(index), reuse(true), value(value), site(site), old(old), - next(next) - { } - - unsigned size; - unsigned index; - bool reuse; - Value* value; - Site* site; - Local* old; - Local* next; -}; - class LogicalInstruction { public: Event* firstEvent; Event* lastEvent; LogicalInstruction* immediatePredecessor; Stack* stack; - Local* locals; + Value** locals; unsigned machineOffset; bool stackSaved; }; @@ -162,18 +142,22 @@ class Junction { class Read { public: - Read(unsigned size, Value* value, Site* target, Read* next, Event* event, - Read* eventNext): - size(size), value(value), target(target), next(next), event(event), - eventNext(eventNext) + Read(): + next(0), event(0), eventNext(0) { } - unsigned size; - Value* value; - Site* target; Read* next; Event* event; Read* eventNext; + + virtual Site* pickSite(Context* c, Value* v) = 0; + + virtual Site* allocateSite(Context* c) = 0; + + virtual void intersect(uint8_t* typeMask, uint64_t* registerMask, + int* frameIndex) = 0; + + virtual bool valid() = 0; }; class Value: public Compiler::Operand { @@ -189,6 +173,11 @@ class Value: public Compiler::Operand { Site* target; }; +enum Pass { + ScanPass, + CompilePass +}; + class Context { public: Context(System* system, Assembler* assembler, Zone* zone, @@ -212,10 +201,10 @@ class Context { nextSequence(0), junctions(0), machineCode(0), - locals(0), - localTable(0), stackReset(false), - constantCompare(CompareNone) + constantCompare(CompareNone), + pass(ScanPass), + stackPadding(0) { for (unsigned i = 0; i < assembler->registerCount(); ++i) { registers[i] = new (zone->allocate(sizeof(Register))) Register(i); @@ -243,10 +232,10 @@ class Context { unsigned nextSequence; Junction* junctions; uint8_t* machineCode; - Local* locals; - Local** localTable; bool stackReset; ConstantCompare constantCompare; + Pass pass; + unsigned stackPadding; }; class PoolPromise: public Promise { @@ -368,11 +357,9 @@ class Event { virtual void compile(Context* c) = 0; - virtual bool skipMove(unsigned) { return false; } - Event* next; Stack* stack; - Local* locals; + Value** locals; CodePromise* promises; Read* reads; unsigned readCount; @@ -452,13 +439,25 @@ clearSites(Context* c, Value* v) v->sites = 0; } +bool +valid(Read* r) +{ + return r and r->valid(); +} + +bool +live(Value* v) +{ + return valid(v->reads); +} + void nextRead(Context* c, Value* v) { // fprintf(stderr, "pop read %p from %p; next: %p\n", v->reads, v, v->reads->next); v->reads = v->reads->next; - if (v->reads == 0) { + if (not live(v)) { clearSites(c, v); } } @@ -709,7 +708,7 @@ decrement(Context* c UNUSED, Register* r) class MemorySite: public Site { public: MemorySite(int base, int offset, int index, unsigned scale): - base(0), index(0), value(base, offset, index, scale) + base(0), index(0), offsetPromise(0), value(base, offset, index, scale) { } void sync(Context* c UNUSED) { @@ -772,40 +771,15 @@ memorySite(Context* c, int base, int offset = 0, int index = NoRegister, MemorySite(base, offset, index, scale); } -bool -matchRegister(Context* c UNUSED, Site* s, uint64_t mask) -{ - assert(c, s->type(c) == RegisterOperand); - - RegisterSite* r = static_cast(s); - if (r->low) { - r->sync(c); - return ((static_cast(1) << r->register_.low) & mask) - and (r->register_.high == NoRegister - or ((static_cast(1) << (r->register_.high + 32)) & mask)); - } else { - return false; - } -} - -bool -match(Context* c, Site* s, uint8_t typeMask, uint64_t registerMask) -{ - OperandType t = s->type(c); - return ((1 << t) & typeMask) - and (t != RegisterOperand or matchRegister(c, s, registerMask)); -} - Site* -targetOrNull(Context* c, Read* r) +targetOrNull(Context* c, Value* v, Read* r) { - Value* v = r->value; if (v->target) { return v->target; - } else if (r->target) { - return r->target->readTarget(c, r); } else { - return 0; + Site* s = r->pickSite(c, v); + if (s) return s; + return r->allocateSite(c); } } @@ -814,77 +788,152 @@ targetOrNull(Context* c, Value* v) { if (v->target) { return v->target; - } else if (v->reads and v->reads->target) { - return v->reads->target->readTarget(c, v->reads); + } else if (live(v)) { + Site* s = r->pickSite(c, v); + if (s) return s; + return r->allocateSite(c); } return 0; } -class AbstractSite: public Site { +Site* +pickSite(Context* c, Value* value, uint8_t typeMask, uint64_t registerMask, + int frameIndex) +{ + Site* site = 0; + unsigned copyCost = 0xFFFFFFFF; + for (Site* s = value->sites; s; s = s->next) { + if (s->match(c, typeMask, registerMask, frameIndex)) { + unsigned v = s->copyCost(c, 0); + if (v < copyCost) { + site = s; + copyCost = v; + } + } + } + return site; +} + +Site* +allocateSite(Context* c, uint8_t typeMask, uint64_t registerMask, + int frameIndex) +{ + if ((typeMask & (1 << RegisterOperand)) and registerMask) { + return freeRegisterSite(c, registerMask); + } else if (frameIndex >= 0) { + return frameSite(c, frameIndex); + } else { + abort(c); + } +} + +class TargetRead: public Read { public: - virtual unsigned copyCost(Context* c, Site*) { - abort(c); - } - - virtual void copyTo(Context* c, unsigned, Site*) { - abort(c); - } - - virtual OperandType type(Context* c) { - abort(c); - } - - virtual Assembler::Operand* asAssemblerOperand(Context* c) { - abort(c); - } -}; - -class VirtualSite: public AbstractSite { - public: - VirtualSite(Value* value, uint8_t typeMask, uint64_t registerMask): - value(value), registerMask(registerMask), typeMask(typeMask) + TargetRead(Value* target, uint8_t typeMask, uint64_t registerMask, + int frameIndex): + target(target), registerMask(registerMask), typeMask(typeMask), + frameIndex(frameIndex) { } - virtual Site* readTarget(Context* c, Read* r) { - if (value) { - Site* s = targetOrNull(c, value); - if (s and match(c, s, typeMask, registerMask)) { + virtual Site* pickSite(Context* c, Value* value) { + return ::pickSite(c, value, typeMask, registerMask, frameIndex); + } + + virtual Site* allocateSite(Context* c) { + if (target) { + Site* s = targetOrNull(c, target); + if (s and s->match(c, typeMask, registerMask, frameIndex)) { return s; } } + return ::allocateSite(c, typeMask, registerMask, frameIndex); + } - Site* site = 0; - unsigned copyCost = 0xFFFFFFFF; - for (Site* s = r->value->sites; s; s = s->next) { - if (match(c, s, typeMask, registerMask)) { - unsigned v = s->copyCost(c, 0); - if (v < copyCost) { - site = s; - copyCost = v; - } - } - } + virtual void intersect(uint8_t* typeMask, uint64_t* registerMask, + int* frameIndex) + { + *typeMask &= this->typeMask; + *registerMask &= this->registerMask; - if (site) { - return site; - } else { - assert(c, typeMask & (1 << RegisterOperand)); - return freeRegisterSite(c, registerMask); + if (*frameIndex == AnyFrameIndex) { + *frameIndex = this->frameIndex; + } else if (this->frameIndex != AnyFrameIndex + and *frameIndex != this->frameIndex) + { + *frameIndex = NoFrameIndex; } } - Value* value; + virtual bool valid() { + return true; + } + + Value* target; uint64_t registerMask; uint8_t typeMask; + int frameIndex; }; -VirtualSite* -virtualSite(Context* c, Value* v = 0, - uint8_t typeMask = ~static_cast(0), - uint64_t registerMask = ~static_cast(0)) +TargetRead* +targetRead(Context* c, Value* target = 0, + uint8_t typeMask = ~static_cast(0), + uint64_t registerMask = ~static_cast(0), + int frameIndex = NoFrameIndex) { - return new (c->zone->allocate(sizeof(VirtualSite))) - VirtualSite(v, typeMask, registerMask); + return new (c->zone->allocate(sizeof(TargetRead))) + TargetRead(target, typeMask, registerMask); +} + +class MultiRead: public Read { + public: + MultiRead(): + reads(0) + { } + + virtual Site* pickSite(Context* c) { + uint8_t typeMask = ~static_cast(0); + uint64_t registerMask = ~static_cast(0); + int frameIndex = AnyFrameIndex; + intersect(&typeMask, ®isterMask, &frameIndex); + + return ::pickSite(c, value, typeMask, registerMask, frameIndex); + } + + virtual Site* allocateSite(Context* c) { + uint8_t typeMask = ~static_cast(0); + uint64_t registerMask = ~static_cast(0); + int frameIndex = AnyFrameIndex; + intersect(&typeMask, ®isterMask, &frameIndex); + + return ::allocateSite(c, typeMask, registerMask, frameIndex); + } + + virtual void intersect(uint8_t* typeMask, uint64_t* registerMask, + int* frameIndex) + { + for (Cell* cell = reads; cell; cell = cell->next) { + Read* r = static_cast(cell->value); + r->intersect(&typeMask, ®isterMask, &frameIndex); + } + } + + virtual bool valid() { + for (Cell* cell = reads; cell; cell = cell->next) { + Read* r = static_cast(cell->value); + if (r->valid()) { + return true; + } + } + return false; + } + + Cell* reads; +}; + +MultiRead* +multiRead(Context* c) +{ + return new (c->zone->allocate(sizeof(MultiRead))) MultiRead; } VirtualSite* @@ -928,72 +977,8 @@ pick(Context* c, Site* sites, Site* target = 0, unsigned* cost = 0) return site; } -unsigned -stackOffset(Context* c) -{ - return c->localFootprint - c->parameterFootprint; -} - -Site* -pushSite(Context* c, unsigned index) -{ - return memorySite - (c, c->assembler->base(), - - (stackOffset(c) + index + 1) * BytesPerWord, NoRegister, 1); -} - -void -pushNow(Context* c, Stack* start, unsigned count) -{ - Stack* segment[count]; - unsigned index = count; - for (Stack* s = start; s and index; s = s->next) { - segment[--index] = s; - } - - for (unsigned i = 0; i < count; ++i) { - Stack* s = segment[i]; - assert(c, not s->pushed); - - if (s->value and s->value->sites) { - Site* source = pick(c, s->value->sites); - - removeMemorySites(c, s->value); - - s->pushSite = pushSite(c, s->index); - addSite(c, 0, s->size * BytesPerWord, s->value, s->pushSite); - - apply(c, Push, s->size * BytesPerWord, source); - } else { - Assembler::Register stack(c->assembler->stack()); - Assembler::Constant offset(resolved(c, s->size * BytesPerWord)); - c->assembler->apply - (Subtract, BytesPerWord, ConstantOperand, &offset, - RegisterOperand, &stack); - } - - if (DebugStack) { - fprintf(stderr, "pushed %p value: %p sites: %p\n", - s, s->value, s->value->sites); - } - - s->pushed = true; - } -} - -void -pushNow(Context* c, Stack* start) -{ - unsigned count = 0; - for (Stack* s = start; s and (not s->pushed); s = s->next) { - ++ count; - } - - pushNow(c, start, count); -} - bool -trySteal(Context* c, Register* r, Stack* stack) +trySteal(Context* c, Register* r, Stack* stack, Value** locals) { assert(c, r->refCount == 0); @@ -1233,11 +1218,8 @@ apply(Context* c, BinaryOperation op, unsigned size, Site* a, Site* b) } void -insertRead(Context* c, Event* event, int sequence, Value* v, - unsigned size, Site* target) +insertRead(Context* c, Event* event, int sequence, Value* v, Read* r) { - Read* r = new (c->zone->allocate(sizeof(Read))) - Read(size, v, target, 0, event, event->reads); event->reads = r; ++ event->readCount; @@ -1266,9 +1248,9 @@ insertRead(Context* c, Event* event, int sequence, Value* v, } void -addRead(Context* c, Value* v, unsigned size, Site* target) +addRead(Context* c, Value* v, Read* r) { - insertRead(c, c->logicalCode[c->logicalIp].lastEvent, -1, v, size, target); + insertRead(c, c->logicalCode[c->logicalIp].lastEvent, -1, v, r); } Site* @@ -1282,7 +1264,7 @@ class PushEvent: public Event { assert(c, s->pushEvent == 0); s->pushEvent = this; - addRead(c, s->value, s->size * BytesPerWord, pushSite(c, this)); + addRead(c, s->value, s->geometry->size * BytesPerWord, pushSite(c, this)); } virtual void compile(Context* c) { @@ -1305,6 +1287,52 @@ class PushEvent: public Event { bool active; }; +void +appendPush(Context* c, Stack* s) +{ + if (DebugAppend) { + fprintf(stderr, "appendPush\n"); + } + + new (c->zone->allocate(sizeof(PushEvent))) PushEvent(c, s); +} + +void +appendPush(Context* c) +{ + appendPush(c, c->state->stack); +} + +class PushedEvent: public Event { + public: + PushedEvent(Context* c, Stack* s): + Event(c), s(s) + { } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "PushedEvent.compile\n"); + } + + assert(c, c->stackPadding == 0); + assert(c, s->geometry->padding == 0); + + s->pushSite = s->value->sites = pushSite(&c, s); + } + + Stack* s; +}; + +void +appendPushed(Context* c, Stack* s) +{ + if (DebugAppend) { + fprintf(stderr, "appendPushed\n"); + } + + new (c->zone->allocate(sizeof(PushedEvent))) PushedEvent(c, s); +} + void push(Context* c, unsigned size, Value* v); @@ -1332,7 +1360,7 @@ cleanStack(Context* c, Stack* stack, Local* locals, Read* reads) for (Stack* s = stack; s; s = s->next) { if (s->pushSite) { - addSite(c, 0, s->size * BytesPerWord, s->value, s->pushSite); + addSite(c, 0, s->geometry->size * BytesPerWord, s->value, s->pushSite); } } @@ -1367,14 +1395,15 @@ class CallEvent: public Event { public: CallEvent(Context* c, Value* address, unsigned flags, TraceHandler* traceHandler, Value* result, unsigned resultSize, - Stack* argumentStack, unsigned argumentCount): + Stack* argumentStack, unsigned argumentCount, unsigned padding): Event(c), address(address), traceHandler(traceHandler), result(result), flags(flags), resultSize(resultSize), - argumentFootprint(0) + argumentFootprint(0), + paddding(padding) { uint32_t mask = ~0; Stack* s = argumentStack; @@ -1388,10 +1417,10 @@ class CallEvent: public Event { } else { target = 0; s->pushEvent->active = true; - argumentFootprint += s->size; + argumentFootprint += s->geometry->size; } - addRead(c, s->value, s->size * BytesPerWord, target); - index += s->size; + addRead(c, s->value, s->geometry->size * BytesPerWord, target); + index += s->geometry->size; s = s->next; } @@ -1400,7 +1429,7 @@ class CallEvent: public Event { for (Stack* s = stack; s; s = s->next) { s->pushEvent->active = true; - addRead(c, s->value, s->size * BytesPerWord, virtualSite + addRead(c, s->value, s->geometry->size * BytesPerWord, virtualSite (c, 0, ~0, (static_cast(mask) << 32) | mask)); } @@ -1413,6 +1442,8 @@ class CallEvent: public Event { } pushNow(c, stack); + + pad(c, padding); UnaryOperation type = ((flags & Compiler::Aligned) ? AlignedCall : Call); apply(c, type, BytesPerWord, address->source); @@ -1423,7 +1454,7 @@ class CallEvent: public Event { cleanStack(c, stack, locals, reads); - if (resultSize and result->reads) { + if (resultSize and live(result)) { addSite(c, 0, resultSize, result, registerSite (c, c->assembler->returnLow(), resultSize > BytesPerWord ? @@ -1441,12 +1472,13 @@ class CallEvent: public Event { unsigned flags; unsigned resultSize; unsigned argumentFootprint; + unsigned padding; }; void appendCall(Context* c, Value* address, unsigned flags, TraceHandler* traceHandler, Value* result, unsigned resultSize, - Stack* argumentStack, unsigned argumentCount) + Stack* argumentStack, unsigned argumentCount, unsigned padding) { if (DebugAppend) { fprintf(stderr, "appendCall\n"); @@ -1454,7 +1486,7 @@ appendCall(Context* c, Value* address, unsigned flags, new (c->zone->allocate(sizeof(CallEvent))) CallEvent(c, address, flags, traceHandler, result, - resultSize, argumentStack, argumentCount); + resultSize, argumentStack, argumentCount, unsigned padding); } class ReturnEvent: public Event { @@ -1515,24 +1547,13 @@ class MoveEvent: public Event { fprintf(stderr, "MoveEvent.compile\n"); } - bool isLoad = src->reads->next == 0; - bool isStore = dst->reads == 0; + bool isLoad = not valid(src->reads->next); + bool isStore = not valid(dst->reads); - Site* target; - unsigned cost; - if (type == Move - and dst->reads - and next == dst->reads->event - and dst->reads->event->skipMove(size)) - { + Site* target = targetOrRegister(c, dst); + unsigned cost = src->source->copyCost(c, target); + if (cost == 0 and (isLoad or isStore)) { target = src->source; - cost = 0; - } else { - target = targetOrRegister(c, dst); - cost = src->source->copyCost(c, target); - if (cost == 0 and (isLoad or isStore)) { - target = src->source; - } } assert(c, isLoad or isStore or target != src->source); @@ -1695,7 +1716,7 @@ preserve(Context* c, Stack* stack, unsigned size, Value* v, Site* s, void maybePreserve(Context* c, Stack* stack, unsigned size, Value* v, Site* s) { - if (v->reads->next and v->sites->next == 0) { + if (valid(v->reads->next) and v->sites->next == 0) { preserve(c, stack, size, v, s, v->reads->next); } } @@ -1741,7 +1762,7 @@ class CombineEvent: public Event { nextRead(c, second); removeSite(c, second, second->source); - if (result->reads) { + if (live(result)) { addSite(c, 0, size, result, second->source); } } @@ -1823,7 +1844,7 @@ class TranslateEvent: public Event { nextRead(c, value); removeSite(c, value, value->source); - if (result->reads) { + if (live(result)) { addSite(c, 0, size, result, value->source); } } @@ -1926,11 +1947,17 @@ appendMemory(Context* c, Value* base, int displacement, Value* index, MemoryEvent(c, base, displacement, index, scale, result); } +Stack* +stack(Context* c, Value* value, StackGeometry* geometry, Stack* next) +{ + return new (c->zone->allocate(sizeof(Stack))) Stack(value, geometry, next); +} + Stack* stack(Context* c, Value* value, unsigned size, unsigned index, Stack* next) { - return new (c->zone->allocate(sizeof(Stack))) - Stack(value, size, index, next); + return stack(c, value, new (c->zone->allocate(sizeof(StackGeometry))) + StackGeometry(size, index), next); } void @@ -1939,8 +1966,8 @@ resetStack(Context* c) unsigned i = 0; Stack* p = 0; for (Stack* s = c->state->stack; s; s = s->next) { - Stack* n = stack(c, value(c), s->size, s->index, 0); - n->value->sites = n->pushSite = pushSite(c, s->index); + Stack* n = stack(c, value(c), s->geometry, 0); + n->value->sites = n->pushSite = pushSite(c, s); n->pushed = true; if (p) { @@ -1950,7 +1977,7 @@ resetStack(Context* c) } p = n; - i += s->size; + i += s->geometry->size; } resetLocals(c); @@ -1970,7 +1997,7 @@ popNow(Context* c, Stack* stack, unsigned count, bool ignore) s->pushed = false; Value* v = s->value; - if (v->reads and v->sites == 0 and (not ignore)) { + if (live(v) and v->sites == 0 and (not ignore)) { ::ignore(c, ignored); Site* target = targetOrRegister(c, v); @@ -1980,15 +2007,15 @@ popNow(Context* c, Stack* stack, unsigned count, bool ignore) s, s->value, target); } - addSite(c, stack, s->size * BytesPerWord, v, target); + addSite(c, stack, s->geometry->size * BytesPerWord, v, target); - apply(c, Pop, BytesPerWord * s->size, target); + apply(c, Pop, BytesPerWord * s->geometry->size, target); } else { if (DebugStack) { fprintf(stderr, "ignore %p value: %p\n", s, v); } - ignored += s->size; + ignored += s->geometry->size; } } else { if (DebugStack) { @@ -1996,7 +2023,9 @@ popNow(Context* c, Stack* stack, unsigned count, bool ignore) } } - i -= s->size; + c->stackPadding -= s->geometry->padding; + ignored += s->geometry->padding; + i -= s->geometry->size; s = s->next; } @@ -2010,7 +2039,7 @@ class StackSyncEvent: public Event { { for (Stack* s = stack; s; s = s->next) { if (s->pushEvent) s->pushEvent->active = true; - addRead(c, s->value, s->size * BytesPerWord, 0); + addRead(c, s->value, s->geometry->size * BytesPerWord, 0); } } @@ -2019,7 +2048,8 @@ class StackSyncEvent: public Event { { for (Stack* s = stack; s; s = s->next) { if (s->pushEvent) s->pushEvent->active = true; - insertRead(c, this, sequence, s->value, s->size * BytesPerWord, 0); + insertRead + (c, this, sequence, s->value, s->geometry->size * BytesPerWord, 0); } } @@ -2157,22 +2187,6 @@ pushSite(Context* c, PushEvent* e) return new (c->zone->allocate(sizeof(PushSite))) PushSite(e); } -void -appendPush(Context* c, Stack* s) -{ - if (DebugAppend) { - fprintf(stderr, "appendPush\n"); - } - - new (c->zone->allocate(sizeof(PushEvent))) PushEvent(c, s); -} - -void -appendPush(Context* c) -{ - appendPush(c, c->state->stack); -} - class PopEvent: public Event { public: PopEvent(Context* c, unsigned count, bool ignore): @@ -2215,7 +2229,7 @@ class ClobberLocalEvent: public Event { for (Local* l = local; l; l = l->old) { Value* v = l->value; Site* s = l->site; - if (v->reads + if (live(v) and v->sites->next == 0 and v->sites == s) { @@ -2258,7 +2272,7 @@ class LocalEvent: public Event { Site* sites = 0; if (local->old) { Value* v = local->old->value; - if (local->old->reuse and v->reads->next == 0) { + if (local->old->reuse and valid(v->reads->next) == 0) { sites = v->sites; } @@ -2266,7 +2280,7 @@ class LocalEvent: public Event { } Value* v = local->value; - if (v->reads) { + if (live(v)) { for (Site* s = sites; s;) { Site* t = s->next; if (s->type(c) != MemoryOperand) { @@ -2363,7 +2377,7 @@ void appendBoundsCheck(Context* c, Value* object, unsigned lengthOffset, Value* index, intptr_t handler) { - if (DebugAppend) { + if (DebugAppend) { fprintf(stderr, "appendBoundsCheck\n"); } @@ -2397,6 +2411,8 @@ compile(Context* c) { Assembler* a = c->assembler; + c->pass = CompilePass; + Assembler::Register base(a->base()); Assembler::Register stack(a->stack()); a->apply(Push, BytesPerWord, RegisterOperand, &base); @@ -2424,7 +2440,8 @@ compile(Context* c) for (Stack* s = e->stack; s; s = s->next) { if (s->value->sites) { assert(c, s->value->sites->next == 0); - s->value->sites->acquire(c, 0, s->size * BytesPerWord, s->value); + s->value->sites->acquire + (c, 0, s->geometry->size * BytesPerWord, s->value); } } } @@ -2513,7 +2530,9 @@ popState(Context* c) Stack* stack(Context* c, Value* value, unsigned size, Stack* next) { - return stack(c, value, size, (next ? next->index + next->size : 0), next); + return stack + (c, value, size, (next ? next->geometry->index + next->geometry->size : 0), + next); } void @@ -2545,7 +2564,7 @@ pop(Context* c, unsigned size UNUSED) Stack* s = c->state->stack; assert(c, ceiling(size, BytesPerWord) == s->size); - appendPop(c, s->size, false); + appendPop(c, s->geometry->size, false); c->state->stack = s->next; return s->value; @@ -2778,33 +2797,38 @@ class MyCompiler: public Compiler { return ::pop(&c, size); } - virtual void pushed(unsigned count) { - for (unsigned i = 0; i < count; ++i) { - Value* v = value(&c); - c.state->stack = ::stack(&c, v, 1, c.state->stack); - c.state->stack->pushed = true; - c.state->stack->pushSite - = v->sites = pushSite(&c, c.state->stack->index); - } + virtual void pushed() { + Value* v = value(&c); + c.state->stack = ::stack(&c, v, 1, c.state->stack); + c.state->stack->pushed = true; + appendPushed(&c, c.state->stack); } - virtual void popped(unsigned count) { - appendPop(&c, count, true); + virtual void popped() { + appendPop(&c, c.state->stack->size, true); - for (unsigned i = count; i;) { - Stack* s = c.state->stack; - c.state->stack = s->next; - i -= s->size; - } + c.state->stack = c.state->stack->next; + } + + virtual StackElement* top() { + return c.state->stack; + } + + virtual unsigned size(StackElement* e) { + return static_cast(e)->geometry->size; + } + + virtual unsigned padding(StackElement* e) { + return static_cast(e)->geometry->padding; } virtual Operand* peek(unsigned size UNUSED, unsigned index) { Stack* s = c.state->stack; for (unsigned i = index; i > 0;) { - i -= s->size; + i -= s->geometry->size; s = s->next; } - assert(&c, s->size == ceiling(size, BytesPerWord)); + assert(&c, s->geometry->size == ceiling(size, BytesPerWord)); return s->value; } @@ -2845,18 +2869,56 @@ class MyCompiler: public Compiler { } Stack* oldStack = c.state->stack; + Stack* bottomArgument = 0; for (int i = index - 1; i >= 0; --i) { ::push(&c, argumentSizes[i], arguments[i]); + if (i == index - 1) { + bottomArgument = c.state->stack; + } } - Stack* argumentStack = c.state->stack; c.state->stack = oldStack; + unsigned padding = c->assembler->stackPadding + (c.state->stack->geometry->index + c.state->stack->geometry->size); + + if (bottomArgument) { + bottomArgument->geometry->padding = padding; + } + Value* result = value(&c); - appendCall(&c, static_cast(address), flags, - traceHandler, result, resultSize, argumentStack, - index); + appendCall(&c, static_cast(address), flags, traceHandler, result, + resultSize, argumentStack, index, argumentCount ? 0 : padding); + + return result; + } + + virtual Operand* stackCall(Operand* address, + unsigned flags, + TraceHandler* traceHandler, + unsigned resultSize, + unsigned argumentFootprint) + { + unsigned padding = c->assembler->stackPadding + (c.state->stack->geometry->index + c.state->stack->geometry->size); + + int footprint = argumentFootprint; + for (Stack* s = c.state->stack; s; s = s->next) { + footprint -= s->geometry->size; + if (footprint == 0) { + s->geometry->padding = padding; + } + + if (s->pushEvent == 0) { + appendPush(&c, s); + } + s->pushEvent->active = true; + } + + Value* result = value(&c); + appendCall(&c, static_cast(address), flags, traceHandler, result, + resultSize, c.state->stack, 0, argumentFootprint ? 0 : padding); return result; } diff --git a/src/compiler.h b/src/compiler.h index 621f941019..b1f1d8f363 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -31,6 +31,7 @@ class Compiler { static const unsigned NoReturn = 1 << 1; class Operand { }; + class StackElement { }; virtual ~Compiler() { } @@ -68,8 +69,11 @@ class Compiler { virtual void push(unsigned size) = 0; virtual void push(unsigned size, Operand* value) = 0; virtual Operand* pop(unsigned size) = 0; - virtual void pushed(unsigned count) = 0; - virtual void popped(unsigned count) = 0; + virtual void pushed() = 0; + virtual void popped() = 0; + virtual StackElement* top() = 0; + virtual unsigned size(StackElement*) = 0; + virtual unsigned padding(StackElement*) = 0; virtual Operand* peek(unsigned size, unsigned index) = 0; virtual Operand* call(Operand* address, @@ -79,6 +83,12 @@ class Compiler { unsigned argumentCount, ...) = 0; + virtual Operand* stackCall(Operand* address, + unsigned flags, + TraceHandler* traceHandler, + unsigned resultSize, + unsigned argumentFootprint) = 0; + virtual void return_(unsigned size, Operand* value) = 0; virtual void storeLocal(unsigned size, Operand* src, unsigned index) = 0; diff --git a/src/machine.cpp b/src/machine.cpp index dbbcde9880..5391af5e86 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -2221,7 +2221,7 @@ isAssignableFrom(Thread* t, object a, object b) return false; } -bool FORCE_ALIGN +bool instanceOf(Thread* t, object class_, object o) { if (o == 0) { diff --git a/src/machine.h b/src/machine.h index 9920e057b0..73a2524cd2 100644 --- a/src/machine.h +++ b/src/machine.h @@ -1507,7 +1507,7 @@ mark(Thread* t, object o, unsigned offset) } } -inline void FORCE_ALIGN +inline void set(Thread* t, object target, unsigned offset, object value) { cast(target, offset) = value; @@ -1694,7 +1694,7 @@ makeExceptionInInitializerError(Thread* t, object cause) return makeExceptionInInitializerError(t, 0, trace, cause); } -inline object FORCE_ALIGN +inline object makeNew(Thread* t, object class_) { assert(t, t->state == Thread::ActiveState); @@ -1709,7 +1709,7 @@ makeNew(Thread* t, object class_) return instance; } -inline object FORCE_ALIGN +inline object makeNewWeakReference(Thread* t, object class_) { assert(t, t->state == Thread::ActiveState); diff --git a/src/x86.cpp b/src/x86.cpp index 9ee58700fe..fffb86149c 100644 --- a/src/x86.cpp +++ b/src/x86.cpp @@ -2030,6 +2030,15 @@ class MyAssembler: public Assembler { } } + virtual unsigned stackPadding(unsigned footprint) { +#if (defined __APPLE__) || (defined __x86_64__) + const unsigned alignment = 16 / BytesPerWord; + return (ceiling(footprint, alignment) * alignment) - footprint; +#else + return 0; +#endif + } + virtual void plan(UnaryOperation op, unsigned size, uint8_t* typeMask, uint64_t* registerMask, bool* thunk) {