diff --git a/makefile b/makefile index b0f5bab1c2..63954a932f 100644 --- a/makefile +++ b/makefile @@ -43,7 +43,7 @@ ar = ar ranlib = ranlib objcopy = objcopy vg = nice valgrind --num-callers=32 --db-attach=yes --freelist-vol=100000000 -vg += --leak-check=full +vg += --leak-check=full --suppressions=valgrind.supp db = gdb --args javac = javac jar = jar @@ -164,7 +164,9 @@ vm-depends = \ $(src)/jnienv.h \ $(src)/machine.h \ $(src)/util.h \ - $(src)/zone.h + $(src)/zone.h \ + $(src)/assembler.h \ + $(src)/compiler.h vm-sources = \ $(src)/$(system).cpp \ @@ -175,7 +177,8 @@ vm-sources = \ $(src)/$(process).cpp \ $(src)/builtin.cpp \ $(src)/jnienv.cpp \ - $(src)/process.cpp + $(src)/process.cpp \ + $(src)/$(asm).cpp vm-asm-sources = $(src)/$(asm).S diff --git a/readme.txt b/readme.txt index 7aa9f8cec6..fd4d549fbc 100644 --- a/readme.txt +++ b/readme.txt @@ -2,7 +2,7 @@ Quick Start ----------- on Linux: - $ export JAVA_HOME=/usr/local/java # or wherever you have Java installed + $ export JAVA_HOME=/usr/local/java # or wherever you have the JDK installed $ make $ build/linux-i386-compile-fast/avian -cp build/test Hello diff --git a/src/assembler.h b/src/assembler.h new file mode 100644 index 0000000000..e597554ee8 --- /dev/null +++ b/src/assembler.h @@ -0,0 +1,190 @@ +#ifndef ASSEMBLER_H +#define ASSEMBLER_H + +#include "system.h" +#include "zone.h" + +namespace vm { + +enum Operation { + Return +}; + +const unsigned OperationCount = Return + 1; + +enum UnaryOperation { + Push, + Pop, + Call, + AlignedCall, + Jump, + JumpIfLess, + JumpIfGreater, + JumpIfLessOrEqual, + JumpIfGreaterOrEqual, + JumpIfEqual, + JumpIfNotEqual, + Negate +}; + +const unsigned UnaryOperationCount = Negate + 1; + +enum BinaryOperation { + LoadAddress, + Move, + MoveZ, + Move4To8, + Swap, + Compare, + Add, + Subtract, + Multiply, + Divide, + Remainder, + ShiftLeft, + ShiftRight, + UnsignedShiftRight, + And, + Or, + Xor +}; + +const unsigned BinaryOperationCount = Xor + 1; + +enum OperandType { + ConstantOperand, + AddressOperand, + RegisterOperand, + MemoryOperand +}; + +const unsigned OperandTypeCount = MemoryOperand + 1; + +const int NoRegister = -1; +const int AnyRegister = -2; + +class Promise { + public: + virtual ~Promise() { } + + virtual int64_t value() = 0; + virtual bool resolved() = 0; +}; + +class ResolvedPromise: public Promise { + public: + ResolvedPromise(int64_t value): value_(value) { } + + virtual int64_t value() { + return value_; + } + + virtual bool resolved() { + return true; + } + + int64_t value_; +}; + +class TraceHandler { + public: + virtual ~TraceHandler() { } + + virtual void handleTrace(Promise* address) = 0; +}; + +class Assembler { + public: + class Operand { }; + + class Constant: public Operand { + public: + Constant(Promise* value): value(value) { } + + Promise* value; + }; + + class Address: public Operand { + public: + Address(Promise* address): address(address) { } + + Promise* address; + }; + + class Register: public Operand { + public: + Register(int low, int high = NoRegister): low(low), high(high) { } + + int low; + int high; + }; + + class Memory: public Operand { + public: + Memory(int base, int offset, int index = NoRegister, unsigned scale = 0): + base(base), offset(offset), index(index), scale(scale) + { } + + int base; + int offset; + int index; + unsigned scale; + }; + + class Client { + public: + virtual ~Client() { } + + virtual int acquireTemporary + (uint32_t mask = ~static_cast(0)) = 0; + virtual void releaseTemporary(int r) = 0; + + virtual void save(int r) = 0; + virtual void restore(int r) = 0; + }; + + virtual ~Assembler() { } + + virtual void setClient(Client* client) = 0; + + virtual unsigned registerCount() = 0; + + virtual int base() = 0; + virtual int stack() = 0; + virtual int thread() = 0; + virtual int returnLow() = 0; + virtual int returnHigh() = 0; + + virtual unsigned argumentRegisterCount() = 0; + virtual int argumentRegister(unsigned index) = 0; + + virtual void plan(UnaryOperation op, unsigned size, uint8_t* typeMask, + uint64_t* registerMask, bool* thunk) = 0; + + virtual void plan(BinaryOperation op, unsigned size, uint8_t* aTypeMask, + uint64_t* aRegisterMask, uint8_t* bTypeMask, + uint64_t* bRegisterMask, bool* thunk) = 0; + + virtual void apply(Operation op) = 0; + + virtual void apply(UnaryOperation op, unsigned size, OperandType type, + Operand* operand) = 0; + + virtual void apply(BinaryOperation op, unsigned size, OperandType aType, + Operand* a, OperandType bType, Operand* b) = 0; + + virtual void writeTo(uint8_t* dst) = 0; + + virtual unsigned length() = 0; + + virtual void updateCall(void* returnAddress, void* newTarget) = 0; + + virtual void dispose() = 0; +}; + +Assembler* +makeAssembler(System* system, Allocator* allocator, Zone* zone); + +} // namespace vm + +#endif//ASSEMBLER_H diff --git a/src/compile.cpp b/src/compile.cpp index 940dbcb956..8e70d23947 100644 --- a/src/compile.cpp +++ b/src/compile.cpp @@ -12,6 +12,7 @@ #include "util.h" #include "vector.h" #include "process.h" +#include "assembler.h" #include "compiler.h" #include "x86.h" @@ -359,7 +360,7 @@ class PoolElement { class Context; -class TraceElement: public Compiler::TraceHandler { +class TraceElement: public TraceHandler { public: TraceElement(Context* context, object target, bool virtualCall, TraceElement* next): @@ -378,83 +379,12 @@ class TraceElement: public Compiler::TraceHandler { Context* context; Promise* address; - intptr_t addressValue; object target; bool virtualCall; TraceElement* next; uintptr_t map[0]; }; -inline Stack* -push(Compiler* c, Stack* s, Operand* v) -{ - if (BytesPerWord == 8) { - return c->push8(s, v); - } else { - return c->push4(s, v); - } -} - -inline Stack* -pop(Compiler* c, Stack* s, Operand* v) -{ - if (BytesPerWord == 8) { - return c->pop8(s, v); - } else { - return c->pop4(s, v); - } -} - -inline void -mov(Compiler* c, Operand* src, Operand* dst) -{ - if (BytesPerWord == 8) { - c->mov8(src, dst); - } else { - c->mov4(src, dst); - } -} - -inline Operand* -result(Compiler* c) -{ - if (BytesPerWord == 8) { - return c->result8(); - } else { - return c->result4(); - } -} - -inline void -returnW(Compiler* c, Operand* v) -{ - if (BytesPerWord == 8) { - c->return8(v); - } else { - c->return4(v); - } -} - -inline void -cmp(Compiler* c, Operand* src, Operand* dst) -{ - if (BytesPerWord == 8) { - c->cmp8(src, dst); - } else { - c->cmp4(src, dst); - } -} - -inline void -and_(Compiler* c, Operand* src, Operand* dst) -{ - if (BytesPerWord == 8) { - c->and8(src, dst); - } else { - c->and4(src, dst); - } -} - enum Event { PushEvent, PopEvent, @@ -482,13 +412,6 @@ frameSize(MyThread* t, object method) return localSize(t, method) + codeMaxStack(t, methodCode(t, method)); } -unsigned -stackMapSizeInWords(MyThread* t, object method) -{ - return ceiling(codeMaxStack(t, methodCode(t, method)), BitsPerWord) - * BytesPerWord; -} - unsigned frameMapSizeInWords(MyThread* t, object method) { @@ -515,11 +438,24 @@ makeRootTable(MyThread* t, Zone* zone, object method) return table; } +enum Thunk { +#define THUNK(s) s##Thunk, + +#include "thunks.cpp" + +#undef THUNK +}; + +const unsigned ThunkCount = gcIfNecessaryThunk + 1; + +intptr_t +getThunk(MyThread* t, Thunk thunk); + class Context { public: class MyProtector: public Thread::Protector { public: - MyProtector(Context* c): Protector(c->t), c(c) { } + MyProtector(Context* c): Protector(c->thread), c(c) { } virtual void visit(Heap::Visitor* v) { v->visit(&(c->method)); @@ -536,10 +472,43 @@ class Context { Context* c; }; - Context(MyThread* t, object method, uint8_t* indirectCaller): - t(t), + class MyClient: public Compiler::Client { + public: + MyClient(MyThread* t): t(t) { } + + virtual intptr_t getThunk(UnaryOperation, unsigned) { + abort(t); + } + + virtual intptr_t getThunk(BinaryOperation op, unsigned size) { + switch (op) { + case Divide: + if (size == 8) { + return ::getThunk(t, divideLongThunk); + } + break; + + case Remainder: + if (size == 8) { + return ::getThunk(t, moduloLongThunk); + } + break; + + default: break; + } + + abort(t); + } + + MyThread* t; + }; + + Context(MyThread* t, object method): + thread(t), zone(t->m->system, t->m->heap, 16 * 1024), - c(makeCompiler(t->m->system, t->m->heap, &zone, indirectCaller)), + assembler(makeAssembler(t->m->system, t->m->heap, &zone)), + client(t), + compiler(makeCompiler(t->m->system, assembler, &zone, &client)), method(method), objectPool(0), traceLog(0), @@ -551,9 +520,11 @@ class Context { { } Context(MyThread* t): - t(t), + thread(t), zone(t->m->system, t->m->heap, LikelyPageSizeInBytes), - c(makeCompiler(t->m->system, t->m->heap, &zone, 0)), + assembler(makeAssembler(t->m->system, t->m->heap, &zone)), + client(t), + compiler(0), method(0), objectPool(0), traceLog(0), @@ -565,12 +536,15 @@ class Context { { } ~Context() { - c->dispose(); + if (compiler) compiler->dispose(); + assembler->dispose(); } - MyThread* t; + MyThread* thread; Zone zone; - Compiler* c; + Assembler* assembler; + MyClient client; + Compiler* compiler; object method; PoolElement* objectPool; TraceElement* traceLog; @@ -584,32 +558,37 @@ class Context { class Frame { public: - Frame(Context* context, uintptr_t* stackMap): + enum StackType { + Integer, + Long, + Object + }; + + Frame(Context* context, uint8_t* stackMap): context(context), - t(context->t), - c(context->c), - stack(0), + t(context->thread), + c(context->compiler), stackMap(stackMap), ip(0), sp(localSize()), level(0) { - memset(stackMap, 0, - stackMapSizeInWords(t, context->method) * BytesPerWord); + memset(stackMap, 0, codeMaxStack(t, methodCode(t, context->method))); } - Frame(Frame* f, uintptr_t* stackMap): + Frame(Frame* f, uint8_t* stackMap): context(f->context), - t(context->t), - c(context->c), - stack(f->stack), + t(context->thread), + c(context->compiler), stackMap(stackMap), ip(f->ip), sp(f->sp), level(f->level + 1) { - memcpy(stackMap, f->stackMap, - stackMapSizeInWords(t, context->method) * BytesPerWord); + c->pushState(); + + memcpy(stackMap, f->stackMap, codeMaxStack + (t, methodCode(t, context->method))); if (level > 1) { context->eventLog.append(PushEvent); @@ -617,17 +596,25 @@ class Frame { } ~Frame() { - if (level > 1 and t->exception == 0) { - context->eventLog.append(PopEvent); + if (t->exception == 0) { + if (level > 0) { + c->saveStack(); + c->popState(); + c->resetStack(); + } + + if (level > 1) { + context->eventLog.append(PopEvent); + } } } - Operand* append(object o) { + Compiler::Operand* append(object o) { Promise* p = c->poolAppend(0); context->objectPool = new (context->zone.allocate(sizeof(PoolElement))) PoolElement(o, p, context->objectPool); - return c->absolute(p); + return c->address(p); } unsigned localSize() { @@ -642,53 +629,51 @@ class Frame { return localSize() + stackSize(); } - void mark(unsigned index) { + void set(unsigned index, uint8_t type) { assert(t, index < frameSize()); - context->eventLog.append(MarkEvent); - context->eventLog.append2(index); + if (type == Object) { + context->eventLog.append(MarkEvent); + context->eventLog.append2(index); + } else { + context->eventLog.append(ClearEvent); + context->eventLog.append2(index); + } int si = index - localSize(); if (si >= 0) { - markBit(stackMap, si); + stackMap[si] = type; } } - void clear(unsigned index) { - assert(t, index < frameSize()); - - context->eventLog.append(ClearEvent); - context->eventLog.append2(index); - - int si = index - localSize(); - if (si >= 0) { - clearBit(stackMap, si); - } - } - - unsigned get(unsigned index) { + uint8_t get(unsigned index) { assert(t, index < frameSize()); int si = index - localSize(); assert(t, si >= 0); - return getBit(stackMap, si); + return stackMap[si]; } void pushedInt() { assert(t, sp + 1 <= frameSize()); - assert(t, get(sp) == 0); - ++ sp; + set(sp++, Integer); + } + + void pushedLong() { + assert(t, sp + 2 <= frameSize()); + set(sp++, Long); + set(sp++, Long); } void pushedObject() { assert(t, sp + 1 <= frameSize()); - mark(sp++); + set(sp++, Object); } void popped(unsigned count) { assert(t, sp >= count); assert(t, sp - count >= localSize()); while (count) { - clear(-- sp); + set(--sp, Integer); -- count; } } @@ -696,55 +681,57 @@ class Frame { void poppedInt() { assert(t, sp >= 1); assert(t, sp - 1 >= localSize()); - assert(t, get(sp - 1) == 0); + assert(t, get(sp - 1) == Integer); -- sp; } + void poppedLong() { + assert(t, sp >= 1); + assert(t, sp - 2 >= localSize()); + assert(t, get(sp - 1) == Long); + assert(t, get(sp - 2) == Long); + sp -= 2; + } + void poppedObject() { assert(t, sp >= 1); assert(t, sp - 1 >= localSize()); - assert(t, get(sp - 1) != 0); - clear(-- sp); + assert(t, get(sp - 1) == Object); + set(--sp, Integer); } void storedInt(unsigned index) { assert(t, index < localSize()); - clear(index); + set(index, Integer); + } + + void storedLong(unsigned index) { + assert(t, index + 1 < localSize()); + set(index, Long); + set(index + 1, Long); } void storedObject(unsigned index) { assert(t, index < localSize()); - mark(index); + set(index, Object); } void dupped() { assert(t, sp + 1 <= frameSize()); assert(t, sp - 1 >= localSize()); - if (get(sp - 1)) { - mark(sp); - } - ++ sp; + set(sp++, get(sp - 1)); } void duppedX1() { assert(t, sp + 1 <= frameSize()); assert(t, sp - 2 >= localSize()); - unsigned b2 = get(sp - 2); - unsigned b1 = get(sp - 1); + uint8_t b2 = get(sp - 2); + uint8_t b1 = get(sp - 1); - if (b2) { - mark(sp - 1); - } else { - clear(sp - 1); - } - - if (b1) { - mark(sp - 2); - mark(sp); - } else { - clear(sp - 2); - } + set(sp - 1, b2); + set(sp - 2, b1); + set(sp , b1); ++ sp; } @@ -753,28 +740,14 @@ class Frame { assert(t, sp + 1 <= frameSize()); assert(t, sp - 3 >= localSize()); - unsigned b3 = get(sp - 3); - unsigned b2 = get(sp - 2); - unsigned b1 = get(sp - 1); + uint8_t b3 = get(sp - 3); + uint8_t b2 = get(sp - 2); + uint8_t b1 = get(sp - 1); - if (b3) { - mark(sp - 2); - } else { - clear(sp - 2); - } - - if (b2) { - mark(sp - 1); - } else { - clear(sp - 1); - } - - if (b1) { - mark(sp - 3); - mark(sp); - } else { - clear(sp - 3); - } + set(sp - 2, b3); + set(sp - 1, b2); + set(sp - 3, b1); + set(sp , b1); ++ sp; } @@ -783,16 +756,11 @@ class Frame { assert(t, sp + 2 <= frameSize()); assert(t, sp - 2 >= localSize()); - unsigned b2 = get(sp - 2); - unsigned b1 = get(sp - 1); + uint8_t b2 = get(sp - 2); + uint8_t b1 = get(sp - 1); - if (b2) { - mark(sp); - } - - if (b1) { - mark(sp + 1); - } + set(sp, b2); + set(sp + 1, b1); sp += 2; } @@ -801,29 +769,15 @@ class Frame { assert(t, sp + 2 <= frameSize()); assert(t, sp - 3 >= localSize()); - unsigned b3 = get(sp - 3); - unsigned b2 = get(sp - 2); - unsigned b1 = get(sp - 1); + uint8_t b3 = get(sp - 3); + uint8_t b2 = get(sp - 2); + uint8_t b1 = get(sp - 1); - if (b3) { - mark(sp - 1); - } else { - clear(sp - 1); - } - - if (b2) { - mark(sp - 3); - mark(sp); - } else { - clear(sp - 3); - } - - if (b1) { - mark(sp - 2); - mark(sp + 1); - } else { - clear(sp - 2); - } + set(sp - 1, b3); + set(sp - 3, b2); + set(sp , b2); + set(sp - 2, b1); + set(sp + 1, b1); sp += 2; } @@ -832,36 +786,17 @@ class Frame { assert(t, sp + 2 <= frameSize()); assert(t, sp - 4 >= localSize()); - unsigned b4 = get(sp - 4); - unsigned b3 = get(sp - 3); - unsigned b2 = get(sp - 2); - unsigned b1 = get(sp - 1); + uint8_t b4 = get(sp - 4); + uint8_t b3 = get(sp - 3); + uint8_t b2 = get(sp - 2); + uint8_t b1 = get(sp - 1); - if (b4) { - mark(sp - 2); - } else { - clear(sp - 2); - } - - if (b3) { - mark(sp - 1); - } else { - clear(sp - 1); - } - - if (b2) { - mark(sp - 4); - mark(sp); - } else { - clear(sp - 4); - } - - if (b1) { - mark(sp - 3); - mark(sp + 1); - } else { - clear(sp - 3); - } + set(sp - 2, b4); + set(sp - 1, b3); + set(sp - 4, b2); + set(sp , b2); + set(sp - 3, b1); + set(sp + 1, b1); sp += 2; } @@ -869,212 +804,132 @@ class Frame { void swapped() { assert(t, sp - 2 >= localSize()); - bool savedBit = get(sp - 1); - if (get(sp - 2)) { - mark(sp - 1); - } else { - clear(sp - 1); - } + uint8_t saved = get(sp - 1); - if (savedBit) { - mark(sp - 2); - } else { - clear(sp - 2); - } + set(sp - 1, get(sp - 2)); + set(sp - 2, saved); } - Operand* machineIp(unsigned logicalIp) { + Compiler::Operand* machineIp(unsigned logicalIp) { return c->promiseConstant(c->machineIp(logicalIp)); } void visitLogicalIp(unsigned ip) { + c->visitLogicalIp(ip); + context->eventLog.append(IpEvent); context->eventLog.append2(ip); } void startLogicalIp(unsigned ip) { c->startLogicalIp(ip); + + context->eventLog.append(IpEvent); + context->eventLog.append2(ip); + this->ip = ip; } - void topIntToLong() { - dup(); - if (BytesPerWord == 4) { - c->mov4To8(c->stack(stack, 0), c->stack(stack, 0)); - } - } - - void topLongToInt() { - mov(c, c->stack(stack, 0), c->stack(stack, 1)); - stack = c->pop(stack, 1); - poppedInt(); - } - - void pushInt(Operand* o) { - stack = push(c, stack, o); + void pushInt(Compiler::Operand* o) { + c->push(4, o); pushedInt(); } - void pushInt1(Operand* o) { - stack = c->push1(stack, o); + void pushAddress(Compiler::Operand* o) { + c->push(BytesPerWord, o); pushedInt(); } - void pushInt2(Operand* o) { - stack = c->push2(stack, o); - pushedInt(); - } - - void pushInt2z(Operand* o) { - stack = c->push2z(stack, o); - pushedInt(); - } - - void pushInt4(Operand* o) { - stack = c->push4(stack, o); - pushedInt(); - } - - void pushAddress(Operand* o) { - stack = push(c, stack, o); - pushedInt(); - } - - void pushObject(Operand* o) { - stack = push(c, stack, o); + void pushObject(Compiler::Operand* o) { + c->push(BytesPerWord, o); pushedObject(); } void pushObject() { - stack = c->pushed(stack, 1); + c->pushed(1); pushedObject(); } - void pushLong(Operand* o) { + void pushLongQuiet(Compiler::Operand* o) { if (BytesPerWord == 8) { - stack = c->push(stack, 1); + c->push(8); } - stack = c->push8(stack, o); + c->push(8, o); + } - pushedInt(); - pushedInt(); + void pushLong(Compiler::Operand* o) { + pushLongQuiet(o); + pushedLong(); } void pop(unsigned count) { popped(count); - stack = c->pop(stack, count); + c->popped(count); } - Operand* topInt() { - assert(t, sp >= 1); - assert(t, sp - 1 >= localSize()); - assert(t, get(sp - 1) == 0); - return c->stack(stack, 0); - } - - Operand* topLong() { - assert(t, sp >= 2); - assert(t, sp - 2 >= localSize()); - assert(t, get(sp - 1) == 0); - assert(t, get(sp - 2) == 0); - return c->stack(stack, 0); - } - - Operand* topObject() { - assert(t, sp >= 1); - assert(t, sp - 1 >= localSize()); - assert(t, get(sp - 1) != 0); - return c->stack(stack, 0); - } - - Operand* popInt() { - Operand* tmp = c->temporary(); - popInt(tmp); - return tmp; - } - - Operand* popInt4() { - Operand* tmp = c->temporary(); - popInt4(tmp); - return tmp; - } - - Operand* popLong() { - Operand* tmp = c->temporary(); - popLong(tmp); - return tmp; - } - - Operand* popObject() { - Operand* tmp = c->temporary(); - popObject(tmp); - return tmp; - } - - void popInt(Operand* o) { - stack = ::pop(c, stack, o); + Compiler::Operand* popInt() { poppedInt(); + return c->pop(4); } - void popInt4(Operand* o) { - stack = c->pop4(stack, o); - poppedInt(); - } - - void popLong(Operand* o) { - stack = c->pop8(stack, o); + Compiler::Operand* popLongQuiet() { + Compiler::Operand* r = c->pop(8); if (BytesPerWord == 8) { - stack = c->pop(stack, 1); + c->pop(8); } - - poppedInt(); - poppedInt(); + return r; } - void popObject(Operand* o) { - stack = ::pop(c, stack, o); + Compiler::Operand* peekLong(unsigned index) { + return c->peek(8, index); + } + + Compiler::Operand* popLong() { + poppedLong(); + return popLongQuiet(); + } + + Compiler::Operand* popObject() { poppedObject(); + return c->pop(BytesPerWord); } void loadInt(unsigned index) { assert(t, index < localSize()); - pushInt(c->memory(c->base(), localOffset(t, index, context->method))); + pushInt(c->loadLocal(BytesPerWord, index)); } void loadLong(unsigned index) { - assert(t, index < static_cast - (localSize() - 1)); - pushLong(c->memory(c->base(), localOffset(t, index + 1, context->method))); + assert(t, index < static_cast(localSize() - 1)); + pushLong(c->loadLocal(8, index + 1)); } void loadObject(unsigned index) { assert(t, index < localSize()); - pushObject(c->memory(c->base(), localOffset(t, index, context->method))); + pushObject(c->loadLocal(BytesPerWord, index)); } void storeInt(unsigned index) { - popInt(c->memory(c->base(), localOffset(t, index, context->method))); + c->storeLocal(BytesPerWord, popInt(), index); storedInt(index); } void storeLong(unsigned index) { - popLong(c->memory(c->base(), localOffset(t, index + 1, context->method))); - storedInt(index); - storedInt(index + 1); + c->storeLocal(8, popLong(), index + 1); + storedLong(index); } void storeObject(unsigned index) { - popObject(c->memory(c->base(), localOffset(t, index, context->method))); + c->storeLocal(BytesPerWord, popObject(), index); storedObject(index); } void storeObjectOrAddress(unsigned index) { - stack = ::pop - (c, stack, c->memory(c->base(), localOffset(t, index, context->method))); + c->storeLocal(BytesPerWord, c->pop(BytesPerWord), index); assert(t, sp >= 1); assert(t, sp - 1 >= localSize()); - if (get(sp - 1)) { + if (get(sp - 1) == Object) { storedObject(index); } else { storedInt(index); @@ -1084,65 +939,125 @@ class Frame { } void dup() { - stack = push(c, stack, c->stack(stack, 0)); + c->push(BytesPerWord, c->peek(BytesPerWord, 0)); + dupped(); } void dupX1() { - stack = push(c, stack, c->stack(stack, 0)); - mov(c, c->stack(stack, 2), c->stack(stack, 1)); - mov(c, c->stack(stack, 0), c->stack(stack, 2)); + Compiler::Operand* s0 = c->pop(BytesPerWord); + Compiler::Operand* s1 = c->pop(BytesPerWord); + + c->push(BytesPerWord, s0); + c->push(BytesPerWord, s1); + c->push(BytesPerWord, s0); duppedX1(); } void dupX2() { - stack = push(c, stack, c->stack(stack, 0)); - mov(c, c->stack(stack, 2), c->stack(stack, 1)); - mov(c, c->stack(stack, 3), c->stack(stack, 2)); - mov(c, c->stack(stack, 0), c->stack(stack, 3)); + Compiler::Operand* s0 = c->pop(BytesPerWord); + + if (get(sp - 2) == Long) { + Compiler::Operand* s1 = popLongQuiet(); + + c->push(BytesPerWord, s0); + pushLongQuiet(s1); + c->push(BytesPerWord, s0); + } else { + Compiler::Operand* s1 = c->pop(BytesPerWord); + Compiler::Operand* s2 = c->pop(BytesPerWord); + + c->push(BytesPerWord, s0); + c->push(BytesPerWord, s2); + c->push(BytesPerWord, s1); + c->push(BytesPerWord, s0); + } duppedX2(); } void dup2() { - stack = push(c, stack, c->stack(stack, 1)); - stack = push(c, stack, c->stack(stack, 1)); + if (get(sp - 1) == Long) { + pushLongQuiet(peekLong(0)); + } else { + Compiler::Operand* s0 = c->pop(BytesPerWord); + Compiler::Operand* s1 = c->pop(BytesPerWord); + + c->push(BytesPerWord, s1); + c->push(BytesPerWord, s0); + c->push(BytesPerWord, s1); + c->push(BytesPerWord, s0); + } dupped2(); } void dup2X1() { - stack = push(c, stack, c->stack(stack, 1)); - stack = push(c, stack, c->stack(stack, 1)); - mov(c, c->stack(stack, 4), c->stack(stack, 2)); - mov(c, c->stack(stack, 1), c->stack(stack, 4)); - mov(c, c->stack(stack, 0), c->stack(stack, 3)); + if (get(sp - 1) == Long) { + Compiler::Operand* s0 = popLongQuiet(); + Compiler::Operand* s1 = c->pop(BytesPerWord); + + pushLongQuiet(s0); + c->push(BytesPerWord, s1); + pushLongQuiet(s0); + } else { + Compiler::Operand* s0 = c->pop(BytesPerWord); + Compiler::Operand* s1 = c->pop(BytesPerWord); + Compiler::Operand* s2 = c->pop(BytesPerWord); + + c->push(BytesPerWord, s1); + c->push(BytesPerWord, s0); + c->push(BytesPerWord, s2); + c->push(BytesPerWord, s1); + c->push(BytesPerWord, s0); + } dupped2X1(); } void dup2X2() { - stack = push(c, stack, c->stack(stack, 1)); - stack = push(c, stack, c->stack(stack, 1)); - mov(c, c->stack(stack, 5), c->stack(stack, 3)); - mov(c, c->stack(stack, 4), c->stack(stack, 2)); - mov(c, c->stack(stack, 1), c->stack(stack, 5)); - mov(c, c->stack(stack, 0), c->stack(stack, 4)); + if (get(sp - 1) == Long) { + Compiler::Operand* s0 = popLongQuiet(); + + if (get(sp - 3) == Long) { + Compiler::Operand* s1 = popLongQuiet(); + + pushLongQuiet(s0); + pushLongQuiet(s1); + pushLongQuiet(s0); + } else { + Compiler::Operand* s1 = c->pop(BytesPerWord); + Compiler::Operand* s2 = c->pop(BytesPerWord); + + pushLongQuiet(s0); + c->push(BytesPerWord, s2); + c->push(BytesPerWord, s1); + pushLongQuiet(s0); + } + } else { + Compiler::Operand* s0 = c->pop(BytesPerWord); + Compiler::Operand* s1 = c->pop(BytesPerWord); + Compiler::Operand* s2 = c->pop(BytesPerWord); + Compiler::Operand* s3 = c->pop(BytesPerWord); + + c->push(BytesPerWord, s1); + c->push(BytesPerWord, s0); + c->push(BytesPerWord, s3); + c->push(BytesPerWord, s2); + c->push(BytesPerWord, s1); + c->push(BytesPerWord, s0); + } dupped2X2(); } void swap() { - Operand* s0 = c->stack(stack, 0); - Operand* s1 = c->stack(stack, 1); - Operand* tmp = c->temporary(); + Compiler::Operand* s0 = c->pop(BytesPerWord); + Compiler::Operand* s1 = c->pop(BytesPerWord); - mov(c, s0, tmp); - mov(c, s1, s0); - mov(c, tmp, s1); - - c->release(tmp); + c->push(BytesPerWord, s0); + c->push(BytesPerWord, s1); swapped(); } @@ -1165,8 +1080,7 @@ class Frame { Context* context; MyThread* t; Compiler* c; - Stack* stack; - uintptr_t* stackMap; + uint8_t* stackMap; unsigned ip; unsigned sp; unsigned level; @@ -1456,13 +1370,13 @@ negateFloat(uint32_t a) return floatToBits(- bitsToFloat(a)); } -int64_t +int64_t FORCE_ALIGN divideLong(int64_t b, int64_t a) { return a / b; } -int64_t +int64_t FORCE_ALIGN moduloLong(int64_t b, int64_t a) { return a % b; @@ -1511,16 +1425,28 @@ longToFloat(int64_t a) } object FORCE_ALIGN -makeBlankObjectArray(Thread* t, object class_, int32_t length) +makeBlankObjectArray(MyThread* t, object class_, int32_t length) { - return makeObjectArray(t, class_, length, true); + if (length >= 0) { + return makeObjectArray(t, class_, length, true); + } else { + object message = makeString(t, "%d", length); + t->exception = makeNegativeArraySizeException(t, message); + unwind(t); + } } object FORCE_ALIGN -makeBlankArray(Thread* t, object (*constructor)(Thread*, uintptr_t, bool), +makeBlankArray(MyThread* t, object (*constructor)(Thread*, uintptr_t, bool), int32_t length) { - return constructor(t, length, true); + if (length >= 0) { + return constructor(t, length, true); + } else { + object message = makeString(t, "%d", length); + t->exception = makeNegativeArraySizeException(t, message); + unwind(t); + } } uintptr_t @@ -1605,8 +1531,8 @@ makeMultidimensionalArray2(MyThread* t, object class_, uintptr_t* stack, } object FORCE_ALIGN -makeMultidimensionalArray(MyThread* t, object class_, uintptr_t* stack, - int32_t dimensions) +makeMultidimensionalArray(MyThread* t, object class_, int32_t dimensions, + uintptr_t* stack) { object r = makeMultidimensionalArray2(t, class_, stack, dimensions); if (UNLIKELY(t->exception)) { @@ -1616,20 +1542,36 @@ makeMultidimensionalArray(MyThread* t, object class_, uintptr_t* stack, } } -void NO_RETURN FORCE_ALIGN -throwArrayIndexOutOfBounds(MyThread* t, object array, int32_t index) +unsigned +traceSize(Thread* t) { - object message = makeString - (t, "array of length %d indexed at %d", arrayLength(t, array), index); - t->exception = makeArrayIndexOutOfBoundsException(t, message); - unwind(t); + class Counter: public Processor::StackVisitor { + public: + Counter(): count(0) { } + + virtual bool visit(Processor::StackWalker*) { + ++ count; + return true; + } + + unsigned count; + } counter; + + t->m->processor->walkStack(t, &counter); + + return FixedSizeOfArray + (counter.count * ArrayElementSizeOfArray) + + (counter.count * FixedSizeOfTraceElement); } void NO_RETURN FORCE_ALIGN -throwNegativeArraySize(MyThread* t, int32_t length) +throwArrayIndexOutOfBounds(MyThread* t) { - object message = makeString(t, "%d", length); - t->exception = makeArrayIndexOutOfBoundsException(t, message); + ensure(t, FixedSizeOfArrayIndexOutOfBoundsException + traceSize(t)); + + t->tracing = true; + t->exception = makeArrayIndexOutOfBoundsException(t, 0); + t->tracing = false; + unwind(t); } @@ -1665,78 +1607,120 @@ gcIfNecessary(MyThread* t) } } -void -pushReturnValue(MyThread* t, Frame* frame, unsigned code) +unsigned +resultSize(MyThread* t, unsigned code) { - Compiler* c = frame->c; - switch (code) { case ByteField: case BooleanField: case CharField: case ShortField: case FloatField: - case IntField: { - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); - } break; + case IntField: + return 4; - case ObjectField: { - Operand* result = ::result(c); - frame->pushObject(result); - c->release(result); - } break; + case ObjectField: + return BytesPerWord; case LongField: - case DoubleField: { - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); - } break; + case DoubleField: + return 8; case VoidField: - break; + return 0; default: abort(t); } } -object -defaultCompiled(MyThread* t); +void +pushReturnValue(MyThread* t, Frame* frame, unsigned code, + Compiler::Operand* result) +{ + switch (code) { + case ByteField: + case BooleanField: + case CharField: + case ShortField: + case FloatField: + case IntField: + return frame->pushInt(result); + + case ObjectField: + return frame->pushObject(result); + + case LongField: + case DoubleField: + return frame->pushLong(result); + + default: + abort(t); + } +} + +bool +emptyMethod(MyThread* t, object method) +{ + return ((methodFlags(t, method) & ACC_NATIVE) == 0) + and (codeLength(t, methodCode(t, method)) == 1) + and (codeBody(t, methodCode(t, method), 0) == return_); +} object -nativeCompiled(MyThread* t); +defaultThunk(MyThread* t); + +object +nativeThunk(MyThread* t); + +object +aioobThunk(MyThread* t); void compileDirectInvoke(MyThread* t, Frame* frame, object target) { Compiler* c = frame->c; - if (methodFlags(t, target) & ACC_NATIVE) { - c->call - (c->constant - (reinterpret_cast - (&singletonBody(t, nativeCompiled(t), 0))), - frame->trace(target, false)); - } else if (methodCompiled(t, target) == defaultCompiled(t)) { - c->alignedCall - (c->constant - (reinterpret_cast - (&singletonBody(t, defaultCompiled(t), 0))), - frame->trace(target, false)); - } else { - c->call - (c->constant - (reinterpret_cast - (&singletonBody(t, methodCompiled(t, target), 0))), - frame->trace(0, false)); + unsigned rSize = resultSize(t, methodReturnCode(t, target)); + + Compiler::Operand* result = 0; + + if (not emptyMethod(t, target)) { + if (methodFlags(t, target) & ACC_NATIVE) { + result = c->call + (c->constant + (reinterpret_cast + (&singletonBody(t, nativeThunk(t), 0))), + 0, + frame->trace(target, false), + rSize, + 0); + } else if (methodCompiled(t, target) == defaultThunk(t)) { + result = c->call + (c->constant + (reinterpret_cast + (&singletonBody(t, defaultThunk(t), 0))), + Compiler::Aligned, + frame->trace(target, false), + rSize, + 0); + } else { + result = c->call + (c->constant + (reinterpret_cast + (&singletonBody(t, methodCompiled(t, target), 0))), + 0, + frame->trace(0, false), + rSize, + 0); + } } frame->pop(methodParameterFootprint(t, target)); - pushReturnValue(t, frame, methodReturnCode(t, target)); + if (rSize) { + pushReturnValue(t, frame, methodReturnCode(t, target), result); + } } void @@ -1746,7 +1730,7 @@ handleMonitorEvent(MyThread* t, Frame* frame, intptr_t function) object method = frame->context->method; if (methodFlags(t, method) & ACC_SYNCHRONIZED) { - Operand* lock; + Compiler::Operand* lock; if (methodFlags(t, method) & ACC_STATIC) { lock = frame->append(methodClass(t, method)); } else { @@ -1754,10 +1738,11 @@ handleMonitorEvent(MyThread* t, Frame* frame, intptr_t function) (c->base(), localOffset(t, savedTargetIndex(t, method), method)); } - c->indirectCall - (c->constant(function), - frame->trace(0, false), - 2, c->thread(), lock); + c->call(c->constant(function), + 0, + frame->trace(0, false), + 0, + 2, c->thread(), lock); } } @@ -1773,27 +1758,29 @@ handleEntrance(MyThread* t, Frame* frame) // save 'this' pointer in case it is overwritten. unsigned index = savedTargetIndex(t, method); - mov(c, c->memory(c->base(), localOffset(t, 0, method)), - c->memory(c->base(), localOffset(t, index, method))); - frame->mark(index); + c->store(BytesPerWord, + c->memory(c->base(), localOffset(t, 0, method)), + c->memory(c->base(), localOffset(t, index, method))); + frame->set(index, Frame::Object); } handleMonitorEvent - (t, frame, reinterpret_cast(acquireMonitorForObject)); + (t, frame, getThunk(t, acquireMonitorForObjectThunk)); } void handleExit(MyThread* t, Frame* frame) { handleMonitorEvent - (t, frame, reinterpret_cast(releaseMonitorForObject)); + (t, frame, getThunk(t, releaseMonitorForObjectThunk)); } void compile(MyThread* t, Frame* initialFrame, unsigned ip, bool exceptionHandler = false) { - uintptr_t stackMap[stackMapSizeInWords(t, initialFrame->context->method)]; + uint8_t stackMap + [codeMaxStack(t, methodCode(t, initialFrame->context->method))]; Frame myFrame(initialFrame, stackMap); Frame* frame = &myFrame; Compiler* c = frame->c; @@ -1803,21 +1790,26 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, PROTECT(t, code); while (ip < codeLength(t, code)) { - frame->visitLogicalIp(ip); - if (context->visitTable[ip] ++) { // we've already visited this part of the code + frame->visitLogicalIp(ip); return; } frame->startLogicalIp(ip); - if (exceptionHandler) { + if (ip == 0) { + handleEntrance(t, frame); + } else if (exceptionHandler) { exceptionHandler = false; + + frame->pushObject(); - c->indirectCall - (c->constant(reinterpret_cast(gcIfNecessary)), + c->call + (c->constant(getThunk(t, gcIfNecessaryThunk)), + 0, frame->trace(0, false), + 0, 1, c->thread()); } @@ -1834,60 +1826,76 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, case iaload: case laload: case saload: { - Operand* index = frame->popInt4(); - Operand* array = frame->popObject(); + Compiler::Operand* index = frame->popInt(); + Compiler::Operand* array = frame->popObject(); if (CheckArrayBounds) { - Operand* load = c->label(); - Operand* throw_ = c->label(); - - c->cmp4(c->constant(0), index); - c->jl(throw_); - - c->cmp4(c->memory(array, ArrayLength, 0, 1), index); - c->jl(load); - - c->mark(throw_); - - c->indirectCallNoReturn - (c->constant(reinterpret_cast(throwArrayIndexOutOfBounds)), - frame->trace(0, false), - 3, c->thread(), array, index); - - c->mark(load); + c->checkBounds(array, ArrayLength, index, reinterpret_cast + (&singletonValue(t, aioobThunk(t), 0))); } - switch (instruction) { - case aaload: - frame->pushObject - (c->memory(array, ArrayBody, index, BytesPerWord)); - break; + if (c->isConstant(index)) { + unsigned i = c->constantValue(index); + switch (instruction) { + case aaload: + frame->pushObject + (c->load + (BytesPerWord, c->memory(array, ArrayBody + (i * BytesPerWord)))); + break; - case faload: - case iaload: - frame->pushInt4(c->memory(array, ArrayBody, index, 4)); - break; + case faload: + case iaload: + frame->pushInt(c->load(4, c->memory(array, ArrayBody + (i * 4)))); + break; - case baload: - frame->pushInt1(c->memory(array, ArrayBody, index, 1)); - break; + case baload: + frame->pushInt(c->load(1, c->memory(array, ArrayBody + i))); + break; - case caload: - frame->pushInt2z(c->memory(array, ArrayBody, index, 2)); - break; + case caload: + frame->pushInt(c->loadz(2, c->memory(array, ArrayBody + (i * 2)))); + break; - case daload: - case laload: - frame->pushLong(c->memory(array, ArrayBody, index, 8)); - break; + case daload: + case laload: + frame->pushLong(c->load(8, c->memory(array, ArrayBody + (i * 8)))); + break; - case saload: - frame->pushInt2(c->memory(array, ArrayBody, index, 2)); - break; + case saload: + frame->pushInt(c->load(2, c->memory(array, ArrayBody + (i * 2)))); + break; + } + } else { + switch (instruction) { + case aaload: + frame->pushObject + (c->load + (BytesPerWord, c->memory(array, ArrayBody, index, BytesPerWord))); + break; + + case faload: + case iaload: + frame->pushInt(c->load(4, c->memory(array, ArrayBody, index, 4))); + break; + + case baload: + frame->pushInt(c->load(1, c->memory(array, ArrayBody, index, 1))); + break; + + case caload: + frame->pushInt(c->loadz(2, c->memory(array, ArrayBody, index, 2))); + break; + + case daload: + case laload: + frame->pushLong(c->load(8, c->memory(array, ArrayBody, index, 8))); + break; + + case saload: + frame->pushInt(c->load(2, c->memory(array, ArrayBody, index, 2))); + break; + } } - - c->release(index); - c->release(array); } break; case aastore: @@ -1898,7 +1906,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, case iastore: case lastore: case sastore: { - Operand* value; + Compiler::Operand* value; if (instruction == dastore or instruction == lastore) { value = frame->popLong(); } else if (instruction == aastore) { @@ -1907,63 +1915,81 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, value = frame->popInt(); } - Operand* index = frame->popInt4(); - Operand* array = frame->popObject(); + Compiler::Operand* index = frame->popInt(); + Compiler::Operand* array = frame->popObject(); if (CheckArrayBounds) { - Operand* store = c->label(); - Operand* throw_ = c->label(); - - c->cmp4(c->constant(0), index); - c->jl(throw_); - - c->cmp4(c->memory(array, ArrayLength, 0, 1), index); - c->jl(store); - - c->mark(throw_); - - c->indirectCallNoReturn - (c->constant(reinterpret_cast(throwArrayIndexOutOfBounds)), - frame->trace(0, false), - 3, c->thread(), array, index); - - c->mark(store); + c->checkBounds(array, ArrayLength, index, reinterpret_cast + (&singletonValue(t, aioobThunk(t), 0))); } - switch (instruction) { - case aastore: { - c->shl4(c->constant(log(BytesPerWord)), index); - c->add4(c->constant(ArrayBody), index); - - c->indirectCall - (c->constant(reinterpret_cast(setMaybeNull)), - frame->trace(0, false), - 4, c->thread(), array, index, value); - } break; + if (c->isConstant(index)) { + unsigned i = c->constantValue(index); + switch (instruction) { + case aastore: { + c->call + (c->constant(getThunk(t, setMaybeNullThunk)), + 0, + frame->trace(0, false), + 0, + 4, c->thread(), array, + c->constant(ArrayBody + (i * BytesPerWord)), + value); + } break; - case fastore: - case iastore: - c->mov4(value, c->memory(array, ArrayBody, index, 4)); - break; + case fastore: + case iastore: + c->store(4, value, c->memory(array, ArrayBody + (i * 4))); + break; - case bastore: - c->mov1(value, c->memory(array, ArrayBody, index, 1)); - break; + case bastore: + c->store(1, value, c->memory(array, ArrayBody + i)); + break; - case castore: - case sastore: - c->mov2(value, c->memory(array, ArrayBody, index, 2)); - break; + case castore: + case sastore: + c->store(2, value, c->memory(array, ArrayBody + (i * 2))); + break; - case dastore: - case lastore: - c->mov8(value, c->memory(array, ArrayBody, index, 8)); - break; + case dastore: + case lastore: + c->store(8, value, c->memory(array, ArrayBody + (i * 8))); + break; + } + } else { + switch (instruction) { + case aastore: { + c->call + (c->constant(getThunk(t, setMaybeNullThunk)), + 0, + frame->trace(0, false), + 0, + 4, c->thread(), array, + c->add(4, c->constant(ArrayBody), + c->shl(4, c->constant(log(BytesPerWord)), index)), + value); + } break; + + case fastore: + case iastore: + c->store(4, value, c->memory(array, ArrayBody, index, 4)); + break; + + case bastore: + c->store(1, value, c->memory(array, ArrayBody, index, 1)); + break; + + case castore: + case sastore: + c->store(2, value, c->memory(array, ArrayBody, index, 2)); + break; + + case dastore: + case lastore: + c->store(8, value, c->memory(array, ArrayBody, index, 8)); + break; + } } - - c->release(value); - c->release(index); - c->release(array); } break; case aconst_null: @@ -1996,43 +2022,26 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, object class_ = resolveClassInPool(t, codePool(t, code), index - 1); if (UNLIKELY(t->exception)) return; - Operand* nonnegative = c->label(); + Compiler::Operand* length = frame->popInt(); - Operand* length = frame->popInt4(); - c->cmp4(c->constant(0), length); - c->jge(nonnegative); - - c->indirectCallNoReturn - (c->constant(reinterpret_cast(throwNegativeArraySize)), - frame->trace(0, false), - 2, c->thread(), length); - - c->mark(nonnegative); - - c->indirectCall - (c->constant(reinterpret_cast(makeBlankObjectArray)), - frame->trace(0, false), - 3, c->thread(), frame->append(class_), length); - - Operand* result = ::result(c); - - c->release(length); - - frame->pushObject(result); - c->release(result); + frame->pushObject + (c->call + (c->constant(getThunk(t, makeBlankObjectArrayThunk)), + 0, + frame->trace(0, false), + BytesPerWord, + 3, c->thread(), frame->append(class_), length)); } break; case areturn: { handleExit(t, frame); - Operand* result = frame->popObject(); - returnW(c, result); - c->release(result); + c->return_(BytesPerWord, frame->popObject()); } return; case arraylength: { - Operand* array = frame->popObject(); - frame->pushInt4(c->memory(array, ArrayLength, 0, 1)); - c->release(array); + frame->pushInt + (c->load + (BytesPerWord, c->memory(frame->popObject(), ArrayLength, 0, 1))); } break; case astore: @@ -2056,12 +2065,12 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, break; case athrow: { - Operand* e = frame->popObject(); - c->indirectCallNoReturn - (c->constant(reinterpret_cast(throw_)), + c->call + (c->constant(getThunk(t, throw_Thunk)), + Compiler::NoReturn, frame->trace(0, false), - 2, c->thread(), e); - c->release(e); + 0, + 2, c->thread(), frame->popObject()); } return; case bipush: @@ -2075,94 +2084,74 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, object class_ = resolveClassInPool(t, codePool(t, code), index - 1); if (UNLIKELY(t->exception)) return; - Operand* instance = frame->topObject(); + Compiler::Operand* instance = c->peek(BytesPerWord, 0); - Operand* classOperand = frame->append(class_); - - c->indirectCall - (c->constant(reinterpret_cast(checkCast)), + c->call + (c->constant(getThunk(t, checkCastThunk)), + 0, frame->trace(0, false), - 3, c->thread(), classOperand, instance); + 0, + 3, c->thread(), frame->append(class_), instance); } break; case d2f: { - Operand* a = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(doubleToFloat)), 2, 0, a); - c->release(a); - - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, doubleToFloatThunk)), + 0, 0, 4, 2, + static_cast(0), frame->popLong())); } break; case d2i: { - Operand* a = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(doubleToInt)), 2, 0, a); - c->release(a); - - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, doubleToIntThunk)), + 0, 0, 4, 2, + static_cast(0), frame->popLong())); } break; case d2l: { - Operand* a = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(doubleToLong)), 2, 0, a); - c->release(a); - - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); + frame->pushLong + (c->call + (c->constant(getThunk(t, doubleToLongThunk)), + 0, 0, 8, 2, + static_cast(0), frame->popLong())); } break; case dadd: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(addDouble)), 4, 0, a, 0, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); + frame->pushLong + (c->call + (c->constant(getThunk(t, addDoubleThunk)), + 0, 0, 8, 4, + static_cast(0), a, + static_cast(0), b)); } break; case dcmpg: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(compareDoublesG)), - 4, 0, a, 0, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, compareDoublesGThunk)), + 0, 0, 4, 4, + static_cast(0), a, + static_cast(0), b)); } break; case dcmpl: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); - c->indirectCall - (c->constant(reinterpret_cast(compareDoublesL)), - 4, 0, a, 0, b); - c->release(a); - c->release(b); - - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, compareDoublesLThunk)), + 0, 0, 4, 4, + static_cast(0), a, + static_cast(0), b)); } break; case dconst_0: @@ -2174,75 +2163,59 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, break; case ddiv: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(divideDouble)), - 4, 0, a, 0, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); + frame->pushLong + (c->call + (c->constant(getThunk(t, divideDoubleThunk)), + 0, 0, 8, 4, + static_cast(0), a, + static_cast(0), b)); } break; case dmul: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(multiplyDouble)), - 4, 0, a, 0, b); - c->release(a); - c->release(b); - - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + + frame->pushLong + (c->call + (c->constant(getThunk(t, multiplyDoubleThunk)), + 0, 0, 8, 4, + static_cast(0), a, + static_cast(0), b)); } break; case dneg: { - Operand* a = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(negateDouble)), 2, 0, a); - c->release(a); - - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); + frame->pushLong + (c->call + (c->constant(getThunk(t, negateDoubleThunk)), + 0, 0, 8, 2, + static_cast(0), frame->popLong())); } break; case vm::drem: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(moduloDouble)), - 4, 0, a, 0, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); + frame->pushLong + (c->call + (c->constant(getThunk(t, moduloDoubleThunk)), + 0, 0, 8, 4, + static_cast(0), a, + static_cast(0), b)); } break; case dsub: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(subtractDouble)), - 4, 0, a, 0, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); + frame->pushLong + (c->call + (c->constant(getThunk(t, subtractDoubleThunk)), + 0, 0, 8, 4, + static_cast(0), a, + static_cast(0), b)); } break; case dup: @@ -2270,81 +2243,54 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, break; case f2d: { - Operand* a = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(floatToDouble)), 1, a); - c->release(a); - - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); + frame->pushLong + (c->call + (c->constant(getThunk(t, floatToDoubleThunk)), + 0, 0, 8, 1, frame->popInt())); } break; case f2i: { - Operand* a = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(floatToInt)), 1, a); - c->release(a); - - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, floatToIntThunk)), + 0, 0, 4, 1, frame->popInt())); } break; case f2l: { - Operand* a = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(floatToLong)), 1, a); - c->release(a); - - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); + frame->pushLong + (c->call + (c->constant(getThunk(t, floatToLongThunk)), + 0, 0, 8, 1, frame->popInt())); } break; case fadd: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(addFloat)), 2, a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, addFloatThunk)), + 0, 0, 4, 2, a, b)); } break; case fcmpg: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(compareFloatsG)), 2, a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, compareFloatsGThunk)), + 0, 0, 4, 2, a, b)); } break; case fcmpl: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(compareFloatsL)), 2, a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, compareFloatsLThunk)), + 0, 0, 4, 2, a, b)); } break; case fconst_0: @@ -2360,71 +2306,50 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, break; case fdiv: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(divideFloat)), 2, a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, divideFloatThunk)), + 0, 0, 4, 2, a, b)); } break; case fmul: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(multiplyFloat)), 2, a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, multiplyFloatThunk)), + 0, 0, 4, 2, a, b)); } break; case fneg: { - Operand* a = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(negateFloat)), 1, a); - c->release(a); - - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, negateFloatThunk)), + 0, 0, 4, 1, frame->popInt())); } break; case vm::frem: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(moduloFloat)), 2, a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, moduloFloatThunk)), + 0, 0, 4, 2, a, b)); } break; case fsub: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(subtractFloat)), 2, a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); + frame->pushInt + (c->call + (c->constant(getThunk(t, subtractFloatThunk)), + 0, 0, 4, 2, a, b)); } break; case getfield: @@ -2434,15 +2359,17 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, object field = resolveField(t, codePool(t, code), index - 1); if (UNLIKELY(t->exception)) return; - Operand* table; + Compiler::Operand* table; if (instruction == getstatic) { if ((classVmFlags(t, fieldClass(t, field)) & NeedInitFlag) and (classVmFlags(t, fieldClass(t, field)) & InitFlag) == 0) { - c->indirectCall - (c->constant(reinterpret_cast(tryInitClass)), + c->call + (c->constant(getThunk(t, tryInitClassThunk)), + 0, frame->trace(0, false), + 0, 2, c->thread(), frame->append(fieldClass(t, field))); } @@ -2454,38 +2381,41 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, switch (fieldCode(t, field)) { case ByteField: case BooleanField: - frame->pushInt1(c->memory(table, fieldOffset(t, field), 0, 1)); + frame->pushInt + (c->load(1, c->memory(table, fieldOffset(t, field), 0, 1))); break; case CharField: - frame->pushInt2z(c->memory(table, fieldOffset(t, field), 0, 1)); + frame->pushInt + (c->loadz(2, c->memory(table, fieldOffset(t, field), 0, 1))); break; case ShortField: - frame->pushInt2(c->memory(table, fieldOffset(t, field), 0, 1)); + frame->pushInt + (c->load(2, c->memory(table, fieldOffset(t, field), 0, 1))); break; case FloatField: case IntField: - frame->pushInt4(c->memory(table, fieldOffset(t, field), 0, 1)); + frame->pushInt + (c->load(4, c->memory(table, fieldOffset(t, field), 0, 1))); break; case DoubleField: case LongField: - frame->pushLong(c->memory(table, fieldOffset(t, field), 0, 1)); + frame->pushLong + (c->load(8, c->memory(table, fieldOffset(t, field), 0, 1))); break; case ObjectField: - frame->pushObject(c->memory(table, fieldOffset(t, field), 0, 1)); + frame->pushObject + (c->load + (BytesPerWord, c->memory(table, fieldOffset(t, field), 0, 1))); break; default: abort(t); } - - if (instruction == getfield) { - c->release(table); - } } break; case goto_: { @@ -2505,58 +2435,45 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, } break; case i2b: { - Operand* top = frame->topInt(); - c->mov1ToW(top, top); + frame->pushInt(c->load(1, frame->popInt())); } break; case i2c: { - Operand* top = frame->topInt(); - c->mov2zToW(top, top); + frame->pushInt(c->loadz(2, frame->popInt())); } break; case i2d: { - Operand* a = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(intToDouble)), 1, a); - - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); - c->release(a); + frame->pushLong + (c->call + (c->constant(getThunk(t, intToDoubleThunk)), + 0, 0, 8, 1, frame->popInt())); } break; case i2f: { - Operand* a = frame->popInt(); - - c->indirectCall - (c->constant(reinterpret_cast(intToFloat)), 1, a); - - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); - c->release(a); + frame->pushInt + (c->call + (c->constant(getThunk(t, intToFloatThunk)), + 0, 0, 4, 1, frame->popInt())); } break; case i2l: - frame->topIntToLong(); + frame->pushLong(c->load4To8(frame->popInt())); break; case i2s: { - Operand* top = frame->topInt(); - c->mov2ToW(top, top); + frame->pushInt(c->load(2, frame->popInt())); } break; case iadd: { - Operand* a = frame->popInt(); - c->add4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->add(4, a, b)); } break; case iand: { - Operand* a = frame->popInt(); - c->and4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->and_(4, a, b)); } break; case iconst_m1: @@ -2588,9 +2505,9 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, break; case idiv: { - Operand* a = frame->popInt(); - c->div4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->div(4, a, b)); } break; case if_acmpeq: @@ -2598,13 +2515,11 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); assert(t, newIp < codeLength(t, code)); - Operand* a = frame->popObject(); - Operand* b = frame->popObject(); - cmp(c, a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popObject(); + Compiler::Operand* b = frame->popObject(); + Compiler::Operand* target = frame->machineIp(newIp); - Operand* target = frame->machineIp(newIp); + c->cmp(BytesPerWord, a, b); if (instruction == if_acmpeq) { c->je(target); } else { @@ -2624,13 +2539,11 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); assert(t, newIp < codeLength(t, code)); - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - c->cmp4(a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + Compiler::Operand* target = frame->machineIp(newIp); - Operand* target = frame->machineIp(newIp); + c->cmp(4, a, b); switch (instruction) { case if_icmpeq: c->je(target); @@ -2665,11 +2578,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); assert(t, newIp < codeLength(t, code)); - Operand* a = frame->popInt(); - c->cmp4(c->constant(0), a); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* target = frame->machineIp(newIp); - Operand* target = frame->machineIp(newIp); + c->cmp(4, c->constant(0), a); switch (instruction) { case ifeq: c->je(target); @@ -2700,11 +2612,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); assert(t, newIp < codeLength(t, code)); - Operand* a = frame->popObject(); - cmp(c, c->constant(0), a); - c->release(a); + Compiler::Operand* a = frame->popObject(); + Compiler::Operand* target = frame->machineIp(newIp); - Operand* target = frame->machineIp(newIp); + c->cmp(BytesPerWord, c->constant(0), a); if (instruction == ifnull) { c->je(target); } else { @@ -2719,8 +2630,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, uint8_t index = codeBody(t, code, ip++); int8_t count = codeBody(t, code, ip++); - c->add4(c->constant(count), - c->memory(c->base(), localOffset(t, index, context->method))); + Compiler::Operand* a = c->memory + (c->base(), localOffset(t, index, context->method)); + + c->storeLocal(4, c->add(4, c->constant(count), a), index); } break; case iload: @@ -2749,13 +2662,13 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, break; case imul: { - Operand* a = frame->popInt(); - c->mul4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->mul(4, a, b)); } break; case ineg: { - c->neg4(frame->topInt()); + frame->pushInt(c->neg(4, frame->popInt())); } break; case instanceof: { @@ -2764,18 +2677,11 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, object class_ = resolveClassInPool(t, codePool(t, code), index - 1); if (UNLIKELY(t->exception)) return; - Operand* instance = frame->popObject(); - - Operand* classOperand = frame->append(class_); - - c->indirectCall - (c->constant(reinterpret_cast(instanceOf)), - 3, c->thread(), classOperand, instance); - - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); - c->release(instance); + frame->pushInt + (c->call + (c->constant(getThunk(t, instanceOfThunk)), + 0, 0, 4, + 3, c->thread(), frame->append(class_), frame->popObject())); } break; case invokeinterface: { @@ -2789,20 +2695,27 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, unsigned instance = parameterFootprint - 1; - c->indirectCall - (c->constant - (reinterpret_cast(findInterfaceMethodFromInstance)), - frame->trace(0, false), - 3, c->thread(), frame->append(target), - c->stack(frame->stack, instance)); + unsigned rSize = resultSize(t, methodReturnCode(t, target)); - Operand* result = ::result(c); - c->call(result, frame->trace(target, true)); - c->release(result); + Compiler::Operand* result = c->call + (c->call + (c->constant + (getThunk(t, findInterfaceMethodFromInstanceThunk)), + 0, + frame->trace(0, false), + BytesPerWord, + 3, c->thread(), frame->append(target), + c->peek(BytesPerWord, instance)), + 0, + frame->trace(target, true), + rSize, + 0); frame->pop(parameterFootprint); - pushReturnValue(t, frame, methodReturnCode(t, target)); + if (rSize) { + pushReturnValue(t, frame, methodReturnCode(t, target), result); + } } break; case invokespecial: { @@ -2838,51 +2751,56 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, unsigned offset = ClassVtable + (methodOffset(t, target) * BytesPerWord); - Operand* instance = c->stack(frame->stack, parameterFootprint - 1); - Operand* class_ = c->temporary(); - - mov(c, c->memory(instance, 0, 0, 1), class_); - and_(c, c->constant(PointerMask), class_); + Compiler::Operand* instance = c->peek + (BytesPerWord, parameterFootprint - 1); - c->call(c->memory(class_, offset, 0, 1), frame->trace(target, true)); + unsigned rSize = resultSize(t, methodReturnCode(t, target)); - c->release(class_); + Compiler::Operand* result = c->call + (c->memory + (c->and_ + (BytesPerWord, c->constant(PointerMask), + c->memory(instance, 0, 0, 1)), offset, 0, 1), + 0, + frame->trace(target, true), + rSize, + 0); frame->pop(parameterFootprint); - pushReturnValue(t, frame, methodReturnCode(t, target)); + if (rSize) { + pushReturnValue(t, frame, methodReturnCode(t, target), result); + } } break; case ior: { - Operand* a = frame->popInt(); - c->or4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->or_(4, a, b)); } break; case irem: { - Operand* a = frame->popInt(); - c->rem4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->rem(4, a, b)); } break; case ireturn: case freturn: { handleExit(t, frame); - Operand* a = frame->popInt(); - c->return4(a); - c->release(a); + c->return_(4, frame->popInt()); } return; case ishl: { - Operand* a = frame->popInt(); - c->shl4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->shl(4, a, b)); } break; case ishr: { - Operand* a = frame->popInt(); - c->shr4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->shr(4, a, b)); } break; case istore: @@ -2911,21 +2829,21 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, break; case isub: { - Operand* a = frame->popInt(); - c->sub4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->sub(4, a, b)); } break; case iushr: { - Operand* a = frame->popInt(); - c->ushr4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->ushr(4, a, b)); } break; case ixor: { - Operand* a = frame->popInt(); - c->xor4(a, frame->topInt()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popInt(); + frame->pushInt(c->xor_(4, a, b)); } break; case jsr: @@ -2952,75 +2870,70 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, } break; case l2d: { - Operand* a = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(longToDouble)), 2, 0, a); - - Operand* result = c->result8(); - frame->pushLong(result); - c->release(result); - c->release(a); + frame->pushLong + (c->call + (c->constant(getThunk(t, longToDoubleThunk)), + 0, 0, 8, 2, + static_cast(0), frame->popLong())); } break; case l2f: { - Operand* a = frame->popLong(); - - c->indirectCall - (c->constant(reinterpret_cast(longToDouble)), 2, 0, a); - - Operand* result = c->result4(); - frame->pushInt(result); - c->release(result); - c->release(a); + frame->pushInt + (c->call + (c->constant(getThunk(t, longToFloatThunk)), + 0, 0, 4, 2, + static_cast(0), frame->popLong())); } break; case l2i: - frame->topLongToInt(); + frame->pushInt(c->load(4, frame->popLong())); break; case ladd: { - Operand* a = frame->popLong(); - c->add8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->add(8, a, b)); } break; case land: { - Operand* a = frame->popLong(); - c->and8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->and_(8, a, b)); } break; case lcmp: { - Operand* next = c->label(); - Operand* less = c->label(); - Operand* greater = c->label(); + Compiler::Operand* next = c->label(); + Compiler::Operand* less = c->label(); + Compiler::Operand* greater = c->label(); - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - Operand* result = c->temporary(); - - c->cmp8(a, b); - c->release(a); - c->release(b); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + + c->cmp(8, a, b); c->jl(less); + c->pushState(); + c->jg(greater); + c->pushState(); - c->mov4(c->constant(0), result); + c->push(4, c->constant(0)); c->jmp(next); - + + c->popState(); c->mark(less); - c->mov4(c->constant(-1), result); + + c->push(4, c->constant(-1)); c->jmp(next); + c->popState(); c->mark(greater); - c->mov4(c->constant(1), result); + + c->push(4, c->constant(1)); c->mark(next); - frame->pushInt(result); - c->release(result); + frame->pushedInt(); } break; case lconst_0: @@ -3071,9 +2984,9 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, } break; case ldiv_: { - Operand* a = frame->popLong(); - c->div8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->div(8, a, b)); } break; case lload: @@ -3102,13 +3015,13 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, break; case lmul: { - Operand* a = frame->popLong(); - c->mul8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->mul(8, a, b)); } break; case lneg: - c->neg8(frame->topLong()); + frame->pushLong(c->neg(8, frame->popLong())); break; case lookupswitch: { @@ -3116,17 +3029,17 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, ip = (ip + 3) & ~3; // pad to four byte boundary - Operand* key = frame->popInt4(); + Compiler::Operand* key = frame->popInt(); uint32_t defaultIp = base + codeReadInt32(t, code, ip); assert(t, defaultIp < codeLength(t, code)); - Operand* default_ = c->absolute + Compiler::Operand* default_ = c->address (c->poolAppendPromise(c->machineIp(defaultIp))); int32_t pairCount = codeReadInt32(t, code, ip); - Operand* start = 0; + Compiler::Operand* start = 0; uint32_t ipTable[pairCount]; for (int32_t i = 0; i < pairCount; ++i) { unsigned index = ip + (i * 8); @@ -3144,15 +3057,11 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, } assert(t, start); - c->indirectCall - (c->constant(reinterpret_cast(lookUpAddress)), - 4, key, start, c->constant(pairCount), default_); - - Operand* result = ::result(c); - c->jmp(result); - c->release(result); - - c->release(key); + c->jmp + (c->call + (c->constant(getThunk(t, lookUpAddressThunk)), + 0, 0, BytesPerWord, + 4, key, start, c->constant(pairCount), default_)); for (int32_t i = 0; i < pairCount; ++i) { compile(t, frame, ipTable[i]); @@ -3163,35 +3072,33 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, } break; case lor: { - Operand* a = frame->popLong(); - c->or8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->or_(8, a, b)); } break; case lrem: { - Operand* a = frame->popLong(); - c->rem8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->rem(8, a, b)); } break; case lreturn: case dreturn: { handleExit(t, frame); - Operand* a = frame->popLong(); - c->return8(a); - c->release(a); + c->return_(8, frame->popLong()); } return; case lshl: { - Operand* a = frame->popInt(); - c->shl8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->shl(8, a, b)); } break; case lshr: { - Operand* a = frame->popInt(); - c->shr8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->shr(8, a, b)); } break; case lstore: @@ -3220,39 +3127,35 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, break; case lsub: { - Operand* a = frame->popLong(); - c->sub8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->sub(8, a, b)); } break; case lushr: { - Operand* a = frame->popInt(); - c->ushr8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popInt(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->ushr(8, a, b)); } break; case lxor: { - Operand* a = frame->popLong(); - c->xor8(a, frame->topLong()); - c->release(a); + Compiler::Operand* a = frame->popLong(); + Compiler::Operand* b = frame->popLong(); + frame->pushLong(c->xor_(8, a, b)); } break; case monitorenter: { - Operand* a = frame->popObject(); - c->indirectCall - (c->constant(reinterpret_cast(acquireMonitorForObject)), - frame->trace(0, false), - 2, c->thread(), a); - c->release(a); + c->call + (c->constant(getThunk(t, acquireMonitorForObjectThunk)), + 0, + frame->trace(0, false), 0, 2, c->thread(), frame->popObject()); } break; case monitorexit: { - Operand* a = frame->popObject(); - c->indirectCall - (c->constant(reinterpret_cast(releaseMonitorForObject)), - frame->trace(0, false), - 2, c->thread(), a); - c->release(a); + c->call + (c->constant(getThunk(t, releaseMonitorForObjectThunk)), + 0, + frame->trace(0, false), 0, 2, c->thread(), frame->popObject()); } break; case multianewarray: { @@ -3263,22 +3166,16 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, if (UNLIKELY(t->exception)) return; PROTECT(t, class_); - Operand* stack = c->temporary(); - mov(c, c->stack(), stack); - - c->indirectCall - (c->constant(reinterpret_cast(makeMultidimensionalArray)), + Compiler::Operand* result = c->call + (c->constant(getThunk(t, makeMultidimensionalArrayThunk)), + 0, frame->trace(0, false), - 4, c->thread(), frame->append(class_), stack, - c->constant(dimensions)); - - c->release(stack); - - Operand* result = ::result(c); + BytesPerWord, + 4, c->thread(), frame->append(class_), c->constant(dimensions), + c->stack()); frame->pop(dimensions); frame->pushObject(result); - c->release(result); } break; case new_: { @@ -3288,38 +3185,28 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, if (UNLIKELY(t->exception)) return; if (classVmFlags(t, class_) & WeakReferenceFlag) { - c->indirectCall - (c->constant(reinterpret_cast(makeNewWeakReference)), - frame->trace(0, false), - 2, c->thread(), frame->append(class_)); + frame->pushObject + (c->call + (c->constant(getThunk(t, makeNewWeakReferenceThunk)), + 0, + frame->trace(0, false), + BytesPerWord, + 2, c->thread(), frame->append(class_))); } else { - c->indirectCall - (c->constant(reinterpret_cast(makeNew)), - frame->trace(0, false), - 2, c->thread(), frame->append(class_)); + frame->pushObject + (c->call + (c->constant(getThunk(t, makeNewThunk)), + 0, + frame->trace(0, false), + BytesPerWord, + 2, c->thread(), frame->append(class_))); } - - Operand* result = ::result(c); - frame->pushObject(result); - c->release(result); } break; case newarray: { uint8_t type = codeBody(t, code, ip++); - Operand* nonnegative = c->label(); - - Operand* length = frame->popInt4(); - c->cmp4(c->constant(0), length); - - c->jge(nonnegative); - - c->indirectCallNoReturn - (c->constant(reinterpret_cast(throwNegativeArraySize)), - frame->trace(0, false), - 2, c->thread(), length); - - c->mark(nonnegative); + Compiler::Operand* length = frame->popInt(); object (*constructor)(Thread*, uintptr_t, bool); switch (type) { @@ -3358,18 +3245,14 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, default: abort(t); } - c->indirectCall - (c->constant(reinterpret_cast(makeBlankArray)), - frame->trace(0, false), - 3, c->thread(), c->constant(reinterpret_cast(constructor)), - length); - - Operand* result = ::result(c); - - c->release(length); - - frame->pushObject(result); - c->release(result); + frame->pushObject + (c->call + (c->constant(getThunk(t, makeBlankArrayThunk)), + 0, + frame->trace(0, false), + BytesPerWord, + 3, c->thread(), c->constant(reinterpret_cast(constructor)), + length)); } break; case nop: break; @@ -3395,16 +3278,18 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, if ((classVmFlags(t, fieldClass(t, field)) & NeedInitFlag) and (classVmFlags(t, fieldClass(t, field)) & InitFlag) == 0) { - c->indirectCall - (c->constant(reinterpret_cast(tryInitClass)), + c->call + (c->constant(getThunk(t, tryInitClassThunk)), + 0, frame->trace(0, false), + 0, 2, c->thread(), frame->append(fieldClass(t, field))); } staticTable = classStaticTable(t, fieldClass(t, field)); } - Operand* value; + Compiler::Operand* value; switch (fieldCode(t, field)) { case ByteField: case BooleanField: @@ -3427,7 +3312,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, default: abort(t); } - Operand* table; + Compiler::Operand* table; if (instruction == putstatic) { table = frame->append(staticTable); @@ -3438,45 +3323,42 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, switch (fieldCode(t, field)) { case ByteField: case BooleanField: - c->mov1(value, c->memory(table, fieldOffset(t, field), 0, 1)); + c->store(1, value, c->memory(table, fieldOffset(t, field), 0, 1)); break; case CharField: case ShortField: - c->mov2(value, c->memory(table, fieldOffset(t, field), 0, 1)); + c->store(2, value, c->memory(table, fieldOffset(t, field), 0, 1)); break; case FloatField: case IntField: - c->mov4(value, c->memory(table, fieldOffset(t, field), 0, 1)); + c->store(4, value, c->memory(table, fieldOffset(t, field), 0, 1)); break; case DoubleField: case LongField: - c->mov8(value, c->memory(table, fieldOffset(t, field), 0, 1)); + c->store(8, value, c->memory(table, fieldOffset(t, field), 0, 1)); break; case ObjectField: if (instruction == putfield) { - c->indirectCall - (c->constant(reinterpret_cast(setMaybeNull)), + c->call + (c->constant(getThunk(t, setMaybeNullThunk)), + 0, frame->trace(0, false), + 0, 4, c->thread(), table, c->constant(fieldOffset(t, field)), value); } else { - c->indirectCall - (c->constant(reinterpret_cast(set)), - 4, c->thread(), table, c->constant(fieldOffset(t, field)), - value); + c->call + (c->constant(getThunk(t, setThunk)), + 0, 0, 0, + 4, c->thread(), table, c->constant(fieldOffset(t, field)), value); } break; default: abort(t); } - - if (instruction == putfield) { - c->release(table); - } - c->release(value); } break; case ret: @@ -3488,8 +3370,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, case return_: handleExit(t, frame); - c->epilogue(); - c->ret(); + c->return_(0, 0); return; case sipush: @@ -3506,15 +3387,13 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, ip = (ip + 3) & ~3; // pad to four byte boundary - Operand* key = frame->popInt4(); - uint32_t defaultIp = base + codeReadInt32(t, code, ip); assert(t, defaultIp < codeLength(t, code)); int32_t bottom = codeReadInt32(t, code, ip); int32_t top = codeReadInt32(t, code, ip); - Operand* start = 0; + Compiler::Operand* start = 0; uint32_t ipTable[top - bottom + 1]; for (int32_t i = 0; i < top - bottom + 1; ++i) { unsigned index = ip + (i * 4); @@ -3530,22 +3409,22 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, } assert(t, start); - Operand* defaultCase = c->label(); + Compiler::Operand* defaultCase = c->label(); + + Compiler::Operand* key = frame->popInt(); - c->cmp4(c->constant(bottom), key); + c->cmp(4, c->constant(bottom), key); c->jl(defaultCase); - c->cmp4(c->constant(top), key); + c->cmp(4, c->constant(top), key); c->jg(defaultCase); - c->sub4(c->constant(bottom), key); - c->jmp(c->memory(start, 0, key, BytesPerWord)); + c->jmp(c->memory(start, 0, c->sub(4, c->constant(bottom), key), + BytesPerWord)); c->mark(defaultCase); c->jmp(frame->machineIp(defaultIp)); - c->release(key); - for (int32_t i = 0; i < top - bottom + 1; ++i) { compile(t, frame, ipTable[i]); if (UNLIKELY(t->exception)) return; @@ -3568,8 +3447,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, uint16_t index = codeReadInt16(t, code, ip); uint16_t count = codeReadInt16(t, code, ip); - c->add4(c->constant(count), - c->memory(c->base(), localOffset(t, index, context->method))); + Compiler::Operand* a = c->memory + (c->base(), localOffset(t, index, context->method)); + + c->storeLocal(4, c->add(4, c->constant(count), a), index); } break; case iload: { @@ -3608,7 +3489,7 @@ void logCompile(const void* code, unsigned size, const char* class_, const char* name, const char* spec) { - fprintf(stderr, "%s.%s%s from %p to %p\n", + fprintf(stderr, "%s.%s%s: %p %p\n", class_, name, spec, code, static_cast(code) + size); } @@ -3637,13 +3518,13 @@ translateExceptionHandlerTable(MyThread* t, Compiler* c, object code, (t, oldTable, i); intArrayBody(t, newIndex, i * 3) - = c->machineIp(exceptionHandlerStart(oldHandler))->value(c) - start; + = c->machineIp(exceptionHandlerStart(oldHandler))->value() - start; intArrayBody(t, newIndex, (i * 3) + 1) - = c->machineIp(exceptionHandlerEnd(oldHandler))->value(c) - start; + = c->machineIp(exceptionHandlerEnd(oldHandler))->value() - start; intArrayBody(t, newIndex, (i * 3) + 2) - = c->machineIp(exceptionHandlerIp(oldHandler))->value(c) - start; + = c->machineIp(exceptionHandlerIp(oldHandler))->value() - start; object type; if (exceptionHandlerCatchType(oldHandler)) { @@ -3676,7 +3557,7 @@ translateLineNumberTable(MyThread* t, Compiler* c, object code, intptr_t start) LineNumber* newLine = lineNumberTableBody(t, newTable, i); lineNumberIp(newLine) - = c->machineIp(lineNumberIp(oldLine))->value(c) - start; + = c->machineIp(lineNumberIp(oldLine))->value() - start; lineNumberLine(newLine) = lineNumberLine(oldLine); } @@ -3806,7 +3687,7 @@ calculateFrameMaps(MyThread* t, Context* context, uintptr_t* originalRoots, return eventIndex; } -Allocator* +Zone* codeZone(MyThread* t); int @@ -3814,9 +3695,9 @@ compareTraceElementPointers(const void* va, const void* vb) { TraceElement* a = *static_cast(va); TraceElement* b = *static_cast(vb); - if (a->addressValue > b->addressValue) { + if (a->address->value() > b->address->value()) { return 1; - } else if (a->addressValue < b->addressValue) { + } else if (a->address->value() < b->address->value()) { return -1; } else { return 0; @@ -3845,12 +3726,18 @@ frameObjectMapSize(MyThread* t, object method, object map) return ceiling(intArrayLength(t, map) * size, 32 + size); } -object -finish(MyThread* t, Context* context, const char* name) +unsigned +codeSingletonSizeInBytes(MyThread*, unsigned codeSizeInBytes) { - Compiler* c = context->c; + unsigned count = ceiling(codeSizeInBytes, BytesPerWord); + unsigned size = count + singletonMaskSize(count); + return pad(SingletonBody + (size * BytesPerWord)); +} - unsigned count = ceiling(c->codeSize() + c->poolSize(), BytesPerWord); +object +allocateCode(MyThread* t, unsigned codeSizeInBytes) +{ + unsigned count = ceiling(codeSizeInBytes, BytesPerWord); unsigned size = count + singletonMaskSize(count); object result = allocate3 (t, codeZone(t), Machine::ImmortalAllocation, @@ -3858,120 +3745,136 @@ finish(MyThread* t, Context* context, const char* name) initSingleton(t, result, size, true); mark(t, result, 0); singletonMask(t, result)[0] = 1; + return result; +} + +object +finish(MyThread* t, Assembler* a, const char* name) +{ + object result = allocateCode(t, a->length()); + uint8_t* start = reinterpret_cast(&singletonValue(t, result, 0)); + + a->writeTo(start); + + if (Verbose) { + logCompile(start, a->length(), 0, name, 0); + } + + return result; +} + +object +finish(MyThread* t, Context* context) +{ + Compiler* c = context->compiler; + + unsigned codeSize = c->compile(); + object result = allocateCode(t, pad(codeSize) + c->poolSize()); + PROTECT(t, result); uint8_t* start = reinterpret_cast(&singletonValue(t, result, 0)); c->writeTo(start); - if (context->method) { - PROTECT(t, result); + translateExceptionHandlerTable(t, c, methodCode(t, context->method), + reinterpret_cast(start)); + if (UNLIKELY(t->exception)) return 0; - translateExceptionHandlerTable(t, c, methodCode(t, context->method), - reinterpret_cast(start)); - if (UNLIKELY(t->exception)) return 0; + translateLineNumberTable(t, c, methodCode(t, context->method), + reinterpret_cast(start)); - translateLineNumberTable(t, c, methodCode(t, context->method), - reinterpret_cast(start)); + { object code = methodCode(t, context->method); - { object code = methodCode(t, context->method); + code = makeCode(t, 0, + codeExceptionHandlerTable(t, code), + codeLineNumberTable(t, code), + codeMaxStack(t, code), + codeMaxLocals(t, code), + 0, false); - code = makeCode(t, 0, - codeExceptionHandlerTable(t, code), - codeLineNumberTable(t, code), - codeMaxStack(t, code), - codeMaxLocals(t, code), - 0, false); + set(t, context->method, MethodCode, code); + } - set(t, context->method, MethodCode, code); + if (context->traceLogCount) { + TraceElement* elements[context->traceLogCount]; + unsigned index = 0; + for (TraceElement* p = context->traceLog; p; p = p->next) { + assert(t, index < context->traceLogCount); + + elements[index++] = p; + + if (p->target) { + insertCallNode + (t, makeCallNode + (t, p->address->value(), p->target, p->virtualCall, 0)); + } } - if (context->traceLogCount) { - TraceElement* elements[context->traceLogCount]; - unsigned index = 0; - for (TraceElement* p = context->traceLog; p; p = p->next) { - assert(t, index < context->traceLogCount); + qsort(elements, context->traceLogCount, sizeof(TraceElement*), + compareTraceElementPointers); - elements[index++] = p; - p->addressValue = p->address->value(c); + unsigned size = frameSize(t, context->method); + object map = makeIntArray + (t, context->traceLogCount + + ceiling(context->traceLogCount * size, 32), + false); - if (p->target) { - insertCallNode - (t, makeCallNode - (t, p->address->value(c), p->target, p->virtualCall, 0)); - } - } + assert(t, intArrayLength(t, map) == context->traceLogCount + + frameObjectMapSize(t, context->method, map)); - qsort(elements, context->traceLogCount, sizeof(TraceElement*), - compareTraceElementPointers); + for (unsigned i = 0; i < context->traceLogCount; ++i) { + TraceElement* p = elements[i]; - unsigned size = frameSize(t, context->method); - object map = makeIntArray - (t, context->traceLogCount - + ceiling(context->traceLogCount * size, 32), - false); - - assert(t, intArrayLength(t, map) == context->traceLogCount - + frameObjectMapSize(t, context->method, map)); - - for (unsigned i = 0; i < context->traceLogCount; ++i) { - TraceElement* p = elements[i]; - - intArrayBody(t, map, i) = static_cast(p->addressValue) - - reinterpret_cast(start); - - for (unsigned j = 0; j < size; ++j) { - unsigned index = ((i * size) + j); - int32_t* v = &intArrayBody - (t, map, context->traceLogCount + (index / 32)); - - if (getBit(p->map, j)) { - *v |= static_cast(1) << (index % 32); - } else { - *v &= ~(static_cast(1) << (index % 32)); - } - } - } - - set(t, methodCode(t, context->method), CodePool, map); - } - - for (PoolElement* p = context->objectPool; p; p = p->next) { - intptr_t offset = p->address->value(c) + intArrayBody(t, map, i) = static_cast(p->address->value()) - reinterpret_cast(start); - singletonMarkObject(t, result, offset / BytesPerWord); + for (unsigned j = 0; j < size; ++j) { + unsigned index = ((i * size) + j); + int32_t* v = &intArrayBody + (t, map, context->traceLogCount + (index / 32)); - set(t, result, SingletonBody + offset, p->value); + if (getBit(p->map, j)) { + *v |= static_cast(1) << (index % 32); + } else { + *v &= ~(static_cast(1) << (index % 32)); + } + } } - if (Verbose) { - logCompile - (start, c->codeSize(), - reinterpret_cast - (&byteArrayBody(t, className(t, methodClass(t, context->method)), 0)), - reinterpret_cast - (&byteArrayBody(t, methodName(t, context->method), 0)), - reinterpret_cast - (&byteArrayBody(t, methodSpec(t, context->method), 0))); - } + set(t, methodCode(t, context->method), CodePool, map); + } - // for debugging: - if (false and - strcmp - (reinterpret_cast - (&byteArrayBody(t, className(t, methodClass(t, context->method)), 0)), - "java/lang/System") == 0 and - strcmp - (reinterpret_cast - (&byteArrayBody(t, methodName(t, context->method), 0)), - "getProperty") == 0) - { - asm("int3"); - } - } else { - if (Verbose) { - logCompile(start, c->codeSize(), 0, name, 0); - } + for (PoolElement* p = context->objectPool; p; p = p->next) { + intptr_t offset = p->address->value() - reinterpret_cast(start); + + singletonMarkObject(t, result, offset / BytesPerWord); + + set(t, result, SingletonBody + offset, p->value); + } + + if (Verbose) { + logCompile + (start, codeSize, + reinterpret_cast + (&byteArrayBody(t, className(t, methodClass(t, context->method)), 0)), + reinterpret_cast + (&byteArrayBody(t, methodName(t, context->method), 0)), + reinterpret_cast + (&byteArrayBody(t, methodSpec(t, context->method), 0))); + } + + // for debugging: + if (false and + strcmp + (reinterpret_cast + (&byteArrayBody(t, className(t, methodClass(t, context->method)), 0)), + "java/lang/String") == 0 and + strcmp + (reinterpret_cast + (&byteArrayBody(t, methodName(t, context->method), 0)), + "compareTo") == 0) + { + asm("int3"); } return result; @@ -3980,25 +3883,23 @@ finish(MyThread* t, Context* context, const char* name) object compile(MyThread* t, Context* context) { - Compiler* c = context->c; + Compiler* c = context->compiler; // fprintf(stderr, "compiling %s.%s%s\n", // &byteArrayBody(t, className(t, methodClass(t, context->method)), 0), // &byteArrayBody(t, methodName(t, context->method), 0), // &byteArrayBody(t, methodSpec(t, context->method), 0)); - c->prologue(); - unsigned footprint = methodParameterFootprint(t, context->method); unsigned locals = localSize(t, context->method); - c->reserve(locals - footprint); + c->init(codeLength(t, methodCode(t, context->method)), footprint, locals); - uintptr_t stackMap[stackMapSizeInWords(t, context->method)]; + uint8_t stackMap[codeMaxStack(t, methodCode(t, context->method))]; Frame frame(context, stackMap); unsigned index = 0; if ((methodFlags(t, context->method) & ACC_STATIC) == 0) { - frame.mark(index++); + frame.set(index++, Frame::Object); } for (MethodSpecIterator it @@ -4009,22 +3910,21 @@ compile(MyThread* t, Context* context) switch (*it.next()) { case 'L': case '[': - frame.mark(index++); + frame.set(index++, Frame::Object); break; case 'J': case 'D': - index += 2; + frame.set(index++, Frame::Long); + frame.set(index++, Frame::Long); break; default: - ++ index; + frame.set(index++, Frame::Integer); break; } } - handleEntrance(t, &frame); - compile(t, &frame, 0); if (UNLIKELY(t->exception)) return 0; @@ -4051,7 +3951,7 @@ compile(MyThread* t, Context* context) visited[i] = true; progress = true; - uintptr_t stackMap[stackMapSizeInWords(t, context->method)]; + uint8_t stackMap[codeMaxStack(t, methodCode(t, context->method))]; Frame frame2(&frame, stackMap); uintptr_t* roots = context->rootTable @@ -4059,19 +3959,17 @@ compile(MyThread* t, Context* context) for (unsigned i = 0; i < localSize(t, context->method); ++ i) { if (getBit(roots, i)) { - frame2.mark(i); + frame2.set(i, Frame::Object); } else { - frame2.clear(i); + frame2.set(i, Frame::Integer); } } - frame2.pushObject(); - for (unsigned i = 1; i < codeMaxStack(t, methodCode(t, context->method)); ++i) { - frame2.clear(localSize(t, context->method) + i); + frame2.set(localSize(t, context->method) + i, Frame::Integer); } compile(t, &frame2, exceptionHandlerIp(eh), true); @@ -4090,7 +3988,7 @@ compile(MyThread* t, Context* context) calculateFrameMaps(t, context, 0, 0); } - return finish(t, context, 0); + return finish(t, context); } void @@ -4123,7 +4021,7 @@ compileMethod2(MyThread* t) = &singletonValue(t, methodCompiled(t, target), 0); } else { Context context(t); - context.c->updateCall + context.assembler->updateCall (reinterpret_cast(callNodeAddress(t, node)), &singletonValue(t, methodCompiled(t, target), 0)); } @@ -4454,39 +4352,44 @@ visitStack(MyThread* t, Heap::Visitor* v) } } -object -compileDefault(MyThread* t, Context* context) +void +saveStackAndBase(MyThread* t, Assembler* a) { - Compiler* c = context->c; + Assembler::Register base(a->base()); + Assembler::Memory baseDst(a->thread(), difference(&(t->base), t)); + a->apply(Move, BytesPerWord, RegisterOperand, &base, + MemoryOperand, &baseDst); - mov(c, c->base(), c->memory(c->thread(), difference(&(t->base), t))); - mov(c, c->stack(), c->memory(c->thread(), difference(&(t->stack), t))); - - c->directCall - (c->constant(reinterpret_cast(compileMethod)), - 1, c->thread()); - - Operand* result = ::result(c); - c->jmp(result); - c->release(result); - - return finish(t, context, "default"); + Assembler::Register stack(a->stack()); + Assembler::Memory stackDst(a->thread(), difference(&(t->stack), t)); + a->apply(Move, BytesPerWord, RegisterOperand, &stack, + MemoryOperand, &stackDst); } -object -compileNative(MyThread* t, Context* context) +void +pushThread(MyThread*, Assembler* a) { - Compiler* c = context->c; + Assembler::Register thread(a->thread()); - mov(c, c->base(), c->memory(c->thread(), difference(&(t->base), t))); - mov(c, c->stack(), c->memory(c->thread(), difference(&(t->stack), t))); + if (a->argumentRegisterCount()) { + Assembler::Register arg(a->argumentRegister(0)); + a->apply(Move, BytesPerWord, RegisterOperand, &thread, + RegisterOperand, &arg); + } else { + a->apply(Push, BytesPerWord, RegisterOperand, &thread); + } +} - c->directCall - (c->constant(reinterpret_cast(invokeNative)), 1, c->thread()); - - c->ret(); - - return finish(t, context, "native"); +void +popThread(MyThread*, Assembler* a) +{ + if (a->argumentRegisterCount() == 0) { + ResolvedPromise bpwPromise(BytesPerWord); + Assembler::Constant bpw(&bpwPromise); + Assembler::Register stack(a->stack()); + a->apply(Add, BytesPerWord, ConstantOperand, &bpw, + RegisterOperand, &stack); + } } class ArgumentList { @@ -4667,27 +4570,6 @@ invoke(Thread* thread, object method, ArgumentList* arguments) return r; } -unsigned -traceSize(Thread* t) -{ - class Counter: public Processor::StackVisitor { - public: - Counter(): count(0) { } - - virtual bool visit(Processor::StackWalker*) { - ++ count; - return true; - } - - unsigned count; - } counter; - - t->m->processor->walkStack(t, &counter); - - return FixedSizeOfArray + (counter.count * ArrayElementSizeOfArray) - + (counter.count * FixedSizeOfTraceElement); -} - class SegFaultHandler: public System::SignalHandler { public: SegFaultHandler(): m(0) { } @@ -4760,14 +4642,13 @@ class MyProcessor: public Processor { MyProcessor(System* s, Allocator* allocator): s(s), allocator(allocator), - defaultCompiled(0), - nativeCompiled(0), + defaultThunk(0), + nativeThunk(0), + aioobThunk(0), callTable(0), callTableSize(0), methodTree(0), methodTreeSentinal(0), - indirectCaller(0), - indirectCallerSize(0), codeAllocator(s), codeZone(s, &codeAllocator, 64 * 1024) { } @@ -4781,22 +4662,6 @@ class MyProcessor: public Processor { return t; } - object getDefaultCompiled(MyThread* t) { - if (defaultCompiled == 0) { - Context context(t); - defaultCompiled = compileDefault(t, &context); - } - return defaultCompiled; - } - - object getNativeCompiled(MyThread* t) { - if (nativeCompiled == 0) { - Context context(t); - nativeCompiled = compileNative(t, &context); - } - return nativeCompiled; - } - virtual object makeMethod(vm::Thread* t, uint8_t vmFlags, @@ -4813,7 +4678,7 @@ class MyProcessor: public Processor { return vm::makeMethod (t, vmFlags, returnCode, parameterCount, parameterFootprint, flags, offset, name, spec, class_, code, - getDefaultCompiled(static_cast(t))); + ::defaultThunk(static_cast(t))); } virtual object @@ -4844,7 +4709,7 @@ class MyProcessor: public Processor { initVtable(Thread* t, object c) { void* compiled = &singletonBody - (t, getDefaultCompiled(static_cast(t)), 0); + (t, ::defaultThunk(static_cast(t)), 0); for (unsigned i = 0; i < classLength(t, c); ++i) { classVtable(t, c, i) = compiled; @@ -4875,8 +4740,10 @@ class MyProcessor: public Processor { MyThread* t = static_cast(vmt); if (t == t->m->rootThread) { - v->visit(&defaultCompiled); - v->visit(&nativeCompiled); + v->visit(&defaultThunk); + v->visit(&nativeThunk); + v->visit(&aioobThunk); + v->visit(&thunkTable); v->visit(&callTable); v->visit(&methodTree); v->visit(&methodTreeSentinal); @@ -5044,13 +4911,13 @@ class MyProcessor: public Processor { virtual object getStackTrace(Thread* vmt, Thread* vmTarget) { MyThread* t = static_cast(vmt); MyThread* target = static_cast(vmTarget); - - processor(t)->getDefaultCompiled(t); - processor(t)->getNativeCompiled(t); + MyProcessor* p = processor(t); class Visitor: public System::ThreadVisitor { public: - Visitor(MyThread* t, MyThread* target): t(t), target(target) { } + Visitor(MyThread* t, MyProcessor* p, MyThread* target): + t(t), p(p), target(target) + { } virtual void visit(void* ip, void* base, void* stack) { void* oldIp = target->ip; @@ -5062,25 +4929,12 @@ class MyProcessor: public Processor { target->base = base; target->stack = stack; } else { - MyProcessor* p = processor(t); + uint8_t* thunkStart = reinterpret_cast + (&singletonValue(t, p->thunkTable, 0)); + uint8_t* thunkEnd = thunkStart + (p->thunkSize * ThunkCount); - uint8_t* default_ = reinterpret_cast - (&singletonValue(t, p->defaultCompiled, 0)); - unsigned defaultSize = singletonLength(t, p->defaultCompiled); - - uint8_t* native = reinterpret_cast - (&singletonValue(t, p->nativeCompiled, 0)); - unsigned nativeSize = singletonLength(t, p->nativeCompiled); - - if ((static_cast(ip) >= p->indirectCaller - and static_cast(ip) - < p->indirectCaller + p->indirectCallerSize) - - or (static_cast(ip) >= default_ - and static_cast(ip) < default_ + defaultSize) - - or (static_cast(ip) >= native - and static_cast(ip) < native + nativeSize)) + if (static_cast(ip) >= thunkStart + and static_cast(ip) < thunkEnd) { target->ip = *static_cast(stack); target->base = base; @@ -5100,9 +4954,10 @@ class MyProcessor: public Processor { } MyThread* t; + MyProcessor* p; MyThread* target; object trace; - } visitor(t, target); + } visitor(t, p, target); t->m->system->visit(t->systemThread, target->systemThread, &visitor); @@ -5117,19 +4972,155 @@ class MyProcessor: public Processor { System* s; Allocator* allocator; - object defaultCompiled; - object nativeCompiled; + object defaultThunk; + object nativeThunk; + object aioobThunk; + object thunkTable; + unsigned thunkSize; object callTable; unsigned callTableSize; object methodTree; object methodTreeSentinal; - uint8_t* indirectCaller; - unsigned indirectCallerSize; SegFaultHandler segFaultHandler; CodeAllocator codeAllocator; Zone codeZone; }; +intptr_t +getThunk(MyThread* t, Thunk thunk) +{ + MyProcessor* p = processor(t); + + return reinterpret_cast + (&singletonValue(t, p->thunkTable, (thunk * p->thunkSize) / BytesPerWord)); +} + +void +compileThunks(MyThread* t, MyProcessor* p) +{ + class ThunkContext { + public: + class MyPromise: public Promise { + public: + MyPromise(): resolved_(false) { } + + virtual int64_t value() { + return value_; + } + + virtual bool resolved() { + return resolved_; + } + + int64_t value_; + bool resolved_; + }; + + ThunkContext(MyThread* t): context(t) { } + + Context context; + MyPromise promise; + }; + + ThunkContext defaultContext(t); + + { Assembler* a = defaultContext.context.assembler; + + saveStackAndBase(t, a); + pushThread(t, a); + + defaultContext.promise.resolved_ = true; + defaultContext.promise.value_ = reinterpret_cast(compileMethod); + + Assembler::Constant proc(&(defaultContext.promise)); + a->apply(Call, BytesPerWord, ConstantOperand, &proc); + + popThread(t, a); + + Assembler::Register result(a->returnLow()); + a->apply(Jump, BytesPerWord, RegisterOperand, &result); + } + + ThunkContext nativeContext(t); + + { Assembler* a = nativeContext.context.assembler; + + saveStackAndBase(t, a); + pushThread(t, a); + + nativeContext.promise.resolved_ = true; + nativeContext.promise.value_ = reinterpret_cast(invokeNative); + + Assembler::Constant proc(&(nativeContext.promise)); + a->apply(Call, BytesPerWord, ConstantOperand, &proc); + + popThread(t, a); + + a->apply(Return); + } + + ThunkContext aioobContext(t); + + { Assembler* a = aioobContext.context.assembler; + + saveStackAndBase(t, a); + pushThread(t, a); + + aioobContext.promise.resolved_ = true; + aioobContext.promise.value_ = reinterpret_cast + (throwArrayIndexOutOfBounds); + + Assembler::Constant proc(&(aioobContext.promise)); + a->apply(Call, BytesPerWord, ConstantOperand, &proc); + } + + ThunkContext tableContext(t); + + { Assembler* a = tableContext.context.assembler; + + saveStackAndBase(t, a); + + Assembler::Constant proc(&(tableContext.promise)); + a->apply(Jump, BytesPerWord, ConstantOperand, &proc); + } + + p->thunkSize = pad(tableContext.context.assembler->length()); + + expect(t, codeZone(t)->ensure + (codeSingletonSizeInBytes + (t, defaultContext.context.assembler->length()) + + codeSingletonSizeInBytes + (t, nativeContext.context.assembler->length()) + + codeSingletonSizeInBytes + (t, aioobContext.context.assembler->length()) + + codeSingletonSizeInBytes + (t, p->thunkSize * ThunkCount))); + + p->defaultThunk = finish(t, defaultContext.context.assembler, "default"); + p->nativeThunk = finish(t, nativeContext.context.assembler, "native"); + p->aioobThunk = finish(t, aioobContext.context.assembler, "aioob"); + + p->thunkTable = allocateCode(t, p->thunkSize * ThunkCount); + uint8_t* start = reinterpret_cast + (&singletonValue(t, p->thunkTable, 0)); + + if (Verbose) { + logCompile(start, p->thunkSize * ThunkCount, 0, "thunkTable", 0); + fprintf(stderr, "thunk size: %d\n", p->thunkSize); + } + + tableContext.promise.resolved_ = true; + +#define THUNK(s) \ + tableContext.promise.value_ = reinterpret_cast(s); \ + tableContext.context.assembler->writeTo(start); \ + start += p->thunkSize; + +#include "thunks.cpp" + +#undef THUNK +} + MyProcessor* processor(MyThread* t) { @@ -5144,22 +5135,7 @@ processor(MyThread* t) set(t, p->methodTree, TreeNodeLeft, p->methodTreeSentinal); set(t, p->methodTree, TreeNodeRight, p->methodTreeSentinal); - Context context(t); - Compiler* c = context.c; - - mov(c, c->base(), c->memory(c->thread(), difference(&(t->base), t))); - mov(c, c->stack(), c->memory(c->thread(), difference(&(t->stack), t))); - - c->jmp(c->indirectTarget()); - - p->indirectCallerSize = c->codeSize(); - p->indirectCaller = static_cast - (p->codeZone.allocate(p->indirectCallerSize)); - c->writeTo(p->indirectCaller); - - if (Verbose) { - logCompile(p->indirectCaller, c->codeSize(), 0, "indirect caller", 0); - } + compileThunks(t, p); p->segFaultHandler.m = t->m; expect(t, t->m->system->success @@ -5170,15 +5146,21 @@ processor(MyThread* t) } object -defaultCompiled(MyThread* t) +defaultThunk(MyThread* t) { - return processor(t)->getDefaultCompiled(t); + return processor(t)->defaultThunk; } object -nativeCompiled(MyThread* t) +nativeThunk(MyThread* t) { - return processor(t)->getNativeCompiled(t); + return processor(t)->nativeThunk; +} + +object +aioobThunk(MyThread* t) +{ + return processor(t)->aioobThunk; } void @@ -5186,21 +5168,21 @@ compile(MyThread* t, object method) { MyProcessor* p = processor(t); - if (methodCompiled(t, method) == p->getDefaultCompiled(t)) { + if (methodCompiled(t, method) == p->defaultThunk) { PROTECT(t, method); ACQUIRE(t, t->m->classLock); - if (methodCompiled(t, method) == p->getDefaultCompiled(t)) { + if (methodCompiled(t, method) == p->defaultThunk) { initClass(t, methodClass(t, method)); if (UNLIKELY(t->exception)) return; - if (methodCompiled(t, method) == p->getDefaultCompiled(t)) { + if (methodCompiled(t, method) == p->defaultThunk) { object compiled; if (methodFlags(t, method) & ACC_NATIVE) { - compiled = p->getNativeCompiled(t); + compiled = p->nativeThunk; } else { - Context context(t, method, p->indirectCaller); + Context context(t, method); compiled = compile(t, &context); if (UNLIKELY(t->exception)) return; } @@ -5325,7 +5307,7 @@ methodTreeSentinal(MyThread* t) return processor(t)->methodTreeSentinal; } -Allocator* +Zone* codeZone(MyThread* t) { return &(processor(t)->codeZone); } diff --git a/src/compiler.cpp b/src/compiler.cpp index 3bdd19e556..9ef5968078 100644 --- a/src/compiler.cpp +++ b/src/compiler.cpp @@ -9,2910 +9,2985 @@ details. */ #include "compiler.h" -#include "vector.h" -#include "zone.h" +#include "assembler.h" using namespace vm; namespace { -enum Register { - NoRegister = -1, - rax = 0, - rcx = 1, - rdx = 2, - rbx = 3, - rsp = 4, - rbp = 5, - rsi = 6, - rdi = 7, - r8 = 8, - r9 = 9, - r10 = 10, - r11 = 11, - r12 = 12, - r13 = 13, - r14 = 14, - r15 = 15, -}; - -const bool Verbose = false; - -const unsigned RegisterCount = BytesPerWord * 2; -const unsigned GprParameterCount = 6; +const bool DebugAppend = false; +const bool DebugCompile = false; +const bool DebugStack = false; +const bool DebugRegisters = false; class Context; -class MyOperand; -class AddressOperand; -class ImmediateOperand; -class AbsoluteOperand; -class RegisterOperand; -class MemoryOperand; -class CodePromise; -class MyPromise; -class RegisterReference; +class Value; +class Stack; +class Site; +class RegisterSite; +class Event; +class PushEvent; +class Read; -int64_t FORCE_ALIGN -divideLong(int64_t a, int64_t b) -{ - return a / b; -} +void NO_RETURN abort(Context*); -int64_t FORCE_ALIGN -moduloLong(int64_t a, int64_t b) -{ - return a % b; -} +void +apply(Context* c, UnaryOperation op, unsigned size, Site* a); -inline bool -isInt8(intptr_t v) -{ - return v == static_cast(v); -} +void +apply(Context* c, BinaryOperation op, unsigned size, Site* a, Site* b); -inline bool -isInt32(intptr_t v) -{ - return v == static_cast(v); -} - -class RegisterNode { +class Site { public: - RegisterNode(Register value, RegisterNode* next): - value(value), next(next) + Site(): next(0) { } + + virtual ~Site() { } + + virtual Site* readTarget(Context*, Read*) { return this; } + + virtual unsigned copyCost(Context*, Site*) = 0; + + virtual void acquire(Context*, Stack*, unsigned, Value*) { } + + virtual void release(Context*) { } + + virtual void freeze(Context*) { } + + virtual void thaw(Context*) { } + + virtual OperandType type(Context*) = 0; + + virtual Assembler::Operand* asAssemblerOperand(Context*) = 0; + + Site* next; +}; + +class Stack { + public: + Stack(Value* value, unsigned size, unsigned index, Stack* next): + value(value), size(size), index(index), next(next), pushEvent(0), + pushSite(0), pushed(false) { } - Register value; - RegisterNode* next; + Value* value; + unsigned size; + unsigned index; + Stack* next; + PushEvent* pushEvent; + Site* pushSite; + bool pushed; }; -class Task { +class State { public: - Task(Task* next): next(next) { } - - virtual ~Task() { } - - virtual void run(Context*, unsigned) = 0; - - Task* next; -}; - -class Event { - public: - Event(Event* next): next(next), task(0) { - if (next) { - count = next->count + 1; - } else { - count = 1; - } - } - - virtual ~Event() { } - - virtual void run(Context*) { } - - Event* next; - Task* task; - unsigned count; -}; - -class Segment { - public: - Segment(int logicalIp, Event* event): - logicalIp(logicalIp), offset(-1), event(event) + State(State* next, Stack* stack): + stack(stack), + next(next) { } - int logicalIp; - int offset; - Event* event; + Stack* stack; + State* next; }; -class MyStack: public Stack { +class Local { public: - MyStack(MyOperand* value, int index, MyStack* next): - value(value), index(index), next(next) + Local(unsigned size, unsigned index, Value* value, Site* site, Local* old, + Local* next): + size(size), index(index), reuse(true), value(value), site(site), old(old), + next(next) { } - MyOperand* value; - int index; - MyStack* next; + unsigned size; + unsigned index; + bool reuse; + Value* value; + Site* site; + Local* old; + Local* next; }; -class RegisterData { +class LogicalInstruction { public: - RegisterData(): reserved(false) { } + Event* firstEvent; + Event* lastEvent; + LogicalInstruction* immediatePredecessor; + Stack* stack; + Local* locals; + unsigned machineOffset; + bool stackSaved; +}; +class Register { + public: + Register(int number): + value(0), site(0), number(number), size(0), refCount(0), + freezeCount(0), reserved(false), pushed(false) + { } + + Value* value; + RegisterSite* site; + int number; + unsigned size; + unsigned refCount; + unsigned freezeCount; bool reserved; + bool pushed; +}; + +class ConstantPoolNode { + public: + ConstantPoolNode(Promise* promise): promise(promise), next(0) { } + + Promise* promise; + ConstantPoolNode* next; +}; + +class Junction { + public: + Junction(unsigned logicalIp, Junction* next): + logicalIp(logicalIp), + next(next) + { } + + unsigned logicalIp; + Junction* next; +}; + +class Read { + public: + Read(unsigned size, Value* value, Site* target, Read* next, Event* event, + Read* eventNext): + size(size), value(value), target(target), next(next), event(event), + eventNext(eventNext) + { } + + unsigned size; + Value* value; + Site* target; + Read* next; + Event* event; + Read* eventNext; +}; + +class Value: public Compiler::Operand { + public: + Value(Site* site, Site* target): + reads(0), lastRead(0), sites(site), source(0), target(target) + { } + + Read* reads; + Read* lastRead; + Site* sites; + Site* source; + Site* target; }; class Context { public: - Context(System* s, Allocator* allocator, Zone* zone, void* indirectCaller): - s(s), - constantPool(s, allocator, BytesPerWord * 32), - plan(s, allocator, 1024), - code(s, allocator, 1024), + Context(System* system, Assembler* assembler, Zone* zone, + Compiler::Client* client): + system(system), + assembler(assembler), zone(zone), - indirectCaller(reinterpret_cast(indirectCaller)), - segmentTable(0), - reserved(0), - codeLength(-1) + client(client), + logicalIp(-1), + state(new (zone->allocate(sizeof(State))) State(0, 0)), + logicalCode(0), + logicalCodeLength(0), + parameterFootprint(0), + localFootprint(0), + registers + (static_cast + (zone->allocate(sizeof(Register*) * assembler->registerCount()))), + firstConstant(0), + lastConstant(0), + constantCount(0), + nextSequence(0), + junctions(0), + machineCode(0), + locals(0), + localTable(0), + stackReset(false) { - plan.appendAddress(new (zone->allocate(sizeof(Segment))) Segment - (-1, new (zone->allocate(sizeof(Event))) Event(0))); + for (unsigned i = 0; i < assembler->registerCount(); ++i) { + registers[i] = new (zone->allocate(sizeof(Register))) Register(i); + } - registers[rsp].reserved = true; - registers[rbp].reserved = true; - registers[rbx].reserved = true; + registers[assembler->base()]->reserved = true; + registers[assembler->stack()]->reserved = true; + registers[assembler->thread()]->reserved = true; } - void dispose() { - plan.dispose(); - code.dispose(); - constantPool.dispose(); - } - - System* s; - Vector constantPool; - Vector plan; - Vector code; + System* system; + Assembler* assembler; Zone* zone; - intptr_t indirectCaller; - Segment** segmentTable; - unsigned reserved; - int codeLength; - RegisterData registers[RegisterCount]; + Compiler::Client* client; + int logicalIp; + State* state; + LogicalInstruction* logicalCode; + unsigned logicalCodeLength; + unsigned parameterFootprint; + unsigned localFootprint; + Register** registers; + ConstantPoolNode* firstConstant; + ConstantPoolNode* lastConstant; + unsigned constantCount; + unsigned nextSequence; + Junction* junctions; + uint8_t* machineCode; + Local* locals; + Local** localTable; + bool stackReset; +}; + +class PoolPromise: public Promise { + public: + PoolPromise(Context* c, int key): c(c), key(key) { } + + virtual int64_t value() { + if (resolved()) { + return reinterpret_cast + (c->machineCode + pad(c->assembler->length()) + (key * BytesPerWord)); + } + + abort(c); + } + + virtual bool resolved() { + return c->machineCode != 0; + } + + Context* c; + int key; +}; + +class CodePromise: public Promise { + public: + CodePromise(Context* c, CodePromise* next): c(c), offset(-1), next(next) { } + + CodePromise(Context* c, int offset): c(c), offset(offset), next(0) { } + + virtual int64_t value() { + if (resolved()) { + return reinterpret_cast(c->machineCode + offset); + } + + abort(c); + } + + virtual bool resolved() { + return c->machineCode != 0 and offset >= 0; + } + + Context* c; + int offset; + CodePromise* next; +}; + +class IpPromise: public Promise { + public: + IpPromise(Context* c, int logicalIp): + c(c), + logicalIp(logicalIp) + { } + + virtual int64_t value() { + if (resolved()) { + return reinterpret_cast + (c->machineCode + c->logicalCode[logicalIp].machineOffset); + } + + abort(c); + } + + virtual bool resolved() { + return c->machineCode != 0; + } + + Context* c; + int logicalIp; }; inline void NO_RETURN abort(Context* c) { - abort(c->s); + abort(c->system); } #ifndef NDEBUG inline void assert(Context* c, bool v) { - assert(c->s, v); + assert(c->system, v); } #endif // not NDEBUG inline void expect(Context* c, bool v) { - expect(c->s, v); + expect(c->system, v); } -class MyPromise: public Promise { +class Event { public: - virtual intptr_t value(Compiler*); + Event(Context* c): + next(0), stack(c->state->stack), locals(c->locals), promises(0), reads(0), + readCount(0), sequence(c->nextSequence++), stackReset(c->stackReset) + { + assert(c, c->logicalIp >= 0); - virtual intptr_t value(Context*) = 0; + LogicalInstruction* i = c->logicalCode + c->logicalIp; + if (i->lastEvent) { + i->lastEvent->next = this; + } else { + i->firstEvent = this; + } + i->lastEvent = this; - virtual bool resolved(Context*) = 0; -}; - -class ResolvedPromise: public MyPromise { - public: - ResolvedPromise(intptr_t value): value_(value) { } - - virtual intptr_t value(Context*) { - return value_; + if (c->stackReset) { +// fprintf(stderr, "stack reset\n"); + c->stackReset = false; + } } - virtual bool resolved(Context*) { - return true; + Event(Context*, unsigned sequence, Stack* stack, Local* locals): + next(0), stack(stack), locals(locals), promises(0), reads(0), readCount(0), + sequence(sequence), stackReset(false) + { } + + virtual ~Event() { } + + virtual void compile(Context* c) = 0; + + virtual bool skipMove(unsigned) { return false; } + + Event* next; + Stack* stack; + Local* locals; + CodePromise* promises; + Read* reads; + unsigned readCount; + unsigned sequence; + bool stackReset; +}; + +int +localOffset(Context* c, int v) +{ + int parameterFootprint = c->parameterFootprint * BytesPerWord; + + v *= BytesPerWord; + if (v < parameterFootprint) { + return (parameterFootprint - v - BytesPerWord) + (BytesPerWord * 2); + } else { + return -(v + BytesPerWord - parameterFootprint); + } +} + +bool +findSite(Context*, Value* v, Site* site) +{ + for (Site* s = v->sites; s; s = s->next) { + if (s == site) return true; + } + return false; +} + +void +addSite(Context* c, Stack* stack, unsigned size, Value* v, Site* s) +{ + if (not findSite(c, v, s)) { +// fprintf(stderr, "add site %p (%d) to %p\n", s, s->type(c), v); + s->acquire(c, stack, size, v); + s->next = v->sites; + v->sites = s; + } +} + +void +removeSite(Context* c, Value* v, Site* s) +{ + for (Site** p = &(v->sites); *p;) { + if (s == *p) { +// fprintf(stderr, "remove site %p (%d) from %p\n", s, s->type(c), v); + s->release(c); + *p = (*p)->next; + break; + } else { + p = &((*p)->next); + } + } +} + +void +removeMemorySites(Context* c, Value* v) +{ + for (Site** p = &(v->sites); *p;) { + if ((*p)->type(c) == MemoryOperand) { +// fprintf(stderr, "remove site %p (%d) from %p\n", *p, (*p)->type(c), v); + (*p)->release(c); + *p = (*p)->next; + break; + } else { + p = &((*p)->next); + } + } +} + +void +clearSites(Context* c, Value* v) +{ + for (Site* s = v->sites; s; s = s->next) { + s->release(c); + } + v->sites = 0; +} + +void +nextRead(Context* c, Value* v) +{ +// fprintf(stderr, "pop read %p from %p; next: %p\n", v->reads, v, v->reads->next); + + v->reads = v->reads->next; + if (v->reads == 0) { + clearSites(c, v); + } +} + +class ConstantSite: public Site { + public: + ConstantSite(Promise* value): value(value) { } + + virtual unsigned copyCost(Context*, Site* s) { + return (s == this ? 0 : 1); } - intptr_t value_; + virtual OperandType type(Context*) { + return ConstantOperand; + } + + virtual Assembler::Operand* asAssemblerOperand(Context*) { + return &value; + } + + Assembler::Constant value; }; +ConstantSite* +constantSite(Context* c, Promise* value) +{ + return new (c->zone->allocate(sizeof(ConstantSite))) ConstantSite(value); +} + ResolvedPromise* -resolved(Context* c, intptr_t value) +resolved(Context* c, int64_t value) { return new (c->zone->allocate(sizeof(ResolvedPromise))) ResolvedPromise(value); } -class PoolPromise: public MyPromise { - public: - PoolPromise(intptr_t key): key(key) { } - - virtual intptr_t value(Context* c) { - if (resolved(c)) { - return reinterpret_cast(c->code.data + c->codeLength + key); - } - - abort(c); - } - - virtual bool resolved(Context* c) { - return c->codeLength >= 0; - } - - intptr_t key; -}; - -class CodePromise: public MyPromise { - public: - CodePromise(): - offset(-1) - { } - - virtual intptr_t value(Context* c) { - if (resolved(c)) { - return reinterpret_cast(c->code.data + offset); - } - - abort(c); - } - - virtual bool resolved(Context*) { - return offset >= 0; - } - - intptr_t offset; -}; - -class IpPromise: public MyPromise { - public: - IpPromise(intptr_t logicalIp): - logicalIp(logicalIp) - { } - - virtual intptr_t value(Context* c) { - if (resolved(c)) { - unsigned bottom = 0; - unsigned top = c->plan.length() / BytesPerWord; - for (unsigned span = top - bottom; span; span = top - bottom) { - unsigned middle = bottom + (span / 2); - Segment* s = c->segmentTable[middle]; - - if (logicalIp == s->logicalIp) { - return reinterpret_cast(c->code.data + s->offset); - } else if (logicalIp < s->logicalIp) { - top = middle; - } else if (logicalIp > s->logicalIp) { - bottom = middle + 1; - } - } - } - - abort(c); - } - - virtual bool resolved(Context* c) { - return c->codeLength >= 0; - } - - intptr_t logicalIp; -}; - -AddressOperand* -address(Context* c, MyPromise* p); - -ImmediateOperand* -immediate(Context* c, int64_t v); - -AbsoluteOperand* -absolute(Context* c, MyPromise* v); - -RegisterOperand* -register_(Context* c, RegisterReference*); - -RegisterOperand* -register_(Context* c, Register = NoRegister, Register = NoRegister); - -MemoryOperand* -memory(Context* c, MyOperand* base, int displacement, - MyOperand* index, unsigned scale); - -class MyOperand: public Operand { - public: - enum Operation { - push1, - push2, - push2z, - push4, - push8, - pop4, - pop8, - call, - alignedCall, - ret, - mov1, - mov2, - mov4, - mov8, - mov1ToW, - mov2ToW, - mov2zToW, - mov4To8, - cmp4, - cmp8, - jl, - jg, - jle, - jge, - je, - jne, - jmp, - add4, - add8, - sub4, - sub8, - mul4, - mul8, - div4, - div8, - rem4, - rem8, - shl4, - shl8, - shr4, - shr8, - ushr4, - ushr8, - and4, - and8, - or4, - or8, - xor4, - xor8, - neg4, - neg8, - addc, - subb - }; - - static const Operation push = (BytesPerWord == 8 ? push8 : push4); - static const Operation pop = (BytesPerWord == 8 ? pop8 : pop4); - static const Operation mov = (BytesPerWord == 8 ? mov8 : mov4); - static const Operation cmp = (BytesPerWord == 8 ? cmp8 : cmp4); - static const Operation add = (BytesPerWord == 8 ? add8 : add4); - static const Operation sub = (BytesPerWord == 8 ? sub8 : sub4); - static const Operation mul = (BytesPerWord == 8 ? mul8 : mul4); - static const Operation neg = (BytesPerWord == 8 ? neg8 : neg4); - - virtual ~MyOperand() { } - - virtual Register asRegister(Context* c) { abort(c); } - - virtual RegisterNode* dependencies(Context*, RegisterNode* next) - { return next; } - - virtual void release(Context*) { /* ignore */ } - - virtual void setLabelValue(Context* c, MyPromise*) { abort(c); } - - virtual void apply(Context*, Operation) = 0; - - virtual void apply(Context*, Operation, MyOperand*) = 0; - - virtual void accept(Context* c, Operation, RegisterOperand*) = 0; - - virtual void accept(Context* c, Operation, ImmediateOperand*) = 0; - - virtual void accept(Context* c, Operation, AddressOperand*) = 0; - - virtual void accept(Context* c, Operation, AbsoluteOperand*) = 0; - - virtual void accept(Context* c, Operation, MemoryOperand*) = 0; -}; - -void -acquire(Context* c, Register v) +ConstantSite* +constantSite(Context* c, int64_t value) { - assert(c, not c->registers[v].reserved); - if (Verbose) { - fprintf(stderr, "acquire %d\n", v); - } - c->registers[v].reserved = true; + return constantSite(c, resolved(c, value)); } -Register -acquire(Context* c) -{ - // we don't yet support using r9-r15 - for (int i = 8/*RegisterCount*/ - 1; i >= 0; --i) { - if (not c->registers[i].reserved) { - acquire(c, static_cast(i)); - return static_cast(i); - } +class AddressSite: public Site { + public: + AddressSite(Promise* address): address(address) { } + + virtual unsigned copyCost(Context*, Site* s) { + return (s == this ? 0 : 3); } - abort(c); + virtual OperandType type(Context*) { + return AddressOperand; + } + + virtual Assembler::Operand* asAssemblerOperand(Context*) { + return &address; + } + + Assembler::Address address; +}; + +AddressSite* +addressSite(Context* c, Promise* address) +{ + return new (c->zone->allocate(sizeof(AddressSite))) AddressSite(address); } void -release(Context* c, Register v) +freeze(Register* r) { - assert(c, c->registers[v].reserved); - if (Verbose) { - fprintf(stderr, "release %d\n", v); + if (DebugRegisters) { + fprintf(stderr, "freeze %d to %d\n", r->number, r->freezeCount + 1); } - c->registers[v].reserved = false; + + ++ r->freezeCount; } -class RegisterReference { +void +thaw(Register* r) +{ + if (DebugRegisters) { + fprintf(stderr, "thaw %d to %d\n", r->number, r->freezeCount - 1); + } + + -- r->freezeCount; +} + +Register* +acquire(Context* c, uint32_t mask, Stack* stack, unsigned newSize, + Value* newValue, RegisterSite* newSite); + +void +release(Context* c, Register* r); + +Register* +validate(Context* c, uint32_t mask, Stack* stack, unsigned size, + Value* value, RegisterSite* site, Register* current); + +class RegisterSite: public Site { public: - RegisterReference(Register value = NoRegister, Register high = NoRegister): - value_(value), defaultValue(value), high_(high), defaultHigh(high), - acquired(true) + RegisterSite(uint64_t mask, Register* low = 0, Register* high = 0): + mask(mask), low(low), high(high), register_(NoRegister, NoRegister) { } - void acquire(Context* c) { - if (defaultValue != NoRegister) { - ::acquire(c, defaultValue); + void sync(Context* c UNUSED) { + assert(c, low); + + register_.low = low->number; + register_.high = (high? high->number : NoRegister); + } + + virtual unsigned copyCost(Context* c, Site* s) { + sync(c); + + if (s and + (this == s or + (s->type(c) == RegisterOperand + and (static_cast(s)->mask + & (static_cast(1) << register_.low)) + and (register_.high == NoRegister + or (static_cast(s)->mask + & (static_cast(1) << (register_.high + 32))))))) + { + return 0; + } else { + return 2; } + } - if (defaultHigh != NoRegister) { - ::acquire(c, defaultHigh); + virtual void acquire(Context* c, Stack* stack, unsigned size, Value* v) { + low = ::validate(c, mask, stack, size, v, this, low); + if (size > BytesPerWord) { + ::freeze(low); + high = ::validate(c, mask >> 32, stack, size, v, this, high); + ::thaw(low); } - - value_ = defaultValue; - high_ = defaultHigh; - - acquired = true; - } - - void release(Context* c) { - assert(c, acquired); - - if (value_ != NoRegister) { - ::release(c, value_); - } - - if (high_ != NoRegister) { - ::release(c, high_); - } - - value_ = NoRegister; - high_ = NoRegister; - - acquired = false; - } - - Register value(Context* c) { - assert(c, acquired); - if (value_ == NoRegister) { - value_ = ::acquire(c); - } - return value_; - } - - Register high(Context* c) { - assert(c, acquired); - if (high_ == NoRegister) { - high_ = ::acquire(c); - } - return high_; - } - - Register value_; - Register defaultValue; - Register high_; - Register defaultHigh; - bool acquired; -}; - -class RegisterOperand: public MyOperand { - public: - RegisterOperand(RegisterReference* reference): - reference(reference) - { } - - Register value(Context* c) { - return reference->value(c); - } - - Register high(Context* c) { - return reference->high(c); - } - - virtual Register asRegister(Context* c) { - return value(c); - } - - virtual RegisterNode* dependencies(Context* c, RegisterNode* next) { - return new (c->zone->allocate(sizeof(RegisterNode))) - RegisterNode(value(c), next); } virtual void release(Context* c) { - reference->release(c); - } + assert(c, low); - virtual void apply(Context*, Operation); - - virtual void apply(Context* c, Operation op, MyOperand* operand) { - operand->accept(c, op, this); - } - - virtual void accept(Context*, Operation, RegisterOperand*); - virtual void accept(Context*, Operation, ImmediateOperand*); - virtual void accept(Context*, Operation, AddressOperand*); - virtual void accept(Context*, Operation, AbsoluteOperand*); - virtual void accept(Context*, Operation, MemoryOperand*); - - RegisterReference* reference; -}; - -class ImmediateOperand: public MyOperand { - public: - ImmediateOperand(int64_t value): - value(value) - { } - - virtual void apply(Context* c, Operation op); - - virtual void apply(Context* c, Operation op, MyOperand* operand) { - operand->accept(c, op, this); - } - - virtual void accept(Context* c, Operation, RegisterOperand*) { abort(c); } - virtual void accept(Context* c, Operation, ImmediateOperand*) { abort(c); } - virtual void accept(Context* c, Operation, AddressOperand*) { abort(c); } - virtual void accept(Context* c, Operation, AbsoluteOperand*) { abort(c); } - virtual void accept(Context* c, Operation, MemoryOperand*) { abort(c); } - - int64_t value; -}; - -class AddressOperand: public MyOperand { - public: - AddressOperand(MyPromise* promise): - promise(promise) - { } - - virtual Register asRegister(Context* c); - - virtual void setLabelValue(Context*, MyPromise*); - - virtual void apply(Context*, Operation); - virtual void apply(Context* c, Operation op, MyOperand* operand) { - operand->accept(c, op, this); - } - - virtual void accept(Context* c, Operation, RegisterOperand*) { abort(c); } - virtual void accept(Context* c, Operation, ImmediateOperand*) { abort(c); } - virtual void accept(Context* c, Operation, AddressOperand*) { abort(c); } - virtual void accept(Context* c, Operation, AbsoluteOperand*) { abort(c); } - virtual void accept(Context* c, Operation, MemoryOperand*) { abort(c); } - - MyPromise* promise; -}; - -class AbsoluteOperand: public MyOperand { - public: - AbsoluteOperand(MyPromise* promise): - promise(promise) - { } - - virtual Register asRegister(Context* c); - - virtual void apply(Context*, Operation); - - virtual void apply(Context* c, Operation op, MyOperand* operand) { - operand->accept(c, op, this); - } - - virtual void accept(Context* c, Operation, RegisterOperand*) { abort(c); } - virtual void accept(Context* c, Operation, ImmediateOperand*) { abort(c); } - virtual void accept(Context* c, Operation, AddressOperand*) { abort(c); } - virtual void accept(Context* c, Operation, AbsoluteOperand*) { abort(c); } - virtual void accept(Context* c, Operation, MemoryOperand*) { abort(c); } - - MyPromise* promise; -}; - -class MemoryOperand: public MyOperand { - public: - MemoryOperand(MyOperand* base, int displacement, MyOperand* index, - unsigned scale): - base(base), - displacement(displacement), - index(index), - scale(scale) - { } - - MemoryOperand* high(Context* c) { - return memory(c, base, displacement + BytesPerWord, index, scale); - } - - virtual Register asRegister(Context*); - - virtual RegisterNode* dependencies(Context* c, RegisterNode* next) { - next = base->dependencies(c, next); - if (index) { - return index->dependencies(c, next); - } else { - return next; + ::release(c, low); + if (high) { + ::release(c, high); } } - virtual void apply(Context*, Operation); + virtual void freeze(Context* c UNUSED) { + assert(c, low); - virtual void apply(Context* c, Operation op, MyOperand* operand) { - operand->accept(c, op, this); - } - - virtual void accept(Context*, Operation, RegisterOperand*); - virtual void accept(Context*, Operation, ImmediateOperand*); - virtual void accept(Context* c, Operation, AddressOperand*) { abort(c); } - virtual void accept(Context*, Operation, AbsoluteOperand*); - virtual void accept(Context*, Operation, MemoryOperand*); - - MyOperand* base; - int displacement; - MyOperand* index; - unsigned scale; -}; - -class CodePromiseTask: public Task { - public: - CodePromiseTask(CodePromise* promise, Task* next): - Task(next), promise(promise) - { } - - virtual void run(Context* c UNUSED, unsigned offset) { - promise->offset = offset; - } - - CodePromise* promise; -}; - -AddressOperand* -address(Context* c, MyPromise* p) -{ - return new (c->zone->allocate(sizeof(AddressOperand))) AddressOperand(p); -} - -ImmediateOperand* -immediate(Context* c, int64_t v) -{ - return new (c->zone->allocate(sizeof(ImmediateOperand))) - ImmediateOperand(v); -} - -AbsoluteOperand* -absolute(Context* c, MyPromise* v) -{ - return new (c->zone->allocate(sizeof(AbsoluteOperand))) AbsoluteOperand(v); -} - -RegisterOperand* -register_(Context* c, RegisterReference* r) -{ - return new (c->zone->allocate(sizeof(RegisterOperand))) - RegisterOperand(r); -} - -RegisterOperand* -register_(Context* c, Register v, Register h) -{ - RegisterReference* r = new (c->zone->allocate(sizeof(RegisterReference))) - RegisterReference(v, h); - return register_(c, r); -} - -MemoryOperand* -memory(Context* c, MyOperand* base, int displacement, - MyOperand* index, unsigned scale) -{ - return new (c->zone->allocate(sizeof(MemoryOperand))) - MemoryOperand(base, displacement, index, scale); -} - -RegisterOperand* -temporary(Context* c) -{ - return register_(c, acquire(c)); -} - -RegisterOperand* -temporary(Context* c, Register v) -{ - acquire(c, v); - return register_(c, v); -} - -RegisterOperand* -temporary(Context* c, Register v, Register h) -{ - acquire(c, v); - acquire(c, h); - return register_(c, v, h); -} - -Segment* -currentSegment(Context* c) -{ - Segment* s; c->plan.get(c->plan.length() - BytesPerWord, &s, BytesPerWord); - return s; -} - -Promise* -machineIp(Context* c) -{ - CodePromise* p = new (c->zone->allocate(sizeof(CodePromise))) CodePromise(); - - Segment* s = currentSegment(c); - s->event->task = new (c->zone->allocate(sizeof(CodePromiseTask))) - CodePromiseTask(p, s->event->task); - - return p; -} - -void -apply(Context* c, MyOperand::Operation op) -{ - switch (op) { - case MyOperand::ret: - c->code.append(0xc3); - break; - - default: abort(c); - } -} - -class OpEvent: public Event { - public: - OpEvent(MyOperand::Operation op, Event* next): - Event(next), op(op) - { } - - virtual void run(Context* c) { - apply(c, op); - } - - MyOperand::Operation op; -}; - -class UnaryOpEvent: public Event { - public: - UnaryOpEvent(MyOperand::Operation op, Operand* operand, Event* next): - Event(next), - op(op), - operand(static_cast(operand)) - { } - - virtual void run(Context* c) { - if (Verbose) { - fprintf(stderr, "unary %d\n", op); - } - operand->apply(c, op); - } - - MyOperand::Operation op; - MyOperand* operand; -}; - -class BinaryOpEvent: public Event { - public: - BinaryOpEvent(MyOperand::Operation op, Operand* a, Operand* b, - Event* next): - Event(next), - op(op), - a(static_cast(a)), - b(static_cast(b)) - { } - - virtual void run(Context* c) { - if (Verbose) { - fprintf(stderr, "binary %d\n", op); - } - a->apply(c, op, b); - } - - MyOperand::Operation op; - MyOperand* a; - MyOperand* b; -}; - -class AcquireEvent: public Event { - public: - AcquireEvent(RegisterOperand* operand, Event* next): - Event(next), - operand(operand) - { } - - virtual void run(Context* c) { - if (Verbose) { - fprintf(stderr, "acquire register\n"); - } - operand->reference->acquire(c); - } - - RegisterOperand* operand; -}; - -class ReleaseEvent: public Event { - public: - ReleaseEvent(Operand* operand, Event* next): - Event(next), - operand(static_cast(operand)) - { } - - virtual void run(Context* c) { - if (Verbose) { - fprintf(stderr, "release register\n"); - } - operand->release(c); - } - - MyOperand* operand; -}; - - -class Movement { - public: - MyOperand* source; - Register destination; - RegisterNode* dependencies; -}; - -void -push(Context* c, Movement* table, unsigned size) -{ - int pushed[size]; - unsigned pushIndex = 0; - for (unsigned i = 0; i < size; ++i) { - Movement* mi = table + i; - for (unsigned j = i + 1; j < size; ++j) { - Movement* mj = table + j; - for (RegisterNode* d = mj->dependencies; d; d = d->next) { - if (mi->destination == d->value) { - mi->source->apply(c, MyOperand::push); - pushed[pushIndex++] = i; - goto loop; - } - } - } - - mi->source->apply - (c, MyOperand::mov, register_(c, mi->destination)); - loop:; - } - - for (int i = pushIndex - 1; i >= 0; --i) { - register_(c, table[pushed[i]].destination)->apply - (c, MyOperand::pop); - } -} - -Register -gpRegister(Context* c, unsigned index) -{ - switch (index) { - case 0: - return rdi; - case 1: - return rsi; - case 2: - return rdx; - case 3: - return rcx; - case 4: - return r8; - case 5: - return r9; - default: - abort(c); - } -} - -class ArgumentEvent: public Event { - public: - ArgumentEvent(MyOperand** arguments, unsigned count, Event* next): - Event(next), - arguments(arguments), - count(count) - { } - - virtual void run(Context* c) { - if (BytesPerWord == 8) { - const unsigned size = min(count, GprParameterCount); - Movement moveTable[size]; - - for (int i = count - 1; i >= 0; --i) { - if (static_cast(i) < GprParameterCount) { - Movement* m = moveTable + (size - i - 1); - m->source = arguments[i]; - m->destination = gpRegister(c, i); - m->dependencies = arguments[i]->dependencies(c, 0); - } else { - arguments[i]->apply(c, MyOperand::push8); - } - } - - push(c, moveTable, size); - } else { - for (int i = count - 1; i >= 0; --i) { - if (i > 0 and arguments[i - 1] == 0) { - arguments[i]->apply(c, MyOperand::push8); - -- i; - } else { - arguments[i]->apply(c, MyOperand::push4); - } - } + ::freeze(low); + if (high) { + ::freeze(high); } } - MyOperand** arguments; - unsigned count; + virtual void thaw(Context* c UNUSED) { + assert(c, low); + + ::thaw(low); + if (high) { + ::thaw(high); + } + } + + virtual OperandType type(Context*) { + return RegisterOperand; + } + + virtual Assembler::Operand* asAssemblerOperand(Context* c) { + sync(c); + return ®ister_; + } + + uint64_t mask; + Register* low; + Register* high; + Assembler::Register register_; }; -void -appendOperation(Context* c, MyOperand::Operation op) +RegisterSite* +registerSite(Context* c, int low, int high = NoRegister) { - Segment* s = currentSegment(c); - s->event = new (c->zone->allocate(sizeof(OpEvent))) - OpEvent(op, s->event); -} + assert(c, low != NoRegister); + assert(c, low < static_cast(c->assembler->registerCount())); + assert(c, high == NoRegister + or high < static_cast(c->assembler->registerCount())); -void -appendOperation(Context* c, MyOperand::Operation op, Operand* operand) -{ - Segment* s = currentSegment(c); - s->event = new (c->zone->allocate(sizeof(UnaryOpEvent))) - UnaryOpEvent(op, operand, s->event); -} - -void -appendOperation(Context* c, MyOperand::Operation op, Operand* a, Operand* b) -{ - Segment* s = currentSegment(c); - s->event = new (c->zone->allocate(sizeof(BinaryOpEvent))) - BinaryOpEvent(op, a, b, s->event); -} - -void -appendAcquire(Context* c, RegisterOperand* operand) -{ - Segment* s = currentSegment(c); - s->event = new (c->zone->allocate(sizeof(AcquireEvent))) - AcquireEvent(operand, s->event); -} - -void -appendRelease(Context* c, Operand* operand) -{ - Segment* s = currentSegment(c); - s->event = new (c->zone->allocate(sizeof(ReleaseEvent))) - ReleaseEvent(operand, s->event); -} - -void -appendArgumentEvent(Context* c, MyOperand** arguments, unsigned count) -{ - Segment* s = currentSegment(c); - s->event = new (c->zone->allocate(sizeof(ArgumentEvent))) - ArgumentEvent(arguments, count, s->event); -} - -void -logStack(Context* c, MyStack* stack) -{ - fprintf(stderr, "ip %3d: ", currentSegment(c)->logicalIp); - - if (stack) { - fprintf(stderr, " %d", - static_cast(stack->value)->displacement); - } - - for (MyStack* s = stack; s; s = s->next) { - fprintf(stderr, "*"); - } - - fprintf(stderr, "\n"); -} - -MyStack* -pushed(Context* c, MyStack* stack) -{ - int index = (stack ? stack->index + 1 : 0); - - MyOperand* value = memory - (c, register_(c, rbp), - (c->reserved + index + 1) * BytesPerWord, 0, 1); - - stack = new (c->zone->allocate(sizeof(MyStack))) - MyStack(value, index, stack); - - if (Verbose) { - logStack(c, stack); - } - - return stack; -} - -MyStack* -push(Context* c, MyStack* stack, MyOperand::Operation op, MyOperand* v) -{ - appendOperation(c, op, v); - - if (BytesPerWord == 4 and op == MyOperand::push8) { - stack = pushed(c, stack); - } - return pushed(c, stack); -} - -MyStack* -pop(Context* c, MyStack* stack, int count) -{ - appendOperation - (c, MyOperand::add, - immediate(c, count * BytesPerWord), - register_(c, rsp)); - - while (count) { - -- count; - assert(c, count >= 0); - stack = stack->next; - } - - if (Verbose) { - logStack(c, stack); - } - - return stack; -} - -MyStack* -pop(Context* c, MyStack* stack, MyOperand::Operation op, MyOperand* dst) -{ - appendOperation(c, op, dst); - - if (BytesPerWord == 4 and op == MyOperand::pop8) { - stack = stack->next; - } - - if (Verbose) { - logStack(c, stack->next); - } - - return stack->next; -} - -void -pushArguments(Context* c, unsigned count, va_list list) -{ - MyOperand** arguments = static_cast - (c->zone->allocate(count * BytesPerWord)); - - unsigned index = 0; - for (unsigned i = 0; i < count; ++i) { - arguments[index] = va_arg(list, MyOperand*); - if (BytesPerWord == 4 or arguments[index]) { - ++ index; - } - } - - appendArgumentEvent(c, arguments, index); -} - -unsigned -argumentFootprint(unsigned count) -{ - if (BytesPerWord == 8) { - if (count > GprParameterCount) { - return (count - GprParameterCount) * BytesPerWord; - } else { - return 0; - } + Register* hr; + if (high == NoRegister) { + hr = 0; } else { - return count * BytesPerWord; + hr = c->registers[high]; } + return new (c->zone->allocate(sizeof(RegisterSite))) + RegisterSite(~static_cast(0), c->registers[low], hr); } -void -rex(Context* c, uint8_t mask, Register r) +RegisterSite* +freeRegisterSite(Context* c, uint64_t mask = ~static_cast(0)) { - if (BytesPerWord == 8) { - c->code.append(mask | ((r & 8) >> 3)); - } + return new (c->zone->allocate(sizeof(RegisterSite))) + RegisterSite(mask); } -void -rex(Context* c) +RegisterSite* +fixedRegisterSite(Context* c, int low, int high = NoRegister) { - rex(c, 0x48, rax); -} - -void -encode(Context* c, uint8_t* instruction, unsigned length, int a, Register b, - int32_t displacement, int index, unsigned scale) -{ - c->code.append(instruction, length); - - uint8_t width; - if (displacement == 0 and b != rbp) { - width = 0; - } else if (isInt8(displacement)) { - width = 0x40; + uint64_t mask; + if (high == NoRegister) { + mask = (~static_cast(0) << 32) + | (static_cast(1) << low); } else { - width = 0x80; + mask = (static_cast(1) << (high + 32)) + | (static_cast(1) << low); } - if (index == -1) { - c->code.append(width | (a << 3) | b); - if (b == rsp) { - c->code.append(0x24); - } - } else { - assert(c, b != rsp); - c->code.append(width | (a << 3) | 4); - c->code.append((log(scale) << 6) | (index << 3) | b); - } - - if (displacement == 0 and b != rbp) { - // do nothing - } else if (isInt8(displacement)) { - c->code.append(displacement); - } else { - c->code.append4(displacement); - } + return new (c->zone->allocate(sizeof(RegisterSite))) + RegisterSite(mask); } -void -encode(Context* c, uint8_t instruction, int a, MemoryOperand* b, bool rex) +Register* +increment(Context* c, int i) { - Register r = b->base->asRegister(c); - int index = b->index ? b->index->asRegister(c) : -1; + Register* r = c->registers[i]; - if (rex) { - ::rex(c); - } - encode(c, &instruction, 1, a, r, b->displacement, index, b->scale); -} - -void -encode2(Context* c, uint16_t instruction, int a, MemoryOperand* b, bool rex) -{ - Register r = b->base->asRegister(c); - int index = b->index ? b->index->asRegister(c) : -1; - - if (rex) { - ::rex(c); - } - uint8_t i[2] = { instruction >> 8, instruction & 0xff }; - encode(c, i, 2, a, r, b->displacement, index, b->scale); -} - -void -RegisterOperand::apply(Context* c, Operation op) -{ - switch (op) { - case call: { - if (value(c) & 8) rex(c, 0x40, value(c)); - c->code.append(0xff); - c->code.append(0xd0 | (value(c) & 7)); - } break; - - case jmp: - c->code.append(0xff); - c->code.append(0xe0 | value(c)); - break; - - case pop4: - case pop8: - if (BytesPerWord == 4 and op == pop8) { - apply(c, pop); - register_(c, high(c))->apply(c, pop); - } else { - c->code.append(0x58 | value(c)); - if (BytesPerWord == 8 and op == pop4) { - accept(c, mov4To8, this); - } - } - break; - - case push4: - case push8: - if (BytesPerWord == 4 and op == push8) { - register_(c, high(c))->apply(c, push); - apply(c, push); - } else { - c->code.append(0x50 | value(c)); - } - break; - - case neg4: - case neg8: - assert(c, BytesPerWord == 8 or op == neg4); // todo - - rex(c); - c->code.append(0xf7); - c->code.append(0xd8 | value(c)); - break; - - default: abort(c); - } -} - -void -RegisterOperand::accept(Context* c, Operation op, - RegisterOperand* operand) -{ - switch (op) { - case add: - rex(c); - c->code.append(0x01); - c->code.append(0xc0 | (operand->value(c) << 3) | value(c)); - break; - - case cmp4: - case cmp8: - if (BytesPerWord == 4 and op == cmp8) { - register_(c, high(c))->accept - (c, cmp, register_(c, operand->high(c))); - - // if the high order bits are equal, we compare the low order - // bits; otherwise, we jump past that comparison - c->code.append(0x0f); - c->code.append(0x85); // jne - c->code.append4(2); - - accept(c, cmp, operand); - } else { - if (op == cmp8) rex(c); - c->code.append(0x39); - c->code.append(0xc0 | (operand->value(c) << 3) | value(c)); - } - break; - - case mov4: - case mov8: - if (BytesPerWord == 4 and op == mov8) { - accept(c, mov, operand); - - register_(c, high(c))->accept - (c, mov, register_(c, operand->high(c))); - } else if (value(c) != operand->value(c)) { - rex(c); - c->code.append(0x89); - c->code.append(0xc0 | (operand->value(c) << 3) | value(c)); - } - break; - - case mov1ToW: - c->code.append(0xbe); - c->code.append(0xc0 | (operand->value(c) << 3) | value(c)); - break; - - case mov2ToW: - c->code.append(0xbf); - c->code.append(0xc0 | (operand->value(c) << 3) | value(c)); - break; - - case mov2zToW: - c->code.append(0xb7); - c->code.append(0xc0 | (operand->value(c) << 3) | value(c)); - break; - - case mov4To8: - assert(c, BytesPerWord == 8); - rex(c); - c->code.append(0x63); - c->code.append(0xc0 | (operand->value(c) << 3) | value(c)); - break; - - case mul4: - case mul8: - assert(c, BytesPerWord == 8 or op == mul4); // todo - - rex(c); - c->code.append(0x0f); - c->code.append(0xaf); - c->code.append(0xc0 | (value(c) << 3) | operand->value(c)); - break; - - case xor4: - rex(c); - c->code.append(0x31); - c->code.append(0xc0 | (operand->value(c) << 3) | value(c)); - break; - - default: abort(c); - } -} - -void -RegisterOperand::accept(Context* c, Operation op, - ImmediateOperand* operand) -{ - switch (op) { - case add4: - case add8: - assert(c, BytesPerWord == 8 or op == add4); // todo - - if (operand->value) { - rex(c); - if (isInt8(operand->value)) { - c->code.append(0x83); - c->code.append(0xc0 | value(c)); - c->code.append(operand->value); - } else if (isInt32(operand->value)) { - c->code.append(0x81); - c->code.append(0xc0 | value(c)); - c->code.append4(operand->value); - } else { - abort(c); - } - } - break; - - case addc: - if (isInt8(operand->value)) { - c->code.append(0x83); - c->code.append(0xd0 | value(c)); - c->code.append(operand->value); - } else { - abort(c); - } - break; - - case and4: - case and8: - assert(c, BytesPerWord == 8 or op == and4); // todo - - rex(c); - if (isInt8(operand->value)) { - c->code.append(0x83); - c->code.append(0xe0 | value(c)); - c->code.append(operand->value); - } else { - assert(c, isInt32(operand->value)); - - c->code.append(0x81); - c->code.append(0xe0 | value(c)); - c->code.append(operand->value); - } - break; - - case cmp4: - case cmp8: { - assert(c, BytesPerWord == 8 or op == cmp4); // todo - - if (op == cmp8) rex(c); - if (isInt8(operand->value)) { - c->code.append(0x83); - c->code.append(0xf8 | value(c)); - c->code.append(operand->value); - } else { - assert(c, isInt32(operand->value)); - - c->code.append(0x81); - c->code.append(0xf8 | value(c)); - c->code.append4(operand->value); - } - } break; - - case mov4: - case mov8: { - assert(c, BytesPerWord == 8 or op == mov4); // todo - - rex(c, 0x48, value(c)); - c->code.append(0xb8 | (value(c) & 7)); - c->code.appendAddress(operand->value); - } break; - - case shl4: - case shl8: { - assert(c, BytesPerWord == 8 or op == shl4); // todo - - if (operand->value) { - rex(c); - if (operand->value == 1) { - c->code.append(0xd1); - c->code.append(0xe0 | value(c)); - } else { - assert(c, isInt8(operand->value)); - - c->code.append(0xc1); - c->code.append(0xe0 | value(c)); - c->code.append(operand->value); - } - } - } break; - - case sub4: - case sub8: { - assert(c, BytesPerWord == 8 or op == sub4); // todo - - if (operand->value) { - rex(c); - if (isInt8(operand->value)) { - c->code.append(0x83); - c->code.append(0xe8 | value(c)); - c->code.append(operand->value); - } else if (isInt32(operand->value)) { - c->code.append(0x81); - c->code.append(0xe8 | value(c)); - c->code.append4(operand->value); - } else { - abort(c); - } - } - } break; - - default: abort(c); - } -} - -ImmediateOperand* -value(Context* c, AddressOperand* operand) -{ - if (c->codeLength >= 0 and operand->promise->resolved(c)) { - return immediate(c, operand->promise->value(c)); - } else { - return immediate(c, 0); - } -} - -void -RegisterOperand::accept(Context* c, Operation op, - AddressOperand* operand) -{ - switch (op) { - case mov: { - accept(c, op, ::value(c, operand)); - } break; - - default: abort(c); - } -} - -void -RegisterOperand::accept(Context* c, Operation op, - MemoryOperand* operand) -{ - switch (op) { - case cmp4: - case cmp8: - assert(c, BytesPerWord == 8 or op == cmp4); // todo - - encode(c, 0x3b, value(c), operand, true); - break; - - case mov4: - case mov8: - if (BytesPerWord == 4 and op == mov8) { - accept(c, mov, operand); - - register_(c, high(c))->accept(c, mov, operand->high(c)); - } else if (BytesPerWord == 8 and op == mov4) { - encode(c, 0x63, value(c), operand, true); - } else { - encode(c, 0x8b, value(c), operand, true); - } - break; - - case mov1ToW: - encode2(c, 0x0fbe, value(c), operand, true); - break; - - case mov2ToW: - encode2(c, 0x0fbf, value(c), operand, true); - break; - - case mov2zToW: - encode2(c, 0x0fb7, value(c), operand, true); - break; - - case mov4To8: - assert(c, BytesPerWord == 8); // todo - - encode(c, 0x63, value(c), operand, true); - break; - - case mul4: - case mul8: - assert(c, BytesPerWord == 8 or op == mul4); // todo - - encode2(c, 0x0faf, value(c), operand, true); - break; - - default: abort(c); - } -} - -ImmediateOperand* -value(Context* c, AbsoluteOperand* operand) -{ - if (c->codeLength >= 0 and operand->promise->resolved(c)) { - return immediate(c, operand->promise->value(c)); - } else { - return immediate(c, 0); - } -} - -void -RegisterOperand::accept(Context* c, Operation op, - AbsoluteOperand* operand) -{ - switch (op) { - case cmp4: - case cmp8: { - assert(c, BytesPerWord == 8 or op == cmp4); // todo - - RegisterOperand* tmp = temporary(c); - tmp->accept(c, mov, ::value(c, operand)); - accept(c, cmp, memory(c, tmp, 0, 0, 1)); - tmp->release(c); - } break; - - case mov4: - case mov8: { - assert(c, BytesPerWord == 8 or op == mov4); // todo - - accept(c, mov, ::value(c, operand)); - accept(c, mov, memory(c, this, 0, 0, 1)); - } break; - - default: abort(c); - } -} - -void -unconditional(Context* c, unsigned jump, AddressOperand* operand) -{ - intptr_t v; - if (c->codeLength >= 0 and operand->promise->resolved(c)) { - uint8_t* instruction = c->code.data + c->code.length(); - v = reinterpret_cast(operand->promise->value(c)) - - instruction - 5; - } else { - v = 0; + if (DebugRegisters) { + fprintf(stderr, "increment %d to %d\n", r->number, r->refCount + 1); } - expect(c, isInt32(v)); - - c->code.append(jump); - c->code.append4(v); -} + ++ r->refCount; -void -conditional(Context* c, unsigned condition, AddressOperand* operand) -{ - intptr_t v; - if (c->codeLength >= 0 and operand->promise->resolved(c)) { - uint8_t* instruction = c->code.data + c->code.length(); - v = reinterpret_cast(operand->promise->value(c)) - - instruction - 6; - } else { - v = 0; - } - - expect(c, isInt32(v)); - - c->code.append(0x0f); - c->code.append(condition); - c->code.append4(v); -} - -void -AddressOperand::setLabelValue(Context*, MyPromise* p) -{ - promise = p; -} - -void -AddressOperand::apply(Context* c, Operation op) -{ - switch (op) { - case alignedCall: { - if (BytesPerWord == 8) { - while ((c->code.length() + 2) % 8) { - c->code.append(0x90); - } - } else { - while ((c->code.length() + 1) % 4) { - c->code.append(0x90); - } - } - apply(c, call); - } break; - - case call: { - if (BytesPerWord == 8) { - RegisterOperand* t = temporary(c, r10); - t->accept(c, mov, this); - t->apply(c, call); - t->release(c); - } else { - unconditional(c, 0xe8, this); - } - } break; - - case jmp: - unconditional(c, 0xe9, this); - break; - - case je: - conditional(c, 0x84, this); - break; - - case jne: - conditional(c, 0x85, this); - break; - - case jg: - conditional(c, 0x8f, this); - break; - - case jge: - conditional(c, 0x8d, this); - break; - - case jl: - conditional(c, 0x8c, this); - break; - - case jle: - conditional(c, 0x8e, this); - break; - - case push4: - case push8: { - assert(c, BytesPerWord == 8 or op == push4); // todo - - RegisterOperand* tmp = temporary(c); - tmp->accept(c, mov, this); - tmp->apply(c, push); - tmp->release(c); - } break; - - default: abort(c); - } -} - -Register -AddressOperand::asRegister(Context* c) -{ - intptr_t v; - if (c->codeLength >= 0) { - v = promise->value(c); - } else { - v = 0; - } - - RegisterOperand* tmp = temporary(c); - tmp->accept(c, mov, immediate(c, v)); - Register r = tmp->value(c); - tmp->release(c); return r; } void -ImmediateOperand::apply(Context* c, Operation op) +decrement(Context* c UNUSED, Register* r) { - switch (op) { - case alignedCall: - case call: - case jmp: - address(c, resolved(c, value))->apply(c, op); - break; + assert(c, r->refCount > 0); - case push4: - case push8: - if (BytesPerWord == 4 and op == push8) { - immediate(c, (value >> 32) & 0xFFFFFFFF)->apply - (c, push); - immediate(c, (value ) & 0xFFFFFFFF)->apply - (c, push); + if (DebugRegisters) { + fprintf(stderr, "decrement %d to %d\n", r->number, r->refCount - 1); + } + + -- r->refCount; +} + +class MemorySite: public Site { + public: + MemorySite(int base, int offset, int index, unsigned scale): + base(0), index(0), value(base, offset, index, scale) + { } + + void sync(Context* c UNUSED) { + assert(c, base); + + value.base = base->number; + value.index = (index? index->number : NoRegister); + } + + virtual unsigned copyCost(Context* c, Site* s) { + sync(c); + + if (s and + (this == s or + (s->type(c) == MemoryOperand + and static_cast(s)->value.base == value.base + and static_cast(s)->value.offset == value.offset + and static_cast(s)->value.index == value.index + and static_cast(s)->value.scale == value.scale))) + { + return 0; } else { - if (isInt8(value)) { - c->code.append(0x6a); - c->code.append(value); - } else if (isInt32(value)) { - c->code.append(0x68); - c->code.append4(value); - } else { - RegisterOperand* tmp = temporary(c); - tmp->accept(c, mov, this); - tmp->apply(c, push); - tmp->release(c); + return 4; + } + } + + virtual void acquire(Context* c, Stack*, unsigned, Value*) { + base = increment(c, value.base); + if (value.index != NoRegister) { + index = increment(c, value.index); + } + } + + virtual void release(Context* c) { + decrement(c, base); + if (index) { + decrement(c, index); + } + } + + virtual OperandType type(Context*) { + return MemoryOperand; + } + + virtual Assembler::Operand* asAssemblerOperand(Context* c) { + sync(c); + return &value; + } + + Register* base; + Register* index; + Assembler::Memory value; +}; + +MemorySite* +memorySite(Context* c, int base, int offset = 0, int index = NoRegister, + unsigned scale = 1) +{ + return new (c->zone->allocate(sizeof(MemorySite))) + MemorySite(base, offset, index, scale); +} + +bool +matchRegister(Context* c UNUSED, Site* s, uint64_t mask) +{ + assert(c, s->type(c) == RegisterOperand); + + RegisterSite* r = static_cast(s); + if (r->low) { + r->sync(c); + return ((static_cast(1) << r->register_.low) & mask) + and (r->register_.high == NoRegister + or ((static_cast(1) << (r->register_.high + 32)) & mask)); + } else { + return false; + } +} + +bool +match(Context* c, Site* s, uint8_t typeMask, uint64_t registerMask) +{ + OperandType t = s->type(c); + return ((1 << t) & typeMask) + and (t != RegisterOperand or matchRegister(c, s, registerMask)); +} + +Site* +targetOrNull(Context* c, Read* r) +{ + Value* v = r->value; + if (v->target) { + return v->target; + } else if (r->target) { + return r->target->readTarget(c, r); + } else { + return 0; + } +} + +Site* +targetOrNull(Context* c, Value* v) +{ + if (v->target) { + return v->target; + } else if (v->reads and v->reads->target) { + return v->reads->target->readTarget(c, v->reads); + } else { + return 0; + } +} + +class AbstractSite: public Site { + public: + virtual unsigned copyCost(Context* c, Site*) { + abort(c); + } + + virtual void copyTo(Context* c, unsigned, Site*) { + abort(c); + } + + virtual OperandType type(Context* c) { + abort(c); + } + + virtual Assembler::Operand* asAssemblerOperand(Context* c) { + abort(c); + } +}; + +class VirtualSite: public AbstractSite { + public: + VirtualSite(Value* value, uint8_t typeMask, uint64_t registerMask): + value(value), registerMask(registerMask), typeMask(typeMask) + { } + + virtual Site* readTarget(Context* c, Read* r) { + if (value) { + Site* s = targetOrNull(c, value); + if (s and match(c, s, typeMask, registerMask)) { + return s; } } - break; - - default: abort(c); - } -} -Register -AbsoluteOperand::asRegister(Context* c) -{ - RegisterOperand* tmp = temporary(c); - tmp->accept(c, MyOperand::mov, this); - Register v = tmp->value(c); - tmp->release(c); - return v; -} - -void -absoluteApply(Context* c, MyOperand::Operation op, - AbsoluteOperand* operand) -{ - RegisterOperand* tmp = temporary(c); - tmp->accept(c, MyOperand::mov, value(c, operand)); - memory(c, tmp, 0, 0, 1)->apply(c, op); - tmp->release(c); -} - -void -AbsoluteOperand::apply(Context* c, Operation op) -{ - switch (op) { - case push: - absoluteApply(c, op, this); - break; - - default: abort(c); - } -} - -Register -MemoryOperand::asRegister(Context* c) -{ - RegisterOperand* tmp = temporary(c); - tmp->accept(c, mov, this); - Register v = tmp->value(c); - tmp->release(c); - return v; -} - -void -MemoryOperand::apply(Context* c, Operation op) -{ - switch (op) { - case call: - encode(c, 0xff, 2, this, false); - break; - - case jmp: - encode(c, 0xff, 4, this, false); - break; - - case neg4: - case neg8: - if (BytesPerWord == 4 and op == neg8) { - RegisterOperand* ax = temporary(c, rax); - RegisterOperand* dx = temporary(c, rdx); - - MemoryOperand* low = this; - MemoryOperand* high = this->high(c); - - ax->accept(c, mov, low); - dx->accept(c, mov, high); - - ax->apply(c, neg); - dx->accept(c, addc, immediate(c, 0)); - dx->apply(c, neg); - - low->accept(c, mov, ax); - high->accept(c, mov, dx); - - ax->release(c); - dx->release(c); - } else { - encode(c, 0xf7, 3, this, true); - } - break; - - case pop4: - case pop8: - if (BytesPerWord == 4 and op == pop8) { - MemoryOperand* low = this; - MemoryOperand* high = this->high(c); - - low->apply(c, pop); - high->apply(c, pop); - } else if (BytesPerWord == 8 and op == pop4) { - abort(c); - } else { - encode(c, 0x8f, 0, this, false); - } - break; - - case push4: - case push8: - if (BytesPerWord == 4 and op == push8) { - MemoryOperand* low = this; - MemoryOperand* high = this->high(c); - - high->apply(c, push); - low->apply(c, push); - } else if (BytesPerWord == 8 and op == push4) { - RegisterOperand* tmp = temporary(c); - tmp->accept(c, mov4, this); - tmp->apply(c, op); - tmp->release(c); - } else { - encode(c, 0xff, 6, this, false); - } - break; - - case push1: - case push2: - case push2z: { - RegisterOperand* tmp = temporary(c); - switch (op) { - case push1: - tmp->accept(c, mov1ToW, this); - break; - - case push2: - tmp->accept(c, mov2ToW, this); - break; - - case push2z: - tmp->accept(c, mov2zToW, this); - break; - - default: abort(c); - } - tmp->apply(c, push); - tmp->release(c); - } break; - - default: abort(c); - } -} - -void -MemoryOperand::accept(Context* c, Operation op, RegisterOperand* operand) -{ - switch (op) { - case and4: - case and8: - if (BytesPerWord == 4 and op == and8) { - accept(c, and4, operand); - high(c)->accept(c, and4, register_(c, operand->high(c))); - } else { - encode(c, 0x21, operand->value(c), this, true); - } - break; - - case add4: - case add8: - if (BytesPerWord == 4 and op == add8) { - RegisterOperand* ax = temporary(c, rax); - RegisterOperand* dx = temporary(c, rdx); - - ax->accept(c, mov, operand); - dx->accept(c, mov, register_(c, operand->high(c))); - - accept(c, add, ax); - high(c)->accept(c, addc, dx); - - ax->release(c); - dx->release(c); - } else { - encode(c, 0x01, operand->value(c), this, true); - } - break; - - case addc: - encode(c, 0x11, operand->value(c), this, true); - break; - - case div4: - case div8: - if (BytesPerWord == 4 and op == div8) { - RegisterOperand* axdx = temporary(c, rax, rdx); - - operand->apply(c, push8); - apply(c, push8); - immediate(c, reinterpret_cast(divideLong))->apply - (c, call); - register_(c, rsp)->accept - (c, add, immediate(c, 16)); - accept(c, mov8, axdx); - - axdx->release(c); - } else { - RegisterOperand* ax = temporary(c, rax); - RegisterOperand* dx = temporary(c, rdx); - ax->accept(c, mov, this); - - rex(c); - c->code.append(0x99); - rex(c); - c->code.append(0xf7); - c->code.append(0xf8 | operand->value(c)); - - accept(c, mov, ax); - - ax->release(c); - dx->release(c); - } - break; - - case mov4: - case mov8: - if (BytesPerWord == 4 and op == mov8) { - accept(c, mov, operand); - - high(c)->accept - (c, mov, register_(c, operand->high(c))); - } else if (BytesPerWord == 8 and op == mov4) { - encode(c, 0x89, operand->value(c), this, false); - } else { - encode(c, 0x89, operand->value(c), this, true); - } - break; - - case mov1: - if (BytesPerWord == 8) { - if (operand->value(c) > rbx) { - encode2(c, 0x4088, operand->value(c), this, false); - } else { - encode(c, 0x88, operand->value(c), this, false); - } - } else { - if (operand->value(c) > rbx) { - RegisterOperand* ax = temporary(c, rax); - ax->accept(c, mov, operand); - accept(c, mov1, register_(c, rax)); - ax->release(c); - } else { - encode(c, 0x88, operand->value(c), this, false); + Site* site = 0; + unsigned copyCost = 0xFFFFFFFF; + for (Site* s = r->value->sites; s; s = s->next) { + if (match(c, s, typeMask, registerMask)) { + unsigned v = s->copyCost(c, 0); + if (v < copyCost) { + site = s; + copyCost = v; + } } } - break; - case mov2: - encode2(c, 0x6689, operand->value(c), this, false); - break; - - case mov4To8: - assert(c, BytesPerWord == 8); - encode(c, 0x89, operand->value(c), this, false); - break; - - case mul4: - case mul8: - if (BytesPerWord == 4 and op == mul8) { - RegisterOperand* tmp = temporary(c, rcx); - RegisterOperand* ax = temporary(c, rax); - RegisterOperand* dx = temporary(c, rdx); - - RegisterOperand* lowSrc = operand; - RegisterOperand* highSrc = register_(c, operand->high(c)); - - MemoryOperand* lowDst = this; - MemoryOperand* highDst = this->high(c); - - tmp->accept(c, mov, highSrc); - tmp->accept(c, mul, lowDst); - ax->accept(c, mov, highDst); - ax->accept(c, mul, lowSrc); - tmp->accept(c, add, ax); - ax->accept(c, mov, lowDst); - - // mul lowSrc,%eax - c->code.append(0xf7); - c->code.append(0xe8 | lowSrc->value(c)); - - dx->accept(c, add, tmp); - - lowDst->accept(c, mov, ax); - highDst->accept(c, mov, dx); - - tmp->release(c); - ax->release(c); - dx->release(c); + if (site) { + return site; } else { - RegisterOperand* tmp = temporary(c); - - tmp->accept(c, mov, this); - tmp->accept(c, mul, operand); - accept(c, mov, tmp); - - tmp->release(c); + assert(c, typeMask & (1 << RegisterOperand)); + return freeRegisterSite(c, registerMask); } - break; + } - case or4: - case or8: - if (BytesPerWord == 4 and op == or8) { - accept(c, or4, operand); - high(c)->accept(c, or4, register_(c, operand->high(c))); + Value* value; + uint64_t registerMask; + uint8_t typeMask; +}; + +VirtualSite* +virtualSite(Context* c, Value* v = 0, + uint8_t typeMask = ~static_cast(0), + uint64_t registerMask = ~static_cast(0)) +{ + return new (c->zone->allocate(sizeof(VirtualSite))) + VirtualSite(v, typeMask, registerMask); +} + +VirtualSite* +anyRegisterSite(Context* c) +{ + return virtualSite(c, 0, 1 << RegisterOperand, ~static_cast(0)); +} + +VirtualSite* +registerOrConstantSite(Context* c) +{ + return virtualSite(c, 0, (1 << RegisterOperand) | (1 << ConstantOperand), + ~static_cast(0)); +} + +Site* +targetOrRegister(Context* c, Value* v) +{ + Site* s = targetOrNull(c, v); + if (s) { + return s; + } else { + return freeRegisterSite(c); + } +} + +Site* +pick(Context* c, Site* sites, Site* target = 0, unsigned* cost = 0) +{ + Site* site = 0; + unsigned copyCost = 0xFFFFFFFF; + for (Site* s = sites; s; s = s->next) { + unsigned v = s->copyCost(c, target); + if (v < copyCost) { + site = s; + copyCost = v; + } + } + + if (cost) *cost = copyCost; + return site; +} + +unsigned +stackOffset(Context* c) +{ + return c->localFootprint - c->parameterFootprint; +} + +Site* +pushSite(Context* c, unsigned index) +{ + return memorySite + (c, c->assembler->base(), + - (stackOffset(c) + index + 1) * BytesPerWord, NoRegister, 1); +} + +void +pushNow(Context* c, Stack* start, unsigned count) +{ + Stack* segment[count]; + unsigned index = count; + for (Stack* s = start; s and index; s = s->next) { + segment[--index] = s; + } + + for (unsigned i = 0; i < count; ++i) { + Stack* s = segment[i]; + assert(c, not s->pushed); + + if (s->value and s->value->sites) { + Site* source = pick(c, s->value->sites); + + removeMemorySites(c, s->value); + + s->pushSite = pushSite(c, s->index); + addSite(c, 0, s->size * BytesPerWord, s->value, s->pushSite); + + apply(c, Push, s->size * BytesPerWord, source); } else { - encode(c, 0x09, operand->value(c), this, true); + Assembler::Register stack(c->assembler->stack()); + Assembler::Constant offset(resolved(c, s->size * BytesPerWord)); + c->assembler->apply + (Subtract, BytesPerWord, ConstantOperand, &offset, + RegisterOperand, &stack); } - break; - case rem4: - case rem8: - if (BytesPerWord == 4 and op == rem8) { - RegisterOperand* axdx = temporary(c, rax, rdx); - - operand->apply(c, push8); - apply(c, push8); - immediate(c, reinterpret_cast(moduloLong))->apply - (c, call); - register_(c, rsp)->accept - (c, add, immediate(c, 16)); - accept(c, mov8, axdx); - - axdx->release(c); - } else { - RegisterOperand* ax = temporary(c, rax); - RegisterOperand* dx = temporary(c, rdx); - ax->accept(c, mov, this); - - rex(c); - c->code.append(0x99); - rex(c); - c->code.append(0xf7); - c->code.append(0xf8 | operand->value(c)); - - accept(c, mov, dx); - - ax->release(c); - dx->release(c); + if (DebugStack) { + fprintf(stderr, "pushed %p value: %p sites: %p\n", + s, s->value, s->value->sites); } - break; - case shl4: - case shl8: { - if (BytesPerWord == 4 and op == shl8) { - RegisterOperand* count = temporary(c, rcx); - RegisterOperand* low = temporary(c); - RegisterOperand* high = temporary(c); - - count->accept(c, mov, operand); - low->accept(c, mov, this); - high->accept(c, mov, this->high(c)); - - // shld - c->code.append(0x0f); - c->code.append(0xa5); - c->code.append(0xc0 | (low->value(c) << 3) | high->value(c)); - - // shl - c->code.append(0xd3); - c->code.append(0xe0 | low->value(c)); - - count->accept(c, cmp, immediate(c, 32)); - c->code.append(0x0f); - c->code.append(0x8c); // jl - c->code.append4(2 + 2); - - high->accept(c, mov, low); // 2 bytes - low->accept(c, xor4, low); // 2 bytes - - this->accept(c, mov, low); - this->high(c)->accept(c, mov, high); - - high->release(c); - low->release(c); - count->release(c); - } else { - RegisterOperand* cx = temporary(c, rcx); - cx->accept(c, mov, operand); - encode(c, 0xd3, 4, this, true); - cx->release(c); - } - } break; - - case shr4: - case shr8: { - if (BytesPerWord == 4 and op == shr8) { - RegisterOperand* count = temporary(c, rcx); - RegisterOperand* low = temporary(c); - RegisterOperand* high = temporary(c); - - count->accept(c, mov, operand); - low->accept(c, mov, this); - high->accept(c, mov, this->high(c)); - - // shrd - c->code.append(0x0f); - c->code.append(0xad); - c->code.append(0xc0 | (high->value(c) << 3) | low->value(c)); - - // sar - c->code.append(0xd3); - c->code.append(0xf8 | high->value(c)); - - count->accept(c, cmp, immediate(c, 32)); - c->code.append(0x0f); - c->code.append(0x8c); // jl - c->code.append4(2 + 3); - - low->accept(c, mov, high); // 2 bytes - // sar 31,high - c->code.append(0xc1); - c->code.append(0xf8 | high->value(c)); - c->code.append(31); - - this->accept(c, mov, low); - this->high(c)->accept(c, mov, high); - - high->release(c); - low->release(c); - count->release(c); - } else { - RegisterOperand* cx = temporary(c, rcx); - cx->accept(c, mov, operand); - encode(c, 0xd3, 7, this, true); - cx->release(c); - } - } break; - - case ushr4: - case ushr8: { - if (BytesPerWord == 4 and op == ushr8) { - RegisterOperand* count = temporary(c, rcx); - RegisterOperand* low = temporary(c); - RegisterOperand* high = temporary(c); - - count->accept(c, mov, operand); - low->accept(c, mov, this); - high->accept(c, mov, this->high(c)); - - // shld - c->code.append(0x0f); - c->code.append(0xa5); - c->code.append(0xc0 | (high->value(c) << 3) | low->value(c)); - - // shr - c->code.append(0xd3); - c->code.append(0xe8 | high->value(c)); - - count->accept(c, cmp, immediate(c, 32)); - c->code.append(0x0f); - c->code.append(0x8c); // jl - c->code.append4(2 + 2); - - low->accept(c, mov, high); // 2 bytes - high->accept(c, xor4, high); // 2 bytes - - this->accept(c, mov, low); - this->high(c)->accept(c, mov, high); - - high->release(c); - low->release(c); - count->release(c); - } else { - RegisterOperand* cx = temporary(c, rcx); - cx->accept(c, mov, operand); - encode(c, 0xd3, 5, this, true); - cx->release(c); - } - } break; - - case sub4: - case sub8: - if (BytesPerWord == 4 and op == sub8) { - RegisterOperand* ax = temporary(c, rax); - RegisterOperand* dx = temporary(c, rdx); - - ax->accept(c, mov, operand); - dx->accept(c, mov, register_(c, operand->high(c))); - - accept(c, sub, ax); - high(c)->accept(c, subb, dx); - - ax->release(c); - dx->release(c); - } else { - encode(c, 0x29, operand->value(c), this, true); - } - break; - - case subb: - encode(c, 0x19, operand->value(c), this, true); - break; - - case xor4: - case xor8: { - if (BytesPerWord == 4 and op == xor8) { - accept(c, xor4, operand); - high(c)->accept(c, xor4, register_(c, operand->high(c))); - } else { - encode(c, 0x31, operand->value(c), this, true); - } - } break; - - default: abort(c); + s->pushed = true; } } void -MemoryOperand::accept(Context* c, Operation op, - ImmediateOperand* operand) +pushNow(Context* c, Stack* start) { - switch (op) { - case add4: - case add8: { - assert(c, BytesPerWord == 8 or op == add4); // todo + unsigned count = 0; + for (Stack* s = start; s and (not s->pushed); s = s->next) { + ++ count; + } - unsigned i = (isInt8(operand->value) ? 0x83 : 0x81); + pushNow(c, start, count); +} - encode(c, i, 0, this, true); - if (isInt8(operand->value)) { - c->code.append(operand->value); - } else if (isInt32(operand->value)) { - c->code.append4(operand->value); +bool +trySteal(Context* c, Register* r, Stack* stack) +{ + assert(c, r->refCount == 0); + + Value* v = r->value; + + if (DebugRegisters) { + fprintf(stderr, "try steal %d from %p: next: %p\n", + r->number, v, v->sites->next); + } + + if (v->sites->next == 0) { + unsigned count = 0; + Stack* start = 0; + for (Stack* s = stack; s and (not s->pushed); s = s->next) { + if (s->value == v) { + start = s; + } + if (start) { + ++ count; + } + } + + if (start) { + if (DebugRegisters) { + fprintf(stderr, "push %p\n", v); + } + pushNow(c, start, count); + } else { + if (DebugRegisters) { + fprintf(stderr, "unable to steal %d from %p\n", r->number, v); + } + return false; + } + } + + removeSite(c, v, r->site); + + return true; +} + +bool +used(Context* c, Register* r) +{ + Value* v = r->value; + return v and findSite(c, v, r->site); +} + +bool +usedExclusively(Context* c, Register* r) +{ + return used(c, r) and r->value->sites->next == 0; +} + +unsigned +registerCost(Context* c, Register* r) +{ + if (r->reserved or r->freezeCount) { + return 6; + } + + unsigned cost = 0; + + if (used(c, r)) { + ++ cost; + if (usedExclusively(c, r)) { + cost += 2; + } + } + + if (r->refCount) { + cost += 2; + } + + return cost; +} + +Register* +pickRegister(Context* c, uint32_t mask) +{ + Register* register_ = 0; + unsigned cost = 5; + for (int i = c->assembler->registerCount() - 1; i >= 0; --i) { + if ((1 << i) & mask) { + Register* r = c->registers[i]; + if ((static_cast(1) << i) == mask) { + return r; + } + + unsigned myCost = registerCost(c, r); + if (myCost < cost) { + register_ = r; + cost = myCost; + } + } + } + + expect(c, register_); + + return register_; +} + +void +swap(Context* c, Register* a, Register* b) +{ + assert(c, a != b); + assert(c, a->number != b->number); + + Assembler::Register ar(a->number); + Assembler::Register br(b->number); + c->assembler->apply + (Swap, BytesPerWord, RegisterOperand, &ar, RegisterOperand, &br); + + c->registers[a->number] = b; + c->registers[b->number] = a; + + int t = a->number; + a->number = b->number; + b->number = t; +} + +Register* +replace(Context* c, Stack* stack, Register* r) +{ + uint32_t mask = (r->freezeCount? r->site->mask : ~0); + + freeze(r); + Register* s = acquire(c, mask, stack, r->size, r->value, r->site); + thaw(r); + + if (DebugRegisters) { + fprintf(stderr, "replace %d with %d\n", r->number, s->number); + } + + swap(c, r, s); + + return s; +} + +Register* +acquire(Context* c, uint32_t mask, Stack* stack, unsigned newSize, + Value* newValue, RegisterSite* newSite) +{ + Register* r = pickRegister(c, mask); + + if (r->reserved) return r; + + if (DebugRegisters) { + fprintf(stderr, "acquire %d, value %p, site %p freeze count %d ref count %d used %d used exclusively %d\n", + r->number, newValue, newSite, r->freezeCount, r->refCount, used(c, r), usedExclusively(c, r)); + } + + if (r->refCount) { + r = replace(c, stack, r); + } else { + Value* oldValue = r->value; + if (oldValue + and oldValue != newValue + and findSite(c, oldValue, r->site)) + { + if (not trySteal(c, r, stack)) { + r = replace(c, stack, r); + } + } + } + + r->size = newSize; + r->value = newValue; + r->site = newSite; + + return r; +} + +void +release(Context*, Register* r) +{ + if (DebugRegisters) { + fprintf(stderr, "release %d\n", r->number); + } + + r->size = 0; + r->value = 0; + r->site = 0; +} + +Register* +validate(Context* c, uint32_t mask, Stack* stack, unsigned size, + Value* value, RegisterSite* site, Register* current) +{ + if (current and (mask & (1 << current->number))) { + if (current->reserved or current->value == value) { + return current; + } + + if (current->value == 0) { + current->size = size; + current->value = value; + current->site = site; + return current; } else { abort(c); } - } break; - - case mov4: - case mov8: { - assert(c, isInt32(operand->value)); // todo - assert(c, BytesPerWord == 8 or op == mov4); // todo - - encode(c, 0xc7, 0, this, true); - c->code.append4(operand->value); - } break; - - default: abort(c); } -} -void -MemoryOperand::accept(Context* c, Operation op, - AbsoluteOperand* operand) -{ - RegisterOperand* tmp = temporary(c); + Register* r = acquire(c, mask, stack, size, value, site); + + if (current and current != r) { + release(c, current); - tmp->accept(c, mov, operand); - accept(c, op, tmp); + Assembler::Register rr(r->number); + Assembler::Register cr(current->number); + c->assembler->apply + (Move, BytesPerWord, RegisterOperand, &cr, RegisterOperand, &rr); + } - tmp->release(c); + return r; } void -MemoryOperand::accept(Context* c, Operation op, - MemoryOperand* operand) +apply(Context* c, UnaryOperation op, unsigned size, Site* a) { - switch (op) { - case mov1ToW: - case mov2ToW: - case mov2zToW: - case mov4To8: { - if (BytesPerWord == 4 and op == mov4To8) { - RegisterOperand* ax = temporary(c, rax); - RegisterOperand* dx = temporary(c, rdx); - - ax->accept(c, mov4, operand); - c->code.append(0x99); // cdq - accept(c, mov8, register_(c, rax, rdx)); - - ax->release(c); - dx->release(c); + OperandType type = a->type(c); + Assembler::Operand* operand = a->asAssemblerOperand(c); + + c->assembler->apply(op, size, type, operand); +} + +void +apply(Context* c, BinaryOperation op, unsigned size, Site* a, Site* b) +{ + OperandType aType = a->type(c); + Assembler::Operand* aOperand = a->asAssemblerOperand(c); + + OperandType bType = b->type(c); + Assembler::Operand* bOperand = b->asAssemblerOperand(c); + + c->assembler->apply(op, size, aType, aOperand, bType, bOperand); +} + +void +insertRead(Context* c, Event* event, int sequence, Value* v, + unsigned size, Site* target) +{ + Read* r = new (c->zone->allocate(sizeof(Read))) + Read(size, v, target, 0, event, event->reads); + event->reads = r; + ++ event->readCount; + + // fprintf(stderr, "add read %p to %p\n", r, v); + + if (sequence >= 0) { + for (Read** p = &(v->reads); *p;) { + if ((*p)->event->sequence > static_cast(sequence)) { + r->next = *p; + *p = r; + break; + } else { + p = &((*p)->next); + } + } + } + + if (r->next == 0) { + if (v->lastRead) { + v->lastRead->next = r; } else { - RegisterOperand* tmp = temporary(c); - tmp->accept(c, op, operand); - accept(c, mov, tmp); - tmp->release(c); + v->reads = r; } - } break; - - case mov4: - case mov8: - case and4: { - RegisterOperand* tmp = temporary(c); - tmp->accept(c, mov, operand); - accept(c, op, tmp); - tmp->release(c); - } break; - - default: abort(c); + v->lastRead = r; } } -int -compareSegmentPointers(const void* a, const void* b) -{ - return (*static_cast(a))->logicalIp - - (*static_cast(b))->logicalIp; -} - void -writeCode(Context* c) +addRead(Context* c, Value* v, unsigned size, Site* target) { - unsigned tableSize = (c->plan.length() / BytesPerWord); + insertRead(c, c->logicalCode[c->logicalIp].lastEvent, -1, v, size, target); +} - if (c->codeLength < 0) { - c->segmentTable = static_cast - (c->zone->allocate(c->plan.length())); - - for (unsigned i = 0; i < tableSize; ++i) { - c->plan.get(i * BytesPerWord, c->segmentTable + i, BytesPerWord); - } - - qsort(c->segmentTable, tableSize, BytesPerWord, compareSegmentPointers); +Site* +pushSite(Context*, PushEvent*); + +class PushEvent: public Event { + public: + PushEvent(Context* c, Stack* s): + Event(c), s(s), active(false) + { + assert(c, s->pushEvent == 0); + + s->pushEvent = this; + addRead(c, s->value, s->size * BytesPerWord, pushSite(c, this)); } - for (unsigned i = 0; i < tableSize; ++i) { - Segment* s = c->segmentTable[i]; - if (Verbose) { - fprintf(stderr, "\nip %d\n", s->logicalIp); + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "PushEvent.compile active: %d\n", active); } - if (c->codeLength >= 0) { - assert(c, s->offset == static_cast(c->code.length())); + if (active) { + pushNow(c, s); + } + + nextRead(c, s->value); + } + + virtual bool skipMove(unsigned size) { + return active and size >= BytesPerWord; + } + + Stack* s; + bool active; +}; + +void +push(Context* c, unsigned size, Value* v); + +void +ignore(Context* c, unsigned count) +{ + if (count) { + Assembler::Register stack(c->assembler->stack()); + Assembler::Constant offset(resolved(c, count * BytesPerWord)); + c->assembler->apply + (Add, BytesPerWord, ConstantOperand, &offset, RegisterOperand, &stack); + } +} + +void +cleanStack(Context* c, Stack* stack, Local* locals, Read* reads) +{ + for (Local* l = locals; l; l = l->next) { + l->reuse = false; + } + + for (Stack* s = stack; s; s = s->next) { + clearSites(c, s->value); + } + + for (Stack* s = stack; s; s = s->next) { + if (s->pushSite) { + addSite(c, 0, s->size * BytesPerWord, s->value, s->pushSite); + } + } + + for (Read* r = reads; r; r = r->eventNext) { + nextRead(c, r->value); + } +} + +void +resetLocals(Context* c) +{ + for (Local* l = c->locals; l; l = l->next) { + c->localTable[l->index] = 0; + } + c->locals = 0; +} + +CodePromise* +codePromise(Context* c, Event* e) +{ + return e->promises = new (c->zone->allocate(sizeof(CodePromise))) + CodePromise(c, e->promises); +} + +CodePromise* +codePromise(Context* c, int offset) +{ + return new (c->zone->allocate(sizeof(CodePromise))) CodePromise(c, offset); +} + +class CallEvent: public Event { + public: + CallEvent(Context* c, Value* address, unsigned flags, + TraceHandler* traceHandler, Value* result, unsigned resultSize, + Stack* argumentStack, unsigned argumentCount): + Event(c), + address(address), + traceHandler(traceHandler), + result(result), + flags(flags), + resultSize(resultSize), + argumentFootprint(0) + { + uint32_t mask = ~0; + Stack* s = argumentStack; + unsigned index = 0; + for (unsigned i = 0; i < argumentCount; ++i) { + Site* target; + if (index < c->assembler->argumentRegisterCount()) { + int r = c->assembler->argumentRegister(index); + target = fixedRegisterSite(c, r); + mask &= ~(1 << r); + } else { + target = 0; + s->pushEvent->active = true; + argumentFootprint += s->size; + } + addRead(c, s->value, s->size * BytesPerWord, target); + index += s->size; + s = s->next; + } + + addRead(c, address, BytesPerWord, virtualSite + (c, 0, ~0, (static_cast(mask) << 32) | mask)); + + for (Stack* s = stack; s; s = s->next) { + s->pushEvent->active = true; + addRead(c, s->value, s->size * BytesPerWord, virtualSite + (c, 0, ~0, (static_cast(mask) << 32) | mask)); + } + + resetLocals(c); + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "CallEvent.compile\n"); + } + + pushNow(c, stack); + + UnaryOperation type = ((flags & Compiler::Aligned) ? AlignedCall : Call); + apply(c, type, BytesPerWord, address->source); + + if (traceHandler) { + traceHandler->handleTrace(codePromise(c, c->assembler->length())); + } + + cleanStack(c, stack, locals, reads); + + if (resultSize and result->reads) { + addSite(c, 0, resultSize, result, registerSite + (c, c->assembler->returnLow(), + resultSize > BytesPerWord ? + c->assembler->returnHigh() : NoRegister)); + } + + if (argumentFootprint and ((flags & Compiler::NoReturn) == 0)) { + ignore(c, argumentFootprint); + } + } + + Value* address; + TraceHandler* traceHandler; + Value* result; + unsigned flags; + unsigned resultSize; + unsigned argumentFootprint; +}; + +void +appendCall(Context* c, Value* address, unsigned flags, + TraceHandler* traceHandler, Value* result, unsigned resultSize, + Stack* argumentStack, unsigned argumentCount) +{ + if (DebugAppend) { + fprintf(stderr, "appendCall\n"); + } + + new (c->zone->allocate(sizeof(CallEvent))) + CallEvent(c, address, flags, traceHandler, result, + resultSize, argumentStack, argumentCount); +} + +class ReturnEvent: public Event { + public: + ReturnEvent(Context* c, unsigned size, Value* value): + Event(c), value(value) + { + if (value) { + addRead(c, value, size, fixedRegisterSite + (c, c->assembler->returnLow(), + size > BytesPerWord ? + c->assembler->returnHigh() : NoRegister)); + } + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "ReturnEvent.compile\n"); + } + + if (value) { + nextRead(c, value); + } + + Assembler::Register base(c->assembler->base()); + Assembler::Register stack(c->assembler->stack()); + + c->assembler->apply(Move, BytesPerWord, RegisterOperand, &base, + RegisterOperand, &stack); + c->assembler->apply(Pop, BytesPerWord, RegisterOperand, &base); + c->assembler->apply(Return); + } + + Value* value; +}; + +void +appendReturn(Context* c, unsigned size, Value* value) +{ + if (DebugAppend) { + fprintf(stderr, "appendReturn\n"); + } + + new (c->zone->allocate(sizeof(ReturnEvent))) ReturnEvent(c, size, value); +} + +class MoveEvent: public Event { + public: + MoveEvent(Context* c, BinaryOperation type, unsigned size, Value* src, + Value* dst, Site* srcTarget, VirtualSite* dstTarget): + Event(c), type(type), size(size), src(src), dst(dst), dstTarget(dstTarget) + { + addRead(c, src, size, srcTarget); + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "MoveEvent.compile\n"); + } + + bool isLoad = src->reads->next == 0; + bool isStore = dst->reads == 0; + + Site* target; + unsigned cost; + if (type == Move + and dst->reads + and next == dst->reads->event + and dst->reads->event->skipMove(size)) + { + target = src->source; + cost = 0; } else { - s->offset = c->code.length(); - } - - Event* events[s->event->count]; - unsigned ei = s->event->count; - for (Event* e = s->event; e; e = e->next) { - events[--ei] = e; + target = targetOrRegister(c, dst); + cost = src->source->copyCost(c, target); + if (cost == 0 and (isLoad or isStore)) { + target = src->source; + } } - for (unsigned ei = 0; ei < s->event->count; ++ei) { - if (Verbose and ei) { - fprintf(stderr, "address %p\n", c->code.data + c->code.length()); + assert(c, isLoad or isStore or target != src->source); + + if (target == src->source) { + removeSite(c, src, target); + } + + if (not isStore) { + addSite(c, stack, size, dst, target); + } + + if (cost or type != Move) { + if (match(c, target, dstTarget->typeMask, dstTarget->registerMask)) { + apply(c, type, size, src->source, target); + } else { + assert(c, dstTarget->typeMask & (1 << RegisterOperand)); + + Site* tmpTarget = freeRegisterSite(c, dstTarget->registerMask); + + addSite(c, stack, size, dst, tmpTarget); + + apply(c, type, size, src->source, tmpTarget); + + if (isStore) { + removeSite(c, dst, tmpTarget); + + apply(c, Move, size, tmpTarget, target); + } else { + removeSite(c, dst, target); + } + } + } + + if (isStore) { + removeSite(c, dst, target); + } + + nextRead(c, src); + } + + BinaryOperation type; + unsigned size; + Value* src; + Value* dst; + VirtualSite* dstTarget; +}; + +void +appendMove(Context* c, BinaryOperation type, unsigned size, Value* src, + Value* dst) +{ + if (DebugAppend) { + fprintf(stderr, "appendMove\n"); + } + + VirtualSite* srcTarget = virtualSite(c, dst); + VirtualSite* dstTarget = virtualSite(c); + bool thunk; + + c->assembler->plan(type, size, + &(srcTarget->typeMask), &(srcTarget->registerMask), + &(dstTarget->typeMask), &(dstTarget->registerMask), + &thunk); + + assert(c, not thunk); // todo + + new (c->zone->allocate(sizeof(MoveEvent))) + MoveEvent(c, type, size, src, dst, srcTarget, dstTarget); +} + +class CompareEvent: public Event { + public: + CompareEvent(Context* c, unsigned size, Value* first, Value* second, + Site* firstTarget, Site* secondTarget): + Event(c), size(size), first(first), second(second) + { + addRead(c, first, size, firstTarget); + addRead(c, second, size, secondTarget); + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "CompareEvent.compile\n"); + } + + apply(c, Compare, size, first->source, second->source); + + nextRead(c, first); + nextRead(c, second); + } + + unsigned size; + Value* first; + Value* second; +}; + +void +appendCompare(Context* c, unsigned size, Value* first, Value* second) +{ + VirtualSite* firstTarget = virtualSite(c); + VirtualSite* secondTarget = virtualSite(c); + bool thunk; + + c->assembler->plan(Compare, size, + &(firstTarget->typeMask), &(firstTarget->registerMask), + &(secondTarget->typeMask), &(secondTarget->registerMask), + &thunk); + + assert(c, not thunk); // todo + + if (DebugAppend) { + fprintf(stderr, "appendCompare\n"); + } + + new (c->zone->allocate(sizeof(CompareEvent))) + CompareEvent(c, size, first, second, firstTarget, secondTarget); +} + +void +preserve(Context* c, Stack* stack, unsigned size, Value* v, Site* s, + Read* read) +{ + assert(c, v->sites == s); + Site* r = targetOrNull(c, read); + if (r == 0 or r == s) r = freeRegisterSite(c); + addSite(c, stack, size, v, r); + apply(c, Move, size, s, r); +} + +void +maybePreserve(Context* c, Stack* stack, unsigned size, Value* v, Site* s) +{ + if (v->reads->next and v->sites->next == 0) { + preserve(c, stack, size, v, s, v->reads->next); + } +} + +class CombineEvent: public Event { + public: + CombineEvent(Context* c, BinaryOperation type, unsigned size, Value* first, + Value* second, Value* result, Site* firstTarget, + Site* secondTarget): + Event(c), type(type), size(size), first(first), second(second), + result(result) + { + // todo: we should really specify the sizes of each operand + // seperately for binary operations. The following is a hack + // until then. + unsigned firstSize; + switch (type) { + case ShiftLeft: + case ShiftRight: + case UnsignedShiftRight: + firstSize = 4; + break; + + default: + firstSize = size; + break; + } + + addRead(c, first, firstSize, firstTarget); + addRead(c, second, size, secondTarget); + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "CombineEvent.compile\n"); + } + + maybePreserve(c, stack, size, second, second->source); + + apply(c, type, size, first->source, second->source); + + nextRead(c, first); + nextRead(c, second); + + removeSite(c, second, second->source); + if (result->reads) { + addSite(c, 0, 0, result, second->source); + } + } + + BinaryOperation type; + unsigned size; + Value* first; + Value* second; + Value* result; +}; + +void +appendStackSync(Context* c); + +Value* +value(Context* c, Site* site = 0, Site* target = 0) +{ + return new (c->zone->allocate(sizeof(Value))) Value(site, target); +} + +void +appendCombine(Context* c, BinaryOperation type, unsigned size, Value* first, + Value* second, Value* result) +{ + VirtualSite* firstTarget = virtualSite(c); + VirtualSite* secondTarget = virtualSite(c, result); + bool thunk; + + c->assembler->plan(type, size, + &(firstTarget->typeMask), &(firstTarget->registerMask), + &(secondTarget->typeMask), &(secondTarget->registerMask), + &thunk); + + if (thunk) { + secondTarget->value = 0; + + Stack* oldStack = c->state->stack; + + ::push(c, size, second); + ::push(c, size, first); + + Stack* argumentStack = c->state->stack; + c->state->stack = oldStack; + + appendCall(c, value(c, constantSite(c, c->client->getThunk(type, size))), + 0, 0, result, size, argumentStack, 2); + } else { + if (DebugAppend) { + fprintf(stderr, "appendCombine\n"); + } + + firstTarget->typeMask &= ~(1 << MemoryOperand); + secondTarget->typeMask &= ~(1 << MemoryOperand); + + new (c->zone->allocate(sizeof(CombineEvent))) + CombineEvent(c, type, size, first, second, result, firstTarget, + secondTarget); + } +} + +class TranslateEvent: public Event { + public: + TranslateEvent(Context* c, UnaryOperation type, unsigned size, Value* value, + Value* result, Site* target): + Event(c), type(type), size(size), value(value), result(result) + { + addRead(c, value, size, target); + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "TranslateEvent.compile\n"); + } + + maybePreserve(c, stack, size, value, value->source); + + apply(c, type, size, value->source); + + nextRead(c, value); + + removeSite(c, value, value->source); + if (result->reads) { + addSite(c, 0, 0, result, value->source); + } + } + + UnaryOperation type; + unsigned size; + Value* value; + Value* result; +}; + +void +appendTranslate(Context* c, UnaryOperation type, unsigned size, Value* value, + Value* result) +{ + if (DebugAppend) { + fprintf(stderr, "appendTranslate\n"); + } + + VirtualSite* target = virtualSite(c, result); + bool thunk; + + c->assembler->plan + (type, size, &(target->typeMask), &(target->registerMask), &thunk); + + assert(c, not thunk); // todo + + target->typeMask &= ~(1 << MemoryOperand); + + new (c->zone->allocate(sizeof(TranslateEvent))) + TranslateEvent(c, type, size, value, result, target); +} + +class MemoryEvent: public Event { + public: + MemoryEvent(Context* c, Value* base, int displacement, Value* index, + unsigned scale, Value* result): + Event(c), base(base), displacement(displacement), index(index), + scale(scale), result(result) + { + addRead(c, base, BytesPerWord, anyRegisterSite(c)); + if (index) addRead(c, index, BytesPerWord, anyRegisterSite(c)); + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "MemoryEvent.compile\n"); + } + + int indexRegister; + if (index) { + assert(c, index->source->type(c) == RegisterOperand); + indexRegister = static_cast(index->source)->register_.low; + } else { + indexRegister = NoRegister; + } + assert(c, base->source->type(c) == RegisterOperand); + int baseRegister = static_cast(base->source)->register_.low; + + nextRead(c, base); + if (index) { + if (BytesPerWord == 8) { + apply(c, Move4To8, 8, index->source, index->source); } - events[ei]->run(c); + nextRead(c, index); + } - if (c->codeLength < 0) { - for (Task* t = events[ei]->task; t; t = t->next) { - t->run(c, c->code.length()); + result->target = memorySite + (c, baseRegister, displacement, indexRegister, scale); + addSite(c, 0, 0, result, result->target); + } + + Value* base; + int displacement; + Value* index; + unsigned scale; + Value* result; +}; + +void +appendMemory(Context* c, Value* base, int displacement, Value* index, + unsigned scale, Value* result) +{ + if (DebugAppend) { + fprintf(stderr, "appendMemory\n"); + } + + new (c->zone->allocate(sizeof(MemoryEvent))) + MemoryEvent(c, base, displacement, index, scale, result); +} + +Stack* +stack(Context* c, Value* value, unsigned size, unsigned index, Stack* next) +{ + return new (c->zone->allocate(sizeof(Stack))) + Stack(value, size, index, next); +} + +void +resetStack(Context* c) +{ + unsigned i = 0; + Stack* p = 0; + for (Stack* s = c->state->stack; s; s = s->next) { + Stack* n = stack(c, value(c), s->size, s->index, 0); + n->value->sites = n->pushSite = pushSite(c, s->index); + n->pushed = true; + + if (p) { + p->next = n; + } else { + c->state->stack = n; + } + p = n; + + i += s->size; + } + + resetLocals(c); + + c->stackReset = true; +} + +void +popNow(Context* c, Stack* stack, unsigned count, bool ignore) +{ + Stack* s = stack; + unsigned ignored = 0; + for (unsigned i = count; i and s;) { + if (s->pushed) { + removeSite(c, s->value, s->pushSite); + s->pushSite = 0; + s->pushed = false; + + if (s->value->reads and (not ignore)) { + ::ignore(c, ignored); + + Site* target = targetOrRegister(c, s->value); + + if (DebugStack) { + fprintf(stderr, "pop %p value: %p target: %p\n", + s, s->value, target); + } + + addSite(c, stack, s->size * BytesPerWord, s->value, target); + + apply(c, Pop, BytesPerWord * s->size, target); + } else { + if (DebugStack) { + fprintf(stderr, "ignore %p value: %p\n", s, s->value); + } + + ignored += s->size; + } + } else { + if (DebugStack) { + fprintf(stderr, "%p not pushed\n", s); + } + } + + i -= s->size; + s = s->next; + } + + ::ignore(c, ignored); +} + +class StackSyncEvent: public Event { + public: + StackSyncEvent(Context* c): + Event(c) + { + for (Stack* s = stack; s; s = s->next) { + if (s->pushEvent) s->pushEvent->active = true; + addRead(c, s->value, s->size * BytesPerWord, 0); + } + } + + StackSyncEvent(Context* c, unsigned sequence, Stack* stack, Local* locals): + Event(c, sequence, stack, locals) + { + for (Stack* s = stack; s; s = s->next) { + if (s->pushEvent) s->pushEvent->active = true; + insertRead(c, this, sequence, s->value, s->size * BytesPerWord, 0); + } + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "StackSyncEvent.compile\n"); + } + + cleanStack(c, stack, locals, reads); + } +}; + +void +appendStackSync(Context* c) +{ + if (DebugAppend) { + fprintf(stderr, "appendStackSync\n"); + } + + new (c->zone->allocate(sizeof(StackSyncEvent))) StackSyncEvent(c); +} + +class BranchEvent: public Event { + public: + BranchEvent(Context* c, UnaryOperation type, Value* address): + Event(c), type(type), address(address) + { + addRead(c, address, BytesPerWord, 0); + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "BranchEvent.compile\n"); + } + + apply(c, type, BytesPerWord, address->source); + + nextRead(c, address); + } + + UnaryOperation type; + Value* address; +}; + +void +appendBranch(Context* c, UnaryOperation type, Value* address) +{ + appendStackSync(c); + + if (DebugAppend) { + fprintf(stderr, "appendBranch\n"); + } + + new (c->zone->allocate(sizeof(BranchEvent))) BranchEvent(c, type, address); + + resetStack(c); +} + +class PushSite: public AbstractSite { + public: + PushSite(PushEvent* event): event(event) { } + + virtual Site* readTarget(Context* c, Read* r) { + if (r->next and (not event->active)) { + return targetOrNull(c, r->next); + } else { + return 0; + } + } + + PushEvent* event; +}; + +Site* +pushSite(Context* c, PushEvent* e) +{ + return new (c->zone->allocate(sizeof(PushSite))) PushSite(e); +} + +void +appendPush(Context* c, Stack* s) +{ + if (DebugAppend) { + fprintf(stderr, "appendPush\n"); + } + + new (c->zone->allocate(sizeof(PushEvent))) PushEvent(c, s); +} + +void +appendPush(Context* c) +{ + appendPush(c, c->state->stack); +} + +class PopEvent: public Event { + public: + PopEvent(Context* c, unsigned count, bool ignore): + Event(c), count(count), ignore(ignore) + { } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "PopEvent.compile\n"); + } + + popNow(c, stack, count, ignore); + } + + unsigned count; + bool ignore; +}; + +void +appendPop(Context* c, unsigned count, bool ignore) +{ + if (DebugAppend) { + fprintf(stderr, "appendPop\n"); + } + + new (c->zone->allocate(sizeof(PopEvent))) PopEvent(c, count, ignore); +} + +class ClobberLocalEvent: public Event { + public: + ClobberLocalEvent(Context* c, unsigned size, Local* local): + Event(c), size(size), local(local) + { } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "ClobberLocalEvent.compile\n"); + } + + for (Local* l = local; l; l = l->old) { + Value* v = l->value; + Site* s = l->site; + if (v->reads + and v->sites->next == 0 + and v->sites == s) + { + preserve(c, stack, size, v, s, v->reads); + } + removeSite(c, v, s); + } + } + + unsigned size; + Local* local; +}; + +void +appendClobberLocal(Context* c, unsigned size, Local* local) +{ + if (DebugAppend) { + fprintf(stderr, "appendClobberLocal\n"); + } + + new (c->zone->allocate(sizeof(ClobberLocalEvent))) + ClobberLocalEvent(c, size, local); +} + +class LocalEvent: public Event { + public: + LocalEvent(Context* c, unsigned size, Local* local): + Event(c), size(size), local(local) + { + if (local->old) { + addRead(c, local->old->value, size, 0); + } + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "LocalEvent.compile\n"); + } + + Site* sites = 0; + if (local->old) { + Value* v = local->old->value; + if (local->old->reuse and v->reads->next == 0) { + sites = v->sites; + } + + nextRead(c, v); + } + + Value* v = local->value; + if (v->reads) { + for (Site* s = sites; s;) { + Site* t = s->next; + if (s->type(c) != MemoryOperand) { + addSite(c, 0, size, v, s); + } + s = t; + } + + addSite(c, 0, size, v, local->site); + } + } + + unsigned size; + Local* local; +}; + +void +appendLocal(Context* c, unsigned size, Local* local) +{ + if (DebugAppend) { + fprintf(stderr, "appendLocal\n"); + } + + new (c->zone->allocate(sizeof(LocalEvent))) LocalEvent(c, size, local); +} + +class BoundsCheckEvent: public Event { + public: + BoundsCheckEvent(Context* c, Value* object, unsigned lengthOffset, + Value* index, intptr_t handler): + Event(c), object(object), lengthOffset(lengthOffset), index(index), + handler(handler) + { + addRead(c, object, BytesPerWord, anyRegisterSite(c)); + addRead(c, index, BytesPerWord, registerOrConstantSite(c)); + } + + virtual void compile(Context* c) { + if (DebugCompile) { + fprintf(stderr, "BoundsCheckEvent.compile\n"); + } + + Assembler* a = c->assembler; + + ConstantSite* constant = 0; + for (Site* s = index->sites; s; s = s->next) { + if (s->type(c) == ConstantOperand) { + constant = static_cast(s); + break; + } + } + + CodePromise* nextPromise = codePromise(c, -1); + CodePromise* outOfBoundsPromise = 0; + + if (constant) { + expect(c, constant->value.value->value() >= 0); + } else { + outOfBoundsPromise = codePromise(c, -1); + + apply(c, Compare, 4, constantSite(c, resolved(c, 0)), index->source); + + Assembler::Constant outOfBoundsConstant(outOfBoundsPromise); + a->apply + (JumpIfLess, BytesPerWord, ConstantOperand, &outOfBoundsConstant); + } + + assert(c, object->source->type(c) == RegisterOperand); + int base = static_cast(object->source)->register_.low; + + Site* length = memorySite(c, base, lengthOffset); + length->acquire(c, 0, 0, 0); + + apply(c, Compare, BytesPerWord, index->source, length); + + length->release(c); + + Assembler::Constant nextConstant(nextPromise); + a->apply(JumpIfGreater, BytesPerWord, ConstantOperand, &nextConstant); + + if (constant == 0) { + outOfBoundsPromise->offset = a->length(); + } + + Assembler::Constant handlerConstant(resolved(c, handler)); + a->apply(Call, BytesPerWord, ConstantOperand, &handlerConstant); + + nextPromise->offset = a->length(); + + nextRead(c, object); + nextRead(c, index); + } + + Value* object; + unsigned lengthOffset; + Value* index; + intptr_t handler; +}; + +void +appendBoundsCheck(Context* c, Value* object, unsigned lengthOffset, + Value* index, intptr_t handler) +{ + if (DebugAppend) { + fprintf(stderr, "appendBoundsCheck\n"); + } + + new (c->zone->allocate(sizeof(BoundsCheckEvent))) BoundsCheckEvent + (c, object, lengthOffset, index, handler); +} + +Site* +readSource(Context* c, Stack* stack, Read* r) +{ + if (r->value->sites == 0) { + return 0; + } + + Site* target = (r->target ? r->target->readTarget(c, r) : 0); + + unsigned copyCost; + Site* site = pick(c, r->value->sites, target, ©Cost); + + if (target and copyCost) { + addSite(c, stack, r->size, r->value, target); + apply(c, Move, r->size, site, target); + return target; + } else { + return site; + } +} + +void +compile(Context* c) +{ + Assembler* a = c->assembler; + + Assembler::Register base(a->base()); + Assembler::Register stack(a->stack()); + a->apply(Push, BytesPerWord, RegisterOperand, &base); + a->apply(Move, BytesPerWord, RegisterOperand, &stack, + RegisterOperand, &base); + + if (stackOffset(c)) { + Assembler::Constant offset(resolved(c, stackOffset(c) * BytesPerWord)); + a->apply(Subtract, BytesPerWord, ConstantOperand, &offset, + RegisterOperand, &stack); + } + + for (unsigned i = 0; i < c->logicalCodeLength; ++i) { + LogicalInstruction* li = c->logicalCode + i; + if (li->firstEvent) { + li->machineOffset = a->length(); + + if (DebugCompile) { + fprintf(stderr, " -- ip: %d\n", i); + } + + for (Event* e = li->firstEvent; e; e = e->next) { + if (e->stackReset) { +// fprintf(stderr, "stack reset\n"); + for (Stack* s = e->stack; s; s = s->next) { + if (s->value->sites) { + assert(c, s->value->sites->next == 0); + s->value->sites->acquire(c, 0, s->size * BytesPerWord, s->value); + } + } + } + + Site* sites[e->readCount]; + unsigned si = 0; + for (Read* r = e->reads; r; r = r->eventNext) { + r->value->source = readSource(c, e->stack, r); + + if (r->value->source) { + assert(c, si < e->readCount); + sites[si++] = r->value->source; + r->value->source->freeze(c); + } + } + + while (si) { + sites[--si]->thaw(c); + } + + e->compile(c); + + for (CodePromise* p = e->promises; p; p = p->next) { + p->offset = a->length(); } } } } - - c->codeLength = pad(c->code.length()); } +unsigned +count(Stack* s) +{ + unsigned c = 0; + while (s) { + ++ c; + s = s->next; + } + return c; +} + +void +pushState(Context* c) +{ + if (DebugAppend) { + unsigned count = 0; for (State* s = c->state; s; s = s->next) ++ count; + fprintf(stderr, "push at level %d\n", count); + count = 0; for (Stack* s = c->state->stack; s; s = s->next) ++ count; + fprintf(stderr, "stack count: %d\n", count); + } + + c->state = new (c->zone->allocate(sizeof(State))) + State(c->state, c->state->stack); +} + +void +saveStack(Context* c) +{ + if (c->logicalIp >= 0 and not c->logicalCode[c->logicalIp].stackSaved) { + c->logicalCode[c->logicalIp].stackSaved = true; + c->logicalCode[c->logicalIp].stack = c->state->stack; + c->logicalCode[c->logicalIp].locals = c->locals; + + if (DebugAppend) { + unsigned count = 0; + for (Stack* s = c->state->stack; s; s = s->next) ++ count; + fprintf(stderr, "stack count after ip %d: %d\n", c->logicalIp, count); + } + } +} + +void +popState(Context* c) +{ + c->state = new (c->zone->allocate(sizeof(State))) + State(c->state->next->next, c->state->next->stack); + + if (DebugAppend) { + unsigned count = 0; for (State* s = c->state; s; s = s->next) ++ count; + fprintf(stderr, "pop to level %d\n", count); + count = 0; for (Stack* s = c->state->stack; s; s = s->next) ++ count; + fprintf(stderr, "stack count: %d\n", count); + } +} + +Stack* +stack(Context* c, Value* value, unsigned size, Stack* next) +{ + return stack(c, value, size, (next ? next->index + next->size : 0), next); +} + +void +push(Context* c, unsigned size, Value* v) +{ + assert(c, ceiling(size, BytesPerWord)); + + c->state->stack = stack(c, v, ceiling(size, BytesPerWord), c->state->stack); + + appendPush(c); +} + +void +addLocal(Context* c, unsigned size, unsigned index, Value* newValue) +{ + unsigned sizeInWords = ceiling(size, BytesPerWord); + + c->localTable[index] = c->locals = new (c->zone->allocate(sizeof(Local))) + Local(sizeInWords, index, newValue, memorySite + (c, c->assembler->base(), localOffset(c, index)), + c->localTable[index], c->locals); + + appendLocal(c, sizeInWords * BytesPerWord, c->locals); +} + +Value* +pop(Context* c, unsigned size UNUSED) +{ + Stack* s = c->state->stack; + assert(c, ceiling(size, BytesPerWord) == s->size); + + appendPop(c, s->size, false); + + c->state->stack = s->next; + return s->value; +} + +void +updateJunctions(Context* c) +{ + for (Junction* j = c->junctions; j; j = j->next) { + LogicalInstruction* i = c->logicalCode + j->logicalIp; + LogicalInstruction* p = i->immediatePredecessor; + + p->lastEvent = p->lastEvent->next + = new (c->zone->allocate(sizeof(StackSyncEvent))) + StackSyncEvent(c, p->lastEvent->sequence, p->stack, p->locals); + } +} + +void +visit(Context* c, unsigned logicalIp) +{ + assert(c, logicalIp < c->logicalCodeLength); + + if (c->logicalIp >= 0 and (not c->stackReset)) { + assert(c, c->logicalCode[logicalIp].immediatePredecessor == 0); + c->logicalCode[logicalIp].immediatePredecessor + = c->logicalCode + c->logicalIp; + } +} + +class Client: public Assembler::Client { + public: + Client(Context* c): c(c) { } + + virtual int acquireTemporary(uint32_t mask) { + int r = pickRegister(c, mask)->number; + save(r); + increment(c, r); + return r; + } + + virtual void releaseTemporary(int r) { + decrement(c, c->registers[r]); + restore(r); + } + + virtual void save(int r) { + if (c->registers[r]->refCount or c->registers[r]->value) { + Assembler::Register operand(r); + c->assembler->apply(Push, BytesPerWord, RegisterOperand, &operand); + c->registers[r]->pushed = true; + } + } + + virtual void restore(int r) { + if (c->registers[r]->pushed) { + Assembler::Register operand(r); + c->assembler->apply(Pop, BytesPerWord, RegisterOperand, &operand); + c->registers[r]->pushed = false; + } + } + + Context* c; +}; + class MyCompiler: public Compiler { public: - MyCompiler(System* s, Allocator* allocator, Zone* zone, - void* indirectCaller): - c(s, allocator, zone, indirectCaller) - { } + MyCompiler(System* s, Assembler* assembler, Zone* zone, + Compiler::Client* compilerClient): + c(s, assembler, zone, compilerClient), client(&c) + { + assembler->setClient(&client); + } + + virtual void pushState() { + ::pushState(&c); + } + + virtual void popState() { + ::popState(&c); + } + + virtual void saveStack() { + ::saveStack(&c); + } + + virtual void resetStack() { + ::resetStack(&c); + } + + virtual void init(unsigned logicalCodeLength, unsigned parameterFootprint, + unsigned localFootprint) + { + c.logicalCodeLength = logicalCodeLength; + c.parameterFootprint = parameterFootprint; + c.localFootprint = localFootprint; + + c.logicalCode = static_cast + (c.zone->allocate(sizeof(LogicalInstruction) * logicalCodeLength)); + memset(c.logicalCode, 0, sizeof(LogicalInstruction) * logicalCodeLength); + + c.localTable = static_cast + (c.zone->allocate(sizeof(Local*) * localFootprint)); + memset(c.localTable, 0, sizeof(Local*) * localFootprint); + } + + virtual void visitLogicalIp(unsigned logicalIp) { + visit(&c, logicalIp); + + c.stackReset = false; + + if (c.logicalCode[logicalIp].immediatePredecessor) { + c.junctions = new (c.zone->allocate(sizeof(Junction))) + Junction(logicalIp, c.junctions); + } + } + + virtual void startLogicalIp(unsigned logicalIp) { + if (DebugAppend) { + fprintf(stderr, " -- ip: %d\n", logicalIp); + } + + visit(&c, logicalIp); + + ::saveStack(&c); + + c.logicalIp = logicalIp; + } virtual Promise* machineIp(unsigned logicalIp) { - return new (c.zone->allocate(sizeof(IpPromise))) IpPromise(logicalIp); + return new (c.zone->allocate(sizeof(IpPromise))) IpPromise(&c, logicalIp); } - virtual Promise* poolAppend(intptr_t v) { - return poolAppendPromise(resolved(&c, v)); + virtual Promise* poolAppend(intptr_t value) { + return poolAppendPromise(resolved(&c, value)); } - virtual Promise* poolAppendPromise(Promise* v) { + virtual Promise* poolAppendPromise(Promise* value) { Promise* p = new (c.zone->allocate(sizeof(PoolPromise))) - PoolPromise(c.constantPool.length()); - c.constantPool.appendAddress(v); + PoolPromise(&c, c.constantCount); + + ConstantPoolNode* constant + = new (c.zone->allocate(sizeof(ConstantPoolNode))) + ConstantPoolNode(value); + + if (c.firstConstant) { + c.lastConstant->next = constant; + } else { + c.firstConstant = constant; + } + c.lastConstant = constant; + ++ c.constantCount; + return p; } - virtual Operand* constant(int64_t v) { - return immediate(&c, v); + virtual Operand* constant(int64_t value) { + return promiseConstant(resolved(&c, value)); } - virtual Operand* promiseConstant(Promise* p) { - return address(&c, static_cast(p)); + virtual Operand* promiseConstant(Promise* value) { + return ::value(&c, ::constantSite(&c, value)); } - virtual Operand* absolute(Promise* p) { - return ::absolute(&c, static_cast(p)); + virtual Operand* address(Promise* address) { + return value(&c, ::addressSite(&c, address)); } - virtual Stack* push(Stack* s, unsigned count) { - appendOperation - (&c, MyOperand::sub, - immediate(&c, count * BytesPerWord), - register_(&c, rsp)); + virtual Operand* memory(Operand* base, + int displacement = 0, + Operand* index = 0, + unsigned scale = 1) + { + Value* result = value(&c); - return pushed(s, count); - } + appendMemory(&c, static_cast(base), displacement, + static_cast(index), scale, result); - virtual Stack* pushed(Stack* s, unsigned count) { - MyStack* stack = static_cast(s); - while (count) { - -- count; - stack = ::pushed(&c, stack); - } - return stack; - } - - virtual Stack* push1(Stack* s, Operand* v) { - return ::push(&c, static_cast(s), MyOperand::push1, - static_cast(v)); - } - - virtual Stack* push2(Stack* s, Operand* v) { - return ::push(&c, static_cast(s), MyOperand::push2, - static_cast(v)); - } - - virtual Stack* push2z(Stack* s, Operand* v) { - return ::push(&c, static_cast(s), MyOperand::push2z, - static_cast(v)); - } - - virtual Stack* push4(Stack* s, Operand* v) { - return ::push(&c, static_cast(s), MyOperand::push4, - static_cast(v)); - } - - virtual Stack* push8(Stack* s, Operand* v) { - return ::push(&c, static_cast(s), MyOperand::push8, - static_cast(v)); - } - - virtual Operand* stack(Stack* s, unsigned index) { - MyStack* stack = static_cast(s); - - while (index) { - -- index; - stack = stack->next; - } - - return stack->value; - } - - virtual Stack* pop(Stack* s, unsigned count) { - return ::pop(&c, static_cast(s), count); - } - - virtual Stack* pop4(Stack* s, Operand* dst) { - return ::pop(&c, static_cast(s), MyOperand::pop4, - static_cast(dst)); - } - - virtual Stack* pop8(Stack* s, Operand* dst) { - return ::pop(&c, static_cast(s), MyOperand::pop8, - static_cast(dst)); + return result; } virtual Operand* stack() { - return register_(&c, rsp); + Site* s = registerSite(&c, c.assembler->stack()); + return value(&c, s, s); } virtual Operand* base() { - return register_(&c, rbp); + Site* s = registerSite(&c, c.assembler->base()); + return value(&c, s, s); } virtual Operand* thread() { - return register_(&c, rbx); + Site* s = registerSite(&c, c.assembler->thread()); + return value(&c, s, s); } - virtual Operand* indirectTarget() { - return register_(&c, rax); + virtual bool isConstant(Operand* a) { + for (Site* s = static_cast(a)->sites; s; s = s->next) { + if (s->type(&c) == ConstantOperand) return true; + } + return false; } - virtual Operand* temporary() { - RegisterOperand* r = register_(&c); - appendAcquire(&c, r); - return r; - } - - virtual void release(Operand* v) { - appendRelease(&c, v); + virtual int64_t constantValue(Operand* a) { + for (Site* s = static_cast(a)->sites; s; s = s->next) { + if (s->type(&c) == ConstantOperand) { + return static_cast(s)->value.value->value(); + } + } + abort(&c); } virtual Operand* label() { - return address(&c, 0); + return value(&c, ::constantSite(&c, static_cast(0))); } Promise* machineIp() { - CodePromise* p = new (c.zone->allocate(sizeof(CodePromise))) CodePromise(); - - Segment* s = currentSegment(&c); - s->event->task = new (c.zone->allocate(sizeof(CodePromiseTask))) - CodePromiseTask(p, s->event->task); - - return p; + return codePromise(&c, c.logicalCode[c.logicalIp].lastEvent); } virtual void mark(Operand* label) { - static_cast(label)->setLabelValue - (&c, static_cast(machineIp())); + appendStackSync(&c); + ::resetStack(&c); + + for (Site* s = static_cast(label)->sites; s; s = s->next) { + if (s->type(&c) == ConstantOperand) { + static_cast(s)->value.value = machineIp(); + return; + } + } + abort(&c); } - virtual void indirectCall - (Operand* address, unsigned argumentCount, ...) + virtual void push(unsigned size) { + assert(&c, ceiling(size, BytesPerWord)); + + c.state->stack = ::stack + (&c, value(&c), ceiling(size, BytesPerWord), c.state->stack); + } + + virtual void push(unsigned size, Operand* value) { + ::push(&c, size, static_cast(value)); + } + + virtual Operand* pop(unsigned size) { + return ::pop(&c, size); + } + + virtual void pushed(unsigned count) { + for (unsigned i = 0; i < count; ++i) { + Value* v = value(&c); + c.state->stack = ::stack(&c, v, 1, c.state->stack); + c.state->stack->pushed = true; +// v->sites = pushSite(&c, c.state->stack->index); + } + } + + virtual void popped(unsigned count) { + appendPop(&c, count, true); + + for (unsigned i = count; i;) { + Stack* s = c.state->stack; + c.state->stack = s->next; + i -= s->size; + } + } + + virtual Operand* peek(unsigned size UNUSED, unsigned index) { + Stack* s = c.state->stack; + for (unsigned i = index; i > 0;) { + i -= s->size; + s = s->next; + } + assert(&c, s->size == ceiling(size, BytesPerWord)); + return s->value; + } + + virtual Operand* call(Operand* address, + unsigned flags, + TraceHandler* traceHandler, + unsigned resultSize, + unsigned argumentCount, + ...) { va_list a; va_start(a, argumentCount); - pushArguments(&c, argumentCount, a); + + unsigned footprint = 0; + unsigned size = BytesPerWord; + Value* arguments[argumentCount]; + unsigned argumentSizes[argumentCount]; + unsigned index = 0; + for (unsigned i = 0; i < argumentCount; ++i) { + Value* o = va_arg(a, Value*); + if (o) { + arguments[index] = o; + argumentSizes[index] = size; + size = BytesPerWord; + ++ index; + } else { + size = 8; + } + ++ footprint; + } + va_end(a); - appendOperation - (&c, MyOperand::mov, address, register_(&c, rax)); - call(immediate(&c, c.indirectCaller), 0); + for (Stack* s = c.state->stack; s; s = s->next) { + if (s->pushEvent == 0) { + appendPush(&c, s); + } + s->pushEvent->active = true; + } - appendOperation - (&c, MyOperand::add, - immediate(&c, argumentFootprint(argumentCount)), - register_(&c, rsp)); + Stack* oldStack = c.state->stack; + + for (int i = index - 1; i >= 0; --i) { + ::push(&c, argumentSizes[i], arguments[i]); + } + + Stack* argumentStack = c.state->stack; + c.state->stack = oldStack; + + Value* result = value(&c); + appendCall(&c, static_cast(address), flags, + traceHandler, result, resultSize, argumentStack, + index); + + return result; } - virtual void indirectCall - (Operand* address, TraceHandler* traceHandler, unsigned argumentCount, ...) + virtual void return_(unsigned size, Operand* value) { + appendReturn(&c, size, static_cast(value)); + } + + virtual void storeLocal(unsigned size, Operand* src, unsigned index) { + assert(&c, index < c.localFootprint); + + if (c.localTable[index]) { + appendClobberLocal(&c, size, c.localTable[index]); + c.localTable[index] = 0; + } + + store(size, src, memory(base(), localOffset(&c, index))); + + // todo: find out why this doesn't work and fix it: +// addLocal(&c, size, index, static_cast(src)); + } + + virtual Operand* loadLocal(unsigned size, unsigned index) { + assert(&c, index < c.localFootprint); + + Value* v = value(&c); + addLocal(&c, size, index, v); + return v; + } + + virtual void checkBounds(Operand* object, unsigned lengthOffset, + Operand* index, intptr_t handler) { - va_list a; va_start(a, argumentCount); - pushArguments(&c, argumentCount, a); - va_end(a); - - appendOperation - (&c, MyOperand::mov, address, register_(&c, rax)); - call(immediate(&c, c.indirectCaller), traceHandler); - - appendOperation - (&c, MyOperand::add, - immediate(&c, argumentFootprint(argumentCount)), - register_(&c, rsp)); + appendBoundsCheck(&c, static_cast(object), + lengthOffset, static_cast(index), handler); } - virtual void indirectCallNoReturn - (Operand* address, TraceHandler* traceHandler, unsigned argumentCount, ...) - { - va_list a; va_start(a, argumentCount); - pushArguments(&c, argumentCount, a); - va_end(a); - - appendOperation - (&c, MyOperand::mov, address, register_(&c, rax)); - - call(immediate(&c, c.indirectCaller), traceHandler); + virtual void store(unsigned size, Operand* src, Operand* dst) { + appendMove(&c, Move, size, static_cast(src), + static_cast(dst)); } - virtual void directCall - (Operand* address, unsigned argumentCount, ...) - { - va_list a; va_start(a, argumentCount); - pushArguments(&c, argumentCount, a); - va_end(a); - - call(address, 0); - - appendOperation - (&c, MyOperand::add, - immediate(&c, argumentFootprint(argumentCount)), - register_(&c, rsp)); + virtual Operand* load(unsigned size, Operand* src) { + Value* dst = value(&c); + appendMove(&c, Move, size, static_cast(src), dst); + return dst; } - virtual Operand* result4() { - RegisterOperand* r = register_(&c, rax); - appendAcquire(&c, r); - return r; + virtual Operand* loadz(unsigned size, Operand* src) { + Value* dst = value(&c); + appendMove(&c, MoveZ, size, static_cast(src), dst); + return dst; } - virtual Operand* result8() { - if (BytesPerWord == 8) { - return result4(); - } else { - RegisterOperand* r = register_(&c, rax, rdx); - appendAcquire(&c, r); - return r; - } + virtual Operand* load4To8(Operand* src) { + Value* dst = value(&c); + appendMove(&c, Move4To8, 8, static_cast(src), dst); + return dst; } - virtual void return4(Operand* v) { - appendOperation(&c, MyOperand::mov, v, register_(&c, rax)); - epilogue(); - ret(); + virtual void cmp(unsigned size, Operand* a, Operand* b) { + appendCompare(&c, size, static_cast(a), + static_cast(b)); } - virtual void return8(Operand* v) { - if (BytesPerWord == 8) { - return4(v); - } else { - appendOperation(&c, MyOperand::mov8, v, register_(&c, rax, rdx)); - epilogue(); - ret(); - } + virtual void jl(Operand* address) { + appendBranch(&c, JumpIfLess, static_cast(address)); } - virtual void call(Operand* v, TraceHandler* traceHandler) { - appendOperation(&c, MyOperand::call, v); - if (traceHandler) { - traceHandler->handleTrace(machineIp()); - } + virtual void jg(Operand* address) { + appendBranch(&c, JumpIfGreater, static_cast(address)); } - virtual void alignedCall(Operand* v, TraceHandler* traceHandler) { - appendOperation(&c, MyOperand::alignedCall, v); - if (traceHandler) { - traceHandler->handleTrace(machineIp()); - } + virtual void jle(Operand* address) { + appendBranch(&c, JumpIfLessOrEqual, static_cast(address)); } - virtual void ret() { - appendOperation(&c, MyOperand::ret); + virtual void jge(Operand* address) { + appendBranch(&c, JumpIfGreaterOrEqual, static_cast(address)); } - virtual void mov1(Operand* src, Operand* dst) { - appendOperation(&c, MyOperand::mov1, src, dst); + virtual void je(Operand* address) { + appendBranch(&c, JumpIfEqual, static_cast(address)); } - virtual void mov2(Operand* src, Operand* dst) { - appendOperation(&c, MyOperand::mov2, src, dst); + virtual void jne(Operand* address) { + appendBranch(&c, JumpIfNotEqual, static_cast(address)); } - virtual void mov4(Operand* src, Operand* dst) { - appendOperation(&c, MyOperand::mov4, src, dst); + virtual void jmp(Operand* address) { + appendBranch(&c, Jump, static_cast(address)); } - virtual void mov8(Operand* src, Operand* dst) { - appendOperation(&c, MyOperand::mov8, src, dst); + virtual Operand* add(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, Add, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void mov1ToW(Operand* src, Operand* dst) { - appendOperation(&c, MyOperand::mov1ToW, src, dst); + virtual Operand* sub(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, Subtract, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void mov2ToW(Operand* src, Operand* dst) { - appendOperation(&c, MyOperand::mov2ToW, src, dst); + virtual Operand* mul(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, Multiply, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void mov2zToW(Operand* src, Operand* dst) { - appendOperation(&c, MyOperand::mov2zToW, src, dst); + virtual Operand* div(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, Divide, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void mov4To8(Operand* src, Operand* dst) { - appendOperation(&c, MyOperand::mov4To8, src, dst); + virtual Operand* rem(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, Remainder, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void cmp4(Operand* subtrahend, Operand* minuend) { - appendOperation(&c, MyOperand::cmp4, subtrahend, minuend); + virtual Operand* shl(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, ShiftLeft, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void cmp8(Operand* subtrahend, Operand* minuend) { - appendOperation(&c, MyOperand::cmp8, subtrahend, minuend); + virtual Operand* shr(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, ShiftRight, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void jl(Operand* v) { - appendOperation(&c, MyOperand::jl, v); + virtual Operand* ushr(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, UnsignedShiftRight, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void jg(Operand* v) { - appendOperation(&c, MyOperand::jg, v); + virtual Operand* and_(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, And, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void jle(Operand* v) { - appendOperation(&c, MyOperand::jle, v); + virtual Operand* or_(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, Or, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void jge(Operand* v) { - appendOperation(&c, MyOperand::jge, v); + virtual Operand* xor_(unsigned size, Operand* a, Operand* b) { + Value* result = value(&c); + appendCombine(&c, Xor, size, static_cast(a), + static_cast(b), result); + return result; } - virtual void je(Operand* v) { - appendOperation(&c, MyOperand::je, v); + virtual Operand* neg(unsigned size, Operand* a) { + Value* result = value(&c); + appendTranslate(&c, Negate, size, static_cast(a), result); + return result; } - virtual void jne(Operand* v) { - appendOperation(&c, MyOperand::jne, v); - } - - virtual void jmp(Operand* v) { - appendOperation(&c, MyOperand::jmp, v); - } - - virtual void add4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::add4, v, dst); - } - - virtual void add8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::add8, v, dst); - } - - virtual void sub4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::sub4, v, dst); - } - - virtual void sub8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::sub8, v, dst); - } - - virtual void mul4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::mul4, v, dst); - } - - virtual void mul8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::mul8, v, dst); - } - - virtual void div4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::div4, v, dst); - } - - virtual void div8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::div8, v, dst); - } - - virtual void rem4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::rem4, v, dst); - } - - virtual void rem8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::rem8, v, dst); - } - - virtual void shl4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::shl4, v, dst); - } - - virtual void shl8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::shl8, v, dst); - } - - virtual void shr4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::shr4, v, dst); - } - - virtual void shr8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::shr8, v, dst); - } - - virtual void ushr4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::ushr4, v, dst); - } - - virtual void ushr8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::ushr8, v, dst); - } - - virtual void and4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::and4, v, dst); - } - - virtual void and8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::and8, v, dst); - } - - virtual void or4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::or4, v, dst); - } - - virtual void or8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::or8, v, dst); - } - - virtual void xor4(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::xor4, v, dst); - } - - virtual void xor8(Operand* v, Operand* dst) { - appendOperation(&c, MyOperand::xor8, v, dst); - } - - virtual void neg4(Operand* v) { - appendOperation(&c, MyOperand::neg4, v); - } - - virtual void neg8(Operand* v) { - appendOperation(&c, MyOperand::neg8, v); - } - - virtual Operand* memory(Operand* base, int displacement, - Operand* index, unsigned scale) - { - return ::memory(&c, static_cast(base), displacement, - static_cast(index), scale); - } - - virtual void prologue() { - appendOperation(&c, MyOperand::push, register_(&c, rbp)); - appendOperation - (&c, MyOperand::mov, register_(&c, rsp), register_(&c, rbp)); - } - - virtual void reserve(unsigned size) { - appendOperation - (&c, MyOperand::sub, immediate(&c, size * BytesPerWord), - register_(&c, rsp)); - - c.reserved = size; - } - - virtual void epilogue() { - appendOperation - (&c, MyOperand::mov, register_(&c, rbp), register_(&c, rsp)); - appendOperation(&c, MyOperand::pop, register_(&c, rbp)); - } - - virtual void startLogicalIp(unsigned ip) { - c.plan.appendAddress - (new (c.zone->allocate(sizeof(Segment))) - Segment(ip, new (c.zone->allocate(sizeof(Event))) Event(0))); - } - - virtual unsigned codeSize() { - if (c.codeLength < 0) { - assert(&c, c.code.length() == 0); - writeCode(&c); - } - return c.codeLength; + virtual unsigned compile() { + updateJunctions(&c); + ::compile(&c); + return c.assembler->length(); } virtual unsigned poolSize() { - return c.constantPool.length(); + return c.constantCount * BytesPerWord; } - virtual void writeTo(uint8_t* out) { - c.code.wrap(out, codeSize()); - writeCode(&c); + virtual void writeTo(uint8_t* dst) { + c.machineCode = dst; + c.assembler->writeTo(dst); - for (unsigned i = 0; i < c.constantPool.length(); i += BytesPerWord) { - Promise* p; c.constantPool.get(i, &p, BytesPerWord); - *reinterpret_cast(out + codeSize() + i) = p->value(this); - } - } - - virtual void updateCall(void* returnAddress, void* newTarget) { - if (BytesPerWord == 8) { - uint8_t* instruction = static_cast(returnAddress) - 13; - assert(&c, instruction[0] == 0x49); - assert(&c, instruction[1] == 0xba); - assert(&c, instruction[10] == 0x41); - assert(&c, instruction[11] == 0xff); - assert(&c, instruction[12] == 0xd2); - assert(&c, reinterpret_cast(instruction + 2) % 8 == 0); - - intptr_t v = reinterpret_cast(newTarget); - memcpy(instruction + 2, &v, 8); - } else { - uint8_t* instruction = static_cast(returnAddress) - 5; - assert(&c, *instruction == 0xE8); - assert(&c, reinterpret_cast(instruction + 1) % 4 == 0); - - int32_t v = static_cast(newTarget) - - static_cast(returnAddress); - memcpy(instruction + 1, &v, 4); + int i = 0; + for (ConstantPoolNode* n = c.firstConstant; n; n = n->next) { + *reinterpret_cast(dst + pad(c.assembler->length()) + i) + = n->promise->value(); + i += BytesPerWord; } } virtual void dispose() { - c.dispose(); + // ignore } Context c; + ::Client client; }; -intptr_t -MyPromise::value(Compiler* compiler) -{ - return value(&(static_cast(compiler)->c)); -} - } // namespace namespace vm { Compiler* -makeCompiler(System* system, Allocator* allocator, Zone* zone, - void* indirectCaller) +makeCompiler(System* system, Assembler* assembler, Zone* zone, + Compiler::Client* client) { return new (zone->allocate(sizeof(MyCompiler))) - MyCompiler(system, allocator, zone, indirectCaller); + MyCompiler(system, assembler, zone, client); } -} // namespace v +} // namespace vm diff --git a/src/compiler.h b/src/compiler.h index 1f95683dd6..6d27a60cc1 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -13,41 +13,46 @@ #include "system.h" #include "zone.h" +#include "assembler.h" namespace vm { -class Operand { }; - -class Stack { }; - -class Compiler; - -class Promise { - public: - virtual ~Promise() { } - - virtual intptr_t value(Compiler*) = 0; -}; - class Compiler { public: - class TraceHandler { + class Client { public: - virtual ~TraceHandler() { } + virtual ~Client() { } - virtual void handleTrace(Promise* address) = 0; + virtual intptr_t getThunk(UnaryOperation op, unsigned size) = 0; + virtual intptr_t getThunk(BinaryOperation op, unsigned size) = 0; }; + + static const unsigned Aligned = 1 << 0; + static const unsigned NoReturn = 1 << 1; + + class Operand { }; virtual ~Compiler() { } + virtual void pushState() = 0; + virtual void popState() = 0; + virtual void saveStack() = 0; + virtual void resetStack() = 0; + + virtual void init(unsigned logicalCodeSize, unsigned parameterFootprint, + unsigned localFootprint) = 0; + + virtual void visitLogicalIp(unsigned logicalIp) = 0; + virtual void startLogicalIp(unsigned logicalIp) = 0; + virtual Promise* machineIp(unsigned logicalIp) = 0; - virtual Promise* poolAppend(intptr_t) = 0; - virtual Promise* poolAppendPromise(Promise*) = 0; + virtual Promise* poolAppend(intptr_t value) = 0; + virtual Promise* poolAppendPromise(Promise* value) = 0; - virtual Operand* constant(int64_t) = 0; - virtual Operand* promiseConstant(Promise*) = 0; - virtual Operand* absolute(Promise*) = 0; + virtual Operand* constant(int64_t value) = 0; + virtual Operand* promiseConstant(Promise* value) = 0; + virtual Operand* address(Promise* address) = 0; virtual Operand* memory(Operand* base, int displacement = 0, Operand* index = 0, @@ -56,104 +61,70 @@ class Compiler { virtual Operand* stack() = 0; virtual Operand* base() = 0; virtual Operand* thread() = 0; - virtual Operand* indirectTarget() = 0; - virtual Operand* temporary() = 0; - virtual Operand* result4() = 0; - virtual Operand* result8() = 0; - virtual void release(Operand*) = 0; + + virtual bool isConstant(Operand* value) = 0; + virtual int64_t constantValue(Operand* value) = 0; virtual Operand* label() = 0; - virtual void mark(Operand*) = 0; + virtual void mark(Operand* label) = 0; - virtual void indirectCall - (Operand* address, unsigned argumentCount, ...) = 0; - virtual void indirectCall - (Operand* address, TraceHandler* traceHandler, - unsigned argumentCount, ...) = 0; - virtual void indirectCallNoReturn - (Operand* address, TraceHandler* traceHandler, - unsigned argumentCount, ...) = 0; - virtual void directCall - (Operand* address, unsigned argumentCount, ...) = 0; + virtual void push(unsigned size) = 0; + virtual void push(unsigned size, Operand* value) = 0; + virtual Operand* pop(unsigned size) = 0; + virtual void pushed(unsigned count) = 0; + virtual void popped(unsigned count) = 0; + virtual Operand* peek(unsigned size, unsigned index) = 0; - virtual void call(Operand*, TraceHandler*) = 0; - virtual void alignedCall(Operand*, TraceHandler*) = 0; - virtual void return4(Operand*) = 0; - virtual void return8(Operand*) = 0; - virtual void ret() = 0; + virtual Operand* call(Operand* address, + unsigned flags, + TraceHandler* traceHandler, + unsigned resultSize, + unsigned argumentCount, + ...) = 0; - virtual Stack* push(Stack*, unsigned count) = 0; - virtual Stack* pushed(Stack*, unsigned count) = 0; - virtual Stack* pop(Stack*, unsigned count) = 0; - virtual Operand* stack(Stack*, unsigned) = 0; + virtual void return_(unsigned size, Operand* value) = 0; - virtual Stack* push1(Stack*, Operand*) = 0; - virtual Stack* push2(Stack*, Operand*) = 0; - virtual Stack* push2z(Stack*, Operand*) = 0; - virtual Stack* push4(Stack*, Operand*) = 0; - virtual Stack* push8(Stack*, Operand*) = 0; - virtual Stack* pop4(Stack*, Operand*) = 0; - virtual Stack* pop8(Stack*, Operand*) = 0; - virtual void mov1(Operand* src, Operand* dst) = 0; - virtual void mov2(Operand* src, Operand* dst) = 0; - virtual void mov4(Operand* src, Operand* dst) = 0; - virtual void mov8(Operand* src, Operand* dst) = 0; - virtual void mov1ToW(Operand* src, Operand* dst) = 0; - virtual void mov2ToW(Operand* src, Operand* dst) = 0; - virtual void mov2zToW(Operand* src, Operand* dst) = 0; - virtual void mov4To8(Operand* src, Operand* dst) = 0; - virtual void cmp4(Operand* subtrahend, Operand* minuend) = 0; - virtual void cmp8(Operand* subtrahend, Operand* minuend) = 0; - virtual void jl(Operand*) = 0; - virtual void jg(Operand*) = 0; - virtual void jle(Operand*) = 0; - virtual void jge(Operand*) = 0; - virtual void je(Operand*) = 0; - virtual void jne(Operand*) = 0; - virtual void jmp(Operand*) = 0; - virtual void add4(Operand* v, Operand* dst) = 0; - virtual void add8(Operand* v, Operand* dst) = 0; - virtual void sub4(Operand* v, Operand* dst) = 0; - virtual void sub8(Operand* v, Operand* dst) = 0; - virtual void mul4(Operand* v, Operand* dst) = 0; - virtual void mul8(Operand* v, Operand* dst) = 0; - virtual void div4(Operand* v, Operand* dst) = 0; - virtual void div8(Operand* v, Operand* dst) = 0; - virtual void rem4(Operand* v, Operand* dst) = 0; - virtual void rem8(Operand* v, Operand* dst) = 0; - virtual void shl4(Operand* v, Operand* dst) = 0; - virtual void shl8(Operand* v, Operand* dst) = 0; - virtual void shr4(Operand* v, Operand* dst) = 0; - virtual void shr8(Operand* v, Operand* dst) = 0; - virtual void ushr4(Operand* v, Operand* dst) = 0; - virtual void ushr8(Operand* v, Operand* dst) = 0; - virtual void and4(Operand* v, Operand* dst) = 0; - virtual void and8(Operand* v, Operand* dst) = 0; - virtual void or4(Operand* v, Operand* dst) = 0; - virtual void or8(Operand* v, Operand* dst) = 0; - virtual void xor4(Operand* v, Operand* dst) = 0; - virtual void xor8(Operand* v, Operand* dst) = 0; - virtual void neg4(Operand*) = 0; - virtual void neg8(Operand*) = 0; + virtual void storeLocal(unsigned size, Operand* src, unsigned index) = 0; + virtual Operand* loadLocal(unsigned size, unsigned index) = 0; - virtual void prologue() = 0; - virtual void reserve(unsigned size) = 0; - virtual void epilogue() = 0; + virtual void checkBounds(Operand* object, unsigned lengthOffset, + Operand* index, intptr_t handler) = 0; - virtual void startLogicalIp(unsigned) = 0; + virtual void store(unsigned size, Operand* src, Operand* dst) = 0; + virtual Operand* load(unsigned size, Operand* src) = 0; + virtual Operand* loadz(unsigned size, Operand* src) = 0; + virtual Operand* load4To8(Operand* src) = 0; + virtual void cmp(unsigned size, Operand* a, Operand* b) = 0; + virtual void jl(Operand* address) = 0; + virtual void jg(Operand* address) = 0; + virtual void jle(Operand* address) = 0; + virtual void jge(Operand* address) = 0; + virtual void je(Operand* address) = 0; + virtual void jne(Operand* address) = 0; + virtual void jmp(Operand* address) = 0; + virtual Operand* add(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* sub(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* mul(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* div(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* rem(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* shl(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* shr(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* ushr(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* and_(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* or_(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* xor_(unsigned size, Operand* a, Operand* b) = 0; + virtual Operand* neg(unsigned size, Operand* a) = 0; - virtual unsigned codeSize() = 0; + virtual unsigned compile() = 0; virtual unsigned poolSize() = 0; - virtual void writeTo(uint8_t*) = 0; - - virtual void updateCall(void* returnAddress, void* newTarget) = 0; + virtual void writeTo(uint8_t* dst) = 0; virtual void dispose() = 0; }; Compiler* -makeCompiler(System* system, Allocator* allocator, Zone* zone, - void* indirectCaller); +makeCompiler(System* system, Assembler* assembler, Zone* zone, + Compiler::Client* client); } // namespace vm diff --git a/src/thunks.cpp b/src/thunks.cpp new file mode 100644 index 0000000000..833ffd8114 --- /dev/null +++ b/src/thunks.cpp @@ -0,0 +1,44 @@ +THUNK(tryInitClass) +THUNK(findInterfaceMethodFromInstance) +THUNK(compareDoublesG) +THUNK(compareDoublesL) +THUNK(compareFloatsG) +THUNK(compareFloatsL) +THUNK(addDouble) +THUNK(subtractDouble) +THUNK(multiplyDouble) +THUNK(divideDouble) +THUNK(moduloDouble) +THUNK(negateDouble) +THUNK(doubleToFloat) +THUNK(doubleToInt) +THUNK(doubleToLong) +THUNK(addFloat) +THUNK(subtractFloat) +THUNK(multiplyFloat) +THUNK(divideFloat) +THUNK(moduloFloat) +THUNK(negateFloat) +THUNK(floatToDouble) +THUNK(floatToInt) +THUNK(floatToLong) +THUNK(intToDouble) +THUNK(intToFloat) +THUNK(longToDouble) +THUNK(longToFloat) +THUNK(divideLong) +THUNK(moduloLong) +THUNK(makeBlankObjectArray) +THUNK(makeBlankArray) +THUNK(lookUpAddress) +THUNK(setMaybeNull) +THUNK(acquireMonitorForObject) +THUNK(releaseMonitorForObject) +THUNK(makeMultidimensionalArray) +THUNK(throw_) +THUNK(checkCast) +THUNK(instanceOf) +THUNK(makeNewWeakReference) +THUNK(makeNew) +THUNK(set) +THUNK(gcIfNecessary) diff --git a/src/x86.cpp b/src/x86.cpp new file mode 100644 index 0000000000..75334f5b84 --- /dev/null +++ b/src/x86.cpp @@ -0,0 +1,2105 @@ +#include "assembler.h" +#include "vector.h" + +using namespace vm; + +#define INDEX1(a, b) ((a) + (UnaryOperationCount * (b))) + +#define CAST1(x) reinterpret_cast(x) + +#define INDEX2(a, b, c) \ + ((a) \ + + (BinaryOperationCount * (b)) \ + + (BinaryOperationCount * OperandTypeCount * (c))) + +#define CAST2(x) reinterpret_cast(x) + +namespace { + +enum { + rax = 0, + rcx = 1, + rdx = 2, + rbx = 3, + rsp = 4, + rbp = 5, + rsi = 6, + rdi = 7, + r8 = 8, + r9 = 9, + r10 = 10, + r11 = 11, + r12 = 12, + r13 = 13, + r14 = 14, + r15 = 15, +}; + +inline bool +isInt8(intptr_t v) +{ + return v == static_cast(v); +} + +inline bool +isInt32(intptr_t v) +{ + return v == static_cast(v); +} + +class Task; + +class Context { + public: + Context(System* s, Allocator* a, Zone* zone): + s(s), zone(zone), client(0), code(s, a, 1024), tasks(0), result(0) + { } + + System* s; + Zone* zone; + Assembler::Client* client; + Vector code; + Task* tasks; + uint8_t* result; +}; + +inline void NO_RETURN +abort(Context* c) +{ + abort(c->s); +} + +#ifndef NDEBUG +inline void +assert(Context* c, bool v) +{ + assert(c->s, v); +} +#endif // not NDEBUG + +inline void +expect(Context* c, bool v) +{ + expect(c->s, v); +} + +ResolvedPromise* +resolved(Context* c, int64_t value) +{ + return new (c->zone->allocate(sizeof(ResolvedPromise))) + ResolvedPromise(value); +} + +class CodePromise: public Promise { + public: + CodePromise(Context* c, unsigned offset): c(c), offset(offset) { } + + virtual int64_t value() { + if (resolved()) { + return reinterpret_cast(c->result + offset); + } + + abort(c); + } + + virtual bool resolved() { + return c->result != 0; + } + + Context* c; + unsigned offset; +}; + +CodePromise* +codePromise(Context* c, unsigned offset) +{ + return new (c->zone->allocate(sizeof(CodePromise))) CodePromise(c, offset); +} + +class Task { + public: + Task(Task* next): next(next) { } + + virtual ~Task() { } + + virtual void run(Context* c) = 0; + + Task* next; +}; + +class OffsetTask: public Task { + public: + OffsetTask(Task* next, Promise* promise, unsigned instructionOffset, + unsigned instructionSize): + Task(next), + promise(promise), + instructionOffset(instructionOffset), + instructionSize(instructionSize) + { } + + virtual void run(Context* c) { + uint8_t* instruction = c->result + instructionOffset; + intptr_t v = reinterpret_cast(promise->value()) + - instruction - instructionSize; + + expect(c, isInt32(v)); + + int32_t v4 = v; + memcpy(instruction + instructionSize - 4, &v4, 4); + } + + Promise* promise; + unsigned instructionOffset; + unsigned instructionSize; +}; + +void +appendOffsetTask(Context* c, Promise* promise, int instructionOffset, + unsigned instructionSize) +{ + c->tasks = new (c->zone->allocate(sizeof(OffsetTask))) OffsetTask + (c->tasks, promise, instructionOffset, instructionSize); +} + +class ImmediateTask: public Task { + public: + ImmediateTask(Task* next, Promise* promise, unsigned offset): + Task(next), + promise(promise), + offset(offset) + { } + + virtual void run(Context* c) { + intptr_t v = promise->value(); + memcpy(c->result + offset, &v, BytesPerWord); + } + + Promise* promise; + unsigned offset; +}; + +void +appendImmediateTask(Context* c, Promise* promise, unsigned offset) +{ + c->tasks = new (c->zone->allocate(sizeof(ImmediateTask))) ImmediateTask + (c->tasks, promise, offset); +} + +void +encode(Context* c, uint8_t* instruction, unsigned length, int a, int b, + int32_t displacement, int index, unsigned scale) +{ + c->code.append(instruction, length); + + uint8_t width; + if (displacement == 0 and b != rbp) { + width = 0; + } else if (isInt8(displacement)) { + width = 0x40; + } else { + width = 0x80; + } + + if (index == -1) { + c->code.append(width | (a << 3) | b); + if (b == rsp) { + c->code.append(0x24); + } + } else { + assert(c, b != rsp); + c->code.append(width | (a << 3) | 4); + c->code.append((log(scale) << 6) | (index << 3) | b); + } + + if (displacement == 0 and b != rbp) { + // do nothing + } else if (isInt8(displacement)) { + c->code.append(displacement); + } else { + c->code.append4(displacement); + } +} + +void +rex(Context* c, uint8_t mask, int r) +{ + if (BytesPerWord == 8) { + c->code.append(mask | ((r & 8) >> 3)); + } +} + +void +rex(Context* c) +{ + rex(c, 0x48, rax); +} + +void +encode(Context* c, uint8_t instruction, int a, Assembler::Memory* b, bool rex) +{ + if (rex) { + ::rex(c); + } + + encode(c, &instruction, 1, a, b->base, b->offset, b->index, b->scale); +} + +void +encode2(Context* c, uint16_t instruction, int a, Assembler::Memory* b, + bool rex) +{ + if (rex) { + ::rex(c); + } + + uint8_t i[2] = { instruction >> 8, instruction & 0xff }; + encode(c, i, 2, a, b->base, b->offset, b->index, b->scale); +} + +typedef void (*OperationType)(Context*); +OperationType +Operations[OperationCount]; + +typedef void (*UnaryOperationType)(Context*, unsigned, Assembler::Operand*); +UnaryOperationType +UnaryOperations[UnaryOperationCount * OperandTypeCount]; + +typedef void (*BinaryOperationType) +(Context*, unsigned, Assembler::Operand*, Assembler::Operand*); +BinaryOperationType +BinaryOperations[BinaryOperationCount * OperandTypeCount * OperandTypeCount]; + +void +return_(Context* c) +{ + c->code.append(0xc3); +} + +void +unconditional(Context* c, unsigned jump, Assembler::Constant* a) +{ + appendOffsetTask(c, a->value, c->code.length(), 5); + + c->code.append(jump); + c->code.append4(0); +} + +void +conditional(Context* c, unsigned condition, Assembler::Constant* a) +{ + appendOffsetTask(c, a->value, c->code.length(), 6); + + c->code.append(0x0f); + c->code.append(condition); + c->code.append4(0); +} + +void +moveCR(Context*, unsigned, Assembler::Constant*, Assembler::Register*); + +void +callR(Context*, unsigned, Assembler::Register*); + +void +callC(Context* c, unsigned size, Assembler::Constant* a) +{ + assert(c, size == BytesPerWord); + + if (BytesPerWord == 8) { + Assembler::Register r(r10); + moveCR(c, size, a, &r); + callR(c, size, &r); + } else { + unconditional(c, 0xe8, a); + } +} + +void +alignedCallC(Context* c, unsigned size, Assembler::Constant* a) +{ + if (BytesPerWord == 8) { + while ((c->code.length() + 2) % 8) { + c->code.append(0x90); + } + } else { + while ((c->code.length() + 1) % 4) { + c->code.append(0x90); + } + } + callC(c, size, a); +} + +void +callR(Context* c, unsigned size UNUSED, Assembler::Register* a) +{ + assert(c, size == BytesPerWord); + + if (a->low & 8) rex(c, 0x40, a->low); + c->code.append(0xff); + c->code.append(0xd0 | (a->low & 7)); +} + +void +callM(Context* c, unsigned size UNUSED, Assembler::Memory* a) +{ + assert(c, size == BytesPerWord); + + encode(c, 0xff, 2, a, false); +} + +void +jumpR(Context* c, unsigned size UNUSED, Assembler::Register* a) +{ + assert(c, size == BytesPerWord); + + if (a->low & 8) rex(c, 0x40, a->low); + c->code.append(0xff); + c->code.append(0xe0 | (a->low & 7)); +} + +void +jumpC(Context* c, unsigned size, Assembler::Constant* a) +{ + assert(c, size == BytesPerWord); + + if (BytesPerWord == 8) { + Assembler::Register r(r10); + moveCR(c, size, a, &r); + jumpR(c, size, &r); + } else { + unconditional(c, 0xe9, a); + } +} + +void +jumpM(Context* c, unsigned size UNUSED, Assembler::Memory* a) +{ + assert(c, size == BytesPerWord); + + encode(c, 0xff, 4, a, false); +} + +void +jumpIfEqualC(Context* c, unsigned size UNUSED, Assembler::Constant* a) +{ + assert(c, size == BytesPerWord); + + conditional(c, 0x84, a); +} + +void +jumpIfNotEqualC(Context* c, unsigned size UNUSED, Assembler::Constant* a) +{ + assert(c, size == BytesPerWord); + + conditional(c, 0x85, a); +} + +void +jumpIfGreaterC(Context* c, unsigned size UNUSED, Assembler::Constant* a) +{ + assert(c, size == BytesPerWord); + + conditional(c, 0x8f, a); +} + +void +jumpIfGreaterOrEqualC(Context* c, unsigned size UNUSED, Assembler::Constant* a) +{ + assert(c, size == BytesPerWord); + + conditional(c, 0x8d, a); +} + +void +jumpIfLessC(Context* c, unsigned size UNUSED, Assembler::Constant* a) +{ + assert(c, size == BytesPerWord); + + conditional(c, 0x8c, a); +} + +void +jumpIfLessOrEqualC(Context* c, unsigned size UNUSED, Assembler::Constant* a) +{ + assert(c, size == BytesPerWord); + + conditional(c, 0x8e, a); +} + +void +pushR(Context*, unsigned, Assembler::Register*); + +void +pushC(Context* c, unsigned size, Assembler::Constant* a) +{ + if (BytesPerWord == 4 and size == 8) { + int64_t v = a->value->value(); + + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + pushC(c, 4, &ah); + pushC(c, 4, &al); + } else { + if (a->value->resolved()) { + int64_t v = a->value->value(); + if (isInt8(v)) { + c->code.append(0x6a); + c->code.append(v); + } else if (isInt32(v)) { + c->code.append(0x68); + c->code.append4(v); + } else { + Assembler::Register tmp(c->client->acquireTemporary()); + moveCR(c, size, a, &tmp); + pushR(c, size, &tmp); + c->client->releaseTemporary(tmp.low); + } + } else { + if (BytesPerWord == 4) { + c->code.append(0x68); + appendImmediateTask(c, a->value, c->code.length()); + c->code.appendAddress(static_cast(0)); + } else { + Assembler::Register tmp(c->client->acquireTemporary()); + moveCR(c, size, a, &tmp); + pushR(c, size, &tmp); + c->client->releaseTemporary(tmp.low); + } + } + } +} + +void +moveAR(Context*, unsigned, Assembler::Address*, Assembler::Register* b); + +void +pushA(Context* c, unsigned size, Assembler::Address* a) +{ + assert(c, BytesPerWord == 8 or size == 4); // todo + + Assembler::Register tmp(c->client->acquireTemporary()); + moveAR(c, size, a, &tmp); + pushR(c, size, &tmp); + c->client->releaseTemporary(tmp.low); +} + +void +pushR(Context* c, unsigned size, Assembler::Register* a) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + + pushR(c, 4, &ah); + pushR(c, 4, a); + } else { + c->code.append(0x50 | a->low); + } +} + +void +pushM(Context* c, unsigned size, Assembler::Memory* a) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Memory ah(a->base, a->offset + 4, a->index, a->scale); + + pushM(c, 4, &ah); + pushM(c, 4, a); + } else { + assert(c, BytesPerWord == 4 or size == 8); + + encode(c, 0xff, 6, a, false); + } +} + +void +move4To8RR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b); + +void +popR(Context* c, unsigned size, Assembler::Register* a) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + + popR(c, 4, a); + popR(c, 4, &ah); + } else { + c->code.append(0x58 | a->low); + if (BytesPerWord == 8 and size == 4) { + move4To8RR(c, 0, a, a); + } + } +} + +void +popM(Context* c, unsigned size, Assembler::Memory* a) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Memory ah(a->base, a->offset + 4, a->index, a->scale); + + popM(c, 4, a); + popM(c, 4, &ah); + } else { + assert(c, BytesPerWord == 4 or size == 8); + + encode(c, 0x8f, 0, a, false); + } +} + +void +addCarryCR(Context* c, unsigned size UNUSED, Assembler::Constant* a, + Assembler::Register* b) +{ + assert(c, BytesPerWord == 8 or size == 4); + + int64_t v = a->value->value(); + if (isInt8(v)) { + c->code.append(0x83); + c->code.append(0xd0 | b->low); + c->code.append(v); + } else { + abort(c); + } +} + +void +moveRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b); + +void +xorRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b); + +void +negateR(Context* c, unsigned size, Assembler::Register* a) +{ + if (BytesPerWord == 4 and size == 8) { + assert(c, a->low == rax and a->high == rdx); + + ResolvedPromise zeroPromise(0); + Assembler::Constant zero(&zeroPromise); + + Assembler::Register ah(a->high); + + negateR(c, 4, a); + addCarryCR(c, 4, &zero, &ah); + negateR(c, 4, &ah); + } else { + if (size == 8) rex(c); + c->code.append(0xf7); + c->code.append(0xd8 | a->low); + } +} + +void +leaMR(Context* c, unsigned size, Assembler::Memory* b, Assembler::Register* a) +{ + if (BytesPerWord == 8 and size == 4) { + encode(c, 0x8d, a->low, b, false); + } else { + assert(c, BytesPerWord == 8 or size == 4); + + encode(c, 0x8d, a->low, b, true); + } +} + +void +moveCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + int64_t v = a->value->value(); + + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + Assembler::Register bh(b->high); + + moveCR(c, 4, &al, b); + moveCR(c, 4, &ah, &bh); + } else { + rex(c, 0x48, b->low); + c->code.append(0xb8 | b->low); + if (a->value->resolved()) { + c->code.appendAddress(a->value->value()); + } else { + appendImmediateTask(c, a->value, c->code.length()); + c->code.appendAddress(static_cast(0)); + } + } +} + +void +moveCM(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Memory* b) +{ + int64_t v = a->value->value(); + + switch (size) { + case 1: + encode(c, 0xc6, 0, b, false); + c->code.append(a->value->value()); + break; + + case 2: + encode2(c, 0x66c7, 0, b, false); + c->code.append2(a->value->value()); + break; + + case 4: + encode(c, 0xc7, 0, b, false); + c->code.append4(a->value->value()); + break; + + case 8: { + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + Assembler::Memory bh(b->base, b->offset + 4, b->index, b->scale); + + moveCM(c, 4, &al, b); + moveCM(c, 4, &ah, &bh); + } break; + + default: abort(c); + } +} + +void +moveRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + Assembler::Register bh(b->high); + + moveRR(c, 4, a, b); + moveRR(c, 4, &ah, &bh); + } else { + switch (size) { + case 1: + if (BytesPerWord == 4 and a->low > rbx) { + assert(c, b->low <= rbx); + + moveRR(c, BytesPerWord, a, b); + moveRR(c, 1, b, b); + } else { + rex(c); + c->code.append(0x0f); + c->code.append(0xbe); + c->code.append(0xc0 | (b->low << 3) | a->low); + } + break; + + case 2: + rex(c); + c->code.append(0x0f); + c->code.append(0xbf); + c->code.append(0xc0 | (b->low << 3) | a->low); + break; + + case 8: + case 4: + if (a->low != b->low) { + rex(c); + c->code.append(0x89); + c->code.append(0xc0 | (a->low << 3) | b->low); + } + break; + } + } +} + +void +moveRM(Context* c, unsigned size, Assembler::Register* a, Assembler::Memory* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + Assembler::Memory bh(b->base, b->offset + 4, b->index, b->scale); + + moveRM(c, 4, a, b); + moveRM(c, 4, &ah, &bh); + } else if (BytesPerWord == 8 and size == 4) { + encode(c, 0x89, a->low, b, false); + } else { + switch (size) { + case 1: + if (BytesPerWord == 8) { + if (a->low > rbx) { + encode2(c, 0x4088, a->low, b, false); + } else { + encode(c, 0x88, a->low, b, false); + } + } else { + assert(c, a->low <= rbx); + + encode(c, 0x88, a->low, b, false); + } + break; + + case 2: + encode2(c, 0x6689, a->low, b, false); + break; + + case BytesPerWord: + encode(c, 0x89, a->low, b, true); + break; + + default: abort(c); + } + } +} + +void +move4To8RR(Context* c, unsigned, Assembler::Register* a, + Assembler::Register* b) +{ + if (BytesPerWord == 8) { + rex(c); + c->code.append(0x63); + c->code.append(0xc0 | (b->low << 3) | a->low); + } else { + if (a->low == rax and b->low == rax and b->high == rdx) { + c->code.append(0x99); // cdq + } else { + assert(c, b->low == rax and b->high == rdx); + + moveRR(c, 4, a, b); + move4To8RR(c, 0, b, b); + } + } +} + +void +moveMR(Context* c, unsigned size, Assembler::Memory* a, Assembler::Register* b) +{ + switch (size) { + case 1: + encode2(c, 0x0fbe, b->low, a, true); + break; + + case 2: + encode2(c, 0x0fbf, b->low, a, true); + break; + + case 4: + case 8: + if (BytesPerWord == 4 and size == 8) { + Assembler::Memory ah(a->base, a->offset + 4, a->index, a->scale); + Assembler::Register bh(b->high); + + moveMR(c, 4, a, b); + moveMR(c, 4, &ah, &bh); + } else if (BytesPerWord == 8 and size == 4) { + encode(c, 0x63, b->low, a, true); + } else { + encode(c, 0x8b, b->low, a, true); + } + break; + + default: abort(c); + } +} + +void +moveAR(Context* c, unsigned size, Assembler::Address* a, + Assembler::Register* b) +{ + assert(c, BytesPerWord == 8 or size == 4); // todo + + Assembler::Constant constant(a->address); + Assembler::Memory memory(b->low, 0, -1, 0); + + moveCR(c, size, &constant, b); + moveMR(c, size, &memory, b); +} + +void +moveAM(Context* c, unsigned size, Assembler::Address* a, + Assembler::Memory* b) +{ + assert(c, BytesPerWord == 8 or size == 4); // todo + + Assembler::Register tmp(c->client->acquireTemporary()); + moveAR(c, size, a, &tmp); + moveRM(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); +} + +void +moveMM(Context* c, unsigned size, Assembler::Memory* a, + Assembler::Memory* b) +{ + if (BytesPerWord == 8 or size <= 4) { + uint32_t mask; + if (BytesPerWord == 4 and size == 1) { + mask = (1 << rax) | (1 << rcx) | (1 << rdx) | (1 << rbx); + } else { + mask = ~static_cast(0); + } + + Assembler::Register tmp(c->client->acquireTemporary(mask)); + moveMR(c, size, a, &tmp); + moveRM(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); + } else { + Assembler::Register tmp(c->client->acquireTemporary(), + c->client->acquireTemporary()); + moveMR(c, size, a, &tmp); + moveRM(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); + c->client->releaseTemporary(tmp.high); + } +} + +void +move4To8MR(Context* c, unsigned, Assembler::Memory* a, Assembler::Register* b) +{ + if (BytesPerWord == 8) { + encode(c, 0x63, b->low, a, true); + } else { + assert(c, b->low == rax and b->high == rdx); + + moveMR(c, 4, a, b); + move4To8RR(c, 0, b, b); + } +} + +void +moveZMR(Context* c, unsigned size, Assembler::Memory* a, + Assembler::Register* b) +{ + switch (size) { + case 2: + encode2(c, 0x0fb7, b->low, a, true); + break; + + default: abort(c); // todo + } +} + +void +moveZRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + switch (size) { + case 2: + rex(c); + c->code.append(0x0f); + c->code.append(0xb7); + c->code.append(0xc0 | (b->low << 3) | a->low); + break; + + default: abort(c); // todo + } +} + +void +swapRR(Context* c, unsigned, Assembler::Register* a, Assembler::Register* b) +{ + rex(c); + c->code.append(0x87); + c->code.append(0xc0 | (b->low << 3) | a->low); +} + +void +addCM(Context* c, unsigned size UNUSED, Assembler::Constant* a, + Assembler::Memory* b) +{ + assert(c, BytesPerWord == 8 or size == 4); // todo + + int64_t v = a->value->value(); + unsigned i = (isInt8(v) ? 0x83 : 0x81); + + encode(c, i, 0, b, true); + if (isInt8(v)) { + c->code.append(v); + } else if (isInt32(v)) { + c->code.append4(v); + } else { + abort(c); + } +} + +void +addCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + int64_t v = a->value->value(); + if (v) { + if (BytesPerWord == 4 and size == 8) { + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + Assembler::Register bh(b->high); + + addCR(c, 4, &al, b); + addCarryCR(c, 4, &ah, &bh); + } else { + if (size == 8) rex(c); + if (isInt8(v)) { + c->code.append(0x83); + c->code.append(0xc0 | b->low); + c->code.append(v); + } else if (isInt32(v)) { + c->code.append(0x81); + c->code.append(0xc0 | b->low); + c->code.append4(v); + } else { + abort(c); + } + } + } +} + +void +subtractBorrowCR(Context* c, unsigned size UNUSED, Assembler::Constant* a, + Assembler::Register* b) +{ + assert(c, BytesPerWord == 8 or size == 4); + + int64_t v = a->value->value(); + if (isInt8(v)) { + c->code.append(0x83); + c->code.append(0xd8 | b->low); + c->code.append(v); + } else { + abort(c); + } +} + +void +subtractCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + int64_t v = a->value->value(); + if (v) { + if (BytesPerWord == 4 and size == 8) { + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + Assembler::Register bh(b->high); + + subtractCR(c, 4, &al, b); + subtractBorrowCR(c, 4, &ah, &bh); + } else { + if (size == 8) rex(c); + if (isInt8(v)) { + c->code.append(0x83); + c->code.append(0xe8 | b->low); + c->code.append(v); + } else if (isInt32(v)) { + c->code.append(0x81); + c->code.append(0xe8 | b->low); + c->code.append4(v); + } else { + abort(c); + } + } + } +} + +void +subtractBorrowRR(Context* c, unsigned size UNUSED, Assembler::Register* a, + Assembler::Register* b) +{ + assert(c, BytesPerWord == 8 or size == 4); + + if (size == 8) rex(c); + c->code.append(0x19); + c->code.append(0xc0 | (a->low << 3) | b->low); +} + +void +subtractRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + Assembler::Register bh(b->high); + + subtractRR(c, 4, a, b); + subtractBorrowRR(c, 4, &ah, &bh); + } else { + if (size == 8) rex(c); + c->code.append(0x29); + c->code.append(0xc0 | (a->low << 3) | b->low); + } +} + +void +addCarryRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + assert(c, BytesPerWord == 8 or size == 4); + + if (size == 8) rex(c); + c->code.append(0x11); + c->code.append(0xc0 | (a->low << 3) | b->low); +} + +void +addRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + Assembler::Register bh(b->high); + + addRR(c, 4, a, b); + addCarryRR(c, 4, &ah, &bh); + } else { + if (size == 8) rex(c); + c->code.append(0x01); + c->code.append(0xc0 | (a->low << 3) | b->low); + } +} + +void +addRM(Context* c, unsigned size UNUSED, Assembler::Register* a, + Assembler::Memory* b) +{ + assert(c, BytesPerWord == 8 or size == 4); + + encode(c, 0x01, a->low, b, true); +} + +void +multiplyRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + assert(c, b->high == rdx); + assert(c, b->low != rax); + assert(c, a->low != rax); + assert(c, a->high != rax); + + c->client->save(rax); + + Assembler::Register axdx(rax, rdx); + Assembler::Register ah(a->high); + Assembler::Register bh(b->high); + + moveRR(c, 4, b, &axdx); + multiplyRR(c, 4, &ah, b); + multiplyRR(c, 4, a, &bh); + addRR(c, 4, &bh, b); + + // mul a->low,%eax%edx + c->code.append(0xf7); + c->code.append(0xe0 | a->low); + + addRR(c, 4, b, &bh); + moveRR(c, 4, &axdx, b); + + c->client->restore(rax); + } else { + if (size == 8) rex(c); + c->code.append(0x0f); + c->code.append(0xaf); + c->code.append(0xc0 | (b->low << 3) | a->low); + } +} + +void +multiplyCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + const uint32_t mask = ~((1 << rax) | (1 << rdx)); + Assembler::Register tmp(c->client->acquireTemporary(mask), + c->client->acquireTemporary(mask)); + + moveCR(c, size, a, &tmp); + multiplyRR(c, size, &tmp, b); + + c->client->releaseTemporary(tmp.low); + c->client->releaseTemporary(tmp.high); + } else { + int64_t v = a->value->value(); + if (v) { + if (isInt32(v)) { + if (size == 8) rex(c); + if (isInt8(v)) { + c->code.append(0x6b); + c->code.append(0xc0 | (b->low << 3) | b->low); + c->code.append(v); + } else { + c->code.append(0x69); + c->code.append(0xc0 | (b->low << 3) | b->low); + c->code.append4(v); + } + } else { + Assembler::Register tmp(c->client->acquireTemporary()); + moveCR(c, size, a, &tmp); + multiplyRR(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); + } + } + } +} + +void +divideRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b UNUSED) +{ + assert(c, BytesPerWord == 8 or size == 4); + + assert(c, b->low == rax); + assert(c, a->low != rdx); + + c->client->save(rdx); + + if (size == 8) rex(c); + c->code.append(0x99); // cdq + if (size == 8) rex(c); + c->code.append(0xf7); + c->code.append(0xf8 | a->low); + + c->client->restore(rdx); +} + +void +divideCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + assert(c, BytesPerWord == 8 or size == 4); + + const uint32_t mask = ~((1 << rax) | (1 << rdx)); + Assembler::Register tmp(c->client->acquireTemporary(mask)); + moveCR(c, size, a, &tmp); + divideRR(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); +} + +void +remainderRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + assert(c, BytesPerWord == 8 or size == 4); + + assert(c, b->low == rax); + assert(c, a->low != rdx); + + c->client->save(rdx); + + if (size == 8) rex(c); + c->code.append(0x99); // cdq + if (size == 8) rex(c); + c->code.append(0xf7); + c->code.append(0xf8 | a->low); + + Assembler::Register dx(rdx); + moveRR(c, BytesPerWord, &dx, b); + + c->client->restore(rdx); +} + +void +remainderCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + assert(c, BytesPerWord == 8 or size == 4); + + const uint32_t mask = ~((1 << rax) | (1 << rdx)); + Assembler::Register tmp(c->client->acquireTemporary(mask)); + moveCR(c, size, a, &tmp); + remainderRR(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); +} + +void +andRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + Assembler::Register bh(b->high); + + andRR(c, 4, a, b); + andRR(c, 4, &ah, &bh); + } else { + if (size == 8) rex(c); + c->code.append(0x21); + c->code.append(0xc0 | (a->low << 3) | b->low); + } +} + +void +andCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + int64_t v = a->value->value(); + + if (BytesPerWord == 4 and size == 8) { + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + Assembler::Register bh(b->high); + + andCR(c, 4, &al, b); + andCR(c, 4, &ah, &bh); + } else { + if (isInt32(v)) { + if (size == 8) rex(c); + if (isInt8(v)) { + c->code.append(0x83); + c->code.append(0xe0 | b->low); + c->code.append(v); + } else { + c->code.append(0x81); + c->code.append(0xe0 | b->low); + c->code.append4(v); + } + } else { + Assembler::Register tmp(c->client->acquireTemporary()); + moveCR(c, size, a, &tmp); + andRR(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); + } + } +} + +void +andCM(Context* c, unsigned size UNUSED, Assembler::Constant* a, + Assembler::Memory* b) +{ + assert(c, BytesPerWord == 8 or size == 4); + + int64_t v = a->value->value(); + + encode(c, isInt8(a->value->value()) ? 0x83 : 0x81, 5, b, true); + if (isInt8(v)) { + c->code.append(v); + } else if (isInt32(v)) { + c->code.append4(v); + } else { + abort(c); + } +} + +void +orRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + Assembler::Register bh(b->high); + + orRR(c, 4, a, b); + orRR(c, 4, &ah, &bh); + } else { + if (size == 8) rex(c); + c->code.append(0x09); + c->code.append(0xc0 | (a->low << 3) | b->low); + } +} + +void +orCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + int64_t v = a->value->value(); + if (v) { + if (BytesPerWord == 4 and size == 8) { + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + Assembler::Register bh(b->high); + + orCR(c, 4, &al, b); + orCR(c, 4, &ah, &bh); + } else { + if (isInt32(v)) { + if (size == 8) rex(c); + if (isInt8(v)) { + c->code.append(0x83); + c->code.append(0xc8 | b->low); + c->code.append(v); + } else { + c->code.append(0x81); + c->code.append(0xc8 | b->low); + c->code.append4(v); + } + } else { + Assembler::Register tmp(c->client->acquireTemporary()); + moveCR(c, size, a, &tmp); + orRR(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); + } + } + } +} + +void +xorRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + Assembler::Register bh(b->high); + + xorRR(c, 4, a, b); + xorRR(c, 4, &ah, &bh); + } else { + if (size == 8) rex(c); + c->code.append(0x31); + c->code.append(0xc0 | (a->low << 3) | b->low); + } +} + +void +xorCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + int64_t v = a->value->value(); + if (v) { + if (BytesPerWord == 4 and size == 8) { + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + Assembler::Register bh(b->high); + + xorCR(c, 4, &al, b); + xorCR(c, 4, &ah, &bh); + } else { + if (isInt32(v)) { + if (size == 8) rex(c); + if (isInt8(v)) { + c->code.append(0x83); + c->code.append(0xf0 | b->low); + c->code.append(v); + } else { + c->code.append(0x81); + c->code.append(0xf0 | b->low); + c->code.append4(v); + } + } else { + Assembler::Register tmp(c->client->acquireTemporary()); + moveCR(c, size, a, &tmp); + xorRR(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); + } + } + } +} + +void +doShift(Context* c, void (*shift) + (Context*, unsigned, Assembler::Register*, Assembler::Register*), + int type, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + int64_t v = a->value->value(); + + if (BytesPerWord == 4 and size == 8) { + c->client->save(rcx); + + Assembler::Register cx(rcx); + moveCR(c, 4, a, &cx); + shift(c, size, &cx, b); + + c->client->restore(rcx); + } else { + if (size == 8) rex(c); + if (v == 1) { + c->code.append(0xd1); + c->code.append(type | b->low); + } else if (isInt8(v)) { + c->code.append(0xc1); + c->code.append(type | b->low); + c->code.append(v); + } else { + abort(c); + } + } +} + +void +compareCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b); + +void +shiftLeftRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + assert(c, a->low == rcx); + + if (BytesPerWord == 4 and size == 8) { + // shld + c->code.append(0x0f); + c->code.append(0xa5); + c->code.append(0xc0 | (b->low << 3) | b->high); + + // shl + c->code.append(0xd3); + c->code.append(0xe0 | b->low); + + ResolvedPromise promise(32); + Assembler::Constant constant(&promise); + compareCR(c, 4, &constant, a); + + c->code.append(0x0f); + c->code.append(0x8c); // jl + c->code.append4(2 + 2); + + Assembler::Register bh(b->high); + moveRR(c, 4, b, &bh); // 2 bytes + xorRR(c, 4, b, b); // 2 bytes + } else { + if (size == 8) rex(c); + c->code.append(0xd3); + c->code.append(0xe0 | b->low); + } +} + +void +shiftLeftCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + doShift(c, shiftLeftRR, 0xe0, size, a, b); +} + +void +shiftRightRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + assert(c, a->low == rcx); + + if (BytesPerWord == 4 and size == 8) { + // shrd + c->code.append(0x0f); + c->code.append(0xad); + c->code.append(0xc0 | (b->high << 3) | b->low); + + // sar + c->code.append(0xd3); + c->code.append(0xf8 | b->high); + + ResolvedPromise promise(32); + Assembler::Constant constant(&promise); + compareCR(c, 4, &constant, a); + + c->code.append(0x0f); + c->code.append(0x8c); // jl + c->code.append4(2 + 3); + + Assembler::Register bh(b->high); + moveRR(c, 4, &bh, b); // 2 bytes + + // sar 31,high + c->code.append(0xc1); + c->code.append(0xf8 | b->high); + c->code.append(31); + } else { + if (size == 8) rex(c); + c->code.append(0xd3); + c->code.append(0xf8 | b->low); + } +} + +void +shiftRightCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + doShift(c, shiftRightRR, 0xf8, size, a, b); +} + +void +unsignedShiftRightRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + assert(c, a->low == rcx); + + if (BytesPerWord == 4 and size == 8) { + // shrd + c->code.append(0x0f); + c->code.append(0xad); + c->code.append(0xc0 | (b->high << 3) | b->low); + + // shr + c->code.append(0xd3); + c->code.append(0xe8 | b->high); + + ResolvedPromise promise(32); + Assembler::Constant constant(&promise); + compareCR(c, 4, &constant, a); + + c->code.append(0x0f); + c->code.append(0x8c); // jl + c->code.append4(2 + 2); + + Assembler::Register bh(b->high); + moveRR(c, 4, &bh, b); // 2 bytes + xorRR(c, 4, &bh, &bh); // 2 bytes + } else { + if (size == 8) rex(c); + c->code.append(0xd3); + c->code.append(0xe8 | b->low); + } +} + +void +unsignedShiftRightCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + doShift(c, unsignedShiftRightRR, 0xe8, size, a, b); +} + +void +multiwordCompare(Context* c, Assembler::Operand* al, Assembler::Operand* ah, + Assembler::Operand* bl, Assembler::Operand* bh, + BinaryOperationType op) +{ + op(c, BytesPerWord, ah, bh); + + // if the high order bits are equal, we compare the low order + // bits; otherwise, we jump past that comparison + c->code.append(0x0f); + c->code.append(0x85); // jne + + unsigned comparisonOffset = c->code.length(); + c->code.append4(0); + + op(c, BytesPerWord, al, bl); + + int32_t comparisonSize = c->code.length() - comparisonOffset - 4; + c->code.set(comparisonOffset, &comparisonSize, 4); +} + +void +compareRR(Context* c, unsigned size, Assembler::Register* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + Assembler::Register bh(b->high); + + multiwordCompare(c, a, &ah, b, &bh, CAST2(compareRR)); + } else { + if (size == 8) rex(c); + c->code.append(0x39); + c->code.append(0xc0 | (a->low << 3) | b->low); + } +} + +void +compareAR(Context* c, unsigned size, Assembler::Address* a, + Assembler::Register* b) +{ + assert(c, BytesPerWord == 8 or size == 4); // todo + + Assembler::Register tmp(c->client->acquireTemporary()); + moveAR(c, size, a, &tmp); + compareRR(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); +} + +void +compareCR(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Register* b) +{ + int64_t v = a->value->value(); + + if (BytesPerWord == 4 and size == 8) { + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + Assembler::Register bh(b->high); + + multiwordCompare(c, &al, &ah, b, &bh, CAST2(compareCR)); + } else { + if (isInt32(v)) { + if (size == 8) rex(c); + if (isInt8(v)) { + c->code.append(0x83); + c->code.append(0xf8 | b->low); + c->code.append(v); + } else { + c->code.append(0x81); + c->code.append(0xf8 | b->low); + c->code.append4(v); + } + } else { + Assembler::Register tmp(c->client->acquireTemporary()); + moveCR(c, size, a, &tmp); + compareRR(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); + } + } +} + +void +compareCM(Context* c, unsigned size, Assembler::Constant* a, + Assembler::Memory* b) +{ + int64_t v = a->value->value(); + + if (BytesPerWord == 4 and size == 8) { + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant al(&low); + + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant ah(&high); + + Assembler::Memory bh(b->base, b->offset + 4, b->index, b->scale); + + multiwordCompare(c, &al, &ah, b, &bh, CAST2(compareCM)); + } else { + encode(c, isInt8(v) ? 0x83 : 0x81, 7, b, true); + + if (isInt8(v)) { + c->code.append(v); + } else if (isInt32(v)) { + c->code.append4(v); + } else { + abort(c); + } + } +} + +void +compareRM(Context* c, unsigned size, Assembler::Register* a, + Assembler::Memory* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + Assembler::Memory bh(b->base, b->offset + 4, b->index, b->scale); + + multiwordCompare(c, a, &ah, b, &bh, CAST2(compareRM)); + } else { + if (BytesPerWord == 8 and size == 4) { + move4To8RR(c, size, a, a); + } + encode(c, 0x39, a->low, b, true); + } +} + +void +compareMR(Context* c, unsigned size, Assembler::Memory* a, + Assembler::Register* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Memory ah(a->base, a->offset + 4, a->index, a->scale); + Assembler::Register bh(b->high); + + multiwordCompare(c, a, &ah, b, &bh, CAST2(compareMR)); + } else { + if (BytesPerWord == 8 and size == 4) { + move4To8RR(c, size, b, b); + } + encode(c, 0x3b, b->low, a, true); + } +} + +void +compareMM(Context* c, unsigned size, Assembler::Memory* a, + Assembler::Memory* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Memory ah(a->base, a->offset + 4, a->index, a->scale); + Assembler::Memory bh(b->base, b->offset + 4, b->index, b->scale); + + multiwordCompare(c, a, &ah, b, &bh, CAST2(compareMM)); + } else { + Assembler::Register tmp(c->client->acquireTemporary()); + moveMR(c, size, a, &tmp); + compareRM(c, size, &tmp, b); + c->client->releaseTemporary(tmp.low); + } +} + +void +compareRC(Context* c, unsigned size, Assembler::Register* a, + Assembler::Constant* b) +{ + if (BytesPerWord == 4 and size == 8) { + Assembler::Register ah(a->high); + + int64_t v = b->value->value(); + + ResolvedPromise low(v & 0xFFFFFFFF); + Assembler::Constant bl(&low); + + ResolvedPromise high((v >> 32) & 0xFFFFFFFF); + Assembler::Constant bh(&high); + + multiwordCompare(c, a, &ah, &bl, &bh, CAST2(compareRC)); + } else { + Assembler::Register tmp(c->client->acquireTemporary()); + moveCR(c, size, b, &tmp); + compareRR(c, size, a, &tmp); + c->client->releaseTemporary(tmp.low); + } +} + +void +populateTables() +{ + Operations[Return] = return_; + + const int Constant = ConstantOperand; + const int Address = AddressOperand; + const int Register = RegisterOperand; + const int Memory = MemoryOperand; + + UnaryOperations[INDEX1(Call, Constant)] = CAST1(callC); + UnaryOperations[INDEX1(Call, Register)] = CAST1(callR); + UnaryOperations[INDEX1(Call, Memory)] = CAST1(callM); + + UnaryOperations[INDEX1(AlignedCall, Constant)] = CAST1(alignedCallC); + + UnaryOperations[INDEX1(Jump, Register)] = CAST1(jumpR); + UnaryOperations[INDEX1(Jump, Constant)] = CAST1(jumpC); + UnaryOperations[INDEX1(Jump, Memory)] = CAST1(jumpM); + + UnaryOperations[INDEX1(JumpIfEqual, Constant)] = CAST1(jumpIfEqualC); + UnaryOperations[INDEX1(JumpIfNotEqual, Constant)] = CAST1(jumpIfNotEqualC); + UnaryOperations[INDEX1(JumpIfGreater, Constant)] = CAST1(jumpIfGreaterC); + UnaryOperations[INDEX1(JumpIfGreaterOrEqual, Constant)] + = CAST1(jumpIfGreaterOrEqualC); + UnaryOperations[INDEX1(JumpIfLess, Constant)] = CAST1(jumpIfLessC); + UnaryOperations[INDEX1(JumpIfLessOrEqual, Constant)] + = CAST1(jumpIfLessOrEqualC); + + UnaryOperations[INDEX1(Push, Constant)] = CAST1(pushC); + UnaryOperations[INDEX1(Push, Address)] = CAST1(pushA); + UnaryOperations[INDEX1(Push, Register)] = CAST1(pushR); + UnaryOperations[INDEX1(Push, Memory)] = CAST1(pushM); + + UnaryOperations[INDEX1(Pop, Register)] = CAST1(popR); + UnaryOperations[INDEX1(Pop, Memory)] = CAST1(popM); + + UnaryOperations[INDEX1(Negate, Register)] = CAST1(negateR); + + BinaryOperations[INDEX2(LoadAddress, Memory, Register)] = CAST2(leaMR); + + BinaryOperations[INDEX2(Move, Constant, Register)] = CAST2(moveCR); + BinaryOperations[INDEX2(Move, Constant, Memory)] = CAST2(moveCM); + BinaryOperations[INDEX2(Move, Register, Memory)] = CAST2(moveRM); + BinaryOperations[INDEX2(Move, Register, Register)] = CAST2(moveRR); + BinaryOperations[INDEX2(Move, Memory, Register)] = CAST2(moveMR); + BinaryOperations[INDEX2(Move, Address, Register)] = CAST2(moveAR); + BinaryOperations[INDEX2(Move, Address, Memory)] = CAST2(moveAM); + BinaryOperations[INDEX2(Move, Memory, Memory)] = CAST2(moveMM); + + BinaryOperations[INDEX2(Move4To8, Register, Register)] = CAST2(move4To8RR); + BinaryOperations[INDEX2(Move4To8, Memory, Register)] = CAST2(move4To8MR); + + BinaryOperations[INDEX2(MoveZ, Memory, Register)] = CAST2(moveZMR); + BinaryOperations[INDEX2(MoveZ, Register, Register)] = CAST2(moveZRR); + + BinaryOperations[INDEX2(Swap, Register, Register)] = CAST2(swapRR); + + BinaryOperations[INDEX2(Add, Constant, Register)] = CAST2(addCR); + BinaryOperations[INDEX2(Add, Register, Register)] = CAST2(addRR); + BinaryOperations[INDEX2(Add, Register, Memory)] = CAST2(addRM); + BinaryOperations[INDEX2(Add, Constant, Memory)] = CAST2(addCM); + + BinaryOperations[INDEX2(Multiply, Register, Register)] = CAST2(multiplyRR); + BinaryOperations[INDEX2(Multiply, Constant, Register)] = CAST2(multiplyCR); + + BinaryOperations[INDEX2(Divide, Register, Register)] = CAST2(divideRR); + BinaryOperations[INDEX2(Divide, Constant, Register)] = CAST2(divideCR); + + BinaryOperations[INDEX2(Remainder, Constant, Register)] = CAST2(remainderCR); + BinaryOperations[INDEX2(Remainder, Register, Register)] = CAST2(remainderRR); + + BinaryOperations[INDEX2(And, Register, Register)] = CAST2(andRR); + BinaryOperations[INDEX2(And, Constant, Register)] = CAST2(andCR); + BinaryOperations[INDEX2(And, Constant, Memory)] = CAST2(andCM); + + BinaryOperations[INDEX2(Or, Register, Register)] = CAST2(orRR); + BinaryOperations[INDEX2(Or, Constant, Register)] = CAST2(orCR); + + BinaryOperations[INDEX2(Xor, Register, Register)] = CAST2(xorRR); + BinaryOperations[INDEX2(Xor, Constant, Register)] = CAST2(xorCR); + + BinaryOperations[INDEX2(ShiftLeft, Register, Register)] = CAST2(shiftLeftRR); + BinaryOperations[INDEX2(ShiftLeft, Constant, Register)] = CAST2(shiftLeftCR); + + BinaryOperations[INDEX2(ShiftRight, Register, Register)] + = CAST2(shiftRightRR); + BinaryOperations[INDEX2(ShiftRight, Constant, Register)] + = CAST2(shiftRightCR); + + BinaryOperations[INDEX2(UnsignedShiftRight, Register, Register)] + = CAST2(unsignedShiftRightRR); + BinaryOperations[INDEX2(UnsignedShiftRight, Constant, Register)] + = CAST2(unsignedShiftRightCR); + + BinaryOperations[INDEX2(Subtract, Constant, Register)] = CAST2(subtractCR); + BinaryOperations[INDEX2(Subtract, Register, Register)] = CAST2(subtractRR); + + BinaryOperations[INDEX2(Compare, Constant, Register)] = CAST2(compareCR); + BinaryOperations[INDEX2(Compare, Register, Constant)] = CAST2(compareRC); + BinaryOperations[INDEX2(Compare, Register, Register)] = CAST2(compareRR); + BinaryOperations[INDEX2(Compare, Address, Register)] = CAST2(compareAR); + BinaryOperations[INDEX2(Compare, Register, Memory)] = CAST2(compareRM); + BinaryOperations[INDEX2(Compare, Memory, Register)] = CAST2(compareMR); + BinaryOperations[INDEX2(Compare, Constant, Memory)] = CAST2(compareCM); + BinaryOperations[INDEX2(Compare, Memory, Memory)] = CAST2(compareMM); +} + +class MyAssembler: public Assembler { + public: + MyAssembler(System* s, Allocator* a, Zone* zone): c(s, a, zone) { + static bool populated = false; + if (not populated) { + populated = true; + populateTables(); + } + } + + virtual void setClient(Client* client) { + assert(&c, c.client == 0); + c.client = client; + } + + virtual unsigned registerCount() { + return 8;//BytesPerWord == 4 ? 8 : 16; + } + + virtual int base() { + return rbp; + } + + virtual int stack() { + return rsp; + } + + virtual int thread() { + return rbx; + } + + virtual int returnLow() { + return rax; + } + + virtual int returnHigh() { + return (BytesPerWord == 4 ? rdx : NoRegister); + } + + virtual unsigned argumentRegisterCount() { + return (BytesPerWord == 4 ? 0 : 6); + } + + virtual int argumentRegister(unsigned index) { + assert(&c, BytesPerWord == 8); + + switch (index) { + case 0: + return rdi; + case 1: + return rsi; + case 2: + return rdx; + case 3: + return rcx; + case 4: + return r8; + case 5: + return r9; + default: + abort(&c); + } + } + + virtual void plan(UnaryOperation op, unsigned size, uint8_t* typeMask, + uint64_t* registerMask, bool* thunk) + { + if (op == Negate and BytesPerWord == 4 and size == 8) { + *typeMask = 1 << RegisterOperand; + *registerMask = (static_cast(1) << (rdx + 32)) + | (static_cast(1) << rax); + } else { + *typeMask = (1 << RegisterOperand) | (1 << MemoryOperand); + *registerMask = ~static_cast(0); + } + *thunk = false; + } + + virtual void plan(BinaryOperation op, unsigned size, uint8_t* aTypeMask, + uint64_t* aRegisterMask, uint8_t* bTypeMask, + uint64_t* bRegisterMask, bool* thunk) + { + *aTypeMask = ~0; + *aRegisterMask = ~static_cast(0); + + *bTypeMask = (1 << RegisterOperand) | (1 << MemoryOperand); + *bRegisterMask = ~static_cast(0); + + *thunk = false; + + switch (op) { + case Compare: + if (BytesPerWord == 8 and size != 8) { + *aTypeMask = ~(1 << MemoryOperand); + *bTypeMask = ~(1 << MemoryOperand); + } else { + *bTypeMask = ~(1 << ConstantOperand); + } + break; + + case Move: + if (BytesPerWord == 4 and size == 1) { + const uint32_t mask + = (1 << rax) | (1 << rcx) | (1 << rdx) | (1 << rbx); + *aRegisterMask = (static_cast(mask) << 32) | mask; + *bRegisterMask = (static_cast(mask) << 32) | mask; + } + break; + + case Move4To8: + if (BytesPerWord == 4) { + const uint32_t mask = ~((1 << rax) | (1 << rdx)); + *aRegisterMask = (static_cast(mask) << 32) | mask; + *bRegisterMask = (static_cast(1) << (rdx + 32)) + | (static_cast(1) << rax); + } + break; + + case Multiply: + if (BytesPerWord == 4 and size == 8) { + const uint32_t mask = ~((1 << rax) | (1 << rdx)); + *aRegisterMask = (static_cast(mask) << 32) | mask; + *bRegisterMask = (static_cast(1) << (rdx + 32)) | mask; + } + break; + + case Divide: + if (BytesPerWord == 4 and size == 8) { + *bTypeMask = ~0; + *thunk = true; + } else { + *aRegisterMask = ~((1 << rax) | (1 << rdx)); + *bRegisterMask = 1 << rax; + } + break; + + case Remainder: + if (BytesPerWord == 4 and size == 8) { + *bTypeMask = ~0; + *thunk = true; + } else { + *aRegisterMask = ~((1 << rax) | (1 << rdx)); + *bRegisterMask = 1 << rax; + } + break; + + case ShiftLeft: + case ShiftRight: + case UnsignedShiftRight: { + *aTypeMask = (1 << RegisterOperand) | (1 << ConstantOperand); + *aRegisterMask = (~static_cast(0) << 32) + | (static_cast(1) << rcx); + const uint32_t mask = ~(1 << rcx); + *bRegisterMask = (static_cast(mask) << 32) | mask; + } break; + + default: + break; + } + } + + virtual void apply(Operation op) { + Operations[op](&c); + } + + virtual void apply(UnaryOperation op, unsigned size, + OperandType type, Operand* operand) + { + UnaryOperations[INDEX1(op, type)](&c, size, operand); + } + + virtual void apply(BinaryOperation op, unsigned size, + OperandType aType, Operand* a, + OperandType bType, Operand* b) + { + BinaryOperations[INDEX2(op, aType, bType)](&c, size, a, b); + } + + virtual void writeTo(uint8_t* dst) { + c.result = dst; + memcpy(dst, c.code.data, c.code.length()); + + for (Task* t = c.tasks; t; t = t->next) { + t->run(&c); + } + } + + virtual unsigned length() { + return c.code.length(); + } + + virtual void updateCall(void* returnAddress, void* newTarget) { + if (BytesPerWord == 8) { + uint8_t* instruction = static_cast(returnAddress) - 13; + assert(&c, instruction[0] == 0x49); + assert(&c, instruction[1] == 0xba); + assert(&c, instruction[10] == 0x41); + assert(&c, instruction[11] == 0xff); + assert(&c, instruction[12] == 0xd2); + assert(&c, reinterpret_cast(instruction + 2) % 8 == 0); + + intptr_t v = reinterpret_cast(newTarget); + memcpy(instruction + 2, &v, 8); + } else { + uint8_t* instruction = static_cast(returnAddress) - 5; + assert(&c, *instruction == 0xE8); + assert(&c, reinterpret_cast(instruction + 1) % 4 == 0); + + int32_t v = static_cast(newTarget) + - static_cast(returnAddress); + memcpy(instruction + 1, &v, 4); + } + } + + virtual void dispose() { + c.code.dispose(); + } + + Context c; +}; + +} // namespace + +namespace vm { + +Assembler* +makeAssembler(System* system, Allocator* allocator, Zone* zone) +{ + return new (zone->allocate(sizeof(MyAssembler))) + MyAssembler(system, allocator, zone); +} + +} // namespace vm diff --git a/test/Floats.java b/test/Floats.java index 4db89a233c..655935d03f 100644 --- a/test/Floats.java +++ b/test/Floats.java @@ -32,5 +32,18 @@ public class Floats { expect(divide(0.5d, 0.5d) == 1.0d); expect(subtract(0.5d, 0.5d) == 0.0d); + + double d = 1d; + expect(((int) d) == 1); + + expect(Math.round(0.4f) == 0); + expect(Math.round(0.5f) == 1); + expect(Math.round(1.0f) == 1); + expect(Math.round(1.9f) == 2); + + expect(Math.round(0.4d) == 0); + expect(Math.round(0.5d) == 1); + expect(Math.round(1.0d) == 1); + expect(Math.round(1.9d) == 2); } } diff --git a/test/Misc.java b/test/Misc.java index d4a2b3251c..483ba73820 100644 --- a/test/Misc.java +++ b/test/Misc.java @@ -66,34 +66,156 @@ public class Misc { return super.toString(); } + private static int zap() { + return 42; + } + + private static int zip() { + return 5 + zap(); + } + + private static int zup() { + return zap() + 5; + } + + private static class Foo { + public int a; + public int b; + public int c; + public int[] array; + } + + private static int bar(int a, int b, int c) { + return a + b + c; + } + + private static long roundUp(long a, long b) { + a += b - 1L; + return a - (a % b); + } + public static void main(String[] args) { + expect(roundUp(156, 2) == 156); + + { Foo foo = new Foo(); + int x = foo.a + foo.b + foo.c; + bar(foo.a, foo.b, foo.c); + } + + { int get_buffer = 2144642881; + int bits_left = 30; + int l = 9; + int code = (((get_buffer >> (bits_left -= (l)))) & ((1<<(l))-1)); + expect(code == 510); + } + + { int width = 8; + int height = 8; + int depth = 24; + int scanlinePad = 4; + + int bytesPerLine = (((width * depth + 7) / 8) + (scanlinePad - 1)) + / scanlinePad * scanlinePad; + expect(bytesPerLine == 24); + } + + { int a = -5; + int b = 2; + expect(a >> b == -5 >> 2); + expect(a >>> b == -5 >>> 2); + expect(a << b == -5 << 2); + expect(a + b == -5 + 2); + expect(a - b == -5 - 2); + expect(a * b == -5 * 2); + expect(a / b == -5 / 2); + expect(a % b == -5 % 2); + expect((a & b) == (-5 & 2)); + expect((a | b) == (-5 | 2)); + expect((a ^ b) == (-5 ^ 2)); + expect(-a == 5); + expect(~a == ~-5); + + a = 5; + b = 2; + expect(a >> b == 5 >> 2); + expect(a >>> b == 5 >>> 2); + expect(a << b == 5 << 2); + expect(a + b == 5 + 2); + expect(a - b == 5 - 2); + expect(a * b == 5 * 2); + expect(a / b == 5 / 2); + expect(a % b == 5 % 2); + expect((a & b) == (5 & 2)); + expect((a | b) == (5 | 2)); + expect((a ^ b) == (5 ^ 2)); + expect(-a == -5); + expect(~a == ~5); + } + + { long a = -5; + long b = 2; + expect(a >> b == -5L >> 2); + expect(a >>> b == -5L >>> 2); + expect(a << b == -5L << 2); + expect(a + b == -5L + 2L); + expect(a - b == -5L - 2L); + expect(a * b == -5L * 2L); + expect(a / b == -5L / 2L); + expect(a % b == -5L % 2L); + expect((a & b) == (-5L & 2L)); + expect((a | b) == (-5L | 2L)); + expect((a ^ b) == (-5L ^ 2L)); + expect(-a == 5L); + expect(~a == ~-5L); + + a = 5; + b = 2; + expect(a >> b == 5L >> 2); + expect(a >>> b == 5L >>> 2); + expect(a << b == 5L << 2); + expect(a + b == 5L + 2L); + expect(a - b == 5L - 2L); + expect(a * b == 5L * 2L); + expect(a / b == 5L / 2L); + expect(a % b == 5L % 2L); + expect((a & b) == (5L & 2L)); + expect((a | b) == (5L | 2L)); + expect((a ^ b) == (5L ^ 2L)); + expect(-a == -5L); + expect(~a == ~5L); + } + byte2 = 0; expect(byte2 == 0); - Misc m = new Misc(); - m.toString(); - expect(Long.valueOf(231L) == 231L); - long x = 231; - expect((x >> 32) == 0); - expect((x >>> 32) == 0); - expect((x << 32) == 992137445376L); + { long x = 231; + expect((x >> 32) == 0); + expect((x >>> 32) == 0); + expect((x << 32) == 992137445376L); - long y = -231; - expect((y >> 32) == 0xffffffffffffffffL); - expect((y >>> 32) == 0xffffffffL); + int shift = 32; + expect((x >> shift) == 0); + expect((x >>> shift) == 0); + expect((x << shift) == 992137445376L); - byte[] array = new byte[8]; - putLong(231, array, 0); - expect((array[0] & 0xff) == 0); - expect((array[1] & 0xff) == 0); - expect((array[2] & 0xff) == 0); - expect((array[3] & 0xff) == 0); - expect((array[4] & 0xff) == 0); - expect((array[5] & 0xff) == 0); - expect((array[6] & 0xff) == 0); - expect((array[7] & 0xff) == 231); + long y = -231; + expect((y >> 32) == 0xffffffffffffffffL); + expect((y >>> 32) == 0xffffffffL); + } + + { byte[] array = new byte[8]; + putLong(231, array, 0); + expect((array[0] & 0xff) == 0); + expect((array[1] & 0xff) == 0); + expect((array[2] & 0xff) == 0); + expect((array[3] & 0xff) == 0); + expect((array[4] & 0xff) == 0); + expect((array[5] & 0xff) == 0); + expect((array[6] & 0xff) == 0); + expect((array[7] & 0xff) == 231); + } java.nio.ByteBuffer buffer = java.nio.ByteBuffer.allocate(8); buffer.putLong(231); @@ -104,32 +226,103 @@ public class Misc { ClassLoader.getSystemClassLoader().toString(); - int a = 2; - int b = 2; - int c = a + b; - - String s = "hello"; - m.foo(s); - m.bar(s); - baz(s); - - m.sync(); - syncStatic(false); - try { - syncStatic(true); - } catch (RuntimeException e) { - e.printStackTrace(); + { int a = 2; + int b = 2; + int c = a + b; } - int d = alpha; - beta = 42; - alpha = 43; - int e = beta; - int f = alpha; - m.gamma = 44; + { Misc m = new Misc(); + m.toString(); - expect(beta == 42); - expect(alpha == 43); - expect(m.gamma == 44); + expect(m.time == 0xffffffffffffffffL); + long t = m.time; + expect(t == 0xffffffffffffffffL); + + String s = "hello"; + m.foo(s); + m.bar(s); + baz(s); + + m.sync(); + syncStatic(false); + try { + syncStatic(true); + } catch (RuntimeException e) { + e.printStackTrace(); + } + + int d = alpha; + beta = 42; + alpha = 43; + int e = beta; + int f = alpha; + m.gamma = 44; + + expect(beta == 42); + expect(alpha == 43); + expect(m.gamma == 44); + } + + expect(zip() == 47); + expect(zup() == 47); + + { int[] array = new int[0]; + Exception exception = null; + try { + int x = array[0]; + } catch (ArrayIndexOutOfBoundsException e) { + exception = e; + } + + expect(exception != null); + } + + { int[] array = new int[3]; + int i = 0; + array[i++] = 1; + array[i++] = 2; + array[i++] = 3; + + expect(array[--i] == 3); + expect(array[--i] == 2); + expect(array[--i] == 1); + } + + { Object[][] array = new Object[1][1]; + expect(array.length == 1); + expect(array[0].length == 1); + } + + { + Object a = new Object(); + Object b = new Object(); + expect(a != b); + + Object c = a; + Object d = b; + expect(c != d); + + c = (c == a) ? b : a; + d = (d == a) ? b : a; + + expect(c != d); + } + + { Foo foo = new Foo(); + foo.array = new int[3]; + foo.a = (foo.a + 1) % foo.array.length; + } + + { int j = 0; + byte[] decodeTable = new byte[256]; + for (int i = 'A'; i <= 'Z'; ++i) decodeTable[i] = (byte) j++; + for (int i = 'a'; i <= 'z'; ++i) decodeTable[i] = (byte) j++; + for (int i = '0'; i <= '9'; ++i) decodeTable[i] = (byte) j++; + decodeTable['+'] = (byte) j++; + decodeTable['/'] = (byte) j++; + decodeTable['='] = 0; + + expect(decodeTable['a'] != 0); + } } } diff --git a/valgrind.supp b/valgrind.supp index 9ca17ffbbf..89e9ef784b 100644 --- a/valgrind.supp +++ b/valgrind.supp @@ -5,6 +5,12 @@ obj:/lib/ld-2.3.6.so } +{ + + Memcheck:Cond + obj:/lib/ld-2.6.so +} + { Memcheck:Addr4