diff --git a/makefile b/makefile index 5cd3d5d06d..90b9cc5c96 100644 --- a/makefile +++ b/makefile @@ -161,6 +161,12 @@ interpreter-sources = \ interpreter-asm-sources = $(src)/$(asm).S ifeq ($(process),compile) + interpreter-depends += \ + $(src)/compiler.h \ + $(src)/vector.h + + interpreter-sources += $(src)/compiler.cpp + interpreter-asm-sources += $(src)/compile.S endif diff --git a/src/compile.cpp b/src/compile.cpp index c92b247619..176af855a1 100644 --- a/src/compile.cpp +++ b/src/compile.cpp @@ -1,14 +1,13 @@ -#include "common.h" -#include "system.h" -#include "constants.h" #include "machine.h" -#include "processor.h" +#include "util.h" +#include "vector.h" #include "process.h" +#include "compiler.h" using namespace vm; extern "C" uint64_t -vmInvoke(void* function, void* stack, unsigned stackSize, +vmInvoke(void* thread, void* function, void* stack, unsigned stackSize, unsigned returnType); extern "C" void @@ -19,949 +18,792 @@ vmJump(void* address, void* base, void* stack); namespace { -const bool Verbose = false; - -const unsigned FrameThread = BytesPerWord * 2; -const unsigned FrameMethod = FrameThread + BytesPerWord; -const unsigned FrameNext = FrameMethod + BytesPerWord; -const unsigned FrameFootprint = BytesPerWord * 3; - -class ArgumentList; - -class Buffer { - public: - Buffer(System* s, unsigned minimumCapacity): - s(s), - data(0), - position(0), - capacity(0), - minimumCapacity(minimumCapacity) - { } - - ~Buffer() { - if (data) { - s->free(data); - } - } - - void ensure(unsigned space) { - if (position + space > capacity) { - unsigned newCapacity = max - (position + space, max(minimumCapacity, capacity * 2)); - uint8_t* newData = static_cast(s->allocate(newCapacity)); - if (data) { - memcpy(newData, data, position); - s->free(data); - } - data = newData; - } - } - - void append(uint8_t v) { - ensure(1); - data[position++] = v; - } - - void append2(uint16_t v) { - ensure(2); - memcpy(data + position, &v, 2); - position += 2; - } - - void append4(uint32_t v) { - ensure(4); - memcpy(data + position, &v, 4); - position += 4; - } - - void set2(unsigned offset, uint32_t v) { - assert(s, offset + 2 <= position); - memcpy(data + offset, &v, 2); - } - - void set4(unsigned offset, uint32_t v) { - assert(s, offset + 4 <= position); - memcpy(data + offset, &v, 4); - } - - uint8_t& get(unsigned offset) { - assert(s, offset + 1 <= position); - return data[offset]; - } - - uint16_t& get2(unsigned offset) { - assert(s, offset + 2 <= position); - return *reinterpret_cast(data + offset); - } - - uint32_t& get4(unsigned offset) { - assert(s, offset + 4 <= position); - return *reinterpret_cast(data + offset); - } - - uintptr_t& getAddress(unsigned offset) { - assert(s, offset + 4 <= position); - return *reinterpret_cast(data + offset); - } - - void appendAddress(uintptr_t v) { - append4(v); - if (BytesPerWord == 8) { - // we have to use the preprocessor here to avoid a warning on - // 32-bit systems -#ifdef __x86_64__ - append4(v >> 32); -#endif - } - } - - unsigned length() { - return position; - } - - void copyTo(void* b) { - if (data) { - memcpy(b, data, position); - } - } - - System* s; - uint8_t* data; - unsigned position; - unsigned capacity; - unsigned minimumCapacity; -}; - -class StackMapper { - public: - static const uint32_t Infinity = ~static_cast(0); - - enum { - Call, - PushLong, - PushInt, - PushObject, - Duplicate, - DuplicateX1, - DuplicateX2, - Duplicate2, - Duplicate2X1, - Duplicate2X2, - Pop, - PopLong, - PopInt, - PopObject, - StoreLong, - StoreInt, - StoreObject, - Return, - Jump, - Branch, - End - }; - - StackMapper(Thread* t, object method): - t(t), - method(method), - ip(-1), - index(codeSize() ? static_cast - (t->m->system->allocate(codeSize() * 4)) : 0), - log(t->m->system, 1024), - callCount(0), - protector(this) - { } - - ~StackMapper() { - if (index) { - t->m->system->free(index); - } - } - - unsigned codeSize() { - return codeLength(t, methodCode(t, method)); - } - - void end(unsigned nextIp) { - if (ip >= 0) { - log.append(End); - log.append2(nextIp); - } - ip = nextIp; - } - - void newIp(unsigned javaIp) { - assert(t, javaIp < codeSize()); - - end(javaIp); - index[javaIp] = log.length(); - } - - void finish() { - end(codeSize()); - } - - void called(unsigned nextMachineIp) { - log.append(Call); - log.append4(nextMachineIp); - ++ callCount; - } - - void pushedLong() { - log.append(PushLong); - } - - void pushedInt() { - log.append(PushInt); - } - - void pushedObject() { - log.append(PushObject); - } - - void dupped() { - log.append(Duplicate); - } - - void duppedX1() { - log.append(DuplicateX1); - } - - void duppedX2() { - log.append(DuplicateX2); - } - - void dupped2() { - log.append(Duplicate2); - } - - void dupped2X1() { - log.append(Duplicate2X1); - } - - void dupped2X2() { - log.append(Duplicate2X2); - } - - void popped(unsigned count) { - log.append(Pop); - log.append(count); - } - - void poppedLong() { - log.append(PopLong); - } - - void poppedInt() { - log.append(PopInt); - } - - void poppedObject() { - log.append(PopObject); - } - - void storedLong(unsigned index) { - if (index >= parameterFootprint()) { - log.append(StoreLong); - log.append2(index - parameterFootprint()); - } - } - - void storedInt(unsigned index) { - if (index >= parameterFootprint()) { - log.append(StoreInt); - log.append2(index - parameterFootprint()); - } - } - - void storedObject(unsigned index) { - if (index >= parameterFootprint()) { - log.append(StoreObject); - log.append2(index - parameterFootprint()); - } - } - - void exited() { - log.append(Return); - } - - void jumped(unsigned targetJavaIp) { - log.append(Jump); - log.append2(targetJavaIp); - } - - void branched(unsigned targetJavaIp) { - log.append(Branch); - log.append2(targetJavaIp); - } - - unsigned parameterFootprint() { - return methodParameterFootprint(t, method); - } - - unsigned localSize() { - return codeMaxLocals(t, methodCode(t, method)) - - parameterFootprint(); - } - - unsigned stackSize() { - return codeMaxStack(t, methodCode(t, method)); - } - - unsigned mapSize() { - return stackSize() + localSize(); - } - - unsigned mapSizeInWords() { - return ceiling(mapSize(), BytesPerWord); - } - - unsigned mapSizeInBytes() { - return mapSizeInWords() * BytesPerWord; - } - - unsigned callTableSize() { - return callCount * (mapSizeInBytes() + BytesPerWord); - } - - void populateCalls(uintptr_t* calls, unsigned& callIndex, uintptr_t* table, - uintptr_t* mask, uintptr_t* initialMap, - unsigned initialSp, unsigned ip) - { - uintptr_t map[mapSizeInWords()]; - memcpy(map, initialMap, mapSizeInBytes()); - - unsigned sp = initialSp; - - while (ip < codeSize()) { - if (getBit(mask, ip)) { - return; - } - - markBit(mask, ip); - memcpy(table + (ip * mapSizeInWords()), map, mapSizeInBytes()); - - unsigned i = index[ip]; - while (true) { - switch (log.get(i++)) { - case Call: { - assert(t, callIndex < callCount); - unsigned machineIp = log.get4(i); i += 4; - calls[callIndex * (mapSizeInWords() + 1)] = machineIp; - memcpy(calls + (callIndex * (mapSizeInWords() + 1)) + 1, - map, mapSizeInBytes()); - ++ callIndex; - -// fprintf(stderr, -// "call stack and locals 0x%x of size %d at %d of %s.%s\n", -// *map, -// mapSize(), -// machineIp, -// &byteArrayBody -// (t, className(t, methodClass(t, method)), 0), -// &byteArrayBody(t, methodName(t, method), 0)); - - } break; - - case PushLong: - assert(t, sp + 2 <= mapSize()); - assert(t, getBit(map, sp) == 0); - assert(t, getBit(map, sp + 1) == 0); - sp += 2; - break; - - case PushInt: - assert(t, sp + 1 <= mapSize()); - assert(t, getBit(map, sp) == 0); - ++ sp; - break; - - case PushObject: - assert(t, sp + 1 <= mapSize()); - markBit(map, sp++); - break; - - case Duplicate: - assert(t, sp + 1 <= mapSize()); - assert(t, sp - 1 >= localSize()); - if (getBit(map, sp - 1)) { - markBit(map, sp); - } - ++ sp; - break; - - case DuplicateX1: { - assert(t, sp + 1 <= mapSize()); - assert(t, sp - 2 >= localSize()); - - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b2) { - markBit(map, sp - 1); - } else { - clearBit(map, sp - 1); - } - - if (b1) { - markBit(map, sp - 2); - markBit(map, sp); - } else { - clearBit(map, sp - 2); - } - - ++ sp; - } break; - - case DuplicateX2: { - assert(t, sp + 1 <= mapSize()); - assert(t, sp - 3 >= localSize()); - - unsigned b3 = getBit(map, sp - 3); - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b3) { - markBit(map, sp - 2); - } else { - clearBit(map, sp - 2); - } - - if (b2) { - markBit(map, sp - 1); - } else { - clearBit(map, sp - 1); - } - - if (b1) { - markBit(map, sp - 3); - markBit(map, sp); - } else { - clearBit(map, sp - 3); - } - - ++ sp; - } break; - - case Duplicate2: { - assert(t, sp + 2 <= mapSize()); - assert(t, sp - 2 >= localSize()); - - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b2) { - markBit(map, sp); - } - - if (b1) { - markBit(map, sp + 1); - } - - sp += 2; - } break; - - case Duplicate2X1: { - assert(t, sp + 2 <= mapSize()); - assert(t, sp - 3 >= localSize()); - - unsigned b3 = getBit(map, sp - 3); - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b3) { - markBit(map, sp - 1); - } else { - clearBit(map, sp - 1); - } - - if (b2) { - markBit(map, sp - 3); - markBit(map, sp); - } else { - clearBit(map, sp - 3); - } - - if (b1) { - markBit(map, sp - 2); - markBit(map, sp + 1); - } else { - clearBit(map, sp - 2); - } - - sp += 2; - } break; - - case Duplicate2X2: { - assert(t, sp + 2 <= mapSize()); - assert(t, sp - 4 >= localSize()); - - unsigned b4 = getBit(map, sp - 4); - unsigned b3 = getBit(map, sp - 3); - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b4) { - markBit(map, sp - 2); - } else { - clearBit(map, sp - 2); - } - - if (b3) { - markBit(map, sp - 1); - } else { - clearBit(map, sp - 1); - } - - if (b2) { - markBit(map, sp - 4); - markBit(map, sp); - } else { - clearBit(map, sp - 4); - } - - if (b1) { - markBit(map, sp - 3); - markBit(map, sp + 1); - } else { - clearBit(map, sp - 3); - } - - sp += 2; - } break; - - case Pop: { - unsigned count = log.get(i++); - assert(t, sp >= count); - assert(t, sp - count >= localSize()); - while (count) { - clearBit(map, -- sp); - -- count; - } - } break; - - case PopLong: - assert(t, sp >= 2); - assert(t, sp - 2 >= localSize()); - assert(t, getBit(map, sp - 1) == 0); - assert(t, getBit(map, sp - 2) == 0); - sp -= 2; - break; - - case PopInt: - assert(t, sp >= 1); - assert(t, sp - 1 >= localSize()); - assert(t, getBit(map, sp - 1) == 0); - -- sp; - break; - - case PopObject: - assert(t, sp >= 1); - assert(t, sp - 1 >= localSize()); - assert(t, getBit(map, sp - 1) != 0); - clearBit(map, -- sp); - break; - - case StoreLong: { - unsigned index = log.get2(i); i += 2; - assert(t, index + 1 < localSize()); - clearBit(map, index); - clearBit(map, index + 1); - } break; - - case StoreInt: { - unsigned index = log.get2(i); i += 2; - assert(t, index < localSize()); - clearBit(map, index); - } break; - - case StoreObject: { - unsigned index = log.get2(i); i += 2; - assert(t, index < localSize()); - markBit(map, index); - } break; - - case Return: - ip = codeSize(); - goto loop; - - case Jump: - ip = log.get2(i); i += 2; - assert(t, ip < codeSize()); - goto loop; - - case Branch: { - unsigned target = log.get2(i); i += 2; - assert(t, target < codeSize()); - populateCalls(calls, callIndex, table, mask, map, sp, target); - } break; - - case End: - goto next; - - default: - abort(t); - } - } - - next: - ip = log.get2(i); - - loop:; - } - } - - static int compareCalls(const void* a, const void* b) { - return (*static_cast(a) > - *static_cast(b) ? 1 : -1); - } - - void writeCallTableTo(uintptr_t* calls) { - uintptr_t* table = static_cast - (t->m->system->allocate(codeSize() * mapSizeInBytes())); - - uintptr_t* mask = static_cast - (t->m->system->allocate - (ceiling(codeSize(), BytesPerWord) * BytesPerWord)); - memset(mask, 0, ceiling(codeSize(), BytesPerWord) * BytesPerWord); - - uintptr_t map[mapSizeInWords()]; - memset(map, 0, mapSizeInBytes()); - - unsigned callIndex = 0; - - populateCalls(calls, callIndex, table, mask, map, localSize(), 0); - - object eht = codeExceptionHandlerTable(t, methodCode(t, method)); - if (eht) { - PROTECT(t, eht); - - for (unsigned i = 0; i < exceptionHandlerTableLength(t, eht); ++i) { - ExceptionHandler* eh = exceptionHandlerTableBody(t, eht, i); - - assert(t, getBit(mask, exceptionHandlerStart(eh))); - - memcpy(map, - table + (exceptionHandlerStart(eh) * mapSizeInWords()), - mapSizeInBytes()); - - for (unsigned j = localSize() + 1; j < mapSize(); ++j) { - clearBit(map, j); - } - - markBit(map, localSize()); - - populateCalls(calls, callIndex, table, mask, map, localSize() + 1, - exceptionHandlerIp(eh)); - } - } - - t->m->system->free(mask); - t->m->system->free(table); - - assert(t, callIndex == callCount); - qsort(calls, callCount, mapSizeInBytes() + BytesPerWord, compareCalls); - } - - Thread* t; - object method; - int ip; - uint32_t* index; - Buffer log; - unsigned callCount; - - class MyProtector: public Thread::Protector { - public: - MyProtector(StackMapper* mapper): Protector(mapper->t), mapper(mapper) { } - - virtual void visit(Heap::Visitor* v) { - v->visit(&(mapper->method)); - } - - StackMapper* mapper; - } protector; -}; - class MyThread: public Thread { public: + class CallTrace { + public: + CallTrace(MyThread* t): + t(t), + base(t->base), + stack(t->stack), + next(t->trace) + { + t->trace = this; + t->base = 0; + t->stack = 0; + } + + ~CallTrace() { + t->stack = stack; + t->base = base; + t->trace = next; + } + + MyThread* t; + void* base; + void* stack; + CallTrace* next; + }; + MyThread(Machine* m, object javaThread, Thread* parent): Thread(m, javaThread, parent), - argumentList(0), + base(0), + stack(0), + trace(0), reference(0) - { - memset(frameData, 0, BytesPerWord * 2); - } + { } - ArgumentList* argumentList; - void* frameData[2]; + void* base; + void* stack; + CallTrace* trace; Reference* reference; }; -inline void* -frameBase(void* frame) +object +resolveTarget(MyThread* t, void* stack, object method) { - return static_cast(frame) - [static_cast(- (FrameFootprint / BytesPerWord) - 2)]; -} + if (methodVirtual(t, method)) { + unsigned parameterFootprint = methodParameterFootprint(t, method); + object class_ = objectClass + (t, reinterpret_cast(stack)[parameterFootprint + 1]); -inline bool -frameValid(void* frame) -{ - return frame != 0; -} + if (classVmFlags(t, class_) & BootstrapFlag) { + resolveClass(t, className(t, class_)); + if (UNLIKELY(t->exception)) return 0; + } -inline void* -frameNext(void* frame) -{ - return static_cast(frameBase(frame))[FrameNext / BytesPerWord]; -} - -inline object& -frameMethod(void* frame) -{ - return static_cast(frameBase(frame))[FrameMethod / BytesPerWord]; -} - -inline void* -frameAddress(void* frame) -{ - return static_cast(frame) - [static_cast(- (FrameFootprint / BytesPerWord) - 1)]; -} - -inline void* -frameReturnAddress(void* frame) -{ - return static_cast(frameBase(frame))[1]; -} - -inline uint8_t* -compiledCode(Compiled* code) -{ - return compiledBody(code); -} - -inline unsigned -compiledLineNumberCount(Thread*, Compiled* code) -{ - return compiledLineNumberTableLength(code) / sizeof(NativeLineNumber); -} - -inline NativeLineNumber* -compiledLineNumber(Thread* t UNUSED, Compiled* code, unsigned index) -{ - assert(t, index < compiledLineNumberCount(t, code)); - return reinterpret_cast - (compiledBody(code) + pad(compiledCodeLength(code))) + index; -} - -inline unsigned -compiledExceptionHandlerCount(Thread*, Compiled* code) -{ - return compiledExceptionHandlerTableLength(code) - / sizeof(NativeExceptionHandler); -} - -inline NativeExceptionHandler* -compiledExceptionHandler(Thread* t UNUSED, Compiled* code, unsigned index) -{ - assert(t, index < compiledExceptionHandlerCount(t, code)); - return reinterpret_cast - (compiledBody(code) - + pad(compiledCodeLength(code)) - + pad(compiledLineNumberTableLength(code))) + index; -} - -inline unsigned -compiledStackMapSize(Thread*, Compiled* code) -{ - return ceiling(compiledMaxStack(code) - + compiledMaxLocals(code) - - compiledParameterFootprint(code), - BytesPerWord) + 1; -} - -inline unsigned -compiledStackMapCount(Thread* t, Compiled* code) -{ - return compiledStackMapTableLength(code) - / (compiledStackMapSize(t, code) * BytesPerWord); -} - -inline uintptr_t* -compiledStackMap(Thread* t, Compiled* code, unsigned index) -{ - assert(t, index < compiledStackMapCount(t, code)); - return reinterpret_cast - (compiledBody(code) - + pad(compiledCodeLength(code)) - + pad(compiledLineNumberTableLength(code)) - + pad(compiledExceptionHandlerTableLength(code))) - + (index * compiledStackMapSize(t, code)); -} - -inline Compiled* -makeCompiled(Thread* t, Buffer* code) -{ - Compiled* c = static_cast - (t->m->system->allocate(sizeof(Compiled) + pad(code->length()))); - - compiledMaxLocals(c) = 0; - compiledMaxStack(c) = 0; - compiledParameterFootprint(c) = 0; - compiledCodeLength(c) = code->length(); - compiledLineNumberTableLength(c) = 0; - compiledExceptionHandlerTableLength(c) = 0; - compiledStackMapTableLength(c) = 0; - - if (code->length()) { - code->copyTo(compiledCode(c)); - } - - return c; -} - -inline Compiled* -makeCompiled(Thread* t, object method, Buffer* code, - NativeLineNumber* lineNumbers, - NativeExceptionHandler* exceptionHandlers, - StackMapper* stackMapper) -{ - unsigned lineNumberCount - = codeLineNumberTable(t, methodCode(t, method)) ? - lineNumberTableLength - (t, codeLineNumberTable(t, methodCode(t, method))) : 0; - - unsigned exceptionHandlerCount - = codeExceptionHandlerTable(t, methodCode(t, method)) ? - exceptionHandlerTableLength - (t, codeExceptionHandlerTable(t, methodCode(t, method))) : 0; - - unsigned maxStack = codeMaxStack(t, methodCode(t, method)); - unsigned maxLocals = codeMaxLocals(t, methodCode(t, method)); - unsigned parameterFootprint = methodParameterFootprint(t, method); - - Compiled* c = static_cast - (t->m->system->allocate(sizeof(Compiled) - + pad(code->length()) - + pad(lineNumberCount - * sizeof(NativeLineNumber)) - + pad(exceptionHandlerCount - * sizeof(NativeExceptionHandler)) - + pad(stackMapper->callTableSize()))); - - compiledMaxLocals(c) = maxLocals; - compiledMaxStack(c) = maxStack; - compiledParameterFootprint(c) = parameterFootprint; - compiledCodeLength(c) = code->length(); - - compiledLineNumberTableLength(c) - = lineNumberCount * sizeof(NativeLineNumber); - - compiledExceptionHandlerTableLength(c) - = exceptionHandlerCount * sizeof(NativeExceptionHandler); - - compiledStackMapTableLength(c) = stackMapper->callTableSize(); - - if (code->length()) { - code->copyTo(compiledCode(c)); - } - - if (lineNumberCount) { - memcpy(compiledLineNumber(t, c, 0), - lineNumbers, - lineNumberCount * sizeof(NativeLineNumber)); - } - - if (exceptionHandlerCount) { - memcpy(compiledExceptionHandler(t, c, 0), - exceptionHandlers, - exceptionHandlerCount * sizeof(NativeExceptionHandler)); - } - - if (stackMapper->callTableSize()) { - stackMapper->writeCallTableTo(compiledStackMap(t, c, 0)); - } - - return c; -} - -inline unsigned -addressOffset(Thread* t, object method, void* address) -{ - Compiled* code = reinterpret_cast(methodCompiled(t, method)); - return static_cast(address) - compiledCode(code); -} - -NativeExceptionHandler* -findExceptionHandler(Thread* t, void* frame) -{ - object method = frameMethod(frame); - Compiled* code = reinterpret_cast(methodCompiled(t, method)); - - for (unsigned i = 0; i < compiledExceptionHandlerCount(t, code); ++i) { - NativeExceptionHandler* handler = compiledExceptionHandler(t, code, i); - unsigned offset = addressOffset(t, method, frameAddress(frame)); - - if (offset - 1 >= nativeExceptionHandlerStart(handler) - and offset - 1 < nativeExceptionHandlerEnd(handler)) - { - object catchType; - if (nativeExceptionHandlerCatchType(handler)) { - catchType = arrayBody - (t, methodCode(t, method), - nativeExceptionHandlerCatchType(handler) - 1); - } else { - catchType = 0; - } - - if (catchType == 0 or instanceOf(t, catchType, t->exception)) { - if (Verbose) { - fprintf(stderr, "exception handler match for %d in %s: " - "start: %d; end: %d; ip: %d\n", - offset, - &byteArrayBody(t, methodName(t, frameMethod(frame)), 0), - nativeExceptionHandlerStart(handler), - nativeExceptionHandlerEnd(handler), - nativeExceptionHandlerIp(handler)); - } - - return handler; - } + if (classFlags(t, methodClass(t, method)) & ACC_INTERFACE) { + return findInterfaceMethod(t, method, class_); + } else { + return findMethod(t, method, class_); } } - return 0; + return method; } -void* -framePointer(void** data) +object +findTraceNode(MyThread* t, void* address); + +void +insertTraceNode(MyThread* t, object node); + +class MyStackWalker: public Processor::StackWalker { + public: + class MyProtector: public Thread::Protector { + public: + MyProtector(MyStackWalker* walker): + Protector(walker->t), walker(walker) + { } + + virtual void visit(Heap::Visitor* v) { + v->visit(&(walker->node)); + v->visit(&(walker->nativeMethod)); + } + + MyStackWalker* walker; + }; + + MyStackWalker(MyThread* t): + t(t), + base(t->base), + stack(t->stack), + trace(t->trace), + node(findTraceNode(t, *static_cast(stack))), + nativeMethod(resolveNativeMethod(t, stack, node)), + protector(this) + { } + + MyStackWalker(MyStackWalker* w): + t(w->t), + base(w->base), + stack(w->stack), + trace(w->trace), + node(w->node), + nativeMethod(w->nativeMethod), + protector(this) + { } + + static object resolveNativeMethod(MyThread* t, void* stack, object node) { + if (node) { + object target = resolveTarget(t, stack, traceNodeTarget(t, node)); + if (target and methodFlags(t, target) & ACC_NATIVE) { + return target; + } + } + return 0; + } + + virtual void walk(Processor::StackVisitor* v) { + if (not v->visit(this)) { + return; + } + + for (MyStackWalker it(this); it.next();) { + MyStackWalker walker(it); + if (not v->visit(&walker)) { + break; + } + } + } + + bool next() { + if (nativeMethod) { + nativeMethod = 0; + } else { + stack = static_cast(base) + 1; + base = *static_cast(base); + node = findTraceNode(t, *static_cast(stack)); + if (node == 0) { + if (trace) { + base = trace->base; + stack = static_cast(trace->stack); + trace = trace->next; + node = findTraceNode(t, *static_cast(stack)); + nativeMethod = resolveNativeMethod(t, stack, node); + } else { + return false; + } + } + } + return true; + } + + virtual object method() { + if (nativeMethod) { + return nativeMethod; + } else { + return traceNodeMethod(t, node); + } + } + + virtual int ip() { + if (nativeMethod) { + return 0; + } else { + return treeNodeKey(t, node); + } + } + + virtual unsigned count() { + class Visitor: public Processor::StackVisitor { + public: + Visitor(): count(0) { } + + virtual bool visit(Processor::StackWalker*) { + ++ count; + return true; + } + + unsigned count; + } v; + + MyStackWalker walker(this); + walker.walk(&v); + + return v.count; + } + + MyThread* t; + void* base; + void* stack; + MyThread::CallTrace* trace; + object node; + object nativeMethod; + MyProtector protector; +}; + +uintptr_t* +makeCodeMask(MyThread* t, unsigned length) { - return data + (FrameFootprint / BytesPerWord) + 2; + unsigned size = ceiling(length, BytesPerWord) * BytesPerWord; + uintptr_t* mask = static_cast(t->m->system->allocate(size)); + memset(mask, 0, size); + return mask; } -void* -frameStart(MyThread* t) +int +localOffset(MyThread* t, int v, object method) { - return t->frameData[0] ? framePointer(t->frameData) : 0; + int parameterFootprint = methodParameterFootprint(t, method) * BytesPerWord; + + v *= BytesPerWord; + if (v < parameterFootprint) { + return (parameterFootprint - v - BytesPerWord) + (BytesPerWord * 2); + } else { + return -(v + BytesPerWord - parameterFootprint); + } } +class PoolElement { + public: + PoolElement(object value, Promise* offset): value(value), offset(offset) { } + + object value; + Promise* offset; +}; + +class TraceElement { + public: + TraceElement(object target, Promise* offset, bool virtualCall): + target(target), offset(offset), virtualCall(virtualCall) { } + + object target; + Promise* offset; + bool virtualCall; + uint8_t map[0]; +}; + +class Frame { + public: + class MyProtector: public Thread::Protector { + public: + MyProtector(Frame* frame): Protector(frame->t), frame(frame) { } + + virtual void visit(Heap::Visitor* v) { + v->visit(&(frame->method)); + + if (next == 0) { + Vector* pool = frame->objectPool; + for (unsigned i = 0; i < pool->length(); i += sizeof(PoolElement)) { + v->visit(&(pool->peek(i)->value)); + } + + Vector* log = frame->traceLog; + unsigned traceSize = traceSizeInBytes(t, frame->method); + for (unsigned i = 0; i < log->length(); i += traceSize) { + v->visit(&(pool->peek(i)->target)); + } + } + } + + Frame* frame; + }; + + Frame(MyThread* t, Compiler* c, object method, uintptr_t* map, + Vector* objectPool, Vector* traceLog): + next(0), + t(t), + c(c), + method(method), + map(map), + objectPool(objectPool), + traceLog(traceLog), + codeMask(makeCodeMask(t, codeLength(t, methodCode(t, method)))), + ip(0), + sp(localSize(t, method)), + protector(this) + { + memset(map, 0, mapSizeInBytes(t, method)); + } + + Frame(Frame* f, uintptr_t* map): + next(f), + t(f->t), + c(f->c), + method(f->method), + map(map), + objectPool(f->objectPool), + traceLog(f->traceLog), + codeMask(f->codeMask), + ip(f->ip), + sp(f->sp), + protector(this) + { + memcpy(map, f->map, mapSizeInBytes(t, method)); + } + + ~Frame() { + t->m->system->free(codeMask); + } + + Operand* append(object o) { + new (objectPool->allocate(sizeof(PoolElement))) + PoolElement(o, c->poolOffset()); + return c->poolAppend(c->constant(0)); + } + + static unsigned parameterFootprint(Thread* t, object method) { + return methodParameterFootprint(t, method); + } + + static unsigned localSize(Thread* t, object method) { + return codeMaxLocals(t, methodCode(t, method)) + - parameterFootprint(t, method); + } + + static unsigned stackSize(Thread* t, object method) { + return codeMaxStack(t, methodCode(t, method)); + } + + static unsigned mapSize(Thread* t, object method) { + return stackSize(t, method) + localSize(t, method); + } + + static unsigned mapSizeInWords(Thread* t, object method) { + return ceiling(mapSize(t, method), BytesPerWord); + } + + static unsigned mapSizeInBytes(Thread* t, object method) { + return mapSizeInWords(t, method) * BytesPerWord; + } + + static unsigned traceSizeInBytes(Thread* t, object method) { + return BytesPerWord + BytesPerWord + 1 + mapSizeInWords(t, method); + } + + void pushedInt() { + assert(t, sp + 1 <= mapSize(t, method)); + assert(t, getBit(map, sp) == 0); + ++ sp; + } + + void pushedObject() { + assert(t, sp + 1 <= mapSize(t, method)); + markBit(map, sp++); + } + + void poppedInt() { + assert(t, sp >= 1); + assert(t, sp - 1 >= localSize(t, method)); + assert(t, getBit(map, sp - 1) == 0); + -- sp; + } + + void poppedObject() { + assert(t, sp >= 1); + assert(t, sp - 1 >= localSize(t, method)); + assert(t, getBit(map, sp - 1) != 0); + clearBit(map, -- sp); + } + + void storedInt(unsigned index) { + assert(t, index < localSize(t, method)); + clearBit(map, index); + } + + void storedObject(unsigned index) { + assert(t, index < localSize(t, method)); + markBit(map, index); + } + + void dupped() { + assert(t, sp + 1 <= mapSize(t, method)); + assert(t, sp - 1 >= localSize(t, method)); + if (getBit(map, sp - 1)) { + markBit(map, sp); + } + ++ sp; + } + + void duppedX1() { + assert(t, sp + 1 <= mapSize(t, method)); + assert(t, sp - 2 >= localSize(t, method)); + + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b2) { + markBit(map, sp - 1); + } else { + clearBit(map, sp - 1); + } + + if (b1) { + markBit(map, sp - 2); + markBit(map, sp); + } else { + clearBit(map, sp - 2); + } + + ++ sp; + } + + void duppedX2() { + assert(t, sp + 1 <= mapSize(t, method)); + assert(t, sp - 3 >= localSize(t, method)); + + unsigned b3 = getBit(map, sp - 3); + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b3) { + markBit(map, sp - 2); + } else { + clearBit(map, sp - 2); + } + + if (b2) { + markBit(map, sp - 1); + } else { + clearBit(map, sp - 1); + } + + if (b1) { + markBit(map, sp - 3); + markBit(map, sp); + } else { + clearBit(map, sp - 3); + } + + ++ sp; + } + + void dupped2() { + assert(t, sp + 2 <= mapSize(t, method)); + assert(t, sp - 2 >= localSize(t, method)); + + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b2) { + markBit(map, sp); + } + + if (b1) { + markBit(map, sp + 1); + } + + sp += 2; + } + + void dupped2X1() { + assert(t, sp + 2 <= mapSize(t, method)); + assert(t, sp - 3 >= localSize(t, method)); + + unsigned b3 = getBit(map, sp - 3); + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b3) { + markBit(map, sp - 1); + } else { + clearBit(map, sp - 1); + } + + if (b2) { + markBit(map, sp - 3); + markBit(map, sp); + } else { + clearBit(map, sp - 3); + } + + if (b1) { + markBit(map, sp - 2); + markBit(map, sp + 1); + } else { + clearBit(map, sp - 2); + } + + sp += 2; + } + + void dupped2X2() { + assert(t, sp + 2 <= mapSize(t, method)); + assert(t, sp - 4 >= localSize(t, method)); + + unsigned b4 = getBit(map, sp - 4); + unsigned b3 = getBit(map, sp - 3); + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b4) { + markBit(map, sp - 2); + } else { + clearBit(map, sp - 2); + } + + if (b3) { + markBit(map, sp - 1); + } else { + clearBit(map, sp - 1); + } + + if (b2) { + markBit(map, sp - 4); + markBit(map, sp); + } else { + clearBit(map, sp - 4); + } + + if (b1) { + markBit(map, sp - 3); + markBit(map, sp + 1); + } else { + clearBit(map, sp - 3); + } + + sp += 2; + } + + void swapped() { + assert(t, sp - 1 >= localSize(t, method)); + assert(t, sp - 2 >= localSize(t, method)); + + bool savedBit = getBit(map, sp - 1); + if (getBit(map, sp - 2)) { + markBit(map, sp - 1); + } else { + clearBit(map, sp - 1); + } + + if (savedBit) { + markBit(map, sp - 2); + } else { + clearBit(map, sp - 2); + } + } + + void trace(object target, bool virtualCall) { + TraceElement* e = new (traceLog->allocate(traceSizeInBytes(t, method))) + TraceElement(target, c->codeOffset(), virtualCall); + memcpy(e->map, map, mapSizeInWords(t, method) * BytesPerWord); + } + + void trace() { + trace(0, false); + } + + void startLogicalIp(unsigned ip) { + c->startLogicalIp(ip); + this->ip = ip; + } + + void pushInt(Operand* o) { + c->push(o); + pushedInt(); + } + + void pushObject(Operand* o) { + c->push(o); + pushedObject(); + } + + void pushLong(Operand* o) { + c->push2(o); + pushedInt(); + pushedInt(); + } + + void pop(unsigned count) { + assert(t, sp >= count); + assert(t, sp - count >= localSize(t, method)); + while (count) { + clearBit(map, -- sp); + -- count; + } + } + + Operand* topInt() { + assert(t, sp >= 1); + assert(t, sp - 1 >= localSize(t, method)); + assert(t, getBit(map, sp - 1) == 0); + return c->stack(0); + } + + Operand* topLong() { + assert(t, sp >= 2); + assert(t, sp - 2 >= localSize(t, method)); + assert(t, getBit(map, sp - 1) == 0); + assert(t, getBit(map, sp - 2) == 0); + return c->stack2(1); + } + + Operand* topObject() { + assert(t, sp >= 1); + assert(t, sp - 1 >= localSize(t, method)); + assert(t, getBit(map, sp - 1) != 0); + return c->stack(0); + } + + Operand* popInt() { + poppedInt(); + return c->pop(); + } + + Operand* popLong() { + poppedInt(); + poppedInt(); + return c->pop2(); + } + + Operand* popObject() { + poppedObject(); + return c->pop(); + } + + void popInt(Operand* o) { + c->pop(o); + poppedInt(); + } + + void popLong(Operand* o) { + c->pop2(o); + poppedInt(); + poppedInt(); + } + + void popObject(Operand* o) { + c->pop(o); + poppedObject(); + } + + void loadInt(unsigned index) { + assert(t, index < localSize(t, method)); + assert(t, getBit(map, index) == 0); + pushInt(c->memory(c->base(), localOffset(t, index, method))); + } + + void loadLong(unsigned index) { + assert(t, index < localSize(t, method) - 1); + assert(t, getBit(map, index) == 0); + assert(t, getBit(map, index + 1) == 0); + pushLong(c->memory(c->base(), localOffset(t, index, method))); + } + + void loadObject(unsigned index) { + assert(t, index < localSize(t, method)); + assert(t, getBit(map, index) != 0); + pushObject(c->memory(c->base(), localOffset(t, index, method))); + } + + void storeInt(unsigned index) { + popInt(c->memory(c->base(), localOffset(t, index, method))); + storedInt(index); + } + + void storeLong(unsigned index) { + popLong(c->memory(c->base(), localOffset(t, index, method))); + storedInt(index); + storedInt(index + 1); + } + + void storeObject(unsigned index) { + popObject(c->memory(c->base(), localOffset(t, index, method))); + storedObject(index); + } + + void increment(unsigned index, unsigned count) { + assert(t, index < localSize(t, method)); + assert(t, getBit(map, index) == 0); + c->add(c->constant(count), + c->memory(c->base(), localOffset(t, index, method))); + } + + void dup() { + c->push(c->stack(0)); + dupped(); + } + + void dupX1() { + Operand* s0 = c->stack(0); + Operand* s1 = c->stack(1); + + c->mov(s0, s1); + c->mov(s1, s0); + c->push(s0); + + duppedX1(); + } + + void dupX2() { + Operand* s0 = c->stack(0); + Operand* s1 = c->stack(1); + Operand* s2 = c->stack(2); + + c->mov(s0, s2); + c->mov(s2, s1); + c->mov(s1, s0); + c->push(s0); + + duppedX2(); + } + + void dup2() { + Operand* s0 = c->stack(0); + + c->push(s0); + c->push(s0); + + dupped2(); + } + + void dup2X1() { + Operand* s0 = c->stack(0); + Operand* s1 = c->stack(1); + Operand* s2 = c->stack(2); + + c->mov(s1, s2); + c->mov(s0, s1); + c->mov(s2, s0); + c->push(s1); + c->push(s0); + + dupped2X1(); + } + + void dup2X2() { + Operand* s0 = c->stack(0); + Operand* s1 = c->stack(1); + Operand* s2 = c->stack(2); + Operand* s3 = c->stack(3); + + c->mov(s1, s3); + c->mov(s0, s2); + c->mov(s3, s1); + c->mov(s2, s0); + c->push(s1); + c->push(s0); + + dupped2X2(); + } + + void swap() { + Operand* s0 = c->stack(0); + Operand* s1 = c->stack(1); + Operand* tmp = c->temporary(); + + c->mov(s0, tmp); + c->mov(s1, s0); + c->mov(tmp, s1); + + c->release(tmp); + + swapped(); + } + + Frame* next; + MyThread* t; + Compiler* c; + object method; + uintptr_t* map; + Vector* objectPool; + Vector* traceLog; + uintptr_t* codeMask; + unsigned ip; + unsigned sp; + MyProtector protector; +}; + void NO_RETURN unwind(MyThread* t) { - for (void* frame = frameStart(t); frameValid(frame); - frame = frameNext(frame)) - { - if ((methodFlags(t, frameMethod(frame)) & ACC_NATIVE) == 0) { - NativeExceptionHandler* eh = findExceptionHandler(t, frame); - if (eh) { - object method = frameMethod(frame); - Compiled* code = reinterpret_cast - (methodCompiled(t, method)); + void* base = t->base; + void** stack = static_cast(t->stack); + while (true) { + void* returnAddress = *stack; + object node = findTraceNode(t, returnAddress); + if (node) { + object method = traceNodeMethod(t, node); + uint8_t* compiled = reinterpret_cast + (&singletonValue(t, methodCompiled(t, method), 0)); - void** stack = static_cast(frameBase(frame)); + ExceptionHandler* handler = findExceptionHandler + (t, method, difference(returnAddress, compiled)); + if (handler) { unsigned parameterFootprint = methodParameterFootprint(t, method); - unsigned localFootprint = compiledMaxLocals(code); + unsigned localFootprint = codeMaxLocals(t, methodCode(t, method)); if (localFootprint > parameterFootprint) { stack -= (localFootprint - parameterFootprint); @@ -970,194 +812,13 @@ unwind(MyThread* t) *(--stack) = t->exception; t->exception = 0; - vmJump(compiledCode(code) + nativeExceptionHandlerIp(eh), - frameBase(frame), - stack); - } - } - - void* next = frameNext(frame); - if (not frameValid(next) - or methodFlags(t, frameMethod(next)) & ACC_NATIVE) - { - vmJump(frameReturnAddress(frame), - *static_cast(frameBase(frame)), - static_cast(frameBase(frame)) + 2); - } - } - abort(t); -} - -void NO_RETURN -throwNew(MyThread* t, object class_) -{ - t->exception = makeNew(t, class_); - object trace = makeTrace(t); - set(t, t->exception, ThrowableTrace, trace); - unwind(t); -} - -void NO_RETURN -throw_(MyThread* t, object o) -{ - if (o) { - t->exception = o; - } else { - t->exception = makeNullPointerException(t); - } - unwind(t); -} - -uintptr_t* -frameStackMap(MyThread* t, void* frame) -{ - Compiled* code = reinterpret_cast - (methodCompiled(t, frameMethod(frame))); - unsigned ip = static_cast(frameAddress(frame)) - - compiledCode(code); - - unsigned bottom = 0; - unsigned top = compiledStackMapCount(t, code); - for (unsigned span = top - bottom; span; span = top - bottom) { - unsigned middle = bottom + (span / 2); - uintptr_t* map = compiledStackMap(t, code, middle); - - if (ip < *map) { - top = middle; - } else if (ip > *map) { - bottom = middle + 1; - } else { - return map + 1; - } - } - - fprintf(stderr, "%d not found in ", ip); - for (unsigned i = 0; i < compiledStackMapCount(t, code); ++i) { - fprintf(stderr, "%"LD" ", *compiledStackMap(t, code, i)); - } - fprintf(stderr, "\n"); - - abort(t); -} - -int -localOffset(MyThread* t, int v, object method) -{ - int parameterFootprint - = methodParameterFootprint(t, method) * BytesPerWord; - - v *= BytesPerWord; - if (v < parameterFootprint) { - return (parameterFootprint - v - BytesPerWord) + (BytesPerWord * 2) - + FrameFootprint; - } else { - return -(v + BytesPerWord - parameterFootprint); - } -} - -inline object* -frameLocalObject(MyThread* t, void* frame, unsigned index) -{ - return reinterpret_cast - (static_cast(frameBase(frame)) - + localOffset(t, index, frameMethod(frame))); -} - -void -visitParameters(MyThread* t, Heap::Visitor* v, void* frame) -{ - object method = frameMethod(frame); - - const char* spec = reinterpret_cast - (&byteArrayBody(t, methodSpec(t, method), 0)); - - unsigned index = 0; - if ((methodFlags(t, method) & ACC_STATIC) == 0) { - v->visit(frameLocalObject(t, frame, index++)); - } - - for (MethodSpecIterator it(t, spec); it.hasNext();) { - switch (*it.next()) { - case 'L': - case '[': - v->visit(frameLocalObject(t, frame, index++)); - break; - - case 'J': - case 'D': - index += 2; - break; - - default: - ++ index; - break; - } - } - - assert(t, index == methodParameterFootprint(t, method)); -} - -void -visitStackAndLocals(MyThread* t, Heap::Visitor* v, void* frame) -{ - Compiled* code = reinterpret_cast - (methodCompiled(t, frameMethod(frame))); - - unsigned parameterFootprint = compiledParameterFootprint(code); - unsigned count = compiledMaxStack(code) - + compiledMaxLocals(code) - - parameterFootprint; - - if (count) { - uintptr_t* stackMap = frameStackMap(t, frame); - -// fprintf(stderr, "visit stack and locals 0x%x of size %d, at %d of %s.%s\n", -// *stackMap, -// count, -// static_cast(frameAddress(frame)) -// - compiledCode(code), -// &byteArrayBody -// (t, className(t, methodClass(t, frameMethod(frame))), 0), -// &byteArrayBody(t, methodName(t, frameMethod(frame)), 0)); - - for (unsigned i = 0; i < count; ++i) { - if (getBit(stackMap, i)) { - v->visit(frameLocalObject(t, frame, i + parameterFootprint)); - } - } - } -} - -void -visitStack(MyThread* t, Heap::Visitor* v) -{ - void* frame = frameStart(t); - - if (frameValid(frame)) { - v->visit(&frameMethod(frame)); - } - - for (; frameValid(frame); frame = frameNext(frame)) { - // we only need to visit the parameters of this method if the - // caller is native. Otherwise, the caller owns them. - void* next = frameNext(frame); - if (frameValid(next)) { - v->visit(&frameMethod(next)); - - if (methodFlags(t, frameMethod(next)) & ACC_NATIVE) { - visitParameters(t, v, frame); + vmJump(compiled + exceptionHandlerIp(handler), base, stack); + } else { + stack = static_cast(base) + 1; + base = *static_cast(base); } } else { - visitParameters(t, v, frame); - } - - object method = frameMethod(frame); - Compiled* code = reinterpret_cast(methodCompiled(t, method)); - - if ((methodFlags(t, method) & ACC_NATIVE) == 0 - and code != t->m->processor->methodStub(t)) - { - visitStackAndLocals(t, v, frame); + vmJump(returnAddress, base, stack + 1); } } } @@ -1266,6 +927,30 @@ moduloDouble(uint64_t b, uint64_t a) return doubleToBits(fmod(bitsToDouble(a), bitsToDouble(b))); } +uint64_t +negateDouble(uint64_t a) +{ + return doubleToBits(- bitsToDouble(a)); +} + +uint32_t +doubleToFloat(int64_t a) +{ + return floatToBits(static_cast(bitsToDouble(a))); +} + +int32_t +doubleToInt(int64_t a) +{ + return static_cast(bitsToDouble(a)); +} + +int64_t +doubleToLong(int64_t a) +{ + return static_cast(bitsToDouble(a)); +} + uint32_t addFloat(uint32_t b, uint32_t a) { @@ -1296,6 +981,12 @@ moduloFloat(uint32_t b, uint32_t a) return floatToBits(fmod(bitsToFloat(a), bitsToFloat(b))); } +uint32_t +negateFloat(uint32_t a) +{ + return floatToBits(- bitsToFloat(a)); +} + int64_t divideLong(int64_t b, int64_t a) { @@ -1308,6 +999,36 @@ moduloLong(int64_t b, int64_t a) return a % b; } +uint64_t +floatToDouble(int32_t a) +{ + return doubleToBits(static_cast(bitsToFloat(a))); +} + +int32_t +floatToInt(int32_t a) +{ + return static_cast(bitsToFloat(a)); +} + +int64_t +floatToLong(int32_t a) +{ + return static_cast(bitsToFloat(a)); +} + +uint64_t +intToDouble(int32_t a) +{ + return doubleToBits(static_cast(a)); +} + +uint32_t +intToFloat(int32_t a) +{ + return floatToBits(static_cast(a)); +} + object makeBlankObjectArray(Thread* t, object class_, int32_t length) { @@ -1321,6 +1042,1863 @@ makeBlankArray(Thread* t, object (*constructor)(Thread*, uintptr_t, bool), return constructor(t, length, true); } +uintptr_t +lookUpAddress(int32_t key, uintptr_t* start, int32_t count, + uintptr_t* default_) +{ + int32_t bottom = 0; + int32_t top = count; + for (int32_t span = top - bottom; span; span = top - bottom) { + int32_t middle = bottom + (span / 2); + uintptr_t* p = start + (middle * 2); + int32_t k = *p; + + if (key < k) { + top = middle; + } else if (key > k) { + bottom = middle + 1; + } else { + return p[1]; + } + } + + return *default_; +} + +void +acquireMonitorForObject(Thread* t, object o) +{ + acquire(t, o); +} + +void +releaseMonitorForObject(Thread* t, object o) +{ + release(t, o); +} + +object +makeMultidimensionalArray2(MyThread* t, object class_, uintptr_t* stack, + int32_t dimensions) +{ + PROTECT(t, class_); + + int32_t counts[dimensions]; + for (int i = dimensions - 1; i >= 0; --i) { + counts[i] = stack[dimensions - i - 1]; + if (UNLIKELY(counts[i] < 0)) { + object message = makeString(t, "%d", counts[i]); + t->exception = makeNegativeArraySizeException(t, message); + return 0; + } + } + + object array = makeArray(t, counts[0], true); + setObjectClass(t, array, class_); + PROTECT(t, array); + + populateMultiArray(t, array, counts, 0, dimensions); + + return array; +} + +object +makeMultidimensionalArray(MyThread* t, object class_, uintptr_t* stack, + int32_t dimensions) +{ + object r = makeMultidimensionalArray2(t, class_, stack, dimensions); + if (UNLIKELY(t->exception)) { + unwind(t); + } else { + return r; + } +} + +void NO_RETURN +throwNew(MyThread* t, object class_) +{ + t->exception = makeNew(t, class_); + object trace = makeTrace(t); + set(t, t->exception, ThrowableTrace, trace); + unwind(t); +} + +void NO_RETURN +throw_(MyThread* t, object o) +{ + if (o) { + t->exception = o; + } else { + t->exception = makeNullPointerException(t); + } + unwind(t); +} + +void +compileThrowNew(MyThread* t, Frame* frame, Machine::Type type) +{ + Operand* class_ = frame->append(arrayBody(t, t->m->types, type)); + Compiler* c = frame->c; + c->indirectCallNoReturn + (c->constant(reinterpret_cast(throwNew)), + 2, c->thread(), class_); + + frame->trace(); +} + +void +pushReturnValue(MyThread* t, Frame* frame, unsigned code, Operand* result) +{ + switch (code) { + case ByteField: + case BooleanField: + case CharField: + case ShortField: + case FloatField: + case IntField: + frame->pushInt(result); + break; + + case ObjectField: + frame->pushObject(result); + break; + + case LongField: + case DoubleField: + frame->pushLong(result); + break; + + case VoidField: + break; + + default: + abort(t); + } +} + +void +compileDirectInvoke(MyThread* t, Frame* frame, object target) +{ + Operand* result = frame->c->alignedCall + (frame->c->constant + (reinterpret_cast + (&singletonBody(t, methodCompiled(t, target), 0)))); + + frame->trace(target, false); + + frame->pop(methodParameterFootprint(t, target)); + + pushReturnValue(t, frame, methodReturnCode(t, target), result); +} + +void +compile(MyThread* t, Frame* initialFrame, unsigned ip) +{ + uintptr_t map[Frame::mapSizeInWords(t, initialFrame->method)]; + Frame myFrame(initialFrame, map); + Frame* frame = &myFrame; + Compiler* c = frame->c; + + object code = methodCode(t, frame->method); + PROTECT(t, code); + + while (ip < codeLength(t, code)) { + if (getBit(frame->codeMask, ip)) { + // we've already visited this part of the code + return; + } + + markBit(frame->codeMask, ip); + + frame->startLogicalIp(ip); + + unsigned instruction = codeBody(t, code, ip++); + + switch (instruction) { + case aaload: + case baload: + case caload: + case daload: + case faload: + case iaload: + case laload: + case saload: { + Operand* next = c->label(); + Operand* outOfBounds = c->label(); + + Operand* index = frame->popInt(); + Operand* array = frame->popObject(); + + c->cmp(c->constant(0), index); + c->jl(outOfBounds); + + c->cmp(c->memory(array, ArrayLength), index); + c->jge(outOfBounds); + + switch (instruction) { + case aaload: + frame->pushObject(c->memory(array, ArrayBody, index, BytesPerWord)); + break; + + case faload: + case iaload: + frame->pushInt(c->select4(c->memory(array, ArrayBody, index, 4))); + break; + + case baload: + frame->pushInt(c->select1(c->memory(array, ArrayBody, index, 1))); + break; + + case caload: + frame->pushInt(c->select2z(c->memory(array, ArrayBody, index, 2))); + break; + + case daload: + case laload: + frame->pushInt(c->select8(c->memory(array, ArrayBody, index, 8))); + break; + + case saload: + frame->pushInt(c->select2(c->memory(array, ArrayBody, index, 2))); + break; + } + + c->jmp(next); + + c->mark(outOfBounds); + compileThrowNew(t, frame, Machine::ArrayIndexOutOfBoundsExceptionType); + + c->mark(next); + } break; + + case aastore: + case bastore: + case castore: + case dastore: + case fastore: + case iastore: + case lastore: + case sastore: { + Operand* next = c->label(); + Operand* outOfBounds = c->label(); + + Operand* value; + if (instruction == dastore or instruction == lastore) { + value = frame->popLong(); + } else if (instruction == aastore) { + value = frame->popObject(); + } else { + value = frame->popInt(); + } + + Operand* index = frame->popInt(); + Operand* array = frame->popObject(); + + c->cmp(c->constant(0), index); + c->jl(outOfBounds); + + c->cmp(c->memory(array, BytesPerWord), index); + c->jge(outOfBounds); + + switch (instruction) { + case aastore: + c->shl(c->constant(log(BytesPerWord)), index); + c->add(c->constant(ArrayBody), index); + + c->directCall + (c->constant(reinterpret_cast(set)), + 4, c->thread(), array, index, value); + + frame->trace(); + break; + + case fastore: + case iastore: + c->mov(value, c->select4(c->memory(array, ArrayBody, index, 4))); + break; + + case bastore: + c->mov(value, c->select1(c->memory(array, ArrayBody, index, 1))); + break; + + case castore: + case sastore: + c->mov(value, c->select2(c->memory(array, ArrayBody, index, 2))); + break; + + case dastore: + case lastore: + c->mov(value, c->select8(c->memory(array, ArrayBody, index, 8))); + break; + } + + c->jmp(next); + + c->mark(outOfBounds); + compileThrowNew(t, frame, Machine::ArrayIndexOutOfBoundsExceptionType); + + c->mark(next); + } break; + + case aconst_null: + frame->pushObject(c->constant(0)); + break; + + case aload: + frame->loadObject(codeBody(t, code, ip++)); + break; + + case aload_0: + frame->loadObject(0); + break; + + case aload_1: + frame->loadObject(1); + break; + + case aload_2: + frame->loadObject(2); + break; + + case aload_3: + frame->loadObject(3); + break; + + case anewarray: { + uint16_t index = codeReadInt16(t, code, ip); + + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + + Operand* nonnegative = c->label(); + + Operand* length = frame->popInt(); + c->cmp(c->constant(0), length); + c->jge(nonnegative); + + compileThrowNew(t, frame, Machine::NegativeArraySizeExceptionType); + + c->mark(nonnegative); + + Operand* r = c->indirectCall + (c->constant(reinterpret_cast(makeBlankObjectArray)), + 3, c->thread(), frame->append(class_), length); + + frame->trace(); + + frame->pushObject(r); + } break; + + case areturn: + c->epilogue(); + c->return_(frame->popObject()); + return; + + case arraylength: + frame->pushInt(c->memory(frame->popObject(), ArrayLength)); + break; + + case astore: + frame->storeObject(codeBody(t, code, ip++)); + break; + + case astore_0: + frame->storeObject(0); + break; + + case astore_1: + frame->storeObject(1); + break; + + case astore_2: + frame->storeObject(2); + break; + + case astore_3: + frame->storeObject(3); + break; + + case athrow: + c->indirectCallNoReturn + (c->constant(reinterpret_cast(throw_)), + 2, c->thread(), frame->popObject()); + + frame->trace(); + break; + + case bipush: + frame->pushInt + (c->constant(static_cast(codeBody(t, code, ip++)))); + break; + + case checkcast: { + uint16_t index = codeReadInt16(t, code, ip); + + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + + Operand* next = c->label(); + + Operand* instance = frame->topObject(); + Operand* tmp = c->temporary(); + + c->mov(instance, tmp); + + c->cmp(c->constant(0), tmp); + c->je(next); + + Operand* classOperand = frame->append(class_); + + c->mov(c->memory(tmp), tmp); + c->and_(c->constant(PointerMask), tmp); + + c->cmp(classOperand, tmp); + c->je(next); + + Operand* result = c->directCall + (c->constant(reinterpret_cast(isAssignableFrom)), + 2, classOperand, tmp); + + c->release(tmp); + + c->cmp(0, result); + c->jne(next); + + compileThrowNew(t, frame, Machine::ClassCastExceptionType); + + c->mark(next); + } break; + + case d2f: { + Operand* a = frame->popLong(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(doubleToFloat)), 1, a)); + } break; + + case d2i: { + Operand* a = frame->popLong(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(doubleToInt)), 1, a)); + } break; + + case d2l: { + Operand* a = frame->popLong(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(doubleToLong)), 1, a)); + } break; + + case dadd: { + Operand* a = frame->popLong(); + Operand* b = frame->popLong(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(addDouble)), 2, a, b)); + } break; + + case dcmpg: { + Operand* a = frame->popLong(); + Operand* b = frame->popLong(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(compareDoublesG)), 2, a, b)); + } break; + + case dcmpl: { + Operand* a = frame->popLong(); + Operand* b = frame->popLong(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(compareDoublesL)), 2, a, b)); + } break; + + case dconst_0: + frame->pushLong(c->constant(doubleToBits(0.0))); + break; + + case dconst_1: + frame->pushLong(c->constant(doubleToBits(1.0))); + break; + + case ddiv: { + Operand* a = frame->popLong(); + Operand* b = frame->popLong(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(divideDouble)), 2, a, b)); + } break; + + case dmul: { + Operand* a = frame->popLong(); + Operand* b = frame->popLong(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(multiplyDouble)), 2, a, b)); + } break; + + case dneg: { + Operand* a = frame->popLong(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(negateDouble)), 1, a)); + } break; + + case vm::drem: { + Operand* a = frame->popLong(); + Operand* b = frame->popLong(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(moduloDouble)), 2, a, b)); + } break; + + case dsub: { + Operand* a = frame->popLong(); + Operand* b = frame->popLong(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(subtractDouble)), 2, a, b)); + } break; + + case dup: + frame->dup(); + break; + + case dup_x1: + frame->dupX1(); + break; + + case dup_x2: + frame->dupX2(); + break; + + case dup2: + frame->dup2(); + break; + + case dup2_x1: + frame->dup2X1(); + break; + + case dup2_x2: + frame->dup2X2(); + break; + + case f2d: { + Operand* a = frame->popInt(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(floatToDouble)), 1, a)); + } break; + + case f2i: { + Operand* a = frame->popInt(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(floatToInt)), 1, a)); + } break; + + case f2l: { + Operand* a = frame->popInt(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(floatToLong)), 1, a)); + } break; + + case fadd: { + Operand* a = frame->popInt(); + Operand* b = frame->popInt(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(addFloat)), 2, a, b)); + } break; + + case fcmpg: { + Operand* a = frame->popInt(); + Operand* b = frame->popInt(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(compareFloatsG)), 2, a, b)); + } break; + + case fcmpl: { + Operand* a = frame->popInt(); + Operand* b = frame->popInt(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(compareFloatsL)), 2, a, b)); + } break; + + case fconst_0: + frame->pushInt(c->constant(floatToBits(0.0))); + break; + + case fconst_1: + frame->pushInt(c->constant(floatToBits(1.0))); + break; + + case fconst_2: + frame->pushInt(c->constant(floatToBits(2.0))); + break; + + case fdiv: { + Operand* a = frame->popInt(); + Operand* b = frame->popInt(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(divideFloat)), 2, a, b)); + } break; + + case fmul: { + Operand* a = frame->popInt(); + Operand* b = frame->popInt(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(multiplyFloat)), 2, a, b)); + } break; + + case fneg: { + Operand* a = frame->popLong(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(negateFloat)), 1, a)); + } break; + + case vm::frem: { + Operand* a = frame->popInt(); + Operand* b = frame->popInt(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(moduloFloat)), 2, a, b)); + } break; + + case fsub: { + Operand* a = frame->popInt(); + Operand* b = frame->popInt(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(subtractFloat)), 2, a, b)); + } break; + + case getfield: + case getstatic: { + uint16_t index = codeReadInt16(t, code, ip); + + object field = resolveField(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + + Operand* table; + + if (instruction == getstatic) { + initClass(t, fieldClass(t, field)); + if (UNLIKELY(t->exception)) return; + + table = frame->append(classStaticTable(t, fieldClass(t, field))); + } else { + table = frame->popObject(); + } + + switch (fieldCode(t, field)) { + case ByteField: + case BooleanField: + frame->pushInt(c->select1(c->memory(table, fieldOffset(t, field)))); + break; + + case CharField: + frame->pushInt(c->select2z(c->memory(table, fieldOffset(t, field)))); + break; + + case ShortField: + frame->pushInt(c->select2(c->memory(table, fieldOffset(t, field)))); + break; + + case FloatField: + case IntField: + frame->pushInt(c->select4(c->memory(table, fieldOffset(t, field)))); + break; + + case DoubleField: + case LongField: + frame->pushLong(c->select8(c->memory(table, fieldOffset(t, field)))); + break; + + case ObjectField: + frame->pushObject(c->memory(c->memory(table, fieldOffset(t, field)))); + break; + + default: + abort(t); + } + } break; + + case goto_: { + uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); + assert(t, newIp < codeLength(t, code)); + + c->jmp(c->logicalIp(newIp)); + ip = newIp; + } break; + + case goto_w: { + uint32_t newIp = (ip - 5) + codeReadInt32(t, code, ip); + assert(t, newIp < codeLength(t, code)); + + c->jmp(c->logicalIp(newIp)); + ip = newIp; + } break; + + case i2b: { + Operand* top = frame->topInt(); + c->mov(c->select1(top), top); + } break; + + case i2c: { + Operand* top = frame->topInt(); + c->mov(c->select2z(top), top); + } break; + + case i2d: { + Operand* a = frame->popInt(); + frame->pushLong + (c->directCall + (c->constant(reinterpret_cast(intToDouble)), 1, a)); + } break; + + case i2f: { + Operand* a = frame->popInt(); + frame->pushInt + (c->directCall + (c->constant(reinterpret_cast(intToFloat)), 1, a)); + } break; + + case i2l: + frame->pushLong(frame->popInt()); + break; + + case i2s: { + Operand* top = frame->topInt(); + c->mov(c->select2(top), top); + } break; + + case iadd: { + Operand* a = frame->popInt(); + c->add(a, frame->topInt()); + } break; + + case iand: { + Operand* a = frame->popInt(); + c->and_(a, frame->topInt()); + } break; + + case iconst_m1: + frame->pushInt(c->constant(-1)); + break; + + case iconst_0: + frame->pushInt(c->constant(0)); + break; + + case iconst_1: + frame->pushInt(c->constant(1)); + break; + + case iconst_2: + frame->pushInt(c->constant(2)); + break; + + case iconst_3: + frame->pushInt(c->constant(3)); + break; + + case iconst_4: + frame->pushInt(c->constant(4)); + break; + + case iconst_5: + frame->pushInt(c->constant(5)); + break; + + case idiv: { + Operand* a = frame->popInt(); + c->div(a, frame->topInt()); + } break; + + case if_acmpeq: + case if_acmpne: { + uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); + assert(t, newIp < codeLength(t, code)); + + Operand* a = frame->popObject(); + Operand* b = frame->popObject(); + c->cmp(a, b); + + Operand* target = c->logicalIp(newIp); + if (instruction == if_acmpeq) { + c->je(target); + } else { + c->jne(target); + } + + compile(t, frame, newIp); + if (UNLIKELY(t->exception)) return; + } break; + + case if_icmpeq: + case if_icmpne: + case if_icmpgt: + case if_icmpge: + case if_icmplt: + case if_icmple: { + uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); + assert(t, newIp < codeLength(t, code)); + + Operand* a = frame->popInt(); + Operand* b = frame->popInt(); + c->cmp(a, b); + + Operand* target = c->logicalIp(newIp); + switch (instruction) { + case if_icmpeq: + c->je(target); + break; + case if_icmpne: + c->jne(target); + break; + case if_icmpgt: + c->jg(target); + break; + case if_icmpge: + c->jge(target); + break; + case if_icmplt: + c->jl(target); + break; + case if_icmple: + c->jle(target); + break; + } + + compile(t, frame, newIp); + if (UNLIKELY(t->exception)) return; + } break; + + case ifeq: + case ifne: + case ifgt: + case ifge: + case iflt: + case ifle: { + uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); + assert(t, newIp < codeLength(t, code)); + + c->cmp(0, frame->popInt()); + + Operand* target = c->logicalIp(newIp); + switch (instruction) { + case ifeq: + c->je(target); + break; + case ifne: + c->jne(target); + break; + case ifgt: + c->jg(target); + break; + case ifge: + c->jge(target); + break; + case iflt: + c->jl(target); + break; + case ifle: + c->jle(target); + break; + } + + compile(t, frame, newIp); + if (UNLIKELY(t->exception)) return; + } break; + + case ifnull: + case ifnonnull: { + uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); + assert(t, newIp < codeLength(t, code)); + + c->cmp(0, frame->popObject()); + + Operand* target = c->logicalIp(newIp); + if (instruction == ifnull) { + c->je(target); + } else { + c->jne(target); + } + + compile(t, frame, newIp); + if (UNLIKELY(t->exception)) return; + } break; + + case iinc: { + uint8_t index = codeBody(t, code, ip++); + int8_t count = codeBody(t, code, ip++); + + frame->increment(index, count); + } break; + + case iload: + case fload: + frame->loadInt(codeBody(t, code, ip++)); + break; + + case iload_0: + case fload_0: + frame->loadInt(0); + break; + + case iload_1: + case fload_1: + frame->loadInt(1); + break; + + case iload_2: + case fload_2: + frame->loadInt(2); + break; + + case iload_3: + case fload_3: + frame->loadInt(3); + break; + + case imul: { + Operand* a = frame->popInt(); + c->mul(a, frame->topInt()); + } break; + + case instanceof: { + uint16_t index = codeReadInt16(t, code, ip); + + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + + Operand* call = c->label(); + Operand* next = c->label(); + Operand* zero = c->label(); + + Operand* instance = frame->topObject(); + Operand* tmp = c->temporary(); + Operand* result = c->temporary(); + + c->mov(instance, tmp); + + c->cmp(c->constant(0), tmp); + c->je(zero); + + Operand* classOperand = frame->append(class_); + + c->mov(c->memory(tmp), tmp); + c->and_(c->constant(PointerMask), tmp); + + c->cmp(classOperand, tmp); + c->jne(call); + + c->mov(c->constant(1), result); + c->jmp(next); + + c->mov + (c->directCall + (c->constant(reinterpret_cast(isAssignableFrom)), + 2, classOperand, tmp), result); + + c->release(tmp); + + c->jmp(next); + + c->mark(zero); + + c->mov(c->constant(0), result); + + c->mark(next); + frame->pushInt(result); + + c->release(result); + } break; + + case invokeinterface: { + uint16_t index = codeReadInt16(t, code, ip); + ip += 2; + + object target = resolveMethod(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + + unsigned parameterFootprint = methodParameterFootprint(t, target); + + unsigned instance = parameterFootprint - 1; + + Operand* found = c->directCall + (c->constant + (reinterpret_cast(findInterfaceMethodFromInstance)), + 3, c->thread(), frame->append(target), c->stack(instance)); + + c->mov(c->memory(found, MethodCompiled), found); + + Operand* result = c->call(c->memory(found, SingletonBody)); + + frame->trace(target, true); + + frame->pop(parameterFootprint); + + pushReturnValue(t, frame, methodReturnCode(t, target), result); + } break; + + case invokespecial: { + uint16_t index = codeReadInt16(t, code, ip); + + object target = resolveMethod(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + + object class_ = methodClass(t, target); + if (isSpecialMethod(t, target, class_)) { + target = findMethod(t, target, classSuper(t, class_)); + } + + compileDirectInvoke(t, frame, target); + } break; + + case invokestatic: { + uint16_t index = codeReadInt16(t, code, ip); + + object target = resolveMethod(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + PROTECT(t, target); + + initClass(t, methodClass(t, target)); + if (UNLIKELY(t->exception)) return; + + compileDirectInvoke(t, frame, target); + } break; + + case invokevirtual: { + uint16_t index = codeReadInt16(t, code, ip); + + object target = resolveMethod(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + + unsigned parameterFootprint = methodParameterFootprint(t, target); + + unsigned offset = ClassVtable + (methodOffset(t, target) * BytesPerWord); + + Operand* instance = c->stack(parameterFootprint - 1); + Operand* class_ = c->temporary(); + + c->mov(c->memory(instance), class_); + c->and_(c->constant(PointerMask), class_); + + Operand* result = c->call(c->memory(class_, offset)); + + frame->trace(target, true); + + c->release(class_); + + frame->pop(parameterFootprint); + + pushReturnValue(t, frame, methodReturnCode(t, target), result); + } break; + + case ior: { + Operand* a = frame->popInt(); + c->or_(a, frame->topInt()); + } break; + + case irem: { + Operand* a = frame->popInt(); + c->rem(a, frame->topInt()); + } break; + + case ireturn: + case freturn: + c->epilogue(); + c->return_(frame->popInt()); + return; + + case ishl: { + Operand* a = frame->popInt(); + c->shl(a, frame->topInt()); + } break; + + case ishr: { + Operand* a = frame->popInt(); + c->shr(a, frame->topInt()); + } break; + + case istore: + case fstore: + frame->storeInt(codeBody(t, code, ip++)); + break; + + case istore_0: + case fstore_0: + frame->storeInt(0); + break; + + case istore_1: + case fstore_1: + frame->storeInt(1); + break; + + case istore_2: + case fstore_2: + frame->storeInt(2); + break; + + case istore_3: + case fstore_3: + frame->storeInt(3); + break; + + case isub: { + Operand* a = frame->popInt(); + c->sub(a, frame->topInt()); + } break; + + case iushr: { + Operand* a = frame->popInt(); + c->ushr(a, frame->topInt()); + } break; + + case ixor: { + Operand* a = frame->popInt(); + c->xor_(a, frame->topInt()); + } break; + + case jsr: + case jsr_w: + case ret: + // see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4381996 + abort(t); + + case l2i: + frame->pushInt(frame->popLong()); + break; + + case ladd: { + Operand* a = frame->popLong(); + c->sub(a, frame->topLong()); + } break; + + case lcmp: { + Operand* next = c->label(); + Operand* less = c->label(); + Operand* greater = c->label(); + + Operand* a = frame->popLong(); + Operand* b = frame->popLong(); + Operand* result = c->temporary(); + + c->cmp(a, b); + c->jl(less); + c->jg(greater); + + c->mov(c->constant(0), result); + c->jmp(next); + + c->mark(less); + c->mov(c->constant(-1), result); + c->jmp(next); + + c->mark(greater); + c->mov(c->constant(1), result); + + c->mark(next); + frame->pushInt(result); + + c->release(result); + } break; + + case lconst_0: + frame->pushLong(c->constant(0)); + break; + + case lconst_1: + frame->pushLong(c->constant(1)); + break; + + case ldc: + case ldc_w: { + uint16_t index; + + if (instruction == ldc) { + index = codeBody(t, code, ip++); + } else { + index = codeReadInt16(t, code, ip); + } + + object pool = codePool(t, code); + + if (singletonIsObject(t, pool, index - 1)) { + object v = singletonObject(t, pool, index - 1); + if (objectClass(t, v) + == arrayBody(t, t->m->types, Machine::ByteArrayType)) + { + object class_ = resolveClassInPool(t, pool, index - 1); + if (UNLIKELY(t->exception)) return; + + frame->pushObject(frame->append(class_)); + } else { + frame->pushObject(frame->append(v)); + } + } else { + frame->pushInt(c->constant(singletonValue(t, pool, index - 1))); + } + } break; + + case ldc2_w: { + uint16_t index = codeReadInt16(t, code, ip); + + object pool = codePool(t, code); + + uint64_t v; + memcpy(&v, &singletonValue(t, pool, index - 1), 8); + frame->pushLong(c->constant(v)); + } break; + + case ldiv_: { + Operand* a = frame->popLong(); + c->div(a, frame->topLong()); + } break; + + case lload: + case dload: + frame->loadLong(codeBody(t, code, ip++)); + break; + + case lload_0: + case dload_0: + frame->loadLong(0); + break; + + case lload_1: + case dload_1: + frame->loadLong(1); + break; + + case lload_2: + case dload_2: + frame->loadLong(2); + break; + + case lload_3: + case dload_3: + frame->loadLong(3); + break; + + case lmul: { + Operand* a = frame->popLong(); + c->mul(a, frame->topLong()); + } break; + + case lneg: + c->neg(frame->topLong()); + break; + + case lookupswitch: { + int32_t base = ip - 1; + + ip = (ip + 3) & ~3; // pad to four byte boundary + + Operand* key = frame->popInt(); + + uint32_t defaultIp = base + codeReadInt32(t, code, ip); + assert(t, defaultIp < codeLength(t, code)); + + compile(t, frame, defaultIp); + if (UNLIKELY(t->exception)) return; + + Operand* default_ = c->poolAppend(c->logicalIp(defaultIp)); + + int32_t pairCount = codeReadInt32(t, code, ip); + + Operand* start; + for (int32_t i = 0; i < pairCount; ++i) { + unsigned index = ip + (i * 8); + int32_t key = codeReadInt32(t, code, index); + uint32_t newIp = base + codeReadInt32(t, code, index); + assert(t, newIp < codeLength(t, code)); + + compile(t, frame, newIp); + if (UNLIKELY(t->exception)) return; + + Operand* result = c->poolAppend(c->constant(key)); + c->poolAppend(c->logicalIp(newIp)); + + if (i == 0) { + start = result; + } + } + + c->jmp + (c->directCall + (c->constant(reinterpret_cast(lookUpAddress)), + 4, key, start, c->constant(pairCount), default_)); + } return; + + case lor: { + Operand* a = frame->popLong(); + c->or_(a, frame->topLong()); + } break; + + case lrem: { + Operand* a = frame->popLong(); + c->rem(a, frame->topLong()); + } break; + + case lreturn: + case dreturn: + c->epilogue(); + c->return_(frame->popLong()); + return; + + case lshl: { + Operand* a = frame->popLong(); + c->shl(a, frame->topLong()); + } break; + + case lshr: { + Operand* a = frame->popLong(); + c->shr(a, frame->topLong()); + } break; + + case lstore: + case dstore: + frame->storeLong(codeBody(t, code, ip++)); + break; + + case lstore_0: + case dstore_0: + frame->storeLong(0); + break; + + case lstore_1: + case dstore_1: + frame->storeLong(1); + break; + + case lstore_2: + case dstore_2: + frame->storeLong(2); + break; + + case lstore_3: + case dstore_3: + frame->storeLong(3); + break; + + case lsub: { + Operand* a = frame->popLong(); + c->sub(a, frame->topLong()); + } break; + + case lushr: { + Operand* a = frame->popLong(); + c->ushr(a, frame->topLong()); + } break; + + case lxor: { + Operand* a = frame->popLong(); + c->xor_(a, frame->topLong()); + } break; + + case monitorenter: { + c->indirectCall + (c->constant(reinterpret_cast(acquireMonitorForObject)), + 2, c->thread(), frame->popObject()); + + frame->trace(); + } break; + + case monitorexit: { + c->indirectCall + (c->constant(reinterpret_cast(releaseMonitorForObject)), + 2, c->thread(), frame->popObject()); + + frame->trace(); + } break; + + case multianewarray: { + uint16_t index = codeReadInt16(t, code, ip); + uint8_t dimensions = codeBody(t, code, ip++); + + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + PROTECT(t, class_); + + Operand* result = c->indirectCall + (c->constant(reinterpret_cast(makeMultidimensionalArray)), + 4, c->thread(), frame->append(class_), c->stack(), + c->constant(dimensions)); + + frame->trace(); + + frame->pop(dimensions); + frame->pushObject(result); + } break; + + case new_: { + uint16_t index = codeReadInt16(t, code, ip); + + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + PROTECT(t, class_); + + initClass(t, class_); + if (UNLIKELY(t->exception)) return; + + Operand* result; + if (classVmFlags(t, class_) & WeakReferenceFlag) { + result = c->indirectCall + (c->constant(reinterpret_cast(makeNewWeakReference)), + 2, c->thread(), frame->append(class_)); + } else { + result = c->indirectCall + (c->constant(reinterpret_cast(makeNew)), + 2, c->thread(), frame->append(class_)); + } + + frame->trace(); + + frame->pushObject(result); + } break; + + case newarray: { + uint8_t type = codeBody(t, code, ip++); + + Operand* nonnegative = c->label(); + + Operand* size = frame->popInt(); + c->cmp(0, size); + c->jge(nonnegative); + + compileThrowNew(t, frame, Machine::NegativeArraySizeExceptionType); + + c->mark(nonnegative); + + object (*constructor)(Thread*, uintptr_t, bool); + switch (type) { + case T_BOOLEAN: + constructor = makeBooleanArray; + break; + + case T_CHAR: + constructor = makeCharArray; + break; + + case T_FLOAT: + constructor = makeFloatArray; + break; + + case T_DOUBLE: + constructor = makeDoubleArray; + break; + + case T_BYTE: + constructor = makeByteArray; + break; + + case T_SHORT: + constructor = makeShortArray; + break; + + case T_INT: + constructor = makeIntArray; + break; + + case T_LONG: + constructor = makeLongArray; + break; + + default: abort(t); + } + + Operand* result = c->indirectCall + (c->constant(reinterpret_cast(makeBlankArray)), + 2, c->constant(reinterpret_cast(constructor)), size); + + frame->trace(); + + frame->pushObject(result); + } break; + + case nop: break; + + case pop_: + frame->pop(1); + break; + + case pop2: + frame->pop(2); + break; + + case putfield: + case putstatic: { + uint16_t index = codeReadInt16(t, code, ip); + + object field = resolveField(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return; + + object staticTable; + + if (instruction == putstatic) { + PROTECT(t, field); + initClass(t, fieldClass(t, field)); + if (UNLIKELY(t->exception)) return; + + staticTable = classStaticTable(t, fieldClass(t, field)); + } + + Operand* value; + switch (fieldCode(t, field)) { + case ByteField: + case BooleanField: + case CharField: + case ShortField: + case FloatField: + case IntField: { + value = frame->popInt(); + } break; + + case DoubleField: + case LongField: { + value = frame->popLong(); + } break; + + case ObjectField: { + value = frame->popLong(); + } break; + + default: abort(t); + } + + Operand* table; + + if (instruction == putstatic) { + table = frame->append(staticTable); + } else { + table = frame->popObject(); + } + + switch (fieldCode(t, field)) { + case ByteField: + case BooleanField: + c->mov(value, c->select1(c->memory(table, fieldOffset(t, field)))); + break; + + case CharField: + case ShortField: + c->mov(value, c->select2(c->memory(table, fieldOffset(t, field)))); + break; + + case FloatField: + case IntField: + c->mov(value, c->select4(c->memory(table, fieldOffset(t, field)))); + break; + + case DoubleField: + case LongField: + c->mov(value, c->select8(c->memory(table, fieldOffset(t, field)))); + break; + + case ObjectField: + c->directCall + (c->constant(reinterpret_cast(set)), + 4, c->thread(), table, fieldOffset(t, field), value); + break; + + default: abort(t); + } + } break; + + case return_: + c->epilogue(); + c->ret(); + return; + + case sipush: + frame->pushInt + (c->constant(static_cast(codeReadInt16(t, code, ip)))); + break; + + case swap: + frame->swap(); + break; + + case tableswitch: { + int32_t base = ip - 1; + + ip = (ip + 3) & ~3; // pad to four byte boundary + + Operand* key = frame->popInt(); + + uint32_t defaultIp = base + codeReadInt32(t, code, ip); + assert(t, defaultIp < codeLength(t, code)); + + compile(t, frame, defaultIp); + if (UNLIKELY(t->exception)) return; + + Operand* default_ = c->poolAppend(c->logicalIp(defaultIp)); + + int32_t bottom = codeReadInt32(t, code, ip); + int32_t top = codeReadInt32(t, code, ip); + + Operand* start; + for (int32_t i = 0; i < bottom - top + 1; ++i) { + unsigned index = ip + (i * 4); + uint32_t newIp = base + codeReadInt32(t, code, index); + assert(t, newIp < codeLength(t, code)); + + compile(t, frame, newIp); + if (UNLIKELY(t->exception)) return; + + Operand* result = c->poolAppend(c->logicalIp(newIp)); + if (i == 0) { + start = result; + } + } + + Operand* defaultCase = c->label(); + + c->cmp(c->constant(bottom), key); + c->jl(defaultCase); + + c->cmp(c->constant(top), key); + c->jg(defaultCase); + + c->shl(c->constant(1), key); + c->jmp(c->memory(start, 0, key, BytesPerWord)); + + c->mark(defaultCase); + c->jmp(default_); + } return; + + case wide: { + switch (codeBody(t, code, ip++)) { + case aload: { + frame->loadObject(codeReadInt16(t, code, ip)); + } break; + + case astore: { + frame->storeObject(codeReadInt16(t, code, ip)); + } break; + + case iinc: { + uint16_t index = codeReadInt16(t, code, ip); + uint16_t count = codeReadInt16(t, code, ip); + + frame->increment(index, count); + } break; + + case iload: { + frame->loadInt(codeReadInt16(t, code, ip)); + } break; + + case istore: { + frame->storeInt(codeReadInt16(t, code, ip)); + } break; + + case lload: { + frame->loadLong(codeReadInt16(t, code, ip)); + } break; + + case lstore: { + frame->storeLong(codeReadInt16(t, code, ip)); + } break; + + case ret: + // see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4381996 + abort(t); + + default: abort(t); + } + } break; + } + } +} + +object +finish(MyThread* t, Compiler* c, object method, Vector* objectPool, + Vector* traceLog) +{ + unsigned count = ceiling(c->size(), BytesPerWord); + unsigned size = count + singletonMaskSize(count); + object result = allocate2(t, size * BytesPerWord, true, true); + initSingleton(t, result, size, true); + singletonMask(t, result)[0] = 1; + + uint8_t* start = reinterpret_cast(&singletonValue(t, result, 0)); + + c->writeTo(start); + + if (method) { + PROTECT(t, method); + + for (unsigned i = 0; i < objectPool->length(); i += sizeof(PoolElement)) { + PoolElement* e = objectPool->peek(i); + + singletonMarkObject + (t, result, e->offset->value(t->m->system) / BytesPerWord); + + set(t, result, SingletonBody + e->offset->value(t->m->system), + e->value); + } + + unsigned traceSize = Frame::traceSizeInBytes(t, method); + unsigned mapSize = Frame::mapSizeInBytes(t, method); + for (unsigned i = 0; i < traceLog->length(); i += traceSize) { + TraceElement* e = traceLog->peek(i); + + object node = makeTraceNode + (t, reinterpret_cast(start + e->offset->value(t->m->system)), + 0, 0, 0, method, e->target, e->virtualCall, mapSize / BytesPerWord, + false); + + if (mapSize) { + memcpy(&traceNodeMap(t, node, 0), e->map, mapSize); + } + + insertTraceNode(t, node); + } + + object code = methodCode(t, method); + PROTECT(t, code); + + { + object oldTable = codeExceptionHandlerTable(t, code); + if (oldTable) { + PROTECT(t, oldTable); + + unsigned length = exceptionHandlerTableLength(t, oldTable); + object newTable = makeExceptionHandlerTable(t, length, false); + for (unsigned i = 0; i < length; ++i) { + ExceptionHandler* oldHandler = exceptionHandlerTableBody + (t, oldTable, i); + ExceptionHandler* newHandler = exceptionHandlerTableBody + (t, newTable, i); + + exceptionHandlerStart(newHandler) + = c->logicalIpToOffset(exceptionHandlerStart(oldHandler)); + + exceptionHandlerEnd(newHandler) + = c->logicalIpToOffset(exceptionHandlerEnd(oldHandler)); + + exceptionHandlerIp(newHandler) + = c->logicalIpToOffset(exceptionHandlerIp(oldHandler)); + + exceptionHandlerCatchType(newHandler) + = exceptionHandlerCatchType(oldHandler); + } + + set(t, code, CodeExceptionHandlerTable, newTable); + } + } + + { + object oldTable = codeLineNumberTable(t, code); + if (oldTable) { + PROTECT(t, oldTable); + + unsigned length = lineNumberTableLength(t, oldTable); + object newTable = makeLineNumberTable(t, length, false); + for (unsigned i = 0; i < length; ++i) { + LineNumber* oldLine = lineNumberTableBody(t, oldTable, i); + LineNumber* newLine = lineNumberTableBody(t, newTable, i); + + lineNumberIp(newLine) = c->logicalIpToOffset(lineNumberIp(oldLine)); + + lineNumberLine(newLine) = lineNumberLine(oldLine); + } + + set(t, code, CodeLineNumberTable, newTable); + } + } + } + + return result; +} + +object +finish(MyThread* t, Compiler* c) +{ + return finish(t, c, 0, 0, 0); +} + +object +compile(MyThread* t, Compiler* c, object method) +{ + PROTECT(t, method); + + c->prologue(); + + object code = methodCode(t, method); + PROTECT(t, code); + + unsigned footprint = methodParameterFootprint(t, method); + unsigned locals = codeMaxLocals(t, code); + c->sub(c->constant((locals - footprint) * BytesPerWord), c->stack()); + + Vector objectPool(t->m->system, 256); + Vector traceLog(t->m->system, 1024); + uintptr_t map[Frame::mapSizeInWords(t, method)]; + Frame frame(t, c, method, map, &objectPool, &traceLog); + + compile(t, &frame, 0); + if (UNLIKELY(t->exception)) return 0; + + object eht = codeExceptionHandlerTable(t, methodCode(t, method)); + if (eht) { + PROTECT(t, eht); + + for (unsigned i = 0; i < exceptionHandlerTableLength(t, eht); ++i) { + ExceptionHandler* eh = exceptionHandlerTableBody(t, eht, i); + + assert(t, getBit(frame.codeMask, exceptionHandlerStart(eh))); + + uintptr_t map[Frame::mapSizeInWords(t, method)]; + Frame frame2(&frame, map); + frame2.pushedObject(); + + compile(t, &frame2, exceptionHandlerIp(eh)); + if (UNLIKELY(t->exception)) return 0; + } + } + + return finish(t, c, method, &objectPool, &traceLog); +} + +void +compile(MyThread* t, object method); + +void* +compileMethod(MyThread* t) +{ + object node = findTraceNode(t, *static_cast(t->stack)); + object target = resolveTarget(t, t->stack, traceNodeTarget(t, node)); + + if (LIKELY(t->exception == 0)) { + compile(t, target); + } + + if (UNLIKELY(t->exception)) { + unwind(t); + } else { + if (not traceNodeVirtualCall(t, node)) { + Compiler* c = makeCompiler(t->m->system, 0); + c->updateCall(reinterpret_cast(treeNodeKey(t, node)), + &singletonValue(t, methodCompiled(t, target), 0)); + c->dispose(); + } + return &singletonValue(t, methodCompiled(t, target), 0); + } +} + uint64_t invokeNative2(MyThread* t, object method) { @@ -1359,9 +2937,8 @@ invokeNative2(MyThread* t, object method) args[argOffset++] = reinterpret_cast(t); types[typeOffset++] = POINTER_TYPE; - uintptr_t* sp = static_cast(frameBase(frameStart(t))) - + (methodParameterFootprint(t, method) + 1) - + (FrameFootprint / BytesPerWord); + uintptr_t* sp = static_cast(t->stack) + + (methodParameterFootprint(t, method) + 1); if (methodFlags(t, method) & ACC_STATIC) { args[argOffset++] = reinterpret_cast(&class_); @@ -1442,9 +3019,16 @@ invokeNative2(MyThread* t, object method) } uint64_t -invokeNative(MyThread* t, object method) +invokeNative(MyThread* t) { - uint64_t result = invokeNative2(t, method); + object node = findTraceNode(t, *static_cast(t->stack)); + object target = resolveTarget(t, t->stack, traceNodeTarget(t, node)); + uint64_t result; + + if (LIKELY(t->exception == 0)) { + result = invokeNative2(t, target); + } + if (UNLIKELY(t->exception)) { unwind(t); } else { @@ -1452,3230 +3036,134 @@ invokeNative(MyThread* t, object method) } } +inline object* +localObject(MyThread* t, void* base, object method, unsigned index) +{ + return reinterpret_cast + (static_cast(base) + localOffset(t, index, method)); +} + void -compileMethod(MyThread* t, object method); - -inline bool -isByte(int32_t v) +visitParameters(MyThread* t, Heap::Visitor* v, void* base, object method) { - return v == static_cast(v); -} + const char* spec = reinterpret_cast + (&byteArrayBody(t, methodSpec(t, method), 0)); -inline bool -isInt32(uintptr_t v) -{ - return v == static_cast(static_cast(v)); -} - -enum Register { - rax = 0, - rcx = 1, - rdx = 2, - rbx = 3, - rsp = 4, - rbp = 5, - rsi = 6, - rdi = 7, - r8 = 8, - r9 = 9, - r10 = 10, - r11 = 11, - r12 = 12, - r13 = 13, - r14 = 14, - r15 = 15, -}; - -enum SSERegister { - xmm0 = 0, - xmm1 = 1, - xmm2 = 2, - xmm3 = 3, - xmm4 = 4, - xmm5 = 5, - xmm6 = 6, - xmm7 = 7 -}; - -class Assembler { - public: - class Label { - public: - static const unsigned Capacity = 8; - - Label(Assembler* a): - code(&(a->code)), - unresolvedCount(0), - mark_(-1) - { } - - void reference() { - if (mark_ == -1) { - expect(code->s, unresolvedCount < Capacity); - unresolved[unresolvedCount] = code->length(); - ++ unresolvedCount; - - code->append4(0); - } else { - code->append4(mark_ - (code->length() + 4)); - } - } - - void mark() { - mark_ = code->length(); - for (unsigned i = 0; i < unresolvedCount; ++i) { - code->set4(unresolved[i], mark_ - (unresolved[i] + 4)); - } - } - - Buffer* code; - unsigned unresolved[Capacity]; - unsigned unresolvedCount; - int mark_; - }; - - Assembler(System* s): - code(s, 1024), - jumps(s, 256) - { } - - void rex() { - if (BytesPerWord == 8) { - code.append(0x48); - } - } - - void mov(Register src, Register dst) { - rex(); - code.append(0x89); - code.append(0xc0 | (src << 3) | dst); - } - - void offsetInstruction(uint8_t instruction, uint8_t zeroPrefix, - uint8_t bytePrefix, uint8_t wordPrefix, - unsigned a, unsigned b, int32_t offset) - { - code.append(instruction); - - uint8_t prefix; - if (offset == 0 and b != rbp) { - prefix = zeroPrefix; - } else if (isByte(offset)) { - prefix = bytePrefix; - } else { - prefix = wordPrefix; - } - - code.append(prefix | (a << 3) | b); - - if (b == rsp) { - code.append(0x24); - } - - if (offset == 0 and b != rbp) { - // do nothing - } else if (isByte(offset)) { - code.append(offset); - } else { - code.append4(offset); - } - } - - void movz1(Register src, Register dst) { - code.append(0x0f); - code.append(0xb6); - code.append(0xc0 | (dst << 3) | src); - } - - void movz1(Register src, int32_t srcOffset, Register dst) { - code.append(0x0f); - offsetInstruction(0xb6, 0, 0x40, 0x80, dst, src, srcOffset); - } - - void movs1(Register src, Register dst) { - code.append(0x0f); - code.append(0xbe); - code.append(0xc0 | (dst << 3) | src); - } - - void movs1(Register src, int32_t srcOffset, Register dst) { - code.append(0x0f); - offsetInstruction(0xbe, 0, 0x40, 0x80, dst, src, srcOffset); - } - - void movz2(Register src, Register dst) { - code.append(0x0f); - code.append(0xb7); - code.append(0xc0 | (dst << 3) | src); - } - - void movz2(Register src, int32_t srcOffset, Register dst) { - code.append(0x0f); - offsetInstruction(0xb7, 0, 0x40, 0x80, dst, src, srcOffset); - } - - void movs2(Register src, Register dst) { - code.append(0x0f); - code.append(0xbf); - code.append(0xc0 | (dst << 3) | src); - } - - void movs2(Register src, int32_t srcOffset, Register dst) { - code.append(0x0f); - offsetInstruction(0xbf, 0, 0x40, 0x80, dst, src, srcOffset); - } - - void movs4(Register src, int32_t srcOffset, Register dst) { - rex(); - offsetInstruction(0x63, 0, 0x40, 0x80, dst, src, srcOffset); - } - - void mov4(Register src, int32_t srcOffset, Register dst) { - offsetInstruction(0x8b, 0, 0x40, 0x80, dst, src, srcOffset); - } - - void mov1(Register src, Register dst, int32_t dstOffset) { - offsetInstruction(0x88, 0, 0x40, 0x80, src, dst, dstOffset); - } - - void mov2(Register src, Register dst, int32_t dstOffset) { - code.append(0x66); - offsetInstruction(0x89, 0, 0x40, 0x80, src, dst, dstOffset); - } - - void mov4(Register src, Register dst, int32_t dstOffset) { - offsetInstruction(0x89, 0, 0x40, 0x80, src, dst, dstOffset); - } - - void mov(Register src, int32_t srcOffset, SSERegister dst) { - code.append(0xf3); - code.append(0x0f); - offsetInstruction(0x7e, 0, 0x40, 0x80, dst, src, srcOffset); - } - - void mov(Register src, int32_t srcOffset, Register dst) { - rex(); - mov4(src, srcOffset, dst); - } - - void mov(Register src, Register dst, int32_t dstOffset) { - rex(); - mov4(src, dst, dstOffset); - } - - void mov(int32_t v, Register dst, int32_t dstOffset) { - rex(); - offsetInstruction(0xc7, 0, 0x40, 0x80, rax, dst, dstOffset); - code.append4(v); - } - - void mov(uintptr_t v, Register dst) { - rex(); - code.append(0xb8 | dst); - code.appendAddress(v); - } - - void alignedMov(uintptr_t v, Register dst) { - while ((code.length() + (BytesPerWord == 8 ? 2 : 1)) % BytesPerWord) { - nop(); - } - rex(); - code.append(0xb8 | dst); - code.appendAddress(v); - } - - void lea(Register src, int32_t srcOffset, Register dst) { - rex(); - offsetInstruction(0x8d, 0, 0x40, 0x80, dst, src, srcOffset); - } - - void nop() { - code.append(0x90); - } - - void push(Register reg) { - code.append(0x50 | reg); - } - - void push(Register reg, int32_t offset) { - offsetInstruction(0xff, 0x30, 0x70, 0xb0, rax, reg, offset); - } - - void push(int32_t v) { - if (isByte(v)) { - code.append(0x6a); - code.append(v); - } else { - code.append(0x68); - code.append4(v); - } - } - - void push4(Register reg, int32_t offset) { - if (BytesPerWord == 8) { - movs4(reg, offset, rsi); - push(rsi); - } else { - push(reg, offset); - } - } - - void pushAddress(uintptr_t v) { - if (BytesPerWord == 8 and not isInt32(v)) { - mov(v, rsi); - push(rsi); - } else { - push(v); - } - } - - void pop(Register dst) { - code.append(0x58 | dst); - } - - void pop(Register dst, int32_t offset) { - offsetInstruction(0x8f, 0, 0x40, 0x80, rax, dst, offset); - } - - void pop4(Register reg, int32_t offset) { - if (BytesPerWord == 8) { - pop(rsi); - mov4(rsi, reg, offset); - } else { - pop(reg, offset); - } - } - - void add(Register src, Register dst) { - rex(); - code.append(0x01); - code.append(0xc0 | (src << 3) | dst); - } - - void add(int32_t v, Register dst) { - assert(code.s, isByte(v)); // todo - - rex(); - code.append(0x83); - code.append(0xc0 | dst); - code.append(v); - } - - void add(int32_t v, Register dst, unsigned offset) { - rex(); - unsigned i = (isByte(v) ? 0x83 : 0x81); - offsetInstruction(i, 0, 0x40, 0x80, rax, dst, offset); - if (isByte(v)) { - code.append(v); - } else { - code.append4(v); - } - } - - void add(Register src, Register dst, unsigned dstOffset) { - rex(); - offsetInstruction(0x01, 0, 0x40, 0x80, src, dst, dstOffset); - } - - void adc(int32_t v, Register dst) { - assert(code.s, isByte(v)); // todo - - rex(); - code.append(0x83); - code.append(0xd0 | dst); - code.append(v); - } - - void adc(Register src, Register dst, unsigned dstOffset) { - rex(); - offsetInstruction(0x11, 0, 0x40, 0x80, src, dst, dstOffset); - } - - void sub(Register src, Register dst, unsigned dstOffset) { - rex(); - offsetInstruction(0x29, 0, 0x40, 0x80, src, dst, dstOffset); - } - - void sub(Register src, Register dst) { - rex(); - code.append(0x29); - code.append(0xc0 | (src << 3) | dst); - } - - void sub(int32_t v, Register dst) { - assert(code.s, isByte(v)); // todo - - rex(); - code.append(0x83); - code.append(0xe8 | dst); - code.append(v); - } - - void sbb(Register src, Register dst, unsigned dstOffset) { - rex(); - offsetInstruction(0x19, 0, 0x40, 0x80, src, dst, dstOffset); - } - - void or_(Register src, Register dst) { - rex(); - code.append(0x09); - code.append(0xc0 | (src << 3) | dst); - } - - void or_(Register src, Register dst, unsigned dstOffset) { - rex(); - offsetInstruction(0x09, 0, 0x40, 0x80, src, dst, dstOffset); - } - - void or_(int32_t v, Register dst) { - assert(code.s, isByte(v)); // todo - - rex(); - code.append(0x83); - code.append(0xc8 | dst); - code.append(v); - } - - void and_(Register src, Register dst) { - rex(); - code.append(0x21); - code.append(0xc0 | (src << 3) | dst); - } - - void and_(Register src, Register dst, unsigned dstOffset) { - rex(); - offsetInstruction(0x21, 0, 0x40, 0x80, src, dst, dstOffset); - } - - void and_(int32_t v, Register dst) { - rex(); - if (isByte(v)) { - code.append(0x83); - code.append(0xe0 | dst); - code.append(v); - } else { - code.append(0x81); - code.append(0xe0 | dst); - code.append(v); - } - } - - void shl(int8_t v, Register dst) { - rex(); - if (v == 1) { - code.append(0xd1); - code.append(0xe0 | dst); - } else { - code.append(0xc1); - code.append(0xe0 | dst); - code.append(v); - } - } - - void shl4_cl(Register dst, unsigned dstOffset) { - offsetInstruction(0xd3, 0x20, 0x60, 0xa0, rax, dst, dstOffset); - } - - void sar4_cl(Register dst, unsigned dstOffset) { - offsetInstruction(0xd3, 0x38, 0x7d, 0xb8, rax, dst, dstOffset); - } - - void shr4_cl(Register dst, unsigned dstOffset) { - offsetInstruction(0xd3, 0x28, 0x6d, 0xa8, rax, dst, dstOffset); - } - - void ret() { - code.append(0xc3); - } - - void jmp(Label& label) { - code.append(0xE9); - label.reference(); - } - - void jmp(unsigned javaIP) { - code.append(0xE9); - - jumps.append4(javaIP); - jumps.append4(code.length()); - - code.append4(0); - } - - void jmp(Register reg) { - code.append(0xff); - code.append(0xe0 | reg); - } - - void conditional(Label& label, unsigned condition) { - code.append(0x0f); - code.append(condition); - label.reference(); - } - - void conditional(unsigned javaIP, unsigned condition) { - code.append(0x0f); - code.append(condition); - - jumps.append4(javaIP); - jumps.append4(code.length()); - - code.append4(0); - } - - void je(Label& label) { - conditional(label, 0x84); - } - - void je(unsigned javaIP) { - conditional(javaIP, 0x84); - } - - void jne(Label& label) { - conditional(label, 0x85); - } - - void jne(unsigned javaIP) { - conditional(javaIP, 0x85); - } - - void jg(Label& label) { - conditional(label, 0x8f); - } - - void jg(unsigned javaIP) { - conditional(javaIP, 0x8f); - } - - void jge(Label& label) { - conditional(label, 0x8d); - } - - void jge(unsigned javaIP) { - conditional(javaIP, 0x8d); - } - - void jl(Label& label) { - conditional(label, 0x8c); - } - - void jl(unsigned javaIP) { - conditional(javaIP, 0x8c); - } - - void jle(Label& label) { - conditional(label, 0x8e); - } - - void jle(unsigned javaIP) { - conditional(javaIP, 0x8e); - } - - void jb(Label& label) { - conditional(label, 0x82); - } - - void ja(Label& label) { - conditional(label, 0x87); - } - - void cmp(int v, Register reg) { - assert(code.s, isByte(v)); // todo - - rex(); - code.append(0x83); - code.append(0xf8 | reg); - code.append(v); - } - - void cmp(Register a, Register b) { - rex(); - code.append(0x39); - code.append(0xc0 | (a << 3) | b); - } - - void call(Register reg) { - code.append(0xff); - code.append(0xd0 | reg); - } - - void cdq() { - code.append(0x99); - } - - void cqo() { - rex(); - cdq(); - } - - void imul4(Register src, unsigned srcOffset, Register dst) { - code.append(0x0f); - offsetInstruction(0xaf, 0, 0x40, 0x80, dst, src, srcOffset); - } - - void imul(Register src, unsigned srcOffset, Register dst) { - rex(); - imul4(src, srcOffset, dst); - } - - void imul(Register src) { - rex(); - code.append(0xf7); - code.append(0xe8 | src); - } - - void idiv(Register src) { - rex(); - code.append(0xf7); - code.append(0xf8 | src); - } - - void mul(Register src, unsigned offset) { - rex(); - offsetInstruction(0xf7, 0x20, 0x60, 0xa0, rax, src, offset); - } - - void neg(Register reg, unsigned offset) { - rex(); - offsetInstruction(0xf7, 0x10, 0x50, 0x90, rax, reg, offset); - } - - void neg(Register reg) { - rex(); - code.append(0xf7); - code.append(0xd8 | reg); - } - - void int3() { - code.append(0xcc); - } - - Buffer code; - Buffer jumps; -}; - -Register -gpRegister(Thread* t, unsigned index) -{ - switch (index) { - case 0: - return rdi; - case 1: - return rsi; - case 2: - return rdx; - case 3: - return rcx; - case 4: - return r8; - case 5: - return r9; - default: - abort(t); - } -} - -SSERegister -sseRegister(Thread* t UNUSED, unsigned index) -{ - assert(t, index < 8); - return static_cast(index); -} - -unsigned -parameterOffset(unsigned index) -{ - return FrameFootprint + ((index + 2) * BytesPerWord); -} - -Compiled* -caller(MyThread* t); - -class Compiler: public Assembler { - public: - Compiler(MyThread* t): - Assembler(t->m->system), - t(t) - { } - - unsigned threadFrameDataOffset() { - return reinterpret_cast(&(t->frameData)) - - reinterpret_cast(t); - } - - Compiled* compileStub() { - push(rbp); - mov(rsp, rbp); - - if (BytesPerWord == 4) { - push(rbp, FrameMethod); - push(rbp, FrameThread); - } else { - mov(rbp, FrameMethod, rsi); - mov(rbp, FrameThread, rdi); - } - - mov(reinterpret_cast(compileMethod), rbx); - callAddress(compiledCode(caller(t))); - - if (BytesPerWord == 4) { - add(BytesPerWord * 2, rsp); - } - - mov(rbp, FrameMethod, rax); - mov(rax, MethodCompiled, rax); // load compiled code - - mov(rbp, rsp); - pop(rbp); - - add(CompiledBody, rax); - jmp(rax); // call compiled code - - return finish(); - } - - Compiled* compileNativeInvoker() { - push(rbp); - mov(rsp, rbp); - - if (BytesPerWord == 4) { - push(rbp, FrameMethod); - push(rbp, FrameThread); - } else { - mov(rbp, FrameMethod, rsi); - mov(rbp, FrameThread, rdi); - } - - mov(reinterpret_cast(invokeNative), rbx); - callAddress(compiledCode(caller(t))); - - if (BytesPerWord == 4) { - add(BytesPerWord * 2, rsp); - } - - mov(rbp, rsp); - pop(rbp); - ret(); - - return finish(); - } - - Compiled* compileCaller() { - mov(rbp, FrameThread, rdi); - mov(rbp, rdi, threadFrameDataOffset()); - mov(rsp, 0, rcx); - mov(rcx, rdi, threadFrameDataOffset() + BytesPerWord); - - jmp(rbx); - - return finish(); - } - - Compiled* finish() { - return makeCompiled(t, &code); - } - - void callAddress(void* function) { - mov(reinterpret_cast(function), rax); - call(rax); - } - - void callAlignedAddress(void* function) { - alignedMov(reinterpret_cast(function), rax); - call(rax); - } - - MyThread* t; -}; - -class JavaCompiler: public Compiler { - public: - JavaCompiler(MyThread* t, object method): - Compiler(t), - method(method), - stackMapper(t, method), - poolRegisterClobbered(true), - machineIPs(static_cast - (t->m->system->allocate - (codeLength(t, methodCode(t, method)) * 4))), - lineNumbers(codeLineNumberTable(t, methodCode(t, method)) ? - static_cast - (t->m->system->allocate - (lineNumberTableLength - (t, codeLineNumberTable(t, methodCode(t, method))) - * sizeof(NativeLineNumber))) : 0), - exceptionHandlers(codeExceptionHandlerTable(t, methodCode(t, method)) ? - static_cast - (t->m->system->allocate - (exceptionHandlerTableLength - (t, codeExceptionHandlerTable - (t, methodCode(t, method))) - * sizeof(NativeExceptionHandler))) : 0), - pool(t->m->system, 256), - protector(this) - { } - - ~JavaCompiler() { - if (machineIPs) { - t->m->system->free(machineIPs); - } - - if (lineNumbers) { - t->m->system->free(lineNumbers); - } - - if (exceptionHandlers) { - t->m->system->free(exceptionHandlers); - } - } - - void pushInt() { - sub(BytesPerWord, rsp); - stackMapper.pushedInt(); - } - - void pushInt(int32_t v) { - push(v); - stackMapper.pushedInt(); - } - - void pushInt(Register r) { - push(r); - stackMapper.pushedInt(); - } - - void pushInt(Register r, int32_t offset) { - push(r, offset); - stackMapper.pushedInt(); - } - - void pushInt4(Register r, int32_t offset) { - push4(r, offset); - stackMapper.pushedInt(); - } - - void pushObject(int32_t v) { - push(v); - stackMapper.pushedObject(); - } - - void pushObject(Register r) { - push(r); - stackMapper.pushedObject(); - } - - void pushObject(Register r, int32_t offset) { - push(r, offset); - stackMapper.pushedObject(); - } - - void pushLongQuiet(uint64_t v) { - if (BytesPerWord == 8) { - pushAddress(v); - sub(8, rsp); - } else { - push((v >> 32) & 0xFFFFFFFF); - push((v ) & 0xFFFFFFFF); - } - } - - void pushLong(uint64_t v) { - pushLongQuiet(v); - stackMapper.pushedLong(); - } - - void pushLong(Register r) { - assert(t, BytesPerWord == 8); - push(r); - sub(8, rsp); - stackMapper.pushedLong(); - } - - void pushLongQuiet(Register r, int32_t offset) { - assert(t, BytesPerWord == 8); - push(r, offset); - sub(8, rsp); - } - - void pushLong(Register r, int32_t offset) { - pushLongQuiet(r, offset); - stackMapper.pushedLong(); - } - - void pushLong(Register low, Register high) { - assert(t, BytesPerWord == 4); - push(high); - push(low); - stackMapper.pushedLong(); - } - - void pushLongQuiet(Register low, int32_t lowOffset, - Register high, int32_t highOffset) - { - assert(t, BytesPerWord == 4); - push(high, highOffset); - push(low, lowOffset); - } - - void pushLong(Register low, int32_t lowOffset, - Register high, int32_t highOffset) - { - pushLongQuiet(low, lowOffset, high, highOffset); - stackMapper.pushedLong(); - } - - void popInt() { - add(BytesPerWord, rsp); - stackMapper.poppedInt(); - } - - void popInt(Register r) { - pop(r); - stackMapper.poppedInt(); - } - - void popInt(Register r, int32_t offset) { - pop(r, offset); - stackMapper.poppedInt(); - } - - void popInt4(Register r, int32_t offset) { - pop4(r, offset); - stackMapper.poppedInt(); - } - - void popObject(Register r) { - pop(r); - stackMapper.poppedObject(); - } - - void popObject(Register r, int32_t offset) { - pop(r, offset); - stackMapper.poppedObject(); - } - - void popLong() { - add(BytesPerWord * 2, rsp); - stackMapper.poppedLong(); - } - - void popLong(Register r) { - assert(t, BytesPerWord == 8); - add(BytesPerWord, rsp); - pop(r); - stackMapper.poppedLong(); - } - - void popLong(Register r, int32_t offset) { - assert(t, BytesPerWord == 8); - add(BytesPerWord, rsp); - pop(r, offset); - stackMapper.poppedLong(); - } - - void popLong(Register low, Register high) { - assert(t, BytesPerWord == 4); - pop(low); - pop(high); - stackMapper.poppedLong(); - } - - void popLong(Register low, int32_t lowOffset, - Register high, int32_t highOffset) - { - assert(t, BytesPerWord == 4); - pop(low, lowOffset); - pop(high, highOffset); - stackMapper.poppedLong(); - } - - void loadInt(unsigned index) { - pushInt(rbp, localOffset(t, index, method)); - } - - void loadObject(unsigned index) { - pushObject(rbp, localOffset(t, index, method)); - } - - void loadLong(unsigned index) { - if (BytesPerWord == 8) { - pushLong(rbp, localOffset(t, index, method)); - } else { - pushLong(rbp, localOffset(t, index + 1, method), - rbp, localOffset(t, index, method)); - } - } - - void storeInt(unsigned index) { - popInt(rbp, localOffset(t, index, method)); - stackMapper.storedInt(index); - } - - void storeObject(unsigned index) { - popObject(rbp, localOffset(t, index, method)); - stackMapper.storedObject(index); - } - - void storeLong(unsigned index) { - if (BytesPerWord == 8) { - popLong(rbp, localOffset(t, index, method)); - } else { - popLong(rbp, localOffset(t, index, method), - rbp, localOffset(t, index + 1, method)); - } - stackMapper.storedLong(index); + unsigned index = 0; + if ((methodFlags(t, method) & ACC_STATIC) == 0) { + v->visit(localObject(t, base, method, index++)); } - void pushReturnValue(unsigned code) { - switch (code) { - case ByteField: - case BooleanField: - case CharField: - case ShortField: - case FloatField: - case IntField: - push(rax); - stackMapper.pushedInt(); + for (MethodSpecIterator it(t, spec); it.hasNext();) { + switch (*it.next()) { + case 'L': + case '[': + v->visit(localObject(t, base, method, index++)); break; - - case ObjectField: - push(rax); - stackMapper.pushedObject(); - break; - - case LongField: - case DoubleField: - push(rax); - push(rdx); - stackMapper.pushedLong(); - break; - - case VoidField: + + case 'J': + case 'D': + index += 2; break; default: - abort(t); + ++ index; + break; } } - void compileDirectInvoke(object target) { - unsigned footprint = FrameFootprint - + (methodParameterFootprint(t, target) * BytesPerWord); - - Compiled* code = reinterpret_cast(methodCompiled(t, target)); - - push(rsp); - push(poolRegister(), poolReference(target)); - push(rbp, FrameThread); - - callAlignedAddress(compiledCode(code)); - - stackMapper.called(this->code.length()); - poolRegisterClobbered = true; - - add(footprint, rsp); // pop arguments - stackMapper.popped(methodParameterFootprint(t, target)); - - pushReturnValue(methodReturnCode(t, target)); - } - - void compileThrowNew(Machine::Type type) { - object class_ = arrayBody(t, t->m->types, type); - - if (BytesPerWord == 8) { - mov(poolRegister(), poolReference(class_), rsi); - mov(rbp, FrameThread, rdi); - } else { - push(poolRegister(), poolReference(class_)); - push(rbp, FrameThread); - } - - indirectCall(reinterpret_cast(throwNew)); - } - - void directCall(void* function) { - callAddress(function); - - poolRegisterClobbered = true; - } - - void indirectCall(void* function) { - mov(reinterpret_cast(function), rbx); - callAddress(compiledCode(caller(t))); - - stackMapper.called(code.length()); - poolRegisterClobbered = true; - } - - void compileCall(bool direct, void* function, Register arg1, Register arg2) { - if (BytesPerWord == 8) { - mov(arg2, rdx); - mov(arg1, rsi); - mov(rbp, FrameThread, rdi); - } else { - push(arg2); - push(arg1); - push(rbp, FrameThread); - } - - if (direct) { - directCall(function); - } else { - indirectCall(function); - } - - if (BytesPerWord == 4) { - add(BytesPerWord * 3, rsp); - } - } - - void compileCall(bool direct, void* function, uintptr_t arg1, Register arg2) - { - if (BytesPerWord == 8) { - mov(arg2, rdx); - mov(arg1, rsi); - mov(rbp, FrameThread, rdi); - } else { - push(arg2); - push(arg1); - push(rbp, FrameThread); - } - - if (direct) { - directCall(function); - } else { - indirectCall(function); - } - - if (BytesPerWord == 4) { - add(BytesPerWord * 3, rsp); - } - } - - void compileCall(bool direct, void* function, object arg1, Register arg2) { - if (BytesPerWord == 8) { - mov(arg2, rdx); - mov(poolRegister(), poolReference(arg1), rsi); - mov(rbp, FrameThread, rdi); - } else { - push(arg2); - push(poolRegister(), poolReference(arg1)); - push(rbp, FrameThread); - } - - if (direct) { - directCall(function); - } else { - indirectCall(function); - } - - if (BytesPerWord == 4) { - add(BytesPerWord * 3, rsp); - } - } - - void compileCall(bool direct, void* function, object arg1, Register arg2, - int32_t arg2offset) - { - if (BytesPerWord == 8) { - mov(arg2, arg2offset, rdx); - mov(poolRegister(), poolReference(arg1), rsi); - mov(rbp, FrameThread, rdi); - } else { - push(arg2, arg2offset); - push(poolRegister(), poolReference(arg1)); - push(rbp, FrameThread); - } - - if (direct) { - directCall(function); - } else { - indirectCall(function); - } - - if (BytesPerWord == 4) { - add(BytesPerWord * 3, rsp); - } - } - - void compileCall(bool direct, void* function, object arg1) { - if (BytesPerWord == 8) { - mov(poolRegister(), poolReference(arg1), rsi); - mov(rbp, FrameThread, rdi); - } else { - push(poolRegister(), poolReference(arg1)); - push(rbp, FrameThread); - } - - if (direct) { - directCall(function); - } else { - indirectCall(function); - } - - if (BytesPerWord == 4) { - add(BytesPerWord * 2, rsp); - } - } - - Compiled* compile() { - object code = methodCode(t, method); - PROTECT(t, code); - - unsigned parameterFootprint - = methodParameterFootprint(t, method) * BytesPerWord; - - unsigned localFootprint = codeMaxLocals(t, code) * BytesPerWord; - - push(rbp); - mov(rsp, rbp); - - if (localFootprint > parameterFootprint) { - // reserve space for local variables - sub(localFootprint - parameterFootprint, rsp); - } - - int lineNumberIndex; - object lnt = codeLineNumberTable(t, code); - if (lnt and lineNumberTableLength(t, lnt)) { - lineNumberIndex = 0; - } else { - lineNumberIndex = -1; - } - - for (unsigned ip = 0; ip < codeLength(t, code);) { - stackMapper.newIp(ip); - machineIPs[ip] = this->code.length(); - - if (lineNumberIndex >= 0) { - object lnt = codeLineNumberTable(t, code); - LineNumber* ln = lineNumberTableBody(t, lnt, lineNumberIndex); - - if (lineNumberIp(ln) == ip) { - nativeLineNumberIp(lineNumbers + lineNumberIndex) - = this->code.length(); - nativeLineNumberLine(lineNumbers + lineNumberIndex) - = lineNumberLine(ln); - if (static_cast(lineNumberIndex) + 1 - < lineNumberTableLength(t, lnt)) - { - ++ lineNumberIndex; - } else { - lineNumberIndex = -1; - } - } - } - - unsigned instruction = codeBody(t, code, ip++); - - switch (instruction) { - case aaload: - case baload: - case caload: - case daload: - case faload: - case iaload: - case laload: - case saload: { - Label next(this); - Label outOfBounds(this); - - popInt(rcx); - popObject(rax); - - cmp(0, rcx); - jl(outOfBounds); - - mov(rax, BytesPerWord, rdx); - cmp(rdx, rcx); - jge(outOfBounds); - - add(BytesPerWord * 2, rax); - - switch (instruction) { - case aaload: - case faload: - case iaload: - shl(log(BytesPerWord), rcx); - add(rcx, rax); - - if (instruction == aaload) { - pushObject(rax, 0); - } else { - pushInt4(rax, 0); - } - break; - - case baload: - add(rcx, rax); - movs1(rax, 0, rax); - pushInt(rax); - break; - - case caload: - shl(1, rcx); - add(rcx, rax); - movz2(rax, 0, rax); - pushInt(rax); - break; - - case daload: - case laload: - shl(3, rcx); - add(rcx, rax); - pushLong(rax, 0); - break; - - case saload: - shl(1, rcx); - add(rcx, rax); - movs2(rax, 0, rax); - pushInt(rax); - break; - } - - jmp(next); - - outOfBounds.mark(); - compileThrowNew(Machine::ArrayIndexOutOfBoundsExceptionType); - - next.mark(); - } break; - - case aastore: - case bastore: - case castore: - case dastore: - case fastore: - case iastore: - case lastore: - case sastore: { - Label next(this); - Label outOfBounds(this); - - if (instruction == dastore or instruction == lastore) { - popLong(rdx, rbx); - } else if (instruction == aastore) { - popObject(rbx); - } else { - popInt(rbx); - } - - popInt(rcx); - popObject(rax); - - cmp(0, rcx); - jl(outOfBounds); - - mov(rax, BytesPerWord, rsi); - cmp(rsi, rcx); - jge(outOfBounds); - - switch (instruction) { - case aastore: - shl(log(BytesPerWord), rcx); - add(ArrayBody, rcx); - - if (BytesPerWord == 8) { - mov(rcx, rdx); - mov(rbx, rcx); - mov(rax, rsi); - mov(rbp, FrameThread, rdi); - } else { - push(rbx); - push(rcx); - push(rax); - push(rbp, FrameThread); - } - - directCall(reinterpret_cast(set)); - - if (BytesPerWord == 4) { - add(BytesPerWord * 4, rsp); - } - break; - - case fastore: - case iastore: - shl(log(BytesPerWord), rcx); - add(ArrayBody, rcx); - add(rcx, rax); - mov(rbx, rax, 0); - break; - - case bastore: - add(ArrayBody, rcx); - add(rcx, rax); - mov1(rbx, rax, 0); - break; - - case castore: - case sastore: - shl(1, rcx); - add(ArrayBody, rcx); - add(rcx, rax); - mov2(rbx, rax, 0); - break; - - case dastore: - case lastore: - shl(3, rcx); - add(ArrayBody, rcx); - add(rcx, rax); - mov4(rbx, rax, 0); - mov4(rdx, rax, 4); - break; - } - - jmp(next); - - outOfBounds.mark(); - compileThrowNew(Machine::ArrayIndexOutOfBoundsExceptionType); - - next.mark(); - } break; - - case aconst_null: - pushObject(0); - break; - - case aload: - loadObject(codeBody(t, code, ip++)); - break; - - case aload_0: - loadObject(0); - break; - - case aload_1: - loadObject(1); - break; - - case aload_2: - loadObject(2); - break; - - case aload_3: - loadObject(3); - break; - - case anewarray: { - uint16_t index = codeReadInt16(t, code, ip); - - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - - Label nonnegative(this); - - popInt(rax); - cmp(0, rax); - jge(nonnegative); - - compileThrowNew(Machine::NegativeArraySizeExceptionType); - - nonnegative.mark(); - - compileCall(false, reinterpret_cast(makeBlankObjectArray), - class_, rax); - - pushObject(rax); - } break; - - case areturn: - popObject(rax); - mov(rbp, rsp); - pop(rbp); - ret(); - stackMapper.exited(); - break; - - case arraylength: - popObject(rax); - pushInt(rax, BytesPerWord); - break; - - case astore: - storeObject(codeBody(t, code, ip++)); - break; - - case astore_0: - storeObject(0); - break; - - case astore_1: - storeObject(1); - break; - - case astore_2: - storeObject(2); - break; - - case astore_3: - storeObject(3); - break; - - case athrow: - if (BytesPerWord == 8) { - popObject(rsi); - mov(rbp, FrameThread, rdi); - } else { - push(rbp, FrameThread); - } - - indirectCall(reinterpret_cast(throw_)); - stackMapper.exited(); - break; - - case bipush: { - pushInt(static_cast(codeBody(t, code, ip++))); - } break; - - case checkcast: { - uint16_t index = codeReadInt16(t, code, ip); - - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - - Label next(this); - - mov(rsp, 0, rax); - cmp(0, rax); - je(next); - - mov(poolRegister(), poolReference(class_), rcx); - mov(rax, 0, rax); - and_(static_cast(PointerMask), rax); - cmp(rcx, rax); - je(next); - - compileCall(true, reinterpret_cast(isAssignableFrom), rcx, rax); - - cmp(0, rax); - jne(next); - - compileThrowNew(Machine::ClassCastExceptionType); - - next.mark(); - } break; - - case dadd: { - if (BytesPerWord == 8) { - popLong(rdi); - popLong(rsi); - directCall(reinterpret_cast(addDouble)); - pushLong(rax); - } else { - directCall(reinterpret_cast(addDouble)); - add(BytesPerWord * 4, rsp); - stackMapper.poppedLong(); - stackMapper.poppedLong(); - pushLong(rax, rdx); - } - } break; - - case dcmpg: { - if (BytesPerWord == 8) { - popLong(rdi); - popLong(rsi); - directCall(reinterpret_cast(compareDoublesG)); - pushInt(rax); - } else { - directCall(reinterpret_cast(compareDoublesG)); - add(BytesPerWord * 4, rsp); - stackMapper.poppedLong(); - stackMapper.poppedLong(); - pushInt(rax); - } - } break; - - case dcmpl: { - if (BytesPerWord == 8) { - popLong(rdi); - popLong(rsi); - directCall(reinterpret_cast(compareDoublesL)); - pushInt(rax); - } else { - directCall(reinterpret_cast(compareDoublesL)); - add(BytesPerWord * 4, rsp); - stackMapper.poppedLong(); - stackMapper.poppedLong(); - pushInt(rax); - } - } break; - - case dconst_0: { - pushLong(doubleToBits(0.0)); - } break; - - case dconst_1: { - pushLong(doubleToBits(1.0)); - } break; - - case ddiv: { - if (BytesPerWord == 8) { - popLong(rdi); - popLong(rsi); - directCall(reinterpret_cast(divideDouble)); - pushLong(rax); - } else { - directCall(reinterpret_cast(divideDouble)); - add(BytesPerWord * 4, rsp); - stackMapper.poppedLong(); - stackMapper.poppedLong(); - pushLong(rax, rdx); - } - } break; - - case dmul: { - if (BytesPerWord == 8) { - popLong(rdi); - popLong(rsi); - directCall(reinterpret_cast(multiplyDouble)); - pushLong(rax); - } else { - directCall(reinterpret_cast(multiplyDouble)); - add(BytesPerWord * 4, rsp); - stackMapper.poppedLong(); - stackMapper.poppedLong(); - pushLong(rax, rdx); - } - } break; - - case vm::drem: { - if (BytesPerWord == 8) { - popLong(rdi); - popLong(rsi); - directCall(reinterpret_cast(moduloDouble)); - pushLong(rax); - } else { - directCall(reinterpret_cast(moduloDouble)); - add(BytesPerWord * 4, rsp); - stackMapper.poppedLong(); - stackMapper.poppedLong(); - pushLong(rax, rdx); - } - } break; - - case dsub: { - if (BytesPerWord == 8) { - popLong(rdi); - popLong(rsi); - directCall(reinterpret_cast(subtractDouble)); - pushLong(rax); - } else { - directCall(reinterpret_cast(subtractDouble)); - add(BytesPerWord * 4, rsp); - stackMapper.poppedLong(); - stackMapper.poppedLong(); - pushLong(rax, rdx); - } - } break; - - case dup: - push(rsp, 0); - stackMapper.dupped(); - break; - - case dup_x1: - mov(rsp, BytesPerWord, rcx); - mov(rsp, 0, rax); - mov(rax, rsp, BytesPerWord); - mov(rcx, rsp, 0); - push(rax); - stackMapper.duppedX1(); - break; - - case dup_x2: - mov(rsp, BytesPerWord * 2, rdx); - mov(rsp, BytesPerWord, rcx); - mov(rsp, 0, rax); - mov(rax, rsp, BytesPerWord * 2); - mov(rdx, rsp, BytesPerWord); - mov(rcx, rsp, 0); - push(rax); - stackMapper.duppedX2(); - break; - - case dup2: - push(rsp, BytesPerWord); - push(rsp, BytesPerWord); - stackMapper.dupped2(); - break; - - case dup2_x1: - mov(rsp, BytesPerWord * 2, rdx); - mov(rsp, BytesPerWord, rcx); - mov(rsp, 0, rax); - mov(rcx, rsp, BytesPerWord * 2); - mov(rax, rsp, BytesPerWord); - mov(rdx, rsp, 0); - push(rcx); - push(rax); - stackMapper.dupped2X1(); - break; - - case dup2_x2: - mov(rsp, BytesPerWord * 3, rbx); - mov(rsp, BytesPerWord * 2, rdx); - mov(rsp, BytesPerWord, rcx); - mov(rsp, 0, rax); - mov(rcx, rsp, BytesPerWord * 3); - mov(rax, rsp, BytesPerWord * 2); - mov(rbx, rsp, BytesPerWord); - mov(rdx, rsp, 0); - push(rcx); - push(rax); - stackMapper.dupped2X2(); - break; - - case fadd: { - if (BytesPerWord == 8) { - popInt(rdi); - popInt(rsi); - directCall(reinterpret_cast(addFloat)); - pushInt(rax); - } else { - directCall(reinterpret_cast(addFloat)); - add(BytesPerWord * 2, rsp); - stackMapper.poppedInt(); - stackMapper.poppedInt(); - pushInt(rax); - } - } break; - - case fcmpg: { - if (BytesPerWord == 8) { - popInt(rdi); - popInt(rsi); - directCall(reinterpret_cast(compareFloatsG)); - pushInt(rax); - } else { - directCall(reinterpret_cast(compareFloatsG)); - add(BytesPerWord * 2, rsp); - stackMapper.poppedInt(); - stackMapper.poppedInt(); - pushInt(rax); - } - } break; - - case fcmpl: { - if (BytesPerWord == 8) { - popInt(rdi); - popInt(rsi); - directCall(reinterpret_cast(compareFloatsL)); - pushInt(rax); - } else { - directCall(reinterpret_cast(compareFloatsL)); - add(BytesPerWord * 2, rsp); - stackMapper.poppedInt(); - stackMapper.poppedInt(); - pushInt(rax); - } - } break; - - case fconst_0: { - pushInt(floatToBits(0.0)); - } break; - - case fconst_1: { - pushInt(floatToBits(1.0)); - } break; - - case fconst_2: { - pushInt(floatToBits(2.0)); - } break; - - case fdiv: { - if (BytesPerWord == 8) { - popInt(rdi); - popInt(rsi); - directCall(reinterpret_cast(divideFloat)); - pushInt(rax); - } else { - directCall(reinterpret_cast(divideFloat)); - add(BytesPerWord * 2, rsp); - stackMapper.poppedInt(); - stackMapper.poppedInt(); - pushInt(rax); - } - } break; - - case fmul: { - if (BytesPerWord == 8) { - popInt(rdi); - popInt(rsi); - directCall(reinterpret_cast(multiplyFloat)); - pushInt(rax); - } else { - directCall(reinterpret_cast(multiplyFloat)); - add(BytesPerWord * 2, rsp); - stackMapper.poppedInt(); - stackMapper.poppedInt(); - pushInt(rax); - } - } break; - - case frem: { - if (BytesPerWord == 8) { - popInt(rdi); - popInt(rsi); - directCall(reinterpret_cast(moduloFloat)); - pushInt(rax); - } else { - directCall(reinterpret_cast(moduloFloat)); - add(BytesPerWord * 2, rsp); - stackMapper.poppedInt(); - stackMapper.poppedInt(); - pushInt(rax); - } - } break; - - case fsub: { - if (BytesPerWord == 8) { - popInt(rdi); - popInt(rsi); - directCall(reinterpret_cast(subtractFloat)); - pushInt(rax); - } else { - directCall(reinterpret_cast(subtractFloat)); - add(BytesPerWord * 2, rsp); - stackMapper.poppedInt(); - stackMapper.poppedInt(); - pushInt(rax); - } - } break; - - case getfield: { - uint16_t index = codeReadInt16(t, code, ip); - - object field = resolveField(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - - popObject(rax); - - switch (fieldCode(t, field)) { - case ByteField: - case BooleanField: - movs1(rax, fieldOffset(t, field), rax); - pushInt(rax); - break; - - case CharField: - movz2(rax, fieldOffset(t, field), rax); - pushInt(rax); - break; - - case ShortField: - movs2(rax, fieldOffset(t, field), rax); - pushInt(rax); - break; - - case FloatField: - case IntField: - pushInt4(rax, fieldOffset(t, field)); - break; - - case DoubleField: - case LongField: - pushLong(rax, fieldOffset(t, field)); - break; - - case ObjectField: - pushObject(rax, fieldOffset(t, field)); - break; - - default: - abort(t); - } - } break; - - case getstatic: { - uint16_t index = codeReadInt16(t, code, ip); - - object field = resolveField(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - PROTECT(t, field); - - initClass(t, fieldClass(t, field)); - if (UNLIKELY(t->exception)) return 0; - - object table = classStaticTable(t, fieldClass(t, field)); - unsigned offset = (fieldOffset(t, field) * BytesPerWord) + ArrayBody; - - mov(poolRegister(), poolReference(table), rax); - - switch (fieldCode(t, field)) { - case ByteField: - case BooleanField: - case CharField: - case ShortField: - case FloatField: - case IntField: { - Label zero(this); - Label next(this); - - mov(rax, offset, rax); - cmp(0, rax); - je(zero); - - push4(rax, IntValue); - jmp(next); - - zero.mark(); - push(0); - - next.mark(); - stackMapper.pushedInt(); - } break; - - case DoubleField: - case LongField: { - Label zero(this); - Label next(this); - - mov(rax, offset, rax); - cmp(0, rax); - je(zero); - - if (BytesPerWord == 8) { - pushLongQuiet(rax, LongValue); - } else { - pushLongQuiet(rax, LongValue); - pushLongQuiet(rax, LongValue + 4); - } - jmp(next); - - zero.mark(); - pushLongQuiet(0); - - next.mark(); - stackMapper.pushedLong(); - } break; - - case ObjectField: { - pushObject(rax, offset); - } break; - - default: abort(t); - } - } break; - - case goto_: { - int16_t offset = codeReadInt16(t, code, ip); - jmp((ip - 3) + offset); - stackMapper.jumped((ip - 3) + offset); - } break; - - case goto_w: { - int32_t offset = codeReadInt32(t, code, ip); - jmp((ip - 5) + offset); - stackMapper.jumped((ip - 5) + offset); - } break; - - case i2b: - mov(rsp, 0, rax); - movs1(rax, rax); - mov(rax, rsp, 0); - break; - - case i2c: - mov(rsp, 0, rax); - movz2(rax, rax); - mov(rax, rsp, 0); - break; - - case i2s: - mov(rsp, 0, rax); - movs2(rax, rax); - mov(rax, rsp, 0); - break; - - case i2l: - if (BytesPerWord == 8) { - pushInt(); - } else { - popInt(rax); - cdq(); - pushLong(rax, rdx); - } - break; - - case iadd: - popInt(rax); - popInt(rcx); - add(rcx, rax); - pushInt(rax); - break; - - case iand: - popInt(rax); - and_(rax, rsp, 0); - break; - - case iconst_m1: - pushInt(-1); - break; - - case iconst_0: - pushInt(0); - break; - - case iconst_1: - pushInt(1); - break; - - case iconst_2: - pushInt(2); - break; - - case iconst_3: - pushInt(3); - break; - - case iconst_4: - pushInt(4); - break; - - case iconst_5: - pushInt(5); - break; - - case vm::idiv: - popInt(rcx); - popInt(rax); - cqo(); - Assembler::idiv(rcx); - pushInt(rax); - break; - - case if_acmpeq: { - int16_t offset = codeReadInt16(t, code, ip); - - popObject(rax); - popObject(rcx); - cmp(rax, rcx); - je((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case if_acmpne: { - int16_t offset = codeReadInt16(t, code, ip); - - popObject(rax); - popObject(rcx); - cmp(rax, rcx); - jne((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case if_icmpeq: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - popInt(rcx); - cmp(rax, rcx); - je((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case if_icmpne: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - popInt(rcx); - cmp(rax, rcx); - jne((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case if_icmpgt: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - popInt(rcx); - cmp(rax, rcx); - jg((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case if_icmpge: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - popInt(rcx); - cmp(rax, rcx); - jge((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case if_icmplt: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - popInt(rcx); - cmp(rax, rcx); - jl((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case if_icmple: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - popInt(rcx); - cmp(rax, rcx); - jle((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case ifeq: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - cmp(0, rax); - je((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case ifnull: { - int16_t offset = codeReadInt16(t, code, ip); - - popObject(rax); - cmp(0, rax); - je((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case ifne: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - cmp(0, rax); - jne((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case ifnonnull: { - int16_t offset = codeReadInt16(t, code, ip); - - popObject(rax); - cmp(0, rax); - jne((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case ifgt: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - cmp(0, rax); - jg((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case ifge: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - cmp(0, rax); - jge((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case iflt: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - cmp(0, rax); - jl((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case ifle: { - int16_t offset = codeReadInt16(t, code, ip); - - popInt(rax); - cmp(0, rax); - jle((ip - 3) + offset); - stackMapper.branched((ip - 3) + offset); - } break; - - case iinc: { - uint8_t index = codeBody(t, code, ip++); - int8_t c = codeBody(t, code, ip++); - - add(c, rbp, localOffset(t, index, method)); - } break; - - case iload: - case fload: - loadInt(codeBody(t, code, ip++)); - break; - - case iload_0: - case fload_0: - loadInt(0); - break; - - case iload_1: - case fload_1: - loadInt(1); - break; - - case iload_2: - case fload_2: - loadInt(2); - break; - - case iload_3: - case fload_3: - loadInt(3); - break; - - case vm::imul: - popInt(rax); - popInt(rcx); - Assembler::imul(rcx); - pushInt(rax); - break; - - case ineg: - neg(rsp, 0); - break; - - case instanceof: { - uint16_t index = codeReadInt16(t, code, ip); - - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - - Label call(this); - Label zero(this); - Label next(this); - - popObject(rax); - cmp(0, rax); - je(zero); - - mov(poolRegister(), poolReference(class_), rcx); - mov(rax, 0, rax); - and_(static_cast(PointerMask), rax); - cmp(rcx, rax); - jne(call); - - push(1); - jmp(next); - - call.mark(); - - compileCall(true, reinterpret_cast(isAssignableFrom), rcx, rax); - - push(rax); - jmp(next); - - zero.mark(); - push(0); - - next.mark(); - - stackMapper.pushedInt(); - } break; - - case invokeinterface: { - uint16_t index = codeReadInt16(t, code, ip); - ip += 2; - - object target = resolveMethod(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - - unsigned parameterFootprint - = methodParameterFootprint(t, target) * BytesPerWord; - - unsigned instance = parameterFootprint - BytesPerWord; - - unsigned footprint = FrameFootprint + parameterFootprint; - - compileCall(false, - reinterpret_cast(findInterfaceMethodFromInstance), - target, rsp, instance); - - push(rsp); - push(rax); - push(rbp, FrameThread); - - mov(rax, MethodCompiled, rax); // load compiled code - add(CompiledBody, rax); - call(rax); // call compiled code - - stackMapper.called(this->code.length()); - poolRegisterClobbered = true; - - add(footprint, rsp); // pop arguments - stackMapper.popped(methodParameterFootprint(t, target)); - - pushReturnValue(methodReturnCode(t, target)); - } break; - - case invokespecial: { - uint16_t index = codeReadInt16(t, code, ip); - - object target = resolveMethod(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - - object class_ = methodClass(t, target); - if (isSpecialMethod(t, target, class_)) { - target = findMethod(t, target, classSuper(t, class_)); - } - - compileDirectInvoke(target); - } break; - - case invokestatic: { - uint16_t index = codeReadInt16(t, code, ip); - - object target = resolveMethod(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - PROTECT(t, target); - - initClass(t, methodClass(t, target)); - if (UNLIKELY(t->exception)) return 0; - - compileDirectInvoke(target); - } break; - - case invokevirtual: { - uint16_t index = codeReadInt16(t, code, ip); - - object target = resolveMethod(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - - unsigned parameterFootprint - = methodParameterFootprint(t, target) * BytesPerWord; - - unsigned instance = parameterFootprint - BytesPerWord; - - unsigned footprint = FrameFootprint + parameterFootprint; - - unsigned offset = ArrayBody + (methodOffset(t, target) * BytesPerWord); - - mov(rsp, instance, rax); // load instance - mov(rax, 0, rax); // load class - and_(static_cast(PointerMask), rax); // clean pointer - mov(rax, ClassVirtualTable, rax); // load vtable - mov(rax, offset, rax); // load method - - push(rsp); - push(rax); - push(rbp, FrameThread); - - mov(rax, MethodCompiled, rax); // load compiled code - add(CompiledBody, rax); - call(rax); // call compiled code - - stackMapper.called(this->code.length()); - poolRegisterClobbered = true; - - add(footprint, rsp); // pop arguments - stackMapper.popped(methodParameterFootprint(t, target)); - - pushReturnValue(methodReturnCode(t, target)); - } break; - - case ior: - popInt(rax); - or_(rax, rsp, 0); - break; - - case irem: - popInt(rcx); - popInt(rax); - cqo(); - Assembler::idiv(rcx); - pushInt(rdx); - break; - - case ireturn: - case freturn: - popInt(rax); - mov(rbp, rsp); - pop(rbp); - ret(); - stackMapper.exited(); - break; - - case ishl: - popInt(rcx); - shl4_cl(rsp, 0); - break; - - case ishr: - popInt(rcx); - sar4_cl(rsp, 0); - break; - - case istore: - case fstore: - storeInt(codeBody(t, code, ip++)); - break; - - case istore_0: - case fstore_0: - storeInt(0); - break; - - case istore_1: - case fstore_1: - storeInt(1); - break; - - case istore_2: - case fstore_2: - storeInt(2); - break; - - case istore_3: - case fstore_3: - storeInt(3); - break; - - case isub: - popInt(rax); - sub(rax, rsp, 0); - break; - - case iushr: - popInt(rcx); - shr4_cl(rsp, 0); - break; - - case l2i: - if (BytesPerWord == 8) { - popInt(); - } else { - popInt(rax); - mov(rax, rsp, 0); - } - break; - - case ladd: - if (BytesPerWord == 8) { - popLong(rax); - add(rax, rsp, BytesPerWord); - } else { - popLong(rax, rdx); - add(rax, rsp, 0); - adc(rdx, rsp, BytesPerWord); - } - break; - - case ldc: - case ldc_w: { - uint16_t index; - - if (instruction == ldc) { - index = codeBody(t, code, ip++); - } else { - index = codeReadInt16(t, code, ip); - } - - object v = arrayBody(t, codePool(t, code), index - 1); - - if (objectClass(t, v) == arrayBody(t, t->m->types, Machine::IntType)) { - pushInt(intValue(t, v)); - } else if (objectClass(t, v) - == arrayBody(t, t->m->types, Machine::FloatType)) - { - pushInt(floatValue(t, v)); - } else if (objectClass(t, v) - == arrayBody(t, t->m->types, Machine::StringType)) - { - pushObject(poolRegister(), poolReference(v)); - } else { - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - - pushObject(poolRegister(), poolReference(class_)); - } - } break; - - case ldc2_w: { - uint16_t index = codeReadInt16(t, code, ip); - - object v = arrayBody(t, codePool(t, code), index - 1); - - if (objectClass(t, v) == arrayBody(t, t->m->types, Machine::LongType)) - { - pushLong(longValue(t, v)); - } else if (objectClass(t, v) - == arrayBody(t, t->m->types, Machine::DoubleType)) - { - pushLong(doubleValue(t, v)); - } else { - abort(t); - } - } break; - - case lconst_0: - pushLong(0); - break; - - case lconst_1: - pushLong(1); - break; - - case lcmp: { - Label next(this); - Label less(this); - Label greater(this); - - if (BytesPerWord == 8) { - popLong(rax); - popLong(rcx); - - cmp(rax, rcx); - jl(less); - jg(greater); - - push(0); - jmp(next); - - less.mark(); - push(-1); - jmp(next); - - greater.mark(); - push(1); - - next.mark(); - - stackMapper.pushedInt(); - } else { - popLong(rax, rdx); - popLong(rcx, rbx); - - cmp(rdx, rbx); - jl(less); - jg(greater); - - cmp(rax, rcx); - jb(less); - ja(greater); - - push(0); - jmp(next); - - less.mark(); - push(-1); - jmp(next); - - greater.mark(); - push(1); - - next.mark(); - - stackMapper.pushedInt(); - } - } break; - - case ldiv_: - if (BytesPerWord == 8) { - popLong(rcx); - popLong(rax); - cqo(); - Assembler::idiv(rcx); - pushLong(rax); - } else { - directCall(reinterpret_cast(divideLong)); - popLong(); - mov(rax, rsp, 0); - mov(rdx, rsp, 4); - } - break; - - case lload: - case dload: - loadLong(codeBody(t, code, ip++)); - break; - - case lload_0: - case dload_0: - loadLong(0); - break; - - case lload_1: - case dload_1: - loadLong(1); - break; - - case lload_2: - case dload_2: - loadLong(2); - break; - - case lload_3: - case dload_3: - loadLong(3); - break; - - case lmul: - if (BytesPerWord == 8) { - popLong(rax); - popLong(rcx); - Assembler::imul(rcx); - pushLong(rax); - } else { - mov(rsp, 4, rcx); - Assembler::imul(rsp, 8, rcx); - mov(rsp, 12, rax); - Assembler::imul(rsp, 0, rax); - add(rax, rcx); - mov(rsp, 8, rax); - mul(rsp, 0); - add(rcx, rdx); - - popLong(); - mov(rax, rsp, 0); - mov(rdx, rsp, 4); - } - break; - - case lneg: - if (BytesPerWord == 8) { - neg(rsp, 8); - } else { - mov(rsp, 0, rax); - mov(rsp, 4, rdx); - neg(rax); - adc(0, rdx); - neg(rdx); - - mov(rax, rsp, 0); - mov(rdx, rsp, 4); - } - break; - - // todo: -// case lookupswitch: { -// int32_t base = ip - 1; -// ip += 3; -// ip -= (ip % 4); - -// int32_t default_ = codeReadInt32(t, code, ip); -// int32_t pairCount = codeReadInt32(t, code, ip); - -// popInt(rax); -// mov(LookupTable(this, base, pairCount, code, ip), rbx); -// mov(pairCount, rcx); -// jmp(test); - -// loop.mark(); -// // tbc - -// test.mark(); -// cmp(0, rcx); -// jne(loop); - -// int32_t key = popInt(t); - -// int32_t bottom = 0; -// int32_t top = pairCount; -// for (int32_t span = top - bottom; span; span = top - bottom) { -// int32_t middle = bottom + (span / 2); -// unsigned index = ip + (middle * 8); - -// int32_t k = codeReadInt32(t, code, index); - -// if (key < k) { -// top = middle; -// } else if (key > k) { -// bottom = middle + 1; -// } else { -// ip = base + codeReadInt32(t, code, index); -// break; -// } -// } - -// ip = base + default_; -// } break; - - case lrem: - if (BytesPerWord == 8) { - popLong(rcx); - popLong(rax); - cqo(); - Assembler::idiv(rcx); - pushLong(rdx); - } else { - directCall(reinterpret_cast(moduloLong)); - popLong(); - mov(rax, rsp, 0); - mov(rdx, rsp, 4); - } - break; - - case lreturn: - case dreturn: - if (BytesPerWord == 8) { - popLong(rax); - } else { - popLong(rax, rdx); - } - mov(rbp, rsp); - pop(rbp); - ret(); - stackMapper.exited(); - break; - - case lstore: - case dstore: - storeLong(codeBody(t, code, ip++)); - break; - - case lstore_0: - case dstore_0: - storeLong(0); - break; - - case lstore_1: - case dstore_1: - storeLong(1); - break; - - case lstore_2: - case dstore_2: - storeLong(2); - break; - - case lstore_3: - case dstore_3: - storeLong(3); - break; - - case lsub: - if (BytesPerWord == 8) { - popLong(rax); - sub(rax, rsp, BytesPerWord); - } else { - popLong(rax, rdx); - sub(rax, rsp, 0); - sbb(rdx, rsp, BytesPerWord); - } - break; - - case new_: { - uint16_t index = codeReadInt16(t, code, ip); - - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - PROTECT(t, class_); - - initClass(t, class_); - if (UNLIKELY(t->exception)) return 0; - - if (classVmFlags(t, class_) & WeakReferenceFlag) { - compileCall(false, reinterpret_cast(makeNewWeakReference), - class_); - } else { - compileCall(false, reinterpret_cast(makeNew), class_); - } - - pushObject(rax); - } break; - - case newarray: { - uint8_t type = codeBody(t, code, ip++); - - Label nonnegative(this); - - popInt(rax); - cmp(0, rax); - jge(nonnegative); - - compileThrowNew(Machine::NegativeArraySizeExceptionType); - - nonnegative.mark(); - - object (*constructor)(Thread*, uintptr_t, bool); - switch (type) { - case T_BOOLEAN: - constructor = makeBooleanArray; - break; - - case T_CHAR: - constructor = makeCharArray; - break; - - case T_FLOAT: - constructor = makeFloatArray; - break; - - case T_DOUBLE: - constructor = makeDoubleArray; - break; - - case T_BYTE: - constructor = makeByteArray; - break; - - case T_SHORT: - constructor = makeShortArray; - break; - - case T_INT: - constructor = makeIntArray; - break; - - case T_LONG: - constructor = makeLongArray; - break; - - default: abort(t); - } - - compileCall(false, reinterpret_cast(makeBlankArray), - reinterpret_cast(constructor), rax); - - pushObject(rax); - } break; - - case pop_: { - add(BytesPerWord, rsp); - stackMapper.popped(1); - } break; - - case putfield: { - uint16_t index = codeReadInt16(t, code, ip); - - object field = resolveField(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - - switch (fieldCode(t, field)) { - case ByteField: - case BooleanField: - case CharField: - case ShortField: - case FloatField: - case IntField: { - popInt(rcx); - popObject(rax); - - switch (fieldCode(t, field)) { - case ByteField: - case BooleanField: - mov1(rcx, rax, fieldOffset(t, field)); - break; - - case CharField: - case ShortField: - mov2(rcx, rax, fieldOffset(t, field)); - break; - - case FloatField: - case IntField: - mov4(rcx, rax, fieldOffset(t, field)); - break; - } - } break; - - case DoubleField: - case LongField: { - if (BytesPerWord == 8) { - popLong(rcx); - popObject(rax); - mov(rcx, rax, fieldOffset(t, field)); - } else { - popLong(rcx, rbx); - popObject(rax); - mov(rcx, rax, fieldOffset(t, field)); - mov(rbx, rax, fieldOffset(t, field) + 4); - } - } break; - - case ObjectField: { - if (BytesPerWord == 8) { - popObject(rcx); - mov(fieldOffset(t, field), rdx); - popObject(rsi); - mov(rbp, FrameThread, rdi); - } else { - popObject(rcx); - popObject(rsi); - push(rcx); - push(fieldOffset(t, field)); - push(rsi); - push(rbp, FrameThread); - } - - directCall(reinterpret_cast(set)); - - if (BytesPerWord == 4) { - add(BytesPerWord * 3, rsp); - } - } break; - - default: abort(t); - } - } break; - - case putstatic: { - uint16_t index = codeReadInt16(t, code, ip); - - object field = resolveField(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return 0; - - initClass(t, fieldClass(t, field)); - if (UNLIKELY(t->exception)) return 0; - - object table = classStaticTable(t, fieldClass(t, field)); - unsigned offset = (fieldOffset(t, field) * BytesPerWord) + ArrayBody; - - switch (fieldCode(t, field)) { - case ByteField: - case BooleanField: - case CharField: - case ShortField: - case FloatField: - case IntField: { - object intType = arrayBody(t, t->m->types, Machine::IntType); - - if (BytesPerWord == 8) { - mov(poolRegister(), poolReference(intType), rsi); - mov(rbp, FrameThread, rdi); - } else { - push(poolRegister(), poolReference(intType)); - push(rbp, FrameThread); - } - - indirectCall(reinterpret_cast(makeNew)); - - if (BytesPerWord == 4) { - add(BytesPerWord * 2, rsp); - } - - popInt4(rax, IntValue); - - if (BytesPerWord == 8) { - mov(rax, rcx); - mov(offset, rdx); - mov(poolRegister(), poolReference(table), rsi); - mov(rbp, FrameThread, rdi); - } else { - push(rax); - push(offset); - push(poolRegister(), poolReference(table)); - push(rbp, FrameThread); - } - - directCall(reinterpret_cast(set)); - - if (BytesPerWord == 4) { - add(BytesPerWord * 3, rsp); - } - } break; - - case DoubleField: - case LongField: { - object longType = arrayBody(t, t->m->types, Machine::LongType); - - if (BytesPerWord == 8) { - mov(poolRegister(), poolReference(longType), rsi); - mov(rbp, FrameThread, rdi); - } else { - push(poolRegister(), poolReference(longType)); - push(rbp, FrameThread); - } - - indirectCall(reinterpret_cast(makeNew)); - - if (BytesPerWord == 8) { - popLong(rax, LongValue); - } else { - add(BytesPerWord * 2, rsp); - popLong(rax, LongValue, - rax, LongValue + 4); - } - - if (BytesPerWord == 8) { - mov(rax, rcx); - mov(offset, rdx); - mov(poolRegister(), poolReference(table), rsi); - mov(rbp, FrameThread, rdi); - } else { - push(rax); - push(offset); - push(poolRegister(), poolReference(table)); - push(rbp, FrameThread); - } - - directCall(reinterpret_cast(set)); - - if (BytesPerWord == 4) { - add(BytesPerWord * 3, rsp); - } - } break; - - case ObjectField: - if (BytesPerWord == 8) { - popObject(rcx); - mov(offset, rdx); - mov(poolRegister(), poolReference(table), rsi); - mov(rbp, FrameThread, rdi); - } else { - push(offset); - push(poolRegister(), poolReference(table)); - push(rbp, FrameThread); - } - - directCall(reinterpret_cast(set)); - - if (BytesPerWord == 4) { - add(BytesPerWord * 3, rsp); - stackMapper.poppedObject(); - } - break; - - default: abort(t); - } - } break; - - case return_: - mov(rbp, rsp); - pop(rbp); - ret(); - stackMapper.exited(); - break; - - case sipush: { - pushInt(static_cast(codeReadInt16(t, code, ip))); - } break; - - // todo: -// case tableswitch: { -// int32_t base = ip - 1; -// ip += 3; -// ip -= (ip % 4); - -// int32_t default_ = codeReadInt32(t, code, ip); -// int32_t bottom = codeReadInt32(t, code, ip); -// int32_t top = codeReadInt32(t, code, ip); - -// popInt(rax); -// cmp(bottom, rax); -// jl(defaultLabel); - -// cmp(top, rax); -// jg(defaultLabel); - -// sub(bottom, rax); -// mov(Address(NoRegister, rax, 4, -// IndexTable(this, base, top - bottom, code, ip)), -// rax); -// jmp(rax); - -// defaultLabel.mark(); -// jmp(base + default_); -// stackMapper.jumped(base + default_); -// } break; - - default: - abort(t); - } - } - - resolveJumps(); - buildExceptionHandlerTable(code); - - return finish(); - } - - void resolveJumps() { - for (unsigned i = 0; i < jumps.length(); i += 8) { - uint32_t ip = jumps.get4(i); - uint32_t offset = jumps.get4(i + 4); - - code.set4(offset, machineIPs[ip] - (offset + 4)); - } - } - - void buildExceptionHandlerTable(object code) { - PROTECT(t, code); - - object eht = codeExceptionHandlerTable(t, code); - if (eht) { - PROTECT(t, eht); - - for (unsigned i = 0; i < exceptionHandlerTableLength(t, eht); ++i) { - ExceptionHandler* eh = exceptionHandlerTableBody(t, eht, i); - - nativeExceptionHandlerStart(exceptionHandlers + i) - = machineIPs[exceptionHandlerStart(eh)]; - - nativeExceptionHandlerEnd(exceptionHandlers + i) - = machineIPs[exceptionHandlerEnd(eh)]; - - nativeExceptionHandlerIp(exceptionHandlers + i) - = machineIPs[exceptionHandlerIp(eh)]; - - unsigned ct = exceptionHandlerCatchType(eh); - object catchType; - if (ct) { - catchType = resolveClassInPool - (t, codePool(t, code), exceptionHandlerCatchType(eh) - 1); - } else { - catchType = 0; - } - - nativeExceptionHandlerCatchType(exceptionHandlers + i) - = (catchType ? (poolReference(catchType) / BytesPerWord) - 1 : 0); - } - } - } - - Compiled* finish() { - stackMapper.finish(); - return makeCompiled(t, method, &code, lineNumbers, exceptionHandlers, - &stackMapper); - } - - object makePool() { - if (pool.length()) { - object array = makeArray(t, pool.length() / BytesPerWord, false); - pool.copyTo(&arrayBody(t, array, 0)); - return array; - } else { - return 0; - } - } - - Register poolRegister() { - return rdi; - } - - uint32_t poolReference(object o) { - if (poolRegisterClobbered) { - mov(rbp, FrameMethod, rdi); - mov(rdi, MethodCode, rdi); - //poolRegisterClobbered = false; - } - pool.appendAddress(reinterpret_cast(o)); - return pool.length() + BytesPerWord; - } - - object method; - StackMapper stackMapper; - bool poolRegisterClobbered; - uint32_t* machineIPs; - NativeLineNumber* lineNumbers; - NativeExceptionHandler* exceptionHandlers; - Buffer pool; - - class MyProtector: public Thread::Protector { - public: - MyProtector(JavaCompiler* c): Protector(c->t), c(c) { } - - virtual void visit(Heap::Visitor* v) { - v->visit(&(c->method)); - - for (unsigned i = 0; i < c->pool.length(); i += BytesPerWord) { - v->visit(reinterpret_cast(&(c->pool.getAddress(i)))); - } - } - - JavaCompiler* c; - } protector; -}; + assert(t, index == methodParameterFootprint(t, method)); +} void -compileMethod2(MyThread* t, object method) +visitStackAndLocals(MyThread* t, Heap::Visitor* v, void* base, object node) { - if (reinterpret_cast(methodCompiled(t, method)) - == t->m->processor->methodStub(t)) - { - PROTECT(t, method); + object method = traceNodeMethod(t, node); - ACQUIRE(t, t->m->classLock); - - if (reinterpret_cast(methodCompiled(t, method)) - == t->m->processor->methodStub(t)) - { - if (Verbose) { - fprintf(stderr, "compiling %s.%s\n", - &byteArrayBody(t, className(t, methodClass(t, method)), 0), - &byteArrayBody(t, methodName(t, method), 0)); - } + unsigned parameterFootprint = methodParameterFootprint(t, method); + unsigned count = codeMaxStack(t, methodCode(t, method)) + + codeMaxLocals(t, methodCode(t, method)) + - parameterFootprint; + + if (count) { + uintptr_t* map = &traceNodeMap(t, node, 0); - JavaCompiler c(t, method); - Compiled* code = c.compile(); - - if (Verbose) { - fprintf(stderr, "compiled %s.%s from %p to %p\n", - &byteArrayBody(t, className(t, methodClass(t, method)), 0), - &byteArrayBody(t, methodName(t, method), 0), - compiledCode(code), - compiledCode(code) + compiledCodeLength(code)); + for (unsigned i = 0; i < count; ++i) { + if (getBit(map, i)) { + v->visit(localObject(t, base, method, i + parameterFootprint)); } - - // for debugging: - if (false and - strcmp(reinterpret_cast - (&byteArrayBody(t, className(t, methodClass(t, method)), 0)), - "Memory") == 0 and - strcmp(reinterpret_cast - (&byteArrayBody(t, methodName(t, method), 0)), - "runningSum") == 0) - { - noop(); - } - - object pool = c.makePool(); - set(t, method, MethodCode, pool); - - methodCompiled(t, method) = reinterpret_cast(code); } } } void -updateCaller(MyThread* t, object method) +visitStack(MyThread* t, Heap::Visitor* v) { - uintptr_t stub = reinterpret_cast - (compiledCode(static_cast(t->m->processor->methodStub(t)))); + void* base = t->base; + void** stack = static_cast(t->stack); + MyThread::CallTrace* trace = t->trace; - Assembler a(t->m->system); - a.mov(stub, rax); - unsigned offset = a.code.length() - BytesPerWord; + while (true) { + object node = findTraceNode(t, *stack); + if (node) { + object method = traceNodeMethod(t, node); + + // we only need to visit the parameters of this method if the + // caller is native. Otherwise, the caller owns them. + object next = findTraceNode(t, static_cast(base)[1]); + if (next == 0) { + visitParameters(t, v, base, method); + } + + visitStackAndLocals(t, v, base, method); - a.call(rax); - - uint8_t* caller = static_cast(frameAddress(frameStart(t))) - - a.code.length(); - if (memcmp(a.code.data, caller, a.code.length()) == 0) { - // it's a direct call - update caller to point to new code - - // address must be aligned on a word boundary for this write to - // be atomic - assert(t, reinterpret_cast(caller + offset) - % BytesPerWord == 0); - - *reinterpret_cast(caller + offset) - = compiledCode(reinterpret_cast(methodCompiled(t, method))); - } + stack = static_cast(base) + 1; + base = *static_cast(base); + } else if (trace) { + base = trace->base; + stack = static_cast(trace->stack); + trace = trace->next; + } else { + break; + } + } } -void -compileMethod(MyThread* t, object method) +object +compileDefault(MyThread* t, Compiler* c) { - { PROTECT(t, method); - compileMethod2(t, method); - - if (LIKELY(t->exception == 0)) { - if (not methodVirtual(t, method)) { - updateCaller(t, method); - } - return; - } - } + c->prologue(); - unwind(t); + c->mov(c->base(), c->memory(c->thread(), difference(&(t->base), t))); + c->mov(c->stack(), c->memory(c->thread(), difference(&(t->stack), t))); + + c->epilogue(); + + c->jmp + (c->directCall + (c->constant(reinterpret_cast(compileMethod)), + 1, c->thread())); + + return finish(t, c); +} + +object +compileNative(MyThread* t, Compiler* c) +{ + c->prologue(); + + c->mov(c->base(), c->memory(c->thread(), difference(&(t->base), t))); + c->mov(c->stack(), c->memory(c->thread(), difference(&(t->stack), t))); + + c->call + (c->directCall + (c->constant(reinterpret_cast(invokeNative)), + 1, c->thread())); + + c->epilogue(); + c->ret(); + + return finish(t, c); } class ArgumentList { @@ -4683,18 +3171,11 @@ class ArgumentList { ArgumentList(Thread* t, uintptr_t* array, bool* objectMask, object this_, const char* spec, bool indirectObjects, va_list arguments): t(static_cast(t)), - next(this->t->argumentList), array(array), objectMask(objectMask), position(0), protector(this) { - this->t->argumentList = this; - - addInt(reinterpret_cast(t)); - addObject(0); // reserve space for method - addInt(0); // reserve space for frame - if (this_) { addObject(this_); } @@ -4726,17 +3207,11 @@ class ArgumentList { ArgumentList(Thread* t, uintptr_t* array, bool* objectMask, object this_, const char* spec, object arguments): t(static_cast(t)), - next(this->t->argumentList), array(array), objectMask(objectMask), position(0), protector(this) { - this->t->argumentList = this; - - addInt(0); // reserve space for trace pointer - addObject(0); // reserve space for method pointer - if (this_) { addObject(this_); } @@ -4763,10 +3238,6 @@ class ArgumentList { } } - ~ArgumentList() { - t->argumentList = next; - } - void addObject(object v) { array[position] = reinterpret_cast(v); objectMask[position] = true; @@ -4787,7 +3258,6 @@ class ArgumentList { } MyThread* t; - ArgumentList* next; uintptr_t* array; bool* objectMask; unsigned position; @@ -4812,32 +3282,21 @@ object invoke(Thread* thread, object method, ArgumentList* arguments) { MyThread* t = static_cast(thread); - - arguments->array[1] = reinterpret_cast(method); unsigned returnCode = methodReturnCode(t, method); unsigned returnType = fieldType(t, returnCode); Reference* reference = t->reference; + MyThread::CallTrace trace(t); - void* frameData[2]; - memcpy(frameData, t->frameData, BytesPerWord * 2); - - if (frameData[0]) { - arguments->array[2] = reinterpret_cast(framePointer(frameData)); - } - - Compiled* code = reinterpret_cast(methodCompiled(t, method)); uint64_t result = vmInvoke - (compiledCode(code), arguments->array, arguments->position * BytesPerWord, - returnType); + (t, &singletonValue(t, methodCompiled(t, method), 0), arguments->array, + arguments->position * BytesPerWord, returnType); while (t->reference != reference) { dispose(t, t->reference); } - memcpy(t->frameData, frameData, BytesPerWord * 2); - object r; switch (returnCode) { case ByteField: @@ -4874,8 +3333,11 @@ class MyProcessor: public Processor { public: MyProcessor(System* s): s(s), - methodStub_(0), - nativeInvoker_(0) + defaultCompiled(0), + nativeCompiled(0), + addressTree(0), + addressTreeSentinal(0), + indirectCaller(0) { } virtual Thread* @@ -4887,76 +3349,80 @@ class MyProcessor: public Processor { return t; } - virtual void* - methodStub(Thread* t) - { - if (methodStub_ == 0) { - Compiler c(static_cast(t)); - methodStub_ = c.compileStub(); - - if (Verbose) { - fprintf(stderr, "compiled method stub from %p to %p\n", - compiledCode(methodStub_), - compiledCode(methodStub_) + compiledCodeLength(methodStub_)); - } + object getDefaultCompiled(MyThread* t) { + if (defaultCompiled == 0) { + Compiler* c = makeCompiler(t->m->system, 0); + defaultCompiled = compileDefault(t, c); + c->dispose(); } - return methodStub_; + return defaultCompiled; } - virtual void* - nativeInvoker(Thread* t) - { - if (nativeInvoker_ == 0) { - Compiler c(static_cast(t)); - nativeInvoker_ = c.compileNativeInvoker(); - - if (Verbose) { - fprintf(stderr, "compiled native invoker from %p to %p\n", - compiledCode(nativeInvoker_), - compiledCode(nativeInvoker_) - + compiledCodeLength(nativeInvoker_)); - } + object getNativeCompiled(MyThread* t) { + if (nativeCompiled == 0) { + Compiler* c = makeCompiler(t->m->system, 0); + nativeCompiled = compileNative(t, c); + c->dispose(); } - return nativeInvoker_; + return nativeCompiled; } - Compiled* - caller(Thread* t) + virtual object + makeMethod(vm::Thread* t, + uint8_t vmFlags, + uint8_t returnCode, + uint8_t parameterCount, + uint8_t parameterFootprint, + uint16_t flags, + uint16_t offset, + object name, + object spec, + object class_, + object code) { - if (caller_ == 0) { - Compiler c(static_cast(t)); - caller_ = c.compileCaller(); + object compiled + = ((flags & ACC_NATIVE) + ? getNativeCompiled(static_cast(t)) + : getDefaultCompiled(static_cast(t))); - if (Verbose) { - fprintf(stderr, "compiled caller from %p to %p\n", - compiledCode(caller_), - compiledCode(caller_) + compiledCodeLength(caller_)); - } - } - return caller_; + return vm::makeMethod + (t, vmFlags, returnCode, parameterCount, parameterFootprint, flags, + offset, name, spec, class_, code, compiled); } - virtual unsigned - parameterFootprint(vm::Thread* t, const char* s, bool static_) + virtual object + makeClass(vm::Thread* t, + uint16_t flags, + uint8_t vmFlags, + uint8_t arrayDimensions, + uint16_t fixedSize, + uint16_t arrayElementSize, + object objectMask, + object name, + object super, + object interfaceTable, + object virtualTable, + object fieldTable, + object methodTable, + object staticTable, + object loader, + unsigned vtableLength) { - unsigned footprint = 0; - for (MethodSpecIterator it(t, s); it.hasNext();) { - switch (*it.next()) { - case 'J': - case 'D': - footprint += 2; - break; + object c = vm::makeClass + (t, flags, vmFlags, arrayDimensions, fixedSize, arrayElementSize, + objectMask, name, super, interfaceTable, virtualTable, fieldTable, + methodTable, staticTable, loader, vtableLength, false); - default: - ++ footprint; - break; - } + for (unsigned i = 0; i < vtableLength; ++i) { + object compiled + = ((flags & ACC_NATIVE) + ? getNativeCompiled(static_cast(t)) + : getDefaultCompiled(static_cast(t))); + + classVtable(t, c, i) = &singletonBody(t, compiled, 0); } - if (not static_) { - ++ footprint; - } - return footprint; + return c; } virtual void @@ -4982,80 +3448,32 @@ class MyProcessor: public Processor { { MyThread* t = static_cast(vmt); - if (t->m->active) { - for (Reference* r = t->reference; r; r = r->next) { - v->visit(&(r->target)); - } - - visitStack(t, v); + if (t == t->m->rootThread) { + v->visit(&defaultCompiled); + v->visit(&nativeCompiled); + v->visit(&addressTree); } + + for (Reference* r = t->reference; r; r = r->next) { + v->visit(&(r->target)); + } + + visitStack(t, v); } - virtual uintptr_t - frameStart(Thread* vmt) + virtual void + walkStack(Thread* vmt, StackVisitor* v) { - return reinterpret_cast - (::frameStart(static_cast(vmt))); - } + MyThread* t = static_cast(vmt); - virtual uintptr_t - frameNext(Thread*, uintptr_t frame) - { - return reinterpret_cast - (::frameNext(reinterpret_cast(frame))); - } - - virtual bool - frameValid(Thread*, uintptr_t frame) - { - return ::frameValid(reinterpret_cast(frame)); - } - - virtual object - frameMethod(Thread*, uintptr_t frame) - { - return ::frameMethod(reinterpret_cast(frame)); - } - - virtual unsigned - frameIp(Thread* t, uintptr_t frame) - { - void* f = reinterpret_cast(frame); - return addressOffset(t, ::frameMethod(f), ::frameAddress(f)); + MyStackWalker walker(t); + walker.walk(v); } virtual int - lineNumber(Thread* t, object method, unsigned ip) + lineNumber(Thread* vmt, object method, int ip) { - if (methodFlags(t, method) & ACC_NATIVE) { - return NativeLine; - } - - Compiled* code = reinterpret_cast(methodCompiled(t, method)); - if (compiledLineNumberCount(t, code)) { - unsigned bottom = 0; - unsigned top = compiledLineNumberCount(t, code); - for (unsigned span = top - bottom; span; span = top - bottom) { - unsigned middle = bottom + (span / 2); - NativeLineNumber* ln = compiledLineNumber(t, code, middle); - - if (ip >= nativeLineNumberIp(ln) - and (middle + 1 == compiledLineNumberCount(t, code) - or ip < nativeLineNumberIp - (compiledLineNumber(t, code, middle + 1)))) - { - return nativeLineNumberLine(ln); - } else if (ip < nativeLineNumberIp(ln)) { - top = middle; - } else if (ip > nativeLineNumberIp(ln)) { - bottom = middle + 1; - } - } - - abort(t); - } else { - return UnknownLine; - } + return findLineNumber(static_cast(vmt), method, ip); } virtual object* @@ -5092,12 +3510,20 @@ class MyProcessor: public Processor { const char* spec = reinterpret_cast (&byteArrayBody(t, methodSpec(t, method), 0)); - unsigned size = methodParameterFootprint(t, method) + FrameFootprint; + unsigned size = methodParameterFootprint(t, method); uintptr_t array[size]; bool objectMask[size]; ArgumentList list(t, array, objectMask, this_, spec, arguments); - return ::invoke(t, method, &list); + PROTECT(t, method); + + compile(static_cast(t), method); + + if (LIKELY(t->exception == 0)) { + return ::invoke(t, method, &list); + } + + return 0; } virtual object @@ -5112,13 +3538,21 @@ class MyProcessor: public Processor { const char* spec = reinterpret_cast (&byteArrayBody(t, methodSpec(t, method), 0)); - unsigned size = methodParameterFootprint(t, method) + FrameFootprint; + unsigned size = methodParameterFootprint(t, method); uintptr_t array[size]; bool objectMask[size]; ArgumentList list (t, array, objectMask, this_, spec, indirectObjects, arguments); - return ::invoke(t, method, &list); + PROTECT(t, method); + + compile(static_cast(t), method); + + if (LIKELY(t->exception == 0)) { + return ::invoke(t, method, &list); + } + + return 0; } virtual object @@ -5128,7 +3562,7 @@ class MyProcessor: public Processor { assert(t, t->state == Thread::ActiveState or t->state == Thread::ExclusiveState); - unsigned size = parameterFootprint(t, methodSpec, false) + FrameFootprint; + unsigned size = parameterFootprint(t, methodSpec, false); uintptr_t array[size]; bool objectMask[size]; ArgumentList list @@ -5138,56 +3572,106 @@ class MyProcessor: public Processor { if (LIKELY(t->exception == 0)) { assert(t, ((methodFlags(t, method) & ACC_STATIC) == 0) xor (this_ == 0)); - return ::invoke(t, method, &list); - } else { - return 0; + PROTECT(t, method); + + compile(static_cast(t), method); + + if (LIKELY(t->exception == 0)) { + return ::invoke(t, method, &list); + } } + + return 0; } virtual void dispose() { - if (methodStub_) { - s->free(methodStub_); - } - - if (nativeInvoker_) { - s->free(nativeInvoker_); - } - - if (caller_) { - s->free(caller_); + if (indirectCaller) { + s->free(indirectCaller); } s->free(this); } System* s; - Compiled* methodStub_; - Compiled* nativeInvoker_; - Compiled* caller_; + object defaultCompiled; + object nativeCompiled; + object addressTree; + object addressTreeSentinal; + void* indirectCaller; }; -Compiled* -caller(MyThread* t) +MyProcessor* +processor(MyThread* t) { - return static_cast(t->m->processor)->caller(t); + MyProcessor* p = static_cast(t->m->processor); + if (p->addressTree == 0) { + ACQUIRE(t, t->m->classLock); + + if (p->addressTree == 0) { + p->addressTreeSentinal = makeTraceNode(t, 0, 0, 0, 0, 0, 0, 0, 0, false); + set(t, p->addressTreeSentinal, TreeNodeLeft, p->addressTreeSentinal); + set(t, p->addressTreeSentinal, TreeNodeRight, p->addressTreeSentinal); + p->addressTree = p->addressTreeSentinal; + + Compiler* c = makeCompiler(t->m->system, 0); + + c->mov(c->base(), c->memory(c->thread(), difference(&(t->base), t))); + c->mov(c->stack(), c->memory(c->thread(), difference(&(t->stack), t))); + + c->jmp(c->indirectTarget()); + + p->indirectCaller = t->m->system->allocate(c->size()); + c->writeTo(p->indirectCaller); + } + } + return p; +} + +void +compile(MyThread* t, object method) +{ + MyProcessor* p = processor(t); + + object stub = p->getDefaultCompiled(t); + + if (methodCompiled(t, method) == stub) { + PROTECT(t, method); + + ACQUIRE(t, t->m->classLock); + + if (methodCompiled(t, method) == stub) { + PROTECT(t, method); + + Compiler* c = makeCompiler(t->m->system, p->indirectCaller); + + object compiled = compile(t, c, method); + set(t, method, MethodCompiled, compiled); + + c->dispose(); + } + } +} + +object +findTraceNode(MyThread* t, void* address) +{ + MyProcessor* p = processor(t); + return treeQuery(t, p->addressTree, reinterpret_cast(address), + p->addressTreeSentinal); +} + +void +insertTraceNode(MyThread* t, object node) +{ + MyProcessor* p = processor(t); + p->addressTree = treeInsert + (t, p->addressTree, node, p->addressTreeSentinal); } } // namespace namespace vm { -void -printJavaTrace(Thread* t, void* base, void* ip) -{ - void* top[] = { base, ip }; - - printTrace - (t, makeRuntimeException - (t, 0, makeTrace - (t, reinterpret_cast - (top + (FrameFootprint / BytesPerWord) + 2)), 0)); -} - Processor* makeProcessor(System* system) { @@ -5195,4 +3679,3 @@ makeProcessor(System* system) } } // namespace vm - diff --git a/src/compile2.cpp b/src/compile2.cpp deleted file mode 100644 index 4b302db6d9..0000000000 --- a/src/compile2.cpp +++ /dev/null @@ -1,3673 +0,0 @@ -#include "machine.h" -#include "util.h" -#include "buffer.h" -#include "process.h" -#include "compiler.h" - -using namespace vm; - -extern "C" uint64_t -vmInvoke(void* thread, void* function, void* stack, unsigned stackSize, - unsigned returnType); - -extern "C" void -vmCall(); - -extern "C" void NO_RETURN -vmJump(void* address, void* base, void* stack); - -namespace { - -class MyThread: public Thread { - public: - class CallTrace { - public: - CallTrace(MyThread* t): - t(t), - base(t->base), - stack(t->stack), - next(t->trace) - { - t->trace = this; - t->base = 0; - t->stack = 0; - } - - ~CallTrace() { - t->stack = stack; - t->base = base; - t->trace = next; - } - - MyThread* t; - void* base; - void* stack; - CallTrace* next; - }; - - MyThread(Machine* m, object javaThread, Thread* parent): - Thread(m, javaThread, parent), - base(0), - stack(0), - trace(0), - reference(0) - { } - - void* base; - void* stack; - CallTrace* trace; - Reference* reference; -}; - -object -resolveTarget(MyThread* t, void* stack, object method) -{ - if (methodVirtual(t, method)) { - unsigned parameterFootprint = methodParameterFootprint(t, method); - object class_ = objectClass - (t, reinterpret_cast(stack)[parameterFootprint + 1]); - - if (classVmFlags(t, class_) & BootstrapFlag) { - resolveClass(t, className(t, class_)); - if (UNLIKELY(t->exception)) return 0; - } - - if (classFlags(t, methodClass(t, method)) & ACC_INTERFACE) { - return findInterfaceMethod(t, method, class_); - } else { - return findMethod(t, method, class_); - } - } - - return method; -} - -object -findTraceNode(MyThread* t, void* address); - -void -insertTraceNode(MyThread* t, object node); - -class MyStackWalker: public Processor::StackWalker { - public: - class MyProtector: public Thread::Protector { - public: - MyProtector(MyStackWalker* walker): - Protector(walker->t), walker(walker) - { } - - virtual void visit(Heap::Visitor* v) { - v->visit(&(walker->node)); - v->visit(&(walker->nativeMethod)); - } - - MyStackWalker* walker; - }; - - MyStackWalker(MyThread* t): - t(t), - base(t->base), - stack(t->stack), - trace(t->trace), - node(findTraceNode(t, *static_cast(stack))), - nativeMethod(resolveNativeMethod(t, stack, node)), - protector(this) - { } - - MyStackWalker(MyStackWalker* w): - t(w->t), - base(w->base), - stack(w->stack), - trace(w->trace), - node(w->node), - nativeMethod(w->nativeMethod), - protector(this) - { } - - static object resolveNativeMethod(MyThread* t, void* stack, object node) { - if (node) { - object target = resolveTarget(t, stack, traceNodeTarget(t, node)); - if (target and methodFlags(t, target) & ACC_NATIVE) { - return target; - } - } - return 0; - } - - virtual void walk(Processor::StackVisitor* v) { - if (not v->visit(this)) { - return; - } - - for (MyStackWalker it(this); it.next();) { - MyStackWalker walker(it); - if (not v->visit(&walker)) { - break; - } - } - } - - bool next() { - if (nativeMethod) { - nativeMethod = 0; - } else { - stack = static_cast(base) + 1; - base = *static_cast(base); - node = findTraceNode(t, *static_cast(stack)); - if (node == 0) { - if (trace) { - base = trace->base; - stack = static_cast(trace->stack); - trace = trace->next; - node = findTraceNode(t, *static_cast(stack)); - nativeMethod = resolveNativeMethod(t, stack, node); - } else { - return false; - } - } - } - return true; - } - - virtual object method() { - if (nativeMethod) { - return nativeMethod; - } else { - return traceNodeMethod(t, node); - } - } - - virtual int ip() { - if (nativeMethod) { - return 0; - } else { - return treeNodeKey(t, node); - } - } - - virtual unsigned count() { - class Visitor: public Processor::StackVisitor { - public: - Visitor(): count(0) { } - - virtual bool visit(Processor::StackWalker*) { - ++ count; - return true; - } - - unsigned count; - } v; - - MyStackWalker walker(this); - walker.walk(&v); - - return v.count; - } - - MyThread* t; - void* base; - void* stack; - MyThread::CallTrace* trace; - object node; - object nativeMethod; - MyProtector protector; -}; - -uintptr_t* -makeCodeMask(MyThread* t, unsigned length) -{ - unsigned size = ceiling(length, BytesPerWord) * BytesPerWord; - uintptr_t* mask = static_cast(t->m->system->allocate(size)); - memset(mask, 0, size); - return mask; -} - -int -localOffset(MyThread* t, int v, object method) -{ - int parameterFootprint = methodParameterFootprint(t, method) * BytesPerWord; - - v *= BytesPerWord; - if (v < parameterFootprint) { - return (parameterFootprint - v - BytesPerWord) + (BytesPerWord * 2); - } else { - return -(v + BytesPerWord - parameterFootprint); - } -} - -class Frame { - public: - class MyProtector: public Thread::Protector { - public: - MyProtector(Frame* frame): Protector(frame->t), frame(frame) { } - - virtual void visit(Heap::Visitor* v) { - v->visit(&(frame->method)); - - if (next == 0) { - Buffer* pool = frame->objectPool; - for (unsigned i = 1; i < pool->length(); i += BytesPerWord * 2) { - v->visit(reinterpret_cast(&(pool->getAddress(i)))); - } - - Buffer* log = frame->traceLog; - unsigned traceSize = traceSizeInBytes(t, frame->method); - for (unsigned i = 1; i < log->length(); i += traceSize) { - v->visit(reinterpret_cast(&(log->getAddress(i)))); - } - } - } - - Frame* frame; - }; - - Frame(MyThread* t, Compiler* c, object method, uintptr_t* map, - Buffer* objectPool, Buffer* traceLog): - next(0), - t(t), - c(c), - method(method), - map(map), - objectPool(objectPool), - traceLog(traceLog), - codeMask(makeCodeMask(t, codeLength(t, methodCode(t, method)))), - ip(0), - sp(localSize(t, method)), - protector(this) - { - memset(map, 0, mapSizeInBytes(t, method)); - } - - Frame(Frame* f, uintptr_t* map): - next(f), - t(f->t), - c(f->c), - method(f->method), - map(map), - objectPool(f->objectPool), - traceLog(f->traceLog), - codeMask(f->codeMask), - ip(f->ip), - sp(f->sp), - protector(this) - { - memcpy(map, f->map, mapSizeInBytes(t, method)); - } - - ~Frame() { - t->m->system->free(codeMask); - } - - Operand* append(object o) { - objectPool->appendAddress(reinterpret_cast(c->poolOffset())); - Operand* result = c->poolAppend(c->constant(0)); - objectPool->appendAddress(reinterpret_cast(o)); - return result; - } - - static unsigned parameterFootprint(Thread* t, object method) { - return methodParameterFootprint(t, method); - } - - static unsigned localSize(Thread* t, object method) { - return codeMaxLocals(t, methodCode(t, method)) - - parameterFootprint(t, method); - } - - static unsigned stackSize(Thread* t, object method) { - return codeMaxStack(t, methodCode(t, method)); - } - - static unsigned mapSize(Thread* t, object method) { - return stackSize(t, method) + localSize(t, method); - } - - static unsigned mapSizeInWords(Thread* t, object method) { - return ceiling(mapSize(t, method), BytesPerWord); - } - - static unsigned mapSizeInBytes(Thread* t, object method) { - return mapSizeInWords(t, method) * BytesPerWord; - } - - static unsigned traceSizeInBytes(Thread* t, object method) { - return BytesPerWord + BytesPerWord + 1 + mapSizeInWords(t, method); - } - - void pushedInt() { - assert(t, sp + 1 <= mapSize(t, method)); - assert(t, getBit(map, sp) == 0); - ++ sp; - } - - void pushedObject() { - assert(t, sp + 1 <= mapSize(t, method)); - markBit(map, sp++); - } - - void poppedInt() { - assert(t, sp >= 1); - assert(t, sp - 1 >= localSize(t, method)); - assert(t, getBit(map, sp - 1) == 0); - -- sp; - } - - void poppedObject() { - assert(t, sp >= 1); - assert(t, sp - 1 >= localSize(t, method)); - assert(t, getBit(map, sp - 1) != 0); - clearBit(map, -- sp); - } - - void storedInt(unsigned index) { - assert(t, index < localSize(t, method)); - clearBit(map, index); - } - - void storedObject(unsigned index) { - assert(t, index < localSize(t, method)); - markBit(map, index); - } - - void dupped() { - assert(t, sp + 1 <= mapSize(t, method)); - assert(t, sp - 1 >= localSize(t, method)); - if (getBit(map, sp - 1)) { - markBit(map, sp); - } - ++ sp; - } - - void duppedX1() { - assert(t, sp + 1 <= mapSize(t, method)); - assert(t, sp - 2 >= localSize(t, method)); - - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b2) { - markBit(map, sp - 1); - } else { - clearBit(map, sp - 1); - } - - if (b1) { - markBit(map, sp - 2); - markBit(map, sp); - } else { - clearBit(map, sp - 2); - } - - ++ sp; - } - - void duppedX2() { - assert(t, sp + 1 <= mapSize(t, method)); - assert(t, sp - 3 >= localSize(t, method)); - - unsigned b3 = getBit(map, sp - 3); - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b3) { - markBit(map, sp - 2); - } else { - clearBit(map, sp - 2); - } - - if (b2) { - markBit(map, sp - 1); - } else { - clearBit(map, sp - 1); - } - - if (b1) { - markBit(map, sp - 3); - markBit(map, sp); - } else { - clearBit(map, sp - 3); - } - - ++ sp; - } - - void dupped2() { - assert(t, sp + 2 <= mapSize(t, method)); - assert(t, sp - 2 >= localSize(t, method)); - - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b2) { - markBit(map, sp); - } - - if (b1) { - markBit(map, sp + 1); - } - - sp += 2; - } - - void dupped2X1() { - assert(t, sp + 2 <= mapSize(t, method)); - assert(t, sp - 3 >= localSize(t, method)); - - unsigned b3 = getBit(map, sp - 3); - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b3) { - markBit(map, sp - 1); - } else { - clearBit(map, sp - 1); - } - - if (b2) { - markBit(map, sp - 3); - markBit(map, sp); - } else { - clearBit(map, sp - 3); - } - - if (b1) { - markBit(map, sp - 2); - markBit(map, sp + 1); - } else { - clearBit(map, sp - 2); - } - - sp += 2; - } - - void dupped2X2() { - assert(t, sp + 2 <= mapSize(t, method)); - assert(t, sp - 4 >= localSize(t, method)); - - unsigned b4 = getBit(map, sp - 4); - unsigned b3 = getBit(map, sp - 3); - unsigned b2 = getBit(map, sp - 2); - unsigned b1 = getBit(map, sp - 1); - - if (b4) { - markBit(map, sp - 2); - } else { - clearBit(map, sp - 2); - } - - if (b3) { - markBit(map, sp - 1); - } else { - clearBit(map, sp - 1); - } - - if (b2) { - markBit(map, sp - 4); - markBit(map, sp); - } else { - clearBit(map, sp - 4); - } - - if (b1) { - markBit(map, sp - 3); - markBit(map, sp + 1); - } else { - clearBit(map, sp - 3); - } - - sp += 2; - } - - void swapped() { - assert(t, sp - 1 >= localSize(t, method)); - assert(t, sp - 2 >= localSize(t, method)); - - bool savedBit = getBit(map, sp - 1); - if (getBit(map, sp - 2)) { - markBit(map, sp - 1); - } else { - clearBit(map, sp - 1); - } - - if (savedBit) { - markBit(map, sp - 2); - } else { - clearBit(map, sp - 2); - } - } - - void trace(object target, bool virtualCall) { - traceLog->appendAddress(reinterpret_cast(target)); - traceLog->appendAddress(reinterpret_cast(c->codeOffset())); - traceLog->append(virtualCall); - traceLog->append(map, mapSizeInWords(t, method) * BytesPerWord); - } - - void trace() { - trace(0, false); - } - - void startLogicalIp(unsigned ip) { - c->startLogicalIp(ip); - this->ip = ip; - } - - void pushInt(Operand* o) { - c->push(o); - pushedInt(); - } - - void pushObject(Operand* o) { - c->push(o); - pushedObject(); - } - - void pushLong(Operand* o) { - c->push2(o); - pushedInt(); - pushedInt(); - } - - void pop(unsigned count) { - assert(t, sp >= count); - assert(t, sp - count >= localSize(t, method)); - while (count) { - clearBit(map, -- sp); - -- count; - } - } - - Operand* topInt() { - assert(t, sp >= 1); - assert(t, sp - 1 >= localSize(t, method)); - assert(t, getBit(map, sp - 1) == 0); - return c->stack(0); - } - - Operand* topLong() { - assert(t, sp >= 2); - assert(t, sp - 2 >= localSize(t, method)); - assert(t, getBit(map, sp - 1) == 0); - assert(t, getBit(map, sp - 2) == 0); - return c->stack2(1); - } - - Operand* topObject() { - assert(t, sp >= 1); - assert(t, sp - 1 >= localSize(t, method)); - assert(t, getBit(map, sp - 1) != 0); - return c->stack(0); - } - - Operand* popInt() { - poppedInt(); - return c->pop(); - } - - Operand* popLong() { - poppedInt(); - poppedInt(); - return c->pop2(); - } - - Operand* popObject() { - poppedObject(); - return c->pop(); - } - - void popInt(Operand* o) { - c->pop(o); - poppedInt(); - } - - void popLong(Operand* o) { - c->pop2(o); - poppedInt(); - poppedInt(); - } - - void popObject(Operand* o) { - c->pop(o); - poppedObject(); - } - - void loadInt(unsigned index) { - assert(t, index < localSize(t, method)); - assert(t, getBit(map, index) == 0); - pushInt(c->memory(c->base(), localOffset(t, index, method))); - } - - void loadLong(unsigned index) { - assert(t, index < localSize(t, method) - 1); - assert(t, getBit(map, index) == 0); - assert(t, getBit(map, index + 1) == 0); - pushLong(c->memory(c->base(), localOffset(t, index, method))); - } - - void loadObject(unsigned index) { - assert(t, index < localSize(t, method)); - assert(t, getBit(map, index) != 0); - pushObject(c->memory(c->base(), localOffset(t, index, method))); - } - - void storeInt(unsigned index) { - popInt(c->memory(c->base(), localOffset(t, index, method))); - storedInt(index); - } - - void storeLong(unsigned index) { - popLong(c->memory(c->base(), localOffset(t, index, method))); - storedInt(index); - storedInt(index + 1); - } - - void storeObject(unsigned index) { - popObject(c->memory(c->base(), localOffset(t, index, method))); - storedObject(index); - } - - void increment(unsigned index, unsigned count) { - assert(t, index < localSize(t, method)); - assert(t, getBit(map, index) == 0); - c->add(c->constant(count), - c->memory(c->base(), localOffset(t, index, method))); - } - - void dup() { - c->push(c->stack(0)); - dupped(); - } - - void dupX1() { - Operand* s0 = c->stack(0); - Operand* s1 = c->stack(1); - - c->mov(s0, s1); - c->mov(s1, s0); - c->push(s0); - - duppedX1(); - } - - void dupX2() { - Operand* s0 = c->stack(0); - Operand* s1 = c->stack(1); - Operand* s2 = c->stack(2); - - c->mov(s0, s2); - c->mov(s2, s1); - c->mov(s1, s0); - c->push(s0); - - duppedX2(); - } - - void dup2() { - Operand* s0 = c->stack(0); - - c->push(s0); - c->push(s0); - - dupped2(); - } - - void dup2X1() { - Operand* s0 = c->stack(0); - Operand* s1 = c->stack(1); - Operand* s2 = c->stack(2); - - c->mov(s1, s2); - c->mov(s0, s1); - c->mov(s2, s0); - c->push(s1); - c->push(s0); - - dupped2X1(); - } - - void dup2X2() { - Operand* s0 = c->stack(0); - Operand* s1 = c->stack(1); - Operand* s2 = c->stack(2); - Operand* s3 = c->stack(3); - - c->mov(s1, s3); - c->mov(s0, s2); - c->mov(s3, s1); - c->mov(s2, s0); - c->push(s1); - c->push(s0); - - dupped2X2(); - } - - void swap() { - Operand* s0 = c->stack(0); - Operand* s1 = c->stack(1); - Operand* tmp = c->temporary(); - - c->mov(s0, tmp); - c->mov(s1, s0); - c->mov(tmp, s1); - - c->release(tmp); - - swapped(); - } - - Frame* next; - MyThread* t; - Compiler* c; - object method; - uintptr_t* map; - Buffer* objectPool; - Buffer* traceLog; - uintptr_t* codeMask; - unsigned ip; - unsigned sp; - MyProtector protector; -}; - -void NO_RETURN -unwind(MyThread* t) -{ - void* base = t->base; - void** stack = static_cast(t->stack); - while (true) { - void* returnAddress = *stack; - object node = findTraceNode(t, returnAddress); - if (node) { - object method = traceNodeMethod(t, node); - uint8_t* compiled = reinterpret_cast - (&singletonValue(t, methodCompiled(t, method), 0)); - - ExceptionHandler* handler = findExceptionHandler - (t, method, difference(returnAddress, compiled)); - - if (handler) { - unsigned parameterFootprint = methodParameterFootprint(t, method); - unsigned localFootprint = codeMaxLocals(t, methodCode(t, method)); - - if (localFootprint > parameterFootprint) { - stack -= (localFootprint - parameterFootprint); - } - - *(--stack) = t->exception; - t->exception = 0; - - vmJump(compiled + exceptionHandlerIp(handler), base, stack); - } else { - stack = static_cast(base) + 1; - base = *static_cast(base); - } - } else { - vmJump(returnAddress, base, stack + 1); - } - } -} - -object -findInterfaceMethodFromInstance(Thread* t, object method, object instance) -{ - return findInterfaceMethod(t, method, objectClass(t, instance)); -} - -intptr_t -compareDoublesG(uint64_t bi, uint64_t ai) -{ - double a = bitsToDouble(ai); - double b = bitsToDouble(bi); - - if (a < b) { - return -1; - } else if (a > b) { - return 1; - } else if (a == b) { - return 0; - } else { - return 1; - } -} - -intptr_t -compareDoublesL(uint64_t bi, uint64_t ai) -{ - double a = bitsToDouble(ai); - double b = bitsToDouble(bi); - - if (a < b) { - return -1; - } else if (a > b) { - return 1; - } else if (a == b) { - return 0; - } else { - return -1; - } -} - -intptr_t -compareFloatsG(uint32_t bi, uint32_t ai) -{ - float a = bitsToFloat(ai); - float b = bitsToFloat(bi); - - if (a < b) { - return -1; - } else if (a > b) { - return 1; - } else if (a == b) { - return 0; - } else { - return 1; - } -} - -intptr_t -compareFloatsL(uint32_t bi, uint32_t ai) -{ - float a = bitsToFloat(ai); - float b = bitsToFloat(bi); - - if (a < b) { - return -1; - } else if (a > b) { - return 1; - } else if (a == b) { - return 0; - } else { - return -1; - } -} - -uint64_t -addDouble(uint64_t b, uint64_t a) -{ - return doubleToBits(bitsToDouble(a) + bitsToDouble(b)); -} - -uint64_t -subtractDouble(uint64_t b, uint64_t a) -{ - return doubleToBits(bitsToDouble(a) - bitsToDouble(b)); -} - -uint64_t -multiplyDouble(uint64_t b, uint64_t a) -{ - return doubleToBits(bitsToDouble(a) * bitsToDouble(b)); -} - -uint64_t -divideDouble(uint64_t b, uint64_t a) -{ - return doubleToBits(bitsToDouble(a) / bitsToDouble(b)); -} - -uint64_t -moduloDouble(uint64_t b, uint64_t a) -{ - return doubleToBits(fmod(bitsToDouble(a), bitsToDouble(b))); -} - -uint64_t -negateDouble(uint64_t a) -{ - return doubleToBits(- bitsToDouble(a)); -} - -uint32_t -doubleToFloat(int64_t a) -{ - return floatToBits(static_cast(bitsToDouble(a))); -} - -int32_t -doubleToInt(int64_t a) -{ - return static_cast(bitsToDouble(a)); -} - -int64_t -doubleToLong(int64_t a) -{ - return static_cast(bitsToDouble(a)); -} - -uint32_t -addFloat(uint32_t b, uint32_t a) -{ - return floatToBits(bitsToFloat(a) + bitsToFloat(b)); -} - -uint32_t -subtractFloat(uint32_t b, uint32_t a) -{ - return floatToBits(bitsToFloat(a) - bitsToFloat(b)); -} - -uint32_t -multiplyFloat(uint32_t b, uint32_t a) -{ - return floatToBits(bitsToFloat(a) * bitsToFloat(b)); -} - -uint32_t -divideFloat(uint32_t b, uint32_t a) -{ - return floatToBits(bitsToFloat(a) / bitsToFloat(b)); -} - -uint32_t -moduloFloat(uint32_t b, uint32_t a) -{ - return floatToBits(fmod(bitsToFloat(a), bitsToFloat(b))); -} - -uint32_t -negateFloat(uint32_t a) -{ - return floatToBits(- bitsToFloat(a)); -} - -int64_t -divideLong(int64_t b, int64_t a) -{ - return a / b; -} - -int64_t -moduloLong(int64_t b, int64_t a) -{ - return a % b; -} - -uint64_t -floatToDouble(int32_t a) -{ - return doubleToBits(static_cast(bitsToFloat(a))); -} - -int32_t -floatToInt(int32_t a) -{ - return static_cast(bitsToFloat(a)); -} - -int64_t -floatToLong(int32_t a) -{ - return static_cast(bitsToFloat(a)); -} - -uint64_t -intToDouble(int32_t a) -{ - return doubleToBits(static_cast(a)); -} - -uint32_t -intToFloat(int32_t a) -{ - return floatToBits(static_cast(a)); -} - -object -makeBlankObjectArray(Thread* t, object class_, int32_t length) -{ - return makeObjectArray(t, class_, length, true); -} - -object -makeBlankArray(Thread* t, object (*constructor)(Thread*, uintptr_t, bool), - int32_t length) -{ - return constructor(t, length, true); -} - -uintptr_t -lookUpAddress(int32_t key, uintptr_t* start, int32_t count, - uintptr_t* default_) -{ - int32_t bottom = 0; - int32_t top = count; - for (int32_t span = top - bottom; span; span = top - bottom) { - int32_t middle = bottom + (span / 2); - uintptr_t* p = start + (middle * 2); - int32_t k = *p; - - if (key < k) { - top = middle; - } else if (key > k) { - bottom = middle + 1; - } else { - return p[1]; - } - } - - return *default_; -} - -void -acquireMonitorForObject(Thread* t, object o) -{ - acquire(t, o); -} - -void -releaseMonitorForObject(Thread* t, object o) -{ - release(t, o); -} - -object -makeMultidimensionalArray2(MyThread* t, object class_, uintptr_t* stack, - int32_t dimensions) -{ - PROTECT(t, class_); - - int32_t counts[dimensions]; - for (int i = dimensions - 1; i >= 0; --i) { - counts[i] = stack[dimensions - i - 1]; - if (UNLIKELY(counts[i] < 0)) { - object message = makeString(t, "%d", counts[i]); - t->exception = makeNegativeArraySizeException(t, message); - return 0; - } - } - - object array = makeArray(t, counts[0], true); - setObjectClass(t, array, class_); - PROTECT(t, array); - - populateMultiArray(t, array, counts, 0, dimensions); - - return array; -} - -object -makeMultidimensionalArray(MyThread* t, object class_, uintptr_t* stack, - int32_t dimensions) -{ - object r = makeMultidimensionalArray2(t, class_, stack, dimensions); - if (UNLIKELY(t->exception)) { - unwind(t); - } else { - return r; - } -} - -void NO_RETURN -throwNew(MyThread* t, object class_) -{ - t->exception = makeNew(t, class_); - object trace = makeTrace(t); - set(t, t->exception, ThrowableTrace, trace); - unwind(t); -} - -void NO_RETURN -throw_(MyThread* t, object o) -{ - if (o) { - t->exception = o; - } else { - t->exception = makeNullPointerException(t); - } - unwind(t); -} - -void -compileThrowNew(MyThread* t, Frame* frame, Machine::Type type) -{ - Operand* class_ = frame->append(arrayBody(t, t->m->types, type)); - Compiler* c = frame->c; - c->indirectCallNoReturn - (c->constant(reinterpret_cast(throwNew)), - 2, c->thread(), class_); - - frame->trace(); -} - -void -pushReturnValue(MyThread* t, Frame* frame, unsigned code, Operand* result) -{ - switch (code) { - case ByteField: - case BooleanField: - case CharField: - case ShortField: - case FloatField: - case IntField: - frame->pushInt(result); - break; - - case ObjectField: - frame->pushObject(result); - break; - - case LongField: - case DoubleField: - frame->pushLong(result); - break; - - case VoidField: - break; - - default: - abort(t); - } -} - -void -compileDirectInvoke(MyThread* t, Frame* frame, object target) -{ - Operand* result = frame->c->alignedCall - (frame->c->constant - (reinterpret_cast - (&singletonBody(t, methodCompiled(t, target), 0)))); - - frame->trace(target, false); - - frame->pop(methodParameterFootprint(t, target)); - - pushReturnValue(t, frame, methodReturnCode(t, target), result); -} - -void -compile(MyThread* t, Frame* initialFrame, unsigned ip) -{ - uintptr_t map[Frame::mapSizeInWords(t, initialFrame->method)]; - Frame myFrame(initialFrame, map); - Frame* frame = &myFrame; - Compiler* c = frame->c; - - object code = methodCode(t, frame->method); - PROTECT(t, code); - - while (ip < codeLength(t, code)) { - if (getBit(frame->codeMask, ip)) { - // we've already visited this part of the code - return; - } - - markBit(frame->codeMask, ip); - - frame->startLogicalIp(ip); - - unsigned instruction = codeBody(t, code, ip++); - - switch (instruction) { - case aaload: - case baload: - case caload: - case daload: - case faload: - case iaload: - case laload: - case saload: { - Operand* next = c->label(); - Operand* outOfBounds = c->label(); - - Operand* index = frame->popInt(); - Operand* array = frame->popObject(); - - c->cmp(c->constant(0), index); - c->jl(outOfBounds); - - c->cmp(c->memory(array, ArrayLength), index); - c->jge(outOfBounds); - - switch (instruction) { - case aaload: - frame->pushObject(c->memory(array, ArrayBody, index, BytesPerWord)); - break; - - case faload: - case iaload: - frame->pushInt(c->select4(c->memory(array, ArrayBody, index, 4))); - break; - - case baload: - frame->pushInt(c->select1(c->memory(array, ArrayBody, index, 1))); - break; - - case caload: - frame->pushInt(c->select2z(c->memory(array, ArrayBody, index, 2))); - break; - - case daload: - case laload: - frame->pushInt(c->select8(c->memory(array, ArrayBody, index, 8))); - break; - - case saload: - frame->pushInt(c->select2(c->memory(array, ArrayBody, index, 2))); - break; - } - - c->jmp(next); - - c->mark(outOfBounds); - compileThrowNew(t, frame, Machine::ArrayIndexOutOfBoundsExceptionType); - - c->mark(next); - } break; - - case aastore: - case bastore: - case castore: - case dastore: - case fastore: - case iastore: - case lastore: - case sastore: { - Operand* next = c->label(); - Operand* outOfBounds = c->label(); - - Operand* value; - if (instruction == dastore or instruction == lastore) { - value = frame->popLong(); - } else if (instruction == aastore) { - value = frame->popObject(); - } else { - value = frame->popInt(); - } - - Operand* index = frame->popInt(); - Operand* array = frame->popObject(); - - c->cmp(c->constant(0), index); - c->jl(outOfBounds); - - c->cmp(c->memory(array, BytesPerWord), index); - c->jge(outOfBounds); - - switch (instruction) { - case aastore: - c->shl(c->constant(log(BytesPerWord)), index); - c->add(c->constant(ArrayBody), index); - - c->directCall - (c->constant(reinterpret_cast(set)), - 4, c->thread(), array, index, value); - - frame->trace(); - break; - - case fastore: - case iastore: - c->mov(value, c->select4(c->memory(array, ArrayBody, index, 4))); - break; - - case bastore: - c->mov(value, c->select1(c->memory(array, ArrayBody, index, 1))); - break; - - case castore: - case sastore: - c->mov(value, c->select2(c->memory(array, ArrayBody, index, 2))); - break; - - case dastore: - case lastore: - c->mov(value, c->select8(c->memory(array, ArrayBody, index, 8))); - break; - } - - c->jmp(next); - - c->mark(outOfBounds); - compileThrowNew(t, frame, Machine::ArrayIndexOutOfBoundsExceptionType); - - c->mark(next); - } break; - - case aconst_null: - frame->pushObject(c->constant(0)); - break; - - case aload: - frame->loadObject(codeBody(t, code, ip++)); - break; - - case aload_0: - frame->loadObject(0); - break; - - case aload_1: - frame->loadObject(1); - break; - - case aload_2: - frame->loadObject(2); - break; - - case aload_3: - frame->loadObject(3); - break; - - case anewarray: { - uint16_t index = codeReadInt16(t, code, ip); - - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - - Operand* nonnegative = c->label(); - - Operand* length = frame->popInt(); - c->cmp(c->constant(0), length); - c->jge(nonnegative); - - compileThrowNew(t, frame, Machine::NegativeArraySizeExceptionType); - - c->mark(nonnegative); - - Operand* r = c->indirectCall - (c->constant(reinterpret_cast(makeBlankObjectArray)), - 3, c->thread(), frame->append(class_), length); - - frame->trace(); - - frame->pushObject(r); - } break; - - case areturn: - c->epilogue(); - c->return_(frame->popObject()); - return; - - case arraylength: - frame->pushInt(c->memory(frame->popObject(), ArrayLength)); - break; - - case astore: - frame->storeObject(codeBody(t, code, ip++)); - break; - - case astore_0: - frame->storeObject(0); - break; - - case astore_1: - frame->storeObject(1); - break; - - case astore_2: - frame->storeObject(2); - break; - - case astore_3: - frame->storeObject(3); - break; - - case athrow: - c->indirectCallNoReturn - (c->constant(reinterpret_cast(throw_)), - 2, c->thread(), frame->popObject()); - - frame->trace(); - break; - - case bipush: - frame->pushInt - (c->constant(static_cast(codeBody(t, code, ip++)))); - break; - - case checkcast: { - uint16_t index = codeReadInt16(t, code, ip); - - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - - Operand* next = c->label(); - - Operand* instance = frame->topObject(); - Operand* tmp = c->temporary(); - - c->mov(instance, tmp); - - c->cmp(c->constant(0), tmp); - c->je(next); - - Operand* classOperand = frame->append(class_); - - c->mov(c->memory(tmp), tmp); - c->and_(c->constant(PointerMask), tmp); - - c->cmp(classOperand, tmp); - c->je(next); - - Operand* result = c->directCall - (c->constant(reinterpret_cast(isAssignableFrom)), - 2, classOperand, tmp); - - c->release(tmp); - - c->cmp(0, result); - c->jne(next); - - compileThrowNew(t, frame, Machine::ClassCastExceptionType); - - c->mark(next); - } break; - - case d2f: { - Operand* a = frame->popLong(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(doubleToFloat)), 1, a)); - } break; - - case d2i: { - Operand* a = frame->popLong(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(doubleToInt)), 1, a)); - } break; - - case d2l: { - Operand* a = frame->popLong(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(doubleToLong)), 1, a)); - } break; - - case dadd: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(addDouble)), 2, a, b)); - } break; - - case dcmpg: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(compareDoublesG)), 2, a, b)); - } break; - - case dcmpl: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(compareDoublesL)), 2, a, b)); - } break; - - case dconst_0: - frame->pushLong(c->constant(doubleToBits(0.0))); - break; - - case dconst_1: - frame->pushLong(c->constant(doubleToBits(1.0))); - break; - - case ddiv: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(divideDouble)), 2, a, b)); - } break; - - case dmul: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(multiplyDouble)), 2, a, b)); - } break; - - case dneg: { - Operand* a = frame->popLong(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(negateDouble)), 1, a)); - } break; - - case vm::drem: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(moduloDouble)), 2, a, b)); - } break; - - case dsub: { - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(subtractDouble)), 2, a, b)); - } break; - - case dup: - frame->dup(); - break; - - case dup_x1: - frame->dupX1(); - break; - - case dup_x2: - frame->dupX2(); - break; - - case dup2: - frame->dup2(); - break; - - case dup2_x1: - frame->dup2X1(); - break; - - case dup2_x2: - frame->dup2X2(); - break; - - case f2d: { - Operand* a = frame->popInt(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(floatToDouble)), 1, a)); - } break; - - case f2i: { - Operand* a = frame->popInt(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(floatToInt)), 1, a)); - } break; - - case f2l: { - Operand* a = frame->popInt(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(floatToLong)), 1, a)); - } break; - - case fadd: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(addFloat)), 2, a, b)); - } break; - - case fcmpg: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(compareFloatsG)), 2, a, b)); - } break; - - case fcmpl: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(compareFloatsL)), 2, a, b)); - } break; - - case fconst_0: - frame->pushInt(c->constant(floatToBits(0.0))); - break; - - case fconst_1: - frame->pushInt(c->constant(floatToBits(1.0))); - break; - - case fconst_2: - frame->pushInt(c->constant(floatToBits(2.0))); - break; - - case fdiv: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(divideFloat)), 2, a, b)); - } break; - - case fmul: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(multiplyFloat)), 2, a, b)); - } break; - - case fneg: { - Operand* a = frame->popLong(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(negateFloat)), 1, a)); - } break; - - case vm::frem: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(moduloFloat)), 2, a, b)); - } break; - - case fsub: { - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(subtractFloat)), 2, a, b)); - } break; - - case getfield: - case getstatic: { - uint16_t index = codeReadInt16(t, code, ip); - - object field = resolveField(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - - Operand* table; - - if (instruction == getstatic) { - initClass(t, fieldClass(t, field)); - if (UNLIKELY(t->exception)) return; - - table = frame->append(classStaticTable(t, fieldClass(t, field))); - } else { - table = frame->popObject(); - } - - switch (fieldCode(t, field)) { - case ByteField: - case BooleanField: - frame->pushInt(c->select1(c->memory(table, fieldOffset(t, field)))); - break; - - case CharField: - frame->pushInt(c->select2z(c->memory(table, fieldOffset(t, field)))); - break; - - case ShortField: - frame->pushInt(c->select2(c->memory(table, fieldOffset(t, field)))); - break; - - case FloatField: - case IntField: - frame->pushInt(c->select4(c->memory(table, fieldOffset(t, field)))); - break; - - case DoubleField: - case LongField: - frame->pushLong(c->select8(c->memory(table, fieldOffset(t, field)))); - break; - - case ObjectField: - frame->pushObject(c->memory(c->memory(table, fieldOffset(t, field)))); - break; - - default: - abort(t); - } - } break; - - case goto_: { - uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); - assert(t, newIp < codeLength(t, code)); - - c->jmp(c->logicalIp(newIp)); - ip = newIp; - } break; - - case goto_w: { - uint32_t newIp = (ip - 5) + codeReadInt32(t, code, ip); - assert(t, newIp < codeLength(t, code)); - - c->jmp(c->logicalIp(newIp)); - ip = newIp; - } break; - - case i2b: { - Operand* top = frame->topInt(); - c->mov(c->select1(top), top); - } break; - - case i2c: { - Operand* top = frame->topInt(); - c->mov(c->select2z(top), top); - } break; - - case i2d: { - Operand* a = frame->popInt(); - frame->pushLong - (c->directCall - (c->constant(reinterpret_cast(intToDouble)), 1, a)); - } break; - - case i2f: { - Operand* a = frame->popInt(); - frame->pushInt - (c->directCall - (c->constant(reinterpret_cast(intToFloat)), 1, a)); - } break; - - case i2l: - frame->pushLong(frame->popInt()); - break; - - case i2s: { - Operand* top = frame->topInt(); - c->mov(c->select2(top), top); - } break; - - case iadd: { - Operand* a = frame->popInt(); - c->add(a, frame->topInt()); - } break; - - case iand: { - Operand* a = frame->popInt(); - c->and_(a, frame->topInt()); - } break; - - case iconst_m1: - frame->pushInt(c->constant(-1)); - break; - - case iconst_0: - frame->pushInt(c->constant(0)); - break; - - case iconst_1: - frame->pushInt(c->constant(1)); - break; - - case iconst_2: - frame->pushInt(c->constant(2)); - break; - - case iconst_3: - frame->pushInt(c->constant(3)); - break; - - case iconst_4: - frame->pushInt(c->constant(4)); - break; - - case iconst_5: - frame->pushInt(c->constant(5)); - break; - - case idiv: { - Operand* a = frame->popInt(); - c->div(a, frame->topInt()); - } break; - - case if_acmpeq: - case if_acmpne: { - uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); - assert(t, newIp < codeLength(t, code)); - - Operand* a = frame->popObject(); - Operand* b = frame->popObject(); - c->cmp(a, b); - - Operand* target = c->logicalIp(newIp); - if (instruction == if_acmpeq) { - c->je(target); - } else { - c->jne(target); - } - - compile(t, frame, newIp); - if (UNLIKELY(t->exception)) return; - } break; - - case if_icmpeq: - case if_icmpne: - case if_icmpgt: - case if_icmpge: - case if_icmplt: - case if_icmple: { - uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); - assert(t, newIp < codeLength(t, code)); - - Operand* a = frame->popInt(); - Operand* b = frame->popInt(); - c->cmp(a, b); - - Operand* target = c->logicalIp(newIp); - switch (instruction) { - case if_icmpeq: - c->je(target); - break; - case if_icmpne: - c->jne(target); - break; - case if_icmpgt: - c->jg(target); - break; - case if_icmpge: - c->jge(target); - break; - case if_icmplt: - c->jl(target); - break; - case if_icmple: - c->jle(target); - break; - } - - compile(t, frame, newIp); - if (UNLIKELY(t->exception)) return; - } break; - - case ifeq: - case ifne: - case ifgt: - case ifge: - case iflt: - case ifle: { - uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); - assert(t, newIp < codeLength(t, code)); - - c->cmp(0, frame->popInt()); - - Operand* target = c->logicalIp(newIp); - switch (instruction) { - case ifeq: - c->je(target); - break; - case ifne: - c->jne(target); - break; - case ifgt: - c->jg(target); - break; - case ifge: - c->jge(target); - break; - case iflt: - c->jl(target); - break; - case ifle: - c->jle(target); - break; - } - - compile(t, frame, newIp); - if (UNLIKELY(t->exception)) return; - } break; - - case ifnull: - case ifnonnull: { - uint32_t newIp = (ip - 3) + codeReadInt16(t, code, ip); - assert(t, newIp < codeLength(t, code)); - - c->cmp(0, frame->popObject()); - - Operand* target = c->logicalIp(newIp); - if (instruction == ifnull) { - c->je(target); - } else { - c->jne(target); - } - - compile(t, frame, newIp); - if (UNLIKELY(t->exception)) return; - } break; - - case iinc: { - uint8_t index = codeBody(t, code, ip++); - int8_t count = codeBody(t, code, ip++); - - frame->increment(index, count); - } break; - - case iload: - case fload: - frame->loadInt(codeBody(t, code, ip++)); - break; - - case iload_0: - case fload_0: - frame->loadInt(0); - break; - - case iload_1: - case fload_1: - frame->loadInt(1); - break; - - case iload_2: - case fload_2: - frame->loadInt(2); - break; - - case iload_3: - case fload_3: - frame->loadInt(3); - break; - - case imul: { - Operand* a = frame->popInt(); - c->mul(a, frame->topInt()); - } break; - - case instanceof: { - uint16_t index = codeReadInt16(t, code, ip); - - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - - Operand* call = c->label(); - Operand* next = c->label(); - Operand* zero = c->label(); - - Operand* instance = frame->topObject(); - Operand* tmp = c->temporary(); - Operand* result = c->temporary(); - - c->mov(instance, tmp); - - c->cmp(c->constant(0), tmp); - c->je(zero); - - Operand* classOperand = frame->append(class_); - - c->mov(c->memory(tmp), tmp); - c->and_(c->constant(PointerMask), tmp); - - c->cmp(classOperand, tmp); - c->jne(call); - - c->mov(c->constant(1), result); - c->jmp(next); - - c->mov - (c->directCall - (c->constant(reinterpret_cast(isAssignableFrom)), - 2, classOperand, tmp), result); - - c->release(tmp); - - c->jmp(next); - - c->mark(zero); - - c->mov(c->constant(0), result); - - c->mark(next); - frame->pushInt(result); - - c->release(result); - } break; - - case invokeinterface: { - uint16_t index = codeReadInt16(t, code, ip); - ip += 2; - - object target = resolveMethod(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - - unsigned parameterFootprint = methodParameterFootprint(t, target); - - unsigned instance = parameterFootprint - 1; - - Operand* found = c->directCall - (c->constant - (reinterpret_cast(findInterfaceMethodFromInstance)), - 3, c->thread(), frame->append(target), c->stack(instance)); - - c->mov(c->memory(found, MethodCompiled), found); - - Operand* result = c->call(c->memory(found, SingletonBody)); - - frame->trace(target, true); - - frame->pop(parameterFootprint); - - pushReturnValue(t, frame, methodReturnCode(t, target), result); - } break; - - case invokespecial: { - uint16_t index = codeReadInt16(t, code, ip); - - object target = resolveMethod(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - - object class_ = methodClass(t, target); - if (isSpecialMethod(t, target, class_)) { - target = findMethod(t, target, classSuper(t, class_)); - } - - compileDirectInvoke(t, frame, target); - } break; - - case invokestatic: { - uint16_t index = codeReadInt16(t, code, ip); - - object target = resolveMethod(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - PROTECT(t, target); - - initClass(t, methodClass(t, target)); - if (UNLIKELY(t->exception)) return; - - compileDirectInvoke(t, frame, target); - } break; - - case invokevirtual: { - uint16_t index = codeReadInt16(t, code, ip); - - object target = resolveMethod(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - - unsigned parameterFootprint = methodParameterFootprint(t, target); - - unsigned offset = ClassVtable + (methodOffset(t, target) * BytesPerWord); - - Operand* instance = c->stack(parameterFootprint - 1); - Operand* class_ = c->temporary(); - - c->mov(c->memory(instance), class_); - c->and_(c->constant(PointerMask), class_); - - Operand* result = c->call(c->memory(class_, offset)); - - frame->trace(target, true); - - c->release(class_); - - frame->pop(parameterFootprint); - - pushReturnValue(t, frame, methodReturnCode(t, target), result); - } break; - - case ior: { - Operand* a = frame->popInt(); - c->or_(a, frame->topInt()); - } break; - - case irem: { - Operand* a = frame->popInt(); - c->rem(a, frame->topInt()); - } break; - - case ireturn: - case freturn: - c->epilogue(); - c->return_(frame->popInt()); - return; - - case ishl: { - Operand* a = frame->popInt(); - c->shl(a, frame->topInt()); - } break; - - case ishr: { - Operand* a = frame->popInt(); - c->shr(a, frame->topInt()); - } break; - - case istore: - case fstore: - frame->storeInt(codeBody(t, code, ip++)); - break; - - case istore_0: - case fstore_0: - frame->storeInt(0); - break; - - case istore_1: - case fstore_1: - frame->storeInt(1); - break; - - case istore_2: - case fstore_2: - frame->storeInt(2); - break; - - case istore_3: - case fstore_3: - frame->storeInt(3); - break; - - case isub: { - Operand* a = frame->popInt(); - c->sub(a, frame->topInt()); - } break; - - case iushr: { - Operand* a = frame->popInt(); - c->ushr(a, frame->topInt()); - } break; - - case ixor: { - Operand* a = frame->popInt(); - c->xor_(a, frame->topInt()); - } break; - - case jsr: - case jsr_w: - case ret: - // see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4381996 - abort(t); - - case l2i: - frame->pushInt(frame->popLong()); - break; - - case ladd: { - Operand* a = frame->popLong(); - c->sub(a, frame->topLong()); - } break; - - case lcmp: { - Operand* next = c->label(); - Operand* less = c->label(); - Operand* greater = c->label(); - - Operand* a = frame->popLong(); - Operand* b = frame->popLong(); - Operand* result = c->temporary(); - - c->cmp(a, b); - c->jl(less); - c->jg(greater); - - c->mov(c->constant(0), result); - c->jmp(next); - - c->mark(less); - c->mov(c->constant(-1), result); - c->jmp(next); - - c->mark(greater); - c->mov(c->constant(1), result); - - c->mark(next); - frame->pushInt(result); - - c->release(result); - } break; - - case lconst_0: - frame->pushLong(c->constant(0)); - break; - - case lconst_1: - frame->pushLong(c->constant(1)); - break; - - case ldc: - case ldc_w: { - uint16_t index; - - if (instruction == ldc) { - index = codeBody(t, code, ip++); - } else { - index = codeReadInt16(t, code, ip); - } - - object pool = codePool(t, code); - - if (singletonIsObject(t, pool, index - 1)) { - object v = singletonObject(t, pool, index - 1); - if (objectClass(t, v) - == arrayBody(t, t->m->types, Machine::ByteArrayType)) - { - object class_ = resolveClassInPool(t, pool, index - 1); - if (UNLIKELY(t->exception)) return; - - frame->pushObject(frame->append(class_)); - } else { - frame->pushObject(frame->append(v)); - } - } else { - frame->pushInt(c->constant(singletonValue(t, pool, index - 1))); - } - } break; - - case ldc2_w: { - uint16_t index = codeReadInt16(t, code, ip); - - object pool = codePool(t, code); - - uint64_t v; - memcpy(&v, &singletonValue(t, pool, index - 1), 8); - frame->pushLong(c->constant(v)); - } break; - - case ldiv_: { - Operand* a = frame->popLong(); - c->div(a, frame->topLong()); - } break; - - case lload: - case dload: - frame->loadLong(codeBody(t, code, ip++)); - break; - - case lload_0: - case dload_0: - frame->loadLong(0); - break; - - case lload_1: - case dload_1: - frame->loadLong(1); - break; - - case lload_2: - case dload_2: - frame->loadLong(2); - break; - - case lload_3: - case dload_3: - frame->loadLong(3); - break; - - case lmul: { - Operand* a = frame->popLong(); - c->mul(a, frame->topLong()); - } break; - - case lneg: - c->neg(frame->topLong()); - break; - - case lookupswitch: { - int32_t base = ip - 1; - - ip = (ip + 3) & ~3; // pad to four byte boundary - - Operand* key = frame->popInt(); - - uint32_t defaultIp = base + codeReadInt32(t, code, ip); - assert(t, defaultIp < codeLength(t, code)); - - compile(t, frame, defaultIp); - if (UNLIKELY(t->exception)) return; - - Operand* default_ = c->poolAppend(c->logicalIp(defaultIp)); - - int32_t pairCount = codeReadInt32(t, code, ip); - - Operand* start; - for (int32_t i = 0; i < pairCount; ++i) { - unsigned index = ip + (i * 8); - int32_t key = codeReadInt32(t, code, index); - uint32_t newIp = base + codeReadInt32(t, code, index); - assert(t, newIp < codeLength(t, code)); - - compile(t, frame, newIp); - if (UNLIKELY(t->exception)) return; - - Operand* result = c->poolAppend(c->constant(key)); - c->poolAppend(c->logicalIp(newIp)); - - if (i == 0) { - start = result; - } - } - - c->jmp - (c->directCall - (c->constant(reinterpret_cast(lookUpAddress)), - 4, key, start, c->constant(pairCount), default_)); - } return; - - case lor: { - Operand* a = frame->popLong(); - c->or_(a, frame->topLong()); - } break; - - case lrem: { - Operand* a = frame->popLong(); - c->rem(a, frame->topLong()); - } break; - - case lreturn: - case dreturn: - c->epilogue(); - c->return_(frame->popLong()); - return; - - case lshl: { - Operand* a = frame->popLong(); - c->shl(a, frame->topLong()); - } break; - - case lshr: { - Operand* a = frame->popLong(); - c->shr(a, frame->topLong()); - } break; - - case lstore: - case dstore: - frame->storeLong(codeBody(t, code, ip++)); - break; - - case lstore_0: - case dstore_0: - frame->storeLong(0); - break; - - case lstore_1: - case dstore_1: - frame->storeLong(1); - break; - - case lstore_2: - case dstore_2: - frame->storeLong(2); - break; - - case lstore_3: - case dstore_3: - frame->storeLong(3); - break; - - case lsub: { - Operand* a = frame->popLong(); - c->sub(a, frame->topLong()); - } break; - - case lushr: { - Operand* a = frame->popLong(); - c->ushr(a, frame->topLong()); - } break; - - case lxor: { - Operand* a = frame->popLong(); - c->xor_(a, frame->topLong()); - } break; - - case monitorenter: { - c->indirectCall - (c->constant(reinterpret_cast(acquireMonitorForObject)), - 2, c->thread(), frame->popObject()); - - frame->trace(); - } break; - - case monitorexit: { - c->indirectCall - (c->constant(reinterpret_cast(releaseMonitorForObject)), - 2, c->thread(), frame->popObject()); - - frame->trace(); - } break; - - case multianewarray: { - uint16_t index = codeReadInt16(t, code, ip); - uint8_t dimensions = codeBody(t, code, ip++); - - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - PROTECT(t, class_); - - Operand* result = c->indirectCall - (c->constant(reinterpret_cast(makeMultidimensionalArray)), - 4, c->thread(), frame->append(class_), c->stack(), - c->constant(dimensions)); - - frame->trace(); - - frame->pop(dimensions); - frame->pushObject(result); - } break; - - case new_: { - uint16_t index = codeReadInt16(t, code, ip); - - object class_ = resolveClassInPool(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - PROTECT(t, class_); - - initClass(t, class_); - if (UNLIKELY(t->exception)) return; - - Operand* result; - if (classVmFlags(t, class_) & WeakReferenceFlag) { - result = c->indirectCall - (c->constant(reinterpret_cast(makeNewWeakReference)), - 2, c->thread(), frame->append(class_)); - } else { - result = c->indirectCall - (c->constant(reinterpret_cast(makeNew)), - 2, c->thread(), frame->append(class_)); - } - - frame->trace(); - - frame->pushObject(result); - } break; - - case newarray: { - uint8_t type = codeBody(t, code, ip++); - - Operand* nonnegative = c->label(); - - Operand* size = frame->popInt(); - c->cmp(0, size); - c->jge(nonnegative); - - compileThrowNew(t, frame, Machine::NegativeArraySizeExceptionType); - - c->mark(nonnegative); - - object (*constructor)(Thread*, uintptr_t, bool); - switch (type) { - case T_BOOLEAN: - constructor = makeBooleanArray; - break; - - case T_CHAR: - constructor = makeCharArray; - break; - - case T_FLOAT: - constructor = makeFloatArray; - break; - - case T_DOUBLE: - constructor = makeDoubleArray; - break; - - case T_BYTE: - constructor = makeByteArray; - break; - - case T_SHORT: - constructor = makeShortArray; - break; - - case T_INT: - constructor = makeIntArray; - break; - - case T_LONG: - constructor = makeLongArray; - break; - - default: abort(t); - } - - Operand* result = c->indirectCall - (c->constant(reinterpret_cast(makeBlankArray)), - 2, c->constant(reinterpret_cast(constructor)), size); - - frame->trace(); - - frame->pushObject(result); - } break; - - case nop: break; - - case pop_: - frame->pop(1); - break; - - case pop2: - frame->pop(2); - break; - - case putfield: - case putstatic: { - uint16_t index = codeReadInt16(t, code, ip); - - object field = resolveField(t, codePool(t, code), index - 1); - if (UNLIKELY(t->exception)) return; - - object staticTable; - - if (instruction == putstatic) { - PROTECT(t, field); - initClass(t, fieldClass(t, field)); - if (UNLIKELY(t->exception)) return; - - staticTable = classStaticTable(t, fieldClass(t, field)); - } - - Operand* value; - switch (fieldCode(t, field)) { - case ByteField: - case BooleanField: - case CharField: - case ShortField: - case FloatField: - case IntField: { - value = frame->popInt(); - } break; - - case DoubleField: - case LongField: { - value = frame->popLong(); - } break; - - case ObjectField: { - value = frame->popLong(); - } break; - - default: abort(t); - } - - Operand* table; - - if (instruction == putstatic) { - table = frame->append(staticTable); - } else { - table = frame->popObject(); - } - - switch (fieldCode(t, field)) { - case ByteField: - case BooleanField: - c->mov(value, c->select1(c->memory(table, fieldOffset(t, field)))); - break; - - case CharField: - case ShortField: - c->mov(value, c->select2(c->memory(table, fieldOffset(t, field)))); - break; - - case FloatField: - case IntField: - c->mov(value, c->select4(c->memory(table, fieldOffset(t, field)))); - break; - - case DoubleField: - case LongField: - c->mov(value, c->select8(c->memory(table, fieldOffset(t, field)))); - break; - - case ObjectField: - c->directCall - (c->constant(reinterpret_cast(set)), - 4, c->thread(), table, fieldOffset(t, field), value); - break; - - default: abort(t); - } - } break; - - case return_: - c->epilogue(); - c->ret(); - return; - - case sipush: - frame->pushInt - (c->constant(static_cast(codeReadInt16(t, code, ip)))); - break; - - case swap: - frame->swap(); - break; - - case tableswitch: { - int32_t base = ip - 1; - - ip = (ip + 3) & ~3; // pad to four byte boundary - - Operand* key = frame->popInt(); - - uint32_t defaultIp = base + codeReadInt32(t, code, ip); - assert(t, defaultIp < codeLength(t, code)); - - compile(t, frame, defaultIp); - if (UNLIKELY(t->exception)) return; - - Operand* default_ = c->poolAppend(c->logicalIp(defaultIp)); - - int32_t bottom = codeReadInt32(t, code, ip); - int32_t top = codeReadInt32(t, code, ip); - - Operand* start; - for (int32_t i = 0; i < bottom - top + 1; ++i) { - unsigned index = ip + (i * 4); - uint32_t newIp = base + codeReadInt32(t, code, index); - assert(t, newIp < codeLength(t, code)); - - compile(t, frame, newIp); - if (UNLIKELY(t->exception)) return; - - Operand* result = c->poolAppend(c->logicalIp(newIp)); - if (i == 0) { - start = result; - } - } - - Operand* defaultCase = c->label(); - - c->cmp(c->constant(bottom), key); - c->jl(defaultCase); - - c->cmp(c->constant(top), key); - c->jg(defaultCase); - - c->shl(c->constant(1), key); - c->jmp(c->memory(start, 0, key, BytesPerWord)); - - c->mark(defaultCase); - c->jmp(default_); - } return; - - case wide: { - switch (codeBody(t, code, ip++)) { - case aload: { - frame->loadObject(codeReadInt16(t, code, ip)); - } break; - - case astore: { - frame->storeObject(codeReadInt16(t, code, ip)); - } break; - - case iinc: { - uint16_t index = codeReadInt16(t, code, ip); - uint16_t count = codeReadInt16(t, code, ip); - - frame->increment(index, count); - } break; - - case iload: { - frame->loadInt(codeReadInt16(t, code, ip)); - } break; - - case istore: { - frame->storeInt(codeReadInt16(t, code, ip)); - } break; - - case lload: { - frame->loadLong(codeReadInt16(t, code, ip)); - } break; - - case lstore: { - frame->storeLong(codeReadInt16(t, code, ip)); - } break; - - case ret: - // see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4381996 - abort(t); - - default: abort(t); - } - } break; - } - } -} - -object -finish(MyThread* t, Compiler* c, object method, Buffer* objectPool, - Buffer* traceLog) -{ - unsigned count = ceiling(c->size(), BytesPerWord); - unsigned size = count + singletonMaskSize(count); - object result = allocate2(t, size * BytesPerWord, true, true); - initSingleton(t, result, size, true); - singletonMask(t, result)[0] = 1; - - uint8_t* start = reinterpret_cast(&singletonValue(t, result, 0)); - - c->writeTo(start); - - if (method) { - PROTECT(t, method); - - for (unsigned i = 0; i < objectPool->length(); i += BytesPerWord * 2) { - Promise* offset = reinterpret_cast - (objectPool->getAddress(i)); - - object value = reinterpret_cast - (objectPool->getAddress(i + BytesPerWord)); - - singletonMarkObject(t, result, offset->value() / BytesPerWord); - set(t, result, SingletonBody + offset->value(), value); - } - - unsigned traceSize = Frame::traceSizeInBytes(t, method); - unsigned mapSize = Frame::mapSizeInBytes(t, method); - for (unsigned i = 0; i < traceLog->length(); i += traceSize) { - object target = reinterpret_cast - (traceLog->getAddress(i)); - - Promise* offset = reinterpret_cast - (traceLog->getAddress(i + BytesPerWord)); - - bool virtualCall = traceLog->get(i + BytesPerWord + BytesPerWord); - - uintptr_t map[mapSize / BytesPerWord]; - traceLog->get(i + BytesPerWord + 2 + 1 + 4, map, mapSize); - - object node = makeTraceNode - (t, reinterpret_cast(start + offset->value()), 0, 0, 0, - method, target, virtualCall, mapSize / BytesPerWord, false); - - if (mapSize) { - memcpy(&traceNodeMap(t, node, 0), map, mapSize); - } - - insertTraceNode(t, node); - } - - object code = methodCode(t, method); - PROTECT(t, code); - - { - object oldTable = codeExceptionHandlerTable(t, code); - if (oldTable) { - PROTECT(t, oldTable); - - unsigned length = exceptionHandlerTableLength(t, oldTable); - object newTable = makeExceptionHandlerTable(t, length, false); - for (unsigned i = 0; i < length; ++i) { - ExceptionHandler* oldHandler = exceptionHandlerTableBody - (t, oldTable, i); - ExceptionHandler* newHandler = exceptionHandlerTableBody - (t, newTable, i); - - exceptionHandlerStart(newHandler) - = c->logicalIpToOffset(exceptionHandlerStart(oldHandler)); - - exceptionHandlerEnd(newHandler) - = c->logicalIpToOffset(exceptionHandlerEnd(oldHandler)); - - exceptionHandlerIp(newHandler) - = c->logicalIpToOffset(exceptionHandlerIp(oldHandler)); - - exceptionHandlerCatchType(newHandler) - = exceptionHandlerCatchType(oldHandler); - } - - set(t, code, CodeExceptionHandlerTable, newTable); - } - } - - { - object oldTable = codeLineNumberTable(t, code); - if (oldTable) { - PROTECT(t, oldTable); - - unsigned length = lineNumberTableLength(t, oldTable); - object newTable = makeLineNumberTable(t, length, false); - for (unsigned i = 0; i < length; ++i) { - LineNumber* oldLine = lineNumberTableBody(t, oldTable, i); - LineNumber* newLine = lineNumberTableBody(t, newTable, i); - - lineNumberIp(newLine) = c->logicalIpToOffset(lineNumberIp(oldLine)); - - lineNumberLine(newLine) = lineNumberLine(oldLine); - } - - set(t, code, CodeLineNumberTable, newTable); - } - } - } - - return result; -} - -object -finish(MyThread* t, Compiler* c) -{ - return finish(t, c, 0, 0, 0); -} - -object -compile(MyThread* t, Compiler* c, object method) -{ - PROTECT(t, method); - - c->prologue(); - - object code = methodCode(t, method); - PROTECT(t, code); - - unsigned footprint = methodParameterFootprint(t, method); - unsigned locals = codeMaxLocals(t, code); - c->sub(c->constant((locals - footprint) * BytesPerWord), c->stack()); - - Buffer objectPool(t->m->system, 256); - Buffer traceLog(t->m->system, 1024); - uintptr_t map[Frame::mapSizeInWords(t, method)]; - Frame frame(t, c, method, map, &objectPool, &traceLog); - - compile(t, &frame, 0); - if (UNLIKELY(t->exception)) return 0; - - object eht = codeExceptionHandlerTable(t, methodCode(t, method)); - if (eht) { - PROTECT(t, eht); - - for (unsigned i = 0; i < exceptionHandlerTableLength(t, eht); ++i) { - ExceptionHandler* eh = exceptionHandlerTableBody(t, eht, i); - - assert(t, getBit(frame.codeMask, exceptionHandlerStart(eh))); - - uintptr_t map[Frame::mapSizeInWords(t, method)]; - Frame frame2(&frame, map); - frame2.pushedObject(); - - compile(t, &frame2, exceptionHandlerIp(eh)); - if (UNLIKELY(t->exception)) return 0; - } - } - - return finish(t, c, method, &objectPool, &traceLog); -} - -void -compile(MyThread* t, object method); - -void* -compileMethod(MyThread* t) -{ - object node = findTraceNode(t, *static_cast(t->stack)); - object target = resolveTarget(t, t->stack, traceNodeTarget(t, node)); - - if (LIKELY(t->exception == 0)) { - compile(t, target); - } - - if (UNLIKELY(t->exception)) { - unwind(t); - } else { - if (not traceNodeVirtualCall(t, node)) { - Compiler* c = makeCompiler(t->m->system, 0); - c->updateCall(reinterpret_cast(treeNodeKey(t, node)), - &singletonValue(t, methodCompiled(t, target), 0)); - c->dispose(); - } - return &singletonValue(t, methodCompiled(t, target), 0); - } -} - -uint64_t -invokeNative2(MyThread* t, object method) -{ - PROTECT(t, method); - - if (objectClass(t, methodCode(t, method)) - == arrayBody(t, t->m->types, Machine::ByteArrayType)) - { - void* function = resolveNativeMethod(t, method); - if (UNLIKELY(function == 0)) { - object message = makeString - (t, "%s", &byteArrayBody(t, methodCode(t, method), 0)); - t->exception = makeUnsatisfiedLinkError(t, message); - return 0; - } - - object p = makePointer(t, function); - set(t, method, MethodCode, p); - } - - object class_ = methodClass(t, method); - PROTECT(t, class_); - - unsigned footprint = methodParameterFootprint(t, method) + 1; - unsigned count = methodParameterCount(t, method) + 1; - if (methodFlags(t, method) & ACC_STATIC) { - ++ footprint; - ++ count; - } - - uintptr_t args[footprint]; - unsigned argOffset = 0; - uint8_t types[count]; - unsigned typeOffset = 0; - - args[argOffset++] = reinterpret_cast(t); - types[typeOffset++] = POINTER_TYPE; - - uintptr_t* sp = static_cast(t->stack) - + (methodParameterFootprint(t, method) + 1); - - if (methodFlags(t, method) & ACC_STATIC) { - args[argOffset++] = reinterpret_cast(&class_); - } else { - args[argOffset++] = reinterpret_cast(sp--); - } - types[typeOffset++] = POINTER_TYPE; - - MethodSpecIterator it - (t, reinterpret_cast - (&byteArrayBody(t, methodSpec(t, method), 0))); - - while (it.hasNext()) { - unsigned type = types[typeOffset++] - = fieldType(t, fieldCode(t, *it.next())); - - switch (type) { - case INT8_TYPE: - case INT16_TYPE: - case INT32_TYPE: - case FLOAT_TYPE: - args[argOffset++] = *(sp--); - break; - - case INT64_TYPE: - case DOUBLE_TYPE: { - if (BytesPerWord == 8) { - uint64_t a = *(sp--); - uint64_t b = *(sp--); - args[argOffset++] = (a << 32) | b; - } else { - memcpy(args + argOffset, sp, 8); - argOffset += 2; - sp -= 2; - } - } break; - - case POINTER_TYPE: { - args[argOffset++] = reinterpret_cast(sp--); - } break; - - default: abort(t); - } - } - - void* function = pointerValue(t, methodCode(t, method)); - unsigned returnType = fieldType(t, methodReturnCode(t, method)); - uint64_t result; - - if (Verbose) { - fprintf(stderr, "invoke native method %s.%s\n", - &byteArrayBody(t, className(t, methodClass(t, method)), 0), - &byteArrayBody(t, methodName(t, method), 0)); - } - - { ENTER(t, Thread::IdleState); - - result = t->m->system->call - (function, - args, - types, - count + 1, - footprint * BytesPerWord, - returnType); - } - - if (Verbose) { - fprintf(stderr, "return from native method %s.%s\n", - &byteArrayBody(t, className(t, methodClass(t, method)), 0), - &byteArrayBody(t, methodName(t, method), 0)); - } - - if (LIKELY(t->exception == 0) and returnType == POINTER_TYPE) { - return result ? *reinterpret_cast(result) : 0; - } else { - return result; - } -} - -uint64_t -invokeNative(MyThread* t) -{ - object node = findTraceNode(t, *static_cast(t->stack)); - object target = resolveTarget(t, t->stack, traceNodeTarget(t, node)); - uint64_t result; - - if (LIKELY(t->exception == 0)) { - result = invokeNative2(t, target); - } - - if (UNLIKELY(t->exception)) { - unwind(t); - } else { - return result; - } -} - -inline object* -localObject(MyThread* t, void* base, object method, unsigned index) -{ - return reinterpret_cast - (static_cast(base) + localOffset(t, index, method)); -} - -void -visitParameters(MyThread* t, Heap::Visitor* v, void* base, object method) -{ - const char* spec = reinterpret_cast - (&byteArrayBody(t, methodSpec(t, method), 0)); - - unsigned index = 0; - if ((methodFlags(t, method) & ACC_STATIC) == 0) { - v->visit(localObject(t, base, method, index++)); - } - - for (MethodSpecIterator it(t, spec); it.hasNext();) { - switch (*it.next()) { - case 'L': - case '[': - v->visit(localObject(t, base, method, index++)); - break; - - case 'J': - case 'D': - index += 2; - break; - - default: - ++ index; - break; - } - } - - assert(t, index == methodParameterFootprint(t, method)); -} - -void -visitStackAndLocals(MyThread* t, Heap::Visitor* v, void* base, object node) -{ - object method = traceNodeMethod(t, node); - - unsigned parameterFootprint = methodParameterFootprint(t, method); - unsigned count = codeMaxStack(t, methodCode(t, method)) - + codeMaxLocals(t, methodCode(t, method)) - - parameterFootprint; - - if (count) { - uintptr_t* map = &traceNodeMap(t, node, 0); - - for (unsigned i = 0; i < count; ++i) { - if (getBit(map, i)) { - v->visit(localObject(t, base, method, i + parameterFootprint)); - } - } - } -} - -void -visitStack(MyThread* t, Heap::Visitor* v) -{ - void* base = t->base; - void** stack = static_cast(t->stack); - MyThread::CallTrace* trace = t->trace; - - while (true) { - object node = findTraceNode(t, *stack); - if (node) { - object method = traceNodeMethod(t, node); - - // we only need to visit the parameters of this method if the - // caller is native. Otherwise, the caller owns them. - object next = findTraceNode(t, static_cast(base)[1]); - if (next == 0) { - visitParameters(t, v, base, method); - } - - visitStackAndLocals(t, v, base, method); - - stack = static_cast(base) + 1; - base = *static_cast(base); - } else if (trace) { - base = trace->base; - stack = static_cast(trace->stack); - trace = trace->next; - } else { - break; - } - } -} - -object -compileDefault(MyThread* t, Compiler* c) -{ - c->prologue(); - - c->mov(c->base(), c->memory(c->thread(), difference(&(t->base), t))); - c->mov(c->stack(), c->memory(c->thread(), difference(&(t->stack), t))); - - c->epilogue(); - - c->jmp - (c->directCall - (c->constant(reinterpret_cast(compileMethod)), - 1, c->thread())); - - return finish(t, c); -} - -object -compileNative(MyThread* t, Compiler* c) -{ - c->prologue(); - - c->mov(c->base(), c->memory(c->thread(), difference(&(t->base), t))); - c->mov(c->stack(), c->memory(c->thread(), difference(&(t->stack), t))); - - c->call - (c->directCall - (c->constant(reinterpret_cast(invokeNative)), - 1, c->thread())); - - c->epilogue(); - c->ret(); - - return finish(t, c); -} - -class ArgumentList { - public: - ArgumentList(Thread* t, uintptr_t* array, bool* objectMask, object this_, - const char* spec, bool indirectObjects, va_list arguments): - t(static_cast(t)), - array(array), - objectMask(objectMask), - position(0), - protector(this) - { - if (this_) { - addObject(this_); - } - - for (MethodSpecIterator it(t, spec); it.hasNext();) { - switch (*it.next()) { - case 'L': - case '[': - if (indirectObjects) { - object* v = va_arg(arguments, object*); - addObject(v ? *v : 0); - } else { - addObject(va_arg(arguments, object)); - } - break; - - case 'J': - case 'D': - addLong(va_arg(arguments, uint64_t)); - break; - - default: - addInt(va_arg(arguments, uint32_t)); - break; - } - } - } - - ArgumentList(Thread* t, uintptr_t* array, bool* objectMask, object this_, - const char* spec, object arguments): - t(static_cast(t)), - array(array), - objectMask(objectMask), - position(0), - protector(this) - { - if (this_) { - addObject(this_); - } - - unsigned index = 0; - for (MethodSpecIterator it(t, spec); it.hasNext();) { - switch (*it.next()) { - case 'L': - case '[': - addObject(objectArrayBody(t, arguments, index++)); - break; - - case 'J': - case 'D': - addLong(cast(objectArrayBody(t, arguments, index++), - BytesPerWord)); - break; - - default: - addInt(cast(objectArrayBody(t, arguments, index++), - BytesPerWord)); - break; - } - } - } - - void addObject(object v) { - array[position] = reinterpret_cast(v); - objectMask[position] = true; - ++ position; - } - - void addInt(uintptr_t v) { - array[position] = v; - objectMask[position] = false; - ++ position; - } - - void addLong(uint64_t v) { - memcpy(array + position, &v, 8); - objectMask[position] = false; - objectMask[position] = false; - position += 2; - } - - MyThread* t; - uintptr_t* array; - bool* objectMask; - unsigned position; - - class MyProtector: public Thread::Protector { - public: - MyProtector(ArgumentList* list): Protector(list->t), list(list) { } - - virtual void visit(Heap::Visitor* v) { - for (unsigned i = 0; i < list->position; ++i) { - if (list->objectMask[i]) { - v->visit(reinterpret_cast(list->array + i)); - } - } - } - - ArgumentList* list; - } protector; -}; - -object -invoke(Thread* thread, object method, ArgumentList* arguments) -{ - MyThread* t = static_cast(thread); - - unsigned returnCode = methodReturnCode(t, method); - unsigned returnType = fieldType(t, returnCode); - - Reference* reference = t->reference; - MyThread::CallTrace trace(t); - - uint64_t result = vmInvoke - (t, &singletonValue(t, methodCompiled(t, method), 0), arguments->array, - arguments->position * BytesPerWord, returnType); - - while (t->reference != reference) { - dispose(t, t->reference); - } - - object r; - switch (returnCode) { - case ByteField: - case BooleanField: - case CharField: - case ShortField: - case FloatField: - case IntField: - r = makeInt(t, result); - break; - - case LongField: - case DoubleField: - r = makeLong(t, result); - break; - - case ObjectField: - r = (result == 0 ? 0 : - *reinterpret_cast(static_cast(result))); - break; - - case VoidField: - r = 0; - break; - - default: - abort(t); - }; - - return r; -} - -class MyProcessor: public Processor { - public: - MyProcessor(System* s): - s(s), - defaultCompiled(0), - nativeCompiled(0), - addressTree(0), - addressTreeSentinal(0), - indirectCaller(0) - { } - - virtual Thread* - makeThread(Machine* m, object javaThread, Thread* parent) - { - MyThread* t = new (s->allocate(sizeof(Thread))) - MyThread(m, javaThread, parent); - t->init(); - return t; - } - - object getDefaultCompiled(MyThread* t) { - if (defaultCompiled == 0) { - Compiler* c = makeCompiler(t->m->system, 0); - defaultCompiled = compileDefault(t, c); - c->dispose(); - } - return defaultCompiled; - } - - object getNativeCompiled(MyThread* t) { - if (nativeCompiled == 0) { - Compiler* c = makeCompiler(t->m->system, 0); - nativeCompiled = compileNative(t, c); - c->dispose(); - } - return nativeCompiled; - } - - virtual object - makeMethod(vm::Thread* t, - uint8_t vmFlags, - uint8_t returnCode, - uint8_t parameterCount, - uint8_t parameterFootprint, - uint16_t flags, - uint16_t offset, - object name, - object spec, - object class_, - object code) - { - object compiled - = ((flags & ACC_NATIVE) - ? getNativeCompiled(static_cast(t)) - : getDefaultCompiled(static_cast(t))); - - return vm::makeMethod - (t, vmFlags, returnCode, parameterCount, parameterFootprint, flags, - offset, name, spec, class_, code, compiled); - } - - virtual object - makeClass(vm::Thread* t, - uint16_t flags, - uint8_t vmFlags, - uint8_t arrayDimensions, - uint16_t fixedSize, - uint16_t arrayElementSize, - object objectMask, - object name, - object super, - object interfaceTable, - object virtualTable, - object fieldTable, - object methodTable, - object staticTable, - object loader, - unsigned vtableLength) - { - object c = vm::makeClass - (t, flags, vmFlags, arrayDimensions, fixedSize, arrayElementSize, - objectMask, name, super, interfaceTable, virtualTable, fieldTable, - methodTable, staticTable, loader, vtableLength, false); - - for (unsigned i = 0; i < vtableLength; ++i) { - object compiled - = ((flags & ACC_NATIVE) - ? getNativeCompiled(static_cast(t)) - : getDefaultCompiled(static_cast(t))); - - classVtable(t, c, i) = &singletonBody(t, compiled, 0); - } - - return c; - } - - virtual void - initClass(Thread* t, object c) - { - PROTECT(t, c); - - ACQUIRE(t, t->m->classLock); - if (classVmFlags(t, c) & NeedInitFlag - and (classVmFlags(t, c) & InitFlag) == 0) - { - classVmFlags(t, c) |= InitFlag; - invoke(t, classInitializer(t, c), 0); - if (t->exception) { - t->exception = makeExceptionInInitializerError(t, t->exception); - } - classVmFlags(t, c) &= ~(NeedInitFlag | InitFlag); - } - } - - virtual void - visitObjects(Thread* vmt, Heap::Visitor* v) - { - MyThread* t = static_cast(vmt); - - if (t == t->m->rootThread) { - v->visit(&defaultCompiled); - v->visit(&nativeCompiled); - v->visit(&addressTree); - } - - for (Reference* r = t->reference; r; r = r->next) { - v->visit(&(r->target)); - } - - visitStack(t, v); - } - - virtual void - walkStack(Thread* vmt, StackVisitor* v) - { - MyThread* t = static_cast(vmt); - - MyStackWalker walker(t); - walker.walk(v); - } - - virtual int - lineNumber(Thread* vmt, object method, int ip) - { - return findLineNumber(static_cast(vmt), method, ip); - } - - virtual object* - makeLocalReference(Thread* vmt, object o) - { - if (o) { - MyThread* t = static_cast(vmt); - - Reference* r = new (t->m->system->allocate(sizeof(Reference))) - Reference(o, &(t->reference)); - - return &(r->target); - } else { - return 0; - } - } - - virtual void - disposeLocalReference(Thread* t, object* r) - { - if (r) { - vm::dispose(t, reinterpret_cast(r)); - } - } - - virtual object - invokeArray(Thread* t, object method, object this_, object arguments) - { - assert(t, t->state == Thread::ActiveState - or t->state == Thread::ExclusiveState); - - assert(t, ((methodFlags(t, method) & ACC_STATIC) == 0) xor (this_ == 0)); - - const char* spec = reinterpret_cast - (&byteArrayBody(t, methodSpec(t, method), 0)); - - unsigned size = methodParameterFootprint(t, method); - uintptr_t array[size]; - bool objectMask[size]; - ArgumentList list(t, array, objectMask, this_, spec, arguments); - - PROTECT(t, method); - - compile(static_cast(t), method); - - if (LIKELY(t->exception == 0)) { - return ::invoke(t, method, &list); - } - - return 0; - } - - virtual object - invokeList(Thread* t, object method, object this_, bool indirectObjects, - va_list arguments) - { - assert(t, t->state == Thread::ActiveState - or t->state == Thread::ExclusiveState); - - assert(t, ((methodFlags(t, method) & ACC_STATIC) == 0) xor (this_ == 0)); - - const char* spec = reinterpret_cast - (&byteArrayBody(t, methodSpec(t, method), 0)); - - unsigned size = methodParameterFootprint(t, method); - uintptr_t array[size]; - bool objectMask[size]; - ArgumentList list - (t, array, objectMask, this_, spec, indirectObjects, arguments); - - PROTECT(t, method); - - compile(static_cast(t), method); - - if (LIKELY(t->exception == 0)) { - return ::invoke(t, method, &list); - } - - return 0; - } - - virtual object - invokeList(Thread* t, const char* className, const char* methodName, - const char* methodSpec, object this_, va_list arguments) - { - assert(t, t->state == Thread::ActiveState - or t->state == Thread::ExclusiveState); - - unsigned size = parameterFootprint(t, methodSpec, false); - uintptr_t array[size]; - bool objectMask[size]; - ArgumentList list - (t, array, objectMask, this_, methodSpec, false, arguments); - - object method = resolveMethod(t, className, methodName, methodSpec); - if (LIKELY(t->exception == 0)) { - assert(t, ((methodFlags(t, method) & ACC_STATIC) == 0) xor (this_ == 0)); - - PROTECT(t, method); - - compile(static_cast(t), method); - - if (LIKELY(t->exception == 0)) { - return ::invoke(t, method, &list); - } - } - - return 0; - } - - virtual void dispose() { - if (indirectCaller) { - s->free(indirectCaller); - } - - s->free(this); - } - - System* s; - object defaultCompiled; - object nativeCompiled; - object addressTree; - object addressTreeSentinal; - void* indirectCaller; -}; - -MyProcessor* -processor(MyThread* t) -{ - MyProcessor* p = static_cast(t->m->processor); - if (p->addressTree == 0) { - ACQUIRE(t, t->m->classLock); - - if (p->addressTree == 0) { - p->addressTreeSentinal = makeTraceNode(t, 0, 0, 0, 0, 0, 0, 0, 0, false); - set(t, p->addressTreeSentinal, TreeNodeLeft, p->addressTreeSentinal); - set(t, p->addressTreeSentinal, TreeNodeRight, p->addressTreeSentinal); - p->addressTree = p->addressTreeSentinal; - - Compiler* c = makeCompiler(t->m->system, 0); - - c->mov(c->base(), c->memory(c->thread(), difference(&(t->base), t))); - c->mov(c->stack(), c->memory(c->thread(), difference(&(t->stack), t))); - - c->jmp(c->indirectTarget()); - - p->indirectCaller = t->m->system->allocate(c->size()); - c->writeTo(p->indirectCaller); - } - } - return p; -} - -void -compile(MyThread* t, object method) -{ - MyProcessor* p = processor(t); - - object stub = p->getDefaultCompiled(t); - - if (methodCompiled(t, method) == stub) { - PROTECT(t, method); - - ACQUIRE(t, t->m->classLock); - - if (methodCompiled(t, method) == stub) { - PROTECT(t, method); - - Compiler* c = makeCompiler(t->m->system, p->indirectCaller); - - object compiled = compile(t, c, method); - set(t, method, MethodCompiled, compiled); - - c->dispose(); - } - } -} - -object -findTraceNode(MyThread* t, void* address) -{ - MyProcessor* p = processor(t); - return treeQuery(t, p->addressTree, reinterpret_cast(address), - p->addressTreeSentinal); -} - -void -insertTraceNode(MyThread* t, object node) -{ - MyProcessor* p = processor(t); - p->addressTree = treeInsert - (t, p->addressTree, node, p->addressTreeSentinal); -} - -} // namespace - -namespace vm { - -Processor* -makeProcessor(System* system) -{ - return new (system->allocate(sizeof(MyProcessor))) MyProcessor(system); -} - -} // namespace vm diff --git a/src/compiler.cpp b/src/compiler.cpp index 3ad0e933b5..c6d2d2981d 100644 --- a/src/compiler.cpp +++ b/src/compiler.cpp @@ -5,6 +5,25 @@ using namespace vm; namespace { +enum Register { + rax = 0, + rcx = 1, + rdx = 2, + rbx = 3, + rsp = 4, + rbp = 5, + rsi = 6, + rdi = 7, + r8 = 8, + r9 = 9, + r10 = 10, + r11 = 11, + r12 = 12, + r13 = 13, + r14 = 14, + r15 = 15, +}; + class IpMapping { public: IpMapping(unsigned ip, unsigned offset): ip(ip), offset(offset) { } @@ -24,7 +43,8 @@ class Context { constantPool(s, BytesPerWord * 32), registerPool(s, BytesPerWord * 8), promises(s, 1024), - indirectCaller(reinterpret_cast(indirectCaller)) + indirectCaller(reinterpret_cast(indirectCaller)), + stackIndex(- BytesPerWord) { } void dispose() { @@ -39,13 +59,14 @@ class Context { System* s; Vector code; - Vector virtualStack; + Vector virtualStack; Vector operands; - Vector ipTable; - Vector constantPool; - Vector registerPool; + Vector ipTable; + Vector constantPool; + Vector registerPool; Vector promises; - intptr_t indirectCaller; + intptr_t indirectCaller; + int stackIndex; }; inline void NO_RETURN @@ -65,7 +86,7 @@ assert(Context* c, bool v) inline void expect(Context* c, bool v) { - expect(c->system, v); + expect(c->s, v); } class MyPromise: public Promise { @@ -78,7 +99,7 @@ class MyPromise: public Promise { MyPromise(intptr_t value): resolved(false), value_(value) { } - virtual unsigned value(System* s) { + virtual unsigned value(System* s UNUSED) { assert(s, resolved); return value_; } @@ -100,7 +121,7 @@ class PoolPromise: public MyPromise { class CodePromise: public MyPromise { public: - PoolPromise(intptr_t value): MyPromise(value) { } + CodePromise(intptr_t value): MyPromise(value) { } virtual PromiseType type() { return CodePromiseType; @@ -109,7 +130,7 @@ class CodePromise: public MyPromise { class IpPromise: public MyPromise { public: - PoolPromise(intptr_t value): MyPromise(value) { } + IpPromise(intptr_t value): MyPromise(value) { } virtual PromiseType type() { return IpPromiseType; @@ -120,8 +141,10 @@ class MyOperand: public Operand { public: enum OperandType { ImmediateOperandType, + AbsoluteOperandType, RegisterOperandType, - MemoryOperandType + MemoryOperandType, + SelectionOperandType }; virtual ~MyOperand() { } @@ -136,10 +159,16 @@ class MyOperand: public Operand { virtual void push(Context* c) { abort(c); } + virtual void pop(Context* c) { abort(c); } + virtual void mov(Context* c, MyOperand*) { abort(c); } virtual void cmp(Context* c, MyOperand*) { abort(c); } + virtual void call(Context* c) { abort(c); } + + virtual void alignedCall(Context* c) { abort(c); } + virtual void jl(Context* c) { abort(c); } virtual void jg(Context* c) { abort(c); } @@ -179,6 +208,141 @@ class MyOperand: public Operand { virtual void neg(Context* c) { abort(c); } }; +inline bool +isInt8(intptr_t v) +{ + return v == static_cast(v); +} + +inline bool +isInt32(intptr_t v) +{ + return v == static_cast(v); +} + +Register +registerValue(Context* c, MyOperand* v); + +Promise* +absoluteValue(Context* c, MyOperand* v); + +void +setAbsoluteValue(Context* c, MyOperand* v, Promise* value); + +MyOperand* +memoryBase(Context* c, MyOperand* v); + +int +memoryDisplacement(Context* c, MyOperand* v); + +MyOperand* +memoryIndex(Context* c, MyOperand* v); + +unsigned +memoryScale(Context* c, MyOperand* v); + +Register +asRegister(Context* c, MyOperand* v); + +void +rex(Context* c) +{ + if (BytesPerWord == 8) { + c->code.append(0x48); + } +} + +void +ret(Context* c) +{ + c->code.append(0xc3); +} + +void +encode(Context* c, uint8_t instruction, uint8_t zeroPrefix, + uint8_t bytePrefix, uint8_t wordPrefix, + Register a, Register b, int32_t offset) +{ + c->code.append(instruction); + + uint8_t prefix; + if (offset == 0 and b != rbp) { + prefix = zeroPrefix; + } else if (isInt8(offset)) { + prefix = bytePrefix; + } else { + prefix = wordPrefix; + } + + c->code.append(prefix | (a << 3) | b); + + if (b == rsp) { + c->code.append(0x24); + } + + if (offset == 0 and b != rbp) { + // do nothing + } else if (isInt8(offset)) { + c->code.append(offset); + } else { + c->code.append4(offset); + } +} + +class RegisterOperand: public MyOperand { + public: + RegisterOperand(Register value): + value(value) + { } + + virtual OperandType type() { + return RegisterOperandType; + } + + virtual unsigned size() { + return sizeof(RegisterOperand); + } + + virtual void push(Context* c) { + c->code.append(0x50 | value); + } + + virtual void mov(Context* c, MyOperand* dst) { + switch (dst->type()) { + case RegisterOperandType: + if (value != registerValue(c, dst)) { + rex(c); + c->code.append(0x89); + c->code.append(0xc0 | (value << 3) | registerValue(c, dst)); + } + break; + + case MemoryOperandType: + rex(c); + encode(c, 0x89, 0, 0x40, 0x80, value, + asRegister(c, memoryBase(c, dst)), + memoryDisplacement(c, dst)); + break; + + default: abort(c); + } + } + + Register value; +}; + +RegisterOperand* +temporary(Context* c) +{ + return c->registerPool.pop(); +} + +void +release(Context* c, RegisterOperand* v) +{ + c->registerPool.push(v); +} + class ImmediateOperand: public MyOperand { public: ImmediateOperand(intptr_t value): @@ -219,9 +383,9 @@ class ImmediateOperand: public MyOperand { case MemoryOperandType: rex(c); encode(c, 0xc7, 0, 0x40, 0x80, rax, - asRegister(memoryBase(c, dst)), + asRegister(c, memoryBase(c, dst)), memoryDisplacement(c, dst)); - c->code.append4(v); + c->code.append4(value); break; default: abort(c); @@ -248,98 +412,13 @@ class AbsoluteOperand: public MyOperand { Promise* value; }; -class RegisterOperand: public MyOperand { - public: - RegisterOperand(Register value): - value(value) - { } - - virtual OperandType type() { - return RegisterOperandType; - } - - virtual unsigned size() { - return sizeof(RegisterOperand); - } - - virtual void push(Context* c) { - c->code.append(0x50 | value); - } - - virtual void mov(Context* c, MyOperand* dst) { - switch (dst->type()) { - case RegisterOperandType: - if (value != registerValue(c, dst)) { - rex(c); - c->code.append(0x89); - c->code.append(0xc0 | (value << 3) | registerValue(c, dst)); - } - break; - - case MemoryOperandType: - rex(c); - encode(c, 0x89, 0, 0x40, 0x80, src, - asRegister(memoryBase(c, dst)), - memoryDisplacement(c, dst)); - break; - - default: abort(c); - } - } - - Register value; -}; - -RegisterOperand* -temporary(Context* c) -{ - return c->registerPool.pop(); -} - -void -release(Context* c, RegisterOperand* v) -{ - return c->registerPool.push(v); -} - -void -encode(Context* c, uint8_t instruction, uint8_t zeroPrefix, - uint8_t bytePrefix, uint8_t wordPrefix, - Register a, Register b, int32_t offset) -{ - c->code.append(instruction); - - uint8_t prefix; - if (offset == 0 and b != rbp) { - prefix = zeroPrefix; - } else if (isByte(offset)) { - prefix = bytePrefix; - } else { - prefix = wordPrefix; - } - - c->code.append(prefix | (a << 3) | b); - - if (b == rsp) { - c->code.append(0x24); - } - - if (offset == 0 and b != rbp) { - // do nothing - } else if (isInt8(offset)) { - c->code.append(offset); - } else { - c->code.append4(offset); - } -} - Register asRegister(Context* c, MyOperand* v) { - if (v->type() == RegisterOperandType) { - return registerValue(v); + if (v->type() == MyOperand::RegisterOperandType) { + return registerValue(c, v); } else { - assert(c, v->type() == MemoryOperandType); + assert(c, v->type() == MyOperand::MemoryOperandType); RegisterOperand* tmp = temporary(c); v->mov(c, tmp); @@ -383,7 +462,7 @@ class MemoryOperand: public MyOperand { case RegisterOperandType: rex(c); encode(c, 0x8b, 0, 0x40, 0x80, registerValue(c, dst), - base, displacement); + asRegister(c, base), displacement); break; case MemoryOperandType: { @@ -393,7 +472,7 @@ class MemoryOperand: public MyOperand { release(c, tmp); } break; - default: abort(t); + default: abort(c); } } @@ -420,8 +499,16 @@ class StackOperand: public MemoryOperand { class SelectionOperand: public MyOperand { public: - SelectionOperand(Compiler::SelectionType type, MyOperand* base): - type(type), base(base) + enum SelectionType { + S1Selection, + S2Selection, + Z2Selection, + S4Selection, + S8Selection + }; + + SelectionOperand(SelectionType type, MyOperand* base): + selectionType(type), base(base) { } virtual OperandType type() { @@ -429,7 +516,7 @@ class SelectionOperand: public MyOperand { } virtual unsigned footprint() { - if (type == Compiler::S8Selection) { + if (selectionType == S8Selection) { return 8; } else { return 4; @@ -440,7 +527,7 @@ class SelectionOperand: public MyOperand { return sizeof(SelectionOperand); } - Compiler::SelectionType type; + SelectionType selectionType; MyOperand* base; }; @@ -451,7 +538,7 @@ immediate(Context* c, intptr_t v) } AbsoluteOperand* -absolute(Context* c, intptr_t v) +absolute(Context* c, Promise* v) { return c->operands.push(AbsoluteOperand(v)); } @@ -476,10 +563,10 @@ stack(Context* c, int displacement) } MyOperand* -selection(Context* c, Compiler::SelectionType type, MyOperand* base) +selection(Context* c, SelectionOperand::SelectionType type, MyOperand* base) { - if ((type == S4Selection and BytesPerWord == 4) - or (type == S8Selection and BytesPerWord == 8)) + if ((type == SelectionOperand::S4Selection and BytesPerWord == 4) + or (type == SelectionOperand::S8Selection and BytesPerWord == 8)) { return base; } else { @@ -488,18 +575,18 @@ selection(Context* c, Compiler::SelectionType type, MyOperand* base) } bool -isStackReference(Context* c, MyOperand* v) +isStackReference(Context*, MyOperand* v) { - return v->type() == MemoryOperandType + return v->type() == MyOperand::MemoryOperandType and static_cast(v)->isStackReference(); } void flushStack(Context* c) { - Stack newVirtualStack; + Vector newVirtualStack(c->s, c->virtualStack.length() * 2); for (unsigned i = 0; i < c->virtualStack.length();) { - MyOperand* v = c->virtualStack.peek(i); + MyOperand* v = c->virtualStack.peek(i); if (not isStackReference(c, v)) { v->push(c); @@ -510,13 +597,30 @@ flushStack(Context* c) newVirtualStack.push(stack(c, c->stackIndex)); } } else { - newVirtualStack.push(v, v->size()); + newVirtualStack.append(v, v->size()); } i += v->size(); } - c->virtualStack.swap(&newVirtualStack); + c->virtualStack.update(&newVirtualStack); +} + +unsigned +pushArguments(Context* c, unsigned count, va_list list) +{ + MyOperand* arguments[count]; + unsigned footprint = 0; + for (unsigned i = 0; i < count; ++i) { + arguments[i] = va_arg(list, MyOperand*); + footprint += pad(arguments[i]->footprint()); + } + + for (int i = count - 1; i >= 0; --i) { + arguments[i]->push(c); + } + + return footprint; } class MyCompiler: public Compiler { @@ -526,16 +630,17 @@ class MyCompiler: public Compiler { { } virtual Promise* poolOffset() { - return c.promises.push(PoolPromise(constantPool.length() / BytesPerWord)); + return c.promises.push + (PoolPromise(c.constantPool.length() / BytesPerWord)); } virtual Promise* codeOffset() { - return c.promises.push(CodePromise(code.length())); + return c.promises.push(CodePromise(c.code.length())); } virtual Operand* poolAppend(Operand* v) { Operand* r = absolute(&c, poolOffset()); - constantPool.push(static_cast(v)); + c.constantPool.push(static_cast(v)); return r; } @@ -553,15 +658,16 @@ class MyCompiler: public Compiler { } virtual Operand* stack(unsigned index) { - return c.virtualStack.peek(stack.size() - index - 1); + return c.virtualStack.peek + (c.virtualStack.length() - ((index + 1) * BytesPerWord)); } virtual Operand* stack2(unsigned index) { - return c.virtualStack.peek(stack.size() - index - 1); + return stack(index); } virtual Operand* pop() { - return pop(c); + return c.virtualStack.pop(); } virtual Operand* pop2() { @@ -570,7 +676,7 @@ class MyCompiler: public Compiler { } virtual void pop(Operand* dst) { - pop(c)->mov(&c, static_cast(dst)); + c.virtualStack.pop()->mov(&c, static_cast(dst)); } virtual void pop2(Operand* dst) { @@ -628,45 +734,49 @@ class MyCompiler: public Compiler { (Operand* address, unsigned argumentCount, ...) { va_list a; va_start(a, argumentCount); - pushArguments(&c, argumentCount, a); + unsigned footprint = pushArguments(&c, argumentCount, a); va_end(a); - static_cast(address)->mov(&c, register_(rax)); - immediate(&c, indirectCaller)->call(&c); + static_cast(address)->mov(&c, register_(&c, rax)); + immediate(&c, c.indirectCaller)->call(&c); - popArguments(&c, argumentCount); + immediate(&c, footprint)->sub(&c, register_(&c, rsp)); + + return register_(&c, rax); } - virtual Operand* indirectCallNoReturn + virtual void indirectCallNoReturn (Operand* address, unsigned argumentCount, ...) { va_list a; va_start(a, argumentCount); pushArguments(&c, argumentCount, a); va_end(a); - static_cast(address)->mov(&c, register_(rax)); - immediate(&c, indirectCaller)->call(&c); + static_cast(address)->mov(&c, register_(&c, rax)); + immediate(&c, c.indirectCaller)->call(&c); } virtual Operand* directCall (Operand* address, unsigned argumentCount, ...) { va_list a; va_start(a, argumentCount); - pushArguments(&c, argumentCount, a); + unsigned footprint = pushArguments(&c, argumentCount, a); va_end(a); static_cast(address)->call(&c); - popArguments(&c, argumentCount); + immediate(&c, footprint)->sub(&c, register_(&c, rsp)); + + return register_(&c, rax); } virtual void return_(Operand* v) { - static_cast(v)->mov(&c, register_(rax)); - ret(c); + static_cast(v)->mov(&c, register_(&c, rax)); + ret(); } virtual void ret() { - ret(c); + ::ret(&c); } virtual void mov(Operand* src, Operand* dst) { @@ -757,11 +867,33 @@ class MyCompiler: public Compiler { virtual Operand* memory(Operand* base, int displacement, Operand* index, unsigned scale) { - return memory(&c, base, displacement, index, scale); + return ::memory(&c, static_cast(base), displacement, + static_cast(index), scale); } - virtual Operand* select(SelectionType type, Operand* v) { - return selection(&c, type, v); + virtual Operand* select1(Operand* v) { + return selection(&c, SelectionOperand::S1Selection, + static_cast(v)); + } + + virtual Operand* select2(Operand* v) { + return selection(&c, SelectionOperand::S2Selection, + static_cast(v)); + } + + virtual Operand* select2z(Operand* v) { + return selection(&c, SelectionOperand::Z2Selection, + static_cast(v)); + } + + virtual Operand* select4(Operand* v) { + return selection(&c, SelectionOperand::S4Selection, + static_cast(v)); + } + + virtual Operand* select8(Operand* v) { + return selection(&c, SelectionOperand::S8Selection, + static_cast(v)); } virtual void prologue() { @@ -775,19 +907,20 @@ class MyCompiler: public Compiler { } virtual void startLogicalIp(unsigned ip) { - c.ipTable.push(IpMapping(ip, code.length())); + c.ipTable.push(IpMapping(ip, c.code.length())); } virtual Operand* logicalIp(unsigned ip) { - return absolute(&c, promises.push(IpPromise(ip))); + return absolute(&c, c.promises.push(IpPromise(ip))); } virtual unsigned logicalIpToOffset(unsigned ip) { unsigned bottom = 0; - unsigned top = c.ipTable.size(); + unsigned top = c.ipTable.length() / sizeof(IpMapping); for (unsigned span = top - bottom; span; span = top - bottom) { unsigned middle = bottom + (span / 2); - IpMapping* mapping = c.ipTable.get(middle); + IpMapping* mapping = c.ipTable.peek + (middle * sizeof(IpMapping)); if (ip == mapping->ip) { return mapping->offset; @@ -798,25 +931,25 @@ class MyCompiler: public Compiler { } } - abort(s); + abort(&c); } virtual unsigned size() { return c.code.length(); } - virtual void writeTo(void* out) { + virtual void writeTo(void*) { // todo } - virtual void updateCall(void* returnAddress, void* newTarget) { + virtual void updateCall(void*, void*) { // todo } virtual void dispose() { c.dispose(); - s->free(this); + c.s->free(this); } Context c; diff --git a/src/compiler.h b/src/compiler.h index 5d897d3cd6..4a588ee4a4 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -16,14 +16,6 @@ class Promise { class Compiler { public: - enum SelectionType { - S1Selection, - S2Selection, - Z2Selection, - S4Selection, - S8Selection - }; - virtual ~Compiler() { } virtual Promise* poolOffset() = 0; @@ -56,7 +48,7 @@ class Compiler { virtual Operand* alignedCall(Operand*) = 0; virtual Operand* indirectCall (Operand* address, unsigned argumentCount, ...) = 0; - virtual Operand* indirectCallNoReturn + virtual void indirectCallNoReturn (Operand* address, unsigned argumentCount, ...) = 0; virtual Operand* directCall (Operand* address, unsigned argumentCount, ...) = 0; @@ -89,7 +81,11 @@ class Compiler { virtual Operand* memory(Operand* base, int displacement = 0, Operand* index = 0, unsigned scale = 1) = 0; - virtual Operand* select(SelectionType, Operand*) = 0; + virtual Operand* select1(Operand*) = 0; + virtual Operand* select2(Operand*) = 0; + virtual Operand* select2z(Operand*) = 0; + virtual Operand* select4(Operand*) = 0; + virtual Operand* select8(Operand*) = 0; virtual void prologue() = 0; virtual void epilogue() = 0; diff --git a/src/old-compile.cpp b/src/old-compile.cpp new file mode 100644 index 0000000000..c92b247619 --- /dev/null +++ b/src/old-compile.cpp @@ -0,0 +1,5198 @@ +#include "common.h" +#include "system.h" +#include "constants.h" +#include "machine.h" +#include "processor.h" +#include "process.h" + +using namespace vm; + +extern "C" uint64_t +vmInvoke(void* function, void* stack, unsigned stackSize, + unsigned returnType); + +extern "C" void +vmCall(); + +extern "C" void NO_RETURN +vmJump(void* address, void* base, void* stack); + +namespace { + +const bool Verbose = false; + +const unsigned FrameThread = BytesPerWord * 2; +const unsigned FrameMethod = FrameThread + BytesPerWord; +const unsigned FrameNext = FrameMethod + BytesPerWord; +const unsigned FrameFootprint = BytesPerWord * 3; + +class ArgumentList; + +class Buffer { + public: + Buffer(System* s, unsigned minimumCapacity): + s(s), + data(0), + position(0), + capacity(0), + minimumCapacity(minimumCapacity) + { } + + ~Buffer() { + if (data) { + s->free(data); + } + } + + void ensure(unsigned space) { + if (position + space > capacity) { + unsigned newCapacity = max + (position + space, max(minimumCapacity, capacity * 2)); + uint8_t* newData = static_cast(s->allocate(newCapacity)); + if (data) { + memcpy(newData, data, position); + s->free(data); + } + data = newData; + } + } + + void append(uint8_t v) { + ensure(1); + data[position++] = v; + } + + void append2(uint16_t v) { + ensure(2); + memcpy(data + position, &v, 2); + position += 2; + } + + void append4(uint32_t v) { + ensure(4); + memcpy(data + position, &v, 4); + position += 4; + } + + void set2(unsigned offset, uint32_t v) { + assert(s, offset + 2 <= position); + memcpy(data + offset, &v, 2); + } + + void set4(unsigned offset, uint32_t v) { + assert(s, offset + 4 <= position); + memcpy(data + offset, &v, 4); + } + + uint8_t& get(unsigned offset) { + assert(s, offset + 1 <= position); + return data[offset]; + } + + uint16_t& get2(unsigned offset) { + assert(s, offset + 2 <= position); + return *reinterpret_cast(data + offset); + } + + uint32_t& get4(unsigned offset) { + assert(s, offset + 4 <= position); + return *reinterpret_cast(data + offset); + } + + uintptr_t& getAddress(unsigned offset) { + assert(s, offset + 4 <= position); + return *reinterpret_cast(data + offset); + } + + void appendAddress(uintptr_t v) { + append4(v); + if (BytesPerWord == 8) { + // we have to use the preprocessor here to avoid a warning on + // 32-bit systems +#ifdef __x86_64__ + append4(v >> 32); +#endif + } + } + + unsigned length() { + return position; + } + + void copyTo(void* b) { + if (data) { + memcpy(b, data, position); + } + } + + System* s; + uint8_t* data; + unsigned position; + unsigned capacity; + unsigned minimumCapacity; +}; + +class StackMapper { + public: + static const uint32_t Infinity = ~static_cast(0); + + enum { + Call, + PushLong, + PushInt, + PushObject, + Duplicate, + DuplicateX1, + DuplicateX2, + Duplicate2, + Duplicate2X1, + Duplicate2X2, + Pop, + PopLong, + PopInt, + PopObject, + StoreLong, + StoreInt, + StoreObject, + Return, + Jump, + Branch, + End + }; + + StackMapper(Thread* t, object method): + t(t), + method(method), + ip(-1), + index(codeSize() ? static_cast + (t->m->system->allocate(codeSize() * 4)) : 0), + log(t->m->system, 1024), + callCount(0), + protector(this) + { } + + ~StackMapper() { + if (index) { + t->m->system->free(index); + } + } + + unsigned codeSize() { + return codeLength(t, methodCode(t, method)); + } + + void end(unsigned nextIp) { + if (ip >= 0) { + log.append(End); + log.append2(nextIp); + } + ip = nextIp; + } + + void newIp(unsigned javaIp) { + assert(t, javaIp < codeSize()); + + end(javaIp); + index[javaIp] = log.length(); + } + + void finish() { + end(codeSize()); + } + + void called(unsigned nextMachineIp) { + log.append(Call); + log.append4(nextMachineIp); + ++ callCount; + } + + void pushedLong() { + log.append(PushLong); + } + + void pushedInt() { + log.append(PushInt); + } + + void pushedObject() { + log.append(PushObject); + } + + void dupped() { + log.append(Duplicate); + } + + void duppedX1() { + log.append(DuplicateX1); + } + + void duppedX2() { + log.append(DuplicateX2); + } + + void dupped2() { + log.append(Duplicate2); + } + + void dupped2X1() { + log.append(Duplicate2X1); + } + + void dupped2X2() { + log.append(Duplicate2X2); + } + + void popped(unsigned count) { + log.append(Pop); + log.append(count); + } + + void poppedLong() { + log.append(PopLong); + } + + void poppedInt() { + log.append(PopInt); + } + + void poppedObject() { + log.append(PopObject); + } + + void storedLong(unsigned index) { + if (index >= parameterFootprint()) { + log.append(StoreLong); + log.append2(index - parameterFootprint()); + } + } + + void storedInt(unsigned index) { + if (index >= parameterFootprint()) { + log.append(StoreInt); + log.append2(index - parameterFootprint()); + } + } + + void storedObject(unsigned index) { + if (index >= parameterFootprint()) { + log.append(StoreObject); + log.append2(index - parameterFootprint()); + } + } + + void exited() { + log.append(Return); + } + + void jumped(unsigned targetJavaIp) { + log.append(Jump); + log.append2(targetJavaIp); + } + + void branched(unsigned targetJavaIp) { + log.append(Branch); + log.append2(targetJavaIp); + } + + unsigned parameterFootprint() { + return methodParameterFootprint(t, method); + } + + unsigned localSize() { + return codeMaxLocals(t, methodCode(t, method)) + - parameterFootprint(); + } + + unsigned stackSize() { + return codeMaxStack(t, methodCode(t, method)); + } + + unsigned mapSize() { + return stackSize() + localSize(); + } + + unsigned mapSizeInWords() { + return ceiling(mapSize(), BytesPerWord); + } + + unsigned mapSizeInBytes() { + return mapSizeInWords() * BytesPerWord; + } + + unsigned callTableSize() { + return callCount * (mapSizeInBytes() + BytesPerWord); + } + + void populateCalls(uintptr_t* calls, unsigned& callIndex, uintptr_t* table, + uintptr_t* mask, uintptr_t* initialMap, + unsigned initialSp, unsigned ip) + { + uintptr_t map[mapSizeInWords()]; + memcpy(map, initialMap, mapSizeInBytes()); + + unsigned sp = initialSp; + + while (ip < codeSize()) { + if (getBit(mask, ip)) { + return; + } + + markBit(mask, ip); + memcpy(table + (ip * mapSizeInWords()), map, mapSizeInBytes()); + + unsigned i = index[ip]; + while (true) { + switch (log.get(i++)) { + case Call: { + assert(t, callIndex < callCount); + unsigned machineIp = log.get4(i); i += 4; + calls[callIndex * (mapSizeInWords() + 1)] = machineIp; + memcpy(calls + (callIndex * (mapSizeInWords() + 1)) + 1, + map, mapSizeInBytes()); + ++ callIndex; + +// fprintf(stderr, +// "call stack and locals 0x%x of size %d at %d of %s.%s\n", +// *map, +// mapSize(), +// machineIp, +// &byteArrayBody +// (t, className(t, methodClass(t, method)), 0), +// &byteArrayBody(t, methodName(t, method), 0)); + + } break; + + case PushLong: + assert(t, sp + 2 <= mapSize()); + assert(t, getBit(map, sp) == 0); + assert(t, getBit(map, sp + 1) == 0); + sp += 2; + break; + + case PushInt: + assert(t, sp + 1 <= mapSize()); + assert(t, getBit(map, sp) == 0); + ++ sp; + break; + + case PushObject: + assert(t, sp + 1 <= mapSize()); + markBit(map, sp++); + break; + + case Duplicate: + assert(t, sp + 1 <= mapSize()); + assert(t, sp - 1 >= localSize()); + if (getBit(map, sp - 1)) { + markBit(map, sp); + } + ++ sp; + break; + + case DuplicateX1: { + assert(t, sp + 1 <= mapSize()); + assert(t, sp - 2 >= localSize()); + + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b2) { + markBit(map, sp - 1); + } else { + clearBit(map, sp - 1); + } + + if (b1) { + markBit(map, sp - 2); + markBit(map, sp); + } else { + clearBit(map, sp - 2); + } + + ++ sp; + } break; + + case DuplicateX2: { + assert(t, sp + 1 <= mapSize()); + assert(t, sp - 3 >= localSize()); + + unsigned b3 = getBit(map, sp - 3); + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b3) { + markBit(map, sp - 2); + } else { + clearBit(map, sp - 2); + } + + if (b2) { + markBit(map, sp - 1); + } else { + clearBit(map, sp - 1); + } + + if (b1) { + markBit(map, sp - 3); + markBit(map, sp); + } else { + clearBit(map, sp - 3); + } + + ++ sp; + } break; + + case Duplicate2: { + assert(t, sp + 2 <= mapSize()); + assert(t, sp - 2 >= localSize()); + + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b2) { + markBit(map, sp); + } + + if (b1) { + markBit(map, sp + 1); + } + + sp += 2; + } break; + + case Duplicate2X1: { + assert(t, sp + 2 <= mapSize()); + assert(t, sp - 3 >= localSize()); + + unsigned b3 = getBit(map, sp - 3); + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b3) { + markBit(map, sp - 1); + } else { + clearBit(map, sp - 1); + } + + if (b2) { + markBit(map, sp - 3); + markBit(map, sp); + } else { + clearBit(map, sp - 3); + } + + if (b1) { + markBit(map, sp - 2); + markBit(map, sp + 1); + } else { + clearBit(map, sp - 2); + } + + sp += 2; + } break; + + case Duplicate2X2: { + assert(t, sp + 2 <= mapSize()); + assert(t, sp - 4 >= localSize()); + + unsigned b4 = getBit(map, sp - 4); + unsigned b3 = getBit(map, sp - 3); + unsigned b2 = getBit(map, sp - 2); + unsigned b1 = getBit(map, sp - 1); + + if (b4) { + markBit(map, sp - 2); + } else { + clearBit(map, sp - 2); + } + + if (b3) { + markBit(map, sp - 1); + } else { + clearBit(map, sp - 1); + } + + if (b2) { + markBit(map, sp - 4); + markBit(map, sp); + } else { + clearBit(map, sp - 4); + } + + if (b1) { + markBit(map, sp - 3); + markBit(map, sp + 1); + } else { + clearBit(map, sp - 3); + } + + sp += 2; + } break; + + case Pop: { + unsigned count = log.get(i++); + assert(t, sp >= count); + assert(t, sp - count >= localSize()); + while (count) { + clearBit(map, -- sp); + -- count; + } + } break; + + case PopLong: + assert(t, sp >= 2); + assert(t, sp - 2 >= localSize()); + assert(t, getBit(map, sp - 1) == 0); + assert(t, getBit(map, sp - 2) == 0); + sp -= 2; + break; + + case PopInt: + assert(t, sp >= 1); + assert(t, sp - 1 >= localSize()); + assert(t, getBit(map, sp - 1) == 0); + -- sp; + break; + + case PopObject: + assert(t, sp >= 1); + assert(t, sp - 1 >= localSize()); + assert(t, getBit(map, sp - 1) != 0); + clearBit(map, -- sp); + break; + + case StoreLong: { + unsigned index = log.get2(i); i += 2; + assert(t, index + 1 < localSize()); + clearBit(map, index); + clearBit(map, index + 1); + } break; + + case StoreInt: { + unsigned index = log.get2(i); i += 2; + assert(t, index < localSize()); + clearBit(map, index); + } break; + + case StoreObject: { + unsigned index = log.get2(i); i += 2; + assert(t, index < localSize()); + markBit(map, index); + } break; + + case Return: + ip = codeSize(); + goto loop; + + case Jump: + ip = log.get2(i); i += 2; + assert(t, ip < codeSize()); + goto loop; + + case Branch: { + unsigned target = log.get2(i); i += 2; + assert(t, target < codeSize()); + populateCalls(calls, callIndex, table, mask, map, sp, target); + } break; + + case End: + goto next; + + default: + abort(t); + } + } + + next: + ip = log.get2(i); + + loop:; + } + } + + static int compareCalls(const void* a, const void* b) { + return (*static_cast(a) > + *static_cast(b) ? 1 : -1); + } + + void writeCallTableTo(uintptr_t* calls) { + uintptr_t* table = static_cast + (t->m->system->allocate(codeSize() * mapSizeInBytes())); + + uintptr_t* mask = static_cast + (t->m->system->allocate + (ceiling(codeSize(), BytesPerWord) * BytesPerWord)); + memset(mask, 0, ceiling(codeSize(), BytesPerWord) * BytesPerWord); + + uintptr_t map[mapSizeInWords()]; + memset(map, 0, mapSizeInBytes()); + + unsigned callIndex = 0; + + populateCalls(calls, callIndex, table, mask, map, localSize(), 0); + + object eht = codeExceptionHandlerTable(t, methodCode(t, method)); + if (eht) { + PROTECT(t, eht); + + for (unsigned i = 0; i < exceptionHandlerTableLength(t, eht); ++i) { + ExceptionHandler* eh = exceptionHandlerTableBody(t, eht, i); + + assert(t, getBit(mask, exceptionHandlerStart(eh))); + + memcpy(map, + table + (exceptionHandlerStart(eh) * mapSizeInWords()), + mapSizeInBytes()); + + for (unsigned j = localSize() + 1; j < mapSize(); ++j) { + clearBit(map, j); + } + + markBit(map, localSize()); + + populateCalls(calls, callIndex, table, mask, map, localSize() + 1, + exceptionHandlerIp(eh)); + } + } + + t->m->system->free(mask); + t->m->system->free(table); + + assert(t, callIndex == callCount); + qsort(calls, callCount, mapSizeInBytes() + BytesPerWord, compareCalls); + } + + Thread* t; + object method; + int ip; + uint32_t* index; + Buffer log; + unsigned callCount; + + class MyProtector: public Thread::Protector { + public: + MyProtector(StackMapper* mapper): Protector(mapper->t), mapper(mapper) { } + + virtual void visit(Heap::Visitor* v) { + v->visit(&(mapper->method)); + } + + StackMapper* mapper; + } protector; +}; + +class MyThread: public Thread { + public: + MyThread(Machine* m, object javaThread, Thread* parent): + Thread(m, javaThread, parent), + argumentList(0), + reference(0) + { + memset(frameData, 0, BytesPerWord * 2); + } + + ArgumentList* argumentList; + void* frameData[2]; + Reference* reference; +}; + +inline void* +frameBase(void* frame) +{ + return static_cast(frame) + [static_cast(- (FrameFootprint / BytesPerWord) - 2)]; +} + +inline bool +frameValid(void* frame) +{ + return frame != 0; +} + +inline void* +frameNext(void* frame) +{ + return static_cast(frameBase(frame))[FrameNext / BytesPerWord]; +} + +inline object& +frameMethod(void* frame) +{ + return static_cast(frameBase(frame))[FrameMethod / BytesPerWord]; +} + +inline void* +frameAddress(void* frame) +{ + return static_cast(frame) + [static_cast(- (FrameFootprint / BytesPerWord) - 1)]; +} + +inline void* +frameReturnAddress(void* frame) +{ + return static_cast(frameBase(frame))[1]; +} + +inline uint8_t* +compiledCode(Compiled* code) +{ + return compiledBody(code); +} + +inline unsigned +compiledLineNumberCount(Thread*, Compiled* code) +{ + return compiledLineNumberTableLength(code) / sizeof(NativeLineNumber); +} + +inline NativeLineNumber* +compiledLineNumber(Thread* t UNUSED, Compiled* code, unsigned index) +{ + assert(t, index < compiledLineNumberCount(t, code)); + return reinterpret_cast + (compiledBody(code) + pad(compiledCodeLength(code))) + index; +} + +inline unsigned +compiledExceptionHandlerCount(Thread*, Compiled* code) +{ + return compiledExceptionHandlerTableLength(code) + / sizeof(NativeExceptionHandler); +} + +inline NativeExceptionHandler* +compiledExceptionHandler(Thread* t UNUSED, Compiled* code, unsigned index) +{ + assert(t, index < compiledExceptionHandlerCount(t, code)); + return reinterpret_cast + (compiledBody(code) + + pad(compiledCodeLength(code)) + + pad(compiledLineNumberTableLength(code))) + index; +} + +inline unsigned +compiledStackMapSize(Thread*, Compiled* code) +{ + return ceiling(compiledMaxStack(code) + + compiledMaxLocals(code) + - compiledParameterFootprint(code), + BytesPerWord) + 1; +} + +inline unsigned +compiledStackMapCount(Thread* t, Compiled* code) +{ + return compiledStackMapTableLength(code) + / (compiledStackMapSize(t, code) * BytesPerWord); +} + +inline uintptr_t* +compiledStackMap(Thread* t, Compiled* code, unsigned index) +{ + assert(t, index < compiledStackMapCount(t, code)); + return reinterpret_cast + (compiledBody(code) + + pad(compiledCodeLength(code)) + + pad(compiledLineNumberTableLength(code)) + + pad(compiledExceptionHandlerTableLength(code))) + + (index * compiledStackMapSize(t, code)); +} + +inline Compiled* +makeCompiled(Thread* t, Buffer* code) +{ + Compiled* c = static_cast + (t->m->system->allocate(sizeof(Compiled) + pad(code->length()))); + + compiledMaxLocals(c) = 0; + compiledMaxStack(c) = 0; + compiledParameterFootprint(c) = 0; + compiledCodeLength(c) = code->length(); + compiledLineNumberTableLength(c) = 0; + compiledExceptionHandlerTableLength(c) = 0; + compiledStackMapTableLength(c) = 0; + + if (code->length()) { + code->copyTo(compiledCode(c)); + } + + return c; +} + +inline Compiled* +makeCompiled(Thread* t, object method, Buffer* code, + NativeLineNumber* lineNumbers, + NativeExceptionHandler* exceptionHandlers, + StackMapper* stackMapper) +{ + unsigned lineNumberCount + = codeLineNumberTable(t, methodCode(t, method)) ? + lineNumberTableLength + (t, codeLineNumberTable(t, methodCode(t, method))) : 0; + + unsigned exceptionHandlerCount + = codeExceptionHandlerTable(t, methodCode(t, method)) ? + exceptionHandlerTableLength + (t, codeExceptionHandlerTable(t, methodCode(t, method))) : 0; + + unsigned maxStack = codeMaxStack(t, methodCode(t, method)); + unsigned maxLocals = codeMaxLocals(t, methodCode(t, method)); + unsigned parameterFootprint = methodParameterFootprint(t, method); + + Compiled* c = static_cast + (t->m->system->allocate(sizeof(Compiled) + + pad(code->length()) + + pad(lineNumberCount + * sizeof(NativeLineNumber)) + + pad(exceptionHandlerCount + * sizeof(NativeExceptionHandler)) + + pad(stackMapper->callTableSize()))); + + compiledMaxLocals(c) = maxLocals; + compiledMaxStack(c) = maxStack; + compiledParameterFootprint(c) = parameterFootprint; + compiledCodeLength(c) = code->length(); + + compiledLineNumberTableLength(c) + = lineNumberCount * sizeof(NativeLineNumber); + + compiledExceptionHandlerTableLength(c) + = exceptionHandlerCount * sizeof(NativeExceptionHandler); + + compiledStackMapTableLength(c) = stackMapper->callTableSize(); + + if (code->length()) { + code->copyTo(compiledCode(c)); + } + + if (lineNumberCount) { + memcpy(compiledLineNumber(t, c, 0), + lineNumbers, + lineNumberCount * sizeof(NativeLineNumber)); + } + + if (exceptionHandlerCount) { + memcpy(compiledExceptionHandler(t, c, 0), + exceptionHandlers, + exceptionHandlerCount * sizeof(NativeExceptionHandler)); + } + + if (stackMapper->callTableSize()) { + stackMapper->writeCallTableTo(compiledStackMap(t, c, 0)); + } + + return c; +} + +inline unsigned +addressOffset(Thread* t, object method, void* address) +{ + Compiled* code = reinterpret_cast(methodCompiled(t, method)); + return static_cast(address) - compiledCode(code); +} + +NativeExceptionHandler* +findExceptionHandler(Thread* t, void* frame) +{ + object method = frameMethod(frame); + Compiled* code = reinterpret_cast(methodCompiled(t, method)); + + for (unsigned i = 0; i < compiledExceptionHandlerCount(t, code); ++i) { + NativeExceptionHandler* handler = compiledExceptionHandler(t, code, i); + unsigned offset = addressOffset(t, method, frameAddress(frame)); + + if (offset - 1 >= nativeExceptionHandlerStart(handler) + and offset - 1 < nativeExceptionHandlerEnd(handler)) + { + object catchType; + if (nativeExceptionHandlerCatchType(handler)) { + catchType = arrayBody + (t, methodCode(t, method), + nativeExceptionHandlerCatchType(handler) - 1); + } else { + catchType = 0; + } + + if (catchType == 0 or instanceOf(t, catchType, t->exception)) { + if (Verbose) { + fprintf(stderr, "exception handler match for %d in %s: " + "start: %d; end: %d; ip: %d\n", + offset, + &byteArrayBody(t, methodName(t, frameMethod(frame)), 0), + nativeExceptionHandlerStart(handler), + nativeExceptionHandlerEnd(handler), + nativeExceptionHandlerIp(handler)); + } + + return handler; + } + } + } + + return 0; +} + +void* +framePointer(void** data) +{ + return data + (FrameFootprint / BytesPerWord) + 2; +} + +void* +frameStart(MyThread* t) +{ + return t->frameData[0] ? framePointer(t->frameData) : 0; +} + +void NO_RETURN +unwind(MyThread* t) +{ + for (void* frame = frameStart(t); frameValid(frame); + frame = frameNext(frame)) + { + if ((methodFlags(t, frameMethod(frame)) & ACC_NATIVE) == 0) { + NativeExceptionHandler* eh = findExceptionHandler(t, frame); + if (eh) { + object method = frameMethod(frame); + Compiled* code = reinterpret_cast + (methodCompiled(t, method)); + + void** stack = static_cast(frameBase(frame)); + + unsigned parameterFootprint = methodParameterFootprint(t, method); + unsigned localFootprint = compiledMaxLocals(code); + + if (localFootprint > parameterFootprint) { + stack -= (localFootprint - parameterFootprint); + } + + *(--stack) = t->exception; + t->exception = 0; + + vmJump(compiledCode(code) + nativeExceptionHandlerIp(eh), + frameBase(frame), + stack); + } + } + + void* next = frameNext(frame); + if (not frameValid(next) + or methodFlags(t, frameMethod(next)) & ACC_NATIVE) + { + vmJump(frameReturnAddress(frame), + *static_cast(frameBase(frame)), + static_cast(frameBase(frame)) + 2); + } + } + abort(t); +} + +void NO_RETURN +throwNew(MyThread* t, object class_) +{ + t->exception = makeNew(t, class_); + object trace = makeTrace(t); + set(t, t->exception, ThrowableTrace, trace); + unwind(t); +} + +void NO_RETURN +throw_(MyThread* t, object o) +{ + if (o) { + t->exception = o; + } else { + t->exception = makeNullPointerException(t); + } + unwind(t); +} + +uintptr_t* +frameStackMap(MyThread* t, void* frame) +{ + Compiled* code = reinterpret_cast + (methodCompiled(t, frameMethod(frame))); + unsigned ip = static_cast(frameAddress(frame)) + - compiledCode(code); + + unsigned bottom = 0; + unsigned top = compiledStackMapCount(t, code); + for (unsigned span = top - bottom; span; span = top - bottom) { + unsigned middle = bottom + (span / 2); + uintptr_t* map = compiledStackMap(t, code, middle); + + if (ip < *map) { + top = middle; + } else if (ip > *map) { + bottom = middle + 1; + } else { + return map + 1; + } + } + + fprintf(stderr, "%d not found in ", ip); + for (unsigned i = 0; i < compiledStackMapCount(t, code); ++i) { + fprintf(stderr, "%"LD" ", *compiledStackMap(t, code, i)); + } + fprintf(stderr, "\n"); + + abort(t); +} + +int +localOffset(MyThread* t, int v, object method) +{ + int parameterFootprint + = methodParameterFootprint(t, method) * BytesPerWord; + + v *= BytesPerWord; + if (v < parameterFootprint) { + return (parameterFootprint - v - BytesPerWord) + (BytesPerWord * 2) + + FrameFootprint; + } else { + return -(v + BytesPerWord - parameterFootprint); + } +} + +inline object* +frameLocalObject(MyThread* t, void* frame, unsigned index) +{ + return reinterpret_cast + (static_cast(frameBase(frame)) + + localOffset(t, index, frameMethod(frame))); +} + +void +visitParameters(MyThread* t, Heap::Visitor* v, void* frame) +{ + object method = frameMethod(frame); + + const char* spec = reinterpret_cast + (&byteArrayBody(t, methodSpec(t, method), 0)); + + unsigned index = 0; + if ((methodFlags(t, method) & ACC_STATIC) == 0) { + v->visit(frameLocalObject(t, frame, index++)); + } + + for (MethodSpecIterator it(t, spec); it.hasNext();) { + switch (*it.next()) { + case 'L': + case '[': + v->visit(frameLocalObject(t, frame, index++)); + break; + + case 'J': + case 'D': + index += 2; + break; + + default: + ++ index; + break; + } + } + + assert(t, index == methodParameterFootprint(t, method)); +} + +void +visitStackAndLocals(MyThread* t, Heap::Visitor* v, void* frame) +{ + Compiled* code = reinterpret_cast + (methodCompiled(t, frameMethod(frame))); + + unsigned parameterFootprint = compiledParameterFootprint(code); + unsigned count = compiledMaxStack(code) + + compiledMaxLocals(code) + - parameterFootprint; + + if (count) { + uintptr_t* stackMap = frameStackMap(t, frame); + +// fprintf(stderr, "visit stack and locals 0x%x of size %d, at %d of %s.%s\n", +// *stackMap, +// count, +// static_cast(frameAddress(frame)) +// - compiledCode(code), +// &byteArrayBody +// (t, className(t, methodClass(t, frameMethod(frame))), 0), +// &byteArrayBody(t, methodName(t, frameMethod(frame)), 0)); + + for (unsigned i = 0; i < count; ++i) { + if (getBit(stackMap, i)) { + v->visit(frameLocalObject(t, frame, i + parameterFootprint)); + } + } + } +} + +void +visitStack(MyThread* t, Heap::Visitor* v) +{ + void* frame = frameStart(t); + + if (frameValid(frame)) { + v->visit(&frameMethod(frame)); + } + + for (; frameValid(frame); frame = frameNext(frame)) { + // we only need to visit the parameters of this method if the + // caller is native. Otherwise, the caller owns them. + void* next = frameNext(frame); + if (frameValid(next)) { + v->visit(&frameMethod(next)); + + if (methodFlags(t, frameMethod(next)) & ACC_NATIVE) { + visitParameters(t, v, frame); + } + } else { + visitParameters(t, v, frame); + } + + object method = frameMethod(frame); + Compiled* code = reinterpret_cast(methodCompiled(t, method)); + + if ((methodFlags(t, method) & ACC_NATIVE) == 0 + and code != t->m->processor->methodStub(t)) + { + visitStackAndLocals(t, v, frame); + } + } +} + +object +findInterfaceMethodFromInstance(Thread* t, object method, object instance) +{ + return findInterfaceMethod(t, method, objectClass(t, instance)); +} + +intptr_t +compareDoublesG(uint64_t bi, uint64_t ai) +{ + double a = bitsToDouble(ai); + double b = bitsToDouble(bi); + + if (a < b) { + return -1; + } else if (a > b) { + return 1; + } else if (a == b) { + return 0; + } else { + return 1; + } +} + +intptr_t +compareDoublesL(uint64_t bi, uint64_t ai) +{ + double a = bitsToDouble(ai); + double b = bitsToDouble(bi); + + if (a < b) { + return -1; + } else if (a > b) { + return 1; + } else if (a == b) { + return 0; + } else { + return -1; + } +} + +intptr_t +compareFloatsG(uint32_t bi, uint32_t ai) +{ + float a = bitsToFloat(ai); + float b = bitsToFloat(bi); + + if (a < b) { + return -1; + } else if (a > b) { + return 1; + } else if (a == b) { + return 0; + } else { + return 1; + } +} + +intptr_t +compareFloatsL(uint32_t bi, uint32_t ai) +{ + float a = bitsToFloat(ai); + float b = bitsToFloat(bi); + + if (a < b) { + return -1; + } else if (a > b) { + return 1; + } else if (a == b) { + return 0; + } else { + return -1; + } +} + +uint64_t +addDouble(uint64_t b, uint64_t a) +{ + return doubleToBits(bitsToDouble(a) + bitsToDouble(b)); +} + +uint64_t +subtractDouble(uint64_t b, uint64_t a) +{ + return doubleToBits(bitsToDouble(a) - bitsToDouble(b)); +} + +uint64_t +multiplyDouble(uint64_t b, uint64_t a) +{ + return doubleToBits(bitsToDouble(a) * bitsToDouble(b)); +} + +uint64_t +divideDouble(uint64_t b, uint64_t a) +{ + return doubleToBits(bitsToDouble(a) / bitsToDouble(b)); +} + +uint64_t +moduloDouble(uint64_t b, uint64_t a) +{ + return doubleToBits(fmod(bitsToDouble(a), bitsToDouble(b))); +} + +uint32_t +addFloat(uint32_t b, uint32_t a) +{ + return floatToBits(bitsToFloat(a) + bitsToFloat(b)); +} + +uint32_t +subtractFloat(uint32_t b, uint32_t a) +{ + return floatToBits(bitsToFloat(a) - bitsToFloat(b)); +} + +uint32_t +multiplyFloat(uint32_t b, uint32_t a) +{ + return floatToBits(bitsToFloat(a) * bitsToFloat(b)); +} + +uint32_t +divideFloat(uint32_t b, uint32_t a) +{ + return floatToBits(bitsToFloat(a) / bitsToFloat(b)); +} + +uint32_t +moduloFloat(uint32_t b, uint32_t a) +{ + return floatToBits(fmod(bitsToFloat(a), bitsToFloat(b))); +} + +int64_t +divideLong(int64_t b, int64_t a) +{ + return a / b; +} + +int64_t +moduloLong(int64_t b, int64_t a) +{ + return a % b; +} + +object +makeBlankObjectArray(Thread* t, object class_, int32_t length) +{ + return makeObjectArray(t, class_, length, true); +} + +object +makeBlankArray(Thread* t, object (*constructor)(Thread*, uintptr_t, bool), + int32_t length) +{ + return constructor(t, length, true); +} + +uint64_t +invokeNative2(MyThread* t, object method) +{ + PROTECT(t, method); + + if (objectClass(t, methodCode(t, method)) + == arrayBody(t, t->m->types, Machine::ByteArrayType)) + { + void* function = resolveNativeMethod(t, method); + if (UNLIKELY(function == 0)) { + object message = makeString + (t, "%s", &byteArrayBody(t, methodCode(t, method), 0)); + t->exception = makeUnsatisfiedLinkError(t, message); + return 0; + } + + object p = makePointer(t, function); + set(t, method, MethodCode, p); + } + + object class_ = methodClass(t, method); + PROTECT(t, class_); + + unsigned footprint = methodParameterFootprint(t, method) + 1; + unsigned count = methodParameterCount(t, method) + 1; + if (methodFlags(t, method) & ACC_STATIC) { + ++ footprint; + ++ count; + } + + uintptr_t args[footprint]; + unsigned argOffset = 0; + uint8_t types[count]; + unsigned typeOffset = 0; + + args[argOffset++] = reinterpret_cast(t); + types[typeOffset++] = POINTER_TYPE; + + uintptr_t* sp = static_cast(frameBase(frameStart(t))) + + (methodParameterFootprint(t, method) + 1) + + (FrameFootprint / BytesPerWord); + + if (methodFlags(t, method) & ACC_STATIC) { + args[argOffset++] = reinterpret_cast(&class_); + } else { + args[argOffset++] = reinterpret_cast(sp--); + } + types[typeOffset++] = POINTER_TYPE; + + MethodSpecIterator it + (t, reinterpret_cast + (&byteArrayBody(t, methodSpec(t, method), 0))); + + while (it.hasNext()) { + unsigned type = types[typeOffset++] + = fieldType(t, fieldCode(t, *it.next())); + + switch (type) { + case INT8_TYPE: + case INT16_TYPE: + case INT32_TYPE: + case FLOAT_TYPE: + args[argOffset++] = *(sp--); + break; + + case INT64_TYPE: + case DOUBLE_TYPE: { + if (BytesPerWord == 8) { + uint64_t a = *(sp--); + uint64_t b = *(sp--); + args[argOffset++] = (a << 32) | b; + } else { + memcpy(args + argOffset, sp, 8); + argOffset += 2; + sp -= 2; + } + } break; + + case POINTER_TYPE: { + args[argOffset++] = reinterpret_cast(sp--); + } break; + + default: abort(t); + } + } + + void* function = pointerValue(t, methodCode(t, method)); + unsigned returnType = fieldType(t, methodReturnCode(t, method)); + uint64_t result; + + if (Verbose) { + fprintf(stderr, "invoke native method %s.%s\n", + &byteArrayBody(t, className(t, methodClass(t, method)), 0), + &byteArrayBody(t, methodName(t, method), 0)); + } + + { ENTER(t, Thread::IdleState); + + result = t->m->system->call + (function, + args, + types, + count + 1, + footprint * BytesPerWord, + returnType); + } + + if (Verbose) { + fprintf(stderr, "return from native method %s.%s\n", + &byteArrayBody(t, className(t, methodClass(t, method)), 0), + &byteArrayBody(t, methodName(t, method), 0)); + } + + if (LIKELY(t->exception == 0) and returnType == POINTER_TYPE) { + return result ? *reinterpret_cast(result) : 0; + } else { + return result; + } +} + +uint64_t +invokeNative(MyThread* t, object method) +{ + uint64_t result = invokeNative2(t, method); + if (UNLIKELY(t->exception)) { + unwind(t); + } else { + return result; + } +} + +void +compileMethod(MyThread* t, object method); + +inline bool +isByte(int32_t v) +{ + return v == static_cast(v); +} + +inline bool +isInt32(uintptr_t v) +{ + return v == static_cast(static_cast(v)); +} + +enum Register { + rax = 0, + rcx = 1, + rdx = 2, + rbx = 3, + rsp = 4, + rbp = 5, + rsi = 6, + rdi = 7, + r8 = 8, + r9 = 9, + r10 = 10, + r11 = 11, + r12 = 12, + r13 = 13, + r14 = 14, + r15 = 15, +}; + +enum SSERegister { + xmm0 = 0, + xmm1 = 1, + xmm2 = 2, + xmm3 = 3, + xmm4 = 4, + xmm5 = 5, + xmm6 = 6, + xmm7 = 7 +}; + +class Assembler { + public: + class Label { + public: + static const unsigned Capacity = 8; + + Label(Assembler* a): + code(&(a->code)), + unresolvedCount(0), + mark_(-1) + { } + + void reference() { + if (mark_ == -1) { + expect(code->s, unresolvedCount < Capacity); + unresolved[unresolvedCount] = code->length(); + ++ unresolvedCount; + + code->append4(0); + } else { + code->append4(mark_ - (code->length() + 4)); + } + } + + void mark() { + mark_ = code->length(); + for (unsigned i = 0; i < unresolvedCount; ++i) { + code->set4(unresolved[i], mark_ - (unresolved[i] + 4)); + } + } + + Buffer* code; + unsigned unresolved[Capacity]; + unsigned unresolvedCount; + int mark_; + }; + + Assembler(System* s): + code(s, 1024), + jumps(s, 256) + { } + + void rex() { + if (BytesPerWord == 8) { + code.append(0x48); + } + } + + void mov(Register src, Register dst) { + rex(); + code.append(0x89); + code.append(0xc0 | (src << 3) | dst); + } + + void offsetInstruction(uint8_t instruction, uint8_t zeroPrefix, + uint8_t bytePrefix, uint8_t wordPrefix, + unsigned a, unsigned b, int32_t offset) + { + code.append(instruction); + + uint8_t prefix; + if (offset == 0 and b != rbp) { + prefix = zeroPrefix; + } else if (isByte(offset)) { + prefix = bytePrefix; + } else { + prefix = wordPrefix; + } + + code.append(prefix | (a << 3) | b); + + if (b == rsp) { + code.append(0x24); + } + + if (offset == 0 and b != rbp) { + // do nothing + } else if (isByte(offset)) { + code.append(offset); + } else { + code.append4(offset); + } + } + + void movz1(Register src, Register dst) { + code.append(0x0f); + code.append(0xb6); + code.append(0xc0 | (dst << 3) | src); + } + + void movz1(Register src, int32_t srcOffset, Register dst) { + code.append(0x0f); + offsetInstruction(0xb6, 0, 0x40, 0x80, dst, src, srcOffset); + } + + void movs1(Register src, Register dst) { + code.append(0x0f); + code.append(0xbe); + code.append(0xc0 | (dst << 3) | src); + } + + void movs1(Register src, int32_t srcOffset, Register dst) { + code.append(0x0f); + offsetInstruction(0xbe, 0, 0x40, 0x80, dst, src, srcOffset); + } + + void movz2(Register src, Register dst) { + code.append(0x0f); + code.append(0xb7); + code.append(0xc0 | (dst << 3) | src); + } + + void movz2(Register src, int32_t srcOffset, Register dst) { + code.append(0x0f); + offsetInstruction(0xb7, 0, 0x40, 0x80, dst, src, srcOffset); + } + + void movs2(Register src, Register dst) { + code.append(0x0f); + code.append(0xbf); + code.append(0xc0 | (dst << 3) | src); + } + + void movs2(Register src, int32_t srcOffset, Register dst) { + code.append(0x0f); + offsetInstruction(0xbf, 0, 0x40, 0x80, dst, src, srcOffset); + } + + void movs4(Register src, int32_t srcOffset, Register dst) { + rex(); + offsetInstruction(0x63, 0, 0x40, 0x80, dst, src, srcOffset); + } + + void mov4(Register src, int32_t srcOffset, Register dst) { + offsetInstruction(0x8b, 0, 0x40, 0x80, dst, src, srcOffset); + } + + void mov1(Register src, Register dst, int32_t dstOffset) { + offsetInstruction(0x88, 0, 0x40, 0x80, src, dst, dstOffset); + } + + void mov2(Register src, Register dst, int32_t dstOffset) { + code.append(0x66); + offsetInstruction(0x89, 0, 0x40, 0x80, src, dst, dstOffset); + } + + void mov4(Register src, Register dst, int32_t dstOffset) { + offsetInstruction(0x89, 0, 0x40, 0x80, src, dst, dstOffset); + } + + void mov(Register src, int32_t srcOffset, SSERegister dst) { + code.append(0xf3); + code.append(0x0f); + offsetInstruction(0x7e, 0, 0x40, 0x80, dst, src, srcOffset); + } + + void mov(Register src, int32_t srcOffset, Register dst) { + rex(); + mov4(src, srcOffset, dst); + } + + void mov(Register src, Register dst, int32_t dstOffset) { + rex(); + mov4(src, dst, dstOffset); + } + + void mov(int32_t v, Register dst, int32_t dstOffset) { + rex(); + offsetInstruction(0xc7, 0, 0x40, 0x80, rax, dst, dstOffset); + code.append4(v); + } + + void mov(uintptr_t v, Register dst) { + rex(); + code.append(0xb8 | dst); + code.appendAddress(v); + } + + void alignedMov(uintptr_t v, Register dst) { + while ((code.length() + (BytesPerWord == 8 ? 2 : 1)) % BytesPerWord) { + nop(); + } + rex(); + code.append(0xb8 | dst); + code.appendAddress(v); + } + + void lea(Register src, int32_t srcOffset, Register dst) { + rex(); + offsetInstruction(0x8d, 0, 0x40, 0x80, dst, src, srcOffset); + } + + void nop() { + code.append(0x90); + } + + void push(Register reg) { + code.append(0x50 | reg); + } + + void push(Register reg, int32_t offset) { + offsetInstruction(0xff, 0x30, 0x70, 0xb0, rax, reg, offset); + } + + void push(int32_t v) { + if (isByte(v)) { + code.append(0x6a); + code.append(v); + } else { + code.append(0x68); + code.append4(v); + } + } + + void push4(Register reg, int32_t offset) { + if (BytesPerWord == 8) { + movs4(reg, offset, rsi); + push(rsi); + } else { + push(reg, offset); + } + } + + void pushAddress(uintptr_t v) { + if (BytesPerWord == 8 and not isInt32(v)) { + mov(v, rsi); + push(rsi); + } else { + push(v); + } + } + + void pop(Register dst) { + code.append(0x58 | dst); + } + + void pop(Register dst, int32_t offset) { + offsetInstruction(0x8f, 0, 0x40, 0x80, rax, dst, offset); + } + + void pop4(Register reg, int32_t offset) { + if (BytesPerWord == 8) { + pop(rsi); + mov4(rsi, reg, offset); + } else { + pop(reg, offset); + } + } + + void add(Register src, Register dst) { + rex(); + code.append(0x01); + code.append(0xc0 | (src << 3) | dst); + } + + void add(int32_t v, Register dst) { + assert(code.s, isByte(v)); // todo + + rex(); + code.append(0x83); + code.append(0xc0 | dst); + code.append(v); + } + + void add(int32_t v, Register dst, unsigned offset) { + rex(); + unsigned i = (isByte(v) ? 0x83 : 0x81); + offsetInstruction(i, 0, 0x40, 0x80, rax, dst, offset); + if (isByte(v)) { + code.append(v); + } else { + code.append4(v); + } + } + + void add(Register src, Register dst, unsigned dstOffset) { + rex(); + offsetInstruction(0x01, 0, 0x40, 0x80, src, dst, dstOffset); + } + + void adc(int32_t v, Register dst) { + assert(code.s, isByte(v)); // todo + + rex(); + code.append(0x83); + code.append(0xd0 | dst); + code.append(v); + } + + void adc(Register src, Register dst, unsigned dstOffset) { + rex(); + offsetInstruction(0x11, 0, 0x40, 0x80, src, dst, dstOffset); + } + + void sub(Register src, Register dst, unsigned dstOffset) { + rex(); + offsetInstruction(0x29, 0, 0x40, 0x80, src, dst, dstOffset); + } + + void sub(Register src, Register dst) { + rex(); + code.append(0x29); + code.append(0xc0 | (src << 3) | dst); + } + + void sub(int32_t v, Register dst) { + assert(code.s, isByte(v)); // todo + + rex(); + code.append(0x83); + code.append(0xe8 | dst); + code.append(v); + } + + void sbb(Register src, Register dst, unsigned dstOffset) { + rex(); + offsetInstruction(0x19, 0, 0x40, 0x80, src, dst, dstOffset); + } + + void or_(Register src, Register dst) { + rex(); + code.append(0x09); + code.append(0xc0 | (src << 3) | dst); + } + + void or_(Register src, Register dst, unsigned dstOffset) { + rex(); + offsetInstruction(0x09, 0, 0x40, 0x80, src, dst, dstOffset); + } + + void or_(int32_t v, Register dst) { + assert(code.s, isByte(v)); // todo + + rex(); + code.append(0x83); + code.append(0xc8 | dst); + code.append(v); + } + + void and_(Register src, Register dst) { + rex(); + code.append(0x21); + code.append(0xc0 | (src << 3) | dst); + } + + void and_(Register src, Register dst, unsigned dstOffset) { + rex(); + offsetInstruction(0x21, 0, 0x40, 0x80, src, dst, dstOffset); + } + + void and_(int32_t v, Register dst) { + rex(); + if (isByte(v)) { + code.append(0x83); + code.append(0xe0 | dst); + code.append(v); + } else { + code.append(0x81); + code.append(0xe0 | dst); + code.append(v); + } + } + + void shl(int8_t v, Register dst) { + rex(); + if (v == 1) { + code.append(0xd1); + code.append(0xe0 | dst); + } else { + code.append(0xc1); + code.append(0xe0 | dst); + code.append(v); + } + } + + void shl4_cl(Register dst, unsigned dstOffset) { + offsetInstruction(0xd3, 0x20, 0x60, 0xa0, rax, dst, dstOffset); + } + + void sar4_cl(Register dst, unsigned dstOffset) { + offsetInstruction(0xd3, 0x38, 0x7d, 0xb8, rax, dst, dstOffset); + } + + void shr4_cl(Register dst, unsigned dstOffset) { + offsetInstruction(0xd3, 0x28, 0x6d, 0xa8, rax, dst, dstOffset); + } + + void ret() { + code.append(0xc3); + } + + void jmp(Label& label) { + code.append(0xE9); + label.reference(); + } + + void jmp(unsigned javaIP) { + code.append(0xE9); + + jumps.append4(javaIP); + jumps.append4(code.length()); + + code.append4(0); + } + + void jmp(Register reg) { + code.append(0xff); + code.append(0xe0 | reg); + } + + void conditional(Label& label, unsigned condition) { + code.append(0x0f); + code.append(condition); + label.reference(); + } + + void conditional(unsigned javaIP, unsigned condition) { + code.append(0x0f); + code.append(condition); + + jumps.append4(javaIP); + jumps.append4(code.length()); + + code.append4(0); + } + + void je(Label& label) { + conditional(label, 0x84); + } + + void je(unsigned javaIP) { + conditional(javaIP, 0x84); + } + + void jne(Label& label) { + conditional(label, 0x85); + } + + void jne(unsigned javaIP) { + conditional(javaIP, 0x85); + } + + void jg(Label& label) { + conditional(label, 0x8f); + } + + void jg(unsigned javaIP) { + conditional(javaIP, 0x8f); + } + + void jge(Label& label) { + conditional(label, 0x8d); + } + + void jge(unsigned javaIP) { + conditional(javaIP, 0x8d); + } + + void jl(Label& label) { + conditional(label, 0x8c); + } + + void jl(unsigned javaIP) { + conditional(javaIP, 0x8c); + } + + void jle(Label& label) { + conditional(label, 0x8e); + } + + void jle(unsigned javaIP) { + conditional(javaIP, 0x8e); + } + + void jb(Label& label) { + conditional(label, 0x82); + } + + void ja(Label& label) { + conditional(label, 0x87); + } + + void cmp(int v, Register reg) { + assert(code.s, isByte(v)); // todo + + rex(); + code.append(0x83); + code.append(0xf8 | reg); + code.append(v); + } + + void cmp(Register a, Register b) { + rex(); + code.append(0x39); + code.append(0xc0 | (a << 3) | b); + } + + void call(Register reg) { + code.append(0xff); + code.append(0xd0 | reg); + } + + void cdq() { + code.append(0x99); + } + + void cqo() { + rex(); + cdq(); + } + + void imul4(Register src, unsigned srcOffset, Register dst) { + code.append(0x0f); + offsetInstruction(0xaf, 0, 0x40, 0x80, dst, src, srcOffset); + } + + void imul(Register src, unsigned srcOffset, Register dst) { + rex(); + imul4(src, srcOffset, dst); + } + + void imul(Register src) { + rex(); + code.append(0xf7); + code.append(0xe8 | src); + } + + void idiv(Register src) { + rex(); + code.append(0xf7); + code.append(0xf8 | src); + } + + void mul(Register src, unsigned offset) { + rex(); + offsetInstruction(0xf7, 0x20, 0x60, 0xa0, rax, src, offset); + } + + void neg(Register reg, unsigned offset) { + rex(); + offsetInstruction(0xf7, 0x10, 0x50, 0x90, rax, reg, offset); + } + + void neg(Register reg) { + rex(); + code.append(0xf7); + code.append(0xd8 | reg); + } + + void int3() { + code.append(0xcc); + } + + Buffer code; + Buffer jumps; +}; + +Register +gpRegister(Thread* t, unsigned index) +{ + switch (index) { + case 0: + return rdi; + case 1: + return rsi; + case 2: + return rdx; + case 3: + return rcx; + case 4: + return r8; + case 5: + return r9; + default: + abort(t); + } +} + +SSERegister +sseRegister(Thread* t UNUSED, unsigned index) +{ + assert(t, index < 8); + return static_cast(index); +} + +unsigned +parameterOffset(unsigned index) +{ + return FrameFootprint + ((index + 2) * BytesPerWord); +} + +Compiled* +caller(MyThread* t); + +class Compiler: public Assembler { + public: + Compiler(MyThread* t): + Assembler(t->m->system), + t(t) + { } + + unsigned threadFrameDataOffset() { + return reinterpret_cast(&(t->frameData)) + - reinterpret_cast(t); + } + + Compiled* compileStub() { + push(rbp); + mov(rsp, rbp); + + if (BytesPerWord == 4) { + push(rbp, FrameMethod); + push(rbp, FrameThread); + } else { + mov(rbp, FrameMethod, rsi); + mov(rbp, FrameThread, rdi); + } + + mov(reinterpret_cast(compileMethod), rbx); + callAddress(compiledCode(caller(t))); + + if (BytesPerWord == 4) { + add(BytesPerWord * 2, rsp); + } + + mov(rbp, FrameMethod, rax); + mov(rax, MethodCompiled, rax); // load compiled code + + mov(rbp, rsp); + pop(rbp); + + add(CompiledBody, rax); + jmp(rax); // call compiled code + + return finish(); + } + + Compiled* compileNativeInvoker() { + push(rbp); + mov(rsp, rbp); + + if (BytesPerWord == 4) { + push(rbp, FrameMethod); + push(rbp, FrameThread); + } else { + mov(rbp, FrameMethod, rsi); + mov(rbp, FrameThread, rdi); + } + + mov(reinterpret_cast(invokeNative), rbx); + callAddress(compiledCode(caller(t))); + + if (BytesPerWord == 4) { + add(BytesPerWord * 2, rsp); + } + + mov(rbp, rsp); + pop(rbp); + ret(); + + return finish(); + } + + Compiled* compileCaller() { + mov(rbp, FrameThread, rdi); + mov(rbp, rdi, threadFrameDataOffset()); + mov(rsp, 0, rcx); + mov(rcx, rdi, threadFrameDataOffset() + BytesPerWord); + + jmp(rbx); + + return finish(); + } + + Compiled* finish() { + return makeCompiled(t, &code); + } + + void callAddress(void* function) { + mov(reinterpret_cast(function), rax); + call(rax); + } + + void callAlignedAddress(void* function) { + alignedMov(reinterpret_cast(function), rax); + call(rax); + } + + MyThread* t; +}; + +class JavaCompiler: public Compiler { + public: + JavaCompiler(MyThread* t, object method): + Compiler(t), + method(method), + stackMapper(t, method), + poolRegisterClobbered(true), + machineIPs(static_cast + (t->m->system->allocate + (codeLength(t, methodCode(t, method)) * 4))), + lineNumbers(codeLineNumberTable(t, methodCode(t, method)) ? + static_cast + (t->m->system->allocate + (lineNumberTableLength + (t, codeLineNumberTable(t, methodCode(t, method))) + * sizeof(NativeLineNumber))) : 0), + exceptionHandlers(codeExceptionHandlerTable(t, methodCode(t, method)) ? + static_cast + (t->m->system->allocate + (exceptionHandlerTableLength + (t, codeExceptionHandlerTable + (t, methodCode(t, method))) + * sizeof(NativeExceptionHandler))) : 0), + pool(t->m->system, 256), + protector(this) + { } + + ~JavaCompiler() { + if (machineIPs) { + t->m->system->free(machineIPs); + } + + if (lineNumbers) { + t->m->system->free(lineNumbers); + } + + if (exceptionHandlers) { + t->m->system->free(exceptionHandlers); + } + } + + void pushInt() { + sub(BytesPerWord, rsp); + stackMapper.pushedInt(); + } + + void pushInt(int32_t v) { + push(v); + stackMapper.pushedInt(); + } + + void pushInt(Register r) { + push(r); + stackMapper.pushedInt(); + } + + void pushInt(Register r, int32_t offset) { + push(r, offset); + stackMapper.pushedInt(); + } + + void pushInt4(Register r, int32_t offset) { + push4(r, offset); + stackMapper.pushedInt(); + } + + void pushObject(int32_t v) { + push(v); + stackMapper.pushedObject(); + } + + void pushObject(Register r) { + push(r); + stackMapper.pushedObject(); + } + + void pushObject(Register r, int32_t offset) { + push(r, offset); + stackMapper.pushedObject(); + } + + void pushLongQuiet(uint64_t v) { + if (BytesPerWord == 8) { + pushAddress(v); + sub(8, rsp); + } else { + push((v >> 32) & 0xFFFFFFFF); + push((v ) & 0xFFFFFFFF); + } + } + + void pushLong(uint64_t v) { + pushLongQuiet(v); + stackMapper.pushedLong(); + } + + void pushLong(Register r) { + assert(t, BytesPerWord == 8); + push(r); + sub(8, rsp); + stackMapper.pushedLong(); + } + + void pushLongQuiet(Register r, int32_t offset) { + assert(t, BytesPerWord == 8); + push(r, offset); + sub(8, rsp); + } + + void pushLong(Register r, int32_t offset) { + pushLongQuiet(r, offset); + stackMapper.pushedLong(); + } + + void pushLong(Register low, Register high) { + assert(t, BytesPerWord == 4); + push(high); + push(low); + stackMapper.pushedLong(); + } + + void pushLongQuiet(Register low, int32_t lowOffset, + Register high, int32_t highOffset) + { + assert(t, BytesPerWord == 4); + push(high, highOffset); + push(low, lowOffset); + } + + void pushLong(Register low, int32_t lowOffset, + Register high, int32_t highOffset) + { + pushLongQuiet(low, lowOffset, high, highOffset); + stackMapper.pushedLong(); + } + + void popInt() { + add(BytesPerWord, rsp); + stackMapper.poppedInt(); + } + + void popInt(Register r) { + pop(r); + stackMapper.poppedInt(); + } + + void popInt(Register r, int32_t offset) { + pop(r, offset); + stackMapper.poppedInt(); + } + + void popInt4(Register r, int32_t offset) { + pop4(r, offset); + stackMapper.poppedInt(); + } + + void popObject(Register r) { + pop(r); + stackMapper.poppedObject(); + } + + void popObject(Register r, int32_t offset) { + pop(r, offset); + stackMapper.poppedObject(); + } + + void popLong() { + add(BytesPerWord * 2, rsp); + stackMapper.poppedLong(); + } + + void popLong(Register r) { + assert(t, BytesPerWord == 8); + add(BytesPerWord, rsp); + pop(r); + stackMapper.poppedLong(); + } + + void popLong(Register r, int32_t offset) { + assert(t, BytesPerWord == 8); + add(BytesPerWord, rsp); + pop(r, offset); + stackMapper.poppedLong(); + } + + void popLong(Register low, Register high) { + assert(t, BytesPerWord == 4); + pop(low); + pop(high); + stackMapper.poppedLong(); + } + + void popLong(Register low, int32_t lowOffset, + Register high, int32_t highOffset) + { + assert(t, BytesPerWord == 4); + pop(low, lowOffset); + pop(high, highOffset); + stackMapper.poppedLong(); + } + + void loadInt(unsigned index) { + pushInt(rbp, localOffset(t, index, method)); + } + + void loadObject(unsigned index) { + pushObject(rbp, localOffset(t, index, method)); + } + + void loadLong(unsigned index) { + if (BytesPerWord == 8) { + pushLong(rbp, localOffset(t, index, method)); + } else { + pushLong(rbp, localOffset(t, index + 1, method), + rbp, localOffset(t, index, method)); + } + } + + void storeInt(unsigned index) { + popInt(rbp, localOffset(t, index, method)); + stackMapper.storedInt(index); + } + + void storeObject(unsigned index) { + popObject(rbp, localOffset(t, index, method)); + stackMapper.storedObject(index); + } + + void storeLong(unsigned index) { + if (BytesPerWord == 8) { + popLong(rbp, localOffset(t, index, method)); + } else { + popLong(rbp, localOffset(t, index, method), + rbp, localOffset(t, index + 1, method)); + } + stackMapper.storedLong(index); + } + + void pushReturnValue(unsigned code) { + switch (code) { + case ByteField: + case BooleanField: + case CharField: + case ShortField: + case FloatField: + case IntField: + push(rax); + stackMapper.pushedInt(); + break; + + case ObjectField: + push(rax); + stackMapper.pushedObject(); + break; + + case LongField: + case DoubleField: + push(rax); + push(rdx); + stackMapper.pushedLong(); + break; + + case VoidField: + break; + + default: + abort(t); + } + } + + void compileDirectInvoke(object target) { + unsigned footprint = FrameFootprint + + (methodParameterFootprint(t, target) * BytesPerWord); + + Compiled* code = reinterpret_cast(methodCompiled(t, target)); + + push(rsp); + push(poolRegister(), poolReference(target)); + push(rbp, FrameThread); + + callAlignedAddress(compiledCode(code)); + + stackMapper.called(this->code.length()); + poolRegisterClobbered = true; + + add(footprint, rsp); // pop arguments + stackMapper.popped(methodParameterFootprint(t, target)); + + pushReturnValue(methodReturnCode(t, target)); + } + + void compileThrowNew(Machine::Type type) { + object class_ = arrayBody(t, t->m->types, type); + + if (BytesPerWord == 8) { + mov(poolRegister(), poolReference(class_), rsi); + mov(rbp, FrameThread, rdi); + } else { + push(poolRegister(), poolReference(class_)); + push(rbp, FrameThread); + } + + indirectCall(reinterpret_cast(throwNew)); + } + + void directCall(void* function) { + callAddress(function); + + poolRegisterClobbered = true; + } + + void indirectCall(void* function) { + mov(reinterpret_cast(function), rbx); + callAddress(compiledCode(caller(t))); + + stackMapper.called(code.length()); + poolRegisterClobbered = true; + } + + void compileCall(bool direct, void* function, Register arg1, Register arg2) { + if (BytesPerWord == 8) { + mov(arg2, rdx); + mov(arg1, rsi); + mov(rbp, FrameThread, rdi); + } else { + push(arg2); + push(arg1); + push(rbp, FrameThread); + } + + if (direct) { + directCall(function); + } else { + indirectCall(function); + } + + if (BytesPerWord == 4) { + add(BytesPerWord * 3, rsp); + } + } + + void compileCall(bool direct, void* function, uintptr_t arg1, Register arg2) + { + if (BytesPerWord == 8) { + mov(arg2, rdx); + mov(arg1, rsi); + mov(rbp, FrameThread, rdi); + } else { + push(arg2); + push(arg1); + push(rbp, FrameThread); + } + + if (direct) { + directCall(function); + } else { + indirectCall(function); + } + + if (BytesPerWord == 4) { + add(BytesPerWord * 3, rsp); + } + } + + void compileCall(bool direct, void* function, object arg1, Register arg2) { + if (BytesPerWord == 8) { + mov(arg2, rdx); + mov(poolRegister(), poolReference(arg1), rsi); + mov(rbp, FrameThread, rdi); + } else { + push(arg2); + push(poolRegister(), poolReference(arg1)); + push(rbp, FrameThread); + } + + if (direct) { + directCall(function); + } else { + indirectCall(function); + } + + if (BytesPerWord == 4) { + add(BytesPerWord * 3, rsp); + } + } + + void compileCall(bool direct, void* function, object arg1, Register arg2, + int32_t arg2offset) + { + if (BytesPerWord == 8) { + mov(arg2, arg2offset, rdx); + mov(poolRegister(), poolReference(arg1), rsi); + mov(rbp, FrameThread, rdi); + } else { + push(arg2, arg2offset); + push(poolRegister(), poolReference(arg1)); + push(rbp, FrameThread); + } + + if (direct) { + directCall(function); + } else { + indirectCall(function); + } + + if (BytesPerWord == 4) { + add(BytesPerWord * 3, rsp); + } + } + + void compileCall(bool direct, void* function, object arg1) { + if (BytesPerWord == 8) { + mov(poolRegister(), poolReference(arg1), rsi); + mov(rbp, FrameThread, rdi); + } else { + push(poolRegister(), poolReference(arg1)); + push(rbp, FrameThread); + } + + if (direct) { + directCall(function); + } else { + indirectCall(function); + } + + if (BytesPerWord == 4) { + add(BytesPerWord * 2, rsp); + } + } + + Compiled* compile() { + object code = methodCode(t, method); + PROTECT(t, code); + + unsigned parameterFootprint + = methodParameterFootprint(t, method) * BytesPerWord; + + unsigned localFootprint = codeMaxLocals(t, code) * BytesPerWord; + + push(rbp); + mov(rsp, rbp); + + if (localFootprint > parameterFootprint) { + // reserve space for local variables + sub(localFootprint - parameterFootprint, rsp); + } + + int lineNumberIndex; + object lnt = codeLineNumberTable(t, code); + if (lnt and lineNumberTableLength(t, lnt)) { + lineNumberIndex = 0; + } else { + lineNumberIndex = -1; + } + + for (unsigned ip = 0; ip < codeLength(t, code);) { + stackMapper.newIp(ip); + machineIPs[ip] = this->code.length(); + + if (lineNumberIndex >= 0) { + object lnt = codeLineNumberTable(t, code); + LineNumber* ln = lineNumberTableBody(t, lnt, lineNumberIndex); + + if (lineNumberIp(ln) == ip) { + nativeLineNumberIp(lineNumbers + lineNumberIndex) + = this->code.length(); + nativeLineNumberLine(lineNumbers + lineNumberIndex) + = lineNumberLine(ln); + if (static_cast(lineNumberIndex) + 1 + < lineNumberTableLength(t, lnt)) + { + ++ lineNumberIndex; + } else { + lineNumberIndex = -1; + } + } + } + + unsigned instruction = codeBody(t, code, ip++); + + switch (instruction) { + case aaload: + case baload: + case caload: + case daload: + case faload: + case iaload: + case laload: + case saload: { + Label next(this); + Label outOfBounds(this); + + popInt(rcx); + popObject(rax); + + cmp(0, rcx); + jl(outOfBounds); + + mov(rax, BytesPerWord, rdx); + cmp(rdx, rcx); + jge(outOfBounds); + + add(BytesPerWord * 2, rax); + + switch (instruction) { + case aaload: + case faload: + case iaload: + shl(log(BytesPerWord), rcx); + add(rcx, rax); + + if (instruction == aaload) { + pushObject(rax, 0); + } else { + pushInt4(rax, 0); + } + break; + + case baload: + add(rcx, rax); + movs1(rax, 0, rax); + pushInt(rax); + break; + + case caload: + shl(1, rcx); + add(rcx, rax); + movz2(rax, 0, rax); + pushInt(rax); + break; + + case daload: + case laload: + shl(3, rcx); + add(rcx, rax); + pushLong(rax, 0); + break; + + case saload: + shl(1, rcx); + add(rcx, rax); + movs2(rax, 0, rax); + pushInt(rax); + break; + } + + jmp(next); + + outOfBounds.mark(); + compileThrowNew(Machine::ArrayIndexOutOfBoundsExceptionType); + + next.mark(); + } break; + + case aastore: + case bastore: + case castore: + case dastore: + case fastore: + case iastore: + case lastore: + case sastore: { + Label next(this); + Label outOfBounds(this); + + if (instruction == dastore or instruction == lastore) { + popLong(rdx, rbx); + } else if (instruction == aastore) { + popObject(rbx); + } else { + popInt(rbx); + } + + popInt(rcx); + popObject(rax); + + cmp(0, rcx); + jl(outOfBounds); + + mov(rax, BytesPerWord, rsi); + cmp(rsi, rcx); + jge(outOfBounds); + + switch (instruction) { + case aastore: + shl(log(BytesPerWord), rcx); + add(ArrayBody, rcx); + + if (BytesPerWord == 8) { + mov(rcx, rdx); + mov(rbx, rcx); + mov(rax, rsi); + mov(rbp, FrameThread, rdi); + } else { + push(rbx); + push(rcx); + push(rax); + push(rbp, FrameThread); + } + + directCall(reinterpret_cast(set)); + + if (BytesPerWord == 4) { + add(BytesPerWord * 4, rsp); + } + break; + + case fastore: + case iastore: + shl(log(BytesPerWord), rcx); + add(ArrayBody, rcx); + add(rcx, rax); + mov(rbx, rax, 0); + break; + + case bastore: + add(ArrayBody, rcx); + add(rcx, rax); + mov1(rbx, rax, 0); + break; + + case castore: + case sastore: + shl(1, rcx); + add(ArrayBody, rcx); + add(rcx, rax); + mov2(rbx, rax, 0); + break; + + case dastore: + case lastore: + shl(3, rcx); + add(ArrayBody, rcx); + add(rcx, rax); + mov4(rbx, rax, 0); + mov4(rdx, rax, 4); + break; + } + + jmp(next); + + outOfBounds.mark(); + compileThrowNew(Machine::ArrayIndexOutOfBoundsExceptionType); + + next.mark(); + } break; + + case aconst_null: + pushObject(0); + break; + + case aload: + loadObject(codeBody(t, code, ip++)); + break; + + case aload_0: + loadObject(0); + break; + + case aload_1: + loadObject(1); + break; + + case aload_2: + loadObject(2); + break; + + case aload_3: + loadObject(3); + break; + + case anewarray: { + uint16_t index = codeReadInt16(t, code, ip); + + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + + Label nonnegative(this); + + popInt(rax); + cmp(0, rax); + jge(nonnegative); + + compileThrowNew(Machine::NegativeArraySizeExceptionType); + + nonnegative.mark(); + + compileCall(false, reinterpret_cast(makeBlankObjectArray), + class_, rax); + + pushObject(rax); + } break; + + case areturn: + popObject(rax); + mov(rbp, rsp); + pop(rbp); + ret(); + stackMapper.exited(); + break; + + case arraylength: + popObject(rax); + pushInt(rax, BytesPerWord); + break; + + case astore: + storeObject(codeBody(t, code, ip++)); + break; + + case astore_0: + storeObject(0); + break; + + case astore_1: + storeObject(1); + break; + + case astore_2: + storeObject(2); + break; + + case astore_3: + storeObject(3); + break; + + case athrow: + if (BytesPerWord == 8) { + popObject(rsi); + mov(rbp, FrameThread, rdi); + } else { + push(rbp, FrameThread); + } + + indirectCall(reinterpret_cast(throw_)); + stackMapper.exited(); + break; + + case bipush: { + pushInt(static_cast(codeBody(t, code, ip++))); + } break; + + case checkcast: { + uint16_t index = codeReadInt16(t, code, ip); + + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + + Label next(this); + + mov(rsp, 0, rax); + cmp(0, rax); + je(next); + + mov(poolRegister(), poolReference(class_), rcx); + mov(rax, 0, rax); + and_(static_cast(PointerMask), rax); + cmp(rcx, rax); + je(next); + + compileCall(true, reinterpret_cast(isAssignableFrom), rcx, rax); + + cmp(0, rax); + jne(next); + + compileThrowNew(Machine::ClassCastExceptionType); + + next.mark(); + } break; + + case dadd: { + if (BytesPerWord == 8) { + popLong(rdi); + popLong(rsi); + directCall(reinterpret_cast(addDouble)); + pushLong(rax); + } else { + directCall(reinterpret_cast(addDouble)); + add(BytesPerWord * 4, rsp); + stackMapper.poppedLong(); + stackMapper.poppedLong(); + pushLong(rax, rdx); + } + } break; + + case dcmpg: { + if (BytesPerWord == 8) { + popLong(rdi); + popLong(rsi); + directCall(reinterpret_cast(compareDoublesG)); + pushInt(rax); + } else { + directCall(reinterpret_cast(compareDoublesG)); + add(BytesPerWord * 4, rsp); + stackMapper.poppedLong(); + stackMapper.poppedLong(); + pushInt(rax); + } + } break; + + case dcmpl: { + if (BytesPerWord == 8) { + popLong(rdi); + popLong(rsi); + directCall(reinterpret_cast(compareDoublesL)); + pushInt(rax); + } else { + directCall(reinterpret_cast(compareDoublesL)); + add(BytesPerWord * 4, rsp); + stackMapper.poppedLong(); + stackMapper.poppedLong(); + pushInt(rax); + } + } break; + + case dconst_0: { + pushLong(doubleToBits(0.0)); + } break; + + case dconst_1: { + pushLong(doubleToBits(1.0)); + } break; + + case ddiv: { + if (BytesPerWord == 8) { + popLong(rdi); + popLong(rsi); + directCall(reinterpret_cast(divideDouble)); + pushLong(rax); + } else { + directCall(reinterpret_cast(divideDouble)); + add(BytesPerWord * 4, rsp); + stackMapper.poppedLong(); + stackMapper.poppedLong(); + pushLong(rax, rdx); + } + } break; + + case dmul: { + if (BytesPerWord == 8) { + popLong(rdi); + popLong(rsi); + directCall(reinterpret_cast(multiplyDouble)); + pushLong(rax); + } else { + directCall(reinterpret_cast(multiplyDouble)); + add(BytesPerWord * 4, rsp); + stackMapper.poppedLong(); + stackMapper.poppedLong(); + pushLong(rax, rdx); + } + } break; + + case vm::drem: { + if (BytesPerWord == 8) { + popLong(rdi); + popLong(rsi); + directCall(reinterpret_cast(moduloDouble)); + pushLong(rax); + } else { + directCall(reinterpret_cast(moduloDouble)); + add(BytesPerWord * 4, rsp); + stackMapper.poppedLong(); + stackMapper.poppedLong(); + pushLong(rax, rdx); + } + } break; + + case dsub: { + if (BytesPerWord == 8) { + popLong(rdi); + popLong(rsi); + directCall(reinterpret_cast(subtractDouble)); + pushLong(rax); + } else { + directCall(reinterpret_cast(subtractDouble)); + add(BytesPerWord * 4, rsp); + stackMapper.poppedLong(); + stackMapper.poppedLong(); + pushLong(rax, rdx); + } + } break; + + case dup: + push(rsp, 0); + stackMapper.dupped(); + break; + + case dup_x1: + mov(rsp, BytesPerWord, rcx); + mov(rsp, 0, rax); + mov(rax, rsp, BytesPerWord); + mov(rcx, rsp, 0); + push(rax); + stackMapper.duppedX1(); + break; + + case dup_x2: + mov(rsp, BytesPerWord * 2, rdx); + mov(rsp, BytesPerWord, rcx); + mov(rsp, 0, rax); + mov(rax, rsp, BytesPerWord * 2); + mov(rdx, rsp, BytesPerWord); + mov(rcx, rsp, 0); + push(rax); + stackMapper.duppedX2(); + break; + + case dup2: + push(rsp, BytesPerWord); + push(rsp, BytesPerWord); + stackMapper.dupped2(); + break; + + case dup2_x1: + mov(rsp, BytesPerWord * 2, rdx); + mov(rsp, BytesPerWord, rcx); + mov(rsp, 0, rax); + mov(rcx, rsp, BytesPerWord * 2); + mov(rax, rsp, BytesPerWord); + mov(rdx, rsp, 0); + push(rcx); + push(rax); + stackMapper.dupped2X1(); + break; + + case dup2_x2: + mov(rsp, BytesPerWord * 3, rbx); + mov(rsp, BytesPerWord * 2, rdx); + mov(rsp, BytesPerWord, rcx); + mov(rsp, 0, rax); + mov(rcx, rsp, BytesPerWord * 3); + mov(rax, rsp, BytesPerWord * 2); + mov(rbx, rsp, BytesPerWord); + mov(rdx, rsp, 0); + push(rcx); + push(rax); + stackMapper.dupped2X2(); + break; + + case fadd: { + if (BytesPerWord == 8) { + popInt(rdi); + popInt(rsi); + directCall(reinterpret_cast(addFloat)); + pushInt(rax); + } else { + directCall(reinterpret_cast(addFloat)); + add(BytesPerWord * 2, rsp); + stackMapper.poppedInt(); + stackMapper.poppedInt(); + pushInt(rax); + } + } break; + + case fcmpg: { + if (BytesPerWord == 8) { + popInt(rdi); + popInt(rsi); + directCall(reinterpret_cast(compareFloatsG)); + pushInt(rax); + } else { + directCall(reinterpret_cast(compareFloatsG)); + add(BytesPerWord * 2, rsp); + stackMapper.poppedInt(); + stackMapper.poppedInt(); + pushInt(rax); + } + } break; + + case fcmpl: { + if (BytesPerWord == 8) { + popInt(rdi); + popInt(rsi); + directCall(reinterpret_cast(compareFloatsL)); + pushInt(rax); + } else { + directCall(reinterpret_cast(compareFloatsL)); + add(BytesPerWord * 2, rsp); + stackMapper.poppedInt(); + stackMapper.poppedInt(); + pushInt(rax); + } + } break; + + case fconst_0: { + pushInt(floatToBits(0.0)); + } break; + + case fconst_1: { + pushInt(floatToBits(1.0)); + } break; + + case fconst_2: { + pushInt(floatToBits(2.0)); + } break; + + case fdiv: { + if (BytesPerWord == 8) { + popInt(rdi); + popInt(rsi); + directCall(reinterpret_cast(divideFloat)); + pushInt(rax); + } else { + directCall(reinterpret_cast(divideFloat)); + add(BytesPerWord * 2, rsp); + stackMapper.poppedInt(); + stackMapper.poppedInt(); + pushInt(rax); + } + } break; + + case fmul: { + if (BytesPerWord == 8) { + popInt(rdi); + popInt(rsi); + directCall(reinterpret_cast(multiplyFloat)); + pushInt(rax); + } else { + directCall(reinterpret_cast(multiplyFloat)); + add(BytesPerWord * 2, rsp); + stackMapper.poppedInt(); + stackMapper.poppedInt(); + pushInt(rax); + } + } break; + + case frem: { + if (BytesPerWord == 8) { + popInt(rdi); + popInt(rsi); + directCall(reinterpret_cast(moduloFloat)); + pushInt(rax); + } else { + directCall(reinterpret_cast(moduloFloat)); + add(BytesPerWord * 2, rsp); + stackMapper.poppedInt(); + stackMapper.poppedInt(); + pushInt(rax); + } + } break; + + case fsub: { + if (BytesPerWord == 8) { + popInt(rdi); + popInt(rsi); + directCall(reinterpret_cast(subtractFloat)); + pushInt(rax); + } else { + directCall(reinterpret_cast(subtractFloat)); + add(BytesPerWord * 2, rsp); + stackMapper.poppedInt(); + stackMapper.poppedInt(); + pushInt(rax); + } + } break; + + case getfield: { + uint16_t index = codeReadInt16(t, code, ip); + + object field = resolveField(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + + popObject(rax); + + switch (fieldCode(t, field)) { + case ByteField: + case BooleanField: + movs1(rax, fieldOffset(t, field), rax); + pushInt(rax); + break; + + case CharField: + movz2(rax, fieldOffset(t, field), rax); + pushInt(rax); + break; + + case ShortField: + movs2(rax, fieldOffset(t, field), rax); + pushInt(rax); + break; + + case FloatField: + case IntField: + pushInt4(rax, fieldOffset(t, field)); + break; + + case DoubleField: + case LongField: + pushLong(rax, fieldOffset(t, field)); + break; + + case ObjectField: + pushObject(rax, fieldOffset(t, field)); + break; + + default: + abort(t); + } + } break; + + case getstatic: { + uint16_t index = codeReadInt16(t, code, ip); + + object field = resolveField(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + PROTECT(t, field); + + initClass(t, fieldClass(t, field)); + if (UNLIKELY(t->exception)) return 0; + + object table = classStaticTable(t, fieldClass(t, field)); + unsigned offset = (fieldOffset(t, field) * BytesPerWord) + ArrayBody; + + mov(poolRegister(), poolReference(table), rax); + + switch (fieldCode(t, field)) { + case ByteField: + case BooleanField: + case CharField: + case ShortField: + case FloatField: + case IntField: { + Label zero(this); + Label next(this); + + mov(rax, offset, rax); + cmp(0, rax); + je(zero); + + push4(rax, IntValue); + jmp(next); + + zero.mark(); + push(0); + + next.mark(); + stackMapper.pushedInt(); + } break; + + case DoubleField: + case LongField: { + Label zero(this); + Label next(this); + + mov(rax, offset, rax); + cmp(0, rax); + je(zero); + + if (BytesPerWord == 8) { + pushLongQuiet(rax, LongValue); + } else { + pushLongQuiet(rax, LongValue); + pushLongQuiet(rax, LongValue + 4); + } + jmp(next); + + zero.mark(); + pushLongQuiet(0); + + next.mark(); + stackMapper.pushedLong(); + } break; + + case ObjectField: { + pushObject(rax, offset); + } break; + + default: abort(t); + } + } break; + + case goto_: { + int16_t offset = codeReadInt16(t, code, ip); + jmp((ip - 3) + offset); + stackMapper.jumped((ip - 3) + offset); + } break; + + case goto_w: { + int32_t offset = codeReadInt32(t, code, ip); + jmp((ip - 5) + offset); + stackMapper.jumped((ip - 5) + offset); + } break; + + case i2b: + mov(rsp, 0, rax); + movs1(rax, rax); + mov(rax, rsp, 0); + break; + + case i2c: + mov(rsp, 0, rax); + movz2(rax, rax); + mov(rax, rsp, 0); + break; + + case i2s: + mov(rsp, 0, rax); + movs2(rax, rax); + mov(rax, rsp, 0); + break; + + case i2l: + if (BytesPerWord == 8) { + pushInt(); + } else { + popInt(rax); + cdq(); + pushLong(rax, rdx); + } + break; + + case iadd: + popInt(rax); + popInt(rcx); + add(rcx, rax); + pushInt(rax); + break; + + case iand: + popInt(rax); + and_(rax, rsp, 0); + break; + + case iconst_m1: + pushInt(-1); + break; + + case iconst_0: + pushInt(0); + break; + + case iconst_1: + pushInt(1); + break; + + case iconst_2: + pushInt(2); + break; + + case iconst_3: + pushInt(3); + break; + + case iconst_4: + pushInt(4); + break; + + case iconst_5: + pushInt(5); + break; + + case vm::idiv: + popInt(rcx); + popInt(rax); + cqo(); + Assembler::idiv(rcx); + pushInt(rax); + break; + + case if_acmpeq: { + int16_t offset = codeReadInt16(t, code, ip); + + popObject(rax); + popObject(rcx); + cmp(rax, rcx); + je((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case if_acmpne: { + int16_t offset = codeReadInt16(t, code, ip); + + popObject(rax); + popObject(rcx); + cmp(rax, rcx); + jne((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case if_icmpeq: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + popInt(rcx); + cmp(rax, rcx); + je((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case if_icmpne: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + popInt(rcx); + cmp(rax, rcx); + jne((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case if_icmpgt: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + popInt(rcx); + cmp(rax, rcx); + jg((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case if_icmpge: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + popInt(rcx); + cmp(rax, rcx); + jge((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case if_icmplt: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + popInt(rcx); + cmp(rax, rcx); + jl((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case if_icmple: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + popInt(rcx); + cmp(rax, rcx); + jle((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case ifeq: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + cmp(0, rax); + je((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case ifnull: { + int16_t offset = codeReadInt16(t, code, ip); + + popObject(rax); + cmp(0, rax); + je((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case ifne: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + cmp(0, rax); + jne((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case ifnonnull: { + int16_t offset = codeReadInt16(t, code, ip); + + popObject(rax); + cmp(0, rax); + jne((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case ifgt: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + cmp(0, rax); + jg((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case ifge: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + cmp(0, rax); + jge((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case iflt: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + cmp(0, rax); + jl((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case ifle: { + int16_t offset = codeReadInt16(t, code, ip); + + popInt(rax); + cmp(0, rax); + jle((ip - 3) + offset); + stackMapper.branched((ip - 3) + offset); + } break; + + case iinc: { + uint8_t index = codeBody(t, code, ip++); + int8_t c = codeBody(t, code, ip++); + + add(c, rbp, localOffset(t, index, method)); + } break; + + case iload: + case fload: + loadInt(codeBody(t, code, ip++)); + break; + + case iload_0: + case fload_0: + loadInt(0); + break; + + case iload_1: + case fload_1: + loadInt(1); + break; + + case iload_2: + case fload_2: + loadInt(2); + break; + + case iload_3: + case fload_3: + loadInt(3); + break; + + case vm::imul: + popInt(rax); + popInt(rcx); + Assembler::imul(rcx); + pushInt(rax); + break; + + case ineg: + neg(rsp, 0); + break; + + case instanceof: { + uint16_t index = codeReadInt16(t, code, ip); + + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + + Label call(this); + Label zero(this); + Label next(this); + + popObject(rax); + cmp(0, rax); + je(zero); + + mov(poolRegister(), poolReference(class_), rcx); + mov(rax, 0, rax); + and_(static_cast(PointerMask), rax); + cmp(rcx, rax); + jne(call); + + push(1); + jmp(next); + + call.mark(); + + compileCall(true, reinterpret_cast(isAssignableFrom), rcx, rax); + + push(rax); + jmp(next); + + zero.mark(); + push(0); + + next.mark(); + + stackMapper.pushedInt(); + } break; + + case invokeinterface: { + uint16_t index = codeReadInt16(t, code, ip); + ip += 2; + + object target = resolveMethod(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + + unsigned parameterFootprint + = methodParameterFootprint(t, target) * BytesPerWord; + + unsigned instance = parameterFootprint - BytesPerWord; + + unsigned footprint = FrameFootprint + parameterFootprint; + + compileCall(false, + reinterpret_cast(findInterfaceMethodFromInstance), + target, rsp, instance); + + push(rsp); + push(rax); + push(rbp, FrameThread); + + mov(rax, MethodCompiled, rax); // load compiled code + add(CompiledBody, rax); + call(rax); // call compiled code + + stackMapper.called(this->code.length()); + poolRegisterClobbered = true; + + add(footprint, rsp); // pop arguments + stackMapper.popped(methodParameterFootprint(t, target)); + + pushReturnValue(methodReturnCode(t, target)); + } break; + + case invokespecial: { + uint16_t index = codeReadInt16(t, code, ip); + + object target = resolveMethod(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + + object class_ = methodClass(t, target); + if (isSpecialMethod(t, target, class_)) { + target = findMethod(t, target, classSuper(t, class_)); + } + + compileDirectInvoke(target); + } break; + + case invokestatic: { + uint16_t index = codeReadInt16(t, code, ip); + + object target = resolveMethod(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + PROTECT(t, target); + + initClass(t, methodClass(t, target)); + if (UNLIKELY(t->exception)) return 0; + + compileDirectInvoke(target); + } break; + + case invokevirtual: { + uint16_t index = codeReadInt16(t, code, ip); + + object target = resolveMethod(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + + unsigned parameterFootprint + = methodParameterFootprint(t, target) * BytesPerWord; + + unsigned instance = parameterFootprint - BytesPerWord; + + unsigned footprint = FrameFootprint + parameterFootprint; + + unsigned offset = ArrayBody + (methodOffset(t, target) * BytesPerWord); + + mov(rsp, instance, rax); // load instance + mov(rax, 0, rax); // load class + and_(static_cast(PointerMask), rax); // clean pointer + mov(rax, ClassVirtualTable, rax); // load vtable + mov(rax, offset, rax); // load method + + push(rsp); + push(rax); + push(rbp, FrameThread); + + mov(rax, MethodCompiled, rax); // load compiled code + add(CompiledBody, rax); + call(rax); // call compiled code + + stackMapper.called(this->code.length()); + poolRegisterClobbered = true; + + add(footprint, rsp); // pop arguments + stackMapper.popped(methodParameterFootprint(t, target)); + + pushReturnValue(methodReturnCode(t, target)); + } break; + + case ior: + popInt(rax); + or_(rax, rsp, 0); + break; + + case irem: + popInt(rcx); + popInt(rax); + cqo(); + Assembler::idiv(rcx); + pushInt(rdx); + break; + + case ireturn: + case freturn: + popInt(rax); + mov(rbp, rsp); + pop(rbp); + ret(); + stackMapper.exited(); + break; + + case ishl: + popInt(rcx); + shl4_cl(rsp, 0); + break; + + case ishr: + popInt(rcx); + sar4_cl(rsp, 0); + break; + + case istore: + case fstore: + storeInt(codeBody(t, code, ip++)); + break; + + case istore_0: + case fstore_0: + storeInt(0); + break; + + case istore_1: + case fstore_1: + storeInt(1); + break; + + case istore_2: + case fstore_2: + storeInt(2); + break; + + case istore_3: + case fstore_3: + storeInt(3); + break; + + case isub: + popInt(rax); + sub(rax, rsp, 0); + break; + + case iushr: + popInt(rcx); + shr4_cl(rsp, 0); + break; + + case l2i: + if (BytesPerWord == 8) { + popInt(); + } else { + popInt(rax); + mov(rax, rsp, 0); + } + break; + + case ladd: + if (BytesPerWord == 8) { + popLong(rax); + add(rax, rsp, BytesPerWord); + } else { + popLong(rax, rdx); + add(rax, rsp, 0); + adc(rdx, rsp, BytesPerWord); + } + break; + + case ldc: + case ldc_w: { + uint16_t index; + + if (instruction == ldc) { + index = codeBody(t, code, ip++); + } else { + index = codeReadInt16(t, code, ip); + } + + object v = arrayBody(t, codePool(t, code), index - 1); + + if (objectClass(t, v) == arrayBody(t, t->m->types, Machine::IntType)) { + pushInt(intValue(t, v)); + } else if (objectClass(t, v) + == arrayBody(t, t->m->types, Machine::FloatType)) + { + pushInt(floatValue(t, v)); + } else if (objectClass(t, v) + == arrayBody(t, t->m->types, Machine::StringType)) + { + pushObject(poolRegister(), poolReference(v)); + } else { + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + + pushObject(poolRegister(), poolReference(class_)); + } + } break; + + case ldc2_w: { + uint16_t index = codeReadInt16(t, code, ip); + + object v = arrayBody(t, codePool(t, code), index - 1); + + if (objectClass(t, v) == arrayBody(t, t->m->types, Machine::LongType)) + { + pushLong(longValue(t, v)); + } else if (objectClass(t, v) + == arrayBody(t, t->m->types, Machine::DoubleType)) + { + pushLong(doubleValue(t, v)); + } else { + abort(t); + } + } break; + + case lconst_0: + pushLong(0); + break; + + case lconst_1: + pushLong(1); + break; + + case lcmp: { + Label next(this); + Label less(this); + Label greater(this); + + if (BytesPerWord == 8) { + popLong(rax); + popLong(rcx); + + cmp(rax, rcx); + jl(less); + jg(greater); + + push(0); + jmp(next); + + less.mark(); + push(-1); + jmp(next); + + greater.mark(); + push(1); + + next.mark(); + + stackMapper.pushedInt(); + } else { + popLong(rax, rdx); + popLong(rcx, rbx); + + cmp(rdx, rbx); + jl(less); + jg(greater); + + cmp(rax, rcx); + jb(less); + ja(greater); + + push(0); + jmp(next); + + less.mark(); + push(-1); + jmp(next); + + greater.mark(); + push(1); + + next.mark(); + + stackMapper.pushedInt(); + } + } break; + + case ldiv_: + if (BytesPerWord == 8) { + popLong(rcx); + popLong(rax); + cqo(); + Assembler::idiv(rcx); + pushLong(rax); + } else { + directCall(reinterpret_cast(divideLong)); + popLong(); + mov(rax, rsp, 0); + mov(rdx, rsp, 4); + } + break; + + case lload: + case dload: + loadLong(codeBody(t, code, ip++)); + break; + + case lload_0: + case dload_0: + loadLong(0); + break; + + case lload_1: + case dload_1: + loadLong(1); + break; + + case lload_2: + case dload_2: + loadLong(2); + break; + + case lload_3: + case dload_3: + loadLong(3); + break; + + case lmul: + if (BytesPerWord == 8) { + popLong(rax); + popLong(rcx); + Assembler::imul(rcx); + pushLong(rax); + } else { + mov(rsp, 4, rcx); + Assembler::imul(rsp, 8, rcx); + mov(rsp, 12, rax); + Assembler::imul(rsp, 0, rax); + add(rax, rcx); + mov(rsp, 8, rax); + mul(rsp, 0); + add(rcx, rdx); + + popLong(); + mov(rax, rsp, 0); + mov(rdx, rsp, 4); + } + break; + + case lneg: + if (BytesPerWord == 8) { + neg(rsp, 8); + } else { + mov(rsp, 0, rax); + mov(rsp, 4, rdx); + neg(rax); + adc(0, rdx); + neg(rdx); + + mov(rax, rsp, 0); + mov(rdx, rsp, 4); + } + break; + + // todo: +// case lookupswitch: { +// int32_t base = ip - 1; +// ip += 3; +// ip -= (ip % 4); + +// int32_t default_ = codeReadInt32(t, code, ip); +// int32_t pairCount = codeReadInt32(t, code, ip); + +// popInt(rax); +// mov(LookupTable(this, base, pairCount, code, ip), rbx); +// mov(pairCount, rcx); +// jmp(test); + +// loop.mark(); +// // tbc + +// test.mark(); +// cmp(0, rcx); +// jne(loop); + +// int32_t key = popInt(t); + +// int32_t bottom = 0; +// int32_t top = pairCount; +// for (int32_t span = top - bottom; span; span = top - bottom) { +// int32_t middle = bottom + (span / 2); +// unsigned index = ip + (middle * 8); + +// int32_t k = codeReadInt32(t, code, index); + +// if (key < k) { +// top = middle; +// } else if (key > k) { +// bottom = middle + 1; +// } else { +// ip = base + codeReadInt32(t, code, index); +// break; +// } +// } + +// ip = base + default_; +// } break; + + case lrem: + if (BytesPerWord == 8) { + popLong(rcx); + popLong(rax); + cqo(); + Assembler::idiv(rcx); + pushLong(rdx); + } else { + directCall(reinterpret_cast(moduloLong)); + popLong(); + mov(rax, rsp, 0); + mov(rdx, rsp, 4); + } + break; + + case lreturn: + case dreturn: + if (BytesPerWord == 8) { + popLong(rax); + } else { + popLong(rax, rdx); + } + mov(rbp, rsp); + pop(rbp); + ret(); + stackMapper.exited(); + break; + + case lstore: + case dstore: + storeLong(codeBody(t, code, ip++)); + break; + + case lstore_0: + case dstore_0: + storeLong(0); + break; + + case lstore_1: + case dstore_1: + storeLong(1); + break; + + case lstore_2: + case dstore_2: + storeLong(2); + break; + + case lstore_3: + case dstore_3: + storeLong(3); + break; + + case lsub: + if (BytesPerWord == 8) { + popLong(rax); + sub(rax, rsp, BytesPerWord); + } else { + popLong(rax, rdx); + sub(rax, rsp, 0); + sbb(rdx, rsp, BytesPerWord); + } + break; + + case new_: { + uint16_t index = codeReadInt16(t, code, ip); + + object class_ = resolveClassInPool(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + PROTECT(t, class_); + + initClass(t, class_); + if (UNLIKELY(t->exception)) return 0; + + if (classVmFlags(t, class_) & WeakReferenceFlag) { + compileCall(false, reinterpret_cast(makeNewWeakReference), + class_); + } else { + compileCall(false, reinterpret_cast(makeNew), class_); + } + + pushObject(rax); + } break; + + case newarray: { + uint8_t type = codeBody(t, code, ip++); + + Label nonnegative(this); + + popInt(rax); + cmp(0, rax); + jge(nonnegative); + + compileThrowNew(Machine::NegativeArraySizeExceptionType); + + nonnegative.mark(); + + object (*constructor)(Thread*, uintptr_t, bool); + switch (type) { + case T_BOOLEAN: + constructor = makeBooleanArray; + break; + + case T_CHAR: + constructor = makeCharArray; + break; + + case T_FLOAT: + constructor = makeFloatArray; + break; + + case T_DOUBLE: + constructor = makeDoubleArray; + break; + + case T_BYTE: + constructor = makeByteArray; + break; + + case T_SHORT: + constructor = makeShortArray; + break; + + case T_INT: + constructor = makeIntArray; + break; + + case T_LONG: + constructor = makeLongArray; + break; + + default: abort(t); + } + + compileCall(false, reinterpret_cast(makeBlankArray), + reinterpret_cast(constructor), rax); + + pushObject(rax); + } break; + + case pop_: { + add(BytesPerWord, rsp); + stackMapper.popped(1); + } break; + + case putfield: { + uint16_t index = codeReadInt16(t, code, ip); + + object field = resolveField(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + + switch (fieldCode(t, field)) { + case ByteField: + case BooleanField: + case CharField: + case ShortField: + case FloatField: + case IntField: { + popInt(rcx); + popObject(rax); + + switch (fieldCode(t, field)) { + case ByteField: + case BooleanField: + mov1(rcx, rax, fieldOffset(t, field)); + break; + + case CharField: + case ShortField: + mov2(rcx, rax, fieldOffset(t, field)); + break; + + case FloatField: + case IntField: + mov4(rcx, rax, fieldOffset(t, field)); + break; + } + } break; + + case DoubleField: + case LongField: { + if (BytesPerWord == 8) { + popLong(rcx); + popObject(rax); + mov(rcx, rax, fieldOffset(t, field)); + } else { + popLong(rcx, rbx); + popObject(rax); + mov(rcx, rax, fieldOffset(t, field)); + mov(rbx, rax, fieldOffset(t, field) + 4); + } + } break; + + case ObjectField: { + if (BytesPerWord == 8) { + popObject(rcx); + mov(fieldOffset(t, field), rdx); + popObject(rsi); + mov(rbp, FrameThread, rdi); + } else { + popObject(rcx); + popObject(rsi); + push(rcx); + push(fieldOffset(t, field)); + push(rsi); + push(rbp, FrameThread); + } + + directCall(reinterpret_cast(set)); + + if (BytesPerWord == 4) { + add(BytesPerWord * 3, rsp); + } + } break; + + default: abort(t); + } + } break; + + case putstatic: { + uint16_t index = codeReadInt16(t, code, ip); + + object field = resolveField(t, codePool(t, code), index - 1); + if (UNLIKELY(t->exception)) return 0; + + initClass(t, fieldClass(t, field)); + if (UNLIKELY(t->exception)) return 0; + + object table = classStaticTable(t, fieldClass(t, field)); + unsigned offset = (fieldOffset(t, field) * BytesPerWord) + ArrayBody; + + switch (fieldCode(t, field)) { + case ByteField: + case BooleanField: + case CharField: + case ShortField: + case FloatField: + case IntField: { + object intType = arrayBody(t, t->m->types, Machine::IntType); + + if (BytesPerWord == 8) { + mov(poolRegister(), poolReference(intType), rsi); + mov(rbp, FrameThread, rdi); + } else { + push(poolRegister(), poolReference(intType)); + push(rbp, FrameThread); + } + + indirectCall(reinterpret_cast(makeNew)); + + if (BytesPerWord == 4) { + add(BytesPerWord * 2, rsp); + } + + popInt4(rax, IntValue); + + if (BytesPerWord == 8) { + mov(rax, rcx); + mov(offset, rdx); + mov(poolRegister(), poolReference(table), rsi); + mov(rbp, FrameThread, rdi); + } else { + push(rax); + push(offset); + push(poolRegister(), poolReference(table)); + push(rbp, FrameThread); + } + + directCall(reinterpret_cast(set)); + + if (BytesPerWord == 4) { + add(BytesPerWord * 3, rsp); + } + } break; + + case DoubleField: + case LongField: { + object longType = arrayBody(t, t->m->types, Machine::LongType); + + if (BytesPerWord == 8) { + mov(poolRegister(), poolReference(longType), rsi); + mov(rbp, FrameThread, rdi); + } else { + push(poolRegister(), poolReference(longType)); + push(rbp, FrameThread); + } + + indirectCall(reinterpret_cast(makeNew)); + + if (BytesPerWord == 8) { + popLong(rax, LongValue); + } else { + add(BytesPerWord * 2, rsp); + popLong(rax, LongValue, + rax, LongValue + 4); + } + + if (BytesPerWord == 8) { + mov(rax, rcx); + mov(offset, rdx); + mov(poolRegister(), poolReference(table), rsi); + mov(rbp, FrameThread, rdi); + } else { + push(rax); + push(offset); + push(poolRegister(), poolReference(table)); + push(rbp, FrameThread); + } + + directCall(reinterpret_cast(set)); + + if (BytesPerWord == 4) { + add(BytesPerWord * 3, rsp); + } + } break; + + case ObjectField: + if (BytesPerWord == 8) { + popObject(rcx); + mov(offset, rdx); + mov(poolRegister(), poolReference(table), rsi); + mov(rbp, FrameThread, rdi); + } else { + push(offset); + push(poolRegister(), poolReference(table)); + push(rbp, FrameThread); + } + + directCall(reinterpret_cast(set)); + + if (BytesPerWord == 4) { + add(BytesPerWord * 3, rsp); + stackMapper.poppedObject(); + } + break; + + default: abort(t); + } + } break; + + case return_: + mov(rbp, rsp); + pop(rbp); + ret(); + stackMapper.exited(); + break; + + case sipush: { + pushInt(static_cast(codeReadInt16(t, code, ip))); + } break; + + // todo: +// case tableswitch: { +// int32_t base = ip - 1; +// ip += 3; +// ip -= (ip % 4); + +// int32_t default_ = codeReadInt32(t, code, ip); +// int32_t bottom = codeReadInt32(t, code, ip); +// int32_t top = codeReadInt32(t, code, ip); + +// popInt(rax); +// cmp(bottom, rax); +// jl(defaultLabel); + +// cmp(top, rax); +// jg(defaultLabel); + +// sub(bottom, rax); +// mov(Address(NoRegister, rax, 4, +// IndexTable(this, base, top - bottom, code, ip)), +// rax); +// jmp(rax); + +// defaultLabel.mark(); +// jmp(base + default_); +// stackMapper.jumped(base + default_); +// } break; + + default: + abort(t); + } + } + + resolveJumps(); + buildExceptionHandlerTable(code); + + return finish(); + } + + void resolveJumps() { + for (unsigned i = 0; i < jumps.length(); i += 8) { + uint32_t ip = jumps.get4(i); + uint32_t offset = jumps.get4(i + 4); + + code.set4(offset, machineIPs[ip] - (offset + 4)); + } + } + + void buildExceptionHandlerTable(object code) { + PROTECT(t, code); + + object eht = codeExceptionHandlerTable(t, code); + if (eht) { + PROTECT(t, eht); + + for (unsigned i = 0; i < exceptionHandlerTableLength(t, eht); ++i) { + ExceptionHandler* eh = exceptionHandlerTableBody(t, eht, i); + + nativeExceptionHandlerStart(exceptionHandlers + i) + = machineIPs[exceptionHandlerStart(eh)]; + + nativeExceptionHandlerEnd(exceptionHandlers + i) + = machineIPs[exceptionHandlerEnd(eh)]; + + nativeExceptionHandlerIp(exceptionHandlers + i) + = machineIPs[exceptionHandlerIp(eh)]; + + unsigned ct = exceptionHandlerCatchType(eh); + object catchType; + if (ct) { + catchType = resolveClassInPool + (t, codePool(t, code), exceptionHandlerCatchType(eh) - 1); + } else { + catchType = 0; + } + + nativeExceptionHandlerCatchType(exceptionHandlers + i) + = (catchType ? (poolReference(catchType) / BytesPerWord) - 1 : 0); + } + } + } + + Compiled* finish() { + stackMapper.finish(); + return makeCompiled(t, method, &code, lineNumbers, exceptionHandlers, + &stackMapper); + } + + object makePool() { + if (pool.length()) { + object array = makeArray(t, pool.length() / BytesPerWord, false); + pool.copyTo(&arrayBody(t, array, 0)); + return array; + } else { + return 0; + } + } + + Register poolRegister() { + return rdi; + } + + uint32_t poolReference(object o) { + if (poolRegisterClobbered) { + mov(rbp, FrameMethod, rdi); + mov(rdi, MethodCode, rdi); + //poolRegisterClobbered = false; + } + pool.appendAddress(reinterpret_cast(o)); + return pool.length() + BytesPerWord; + } + + object method; + StackMapper stackMapper; + bool poolRegisterClobbered; + uint32_t* machineIPs; + NativeLineNumber* lineNumbers; + NativeExceptionHandler* exceptionHandlers; + Buffer pool; + + class MyProtector: public Thread::Protector { + public: + MyProtector(JavaCompiler* c): Protector(c->t), c(c) { } + + virtual void visit(Heap::Visitor* v) { + v->visit(&(c->method)); + + for (unsigned i = 0; i < c->pool.length(); i += BytesPerWord) { + v->visit(reinterpret_cast(&(c->pool.getAddress(i)))); + } + } + + JavaCompiler* c; + } protector; +}; + +void +compileMethod2(MyThread* t, object method) +{ + if (reinterpret_cast(methodCompiled(t, method)) + == t->m->processor->methodStub(t)) + { + PROTECT(t, method); + + ACQUIRE(t, t->m->classLock); + + if (reinterpret_cast(methodCompiled(t, method)) + == t->m->processor->methodStub(t)) + { + if (Verbose) { + fprintf(stderr, "compiling %s.%s\n", + &byteArrayBody(t, className(t, methodClass(t, method)), 0), + &byteArrayBody(t, methodName(t, method), 0)); + } + + JavaCompiler c(t, method); + Compiled* code = c.compile(); + + if (Verbose) { + fprintf(stderr, "compiled %s.%s from %p to %p\n", + &byteArrayBody(t, className(t, methodClass(t, method)), 0), + &byteArrayBody(t, methodName(t, method), 0), + compiledCode(code), + compiledCode(code) + compiledCodeLength(code)); + } + + // for debugging: + if (false and + strcmp(reinterpret_cast + (&byteArrayBody(t, className(t, methodClass(t, method)), 0)), + "Memory") == 0 and + strcmp(reinterpret_cast + (&byteArrayBody(t, methodName(t, method), 0)), + "runningSum") == 0) + { + noop(); + } + + object pool = c.makePool(); + set(t, method, MethodCode, pool); + + methodCompiled(t, method) = reinterpret_cast(code); + } + } +} + +void +updateCaller(MyThread* t, object method) +{ + uintptr_t stub = reinterpret_cast + (compiledCode(static_cast(t->m->processor->methodStub(t)))); + + Assembler a(t->m->system); + a.mov(stub, rax); + unsigned offset = a.code.length() - BytesPerWord; + + a.call(rax); + + uint8_t* caller = static_cast(frameAddress(frameStart(t))) + - a.code.length(); + if (memcmp(a.code.data, caller, a.code.length()) == 0) { + // it's a direct call - update caller to point to new code + + // address must be aligned on a word boundary for this write to + // be atomic + assert(t, reinterpret_cast(caller + offset) + % BytesPerWord == 0); + + *reinterpret_cast(caller + offset) + = compiledCode(reinterpret_cast(methodCompiled(t, method))); + } +} + +void +compileMethod(MyThread* t, object method) +{ + { PROTECT(t, method); + compileMethod2(t, method); + + if (LIKELY(t->exception == 0)) { + if (not methodVirtual(t, method)) { + updateCaller(t, method); + } + return; + } + } + + unwind(t); +} + +class ArgumentList { + public: + ArgumentList(Thread* t, uintptr_t* array, bool* objectMask, object this_, + const char* spec, bool indirectObjects, va_list arguments): + t(static_cast(t)), + next(this->t->argumentList), + array(array), + objectMask(objectMask), + position(0), + protector(this) + { + this->t->argumentList = this; + + addInt(reinterpret_cast(t)); + addObject(0); // reserve space for method + addInt(0); // reserve space for frame + + if (this_) { + addObject(this_); + } + + for (MethodSpecIterator it(t, spec); it.hasNext();) { + switch (*it.next()) { + case 'L': + case '[': + if (indirectObjects) { + object* v = va_arg(arguments, object*); + addObject(v ? *v : 0); + } else { + addObject(va_arg(arguments, object)); + } + break; + + case 'J': + case 'D': + addLong(va_arg(arguments, uint64_t)); + break; + + default: + addInt(va_arg(arguments, uint32_t)); + break; + } + } + } + + ArgumentList(Thread* t, uintptr_t* array, bool* objectMask, object this_, + const char* spec, object arguments): + t(static_cast(t)), + next(this->t->argumentList), + array(array), + objectMask(objectMask), + position(0), + protector(this) + { + this->t->argumentList = this; + + addInt(0); // reserve space for trace pointer + addObject(0); // reserve space for method pointer + + if (this_) { + addObject(this_); + } + + unsigned index = 0; + for (MethodSpecIterator it(t, spec); it.hasNext();) { + switch (*it.next()) { + case 'L': + case '[': + addObject(objectArrayBody(t, arguments, index++)); + break; + + case 'J': + case 'D': + addLong(cast(objectArrayBody(t, arguments, index++), + BytesPerWord)); + break; + + default: + addInt(cast(objectArrayBody(t, arguments, index++), + BytesPerWord)); + break; + } + } + } + + ~ArgumentList() { + t->argumentList = next; + } + + void addObject(object v) { + array[position] = reinterpret_cast(v); + objectMask[position] = true; + ++ position; + } + + void addInt(uintptr_t v) { + array[position] = v; + objectMask[position] = false; + ++ position; + } + + void addLong(uint64_t v) { + memcpy(array + position, &v, 8); + objectMask[position] = false; + objectMask[position] = false; + position += 2; + } + + MyThread* t; + ArgumentList* next; + uintptr_t* array; + bool* objectMask; + unsigned position; + + class MyProtector: public Thread::Protector { + public: + MyProtector(ArgumentList* list): Protector(list->t), list(list) { } + + virtual void visit(Heap::Visitor* v) { + for (unsigned i = 0; i < list->position; ++i) { + if (list->objectMask[i]) { + v->visit(reinterpret_cast(list->array + i)); + } + } + } + + ArgumentList* list; + } protector; +}; + +object +invoke(Thread* thread, object method, ArgumentList* arguments) +{ + MyThread* t = static_cast(thread); + + arguments->array[1] = reinterpret_cast(method); + + unsigned returnCode = methodReturnCode(t, method); + unsigned returnType = fieldType(t, returnCode); + + Reference* reference = t->reference; + + void* frameData[2]; + memcpy(frameData, t->frameData, BytesPerWord * 2); + + if (frameData[0]) { + arguments->array[2] = reinterpret_cast(framePointer(frameData)); + } + + Compiled* code = reinterpret_cast(methodCompiled(t, method)); + uint64_t result = vmInvoke + (compiledCode(code), arguments->array, arguments->position * BytesPerWord, + returnType); + + while (t->reference != reference) { + dispose(t, t->reference); + } + + memcpy(t->frameData, frameData, BytesPerWord * 2); + + object r; + switch (returnCode) { + case ByteField: + case BooleanField: + case CharField: + case ShortField: + case FloatField: + case IntField: + r = makeInt(t, result); + break; + + case LongField: + case DoubleField: + r = makeLong(t, result); + break; + + case ObjectField: + r = (result == 0 ? 0 : + *reinterpret_cast(static_cast(result))); + break; + + case VoidField: + r = 0; + break; + + default: + abort(t); + }; + + return r; +} + +class MyProcessor: public Processor { + public: + MyProcessor(System* s): + s(s), + methodStub_(0), + nativeInvoker_(0) + { } + + virtual Thread* + makeThread(Machine* m, object javaThread, Thread* parent) + { + MyThread* t = new (s->allocate(sizeof(Thread))) + MyThread(m, javaThread, parent); + t->init(); + return t; + } + + virtual void* + methodStub(Thread* t) + { + if (methodStub_ == 0) { + Compiler c(static_cast(t)); + methodStub_ = c.compileStub(); + + if (Verbose) { + fprintf(stderr, "compiled method stub from %p to %p\n", + compiledCode(methodStub_), + compiledCode(methodStub_) + compiledCodeLength(methodStub_)); + } + } + return methodStub_; + } + + virtual void* + nativeInvoker(Thread* t) + { + if (nativeInvoker_ == 0) { + Compiler c(static_cast(t)); + nativeInvoker_ = c.compileNativeInvoker(); + + if (Verbose) { + fprintf(stderr, "compiled native invoker from %p to %p\n", + compiledCode(nativeInvoker_), + compiledCode(nativeInvoker_) + + compiledCodeLength(nativeInvoker_)); + } + } + return nativeInvoker_; + } + + Compiled* + caller(Thread* t) + { + if (caller_ == 0) { + Compiler c(static_cast(t)); + caller_ = c.compileCaller(); + + if (Verbose) { + fprintf(stderr, "compiled caller from %p to %p\n", + compiledCode(caller_), + compiledCode(caller_) + compiledCodeLength(caller_)); + } + } + return caller_; + } + + virtual unsigned + parameterFootprint(vm::Thread* t, const char* s, bool static_) + { + unsigned footprint = 0; + for (MethodSpecIterator it(t, s); it.hasNext();) { + switch (*it.next()) { + case 'J': + case 'D': + footprint += 2; + break; + + default: + ++ footprint; + break; + } + } + + if (not static_) { + ++ footprint; + } + return footprint; + } + + virtual void + initClass(Thread* t, object c) + { + PROTECT(t, c); + + ACQUIRE(t, t->m->classLock); + if (classVmFlags(t, c) & NeedInitFlag + and (classVmFlags(t, c) & InitFlag) == 0) + { + classVmFlags(t, c) |= InitFlag; + invoke(t, classInitializer(t, c), 0); + if (t->exception) { + t->exception = makeExceptionInInitializerError(t, t->exception); + } + classVmFlags(t, c) &= ~(NeedInitFlag | InitFlag); + } + } + + virtual void + visitObjects(Thread* vmt, Heap::Visitor* v) + { + MyThread* t = static_cast(vmt); + + if (t->m->active) { + for (Reference* r = t->reference; r; r = r->next) { + v->visit(&(r->target)); + } + + visitStack(t, v); + } + } + + virtual uintptr_t + frameStart(Thread* vmt) + { + return reinterpret_cast + (::frameStart(static_cast(vmt))); + } + + virtual uintptr_t + frameNext(Thread*, uintptr_t frame) + { + return reinterpret_cast + (::frameNext(reinterpret_cast(frame))); + } + + virtual bool + frameValid(Thread*, uintptr_t frame) + { + return ::frameValid(reinterpret_cast(frame)); + } + + virtual object + frameMethod(Thread*, uintptr_t frame) + { + return ::frameMethod(reinterpret_cast(frame)); + } + + virtual unsigned + frameIp(Thread* t, uintptr_t frame) + { + void* f = reinterpret_cast(frame); + return addressOffset(t, ::frameMethod(f), ::frameAddress(f)); + } + + virtual int + lineNumber(Thread* t, object method, unsigned ip) + { + if (methodFlags(t, method) & ACC_NATIVE) { + return NativeLine; + } + + Compiled* code = reinterpret_cast(methodCompiled(t, method)); + if (compiledLineNumberCount(t, code)) { + unsigned bottom = 0; + unsigned top = compiledLineNumberCount(t, code); + for (unsigned span = top - bottom; span; span = top - bottom) { + unsigned middle = bottom + (span / 2); + NativeLineNumber* ln = compiledLineNumber(t, code, middle); + + if (ip >= nativeLineNumberIp(ln) + and (middle + 1 == compiledLineNumberCount(t, code) + or ip < nativeLineNumberIp + (compiledLineNumber(t, code, middle + 1)))) + { + return nativeLineNumberLine(ln); + } else if (ip < nativeLineNumberIp(ln)) { + top = middle; + } else if (ip > nativeLineNumberIp(ln)) { + bottom = middle + 1; + } + } + + abort(t); + } else { + return UnknownLine; + } + } + + virtual object* + makeLocalReference(Thread* vmt, object o) + { + if (o) { + MyThread* t = static_cast(vmt); + + Reference* r = new (t->m->system->allocate(sizeof(Reference))) + Reference(o, &(t->reference)); + + return &(r->target); + } else { + return 0; + } + } + + virtual void + disposeLocalReference(Thread* t, object* r) + { + if (r) { + vm::dispose(t, reinterpret_cast(r)); + } + } + + virtual object + invokeArray(Thread* t, object method, object this_, object arguments) + { + assert(t, t->state == Thread::ActiveState + or t->state == Thread::ExclusiveState); + + assert(t, ((methodFlags(t, method) & ACC_STATIC) == 0) xor (this_ == 0)); + + const char* spec = reinterpret_cast + (&byteArrayBody(t, methodSpec(t, method), 0)); + + unsigned size = methodParameterFootprint(t, method) + FrameFootprint; + uintptr_t array[size]; + bool objectMask[size]; + ArgumentList list(t, array, objectMask, this_, spec, arguments); + + return ::invoke(t, method, &list); + } + + virtual object + invokeList(Thread* t, object method, object this_, bool indirectObjects, + va_list arguments) + { + assert(t, t->state == Thread::ActiveState + or t->state == Thread::ExclusiveState); + + assert(t, ((methodFlags(t, method) & ACC_STATIC) == 0) xor (this_ == 0)); + + const char* spec = reinterpret_cast + (&byteArrayBody(t, methodSpec(t, method), 0)); + + unsigned size = methodParameterFootprint(t, method) + FrameFootprint; + uintptr_t array[size]; + bool objectMask[size]; + ArgumentList list + (t, array, objectMask, this_, spec, indirectObjects, arguments); + + return ::invoke(t, method, &list); + } + + virtual object + invokeList(Thread* t, const char* className, const char* methodName, + const char* methodSpec, object this_, va_list arguments) + { + assert(t, t->state == Thread::ActiveState + or t->state == Thread::ExclusiveState); + + unsigned size = parameterFootprint(t, methodSpec, false) + FrameFootprint; + uintptr_t array[size]; + bool objectMask[size]; + ArgumentList list + (t, array, objectMask, this_, methodSpec, false, arguments); + + object method = resolveMethod(t, className, methodName, methodSpec); + if (LIKELY(t->exception == 0)) { + assert(t, ((methodFlags(t, method) & ACC_STATIC) == 0) xor (this_ == 0)); + + return ::invoke(t, method, &list); + } else { + return 0; + } + } + + virtual void dispose() { + if (methodStub_) { + s->free(methodStub_); + } + + if (nativeInvoker_) { + s->free(nativeInvoker_); + } + + if (caller_) { + s->free(caller_); + } + + s->free(this); + } + + System* s; + Compiled* methodStub_; + Compiled* nativeInvoker_; + Compiled* caller_; +}; + +Compiled* +caller(MyThread* t) +{ + return static_cast(t->m->processor)->caller(t); +} + +} // namespace + +namespace vm { + +void +printJavaTrace(Thread* t, void* base, void* ip) +{ + void* top[] = { base, ip }; + + printTrace + (t, makeRuntimeException + (t, 0, makeTrace + (t, reinterpret_cast + (top + (FrameFootprint / BytesPerWord) + 2)), 0)); +} + +Processor* +makeProcessor(System* system) +{ + return new (system->allocate(sizeof(MyProcessor))) MyProcessor(system); +} + +} // namespace vm + diff --git a/src/vector.h b/src/vector.h index b1b78ab7e3..11767c1f25 100644 --- a/src/vector.h +++ b/src/vector.h @@ -5,7 +5,6 @@ namespace vm { -template class Vector { public: Vector(System* s, unsigned minimumCapacity): @@ -40,15 +39,13 @@ class Vector { } void get(unsigned offset, void* dst, unsigned size) { - assert(s, offset >= 0); assert(s, offset + size <= position); - mempcy(dst, data + offset, size); + memcpy(dst, data + offset, size); } void set(unsigned offset, const void* src, unsigned size) { - assert(s, offset >= 0); assert(s, offset + size <= position); - mempcy(data + offset, src, size); + memcpy(data + offset, src, size); } void pop(void* dst, unsigned size) { @@ -56,14 +53,19 @@ class Vector { position -= size; } - void* append(const void* p, unsigned size) { + void* allocate(unsigned size) { ensure(size); void* r = data + position; - memcpy(r, p, size); position += size; return r; } + void* append(const void* p, unsigned size) { + void* r = allocate(size); + memcpy(r, p, size); + return r; + } + void append(uint8_t v) { append(&v, 1); } @@ -80,16 +82,35 @@ class Vector { append(&v, BytesPerWord); } - template - C* push(const C& v) { - return static_cast(append(&v, sizeof(C))); + unsigned length() { + return position; } - template - C pop() { - C r; pop(&r, sizeof(C)); + template + T* peek(unsigned offset) { + assert(s, offset + sizeof(T) <= position); + return reinterpret_cast(data + offset); + } + + template + T* push(const T& v) { + return static_cast(append(&v, sizeof(T))); + } + + template + T pop() { + T r; pop(&r, sizeof(T)); return r; } + + void update(Vector* v) { + dispose(); + + data = v->data; + position = v->position; + capacity = v->capacity; + minimumCapacity = v->minimumCapacity; + } System* s; uint8_t* data;