diff --git a/src/arch.h b/src/arch.h index ae613297b6..ae2039296f 100644 --- a/src/arch.h +++ b/src/arch.h @@ -14,7 +14,8 @@ #include "common.h" extern "C" void NO_RETURN -vmJump(void* address, void* base, void* stack, void* thread); +vmJump(void* address, void* base, void* stack, void* thread, + uintptr_t returnLow, uintptr_t returnHigh); #if (defined __i386__) || (defined __x86_64__) # include "x86.h" diff --git a/src/compile-x86.S b/src/compile-x86.S index 58b370ae18..456acc2228 100644 --- a/src/compile-x86.S +++ b/src/compile-x86.S @@ -15,6 +15,19 @@ .text #ifdef __x86_64__ + +#define THREAD_CONTINUATION 168 +#define THREAD_EXCEPTION 64 +#define THREAD_EXCEPTION_STACK 176 +#define THREAD_EXCEPTION_OFFSET 184 +#define THREAD_EXCEPTION_HANDLER 192 + +#define CONTINUATION_NEXT 8 +#define CONTINUATION_ADDRESS 24 +#define CONTINUATION_RETURN_ADDRESS_OFFSET 32 +#define CONTINUATION_FRAME_POINTER_OFFSET 40 +#define CONTINUATION_LENGTH 48 +#define CONTINUATION_BODY 56 .globl vmInvoke vmInvoke: @@ -65,12 +78,13 @@ LOCAL(vmInvoke_returnAddress): // restore stack pointer movq %rbp,%rsp - // call the next continuation frame, if any + // call the next continuation, if any movq THREAD_CONTINUATION(%rbx),%rcx cmpq $0,%rcx je LOCAL(vmInvoke_exit) - movq CONTINUATION_SIZE(%rcx),%rsi + movq CONTINUATION_LENGTH(%rcx),%rsi + shrq $3,%rsi subq %rsi,%rsp movq CONTINUATION_BODY(%rcx),%rdi @@ -88,7 +102,8 @@ LOCAL(vmInvoke_continuationTest): jb LOCAL(vmInvoke_continuationLoop) movq CONTINUATION_RETURN_ADDRESS_OFFSET(%rcx),%rdi - movq LOCAL(vmInvoke_returnAddress),(%rsp,%rdi,1) + movq LOCAL(vmInvoke_returnAddress)@GOTPCREL(%rip),%r10 + movq %r10,(%rsp,%rdi,1) movq CONTINUATION_FRAME_POINTER_OFFSET(%rcx),%rdi movq %rbp,(%rsp,%rdi,1) @@ -100,8 +115,10 @@ LOCAL(vmInvoke_continuationTest): // call the continuation unless we're handling an exception movq THREAD_EXCEPTION(%rbx),%rsi cmpq $0,%rsi - je *CONTINUATION_ADDRESS(%rcx) + jne LOCAL(vmInvoke_handleException) + jmp *CONTINUATION_ADDRESS(%rcx) +LOCAL(vmInvoke_handleException): // we're handling an exception - call the exception handler instead movq $0,THREAD_EXCEPTION(%rbx) movq THREAD_EXCEPTION_STACK(%rbx),%rsp @@ -137,7 +154,8 @@ vmCallWithContinuation: movq %rdi,%rdx movq %r8,%rbp movq %r9,%rsp - movq LOCAL(vmInvoke_returnAddress),(%rsp) + movq LOCAL(vmInvoke_returnAddress)@GOTPCREL(%rip),%r10 + movq %r10,(%rsp) movq %rcx,8(%rsp) movq %rdx,16(%rsp) jmp *%rsi diff --git a/src/compile.cpp b/src/compile.cpp index faa41d5fd5..13ad7ee73b 100644 --- a/src/compile.cpp +++ b/src/compile.cpp @@ -22,6 +22,10 @@ extern "C" uint64_t vmInvoke(void* thread, void* function, void* arguments, unsigned argumentFootprint, unsigned frameSize, unsigned returnType); +extern "C" uint64_t +vmCallWithContinuation(void* thread, void* function, void* targetObject, + object continuation, void* base, void* stack); + extern "C" void vmCall(); @@ -118,7 +122,7 @@ parameterOffset(MyThread* t, object method) } object -resolveThisPointer(MyThread* t, void* stack, object method) +resolveThisPointer(MyThread* t, void* stack) { return reinterpret_cast(stack) [t->arch->frameFooterSize() + t->arch->frameReturnAddressSize()]; @@ -127,7 +131,7 @@ resolveThisPointer(MyThread* t, void* stack, object method) object resolveTarget(MyThread* t, void* stack, object method) { - object class_ = objectClass(t, resolveThisPointer(t, stack, method)); + object class_ = objectClass(t, resolveThisPointer(t, stack)); if (classVmFlags(t, class_) & BootstrapFlag) { PROTECT(t, method); @@ -211,6 +215,7 @@ class MyStackWalker: public Processor::StackWalker { enum State { Start, Next, + Continuation, Method, NativeMethod, Finish @@ -351,7 +356,7 @@ class MyStackWalker: public Processor::StackWalker { virtual int ip() { switch (state) { case Continuation: - return continuationAddress(t, continuation) + return reinterpret_cast(continuationAddress(t, continuation)) - methodCompiled(t, continuationMethod(t, continuation)); case Method: @@ -1359,7 +1364,7 @@ findExceptionHandler(Thread* t, object method, void* ip) } void -releaseLock(MyThread* t, object method) +releaseLock(MyThread* t, object method, void* stack) { if (methodFlags(t, method) & ACC_SYNCHRONIZED) { object lock; @@ -1413,7 +1418,7 @@ findUnwindTarget(MyThread* t, void** targetIp, void** targetBase, t->arch->nextFrame(&stack, &base); ip = t->arch->frameIp(stack); - releaseLock(t, method); + releaseLock(t, method, stack); } } else { *targetIp = ip; @@ -1430,15 +1435,19 @@ findUnwindTarget(MyThread* t, void** targetIp, void** targetBase, t->exceptionHandler = handler; t->exceptionStack = stackForFrame - (t, *targetStack, continuationMethod(t, t->continuation)); + (t, stack, continuationMethod(t, t->continuation)); t->exceptionOffset = localOffset(t, localSize(t, method), method); break; } else { - releaseLock(t, continuationMethod(t, t->continuation)); + releaseLock(t, continuationMethod(t, t->continuation), + reinterpret_cast(t->continuation) + + ContinuationBody + + continuationReturnAddressOffset(t, t->continuation) + -t->arch->returnAddressOffset()); } - t->continuation = continuationNext(t, t->continuation) + t->continuation = continuationNext(t, t->continuation); } } } @@ -1470,7 +1479,7 @@ makeCurrentContinuation(MyThread* t, void** targetIp, void** targetBase, if (method) { PROTECT(t, method); - void** top = static_cast(stack) - t->arch->frameHeader();; + void** top = static_cast(stack) - t->arch->frameHeaderSize(); unsigned argumentFootprint = t->arch->argumentFootprint(methodParameterFootprint(t, target)); unsigned alignment = t->arch->stackAlignmentInWords(); @@ -1526,7 +1535,7 @@ unwind(MyThread* t) void* base; void* stack; findUnwindTarget(t, &ip, &base, &stack); - vmJump(ip, base, stack, t); + vmJump(ip, base, stack, t, 0, 0); } object& @@ -4820,7 +4829,40 @@ compileVirtualMethod(MyThread* t) } } -inline uint64_t +void +resolveNative(MyThread* t, object method) +{ + PROTECT(t, method); + + assert(t, methodFlags(t, method) & ACC_NATIVE); + + initClass(t, methodClass(t, method)); + + if (LIKELY(t->exception == 0) + and methodCompiled(t, method) == defaultThunk(t)) + { + void* function = resolveNativeMethod(t, method); + if (UNLIKELY(function == 0)) { + object message = makeString + (t, "%s.%s%s", + &byteArrayBody(t, className(t, methodClass(t, method)), 0), + &byteArrayBody(t, methodName(t, method), 0), + &byteArrayBody(t, methodSpec(t, method), 0)); + t->exception = makeUnsatisfiedLinkError(t, message); + return; + } + + // ensure other threads see updated methodVmFlags before + // methodCompiled, since we don't want them using the slow calling + // convention on a function that expects the fast calling + // convention: + memoryBarrier(); + + methodCompiled(t, method) = reinterpret_cast(function); + } +} + +uint64_t invokeNativeFast(MyThread* t, object method) { return reinterpret_cast(methodCompiled(t, method)) @@ -5200,7 +5242,8 @@ walkContinuationBody(MyThread* t, Heap::Walker* w, object c, int start) int bodyStart = max(0, start - ContinuationReferenceCount); - object method = t->m->heap->follow(continuationMethod(t, c)); + object method = static_cast + (t->m->heap->follow(continuationMethod(t, c))); unsigned count = frameMapSizeInBits(t, method); if (count) { @@ -5219,7 +5262,7 @@ walkContinuationBody(MyThread* t, Heap::Walker* w, object c, int start) ((continuationFramePointerOffset(t, c) / BytesPerWord) - t->arch->framePointerOffset() - stackOffsetFromFrame(t, method) - + localOffsetFromStack(c, i, method))) + + localOffsetFromStack(t, i, method))) { return; } @@ -5241,11 +5284,14 @@ callWithContinuation(MyThread* t, object method, object this_, } vmCallWithContinuation - (t, methodAddress(t, method), this_, continuation, base, + (t, reinterpret_cast(methodAddress(t, method)), + this_, + continuation, + base, static_cast(stack) - t->arch->argumentFootprint(methodParameterFootprint(t, method)) - t->arch->frameFooterSize() - - t->arch->returnAddressSize()); + - t->arch->frameReturnAddressSize()); } class ArgumentList { @@ -5516,6 +5562,7 @@ class MyProcessor: public Processor { MyThread* t = new (m->heap->allocate(sizeof(MyThread))) MyThread(m, javaThread, static_cast(parent)); t->init(); + return t; } @@ -5905,23 +5952,23 @@ class MyProcessor: public Processor { (t->m->system->handleSegFault(&segFaultHandler))); } - virtual void callWithCurrentContinuation(Thread* t, object method, + virtual void callWithCurrentContinuation(Thread* vmt, object method, object this_) { - object coninuation; + MyThread* t = static_cast(vmt); + + object continuation; void* base; void* stack; { PROTECT(t, method); PROTECT(t, this_); - compile(static_cast(t), - ::codeAllocator(static_cast(t)), 0, method); + compile(t, ::codeAllocator(t), 0, method); if (LIKELY(t->exception == 0)) { void* ip; - continuation = makeCurrentContinuation - (static_cast(t), &ip, &base, &stack); + continuation = makeCurrentContinuation(t, &ip, &base, &stack); } } @@ -5930,8 +5977,11 @@ class MyProcessor: public Processor { } } - virtual void callContinuation(Thread* t, object continuation, object result) + virtual void callContinuation(Thread* vmt, object continuation, + object result) { + MyThread* t = static_cast(vmt); + assert(t, t->exception == 0); void* ip; @@ -5943,7 +5993,8 @@ class MyProcessor: public Processor { t->trace->targetMethod = 0; t->continuation = continuation; - vmJump(ip, base, stack, t, result, 0); + + vmJump(ip, base, stack, t, reinterpret_cast(result), 0); } virtual void walkContinuationBody(Thread* t, Heap::Walker* w, object o, @@ -6401,7 +6452,7 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p, Assembler::Register class_(t->arch->virtualCallTarget()); Assembler::Memory virtualCallTargetSrc (t->arch->stack(), - t->arch->frameFooterSize() + t->arch->returnAddressSize()); + t->arch->frameFooterSize() + t->arch->frameReturnAddressSize()); a->apply(Move, BytesPerWord, MemoryOperand, &virtualCallTargetSrc, BytesPerWord, RegisterOperand, &class_); diff --git a/src/compiler.cpp b/src/compiler.cpp index b65df7849a..249f247f2a 100644 --- a/src/compiler.cpp +++ b/src/compiler.cpp @@ -2420,7 +2420,7 @@ class CallEvent: public Event { returnAddressSurrogate = stack->value; addRead(c, this, stack->value, anyRegisterRead(c)); } else if (static_cast(frameIndex) == framePointerIndex) { - framePointerSurrogate = s->value; + framePointerSurrogate = stack->value; addRead(c, this, stack->value, anyRegisterRead(c)); } else { addRead(c, this, stack->value, read diff --git a/src/interpret.cpp b/src/interpret.cpp index e87b58beb8..fcc2621d90 100644 --- a/src/interpret.cpp +++ b/src/interpret.cpp @@ -468,6 +468,12 @@ resolveNativeMethodData(Thread* t, object method) if (LIKELY(p)) { PROTECT(t, method); object data = makeNativeMethodData(t, method, p); + + // ensure other threads see updated methodVmFlags before + // methodCode, and that the native method data is initialized + // before it is visible to those threads: + memoryBarrier(); + set(t, method, MethodCode, data); return data; } else { diff --git a/src/machine.cpp b/src/machine.cpp index 98f8d484d1..b4c877f4da 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -2757,8 +2757,6 @@ collect(Thread* t, Heap::CollectionType type) Machine* m = t->m; - m->continuationClass = arrayBody(t, t->m->types, Machine::ContinuationType); - m->unsafe = true; m->heap->collect(type, footprint(m->rootThread)); m->unsafe = false; diff --git a/src/machine.h b/src/machine.h index 1018ae2aac..c20ef8521e 100644 --- a/src/machine.h +++ b/src/machine.h @@ -92,8 +92,7 @@ const unsigned HasFinalMemberFlag = 1 << 6; const unsigned ClassInitFlag = 1 << 0; const unsigned CompiledFlag = 1 << 1; const unsigned ConstructorFlag = 1 << 2; -const unsigned NativeResolved = 1 << 3; -const unsigned FastNative = 1 << 4; +const unsigned FastNative = 1 << 3; typedef Machine JavaVM; typedef Thread JNIEnv; @@ -1326,7 +1325,7 @@ class Thread { #endif // VM_STRESS }; -typedef uint64_t (*FastNativeFunction)(Thread*, object, uintptr_t*) = 0; +typedef uint64_t (*FastNativeFunction)(Thread*, object, uintptr_t*); inline object objectClass(Thread*, object o) diff --git a/src/posix.cpp b/src/posix.cpp index 9d5e112773..b2b3800eee 100644 --- a/src/posix.cpp +++ b/src/posix.cpp @@ -831,7 +831,7 @@ handleSignal(int signal, siginfo_t* info, void* context) sigaddset(&set, SegFaultSignal); sigprocmask(SIG_UNBLOCK, &set, 0); - vmJump(ip, base, stack, thread); + vmJump(ip, base, stack, thread, 0, 0); } } break; diff --git a/src/process.cpp b/src/process.cpp index 51bf3036bc..83418a969c 100644 --- a/src/process.cpp +++ b/src/process.cpp @@ -199,17 +199,16 @@ resolveNativeMethod(Thread* t, object method, const char* prefix, namespace vm { void* -resolveNativeMethod2(Thread* t, object method) +resolveNativeMethod(Thread* t, object method) { void* p = ::resolveNativeMethod(t, method, "Java_", 5); if (p) { - methodVmFlags(t, method) |= NativeResolved; return p; } p = ::resolveNativeMethod(t, method, "Avian_", 6); if (p) { - methodVmFlags(t, method) |= NativeResolved & FastNative; + methodVmFlags(t, method) |= FastNative; return p; } diff --git a/src/process.h b/src/process.h index ac7e052413..ad58e22c67 100644 --- a/src/process.h +++ b/src/process.h @@ -126,35 +126,7 @@ isSpecialMethod(Thread* t, object method, object class_) } void* -resolveNativeMethod2(Thread* t, object method); - -inline void -resolveNativeMethod(MyThread* t, object method) -{ - PROTECT(t, method); - - assert(t, methodFlags(t, method) & ACC_NATIVE); - - initClass(t, methodClass(t, method)); - if (UNLIKELY(t->exception)) return 0; - - unsigned flags = methodVmFlags(t, method); - uintptr_t address = methodCompiled(t, method); - if ((flags & NativeResolved) == 0 or address == defaultThunk(t)) { - void* function = resolveNativeMethod2(t, method); - if (UNLIKELY(function == 0)) { - object message = makeString - (t, "%s.%s%s", - &byteArrayBody(t, className(t, methodClass(t, method)), 0), - &byteArrayBody(t, methodName(t, method), 0), - &byteArrayBody(t, methodSpec(t, method), 0)); - t->exception = makeUnsatisfiedLinkError(t, message); - return; - } - - methodCompiled(t, method) = reinterpret_cast(function); - } -} +resolveNativeMethod(Thread* t, object method); inline object findInterfaceMethod(Thread* t, object method, object class_) diff --git a/src/processor.h b/src/processor.h index 5f7b7ba00b..a24a6197d0 100644 --- a/src/processor.h +++ b/src/processor.h @@ -140,7 +140,7 @@ class Processor { callContinuation(Thread* t, object continuation, object result) = 0; virtual void - walkContiuationBody(Thread* t, Heap::Walker* w, object o, unsigned start) + walkContinuationBody(Thread* t, Heap::Walker* w, object o, unsigned start) = 0; object diff --git a/src/x86.S b/src/x86.S index a4f873f48d..3740a21f41 100644 --- a/src/x86.S +++ b/src/x86.S @@ -115,6 +115,8 @@ LOCAL(exit): .globl vmJump vmJump: + movq %r8,%rax + movq %r9,%rdx movq %rsi,%rbp movq %rdx,%rsp movq %rcx,%rbx