Support for creating and joining Java threads within SGX Avian. (#123)

* Add threading support to SGX Avian.
* Handle contract verification exceptions using uncaught exception handler.
* Indent uniformly to 4 spaces.
* Add comments for some of the uses of SGX synchronisation primitives.

sgx-jvm/avian/classpath/avian/SystemClassLoader.java

@@ -39,13 +39,13 @@ public class SystemClassLoader extends ClassLoader {
 
   private native VMClass findLoadedVMClass(String name);
 
-  private static native void startBlacklisting0();
+  private static native void startBlacklisting0(Thread t);
 
-  public void startBlacklisting() {
+  public void startBlacklisting(Thread t) {
     if (isForbidden("java/util/regex/Pattern$Test")) {
       throw new IllegalStateException("Impossible!");
     }
-    startBlacklisting0();
+    startBlacklisting0(t);
   }
 
   private static final Set<Pattern> BLACKLIST = Collections.unmodifiableSet(setOf(

sgx-jvm/avian/corda.pro

@@ -1,5 +1,9 @@
 # proguard include file (http://proguard.sourceforge.net)
 
+-keep class avian.SystemClassLoader {
+   public void startBlacklisting(java.lang.Thread);
+}
+
 # We need these for Corda deserialisation:
 -keep class sun.security.ec.ECPublicKeyImpl
 -keep class sun.security.ec.ECPrivateKeyImpl

sgx-jvm/avian/makefile

@@ -533,7 +533,8 @@ system = posix
 asm = x86
 
 ifeq ($(system),sgx)
-    cflags += -DSGX
+    cflags += -DSGX -I../linux-sgx/common/inc -I../jvm-enclave/common
+    lflags += $(shared)
 endif
 
 pointer-size = 8

sgx-jvm/avian/src/avian/machine.h

@@ -158,7 +158,12 @@ const unsigned FixedFootprintThresholdInBytes = ThreadHeapPoolSize
 
 // number of zombie threads which may accumulate before we force a GC
 // to clean them up:
-const unsigned ZombieCollectionThreshold = 16;
+#ifdef SGX
+#  define ZOMBIE_THRESHOLD  0
+#else
+#  define ZOMBIE_THRESHOLD  16
+#endif
+const unsigned ZombieCollectionThreshold = ZOMBIE_THRESHOLD;
 
 enum FieldCode {
   VoidField,
@@ -1342,7 +1347,6 @@ class Thread {
   }
 
   bool isBlacklisting();
-  void startBlacklisting();
 
   JNIEnvVTable* vtable;
   Machine* m;
@@ -1834,15 +1838,6 @@ inline uint64_t runThread(Thread* t, uintptr_t*)
 inline bool startThread(Thread* t, Thread* p)
 {
   p->setFlag(Thread::JoinFlag);
-#ifdef SGX
-  static const char16_t *nameToSkip = u"Reference Handler";
-  if (p->javaThread->name()->length(t) == 17) {
-    if (!memcmp(nameToSkip, cast<GcCharArray>(t, p->javaThread->name()->data())->body().begin(), 17 * 2)) {
-      printf("Skipping start of reference handler thread\n");
-      return true;
-    }
-  }
-#endif
   return t->m->system->success(t->m->system->start(&(p->runnable)));
 }
 

sgx-jvm/avian/src/builtin.cpp

@@ -184,9 +184,15 @@ extern "C" AVIAN_EXPORT int64_t JNICALL
 }
 
 extern "C" AVIAN_EXPORT void JNICALL
-    Avian_avian_SystemClassLoader_startBlacklisting0(Thread* t, object, uintptr_t*)
+    Avian_avian_SystemClassLoader_startBlacklisting0(Thread *t,
+                                                     object,
+                                                     uintptr_t* arguments)
 {
-  t->startBlacklisting();
+  GcThread *jt = cast<GcThread>(t, reinterpret_cast<object>(arguments[0]));
+  if (jt == NULL) {
+    throwNew(t, GcNullPointerException::Type);
+  }
+  jt->setBlacklisting(t, 1);
 }
 
 extern "C" AVIAN_EXPORT int64_t JNICALL

sgx-jvm/avian/src/jnienv.cpp

@@ -70,7 +70,6 @@ jint JNICALL DetachCurrentThread(Machine* m)
 
 uint64_t destroyJavaVM(Thread* t, uintptr_t*)
 {
-#ifndef SGX  
   // wait for other non-daemon threads to exit
   {
     ACQUIRE(t, t->m->stateLock);
@@ -78,13 +77,11 @@ uint64_t destroyJavaVM(Thread* t, uintptr_t*)
       t->m->stateLock->wait(t->systemThread, 0);
     }
   }
-#endif
   {
     ENTER(t, Thread::ActiveState);
 
     t->m->classpath->shutDown(t);
   }
-#ifndef SGX
   // wait again in case the Classpath::shutDown process started new
   // threads:
   {
@@ -95,7 +92,6 @@ uint64_t destroyJavaVM(Thread* t, uintptr_t*)
 
     enter(t, Thread::ExclusiveState);
   }
-#endif
   shutDown(t);
 
   return 1;

sgx-jvm/avian/src/machine.cpp

@@ -3969,13 +3969,6 @@ bool Thread::isBlacklisting()
   return (javaThread != NULL) && javaThread->blacklisting();
 }
 
-void Thread::startBlacklisting()
-{
-  if (javaThread != NULL) {
-    javaThread->setBlacklisting(this, 1);
-  }
-}
-
 void shutDown(Thread* t)
 {
   ACQUIRE(t, t->m->shutdownLock);

sgx-jvm/avian/src/system/sgx.cpp

@@ -14,7 +14,10 @@
 #include <avian/system/memory.h>
 #include <avian/util/math.h>
 
+#include <sgx_thread_completion.h>
+
 #define PATH_MAX 256
+#define ACQUIRE(x) MutexResource MAKE_NAME(mutexResource_)(x)
 
 using namespace vm;
 using namespace avian::util;
@@ -26,7 +29,32 @@ extern "C" __attribute__((weak)) const uint8_t* embedded_file_app_jar(size_t* si
 
 extern "C" const uint8_t* javahomeJar(size_t* size);
 
+typedef struct _thread_data_t thread_data_t;
+extern "C" __attribute__((weak)) thread_data_t* start_thread(void (*routine)(void *), void *param, sgx_thread_completion *completion);
+extern "C" __attribute__((weak)) thread_data_t* get_thread_data();
+
+static void run(void* r)
+{
+  static_cast<System::Runnable*>(r)->run();
+}
+
 namespace {
+    class MutexResource {
+    public:
+      MutexResource(sgx_thread_mutex_t& m) noexcept : _m(&m)
+      {
+        sgx_thread_mutex_lock(_m);
+      }
+
+      ~MutexResource() noexcept
+      {
+        sgx_thread_mutex_unlock(_m);
+      }
+
+    private:
+      sgx_thread_mutex_t* _m;
+    };
+
     void abort_with(const char *msg) {
         printf("%s\n", msg);
         while(true);
@@ -35,81 +63,149 @@ namespace {
     class MySystem;
     MySystem* globalSystem;
     const bool Verbose = false;
+    const unsigned Notified = 1 << 0;
 
     class MySystem : public System {
     public:
         class Thread : public System::Thread {
         public:
-            Thread* next;
-
-            Thread(System* s UNUSED, System::Runnable* r UNUSED) : next(0)
+            Thread(System* s, System::Runnable* r) : s(s), r(r), next(0), flags(0)
             {
+                sgx_thread_mutex_init(&mutex, 0);
+                sgx_thread_cond_init(&condition, 0);
             }
 
             virtual void interrupt()
             {
-                printf("Thread::Interrupt()\n");
+                ACQUIRE(mutex);
+
+                r->setInterrupted(true);
+
+                int rv UNUSED = sgx_thread_cond_signal(&condition);
+                expect(s, rv == 0);
             }
 
             virtual bool getAndClearInterrupted()
             {
-                printf("Thread::getAndClearInterrupted()\n");
-                return false;
+                ACQUIRE(mutex);
+
+                bool interrupted = r->interrupted();
+
+                r->setInterrupted(false);
+
+                return interrupted;
             }
 
             virtual void join()
             {
-                printf("Thread::Join()\n");
+                completion.wait();
             }
 
             virtual void dispose()
             {
+                sgx_thread_mutex_destroy(&mutex);
+                sgx_thread_cond_destroy(&condition);
+                ::free(this);
             }
+
+            thread_data_t* thread;
+            sgx_thread_completion completion;
+
+            /*
+             * The mutex protects this thread object's internal
+             * "state", and the condition wakes the thread when
+             * it is waiting on a monitor lock.
+             */
+            sgx_thread_mutex_t mutex;
+            sgx_thread_cond_t condition;
+
+            System* s;
+            System::Runnable* r;
+            Thread* next;
+            unsigned flags;
         };
 
         class Mutex : public System::Mutex {
         public:
             Mutex(System* s) : s(s)
             {
-
+                sgx_thread_mutex_init(&mutex, 0);
             }
 
             virtual void acquire()
             {
-
+                sgx_thread_mutex_lock(&mutex);
             }
 
             virtual void release()
             {
-
+                sgx_thread_mutex_unlock(&mutex);
             }
 
             virtual void dispose()
             {
-
+                sgx_thread_mutex_destroy(&mutex);
+                ::free(this);
             }
 
+        private:
             System* s;
+            sgx_thread_mutex_t mutex;
         };
 
         class Monitor : public System::Monitor {
         public:
             Monitor(System* s) : s(s), owner_(0), first(0), last(0), depth(0)
             {
-
+                sgx_thread_mutex_init(&mutex, 0);
             }
 
-            virtual bool tryAcquire(System::Thread* context UNUSED)
+            virtual bool tryAcquire(System::Thread* context)
             {
+                Thread* t = static_cast<Thread*>(context);
+
+                if (owner_ == t) {
+                    ++depth;
                     return true;
+                } else {
+                    switch (sgx_thread_mutex_trylock(&mutex)) {
+                    case EBUSY:
+                        return false;
+
+                    case 0:
+                        owner_ = t;
+                        ++depth;
+                        return true;
+
+                    default:
+                        sysAbort(s);
+                    }
+                }
             }
 
-            virtual void acquire(System::Thread* context UNUSED)
+            virtual void acquire(System::Thread* context)
             {
+                Thread* t = static_cast<Thread*>(context);
+
+                if (owner_ != t) {
+                    sgx_thread_mutex_lock(&mutex);
+                    owner_ = t;
+                }
+                ++depth;
             }
 
-            virtual void release(System::Thread* context UNUSED)
+            virtual void release(System::Thread* context)
             {
+                Thread* t = static_cast<Thread*>(context);
+
+                if (owner_ == t) {
+                    if (--depth == 0) {
+                        owner_ = 0;
+                        sgx_thread_mutex_unlock(&mutex);
+                    }
+                } else {
+                    sysAbort(s);
+                }
             }
 
             void append(Thread* t)
@@ -157,33 +253,125 @@ namespace {
                 }
             }
 
-            virtual void wait(System::Thread* context UNUSED, int64_t time UNUSED)
+            virtual void wait(System::Thread* context, int64_t time)
             {
                 wait(context, time, false);
             }
 
-            virtual bool waitAndClearInterrupted(System::Thread* context UNUSED, int64_t time UNUSED)
+            virtual bool waitAndClearInterrupted(System::Thread* context, int64_t time)
             {
                 return wait(context, time, true);
             }
 
-            bool wait(System::Thread* context UNUSED, int64_t time UNUSED, bool clearInterrupted UNUSED)
+            bool wait(System::Thread* context, int64_t time UNUSED, bool clearInterrupted)
             {
-                abort_with("STUB: Thread::Wait()");
-                return true;
+                Thread* t = static_cast<Thread*>(context);
+
+                if (owner_ == t) {
+                    // Initialized here to make gcc 4.2 a happy compiler
+                    bool interrupted = false;
+                    bool notified = false;
+                    unsigned depth = 0;
+
+                    {
+                        ACQUIRE(t->mutex);
+
+                        expect(s, (t->flags & Notified) == 0);
+
+                        interrupted = t->r->interrupted();
+                        if (interrupted and clearInterrupted) {
+                            t->r->setInterrupted(false);
                         }
 
-            void doNotify(Thread* t UNUSED)
-            {
-                printf("STUB: Thread::Notify()\n");
+                        append(t);
+
+                        depth = this->depth;
+                        this->depth = 0;
+                        owner_ = 0;
+                        sgx_thread_mutex_unlock(&mutex);
+
+                        if (not interrupted) {
+                            int rv UNUSED = sgx_thread_cond_wait(&(t->condition), &(t->mutex));
+                            expect(s, rv == 0 or rv == EINTR);
+
+                            interrupted = t->r->interrupted();
+                            if (interrupted and clearInterrupted) {
+                                t->r->setInterrupted(false);
+                            }
                         }
 
-            virtual void notify(System::Thread* context UNUSED)
-            {
+                        notified = ((t->flags & Notified) != 0);
                     }
 
-            virtual void notifyAll(System::Thread* context UNUSED)
+                    sgx_thread_mutex_lock(&mutex);
+
                     {
+                        ACQUIRE(t->mutex);
+                        t->flags = 0;
+                    }
+
+                    if (not notified) {
+                        remove(t);
+                    } else {
+#ifndef NDEBUG
+                        for (Thread* x = first; x; x = x->next) {
+                            expect(s, t != x);
+                        }
+#endif
+                    }
+
+                    t->next = 0;
+
+                    owner_ = t;
+                    this->depth = depth;
+
+                    return interrupted;
+                } else {
+                    sysAbort(s);
+                }
+            }
+
+            void doNotify(Thread* t)
+            {
+                ACQUIRE(t->mutex);
+
+                t->flags |= Notified;
+                int rv UNUSED = sgx_thread_cond_signal(&(t->condition));
+                expect(s, rv == 0);
+            }
+
+            virtual void notify(System::Thread* context)
+            {
+                Thread* t = static_cast<Thread*>(context);
+
+                if (owner_ == t) {
+                    if (first) {
+                        Thread* t = first;
+                        first = first->next;
+                        if (t == last) {
+                            expect(s, first == 0);
+                            last = 0;
+                        }
+
+                        doNotify(t);
+                    }
+                } else {
+                    sysAbort(s);
+                }
+            }
+
+            virtual void notifyAll(System::Thread* context)
+            {
+                Thread* t = static_cast<Thread*>(context);
+
+                if (owner_ == t) {
+                    for (Thread* t = first; t; t = t->next) {
+                        doNotify(t);
+                    }
+                    first = last = 0;
+                } else {
+                    sysAbort(s);
+                }
             }
 
             virtual System::Thread* owner()
@@ -194,32 +382,37 @@ namespace {
             virtual void dispose()
             {
                 expect(s, owner_ == 0);
+                sgx_thread_mutex_destroy(&mutex);
                 ::free(this);
             }
 
+        private:
             System* s;
+            sgx_thread_mutex_t mutex;
             Thread* owner_;
             Thread* first;
             Thread* last;
             unsigned depth;
         };
 
+        // This implementation of thread-local storage
+        // for SGX only works because we only create
+        // one instance of this class.
         class Local : public System::Local {
         public:
-            void *value;
-
             Local(System* s) : s(s)
             {
             }
 
             virtual void* get()
             {
-                return value;
+                return data;
             }
 
             virtual void set(void* p)
             {
-                value = p;
+                expect(s, data == NULL);
+                data = p;
             }
 
             virtual void dispose()
@@ -227,7 +420,10 @@ namespace {
                 ::free(this);
             }
 
+        private:
             System* s;
+            // Requires __get_tls_addr() in libsgx_trts
+            static thread_local void *data;
         };
 
         class Region : public System::Region {
@@ -256,6 +452,7 @@ namespace {
                 ::free(this);
             }
 
+        private:
             System* s;
             uint8_t* start_;
             size_t length_;
@@ -329,6 +526,7 @@ namespace {
                 ::free(this);
             }
 
+        private:
             System* s;
             System::Library* next_;
         };
@@ -371,7 +569,10 @@ namespace {
 
         virtual Status attach(Runnable* r)
         {
+            // This system thread will never be joined because it was not
+            // created using startThread() and so does not have JoinFlag set.
             Thread* t = new (allocate(this, sizeof(Thread))) Thread(this, r);
+            t->thread = get_thread_data();
             r->attach(t);
             return 0;
         }
@@ -380,10 +581,7 @@ namespace {
         {
             Thread* t = new (allocate(this, sizeof(Thread))) Thread(this, r);
             r->attach(t);
-            printf("System::start (thread!!)\n");
-            // We implement threads as blocking calls! This is of course totally wrong, but with extra threads
-            // patched out in a few places, it's hopefully sufficient.
-            r->run();
+            t->thread = start_thread(&run, r, &t->completion);
             return 0;
         }
 
@@ -509,6 +707,8 @@ namespace {
         Thread* visitTarget;
         System::Monitor* visitLock;
     };
+
+    thread_local void* MySystem::Local::data;
 }  // namespace
 
 namespace vm {

sgx-jvm/jvm-enclave/Makefile

@@ -30,3 +30,4 @@ libuntrusted_corda_sgx.so: jni/build/Makefile ../../verify-enclave/build/native/
 .PHONY: clean
 clean:
 	$(RM) -r {standalone,common,enclave,jni}/build
+	../../gradlew --project-dir=../.. verify-enclave:clean

sgx-jvm/jvm-enclave/common/CMakeLists.txt

@@ -5,6 +5,7 @@ set(SGX_USE_HARDWARE TRUE CACHE STRING "")
 set(SGX_SDK ${CMAKE_CURRENT_SOURCE_DIR}/../../linux-sgx CACHE STRING "")
 set(SGX_LIBRARY_PATH ${SGX_SDK}/build/linux CACHE STRING "")
 set(SGX_SDK_INCLUDE ${SGX_SDK}/common/inc CACHE STRING "")
+set(COMMON_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "")
 set(GENERATED_RPC_DIR ${CMAKE_CURRENT_BINARY_DIR}/rpc CACHE STRING "")
 set(PROGUARD_JAR_PATH /usr/share/proguard6.0beta1/lib/proguard.jar CACHE STRING "")
 set(DEPENDENCIES_LIBRARY_PATH /usr/lib/x86_64-linux-gnu CACHE STRING "")

sgx-jvm/jvm-enclave/common/sgx_thread_completion.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <sgx_thread.h>
+
+class sgx_thread_completion {
+    bool completed;
+    sgx_thread_mutex_t mutex;
+    sgx_thread_cond_t thread_complete;
+
+public:
+    sgx_thread_completion() noexcept : completed(false) {
+        sgx_thread_mutex_init(&mutex, NULL);
+        sgx_thread_cond_init(&thread_complete, NULL);
+    }
+    ~sgx_thread_completion() noexcept {
+        sgx_thread_cond_destroy(&thread_complete);
+        sgx_thread_mutex_destroy(&mutex);
+    }
+    void complete() noexcept;
+    void wait() noexcept;
+};
+

sgx-jvm/jvm-enclave/common/sgx_thread_mutex_guard.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <sgx_thread.h>
+
+class sgx_thread_mutex_guard {
+    sgx_thread_mutex_t * const mutex;
+public:
+    sgx_thread_mutex_guard(sgx_thread_mutex_t *mutex) noexcept : mutex(mutex) {
+        sgx_thread_mutex_lock(mutex);
+    }
+    ~sgx_thread_mutex_guard() noexcept {
+        sgx_thread_mutex_unlock(mutex);
+    }
+};
+

sgx-jvm/jvm-enclave/enclave/CMakeLists.txt

@@ -98,9 +98,9 @@ target_include_directories(os_support PUBLIC ${SGX_SDK_INCLUDE} ${AVIAN_PATH}/in
 add_dependencies(os_support GENERATED_EDL)
 
 # Build the enclave into a static archive. We switch off standard headers to make sure SGX headers are used.
-add_library(enclave_objects enclave.cpp enclave_start_thread.cpp ${GENERATED_RPC_DIR}/java_t.c ${CMAKE_CURRENT_BINARY_DIR}/boot-jar.o ${CMAKE_CURRENT_BINARY_DIR}/app-jar.o)
+add_library(enclave_objects enclave.cpp enclave_start_thread.cpp sgx_thread_completion.cpp ${GENERATED_RPC_DIR}/java_t.c ${CMAKE_CURRENT_BINARY_DIR}/boot-jar.o ${CMAKE_CURRENT_BINARY_DIR}/app-jar.o)
 add_dependencies(enclave_objects GENERATED_EDL)
-target_include_directories(enclave_objects PUBLIC ${SGX_SDK_INCLUDE} ${SGX_SDK_INCLUDE}/tlibc ${SGX_SDK}/sdk/tlibstdcxx/stlport ${AVIAN_PATH}/include ${AVIAN_PATH}/src ${GENERATED_RPC_DIR})
+target_include_directories(enclave_objects PUBLIC ${COMMON_INCLUDE} ${SGX_SDK_INCLUDE} ${SGX_SDK_INCLUDE}/tlibc ${SGX_SDK}/sdk/tlibstdcxx/stlport ${AVIAN_PATH}/include ${AVIAN_PATH}/src ${GENERATED_RPC_DIR})
 target_compile_options(enclave_objects PUBLIC -nostdinc)
 
 # Linker flags:

sgx-jvm/jvm-enclave/enclave/enclave_start_thread.cpp

@@ -1,35 +1,47 @@
 #include "enclave_start_thread.h"
+#include <sgx_thread_mutex_guard.h>
+#include <sgx_thread_completion.h>
+#include <sgx_trts.h>
+#include <java_t.h>
 
 #include "aex_assert.h"
+#include <cstdlib>
 #include <map>
 
 struct new_thread_data {
     void *param;
     void (*thread_routine)(void *);
     sgx_thread_cond_t *thread_started;
+    sgx_thread_completion *thread_completed;
 };
 
+typedef unsigned int nonce_t;
 static sgx_thread_mutex_t new_thread_map_mutex;
-static std::map<unsigned int, new_thread_data> new_thread_map;
+static std::map<nonce_t, new_thread_data> new_thread_map;
 static sgx_thread_mutex_t started_thread_data_map_mutex;
-static std::map<unsigned int, thread_data_t *> started_thread_data_map;
+static std::map<nonce_t, thread_data_t *> started_thread_data_map;
 struct ThreadMutexInit {
-    ThreadMutexInit() {
+    ThreadMutexInit() noexcept {
         sgx_thread_mutex_init(&new_thread_map_mutex, NULL);
         sgx_thread_mutex_init(&started_thread_data_map_mutex, NULL);
     }
 };
 static ThreadMutexInit _thread_mutex_init;
 
-thread_data_t *start_thread(void (*routine)(void *), void *param) {
-    unsigned int nonce;
+thread_data_t *start_thread(void (*routine)(void *), void *param, sgx_thread_completion *thread_completed) {
+    nonce_t nonce;
     aex_assert(SGX_SUCCESS == sgx_read_rand((unsigned char*)&nonce, sizeof(nonce)));
     sgx_thread_cond_t thread_started;
     sgx_thread_cond_init(&thread_started, NULL);
     sgx_thread_mutex_t thread_started_mutex;
     sgx_thread_mutex_init(&thread_started_mutex, NULL);
     sgx_thread_mutex_guard thread_started_guard(&thread_started_mutex);
-    new_thread_data thread_init_data = { param, routine, &thread_started };
+    new_thread_data thread_init_data = {
+        .param = param,
+        .thread_routine = routine,
+        .thread_started = &thread_started,
+        .thread_completed = thread_completed
+    };
     {
         sgx_thread_mutex_guard new_thread_map_guard(&new_thread_map_mutex);
         aex_assert(new_thread_map.find(nonce) == new_thread_map.end());
@@ -67,4 +79,8 @@ void create_new_thread(unsigned int nonce) {
     }
     sgx_thread_cond_signal(thread_init_data.thread_started);
     thread_init_data.thread_routine(thread_init_data.param);
+    if (thread_init_data.thread_completed != NULL) {
+        thread_init_data.thread_completed->complete();
     }
+}
+

sgx-jvm/jvm-enclave/enclave/enclave_start_thread.h

@@ -1,19 +1,10 @@
 #pragma once
 
-#include "sgx_thread.h"
-#include "sgx_trts.h"
-#include "java_t.h"
-#include <stdlib.h>
-#include "internal/global_data.h"
+#include "internal/thread_data.h"
 
-thread_data_t *start_thread(void (*routine)(void *), void *param);
+class sgx_thread_completion;
 
-struct sgx_thread_mutex_guard {
-    sgx_thread_mutex_t * const mutex;
-    sgx_thread_mutex_guard(sgx_thread_mutex_t *mutex) : mutex(mutex) {
-        sgx_thread_mutex_lock(mutex);
+extern "C" {
+    thread_data_t *start_thread(void (*routine)(void *), void *param, sgx_thread_completion*);
 }
-    ~sgx_thread_mutex_guard() {
-        sgx_thread_mutex_unlock(mutex);
-    }
-};
+

sgx-jvm/jvm-enclave/enclave/sgx_thread_completion.cpp

@@ -0,0 +1,16 @@
+#include <sgx_thread_completion.h>
+
+void sgx_thread_completion::complete() noexcept {
+    sgx_thread_mutex_lock(&mutex);
+    completed = true;
+    sgx_thread_mutex_unlock(&mutex);
+    sgx_thread_cond_signal(&thread_complete);
+}
+
+void sgx_thread_completion::wait() noexcept {
+    sgx_thread_mutex_lock(&mutex);
+    if (!completed) {
+        sgx_thread_cond_wait(&thread_complete, &mutex);
+    }
+    sgx_thread_mutex_unlock(&mutex);
+}

verify-enclave/src/main/kotlin/com/r3/enclaves/txverify/Enclavelet.kt

@@ -4,9 +4,7 @@ package com.r3.enclaves.txverify
 
 import com.esotericsoftware.minlog.Log
 import net.corda.core.contracts.Attachment
-import net.corda.core.serialization.CordaSerializable
-import net.corda.core.serialization.SerializedBytes
-import net.corda.core.serialization.deserialize
+import net.corda.core.serialization.*
 import net.corda.core.transactions.LedgerTransaction
 import net.corda.core.transactions.WireTransaction
 import java.io.File
@@ -44,18 +42,26 @@ class TransactionVerificationRequest(private val wtxToVerify: SerializedBytes<Wi
  */
 @Throws(Exception::class)
 fun verifyInEnclave(reqBytes: ByteArray) {
+    var ex: Throwable? = null
     val ltx = deserialise(reqBytes)
     // Prevent this thread from linking new classes against any
     // blacklisted classes, e.g. ones needed by Kryo or by the
     // JVM itself. Note that java.lang.Thread is also blacklisted.
-    startClassBlacklisting()
+    Thread {
         ltx.verify()
+    }.apply {
+        startClassBlacklisting(this)
+        setUncaughtExceptionHandler { _, e -> ex = e }
+        start()
+        join()
+    }
+    throw ex ?: return
 }
 
-private fun startClassBlacklisting() {
+private fun startClassBlacklisting(t: Thread) {
     val systemClassLoader = ClassLoader.getSystemClassLoader()
-    systemClassLoader.javaClass.getMethod("startBlacklisting").apply {
-        invoke(systemClassLoader)
+    systemClassLoader.javaClass.getMethod("startBlacklisting", t.javaClass).apply {
+        invoke(systemClassLoader, t)
     }
 }
 

verify-enclave/src/test/kotlin/com/r3/enclaves/DummySystemClassLoader.kt

@@ -1,6 +1,6 @@
 package com.r3.enclaves
 
-@Suppress("unused")
+@Suppress("unused", "unused_parameter")
 class DummySystemClassLoader(parent: ClassLoader) : ClassLoader(parent) {
-    fun startBlacklisting() {}
+    fun startBlacklisting(t: Thread) {}
 }