make sure a busy-waiting loop can't block the GC (and hence the whole VM)

src/compile.cpp

@@ -970,7 +970,7 @@ enum Thunk {
 #undef THUNK
 };
 
-const unsigned ThunkCount = gcIfNecessaryThunk + 1;
+const unsigned ThunkCount = idleIfNecessaryThunk + 1;
 
 intptr_t
 getThunk(MyThread* t, Thunk thunk);
@@ -3298,6 +3298,11 @@ gcIfNecessary(MyThread* t)
   }
 }
 
+void idleIfNecessary(MyThread* t)
+{
+  ENTER(t, Thread::IdleState);
+}
+
 unsigned
 resultSize(MyThread* t, unsigned code)
 {
@@ -3413,6 +3418,16 @@ useLongJump(MyThread* t, uintptr_t target)
     or (target < start && (end - target) > reach);
 }
 
+void compileSafePoint(MyThread* t, Compiler* c, Frame* frame) {
+  c->call
+    (c->constant(getThunk(t, idleIfNecessaryThunk), Compiler::AddressType),
+     0,
+     frame->trace(0, 0),
+     0,
+     Compiler::VoidType,
+     1, c->register_(t->arch->thread()));
+}
+
 Compiler::Operand*
 compileDirectInvoke(MyThread* t, Frame* frame, object target, bool tailCall,
                     bool useThunk, unsigned rSize, avian::codegen::Promise* addressPromise)
@@ -4951,6 +4966,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned initialIp,
       uint32_t newIp = (ip - 3) + offset;
       assert(t, newIp < codeLength(t, code));
 
+      if(newIp <= ip) {
+        compileSafePoint(t, c, frame);
+      }
+
       c->jmp(frame->machineIp(newIp));
       ip = newIp;
     } break;
@@ -4960,6 +4979,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned initialIp,
       uint32_t newIp = (ip - 5) + offset;
       assert(t, newIp < codeLength(t, code));
 
+      if(newIp <= ip) {
+        compileSafePoint(t, c, frame);
+      }
+
       c->jmp(frame->machineIp(newIp));
       ip = newIp;
     } break;
@@ -5048,6 +5071,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned initialIp,
       uint32_t offset = codeReadInt16(t, code, ip);
       newIp = (ip - 3) + offset;
       assert(t, newIp < codeLength(t, code));
+
+      if(newIp <= ip) {
+        compileSafePoint(t, c, frame);
+      }
         
       Compiler::Operand* a = frame->popObject();
       Compiler::Operand* b = frame->popObject();
@@ -5069,6 +5096,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned initialIp,
       uint32_t offset = codeReadInt16(t, code, ip);
       newIp = (ip - 3) + offset;
       assert(t, newIp < codeLength(t, code));
+
+      if(newIp <= ip) {
+        compileSafePoint(t, c, frame);
+      }
         
       Compiler::Operand* a = frame->popInt();
       Compiler::Operand* b = frame->popInt();
@@ -5110,6 +5141,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned initialIp,
 
       Compiler::Operand* target = frame->machineIp(newIp);
 
+      if(newIp <= ip) {
+        compileSafePoint(t, c, frame);
+      }
+
       Compiler::Operand* a = c->constant(0, Compiler::IntegerType);
       Compiler::Operand* b = frame->popInt();
 
@@ -5143,6 +5178,10 @@ compile(MyThread* t, Frame* initialFrame, unsigned initialIp,
       newIp = (ip - 3) + offset;
       assert(t, newIp < codeLength(t, code));
 
+      if(newIp <= ip) {
+        compileSafePoint(t, c, frame);
+      }
+
       Compiler::Operand* a = c->constant(0, Compiler::ObjectType);
       Compiler::Operand* b = frame->popObject();
       Compiler::Operand* target = frame->machineIp(newIp);

src/thunks.cpp

@@ -70,3 +70,4 @@ THUNK(set)
 THUNK(getJClass64)
 THUNK(getJClassFromReference)
 THUNK(gcIfNecessary)
+THUNK(idleIfNecessary)

test/Busy.java

@@ -0,0 +1,25 @@
+public class Busy {
+  private static volatile int foo = 0;
+  private static volatile boolean go;
+
+  public static void main(String[] args) {
+    final Object lock = new Object();
+
+    synchronized (lock) {
+      new Thread() {
+        public void run() {
+          while (foo < 100) {
+            go = true;
+          }
+        }
+      }.start();
+
+      while (foo < 100) {
+        while (! go) { }
+        go = false;
+        byte[] array = new byte[256 * 1024];
+        ++ foo;
+      }
+    }
+  }
+}