From f8d3494b1cd7ca96e28ab6767929d8b5f4a00980 Mon Sep 17 00:00:00 2001
From: Joel Dice <joel.dice@gmail.com>
Date: Fri, 5 Oct 2012 10:06:01 -0600
Subject: [PATCH] clear any weak/soft/phantom references to finalizable objects
 before queuing

If we don't clear these references, we risk finalizing objects which
can still be reached by one of the special reference types.

It's a bit of a chicken-and-egg problem.  We need to visit finalizable
objects before visiting weak references, since some of the weak
references and/or their targets may become reachable once the
finalizable objects are visited.  However, that ordering means we have
no efficient way of distinguishing between objects which are reachable
from one or more normal GC roots and those which are only reachable
via the finalization queue.  The solution is to clear all weak
references to finalizable objects before visiting them.
---
 src/heap.cpp    |  1 +
 src/machine.cpp | 45 +++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/src/heap.cpp b/src/heap.cpp
index b46faeda01..2dc23e904e 100644
--- a/src/heap.cpp
+++ b/src/heap.cpp
@@ -1077,6 +1077,7 @@ update3(Context* c, void* o, bool* needsVisit)
         fprintf(stderr, "mark fixie %p\n", f);
       }
       f->marked(true);
+      f->dead(false);
       f->move(c, &(c->markedFixies));
     }
     *needsVisit = false;
diff --git a/src/machine.cpp b/src/machine.cpp
index 09a314b9f2..743f56e41a 100644
--- a/src/machine.cpp
+++ b/src/machine.cpp
@@ -474,6 +474,25 @@ referenceTargetReachable(Thread* t, Heap::Visitor* v, object* p)
   }
 }
 
+bool
+isFinalizable(Thread* t, object o)
+{
+  return t->m->heap->status(o) == Heap::Unreachable
+    and (classVmFlags
+         (t, static_cast<object>(t->m->heap->follow(objectClass(t, o))))
+         & HasFinalizerFlag);
+}
+
+void
+clearTargetIfFinalizable(Thread* t, object r)
+{
+  if (isFinalizable
+      (t, static_cast<object>(t->m->heap->follow(jreferenceTarget(t, r)))))
+  {
+    jreferenceTarget(t, r) = 0;
+  }
+}
+
 void
 postVisit(Thread* t, Heap::Visitor* v)
 {
@@ -482,6 +501,30 @@ postVisit(Thread* t, Heap::Visitor* v)
 
   assert(t, m->finalizeQueue == 0);
 
+  m->heap->postVisit();
+
+  for (object p = m->weakReferences; p;) {
+    object r = static_cast<object>(m->heap->follow(p));
+    p = jreferenceVmNext(t, r);
+    clearTargetIfFinalizable(t, r);
+  }
+
+  if (major) {
+    for (object p = m->tenuredWeakReferences; p;) {
+      object r = static_cast<object>(m->heap->follow(p));
+      p = jreferenceVmNext(t, r);
+      clearTargetIfFinalizable(t, r);
+    }
+  }
+
+  for (Reference* r = m->jniReferences; r; r = r->next) {
+    if (r->weak and isFinalizable
+        (t, static_cast<object>(t->m->heap->follow(r->target))))
+    {
+      r->target = 0;
+    }
+  }
+
   object firstNewTenuredFinalizer = 0;
   object lastNewTenuredFinalizer = 0;
 
@@ -594,8 +637,6 @@ postVisit(Thread* t, Heap::Visitor* v)
     m->tenuredWeakReferences = firstNewTenuredWeakReference;
   }
 
-  m->heap->postVisit();
-
   for (Reference* r = m->jniReferences; r; r = r->next) {
     if (r->weak) {
       if (m->heap->status(r->target) == Heap::Unreachable) {