diff --git a/src/common.h b/src/common.h index b9990a674f..535e231b6c 100644 --- a/src/common.h +++ b/src/common.h @@ -325,6 +325,14 @@ markBit(uintptr_t* map, unsigned i) map[wordOf(i)] |= static_cast(1) << bitOf(i); } +inline void +markBitAtomic(uintptr_t* map, unsigned i) +{ + uintptr_t* p = map + wordOf(i); + uintptr_t v = static_cast(1) << bitOf(i); + while (not __sync_bool_compare_and_swap(p, *p, *p | v)) { } +} + inline void clearBit(uintptr_t* map, unsigned i) { diff --git a/src/heap.cpp b/src/heap.cpp index 5067909498..11c3c4ceda 100644 --- a/src/heap.cpp +++ b/src/heap.cpp @@ -303,6 +303,13 @@ class Segment { if (child) child->set(p, v); } + void markAtomic(void* p) { + assert(segment->context, bitsPerRecord == 1); + markBitAtomic(data, indexOf(p)); + assert(segment->context, getBit(data, indexOf(p))); + if (child) child->markAtomic(p); + } + unsigned get(void* p) { return getBits(data, bitsPerRecord, indexOf(p)); } @@ -1013,8 +1020,12 @@ void markDirty(Context* c, Fixie* f) { if (not f->dirty) { - f->dirty = true; - f->move(c, &(c->dirtyTenuredFixies)); + ACQUIRE(c->lock); + + if (not f->dirty) { + f->dirty = true; + f->move(c, &(c->dirtyTenuredFixies)); + } } } @@ -1784,7 +1795,7 @@ class MyHeap: public Heap { Fixie(&c, sizeInWords, objectMask, 0, true))->body(); } - virtual bool needsMark(void* p) { + bool needsMark(void* p) { assert(&c, c.client->isFixed(p) or (not immortalHeapContains(&c, p))); if (c.client->isFixed(p)) { @@ -1794,11 +1805,6 @@ class MyHeap: public Heap { } } - virtual bool needsMark(void* p, unsigned offset) { - return needsMark(p) and targetNeedsMark - (mask(*(static_cast(p) + offset))); - } - bool targetNeedsMark(void* target) { return target and not c.gen2.contains(target) @@ -1809,38 +1815,41 @@ class MyHeap: public Heap { } virtual void mark(void* p, unsigned offset, unsigned count) { - if (c.client->isFixed(p)) { - Fixie* f = fixie(p); - assert(&c, offset == 0 or f->hasMask); + if (needsMark(p)) { + if (c.client->isFixed(p)) { + Fixie* f = fixie(p); + assert(&c, offset == 0 or f->hasMask); - bool dirty = false; - for (unsigned i = 0; i < count; ++i) { - void** target = static_cast(p) + offset + i; - if (targetNeedsMark(mask(*target))) { - if (DebugFixies) { - fprintf(stderr, "dirty fixie %p at %d (%p): %p\n", - f, offset, f->body() + offset, mask(*target)); + bool dirty = false; + for (unsigned i = 0; i < count; ++i) { + void** target = static_cast(p) + offset + i; + if (targetNeedsMark(mask(*target))) { + if (DebugFixies) { + fprintf(stderr, "dirty fixie %p at %d (%p): %p\n", + f, offset, f->body() + offset, mask(*target)); + } + + dirty = true; + markBitAtomic(f->mask(), offset + i); + assert(&c, getBit(f->mask(), offset + i)); } - - dirty = true; - markBit(f->mask(), offset + i); } - } - if (dirty) markDirty(&c, f); - } else { - Segment::Map* map; - if (c.gen2.contains(p)) { - map = &(c.heapMap); + if (dirty) markDirty(&c, f); } else { - assert(&c, c.nextGen2.contains(p)); - map = &(c.nextHeapMap); - } + Segment::Map* map; + if (c.gen2.contains(p)) { + map = &(c.heapMap); + } else { + assert(&c, c.nextGen2.contains(p)); + map = &(c.nextHeapMap); + } - for (unsigned i = 0; i < count; ++i) { - void** target = static_cast(p) + offset + i; - if (targetNeedsMark(mask(*target))) { - map->set(target); + for (unsigned i = 0; i < count; ++i) { + void** target = static_cast(p) + offset + i; + if (targetNeedsMark(mask(*target))) { + map->markAtomic(target); + } } } } diff --git a/src/heap.h b/src/heap.h index 28f1533d8f..769bc1ec4b 100644 --- a/src/heap.h +++ b/src/heap.h @@ -59,8 +59,6 @@ class Heap: public Allocator { virtual void* allocateImmortalFixed(Allocator* allocator, unsigned sizeInWords, bool objectMask, unsigned* totalInBytes) = 0; - virtual bool needsMark(void* p) = 0; - virtual bool needsMark(void* p, unsigned offset) = 0; virtual void mark(void* p, unsigned offset, unsigned count) = 0; virtual void pad(void* p) = 0; virtual void* follow(void* p) = 0; diff --git a/src/machine.h b/src/machine.h index 1bc04a9e9e..598a72e424 100644 --- a/src/machine.h +++ b/src/machine.h @@ -1584,19 +1584,13 @@ allocate(Thread* t, unsigned sizeInBytes, bool objectMask) inline void mark(Thread* t, object o, unsigned offset, unsigned count) { - if (t->m->heap->needsMark(o)) { - ACQUIRE_RAW(t, t->m->heapLock); - t->m->heap->mark(o, offset / BytesPerWord, count); - } + t->m->heap->mark(o, offset / BytesPerWord, count); } inline void mark(Thread* t, object o, unsigned offset) { - if (t->m->heap->needsMark(o, offset / BytesPerWord)) { - ACQUIRE_RAW(t, t->m->heapLock); - t->m->heap->mark(o, offset / BytesPerWord, 1); - } + t->m->heap->mark(o, offset / BytesPerWord, 1); } inline void