From 7f1837fecd93706688e878259a0bd40341d34a4d Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Sun, 28 Oct 2007 13:14:53 -0600 Subject: [PATCH] move fixed object (mark and sweep) support into heap.cpp and refine algorithms for determining when and how much to GC --- makefile | 2 +- src/common.h | 4 - src/compile.cpp | 20 +- src/heap.cpp | 298 +++++++++++++++++++++++++--- src/heap.h | 13 +- src/interpret.cpp | 4 +- src/jnienv.cpp | 5 +- src/machine.cpp | 480 ++++++++++++++-------------------------------- src/machine.h | 137 ++----------- 9 files changed, 463 insertions(+), 500 deletions(-) diff --git a/makefile b/makefile index 34f65964bb..405454ece6 100644 --- a/makefile +++ b/makefile @@ -28,7 +28,7 @@ src = src classpath = classpath test = test -input = $(test-build)/Enums.class +input = $(test-build)/GC.class build-cxx = g++ build-cc = gcc diff --git a/src/common.h b/src/common.h index 9eced04931..625cf2c076 100644 --- a/src/common.h +++ b/src/common.h @@ -232,10 +232,6 @@ bitsToFloat(uint32_t bits) return f; } -// an object must survive TenureThreshold + 2 garbage collections -// before being copied to gen2 (muat be at least 1): -const unsigned TenureThreshold = 3; - class Machine; class Thread; diff --git a/src/compile.cpp b/src/compile.cpp index 0696afd16e..d3b6582911 100644 --- a/src/compile.cpp +++ b/src/compile.cpp @@ -674,7 +674,7 @@ class StackMapper { MyProtector(StackMapper* mapper): Protector(mapper->t), mapper(mapper) { } virtual void visit(Heap::Visitor* v) { - vm::visit(t, v, &(mapper->method)); + v->visit(&(mapper->method)); } StackMapper* mapper; @@ -1073,14 +1073,14 @@ visitParameters(MyThread* t, Heap::Visitor* v, void* frame) unsigned index = 0; if ((methodFlags(t, method) & ACC_STATIC) == 0) { - visit(t, v, frameLocalObject(t, frame, index++)); + v->visit(frameLocalObject(t, frame, index++)); } for (MethodSpecIterator it(t, spec); it.hasNext();) { switch (*it.next()) { case 'L': case '[': - visit(t, v, frameLocalObject(t, frame, index++)); + v->visit(frameLocalObject(t, frame, index++)); break; case 'J': @@ -1122,7 +1122,7 @@ visitStackAndLocals(MyThread* t, Heap::Visitor* v, void* frame) for (unsigned i = 0; i < count; ++i) { if (getBit(stackMap, i)) { - visit(t, v, frameLocalObject(t, frame, i + parameterFootprint)); + v->visit(frameLocalObject(t, frame, i + parameterFootprint)); } } } @@ -1134,7 +1134,7 @@ visitStack(MyThread* t, Heap::Visitor* v) void* frame = frameStart(t); if (frameValid(frame)) { - visit(t, v, &frameMethod(frame)); + v->visit(&frameMethod(frame)); } for (; frameValid(frame); frame = frameNext(frame)) { @@ -1142,7 +1142,7 @@ visitStack(MyThread* t, Heap::Visitor* v) // caller is native. Otherwise, the caller owns them. void* next = frameNext(frame); if (frameValid(next)) { - visit(t, v, &frameMethod(next)); + v->visit(&frameMethod(next)); if (methodFlags(t, frameMethod(next)) & ACC_NATIVE) { visitParameters(t, v, frame); @@ -4573,10 +4573,10 @@ class JavaCompiler: public Compiler { MyProtector(JavaCompiler* c): Protector(c->t), c(c) { } virtual void visit(Heap::Visitor* v) { - vm::visit(t, v, &(c->method)); + v->visit(&(c->method)); for (unsigned i = 0; i < c->pool.length(); i += BytesPerWord) { - vm::visit(t, v, reinterpret_cast(&(c->pool.getAddress(i)))); + v->visit(reinterpret_cast(&(c->pool.getAddress(i)))); } } @@ -4799,7 +4799,7 @@ class ArgumentList { virtual void visit(Heap::Visitor* v) { for (unsigned i = 0; i < list->position; ++i) { if (list->objectMask[i]) { - vm::visit(t, v, reinterpret_cast(list->array + i)); + v->visit(reinterpret_cast(list->array + i)); } } } @@ -4984,7 +4984,7 @@ class MyProcessor: public Processor { if (t->m->active) { for (Reference* r = t->reference; r; r = r->next) { - visit(t, v, &(r->target)); + v->visit(&(r->target)); } visitStack(t, v); diff --git a/src/heap.cpp b/src/heap.cpp index 5e8f76830d..81162a7f5c 100644 --- a/src/heap.cpp +++ b/src/heap.cpp @@ -6,6 +6,12 @@ using namespace vm; namespace { +// an object must survive TenureThreshold + 2 garbage collections +// before being copied to gen2 (muat be at least 1): +const unsigned TenureThreshold = 3; + +const unsigned FixieTenureThreshold = TenureThreshold + 2; + const unsigned Top = ~static_cast(0); const unsigned InitialGen2CapacityInBytes = 4 * 1024 * 1024; @@ -381,11 +387,77 @@ class Segment { } }; +class Fixie { + public: + Fixie(unsigned size, bool hasMask, Fixie** handle): + age(0), + hasMask(hasMask), + marked(false), + dirty(false) + { + memset(mask(size), 0, maskSize(size, hasMask)); + add(handle); + if (Debug) { + fprintf(stderr, "make %p\n", this); + } + } + + void add(Fixie** handle) { + this->handle = handle; + next = *handle; + if (next) next->handle = &next; + *handle = this; + } + + void remove() { + *handle = next; + if (next) next->handle = handle; + } + + void move(Fixie** handle) { + remove(); + add(handle); + } + + uintptr_t* mask(unsigned size) { + return body + size; + } + + static unsigned maskSize(unsigned size, bool hasMask) { + return hasMask * ceiling(size, BytesPerWord) * BytesPerWord; + } + + static unsigned totalSize(unsigned size, bool hasMask) { + return sizeof(Fixie) + (size * BytesPerWord) + maskSize(size, hasMask); + } + + unsigned totalSize(unsigned size) { + return totalSize(size, hasMask); + } + + uint8_t age; + bool hasMask; + bool marked; + bool dirty; + Fixie* next; + Fixie** handle; + uintptr_t body[0]; +}; + +Fixie* +fixie(void* body) +{ + return static_cast(body) - 1; +} + +void +free(Context* c, Fixie** fixies); + class Context { public: - Context(System* system): + Context(System* system, Heap::Client* client): system(system), - client(0), + client(client), ageMap(&gen1, log(TenureThreshold), 1, 0, false), gen1(this, &ageMap, 0, 0), @@ -406,10 +478,17 @@ class Context { gen2Base(0), tenureFootprint(0), + fixieTenureFootprint(0), gen1padding(0), gen2padding(0), mode(Heap::MinorCollection), + fixies(0), + tenuredFixies(0), + dirtyFixies(0), + markedFixies(0), + visitedFixies(0), + lastCollectionTime(system->now()), totalCollectionTime(0), totalTime(0) @@ -420,6 +499,10 @@ class Context { nextGen1.dispose(); gen2.dispose(); nextGen2.dispose(); + free(this, &tenuredFixies); + free(this, &dirtyFixies); + free(this, &fixies); + client->dispose(); } System* system; @@ -444,11 +527,18 @@ class Context { unsigned gen2Base; unsigned tenureFootprint; + unsigned fixieTenureFootprint; unsigned gen1padding; unsigned gen2padding; Heap::CollectionType mode; + Fixie* fixies; + Fixie* tenuredFixies; + Fixie* dirtyFixies; + Fixie* markedFixies; + Fixie* visitedFixies; + int64_t lastCollectionTime; int64_t totalCollectionTime; int64_t totalTime; @@ -568,6 +658,62 @@ bitset(Context* c UNUSED, void* o) return &cast(o, BytesPerWord * 2); } +void +free(Context* c, Fixie** fixies) +{ + for (Fixie** p = fixies; *p;) { + Fixie* f = *p; + *p = f->next; + if (Debug) { + fprintf(stderr, "free %p\n", f); + } + c->system->free(f); + } +} + +void +sweepFixies(Context* c) +{ + assert(c, c->markedFixies == 0); + + if (c->mode == Heap::MajorCollection) { + free(c, &(c->tenuredFixies)); + free(c, &(c->dirtyFixies)); + } + free(c, &(c->fixies)); + + for (Fixie** p = &(c->visitedFixies); *p;) { + Fixie* f = *p; + *p = f->next; + + unsigned size = c->client->sizeInWords(f->body); + + ++ f->age; + if (f->age > FixieTenureThreshold) { + f->age = FixieTenureThreshold; + } else if (static_cast(f->age + 1) == FixieTenureThreshold) { + c->fixieTenureFootprint += f->totalSize(size); + } + + if (f->age == FixieTenureThreshold) { + if (Debug) { + fprintf(stderr, "tenure %p\n", f); + } + f->move(&(c->tenuredFixies)); + + if (f->dirty) { + uintptr_t* mask = f->mask(size); + memset(mask, 0, ceiling(size, BytesPerWord) * BytesPerWord); + f->dirty = false; + } + } else { + f->move(&(c->fixies)); + } + + f->marked = false; + } +} + inline void* copyTo(Context* c, Segment* s, void* o, unsigned size) { @@ -644,7 +790,18 @@ update3(Context* c, void* o, bool* needsVisit) if (wasCollected(c, o)) { *needsVisit = false; return follow(c, o); - } else if (c->client->checkFixed(o)) { + } else if (c->client->isFixed(o)) { + Fixie* f = fixie(o); + if ((not f->marked) + and (c->mode == Heap::MajorCollection + or f->age < FixieTenureThreshold)) + { + if (Debug) { + fprintf(stderr, "mark %p\n", f); + } + f->marked = true; + f->move(&(c->markedFixies)); + } *needsVisit = false; return o; } else { @@ -992,6 +1149,64 @@ collect(Context* c, void** p) } } +void +visitDirtyFixies(Context* c) +{ + for (Fixie** p = &(c->dirtyFixies); *p;) { + Fixie* f = *p; + *p = f->next; + + unsigned size = c->client->sizeInWords(f->body); + uintptr_t* mask = f->mask(size); + for (unsigned word = 0; word < wordOf(size); ++ word) { + if (mask[word]) { + for (unsigned bit = 0; bit < bitOf(size); ++ bit) { + unsigned index = indexOf(word, bit); + if (getBit(mask, index)) { + collect(c, reinterpret_cast(f->body) + index); + } + } + } + } + + memset(mask, 0, ceiling(size, BytesPerWord) * BytesPerWord); + f->dirty = false; + f->move(&(c->tenuredFixies)); + } +} + +void +visitMarkedFixies(Context* c) +{ + for (Fixie** p = &(c->markedFixies); *p;) { + Fixie* f = *p; + *p = f->next; + + if (Debug) { + fprintf(stderr, "visit %p\n", f); + } + + class Walker: public Heap::Walker { + public: + Walker(Context* c, void** p): + c(c), p(p) + { } + + virtual bool visit(unsigned offset) { + collect(c, p + offset); + return true; + } + + Context* c; + void** p; + } w(c, reinterpret_cast(f->body)); + + c->client->walk(reinterpret_cast(f->body), &w); + + f->move(&(c->visitedFixies)); + } +} + void collect(Context* c, Segment::Map* map, unsigned start, unsigned end, bool* dirty, bool expectDirty UNUSED) @@ -1038,6 +1253,7 @@ collect2(Context* c) { c->gen2Base = Top; c->tenureFootprint = 0; + c->fixieTenureFootprint = 0; c->gen1padding = 0; c->gen2padding = 0; @@ -1048,12 +1264,17 @@ collect2(Context* c) collect(c, &(c->heapMap), start, end, &dirty, false); } + if (c->mode == Heap::MinorCollection) { + visitDirtyFixies(c); + } + class Visitor : public Heap::Visitor { public: Visitor(Context* c): c(c) { } virtual void visit(void* p) { collect(c, static_cast(p)); + visitMarkedFixies(c); } Context* c; @@ -1065,7 +1286,9 @@ collect2(Context* c) void collect(Context* c, unsigned footprint) { - if (c->tenureFootprint > c->gen2.remaining()) { + if (c->tenureFootprint > c->gen2.remaining() + or c->fixieTenureFootprint) + { c->mode = Heap::MajorCollection; } @@ -1092,6 +1315,8 @@ collect(Context* c, unsigned footprint) c->gen2.replaceWith(&(c->nextGen2)); } + sweepFixies(c); + if (Verbose) { int64_t now = c->system->now(); int64_t collection = now - then; @@ -1114,32 +1339,61 @@ collect(Context* c, unsigned footprint) class MyHeap: public Heap { public: - MyHeap(System* system): c(system) { } + MyHeap(System* system, Heap::Client* client): c(system, client) { } - virtual void collect(CollectionType type, Client* client, unsigned footprint) - { + virtual void collect(CollectionType type, unsigned footprint) { c.mode = type; - c.client = client; ::collect(&c, footprint); } - virtual bool needsMark(void* p) { - return *static_cast(p) - and c.gen2.contains(p) - and not c.gen2.contains(*static_cast(p)); + virtual void* allocateFixed(unsigned sizeInWords, bool objectMask, + unsigned* totalInBytes) + { + *totalInBytes = Fixie::totalSize(sizeInWords, objectMask); + return (new (c.system->allocate(*totalInBytes)) + Fixie(sizeInWords, objectMask, &(c.fixies)))->body; } - virtual void mark(void* p) { - if (Debug) { - fprintf(stderr, "mark %p (%s) at %p (%s)\n", - *static_cast(p), - segment(&c, *static_cast(p)), - p, - segment(&c, p)); + virtual bool needsMark(void* p) { + if (c.client->isFixed(p)) { + return fixie(p)->age == FixieTenureThreshold; + } else { + return c.gen2.contains(p); } + } - c.heapMap.set(p); + bool targetNeedsMark(void* target) { + return target + and not c.gen2.contains(target) + and not (c.client->isFixed(target) + and fixie(target)->age == FixieTenureThreshold); + } + + virtual void mark(void* p, unsigned offset, unsigned count) { + if (c.client->isFixed(p)) { + Fixie* f = fixie(p); + unsigned size = c.client->sizeInWords(p); + + for (unsigned i = 0; i < count; ++i) { + void** target = static_cast(p) + offset + i; + if (targetNeedsMark(*target)) { + f->dirty = true; + markBit(f->mask(size), offset + i); + } + } + + if (f->dirty) { + f->move(&(c.dirtyFixies)); + } + } else { + for (unsigned i = 0; i < count; ++i) { + void** target = static_cast(p) + offset + i; + if (targetNeedsMark(*target)) { + c.heapMap.set(target); + } + } + } } virtual void pad(void* p, unsigned extra) { @@ -1207,9 +1461,9 @@ class MyHeap: public Heap { namespace vm { Heap* -makeHeap(System* system) +makeHeap(System* system, Heap::Client* client) { - return new (system->allocate(sizeof(MyHeap))) MyHeap(system); + return new (system->allocate(sizeof(MyHeap))) MyHeap(system, client); } } // namespace vm diff --git a/src/heap.h b/src/heap.h index 9610827e95..7339c64221 100644 --- a/src/heap.h +++ b/src/heap.h @@ -35,17 +35,20 @@ class Heap { public: virtual ~Client() { } virtual void visitRoots(Visitor*) = 0; - virtual bool checkFixed(void*) = 0; + virtual bool isFixed(void*) = 0; + virtual unsigned sizeInWords(void*) = 0; virtual unsigned copiedSizeInWords(void*) = 0; virtual void copy(void*, void*) = 0; virtual void walk(void*, Walker*) = 0; + virtual void dispose() = 0; }; virtual ~Heap() { } - virtual void collect(CollectionType type, Client* client, unsigned footprint) - = 0; + virtual void collect(CollectionType type, unsigned footprint) = 0; + virtual void* allocateFixed(unsigned sizeInWords, bool objectMask, + unsigned* totalInBytes) = 0; virtual bool needsMark(void* p) = 0; - virtual void mark(void* p) = 0; + virtual void mark(void* p, unsigned offset, unsigned count) = 0; virtual void pad(void* p, unsigned extra) = 0; virtual void* follow(void* p) = 0; virtual Status status(void* p) = 0; @@ -53,7 +56,7 @@ class Heap { virtual void dispose() = 0; }; -Heap* makeHeap(System* system); +Heap* makeHeap(System* system, Heap::Client* client); } // namespace vm diff --git a/src/interpret.cpp b/src/interpret.cpp index 79bb29d5f5..287d3d5bb0 100644 --- a/src/interpret.cpp +++ b/src/interpret.cpp @@ -2955,11 +2955,11 @@ class MyProcessor: public Processor { { Thread* t = static_cast(vmt); - visit(t, v, &(t->code)); + v->visit(&(t->code)); for (unsigned i = 0; i < t->sp; ++i) { if (t->stack[i * 2] == ObjectTag) { - visit(t, v, reinterpret_cast(t->stack + (i * 2) + 1)); + v->visit(reinterpret_cast(t->stack + (i * 2) + 1)); } } } diff --git a/src/jnienv.cpp b/src/jnienv.cpp index 0778437970..7793494ec3 100644 --- a/src/jnienv.cpp +++ b/src/jnienv.cpp @@ -22,7 +22,6 @@ DestroyJavaVM(Machine* m) { System* s = m->system; Processor* p = m->processor; - Heap* h = m->heap; Finder* f = m->finder; Thread* t = m->rootThread; @@ -32,7 +31,6 @@ DestroyJavaVM(Machine* m) m->dispose(); p->dispose(); - h->dispose(); f->dispose(); s->dispose(); @@ -1884,10 +1882,9 @@ JNI_CreateJavaVM(Machine** m, Thread** t, void* args) BUILTIN_CLASSPATH, s->pathSeparator(), a->classpath); Finder* f = makeFinder(s, classpath); - Heap* h = makeHeap(s); Processor* p = makeProcessor(s); - *m = new (s->allocate(sizeof(Machine))) Machine(s, h, f, p); + *m = new (s->allocate(sizeof(Machine))) Machine(s, f, p); if (a->properties) { for (const char** p = a->properties; *p; ++p) { diff --git a/src/machine.cpp b/src/machine.cpp index e662d7f99b..511741c490 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -126,8 +126,8 @@ void visitRoots(Thread* t, Heap::Visitor* v) { if (t->state != Thread::ZombieState) { - visit(t, v, &(t->javaThread)); - visit(t, v, &(t->exception)); + v->visit(&(t->javaThread)); + v->visit(&(t->exception)); t->m->processor->visitObjects(t, v); @@ -210,7 +210,7 @@ walk(Thread* t, object o, Heap::Walker* w) void finalizerTargetUnreachable(Thread* t, Heap::Visitor* v, object* p) { - visit(t, v, &finalizerTarget(t, *p)); + v->visit(&finalizerTarget(t, *p)); object finalizer = *p; *p = finalizerNext(t, finalizer); @@ -226,7 +226,7 @@ referenceTargetUnreachable(Thread* t, Heap::Visitor* v, object* p) jreferenceTarget(t, *p), *p); } - visit(t, v, p); + v->visit(p); jreferenceTarget(t, *p) = 0; if (jreferenceQueue(t, *p) @@ -234,7 +234,7 @@ referenceTargetUnreachable(Thread* t, Heap::Visitor* v, object* p) { // queue is reachable - add the reference - visit(t, v, &jreferenceQueue(t, *p)); + v->visit(&jreferenceQueue(t, *p)); object q = jreferenceQueue(t, *p); @@ -278,89 +278,13 @@ referenceTargetReachable(Thread* t, Heap::Visitor* v, object* p) jreferenceTarget(t, *p), *p); } - visit(t, v, p); - visit(t, v, &jreferenceTarget(t, *p)); + v->visit(p); + v->visit(&jreferenceTarget(t, *p)); if (t->m->heap->status(jreferenceQueue(t, *p)) == Heap::Unreachable) { jreferenceQueue(t, *p) = 0; } else { - visit(t, v, &jreferenceQueue(t, *p)); - } -} - -void -freeFixies(Thread* t, object* fixies) -{ - for (object* p = fixies; *p;) { - object o = *p; - *p = fixedNext(t, o); - - t->m->system->free(fixedStart(t, o)); - } -} - -void -sweepFixies(Thread* t) -{ - Machine* m = t->m; - - assert(t, m->markedFixies == 0); - - if (m->heap->collectionType() == Heap::MajorCollection) { - freeFixies(t, &(m->tenuredFixies)); - freeFixies(t, &(m->dirtyFixies)); - } - freeFixies(t, &(m->fixies)); - - for (object* p = &(m->visitedFixies); *p;) { - object o = *p; - *p = fixedNext(t, o); - - fixedAge(t, o) = (fixedAge(t, o) + 1); - if (fixedAge(t, o) > TenureThreshold) { - fixedAge(t, o) = TenureThreshold; - } - - if (fixedAge(t, o) == TenureThreshold) { - fixedMove(t, o, &(m->tenuredFixies)); - - if (fixedDirty(t, o)) { - unsigned size = baseSize(t, o, objectClass(t, o)); - uintptr_t* mask = fixedMask(t, o, size); - memset(mask, 0, ceiling(size, BytesPerWord) * BytesPerWord); - fixedDirty(t, o) = 0; - } - } else { - fixedMove(t, o, &(m->fixies)); - } - - fixedMarked(t, o) = 0; - } -} - -void -visitDirtyFixies(Thread* t, Heap::Visitor* v) -{ - for (object* p = &(t->m->dirtyFixies); *p;) { - object o = *p; - *p = fixedNext(t, o); - - unsigned size = baseSize(t, o, objectClass(t, o)); - uintptr_t* mask = fixedMask(t, o, size); - for (unsigned word = 0; word < wordOf(size); ++ word) { - if (mask[word]) { - for (unsigned bit = 0; bit < bitOf(size); ++ bit) { - unsigned index = indexOf(word, bit); - if (getBit(mask, index)) { - visit(t, v, &cast(o, index)); - } - } - } - } - - memset(mask, 0, ceiling(size, BytesPerWord) * BytesPerWord); - fixedDirty(t, o) = 0; - fixedMove(t, o, &(t->m->tenuredFixies)); + v->visit(&jreferenceQueue(t, *p)); } } @@ -371,27 +295,27 @@ postVisit(Thread* t, Heap::Visitor* v) bool major = m->heap->collectionType() == Heap::MajorCollection; for (object* p = &(m->finalizeQueue); *p; p = &(finalizerNext(t, *p))) { - visit(t, v, p); - visit(t, v, &finalizerTarget(t, *p)); + v->visit(p); + v->visit(&finalizerTarget(t, *p)); } for (object* p = &(m->finalizeQueue); *p; p = &(finalizerNext(t, *p))) { - visit(t, v, p); - visit(t, v, &finalizerTarget(t, *p)); + v->visit(p); + v->visit(&finalizerTarget(t, *p)); } object firstNewTenuredFinalizer = 0; object lastNewTenuredFinalizer = 0; for (object* p = &(m->finalizers); *p;) { - visit(t, v, p); + v->visit(p); if (m->heap->status(finalizerTarget(t, *p)) == Heap::Unreachable) { // target is unreachable - queue it up for finalization finalizerTargetUnreachable(t, v, p); } else { // target is reachable - visit(t, v, &finalizerTarget(t, *p)); + v->visit(&finalizerTarget(t, *p)); if (m->heap->status(*p) == Heap::Tenured) { // the finalizer is tenured, so we remove it from @@ -448,14 +372,14 @@ postVisit(Thread* t, Heap::Visitor* v) if (major) { for (object* p = &(m->tenuredFinalizers); *p;) { - visit(t, v, p); + v->visit(p); if (m->heap->status(finalizerTarget(t, *p)) == Heap::Unreachable) { // target is unreachable - queue it up for finalization finalizerTargetUnreachable(t, v, p); } else { // target is reachable - visit(t, v, &finalizerTarget(t, *p)); + v->visit(&finalizerTarget(t, *p)); p = &finalizerNext(t, *p); } } @@ -500,7 +424,6 @@ postCollect(Thread* t) t->heap = t->defaultHeap; t->heapOffset = 0; t->heapIndex = 0; - t->allocatedLarge = false; for (Thread* c = t->child; c; c = c->peer) { postCollect(c); @@ -1505,15 +1428,109 @@ invoke(Thread* t, const char* className, int argc, const char** argv) (t, className, "main", "([Ljava/lang/String;)V", 0, args); } +class HeapClient: public Heap::Client { + public: + HeapClient(Machine* m): m(m) { } + + virtual void visitRoots(Heap::Visitor* v) { + v->visit(&(m->loader)); + v->visit(&(m->bootstrapClassMap)); + v->visit(&(m->monitorMap)); + v->visit(&(m->stringMap)); + v->visit(&(m->types)); + v->visit(&(m->jniInterfaceTable)); + + for (Reference* r = m->jniReferences; r; r = r->next) { + v->visit(&(r->target)); + } + + for (Thread* t = m->rootThread; t; t = t->peer) { + ::visitRoots(t, v); + } + + postVisit(m->rootThread, v); + } + + virtual bool isFixed(void* p) { + return objectFixed(m->rootThread, static_cast(p)); + } + + virtual unsigned sizeInWords(void* p) { + Thread* t = m->rootThread; + + object o = static_cast(m->heap->follow(mask(p))); + + unsigned n = baseSize(t, o, static_cast + (m->heap->follow(objectClass(t, o)))); + + if (objectExtended(t, o)) { + ++ n; + } + + return n; + } + + virtual unsigned copiedSizeInWords(void* p) { + Thread* t = m->rootThread; + + object o = static_cast(m->heap->follow(mask(p))); + assert(t, not objectFixed(t, o)); + + unsigned n = baseSize(t, o, static_cast + (m->heap->follow(objectClass(t, o)))); + + if (objectExtended(t, o) or hashTaken(t, o)) { + ++ n; + } + + return n; + } + + virtual void copy(void* srcp, void* dstp) { + Thread* t = m->rootThread; + + object src = static_cast(m->heap->follow(mask(srcp))); + assert(t, not objectFixed(t, src)); + + object class_ = static_cast + (m->heap->follow(objectClass(t, src))); + + unsigned base = baseSize(t, src, class_); + unsigned n = extendedSize(t, src, base); + + object dst = static_cast(dstp); + + memcpy(dst, src, n * BytesPerWord); + + if (hashTaken(t, src)) { + cast(dst, 0) &= PointerMask; + cast(dst, 0) |= ExtendedMark; + extendedWord(t, dst, base) = takeHash(t, src); + } + } + + virtual void walk(void* p, Heap::Walker* w) { + object o = static_cast(m->heap->follow(mask(p))); + ::walk(m->rootThread, o, w); + } + + virtual void dispose() { + m->system->free(this); + } + + private: + Machine* m; +}; + } // namespace namespace vm { -Machine::Machine(System* system, Heap* heap, Finder* finder, - Processor* processor): +Machine::Machine(System* system, Finder* finder, Processor* processor): vtable(&javaVMVTable), system(system), - heap(heap), + heap(makeHeap(system, new (system->allocate(sizeof(HeapClient))) + HeapClient(this))), finder(finder), processor(processor), rootThread(0), @@ -1522,6 +1539,7 @@ Machine::Machine(System* system, Heap* heap, Finder* finder, builtins(0), activeCount(0), liveCount(0), + fixedFootprint(0), localThread(0), stateLock(0), heapLock(0), @@ -1539,11 +1557,6 @@ Machine::Machine(System* system, Heap* heap, Finder* finder, finalizeQueue(0), weakReferences(0), tenuredWeakReferences(0), - fixies(0), - tenuredFixies(0), - dirtyFixies(0), - markedFixies(0), - visitedFixies(0), unsafe(false), heapPoolIndex(0) { @@ -1583,6 +1596,8 @@ Machine::dispose() system->free(heapPool[i]); } + heap->dispose(); + system->free(this); } @@ -1593,7 +1608,6 @@ Thread::Thread(Machine* m, object javaThread, Thread* parent): peer((parent ? parent->child : 0)), child(0), state(NoState), - allocatedLarge(false), criticalLevel(0), systemThread(0), javaThread(javaThread), @@ -1769,10 +1783,6 @@ exit(Thread* t) function(t, finalizerTarget(t, f)); } - freeFixies(t, &(t->m->tenuredFixies)); - freeFixies(t, &(t->m->dirtyFixies)); - freeFixies(t, &(t->m->fixies)); - disposeAll(t, t->m->rootThread); } @@ -1887,44 +1897,7 @@ enter(Thread* t, Thread::State s) } object -allocateFixed(Thread* t, unsigned sizeInBytes, bool objectMask) -{ - ENTER(t, Thread::ExclusiveState); - - unsigned mask = objectMask - * ceiling(sizeInBytes / BytesPerWord, BytesPerWord) - * BytesPerWord; - unsigned total = sizeInBytes + FixedFootprint + mask; - - uint8_t* p = static_cast(t->m->system->tryAllocate(total)); - if (p == 0) { - collect(t, Heap::MajorCollection); - - p = static_cast(t->m->system->allocate(total)); - } - - memset(p + FixedFootprint + sizeInBytes, 0, mask); - - object o = reinterpret_cast(p + FixedFootprint); - - cast(o, 0) = FixedMark; - fixedAge(t, o) = 0; - fixedMarked(t, o) = 0; - fixedDirty(t, o) = 0; - fixedAdd(t, o, &(t->m->fixies)); - - return o; -} - -object -allocateLarge(Thread* t, unsigned sizeInBytes, bool objectMask) -{ - t->allocatedLarge = true; - return allocateFixed(t, sizeInBytes, objectMask); -} - -object -allocate2(Thread* t, unsigned sizeInBytes, bool objectMask) +allocate2(Thread* t, unsigned sizeInBytes, bool objectMask, bool fixed) { ACQUIRE_RAW(t, t->m->stateLock); @@ -1934,14 +1907,17 @@ allocate2(Thread* t, unsigned sizeInBytes, bool objectMask) ENTER(t, Thread::IdleState); } - if (sizeInBytes <= Thread::HeapSizeInBytes - and t->heapIndex + ceiling(sizeInBytes, BytesPerWord) - >= Thread::HeapSizeInWords) + if (fixed) { + if (t->m->fixedFootprint + sizeInBytes + > Machine::FixedFootprintThresholdInBytes) + { + t->heap = 0; + } + } else if (t->heapIndex + ceiling(sizeInBytes, BytesPerWord) + >= Thread::HeapSizeInWords) { t->heap = 0; - if ((not t->allocatedLarge) - and t->m->heapPoolIndex < Machine::HeapPoolSize) - { + if (t->m->heapPoolIndex < Machine::HeapPoolSize) { t->heap = static_cast (t->m->system->tryAllocate(Thread::HeapSizeInBytes)); if (t->heap) { @@ -1950,15 +1926,24 @@ allocate2(Thread* t, unsigned sizeInBytes, bool objectMask) t->heapIndex = 0; } } - - if (t->heap == 0) { - ENTER(t, Thread::ExclusiveState); - collect(t, Heap::MinorCollection); - } } - if (sizeInBytes > Thread::HeapSizeInBytes) { - return allocateLarge(t, sizeInBytes, objectMask); + if (t->heap == 0) { + ENTER(t, Thread::ExclusiveState); + collect(t, Heap::MinorCollection); + } + + if (fixed) { + unsigned total; + object o = static_cast + (t->m->heap->allocateFixed + (ceiling(sizeInBytes, BytesPerWord), objectMask, &total)); + + cast(o, 0) = FixedMark; + + t->m->fixedFootprint += total; + + return o; } else { return allocateSmall(t, sizeInBytes); } @@ -2705,107 +2690,8 @@ collect(Thread* t, Heap::CollectionType type) { Machine* m = t->m; - class Client: public Heap::Client { - public: - Client(Machine* m): m(m) { } - - virtual void visitRoots(Heap::Visitor* v) { - Thread* t = m->rootThread; - - visit(t, v, &(m->loader)); - visit(t, v, &(m->bootstrapClassMap)); - visit(t, v, &(m->monitorMap)); - visit(t, v, &(m->stringMap)); - visit(t, v, &(m->types)); - visit(t, v, &(m->jniInterfaceTable)); - - for (Reference* r = m->jniReferences; r; r = r->next) { - visit(t, v, &(r->target)); - } - - for (Thread* t = m->rootThread; t; t = t->peer) { - ::visitRoots(t, v); - } - - if (m->heap->collectionType() == Heap::MinorCollection) { - visitDirtyFixies(t, v); - } - - postVisit(m->rootThread, v); - } - - virtual bool checkFixed(void* p) { - Thread* t = m->rootThread; - object o = static_cast(p); - - if (objectFixed(t, o)) { - if ((not fixedMarked(t, o)) - and m->heap->collectionType() == Heap::MajorCollection - or fixedAge(t, o) < TenureThreshold) - { - fixedMarked(t, o) = 1; - fixedMove(t, o, &(m->markedFixies)); - } - return true; - } else { - return false; - } - } - - virtual unsigned copiedSizeInWords(void* p) { - Thread* t = m->rootThread; - - object o = static_cast(m->heap->follow(mask(p))); - assert(t, not objectFixed(t, o)); - - unsigned n = baseSize(t, o, static_cast - (m->heap->follow(objectClass(t, o)))); - - if (objectExtended(t, o) or hashTaken(t, o)) { - ++ n; - } - - return n; - } - - virtual void copy(void* srcp, void* dstp) { - Thread* t = m->rootThread; - - object src = static_cast(m->heap->follow(mask(srcp))); - assert(t, not objectFixed(t, src)); - - object class_ = static_cast - (m->heap->follow(objectClass(t, src))); - - unsigned base = baseSize(t, src, class_); - unsigned n = extendedSize(t, src, base); - - object dst = static_cast(dstp); - - memcpy(dst, src, n * BytesPerWord); - - if (hashTaken(t, src)) { - cast(dst, 0) &= PointerMask; - cast(dst, 0) |= ExtendedMark; - extendedWord(t, dst, base) = takeHash(t, src); - } - } - - virtual void walk(void* p, Heap::Walker* w) { - Thread* t = m->rootThread; - - object o = static_cast(m->heap->follow(mask(p))); - assert(t, not objectFixed(t, o)); - - ::walk(m->rootThread, o, w); - } - - private: - Machine* m; - } it(m); - m->unsafe = true; - m->heap->collect(type, &it, footprint(m->rootThread)); + m->heap->collect(type, footprint(m->rootThread)); m->unsafe = false; postCollect(m->rootThread); @@ -2824,7 +2710,7 @@ collect(Thread* t, Heap::CollectionType type) } m->heapPoolIndex = 0; - sweepFixies(t); + m->fixedFootprint = 0; } void @@ -2907,84 +2793,6 @@ makeTrace(Thread* t, uintptr_t start) return trace; } -void -mark(Thread* t, object o, unsigned offset) -{ - if (objectFixed(t, o)) { - if (fixedAge(t, o) == TenureThreshold) { - ACQUIRE_RAW(t, t->m->heapLock); - - markBit(fixedMask(t, o), offset / BytesPerWord); - - fixedDirty(t, o) = 1; - fixedMove(t, o, &(t->m->dirtyFixies)); - } - } else if (t->m->heap->needsMark(&cast(o, offset))) { - ACQUIRE_RAW(t, t->m->heapLock); - - t->m->heap->mark(&cast(o, offset)); - } -} - -void -mark(Thread* t, object o, unsigned offset, unsigned count) -{ - if (objectFixed(t, o)) { - if (fixedAge(t, o) == TenureThreshold) { - ACQUIRE_RAW(t, t->m->heapLock); - - uintptr_t* mask = fixedMask(t, o); - - for (unsigned i = 0; i < count; ++i) { - markBit(mask, (offset / BytesPerWord) + i); - } - - fixedDirty(t, o) = 1; - fixedMove(t, o, &(t->m->dirtyFixies)); - } - } else { - ACQUIRE_RAW(t, t->m->heapLock); - - for (unsigned i = 0; i < count; ++i) { - unsigned j = offset + (i * BytesPerWord); - if (t->m->heap->needsMark(&cast(o, j))) { - t->m->heap->mark(&cast(o, j)); - } - } - } -} - -void -visit(Thread* t, Heap::Visitor* v, object* p) -{ - v->visit(p); - - for (object* p = &(t->m->markedFixies); *p;) { - object o = *p; - *p = fixedNext(t, o); - - class Walker: public Heap::Walker { - public: - Walker(Thread* t, Heap::Visitor* v, object o): - t(t), v(v), o(o) - { } - - virtual bool visit(unsigned offset) { - ::visit(t, v, &cast(o, offset * BytesPerWord)); - return true; - } - - Thread* t; - Heap::Visitor* v; - object o; - } w(t, v, o); - - walk(t, o, &w); - - fixedMove(t, o, &(t->m->visitedFixies)); - } -} - void noop() { } diff --git a/src/machine.h b/src/machine.h index 94890201fd..e1265cca48 100644 --- a/src/machine.h +++ b/src/machine.h @@ -34,13 +34,6 @@ const uintptr_t HashTakenMark = 1; const uintptr_t ExtendedMark = 2; const uintptr_t FixedMark = 3; -const unsigned FixedAge = 0; -const unsigned FixedMarked = 1; -const unsigned FixedDirty = 2; -const unsigned FixedHandle = BytesPerWord; -const unsigned FixedNext = BytesPerWord * 2; -const unsigned FixedFootprint = BytesPerWord * 3; - enum FieldCode { VoidField, ByteField, @@ -1116,7 +1109,7 @@ class Machine { ActiveState }; - Machine(System* system, Heap* heap, Finder* finder, Processor* processor); + Machine(System* system, Finder* finder, Processor* processor); ~Machine() { dispose(); @@ -1124,6 +1117,8 @@ class Machine { static const unsigned HeapPoolSize = 8; + static const unsigned FixedFootprintThresholdInBytes = 256 * 1024; + void dispose(); JavaVMVTable* vtable; @@ -1137,6 +1132,7 @@ class Machine { const char* builtins; unsigned activeCount; unsigned liveCount; + unsigned fixedFootprint; System::Local* localThread; System::Monitor* stateLock; System::Monitor* heapLock; @@ -1154,11 +1150,6 @@ class Machine { object finalizeQueue; object weakReferences; object tenuredWeakReferences; - object fixies; - object tenuredFixies; - object dirtyFixies; - object markedFixies; - object visitedFixies; bool unsafe; JavaVMVTable javaVMVTable; JNIEnvVTable jniEnvVTable; @@ -1175,9 +1166,6 @@ threadInterrupted(Thread* t, object thread); void enterActiveState(Thread* t); -void -visit(Thread* t, Heap::Visitor* v, object* p); - class Thread { public: enum State { @@ -1211,7 +1199,7 @@ class Thread { SingleProtector(Thread* t, object* p): Protector(t), p(p) { } virtual void visit(Heap::Visitor* v) { - vm::visit(t, v, p); + v->visit(p); } object* p; @@ -1266,7 +1254,6 @@ class Thread { Thread* peer; Thread* child; State state; - bool allocatedLarge; unsigned criticalLevel; System::Thread* systemThread; object javaThread; @@ -1422,10 +1409,7 @@ expect(Thread* t, bool v) } object -allocateFixed(Thread* t, unsigned sizeInBytes, bool objectMask); - -object -allocate2(Thread* t, unsigned sizeInBytes, bool objectMask); +allocate2(Thread* t, unsigned sizeInBytes, bool objectMask, bool fixed); inline object allocateSmall(Thread* t, unsigned sizeInBytes) @@ -1445,17 +1429,27 @@ allocate(Thread* t, unsigned sizeInBytes, bool objectMask) >= Thread::HeapSizeInWords or t->m->exclusive)) { - return allocate2(t, sizeInBytes, objectMask); + return allocate2(t, sizeInBytes, objectMask, + sizeInBytes > Thread::HeapSizeInBytes); } else { return allocateSmall(t, sizeInBytes); } } -void -mark(Thread* t, object target, unsigned offset); +inline void +mark(Thread* t, object o, unsigned offset, unsigned count) +{ + if (t->m->heap->needsMark(o)) { + ACQUIRE_RAW(t, t->m->heapLock); + t->m->heap->mark(o, offset / BytesPerWord, count); + } +} -void -mark(Thread* t, object target, unsigned offset, unsigned count); +inline void +mark(Thread* t, object o, unsigned offset) +{ + mark(t, o, offset, 1); +} inline void set(Thread* t, object target, unsigned offset, object value) @@ -1505,95 +1499,6 @@ baseSize(Thread* t, object o, object class_) BytesPerWord); } -inline void* -fixedStart(Thread* t UNUSED, object o) -{ - assert(t, objectFixed(t, o)); - return &cast(o, - FixedFootprint); -} - -inline uint8_t& -fixedAge(Thread* t UNUSED, object o) -{ - assert(t, objectFixed(t, o)); - return cast(o, - (FixedFootprint - FixedAge)); -} - -inline uint8_t& -fixedMarked(Thread* t UNUSED, object o) -{ - assert(t, objectFixed(t, o)); - return cast(o, - (FixedFootprint - FixedMarked)); -} - -inline uint8_t& -fixedDirty(Thread* t UNUSED, object o) -{ - assert(t, objectFixed(t, o)); - return cast(o, - (FixedFootprint - FixedDirty)); -} - -inline object*& -fixedHandle(Thread* t UNUSED, object o) -{ - assert(t, objectFixed(t, o)); - return cast(o, - (FixedFootprint - FixedHandle)); -} - -inline object& -fixedNext(Thread* t UNUSED, object o) -{ - assert(t, objectFixed(t, o)); - return cast(o, - (FixedFootprint - FixedNext)); -} - -inline void -fixedAdd(Thread* t, object o, object* handle) -{ -// fprintf(stderr, "add %p to %s\n", o, -// handle == &(t->m->fixies) ? "fixies" : -// handle == &(t->m->tenuredFixies) ? "tenured" : -// handle == &(t->m->dirtyFixies) ? "dirty" : -// handle == &(t->m->markedFixies) ? "marked" : -// handle == &(t->m->visitedFixies) ? "visited" : "unknown"); - - fixedHandle(t, o) = handle; - fixedNext(t, o) = *handle; - if (*handle) { - fixedHandle(t, *handle) = &fixedNext(t, o); - } - *handle = o; -} - -inline void -fixedRemove(Thread* t, object o) -{ - *fixedHandle(t, o) = fixedNext(t, o); - if (fixedNext(t, o)) { - fixedHandle(t, fixedNext(t, o)) = fixedHandle(t, o); - } -} - -inline void -fixedMove(Thread* t, object o, object* handle) -{ - fixedRemove(t, o); - fixedAdd(t, o, handle); -} - -inline uintptr_t* -fixedMask(Thread* t UNUSED, object o, unsigned size) -{ - assert(t, objectFixed(t, o)); - return &cast(o, size * BytesPerWord); -} - -inline uintptr_t* -fixedMask(Thread* t, object o) -{ - return fixedMask(t, o, baseSize(t, o, objectClass(t, o))); -} - object makeTrace(Thread* t, uintptr_t start);